import{S as _d,i as yd,s as $d,e as l,k as c,w,t as n,M as vd,c as o,d as t,m as h,x as b,a as r,h as a,b as _,N as jd,G as s,g as p,y as j,o as m,p as bo,q as g,B as k,v as Ed,n as jo}from"../../chunks/vendor-hf-doc-builder.js";import{T as Uc}from"../../chunks/Tip-hf-doc-builder.js";import{Y as ko}from"../../chunks/Youtube-hf-doc-builder.js";import{I as Pt}from"../../chunks/IconCopyLink-hf-doc-builder.js";import{C as $}from"../../chunks/CodeBlock-hf-doc-builder.js";import{C as kd}from"../../chunks/CourseFloatingBanner-hf-doc-builder.js";import{F as Od}from"../../chunks/FrameworkSwitchCourse-hf-doc-builder.js";function Id(D){let f,x;return f=new kd({props:{chapter:6,classNames:"absolute z-10 right-0 top-0",notebooks:[{label:"Google Colab",value:"https://colab.research.google.com/github/huggingface/notebooks/blob/master/course/en/chapter6/section3_tf.ipynb"},{label:"Aws Studio",value:"https://studiolab.sagemaker.aws/import/github/huggingface/notebooks/blob/master/course/en/chapter6/section3_tf.ipynb"}]}}),{c(){w(f.$$.fragment)},l(d){b(f.$$.fragment,d)},m(d,y){j(f,d,y),x=!0},i(d){x||(g(f.$$.fragment,d),x=!0)},o(d){m(f.$$.fragment,d),x=!1},d(d){k(f,d)}}}function Cd(D){let f,x;return f=new kd({props:{chapter:6,classNames:"absolute z-10 right-0 top-0",notebooks:[{label:"Google Colab",value:"https://colab.research.google.com/github/huggingface/notebooks/blob/master/course/en/chapter6/section3_pt.ipynb"},{label:"Aws Studio",value:"https://studiolab.sagemaker.aws/import/github/huggingface/notebooks/blob/master/course/en/chapter6/section3_pt.ipynb"}]}}),{c(){w(f.$$.fragment)},l(d){b(f.$$.fragment,d)},m(d,y){j(f,d,y),x=!0},i(d){x||(g(f.$$.fragment,d),x=!0)},o(d){m(f.$$.fragment,d),x=!1},d(d){k(f,d)}}}function zd(D){let f,x;return{c(){f=l("p"),x=n("\u26A0\uFE0F When tokenizing a single sentence, you won\u2019t always see a difference in speed between the slow and fast versions of the same tokenizer. In fact, the fast version might actually be slower! It\u2019s only when tokenizing lots of texts in parallel at the same time that you will be able to clearly see the difference.")},l(d){f=o(d,"P",{});var y=r(f);x=a(y,"\u26A0\uFE0F When tokenizing a single sentence, you won\u2019t always see a difference in speed between the slow and fast versions of the same tokenizer. In fact, the fast version might actually be slower! It\u2019s only when tokenizing lots of texts in parallel at the same time that you will be able to clearly see the difference."),y.forEach(t)},m(d,y){p(d,f,y),s(f,x)},d(d){d&&t(f)}}}function Td(D){let f,x,d,y,P,E,F,A,M,S,Y,I,B,z;return{c(){f=l("p"),x=n("The notion of what a word is is complicated. For instance, does \u201CI\u2019ll\u201D (a contraction of \u201CI will\u201D) count as one or two words? It actually depends on the tokenizer and the pre-tokenization operation it applies. Some tokenizers just split on spaces, so they will consider this as one word. Others use punctuation on top of spaces, so will consider it two words."),d=c(),y=l("p"),P=n("\u270F\uFE0F "),E=l("strong"),F=n("Try it out!"),A=n(" Create a tokenizer from the "),M=l("code"),S=n("bert-base-cased"),Y=n(" and "),I=l("code"),B=n("roberta-base"),z=n(" checkpoints and tokenize \u201D81s\u201D with them. What do you observe? What are the word IDs?")},l(T){f=o(T,"P",{});var O=r(f);x=a(O,"The notion of what a word is is complicated. For instance, does \u201CI\u2019ll\u201D (a contraction of \u201CI will\u201D) count as one or two words? It actually depends on the tokenizer and the pre-tokenization operation it applies. Some tokenizers just split on spaces, so they will consider this as one word. Others use punctuation on top of spaces, so will consider it two words."),O.forEach(t),d=h(T),y=o(T,"P",{});var X=r(y);P=a(X,"\u270F\uFE0F "),E=o(X,"STRONG",{});var q=r(E);F=a(q,"Try it out!"),q.forEach(t),A=a(X," Create a tokenizer from the "),M=o(X,"CODE",{});var V=r(M);S=a(V,"bert-base-cased"),V.forEach(t),Y=a(X," and "),I=o(X,"CODE",{});var H=r(I);B=a(H,"roberta-base"),H.forEach(t),z=a(X," checkpoints and tokenize \u201D81s\u201D with them. What do you observe? What are the word IDs?"),X.forEach(t)},m(T,O){p(T,f,O),s(f,x),p(T,d,O),p(T,y,O),s(y,P),s(y,E),s(E,F),s(y,A),s(y,M),s(M,S),s(y,Y),s(y,I),s(I,B),s(y,z)},d(T){T&&t(f),T&&t(d),T&&t(y)}}}function Dd(D){let f,x,d,y,P;return{c(){f=l("p"),x=n("\u270F\uFE0F "),d=l("strong"),y=n("Try it out!"),P=n(" Create your own example text and see if you can understand which tokens are associated with word ID, and also how to extract the character spans for a single word. For bonus points, try using two sentences as input and see if the sentence IDs make sense to you.")},l(E){f=o(E,"P",{});var F=r(f);x=a(F,"\u270F\uFE0F "),d=o(F,"STRONG",{});var A=r(d);y=a(A,"Try it out!"),A.forEach(t),P=a(F," Create your own example text and see if you can understand which tokens are associated with word ID, and also how to extract the character spans for a single word. For bonus points, try using two sentences as input and see if the sentence IDs make sense to you."),F.forEach(t)},m(E,F){p(E,f,F),s(f,x),s(f,d),s(d,y),s(f,P)},d(E){E&&t(f)}}}function Pd(D){let f,x;return f=new ko({props:{id:"PrX4CjrVnNc"}}),{c(){w(f.$$.fragment)},l(d){b(f.$$.fragment,d)},m(d,y){j(f,d,y),x=!0},i(d){x||(g(f.$$.fragment,d),x=!0)},o(d){m(f.$$.fragment,d),x=!1},d(d){k(f,d)}}}function Sd(D){let f,x;return f=new ko({props:{id:"0E7ltQB7fM8"}}),{c(){w(f.$$.fragment)},l(d){b(f.$$.fragment,d)},m(d,y){j(f,d,y),x=!0},i(d){x||(g(f.$$.fragment,d),x=!0)},o(d){m(f.$$.fragment,d),x=!1},d(d){k(f,d)}}}function Fd(D){let f,x,d,y,P,E,F,A,M,S,Y,I,B,z,T,O,X,q,V,H,Z;return S=new $({props:{code:`from transformers import AutoTokenizer, TFAutoModelForTokenClassification

model_checkpoint = "dbmdz/bert-large-cased-finetuned-conll03-english"
tokenizer = AutoTokenizer.from_pretrained(model_checkpoint)
model = TFAutoModelForTokenClassification.from_pretrained(model_checkpoint)

example = "My name is Sylvain and I work at Hugging Face in Brooklyn."
inputs = tokenizer(example, return_tensors="tf")
outputs = model(**inputs)`,highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoTokenizer, TFAutoModelForTokenClassification

model_checkpoint = <span class="hljs-string">&quot;dbmdz/bert-large-cased-finetuned-conll03-english&quot;</span>
tokenizer = AutoTokenizer.from_pretrained(model_checkpoint)
model = TFAutoModelForTokenClassification.from_pretrained(model_checkpoint)

example = <span class="hljs-string">&quot;My name is Sylvain and I work at Hugging Face in Brooklyn.&quot;</span>
inputs = tokenizer(example, return_tensors=<span class="hljs-string">&quot;tf&quot;</span>)
outputs = model(**inputs)`}}),q=new $({props:{code:`print(inputs["input_ids"].shape)
print(outputs.logits.shape)`,highlighted:`<span class="hljs-built_in">print</span>(inputs[<span class="hljs-string">&quot;input_ids&quot;</span>].shape)
<span class="hljs-built_in">print</span>(outputs.logits.shape)`}}),H=new $({props:{code:`(1, 19)
(1, 19, 9)`,highlighted:`(<span class="hljs-number">1</span>, <span class="hljs-number">19</span>)
(<span class="hljs-number">1</span>, <span class="hljs-number">19</span>, <span class="hljs-number">9</span>)`}}),{c(){f=l("p"),x=n("First we need to tokenize our input and pass it through the model. This is done exactly as in "),d=l("a"),y=n("Chapter 2"),P=n("; we instantiate the tokenizer and the model using the "),E=l("code"),F=n("TFAutoXxx"),A=n(" classes and then use them on our example:"),M=c(),w(S.$$.fragment),Y=c(),I=l("p"),B=n("Since we\u2019re using "),z=l("code"),T=n("TFAutoModelForTokenClassification"),O=n(" here, we get one set of logits for each token in the input sequence:"),X=c(),w(q.$$.fragment),V=c(),w(H.$$.fragment),this.h()},l(u){f=o(u,"P",{});var v=r(f);x=a(v,"First we need to tokenize our input and pass it through the model. This is done exactly as in "),d=o(v,"A",{href:!0});var de=r(d);y=a(de,"Chapter 2"),de.forEach(t),P=a(v,"; we instantiate the tokenizer and the model using the "),E=o(v,"CODE",{});var Ee=r(E);F=a(Ee,"TFAutoXxx"),Ee.forEach(t),A=a(v," classes and then use them on our example:"),v.forEach(t),M=h(u),b(S.$$.fragment,u),Y=h(u),I=o(u,"P",{});var le=r(I);B=a(le,"Since we\u2019re using "),z=o(le,"CODE",{});var re=r(z);T=a(re,"TFAutoModelForTokenClassification"),re.forEach(t),O=a(le," here, we get one set of logits for each token in the input sequence:"),le.forEach(t),X=h(u),b(q.$$.fragment,u),V=h(u),b(H.$$.fragment,u),this.h()},h(){_(d,"href","/course/chapter2")},m(u,v){p(u,f,v),s(f,x),s(f,d),s(d,y),s(f,P),s(f,E),s(E,F),s(f,A),p(u,M,v),j(S,u,v),p(u,Y,v),p(u,I,v),s(I,B),s(I,z),s(z,T),s(I,O),p(u,X,v),j(q,u,v),p(u,V,v),j(H,u,v),Z=!0},i(u){Z||(g(S.$$.fragment,u),g(q.$$.fragment,u),g(H.$$.fragment,u),Z=!0)},o(u){m(S.$$.fragment,u),m(q.$$.fragment,u),m(H.$$.fragment,u),Z=!1},d(u){u&&t(f),u&&t(M),k(S,u),u&&t(Y),u&&t(I),u&&t(X),k(q,u),u&&t(V),k(H,u)}}}function qd(D){let f,x,d,y,P,E,F,A,M,S,Y,I,B,z,T,O,X,q,V,H,Z;return S=new $({props:{code:`from transformers import AutoTokenizer, AutoModelForTokenClassification

model_checkpoint = "dbmdz/bert-large-cased-finetuned-conll03-english"
tokenizer = AutoTokenizer.from_pretrained(model_checkpoint)
model = AutoModelForTokenClassification.from_pretrained(model_checkpoint)

example = "My name is Sylvain and I work at Hugging Face in Brooklyn."
inputs = tokenizer(example, return_tensors="pt")
outputs = model(**inputs)`,highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoTokenizer, AutoModelForTokenClassification

model_checkpoint = <span class="hljs-string">&quot;dbmdz/bert-large-cased-finetuned-conll03-english&quot;</span>
tokenizer = AutoTokenizer.from_pretrained(model_checkpoint)
model = AutoModelForTokenClassification.from_pretrained(model_checkpoint)

example = <span class="hljs-string">&quot;My name is Sylvain and I work at Hugging Face in Brooklyn.&quot;</span>
inputs = tokenizer(example, return_tensors=<span class="hljs-string">&quot;pt&quot;</span>)
outputs = model(**inputs)`}}),q=new $({props:{code:`print(inputs["input_ids"].shape)
print(outputs.logits.shape)`,highlighted:`<span class="hljs-built_in">print</span>(inputs[<span class="hljs-string">&quot;input_ids&quot;</span>].shape)
<span class="hljs-built_in">print</span>(outputs.logits.shape)`}}),H=new $({props:{code:`torch.Size([1, 19])
torch.Size([1, 19, 9])`,highlighted:`torch.Size([<span class="hljs-number">1</span>, <span class="hljs-number">19</span>])
torch.Size([<span class="hljs-number">1</span>, <span class="hljs-number">19</span>, <span class="hljs-number">9</span>])`}}),{c(){f=l("p"),x=n("First we need to tokenize our input and pass it through the model. This is done exactly as in "),d=l("a"),y=n("Chapter 2"),P=n("; we instantiate the tokenizer and the model using the "),E=l("code"),F=n("AutoXxx"),A=n(" classes and then use them on our example:"),M=c(),w(S.$$.fragment),Y=c(),I=l("p"),B=n("Since we\u2019re using "),z=l("code"),T=n("AutoModelForTokenClassification"),O=n(" here, we get one set of logits for each token in the input sequence:"),X=c(),w(q.$$.fragment),V=c(),w(H.$$.fragment),this.h()},l(u){f=o(u,"P",{});var v=r(f);x=a(v,"First we need to tokenize our input and pass it through the model. This is done exactly as in "),d=o(v,"A",{href:!0});var de=r(d);y=a(de,"Chapter 2"),de.forEach(t),P=a(v,"; we instantiate the tokenizer and the model using the "),E=o(v,"CODE",{});var Ee=r(E);F=a(Ee,"AutoXxx"),Ee.forEach(t),A=a(v," classes and then use them on our example:"),v.forEach(t),M=h(u),b(S.$$.fragment,u),Y=h(u),I=o(u,"P",{});var le=r(I);B=a(le,"Since we\u2019re using "),z=o(le,"CODE",{});var re=r(z);T=a(re,"AutoModelForTokenClassification"),re.forEach(t),O=a(le," here, we get one set of logits for each token in the input sequence:"),le.forEach(t),X=h(u),b(q.$$.fragment,u),V=h(u),b(H.$$.fragment,u),this.h()},h(){_(d,"href","/course/chapter2")},m(u,v){p(u,f,v),s(f,x),s(f,d),s(d,y),s(f,P),s(f,E),s(E,F),s(f,A),p(u,M,v),j(S,u,v),p(u,Y,v),p(u,I,v),s(I,B),s(I,z),s(z,T),s(I,O),p(u,X,v),j(q,u,v),p(u,V,v),j(H,u,v),Z=!0},i(u){Z||(g(S.$$.fragment,u),g(q.$$.fragment,u),g(H.$$.fragment,u),Z=!0)},o(u){m(S.$$.fragment,u),m(q.$$.fragment,u),m(H.$$.fragment,u),Z=!1},d(u){u&&t(f),u&&t(M),k(S,u),u&&t(Y),u&&t(I),u&&t(X),k(q,u),u&&t(V),k(H,u)}}}function Rd(D){let f,x;return f=new $({props:{code:`import tensorflow as tf

probabilities = tf.math.softmax(outputs.logits, axis=-1)[0]
probabilities = probabilities.numpy().tolist()
predictions = tf.math.argmax(outputs.logits, axis=-1)[0]
predictions = predictions.numpy().tolist()
print(predictions)`,highlighted:`<span class="hljs-keyword">import</span> tensorflow <span class="hljs-keyword">as</span> tf

probabilities = tf.math.softmax(outputs.logits, axis=-<span class="hljs-number">1</span>)[<span class="hljs-number">0</span>]
probabilities = probabilities.numpy().tolist()
predictions = tf.math.argmax(outputs.logits, axis=-<span class="hljs-number">1</span>)[<span class="hljs-number">0</span>]
predictions = predictions.numpy().tolist()
<span class="hljs-built_in">print</span>(predictions)`}}),{c(){w(f.$$.fragment)},l(d){b(f.$$.fragment,d)},m(d,y){j(f,d,y),x=!0},i(d){x||(g(f.$$.fragment,d),x=!0)},o(d){m(f.$$.fragment,d),x=!1},d(d){k(f,d)}}}function Bd(D){let f,x;return f=new $({props:{code:`import torch

probabilities = torch.nn.functional.softmax(outputs.logits, dim=-1)[0].tolist()
predictions = outputs.logits.argmax(dim=-1)[0].tolist()
print(predictions)`,highlighted:`<span class="hljs-keyword">import</span> torch

probabilities = torch.nn.functional.softmax(outputs.logits, dim=-<span class="hljs-number">1</span>)[<span class="hljs-number">0</span>].tolist()
predictions = outputs.logits.argmax(dim=-<span class="hljs-number">1</span>)[<span class="hljs-number">0</span>].tolist()
<span class="hljs-built_in">print</span>(predictions)`}}),{c(){w(f.$$.fragment)},l(d){b(f.$$.fragment,d)},m(d,y){j(f,d,y),x=!0},i(d){x||(g(f.$$.fragment,d),x=!0)},o(d){m(f.$$.fragment,d),x=!1},d(d){k(f,d)}}}function Ad(D){let f,x,d,y,P,E,F,A,M,S,Y,I,B,z,T,O,X,q,V,H,Z,u,v,de,Ee,le,re,_o,yo,Ca,ls,za,qe,$o,Zs,vo,Eo,Ta,Re,St,Oe,et,Oo,Io,st,Co,zo,Ft,To,os,Ie,tt,qt,Do,Po,nt,So,Fo,at,qo,Ro,Ce,lt,Rt,Bo,Ao,ot,Xo,Ho,rt,Go,Da,Be,Pa,ze,Ae,Bt,rs,Mo,At,No,Sa,is,Fa,Xe,Wo,Xt,Lo,Uo,qa,He,Yo,Ht,Vo,Qo,Ra,it,Jo,Ba,ps,Aa,Ge,Ko,Gt,Zo,er,Xa,cs,Ha,ee,sr,Mt,tr,nr,Nt,ar,lr,Wt,or,rr,Lt,ir,pr,Ga,hs,Ma,fs,Na,Me,cr,Ut,hr,fr,Wa,ds,La,us,Ua,pt,dr,Ya,ms,Va,gs,Qa,je,ur,Yt,mr,gr,Vt,xr,wr,Ja,xs,Ka,ws,Za,Q,br,Qt,jr,kr,Jt,_r,yr,Kt,$r,vr,Zt,Er,Or,en,Ir,Cr,el,Ne,sl,ke,zr,sn,Tr,Dr,tn,Pr,Sr,tl,N,Fr,nn,qr,Rr,an,Br,Ar,ln,Xr,Hr,on,Gr,Mr,rn,Nr,Wr,pn,Lr,Ur,nl,bs,al,js,ll,_e,Yr,cn,Vr,Qr,hn,Jr,Kr,ol,We,rl,Te,Le,fn,ks,Zr,_s,ei,dn,si,ti,il,se,ni,ct,ai,li,un,oi,ri,ht,ii,pi,mn,ci,hi,pl,ue,me,ft,De,Ue,gn,ys,fi,xn,di,cl,Ye,ui,$s,wn,mi,gi,hl,vs,fl,Es,dl,dt,xi,ul,Os,ml,Is,gl,W,wi,bn,bi,ji,jn,ki,_i,kn,yi,$i,_n,vi,Ei,yn,Oi,Ii,$n,Ci,zi,xl,ye,Ve,vn,Ti,Di,En,Pi,Si,Fi,ut,On,qi,Ri,Bi,Qe,In,Ai,Xi,Cn,Hi,Gi,wl,Je,Mi,zn,Ni,Wi,bl,Pe,Ke,Tn,Cs,Li,Dn,Ui,jl,ge,xe,mt,gt,Yi,kl,we,be,xt,zs,_l,Ze,Vi,Pn,Qi,Ji,yl,Ts,$l,Ds,vl,C,Ki,Sn,Zi,ep,Fn,sp,tp,qn,np,ap,Rn,lp,op,Bn,rp,ip,An,pp,cp,Xn,hp,fp,Hn,dp,up,Gn,mp,gp,Mn,xp,wp,Nn,bp,jp,El,G,kp,Wn,_p,yp,Ln,$p,vp,Un,Ep,Op,Yn,Ip,Cp,Vn,zp,Tp,Qn,Dp,Pp,Jn,Sp,Fp,Kn,qp,Rp,Ol,Se,Ps,Yc,Bp,Ss,Vc,Il,es,Ap,Zn,Xp,Hp,Cl,Fs,zl,qs,Tl,ie,Gp,ea,Mp,Np,sa,Wp,Lp,ta,Up,Yp,Dl,Rs,Pl,Bs,Sl,pe,Vp,na,Qp,Jp,aa,Kp,Zp,la,ec,sc,Fl,As,ql,ss,tc,oa,nc,ac,Rl,Xs,Bl,wt,lc,Al,Hs,Xl,Gs,Hl,bt,oc,Gl,Fe,ts,ra,Ms,rc,ia,ic,Ml,L,pc,pa,cc,hc,ca,fc,dc,ha,uc,mc,fa,gc,xc,da,wc,bc,ua,jc,kc,Nl,J,_c,ma,yc,$c,ga,vc,Ec,xa,Oc,Ic,wa,Cc,zc,ba,Tc,Dc,Wl,Ns,Ll,Ws,Ul,K,Pc,ja,Sc,Fc,ka,qc,Rc,_a,Bc,Ac,ya,Xc,Hc,$a,Gc,Mc,Yl,Ls,Vl,jt,Nc,Ql,Us,Jl,kt,Wc,Kl;d=new Od({props:{fw:D[0]}}),A=new Pt({});const Qc=[Cd,Id],Ys=[];function Jc(e,i){return e[0]==="pt"?0:1}B=Jc(D),z=Ys[B]=Qc[B](D),ls=new ko({props:{id:"g8quOxoqhHQ"}}),Be=new Uc({props:{warning:!0,$$slots:{default:[zd]},$$scope:{ctx:D}}}),rs=new Pt({}),is=new ko({props:{id:"3umI3tm27Vw"}}),ps=new $({props:{code:`from transformers import AutoTokenizer

tokenizer = AutoTokenizer.from_pretrained("bert-base-cased")
example = "My name is Sylvain and I work at Hugging Face in Brooklyn."
encoding = tokenizer(example)
print(type(encoding))`,highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoTokenizer

tokenizer = AutoTokenizer.from_pretrained(<span class="hljs-string">&quot;bert-base-cased&quot;</span>)
example = <span class="hljs-string">&quot;My name is Sylvain and I work at Hugging Face in Brooklyn.&quot;</span>
encoding = tokenizer(example)
<span class="hljs-built_in">print</span>(<span class="hljs-built_in">type</span>(encoding))`}}),cs=new $({props:{code:"<class 'transformers.tokenization_utils_base.BatchEncoding'>",highlighted:'&lt;<span class="hljs-keyword">class</span> <span class="hljs-string">&#x27;transformers.tokenization_utils_base.BatchEncoding&#x27;</span>&gt;'}}),hs=new $({props:{code:"tokenizer.is_fast",highlighted:"tokenizer.is_fast"}}),fs=new $({props:{code:"True",highlighted:'<span class="hljs-literal">True</span>'}}),ds=new $({props:{code:"encoding.is_fast",highlighted:"encoding.is_fast"}}),us=new $({props:{code:"True",highlighted:'<span class="hljs-literal">True</span>'}}),ms=new $({props:{code:"encoding.tokens()",highlighted:"encoding.tokens()"}}),gs=new $({props:{code:`['[CLS]', 'My', 'name', 'is', 'S', '##yl', '##va', '##in', 'and', 'I', 'work', 'at', 'Hu', '##gging', 'Face', 'in',
 'Brooklyn', '.', '[SEP]']`,highlighted:`[<span class="hljs-string">&#x27;[CLS]&#x27;</span>, <span class="hljs-string">&#x27;My&#x27;</span>, <span class="hljs-string">&#x27;name&#x27;</span>, <span class="hljs-string">&#x27;is&#x27;</span>, <span class="hljs-string">&#x27;S&#x27;</span>, <span class="hljs-string">&#x27;##yl&#x27;</span>, <span class="hljs-string">&#x27;##va&#x27;</span>, <span class="hljs-string">&#x27;##in&#x27;</span>, <span class="hljs-string">&#x27;and&#x27;</span>, <span class="hljs-string">&#x27;I&#x27;</span>, <span class="hljs-string">&#x27;work&#x27;</span>, <span class="hljs-string">&#x27;at&#x27;</span>, <span class="hljs-string">&#x27;Hu&#x27;</span>, <span class="hljs-string">&#x27;##gging&#x27;</span>, <span class="hljs-string">&#x27;Face&#x27;</span>, <span class="hljs-string">&#x27;in&#x27;</span>,
 <span class="hljs-string">&#x27;Brooklyn&#x27;</span>, <span class="hljs-string">&#x27;.&#x27;</span>, <span class="hljs-string">&#x27;[SEP]&#x27;</span>]`}}),xs=new $({props:{code:"encoding.word_ids()",highlighted:"encoding.word_ids()"}}),ws=new $({props:{code:"[None, 0, 1, 2, 3, 3, 3, 3, 4, 5, 6, 7, 8, 8, 9, 10, 11, 12, None]",highlighted:'[<span class="hljs-literal">None</span>, <span class="hljs-number">0</span>, <span class="hljs-number">1</span>, <span class="hljs-number">2</span>, <span class="hljs-number">3</span>, <span class="hljs-number">3</span>, <span class="hljs-number">3</span>, <span class="hljs-number">3</span>, <span class="hljs-number">4</span>, <span class="hljs-number">5</span>, <span class="hljs-number">6</span>, <span class="hljs-number">7</span>, <span class="hljs-number">8</span>, <span class="hljs-number">8</span>, <span class="hljs-number">9</span>, <span class="hljs-number">10</span>, <span class="hljs-number">11</span>, <span class="hljs-number">12</span>, <span class="hljs-literal">None</span>]'}}),Ne=new Uc({props:{$$slots:{default:[Td]},$$scope:{ctx:D}}}),bs=new $({props:{code:`start, end = encoding.word_to_chars(3)
example[start:end]`,highlighted:`start, end = encoding.word_to_chars(<span class="hljs-number">3</span>)
example[start:end]`}}),js=new $({props:{code:"Sylvain",highlighted:"Sylvain"}}),We=new Uc({props:{$$slots:{default:[Dd]},$$scope:{ctx:D}}}),ks=new Pt({});const Kc=[Sd,Pd],Vs=[];function Zc(e,i){return e[0]==="pt"?0:1}ue=Zc(D),me=Vs[ue]=Kc[ue](D),ys=new Pt({}),vs=new $({props:{code:`from transformers import pipeline

token_classifier = pipeline("token-classification")
token_classifier("My name is Sylvain and I work at Hugging Face in Brooklyn.")`,highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> pipeline

token_classifier = pipeline(<span class="hljs-string">&quot;token-classification&quot;</span>)
token_classifier(<span class="hljs-string">&quot;My name is Sylvain and I work at Hugging Face in Brooklyn.&quot;</span>)`}}),Es=new $({props:{code:`[{'entity': 'I-PER', 'score': 0.9993828, 'index': 4, 'word': 'S', 'start': 11, 'end': 12},
 {'entity': 'I-PER', 'score': 0.99815476, 'index': 5, 'word': '##yl', 'start': 12, 'end': 14},
 {'entity': 'I-PER', 'score': 0.99590725, 'index': 6, 'word': '##va', 'start': 14, 'end': 16},
 {'entity': 'I-PER', 'score': 0.9992327, 'index': 7, 'word': '##in', 'start': 16, 'end': 18},
 {'entity': 'I-ORG', 'score': 0.97389334, 'index': 12, 'word': 'Hu', 'start': 33, 'end': 35},
 {'entity': 'I-ORG', 'score': 0.976115, 'index': 13, 'word': '##gging', 'start': 35, 'end': 40},
 {'entity': 'I-ORG', 'score': 0.98879766, 'index': 14, 'word': 'Face', 'start': 41, 'end': 45},
 {'entity': 'I-LOC', 'score': 0.99321055, 'index': 16, 'word': 'Brooklyn', 'start': 49, 'end': 57}]`,highlighted:`[{<span class="hljs-string">&#x27;entity&#x27;</span>: <span class="hljs-string">&#x27;I-PER&#x27;</span>, <span class="hljs-string">&#x27;score&#x27;</span>: <span class="hljs-number">0.9993828</span>, <span class="hljs-string">&#x27;index&#x27;</span>: <span class="hljs-number">4</span>, <span class="hljs-string">&#x27;word&#x27;</span>: <span class="hljs-string">&#x27;S&#x27;</span>, <span class="hljs-string">&#x27;start&#x27;</span>: <span class="hljs-number">11</span>, <span class="hljs-string">&#x27;end&#x27;</span>: <span class="hljs-number">12</span>},
 {<span class="hljs-string">&#x27;entity&#x27;</span>: <span class="hljs-string">&#x27;I-PER&#x27;</span>, <span class="hljs-string">&#x27;score&#x27;</span>: <span class="hljs-number">0.99815476</span>, <span class="hljs-string">&#x27;index&#x27;</span>: <span class="hljs-number">5</span>, <span class="hljs-string">&#x27;word&#x27;</span>: <span class="hljs-string">&#x27;##yl&#x27;</span>, <span class="hljs-string">&#x27;start&#x27;</span>: <span class="hljs-number">12</span>, <span class="hljs-string">&#x27;end&#x27;</span>: <span class="hljs-number">14</span>},
 {<span class="hljs-string">&#x27;entity&#x27;</span>: <span class="hljs-string">&#x27;I-PER&#x27;</span>, <span class="hljs-string">&#x27;score&#x27;</span>: <span class="hljs-number">0.99590725</span>, <span class="hljs-string">&#x27;index&#x27;</span>: <span class="hljs-number">6</span>, <span class="hljs-string">&#x27;word&#x27;</span>: <span class="hljs-string">&#x27;##va&#x27;</span>, <span class="hljs-string">&#x27;start&#x27;</span>: <span class="hljs-number">14</span>, <span class="hljs-string">&#x27;end&#x27;</span>: <span class="hljs-number">16</span>},
 {<span class="hljs-string">&#x27;entity&#x27;</span>: <span class="hljs-string">&#x27;I-PER&#x27;</span>, <span class="hljs-string">&#x27;score&#x27;</span>: <span class="hljs-number">0.9992327</span>, <span class="hljs-string">&#x27;index&#x27;</span>: <span class="hljs-number">7</span>, <span class="hljs-string">&#x27;word&#x27;</span>: <span class="hljs-string">&#x27;##in&#x27;</span>, <span class="hljs-string">&#x27;start&#x27;</span>: <span class="hljs-number">16</span>, <span class="hljs-string">&#x27;end&#x27;</span>: <span class="hljs-number">18</span>},
 {<span class="hljs-string">&#x27;entity&#x27;</span>: <span class="hljs-string">&#x27;I-ORG&#x27;</span>, <span class="hljs-string">&#x27;score&#x27;</span>: <span class="hljs-number">0.97389334</span>, <span class="hljs-string">&#x27;index&#x27;</span>: <span class="hljs-number">12</span>, <span class="hljs-string">&#x27;word&#x27;</span>: <span class="hljs-string">&#x27;Hu&#x27;</span>, <span class="hljs-string">&#x27;start&#x27;</span>: <span class="hljs-number">33</span>, <span class="hljs-string">&#x27;end&#x27;</span>: <span class="hljs-number">35</span>},
 {<span class="hljs-string">&#x27;entity&#x27;</span>: <span class="hljs-string">&#x27;I-ORG&#x27;</span>, <span class="hljs-string">&#x27;score&#x27;</span>: <span class="hljs-number">0.976115</span>, <span class="hljs-string">&#x27;index&#x27;</span>: <span class="hljs-number">13</span>, <span class="hljs-string">&#x27;word&#x27;</span>: <span class="hljs-string">&#x27;##gging&#x27;</span>, <span class="hljs-string">&#x27;start&#x27;</span>: <span class="hljs-number">35</span>, <span class="hljs-string">&#x27;end&#x27;</span>: <span class="hljs-number">40</span>},
 {<span class="hljs-string">&#x27;entity&#x27;</span>: <span class="hljs-string">&#x27;I-ORG&#x27;</span>, <span class="hljs-string">&#x27;score&#x27;</span>: <span class="hljs-number">0.98879766</span>, <span class="hljs-string">&#x27;index&#x27;</span>: <span class="hljs-number">14</span>, <span class="hljs-string">&#x27;word&#x27;</span>: <span class="hljs-string">&#x27;Face&#x27;</span>, <span class="hljs-string">&#x27;start&#x27;</span>: <span class="hljs-number">41</span>, <span class="hljs-string">&#x27;end&#x27;</span>: <span class="hljs-number">45</span>},
 {<span class="hljs-string">&#x27;entity&#x27;</span>: <span class="hljs-string">&#x27;I-LOC&#x27;</span>, <span class="hljs-string">&#x27;score&#x27;</span>: <span class="hljs-number">0.99321055</span>, <span class="hljs-string">&#x27;index&#x27;</span>: <span class="hljs-number">16</span>, <span class="hljs-string">&#x27;word&#x27;</span>: <span class="hljs-string">&#x27;Brooklyn&#x27;</span>, <span class="hljs-string">&#x27;start&#x27;</span>: <span class="hljs-number">49</span>, <span class="hljs-string">&#x27;end&#x27;</span>: <span class="hljs-number">57</span>}]`}}),Os=new $({props:{code:`from transformers import pipeline

token_classifier = pipeline("token-classification", aggregation_strategy="simple")
token_classifier("My name is Sylvain and I work at Hugging Face in Brooklyn.")`,highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> pipeline

token_classifier = pipeline(<span class="hljs-string">&quot;token-classification&quot;</span>, aggregation_strategy=<span class="hljs-string">&quot;simple&quot;</span>)
token_classifier(<span class="hljs-string">&quot;My name is Sylvain and I work at Hugging Face in Brooklyn.&quot;</span>)`}}),Is=new $({props:{code:`[{'entity_group': 'PER', 'score': 0.9981694, 'word': 'Sylvain', 'start': 11, 'end': 18},
 {'entity_group': 'ORG', 'score': 0.97960204, 'word': 'Hugging Face', 'start': 33, 'end': 45},
 {'entity_group': 'LOC', 'score': 0.99321055, 'word': 'Brooklyn', 'start': 49, 'end': 57}]`,highlighted:`[{<span class="hljs-string">&#x27;entity_group&#x27;</span>: <span class="hljs-string">&#x27;PER&#x27;</span>, <span class="hljs-string">&#x27;score&#x27;</span>: <span class="hljs-number">0.9981694</span>, <span class="hljs-string">&#x27;word&#x27;</span>: <span class="hljs-string">&#x27;Sylvain&#x27;</span>, <span class="hljs-string">&#x27;start&#x27;</span>: <span class="hljs-number">11</span>, <span class="hljs-string">&#x27;end&#x27;</span>: <span class="hljs-number">18</span>},
 {<span class="hljs-string">&#x27;entity_group&#x27;</span>: <span class="hljs-string">&#x27;ORG&#x27;</span>, <span class="hljs-string">&#x27;score&#x27;</span>: <span class="hljs-number">0.97960204</span>, <span class="hljs-string">&#x27;word&#x27;</span>: <span class="hljs-string">&#x27;Hugging Face&#x27;</span>, <span class="hljs-string">&#x27;start&#x27;</span>: <span class="hljs-number">33</span>, <span class="hljs-string">&#x27;end&#x27;</span>: <span class="hljs-number">45</span>},
 {<span class="hljs-string">&#x27;entity_group&#x27;</span>: <span class="hljs-string">&#x27;LOC&#x27;</span>, <span class="hljs-string">&#x27;score&#x27;</span>: <span class="hljs-number">0.99321055</span>, <span class="hljs-string">&#x27;word&#x27;</span>: <span class="hljs-string">&#x27;Brooklyn&#x27;</span>, <span class="hljs-string">&#x27;start&#x27;</span>: <span class="hljs-number">49</span>, <span class="hljs-string">&#x27;end&#x27;</span>: <span class="hljs-number">57</span>}]`}}),Cs=new Pt({});const eh=[qd,Fd],Qs=[];function sh(e,i){return e[0]==="pt"?0:1}ge=sh(D),xe=Qs[ge]=eh[ge](D);const th=[Bd,Rd],Js=[];function nh(e,i){return e[0]==="pt"?0:1}return we=nh(D),be=Js[we]=th[we](D),zs=new $({props:{code:"[0, 0, 0, 0, 4, 4, 4, 4, 0, 0, 0, 0, 6, 6, 6, 0, 8, 0, 0]",highlighted:'[<span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">4</span>, <span class="hljs-number">4</span>, <span class="hljs-number">4</span>, <span class="hljs-number">4</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">6</span>, <span class="hljs-number">6</span>, <span class="hljs-number">6</span>, <span class="hljs-number">0</span>, <span class="hljs-number">8</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>]'}}),Ts=new $({props:{code:"model.config.id2label",highlighted:"model.config.id2label"}}),Ds=new $({props:{code:`{0: 'O',
 1: 'B-MISC',
 2: 'I-MISC',
 3: 'B-PER',
 4: 'I-PER',
 5: 'B-ORG',
 6: 'I-ORG',
 7: 'B-LOC',
 8: 'I-LOC'}`,highlighted:`{<span class="hljs-number">0</span>: <span class="hljs-string">&#x27;O&#x27;</span>,
 <span class="hljs-number">1</span>: <span class="hljs-string">&#x27;B-MISC&#x27;</span>,
 <span class="hljs-number">2</span>: <span class="hljs-string">&#x27;I-MISC&#x27;</span>,
 <span class="hljs-number">3</span>: <span class="hljs-string">&#x27;B-PER&#x27;</span>,
 <span class="hljs-number">4</span>: <span class="hljs-string">&#x27;I-PER&#x27;</span>,
 <span class="hljs-number">5</span>: <span class="hljs-string">&#x27;B-ORG&#x27;</span>,
 <span class="hljs-number">6</span>: <span class="hljs-string">&#x27;I-ORG&#x27;</span>,
 <span class="hljs-number">7</span>: <span class="hljs-string">&#x27;B-LOC&#x27;</span>,
 <span class="hljs-number">8</span>: <span class="hljs-string">&#x27;I-LOC&#x27;</span>}`}}),Fs=new $({props:{code:`results = []
tokens = inputs.tokens()

for idx, pred in enumerate(predictions):
    label = model.config.id2label[pred]
    if label != "O":
        results.append(
            {"entity": label, "score": probabilities[idx][pred], "word": tokens[idx]}
        )

print(results)`,highlighted:`results = []
tokens = inputs.tokens()

<span class="hljs-keyword">for</span> idx, pred <span class="hljs-keyword">in</span> <span class="hljs-built_in">enumerate</span>(predictions):
    label = model.config.id2label[pred]
    <span class="hljs-keyword">if</span> label != <span class="hljs-string">&quot;O&quot;</span>:
        results.append(
            {<span class="hljs-string">&quot;entity&quot;</span>: label, <span class="hljs-string">&quot;score&quot;</span>: probabilities[idx][pred], <span class="hljs-string">&quot;word&quot;</span>: tokens[idx]}
        )

<span class="hljs-built_in">print</span>(results)`}}),qs=new $({props:{code:`[{'entity': 'I-PER', 'score': 0.9993828, 'index': 4, 'word': 'S'},
 {'entity': 'I-PER', 'score': 0.99815476, 'index': 5, 'word': '##yl'},
 {'entity': 'I-PER', 'score': 0.99590725, 'index': 6, 'word': '##va'},
 {'entity': 'I-PER', 'score': 0.9992327, 'index': 7, 'word': '##in'},
 {'entity': 'I-ORG', 'score': 0.97389334, 'index': 12, 'word': 'Hu'},
 {'entity': 'I-ORG', 'score': 0.976115, 'index': 13, 'word': '##gging'},
 {'entity': 'I-ORG', 'score': 0.98879766, 'index': 14, 'word': 'Face'},
 {'entity': 'I-LOC', 'score': 0.99321055, 'index': 16, 'word': 'Brooklyn'}]`,highlighted:`[{<span class="hljs-string">&#x27;entity&#x27;</span>: <span class="hljs-string">&#x27;I-PER&#x27;</span>, <span class="hljs-string">&#x27;score&#x27;</span>: <span class="hljs-number">0.9993828</span>, <span class="hljs-string">&#x27;index&#x27;</span>: <span class="hljs-number">4</span>, <span class="hljs-string">&#x27;word&#x27;</span>: <span class="hljs-string">&#x27;S&#x27;</span>},
 {<span class="hljs-string">&#x27;entity&#x27;</span>: <span class="hljs-string">&#x27;I-PER&#x27;</span>, <span class="hljs-string">&#x27;score&#x27;</span>: <span class="hljs-number">0.99815476</span>, <span class="hljs-string">&#x27;index&#x27;</span>: <span class="hljs-number">5</span>, <span class="hljs-string">&#x27;word&#x27;</span>: <span class="hljs-string">&#x27;##yl&#x27;</span>},
 {<span class="hljs-string">&#x27;entity&#x27;</span>: <span class="hljs-string">&#x27;I-PER&#x27;</span>, <span class="hljs-string">&#x27;score&#x27;</span>: <span class="hljs-number">0.99590725</span>, <span class="hljs-string">&#x27;index&#x27;</span>: <span class="hljs-number">6</span>, <span class="hljs-string">&#x27;word&#x27;</span>: <span class="hljs-string">&#x27;##va&#x27;</span>},
 {<span class="hljs-string">&#x27;entity&#x27;</span>: <span class="hljs-string">&#x27;I-PER&#x27;</span>, <span class="hljs-string">&#x27;score&#x27;</span>: <span class="hljs-number">0.9992327</span>, <span class="hljs-string">&#x27;index&#x27;</span>: <span class="hljs-number">7</span>, <span class="hljs-string">&#x27;word&#x27;</span>: <span class="hljs-string">&#x27;##in&#x27;</span>},
 {<span class="hljs-string">&#x27;entity&#x27;</span>: <span class="hljs-string">&#x27;I-ORG&#x27;</span>, <span class="hljs-string">&#x27;score&#x27;</span>: <span class="hljs-number">0.97389334</span>, <span class="hljs-string">&#x27;index&#x27;</span>: <span class="hljs-number">12</span>, <span class="hljs-string">&#x27;word&#x27;</span>: <span class="hljs-string">&#x27;Hu&#x27;</span>},
 {<span class="hljs-string">&#x27;entity&#x27;</span>: <span class="hljs-string">&#x27;I-ORG&#x27;</span>, <span class="hljs-string">&#x27;score&#x27;</span>: <span class="hljs-number">0.976115</span>, <span class="hljs-string">&#x27;index&#x27;</span>: <span class="hljs-number">13</span>, <span class="hljs-string">&#x27;word&#x27;</span>: <span class="hljs-string">&#x27;##gging&#x27;</span>},
 {<span class="hljs-string">&#x27;entity&#x27;</span>: <span class="hljs-string">&#x27;I-ORG&#x27;</span>, <span class="hljs-string">&#x27;score&#x27;</span>: <span class="hljs-number">0.98879766</span>, <span class="hljs-string">&#x27;index&#x27;</span>: <span class="hljs-number">14</span>, <span class="hljs-string">&#x27;word&#x27;</span>: <span class="hljs-string">&#x27;Face&#x27;</span>},
 {<span class="hljs-string">&#x27;entity&#x27;</span>: <span class="hljs-string">&#x27;I-LOC&#x27;</span>, <span class="hljs-string">&#x27;score&#x27;</span>: <span class="hljs-number">0.99321055</span>, <span class="hljs-string">&#x27;index&#x27;</span>: <span class="hljs-number">16</span>, <span class="hljs-string">&#x27;word&#x27;</span>: <span class="hljs-string">&#x27;Brooklyn&#x27;</span>}]`}}),Rs=new $({props:{code:`inputs_with_offsets = tokenizer(example, return_offsets_mapping=True)
inputs_with_offsets["offset_mapping"]`,highlighted:`inputs_with_offsets = tokenizer(example, return_offsets_mapping=<span class="hljs-literal">True</span>)
inputs_with_offsets[<span class="hljs-string">&quot;offset_mapping&quot;</span>]`}}),Bs=new $({props:{code:`[(0, 0), (0, 2), (3, 7), (8, 10), (11, 12), (12, 14), (14, 16), (16, 18), (19, 22), (23, 24), (25, 29), (30, 32),
 (33, 35), (35, 40), (41, 45), (46, 48), (49, 57), (57, 58), (0, 0)]`,highlighted:`[(<span class="hljs-number">0</span>, <span class="hljs-number">0</span>), (<span class="hljs-number">0</span>, <span class="hljs-number">2</span>), (<span class="hljs-number">3</span>, <span class="hljs-number">7</span>), (<span class="hljs-number">8</span>, <span class="hljs-number">10</span>), (<span class="hljs-number">11</span>, <span class="hljs-number">12</span>), (<span class="hljs-number">12</span>, <span class="hljs-number">14</span>), (<span class="hljs-number">14</span>, <span class="hljs-number">16</span>), (<span class="hljs-number">16</span>, <span class="hljs-number">18</span>), (<span class="hljs-number">19</span>, <span class="hljs-number">22</span>), (<span class="hljs-number">23</span>, <span class="hljs-number">24</span>), (<span class="hljs-number">25</span>, <span class="hljs-number">29</span>), (<span class="hljs-number">30</span>, <span class="hljs-number">32</span>),
 (<span class="hljs-number">33</span>, <span class="hljs-number">35</span>), (<span class="hljs-number">35</span>, <span class="hljs-number">40</span>), (<span class="hljs-number">41</span>, <span class="hljs-number">45</span>), (<span class="hljs-number">46</span>, <span class="hljs-number">48</span>), (<span class="hljs-number">49</span>, <span class="hljs-number">57</span>), (<span class="hljs-number">57</span>, <span class="hljs-number">58</span>), (<span class="hljs-number">0</span>, <span class="hljs-number">0</span>)]`}}),As=new $({props:{code:"example[12:14]",highlighted:'example[<span class="hljs-number">12</span>:<span class="hljs-number">14</span>]'}}),Xs=new $({props:{code:"yl",highlighted:"yl"}}),Hs=new $({props:{code:`results = []
inputs_with_offsets = tokenizer(example, return_offsets_mapping=True)
tokens = inputs_with_offsets.tokens()
offsets = inputs_with_offsets["offset_mapping"]

for idx, pred in enumerate(predictions):
    label = model.config.id2label[pred]
    if label != "O":
        start, end = offsets[idx]
        results.append(
            {
                "entity": label,
                "score": probabilities[idx][pred],
                "word": tokens[idx],
                "start": start,
                "end": end,
            }
        )

print(results)`,highlighted:`results = []
inputs_with_offsets = tokenizer(example, return_offsets_mapping=<span class="hljs-literal">True</span>)
tokens = inputs_with_offsets.tokens()
offsets = inputs_with_offsets[<span class="hljs-string">&quot;offset_mapping&quot;</span>]

<span class="hljs-keyword">for</span> idx, pred <span class="hljs-keyword">in</span> <span class="hljs-built_in">enumerate</span>(predictions):
    label = model.config.id2label[pred]
    <span class="hljs-keyword">if</span> label != <span class="hljs-string">&quot;O&quot;</span>:
        start, end = offsets[idx]
        results.append(
            {
                <span class="hljs-string">&quot;entity&quot;</span>: label,
                <span class="hljs-string">&quot;score&quot;</span>: probabilities[idx][pred],
                <span class="hljs-string">&quot;word&quot;</span>: tokens[idx],
                <span class="hljs-string">&quot;start&quot;</span>: start,
                <span class="hljs-string">&quot;end&quot;</span>: end,
            }
        )

<span class="hljs-built_in">print</span>(results)`}}),Gs=new $({props:{code:`[{'entity': 'I-PER', 'score': 0.9993828, 'index': 4, 'word': 'S', 'start': 11, 'end': 12},
 {'entity': 'I-PER', 'score': 0.99815476, 'index': 5, 'word': '##yl', 'start': 12, 'end': 14},
 {'entity': 'I-PER', 'score': 0.99590725, 'index': 6, 'word': '##va', 'start': 14, 'end': 16},
 {'entity': 'I-PER', 'score': 0.9992327, 'index': 7, 'word': '##in', 'start': 16, 'end': 18},
 {'entity': 'I-ORG', 'score': 0.97389334, 'index': 12, 'word': 'Hu', 'start': 33, 'end': 35},
 {'entity': 'I-ORG', 'score': 0.976115, 'index': 13, 'word': '##gging', 'start': 35, 'end': 40},
 {'entity': 'I-ORG', 'score': 0.98879766, 'index': 14, 'word': 'Face', 'start': 41, 'end': 45},
 {'entity': 'I-LOC', 'score': 0.99321055, 'index': 16, 'word': 'Brooklyn', 'start': 49, 'end': 57}]`,highlighted:`[{<span class="hljs-string">&#x27;entity&#x27;</span>: <span class="hljs-string">&#x27;I-PER&#x27;</span>, <span class="hljs-string">&#x27;score&#x27;</span>: <span class="hljs-number">0.9993828</span>, <span class="hljs-string">&#x27;index&#x27;</span>: <span class="hljs-number">4</span>, <span class="hljs-string">&#x27;word&#x27;</span>: <span class="hljs-string">&#x27;S&#x27;</span>, <span class="hljs-string">&#x27;start&#x27;</span>: <span class="hljs-number">11</span>, <span class="hljs-string">&#x27;end&#x27;</span>: <span class="hljs-number">12</span>},
 {<span class="hljs-string">&#x27;entity&#x27;</span>: <span class="hljs-string">&#x27;I-PER&#x27;</span>, <span class="hljs-string">&#x27;score&#x27;</span>: <span class="hljs-number">0.99815476</span>, <span class="hljs-string">&#x27;index&#x27;</span>: <span class="hljs-number">5</span>, <span class="hljs-string">&#x27;word&#x27;</span>: <span class="hljs-string">&#x27;##yl&#x27;</span>, <span class="hljs-string">&#x27;start&#x27;</span>: <span class="hljs-number">12</span>, <span class="hljs-string">&#x27;end&#x27;</span>: <span class="hljs-number">14</span>},
 {<span class="hljs-string">&#x27;entity&#x27;</span>: <span class="hljs-string">&#x27;I-PER&#x27;</span>, <span class="hljs-string">&#x27;score&#x27;</span>: <span class="hljs-number">0.99590725</span>, <span class="hljs-string">&#x27;index&#x27;</span>: <span class="hljs-number">6</span>, <span class="hljs-string">&#x27;word&#x27;</span>: <span class="hljs-string">&#x27;##va&#x27;</span>, <span class="hljs-string">&#x27;start&#x27;</span>: <span class="hljs-number">14</span>, <span class="hljs-string">&#x27;end&#x27;</span>: <span class="hljs-number">16</span>},
 {<span class="hljs-string">&#x27;entity&#x27;</span>: <span class="hljs-string">&#x27;I-PER&#x27;</span>, <span class="hljs-string">&#x27;score&#x27;</span>: <span class="hljs-number">0.9992327</span>, <span class="hljs-string">&#x27;index&#x27;</span>: <span class="hljs-number">7</span>, <span class="hljs-string">&#x27;word&#x27;</span>: <span class="hljs-string">&#x27;##in&#x27;</span>, <span class="hljs-string">&#x27;start&#x27;</span>: <span class="hljs-number">16</span>, <span class="hljs-string">&#x27;end&#x27;</span>: <span class="hljs-number">18</span>},
 {<span class="hljs-string">&#x27;entity&#x27;</span>: <span class="hljs-string">&#x27;I-ORG&#x27;</span>, <span class="hljs-string">&#x27;score&#x27;</span>: <span class="hljs-number">0.97389334</span>, <span class="hljs-string">&#x27;index&#x27;</span>: <span class="hljs-number">12</span>, <span class="hljs-string">&#x27;word&#x27;</span>: <span class="hljs-string">&#x27;Hu&#x27;</span>, <span class="hljs-string">&#x27;start&#x27;</span>: <span class="hljs-number">33</span>, <span class="hljs-string">&#x27;end&#x27;</span>: <span class="hljs-number">35</span>},
 {<span class="hljs-string">&#x27;entity&#x27;</span>: <span class="hljs-string">&#x27;I-ORG&#x27;</span>, <span class="hljs-string">&#x27;score&#x27;</span>: <span class="hljs-number">0.976115</span>, <span class="hljs-string">&#x27;index&#x27;</span>: <span class="hljs-number">13</span>, <span class="hljs-string">&#x27;word&#x27;</span>: <span class="hljs-string">&#x27;##gging&#x27;</span>, <span class="hljs-string">&#x27;start&#x27;</span>: <span class="hljs-number">35</span>, <span class="hljs-string">&#x27;end&#x27;</span>: <span class="hljs-number">40</span>},
 {<span class="hljs-string">&#x27;entity&#x27;</span>: <span class="hljs-string">&#x27;I-ORG&#x27;</span>, <span class="hljs-string">&#x27;score&#x27;</span>: <span class="hljs-number">0.98879766</span>, <span class="hljs-string">&#x27;index&#x27;</span>: <span class="hljs-number">14</span>, <span class="hljs-string">&#x27;word&#x27;</span>: <span class="hljs-string">&#x27;Face&#x27;</span>, <span class="hljs-string">&#x27;start&#x27;</span>: <span class="hljs-number">41</span>, <span class="hljs-string">&#x27;end&#x27;</span>: <span class="hljs-number">45</span>},
 {<span class="hljs-string">&#x27;entity&#x27;</span>: <span class="hljs-string">&#x27;I-LOC&#x27;</span>, <span class="hljs-string">&#x27;score&#x27;</span>: <span class="hljs-number">0.99321055</span>, <span class="hljs-string">&#x27;index&#x27;</span>: <span class="hljs-number">16</span>, <span class="hljs-string">&#x27;word&#x27;</span>: <span class="hljs-string">&#x27;Brooklyn&#x27;</span>, <span class="hljs-string">&#x27;start&#x27;</span>: <span class="hljs-number">49</span>, <span class="hljs-string">&#x27;end&#x27;</span>: <span class="hljs-number">57</span>}]`}}),Ms=new Pt({}),Ns=new $({props:{code:"example[33:45]",highlighted:'example[<span class="hljs-number">33</span>:<span class="hljs-number">45</span>]'}}),Ws=new $({props:{code:"Hugging Face",highlighted:"Hugging Face"}}),Ls=new $({props:{code:`import numpy as np

results = []
inputs_with_offsets = tokenizer(example, return_offsets_mapping=True)
tokens = inputs_with_offsets.tokens()
offsets = inputs_with_offsets["offset_mapping"]

idx = 0
while idx < len(predictions):
    pred = predictions[idx]
    label = model.config.id2label[pred]
    if label != "O":
        # Remove the B- or I-
        label = label[2:]
        start, _ = offsets[idx]

        # Grab all the tokens labeled with I-label
        all_scores = []
        while (
            idx < len(predictions)
            and model.config.id2label[predictions[idx]] == f"I-{label}"
        ):
            all_scores.append(probabilities[idx][pred])
            _, end = offsets[idx]
            idx += 1

        # The score is the mean of all the scores of the tokens in that grouped entity
        score = np.mean(all_scores).item()
        word = example[start:end]
        results.append(
            {
                "entity_group": label,
                "score": score,
                "word": word,
                "start": start,
                "end": end,
            }
        )
    idx += 1

print(results)`,highlighted:`<span class="hljs-keyword">import</span> numpy <span class="hljs-keyword">as</span> np

results = []
inputs_with_offsets = tokenizer(example, return_offsets_mapping=<span class="hljs-literal">True</span>)
tokens = inputs_with_offsets.tokens()
offsets = inputs_with_offsets[<span class="hljs-string">&quot;offset_mapping&quot;</span>]

idx = <span class="hljs-number">0</span>
<span class="hljs-keyword">while</span> idx &lt; <span class="hljs-built_in">len</span>(predictions):
    pred = predictions[idx]
    label = model.config.id2label[pred]
    <span class="hljs-keyword">if</span> label != <span class="hljs-string">&quot;O&quot;</span>:
        <span class="hljs-comment"># Remove the B- or I-</span>
        label = label[<span class="hljs-number">2</span>:]
        start, _ = offsets[idx]

        <span class="hljs-comment"># Grab all the tokens labeled with I-label</span>
        all_scores = []
        <span class="hljs-keyword">while</span> (
            idx &lt; <span class="hljs-built_in">len</span>(predictions)
            <span class="hljs-keyword">and</span> model.config.id2label[predictions[idx]] == <span class="hljs-string">f&quot;I-<span class="hljs-subst">{label}</span>&quot;</span>
        ):
            all_scores.append(probabilities[idx][pred])
            _, end = offsets[idx]
            idx += <span class="hljs-number">1</span>

        <span class="hljs-comment"># The score is the mean of all the scores of the tokens in that grouped entity</span>
        score = np.mean(all_scores).item()
        word = example[start:end]
        results.append(
            {
                <span class="hljs-string">&quot;entity_group&quot;</span>: label,
                <span class="hljs-string">&quot;score&quot;</span>: score,
                <span class="hljs-string">&quot;word&quot;</span>: word,
                <span class="hljs-string">&quot;start&quot;</span>: start,
                <span class="hljs-string">&quot;end&quot;</span>: end,
            }
        )
    idx += <span class="hljs-number">1</span>

<span class="hljs-built_in">print</span>(results)`}}),Us=new $({props:{code:`[{'entity_group': 'PER', 'score': 0.9981694, 'word': 'Sylvain', 'start': 11, 'end': 18},
 {'entity_group': 'ORG', 'score': 0.97960204, 'word': 'Hugging Face', 'start': 33, 'end': 45},
 {'entity_group': 'LOC', 'score': 0.99321055, 'word': 'Brooklyn', 'start': 49, 'end': 57}]`,highlighted:`[{<span class="hljs-string">&#x27;entity_group&#x27;</span>: <span class="hljs-string">&#x27;PER&#x27;</span>, <span class="hljs-string">&#x27;score&#x27;</span>: <span class="hljs-number">0.9981694</span>, <span class="hljs-string">&#x27;word&#x27;</span>: <span class="hljs-string">&#x27;Sylvain&#x27;</span>, <span class="hljs-string">&#x27;start&#x27;</span>: <span class="hljs-number">11</span>, <span class="hljs-string">&#x27;end&#x27;</span>: <span class="hljs-number">18</span>},
 {<span class="hljs-string">&#x27;entity_group&#x27;</span>: <span class="hljs-string">&#x27;ORG&#x27;</span>, <span class="hljs-string">&#x27;score&#x27;</span>: <span class="hljs-number">0.97960204</span>, <span class="hljs-string">&#x27;word&#x27;</span>: <span class="hljs-string">&#x27;Hugging Face&#x27;</span>, <span class="hljs-string">&#x27;start&#x27;</span>: <span class="hljs-number">33</span>, <span class="hljs-string">&#x27;end&#x27;</span>: <span class="hljs-number">45</span>},
 {<span class="hljs-string">&#x27;entity_group&#x27;</span>: <span class="hljs-string">&#x27;LOC&#x27;</span>, <span class="hljs-string">&#x27;score&#x27;</span>: <span class="hljs-number">0.99321055</span>, <span class="hljs-string">&#x27;word&#x27;</span>: <span class="hljs-string">&#x27;Brooklyn&#x27;</span>, <span class="hljs-string">&#x27;start&#x27;</span>: <span class="hljs-number">49</span>, <span class="hljs-string">&#x27;end&#x27;</span>: <span class="hljs-number">57</span>}]`}}),{c(){f=l("meta"),x=c(),w(d.$$.fragment),y=c(),P=l("h1"),E=l("a"),F=l("span"),w(A.$$.fragment),M=c(),S=l("span"),Y=n("Fast tokenizers' special powers"),I=c(),z.c(),T=c(),O=l("p"),X=n("In this section we will take a closer look at the capabilities of the tokenizers in \u{1F917} Transformers. Up to now we have only used them to tokenize inputs or decode IDs back into text, but tokenizers \u2014 especially those backed by the \u{1F917} Tokenizers library \u2014 can do a lot more. To illustrate these additional features, we will explore how to reproduce the results of the "),q=l("code"),V=n("token-classification"),H=n(" (that we called "),Z=l("code"),u=n("ner"),v=n(") and "),de=l("code"),Ee=n("question-answering"),le=n(" pipelines that we first encountered in "),re=l("a"),_o=n("Chapter 1"),yo=n("."),Ca=c(),w(ls.$$.fragment),za=c(),qe=l("p"),$o=n("In the following discussion, we will often make the distinction between \u201Cslow\u201D and \u201Cfast\u201D tokenizers. Slow tokenizers are those written in Python inside the \u{1F917} Transformers library, while the fast versions are the ones provided by \u{1F917} Tokenizers, which are written in Rust. If you remember the table from "),Zs=l("a"),vo=n("Chapter 5"),Eo=n(" that reported how long it took a fast and a slow tokenizer to tokenize the Drug Review Dataset, you should have an idea of why we call them fast and slow:"),Ta=c(),Re=l("table"),St=l("thead"),Oe=l("tr"),et=l("th"),Oo=n("Fast tokenizer"),Io=c(),st=l("th"),Co=n("Slow tokenizer"),zo=c(),Ft=l("th"),To=c(),os=l("tbody"),Ie=l("tr"),tt=l("td"),qt=l("code"),Do=n("batched=True"),Po=c(),nt=l("td"),So=n("10.8s"),Fo=c(),at=l("td"),qo=n("4min41s"),Ro=c(),Ce=l("tr"),lt=l("td"),Rt=l("code"),Bo=n("batched=False"),Ao=c(),ot=l("td"),Xo=n("59.2s"),Ho=c(),rt=l("td"),Go=n("5min3s"),Da=c(),w(Be.$$.fragment),Pa=c(),ze=l("h2"),Ae=l("a"),Bt=l("span"),w(rs.$$.fragment),Mo=c(),At=l("span"),No=n("Batch encoding"),Sa=c(),w(is.$$.fragment),Fa=c(),Xe=l("p"),Wo=n("The output of a tokenizer isn\u2019t a simple Python dictionary; what we get is actually a special "),Xt=l("code"),Lo=n("BatchEncoding"),Uo=n(" object. It\u2019s a subclass of a dictionary (which is why we were able to index into that result without any problem before), but with additional methods that are mostly used by fast tokenizers."),qa=c(),He=l("p"),Yo=n("Besides their parallelization capabilities, the key functionality of fast tokenizers is that they always keep track of the original span of texts the final tokens come from \u2014 a feature we call "),Ht=l("em"),Vo=n("offset mapping"),Qo=n(". This in turn unlocks features like mapping each word to the tokens it generated or mapping each character of the original text to the token it\u2019s inside, and vice versa."),Ra=c(),it=l("p"),Jo=n("Let\u2019s take a look at an example:"),Ba=c(),w(ps.$$.fragment),Aa=c(),Ge=l("p"),Ko=n("As mentioned previously, we get a "),Gt=l("code"),Zo=n("BatchEncoding"),er=n(" object in the tokenizer\u2019s output:"),Xa=c(),w(cs.$$.fragment),Ha=c(),ee=l("p"),sr=n("Since the "),Mt=l("code"),tr=n("AutoTokenizer"),nr=n(" class picks a fast tokenizer by default, we can use the additional methods this "),Nt=l("code"),ar=n("BatchEncoding"),lr=n(" object provides. We have two ways to check if our tokenizer is a fast or a slow one. We can either check the attribute "),Wt=l("code"),or=n("is_fast"),rr=n(" of the "),Lt=l("code"),ir=n("tokenizer"),pr=n(":"),Ga=c(),w(hs.$$.fragment),Ma=c(),w(fs.$$.fragment),Na=c(),Me=l("p"),cr=n("or check the same attribute of our "),Ut=l("code"),hr=n("encoding"),fr=n(":"),Wa=c(),w(ds.$$.fragment),La=c(),w(us.$$.fragment),Ua=c(),pt=l("p"),dr=n("Let\u2019s see what a fast tokenizer enables us to do. First, we can access the tokens without having to convert the IDs back to tokens:"),Ya=c(),w(ms.$$.fragment),Va=c(),w(gs.$$.fragment),Qa=c(),je=l("p"),ur=n("In this case the token at index 5 is "),Yt=l("code"),mr=n("##yl"),gr=n(", which is part of the word \u201CSylvain\u201D in the original sentence. We can also use the "),Vt=l("code"),xr=n("word_ids()"),wr=n(" method to get the index of the word each token comes from:"),Ja=c(),w(xs.$$.fragment),Ka=c(),w(ws.$$.fragment),Za=c(),Q=l("p"),br=n("We can see that the tokenizer\u2019s special tokens "),Qt=l("code"),jr=n("[CLS]"),kr=n(" and "),Jt=l("code"),_r=n("[SEP]"),yr=n(" are mapped to "),Kt=l("code"),$r=n("None"),vr=n(", and then each token is mapped to the word it originates from. This is especially useful to determine if a token is at the start of a word or if two tokens are in the same word. We could rely on the "),Zt=l("code"),Er=n("##"),Or=n(" prefix for that, but it only works for BERT-like tokenizers; this method works for any type of tokenizer as long as it\u2019s a fast one. In the next chapter, we\u2019ll see how we can use this capability to apply the labels we have for each word properly to the tokens in tasks like named entity recognition (NER) and part-of-speech (POS) tagging. We can also use it to mask all the tokens coming from the same word in masked language modeling (a technique called "),en=l("em"),Ir=n("whole word masking"),Cr=n(")."),el=c(),w(Ne.$$.fragment),sl=c(),ke=l("p"),zr=n("Similarly, there is a "),sn=l("code"),Tr=n("sentence_ids()"),Dr=n(" method that we can use to map a token to the sentence it came from (though in this case, the "),tn=l("code"),Pr=n("token_type_ids"),Sr=n(" returned by the tokenizer can give us the same information)."),tl=c(),N=l("p"),Fr=n("Lastly, we can map any word or token to characters in the original text, and vice versa, via the "),nn=l("code"),qr=n("word_to_chars()"),Rr=n(" or "),an=l("code"),Br=n("token_to_chars()"),Ar=n(" and "),ln=l("code"),Xr=n("char_to_word()"),Hr=n(" or "),on=l("code"),Gr=n("char_to_token()"),Mr=n(" methods. For instance, the "),rn=l("code"),Nr=n("word_ids()"),Wr=n(" method told us that "),pn=l("code"),Lr=n("##yl"),Ur=n(" is part of the word at index 3, but which word is it in the sentence? We can find out like this:"),nl=c(),w(bs.$$.fragment),al=c(),w(js.$$.fragment),ll=c(),_e=l("p"),Yr=n("As we mentioned previously, this is all powered by the fact the fast tokenizer keeps track of the span of text each token comes from in a list of "),cn=l("em"),Vr=n("offsets"),Qr=n(". To illustrate their use, next we\u2019ll show you how to replicate the results of the "),hn=l("code"),Jr=n("token-classification"),Kr=n(" pipeline manually."),ol=c(),w(We.$$.fragment),rl=c(),Te=l("h2"),Le=l("a"),fn=l("span"),w(ks.$$.fragment),Zr=c(),_s=l("span"),ei=n("Inside the "),dn=l("code"),si=n("token-classification"),ti=n(" pipeline"),il=c(),se=l("p"),ni=n("In "),ct=l("a"),ai=n("Chapter 1"),li=n(" we got our first taste of applying NER \u2014 where the task is to identify which parts of the text correspond to entities like persons, locations, or organizations \u2014 with the \u{1F917} Transformers "),un=l("code"),oi=n("pipeline()"),ri=n(" function. Then, in "),ht=l("a"),ii=n("Chapter 2"),pi=n(", we saw how a pipeline groups together the three stages necessary to get the predictions from a raw text: tokenization, passing the inputs through the model, and post-processing. The first two steps in the "),mn=l("code"),ci=n("token-classification"),hi=n(" pipeline are the same as in any other pipeline, but the post-processing is a little more complex \u2014 let\u2019s see how!"),pl=c(),me.c(),ft=c(),De=l("h3"),Ue=l("a"),gn=l("span"),w(ys.$$.fragment),fi=c(),xn=l("span"),di=n("Getting the base results with the pipeline"),cl=c(),Ye=l("p"),ui=n("First, let\u2019s grab a token classification pipeline so we can get some results to compare manually. The model used by default is "),$s=l("a"),wn=l("code"),mi=n("dbmdz/bert-large-cased-finetuned-conll03-english"),gi=n("; it performs NER on sentences:"),hl=c(),w(vs.$$.fragment),fl=c(),w(Es.$$.fragment),dl=c(),dt=l("p"),xi=n("The model properly identified each token generated by \u201CSylvain\u201D as a person, each token generated by \u201CHugging Face\u201D as an organization, and the token \u201CBrooklyn\u201D as a location. We can also ask the pipeline to group together the tokens that correspond to the same entity:"),ul=c(),w(Os.$$.fragment),ml=c(),w(Is.$$.fragment),gl=c(),W=l("p"),wi=n("The "),bn=l("code"),bi=n("aggregation_strategy"),ji=n(" picked will change the scores computed for each grouped entity. With "),jn=l("code"),ki=n('"simple"'),_i=n(" the score is just the mean of the scores of each token in the given entity: for instance, the score of \u201CSylvain\u201D is the mean of the scores we saw in the previous example for the tokens "),kn=l("code"),yi=n("S"),$i=n(", "),_n=l("code"),vi=n("##yl"),Ei=n(", "),yn=l("code"),Oi=n("##va"),Ii=n(", and "),$n=l("code"),Ci=n("##in"),zi=n(". Other strategies available are:"),xl=c(),ye=l("ul"),Ve=l("li"),vn=l("code"),Ti=n('"first"'),Di=n(", where the score of each entity is the score of the first token of that entity (so for \u201CSylvain\u201D it would be 0.993828, the score of the token "),En=l("code"),Pi=n("S"),Si=n(")"),Fi=c(),ut=l("li"),On=l("code"),qi=n('"max"'),Ri=n(", where the score of each entity is the maximum score of the tokens in that entity (so for \u201CHugging Face\u201D it would be 0.98879766, the score of \u201CFace\u201D)"),Bi=c(),Qe=l("li"),In=l("code"),Ai=n('"average"'),Xi=n(", where the score of each entity is the average of the scores of the words composing that entity (so for \u201CSylvain\u201D there would be no difference from the "),Cn=l("code"),Hi=n('"simple"'),Gi=n(" strategy, but \u201CHugging Face\u201D would have a score of 0.9819, the average of the scores for \u201CHugging\u201D, 0.975, and \u201CFace\u201D, 0.98879)"),wl=c(),Je=l("p"),Mi=n("Now let\u2019s see how to obtain these results without using the "),zn=l("code"),Ni=n("pipeline()"),Wi=n(" function!"),bl=c(),Pe=l("h3"),Ke=l("a"),Tn=l("span"),w(Cs.$$.fragment),Li=c(),Dn=l("span"),Ui=n("From inputs to predictions"),jl=c(),xe.c(),mt=c(),gt=l("p"),Yi=n("We have a batch with 1 sequence of 19 tokens and the model has 9 different labels, so the output of the model has a shape of 1 x 19 x 9. Like for the text classification pipeline, we use a softmax function to convert those logits to probabilities, and we take the argmax to get predictions (note that we can take the argmax on the logits because the softmax does not change the order):"),kl=c(),be.c(),xt=c(),w(zs.$$.fragment),_l=c(),Ze=l("p"),Vi=n("The "),Pn=l("code"),Qi=n("model.config.id2label"),Ji=n(" attribute contains the mapping of indexes to labels that we can use to make sense of the predictions:"),yl=c(),w(Ts.$$.fragment),$l=c(),w(Ds.$$.fragment),vl=c(),C=l("p"),Ki=n("As we saw earlier, there are 9 labels: "),Sn=l("code"),Zi=n("O"),ep=n(" is the label for the tokens that are not in any named entity (it stands for \u201Coutside\u201D), and we then have two labels for each type of entity (miscellaneous, person, organization, and location). The label "),Fn=l("code"),sp=n("B-XXX"),tp=n(" indicates the token is at the beginning of an entity "),qn=l("code"),np=n("XXX"),ap=n(" and the label "),Rn=l("code"),lp=n("I-XXX"),op=n(" indicates the token is inside the entity "),Bn=l("code"),rp=n("XXX"),ip=n(". For instance, in the current example we would expect our model to classify the token "),An=l("code"),pp=n("S"),cp=n(" as "),Xn=l("code"),hp=n("B-PER"),fp=n(" (beginning of a person entity) and the tokens "),Hn=l("code"),dp=n("##yl"),up=n(", "),Gn=l("code"),mp=n("##va"),gp=n(" and "),Mn=l("code"),xp=n("##in"),wp=n(" as "),Nn=l("code"),bp=n("I-PER"),jp=n(" (inside a person entity)."),El=c(),G=l("p"),kp=n("You might think the model was wrong in this case as it gave the label "),Wn=l("code"),_p=n("I-PER"),yp=n(" to all four of these tokens, but that\u2019s not entirely true. There are actually two formats for those "),Ln=l("code"),$p=n("B-"),vp=n(" and "),Un=l("code"),Ep=n("I-"),Op=n(" labels: "),Yn=l("em"),Ip=n("IOB1"),Cp=n(" and "),Vn=l("em"),zp=n("IOB2"),Tp=n(". The IOB2 format (in pink below), is the one we introduced whereas in the IOB1 format (in blue), the labels beginning with "),Qn=l("code"),Dp=n("B-"),Pp=n(" are only ever used to separate two adjacent entities of the same type. The model we are using was fine-tuned on a dataset using that format, which is why it assigns the label "),Jn=l("code"),Sp=n("I-PER"),Fp=n(" to the "),Kn=l("code"),qp=n("S"),Rp=n(" token."),Ol=c(),Se=l("div"),Ps=l("img"),Bp=c(),Ss=l("img"),Il=c(),es=l("p"),Ap=n("With this map, we are ready to reproduce (almost entirely) the results of the first pipeline \u2014 we can just grab the score and label of each token that was not classified as "),Zn=l("code"),Xp=n("O"),Hp=n(":"),Cl=c(),w(Fs.$$.fragment),zl=c(),w(qs.$$.fragment),Tl=c(),ie=l("p"),Gp=n("This is very similar to what we had before, with one exception: the pipeline also gave us information about the "),ea=l("code"),Mp=n("start"),Np=n(" and "),sa=l("code"),Wp=n("end"),Lp=n(" of each entity in the original sentence. This is where our offset mapping will come into play. To get the offsets, we just have to set "),ta=l("code"),Up=n("return_offsets_mapping=True"),Yp=n(" when we apply the tokenizer to our inputs:"),Dl=c(),w(Rs.$$.fragment),Pl=c(),w(Bs.$$.fragment),Sl=c(),pe=l("p"),Vp=n("Each tuple is the span of text corresponding to each token, where "),na=l("code"),Qp=n("(0, 0)"),Jp=n(" is reserved for the special tokens. We saw before that the token at index 5 is "),aa=l("code"),Kp=n("##yl"),Zp=n(", which has "),la=l("code"),ec=n("(12, 14)"),sc=n(" as offsets here. If we grab the corresponding slice in our example:"),Fl=c(),w(As.$$.fragment),ql=c(),ss=l("p"),tc=n("we get the proper span of text without the "),oa=l("code"),nc=n("##"),ac=n(":"),Rl=c(),w(Xs.$$.fragment),Bl=c(),wt=l("p"),lc=n("Using this, we can now complete the previous results:"),Al=c(),w(Hs.$$.fragment),Xl=c(),w(Gs.$$.fragment),Hl=c(),bt=l("p"),oc=n("This is the same as what we got from the first pipeline!"),Gl=c(),Fe=l("h3"),ts=l("a"),ra=l("span"),w(Ms.$$.fragment),rc=c(),ia=l("span"),ic=n("Grouping entities"),Ml=c(),L=l("p"),pc=n("Using the offsets to determine the start and end keys for each entity is handy, but that information isn\u2019t strictly necessary. When we want to group the entities together, however, the offsets will save us a lot of messy code. For example, if we wanted to group together the tokens "),pa=l("code"),cc=n("Hu"),hc=n(", "),ca=l("code"),fc=n("##gging"),dc=n(", and "),ha=l("code"),uc=n("Face"),mc=n(", we could make special rules that say the first two should be attached while removing the "),fa=l("code"),gc=n("##"),xc=n(", and the "),da=l("code"),wc=n("Face"),bc=n(" should be added with a space since it does not begin with "),ua=l("code"),jc=n("##"),kc=n(" \u2014 but that would only work for this particular type of tokenizer. We would have to write another set of rules for a SentencePiece or a Byte-Pair-Encoding tokenizer (discussed later in this chapter)."),Nl=c(),J=l("p"),_c=n("With the offsets, all that custom code goes away: we just can take the span in the original text that begins with the first token and ends with the last token. So, in the case of the tokens "),ma=l("code"),yc=n("Hu"),$c=n(", "),ga=l("code"),vc=n("##gging"),Ec=n(", and "),xa=l("code"),Oc=n("Face"),Ic=n(", we should start at character 33 (the beginning of "),wa=l("code"),Cc=n("Hu"),zc=n(") and end before character 45 (the end of "),ba=l("code"),Tc=n("Face"),Dc=n("):"),Wl=c(),w(Ns.$$.fragment),Ll=c(),w(Ws.$$.fragment),Ul=c(),K=l("p"),Pc=n("To write the code that post-processes the predictions while grouping entities, we will group together entities that are consecutive and labeled with "),ja=l("code"),Sc=n("I-XXX"),Fc=n(", except for the first one, which can be labeled as "),ka=l("code"),qc=n("B-XXX"),Rc=n(" or "),_a=l("code"),Bc=n("I-XXX"),Ac=n(" (so, we stop grouping an entity when we get a "),ya=l("code"),Xc=n("O"),Hc=n(", a new type of entity, or a "),$a=l("code"),Gc=n("B-XXX"),Mc=n(" that tells us an entity of the same type is starting):"),Yl=c(),w(Ls.$$.fragment),Vl=c(),jt=l("p"),Nc=n("And we get the same results as with our second pipeline!"),Ql=c(),w(Us.$$.fragment),Jl=c(),kt=l("p"),Wc=n("Another example of a task where these offsets are extremely useful is question answering. Diving into that pipeline, which we\u2019ll do in the next section, will also enable us to take a look at one last feature of the tokenizers in the \u{1F917} Transformers library: dealing with overflowing tokens when we truncate an input to a given length."),this.h()},l(e){const i=vd('[data-svelte="svelte-1phssyn"]',document.head);f=o(i,"META",{name:!0,content:!0}),i.forEach(t),x=h(e),b(d.$$.fragment,e),y=h(e),P=o(e,"H1",{class:!0});var Ks=r(P);E=o(Ks,"A",{id:!0,class:!0,href:!0});var _t=r(E);F=o(_t,"SPAN",{});var va=r(F);b(A.$$.fragment,va),va.forEach(t),_t.forEach(t),M=h(Ks),S=o(Ks,"SPAN",{});var Ea=r(S);Y=a(Ea,"Fast tokenizers' special powers"),Ea.forEach(t),Ks.forEach(t),I=h(e),z.l(e),T=h(e),O=o(e,"P",{});var oe=r(O);X=a(oe,"In this section we will take a closer look at the capabilities of the tokenizers in \u{1F917} Transformers. Up to now we have only used them to tokenize inputs or decode IDs back into text, but tokenizers \u2014 especially those backed by the \u{1F917} Tokenizers library \u2014 can do a lot more. To illustrate these additional features, we will explore how to reproduce the results of the "),q=o(oe,"CODE",{});var yt=r(q);V=a(yt,"token-classification"),yt.forEach(t),H=a(oe," (that we called "),Z=o(oe,"CODE",{});var $t=r(Z);u=a($t,"ner"),$t.forEach(t),v=a(oe,") and "),de=o(oe,"CODE",{});var vt=r(de);Ee=a(vt,"question-answering"),vt.forEach(t),le=a(oe," pipelines that we first encountered in "),re=o(oe,"A",{href:!0});var ah=r(re);_o=a(ah,"Chapter 1"),ah.forEach(t),yo=a(oe,"."),oe.forEach(t),Ca=h(e),b(ls.$$.fragment,e),za=h(e),qe=o(e,"P",{});var Zl=r(qe);$o=a(Zl,"In the following discussion, we will often make the distinction between \u201Cslow\u201D and \u201Cfast\u201D tokenizers. Slow tokenizers are those written in Python inside the \u{1F917} Transformers library, while the fast versions are the ones provided by \u{1F917} Tokenizers, which are written in Rust. If you remember the table from "),Zs=o(Zl,"A",{href:!0});var lh=r(Zs);vo=a(lh,"Chapter 5"),lh.forEach(t),Eo=a(Zl," that reported how long it took a fast and a slow tokenizer to tokenize the Drug Review Dataset, you should have an idea of why we call them fast and slow:"),Zl.forEach(t),Ta=h(e),Re=o(e,"TABLE",{});var eo=r(Re);St=o(eo,"THEAD",{});var oh=r(St);Oe=o(oh,"TR",{});var Et=r(Oe);et=o(Et,"TH",{align:!0});var rh=r(et);Oo=a(rh,"Fast tokenizer"),rh.forEach(t),Io=h(Et),st=o(Et,"TH",{align:!0});var ih=r(st);Co=a(ih,"Slow tokenizer"),ih.forEach(t),zo=h(Et),Ft=o(Et,"TH",{align:!0}),r(Ft).forEach(t),Et.forEach(t),oh.forEach(t),To=h(eo),os=o(eo,"TBODY",{});var so=r(os);Ie=o(so,"TR",{});var Ot=r(Ie);tt=o(Ot,"TD",{align:!0});var ph=r(tt);qt=o(ph,"CODE",{});var ch=r(qt);Do=a(ch,"batched=True"),ch.forEach(t),ph.forEach(t),Po=h(Ot),nt=o(Ot,"TD",{align:!0});var hh=r(nt);So=a(hh,"10.8s"),hh.forEach(t),Fo=h(Ot),at=o(Ot,"TD",{align:!0});var fh=r(at);qo=a(fh,"4min41s"),fh.forEach(t),Ot.forEach(t),Ro=h(so),Ce=o(so,"TR",{});var It=r(Ce);lt=o(It,"TD",{align:!0});var dh=r(lt);Rt=o(dh,"CODE",{});var uh=r(Rt);Bo=a(uh,"batched=False"),uh.forEach(t),dh.forEach(t),Ao=h(It),ot=o(It,"TD",{align:!0});var mh=r(ot);Xo=a(mh,"59.2s"),mh.forEach(t),Ho=h(It),rt=o(It,"TD",{align:!0});var gh=r(rt);Go=a(gh,"5min3s"),gh.forEach(t),It.forEach(t),so.forEach(t),eo.forEach(t),Da=h(e),b(Be.$$.fragment,e),Pa=h(e),ze=o(e,"H2",{class:!0});var to=r(ze);Ae=o(to,"A",{id:!0,class:!0,href:!0});var xh=r(Ae);Bt=o(xh,"SPAN",{});var wh=r(Bt);b(rs.$$.fragment,wh),wh.forEach(t),xh.forEach(t),Mo=h(to),At=o(to,"SPAN",{});var bh=r(At);No=a(bh,"Batch encoding"),bh.forEach(t),to.forEach(t),Sa=h(e),b(is.$$.fragment,e),Fa=h(e),Xe=o(e,"P",{});var no=r(Xe);Wo=a(no,"The output of a tokenizer isn\u2019t a simple Python dictionary; what we get is actually a special "),Xt=o(no,"CODE",{});var jh=r(Xt);Lo=a(jh,"BatchEncoding"),jh.forEach(t),Uo=a(no," object. It\u2019s a subclass of a dictionary (which is why we were able to index into that result without any problem before), but with additional methods that are mostly used by fast tokenizers."),no.forEach(t),qa=h(e),He=o(e,"P",{});var ao=r(He);Yo=a(ao,"Besides their parallelization capabilities, the key functionality of fast tokenizers is that they always keep track of the original span of texts the final tokens come from \u2014 a feature we call "),Ht=o(ao,"EM",{});var kh=r(Ht);Vo=a(kh,"offset mapping"),kh.forEach(t),Qo=a(ao,". This in turn unlocks features like mapping each word to the tokens it generated or mapping each character of the original text to the token it\u2019s inside, and vice versa."),ao.forEach(t),Ra=h(e),it=o(e,"P",{});var _h=r(it);Jo=a(_h,"Let\u2019s take a look at an example:"),_h.forEach(t),Ba=h(e),b(ps.$$.fragment,e),Aa=h(e),Ge=o(e,"P",{});var lo=r(Ge);Ko=a(lo,"As mentioned previously, we get a "),Gt=o(lo,"CODE",{});var yh=r(Gt);Zo=a(yh,"BatchEncoding"),yh.forEach(t),er=a(lo," object in the tokenizer\u2019s output:"),lo.forEach(t),Xa=h(e),b(cs.$$.fragment,e),Ha=h(e),ee=o(e,"P",{});var $e=r(ee);sr=a($e,"Since the "),Mt=o($e,"CODE",{});var $h=r(Mt);tr=a($h,"AutoTokenizer"),$h.forEach(t),nr=a($e," class picks a fast tokenizer by default, we can use the additional methods this "),Nt=o($e,"CODE",{});var vh=r(Nt);ar=a(vh,"BatchEncoding"),vh.forEach(t),lr=a($e," object provides. We have two ways to check if our tokenizer is a fast or a slow one. We can either check the attribute "),Wt=o($e,"CODE",{});var Eh=r(Wt);or=a(Eh,"is_fast"),Eh.forEach(t),rr=a($e," of the "),Lt=o($e,"CODE",{});var Oh=r(Lt);ir=a(Oh,"tokenizer"),Oh.forEach(t),pr=a($e,":"),$e.forEach(t),Ga=h(e),b(hs.$$.fragment,e),Ma=h(e),b(fs.$$.fragment,e),Na=h(e),Me=o(e,"P",{});var oo=r(Me);cr=a(oo,"or check the same attribute of our "),Ut=o(oo,"CODE",{});var Ih=r(Ut);hr=a(Ih,"encoding"),Ih.forEach(t),fr=a(oo,":"),oo.forEach(t),Wa=h(e),b(ds.$$.fragment,e),La=h(e),b(us.$$.fragment,e),Ua=h(e),pt=o(e,"P",{});var Ch=r(pt);dr=a(Ch,"Let\u2019s see what a fast tokenizer enables us to do. First, we can access the tokens without having to convert the IDs back to tokens:"),Ch.forEach(t),Ya=h(e),b(ms.$$.fragment,e),Va=h(e),b(gs.$$.fragment,e),Qa=h(e),je=o(e,"P",{});var Ct=r(je);ur=a(Ct,"In this case the token at index 5 is "),Yt=o(Ct,"CODE",{});var zh=r(Yt);mr=a(zh,"##yl"),zh.forEach(t),gr=a(Ct,", which is part of the word \u201CSylvain\u201D in the original sentence. We can also use the "),Vt=o(Ct,"CODE",{});var Th=r(Vt);xr=a(Th,"word_ids()"),Th.forEach(t),wr=a(Ct," method to get the index of the word each token comes from:"),Ct.forEach(t),Ja=h(e),b(xs.$$.fragment,e),Ka=h(e),b(ws.$$.fragment,e),Za=h(e),Q=o(e,"P",{});var ce=r(Q);br=a(ce,"We can see that the tokenizer\u2019s special tokens "),Qt=o(ce,"CODE",{});var Dh=r(Qt);jr=a(Dh,"[CLS]"),Dh.forEach(t),kr=a(ce," and "),Jt=o(ce,"CODE",{});var Ph=r(Jt);_r=a(Ph,"[SEP]"),Ph.forEach(t),yr=a(ce," are mapped to "),Kt=o(ce,"CODE",{});var Sh=r(Kt);$r=a(Sh,"None"),Sh.forEach(t),vr=a(ce,", and then each token is mapped to the word it originates from. This is especially useful to determine if a token is at the start of a word or if two tokens are in the same word. We could rely on the "),Zt=o(ce,"CODE",{});var Fh=r(Zt);Er=a(Fh,"##"),Fh.forEach(t),Or=a(ce," prefix for that, but it only works for BERT-like tokenizers; this method works for any type of tokenizer as long as it\u2019s a fast one. In the next chapter, we\u2019ll see how we can use this capability to apply the labels we have for each word properly to the tokens in tasks like named entity recognition (NER) and part-of-speech (POS) tagging. We can also use it to mask all the tokens coming from the same word in masked language modeling (a technique called "),en=o(ce,"EM",{});var qh=r(en);Ir=a(qh,"whole word masking"),qh.forEach(t),Cr=a(ce,")."),ce.forEach(t),el=h(e),b(Ne.$$.fragment,e),sl=h(e),ke=o(e,"P",{});var zt=r(ke);zr=a(zt,"Similarly, there is a "),sn=o(zt,"CODE",{});var Rh=r(sn);Tr=a(Rh,"sentence_ids()"),Rh.forEach(t),Dr=a(zt," method that we can use to map a token to the sentence it came from (though in this case, the "),tn=o(zt,"CODE",{});var Bh=r(tn);Pr=a(Bh,"token_type_ids"),Bh.forEach(t),Sr=a(zt," returned by the tokenizer can give us the same information)."),zt.forEach(t),tl=h(e),N=o(e,"P",{});var te=r(N);Fr=a(te,"Lastly, we can map any word or token to characters in the original text, and vice versa, via the "),nn=o(te,"CODE",{});var Ah=r(nn);qr=a(Ah,"word_to_chars()"),Ah.forEach(t),Rr=a(te," or "),an=o(te,"CODE",{});var Xh=r(an);Br=a(Xh,"token_to_chars()"),Xh.forEach(t),Ar=a(te," and "),ln=o(te,"CODE",{});var Hh=r(ln);Xr=a(Hh,"char_to_word()"),Hh.forEach(t),Hr=a(te," or "),on=o(te,"CODE",{});var Gh=r(on);Gr=a(Gh,"char_to_token()"),Gh.forEach(t),Mr=a(te," methods. For instance, the "),rn=o(te,"CODE",{});var Mh=r(rn);Nr=a(Mh,"word_ids()"),Mh.forEach(t),Wr=a(te," method told us that "),pn=o(te,"CODE",{});var Nh=r(pn);Lr=a(Nh,"##yl"),Nh.forEach(t),Ur=a(te," is part of the word at index 3, but which word is it in the sentence? We can find out like this:"),te.forEach(t),nl=h(e),b(bs.$$.fragment,e),al=h(e),b(js.$$.fragment,e),ll=h(e),_e=o(e,"P",{});var Tt=r(_e);Yr=a(Tt,"As we mentioned previously, this is all powered by the fact the fast tokenizer keeps track of the span of text each token comes from in a list of "),cn=o(Tt,"EM",{});var Wh=r(cn);Vr=a(Wh,"offsets"),Wh.forEach(t),Qr=a(Tt,". To illustrate their use, next we\u2019ll show you how to replicate the results of the "),hn=o(Tt,"CODE",{});var Lh=r(hn);Jr=a(Lh,"token-classification"),Lh.forEach(t),Kr=a(Tt," pipeline manually."),Tt.forEach(t),ol=h(e),b(We.$$.fragment,e),rl=h(e),Te=o(e,"H2",{class:!0});var ro=r(Te);Le=o(ro,"A",{id:!0,class:!0,href:!0});var Uh=r(Le);fn=o(Uh,"SPAN",{});var Yh=r(fn);b(ks.$$.fragment,Yh),Yh.forEach(t),Uh.forEach(t),Zr=h(ro),_s=o(ro,"SPAN",{});var io=r(_s);ei=a(io,"Inside the "),dn=o(io,"CODE",{});var Vh=r(dn);si=a(Vh,"token-classification"),Vh.forEach(t),ti=a(io," pipeline"),io.forEach(t),ro.forEach(t),il=h(e),se=o(e,"P",{});var ve=r(se);ni=a(ve,"In "),ct=o(ve,"A",{href:!0});var Qh=r(ct);ai=a(Qh,"Chapter 1"),Qh.forEach(t),li=a(ve," we got our first taste of applying NER \u2014 where the task is to identify which parts of the text correspond to entities like persons, locations, or organizations \u2014 with the \u{1F917} Transformers "),un=o(ve,"CODE",{});var Jh=r(un);oi=a(Jh,"pipeline()"),Jh.forEach(t),ri=a(ve," function. Then, in "),ht=o(ve,"A",{href:!0});var Kh=r(ht);ii=a(Kh,"Chapter 2"),Kh.forEach(t),pi=a(ve,", we saw how a pipeline groups together the three stages necessary to get the predictions from a raw text: tokenization, passing the inputs through the model, and post-processing. The first two steps in the "),mn=o(ve,"CODE",{});var Zh=r(mn);ci=a(Zh,"token-classification"),Zh.forEach(t),hi=a(ve," pipeline are the same as in any other pipeline, but the post-processing is a little more complex \u2014 let\u2019s see how!"),ve.forEach(t),pl=h(e),me.l(e),ft=h(e),De=o(e,"H3",{class:!0});var po=r(De);Ue=o(po,"A",{id:!0,class:!0,href:!0});var ef=r(Ue);gn=o(ef,"SPAN",{});var sf=r(gn);b(ys.$$.fragment,sf),sf.forEach(t),ef.forEach(t),fi=h(po),xn=o(po,"SPAN",{});var tf=r(xn);di=a(tf,"Getting the base results with the pipeline"),tf.forEach(t),po.forEach(t),cl=h(e),Ye=o(e,"P",{});var co=r(Ye);ui=a(co,"First, let\u2019s grab a token classification pipeline so we can get some results to compare manually. The model used by default is "),$s=o(co,"A",{href:!0,rel:!0});var nf=r($s);wn=o(nf,"CODE",{});var af=r(wn);mi=a(af,"dbmdz/bert-large-cased-finetuned-conll03-english"),af.forEach(t),nf.forEach(t),gi=a(co,"; it performs NER on sentences:"),co.forEach(t),hl=h(e),b(vs.$$.fragment,e),fl=h(e),b(Es.$$.fragment,e),dl=h(e),dt=o(e,"P",{});var lf=r(dt);xi=a(lf,"The model properly identified each token generated by \u201CSylvain\u201D as a person, each token generated by \u201CHugging Face\u201D as an organization, and the token \u201CBrooklyn\u201D as a location. We can also ask the pipeline to group together the tokens that correspond to the same entity:"),lf.forEach(t),ul=h(e),b(Os.$$.fragment,e),ml=h(e),b(Is.$$.fragment,e),gl=h(e),W=o(e,"P",{});var ne=r(W);wi=a(ne,"The "),bn=o(ne,"CODE",{});var of=r(bn);bi=a(of,"aggregation_strategy"),of.forEach(t),ji=a(ne," picked will change the scores computed for each grouped entity. With "),jn=o(ne,"CODE",{});var rf=r(jn);ki=a(rf,'"simple"'),rf.forEach(t),_i=a(ne," the score is just the mean of the scores of each token in the given entity: for instance, the score of \u201CSylvain\u201D is the mean of the scores we saw in the previous example for the tokens "),kn=o(ne,"CODE",{});var pf=r(kn);yi=a(pf,"S"),pf.forEach(t),$i=a(ne,", "),_n=o(ne,"CODE",{});var cf=r(_n);vi=a(cf,"##yl"),cf.forEach(t),Ei=a(ne,", "),yn=o(ne,"CODE",{});var hf=r(yn);Oi=a(hf,"##va"),hf.forEach(t),Ii=a(ne,", and "),$n=o(ne,"CODE",{});var ff=r($n);Ci=a(ff,"##in"),ff.forEach(t),zi=a(ne,". Other strategies available are:"),ne.forEach(t),xl=h(e),ye=o(e,"UL",{});var Dt=r(ye);Ve=o(Dt,"LI",{});var Oa=r(Ve);vn=o(Oa,"CODE",{});var df=r(vn);Ti=a(df,'"first"'),df.forEach(t),Di=a(Oa,", where the score of each entity is the score of the first token of that entity (so for \u201CSylvain\u201D it would be 0.993828, the score of the token "),En=o(Oa,"CODE",{});var uf=r(En);Pi=a(uf,"S"),uf.forEach(t),Si=a(Oa,")"),Oa.forEach(t),Fi=h(Dt),ut=o(Dt,"LI",{});var Lc=r(ut);On=o(Lc,"CODE",{});var mf=r(On);qi=a(mf,'"max"'),mf.forEach(t),Ri=a(Lc,", where the score of each entity is the maximum score of the tokens in that entity (so for \u201CHugging Face\u201D it would be 0.98879766, the score of \u201CFace\u201D)"),Lc.forEach(t),Bi=h(Dt),Qe=o(Dt,"LI",{});var Ia=r(Qe);In=o(Ia,"CODE",{});var gf=r(In);Ai=a(gf,'"average"'),gf.forEach(t),Xi=a(Ia,", where the score of each entity is the average of the scores of the words composing that entity (so for \u201CSylvain\u201D there would be no difference from the "),Cn=o(Ia,"CODE",{});var xf=r(Cn);Hi=a(xf,'"simple"'),xf.forEach(t),Gi=a(Ia," strategy, but \u201CHugging Face\u201D would have a score of 0.9819, the average of the scores for \u201CHugging\u201D, 0.975, and \u201CFace\u201D, 0.98879)"),Ia.forEach(t),Dt.forEach(t),wl=h(e),Je=o(e,"P",{});var ho=r(Je);Mi=a(ho,"Now let\u2019s see how to obtain these results without using the "),zn=o(ho,"CODE",{});var wf=r(zn);Ni=a(wf,"pipeline()"),wf.forEach(t),Wi=a(ho," function!"),ho.forEach(t),bl=h(e),Pe=o(e,"H3",{class:!0});var fo=r(Pe);Ke=o(fo,"A",{id:!0,class:!0,href:!0});var bf=r(Ke);Tn=o(bf,"SPAN",{});var jf=r(Tn);b(Cs.$$.fragment,jf),jf.forEach(t),bf.forEach(t),Li=h(fo),Dn=o(fo,"SPAN",{});var kf=r(Dn);Ui=a(kf,"From inputs to predictions"),kf.forEach(t),fo.forEach(t),jl=h(e),xe.l(e),mt=h(e),gt=o(e,"P",{});var _f=r(gt);Yi=a(_f,"We have a batch with 1 sequence of 19 tokens and the model has 9 different labels, so the output of the model has a shape of 1 x 19 x 9. Like for the text classification pipeline, we use a softmax function to convert those logits to probabilities, and we take the argmax to get predictions (note that we can take the argmax on the logits because the softmax does not change the order):"),_f.forEach(t),kl=h(e),be.l(e),xt=h(e),b(zs.$$.fragment,e),_l=h(e),Ze=o(e,"P",{});var uo=r(Ze);Vi=a(uo,"The "),Pn=o(uo,"CODE",{});var yf=r(Pn);Qi=a(yf,"model.config.id2label"),yf.forEach(t),Ji=a(uo," attribute contains the mapping of indexes to labels that we can use to make sense of the predictions:"),uo.forEach(t),yl=h(e),b(Ts.$$.fragment,e),$l=h(e),b(Ds.$$.fragment,e),vl=h(e),C=o(e,"P",{});var R=r(C);Ki=a(R,"As we saw earlier, there are 9 labels: "),Sn=o(R,"CODE",{});var $f=r(Sn);Zi=a($f,"O"),$f.forEach(t),ep=a(R," is the label for the tokens that are not in any named entity (it stands for \u201Coutside\u201D), and we then have two labels for each type of entity (miscellaneous, person, organization, and location). The label "),Fn=o(R,"CODE",{});var vf=r(Fn);sp=a(vf,"B-XXX"),vf.forEach(t),tp=a(R," indicates the token is at the beginning of an entity "),qn=o(R,"CODE",{});var Ef=r(qn);np=a(Ef,"XXX"),Ef.forEach(t),ap=a(R," and the label "),Rn=o(R,"CODE",{});var Of=r(Rn);lp=a(Of,"I-XXX"),Of.forEach(t),op=a(R," indicates the token is inside the entity "),Bn=o(R,"CODE",{});var If=r(Bn);rp=a(If,"XXX"),If.forEach(t),ip=a(R,". For instance, in the current example we would expect our model to classify the token "),An=o(R,"CODE",{});var Cf=r(An);pp=a(Cf,"S"),Cf.forEach(t),cp=a(R," as "),Xn=o(R,"CODE",{});var zf=r(Xn);hp=a(zf,"B-PER"),zf.forEach(t),fp=a(R," (beginning of a person entity) and the tokens "),Hn=o(R,"CODE",{});var Tf=r(Hn);dp=a(Tf,"##yl"),Tf.forEach(t),up=a(R,", "),Gn=o(R,"CODE",{});var Df=r(Gn);mp=a(Df,"##va"),Df.forEach(t),gp=a(R," and "),Mn=o(R,"CODE",{});var Pf=r(Mn);xp=a(Pf,"##in"),Pf.forEach(t),wp=a(R," as "),Nn=o(R,"CODE",{});var Sf=r(Nn);bp=a(Sf,"I-PER"),Sf.forEach(t),jp=a(R," (inside a person entity)."),R.forEach(t),El=h(e),G=o(e,"P",{});var U=r(G);kp=a(U,"You might think the model was wrong in this case as it gave the label "),Wn=o(U,"CODE",{});var Ff=r(Wn);_p=a(Ff,"I-PER"),Ff.forEach(t),yp=a(U," to all four of these tokens, but that\u2019s not entirely true. There are actually two formats for those "),Ln=o(U,"CODE",{});var qf=r(Ln);$p=a(qf,"B-"),qf.forEach(t),vp=a(U," and "),Un=o(U,"CODE",{});var Rf=r(Un);Ep=a(Rf,"I-"),Rf.forEach(t),Op=a(U," labels: "),Yn=o(U,"EM",{});var Bf=r(Yn);Ip=a(Bf,"IOB1"),Bf.forEach(t),Cp=a(U," and "),Vn=o(U,"EM",{});var Af=r(Vn);zp=a(Af,"IOB2"),Af.forEach(t),Tp=a(U,". The IOB2 format (in pink below), is the one we introduced whereas in the IOB1 format (in blue), the labels beginning with "),Qn=o(U,"CODE",{});var Xf=r(Qn);Dp=a(Xf,"B-"),Xf.forEach(t),Pp=a(U," are only ever used to separate two adjacent entities of the same type. The model we are using was fine-tuned on a dataset using that format, which is why it assigns the label "),Jn=o(U,"CODE",{});var Hf=r(Jn);Sp=a(Hf,"I-PER"),Hf.forEach(t),Fp=a(U," to the "),Kn=o(U,"CODE",{});var Gf=r(Kn);qp=a(Gf,"S"),Gf.forEach(t),Rp=a(U," token."),U.forEach(t),Ol=h(e),Se=o(e,"DIV",{class:!0});var mo=r(Se);Ps=o(mo,"IMG",{class:!0,src:!0,alt:!0}),Bp=h(mo),Ss=o(mo,"IMG",{class:!0,src:!0,alt:!0}),mo.forEach(t),Il=h(e),es=o(e,"P",{});var go=r(es);Ap=a(go,"With this map, we are ready to reproduce (almost entirely) the results of the first pipeline \u2014 we can just grab the score and label of each token that was not classified as "),Zn=o(go,"CODE",{});var Mf=r(Zn);Xp=a(Mf,"O"),Mf.forEach(t),Hp=a(go,":"),go.forEach(t),Cl=h(e),b(Fs.$$.fragment,e),zl=h(e),b(qs.$$.fragment,e),Tl=h(e),ie=o(e,"P",{});var ns=r(ie);Gp=a(ns,"This is very similar to what we had before, with one exception: the pipeline also gave us information about the "),ea=o(ns,"CODE",{});var Nf=r(ea);Mp=a(Nf,"start"),Nf.forEach(t),Np=a(ns," and "),sa=o(ns,"CODE",{});var Wf=r(sa);Wp=a(Wf,"end"),Wf.forEach(t),Lp=a(ns," of each entity in the original sentence. This is where our offset mapping will come into play. To get the offsets, we just have to set "),ta=o(ns,"CODE",{});var Lf=r(ta);Up=a(Lf,"return_offsets_mapping=True"),Lf.forEach(t),Yp=a(ns," when we apply the tokenizer to our inputs:"),ns.forEach(t),Dl=h(e),b(Rs.$$.fragment,e),Pl=h(e),b(Bs.$$.fragment,e),Sl=h(e),pe=o(e,"P",{});var as=r(pe);Vp=a(as,"Each tuple is the span of text corresponding to each token, where "),na=o(as,"CODE",{});var Uf=r(na);Qp=a(Uf,"(0, 0)"),Uf.forEach(t),Jp=a(as," is reserved for the special tokens. We saw before that the token at index 5 is "),aa=o(as,"CODE",{});var Yf=r(aa);Kp=a(Yf,"##yl"),Yf.forEach(t),Zp=a(as,", which has "),la=o(as,"CODE",{});var Vf=r(la);ec=a(Vf,"(12, 14)"),Vf.forEach(t),sc=a(as," as offsets here. If we grab the corresponding slice in our example:"),as.forEach(t),Fl=h(e),b(As.$$.fragment,e),ql=h(e),ss=o(e,"P",{});var xo=r(ss);tc=a(xo,"we get the proper span of text without the "),oa=o(xo,"CODE",{});var Qf=r(oa);nc=a(Qf,"##"),Qf.forEach(t),ac=a(xo,":"),xo.forEach(t),Rl=h(e),b(Xs.$$.fragment,e),Bl=h(e),wt=o(e,"P",{});var Jf=r(wt);lc=a(Jf,"Using this, we can now complete the previous results:"),Jf.forEach(t),Al=h(e),b(Hs.$$.fragment,e),Xl=h(e),b(Gs.$$.fragment,e),Hl=h(e),bt=o(e,"P",{});var Kf=r(bt);oc=a(Kf,"This is the same as what we got from the first pipeline!"),Kf.forEach(t),Gl=h(e),Fe=o(e,"H3",{class:!0});var wo=r(Fe);ts=o(wo,"A",{id:!0,class:!0,href:!0});var Zf=r(ts);ra=o(Zf,"SPAN",{});var ed=r(ra);b(Ms.$$.fragment,ed),ed.forEach(t),Zf.forEach(t),rc=h(wo),ia=o(wo,"SPAN",{});var sd=r(ia);ic=a(sd,"Grouping entities"),sd.forEach(t),wo.forEach(t),Ml=h(e),L=o(e,"P",{});var ae=r(L);pc=a(ae,"Using the offsets to determine the start and end keys for each entity is handy, but that information isn\u2019t strictly necessary. When we want to group the entities together, however, the offsets will save us a lot of messy code. For example, if we wanted to group together the tokens "),pa=o(ae,"CODE",{});var td=r(pa);cc=a(td,"Hu"),td.forEach(t),hc=a(ae,", "),ca=o(ae,"CODE",{});var nd=r(ca);fc=a(nd,"##gging"),nd.forEach(t),dc=a(ae,", and "),ha=o(ae,"CODE",{});var ad=r(ha);uc=a(ad,"Face"),ad.forEach(t),mc=a(ae,", we could make special rules that say the first two should be attached while removing the "),fa=o(ae,"CODE",{});var ld=r(fa);gc=a(ld,"##"),ld.forEach(t),xc=a(ae,", and the "),da=o(ae,"CODE",{});var od=r(da);wc=a(od,"Face"),od.forEach(t),bc=a(ae," should be added with a space since it does not begin with "),ua=o(ae,"CODE",{});var rd=r(ua);jc=a(rd,"##"),rd.forEach(t),kc=a(ae," \u2014 but that would only work for this particular type of tokenizer. We would have to write another set of rules for a SentencePiece or a Byte-Pair-Encoding tokenizer (discussed later in this chapter)."),ae.forEach(t),Nl=h(e),J=o(e,"P",{});var he=r(J);_c=a(he,"With the offsets, all that custom code goes away: we just can take the span in the original text that begins with the first token and ends with the last token. So, in the case of the tokens "),ma=o(he,"CODE",{});var id=r(ma);yc=a(id,"Hu"),id.forEach(t),$c=a(he,", "),ga=o(he,"CODE",{});var pd=r(ga);vc=a(pd,"##gging"),pd.forEach(t),Ec=a(he,", and "),xa=o(he,"CODE",{});var cd=r(xa);Oc=a(cd,"Face"),cd.forEach(t),Ic=a(he,", we should start at character 33 (the beginning of "),wa=o(he,"CODE",{});var hd=r(wa);Cc=a(hd,"Hu"),hd.forEach(t),zc=a(he,") and end before character 45 (the end of "),ba=o(he,"CODE",{});var fd=r(ba);Tc=a(fd,"Face"),fd.forEach(t),Dc=a(he,"):"),he.forEach(t),Wl=h(e),b(Ns.$$.fragment,e),Ll=h(e),b(Ws.$$.fragment,e),Ul=h(e),K=o(e,"P",{});var fe=r(K);Pc=a(fe,"To write the code that post-processes the predictions while grouping entities, we will group together entities that are consecutive and labeled with "),ja=o(fe,"CODE",{});var dd=r(ja);Sc=a(dd,"I-XXX"),dd.forEach(t),Fc=a(fe,", except for the first one, which can be labeled as "),ka=o(fe,"CODE",{});var ud=r(ka);qc=a(ud,"B-XXX"),ud.forEach(t),Rc=a(fe," or "),_a=o(fe,"CODE",{});var md=r(_a);Bc=a(md,"I-XXX"),md.forEach(t),Ac=a(fe," (so, we stop grouping an entity when we get a "),ya=o(fe,"CODE",{});var gd=r(ya);Xc=a(gd,"O"),gd.forEach(t),Hc=a(fe,", a new type of entity, or a "),$a=o(fe,"CODE",{});var xd=r($a);Gc=a(xd,"B-XXX"),xd.forEach(t),Mc=a(fe," that tells us an entity of the same type is starting):"),fe.forEach(t),Yl=h(e),b(Ls.$$.fragment,e),Vl=h(e),jt=o(e,"P",{});var wd=r(jt);Nc=a(wd,"And we get the same results as with our second pipeline!"),wd.forEach(t),Ql=h(e),b(Us.$$.fragment,e),Jl=h(e),kt=o(e,"P",{});var bd=r(kt);Wc=a(bd,"Another example of a task where these offsets are extremely useful is question answering. Diving into that pipeline, which we\u2019ll do in the next section, will also enable us to take a look at one last feature of the tokenizers in the \u{1F917} Transformers library: dealing with overflowing tokens when we truncate an input to a given length."),bd.forEach(t),this.h()},h(){_(f,"name","hf:doc:metadata"),_(f,"content",JSON.stringify(Xd)),_(E,"id","fast-tokenizers-special-powers"),_(E,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),_(E,"href","#fast-tokenizers-special-powers"),_(P,"class","relative group"),_(re,"href","/course/chapter1"),_(Zs,"href","/course/chapter5/3"),_(et,"align","center"),_(st,"align","center"),_(Ft,"align","center"),_(tt,"align","center"),_(nt,"align","center"),_(at,"align","center"),_(lt,"align","center"),_(ot,"align","center"),_(rt,"align","center"),_(Ae,"id","batch-encoding"),_(Ae,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),_(Ae,"href","#batch-encoding"),_(ze,"class","relative group"),_(Le,"id","inside-the-tokenclassification-pipeline"),_(Le,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),_(Le,"href","#inside-the-tokenclassification-pipeline"),_(Te,"class","relative group"),_(ct,"href","/course/chapter1"),_(ht,"href","/course/chapter2"),_(Ue,"id","getting-the-base-results-with-the-pipeline"),_(Ue,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),_(Ue,"href","#getting-the-base-results-with-the-pipeline"),_(De,"class","relative group"),_($s,"href","https://huggingface.co/dbmdz/bert-large-cased-finetuned-conll03-english"),_($s,"rel","nofollow"),_(Ke,"id","from-inputs-to-predictions"),_(Ke,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),_(Ke,"href","#from-inputs-to-predictions"),_(Pe,"class","relative group"),_(Ps,"class","block dark:hidden"),jd(Ps.src,Yc="https://huggingface.co/datasets/huggingface-course/documentation-images/resolve/main/en/chapter6/IOB_versions.svg")||_(Ps,"src",Yc),_(Ps,"alt","IOB1 vs IOB2 format"),_(Ss,"class","hidden dark:block"),jd(Ss.src,Vc="https://huggingface.co/datasets/huggingface-course/documentation-images/resolve/main/en/chapter6/IOB_versions-dark.svg")||_(Ss,"src",Vc),_(Ss,"alt","IOB1 vs IOB2 format"),_(Se,"class","flex justify-center"),_(ts,"id","grouping-entities"),_(ts,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),_(ts,"href","#grouping-entities"),_(Fe,"class","relative group")},m(e,i){s(document.head,f),p(e,x,i),j(d,e,i),p(e,y,i),p(e,P,i),s(P,E),s(E,F),j(A,F,null),s(P,M),s(P,S),s(S,Y),p(e,I,i),Ys[B].m(e,i),p(e,T,i),p(e,O,i),s(O,X),s(O,q),s(q,V),s(O,H),s(O,Z),s(Z,u),s(O,v),s(O,de),s(de,Ee),s(O,le),s(O,re),s(re,_o),s(O,yo),p(e,Ca,i),j(ls,e,i),p(e,za,i),p(e,qe,i),s(qe,$o),s(qe,Zs),s(Zs,vo),s(qe,Eo),p(e,Ta,i),p(e,Re,i),s(Re,St),s(St,Oe),s(Oe,et),s(et,Oo),s(Oe,Io),s(Oe,st),s(st,Co),s(Oe,zo),s(Oe,Ft),s(Re,To),s(Re,os),s(os,Ie),s(Ie,tt),s(tt,qt),s(qt,Do),s(Ie,Po),s(Ie,nt),s(nt,So),s(Ie,Fo),s(Ie,at),s(at,qo),s(os,Ro),s(os,Ce),s(Ce,lt),s(lt,Rt),s(Rt,Bo),s(Ce,Ao),s(Ce,ot),s(ot,Xo),s(Ce,Ho),s(Ce,rt),s(rt,Go),p(e,Da,i),j(Be,e,i),p(e,Pa,i),p(e,ze,i),s(ze,Ae),s(Ae,Bt),j(rs,Bt,null),s(ze,Mo),s(ze,At),s(At,No),p(e,Sa,i),j(is,e,i),p(e,Fa,i),p(e,Xe,i),s(Xe,Wo),s(Xe,Xt),s(Xt,Lo),s(Xe,Uo),p(e,qa,i),p(e,He,i),s(He,Yo),s(He,Ht),s(Ht,Vo),s(He,Qo),p(e,Ra,i),p(e,it,i),s(it,Jo),p(e,Ba,i),j(ps,e,i),p(e,Aa,i),p(e,Ge,i),s(Ge,Ko),s(Ge,Gt),s(Gt,Zo),s(Ge,er),p(e,Xa,i),j(cs,e,i),p(e,Ha,i),p(e,ee,i),s(ee,sr),s(ee,Mt),s(Mt,tr),s(ee,nr),s(ee,Nt),s(Nt,ar),s(ee,lr),s(ee,Wt),s(Wt,or),s(ee,rr),s(ee,Lt),s(Lt,ir),s(ee,pr),p(e,Ga,i),j(hs,e,i),p(e,Ma,i),j(fs,e,i),p(e,Na,i),p(e,Me,i),s(Me,cr),s(Me,Ut),s(Ut,hr),s(Me,fr),p(e,Wa,i),j(ds,e,i),p(e,La,i),j(us,e,i),p(e,Ua,i),p(e,pt,i),s(pt,dr),p(e,Ya,i),j(ms,e,i),p(e,Va,i),j(gs,e,i),p(e,Qa,i),p(e,je,i),s(je,ur),s(je,Yt),s(Yt,mr),s(je,gr),s(je,Vt),s(Vt,xr),s(je,wr),p(e,Ja,i),j(xs,e,i),p(e,Ka,i),j(ws,e,i),p(e,Za,i),p(e,Q,i),s(Q,br),s(Q,Qt),s(Qt,jr),s(Q,kr),s(Q,Jt),s(Jt,_r),s(Q,yr),s(Q,Kt),s(Kt,$r),s(Q,vr),s(Q,Zt),s(Zt,Er),s(Q,Or),s(Q,en),s(en,Ir),s(Q,Cr),p(e,el,i),j(Ne,e,i),p(e,sl,i),p(e,ke,i),s(ke,zr),s(ke,sn),s(sn,Tr),s(ke,Dr),s(ke,tn),s(tn,Pr),s(ke,Sr),p(e,tl,i),p(e,N,i),s(N,Fr),s(N,nn),s(nn,qr),s(N,Rr),s(N,an),s(an,Br),s(N,Ar),s(N,ln),s(ln,Xr),s(N,Hr),s(N,on),s(on,Gr),s(N,Mr),s(N,rn),s(rn,Nr),s(N,Wr),s(N,pn),s(pn,Lr),s(N,Ur),p(e,nl,i),j(bs,e,i),p(e,al,i),j(js,e,i),p(e,ll,i),p(e,_e,i),s(_e,Yr),s(_e,cn),s(cn,Vr),s(_e,Qr),s(_e,hn),s(hn,Jr),s(_e,Kr),p(e,ol,i),j(We,e,i),p(e,rl,i),p(e,Te,i),s(Te,Le),s(Le,fn),j(ks,fn,null),s(Te,Zr),s(Te,_s),s(_s,ei),s(_s,dn),s(dn,si),s(_s,ti),p(e,il,i),p(e,se,i),s(se,ni),s(se,ct),s(ct,ai),s(se,li),s(se,un),s(un,oi),s(se,ri),s(se,ht),s(ht,ii),s(se,pi),s(se,mn),s(mn,ci),s(se,hi),p(e,pl,i),Vs[ue].m(e,i),p(e,ft,i),p(e,De,i),s(De,Ue),s(Ue,gn),j(ys,gn,null),s(De,fi),s(De,xn),s(xn,di),p(e,cl,i),p(e,Ye,i),s(Ye,ui),s(Ye,$s),s($s,wn),s(wn,mi),s(Ye,gi),p(e,hl,i),j(vs,e,i),p(e,fl,i),j(Es,e,i),p(e,dl,i),p(e,dt,i),s(dt,xi),p(e,ul,i),j(Os,e,i),p(e,ml,i),j(Is,e,i),p(e,gl,i),p(e,W,i),s(W,wi),s(W,bn),s(bn,bi),s(W,ji),s(W,jn),s(jn,ki),s(W,_i),s(W,kn),s(kn,yi),s(W,$i),s(W,_n),s(_n,vi),s(W,Ei),s(W,yn),s(yn,Oi),s(W,Ii),s(W,$n),s($n,Ci),s(W,zi),p(e,xl,i),p(e,ye,i),s(ye,Ve),s(Ve,vn),s(vn,Ti),s(Ve,Di),s(Ve,En),s(En,Pi),s(Ve,Si),s(ye,Fi),s(ye,ut),s(ut,On),s(On,qi),s(ut,Ri),s(ye,Bi),s(ye,Qe),s(Qe,In),s(In,Ai),s(Qe,Xi),s(Qe,Cn),s(Cn,Hi),s(Qe,Gi),p(e,wl,i),p(e,Je,i),s(Je,Mi),s(Je,zn),s(zn,Ni),s(Je,Wi),p(e,bl,i),p(e,Pe,i),s(Pe,Ke),s(Ke,Tn),j(Cs,Tn,null),s(Pe,Li),s(Pe,Dn),s(Dn,Ui),p(e,jl,i),Qs[ge].m(e,i),p(e,mt,i),p(e,gt,i),s(gt,Yi),p(e,kl,i),Js[we].m(e,i),p(e,xt,i),j(zs,e,i),p(e,_l,i),p(e,Ze,i),s(Ze,Vi),s(Ze,Pn),s(Pn,Qi),s(Ze,Ji),p(e,yl,i),j(Ts,e,i),p(e,$l,i),j(Ds,e,i),p(e,vl,i),p(e,C,i),s(C,Ki),s(C,Sn),s(Sn,Zi),s(C,ep),s(C,Fn),s(Fn,sp),s(C,tp),s(C,qn),s(qn,np),s(C,ap),s(C,Rn),s(Rn,lp),s(C,op),s(C,Bn),s(Bn,rp),s(C,ip),s(C,An),s(An,pp),s(C,cp),s(C,Xn),s(Xn,hp),s(C,fp),s(C,Hn),s(Hn,dp),s(C,up),s(C,Gn),s(Gn,mp),s(C,gp),s(C,Mn),s(Mn,xp),s(C,wp),s(C,Nn),s(Nn,bp),s(C,jp),p(e,El,i),p(e,G,i),s(G,kp),s(G,Wn),s(Wn,_p),s(G,yp),s(G,Ln),s(Ln,$p),s(G,vp),s(G,Un),s(Un,Ep),s(G,Op),s(G,Yn),s(Yn,Ip),s(G,Cp),s(G,Vn),s(Vn,zp),s(G,Tp),s(G,Qn),s(Qn,Dp),s(G,Pp),s(G,Jn),s(Jn,Sp),s(G,Fp),s(G,Kn),s(Kn,qp),s(G,Rp),p(e,Ol,i),p(e,Se,i),s(Se,Ps),s(Se,Bp),s(Se,Ss),p(e,Il,i),p(e,es,i),s(es,Ap),s(es,Zn),s(Zn,Xp),s(es,Hp),p(e,Cl,i),j(Fs,e,i),p(e,zl,i),j(qs,e,i),p(e,Tl,i),p(e,ie,i),s(ie,Gp),s(ie,ea),s(ea,Mp),s(ie,Np),s(ie,sa),s(sa,Wp),s(ie,Lp),s(ie,ta),s(ta,Up),s(ie,Yp),p(e,Dl,i),j(Rs,e,i),p(e,Pl,i),j(Bs,e,i),p(e,Sl,i),p(e,pe,i),s(pe,Vp),s(pe,na),s(na,Qp),s(pe,Jp),s(pe,aa),s(aa,Kp),s(pe,Zp),s(pe,la),s(la,ec),s(pe,sc),p(e,Fl,i),j(As,e,i),p(e,ql,i),p(e,ss,i),s(ss,tc),s(ss,oa),s(oa,nc),s(ss,ac),p(e,Rl,i),j(Xs,e,i),p(e,Bl,i),p(e,wt,i),s(wt,lc),p(e,Al,i),j(Hs,e,i),p(e,Xl,i),j(Gs,e,i),p(e,Hl,i),p(e,bt,i),s(bt,oc),p(e,Gl,i),p(e,Fe,i),s(Fe,ts),s(ts,ra),j(Ms,ra,null),s(Fe,rc),s(Fe,ia),s(ia,ic),p(e,Ml,i),p(e,L,i),s(L,pc),s(L,pa),s(pa,cc),s(L,hc),s(L,ca),s(ca,fc),s(L,dc),s(L,ha),s(ha,uc),s(L,mc),s(L,fa),s(fa,gc),s(L,xc),s(L,da),s(da,wc),s(L,bc),s(L,ua),s(ua,jc),s(L,kc),p(e,Nl,i),p(e,J,i),s(J,_c),s(J,ma),s(ma,yc),s(J,$c),s(J,ga),s(ga,vc),s(J,Ec),s(J,xa),s(xa,Oc),s(J,Ic),s(J,wa),s(wa,Cc),s(J,zc),s(J,ba),s(ba,Tc),s(J,Dc),p(e,Wl,i),j(Ns,e,i),p(e,Ll,i),j(Ws,e,i),p(e,Ul,i),p(e,K,i),s(K,Pc),s(K,ja),s(ja,Sc),s(K,Fc),s(K,ka),s(ka,qc),s(K,Rc),s(K,_a),s(_a,Bc),s(K,Ac),s(K,ya),s(ya,Xc),s(K,Hc),s(K,$a),s($a,Gc),s(K,Mc),p(e,Yl,i),j(Ls,e,i),p(e,Vl,i),p(e,jt,i),s(jt,Nc),p(e,Ql,i),j(Us,e,i),p(e,Jl,i),p(e,kt,i),s(kt,Wc),Kl=!0},p(e,[i]){const Ks={};i&1&&(Ks.fw=e[0]),d.$set(Ks);let _t=B;B=Jc(e),B!==_t&&(jo(),m(Ys[_t],1,1,()=>{Ys[_t]=null}),bo(),z=Ys[B],z||(z=Ys[B]=Qc[B](e),z.c()),g(z,1),z.m(T.parentNode,T));const va={};i&2&&(va.$$scope={dirty:i,ctx:e}),Be.$set(va);const Ea={};i&2&&(Ea.$$scope={dirty:i,ctx:e}),Ne.$set(Ea);const oe={};i&2&&(oe.$$scope={dirty:i,ctx:e}),We.$set(oe);let yt=ue;ue=Zc(e),ue!==yt&&(jo(),m(Vs[yt],1,1,()=>{Vs[yt]=null}),bo(),me=Vs[ue],me||(me=Vs[ue]=Kc[ue](e),me.c()),g(me,1),me.m(ft.parentNode,ft));let $t=ge;ge=sh(e),ge!==$t&&(jo(),m(Qs[$t],1,1,()=>{Qs[$t]=null}),bo(),xe=Qs[ge],xe||(xe=Qs[ge]=eh[ge](e),xe.c()),g(xe,1),xe.m(mt.parentNode,mt));let vt=we;we=nh(e),we!==vt&&(jo(),m(Js[vt],1,1,()=>{Js[vt]=null}),bo(),be=Js[we],be||(be=Js[we]=th[we](e),be.c()),g(be,1),be.m(xt.parentNode,xt))},i(e){Kl||(g(d.$$.fragment,e),g(A.$$.fragment,e),g(z),g(ls.$$.fragment,e),g(Be.$$.fragment,e),g(rs.$$.fragment,e),g(is.$$.fragment,e),g(ps.$$.fragment,e),g(cs.$$.fragment,e),g(hs.$$.fragment,e),g(fs.$$.fragment,e),g(ds.$$.fragment,e),g(us.$$.fragment,e),g(ms.$$.fragment,e),g(gs.$$.fragment,e),g(xs.$$.fragment,e),g(ws.$$.fragment,e),g(Ne.$$.fragment,e),g(bs.$$.fragment,e),g(js.$$.fragment,e),g(We.$$.fragment,e),g(ks.$$.fragment,e),g(me),g(ys.$$.fragment,e),g(vs.$$.fragment,e),g(Es.$$.fragment,e),g(Os.$$.fragment,e),g(Is.$$.fragment,e),g(Cs.$$.fragment,e),g(xe),g(be),g(zs.$$.fragment,e),g(Ts.$$.fragment,e),g(Ds.$$.fragment,e),g(Fs.$$.fragment,e),g(qs.$$.fragment,e),g(Rs.$$.fragment,e),g(Bs.$$.fragment,e),g(As.$$.fragment,e),g(Xs.$$.fragment,e),g(Hs.$$.fragment,e),g(Gs.$$.fragment,e),g(Ms.$$.fragment,e),g(Ns.$$.fragment,e),g(Ws.$$.fragment,e),g(Ls.$$.fragment,e),g(Us.$$.fragment,e),Kl=!0)},o(e){m(d.$$.fragment,e),m(A.$$.fragment,e),m(z),m(ls.$$.fragment,e),m(Be.$$.fragment,e),m(rs.$$.fragment,e),m(is.$$.fragment,e),m(ps.$$.fragment,e),m(cs.$$.fragment,e),m(hs.$$.fragment,e),m(fs.$$.fragment,e),m(ds.$$.fragment,e),m(us.$$.fragment,e),m(ms.$$.fragment,e),m(gs.$$.fragment,e),m(xs.$$.fragment,e),m(ws.$$.fragment,e),m(Ne.$$.fragment,e),m(bs.$$.fragment,e),m(js.$$.fragment,e),m(We.$$.fragment,e),m(ks.$$.fragment,e),m(me),m(ys.$$.fragment,e),m(vs.$$.fragment,e),m(Es.$$.fragment,e),m(Os.$$.fragment,e),m(Is.$$.fragment,e),m(Cs.$$.fragment,e),m(xe),m(be),m(zs.$$.fragment,e),m(Ts.$$.fragment,e),m(Ds.$$.fragment,e),m(Fs.$$.fragment,e),m(qs.$$.fragment,e),m(Rs.$$.fragment,e),m(Bs.$$.fragment,e),m(As.$$.fragment,e),m(Xs.$$.fragment,e),m(Hs.$$.fragment,e),m(Gs.$$.fragment,e),m(Ms.$$.fragment,e),m(Ns.$$.fragment,e),m(Ws.$$.fragment,e),m(Ls.$$.fragment,e),m(Us.$$.fragment,e),Kl=!1},d(e){t(f),e&&t(x),k(d,e),e&&t(y),e&&t(P),k(A),e&&t(I),Ys[B].d(e),e&&t(T),e&&t(O),e&&t(Ca),k(ls,e),e&&t(za),e&&t(qe),e&&t(Ta),e&&t(Re),e&&t(Da),k(Be,e),e&&t(Pa),e&&t(ze),k(rs),e&&t(Sa),k(is,e),e&&t(Fa),e&&t(Xe),e&&t(qa),e&&t(He),e&&t(Ra),e&&t(it),e&&t(Ba),k(ps,e),e&&t(Aa),e&&t(Ge),e&&t(Xa),k(cs,e),e&&t(Ha),e&&t(ee),e&&t(Ga),k(hs,e),e&&t(Ma),k(fs,e),e&&t(Na),e&&t(Me),e&&t(Wa),k(ds,e),e&&t(La),k(us,e),e&&t(Ua),e&&t(pt),e&&t(Ya),k(ms,e),e&&t(Va),k(gs,e),e&&t(Qa),e&&t(je),e&&t(Ja),k(xs,e),e&&t(Ka),k(ws,e),e&&t(Za),e&&t(Q),e&&t(el),k(Ne,e),e&&t(sl),e&&t(ke),e&&t(tl),e&&t(N),e&&t(nl),k(bs,e),e&&t(al),k(js,e),e&&t(ll),e&&t(_e),e&&t(ol),k(We,e),e&&t(rl),e&&t(Te),k(ks),e&&t(il),e&&t(se),e&&t(pl),Vs[ue].d(e),e&&t(ft),e&&t(De),k(ys),e&&t(cl),e&&t(Ye),e&&t(hl),k(vs,e),e&&t(fl),k(Es,e),e&&t(dl),e&&t(dt),e&&t(ul),k(Os,e),e&&t(ml),k(Is,e),e&&t(gl),e&&t(W),e&&t(xl),e&&t(ye),e&&t(wl),e&&t(Je),e&&t(bl),e&&t(Pe),k(Cs),e&&t(jl),Qs[ge].d(e),e&&t(mt),e&&t(gt),e&&t(kl),Js[we].d(e),e&&t(xt),k(zs,e),e&&t(_l),e&&t(Ze),e&&t(yl),k(Ts,e),e&&t($l),k(Ds,e),e&&t(vl),e&&t(C),e&&t(El),e&&t(G),e&&t(Ol),e&&t(Se),e&&t(Il),e&&t(es),e&&t(Cl),k(Fs,e),e&&t(zl),k(qs,e),e&&t(Tl),e&&t(ie),e&&t(Dl),k(Rs,e),e&&t(Pl),k(Bs,e),e&&t(Sl),e&&t(pe),e&&t(Fl),k(As,e),e&&t(ql),e&&t(ss),e&&t(Rl),k(Xs,e),e&&t(Bl),e&&t(wt),e&&t(Al),k(Hs,e),e&&t(Xl),k(Gs,e),e&&t(Hl),e&&t(bt),e&&t(Gl),e&&t(Fe),k(Ms),e&&t(Ml),e&&t(L),e&&t(Nl),e&&t(J),e&&t(Wl),k(Ns,e),e&&t(Ll),k(Ws,e),e&&t(Ul),e&&t(K),e&&t(Yl),k(Ls,e),e&&t(Vl),e&&t(jt),e&&t(Ql),k(Us,e),e&&t(Jl),e&&t(kt)}}}const Xd={local:"fast-tokenizers-special-powers",sections:[{local:"batch-encoding",title:"Batch encoding"},{local:"inside-the-tokenclassification-pipeline",sections:[{local:"getting-the-base-results-with-the-pipeline",title:"Getting the base results with the pipeline"},{local:"from-inputs-to-predictions",title:"From inputs to predictions"},{local:"grouping-entities",title:"Grouping entities"}],title:"Inside the `token-classification` pipeline"}],title:"Fast tokenizers' special powers"};function Hd(D,f,x){let d="pt";return Ed(()=>{const y=new URLSearchParams(window.location.search);x(0,d=y.get("fw")||"pt")}),[d]}class Vd extends _d{constructor(f){super();yd(this,f,Hd,Ad,$d,{})}}export{Vd as default,Xd as metadata};
