import{S as Qx,i as Jx,s as Ux,e as l,k as u,w as b,t as n,M as Yx,c as r,d as t,m as c,x as j,a as o,h as a,b as k,N as Gx,F as e,g as p,y as _,o as f,p as To,q as h,B as E,v as Kx,n as Ro}from"../../chunks/vendor-1e8b365d.js";import{T as vf}from"../../chunks/Tip-62b14c6e.js";import{Y as So}from"../../chunks/Youtube-c2a8cc39.js";import{I as Ut}from"../../chunks/IconCopyLink-483c28ba.js";import{C as q}from"../../chunks/CodeBlock-e5764662.js";import{D as Vx}from"../../chunks/DocNotebookDropdown-37d928d3.js";import{F as Wx}from"../../chunks/FrameworkSwitchCourse-7f8f0f31.js";function Zx(X){let d,g;return d=new Vx({props:{classNames:"absolute z-10 right-0 top-0",options:[{label:"Google Colab",value:"https://colab.research.google.com/github/huggingface/notebooks/blob/master/course/chapter6/section3_tf.ipynb"},{label:"Aws Studio",value:"https://studiolab.sagemaker.aws/import/github/huggingface/notebooks/blob/master/course/chapter6/section3_tf.ipynb"}]}}),{c(){b(d.$$.fragment)},l(m){j(d.$$.fragment,m)},m(m,y){_(d,m,y),g=!0},i(m){g||(h(d.$$.fragment,m),g=!0)},o(m){f(d.$$.fragment,m),g=!1},d(m){E(d,m)}}}function e7(X){let d,g;return d=new Vx({props:{classNames:"absolute z-10 right-0 top-0",options:[{label:"Google Colab",value:"https://colab.research.google.com/github/huggingface/notebooks/blob/master/course/chapter6/section3_pt.ipynb"},{label:"Aws Studio",value:"https://studiolab.sagemaker.aws/import/github/huggingface/notebooks/blob/master/course/chapter6/section3_pt.ipynb"}]}}),{c(){b(d.$$.fragment)},l(m){j(d.$$.fragment,m)},m(m,y){_(d,m,y),g=!0},i(m){g||(h(d.$$.fragment,m),g=!0)},o(m){f(d.$$.fragment,m),g=!1},d(m){E(d,m)}}}function s7(X){let d,g,m,y,R;return{c(){d=l("p"),g=n("\u26A0\uFE0F Lors de la tokenisation d\u2019une seule phrase, vous ne verrez pas toujours une diff\xE9rence de vitesse entre les versions lente et rapide d\u2019un m\xEAme "),m=l("em"),y=n("tokenizer"),R=n(". En fait, la version rapide peut m\xEAme \xEAtre plus lente ! Ce n\u2019est que lorsque vous tokenisez des batchs de textes en parall\xE8le et en m\xEAme temps que vous pourrez voir clairement la diff\xE9rence.")},l(w){d=r(w,"P",{});var S=o(d);g=a(S,"\u26A0\uFE0F Lors de la tokenisation d\u2019une seule phrase, vous ne verrez pas toujours une diff\xE9rence de vitesse entre les versions lente et rapide d\u2019un m\xEAme "),m=r(S,"EM",{});var N=o(m);y=a(N,"tokenizer"),N.forEach(t),R=a(S,". En fait, la version rapide peut m\xEAme \xEAtre plus lente ! Ce n\u2019est que lorsque vous tokenisez des batchs de textes en parall\xE8le et en m\xEAme temps que vous pourrez voir clairement la diff\xE9rence."),S.forEach(t)},m(w,S){p(w,d,S),e(d,g),e(d,m),e(m,y),e(d,R)},d(w){w&&t(d)}}}function t7(X){let d,g,m,y,R,w,S,N,L,C,ne,Q,M,H,D,z,se,te,ge,ae,J,pe,K;return{c(){d=l("p"),g=n("La notion de ce qu\u2019est un mot est compliqu\xE9e. Par exemple, est-ce que \xAB I will \xBB (contraction de \xAB I will \xBB) compte pour un ou deux mots ? Cela d\xE9pend en fait du "),m=l("em"),y=n("tokenizer"),R=n(" et de l\u2019op\xE9ration de pr\xE9-tok\xE9nisation qu\u2019il applique. Certains "),w=l("em"),S=n("tokenizer"),N=n(" se contentent de s\xE9parer les espaces et consid\xE8rent donc qu\u2019il s\u2019agit d\u2019un seul mot. D\u2019autres utilisent la ponctuation en plus des espaces et consid\xE8rent donc qu\u2019il s\u2019agit de deux mots."),L=u(),C=l("p"),ne=n("\u270F\uFE0F "),Q=l("strong"),M=n("Essayez !"),H=n(" Cr\xE9ez un "),D=l("em"),z=n("tokenizer"),se=n(" \xE0 partir des points de contr\xF4le "),te=l("code"),ge=n("bert-base-cased"),ae=n(" et "),J=l("code"),pe=n("roberta-base"),K=n(" et tokenisez \xAB 81s \xBB avec eux. Qu\u2019observez-vous ? Quels sont les identifiants des mots ?")},l(T){d=r(T,"P",{});var x=o(d);g=a(x,"La notion de ce qu\u2019est un mot est compliqu\xE9e. Par exemple, est-ce que \xAB I will \xBB (contraction de \xAB I will \xBB) compte pour un ou deux mots ? Cela d\xE9pend en fait du "),m=r(x,"EM",{});var P=o(m);y=a(P,"tokenizer"),P.forEach(t),R=a(x," et de l\u2019op\xE9ration de pr\xE9-tok\xE9nisation qu\u2019il applique. Certains "),w=r(x,"EM",{});var ue=o(w);S=a(ue,"tokenizer"),ue.forEach(t),N=a(x," se contentent de s\xE9parer les espaces et consid\xE8rent donc qu\u2019il s\u2019agit d\u2019un seul mot. D\u2019autres utilisent la ponctuation en plus des espaces et consid\xE8rent donc qu\u2019il s\u2019agit de deux mots."),x.forEach(t),L=c(T),C=r(T,"P",{});var v=o(C);ne=a(v,"\u270F\uFE0F "),Q=r(v,"STRONG",{});var B=o(Q);M=a(B,"Essayez !"),B.forEach(t),H=a(v," Cr\xE9ez un "),D=r(v,"EM",{});var ce=o(D);z=a(ce,"tokenizer"),ce.forEach(t),se=a(v," \xE0 partir des points de contr\xF4le "),te=r(v,"CODE",{});var ke=o(te);ge=a(ke,"bert-base-cased"),ke.forEach(t),ae=a(v," et "),J=r(v,"CODE",{});var He=o(J);pe=a(He,"roberta-base"),He.forEach(t),K=a(v," et tokenisez \xAB 81s \xBB avec eux. Qu\u2019observez-vous ? Quels sont les identifiants des mots ?"),v.forEach(t)},m(T,x){p(T,d,x),e(d,g),e(d,m),e(m,y),e(d,R),e(d,w),e(w,S),e(d,N),p(T,L,x),p(T,C,x),e(C,ne),e(C,Q),e(Q,M),e(C,H),e(C,D),e(D,z),e(C,se),e(C,te),e(te,ge),e(C,ae),e(C,J),e(J,pe),e(C,K)},d(T){T&&t(d),T&&t(L),T&&t(C)}}}function n7(X){let d,g,m,y,R,w,S,N;return{c(){d=l("p"),g=n("\u270F\uFE0F "),m=l("strong"),y=n("Essayez !"),R=n(" Cr\xE9ez votre propre texte d\u2019exemple et voyez si vous pouvez comprendre quels "),w=l("em"),S=n("tokens"),N=n(" sont associ\xE9s \xE0 l\u2019ID du mot, et aussi comment extraire les port\xE9es de caract\xE8res pour un seul mot. Pour obtenir des points bonus, essayez d\u2019utiliser deux phrases en entr\xE9e et voyez si les identifiants de phrase ont un sens pour vous.")},l(L){d=r(L,"P",{});var C=o(d);g=a(C,"\u270F\uFE0F "),m=r(C,"STRONG",{});var ne=o(m);y=a(ne,"Essayez !"),ne.forEach(t),R=a(C," Cr\xE9ez votre propre texte d\u2019exemple et voyez si vous pouvez comprendre quels "),w=r(C,"EM",{});var Q=o(w);S=a(Q,"tokens"),Q.forEach(t),N=a(C," sont associ\xE9s \xE0 l\u2019ID du mot, et aussi comment extraire les port\xE9es de caract\xE8res pour un seul mot. Pour obtenir des points bonus, essayez d\u2019utiliser deux phrases en entr\xE9e et voyez si les identifiants de phrase ont un sens pour vous."),C.forEach(t)},m(L,C){p(L,d,C),e(d,g),e(d,m),e(m,y),e(d,R),e(d,w),e(w,S),e(d,N)},d(L){L&&t(d)}}}function a7(X){let d,g;return d=new So({props:{id:"PrX4CjrVnNc"}}),{c(){b(d.$$.fragment)},l(m){j(d.$$.fragment,m)},m(m,y){_(d,m,y),g=!0},i(m){g||(h(d.$$.fragment,m),g=!0)},o(m){f(d.$$.fragment,m),g=!1},d(m){E(d,m)}}}function l7(X){let d,g;return d=new So({props:{id:"0E7ltQB7fM8"}}),{c(){b(d.$$.fragment)},l(m){j(d.$$.fragment,m)},m(m,y){_(d,m,y),g=!0},i(m){g||(h(d.$$.fragment,m),g=!0)},o(m){f(d.$$.fragment,m),g=!1},d(m){E(d,m)}}}function r7(X){let d,g,m,y,R,w,S,N,L,C,ne,Q,M,H,D,z,se,te,ge,ae,J,pe,K,T,x,P,ue;return M=new q({props:{code:`from transformers import AutoTokenizer, TFAutoModelForTokenClassification

model_checkpoint = "dbmdz/bert-large-cased-finetuned-conll03-english"
tokenizer = AutoTokenizer.from_pretrained(model_checkpoint)
model = TFAutoModelForTokenClassification.from_pretrained(model_checkpoint)

example = "My name is Sylvain and I work at Hugging Face in Brooklyn."
inputs = tokenizer(example, return_tensors="tf")
outputs = model(**inputs)`,highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoTokenizer, TFAutoModelForTokenClassification

model_checkpoint = <span class="hljs-string">&quot;dbmdz/bert-large-cased-finetuned-conll03-english&quot;</span>
tokenizer = AutoTokenizer.from_pretrained(model_checkpoint)
model = TFAutoModelForTokenClassification.from_pretrained(model_checkpoint)

example = <span class="hljs-string">&quot;My name is Sylvain and I work at Hugging Face in Brooklyn.&quot;</span>
inputs = tokenizer(example, return_tensors=<span class="hljs-string">&quot;tf&quot;</span>)
outputs = model(**inputs)`}}),T=new q({props:{code:`print(inputs["input_ids"].shape)
print(outputs.logits.shape)`,highlighted:`<span class="hljs-built_in">print</span>(inputs[<span class="hljs-string">&quot;input_ids&quot;</span>].shape)
<span class="hljs-built_in">print</span>(outputs.logits.shape)`}}),P=new q({props:{code:`(1, 19)
(1, 19, 9)`,highlighted:`(<span class="hljs-number">1</span>, <span class="hljs-number">19</span>)
(<span class="hljs-number">1</span>, <span class="hljs-number">19</span>, <span class="hljs-number">9</span>)`}}),{c(){d=l("p"),g=n("D\u2019abord, nous devons tokeniser notre entr\xE9e et la faire passer dans le mod\xE8le. Cela se fait exactement comme dans le "),m=l("a"),y=n("Chapitre 2"),R=n(". Nous instancions le "),w=l("em"),S=n("tokenizer"),N=n(" et le mod\xE8le en utilisant les classes "),L=l("code"),C=n("TFAutoXxx"),ne=n(" et les utilisons ensuite dans notre exemple :"),Q=u(),b(M.$$.fragment),H=u(),D=l("p"),z=n("Puisque nous utilisons "),se=l("code"),te=n("TFAutoModelForTokenClassification"),ge=n(" ici, nous obtenons un ensemble de logits pour chaque "),ae=l("em"),J=n("token"),pe=n(" dans la s\xE9quence d\u2019entr\xE9e :"),K=u(),b(T.$$.fragment),x=u(),b(P.$$.fragment),this.h()},l(v){d=r(v,"P",{});var B=o(d);g=a(B,"D\u2019abord, nous devons tokeniser notre entr\xE9e et la faire passer dans le mod\xE8le. Cela se fait exactement comme dans le "),m=r(B,"A",{href:!0});var ce=o(m);y=a(ce,"Chapitre 2"),ce.forEach(t),R=a(B,". Nous instancions le "),w=r(B,"EM",{});var ke=o(w);S=a(ke,"tokenizer"),ke.forEach(t),N=a(B," et le mod\xE8le en utilisant les classes "),L=r(B,"CODE",{});var He=o(L);C=a(He,"TFAutoXxx"),He.forEach(t),ne=a(B," et les utilisons ensuite dans notre exemple :"),B.forEach(t),Q=c(v),j(M.$$.fragment,v),H=c(v),D=r(v,"P",{});var $e=o(D);z=a($e,"Puisque nous utilisons "),se=r($e,"CODE",{});var xt=o(se);te=a(xt,"TFAutoModelForTokenClassification"),xt.forEach(t),ge=a($e," ici, nous obtenons un ensemble de logits pour chaque "),ae=r($e,"EM",{});var gt=o(ae);J=a(gt,"token"),gt.forEach(t),pe=a($e," dans la s\xE9quence d\u2019entr\xE9e :"),$e.forEach(t),K=c(v),j(T.$$.fragment,v),x=c(v),j(P.$$.fragment,v),this.h()},h(){k(m,"href","/course/frchapter2")},m(v,B){p(v,d,B),e(d,g),e(d,m),e(m,y),e(d,R),e(d,w),e(w,S),e(d,N),e(d,L),e(L,C),e(d,ne),p(v,Q,B),_(M,v,B),p(v,H,B),p(v,D,B),e(D,z),e(D,se),e(se,te),e(D,ge),e(D,ae),e(ae,J),e(D,pe),p(v,K,B),_(T,v,B),p(v,x,B),_(P,v,B),ue=!0},i(v){ue||(h(M.$$.fragment,v),h(T.$$.fragment,v),h(P.$$.fragment,v),ue=!0)},o(v){f(M.$$.fragment,v),f(T.$$.fragment,v),f(P.$$.fragment,v),ue=!1},d(v){v&&t(d),v&&t(Q),E(M,v),v&&t(H),v&&t(D),v&&t(K),E(T,v),v&&t(x),E(P,v)}}}function o7(X){let d,g,m,y,R,w,S,N,L,C,ne,Q,M,H,D,z,se,te,ge,ae,J,pe,K,T;return M=new q({props:{code:`from transformers import AutoTokenizer, AutoModelForTokenClassification

model_checkpoint = "dbmdz/bert-large-cased-finetuned-conll03-english"
tokenizer = AutoTokenizer.from_pretrained(model_checkpoint)
model = AutoModelForTokenClassification.from_pretrained(model_checkpoint)

example = "My name is Sylvain and I work at Hugging Face in Brooklyn."
inputs = tokenizer(example, return_tensors="pt")
outputs = model(**inputs)`,highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoTokenizer, AutoModelForTokenClassification

model_checkpoint = <span class="hljs-string">&quot;dbmdz/bert-large-cased-finetuned-conll03-english&quot;</span>
tokenizer = AutoTokenizer.from_pretrained(model_checkpoint)
model = AutoModelForTokenClassification.from_pretrained(model_checkpoint)

example = <span class="hljs-string">&quot;My name is Sylvain and I work at Hugging Face in Brooklyn.&quot;</span>
inputs = tokenizer(example, return_tensors=<span class="hljs-string">&quot;pt&quot;</span>)
outputs = model(**inputs)`}}),J=new q({props:{code:`print(inputs["input_ids"].shape)
print(outputs.logits.shape)`,highlighted:`<span class="hljs-built_in">print</span>(inputs[<span class="hljs-string">&quot;input_ids&quot;</span>].shape)
<span class="hljs-built_in">print</span>(outputs.logits.shape)`}}),K=new q({props:{code:`torch.Size([1, 19])
torch.Size([1, 19, 9])`,highlighted:`torch.Size([<span class="hljs-number">1</span>, <span class="hljs-number">19</span>])
torch.Size([<span class="hljs-number">1</span>, <span class="hljs-number">19</span>, <span class="hljs-number">9</span>])`}}),{c(){d=l("p"),g=n("D\u2019abord, nous devons tokeniser notre entr\xE9e et la faire passer dans le mod\xE8le. Cela se fait exactement comme dans le "),m=l("a"),y=n("Chapitre 2"),R=n(". Nous instancions le "),w=l("em"),S=n("tokenizer"),N=n(" et le mod\xE8le en utilisant les classes "),L=l("code"),C=n("TFAutoXxx"),ne=n(" et les utilisons ensuite dans notre exemple :"),Q=u(),b(M.$$.fragment),H=u(),D=l("p"),z=n("Puisque nous utilisons "),se=l("code"),te=n("AutoModelForTokenClassification"),ge=n(" ici, nous obtenons un ensemble de logits pour chaque token dans la s\xE9quence d\u2019entr\xE9e :"),ae=u(),b(J.$$.fragment),pe=u(),b(K.$$.fragment),this.h()},l(x){d=r(x,"P",{});var P=o(d);g=a(P,"D\u2019abord, nous devons tokeniser notre entr\xE9e et la faire passer dans le mod\xE8le. Cela se fait exactement comme dans le "),m=r(P,"A",{href:!0});var ue=o(m);y=a(ue,"Chapitre 2"),ue.forEach(t),R=a(P,". Nous instancions le "),w=r(P,"EM",{});var v=o(w);S=a(v,"tokenizer"),v.forEach(t),N=a(P," et le mod\xE8le en utilisant les classes "),L=r(P,"CODE",{});var B=o(L);C=a(B,"TFAutoXxx"),B.forEach(t),ne=a(P," et les utilisons ensuite dans notre exemple :"),P.forEach(t),Q=c(x),j(M.$$.fragment,x),H=c(x),D=r(x,"P",{});var ce=o(D);z=a(ce,"Puisque nous utilisons "),se=r(ce,"CODE",{});var ke=o(se);te=a(ke,"AutoModelForTokenClassification"),ke.forEach(t),ge=a(ce," ici, nous obtenons un ensemble de logits pour chaque token dans la s\xE9quence d\u2019entr\xE9e :"),ce.forEach(t),ae=c(x),j(J.$$.fragment,x),pe=c(x),j(K.$$.fragment,x),this.h()},h(){k(m,"href","/course/frchapter2")},m(x,P){p(x,d,P),e(d,g),e(d,m),e(m,y),e(d,R),e(d,w),e(w,S),e(d,N),e(d,L),e(L,C),e(d,ne),p(x,Q,P),_(M,x,P),p(x,H,P),p(x,D,P),e(D,z),e(D,se),e(se,te),e(D,ge),p(x,ae,P),_(J,x,P),p(x,pe,P),_(K,x,P),T=!0},i(x){T||(h(M.$$.fragment,x),h(J.$$.fragment,x),h(K.$$.fragment,x),T=!0)},o(x){f(M.$$.fragment,x),f(J.$$.fragment,x),f(K.$$.fragment,x),T=!1},d(x){x&&t(d),x&&t(Q),E(M,x),x&&t(H),x&&t(D),x&&t(ae),E(J,x),x&&t(pe),E(K,x)}}}function i7(X){let d,g;return d=new q({props:{code:`import tensorflow as tf

probabilities = tf.math.softmax(outputs.logits, axis=-1)[0]
probabilities = probabilities.numpy().tolist()
predictions = tf.math.argmax(outputs.logits, axis=-1)[0]
predictions = predictions.numpy().tolist()
print(predictions)`,highlighted:`<span class="hljs-keyword">import</span> tensorflow <span class="hljs-keyword">as</span> tf

probabilities = tf.math.softmax(outputs.logits, axis=-<span class="hljs-number">1</span>)[<span class="hljs-number">0</span>]
probabilities = probabilities.numpy().tolist()
predictions = tf.math.argmax(outputs.logits, axis=-<span class="hljs-number">1</span>)[<span class="hljs-number">0</span>]
predictions = predictions.numpy().tolist()
<span class="hljs-built_in">print</span>(predictions)`}}),{c(){b(d.$$.fragment)},l(m){j(d.$$.fragment,m)},m(m,y){_(d,m,y),g=!0},i(m){g||(h(d.$$.fragment,m),g=!0)},o(m){f(d.$$.fragment,m),g=!1},d(m){E(d,m)}}}function p7(X){let d,g;return d=new q({props:{code:`import torch

probabilities = torch.nn.functional.softmax(outputs.logits, dim=-1)[0].tolist()
predictions = outputs.logits.argmax(dim=-1)[0].tolist()
print(predictions)`,highlighted:`<span class="hljs-keyword">import</span> torch

probabilities = torch.nn.functional.softmax(outputs.logits, dim=-<span class="hljs-number">1</span>)[<span class="hljs-number">0</span>].tolist()
predictions = outputs.logits.argmax(dim=-<span class="hljs-number">1</span>)[<span class="hljs-number">0</span>].tolist()
<span class="hljs-built_in">print</span>(predictions)`}}),{c(){b(d.$$.fragment)},l(m){j(d.$$.fragment,m)},m(m,y){_(d,m,y),g=!0},i(m){g||(h(d.$$.fragment,m),g=!0)},o(m){f(d.$$.fragment,m),g=!1},d(m){E(d,m)}}}function u7(X){let d,g,m,y,R,w,S,N,L,C,ne,Q,M,H,D,z,se,te,ge,ae,J,pe,K,T,x,P,ue,v,B,ce,ke,He,$e,xt,gt,Yt,Bo,Ao,vt,Fo,No,Ul,_s,Yl,W,Lo,Kt,Xo,Ho,Wt,Go,Vo,Zt,Qo,Jo,en,Uo,Yo,bt,Ko,Wo,sn,Zo,ei,tn,si,ti,Kl,es,nn,Ge,Es,an,ni,ai,li,ks,ln,ri,oi,ii,rn,pi,$s,Ve,jt,on,ui,ci,_t,di,mi,Et,fi,hi,Qe,kt,pn,xi,gi,$t,vi,bi,qt,ji,Wl,ss,Zl,Je,ts,un,qs,_i,cn,Ei,er,ys,sr,qe,ki,dn,$i,qi,mn,yi,wi,fn,Ci,Oi,tr,de,zi,hn,Pi,Di,xn,Ii,Mi,gn,Ti,Ri,vn,Si,Bi,bn,Ai,Fi,nr,yt,Ni,ar,ws,lr,Se,Li,jn,Xi,Hi,_n,Gi,Vi,rr,Cs,or,le,Qi,En,Ji,Ui,kn,Yi,Ki,$n,Wi,Zi,qn,ep,sp,yn,tp,np,wn,ap,lp,ir,Os,pr,zs,ur,ns,rp,Cn,op,ip,cr,Ps,dr,Ds,mr,ye,pp,On,up,cp,zn,dp,mp,Pn,fp,hp,fr,Is,hr,Ms,xr,ve,xp,Dn,gp,vp,In,bp,jp,Mn,_p,Ep,Tn,kp,$p,gr,Ts,vr,Rs,br,I,qp,Rn,yp,wp,Sn,Cp,Op,Bn,zp,Pp,An,Dp,Ip,Fn,Mp,Tp,Nn,Rp,Sp,Ln,Bp,Ap,Xn,Fp,Np,Hn,Lp,Xp,Gn,Hp,Gp,Vn,Vp,Qp,Qn,Jp,Up,Jn,Yp,Kp,jr,as,_r,we,Wp,Un,Zp,eu,Yn,su,tu,Kn,nu,au,Er,re,lu,Wn,ru,ou,Zn,iu,pu,ea,uu,cu,sa,du,mu,ta,fu,hu,na,xu,gu,kr,Ss,$r,Bs,qr,Be,vu,aa,bu,ju,la,_u,Eu,yr,ls,wr,Ue,rs,ra,As,ku,wt,$u,oa,qu,Cr,me,yu,Ct,wu,Cu,ia,Ou,zu,pa,Pu,Du,Ot,Iu,Mu,ua,Tu,Ru,Or,Pe,De,zt,Ye,os,ca,Fs,Su,da,Bu,zr,is,Au,Ns,ma,Fu,Nu,Pr,Ls,Dr,Xs,Ir,ps,Lu,fa,Xu,Hu,Mr,Hs,Tr,Gs,Rr,Z,Gu,ha,Vu,Qu,xa,Ju,Uu,ga,Yu,Ku,va,Wu,Zu,ba,ec,sc,ja,tc,nc,_a,ac,lc,Sr,Ae,us,Ea,rc,oc,ka,ic,pc,uc,Pt,$a,cc,dc,mc,cs,qa,fc,hc,ya,xc,gc,Br,ds,vc,wa,bc,jc,Ar,Ke,ms,Ca,Vs,_c,Oa,Ec,Fr,Ie,Me,Dt,fs,kc,za,$c,qc,Nr,Te,Re,It,Qs,Lr,hs,yc,Pa,wc,Cc,Xr,Js,Hr,Us,Gr,$,Oc,Da,zc,Pc,Ia,Dc,Ic,Ma,Mc,Tc,Ta,Rc,Sc,Ra,Bc,Ac,Sa,Fc,Nc,Ba,Lc,Xc,Aa,Hc,Gc,Fa,Vc,Qc,Na,Jc,Uc,La,Yc,Kc,Xa,Wc,Zc,Ha,ed,sd,Ga,td,nd,Va,ad,ld,Qa,rd,od,Ja,id,pd,Vr,A,ud,Ua,cd,dd,Ya,md,fd,Ka,hd,xd,Wa,gd,vd,Za,bd,jd,el,_d,Ed,sl,kd,$d,tl,qd,yd,nl,wd,Cd,al,Od,zd,ll,Pd,Dd,Qr,We,Ys,bf,Id,Ks,jf,Jr,Fe,Md,rl,Td,Rd,ol,Sd,Bd,Ur,Ws,Yr,Zs,Kr,be,Ad,il,Fd,Nd,pl,Ld,Xd,ul,Hd,Gd,cl,Vd,Qd,Wr,et,Zr,st,eo,oe,Jd,dl,Ud,Yd,ml,Kd,Wd,fl,Zd,em,hl,sm,tm,xl,nm,am,gl,lm,rm,so,tt,to,xs,om,vl,im,pm,no,nt,ao,Mt,um,lo,at,ro,lt,oo,Tt,cm,io,Ze,gs,bl,rt,dm,jl,mm,po,G,fm,_l,hm,xm,El,gm,vm,kl,bm,jm,$l,_m,Em,ql,km,$m,yl,qm,ym,wl,wm,Cm,Cl,Om,zm,Ol,Pm,Dm,uo,U,Im,zl,Mm,Tm,Pl,Rm,Sm,Dl,Bm,Am,Il,Fm,Nm,Ml,Lm,Xm,Tl,Hm,Gm,Rl,Vm,Qm,Sl,Jm,Um,co,ot,mo,it,fo,fe,Ym,Bl,Km,Wm,Al,Zm,ef,Fl,sf,tf,Nl,nf,af,Ll,lf,rf,ho,pt,xo,Rt,of,go,ut,vo,Ne,pf,Xl,uf,cf,Hl,df,mf,bo;m=new Wx({props:{fw:X[0]}}),N=new Ut({});const _f=[e7,Zx],ct=[];function Ef(s,i){return s[0]==="pt"?0:1}M=Ef(X),H=ct[M]=_f[M](X),_s=new So({props:{id:"g8quOxoqhHQ"}}),ss=new vf({props:{warning:!0,$$slots:{default:[s7]},$$scope:{ctx:X}}}),qs=new Ut({}),ys=new So({props:{id:"3umI3tm27Vw"}}),ws=new q({props:{code:`from transformers import AutoTokenizer

tokenizer = AutoTokenizer.from_pretrained("bert-base-cased")
example = "My name is Sylvain and I work at Hugging Face in Brooklyn."  # "Je m'appelle Sylvain et je travaille chez Hugging Face \xE0 Brooklyn."
encoding = tokenizer(example)
print(type(encoding))`,highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoTokenizer

tokenizer = AutoTokenizer.from_pretrained(<span class="hljs-string">&quot;bert-base-cased&quot;</span>)
example = <span class="hljs-string">&quot;My name is Sylvain and I work at Hugging Face in Brooklyn.&quot;</span>  <span class="hljs-comment"># &quot;Je m&#x27;appelle Sylvain et je travaille chez Hugging Face \xE0 Brooklyn.&quot;</span>
encoding = tokenizer(example)
<span class="hljs-built_in">print</span>(<span class="hljs-built_in">type</span>(encoding))`}}),Cs=new q({props:{code:"<class 'transformers.tokenization_utils_base.BatchEncoding'>",highlighted:'&lt;<span class="hljs-keyword">class</span> <span class="hljs-string">&#x27;transformers.tokenization_utils_base.BatchEncoding&#x27;</span>&gt;'}}),Os=new q({props:{code:"tokenizer.is_fast",highlighted:"tokenizer.is_fast"}}),zs=new q({props:{code:"True",highlighted:'<span class="hljs-literal">True</span>'}}),Ps=new q({props:{code:"encoding.is_fast",highlighted:"encoding.is_fast"}}),Ds=new q({props:{code:"True",highlighted:'<span class="hljs-literal">True</span>'}}),Is=new q({props:{code:"encoding.tokens()",highlighted:"encoding.tokens()"}}),Ms=new q({props:{code:`['[CLS]', 'My', 'name', 'is', 'S', '##yl', '##va', '##in', 'and', 'I', 'work', 'at', 'Hu', '##gging', 'Face', 'in',
 'Brooklyn', '.', '[SEP]']`,highlighted:`[<span class="hljs-string">&#x27;[CLS]&#x27;</span>, <span class="hljs-string">&#x27;My&#x27;</span>, <span class="hljs-string">&#x27;name&#x27;</span>, <span class="hljs-string">&#x27;is&#x27;</span>, <span class="hljs-string">&#x27;S&#x27;</span>, <span class="hljs-string">&#x27;##yl&#x27;</span>, <span class="hljs-string">&#x27;##va&#x27;</span>, <span class="hljs-string">&#x27;##in&#x27;</span>, <span class="hljs-string">&#x27;and&#x27;</span>, <span class="hljs-string">&#x27;I&#x27;</span>, <span class="hljs-string">&#x27;work&#x27;</span>, <span class="hljs-string">&#x27;at&#x27;</span>, <span class="hljs-string">&#x27;Hu&#x27;</span>, <span class="hljs-string">&#x27;##gging&#x27;</span>, <span class="hljs-string">&#x27;Face&#x27;</span>, <span class="hljs-string">&#x27;in&#x27;</span>,
 <span class="hljs-string">&#x27;Brooklyn&#x27;</span>, <span class="hljs-string">&#x27;.&#x27;</span>, <span class="hljs-string">&#x27;[SEP]&#x27;</span>]`}}),Ts=new q({props:{code:"encoding.word_ids()",highlighted:"encoding.word_ids()"}}),Rs=new q({props:{code:"[None, 0, 1, 2, 3, 3, 3, 3, 4, 5, 6, 7, 8, 8, 9, 10, 11, 12, None]",highlighted:'[<span class="hljs-literal">None</span>, <span class="hljs-number">0</span>, <span class="hljs-number">1</span>, <span class="hljs-number">2</span>, <span class="hljs-number">3</span>, <span class="hljs-number">3</span>, <span class="hljs-number">3</span>, <span class="hljs-number">3</span>, <span class="hljs-number">4</span>, <span class="hljs-number">5</span>, <span class="hljs-number">6</span>, <span class="hljs-number">7</span>, <span class="hljs-number">8</span>, <span class="hljs-number">8</span>, <span class="hljs-number">9</span>, <span class="hljs-number">10</span>, <span class="hljs-number">11</span>, <span class="hljs-number">12</span>, <span class="hljs-literal">None</span>]'}}),as=new vf({props:{$$slots:{default:[t7]},$$scope:{ctx:X}}}),Ss=new q({props:{code:`start, end = encoding.word_to_chars(3)
example[start:end]`,highlighted:`start, end = encoding.word_to_chars(<span class="hljs-number">3</span>)
example[start:end]`}}),Bs=new q({props:{code:"Sylvain",highlighted:"Sylvain"}}),ls=new vf({props:{$$slots:{default:[n7]},$$scope:{ctx:X}}}),As=new Ut({});const kf=[l7,a7],dt=[];function $f(s,i){return s[0]==="pt"?0:1}Pe=$f(X),De=dt[Pe]=kf[Pe](X),Fs=new Ut({}),Ls=new q({props:{code:`from transformers import pipeline

token_classifier = pipeline("token-classification")
token_classifier("My name is Sylvain and I work at Hugging Face in Brooklyn.")`,highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> pipeline

token_classifier = pipeline(<span class="hljs-string">&quot;token-classification&quot;</span>)
token_classifier(<span class="hljs-string">&quot;My name is Sylvain and I work at Hugging Face in Brooklyn.&quot;</span>)`}}),Xs=new q({props:{code:`[{'entity': 'I-PER', 'score': 0.9993828, 'index': 4, 'word': 'S', 'start': 11, 'end': 12},
 {'entity': 'I-PER', 'score': 0.99815476, 'index': 5, 'word': '##yl', 'start': 12, 'end': 14},
 {'entity': 'I-PER', 'score': 0.99590725, 'index': 6, 'word': '##va', 'start': 14, 'end': 16},
 {'entity': 'I-PER', 'score': 0.9992327, 'index': 7, 'word': '##in', 'start': 16, 'end': 18},
 {'entity': 'I-ORG', 'score': 0.97389334, 'index': 12, 'word': 'Hu', 'start': 33, 'end': 35},
 {'entity': 'I-ORG', 'score': 0.976115, 'index': 13, 'word': '##gging', 'start': 35, 'end': 40},
 {'entity': 'I-ORG', 'score': 0.98879766, 'index': 14, 'word': 'Face', 'start': 41, 'end': 45},
 {'entity': 'I-LOC', 'score': 0.99321055, 'index': 16, 'word': 'Brooklyn', 'start': 49, 'end': 57}]`,highlighted:`[{<span class="hljs-string">&#x27;entity&#x27;</span>: <span class="hljs-string">&#x27;I-PER&#x27;</span>, <span class="hljs-string">&#x27;score&#x27;</span>: <span class="hljs-number">0.9993828</span>, <span class="hljs-string">&#x27;index&#x27;</span>: <span class="hljs-number">4</span>, <span class="hljs-string">&#x27;word&#x27;</span>: <span class="hljs-string">&#x27;S&#x27;</span>, <span class="hljs-string">&#x27;start&#x27;</span>: <span class="hljs-number">11</span>, <span class="hljs-string">&#x27;end&#x27;</span>: <span class="hljs-number">12</span>},
 {<span class="hljs-string">&#x27;entity&#x27;</span>: <span class="hljs-string">&#x27;I-PER&#x27;</span>, <span class="hljs-string">&#x27;score&#x27;</span>: <span class="hljs-number">0.99815476</span>, <span class="hljs-string">&#x27;index&#x27;</span>: <span class="hljs-number">5</span>, <span class="hljs-string">&#x27;word&#x27;</span>: <span class="hljs-string">&#x27;##yl&#x27;</span>, <span class="hljs-string">&#x27;start&#x27;</span>: <span class="hljs-number">12</span>, <span class="hljs-string">&#x27;end&#x27;</span>: <span class="hljs-number">14</span>},
 {<span class="hljs-string">&#x27;entity&#x27;</span>: <span class="hljs-string">&#x27;I-PER&#x27;</span>, <span class="hljs-string">&#x27;score&#x27;</span>: <span class="hljs-number">0.99590725</span>, <span class="hljs-string">&#x27;index&#x27;</span>: <span class="hljs-number">6</span>, <span class="hljs-string">&#x27;word&#x27;</span>: <span class="hljs-string">&#x27;##va&#x27;</span>, <span class="hljs-string">&#x27;start&#x27;</span>: <span class="hljs-number">14</span>, <span class="hljs-string">&#x27;end&#x27;</span>: <span class="hljs-number">16</span>},
 {<span class="hljs-string">&#x27;entity&#x27;</span>: <span class="hljs-string">&#x27;I-PER&#x27;</span>, <span class="hljs-string">&#x27;score&#x27;</span>: <span class="hljs-number">0.9992327</span>, <span class="hljs-string">&#x27;index&#x27;</span>: <span class="hljs-number">7</span>, <span class="hljs-string">&#x27;word&#x27;</span>: <span class="hljs-string">&#x27;##in&#x27;</span>, <span class="hljs-string">&#x27;start&#x27;</span>: <span class="hljs-number">16</span>, <span class="hljs-string">&#x27;end&#x27;</span>: <span class="hljs-number">18</span>},
 {<span class="hljs-string">&#x27;entity&#x27;</span>: <span class="hljs-string">&#x27;I-ORG&#x27;</span>, <span class="hljs-string">&#x27;score&#x27;</span>: <span class="hljs-number">0.97389334</span>, <span class="hljs-string">&#x27;index&#x27;</span>: <span class="hljs-number">12</span>, <span class="hljs-string">&#x27;word&#x27;</span>: <span class="hljs-string">&#x27;Hu&#x27;</span>, <span class="hljs-string">&#x27;start&#x27;</span>: <span class="hljs-number">33</span>, <span class="hljs-string">&#x27;end&#x27;</span>: <span class="hljs-number">35</span>},
 {<span class="hljs-string">&#x27;entity&#x27;</span>: <span class="hljs-string">&#x27;I-ORG&#x27;</span>, <span class="hljs-string">&#x27;score&#x27;</span>: <span class="hljs-number">0.976115</span>, <span class="hljs-string">&#x27;index&#x27;</span>: <span class="hljs-number">13</span>, <span class="hljs-string">&#x27;word&#x27;</span>: <span class="hljs-string">&#x27;##gging&#x27;</span>, <span class="hljs-string">&#x27;start&#x27;</span>: <span class="hljs-number">35</span>, <span class="hljs-string">&#x27;end&#x27;</span>: <span class="hljs-number">40</span>},
 {<span class="hljs-string">&#x27;entity&#x27;</span>: <span class="hljs-string">&#x27;I-ORG&#x27;</span>, <span class="hljs-string">&#x27;score&#x27;</span>: <span class="hljs-number">0.98879766</span>, <span class="hljs-string">&#x27;index&#x27;</span>: <span class="hljs-number">14</span>, <span class="hljs-string">&#x27;word&#x27;</span>: <span class="hljs-string">&#x27;Face&#x27;</span>, <span class="hljs-string">&#x27;start&#x27;</span>: <span class="hljs-number">41</span>, <span class="hljs-string">&#x27;end&#x27;</span>: <span class="hljs-number">45</span>},
 {<span class="hljs-string">&#x27;entity&#x27;</span>: <span class="hljs-string">&#x27;I-LOC&#x27;</span>, <span class="hljs-string">&#x27;score&#x27;</span>: <span class="hljs-number">0.99321055</span>, <span class="hljs-string">&#x27;index&#x27;</span>: <span class="hljs-number">16</span>, <span class="hljs-string">&#x27;word&#x27;</span>: <span class="hljs-string">&#x27;Brooklyn&#x27;</span>, <span class="hljs-string">&#x27;start&#x27;</span>: <span class="hljs-number">49</span>, <span class="hljs-string">&#x27;end&#x27;</span>: <span class="hljs-number">57</span>}]`}}),Hs=new q({props:{code:`from transformers import pipeline

token_classifier = pipeline("token-classification", aggregation_strategy="simple")
token_classifier("My name is Sylvain and I work at Hugging Face in Brooklyn.")`,highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> pipeline

token_classifier = pipeline(<span class="hljs-string">&quot;token-classification&quot;</span>, aggregation_strategy=<span class="hljs-string">&quot;simple&quot;</span>)
token_classifier(<span class="hljs-string">&quot;My name is Sylvain and I work at Hugging Face in Brooklyn.&quot;</span>)`}}),Gs=new q({props:{code:`[{'entity_group': 'PER', 'score': 0.9981694, 'word': 'Sylvain', 'start': 11, 'end': 18},
 {'entity_group': 'ORG', 'score': 0.97960204, 'word': 'Hugging Face', 'start': 33, 'end': 45},
 {'entity_group': 'LOC', 'score': 0.99321055, 'word': 'Brooklyn', 'start': 49, 'end': 57}]`,highlighted:`[{<span class="hljs-string">&#x27;entity_group&#x27;</span>: <span class="hljs-string">&#x27;PER&#x27;</span>, <span class="hljs-string">&#x27;score&#x27;</span>: <span class="hljs-number">0.9981694</span>, <span class="hljs-string">&#x27;word&#x27;</span>: <span class="hljs-string">&#x27;Sylvain&#x27;</span>, <span class="hljs-string">&#x27;start&#x27;</span>: <span class="hljs-number">11</span>, <span class="hljs-string">&#x27;end&#x27;</span>: <span class="hljs-number">18</span>},
 {<span class="hljs-string">&#x27;entity_group&#x27;</span>: <span class="hljs-string">&#x27;ORG&#x27;</span>, <span class="hljs-string">&#x27;score&#x27;</span>: <span class="hljs-number">0.97960204</span>, <span class="hljs-string">&#x27;word&#x27;</span>: <span class="hljs-string">&#x27;Hugging Face&#x27;</span>, <span class="hljs-string">&#x27;start&#x27;</span>: <span class="hljs-number">33</span>, <span class="hljs-string">&#x27;end&#x27;</span>: <span class="hljs-number">45</span>},
 {<span class="hljs-string">&#x27;entity_group&#x27;</span>: <span class="hljs-string">&#x27;LOC&#x27;</span>, <span class="hljs-string">&#x27;score&#x27;</span>: <span class="hljs-number">0.99321055</span>, <span class="hljs-string">&#x27;word&#x27;</span>: <span class="hljs-string">&#x27;Brooklyn&#x27;</span>, <span class="hljs-string">&#x27;start&#x27;</span>: <span class="hljs-number">49</span>, <span class="hljs-string">&#x27;end&#x27;</span>: <span class="hljs-number">57</span>}]`}}),Vs=new Ut({});const qf=[o7,r7],mt=[];function yf(s,i){return s[0]==="pt"?0:1}Ie=yf(X),Me=mt[Ie]=qf[Ie](X);const wf=[p7,i7],ft=[];function Cf(s,i){return s[0]==="pt"?0:1}return Te=Cf(X),Re=ft[Te]=wf[Te](X),Qs=new q({props:{code:"[0, 0, 0, 0, 4, 4, 4, 4, 0, 0, 0, 0, 6, 6, 6, 0, 8, 0, 0]",highlighted:'[<span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">4</span>, <span class="hljs-number">4</span>, <span class="hljs-number">4</span>, <span class="hljs-number">4</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">6</span>, <span class="hljs-number">6</span>, <span class="hljs-number">6</span>, <span class="hljs-number">0</span>, <span class="hljs-number">8</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>]'}}),Js=new q({props:{code:"model.config.id2label",highlighted:"model.config.id2label"}}),Us=new q({props:{code:`{0: 'O',
 1: 'B-MISC',
 2: 'I-MISC',
 3: 'B-PER',
 4: 'I-PER',
 5: 'B-ORG',
 6: 'I-ORG',
 7: 'B-LOC',
 8: 'I-LOC'}`,highlighted:`{<span class="hljs-number">0</span>: <span class="hljs-string">&#x27;O&#x27;</span>,
 <span class="hljs-number">1</span>: <span class="hljs-string">&#x27;B-MISC&#x27;</span>,
 <span class="hljs-number">2</span>: <span class="hljs-string">&#x27;I-MISC&#x27;</span>,
 <span class="hljs-number">3</span>: <span class="hljs-string">&#x27;B-PER&#x27;</span>,
 <span class="hljs-number">4</span>: <span class="hljs-string">&#x27;I-PER&#x27;</span>,
 <span class="hljs-number">5</span>: <span class="hljs-string">&#x27;B-ORG&#x27;</span>,
 <span class="hljs-number">6</span>: <span class="hljs-string">&#x27;I-ORG&#x27;</span>,
 <span class="hljs-number">7</span>: <span class="hljs-string">&#x27;B-LOC&#x27;</span>,
 <span class="hljs-number">8</span>: <span class="hljs-string">&#x27;I-LOC&#x27;</span>}`}}),Ws=new q({props:{code:`results = []
tokens = inputs.tokens()

for idx, pred in enumerate(predictions):
    label = model.config.id2label[pred]
    if label != "O":
        results.append(
            {"entity": label, "score": probabilities[idx][pred], "word": tokens[idx]}
        )

print(results)`,highlighted:`results = []
tokens = inputs.tokens()

<span class="hljs-keyword">for</span> idx, pred <span class="hljs-keyword">in</span> <span class="hljs-built_in">enumerate</span>(predictions):
    label = model.config.id2label[pred]
    <span class="hljs-keyword">if</span> label != <span class="hljs-string">&quot;O&quot;</span>:
        results.append(
            {<span class="hljs-string">&quot;entity&quot;</span>: label, <span class="hljs-string">&quot;score&quot;</span>: probabilities[idx][pred], <span class="hljs-string">&quot;word&quot;</span>: tokens[idx]}
        )

<span class="hljs-built_in">print</span>(results)`}}),Zs=new q({props:{code:`[{'entity': 'I-PER', 'score': 0.9993828, 'index': 4, 'word': 'S'},
 {'entity': 'I-PER', 'score': 0.99815476, 'index': 5, 'word': '##yl'},
 {'entity': 'I-PER', 'score': 0.99590725, 'index': 6, 'word': '##va'},
 {'entity': 'I-PER', 'score': 0.9992327, 'index': 7, 'word': '##in'},
 {'entity': 'I-ORG', 'score': 0.97389334, 'index': 12, 'word': 'Hu'},
 {'entity': 'I-ORG', 'score': 0.976115, 'index': 13, 'word': '##gging'},
 {'entity': 'I-ORG', 'score': 0.98879766, 'index': 14, 'word': 'Face'},
 {'entity': 'I-LOC', 'score': 0.99321055, 'index': 16, 'word': 'Brooklyn'}]`,highlighted:`[{<span class="hljs-string">&#x27;entity&#x27;</span>: <span class="hljs-string">&#x27;I-PER&#x27;</span>, <span class="hljs-string">&#x27;score&#x27;</span>: <span class="hljs-number">0.9993828</span>, <span class="hljs-string">&#x27;index&#x27;</span>: <span class="hljs-number">4</span>, <span class="hljs-string">&#x27;word&#x27;</span>: <span class="hljs-string">&#x27;S&#x27;</span>},
 {<span class="hljs-string">&#x27;entity&#x27;</span>: <span class="hljs-string">&#x27;I-PER&#x27;</span>, <span class="hljs-string">&#x27;score&#x27;</span>: <span class="hljs-number">0.99815476</span>, <span class="hljs-string">&#x27;index&#x27;</span>: <span class="hljs-number">5</span>, <span class="hljs-string">&#x27;word&#x27;</span>: <span class="hljs-string">&#x27;##yl&#x27;</span>},
 {<span class="hljs-string">&#x27;entity&#x27;</span>: <span class="hljs-string">&#x27;I-PER&#x27;</span>, <span class="hljs-string">&#x27;score&#x27;</span>: <span class="hljs-number">0.99590725</span>, <span class="hljs-string">&#x27;index&#x27;</span>: <span class="hljs-number">6</span>, <span class="hljs-string">&#x27;word&#x27;</span>: <span class="hljs-string">&#x27;##va&#x27;</span>},
 {<span class="hljs-string">&#x27;entity&#x27;</span>: <span class="hljs-string">&#x27;I-PER&#x27;</span>, <span class="hljs-string">&#x27;score&#x27;</span>: <span class="hljs-number">0.9992327</span>, <span class="hljs-string">&#x27;index&#x27;</span>: <span class="hljs-number">7</span>, <span class="hljs-string">&#x27;word&#x27;</span>: <span class="hljs-string">&#x27;##in&#x27;</span>},
 {<span class="hljs-string">&#x27;entity&#x27;</span>: <span class="hljs-string">&#x27;I-ORG&#x27;</span>, <span class="hljs-string">&#x27;score&#x27;</span>: <span class="hljs-number">0.97389334</span>, <span class="hljs-string">&#x27;index&#x27;</span>: <span class="hljs-number">12</span>, <span class="hljs-string">&#x27;word&#x27;</span>: <span class="hljs-string">&#x27;Hu&#x27;</span>},
 {<span class="hljs-string">&#x27;entity&#x27;</span>: <span class="hljs-string">&#x27;I-ORG&#x27;</span>, <span class="hljs-string">&#x27;score&#x27;</span>: <span class="hljs-number">0.976115</span>, <span class="hljs-string">&#x27;index&#x27;</span>: <span class="hljs-number">13</span>, <span class="hljs-string">&#x27;word&#x27;</span>: <span class="hljs-string">&#x27;##gging&#x27;</span>},
 {<span class="hljs-string">&#x27;entity&#x27;</span>: <span class="hljs-string">&#x27;I-ORG&#x27;</span>, <span class="hljs-string">&#x27;score&#x27;</span>: <span class="hljs-number">0.98879766</span>, <span class="hljs-string">&#x27;index&#x27;</span>: <span class="hljs-number">14</span>, <span class="hljs-string">&#x27;word&#x27;</span>: <span class="hljs-string">&#x27;Face&#x27;</span>},
 {<span class="hljs-string">&#x27;entity&#x27;</span>: <span class="hljs-string">&#x27;I-LOC&#x27;</span>, <span class="hljs-string">&#x27;score&#x27;</span>: <span class="hljs-number">0.99321055</span>, <span class="hljs-string">&#x27;index&#x27;</span>: <span class="hljs-number">16</span>, <span class="hljs-string">&#x27;word&#x27;</span>: <span class="hljs-string">&#x27;Brooklyn&#x27;</span>}]`}}),et=new q({props:{code:`inputs_with_offsets = tokenizer(example, return_offsets_mapping=True)
inputs_with_offsets["offset_mapping"]`,highlighted:`inputs_with_offsets = tokenizer(example, return_offsets_mapping=<span class="hljs-literal">True</span>)
inputs_with_offsets[<span class="hljs-string">&quot;offset_mapping&quot;</span>]`}}),st=new q({props:{code:`[(0, 0), (0, 2), (3, 7), (8, 10), (11, 12), (12, 14), (14, 16), (16, 18), (19, 22), (23, 24), (25, 29), (30, 32),
 (33, 35), (35, 40), (41, 45), (46, 48), (49, 57), (57, 58), (0, 0)]`,highlighted:`[(<span class="hljs-number">0</span>, <span class="hljs-number">0</span>), (<span class="hljs-number">0</span>, <span class="hljs-number">2</span>), (<span class="hljs-number">3</span>, <span class="hljs-number">7</span>), (<span class="hljs-number">8</span>, <span class="hljs-number">10</span>), (<span class="hljs-number">11</span>, <span class="hljs-number">12</span>), (<span class="hljs-number">12</span>, <span class="hljs-number">14</span>), (<span class="hljs-number">14</span>, <span class="hljs-number">16</span>), (<span class="hljs-number">16</span>, <span class="hljs-number">18</span>), (<span class="hljs-number">19</span>, <span class="hljs-number">22</span>), (<span class="hljs-number">23</span>, <span class="hljs-number">24</span>), (<span class="hljs-number">25</span>, <span class="hljs-number">29</span>), (<span class="hljs-number">30</span>, <span class="hljs-number">32</span>),
 (<span class="hljs-number">33</span>, <span class="hljs-number">35</span>), (<span class="hljs-number">35</span>, <span class="hljs-number">40</span>), (<span class="hljs-number">41</span>, <span class="hljs-number">45</span>), (<span class="hljs-number">46</span>, <span class="hljs-number">48</span>), (<span class="hljs-number">49</span>, <span class="hljs-number">57</span>), (<span class="hljs-number">57</span>, <span class="hljs-number">58</span>), (<span class="hljs-number">0</span>, <span class="hljs-number">0</span>)]`}}),tt=new q({props:{code:"example[12:14]",highlighted:'example[<span class="hljs-number">12</span>:<span class="hljs-number">14</span>]'}}),nt=new q({props:{code:"yl",highlighted:"yl"}}),at=new q({props:{code:`results = []
inputs_with_offsets = tokenizer(example, return_offsets_mapping=True)
tokens = inputs_with_offsets.tokens()
offsets = inputs_with_offsets["offset_mapping"]

for idx, pred in enumerate(predictions):
    label = model.config.id2label[pred]
    if label != "O":
        start, end = offsets[idx]
        results.append(
            {
                "entity": label,
                "score": probabilities[idx][pred],
                "word": tokens[idx],
                "start": start,
                "end": end,
            }
        )

print(results)`,highlighted:`results = []
inputs_with_offsets = tokenizer(example, return_offsets_mapping=<span class="hljs-literal">True</span>)
tokens = inputs_with_offsets.tokens()
offsets = inputs_with_offsets[<span class="hljs-string">&quot;offset_mapping&quot;</span>]

<span class="hljs-keyword">for</span> idx, pred <span class="hljs-keyword">in</span> <span class="hljs-built_in">enumerate</span>(predictions):
    label = model.config.id2label[pred]
    <span class="hljs-keyword">if</span> label != <span class="hljs-string">&quot;O&quot;</span>:
        start, end = offsets[idx]
        results.append(
            {
                <span class="hljs-string">&quot;entity&quot;</span>: label,
                <span class="hljs-string">&quot;score&quot;</span>: probabilities[idx][pred],
                <span class="hljs-string">&quot;word&quot;</span>: tokens[idx],
                <span class="hljs-string">&quot;start&quot;</span>: start,
                <span class="hljs-string">&quot;end&quot;</span>: end,
            }
        )

<span class="hljs-built_in">print</span>(results)`}}),lt=new q({props:{code:`[{'entity': 'I-PER', 'score': 0.9993828, 'index': 4, 'word': 'S', 'start': 11, 'end': 12},
 {'entity': 'I-PER', 'score': 0.99815476, 'index': 5, 'word': '##yl', 'start': 12, 'end': 14},
 {'entity': 'I-PER', 'score': 0.99590725, 'index': 6, 'word': '##va', 'start': 14, 'end': 16},
 {'entity': 'I-PER', 'score': 0.9992327, 'index': 7, 'word': '##in', 'start': 16, 'end': 18},
 {'entity': 'I-ORG', 'score': 0.97389334, 'index': 12, 'word': 'Hu', 'start': 33, 'end': 35},
 {'entity': 'I-ORG', 'score': 0.976115, 'index': 13, 'word': '##gging', 'start': 35, 'end': 40},
 {'entity': 'I-ORG', 'score': 0.98879766, 'index': 14, 'word': 'Face', 'start': 41, 'end': 45},
 {'entity': 'I-LOC', 'score': 0.99321055, 'index': 16, 'word': 'Brooklyn', 'start': 49, 'end': 57}]`,highlighted:`[{<span class="hljs-string">&#x27;entity&#x27;</span>: <span class="hljs-string">&#x27;I-PER&#x27;</span>, <span class="hljs-string">&#x27;score&#x27;</span>: <span class="hljs-number">0.9993828</span>, <span class="hljs-string">&#x27;index&#x27;</span>: <span class="hljs-number">4</span>, <span class="hljs-string">&#x27;word&#x27;</span>: <span class="hljs-string">&#x27;S&#x27;</span>, <span class="hljs-string">&#x27;start&#x27;</span>: <span class="hljs-number">11</span>, <span class="hljs-string">&#x27;end&#x27;</span>: <span class="hljs-number">12</span>},
 {<span class="hljs-string">&#x27;entity&#x27;</span>: <span class="hljs-string">&#x27;I-PER&#x27;</span>, <span class="hljs-string">&#x27;score&#x27;</span>: <span class="hljs-number">0.99815476</span>, <span class="hljs-string">&#x27;index&#x27;</span>: <span class="hljs-number">5</span>, <span class="hljs-string">&#x27;word&#x27;</span>: <span class="hljs-string">&#x27;##yl&#x27;</span>, <span class="hljs-string">&#x27;start&#x27;</span>: <span class="hljs-number">12</span>, <span class="hljs-string">&#x27;end&#x27;</span>: <span class="hljs-number">14</span>},
 {<span class="hljs-string">&#x27;entity&#x27;</span>: <span class="hljs-string">&#x27;I-PER&#x27;</span>, <span class="hljs-string">&#x27;score&#x27;</span>: <span class="hljs-number">0.99590725</span>, <span class="hljs-string">&#x27;index&#x27;</span>: <span class="hljs-number">6</span>, <span class="hljs-string">&#x27;word&#x27;</span>: <span class="hljs-string">&#x27;##va&#x27;</span>, <span class="hljs-string">&#x27;start&#x27;</span>: <span class="hljs-number">14</span>, <span class="hljs-string">&#x27;end&#x27;</span>: <span class="hljs-number">16</span>},
 {<span class="hljs-string">&#x27;entity&#x27;</span>: <span class="hljs-string">&#x27;I-PER&#x27;</span>, <span class="hljs-string">&#x27;score&#x27;</span>: <span class="hljs-number">0.9992327</span>, <span class="hljs-string">&#x27;index&#x27;</span>: <span class="hljs-number">7</span>, <span class="hljs-string">&#x27;word&#x27;</span>: <span class="hljs-string">&#x27;##in&#x27;</span>, <span class="hljs-string">&#x27;start&#x27;</span>: <span class="hljs-number">16</span>, <span class="hljs-string">&#x27;end&#x27;</span>: <span class="hljs-number">18</span>},
 {<span class="hljs-string">&#x27;entity&#x27;</span>: <span class="hljs-string">&#x27;I-ORG&#x27;</span>, <span class="hljs-string">&#x27;score&#x27;</span>: <span class="hljs-number">0.97389334</span>, <span class="hljs-string">&#x27;index&#x27;</span>: <span class="hljs-number">12</span>, <span class="hljs-string">&#x27;word&#x27;</span>: <span class="hljs-string">&#x27;Hu&#x27;</span>, <span class="hljs-string">&#x27;start&#x27;</span>: <span class="hljs-number">33</span>, <span class="hljs-string">&#x27;end&#x27;</span>: <span class="hljs-number">35</span>},
 {<span class="hljs-string">&#x27;entity&#x27;</span>: <span class="hljs-string">&#x27;I-ORG&#x27;</span>, <span class="hljs-string">&#x27;score&#x27;</span>: <span class="hljs-number">0.976115</span>, <span class="hljs-string">&#x27;index&#x27;</span>: <span class="hljs-number">13</span>, <span class="hljs-string">&#x27;word&#x27;</span>: <span class="hljs-string">&#x27;##gging&#x27;</span>, <span class="hljs-string">&#x27;start&#x27;</span>: <span class="hljs-number">35</span>, <span class="hljs-string">&#x27;end&#x27;</span>: <span class="hljs-number">40</span>},
 {<span class="hljs-string">&#x27;entity&#x27;</span>: <span class="hljs-string">&#x27;I-ORG&#x27;</span>, <span class="hljs-string">&#x27;score&#x27;</span>: <span class="hljs-number">0.98879766</span>, <span class="hljs-string">&#x27;index&#x27;</span>: <span class="hljs-number">14</span>, <span class="hljs-string">&#x27;word&#x27;</span>: <span class="hljs-string">&#x27;Face&#x27;</span>, <span class="hljs-string">&#x27;start&#x27;</span>: <span class="hljs-number">41</span>, <span class="hljs-string">&#x27;end&#x27;</span>: <span class="hljs-number">45</span>},
 {<span class="hljs-string">&#x27;entity&#x27;</span>: <span class="hljs-string">&#x27;I-LOC&#x27;</span>, <span class="hljs-string">&#x27;score&#x27;</span>: <span class="hljs-number">0.99321055</span>, <span class="hljs-string">&#x27;index&#x27;</span>: <span class="hljs-number">16</span>, <span class="hljs-string">&#x27;word&#x27;</span>: <span class="hljs-string">&#x27;Brooklyn&#x27;</span>, <span class="hljs-string">&#x27;start&#x27;</span>: <span class="hljs-number">49</span>, <span class="hljs-string">&#x27;end&#x27;</span>: <span class="hljs-number">57</span>}]`}}),rt=new Ut({}),ot=new q({props:{code:"example[33:45]",highlighted:'example[<span class="hljs-number">33</span>:<span class="hljs-number">45</span>]'}}),it=new q({props:{code:"Hugging Face",highlighted:"Hugging Face"}}),pt=new q({props:{code:`import numpy as np

results = []
inputs_with_offsets = tokenizer(example, return_offsets_mapping=True)
tokens = inputs_with_offsets.tokens()
offsets = inputs_with_offsets["offset_mapping"]

idx = 0
while idx < len(predictions):
    pred = predictions[idx]
    label = model.config.id2label[pred]
    if label != "O":
        # Enlever le B- ou le I-
        label = label[2:]
        start, _ = offsets[idx]

        # R\xE9cup\xE9rer tous les tokens \xE9tiquet\xE9s avec I-label
        all_scores = []
        while (
            idx < len(predictions)
            and model.config.id2label[predictions[idx]] == f"I-{label}"
        ):
            all_scores.append(probabilities[idx][pred])
            _, end = offsets[idx]
            idx += 1

        # Le score est la moyenne de tous les scores des tokens dans cette entit\xE9 group\xE9e.
        score = np.mean(all_scores).item()
        word = example[start:end]
        results.append(
            {
                "entity_group": label,
                "score": score,
                "word": word,
                "start": start,
                "end": end,
            }
        )
    idx += 1

print(results)`,highlighted:`<span class="hljs-keyword">import</span> numpy <span class="hljs-keyword">as</span> np

results = []
inputs_with_offsets = tokenizer(example, return_offsets_mapping=<span class="hljs-literal">True</span>)
tokens = inputs_with_offsets.tokens()
offsets = inputs_with_offsets[<span class="hljs-string">&quot;offset_mapping&quot;</span>]

idx = <span class="hljs-number">0</span>
<span class="hljs-keyword">while</span> idx &lt; <span class="hljs-built_in">len</span>(predictions):
    pred = predictions[idx]
    label = model.config.id2label[pred]
    <span class="hljs-keyword">if</span> label != <span class="hljs-string">&quot;O&quot;</span>:
        <span class="hljs-comment"># Enlever le B- ou le I-</span>
        label = label[<span class="hljs-number">2</span>:]
        start, _ = offsets[idx]

        <span class="hljs-comment"># R\xE9cup\xE9rer tous les tokens \xE9tiquet\xE9s avec I-label</span>
        all_scores = []
        <span class="hljs-keyword">while</span> (
            idx &lt; <span class="hljs-built_in">len</span>(predictions)
            <span class="hljs-keyword">and</span> model.config.id2label[predictions[idx]] == <span class="hljs-string">f&quot;I-<span class="hljs-subst">{label}</span>&quot;</span>
        ):
            all_scores.append(probabilities[idx][pred])
            _, end = offsets[idx]
            idx += <span class="hljs-number">1</span>

        <span class="hljs-comment"># Le score est la moyenne de tous les scores des tokens dans cette entit\xE9 group\xE9e.</span>
        score = np.mean(all_scores).item()
        word = example[start:end]
        results.append(
            {
                <span class="hljs-string">&quot;entity_group&quot;</span>: label,
                <span class="hljs-string">&quot;score&quot;</span>: score,
                <span class="hljs-string">&quot;word&quot;</span>: word,
                <span class="hljs-string">&quot;start&quot;</span>: start,
                <span class="hljs-string">&quot;end&quot;</span>: end,
            }
        )
    idx += <span class="hljs-number">1</span>

<span class="hljs-built_in">print</span>(results)`}}),ut=new q({props:{code:`[{'entity_group': 'PER', 'score': 0.9981694, 'word': 'Sylvain', 'start': 11, 'end': 18},
 {'entity_group': 'ORG', 'score': 0.97960204, 'word': 'Hugging Face', 'start': 33, 'end': 45},
 {'entity_group': 'LOC', 'score': 0.99321055, 'word': 'Brooklyn', 'start': 49, 'end': 57}]`,highlighted:`[{<span class="hljs-string">&#x27;entity_group&#x27;</span>: <span class="hljs-string">&#x27;PER&#x27;</span>, <span class="hljs-string">&#x27;score&#x27;</span>: <span class="hljs-number">0.9981694</span>, <span class="hljs-string">&#x27;word&#x27;</span>: <span class="hljs-string">&#x27;Sylvain&#x27;</span>, <span class="hljs-string">&#x27;start&#x27;</span>: <span class="hljs-number">11</span>, <span class="hljs-string">&#x27;end&#x27;</span>: <span class="hljs-number">18</span>},
 {<span class="hljs-string">&#x27;entity_group&#x27;</span>: <span class="hljs-string">&#x27;ORG&#x27;</span>, <span class="hljs-string">&#x27;score&#x27;</span>: <span class="hljs-number">0.97960204</span>, <span class="hljs-string">&#x27;word&#x27;</span>: <span class="hljs-string">&#x27;Hugging Face&#x27;</span>, <span class="hljs-string">&#x27;start&#x27;</span>: <span class="hljs-number">33</span>, <span class="hljs-string">&#x27;end&#x27;</span>: <span class="hljs-number">45</span>},
 {<span class="hljs-string">&#x27;entity_group&#x27;</span>: <span class="hljs-string">&#x27;LOC&#x27;</span>, <span class="hljs-string">&#x27;score&#x27;</span>: <span class="hljs-number">0.99321055</span>, <span class="hljs-string">&#x27;word&#x27;</span>: <span class="hljs-string">&#x27;Brooklyn&#x27;</span>, <span class="hljs-string">&#x27;start&#x27;</span>: <span class="hljs-number">49</span>, <span class="hljs-string">&#x27;end&#x27;</span>: <span class="hljs-number">57</span>}]`}}),{c(){d=l("meta"),g=u(),b(m.$$.fragment),y=u(),R=l("h1"),w=l("a"),S=l("span"),b(N.$$.fragment),L=u(),C=l("span"),ne=n("Pouvoirs sp\xE9ciaux des *tokenizers* rapides"),Q=u(),H.c(),D=u(),z=l("p"),se=n("Dans cette section, nous allons examiner de plus pr\xE8s les capacit\xE9s des tokenizers dans \u{1F917} "),te=l("em"),ge=n("Transformers"),ae=n(`.
Jusqu\u2019\xE0 pr\xE9sent, nous ne les avons utilis\xE9s que pour `),J=l("em"),pe=n("tokeniser"),K=n(" les entr\xE9es ou d\xE9coder les identifiants pour les retranscrire en texte, mais les "),T=l("em"),x=n("tokenizers"),P=n(", surtout ceux soutenus par la biblioth\xE8que \u{1F917} "),ue=l("em"),v=n("Tokenizers"),B=n(", peuvent faire beaucoup plus. Pour illustrer ces fonctionnalit\xE9s suppl\xE9mentaires, nous allons explorer comment reproduire les r\xE9sultats des pipelines "),ce=l("code"),ke=n("token-classification"),He=n(" (que nous avons appel\xE9 "),$e=l("code"),xt=n("ner"),gt=n(") et "),Yt=l("code"),Bo=n("question-answering"),Ao=n(" que nous avons rencontr\xE9s pour la premi\xE8re fois dans "),vt=l("a"),Fo=n("Chapter 1"),No=n("."),Ul=u(),b(_s.$$.fragment),Yl=u(),W=l("p"),Lo=n("Dans la discussion qui suit, nous ferons souvent la distinction entre les "),Kt=l("em"),Xo=n("tokenizers"),Ho=n(" \xAB lents \xBB et \xAB rapides \xBB. Les "),Wt=l("em"),Go=n("tokenizers"),Vo=n(" lents sont ceux \xE9crits en Python \xE0 l\u2019int\xE9rieur de la biblioth\xE8que \u{1F917} "),Zt=l("em"),Qo=n("Transformers"),Jo=n(", tandis que les versions rapides sont celles fournies par \u{1F917} "),en=l("em"),Uo=n("Tokenizers"),Yo=n(",  qui sont \xE9crites en Rust. Si vous vous souvenez du tableau du "),bt=l("a"),Ko=n("Chapitre 5"),Wo=n(" qui indiquait combien de temps il fallait \xE0 un "),sn=l("em"),Zo=n("tokenizer"),ei=n(" rapide et \xE0 un "),tn=l("em"),si=n("tokenizer"),ti=n(" lent pour tokeniser le jeu de donn\xE9es Drug Review, vous devriez avoir une id\xE9e de la raison pour laquelle nous les appelons rapides et lents :"),Kl=u(),es=l("table"),nn=l("thead"),Ge=l("tr"),Es=l("th"),an=l("em"),ni=n("Tokenizer"),ai=n(" rapide"),li=u(),ks=l("th"),ln=l("em"),ri=n("Tokenizer"),oi=n(" lent"),ii=u(),rn=l("th"),pi=u(),$s=l("tbody"),Ve=l("tr"),jt=l("td"),on=l("code"),ui=n("batched=True"),ci=u(),_t=l("td"),di=n("10.8s"),mi=u(),Et=l("td"),fi=n("4min41s"),hi=u(),Qe=l("tr"),kt=l("td"),pn=l("code"),xi=n("batched=False"),gi=u(),$t=l("td"),vi=n("59.2s"),bi=u(),qt=l("td"),ji=n("5min3s"),Wl=u(),b(ss.$$.fragment),Zl=u(),Je=l("h2"),ts=l("a"),un=l("span"),b(qs.$$.fragment),_i=u(),cn=l("span"),Ei=n("*BatchEncoding*"),er=u(),b(ys.$$.fragment),sr=u(),qe=l("p"),ki=n("La sortie d\u2019un "),dn=l("em"),$i=n("tokenizer"),qi=n(" n\u2019est pas un simple dictionnaire Python. Ce que nous obtenons est en fait un objet sp\xE9cial "),mn=l("code"),yi=n("BatchEncoding"),wi=n(". C\u2019est une sous-classe d\u2019un dictionnaire (c\u2019est pourquoi nous avons pu indexer ce r\xE9sultat sans probl\xE8me auparavant), mais avec des m\xE9thodes suppl\xE9mentaires qui sont principalement utilis\xE9es par les "),fn=l("em"),Ci=n("tokenizers"),Oi=n(" rapides."),tr=u(),de=l("p"),zi=n("En plus de leurs capacit\xE9s de parall\xE9lisation, la fonctionnalit\xE9 cl\xE9 des "),hn=l("em"),Pi=n("tokenizers"),Di=n(" rapides est qu\u2019ils gardent toujours la trace de l\u2019\xE9tendue originale des textes d\u2019o\xF9 proviennent les "),xn=l("em"),Ii=n("tokens"),Mi=n(" finaux, une fonctionnalit\xE9 que nous appelons "),gn=l("em"),Ti=n("mapping offset"),Ri=n(". Cela permet de d\xE9bloquer des fonctionnalit\xE9s telles que le mappage de chaque mot aux "),vn=l("em"),Si=n("tokens"),Bi=n(" qu\u2019il a g\xE9n\xE9r\xE9s ou le mappage de chaque caract\xE8re du texte original au "),bn=l("em"),Ai=n("token"),Fi=n(" qu\u2019il contient, et vice versa."),nr=u(),yt=l("p"),Ni=n("Prenons un exemple :"),ar=u(),b(ws.$$.fragment),lr=u(),Se=l("p"),Li=n("Comme mentionn\xE9 pr\xE9c\xE9demment, nous obtenons un objet "),jn=l("code"),Xi=n("BatchEncoding"),Hi=n(" dans la sortie du "),_n=l("em"),Gi=n("tokenizer"),Vi=n(" :"),rr=u(),b(Cs.$$.fragment),or=u(),le=l("p"),Qi=n("Puisque la classe "),En=l("code"),Ji=n("AutoTokenizer"),Ui=n(" choisit un "),kn=l("em"),Yi=n("tokenizer"),Ki=n(" rapide par d\xE9faut, nous pouvons utiliser les m\xE9thodes suppl\xE9mentaires que cet objet "),$n=l("code"),Wi=n("BatchEncoding"),Zi=n(" fournit. Nous avons deux fa\xE7ons de v\xE9rifier si notre "),qn=l("em"),ep=n("tokenizer"),sp=n(" est rapide ou lent. Nous pouvons soit v\xE9rifier l\u2019attribut "),yn=l("code"),tp=n("is_fast"),np=n(" du "),wn=l("em"),ap=n("tokenizer"),lp=n(" :"),ir=u(),b(Os.$$.fragment),pr=u(),b(zs.$$.fragment),ur=u(),ns=l("p"),rp=n("ou v\xE9rifiez le m\xEAme attribut de notre "),Cn=l("code"),op=n("encoding"),ip=n(" :"),cr=u(),b(Ps.$$.fragment),dr=u(),b(Ds.$$.fragment),mr=u(),ye=l("p"),pp=n("Voyons ce qu\u2019un "),On=l("em"),up=n("tokenizer"),cp=n(" rapide nous permet de faire. Tout d\u2019abord, nous pouvons acc\xE9der aux "),zn=l("em"),dp=n("tokens"),mp=n(" sans avoir \xE0 reconvertir les ID en "),Pn=l("em"),fp=n("tokens"),hp=n(" :"),fr=u(),b(Is.$$.fragment),hr=u(),b(Ms.$$.fragment),xr=u(),ve=l("p"),xp=n("Dans ce cas, le "),Dn=l("em"),gp=n("token"),vp=n(" \xE0 l\u2019index 5 est "),In=l("code"),bp=n("##yl"),jp=n(", qui fait partie du mot \xAB Sylvain \xBB dans la phrase originale. Nous pouvons \xE9galement utiliser la m\xE9thode "),Mn=l("code"),_p=n("word_ids()"),Ep=n(" pour obtenir l\u2019index du mot dont provient chaque "),Tn=l("em"),kp=n("token"),$p=n(" :"),gr=u(),b(Ts.$$.fragment),vr=u(),b(Rs.$$.fragment),br=u(),I=l("p"),qp=n("On peut voir que les "),Rn=l("em"),yp=n("tokens"),wp=n(" sp\xE9ciaux du tokenizer "),Sn=l("code"),Cp=n("[CLS]"),Op=n(" et "),Bn=l("code"),zp=n("[SEP]"),Pp=n(" sont mis en correspondance avec "),An=l("code"),Dp=n("None"),Ip=n(", puis chaque "),Fn=l("em"),Mp=n("token"),Tp=n(" est mis en correspondance avec le mot dont il provient. Ceci est particuli\xE8rement utile pour d\xE9terminer si un "),Nn=l("em"),Rp=n("token"),Sp=n(" est au d\xE9but d\u2019un mot ou si deux "),Ln=l("em"),Bp=n("tokens"),Ap=n(" sont dans le m\xEAme mot. Nous pourrions nous appuyer sur le pr\xE9fixe "),Xn=l("code"),Fp=n("##"),Np=n(" pour cela, mais il ne fonctionne que pour les "),Hn=l("em"),Lp=n("tokenizers"),Xp=n(" de type BERT. Cette m\xE9thode fonctionne pour n\u2019importe quel type de "),Gn=l("em"),Hp=n("tokenizer"),Gp=n(", du moment qu\u2019il est rapide. Dans le chapitre suivant, nous verrons comment utiliser cette capacit\xE9 pour appliquer correctement les \xE9tiquettes que nous avons pour chaque mot aux "),Vn=l("em"),Vp=n("tokens"),Qp=n(" dans des t\xE2ches comme la reconnaissance d\u2019entit\xE9s nomm\xE9es (NER) et le marquage POS (Part-of-speech). Nous pouvons \xE9galement l\u2019utiliser pour masquer tous les "),Qn=l("em"),Jp=n("tokens"),Up=n(" provenant du m\xEAme mot dans la mod\xE9lisation du langage masqu\xE9 (une technique appel\xE9e "),Jn=l("em"),Yp=n("whole word masking"),Kp=n(")."),jr=u(),b(as.$$.fragment),_r=u(),we=l("p"),Wp=n("De m\xEAme, il existe une m\xE9thode "),Un=l("code"),Zp=n("sentence_ids()"),eu=n(" que nous pouvons utiliser pour associer un token \xE0 la phrase dont il provient (bien que dans ce cas, le "),Yn=l("code"),su=n("token_type_ids"),tu=n(" retourn\xE9 par le "),Kn=l("em"),nu=n("tokenizer"),au=n(" peut nous donner la m\xEAme information)."),Er=u(),re=l("p"),lu=n("Enfin, nous pouvons faire correspondre n\u2019importe quel mot ou jeton aux caract\xE8res du texte d\u2019origine, et vice versa, gr\xE2ce aux m\xE9thodes "),Wn=l("code"),ru=n("word_to_chars()"),ou=n(" ou "),Zn=l("code"),iu=n("token_to_chars()"),pu=n(" et "),ea=l("code"),uu=n("char_to_word()"),cu=n(" ou "),sa=l("code"),du=n("char_to_token()"),mu=n(". Par exemple, la m\xE9thode "),ta=l("code"),fu=n("word_ids()"),hu=n(" nous a dit que "),na=l("code"),xu=n("##yl"),gu=n(" fait partie du mot \xE0 l\u2019indice 3, mais de quel mot s\u2019agit-il dans la phrase ? Nous pouvons le d\xE9couvrir comme ceci :"),kr=u(),b(Ss.$$.fragment),$r=u(),b(Bs.$$.fragment),qr=u(),Be=l("p"),vu=n("Comme nous l\u2019avons mentionn\xE9 pr\xE9c\xE9demment, tout ceci est rendu possible par le fait que le tokenizer rapide garde la trace de la partie du texte d\u2019o\xF9 provient chaque "),aa=l("em"),bu=n("token dans une liste de "),ju=n("offsets*. Pour illustrer leur utilisation, nous allons maintenant vous montrer comment reproduire manuellement les r\xE9sultats du pipeline "),la=l("code"),_u=n("token-classification"),Eu=n("."),yr=u(),b(ls.$$.fragment),wr=u(),Ue=l("h2"),rs=l("a"),ra=l("span"),b(As.$$.fragment),ku=u(),wt=l("span"),$u=n("A l'int\xE9rieur du pipeline "),oa=l("code"),qu=n("token-classification"),Cr=u(),me=l("p"),yu=n("Dans le "),Ct=l("a"),wu=n("Chapitre 1"),Cu=n(", nous avons eu un premier aper\xE7u de l\u2019application de NER (o\xF9 la t\xE2che est d\u2019identifier les parties du texte qui correspondent \xE0 des entit\xE9s telles que des personnes, des lieux ou des organisations) avec la fonction \u{1F917} "),ia=l("em"),Ou=n("Transformers"),zu=u(),pa=l("code"),Pu=n("pipeline()"),Du=n(". Puis, dans "),Ot=l("a"),Iu=n("Chapitre 2"),Mu=n(", nous avons vu comment un pipeline regroupe les trois \xE9tapes n\xE9cessaires pour obtenir les pr\xE9dictions \xE0 partir d\u2019un texte brut : la tokenisation, le passage des entr\xE9es dans le mod\xE8le et le post-traitement. Les deux premi\xE8res \xE9tapes du pipeline de "),ua=l("code"),Tu=n("token-classification"),Ru=n(" sont les m\xEAmes que dans tout autre pipeline, mais le post-traitement est un peu plus complexe. Voyons comment !"),Or=u(),De.c(),zt=u(),Ye=l("h3"),os=l("a"),ca=l("span"),b(Fs.$$.fragment),Su=u(),da=l("span"),Bu=n("Obtenir les r\xE9sultats de base avec le pipeline"),zr=u(),is=l("p"),Au=n("Tout d\u2019abord, prenons un pipeline de classification de tokens afin d\u2019obtenir des r\xE9sultats \xE0 comparer manuellement. Le mod\xE8le utilis\xE9 par d\xE9faut est "),Ns=l("a"),ma=l("code"),Fu=n("dbmdz/bert-large-cased-finetuned-conll03-english"),Nu=n(". Il effectue un NER sur les phrases :"),Pr=u(),b(Ls.$$.fragment),Dr=u(),b(Xs.$$.fragment),Ir=u(),ps=l("p"),Lu=n("Le mod\xE8le a correctement identifi\xE9 chaque token g\xE9n\xE9r\xE9 par \xAB Sylvain \xBB comme une personne, chaque token g\xE9n\xE9r\xE9 par \xAB Hugging Face \xBB comme une organisation, et le "),fa=l("em"),Xu=n("token"),Hu=n(" \xAB Brooklyn \xBB comme un lieu. Nous pouvons \xE9galement demander au pipeline de regrouper les tokens qui correspondent \xE0 la m\xEAme entit\xE9 :"),Mr=u(),b(Hs.$$.fragment),Tr=u(),b(Gs.$$.fragment),Rr=u(),Z=l("p"),Gu=n("La "),ha=l("code"),Vu=n("aggregation_strategy"),Qu=n(" choisie va changer les scores calcul\xE9s pour chaque entit\xE9 group\xE9e. Avec "),xa=l("code"),Ju=n('"simple"'),Uu=n(" le score est juste la moyenne des scores de chaque "),ga=l("em"),Yu=n("token"),Ku=n(" dans l\u2019entit\xE9 donn\xE9e : par exemple, le score de \xAB Sylvain \xBB est la moyenne des scores que nous avons vu dans l\u2019exemple pr\xE9c\xE9dent pour les tokens "),va=l("code"),Wu=n("S"),Zu=n(", "),ba=l("code"),ec=n("##yl"),sc=n(", "),ja=l("code"),tc=n("##va"),nc=n(", et "),_a=l("code"),ac=n("##in"),lc=n(". D\u2019autres strat\xE9gies sont disponibles :"),Sr=u(),Ae=l("ul"),us=l("li"),Ea=l("code"),rc=n('"first"'),oc=n(", o\xF9 le score de chaque entit\xE9 est le score du premier token de cette entit\xE9 (donc pour \xAB Sylvain \xBB ce serait 0.993828, le score du token "),ka=l("code"),ic=n("S"),pc=n(")"),uc=u(),Pt=l("li"),$a=l("code"),cc=n('"max"'),dc=n(", o\xF9 le score de chaque entit\xE9 est le score maximal des tokens de cette entit\xE9 (ainsi, pour \xAB Hugging Face \xBB, le score de \xAB Face \xBB serait de 0,98879766)."),mc=u(),cs=l("li"),qa=l("code"),fc=n('"moyenne"'),hc=n(`, o\xF9 le score de chaque entit\xE9 est la moyenne des scores des mots qui composent cette entit\xE9 (ainsi, pour \xAB Sylvain \xBB,
il n\u2019y aurait pas de diff\xE9rence avec la strat\xE9gie `),ya=l("code"),xc=n('"simple"'),gc=n(", mais \u201C\xC9treinte du visage\u201D aurait un score de 0,9819, la moyenne des scores de \xAB Hugging Face \xBB, 0,975, et \xAB Face \xBB, 0,98879)."),Br=u(),ds=l("p"),vc=n("Voyons maintenant comment obtenir ces r\xE9sultats sans utiliser la fonction "),wa=l("code"),bc=n("pipeline()"),jc=n(" !"),Ar=u(),Ke=l("h3"),ms=l("a"),Ca=l("span"),b(Vs.$$.fragment),_c=u(),Oa=l("span"),Ec=n("Des entr\xE9es aux pr\xE9dictions"),Fr=u(),Me.c(),Dt=u(),fs=l("p"),kc=n("Nous avons un batch avec 1 s\xE9quence de 19 "),za=l("em"),$c=n("tokens"),qc=n(" et le mod\xE8le a 9 \xE9tiquettes diff\xE9rentes, donc la sortie du mod\xE8le a une forme de 1 x 19 x 9. Comme pour le pipeline de classification de texte, nous utilisons une fonction softmax pour convertir ces logits en probabilit\xE9s, et nous prenons l\u2019argmax pour obtenir des pr\xE9dictions (notez que nous pouvons prendre l\u2019argmax sur les logits parce que la softmax ne change pas l\u2019ordre) :"),Nr=u(),Re.c(),It=u(),b(Qs.$$.fragment),Lr=u(),hs=l("p"),yc=n("L\u2019attribut "),Pa=l("code"),wc=n("model.config.id2label"),Cc=n(" contient la correspondance entre les index et les \xE9tiquettes que nous pouvons utiliser pour donner un sens aux pr\xE9dictions :"),Xr=u(),b(Js.$$.fragment),Hr=u(),b(Us.$$.fragment),Gr=u(),$=l("p"),Oc=n("Comme nous l\u2019avons vu pr\xE9c\xE9demment, il y a 9 \xE9tiquettes : "),Da=l("code"),zc=n("O"),Pc=n(" est le label pour les "),Ia=l("em"),Dc=n("tokens"),Ic=n(" qui ne sont dans aucune entit\xE9 nomm\xE9e (il signifie "),Ma=l("em"),Mc=n("outside"),Tc=n(") et nous avons ensuite deux labels pour chaque type d\u2019entit\xE9 (divers, personne, organisation, et lieu). L\u2019\xE9tiquette "),Ta=l("code"),Rc=n("B-XXX"),Sc=n(" indique que le "),Ra=l("em"),Bc=n("token"),Ac=n(" est au d\xE9but d\u2019une entit\xE9 "),Sa=l("code"),Fc=n("XXX"),Nc=n(" et l\u2019\xE9tiquette "),Ba=l("code"),Lc=n("I-XXX"),Xc=n(" indique que le "),Aa=l("em"),Hc=n("token"),Gc=n(" est \xE0 l\u2019int\xE9rieur de l\u2019entit\xE9 "),Fa=l("code"),Vc=n("XXX"),Qc=n(". Par exemple, dans l\u2019exemple actuel, nous nous attendons \xE0 ce que notre mod\xE8le classe le "),Na=l("em"),Jc=n("token"),Uc=u(),La=l("code"),Yc=n("S"),Kc=n(" comme "),Xa=l("code"),Wc=n("B-PER"),Zc=n(" (d\xE9but d\u2019une entit\xE9 personne) et les "),Ha=l("em"),ed=n("tokens"),sd=u(),Ga=l("code"),td=n("##yl"),nd=n(", "),Va=l("code"),ad=n("##va"),ld=n(" et "),Qa=l("code"),rd=n("##in"),od=n(" comme "),Ja=l("code"),id=n("I-PER"),pd=n(" (\xE0 l\u2019int\xE9rieur d\u2019une entit\xE9 personne)."),Vr=u(),A=l("p"),ud=n("Vous pourriez penser que le mod\xE8le s\u2019est tromp\xE9 dans ce cas, car il a attribu\xE9 l\u2019\xE9tiquette "),Ua=l("code"),cd=n("I-PER"),dd=n(" \xE0 ces quatre "),Ya=l("em"),md=n("tokens"),fd=n(", mais ce n\u2019est pas tout \xE0 fait vrai. Il existe en fait deux formats pour ces \xE9tiquettes "),Ka=l("code"),hd=n("B-"),xd=n(" et "),Wa=l("code"),gd=n("I-"),vd=n(" : "),Za=l("em"),bd=n("IOB1"),jd=n(" et "),el=l("em"),_d=n("IOB2"),Ed=n(". Le format IOB2 (en rose ci-dessous) est celui que nous avons introduit alors que dans le format IOB1 (en bleu), les \xE9tiquettes commen\xE7ant par "),sl=l("code"),kd=n("B-"),$d=n(" ne sont jamais utilis\xE9es que pour s\xE9parer deux entit\xE9s adjacentes du m\xEAme type. Le mod\xE8le que nous utilisons a \xE9t\xE9 "),tl=l("em"),qd=n("finetun\xE9"),yd=n(" sur un jeu de donn\xE9es utilisant ce format, c\u2019est pourquoi il attribue le label "),nl=l("code"),wd=n("I-PER"),Cd=n(" au "),al=l("em"),Od=n("token"),zd=u(),ll=l("code"),Pd=n("S"),Dd=n("."),Qr=u(),We=l("div"),Ys=l("img"),Id=u(),Ks=l("img"),Jr=u(),Fe=l("p"),Md=n("Avec cette carte, nous sommes pr\xEAts \xE0 reproduire (presque enti\xE8rement) les r\xE9sultats du premier pipeline. Nous pouvons simplement r\xE9cup\xE9rer le score et le label de chaque "),rl=l("em"),Td=n("token"),Rd=n(" qui n\u2019a pas \xE9t\xE9 class\xE9 comme "),ol=l("code"),Sd=n("O"),Bd=n(" :"),Ur=u(),b(Ws.$$.fragment),Yr=u(),b(Zs.$$.fragment),Kr=u(),be=l("p"),Ad=n("C\u2019est tr\xE8s similaire \xE0 ce que nous avions avant, \xE0 une exception pr\xE8s : le pipeline nous a aussi donn\xE9 des informations sur le "),il=l("code"),Fd=n("d\xE9but"),Nd=n(" et la "),pl=l("code"),Ld=n("fin"),Xd=n(" de chaque entit\xE9 dans la phrase originale. C\u2019est l\xE0 que notre mappage de d\xE9calage va entrer en jeu. Pour obtenir les d\xE9calages, il suffit de d\xE9finir "),ul=l("code"),Hd=n("return_offsets_mapping=True"),Gd=n(" lorsque nous appliquons le "),cl=l("em"),Vd=n("tokenizer"),Qd=n(" \xE0 nos entr\xE9es :"),Wr=u(),b(et.$$.fragment),Zr=u(),b(st.$$.fragment),eo=u(),oe=l("p"),Jd=n("Chaque "),dl=l("em"),Ud=n("tuple"),Yd=n(" est l\u2019empan de texte correspondant \xE0 chaque token, o\xF9 "),ml=l("code"),Kd=n("(0, 0)"),Wd=n(" est r\xE9serv\xE9 aux "),fl=l("em"),Zd=n("tokens"),em=n(" sp\xE9ciaux. Nous avons vu pr\xE9c\xE9demment que le token \xE0 l\u2019index 5 est "),hl=l("code"),sm=n("##yl"),tm=n(", qui a "),xl=l("code"),nm=n("(12, 14)"),am=n(" comme "),gl=l("em"),lm=n("offsets"),rm=n(" ici. Si on prend la tranche correspondante dans notre exemple :"),so=u(),b(tt.$$.fragment),to=u(),xs=l("p"),om=n("nous obtenons le bon espace de texte sans le "),vl=l("code"),im=n("##"),pm=n(" :"),no=u(),b(nt.$$.fragment),ao=u(),Mt=l("p"),um=n("En utilisant cela, nous pouvons maintenant compl\xE9ter les r\xE9sultats pr\xE9c\xE9dents :"),lo=u(),b(at.$$.fragment),ro=u(),b(lt.$$.fragment),oo=u(),Tt=l("p"),cm=n("C\u2019est la m\xEAme chose que ce que nous avons obtenu avec le premier pipeline !"),io=u(),Ze=l("h3"),gs=l("a"),bl=l("span"),b(rt.$$.fragment),dm=u(),jl=l("span"),mm=n("Regroupement des entit\xE9s"),po=u(),G=l("p"),fm=n("L\u2019utilisation des offsets pour d\xE9terminer les cl\xE9s de d\xE9but et de fin pour chaque entit\xE9 est pratique mais cette information n\u2019est pas strictement n\xE9cessaire. Cependant, lorsque nous voulons regrouper les entit\xE9s, les offsets nous \xE9pargnent un batch de code compliqu\xE9. Par exemple, si nous voulions regrouper les "),_l=l("em"),hm=n("tokens"),xm=u(),El=l("code"),gm=n("Hu"),vm=n(", "),kl=l("code"),bm=n("##gging"),jm=n(", et "),$l=l("code"),_m=n("Face"),Em=n(", nous pourrions \xE9tablir des r\xE8gles sp\xE9ciales disant que les deux premiers devraient \xEAtre attach\xE9s tout en enlevant le "),ql=l("code"),km=n("##"),$m=n(", et le "),yl=l("code"),qm=n("Face"),ym=n(" devrait \xEAtre ajout\xE9 avec un espace puisqu\u2019il ne commence pas par "),wl=l("code"),wm=n("##"),Cm=n(" mais cela ne fonctionnerait que pour ce type particulier de "),Cl=l("em"),Om=n("tokenizer"),zm=n(". Il faudrait \xE9crire un autre ensemble de r\xE8gles pour un "),Ol=l("em"),Pm=n("tokenizer"),Dm=n(" de type SentencePiece ou Byte-Pair-Encoding (voir plus loin dans ce chapitre)."),uo=u(),U=l("p"),Im=n("Avec les "),zl=l("em"),Mm=n("offsets"),Tm=n(", tout ce code personnalis\xE9 dispara\xEEt : il suffit de prendre l\u2019intervalle du texte original qui commence par le premier "),Pl=l("em"),Rm=n("token"),Sm=n(" et se termine par le dernier "),Dl=l("em"),Bm=n("token"),Am=n(". Ainsi, dans le cas des tokens "),Il=l("code"),Fm=n("Hu"),Nm=n(", "),Ml=l("code"),Lm=n("##gging"),Xm=n(", et "),Tl=l("code"),Hm=n("Face"),Gm=n(", nous devrions commencer au caract\xE8re 33 (le d\xE9but de "),Rl=l("code"),Vm=n("Hu"),Qm=n(") et finir avant le caract\xE8re 45 (la fin de "),Sl=l("code"),Jm=n("Face"),Um=n(") :"),co=u(),b(ot.$$.fragment),mo=u(),b(it.$$.fragment),fo=u(),fe=l("p"),Ym=n("Pour \xE9crire le code qui post-traite les pr\xE9dictions tout en regroupant les entit\xE9s, nous regrouperons les entit\xE9s qui sont cons\xE9cutives et \xE9tiquet\xE9es avec "),Bl=l("code"),Km=n("I-XXX"),Wm=n(", \xE0 l\u2019exception de la premi\xE8re, qui peut \xEAtre \xE9tiquet\xE9e comme "),Al=l("code"),Zm=n("B-XXX"),ef=n(" ou "),Fl=l("code"),sf=n("I-XXX"),tf=n(" (ainsi, nous arr\xEAtons de regrouper une entit\xE9 lorsque nous obtenons un "),Nl=l("code"),nf=n("O"),af=n(", un nouveau type d\u2019entit\xE9, ou un "),Ll=l("code"),lf=n("B-XXX"),rf=n(" qui nous indique qu\u2019une entit\xE9 du m\xEAme type commence) :"),ho=u(),b(pt.$$.fragment),xo=u(),Rt=l("p"),of=n("Et nous obtenons les m\xEAmes r\xE9sultats qu\u2019avec notre deuxi\xE8me pipeline !"),go=u(),b(ut.$$.fragment),vo=u(),Ne=l("p"),pf=n("Un autre exemple de t\xE2che o\xF9 ces d\xE9calages sont extr\xEAmement utiles est la r\xE9ponse aux questions. Plonger dans ce pipeline, ce que nous ferons dans la section suivante, nous permettra \xE9galement de jeter un coup d\u2019\u0153il \xE0 une derni\xE8re caract\xE9ristique des "),Xl=l("em"),uf=n("tokenizers"),cf=n(" de la biblioth\xE8que \u{1F917} "),Hl=l("em"),df=n("Transformers"),mf=n(" : la gestion des tokens qui d\xE9bordent lorsque nous tronquons une entr\xE9e \xE0 une longueur donn\xE9e."),this.h()},l(s){const i=Yx('[data-svelte="svelte-1phssyn"]',document.head);d=r(i,"META",{name:!0,content:!0}),i.forEach(t),g=c(s),j(m.$$.fragment,s),y=c(s),R=r(s,"H1",{class:!0});var ht=o(R);w=r(ht,"A",{id:!0,class:!0,href:!0});var St=o(w);S=r(St,"SPAN",{});var Gl=o(S);j(N.$$.fragment,Gl),Gl.forEach(t),St.forEach(t),L=c(ht),C=r(ht,"SPAN",{});var Vl=o(C);ne=a(Vl,"Pouvoirs sp\xE9ciaux des *tokenizers* rapides"),Vl.forEach(t),ht.forEach(t),Q=c(s),H.l(s),D=c(s),z=r(s,"P",{});var Y=o(z);se=a(Y,"Dans cette section, nous allons examiner de plus pr\xE8s les capacit\xE9s des tokenizers dans \u{1F917} "),te=r(Y,"EM",{});var Bt=o(te);ge=a(Bt,"Transformers"),Bt.forEach(t),ae=a(Y,`.
Jusqu\u2019\xE0 pr\xE9sent, nous ne les avons utilis\xE9s que pour `),J=r(Y,"EM",{});var At=o(J);pe=a(At,"tokeniser"),At.forEach(t),K=a(Y," les entr\xE9es ou d\xE9coder les identifiants pour les retranscrire en texte, mais les "),T=r(Y,"EM",{});var Ft=o(T);x=a(Ft,"tokenizers"),Ft.forEach(t),P=a(Y,", surtout ceux soutenus par la biblioth\xE8que \u{1F917} "),ue=r(Y,"EM",{});var Of=o(ue);v=a(Of,"Tokenizers"),Of.forEach(t),B=a(Y,", peuvent faire beaucoup plus. Pour illustrer ces fonctionnalit\xE9s suppl\xE9mentaires, nous allons explorer comment reproduire les r\xE9sultats des pipelines "),ce=r(Y,"CODE",{});var zf=o(ce);ke=a(zf,"token-classification"),zf.forEach(t),He=a(Y," (que nous avons appel\xE9 "),$e=r(Y,"CODE",{});var Pf=o($e);xt=a(Pf,"ner"),Pf.forEach(t),gt=a(Y,") et "),Yt=r(Y,"CODE",{});var Df=o(Yt);Bo=a(Df,"question-answering"),Df.forEach(t),Ao=a(Y," que nous avons rencontr\xE9s pour la premi\xE8re fois dans "),vt=r(Y,"A",{href:!0});var If=o(vt);Fo=a(If,"Chapter 1"),If.forEach(t),No=a(Y,"."),Y.forEach(t),Ul=c(s),j(_s.$$.fragment,s),Yl=c(s),W=r(s,"P",{});var he=o(W);Lo=a(he,"Dans la discussion qui suit, nous ferons souvent la distinction entre les "),Kt=r(he,"EM",{});var Mf=o(Kt);Xo=a(Mf,"tokenizers"),Mf.forEach(t),Ho=a(he," \xAB lents \xBB et \xAB rapides \xBB. Les "),Wt=r(he,"EM",{});var Tf=o(Wt);Go=a(Tf,"tokenizers"),Tf.forEach(t),Vo=a(he," lents sont ceux \xE9crits en Python \xE0 l\u2019int\xE9rieur de la biblioth\xE8que \u{1F917} "),Zt=r(he,"EM",{});var Rf=o(Zt);Qo=a(Rf,"Transformers"),Rf.forEach(t),Jo=a(he,", tandis que les versions rapides sont celles fournies par \u{1F917} "),en=r(he,"EM",{});var Sf=o(en);Uo=a(Sf,"Tokenizers"),Sf.forEach(t),Yo=a(he,",  qui sont \xE9crites en Rust. Si vous vous souvenez du tableau du "),bt=r(he,"A",{href:!0});var Bf=o(bt);Ko=a(Bf,"Chapitre 5"),Bf.forEach(t),Wo=a(he," qui indiquait combien de temps il fallait \xE0 un "),sn=r(he,"EM",{});var Af=o(sn);Zo=a(Af,"tokenizer"),Af.forEach(t),ei=a(he," rapide et \xE0 un "),tn=r(he,"EM",{});var Ff=o(tn);si=a(Ff,"tokenizer"),Ff.forEach(t),ti=a(he," lent pour tokeniser le jeu de donn\xE9es Drug Review, vous devriez avoir une id\xE9e de la raison pour laquelle nous les appelons rapides et lents :"),he.forEach(t),Kl=c(s),es=r(s,"TABLE",{});var jo=o(es);nn=r(jo,"THEAD",{});var Nf=o(nn);Ge=r(Nf,"TR",{});var Nt=o(Ge);Es=r(Nt,"TH",{align:!0});var ff=o(Es);an=r(ff,"EM",{});var Lf=o(an);ni=a(Lf,"Tokenizer"),Lf.forEach(t),ai=a(ff," rapide"),ff.forEach(t),li=c(Nt),ks=r(Nt,"TH",{align:!0});var hf=o(ks);ln=r(hf,"EM",{});var Xf=o(ln);ri=a(Xf,"Tokenizer"),Xf.forEach(t),oi=a(hf," lent"),hf.forEach(t),ii=c(Nt),rn=r(Nt,"TH",{align:!0}),o(rn).forEach(t),Nt.forEach(t),Nf.forEach(t),pi=c(jo),$s=r(jo,"TBODY",{});var _o=o($s);Ve=r(_o,"TR",{});var Lt=o(Ve);jt=r(Lt,"TD",{align:!0});var Hf=o(jt);on=r(Hf,"CODE",{});var Gf=o(on);ui=a(Gf,"batched=True"),Gf.forEach(t),Hf.forEach(t),ci=c(Lt),_t=r(Lt,"TD",{align:!0});var Vf=o(_t);di=a(Vf,"10.8s"),Vf.forEach(t),mi=c(Lt),Et=r(Lt,"TD",{align:!0});var Qf=o(Et);fi=a(Qf,"4min41s"),Qf.forEach(t),Lt.forEach(t),hi=c(_o),Qe=r(_o,"TR",{});var Xt=o(Qe);kt=r(Xt,"TD",{align:!0});var Jf=o(kt);pn=r(Jf,"CODE",{});var Uf=o(pn);xi=a(Uf,"batched=False"),Uf.forEach(t),Jf.forEach(t),gi=c(Xt),$t=r(Xt,"TD",{align:!0});var Yf=o($t);vi=a(Yf,"59.2s"),Yf.forEach(t),bi=c(Xt),qt=r(Xt,"TD",{align:!0});var Kf=o(qt);ji=a(Kf,"5min3s"),Kf.forEach(t),Xt.forEach(t),_o.forEach(t),jo.forEach(t),Wl=c(s),j(ss.$$.fragment,s),Zl=c(s),Je=r(s,"H2",{class:!0});var Eo=o(Je);ts=r(Eo,"A",{id:!0,class:!0,href:!0});var Wf=o(ts);un=r(Wf,"SPAN",{});var Zf=o(un);j(qs.$$.fragment,Zf),Zf.forEach(t),Wf.forEach(t),_i=c(Eo),cn=r(Eo,"SPAN",{});var eh=o(cn);Ei=a(eh,"*BatchEncoding*"),eh.forEach(t),Eo.forEach(t),er=c(s),j(ys.$$.fragment,s),sr=c(s),qe=r(s,"P",{});var vs=o(qe);ki=a(vs,"La sortie d\u2019un "),dn=r(vs,"EM",{});var sh=o(dn);$i=a(sh,"tokenizer"),sh.forEach(t),qi=a(vs," n\u2019est pas un simple dictionnaire Python. Ce que nous obtenons est en fait un objet sp\xE9cial "),mn=r(vs,"CODE",{});var th=o(mn);yi=a(th,"BatchEncoding"),th.forEach(t),wi=a(vs,". C\u2019est une sous-classe d\u2019un dictionnaire (c\u2019est pourquoi nous avons pu indexer ce r\xE9sultat sans probl\xE8me auparavant), mais avec des m\xE9thodes suppl\xE9mentaires qui sont principalement utilis\xE9es par les "),fn=r(vs,"EM",{});var nh=o(fn);Ci=a(nh,"tokenizers"),nh.forEach(t),Oi=a(vs," rapides."),vs.forEach(t),tr=c(s),de=r(s,"P",{});var Ce=o(de);zi=a(Ce,"En plus de leurs capacit\xE9s de parall\xE9lisation, la fonctionnalit\xE9 cl\xE9 des "),hn=r(Ce,"EM",{});var ah=o(hn);Pi=a(ah,"tokenizers"),ah.forEach(t),Di=a(Ce," rapides est qu\u2019ils gardent toujours la trace de l\u2019\xE9tendue originale des textes d\u2019o\xF9 proviennent les "),xn=r(Ce,"EM",{});var lh=o(xn);Ii=a(lh,"tokens"),lh.forEach(t),Mi=a(Ce," finaux, une fonctionnalit\xE9 que nous appelons "),gn=r(Ce,"EM",{});var rh=o(gn);Ti=a(rh,"mapping offset"),rh.forEach(t),Ri=a(Ce,". Cela permet de d\xE9bloquer des fonctionnalit\xE9s telles que le mappage de chaque mot aux "),vn=r(Ce,"EM",{});var oh=o(vn);Si=a(oh,"tokens"),oh.forEach(t),Bi=a(Ce," qu\u2019il a g\xE9n\xE9r\xE9s ou le mappage de chaque caract\xE8re du texte original au "),bn=r(Ce,"EM",{});var ih=o(bn);Ai=a(ih,"token"),ih.forEach(t),Fi=a(Ce," qu\u2019il contient, et vice versa."),Ce.forEach(t),nr=c(s),yt=r(s,"P",{});var ph=o(yt);Ni=a(ph,"Prenons un exemple :"),ph.forEach(t),ar=c(s),j(ws.$$.fragment,s),lr=c(s),Se=r(s,"P",{});var Ht=o(Se);Li=a(Ht,"Comme mentionn\xE9 pr\xE9c\xE9demment, nous obtenons un objet "),jn=r(Ht,"CODE",{});var uh=o(jn);Xi=a(uh,"BatchEncoding"),uh.forEach(t),Hi=a(Ht," dans la sortie du "),_n=r(Ht,"EM",{});var ch=o(_n);Gi=a(ch,"tokenizer"),ch.forEach(t),Vi=a(Ht," :"),Ht.forEach(t),rr=c(s),j(Cs.$$.fragment,s),or=c(s),le=r(s,"P",{});var je=o(le);Qi=a(je,"Puisque la classe "),En=r(je,"CODE",{});var dh=o(En);Ji=a(dh,"AutoTokenizer"),dh.forEach(t),Ui=a(je," choisit un "),kn=r(je,"EM",{});var mh=o(kn);Yi=a(mh,"tokenizer"),mh.forEach(t),Ki=a(je," rapide par d\xE9faut, nous pouvons utiliser les m\xE9thodes suppl\xE9mentaires que cet objet "),$n=r(je,"CODE",{});var fh=o($n);Wi=a(fh,"BatchEncoding"),fh.forEach(t),Zi=a(je," fournit. Nous avons deux fa\xE7ons de v\xE9rifier si notre "),qn=r(je,"EM",{});var hh=o(qn);ep=a(hh,"tokenizer"),hh.forEach(t),sp=a(je," est rapide ou lent. Nous pouvons soit v\xE9rifier l\u2019attribut "),yn=r(je,"CODE",{});var xh=o(yn);tp=a(xh,"is_fast"),xh.forEach(t),np=a(je," du "),wn=r(je,"EM",{});var gh=o(wn);ap=a(gh,"tokenizer"),gh.forEach(t),lp=a(je," :"),je.forEach(t),ir=c(s),j(Os.$$.fragment,s),pr=c(s),j(zs.$$.fragment,s),ur=c(s),ns=r(s,"P",{});var ko=o(ns);rp=a(ko,"ou v\xE9rifiez le m\xEAme attribut de notre "),Cn=r(ko,"CODE",{});var vh=o(Cn);op=a(vh,"encoding"),vh.forEach(t),ip=a(ko," :"),ko.forEach(t),cr=c(s),j(Ps.$$.fragment,s),dr=c(s),j(Ds.$$.fragment,s),mr=c(s),ye=r(s,"P",{});var bs=o(ye);pp=a(bs,"Voyons ce qu\u2019un "),On=r(bs,"EM",{});var bh=o(On);up=a(bh,"tokenizer"),bh.forEach(t),cp=a(bs," rapide nous permet de faire. Tout d\u2019abord, nous pouvons acc\xE9der aux "),zn=r(bs,"EM",{});var jh=o(zn);dp=a(jh,"tokens"),jh.forEach(t),mp=a(bs," sans avoir \xE0 reconvertir les ID en "),Pn=r(bs,"EM",{});var _h=o(Pn);fp=a(_h,"tokens"),_h.forEach(t),hp=a(bs," :"),bs.forEach(t),fr=c(s),j(Is.$$.fragment,s),hr=c(s),j(Ms.$$.fragment,s),xr=c(s),ve=r(s,"P",{});var Le=o(ve);xp=a(Le,"Dans ce cas, le "),Dn=r(Le,"EM",{});var Eh=o(Dn);gp=a(Eh,"token"),Eh.forEach(t),vp=a(Le," \xE0 l\u2019index 5 est "),In=r(Le,"CODE",{});var kh=o(In);bp=a(kh,"##yl"),kh.forEach(t),jp=a(Le,", qui fait partie du mot \xAB Sylvain \xBB dans la phrase originale. Nous pouvons \xE9galement utiliser la m\xE9thode "),Mn=r(Le,"CODE",{});var $h=o(Mn);_p=a($h,"word_ids()"),$h.forEach(t),Ep=a(Le," pour obtenir l\u2019index du mot dont provient chaque "),Tn=r(Le,"EM",{});var qh=o(Tn);kp=a(qh,"token"),qh.forEach(t),$p=a(Le," :"),Le.forEach(t),gr=c(s),j(Ts.$$.fragment,s),vr=c(s),j(Rs.$$.fragment,s),br=c(s),I=r(s,"P",{});var F=o(I);qp=a(F,"On peut voir que les "),Rn=r(F,"EM",{});var yh=o(Rn);yp=a(yh,"tokens"),yh.forEach(t),wp=a(F," sp\xE9ciaux du tokenizer "),Sn=r(F,"CODE",{});var wh=o(Sn);Cp=a(wh,"[CLS]"),wh.forEach(t),Op=a(F," et "),Bn=r(F,"CODE",{});var Ch=o(Bn);zp=a(Ch,"[SEP]"),Ch.forEach(t),Pp=a(F," sont mis en correspondance avec "),An=r(F,"CODE",{});var Oh=o(An);Dp=a(Oh,"None"),Oh.forEach(t),Ip=a(F,", puis chaque "),Fn=r(F,"EM",{});var zh=o(Fn);Mp=a(zh,"token"),zh.forEach(t),Tp=a(F," est mis en correspondance avec le mot dont il provient. Ceci est particuli\xE8rement utile pour d\xE9terminer si un "),Nn=r(F,"EM",{});var Ph=o(Nn);Rp=a(Ph,"token"),Ph.forEach(t),Sp=a(F," est au d\xE9but d\u2019un mot ou si deux "),Ln=r(F,"EM",{});var Dh=o(Ln);Bp=a(Dh,"tokens"),Dh.forEach(t),Ap=a(F," sont dans le m\xEAme mot. Nous pourrions nous appuyer sur le pr\xE9fixe "),Xn=r(F,"CODE",{});var Ih=o(Xn);Fp=a(Ih,"##"),Ih.forEach(t),Np=a(F," pour cela, mais il ne fonctionne que pour les "),Hn=r(F,"EM",{});var Mh=o(Hn);Lp=a(Mh,"tokenizers"),Mh.forEach(t),Xp=a(F," de type BERT. Cette m\xE9thode fonctionne pour n\u2019importe quel type de "),Gn=r(F,"EM",{});var Th=o(Gn);Hp=a(Th,"tokenizer"),Th.forEach(t),Gp=a(F,", du moment qu\u2019il est rapide. Dans le chapitre suivant, nous verrons comment utiliser cette capacit\xE9 pour appliquer correctement les \xE9tiquettes que nous avons pour chaque mot aux "),Vn=r(F,"EM",{});var Rh=o(Vn);Vp=a(Rh,"tokens"),Rh.forEach(t),Qp=a(F," dans des t\xE2ches comme la reconnaissance d\u2019entit\xE9s nomm\xE9es (NER) et le marquage POS (Part-of-speech). Nous pouvons \xE9galement l\u2019utiliser pour masquer tous les "),Qn=r(F,"EM",{});var Sh=o(Qn);Jp=a(Sh,"tokens"),Sh.forEach(t),Up=a(F," provenant du m\xEAme mot dans la mod\xE9lisation du langage masqu\xE9 (une technique appel\xE9e "),Jn=r(F,"EM",{});var Bh=o(Jn);Yp=a(Bh,"whole word masking"),Bh.forEach(t),Kp=a(F,")."),F.forEach(t),jr=c(s),j(as.$$.fragment,s),_r=c(s),we=r(s,"P",{});var js=o(we);Wp=a(js,"De m\xEAme, il existe une m\xE9thode "),Un=r(js,"CODE",{});var Ah=o(Un);Zp=a(Ah,"sentence_ids()"),Ah.forEach(t),eu=a(js," que nous pouvons utiliser pour associer un token \xE0 la phrase dont il provient (bien que dans ce cas, le "),Yn=r(js,"CODE",{});var Fh=o(Yn);su=a(Fh,"token_type_ids"),Fh.forEach(t),tu=a(js," retourn\xE9 par le "),Kn=r(js,"EM",{});var Nh=o(Kn);nu=a(Nh,"tokenizer"),Nh.forEach(t),au=a(js," peut nous donner la m\xEAme information)."),js.forEach(t),Er=c(s),re=r(s,"P",{});var _e=o(re);lu=a(_e,"Enfin, nous pouvons faire correspondre n\u2019importe quel mot ou jeton aux caract\xE8res du texte d\u2019origine, et vice versa, gr\xE2ce aux m\xE9thodes "),Wn=r(_e,"CODE",{});var Lh=o(Wn);ru=a(Lh,"word_to_chars()"),Lh.forEach(t),ou=a(_e," ou "),Zn=r(_e,"CODE",{});var Xh=o(Zn);iu=a(Xh,"token_to_chars()"),Xh.forEach(t),pu=a(_e," et "),ea=r(_e,"CODE",{});var Hh=o(ea);uu=a(Hh,"char_to_word()"),Hh.forEach(t),cu=a(_e," ou "),sa=r(_e,"CODE",{});var Gh=o(sa);du=a(Gh,"char_to_token()"),Gh.forEach(t),mu=a(_e,". Par exemple, la m\xE9thode "),ta=r(_e,"CODE",{});var Vh=o(ta);fu=a(Vh,"word_ids()"),Vh.forEach(t),hu=a(_e," nous a dit que "),na=r(_e,"CODE",{});var Qh=o(na);xu=a(Qh,"##yl"),Qh.forEach(t),gu=a(_e," fait partie du mot \xE0 l\u2019indice 3, mais de quel mot s\u2019agit-il dans la phrase ? Nous pouvons le d\xE9couvrir comme ceci :"),_e.forEach(t),kr=c(s),j(Ss.$$.fragment,s),$r=c(s),j(Bs.$$.fragment,s),qr=c(s),Be=r(s,"P",{});var Gt=o(Be);vu=a(Gt,"Comme nous l\u2019avons mentionn\xE9 pr\xE9c\xE9demment, tout ceci est rendu possible par le fait que le tokenizer rapide garde la trace de la partie du texte d\u2019o\xF9 provient chaque "),aa=r(Gt,"EM",{});var Jh=o(aa);bu=a(Jh,"token dans une liste de "),Jh.forEach(t),ju=a(Gt,"offsets*. Pour illustrer leur utilisation, nous allons maintenant vous montrer comment reproduire manuellement les r\xE9sultats du pipeline "),la=r(Gt,"CODE",{});var Uh=o(la);_u=a(Uh,"token-classification"),Uh.forEach(t),Eu=a(Gt,"."),Gt.forEach(t),yr=c(s),j(ls.$$.fragment,s),wr=c(s),Ue=r(s,"H2",{class:!0});var $o=o(Ue);rs=r($o,"A",{id:!0,class:!0,href:!0});var Yh=o(rs);ra=r(Yh,"SPAN",{});var Kh=o(ra);j(As.$$.fragment,Kh),Kh.forEach(t),Yh.forEach(t),ku=c($o),wt=r($o,"SPAN",{});var xf=o(wt);$u=a(xf,"A l'int\xE9rieur du pipeline "),oa=r(xf,"CODE",{});var Wh=o(oa);qu=a(Wh,"token-classification"),Wh.forEach(t),xf.forEach(t),$o.forEach(t),Cr=c(s),me=r(s,"P",{});var Oe=o(me);yu=a(Oe,"Dans le "),Ct=r(Oe,"A",{href:!0});var Zh=o(Ct);wu=a(Zh,"Chapitre 1"),Zh.forEach(t),Cu=a(Oe,", nous avons eu un premier aper\xE7u de l\u2019application de NER (o\xF9 la t\xE2che est d\u2019identifier les parties du texte qui correspondent \xE0 des entit\xE9s telles que des personnes, des lieux ou des organisations) avec la fonction \u{1F917} "),ia=r(Oe,"EM",{});var e2=o(ia);Ou=a(e2,"Transformers"),e2.forEach(t),zu=c(Oe),pa=r(Oe,"CODE",{});var s2=o(pa);Pu=a(s2,"pipeline()"),s2.forEach(t),Du=a(Oe,". Puis, dans "),Ot=r(Oe,"A",{href:!0});var t2=o(Ot);Iu=a(t2,"Chapitre 2"),t2.forEach(t),Mu=a(Oe,", nous avons vu comment un pipeline regroupe les trois \xE9tapes n\xE9cessaires pour obtenir les pr\xE9dictions \xE0 partir d\u2019un texte brut : la tokenisation, le passage des entr\xE9es dans le mod\xE8le et le post-traitement. Les deux premi\xE8res \xE9tapes du pipeline de "),ua=r(Oe,"CODE",{});var n2=o(ua);Tu=a(n2,"token-classification"),n2.forEach(t),Ru=a(Oe," sont les m\xEAmes que dans tout autre pipeline, mais le post-traitement est un peu plus complexe. Voyons comment !"),Oe.forEach(t),Or=c(s),De.l(s),zt=c(s),Ye=r(s,"H3",{class:!0});var qo=o(Ye);os=r(qo,"A",{id:!0,class:!0,href:!0});var a2=o(os);ca=r(a2,"SPAN",{});var l2=o(ca);j(Fs.$$.fragment,l2),l2.forEach(t),a2.forEach(t),Su=c(qo),da=r(qo,"SPAN",{});var r2=o(da);Bu=a(r2,"Obtenir les r\xE9sultats de base avec le pipeline"),r2.forEach(t),qo.forEach(t),zr=c(s),is=r(s,"P",{});var yo=o(is);Au=a(yo,"Tout d\u2019abord, prenons un pipeline de classification de tokens afin d\u2019obtenir des r\xE9sultats \xE0 comparer manuellement. Le mod\xE8le utilis\xE9 par d\xE9faut est "),Ns=r(yo,"A",{href:!0,rel:!0});var o2=o(Ns);ma=r(o2,"CODE",{});var i2=o(ma);Fu=a(i2,"dbmdz/bert-large-cased-finetuned-conll03-english"),i2.forEach(t),o2.forEach(t),Nu=a(yo,". Il effectue un NER sur les phrases :"),yo.forEach(t),Pr=c(s),j(Ls.$$.fragment,s),Dr=c(s),j(Xs.$$.fragment,s),Ir=c(s),ps=r(s,"P",{});var wo=o(ps);Lu=a(wo,"Le mod\xE8le a correctement identifi\xE9 chaque token g\xE9n\xE9r\xE9 par \xAB Sylvain \xBB comme une personne, chaque token g\xE9n\xE9r\xE9 par \xAB Hugging Face \xBB comme une organisation, et le "),fa=r(wo,"EM",{});var p2=o(fa);Xu=a(p2,"token"),p2.forEach(t),Hu=a(wo," \xAB Brooklyn \xBB comme un lieu. Nous pouvons \xE9galement demander au pipeline de regrouper les tokens qui correspondent \xE0 la m\xEAme entit\xE9 :"),wo.forEach(t),Mr=c(s),j(Hs.$$.fragment,s),Tr=c(s),j(Gs.$$.fragment,s),Rr=c(s),Z=r(s,"P",{});var xe=o(Z);Gu=a(xe,"La "),ha=r(xe,"CODE",{});var u2=o(ha);Vu=a(u2,"aggregation_strategy"),u2.forEach(t),Qu=a(xe," choisie va changer les scores calcul\xE9s pour chaque entit\xE9 group\xE9e. Avec "),xa=r(xe,"CODE",{});var c2=o(xa);Ju=a(c2,'"simple"'),c2.forEach(t),Uu=a(xe," le score est juste la moyenne des scores de chaque "),ga=r(xe,"EM",{});var d2=o(ga);Yu=a(d2,"token"),d2.forEach(t),Ku=a(xe," dans l\u2019entit\xE9 donn\xE9e : par exemple, le score de \xAB Sylvain \xBB est la moyenne des scores que nous avons vu dans l\u2019exemple pr\xE9c\xE9dent pour les tokens "),va=r(xe,"CODE",{});var m2=o(va);Wu=a(m2,"S"),m2.forEach(t),Zu=a(xe,", "),ba=r(xe,"CODE",{});var f2=o(ba);ec=a(f2,"##yl"),f2.forEach(t),sc=a(xe,", "),ja=r(xe,"CODE",{});var h2=o(ja);tc=a(h2,"##va"),h2.forEach(t),nc=a(xe,", et "),_a=r(xe,"CODE",{});var x2=o(_a);ac=a(x2,"##in"),x2.forEach(t),lc=a(xe,". D\u2019autres strat\xE9gies sont disponibles :"),xe.forEach(t),Sr=c(s),Ae=r(s,"UL",{});var Vt=o(Ae);us=r(Vt,"LI",{});var Ql=o(us);Ea=r(Ql,"CODE",{});var g2=o(Ea);rc=a(g2,'"first"'),g2.forEach(t),oc=a(Ql,", o\xF9 le score de chaque entit\xE9 est le score du premier token de cette entit\xE9 (donc pour \xAB Sylvain \xBB ce serait 0.993828, le score du token "),ka=r(Ql,"CODE",{});var v2=o(ka);ic=a(v2,"S"),v2.forEach(t),pc=a(Ql,")"),Ql.forEach(t),uc=c(Vt),Pt=r(Vt,"LI",{});var gf=o(Pt);$a=r(gf,"CODE",{});var b2=o($a);cc=a(b2,'"max"'),b2.forEach(t),dc=a(gf,", o\xF9 le score de chaque entit\xE9 est le score maximal des tokens de cette entit\xE9 (ainsi, pour \xAB Hugging Face \xBB, le score de \xAB Face \xBB serait de 0,98879766)."),gf.forEach(t),mc=c(Vt),cs=r(Vt,"LI",{});var Jl=o(cs);qa=r(Jl,"CODE",{});var j2=o(qa);fc=a(j2,'"moyenne"'),j2.forEach(t),hc=a(Jl,`, o\xF9 le score de chaque entit\xE9 est la moyenne des scores des mots qui composent cette entit\xE9 (ainsi, pour \xAB Sylvain \xBB,
il n\u2019y aurait pas de diff\xE9rence avec la strat\xE9gie `),ya=r(Jl,"CODE",{});var _2=o(ya);xc=a(_2,'"simple"'),_2.forEach(t),gc=a(Jl,", mais \u201C\xC9treinte du visage\u201D aurait un score de 0,9819, la moyenne des scores de \xAB Hugging Face \xBB, 0,975, et \xAB Face \xBB, 0,98879)."),Jl.forEach(t),Vt.forEach(t),Br=c(s),ds=r(s,"P",{});var Co=o(ds);vc=a(Co,"Voyons maintenant comment obtenir ces r\xE9sultats sans utiliser la fonction "),wa=r(Co,"CODE",{});var E2=o(wa);bc=a(E2,"pipeline()"),E2.forEach(t),jc=a(Co," !"),Co.forEach(t),Ar=c(s),Ke=r(s,"H3",{class:!0});var Oo=o(Ke);ms=r(Oo,"A",{id:!0,class:!0,href:!0});var k2=o(ms);Ca=r(k2,"SPAN",{});var $2=o(Ca);j(Vs.$$.fragment,$2),$2.forEach(t),k2.forEach(t),_c=c(Oo),Oa=r(Oo,"SPAN",{});var q2=o(Oa);Ec=a(q2,"Des entr\xE9es aux pr\xE9dictions"),q2.forEach(t),Oo.forEach(t),Fr=c(s),Me.l(s),Dt=c(s),fs=r(s,"P",{});var zo=o(fs);kc=a(zo,"Nous avons un batch avec 1 s\xE9quence de 19 "),za=r(zo,"EM",{});var y2=o(za);$c=a(y2,"tokens"),y2.forEach(t),qc=a(zo," et le mod\xE8le a 9 \xE9tiquettes diff\xE9rentes, donc la sortie du mod\xE8le a une forme de 1 x 19 x 9. Comme pour le pipeline de classification de texte, nous utilisons une fonction softmax pour convertir ces logits en probabilit\xE9s, et nous prenons l\u2019argmax pour obtenir des pr\xE9dictions (notez que nous pouvons prendre l\u2019argmax sur les logits parce que la softmax ne change pas l\u2019ordre) :"),zo.forEach(t),Nr=c(s),Re.l(s),It=c(s),j(Qs.$$.fragment,s),Lr=c(s),hs=r(s,"P",{});var Po=o(hs);yc=a(Po,"L\u2019attribut "),Pa=r(Po,"CODE",{});var w2=o(Pa);wc=a(w2,"model.config.id2label"),w2.forEach(t),Cc=a(Po," contient la correspondance entre les index et les \xE9tiquettes que nous pouvons utiliser pour donner un sens aux pr\xE9dictions :"),Po.forEach(t),Xr=c(s),j(Js.$$.fragment,s),Hr=c(s),j(Us.$$.fragment,s),Gr=c(s),$=r(s,"P",{});var O=o($);Oc=a(O,"Comme nous l\u2019avons vu pr\xE9c\xE9demment, il y a 9 \xE9tiquettes : "),Da=r(O,"CODE",{});var C2=o(Da);zc=a(C2,"O"),C2.forEach(t),Pc=a(O," est le label pour les "),Ia=r(O,"EM",{});var O2=o(Ia);Dc=a(O2,"tokens"),O2.forEach(t),Ic=a(O," qui ne sont dans aucune entit\xE9 nomm\xE9e (il signifie "),Ma=r(O,"EM",{});var z2=o(Ma);Mc=a(z2,"outside"),z2.forEach(t),Tc=a(O,") et nous avons ensuite deux labels pour chaque type d\u2019entit\xE9 (divers, personne, organisation, et lieu). L\u2019\xE9tiquette "),Ta=r(O,"CODE",{});var P2=o(Ta);Rc=a(P2,"B-XXX"),P2.forEach(t),Sc=a(O," indique que le "),Ra=r(O,"EM",{});var D2=o(Ra);Bc=a(D2,"token"),D2.forEach(t),Ac=a(O," est au d\xE9but d\u2019une entit\xE9 "),Sa=r(O,"CODE",{});var I2=o(Sa);Fc=a(I2,"XXX"),I2.forEach(t),Nc=a(O," et l\u2019\xE9tiquette "),Ba=r(O,"CODE",{});var M2=o(Ba);Lc=a(M2,"I-XXX"),M2.forEach(t),Xc=a(O," indique que le "),Aa=r(O,"EM",{});var T2=o(Aa);Hc=a(T2,"token"),T2.forEach(t),Gc=a(O," est \xE0 l\u2019int\xE9rieur de l\u2019entit\xE9 "),Fa=r(O,"CODE",{});var R2=o(Fa);Vc=a(R2,"XXX"),R2.forEach(t),Qc=a(O,". Par exemple, dans l\u2019exemple actuel, nous nous attendons \xE0 ce que notre mod\xE8le classe le "),Na=r(O,"EM",{});var S2=o(Na);Jc=a(S2,"token"),S2.forEach(t),Uc=c(O),La=r(O,"CODE",{});var B2=o(La);Yc=a(B2,"S"),B2.forEach(t),Kc=a(O," comme "),Xa=r(O,"CODE",{});var A2=o(Xa);Wc=a(A2,"B-PER"),A2.forEach(t),Zc=a(O," (d\xE9but d\u2019une entit\xE9 personne) et les "),Ha=r(O,"EM",{});var F2=o(Ha);ed=a(F2,"tokens"),F2.forEach(t),sd=c(O),Ga=r(O,"CODE",{});var N2=o(Ga);td=a(N2,"##yl"),N2.forEach(t),nd=a(O,", "),Va=r(O,"CODE",{});var L2=o(Va);ad=a(L2,"##va"),L2.forEach(t),ld=a(O," et "),Qa=r(O,"CODE",{});var X2=o(Qa);rd=a(X2,"##in"),X2.forEach(t),od=a(O," comme "),Ja=r(O,"CODE",{});var H2=o(Ja);id=a(H2,"I-PER"),H2.forEach(t),pd=a(O," (\xE0 l\u2019int\xE9rieur d\u2019une entit\xE9 personne)."),O.forEach(t),Vr=c(s),A=r(s,"P",{});var V=o(A);ud=a(V,"Vous pourriez penser que le mod\xE8le s\u2019est tromp\xE9 dans ce cas, car il a attribu\xE9 l\u2019\xE9tiquette "),Ua=r(V,"CODE",{});var G2=o(Ua);cd=a(G2,"I-PER"),G2.forEach(t),dd=a(V," \xE0 ces quatre "),Ya=r(V,"EM",{});var V2=o(Ya);md=a(V2,"tokens"),V2.forEach(t),fd=a(V,", mais ce n\u2019est pas tout \xE0 fait vrai. Il existe en fait deux formats pour ces \xE9tiquettes "),Ka=r(V,"CODE",{});var Q2=o(Ka);hd=a(Q2,"B-"),Q2.forEach(t),xd=a(V," et "),Wa=r(V,"CODE",{});var J2=o(Wa);gd=a(J2,"I-"),J2.forEach(t),vd=a(V," : "),Za=r(V,"EM",{});var U2=o(Za);bd=a(U2,"IOB1"),U2.forEach(t),jd=a(V," et "),el=r(V,"EM",{});var Y2=o(el);_d=a(Y2,"IOB2"),Y2.forEach(t),Ed=a(V,". Le format IOB2 (en rose ci-dessous) est celui que nous avons introduit alors que dans le format IOB1 (en bleu), les \xE9tiquettes commen\xE7ant par "),sl=r(V,"CODE",{});var K2=o(sl);kd=a(K2,"B-"),K2.forEach(t),$d=a(V," ne sont jamais utilis\xE9es que pour s\xE9parer deux entit\xE9s adjacentes du m\xEAme type. Le mod\xE8le que nous utilisons a \xE9t\xE9 "),tl=r(V,"EM",{});var W2=o(tl);qd=a(W2,"finetun\xE9"),W2.forEach(t),yd=a(V," sur un jeu de donn\xE9es utilisant ce format, c\u2019est pourquoi il attribue le label "),nl=r(V,"CODE",{});var Z2=o(nl);wd=a(Z2,"I-PER"),Z2.forEach(t),Cd=a(V," au "),al=r(V,"EM",{});var ex=o(al);Od=a(ex,"token"),ex.forEach(t),zd=c(V),ll=r(V,"CODE",{});var sx=o(ll);Pd=a(sx,"S"),sx.forEach(t),Dd=a(V,"."),V.forEach(t),Qr=c(s),We=r(s,"DIV",{class:!0});var Do=o(We);Ys=r(Do,"IMG",{class:!0,src:!0,alt:!0}),Id=c(Do),Ks=r(Do,"IMG",{class:!0,src:!0,alt:!0}),Do.forEach(t),Jr=c(s),Fe=r(s,"P",{});var Qt=o(Fe);Md=a(Qt,"Avec cette carte, nous sommes pr\xEAts \xE0 reproduire (presque enti\xE8rement) les r\xE9sultats du premier pipeline. Nous pouvons simplement r\xE9cup\xE9rer le score et le label de chaque "),rl=r(Qt,"EM",{});var tx=o(rl);Td=a(tx,"token"),tx.forEach(t),Rd=a(Qt," qui n\u2019a pas \xE9t\xE9 class\xE9 comme "),ol=r(Qt,"CODE",{});var nx=o(ol);Sd=a(nx,"O"),nx.forEach(t),Bd=a(Qt," :"),Qt.forEach(t),Ur=c(s),j(Ws.$$.fragment,s),Yr=c(s),j(Zs.$$.fragment,s),Kr=c(s),be=r(s,"P",{});var Xe=o(be);Ad=a(Xe,"C\u2019est tr\xE8s similaire \xE0 ce que nous avions avant, \xE0 une exception pr\xE8s : le pipeline nous a aussi donn\xE9 des informations sur le "),il=r(Xe,"CODE",{});var ax=o(il);Fd=a(ax,"d\xE9but"),ax.forEach(t),Nd=a(Xe," et la "),pl=r(Xe,"CODE",{});var lx=o(pl);Ld=a(lx,"fin"),lx.forEach(t),Xd=a(Xe," de chaque entit\xE9 dans la phrase originale. C\u2019est l\xE0 que notre mappage de d\xE9calage va entrer en jeu. Pour obtenir les d\xE9calages, il suffit de d\xE9finir "),ul=r(Xe,"CODE",{});var rx=o(ul);Hd=a(rx,"return_offsets_mapping=True"),rx.forEach(t),Gd=a(Xe," lorsque nous appliquons le "),cl=r(Xe,"EM",{});var ox=o(cl);Vd=a(ox,"tokenizer"),ox.forEach(t),Qd=a(Xe," \xE0 nos entr\xE9es :"),Xe.forEach(t),Wr=c(s),j(et.$$.fragment,s),Zr=c(s),j(st.$$.fragment,s),eo=c(s),oe=r(s,"P",{});var Ee=o(oe);Jd=a(Ee,"Chaque "),dl=r(Ee,"EM",{});var ix=o(dl);Ud=a(ix,"tuple"),ix.forEach(t),Yd=a(Ee," est l\u2019empan de texte correspondant \xE0 chaque token, o\xF9 "),ml=r(Ee,"CODE",{});var px=o(ml);Kd=a(px,"(0, 0)"),px.forEach(t),Wd=a(Ee," est r\xE9serv\xE9 aux "),fl=r(Ee,"EM",{});var ux=o(fl);Zd=a(ux,"tokens"),ux.forEach(t),em=a(Ee," sp\xE9ciaux. Nous avons vu pr\xE9c\xE9demment que le token \xE0 l\u2019index 5 est "),hl=r(Ee,"CODE",{});var cx=o(hl);sm=a(cx,"##yl"),cx.forEach(t),tm=a(Ee,", qui a "),xl=r(Ee,"CODE",{});var dx=o(xl);nm=a(dx,"(12, 14)"),dx.forEach(t),am=a(Ee," comme "),gl=r(Ee,"EM",{});var mx=o(gl);lm=a(mx,"offsets"),mx.forEach(t),rm=a(Ee," ici. Si on prend la tranche correspondante dans notre exemple :"),Ee.forEach(t),so=c(s),j(tt.$$.fragment,s),to=c(s),xs=r(s,"P",{});var Io=o(xs);om=a(Io,"nous obtenons le bon espace de texte sans le "),vl=r(Io,"CODE",{});var fx=o(vl);im=a(fx,"##"),fx.forEach(t),pm=a(Io," :"),Io.forEach(t),no=c(s),j(nt.$$.fragment,s),ao=c(s),Mt=r(s,"P",{});var hx=o(Mt);um=a(hx,"En utilisant cela, nous pouvons maintenant compl\xE9ter les r\xE9sultats pr\xE9c\xE9dents :"),hx.forEach(t),lo=c(s),j(at.$$.fragment,s),ro=c(s),j(lt.$$.fragment,s),oo=c(s),Tt=r(s,"P",{});var xx=o(Tt);cm=a(xx,"C\u2019est la m\xEAme chose que ce que nous avons obtenu avec le premier pipeline !"),xx.forEach(t),io=c(s),Ze=r(s,"H3",{class:!0});var Mo=o(Ze);gs=r(Mo,"A",{id:!0,class:!0,href:!0});var gx=o(gs);bl=r(gx,"SPAN",{});var vx=o(bl);j(rt.$$.fragment,vx),vx.forEach(t),gx.forEach(t),dm=c(Mo),jl=r(Mo,"SPAN",{});var bx=o(jl);mm=a(bx,"Regroupement des entit\xE9s"),bx.forEach(t),Mo.forEach(t),po=c(s),G=r(s,"P",{});var ee=o(G);fm=a(ee,"L\u2019utilisation des offsets pour d\xE9terminer les cl\xE9s de d\xE9but et de fin pour chaque entit\xE9 est pratique mais cette information n\u2019est pas strictement n\xE9cessaire. Cependant, lorsque nous voulons regrouper les entit\xE9s, les offsets nous \xE9pargnent un batch de code compliqu\xE9. Par exemple, si nous voulions regrouper les "),_l=r(ee,"EM",{});var jx=o(_l);hm=a(jx,"tokens"),jx.forEach(t),xm=c(ee),El=r(ee,"CODE",{});var _x=o(El);gm=a(_x,"Hu"),_x.forEach(t),vm=a(ee,", "),kl=r(ee,"CODE",{});var Ex=o(kl);bm=a(Ex,"##gging"),Ex.forEach(t),jm=a(ee,", et "),$l=r(ee,"CODE",{});var kx=o($l);_m=a(kx,"Face"),kx.forEach(t),Em=a(ee,", nous pourrions \xE9tablir des r\xE8gles sp\xE9ciales disant que les deux premiers devraient \xEAtre attach\xE9s tout en enlevant le "),ql=r(ee,"CODE",{});var $x=o(ql);km=a($x,"##"),$x.forEach(t),$m=a(ee,", et le "),yl=r(ee,"CODE",{});var qx=o(yl);qm=a(qx,"Face"),qx.forEach(t),ym=a(ee," devrait \xEAtre ajout\xE9 avec un espace puisqu\u2019il ne commence pas par "),wl=r(ee,"CODE",{});var yx=o(wl);wm=a(yx,"##"),yx.forEach(t),Cm=a(ee," mais cela ne fonctionnerait que pour ce type particulier de "),Cl=r(ee,"EM",{});var wx=o(Cl);Om=a(wx,"tokenizer"),wx.forEach(t),zm=a(ee,". Il faudrait \xE9crire un autre ensemble de r\xE8gles pour un "),Ol=r(ee,"EM",{});var Cx=o(Ol);Pm=a(Cx,"tokenizer"),Cx.forEach(t),Dm=a(ee," de type SentencePiece ou Byte-Pair-Encoding (voir plus loin dans ce chapitre)."),ee.forEach(t),uo=c(s),U=r(s,"P",{});var ie=o(U);Im=a(ie,"Avec les "),zl=r(ie,"EM",{});var Ox=o(zl);Mm=a(Ox,"offsets"),Ox.forEach(t),Tm=a(ie,", tout ce code personnalis\xE9 dispara\xEEt : il suffit de prendre l\u2019intervalle du texte original qui commence par le premier "),Pl=r(ie,"EM",{});var zx=o(Pl);Rm=a(zx,"token"),zx.forEach(t),Sm=a(ie," et se termine par le dernier "),Dl=r(ie,"EM",{});var Px=o(Dl);Bm=a(Px,"token"),Px.forEach(t),Am=a(ie,". Ainsi, dans le cas des tokens "),Il=r(ie,"CODE",{});var Dx=o(Il);Fm=a(Dx,"Hu"),Dx.forEach(t),Nm=a(ie,", "),Ml=r(ie,"CODE",{});var Ix=o(Ml);Lm=a(Ix,"##gging"),Ix.forEach(t),Xm=a(ie,", et "),Tl=r(ie,"CODE",{});var Mx=o(Tl);Hm=a(Mx,"Face"),Mx.forEach(t),Gm=a(ie,", nous devrions commencer au caract\xE8re 33 (le d\xE9but de "),Rl=r(ie,"CODE",{});var Tx=o(Rl);Vm=a(Tx,"Hu"),Tx.forEach(t),Qm=a(ie,") et finir avant le caract\xE8re 45 (la fin de "),Sl=r(ie,"CODE",{});var Rx=o(Sl);Jm=a(Rx,"Face"),Rx.forEach(t),Um=a(ie,") :"),ie.forEach(t),co=c(s),j(ot.$$.fragment,s),mo=c(s),j(it.$$.fragment,s),fo=c(s),fe=r(s,"P",{});var ze=o(fe);Ym=a(ze,"Pour \xE9crire le code qui post-traite les pr\xE9dictions tout en regroupant les entit\xE9s, nous regrouperons les entit\xE9s qui sont cons\xE9cutives et \xE9tiquet\xE9es avec "),Bl=r(ze,"CODE",{});var Sx=o(Bl);Km=a(Sx,"I-XXX"),Sx.forEach(t),Wm=a(ze,", \xE0 l\u2019exception de la premi\xE8re, qui peut \xEAtre \xE9tiquet\xE9e comme "),Al=r(ze,"CODE",{});var Bx=o(Al);Zm=a(Bx,"B-XXX"),Bx.forEach(t),ef=a(ze," ou "),Fl=r(ze,"CODE",{});var Ax=o(Fl);sf=a(Ax,"I-XXX"),Ax.forEach(t),tf=a(ze," (ainsi, nous arr\xEAtons de regrouper une entit\xE9 lorsque nous obtenons un "),Nl=r(ze,"CODE",{});var Fx=o(Nl);nf=a(Fx,"O"),Fx.forEach(t),af=a(ze,", un nouveau type d\u2019entit\xE9, ou un "),Ll=r(ze,"CODE",{});var Nx=o(Ll);lf=a(Nx,"B-XXX"),Nx.forEach(t),rf=a(ze," qui nous indique qu\u2019une entit\xE9 du m\xEAme type commence) :"),ze.forEach(t),ho=c(s),j(pt.$$.fragment,s),xo=c(s),Rt=r(s,"P",{});var Lx=o(Rt);of=a(Lx,"Et nous obtenons les m\xEAmes r\xE9sultats qu\u2019avec notre deuxi\xE8me pipeline !"),Lx.forEach(t),go=c(s),j(ut.$$.fragment,s),vo=c(s),Ne=r(s,"P",{});var Jt=o(Ne);pf=a(Jt,"Un autre exemple de t\xE2che o\xF9 ces d\xE9calages sont extr\xEAmement utiles est la r\xE9ponse aux questions. Plonger dans ce pipeline, ce que nous ferons dans la section suivante, nous permettra \xE9galement de jeter un coup d\u2019\u0153il \xE0 une derni\xE8re caract\xE9ristique des "),Xl=r(Jt,"EM",{});var Xx=o(Xl);uf=a(Xx,"tokenizers"),Xx.forEach(t),cf=a(Jt," de la biblioth\xE8que \u{1F917} "),Hl=r(Jt,"EM",{});var Hx=o(Hl);df=a(Hx,"Transformers"),Hx.forEach(t),mf=a(Jt," : la gestion des tokens qui d\xE9bordent lorsque nous tronquons une entr\xE9e \xE0 une longueur donn\xE9e."),Jt.forEach(t),this.h()},h(){k(d,"name","hf:doc:metadata"),k(d,"content",JSON.stringify(c7)),k(w,"id","pouvoirs-spciaux-des-tokenizers-rapides"),k(w,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),k(w,"href","#pouvoirs-spciaux-des-tokenizers-rapides"),k(R,"class","relative group"),k(vt,"href","/course/fr/chapter1"),k(bt,"href","/course/fr/chapter5/3"),k(Es,"align","center"),k(ks,"align","center"),k(rn,"align","center"),k(jt,"align","center"),k(_t,"align","center"),k(Et,"align","center"),k(kt,"align","center"),k($t,"align","center"),k(qt,"align","center"),k(ts,"id","batchencoding"),k(ts,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),k(ts,"href","#batchencoding"),k(Je,"class","relative group"),k(rs,"id","a-lintrieur-du-pipeline-tokenclassification"),k(rs,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),k(rs,"href","#a-lintrieur-du-pipeline-tokenclassification"),k(Ue,"class","relative group"),k(Ct,"href","/course/fr/chapter1"),k(Ot,"href","/course/fr/chapter2"),k(os,"id","obtenir-les-rsultats-de-base-avec-le-pipeline"),k(os,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),k(os,"href","#obtenir-les-rsultats-de-base-avec-le-pipeline"),k(Ye,"class","relative group"),k(Ns,"href","https://huggingface.co/dbmdz/bert-large-cased-finetuned-conll03-english"),k(Ns,"rel","nofollow"),k(ms,"id","des-entres-aux-prdictions"),k(ms,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),k(ms,"href","#des-entres-aux-prdictions"),k(Ke,"class","relative group"),k(Ys,"class","block dark:hidden"),Gx(Ys.src,bf="https://huggingface.co/datasets/huggingface-course/documentation-images/resolve/main/en/chapter6/IOB_versions.svg")||k(Ys,"src",bf),k(Ys,"alt","IOB1 vs IOB2 format"),k(Ks,"class","hidden dark:block"),Gx(Ks.src,jf="https://huggingface.co/datasets/huggingface-course/documentation-images/resolve/main/en/chapter6/IOB_versions-dark.svg")||k(Ks,"src",jf),k(Ks,"alt","IOB1 vs IOB2 format"),k(We,"class","flex justify-center"),k(gs,"id","regroupement-des-entits"),k(gs,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),k(gs,"href","#regroupement-des-entits"),k(Ze,"class","relative group")},m(s,i){e(document.head,d),p(s,g,i),_(m,s,i),p(s,y,i),p(s,R,i),e(R,w),e(w,S),_(N,S,null),e(R,L),e(R,C),e(C,ne),p(s,Q,i),ct[M].m(s,i),p(s,D,i),p(s,z,i),e(z,se),e(z,te),e(te,ge),e(z,ae),e(z,J),e(J,pe),e(z,K),e(z,T),e(T,x),e(z,P),e(z,ue),e(ue,v),e(z,B),e(z,ce),e(ce,ke),e(z,He),e(z,$e),e($e,xt),e(z,gt),e(z,Yt),e(Yt,Bo),e(z,Ao),e(z,vt),e(vt,Fo),e(z,No),p(s,Ul,i),_(_s,s,i),p(s,Yl,i),p(s,W,i),e(W,Lo),e(W,Kt),e(Kt,Xo),e(W,Ho),e(W,Wt),e(Wt,Go),e(W,Vo),e(W,Zt),e(Zt,Qo),e(W,Jo),e(W,en),e(en,Uo),e(W,Yo),e(W,bt),e(bt,Ko),e(W,Wo),e(W,sn),e(sn,Zo),e(W,ei),e(W,tn),e(tn,si),e(W,ti),p(s,Kl,i),p(s,es,i),e(es,nn),e(nn,Ge),e(Ge,Es),e(Es,an),e(an,ni),e(Es,ai),e(Ge,li),e(Ge,ks),e(ks,ln),e(ln,ri),e(ks,oi),e(Ge,ii),e(Ge,rn),e(es,pi),e(es,$s),e($s,Ve),e(Ve,jt),e(jt,on),e(on,ui),e(Ve,ci),e(Ve,_t),e(_t,di),e(Ve,mi),e(Ve,Et),e(Et,fi),e($s,hi),e($s,Qe),e(Qe,kt),e(kt,pn),e(pn,xi),e(Qe,gi),e(Qe,$t),e($t,vi),e(Qe,bi),e(Qe,qt),e(qt,ji),p(s,Wl,i),_(ss,s,i),p(s,Zl,i),p(s,Je,i),e(Je,ts),e(ts,un),_(qs,un,null),e(Je,_i),e(Je,cn),e(cn,Ei),p(s,er,i),_(ys,s,i),p(s,sr,i),p(s,qe,i),e(qe,ki),e(qe,dn),e(dn,$i),e(qe,qi),e(qe,mn),e(mn,yi),e(qe,wi),e(qe,fn),e(fn,Ci),e(qe,Oi),p(s,tr,i),p(s,de,i),e(de,zi),e(de,hn),e(hn,Pi),e(de,Di),e(de,xn),e(xn,Ii),e(de,Mi),e(de,gn),e(gn,Ti),e(de,Ri),e(de,vn),e(vn,Si),e(de,Bi),e(de,bn),e(bn,Ai),e(de,Fi),p(s,nr,i),p(s,yt,i),e(yt,Ni),p(s,ar,i),_(ws,s,i),p(s,lr,i),p(s,Se,i),e(Se,Li),e(Se,jn),e(jn,Xi),e(Se,Hi),e(Se,_n),e(_n,Gi),e(Se,Vi),p(s,rr,i),_(Cs,s,i),p(s,or,i),p(s,le,i),e(le,Qi),e(le,En),e(En,Ji),e(le,Ui),e(le,kn),e(kn,Yi),e(le,Ki),e(le,$n),e($n,Wi),e(le,Zi),e(le,qn),e(qn,ep),e(le,sp),e(le,yn),e(yn,tp),e(le,np),e(le,wn),e(wn,ap),e(le,lp),p(s,ir,i),_(Os,s,i),p(s,pr,i),_(zs,s,i),p(s,ur,i),p(s,ns,i),e(ns,rp),e(ns,Cn),e(Cn,op),e(ns,ip),p(s,cr,i),_(Ps,s,i),p(s,dr,i),_(Ds,s,i),p(s,mr,i),p(s,ye,i),e(ye,pp),e(ye,On),e(On,up),e(ye,cp),e(ye,zn),e(zn,dp),e(ye,mp),e(ye,Pn),e(Pn,fp),e(ye,hp),p(s,fr,i),_(Is,s,i),p(s,hr,i),_(Ms,s,i),p(s,xr,i),p(s,ve,i),e(ve,xp),e(ve,Dn),e(Dn,gp),e(ve,vp),e(ve,In),e(In,bp),e(ve,jp),e(ve,Mn),e(Mn,_p),e(ve,Ep),e(ve,Tn),e(Tn,kp),e(ve,$p),p(s,gr,i),_(Ts,s,i),p(s,vr,i),_(Rs,s,i),p(s,br,i),p(s,I,i),e(I,qp),e(I,Rn),e(Rn,yp),e(I,wp),e(I,Sn),e(Sn,Cp),e(I,Op),e(I,Bn),e(Bn,zp),e(I,Pp),e(I,An),e(An,Dp),e(I,Ip),e(I,Fn),e(Fn,Mp),e(I,Tp),e(I,Nn),e(Nn,Rp),e(I,Sp),e(I,Ln),e(Ln,Bp),e(I,Ap),e(I,Xn),e(Xn,Fp),e(I,Np),e(I,Hn),e(Hn,Lp),e(I,Xp),e(I,Gn),e(Gn,Hp),e(I,Gp),e(I,Vn),e(Vn,Vp),e(I,Qp),e(I,Qn),e(Qn,Jp),e(I,Up),e(I,Jn),e(Jn,Yp),e(I,Kp),p(s,jr,i),_(as,s,i),p(s,_r,i),p(s,we,i),e(we,Wp),e(we,Un),e(Un,Zp),e(we,eu),e(we,Yn),e(Yn,su),e(we,tu),e(we,Kn),e(Kn,nu),e(we,au),p(s,Er,i),p(s,re,i),e(re,lu),e(re,Wn),e(Wn,ru),e(re,ou),e(re,Zn),e(Zn,iu),e(re,pu),e(re,ea),e(ea,uu),e(re,cu),e(re,sa),e(sa,du),e(re,mu),e(re,ta),e(ta,fu),e(re,hu),e(re,na),e(na,xu),e(re,gu),p(s,kr,i),_(Ss,s,i),p(s,$r,i),_(Bs,s,i),p(s,qr,i),p(s,Be,i),e(Be,vu),e(Be,aa),e(aa,bu),e(Be,ju),e(Be,la),e(la,_u),e(Be,Eu),p(s,yr,i),_(ls,s,i),p(s,wr,i),p(s,Ue,i),e(Ue,rs),e(rs,ra),_(As,ra,null),e(Ue,ku),e(Ue,wt),e(wt,$u),e(wt,oa),e(oa,qu),p(s,Cr,i),p(s,me,i),e(me,yu),e(me,Ct),e(Ct,wu),e(me,Cu),e(me,ia),e(ia,Ou),e(me,zu),e(me,pa),e(pa,Pu),e(me,Du),e(me,Ot),e(Ot,Iu),e(me,Mu),e(me,ua),e(ua,Tu),e(me,Ru),p(s,Or,i),dt[Pe].m(s,i),p(s,zt,i),p(s,Ye,i),e(Ye,os),e(os,ca),_(Fs,ca,null),e(Ye,Su),e(Ye,da),e(da,Bu),p(s,zr,i),p(s,is,i),e(is,Au),e(is,Ns),e(Ns,ma),e(ma,Fu),e(is,Nu),p(s,Pr,i),_(Ls,s,i),p(s,Dr,i),_(Xs,s,i),p(s,Ir,i),p(s,ps,i),e(ps,Lu),e(ps,fa),e(fa,Xu),e(ps,Hu),p(s,Mr,i),_(Hs,s,i),p(s,Tr,i),_(Gs,s,i),p(s,Rr,i),p(s,Z,i),e(Z,Gu),e(Z,ha),e(ha,Vu),e(Z,Qu),e(Z,xa),e(xa,Ju),e(Z,Uu),e(Z,ga),e(ga,Yu),e(Z,Ku),e(Z,va),e(va,Wu),e(Z,Zu),e(Z,ba),e(ba,ec),e(Z,sc),e(Z,ja),e(ja,tc),e(Z,nc),e(Z,_a),e(_a,ac),e(Z,lc),p(s,Sr,i),p(s,Ae,i),e(Ae,us),e(us,Ea),e(Ea,rc),e(us,oc),e(us,ka),e(ka,ic),e(us,pc),e(Ae,uc),e(Ae,Pt),e(Pt,$a),e($a,cc),e(Pt,dc),e(Ae,mc),e(Ae,cs),e(cs,qa),e(qa,fc),e(cs,hc),e(cs,ya),e(ya,xc),e(cs,gc),p(s,Br,i),p(s,ds,i),e(ds,vc),e(ds,wa),e(wa,bc),e(ds,jc),p(s,Ar,i),p(s,Ke,i),e(Ke,ms),e(ms,Ca),_(Vs,Ca,null),e(Ke,_c),e(Ke,Oa),e(Oa,Ec),p(s,Fr,i),mt[Ie].m(s,i),p(s,Dt,i),p(s,fs,i),e(fs,kc),e(fs,za),e(za,$c),e(fs,qc),p(s,Nr,i),ft[Te].m(s,i),p(s,It,i),_(Qs,s,i),p(s,Lr,i),p(s,hs,i),e(hs,yc),e(hs,Pa),e(Pa,wc),e(hs,Cc),p(s,Xr,i),_(Js,s,i),p(s,Hr,i),_(Us,s,i),p(s,Gr,i),p(s,$,i),e($,Oc),e($,Da),e(Da,zc),e($,Pc),e($,Ia),e(Ia,Dc),e($,Ic),e($,Ma),e(Ma,Mc),e($,Tc),e($,Ta),e(Ta,Rc),e($,Sc),e($,Ra),e(Ra,Bc),e($,Ac),e($,Sa),e(Sa,Fc),e($,Nc),e($,Ba),e(Ba,Lc),e($,Xc),e($,Aa),e(Aa,Hc),e($,Gc),e($,Fa),e(Fa,Vc),e($,Qc),e($,Na),e(Na,Jc),e($,Uc),e($,La),e(La,Yc),e($,Kc),e($,Xa),e(Xa,Wc),e($,Zc),e($,Ha),e(Ha,ed),e($,sd),e($,Ga),e(Ga,td),e($,nd),e($,Va),e(Va,ad),e($,ld),e($,Qa),e(Qa,rd),e($,od),e($,Ja),e(Ja,id),e($,pd),p(s,Vr,i),p(s,A,i),e(A,ud),e(A,Ua),e(Ua,cd),e(A,dd),e(A,Ya),e(Ya,md),e(A,fd),e(A,Ka),e(Ka,hd),e(A,xd),e(A,Wa),e(Wa,gd),e(A,vd),e(A,Za),e(Za,bd),e(A,jd),e(A,el),e(el,_d),e(A,Ed),e(A,sl),e(sl,kd),e(A,$d),e(A,tl),e(tl,qd),e(A,yd),e(A,nl),e(nl,wd),e(A,Cd),e(A,al),e(al,Od),e(A,zd),e(A,ll),e(ll,Pd),e(A,Dd),p(s,Qr,i),p(s,We,i),e(We,Ys),e(We,Id),e(We,Ks),p(s,Jr,i),p(s,Fe,i),e(Fe,Md),e(Fe,rl),e(rl,Td),e(Fe,Rd),e(Fe,ol),e(ol,Sd),e(Fe,Bd),p(s,Ur,i),_(Ws,s,i),p(s,Yr,i),_(Zs,s,i),p(s,Kr,i),p(s,be,i),e(be,Ad),e(be,il),e(il,Fd),e(be,Nd),e(be,pl),e(pl,Ld),e(be,Xd),e(be,ul),e(ul,Hd),e(be,Gd),e(be,cl),e(cl,Vd),e(be,Qd),p(s,Wr,i),_(et,s,i),p(s,Zr,i),_(st,s,i),p(s,eo,i),p(s,oe,i),e(oe,Jd),e(oe,dl),e(dl,Ud),e(oe,Yd),e(oe,ml),e(ml,Kd),e(oe,Wd),e(oe,fl),e(fl,Zd),e(oe,em),e(oe,hl),e(hl,sm),e(oe,tm),e(oe,xl),e(xl,nm),e(oe,am),e(oe,gl),e(gl,lm),e(oe,rm),p(s,so,i),_(tt,s,i),p(s,to,i),p(s,xs,i),e(xs,om),e(xs,vl),e(vl,im),e(xs,pm),p(s,no,i),_(nt,s,i),p(s,ao,i),p(s,Mt,i),e(Mt,um),p(s,lo,i),_(at,s,i),p(s,ro,i),_(lt,s,i),p(s,oo,i),p(s,Tt,i),e(Tt,cm),p(s,io,i),p(s,Ze,i),e(Ze,gs),e(gs,bl),_(rt,bl,null),e(Ze,dm),e(Ze,jl),e(jl,mm),p(s,po,i),p(s,G,i),e(G,fm),e(G,_l),e(_l,hm),e(G,xm),e(G,El),e(El,gm),e(G,vm),e(G,kl),e(kl,bm),e(G,jm),e(G,$l),e($l,_m),e(G,Em),e(G,ql),e(ql,km),e(G,$m),e(G,yl),e(yl,qm),e(G,ym),e(G,wl),e(wl,wm),e(G,Cm),e(G,Cl),e(Cl,Om),e(G,zm),e(G,Ol),e(Ol,Pm),e(G,Dm),p(s,uo,i),p(s,U,i),e(U,Im),e(U,zl),e(zl,Mm),e(U,Tm),e(U,Pl),e(Pl,Rm),e(U,Sm),e(U,Dl),e(Dl,Bm),e(U,Am),e(U,Il),e(Il,Fm),e(U,Nm),e(U,Ml),e(Ml,Lm),e(U,Xm),e(U,Tl),e(Tl,Hm),e(U,Gm),e(U,Rl),e(Rl,Vm),e(U,Qm),e(U,Sl),e(Sl,Jm),e(U,Um),p(s,co,i),_(ot,s,i),p(s,mo,i),_(it,s,i),p(s,fo,i),p(s,fe,i),e(fe,Ym),e(fe,Bl),e(Bl,Km),e(fe,Wm),e(fe,Al),e(Al,Zm),e(fe,ef),e(fe,Fl),e(Fl,sf),e(fe,tf),e(fe,Nl),e(Nl,nf),e(fe,af),e(fe,Ll),e(Ll,lf),e(fe,rf),p(s,ho,i),_(pt,s,i),p(s,xo,i),p(s,Rt,i),e(Rt,of),p(s,go,i),_(ut,s,i),p(s,vo,i),p(s,Ne,i),e(Ne,pf),e(Ne,Xl),e(Xl,uf),e(Ne,cf),e(Ne,Hl),e(Hl,df),e(Ne,mf),bo=!0},p(s,[i]){const ht={};i&1&&(ht.fw=s[0]),m.$set(ht);let St=M;M=Ef(s),M!==St&&(Ro(),f(ct[St],1,1,()=>{ct[St]=null}),To(),H=ct[M],H||(H=ct[M]=_f[M](s),H.c()),h(H,1),H.m(D.parentNode,D));const Gl={};i&2&&(Gl.$$scope={dirty:i,ctx:s}),ss.$set(Gl);const Vl={};i&2&&(Vl.$$scope={dirty:i,ctx:s}),as.$set(Vl);const Y={};i&2&&(Y.$$scope={dirty:i,ctx:s}),ls.$set(Y);let Bt=Pe;Pe=$f(s),Pe!==Bt&&(Ro(),f(dt[Bt],1,1,()=>{dt[Bt]=null}),To(),De=dt[Pe],De||(De=dt[Pe]=kf[Pe](s),De.c()),h(De,1),De.m(zt.parentNode,zt));let At=Ie;Ie=yf(s),Ie!==At&&(Ro(),f(mt[At],1,1,()=>{mt[At]=null}),To(),Me=mt[Ie],Me||(Me=mt[Ie]=qf[Ie](s),Me.c()),h(Me,1),Me.m(Dt.parentNode,Dt));let Ft=Te;Te=Cf(s),Te!==Ft&&(Ro(),f(ft[Ft],1,1,()=>{ft[Ft]=null}),To(),Re=ft[Te],Re||(Re=ft[Te]=wf[Te](s),Re.c()),h(Re,1),Re.m(It.parentNode,It))},i(s){bo||(h(m.$$.fragment,s),h(N.$$.fragment,s),h(H),h(_s.$$.fragment,s),h(ss.$$.fragment,s),h(qs.$$.fragment,s),h(ys.$$.fragment,s),h(ws.$$.fragment,s),h(Cs.$$.fragment,s),h(Os.$$.fragment,s),h(zs.$$.fragment,s),h(Ps.$$.fragment,s),h(Ds.$$.fragment,s),h(Is.$$.fragment,s),h(Ms.$$.fragment,s),h(Ts.$$.fragment,s),h(Rs.$$.fragment,s),h(as.$$.fragment,s),h(Ss.$$.fragment,s),h(Bs.$$.fragment,s),h(ls.$$.fragment,s),h(As.$$.fragment,s),h(De),h(Fs.$$.fragment,s),h(Ls.$$.fragment,s),h(Xs.$$.fragment,s),h(Hs.$$.fragment,s),h(Gs.$$.fragment,s),h(Vs.$$.fragment,s),h(Me),h(Re),h(Qs.$$.fragment,s),h(Js.$$.fragment,s),h(Us.$$.fragment,s),h(Ws.$$.fragment,s),h(Zs.$$.fragment,s),h(et.$$.fragment,s),h(st.$$.fragment,s),h(tt.$$.fragment,s),h(nt.$$.fragment,s),h(at.$$.fragment,s),h(lt.$$.fragment,s),h(rt.$$.fragment,s),h(ot.$$.fragment,s),h(it.$$.fragment,s),h(pt.$$.fragment,s),h(ut.$$.fragment,s),bo=!0)},o(s){f(m.$$.fragment,s),f(N.$$.fragment,s),f(H),f(_s.$$.fragment,s),f(ss.$$.fragment,s),f(qs.$$.fragment,s),f(ys.$$.fragment,s),f(ws.$$.fragment,s),f(Cs.$$.fragment,s),f(Os.$$.fragment,s),f(zs.$$.fragment,s),f(Ps.$$.fragment,s),f(Ds.$$.fragment,s),f(Is.$$.fragment,s),f(Ms.$$.fragment,s),f(Ts.$$.fragment,s),f(Rs.$$.fragment,s),f(as.$$.fragment,s),f(Ss.$$.fragment,s),f(Bs.$$.fragment,s),f(ls.$$.fragment,s),f(As.$$.fragment,s),f(De),f(Fs.$$.fragment,s),f(Ls.$$.fragment,s),f(Xs.$$.fragment,s),f(Hs.$$.fragment,s),f(Gs.$$.fragment,s),f(Vs.$$.fragment,s),f(Me),f(Re),f(Qs.$$.fragment,s),f(Js.$$.fragment,s),f(Us.$$.fragment,s),f(Ws.$$.fragment,s),f(Zs.$$.fragment,s),f(et.$$.fragment,s),f(st.$$.fragment,s),f(tt.$$.fragment,s),f(nt.$$.fragment,s),f(at.$$.fragment,s),f(lt.$$.fragment,s),f(rt.$$.fragment,s),f(ot.$$.fragment,s),f(it.$$.fragment,s),f(pt.$$.fragment,s),f(ut.$$.fragment,s),bo=!1},d(s){t(d),s&&t(g),E(m,s),s&&t(y),s&&t(R),E(N),s&&t(Q),ct[M].d(s),s&&t(D),s&&t(z),s&&t(Ul),E(_s,s),s&&t(Yl),s&&t(W),s&&t(Kl),s&&t(es),s&&t(Wl),E(ss,s),s&&t(Zl),s&&t(Je),E(qs),s&&t(er),E(ys,s),s&&t(sr),s&&t(qe),s&&t(tr),s&&t(de),s&&t(nr),s&&t(yt),s&&t(ar),E(ws,s),s&&t(lr),s&&t(Se),s&&t(rr),E(Cs,s),s&&t(or),s&&t(le),s&&t(ir),E(Os,s),s&&t(pr),E(zs,s),s&&t(ur),s&&t(ns),s&&t(cr),E(Ps,s),s&&t(dr),E(Ds,s),s&&t(mr),s&&t(ye),s&&t(fr),E(Is,s),s&&t(hr),E(Ms,s),s&&t(xr),s&&t(ve),s&&t(gr),E(Ts,s),s&&t(vr),E(Rs,s),s&&t(br),s&&t(I),s&&t(jr),E(as,s),s&&t(_r),s&&t(we),s&&t(Er),s&&t(re),s&&t(kr),E(Ss,s),s&&t($r),E(Bs,s),s&&t(qr),s&&t(Be),s&&t(yr),E(ls,s),s&&t(wr),s&&t(Ue),E(As),s&&t(Cr),s&&t(me),s&&t(Or),dt[Pe].d(s),s&&t(zt),s&&t(Ye),E(Fs),s&&t(zr),s&&t(is),s&&t(Pr),E(Ls,s),s&&t(Dr),E(Xs,s),s&&t(Ir),s&&t(ps),s&&t(Mr),E(Hs,s),s&&t(Tr),E(Gs,s),s&&t(Rr),s&&t(Z),s&&t(Sr),s&&t(Ae),s&&t(Br),s&&t(ds),s&&t(Ar),s&&t(Ke),E(Vs),s&&t(Fr),mt[Ie].d(s),s&&t(Dt),s&&t(fs),s&&t(Nr),ft[Te].d(s),s&&t(It),E(Qs,s),s&&t(Lr),s&&t(hs),s&&t(Xr),E(Js,s),s&&t(Hr),E(Us,s),s&&t(Gr),s&&t($),s&&t(Vr),s&&t(A),s&&t(Qr),s&&t(We),s&&t(Jr),s&&t(Fe),s&&t(Ur),E(Ws,s),s&&t(Yr),E(Zs,s),s&&t(Kr),s&&t(be),s&&t(Wr),E(et,s),s&&t(Zr),E(st,s),s&&t(eo),s&&t(oe),s&&t(so),E(tt,s),s&&t(to),s&&t(xs),s&&t(no),E(nt,s),s&&t(ao),s&&t(Mt),s&&t(lo),E(at,s),s&&t(ro),E(lt,s),s&&t(oo),s&&t(Tt),s&&t(io),s&&t(Ze),E(rt),s&&t(po),s&&t(G),s&&t(uo),s&&t(U),s&&t(co),E(ot,s),s&&t(mo),E(it,s),s&&t(fo),s&&t(fe),s&&t(ho),E(pt,s),s&&t(xo),s&&t(Rt),s&&t(go),E(ut,s),s&&t(vo),s&&t(Ne)}}}const c7={local:"pouvoirs-spciaux-des-tokenizers-rapides",sections:[{local:"batchencoding",title:"*BatchEncoding*"},{local:"a-lintrieur-du-pipeline-tokenclassification",sections:[{local:"obtenir-les-rsultats-de-base-avec-le-pipeline",title:"Obtenir les r\xE9sultats de base avec le pipeline"},{local:"des-entres-aux-prdictions",title:"Des entr\xE9es aux pr\xE9dictions"},{local:"regroupement-des-entits",title:"Regroupement des entit\xE9s"}],title:"A l'int\xE9rieur du pipeline `token-classification`"}],title:"Pouvoirs sp\xE9ciaux des *tokenizers* rapides"};function d7(X,d,g){let m="pt";return Kx(()=>{const y=new URLSearchParams(window.location.search);g(0,m=y.get("fw")||"pt")}),[m]}class j7 extends Qx{constructor(d){super();Jx(this,d,d7,u7,Ux,{})}}export{j7 as default,c7 as metadata};
