import{S as Yg,i as Kg,s as Wg,e as l,k as c,w as v,t,M as Zg,c as r,d as a,m as d,x as b,a as o,h as n,b as E,N as Jg,G as e,g as p,y as _,o as h,p as di,q as x,B as j,v as ev,n as mi}from"../../chunks/vendor-hf-doc-builder.js";import{T as Uh}from"../../chunks/Tip-hf-doc-builder.js";import{Y as fi}from"../../chunks/Youtube-hf-doc-builder.js";import{I as sn}from"../../chunks/IconCopyLink-hf-doc-builder.js";import{C as y}from"../../chunks/CodeBlock-hf-doc-builder.js";import{D as Ug}from"../../chunks/DocNotebookDropdown-hf-doc-builder.js";import{F as sv}from"../../chunks/FrameworkSwitchCourse-hf-doc-builder.js";function tv(L){let u,g;return u=new Ug({props:{classNames:"absolute z-10 right-0 top-0",options:[{label:"Google Colab",value:"https://colab.research.google.com/github/huggingface/notebooks/blob/master/course/chapter6/section3_tf.ipynb"},{label:"Aws Studio",value:"https://studiolab.sagemaker.aws/import/github/huggingface/notebooks/blob/master/course/chapter6/section3_tf.ipynb"}]}}),{c(){v(u.$$.fragment)},l(m){b(u.$$.fragment,m)},m(m,w){_(u,m,w),g=!0},i(m){g||(x(u.$$.fragment,m),g=!0)},o(m){h(u.$$.fragment,m),g=!1},d(m){j(u,m)}}}function nv(L){let u,g;return u=new Ug({props:{classNames:"absolute z-10 right-0 top-0",options:[{label:"Google Colab",value:"https://colab.research.google.com/github/huggingface/notebooks/blob/master/course/chapter6/section3_pt.ipynb"},{label:"Aws Studio",value:"https://studiolab.sagemaker.aws/import/github/huggingface/notebooks/blob/master/course/chapter6/section3_pt.ipynb"}]}}),{c(){v(u.$$.fragment)},l(m){b(u.$$.fragment,m)},m(m,w){_(u,m,w),g=!0},i(m){g||(x(u.$$.fragment,m),g=!0)},o(m){h(u.$$.fragment,m),g=!1},d(m){j(u,m)}}}function av(L){let u,g,m,w,D;return{c(){u=l("p"),g=t("\u26A0\uFE0F Lors de la tokenisation d\u2019une seule phrase, vous ne verrez pas toujours une diff\xE9rence de vitesse entre les versions lente et rapide d\u2019un m\xEAme "),m=l("em"),w=t("tokenizer"),D=t(". En fait, la version rapide peut m\xEAme \xEAtre plus lente ! Ce n\u2019est que lorsque vous tokenisez beaucoup de textes en parall\xE8le et en m\xEAme temps que vous pourrez clairement voir la diff\xE9rence.")},l(C){u=r(C,"P",{});var T=o(u);g=n(T,"\u26A0\uFE0F Lors de la tokenisation d\u2019une seule phrase, vous ne verrez pas toujours une diff\xE9rence de vitesse entre les versions lente et rapide d\u2019un m\xEAme "),m=r(T,"EM",{});var F=o(m);w=n(F,"tokenizer"),F.forEach(a),D=n(T,". En fait, la version rapide peut m\xEAme \xEAtre plus lente ! Ce n\u2019est que lorsque vous tokenisez beaucoup de textes en parall\xE8le et en m\xEAme temps que vous pourrez clairement voir la diff\xE9rence."),T.forEach(a)},m(C,T){p(C,u,T),e(u,g),e(u,m),e(m,w),e(u,D)},d(C){C&&a(u)}}}function lv(L){let u,g,m,w,D,C,T,F,B,R,ae,X,Q,ue,P,J,A;return{c(){u=l("p"),g=t("\u270F\uFE0F "),m=l("strong"),w=t("Essayez !"),D=t(" Cr\xE9ez un "),C=l("em"),T=t("tokenizer"),F=t(" \xE0 partir des "),B=l("i"),R=t("checkpoints"),ae=c(),X=l("code"),Q=t("bert-base-cased"),ue=t(" et "),P=l("code"),J=t("roberta-base"),A=t(" et tokenisez \xAB 81s \xBB avec. Qu\u2019observez-vous ? Quels sont les identifiants des mots ?")},l(W){u=r(W,"P",{});var k=o(u);g=n(k,"\u270F\uFE0F "),m=r(k,"STRONG",{});var le=o(m);w=n(le,"Essayez !"),le.forEach(a),D=n(k," Cr\xE9ez un "),C=r(k,"EM",{});var ce=o(C);T=n(ce,"tokenizer"),ce.forEach(a),F=n(k," \xE0 partir des "),B=r(k,"I",{});var be=o(B);R=n(be,"checkpoints"),be.forEach(a),ae=d(k),X=r(k,"CODE",{});var de=o(X);Q=n(de,"bert-base-cased"),de.forEach(a),ue=n(k," et "),P=r(k,"CODE",{});var H=o(P);J=n(H,"roberta-base"),H.forEach(a),A=n(k," et tokenisez \xAB 81s \xBB avec. Qu\u2019observez-vous ? Quels sont les identifiants des mots ?"),k.forEach(a)},m(W,k){p(W,u,k),e(u,g),e(u,m),e(m,w),e(u,D),e(u,C),e(C,T),e(u,F),e(u,B),e(B,R),e(u,ae),e(u,X),e(X,Q),e(u,ue),e(u,P),e(P,J),e(u,A)},d(W){W&&a(u)}}}function rv(L){let u,g,m,w,D,C,T,F;return{c(){u=l("p"),g=t("\u270F\uFE0F "),m=l("strong"),w=t("Essayez !"),D=t(" R\xE9digez votre propre texte et voyez si vous pouvez comprendre quels "),C=l("em"),T=t("tokens"),F=t(" sont associ\xE9s \xE0 l\u2019identifiant du mot et comment extraire les \xE9tendues de caract\xE8res pour un seul mot. Pour obtenir des points bonus, essayez d\u2019utiliser deux phrases en entr\xE9e et voyez si les identifiants ont un sens pour vous.")},l(B){u=r(B,"P",{});var R=o(u);g=n(R,"\u270F\uFE0F "),m=r(R,"STRONG",{});var ae=o(m);w=n(ae,"Essayez !"),ae.forEach(a),D=n(R," R\xE9digez votre propre texte et voyez si vous pouvez comprendre quels "),C=r(R,"EM",{});var X=o(C);T=n(X,"tokens"),X.forEach(a),F=n(R," sont associ\xE9s \xE0 l\u2019identifiant du mot et comment extraire les \xE9tendues de caract\xE8res pour un seul mot. Pour obtenir des points bonus, essayez d\u2019utiliser deux phrases en entr\xE9e et voyez si les identifiants ont un sens pour vous."),R.forEach(a)},m(B,R){p(B,u,R),e(u,g),e(u,m),e(m,w),e(u,D),e(u,C),e(C,T),e(u,F)},d(B){B&&a(u)}}}function ov(L){let u,g;return u=new fi({props:{id:"PrX4CjrVnNc"}}),{c(){v(u.$$.fragment)},l(m){b(u.$$.fragment,m)},m(m,w){_(u,m,w),g=!0},i(m){g||(x(u.$$.fragment,m),g=!0)},o(m){h(u.$$.fragment,m),g=!1},d(m){j(u,m)}}}function iv(L){let u,g;return u=new fi({props:{id:"0E7ltQB7fM8"}}),{c(){v(u.$$.fragment)},l(m){b(u.$$.fragment,m)},m(m,w){_(u,m,w),g=!0},i(m){g||(x(u.$$.fragment,m),g=!0)},o(m){h(u.$$.fragment,m),g=!1},d(m){j(u,m)}}}function pv(L){let u,g,m,w,D,C,T,F,B,R,ae,X,Q,ue,P,J,A,W,k,le,ce,be,de,H,we,te,ve;return Q=new y({props:{code:`from transformers import AutoTokenizer, TFAutoModelForTokenClassification

model_checkpoint = "dbmdz/bert-large-cased-finetuned-conll03-english"
tokenizer = AutoTokenizer.from_pretrained(model_checkpoint)
model = TFAutoModelForTokenClassification.from_pretrained(model_checkpoint)

example = "My name is Sylvain and I work at Hugging Face in Brooklyn."
inputs = tokenizer(example, return_tensors="tf")
outputs = model(**inputs)`,highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoTokenizer, TFAutoModelForTokenClassification

model_checkpoint = <span class="hljs-string">&quot;dbmdz/bert-large-cased-finetuned-conll03-english&quot;</span>
tokenizer = AutoTokenizer.from_pretrained(model_checkpoint)
model = TFAutoModelForTokenClassification.from_pretrained(model_checkpoint)

example = <span class="hljs-string">&quot;My name is Sylvain and I work at Hugging Face in Brooklyn.&quot;</span>
inputs = tokenizer(example, return_tensors=<span class="hljs-string">&quot;tf&quot;</span>)
outputs = model(**inputs)`}}),H=new y({props:{code:`print(inputs["input_ids"].shape)
print(outputs.logits.shape)`,highlighted:`<span class="hljs-built_in">print</span>(inputs[<span class="hljs-string">&quot;input_ids&quot;</span>].shape)
<span class="hljs-built_in">print</span>(outputs.logits.shape)`}}),te=new y({props:{code:`(1, 19)
(1, 19, 9)`,highlighted:`(<span class="hljs-number">1</span>, <span class="hljs-number">19</span>)
(<span class="hljs-number">1</span>, <span class="hljs-number">19</span>, <span class="hljs-number">9</span>)`}}),{c(){u=l("p"),g=t("D\u2019abord, nous devons tokeniser notre entr\xE9e et la faire passer dans le mod\xE8le. Cela se fait exactement comme dans le "),m=l("a"),w=t("chapitre 2"),D=t(". Nous instancions le "),C=l("em"),T=t("tokenizer"),F=t(" et le mod\xE8le en utilisant les classes "),B=l("code"),R=t("TFAutoXxx"),ae=t(" et les utilisons ensuite dans notre exemple :"),X=c(),v(Q.$$.fragment),ue=c(),P=l("p"),J=t("Puisque nous utilisons "),A=l("code"),W=t("TFAutoModelForTokenClassification"),k=t(", nous obtenons un ensemble de logits pour chaque "),le=l("em"),ce=t("token"),be=t(" dans la s\xE9quence d\u2019entr\xE9e :"),de=c(),v(H.$$.fragment),we=c(),v(te.$$.fragment),this.h()},l(f){u=r(f,"P",{});var $=o(u);g=n($,"D\u2019abord, nous devons tokeniser notre entr\xE9e et la faire passer dans le mod\xE8le. Cela se fait exactement comme dans le "),m=r($,"A",{href:!0});var Ie=o(m);w=n(Ie,"chapitre 2"),Ie.forEach(a),D=n($,". Nous instancions le "),C=r($,"EM",{});var Ke=o(C);T=n(Ke,"tokenizer"),Ke.forEach(a),F=n($," et le mod\xE8le en utilisant les classes "),B=r($,"CODE",{});var We=o(B);R=n(We,"TFAutoXxx"),We.forEach(a),ae=n($," et les utilisons ensuite dans notre exemple :"),$.forEach(a),X=d(f),b(Q.$$.fragment,f),ue=d(f),P=r(f,"P",{});var ne=o(P);J=n(ne,"Puisque nous utilisons "),A=r(ne,"CODE",{});var Ze=o(A);W=n(Ze,"TFAutoModelForTokenClassification"),Ze.forEach(a),k=n(ne,", nous obtenons un ensemble de logits pour chaque "),le=r(ne,"EM",{});var es=o(le);ce=n(es,"token"),es.forEach(a),be=n(ne," dans la s\xE9quence d\u2019entr\xE9e :"),ne.forEach(a),de=d(f),b(H.$$.fragment,f),we=d(f),b(te.$$.fragment,f),this.h()},h(){E(m,"href","/course/fr/chapter2")},m(f,$){p(f,u,$),e(u,g),e(u,m),e(m,w),e(u,D),e(u,C),e(C,T),e(u,F),e(u,B),e(B,R),e(u,ae),p(f,X,$),_(Q,f,$),p(f,ue,$),p(f,P,$),e(P,J),e(P,A),e(A,W),e(P,k),e(P,le),e(le,ce),e(P,be),p(f,de,$),_(H,f,$),p(f,we,$),_(te,f,$),ve=!0},i(f){ve||(x(Q.$$.fragment,f),x(H.$$.fragment,f),x(te.$$.fragment,f),ve=!0)},o(f){h(Q.$$.fragment,f),h(H.$$.fragment,f),h(te.$$.fragment,f),ve=!1},d(f){f&&a(u),f&&a(X),j(Q,f),f&&a(ue),f&&a(P),f&&a(de),j(H,f),f&&a(we),j(te,f)}}}function uv(L){let u,g,m,w,D,C,T,F,B,R,ae,X,Q,ue,P,J,A,W,k,le,ce,be,de,H,we,te,ve;return Q=new y({props:{code:`from transformers import AutoTokenizer, AutoModelForTokenClassification

model_checkpoint = "dbmdz/bert-large-cased-finetuned-conll03-english"
tokenizer = AutoTokenizer.from_pretrained(model_checkpoint)
model = AutoModelForTokenClassification.from_pretrained(model_checkpoint)

example = "My name is Sylvain and I work at Hugging Face in Brooklyn."
inputs = tokenizer(example, return_tensors="pt")
outputs = model(**inputs)`,highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoTokenizer, AutoModelForTokenClassification

model_checkpoint = <span class="hljs-string">&quot;dbmdz/bert-large-cased-finetuned-conll03-english&quot;</span>
tokenizer = AutoTokenizer.from_pretrained(model_checkpoint)
model = AutoModelForTokenClassification.from_pretrained(model_checkpoint)

example = <span class="hljs-string">&quot;My name is Sylvain and I work at Hugging Face in Brooklyn.&quot;</span>
inputs = tokenizer(example, return_tensors=<span class="hljs-string">&quot;pt&quot;</span>)
outputs = model(**inputs)`}}),H=new y({props:{code:`print(inputs["input_ids"].shape)
print(outputs.logits.shape)`,highlighted:`<span class="hljs-built_in">print</span>(inputs[<span class="hljs-string">&quot;input_ids&quot;</span>].shape)
<span class="hljs-built_in">print</span>(outputs.logits.shape)`}}),te=new y({props:{code:`torch.Size([1, 19])
torch.Size([1, 19, 9])`,highlighted:`torch.Size([<span class="hljs-number">1</span>, <span class="hljs-number">19</span>])
torch.Size([<span class="hljs-number">1</span>, <span class="hljs-number">19</span>, <span class="hljs-number">9</span>])`}}),{c(){u=l("p"),g=t("D\u2019abord, nous devons tokeniser notre entr\xE9e et la faire passer dans le mod\xE8le. Cela se fait exactement comme dans le "),m=l("a"),w=t("chapitre 2"),D=t(". Nous instancions le "),C=l("em"),T=t("tokenizer"),F=t(" et le mod\xE8le en utilisant les classes "),B=l("code"),R=t("TFAutoXxx"),ae=t(" et les utilisons ensuite dans notre exemple :"),X=c(),v(Q.$$.fragment),ue=c(),P=l("p"),J=t("Puisque nous utilisons "),A=l("code"),W=t("AutoModelForTokenClassification"),k=t(", nous obtenons un ensemble de logits pour chaque "),le=l("em"),ce=t("token"),be=t(" dans la s\xE9quence d\u2019entr\xE9e :"),de=c(),v(H.$$.fragment),we=c(),v(te.$$.fragment),this.h()},l(f){u=r(f,"P",{});var $=o(u);g=n($,"D\u2019abord, nous devons tokeniser notre entr\xE9e et la faire passer dans le mod\xE8le. Cela se fait exactement comme dans le "),m=r($,"A",{href:!0});var Ie=o(m);w=n(Ie,"chapitre 2"),Ie.forEach(a),D=n($,". Nous instancions le "),C=r($,"EM",{});var Ke=o(C);T=n(Ke,"tokenizer"),Ke.forEach(a),F=n($," et le mod\xE8le en utilisant les classes "),B=r($,"CODE",{});var We=o(B);R=n(We,"TFAutoXxx"),We.forEach(a),ae=n($," et les utilisons ensuite dans notre exemple :"),$.forEach(a),X=d(f),b(Q.$$.fragment,f),ue=d(f),P=r(f,"P",{});var ne=o(P);J=n(ne,"Puisque nous utilisons "),A=r(ne,"CODE",{});var Ze=o(A);W=n(Ze,"AutoModelForTokenClassification"),Ze.forEach(a),k=n(ne,", nous obtenons un ensemble de logits pour chaque "),le=r(ne,"EM",{});var es=o(le);ce=n(es,"token"),es.forEach(a),be=n(ne," dans la s\xE9quence d\u2019entr\xE9e :"),ne.forEach(a),de=d(f),b(H.$$.fragment,f),we=d(f),b(te.$$.fragment,f),this.h()},h(){E(m,"href","/course/fr/chapter2")},m(f,$){p(f,u,$),e(u,g),e(u,m),e(m,w),e(u,D),e(u,C),e(C,T),e(u,F),e(u,B),e(B,R),e(u,ae),p(f,X,$),_(Q,f,$),p(f,ue,$),p(f,P,$),e(P,J),e(P,A),e(A,W),e(P,k),e(P,le),e(le,ce),e(P,be),p(f,de,$),_(H,f,$),p(f,we,$),_(te,f,$),ve=!0},i(f){ve||(x(Q.$$.fragment,f),x(H.$$.fragment,f),x(te.$$.fragment,f),ve=!0)},o(f){h(Q.$$.fragment,f),h(H.$$.fragment,f),h(te.$$.fragment,f),ve=!1},d(f){f&&a(u),f&&a(X),j(Q,f),f&&a(ue),f&&a(P),f&&a(de),j(H,f),f&&a(we),j(te,f)}}}function cv(L){let u,g;return u=new y({props:{code:`import tensorflow as tf

probabilities = tf.math.softmax(outputs.logits, axis=-1)[0]
probabilities = probabilities.numpy().tolist()
predictions = tf.math.argmax(outputs.logits, axis=-1)[0]
predictions = predictions.numpy().tolist()
print(predictions)`,highlighted:`<span class="hljs-keyword">import</span> tensorflow <span class="hljs-keyword">as</span> tf

probabilities = tf.math.softmax(outputs.logits, axis=-<span class="hljs-number">1</span>)[<span class="hljs-number">0</span>]
probabilities = probabilities.numpy().tolist()
predictions = tf.math.argmax(outputs.logits, axis=-<span class="hljs-number">1</span>)[<span class="hljs-number">0</span>]
predictions = predictions.numpy().tolist()
<span class="hljs-built_in">print</span>(predictions)`}}),{c(){v(u.$$.fragment)},l(m){b(u.$$.fragment,m)},m(m,w){_(u,m,w),g=!0},i(m){g||(x(u.$$.fragment,m),g=!0)},o(m){h(u.$$.fragment,m),g=!1},d(m){j(u,m)}}}function dv(L){let u,g;return u=new y({props:{code:`import torch

probabilities = torch.nn.functional.softmax(outputs.logits, dim=-1)[0].tolist()
predictions = outputs.logits.argmax(dim=-1)[0].tolist()
print(predictions)`,highlighted:`<span class="hljs-keyword">import</span> torch

probabilities = torch.nn.functional.softmax(outputs.logits, dim=-<span class="hljs-number">1</span>)[<span class="hljs-number">0</span>].tolist()
predictions = outputs.logits.argmax(dim=-<span class="hljs-number">1</span>)[<span class="hljs-number">0</span>].tolist()
<span class="hljs-built_in">print</span>(predictions)`}}),{c(){v(u.$$.fragment)},l(m){b(u.$$.fragment,m)},m(m,w){_(u,m,w),g=!0},i(m){g||(x(u.$$.fragment,m),g=!0)},o(m){h(u.$$.fragment,m),g=!1},d(m){j(u,m)}}}function mv(L){let u,g,m,w,D,C,T,F,B,R,ae,X,Q,ue,P,J,A,W,k,le,ce,be,de,H,we,te,ve,f,$,Ie,Ke,We,ne,Ze,es,tn,hi,xi,nn,gi,vi,qt,bi,_i,wr,Cs,Cr,U,ji,an,Ei,ki,ln,$i,qi,rn,yi,wi,on,Ci,Oi,yt,zi,Pi,pn,Mi,Ii,un,Di,Ti,cn,Ri,Si,Or,us,dn,ss,Os,mn,Bi,Ai,Fi,zs,fn,Ni,Li,Xi,hn,Hi,Ps,ts,wt,xn,Gi,Vi,Ct,Qi,Ji,Ot,Ui,Yi,ns,zt,gn,Ki,Wi,Pt,Zi,ep,Mt,sp,zr,cs,Pr,as,ds,vn,Ms,tp,It,np,bn,ap,Mr,Is,Ir,Ce,lp,_n,rp,op,jn,ip,pp,En,up,cp,Dr,me,dp,kn,mp,fp,$n,hp,xp,qn,gp,vp,yn,bp,_p,wn,jp,Ep,Tr,Dt,kp,Rr,Ds,Sr,Fe,$p,Cn,qp,yp,On,wp,Cp,Br,Ts,Ar,re,Op,zn,zp,Pp,Pn,Mp,Ip,Mn,Dp,Tp,In,Rp,Sp,Dn,Bp,Ap,Tn,Fp,Np,Fr,Rs,Nr,Ss,Lr,ms,Lp,Rn,Xp,Hp,Xr,Bs,Hr,As,Gr,Oe,Gp,Sn,Vp,Qp,Bn,Jp,Up,An,Yp,Kp,Vr,Fs,Qr,Ns,Jr,_e,Wp,Fn,Zp,eu,Nn,su,tu,Ln,nu,au,Xn,lu,ru,Ur,Ls,Yr,Xs,Kr,O,ou,Hn,iu,pu,Gn,uu,cu,Vn,du,mu,Qn,fu,hu,Jn,xu,gu,Un,vu,bu,Yn,_u,ju,Kn,Eu,ku,Wn,$u,qu,Zn,yu,wu,ea,Cu,Ou,sa,zu,Pu,ta,Mu,Iu,na,Du,Tu,aa,Ru,Su,Wr,Ne,Bu,la,Au,Fu,ra,Nu,Lu,Zr,fs,eo,je,Xu,oa,Hu,Gu,ia,Vu,Qu,pa,Ju,Uu,ua,Yu,Ku,so,Z,Wu,ca,Zu,ec,da,sc,tc,ma,nc,ac,fa,lc,rc,ha,oc,ic,xa,pc,uc,ga,cc,dc,to,Hs,no,Gs,ao,Ee,mc,va,fc,hc,ba,xc,gc,_a,vc,bc,ja,_c,jc,lo,hs,ro,ls,xs,Ea,Vs,Ec,Tt,kc,ka,$c,oo,fe,qc,Rt,yc,wc,$a,Cc,Oc,qa,zc,Pc,St,Mc,Ic,ya,Dc,Tc,io,De,Te,Bt,rs,gs,wa,Qs,Rc,Ca,Sc,po,Le,Bc,Oa,Ac,Fc,Js,za,Nc,Lc,uo,Us,co,Ys,mo,ke,Xc,Pa,Hc,Gc,Ma,Vc,Qc,Ia,Jc,Uc,Da,Yc,Kc,fo,Ks,ho,Ws,xo,ee,Wc,Ta,Zc,ed,Ra,sd,td,Sa,nd,ad,Ba,ld,rd,Aa,od,id,Fa,pd,ud,Na,cd,dd,go,Xe,He,La,md,fd,Xa,hd,xd,Ha,gd,vd,bd,vs,Ga,_d,jd,Va,Ed,kd,$d,bs,Qa,qd,yd,Ja,wd,Cd,vo,_s,Od,Ua,zd,Pd,bo,os,js,Ya,Zs,Md,Ka,Id,_o,Re,Se,At,Es,Dd,Wa,Td,Rd,jo,Be,Ae,Ft,et,Eo,ks,Sd,Za,Bd,Ad,ko,st,$o,tt,qo,q,Fd,el,Nd,Ld,sl,Xd,Hd,tl,Gd,Vd,nl,Qd,Jd,al,Ud,Yd,ll,Kd,Wd,rl,Zd,em,ol,sm,tm,il,nm,am,pl,lm,rm,ul,om,im,cl,pm,um,dl,cm,dm,ml,mm,fm,fl,hm,xm,hl,gm,vm,xl,bm,_m,yo,S,jm,gl,Em,km,vl,$m,qm,bl,ym,wm,_l,Cm,Om,jl,zm,Pm,El,Mm,Im,kl,Dm,Tm,$l,Rm,Sm,ql,Bm,Am,yl,Fm,Nm,wl,Lm,Xm,wo,is,nt,Yh,Hm,at,Kh,Co,Ge,Gm,Cl,Vm,Qm,Ol,Jm,Um,Oo,lt,zo,rt,Po,oe,Ym,zl,Km,Wm,Pl,Zm,ef,Ml,sf,tf,Il,nf,af,Dl,lf,rf,Tl,of,pf,Mo,ot,Io,it,Do,Y,uf,Rl,cf,df,Sl,mf,ff,Bl,hf,xf,Al,gf,vf,Fl,bf,_f,Nl,jf,Ef,Ll,kf,$f,Xl,qf,yf,To,pt,Ro,$s,wf,Hl,Cf,Of,So,ut,Bo,Nt,zf,Ao,ct,Fo,dt,No,Lt,Pf,Lo,ps,qs,Gl,mt,Mf,Vl,If,Xo,I,Df,Ql,Tf,Rf,Jl,Sf,Bf,Ul,Af,Ff,Yl,Nf,Lf,Kl,Xf,Hf,Wl,Gf,Vf,Zl,Qf,Jf,er,Uf,Yf,sr,Kf,Wf,tr,Zf,eh,nr,sh,th,ar,nh,ah,Ho,G,lh,lr,rh,oh,rr,ih,ph,or,uh,ch,ir,dh,mh,pr,fh,hh,ur,xh,gh,cr,vh,bh,dr,_h,jh,mr,Eh,kh,Go,ft,Vo,ht,Qo,he,$h,fr,qh,yh,hr,wh,Ch,xr,Oh,zh,gr,Ph,Mh,vr,Ih,Dh,Jo,xt,Uo,Xt,Th,Yo,gt,Ko,$e,Rh,br,Sh,Bh,_r,Ah,Fh,jr,Nh,Lh,Er,Xh,Hh,Wo;m=new sv({props:{fw:L[0]}}),F=new sn({});const Wh=[nv,tv],vt=[];function Zh(s,i){return s[0]==="pt"?0:1}J=Zh(L),A=vt[J]=Wh[J](L),Cs=new fi({props:{id:"g8quOxoqhHQ"}}),cs=new Uh({props:{warning:!0,$$slots:{default:[av]},$$scope:{ctx:L}}}),Ms=new sn({}),Is=new fi({props:{id:"3umI3tm27Vw"}}),Ds=new y({props:{code:`from transformers import AutoTokenizer

tokenizer = AutoTokenizer.from_pretrained("bert-base-cased")
example = "My name is Sylvain and I work at Hugging Face in Brooklyn."
# Je m'appelle Sylvain et je travaille chez Hugging Face \xE0 Brooklyn.
encoding = tokenizer(example)
print(type(encoding))`,highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoTokenizer

tokenizer = AutoTokenizer.from_pretrained(<span class="hljs-string">&quot;bert-base-cased&quot;</span>)
example = <span class="hljs-string">&quot;My name is Sylvain and I work at Hugging Face in Brooklyn.&quot;</span>
<span class="hljs-comment"># Je m&#x27;appelle Sylvain et je travaille chez Hugging Face \xE0 Brooklyn.</span>
encoding = tokenizer(example)
<span class="hljs-built_in">print</span>(<span class="hljs-built_in">type</span>(encoding))`}}),Ts=new y({props:{code:"<class 'transformers.tokenization_utils_base.BatchEncoding'>",highlighted:'&lt;<span class="hljs-keyword">class</span> <span class="hljs-string">&#x27;transformers.tokenization_utils_base.BatchEncoding&#x27;</span>&gt;'}}),Rs=new y({props:{code:"tokenizer.is_fast",highlighted:"tokenizer.is_fast"}}),Ss=new y({props:{code:"True",highlighted:'<span class="hljs-literal">True</span>'}}),Bs=new y({props:{code:"encoding.is_fast",highlighted:"encoding.is_fast"}}),As=new y({props:{code:"True",highlighted:'<span class="hljs-literal">True</span>'}}),Fs=new y({props:{code:"encoding.tokens()",highlighted:"encoding.tokens()"}}),Ns=new y({props:{code:`['[CLS]', 'My', 'name', 'is', 'S', '##yl', '##va', '##in', 'and', 'I', 'work', 'at', 'Hu', '##gging', 'Face', 'in',
 'Brooklyn', '.', '[SEP]']`,highlighted:`[<span class="hljs-string">&#x27;[CLS]&#x27;</span>, <span class="hljs-string">&#x27;My&#x27;</span>, <span class="hljs-string">&#x27;name&#x27;</span>, <span class="hljs-string">&#x27;is&#x27;</span>, <span class="hljs-string">&#x27;S&#x27;</span>, <span class="hljs-string">&#x27;##yl&#x27;</span>, <span class="hljs-string">&#x27;##va&#x27;</span>, <span class="hljs-string">&#x27;##in&#x27;</span>, <span class="hljs-string">&#x27;and&#x27;</span>, <span class="hljs-string">&#x27;I&#x27;</span>, <span class="hljs-string">&#x27;work&#x27;</span>, <span class="hljs-string">&#x27;at&#x27;</span>, <span class="hljs-string">&#x27;Hu&#x27;</span>, <span class="hljs-string">&#x27;##gging&#x27;</span>, <span class="hljs-string">&#x27;Face&#x27;</span>, <span class="hljs-string">&#x27;in&#x27;</span>,
 <span class="hljs-string">&#x27;Brooklyn&#x27;</span>, <span class="hljs-string">&#x27;.&#x27;</span>, <span class="hljs-string">&#x27;[SEP]&#x27;</span>]`}}),Ls=new y({props:{code:"encoding.word_ids()",highlighted:"encoding.word_ids()"}}),Xs=new y({props:{code:"[None, 0, 1, 2, 3, 3, 3, 3, 4, 5, 6, 7, 8, 8, 9, 10, 11, 12, None]",highlighted:'[<span class="hljs-literal">None</span>, <span class="hljs-number">0</span>, <span class="hljs-number">1</span>, <span class="hljs-number">2</span>, <span class="hljs-number">3</span>, <span class="hljs-number">3</span>, <span class="hljs-number">3</span>, <span class="hljs-number">3</span>, <span class="hljs-number">4</span>, <span class="hljs-number">5</span>, <span class="hljs-number">6</span>, <span class="hljs-number">7</span>, <span class="hljs-number">8</span>, <span class="hljs-number">8</span>, <span class="hljs-number">9</span>, <span class="hljs-number">10</span>, <span class="hljs-number">11</span>, <span class="hljs-number">12</span>, <span class="hljs-literal">None</span>]'}}),fs=new Uh({props:{$$slots:{default:[lv]},$$scope:{ctx:L}}}),Hs=new y({props:{code:`start, end = encoding.word_to_chars(3)
example[start:end]`,highlighted:`start, end = encoding.word_to_chars(<span class="hljs-number">3</span>)
example[start:end]`}}),Gs=new y({props:{code:"Sylvain",highlighted:"Sylvain"}}),hs=new Uh({props:{$$slots:{default:[rv]},$$scope:{ctx:L}}}),Vs=new sn({});const e2=[iv,ov],bt=[];function s2(s,i){return s[0]==="pt"?0:1}De=s2(L),Te=bt[De]=e2[De](L),Qs=new sn({}),Us=new y({props:{code:`from transformers import pipeline

token_classifier = pipeline("token-classification")
token_classifier("My name is Sylvain and I work at Hugging Face in Brooklyn.")`,highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> pipeline

token_classifier = pipeline(<span class="hljs-string">&quot;token-classification&quot;</span>)
token_classifier(<span class="hljs-string">&quot;My name is Sylvain and I work at Hugging Face in Brooklyn.&quot;</span>)`}}),Ys=new y({props:{code:`[{'entity': 'I-PER', 'score': 0.9993828, 'index': 4, 'word': 'S', 'start': 11, 'end': 12},
 {'entity': 'I-PER', 'score': 0.99815476, 'index': 5, 'word': '##yl', 'start': 12, 'end': 14},
 {'entity': 'I-PER', 'score': 0.99590725, 'index': 6, 'word': '##va', 'start': 14, 'end': 16},
 {'entity': 'I-PER', 'score': 0.9992327, 'index': 7, 'word': '##in', 'start': 16, 'end': 18},
 {'entity': 'I-ORG', 'score': 0.97389334, 'index': 12, 'word': 'Hu', 'start': 33, 'end': 35},
 {'entity': 'I-ORG', 'score': 0.976115, 'index': 13, 'word': '##gging', 'start': 35, 'end': 40},
 {'entity': 'I-ORG', 'score': 0.98879766, 'index': 14, 'word': 'Face', 'start': 41, 'end': 45},
 {'entity': 'I-LOC', 'score': 0.99321055, 'index': 16, 'word': 'Brooklyn', 'start': 49, 'end': 57}]`,highlighted:`[{<span class="hljs-string">&#x27;entity&#x27;</span>: <span class="hljs-string">&#x27;I-PER&#x27;</span>, <span class="hljs-string">&#x27;score&#x27;</span>: <span class="hljs-number">0.9993828</span>, <span class="hljs-string">&#x27;index&#x27;</span>: <span class="hljs-number">4</span>, <span class="hljs-string">&#x27;word&#x27;</span>: <span class="hljs-string">&#x27;S&#x27;</span>, <span class="hljs-string">&#x27;start&#x27;</span>: <span class="hljs-number">11</span>, <span class="hljs-string">&#x27;end&#x27;</span>: <span class="hljs-number">12</span>},
 {<span class="hljs-string">&#x27;entity&#x27;</span>: <span class="hljs-string">&#x27;I-PER&#x27;</span>, <span class="hljs-string">&#x27;score&#x27;</span>: <span class="hljs-number">0.99815476</span>, <span class="hljs-string">&#x27;index&#x27;</span>: <span class="hljs-number">5</span>, <span class="hljs-string">&#x27;word&#x27;</span>: <span class="hljs-string">&#x27;##yl&#x27;</span>, <span class="hljs-string">&#x27;start&#x27;</span>: <span class="hljs-number">12</span>, <span class="hljs-string">&#x27;end&#x27;</span>: <span class="hljs-number">14</span>},
 {<span class="hljs-string">&#x27;entity&#x27;</span>: <span class="hljs-string">&#x27;I-PER&#x27;</span>, <span class="hljs-string">&#x27;score&#x27;</span>: <span class="hljs-number">0.99590725</span>, <span class="hljs-string">&#x27;index&#x27;</span>: <span class="hljs-number">6</span>, <span class="hljs-string">&#x27;word&#x27;</span>: <span class="hljs-string">&#x27;##va&#x27;</span>, <span class="hljs-string">&#x27;start&#x27;</span>: <span class="hljs-number">14</span>, <span class="hljs-string">&#x27;end&#x27;</span>: <span class="hljs-number">16</span>},
 {<span class="hljs-string">&#x27;entity&#x27;</span>: <span class="hljs-string">&#x27;I-PER&#x27;</span>, <span class="hljs-string">&#x27;score&#x27;</span>: <span class="hljs-number">0.9992327</span>, <span class="hljs-string">&#x27;index&#x27;</span>: <span class="hljs-number">7</span>, <span class="hljs-string">&#x27;word&#x27;</span>: <span class="hljs-string">&#x27;##in&#x27;</span>, <span class="hljs-string">&#x27;start&#x27;</span>: <span class="hljs-number">16</span>, <span class="hljs-string">&#x27;end&#x27;</span>: <span class="hljs-number">18</span>},
 {<span class="hljs-string">&#x27;entity&#x27;</span>: <span class="hljs-string">&#x27;I-ORG&#x27;</span>, <span class="hljs-string">&#x27;score&#x27;</span>: <span class="hljs-number">0.97389334</span>, <span class="hljs-string">&#x27;index&#x27;</span>: <span class="hljs-number">12</span>, <span class="hljs-string">&#x27;word&#x27;</span>: <span class="hljs-string">&#x27;Hu&#x27;</span>, <span class="hljs-string">&#x27;start&#x27;</span>: <span class="hljs-number">33</span>, <span class="hljs-string">&#x27;end&#x27;</span>: <span class="hljs-number">35</span>},
 {<span class="hljs-string">&#x27;entity&#x27;</span>: <span class="hljs-string">&#x27;I-ORG&#x27;</span>, <span class="hljs-string">&#x27;score&#x27;</span>: <span class="hljs-number">0.976115</span>, <span class="hljs-string">&#x27;index&#x27;</span>: <span class="hljs-number">13</span>, <span class="hljs-string">&#x27;word&#x27;</span>: <span class="hljs-string">&#x27;##gging&#x27;</span>, <span class="hljs-string">&#x27;start&#x27;</span>: <span class="hljs-number">35</span>, <span class="hljs-string">&#x27;end&#x27;</span>: <span class="hljs-number">40</span>},
 {<span class="hljs-string">&#x27;entity&#x27;</span>: <span class="hljs-string">&#x27;I-ORG&#x27;</span>, <span class="hljs-string">&#x27;score&#x27;</span>: <span class="hljs-number">0.98879766</span>, <span class="hljs-string">&#x27;index&#x27;</span>: <span class="hljs-number">14</span>, <span class="hljs-string">&#x27;word&#x27;</span>: <span class="hljs-string">&#x27;Face&#x27;</span>, <span class="hljs-string">&#x27;start&#x27;</span>: <span class="hljs-number">41</span>, <span class="hljs-string">&#x27;end&#x27;</span>: <span class="hljs-number">45</span>},
 {<span class="hljs-string">&#x27;entity&#x27;</span>: <span class="hljs-string">&#x27;I-LOC&#x27;</span>, <span class="hljs-string">&#x27;score&#x27;</span>: <span class="hljs-number">0.99321055</span>, <span class="hljs-string">&#x27;index&#x27;</span>: <span class="hljs-number">16</span>, <span class="hljs-string">&#x27;word&#x27;</span>: <span class="hljs-string">&#x27;Brooklyn&#x27;</span>, <span class="hljs-string">&#x27;start&#x27;</span>: <span class="hljs-number">49</span>, <span class="hljs-string">&#x27;end&#x27;</span>: <span class="hljs-number">57</span>}]`}}),Ks=new y({props:{code:`from transformers import pipeline

token_classifier = pipeline("token-classification", aggregation_strategy="simple")
token_classifier("My name is Sylvain and I work at Hugging Face in Brooklyn.")`,highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> pipeline

token_classifier = pipeline(<span class="hljs-string">&quot;token-classification&quot;</span>, aggregation_strategy=<span class="hljs-string">&quot;simple&quot;</span>)
token_classifier(<span class="hljs-string">&quot;My name is Sylvain and I work at Hugging Face in Brooklyn.&quot;</span>)`}}),Ws=new y({props:{code:`[{'entity_group': 'PER', 'score': 0.9981694, 'word': 'Sylvain', 'start': 11, 'end': 18},
 {'entity_group': 'ORG', 'score': 0.97960204, 'word': 'Hugging Face', 'start': 33, 'end': 45},
 {'entity_group': 'LOC', 'score': 0.99321055, 'word': 'Brooklyn', 'start': 49, 'end': 57}]`,highlighted:`[{<span class="hljs-string">&#x27;entity_group&#x27;</span>: <span class="hljs-string">&#x27;PER&#x27;</span>, <span class="hljs-string">&#x27;score&#x27;</span>: <span class="hljs-number">0.9981694</span>, <span class="hljs-string">&#x27;word&#x27;</span>: <span class="hljs-string">&#x27;Sylvain&#x27;</span>, <span class="hljs-string">&#x27;start&#x27;</span>: <span class="hljs-number">11</span>, <span class="hljs-string">&#x27;end&#x27;</span>: <span class="hljs-number">18</span>},
 {<span class="hljs-string">&#x27;entity_group&#x27;</span>: <span class="hljs-string">&#x27;ORG&#x27;</span>, <span class="hljs-string">&#x27;score&#x27;</span>: <span class="hljs-number">0.97960204</span>, <span class="hljs-string">&#x27;word&#x27;</span>: <span class="hljs-string">&#x27;Hugging Face&#x27;</span>, <span class="hljs-string">&#x27;start&#x27;</span>: <span class="hljs-number">33</span>, <span class="hljs-string">&#x27;end&#x27;</span>: <span class="hljs-number">45</span>},
 {<span class="hljs-string">&#x27;entity_group&#x27;</span>: <span class="hljs-string">&#x27;LOC&#x27;</span>, <span class="hljs-string">&#x27;score&#x27;</span>: <span class="hljs-number">0.99321055</span>, <span class="hljs-string">&#x27;word&#x27;</span>: <span class="hljs-string">&#x27;Brooklyn&#x27;</span>, <span class="hljs-string">&#x27;start&#x27;</span>: <span class="hljs-number">49</span>, <span class="hljs-string">&#x27;end&#x27;</span>: <span class="hljs-number">57</span>}]`}}),Zs=new sn({});const t2=[uv,pv],_t=[];function n2(s,i){return s[0]==="pt"?0:1}Re=n2(L),Se=_t[Re]=t2[Re](L);const a2=[dv,cv],jt=[];function l2(s,i){return s[0]==="pt"?0:1}return Be=l2(L),Ae=jt[Be]=a2[Be](L),et=new y({props:{code:"[0, 0, 0, 0, 4, 4, 4, 4, 0, 0, 0, 0, 6, 6, 6, 0, 8, 0, 0]",highlighted:'[<span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">4</span>, <span class="hljs-number">4</span>, <span class="hljs-number">4</span>, <span class="hljs-number">4</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">6</span>, <span class="hljs-number">6</span>, <span class="hljs-number">6</span>, <span class="hljs-number">0</span>, <span class="hljs-number">8</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>]'}}),st=new y({props:{code:"model.config.id2label",highlighted:"model.config.id2label"}}),tt=new y({props:{code:`{0: 'O',
 1: 'B-MISC',
 2: 'I-MISC',
 3: 'B-PER',
 4: 'I-PER',
 5: 'B-ORG',
 6: 'I-ORG',
 7: 'B-LOC',
 8: 'I-LOC'}`,highlighted:`{<span class="hljs-number">0</span>: <span class="hljs-string">&#x27;O&#x27;</span>,
 <span class="hljs-number">1</span>: <span class="hljs-string">&#x27;B-MISC&#x27;</span>,
 <span class="hljs-number">2</span>: <span class="hljs-string">&#x27;I-MISC&#x27;</span>,
 <span class="hljs-number">3</span>: <span class="hljs-string">&#x27;B-PER&#x27;</span>,
 <span class="hljs-number">4</span>: <span class="hljs-string">&#x27;I-PER&#x27;</span>,
 <span class="hljs-number">5</span>: <span class="hljs-string">&#x27;B-ORG&#x27;</span>,
 <span class="hljs-number">6</span>: <span class="hljs-string">&#x27;I-ORG&#x27;</span>,
 <span class="hljs-number">7</span>: <span class="hljs-string">&#x27;B-LOC&#x27;</span>,
 <span class="hljs-number">8</span>: <span class="hljs-string">&#x27;I-LOC&#x27;</span>}`}}),lt=new y({props:{code:`results = []
tokens = inputs.tokens()

for idx, pred in enumerate(predictions):
    label = model.config.id2label[pred]
    if label != "O":
        results.append(
            {"entity": label, "score": probabilities[idx][pred], "word": tokens[idx]}
        )

print(results)`,highlighted:`results = []
tokens = inputs.tokens()

<span class="hljs-keyword">for</span> idx, pred <span class="hljs-keyword">in</span> <span class="hljs-built_in">enumerate</span>(predictions):
    label = model.config.id2label[pred]
    <span class="hljs-keyword">if</span> label != <span class="hljs-string">&quot;O&quot;</span>:
        results.append(
            {<span class="hljs-string">&quot;entity&quot;</span>: label, <span class="hljs-string">&quot;score&quot;</span>: probabilities[idx][pred], <span class="hljs-string">&quot;word&quot;</span>: tokens[idx]}
        )

<span class="hljs-built_in">print</span>(results)`}}),rt=new y({props:{code:`[{'entity': 'I-PER', 'score': 0.9993828, 'index': 4, 'word': 'S'},
 {'entity': 'I-PER', 'score': 0.99815476, 'index': 5, 'word': '##yl'},
 {'entity': 'I-PER', 'score': 0.99590725, 'index': 6, 'word': '##va'},
 {'entity': 'I-PER', 'score': 0.9992327, 'index': 7, 'word': '##in'},
 {'entity': 'I-ORG', 'score': 0.97389334, 'index': 12, 'word': 'Hu'},
 {'entity': 'I-ORG', 'score': 0.976115, 'index': 13, 'word': '##gging'},
 {'entity': 'I-ORG', 'score': 0.98879766, 'index': 14, 'word': 'Face'},
 {'entity': 'I-LOC', 'score': 0.99321055, 'index': 16, 'word': 'Brooklyn'}]`,highlighted:`[{<span class="hljs-string">&#x27;entity&#x27;</span>: <span class="hljs-string">&#x27;I-PER&#x27;</span>, <span class="hljs-string">&#x27;score&#x27;</span>: <span class="hljs-number">0.9993828</span>, <span class="hljs-string">&#x27;index&#x27;</span>: <span class="hljs-number">4</span>, <span class="hljs-string">&#x27;word&#x27;</span>: <span class="hljs-string">&#x27;S&#x27;</span>},
 {<span class="hljs-string">&#x27;entity&#x27;</span>: <span class="hljs-string">&#x27;I-PER&#x27;</span>, <span class="hljs-string">&#x27;score&#x27;</span>: <span class="hljs-number">0.99815476</span>, <span class="hljs-string">&#x27;index&#x27;</span>: <span class="hljs-number">5</span>, <span class="hljs-string">&#x27;word&#x27;</span>: <span class="hljs-string">&#x27;##yl&#x27;</span>},
 {<span class="hljs-string">&#x27;entity&#x27;</span>: <span class="hljs-string">&#x27;I-PER&#x27;</span>, <span class="hljs-string">&#x27;score&#x27;</span>: <span class="hljs-number">0.99590725</span>, <span class="hljs-string">&#x27;index&#x27;</span>: <span class="hljs-number">6</span>, <span class="hljs-string">&#x27;word&#x27;</span>: <span class="hljs-string">&#x27;##va&#x27;</span>},
 {<span class="hljs-string">&#x27;entity&#x27;</span>: <span class="hljs-string">&#x27;I-PER&#x27;</span>, <span class="hljs-string">&#x27;score&#x27;</span>: <span class="hljs-number">0.9992327</span>, <span class="hljs-string">&#x27;index&#x27;</span>: <span class="hljs-number">7</span>, <span class="hljs-string">&#x27;word&#x27;</span>: <span class="hljs-string">&#x27;##in&#x27;</span>},
 {<span class="hljs-string">&#x27;entity&#x27;</span>: <span class="hljs-string">&#x27;I-ORG&#x27;</span>, <span class="hljs-string">&#x27;score&#x27;</span>: <span class="hljs-number">0.97389334</span>, <span class="hljs-string">&#x27;index&#x27;</span>: <span class="hljs-number">12</span>, <span class="hljs-string">&#x27;word&#x27;</span>: <span class="hljs-string">&#x27;Hu&#x27;</span>},
 {<span class="hljs-string">&#x27;entity&#x27;</span>: <span class="hljs-string">&#x27;I-ORG&#x27;</span>, <span class="hljs-string">&#x27;score&#x27;</span>: <span class="hljs-number">0.976115</span>, <span class="hljs-string">&#x27;index&#x27;</span>: <span class="hljs-number">13</span>, <span class="hljs-string">&#x27;word&#x27;</span>: <span class="hljs-string">&#x27;##gging&#x27;</span>},
 {<span class="hljs-string">&#x27;entity&#x27;</span>: <span class="hljs-string">&#x27;I-ORG&#x27;</span>, <span class="hljs-string">&#x27;score&#x27;</span>: <span class="hljs-number">0.98879766</span>, <span class="hljs-string">&#x27;index&#x27;</span>: <span class="hljs-number">14</span>, <span class="hljs-string">&#x27;word&#x27;</span>: <span class="hljs-string">&#x27;Face&#x27;</span>},
 {<span class="hljs-string">&#x27;entity&#x27;</span>: <span class="hljs-string">&#x27;I-LOC&#x27;</span>, <span class="hljs-string">&#x27;score&#x27;</span>: <span class="hljs-number">0.99321055</span>, <span class="hljs-string">&#x27;index&#x27;</span>: <span class="hljs-number">16</span>, <span class="hljs-string">&#x27;word&#x27;</span>: <span class="hljs-string">&#x27;Brooklyn&#x27;</span>}]`}}),ot=new y({props:{code:`inputs_with_offsets = tokenizer(example, return_offsets_mapping=True)
inputs_with_offsets["offset_mapping"]`,highlighted:`inputs_with_offsets = tokenizer(example, return_offsets_mapping=<span class="hljs-literal">True</span>)
inputs_with_offsets[<span class="hljs-string">&quot;offset_mapping&quot;</span>]`}}),it=new y({props:{code:`[(0, 0), (0, 2), (3, 7), (8, 10), (11, 12), (12, 14), (14, 16), (16, 18), (19, 22), (23, 24), (25, 29), (30, 32),
 (33, 35), (35, 40), (41, 45), (46, 48), (49, 57), (57, 58), (0, 0)]`,highlighted:`[(<span class="hljs-number">0</span>, <span class="hljs-number">0</span>), (<span class="hljs-number">0</span>, <span class="hljs-number">2</span>), (<span class="hljs-number">3</span>, <span class="hljs-number">7</span>), (<span class="hljs-number">8</span>, <span class="hljs-number">10</span>), (<span class="hljs-number">11</span>, <span class="hljs-number">12</span>), (<span class="hljs-number">12</span>, <span class="hljs-number">14</span>), (<span class="hljs-number">14</span>, <span class="hljs-number">16</span>), (<span class="hljs-number">16</span>, <span class="hljs-number">18</span>), (<span class="hljs-number">19</span>, <span class="hljs-number">22</span>), (<span class="hljs-number">23</span>, <span class="hljs-number">24</span>), (<span class="hljs-number">25</span>, <span class="hljs-number">29</span>), (<span class="hljs-number">30</span>, <span class="hljs-number">32</span>),
 (<span class="hljs-number">33</span>, <span class="hljs-number">35</span>), (<span class="hljs-number">35</span>, <span class="hljs-number">40</span>), (<span class="hljs-number">41</span>, <span class="hljs-number">45</span>), (<span class="hljs-number">46</span>, <span class="hljs-number">48</span>), (<span class="hljs-number">49</span>, <span class="hljs-number">57</span>), (<span class="hljs-number">57</span>, <span class="hljs-number">58</span>), (<span class="hljs-number">0</span>, <span class="hljs-number">0</span>)]`}}),pt=new y({props:{code:"example[12:14]",highlighted:'example[<span class="hljs-number">12</span>:<span class="hljs-number">14</span>]'}}),ut=new y({props:{code:"yl",highlighted:"yl"}}),ct=new y({props:{code:`results = []
inputs_with_offsets = tokenizer(example, return_offsets_mapping=True)
tokens = inputs_with_offsets.tokens()
offsets = inputs_with_offsets["offset_mapping"]

for idx, pred in enumerate(predictions):
    label = model.config.id2label[pred]
    if label != "O":
        start, end = offsets[idx]
        results.append(
            {
                "entity": label,
                "score": probabilities[idx][pred],
                "word": tokens[idx],
                "start": start,
                "end": end,
            }
        )

print(results)`,highlighted:`results = []
inputs_with_offsets = tokenizer(example, return_offsets_mapping=<span class="hljs-literal">True</span>)
tokens = inputs_with_offsets.tokens()
offsets = inputs_with_offsets[<span class="hljs-string">&quot;offset_mapping&quot;</span>]

<span class="hljs-keyword">for</span> idx, pred <span class="hljs-keyword">in</span> <span class="hljs-built_in">enumerate</span>(predictions):
    label = model.config.id2label[pred]
    <span class="hljs-keyword">if</span> label != <span class="hljs-string">&quot;O&quot;</span>:
        start, end = offsets[idx]
        results.append(
            {
                <span class="hljs-string">&quot;entity&quot;</span>: label,
                <span class="hljs-string">&quot;score&quot;</span>: probabilities[idx][pred],
                <span class="hljs-string">&quot;word&quot;</span>: tokens[idx],
                <span class="hljs-string">&quot;start&quot;</span>: start,
                <span class="hljs-string">&quot;end&quot;</span>: end,
            }
        )

<span class="hljs-built_in">print</span>(results)`}}),dt=new y({props:{code:`[{'entity': 'I-PER', 'score': 0.9993828, 'index': 4, 'word': 'S', 'start': 11, 'end': 12},
 {'entity': 'I-PER', 'score': 0.99815476, 'index': 5, 'word': '##yl', 'start': 12, 'end': 14},
 {'entity': 'I-PER', 'score': 0.99590725, 'index': 6, 'word': '##va', 'start': 14, 'end': 16},
 {'entity': 'I-PER', 'score': 0.9992327, 'index': 7, 'word': '##in', 'start': 16, 'end': 18},
 {'entity': 'I-ORG', 'score': 0.97389334, 'index': 12, 'word': 'Hu', 'start': 33, 'end': 35},
 {'entity': 'I-ORG', 'score': 0.976115, 'index': 13, 'word': '##gging', 'start': 35, 'end': 40},
 {'entity': 'I-ORG', 'score': 0.98879766, 'index': 14, 'word': 'Face', 'start': 41, 'end': 45},
 {'entity': 'I-LOC', 'score': 0.99321055, 'index': 16, 'word': 'Brooklyn', 'start': 49, 'end': 57}]`,highlighted:`[{<span class="hljs-string">&#x27;entity&#x27;</span>: <span class="hljs-string">&#x27;I-PER&#x27;</span>, <span class="hljs-string">&#x27;score&#x27;</span>: <span class="hljs-number">0.9993828</span>, <span class="hljs-string">&#x27;index&#x27;</span>: <span class="hljs-number">4</span>, <span class="hljs-string">&#x27;word&#x27;</span>: <span class="hljs-string">&#x27;S&#x27;</span>, <span class="hljs-string">&#x27;start&#x27;</span>: <span class="hljs-number">11</span>, <span class="hljs-string">&#x27;end&#x27;</span>: <span class="hljs-number">12</span>},
 {<span class="hljs-string">&#x27;entity&#x27;</span>: <span class="hljs-string">&#x27;I-PER&#x27;</span>, <span class="hljs-string">&#x27;score&#x27;</span>: <span class="hljs-number">0.99815476</span>, <span class="hljs-string">&#x27;index&#x27;</span>: <span class="hljs-number">5</span>, <span class="hljs-string">&#x27;word&#x27;</span>: <span class="hljs-string">&#x27;##yl&#x27;</span>, <span class="hljs-string">&#x27;start&#x27;</span>: <span class="hljs-number">12</span>, <span class="hljs-string">&#x27;end&#x27;</span>: <span class="hljs-number">14</span>},
 {<span class="hljs-string">&#x27;entity&#x27;</span>: <span class="hljs-string">&#x27;I-PER&#x27;</span>, <span class="hljs-string">&#x27;score&#x27;</span>: <span class="hljs-number">0.99590725</span>, <span class="hljs-string">&#x27;index&#x27;</span>: <span class="hljs-number">6</span>, <span class="hljs-string">&#x27;word&#x27;</span>: <span class="hljs-string">&#x27;##va&#x27;</span>, <span class="hljs-string">&#x27;start&#x27;</span>: <span class="hljs-number">14</span>, <span class="hljs-string">&#x27;end&#x27;</span>: <span class="hljs-number">16</span>},
 {<span class="hljs-string">&#x27;entity&#x27;</span>: <span class="hljs-string">&#x27;I-PER&#x27;</span>, <span class="hljs-string">&#x27;score&#x27;</span>: <span class="hljs-number">0.9992327</span>, <span class="hljs-string">&#x27;index&#x27;</span>: <span class="hljs-number">7</span>, <span class="hljs-string">&#x27;word&#x27;</span>: <span class="hljs-string">&#x27;##in&#x27;</span>, <span class="hljs-string">&#x27;start&#x27;</span>: <span class="hljs-number">16</span>, <span class="hljs-string">&#x27;end&#x27;</span>: <span class="hljs-number">18</span>},
 {<span class="hljs-string">&#x27;entity&#x27;</span>: <span class="hljs-string">&#x27;I-ORG&#x27;</span>, <span class="hljs-string">&#x27;score&#x27;</span>: <span class="hljs-number">0.97389334</span>, <span class="hljs-string">&#x27;index&#x27;</span>: <span class="hljs-number">12</span>, <span class="hljs-string">&#x27;word&#x27;</span>: <span class="hljs-string">&#x27;Hu&#x27;</span>, <span class="hljs-string">&#x27;start&#x27;</span>: <span class="hljs-number">33</span>, <span class="hljs-string">&#x27;end&#x27;</span>: <span class="hljs-number">35</span>},
 {<span class="hljs-string">&#x27;entity&#x27;</span>: <span class="hljs-string">&#x27;I-ORG&#x27;</span>, <span class="hljs-string">&#x27;score&#x27;</span>: <span class="hljs-number">0.976115</span>, <span class="hljs-string">&#x27;index&#x27;</span>: <span class="hljs-number">13</span>, <span class="hljs-string">&#x27;word&#x27;</span>: <span class="hljs-string">&#x27;##gging&#x27;</span>, <span class="hljs-string">&#x27;start&#x27;</span>: <span class="hljs-number">35</span>, <span class="hljs-string">&#x27;end&#x27;</span>: <span class="hljs-number">40</span>},
 {<span class="hljs-string">&#x27;entity&#x27;</span>: <span class="hljs-string">&#x27;I-ORG&#x27;</span>, <span class="hljs-string">&#x27;score&#x27;</span>: <span class="hljs-number">0.98879766</span>, <span class="hljs-string">&#x27;index&#x27;</span>: <span class="hljs-number">14</span>, <span class="hljs-string">&#x27;word&#x27;</span>: <span class="hljs-string">&#x27;Face&#x27;</span>, <span class="hljs-string">&#x27;start&#x27;</span>: <span class="hljs-number">41</span>, <span class="hljs-string">&#x27;end&#x27;</span>: <span class="hljs-number">45</span>},
 {<span class="hljs-string">&#x27;entity&#x27;</span>: <span class="hljs-string">&#x27;I-LOC&#x27;</span>, <span class="hljs-string">&#x27;score&#x27;</span>: <span class="hljs-number">0.99321055</span>, <span class="hljs-string">&#x27;index&#x27;</span>: <span class="hljs-number">16</span>, <span class="hljs-string">&#x27;word&#x27;</span>: <span class="hljs-string">&#x27;Brooklyn&#x27;</span>, <span class="hljs-string">&#x27;start&#x27;</span>: <span class="hljs-number">49</span>, <span class="hljs-string">&#x27;end&#x27;</span>: <span class="hljs-number">57</span>}]`}}),mt=new sn({}),ft=new y({props:{code:"example[33:45]",highlighted:'example[<span class="hljs-number">33</span>:<span class="hljs-number">45</span>]'}}),ht=new y({props:{code:"Hugging Face",highlighted:"Hugging Face"}}),xt=new y({props:{code:`import numpy as np

results = []
inputs_with_offsets = tokenizer(example, return_offsets_mapping=True)
tokens = inputs_with_offsets.tokens()
offsets = inputs_with_offsets["offset_mapping"]

idx = 0
while idx < len(predictions):
    pred = predictions[idx]
    label = model.config.id2label[pred]
    if label != "O":
        # Enlever le B- ou le I-
        label = label[2:]
        start, _ = offsets[idx]

        # R\xE9cup\xE9rer tous les tokens \xE9tiquet\xE9s avec I-label
        all_scores = []
        while (
            idx < len(predictions)
            and model.config.id2label[predictions[idx]] == f"I-{label}"
        ):
            all_scores.append(probabilities[idx][pred])
            _, end = offsets[idx]
            idx += 1

        # Le score est la moyenne de tous les scores des tokens dans cette entit\xE9 group\xE9e
        score = np.mean(all_scores).item()
        word = example[start:end]
        results.append(
            {
                "entity_group": label,
                "score": score,
                "word": word,
                "start": start,
                "end": end,
            }
        )
    idx += 1

print(results)`,highlighted:`<span class="hljs-keyword">import</span> numpy <span class="hljs-keyword">as</span> np

results = []
inputs_with_offsets = tokenizer(example, return_offsets_mapping=<span class="hljs-literal">True</span>)
tokens = inputs_with_offsets.tokens()
offsets = inputs_with_offsets[<span class="hljs-string">&quot;offset_mapping&quot;</span>]

idx = <span class="hljs-number">0</span>
<span class="hljs-keyword">while</span> idx &lt; <span class="hljs-built_in">len</span>(predictions):
    pred = predictions[idx]
    label = model.config.id2label[pred]
    <span class="hljs-keyword">if</span> label != <span class="hljs-string">&quot;O&quot;</span>:
        <span class="hljs-comment"># Enlever le B- ou le I-</span>
        label = label[<span class="hljs-number">2</span>:]
        start, _ = offsets[idx]

        <span class="hljs-comment"># R\xE9cup\xE9rer tous les tokens \xE9tiquet\xE9s avec I-label</span>
        all_scores = []
        <span class="hljs-keyword">while</span> (
            idx &lt; <span class="hljs-built_in">len</span>(predictions)
            <span class="hljs-keyword">and</span> model.config.id2label[predictions[idx]] == <span class="hljs-string">f&quot;I-<span class="hljs-subst">{label}</span>&quot;</span>
        ):
            all_scores.append(probabilities[idx][pred])
            _, end = offsets[idx]
            idx += <span class="hljs-number">1</span>

        <span class="hljs-comment"># Le score est la moyenne de tous les scores des tokens dans cette entit\xE9 group\xE9e</span>
        score = np.mean(all_scores).item()
        word = example[start:end]
        results.append(
            {
                <span class="hljs-string">&quot;entity_group&quot;</span>: label,
                <span class="hljs-string">&quot;score&quot;</span>: score,
                <span class="hljs-string">&quot;word&quot;</span>: word,
                <span class="hljs-string">&quot;start&quot;</span>: start,
                <span class="hljs-string">&quot;end&quot;</span>: end,
            }
        )
    idx += <span class="hljs-number">1</span>

<span class="hljs-built_in">print</span>(results)`}}),gt=new y({props:{code:`[{'entity_group': 'PER', 'score': 0.9981694, 'word': 'Sylvain', 'start': 11, 'end': 18},
 {'entity_group': 'ORG', 'score': 0.97960204, 'word': 'Hugging Face', 'start': 33, 'end': 45},
 {'entity_group': 'LOC', 'score': 0.99321055, 'word': 'Brooklyn', 'start': 49, 'end': 57}]`,highlighted:`[{<span class="hljs-string">&#x27;entity_group&#x27;</span>: <span class="hljs-string">&#x27;PER&#x27;</span>, <span class="hljs-string">&#x27;score&#x27;</span>: <span class="hljs-number">0.9981694</span>, <span class="hljs-string">&#x27;word&#x27;</span>: <span class="hljs-string">&#x27;Sylvain&#x27;</span>, <span class="hljs-string">&#x27;start&#x27;</span>: <span class="hljs-number">11</span>, <span class="hljs-string">&#x27;end&#x27;</span>: <span class="hljs-number">18</span>},
 {<span class="hljs-string">&#x27;entity_group&#x27;</span>: <span class="hljs-string">&#x27;ORG&#x27;</span>, <span class="hljs-string">&#x27;score&#x27;</span>: <span class="hljs-number">0.97960204</span>, <span class="hljs-string">&#x27;word&#x27;</span>: <span class="hljs-string">&#x27;Hugging Face&#x27;</span>, <span class="hljs-string">&#x27;start&#x27;</span>: <span class="hljs-number">33</span>, <span class="hljs-string">&#x27;end&#x27;</span>: <span class="hljs-number">45</span>},
 {<span class="hljs-string">&#x27;entity_group&#x27;</span>: <span class="hljs-string">&#x27;LOC&#x27;</span>, <span class="hljs-string">&#x27;score&#x27;</span>: <span class="hljs-number">0.99321055</span>, <span class="hljs-string">&#x27;word&#x27;</span>: <span class="hljs-string">&#x27;Brooklyn&#x27;</span>, <span class="hljs-string">&#x27;start&#x27;</span>: <span class="hljs-number">49</span>, <span class="hljs-string">&#x27;end&#x27;</span>: <span class="hljs-number">57</span>}]`}}),{c(){u=l("meta"),g=c(),v(m.$$.fragment),w=c(),D=l("h1"),C=l("a"),T=l("span"),v(F.$$.fragment),B=c(),R=l("span"),ae=t("Pouvoirs sp\xE9ciaux des "),X=l("i"),Q=t("tokenizers"),ue=t(" rapides"),P=c(),A.c(),W=c(),k=l("p"),le=t("Dans cette section, nous allons examiner de plus pr\xE8s les capacit\xE9s des "),ce=l("em"),be=t("tokenizers"),de=t(" dans \u{1F917} "),H=l("em"),we=t("Transformers"),te=t(`.
Jusqu\u2019\xE0 pr\xE9sent, nous ne les avons utilis\xE9s que pour tokeniser les entr\xE9es ou d\xE9coder les identifiants pour revenir \xE0 du texte. Mais les `),ve=l("em"),f=t("tokenizers"),$=t(", et surtout ceux soutenus par la biblioth\xE8que \u{1F917} "),Ie=l("em"),Ke=t("Tokenizers"),We=t(", peuvent faire beaucoup plus. Pour illustrer ces fonctionnalit\xE9s suppl\xE9mentaires, nous allons explorer comment reproduire les r\xE9sultats des pipelines "),ne=l("code"),Ze=t("token-classification"),es=t(" (que nous avons appel\xE9 "),tn=l("code"),hi=t("ner"),xi=t(") et "),nn=l("code"),gi=t("question-answering"),vi=t(" que nous avons rencontr\xE9s pour la premi\xE8re fois dans le "),qt=l("a"),bi=t("chapitre 1"),_i=t("."),wr=c(),v(Cs.$$.fragment),Cr=c(),U=l("p"),ji=t("Dans la discussion qui suit, nous ferons souvent la distinction entre les "),an=l("em"),Ei=t("tokenizers"),ki=t(" \xAB lents \xBB et les \xAB rapides \xBB. Les "),ln=l("em"),$i=t("tokenizers"),qi=t(" lents sont ceux \xE9crits en Python \xE0 l\u2019int\xE9rieur de la biblioth\xE8que \u{1F917} "),rn=l("em"),yi=t("Transformers"),wi=t(", tandis que les rapides sont ceux fournis par \u{1F917} "),on=l("em"),Ci=t("Tokenizers"),Oi=t(" et sont cod\xE9s en Rust. Si vous vous souvenez du tableau du "),yt=l("a"),zi=t("chapitre 5"),Pi=t(" qui indiquait combien de temps il fallait \xE0 un "),pn=l("em"),Mi=t("tokenizer"),Ii=t(" rapide et \xE0 un "),un=l("em"),Di=t("tokenizer"),Ti=t(" lent pour tokeniser le jeu de donn\xE9es "),cn=l("em"),Ri=t("Drug Review"),Si=t(", vous devriez avoir une id\xE9e de la raison pour laquelle nous les appelons rapides et lents :"),Or=c(),us=l("table"),dn=l("thead"),ss=l("tr"),Os=l("th"),mn=l("em"),Bi=t("Tokenizer"),Ai=t(" rapide"),Fi=c(),zs=l("th"),fn=l("em"),Ni=t("Tokenizer"),Li=t(" lent"),Xi=c(),hn=l("th"),Hi=c(),Ps=l("tbody"),ts=l("tr"),wt=l("td"),xn=l("code"),Gi=t("batched=True"),Vi=c(),Ct=l("td"),Qi=t("10.8s"),Ji=c(),Ot=l("td"),Ui=t("4min41s"),Yi=c(),ns=l("tr"),zt=l("td"),gn=l("code"),Ki=t("batched=False"),Wi=c(),Pt=l("td"),Zi=t("59.2s"),ep=c(),Mt=l("td"),sp=t("5min3s"),zr=c(),v(cs.$$.fragment),Pr=c(),as=l("h2"),ds=l("a"),vn=l("span"),v(Ms.$$.fragment),tp=c(),It=l("span"),np=t("L'objet "),bn=l("i"),ap=t("BatchEncoding"),Mr=c(),v(Is.$$.fragment),Ir=c(),Ce=l("p"),lp=t("La sortie d\u2019un "),_n=l("em"),rp=t("tokenizer"),op=t(" n\u2019est pas un simple dictionnaire Python. Ce que nous obtenons est en fait un objet sp\xE9cial "),jn=l("code"),ip=t("BatchEncoding"),pp=t(". C\u2019est une sous-classe d\u2019un dictionnaire (c\u2019est pourquoi nous avons pu indexer ce r\xE9sultat sans probl\xE8me auparavant), mais avec des m\xE9thodes suppl\xE9mentaires qui sont principalement utilis\xE9es par les "),En=l("em"),up=t("tokenizers"),cp=t(" rapides."),Dr=c(),me=l("p"),dp=t("En plus de leurs capacit\xE9s de parall\xE9lisation, la fonctionnalit\xE9 cl\xE9 des "),kn=l("em"),mp=t("tokenizers"),fp=t(" rapides est qu\u2019ils gardent toujours la trace de l\u2019\xE9tendue originale des textes d\u2019o\xF9 proviennent les "),$n=l("em"),hp=t("tokens"),xp=t(" finaux, une fonctionnalit\xE9 que nous appelons "),qn=l("em"),gp=t("mapping offset"),vp=t(". Cela permet de d\xE9bloquer des fonctionnalit\xE9s telles que le mappage de chaque mot aux "),yn=l("em"),bp=t("tokens"),_p=t(" qu\u2019il a g\xE9n\xE9r\xE9s ou le mappage de chaque caract\xE8re du texte original au "),wn=l("em"),jp=t("token"),Ep=t(" qu\u2019il contient, et vice versa."),Tr=c(),Dt=l("p"),kp=t("Prenons un exemple :"),Rr=c(),v(Ds.$$.fragment),Sr=c(),Fe=l("p"),$p=t("Comme mentionn\xE9 pr\xE9c\xE9demment, nous obtenons un objet "),Cn=l("code"),qp=t("BatchEncoding"),yp=t(" dans la sortie du "),On=l("em"),wp=t("tokenizer"),Cp=t(" :"),Br=c(),v(Ts.$$.fragment),Ar=c(),re=l("p"),Op=t("Puisque la classe "),zn=l("code"),zp=t("AutoTokenizer"),Pp=t(" choisit un "),Pn=l("em"),Mp=t("tokenizer"),Ip=t(" rapide par d\xE9faut, nous pouvons utiliser les m\xE9thodes suppl\xE9mentaires que cet objet "),Mn=l("code"),Dp=t("BatchEncoding"),Tp=t(" fournit. Nous avons deux fa\xE7ons de v\xE9rifier si notre "),In=l("em"),Rp=t("tokenizer"),Sp=t(" est rapide ou lent. Nous pouvons soit v\xE9rifier l\u2019attribut "),Dn=l("code"),Bp=t("is_fast"),Ap=t(" du "),Tn=l("em"),Fp=t("tokenizer"),Np=t(" comme suit :"),Fr=c(),v(Rs.$$.fragment),Nr=c(),v(Ss.$$.fragment),Lr=c(),ms=l("p"),Lp=t("soit v\xE9rifier le m\xEAme attribut mais avec notre "),Rn=l("code"),Xp=t("encoding"),Hp=t(" :"),Xr=c(),v(Bs.$$.fragment),Hr=c(),v(As.$$.fragment),Gr=c(),Oe=l("p"),Gp=t("Voyons ce qu\u2019un "),Sn=l("em"),Vp=t("tokenizer"),Qp=t(" rapide nous permet de faire. Tout d\u2019abord, nous pouvons acc\xE9der aux "),Bn=l("em"),Jp=t("tokens"),Up=t(" sans avoir \xE0 reconvertir les identifiants en "),An=l("em"),Yp=t("tokens"),Kp=t(" :"),Vr=c(),v(Fs.$$.fragment),Qr=c(),v(Ns.$$.fragment),Jr=c(),_e=l("p"),Wp=t("Dans ce cas, le "),Fn=l("em"),Zp=t("token"),eu=t(" \xE0 l\u2019index 5 est "),Nn=l("code"),su=t("##yl"),tu=t(" et fait partie du mot \xAB Sylvain \xBB dans la phrase originale. Nous pouvons \xE9galement utiliser la m\xE9thode "),Ln=l("code"),nu=t("word_ids()"),au=t(" pour obtenir l\u2019index du mot dont provient chaque "),Xn=l("em"),lu=t("token"),ru=t(" :"),Ur=c(),v(Ls.$$.fragment),Yr=c(),v(Xs.$$.fragment),Kr=c(),O=l("p"),ou=t("On peut voir que les "),Hn=l("em"),iu=t("tokens"),pu=t(" sp\xE9ciaux du "),Gn=l("em"),uu=t("tokenizer"),cu=t(", "),Vn=l("code"),du=t("[CLS]"),mu=t(" et "),Qn=l("code"),fu=t("[SEP]"),hu=t(", sont mis en correspondance avec "),Jn=l("code"),xu=t("None"),gu=t(" et que chaque "),Un=l("em"),vu=t("token"),bu=t(" est mis en correspondance avec le mot dont il provient. Ceci est particuli\xE8rement utile pour d\xE9terminer si un "),Yn=l("em"),_u=t("token"),ju=t(" est au d\xE9but d\u2019un mot ou si deux "),Kn=l("em"),Eu=t("tokens"),ku=t(" sont dans le m\xEAme mot. Nous pourrions nous appuyer sur le pr\xE9fixe "),Wn=l("code"),$u=t("##"),qu=t(" pour cela, mais il ne fonctionne que pour les "),Zn=l("em"),yu=t("tokenizers"),wu=t(" de type BERT. Cette m\xE9thode fonctionne pour n\u2019importe quel type de "),ea=l("em"),Cu=t("tokenizer"),Ou=t(", du moment qu\u2019il est rapide. Dans le chapitre suivant, nous verrons comment utiliser cette capacit\xE9 pour appliquer correctement les \xE9tiquettes que nous avons pour chaque mot aux "),sa=l("em"),zu=t("tokens"),Pu=t(" dans des t\xE2ches comme la reconnaissance d\u2019entit\xE9s nomm\xE9es et le POS ("),ta=l("em"),Mu=t("Part-of-speech"),Iu=t("). Nous pouvons \xE9galement l\u2019utiliser pour masquer tous les "),na=l("em"),Du=t("tokens"),Tu=t(" provenant du m\xEAme mot dans la mod\xE9lisation du langage masqu\xE9 (une technique appel\xE9e "),aa=l("em"),Ru=t("whole word masking"),Su=t(")."),Wr=c(),Ne=l("p"),Bu=t("La notion de ce qu\u2019est un mot est compliqu\xE9e. Par exemple, est-ce que \xAB I\u2019ll \xBB (contraction de \xAB I will \xBB) compte pour un ou deux mots ? Cela d\xE9pend en fait du "),la=l("em"),Au=t("tokenizer"),Fu=t(" et de l\u2019op\xE9ration de pr\xE9tok\xE9nisation qu\u2019il applique. Certains "),ra=l("em"),Nu=t("tokenizer"),Lu=t(" se contentent de s\xE9parer les espaces et consid\xE8rent donc qu\u2019il s\u2019agit d\u2019un seul mot. D\u2019autres utilisent la ponctuation en plus des espaces et consid\xE8rent donc qu\u2019il s\u2019agit de deux mots."),Zr=c(),v(fs.$$.fragment),eo=c(),je=l("p"),Xu=t("De m\xEAme, il existe une m\xE9thode "),oa=l("code"),Hu=t("sentence_ids()"),Gu=t(" que nous pouvons utiliser pour associer un "),ia=l("em"),Vu=t("token"),Qu=t(" \xE0 la phrase dont il provient (bien que dans ce cas, le "),pa=l("code"),Ju=t("token_type_ids"),Uu=t(" retourn\xE9 par le "),ua=l("em"),Yu=t("tokenizer"),Ku=t(" peut nous donner la m\xEAme information)."),so=c(),Z=l("p"),Wu=t("Enfin, nous pouvons faire correspondre n\u2019importe quel mot ou "),ca=l("em"),Zu=t("token"),ec=t(" aux caract\xE8res du texte d\u2019origine (et vice versa) gr\xE2ce aux m\xE9thodes "),da=l("code"),sc=t("word_to_chars()"),tc=t(" ou "),ma=l("code"),nc=t("token_to_chars()"),ac=t(" et "),fa=l("code"),lc=t("char_to_word()"),rc=t(" ou "),ha=l("code"),oc=t("char_to_token()"),ic=t(". Par exemple, la m\xE9thode "),xa=l("code"),pc=t("word_ids()"),uc=t(" nous a dit que "),ga=l("code"),cc=t("##yl"),dc=t(" fait partie du mot \xE0 l\u2019indice 3, mais de quel mot s\u2019agit-il dans la phrase ? Nous pouvons le d\xE9couvrir comme ceci :"),to=c(),v(Hs.$$.fragment),no=c(),v(Gs.$$.fragment),ao=c(),Ee=l("p"),mc=t("Comme nous l\u2019avons mentionn\xE9 pr\xE9c\xE9demment, tout ceci est rendu possible par le fait que le "),va=l("em"),fc=t("tokenizer"),hc=t(" rapide garde la trace de la partie du texte d\u2019o\xF9 provient chaque "),ba=l("em"),xc=t("token"),gc=t(" dans une liste d\u2019"),_a=l("em"),vc=t("offsets"),bc=t(". Pour illustrer leur utilisation, nous allons maintenant vous montrer comment reproduire manuellement les r\xE9sultats du pipeline "),ja=l("code"),_c=t("token-classification"),jc=t("."),lo=c(),v(hs.$$.fragment),ro=c(),ls=l("h2"),xs=l("a"),Ea=l("span"),v(Vs.$$.fragment),Ec=c(),Tt=l("span"),kc=t("A l'int\xE9rieur du pipeline "),ka=l("code"),$c=t("token-classification"),oo=c(),fe=l("p"),qc=t("Dans le "),Rt=l("a"),yc=t("chapitre 1"),wc=t(", nous avons eu un premier aper\xE7u de la NER (o\xF9 la t\xE2che est d\u2019identifier les parties du texte qui correspondent \xE0 des entit\xE9s telles que des personnes, des lieux ou des organisations) avec la fonction "),$a=l("code"),Cc=t("pipeline()"),Oc=t(" de \u{1F917} "),qa=l("em"),zc=t("Transformers"),Pc=t(". Puis, dans le "),St=l("a"),Mc=t("chapitre 2"),Ic=t(", nous avons vu comment un pipeline regroupe les trois \xE9tapes n\xE9cessaires pour obtenir les pr\xE9dictions \xE0 partir d\u2019un texte brut : la tokenisation, le passage des entr\xE9es dans le mod\xE8le et le post-traitement. Les deux premi\xE8res \xE9tapes du pipeline de "),ya=l("code"),Dc=t("token-classification"),Tc=t(" sont les m\xEAmes que dans tout autre pipeline mais le post-traitement est un peu plus complexe. Voyons comment !"),io=c(),Te.c(),Bt=c(),rs=l("h3"),gs=l("a"),wa=l("span"),v(Qs.$$.fragment),Rc=c(),Ca=l("span"),Sc=t("Obtenir les r\xE9sultats de base avec le pipeline"),po=c(),Le=l("p"),Bc=t("Tout d\u2019abord, prenons un pipeline de classification de "),Oa=l("em"),Ac=t("tokens"),Fc=t(" afin d\u2019obtenir des r\xE9sultats \xE0 comparer manuellement. Le mod\xE8le utilis\xE9 par d\xE9faut est "),Js=l("a"),za=l("code"),Nc=t("dbmdz/bert-large-cased-finetuned-conll03-english"),Lc=t(". Il effectue une NER sur les phrases :"),uo=c(),v(Us.$$.fragment),co=c(),v(Ys.$$.fragment),mo=c(),ke=l("p"),Xc=t("Le mod\xE8le a correctement identifi\xE9 chaque "),Pa=l("em"),Hc=t("token"),Gc=t(" g\xE9n\xE9r\xE9 par \xAB Sylvain \xBB comme une personne, chaque "),Ma=l("em"),Vc=t("token"),Qc=t(" g\xE9n\xE9r\xE9 par \xAB Hugging Face \xBB comme une organisation, et le "),Ia=l("em"),Jc=t("token"),Uc=t(" \xAB Brooklyn \xBB comme un lieu. Nous pouvons \xE9galement demander au pipeline de regrouper les "),Da=l("em"),Yc=t("tokens"),Kc=t(" qui correspondent \xE0 la m\xEAme entit\xE9 :"),fo=c(),v(Ks.$$.fragment),ho=c(),v(Ws.$$.fragment),xo=c(),ee=l("p"),Wc=t("La propri\xE9t\xE9 "),Ta=l("code"),Zc=t("aggregation_strategy"),ed=t(" choisie va changer les scores calcul\xE9s pour chaque entit\xE9 group\xE9e. Avec "),Ra=l("code"),sd=t('"simple"'),td=t(" le score est juste la moyenne des scores de chaque "),Sa=l("em"),nd=t("token"),ad=t(" dans l\u2019entit\xE9 donn\xE9e. Par exemple, le score de \xAB Sylvain \xBB est la moyenne des scores que nous avons vu dans l\u2019exemple pr\xE9c\xE9dent pour les tokens "),Ba=l("code"),ld=t("S"),rd=t(", "),Aa=l("code"),od=t("##yl"),id=t(", "),Fa=l("code"),pd=t("##va"),ud=t(", et "),Na=l("code"),cd=t("##in"),dd=t(". D\u2019autres strat\xE9gies sont disponibles :"),go=c(),Xe=l("ul"),He=l("li"),La=l("code"),md=t('"first"'),fd=t(", o\xF9 le score de chaque entit\xE9 est le score du premier "),Xa=l("em"),hd=t("token"),xd=t(" de cette entit\xE9 (donc pour \xAB Sylvain \xBB ce serait 0.993828, le score du token "),Ha=l("code"),gd=t("S"),vd=t(")"),bd=c(),vs=l("li"),Ga=l("code"),_d=t('"max"'),jd=t(", o\xF9 le score de chaque entit\xE9 est le score maximal des "),Va=l("em"),Ed=t("tokens"),kd=t(" de cette entit\xE9 (ainsi, pour \xAB Hugging Face \xBB, le score de \xAB Face \xBB serait de 0,98879766)."),$d=c(),bs=l("li"),Qa=l("code"),qd=t('"average"'),yd=t(", o\xF9 le score de chaque entit\xE9 est la moyenne des scores des mots qui composent cette entit\xE9 (ainsi, pour \xAB Sylvain \xBB, il n\u2019y aurait pas de diff\xE9rence avec la strat\xE9gie "),Ja=l("code"),wd=t('"simple"'),Cd=t(", mais \u201CHugging Face\u201D aurait un score de 0,9819, la moyenne des scores de \xAB Hugging \xBB, 0,975, et \xAB Face \xBB, 0,98879)."),vo=c(),_s=l("p"),Od=t("Voyons maintenant comment obtenir ces r\xE9sultats sans utiliser la fonction "),Ua=l("code"),zd=t("pipeline()"),Pd=t(" !"),bo=c(),os=l("h3"),js=l("a"),Ya=l("span"),v(Zs.$$.fragment),Md=c(),Ka=l("span"),Id=t("Des entr\xE9es aux pr\xE9dictions"),_o=c(),Se.c(),At=c(),Es=l("p"),Dd=t("Nous avons un batch avec 1 s\xE9quence de 19 "),Wa=l("em"),Td=t("tokens"),Rd=t(" et le mod\xE8le a 9 \xE9tiquettes diff\xE9rentes. Ainsi, la sortie du mod\xE8le a une forme de 1 x 19 x 9. Comme pour le pipeline de classification de texte, nous utilisons une fonction softmax pour convertir ces logits en probabilit\xE9s et nous prenons l\u2019argmax pour obtenir des pr\xE9dictions (notez que nous pouvons prendre l\u2019argmax sur les logits car la fonction softmax ne change pas l\u2019ordre) :"),jo=c(),Ae.c(),Ft=c(),v(et.$$.fragment),Eo=c(),ks=l("p"),Sd=t("L\u2019attribut "),Za=l("code"),Bd=t("model.config.id2label"),Ad=t(" contient la correspondance entre les index et les \xE9tiquettes que nous pouvons utiliser pour donner un sens aux pr\xE9dictions :"),ko=c(),v(st.$$.fragment),$o=c(),v(tt.$$.fragment),qo=c(),q=l("p"),Fd=t("Comme nous l\u2019avons vu pr\xE9c\xE9demment, il y a 9 \xE9tiquettes : "),el=l("code"),Nd=t("O"),Ld=t(" est le label pour les "),sl=l("em"),Xd=t("tokens"),Hd=t(" qui ne sont dans aucune entit\xE9 nomm\xE9e (il signifie "),tl=l("em"),Gd=t("outside"),Vd=t(" (en dehors)) et nous avons ensuite deux labels pour chaque type d\u2019entit\xE9 (divers, personne, organisation et lieu). L\u2019\xE9tiquette "),nl=l("code"),Qd=t("B-XXX"),Jd=t(" indique que le "),al=l("em"),Ud=t("token"),Yd=t(" est au d\xE9but d\u2019une entit\xE9 "),ll=l("code"),Kd=t("XXX"),Wd=t(" et l\u2019\xE9tiquette "),rl=l("code"),Zd=t("I-XXX"),em=t(" indique que le "),ol=l("em"),sm=t("token"),tm=t(" est \xE0 l\u2019int\xE9rieur de l\u2019entit\xE9 "),il=l("code"),nm=t("XXX"),am=t(". Par exemple, dans l\u2019exemple actuel, nous nous attendons \xE0 ce que notre mod\xE8le classe le "),pl=l("em"),lm=t("token"),rm=c(),ul=l("code"),om=t("S"),im=t(" comme "),cl=l("code"),pm=t("B-PER"),um=t(" (d\xE9but d\u2019une entit\xE9 personne) et les "),dl=l("em"),cm=t("tokens"),dm=c(),ml=l("code"),mm=t("##yl"),fm=t(", "),fl=l("code"),hm=t("##va"),xm=t(" et "),hl=l("code"),gm=t("##in"),vm=t(" comme "),xl=l("code"),bm=t("I-PER"),_m=t(" (\xE0 l\u2019int\xE9rieur d\u2019une entit\xE9 personne)."),yo=c(),S=l("p"),jm=t("Vous pourriez penser que le mod\xE8le s\u2019est tromp\xE9 ici car il a attribu\xE9 l\u2019\xE9tiquette "),gl=l("code"),Em=t("I-PER"),km=t(" \xE0 ces quatre "),vl=l("em"),$m=t("tokens"),qm=t(" mais ce n\u2019est pas tout \xE0 fait vrai. Il existe en fait deux formats pour ces \xE9tiquettes "),bl=l("code"),ym=t("B-"),wm=t(" et "),_l=l("code"),Cm=t("I-"),Om=t(" : "),jl=l("em"),zm=t("IOB1"),Pm=t(" et "),El=l("em"),Mm=t("IOB2"),Im=t(". Le format IOB2 (en rose ci-dessous) est celui que nous avons introduit alors que dans le format IOB1 (en bleu), les \xE9tiquettes commen\xE7ant par "),kl=l("code"),Dm=t("B-"),Tm=t(" ne sont jamais utilis\xE9es que pour s\xE9parer deux entit\xE9s adjacentes du m\xEAme type. Le mod\xE8le que nous utilisons a \xE9t\xE9 "),$l=l("em"),Rm=t("finetun\xE9"),Sm=t(" sur un jeu de donn\xE9es utilisant ce format, c\u2019est pourquoi il attribue le label "),ql=l("code"),Bm=t("I-PER"),Am=t(" au "),yl=l("em"),Fm=t("token"),Nm=c(),wl=l("code"),Lm=t("S"),Xm=t("."),wo=c(),is=l("div"),nt=l("img"),Hm=c(),at=l("img"),Co=c(),Ge=l("p"),Gm=t("Nous sommes \xE0 pr\xE9sent pr\xEAts \xE0 reproduire (presque enti\xE8rement) les r\xE9sultats du premier pipeline. Nous pouvons simplement r\xE9cup\xE9rer le score et le label de chaque "),Cl=l("em"),Vm=t("token"),Qm=t(" qui n\u2019a pas \xE9t\xE9 class\xE9 comme "),Ol=l("code"),Jm=t("O"),Um=t(" :"),Oo=c(),v(lt.$$.fragment),zo=c(),v(rt.$$.fragment),Po=c(),oe=l("p"),Ym=t("C\u2019est tr\xE8s similaire \xE0 ce que nous avions avant, \xE0 une exception pr\xE8s : le pipeline nous a aussi donn\xE9 des informations sur le "),zl=l("code"),Km=t("d\xE9but"),Wm=t(" et la "),Pl=l("code"),Zm=t("fin"),ef=t(" de chaque entit\xE9 dans la phrase originale. C\u2019est l\xE0 que notre "),Ml=l("em"),sf=t("offset mapping"),tf=t(" va entrer en jeu. Pour obtenir les "),Il=l("em"),nf=t("offsets"),af=t(", il suffit de d\xE9finir "),Dl=l("code"),lf=t("return_offsets_mapping=True"),rf=t(" lorsque nous appliquons le "),Tl=l("em"),of=t("tokenizer"),pf=t(" \xE0 nos entr\xE9es :"),Mo=c(),v(ot.$$.fragment),Io=c(),v(it.$$.fragment),Do=c(),Y=l("p"),uf=t("Chaque "),Rl=l("em"),cf=t("tuple"),df=t(" est l\u2019\xE9tendue de texte correspondant \xE0 chaque "),Sl=l("em"),mf=t("token"),ff=t(" o\xF9 "),Bl=l("code"),hf=t("(0, 0)"),xf=t(" est r\xE9serv\xE9 aux "),Al=l("em"),gf=t("tokens"),vf=t(" sp\xE9ciaux. Nous avons vu pr\xE9c\xE9demment que le "),Fl=l("em"),bf=t("token"),_f=t(" \xE0 l\u2019index 5 est "),Nl=l("code"),jf=t("##yl"),Ef=t(", qui a "),Ll=l("code"),kf=t("(12, 14)"),$f=t(" comme "),Xl=l("em"),qf=t("offsets"),yf=t(" ici. Si on prend la tranche correspondante dans notre exemple :"),To=c(),v(pt.$$.fragment),Ro=c(),$s=l("p"),wf=t("nous obtenons le bon espace de texte sans le "),Hl=l("code"),Cf=t("##"),Of=t(" :"),So=c(),v(ut.$$.fragment),Bo=c(),Nt=l("p"),zf=t("En utilisant cela, nous pouvons maintenant compl\xE9ter les r\xE9sultats pr\xE9c\xE9dents :"),Ao=c(),v(ct.$$.fragment),Fo=c(),v(dt.$$.fragment),No=c(),Lt=l("p"),Pf=t("C\u2019est la m\xEAme chose que ce que nous avons obtenu avec le premier pipeline !"),Lo=c(),ps=l("h3"),qs=l("a"),Gl=l("span"),v(mt.$$.fragment),Mf=c(),Vl=l("span"),If=t("Regroupement des entit\xE9s"),Xo=c(),I=l("p"),Df=t("L\u2019utilisation des "),Ql=l("em"),Tf=t("offsets"),Rf=t(" pour d\xE9terminer les cl\xE9s de d\xE9but et de fin pour chaque entit\xE9 est pratique mais cette information n\u2019est pas strictement n\xE9cessaire. Cependant, lorsque nous voulons regrouper les entit\xE9s, les "),Jl=l("em"),Sf=t("offsets"),Bf=t(" nous \xE9pargnent un batch de code compliqu\xE9. Par exemple, si nous voulions regrouper les "),Ul=l("em"),Af=t("tokens"),Ff=c(),Yl=l("code"),Nf=t("Hu"),Lf=t(", "),Kl=l("code"),Xf=t("##gging"),Hf=t(", et "),Wl=l("code"),Gf=t("Face"),Vf=t(", nous pourrions \xE9tablir des r\xE8gles sp\xE9ciales disant que les deux premiers devraient \xEAtre attach\xE9s tout en enlevant le "),Zl=l("code"),Qf=t("##"),Jf=t(", et le "),er=l("code"),Uf=t("Face"),Yf=t(" devrait \xEAtre ajout\xE9 avec un espace puisqu\u2019il ne commence pas par "),sr=l("code"),Kf=t("##"),Wf=t(" mais cela ne fonctionnerait que pour ce type particulier de "),tr=l("em"),Zf=t("tokenizer"),eh=t(". Il faudrait \xE9crire un autre ensemble de r\xE8gles pour un "),nr=l("em"),sh=t("tokenizer"),th=t(" de type SentencePiece ou "),ar=l("em"),nh=t("Byte-Pair-Encoding"),ah=t(" (voir plus loin dans ce chapitre)."),Ho=c(),G=l("p"),lh=t("Avec les "),lr=l("em"),rh=t("offsets"),oh=t(", tout ce code personnalis\xE9 dispara\xEEt : il suffit de prendre l\u2019intervalle du texte original qui commence par le premier "),rr=l("em"),ih=t("token"),ph=t(" et se termine par le dernier "),or=l("em"),uh=t("token"),ch=t(". Ainsi, dans le cas des "),ir=l("em"),dh=t("tokens"),mh=c(),pr=l("code"),fh=t("Hu"),hh=t(", "),ur=l("code"),xh=t("##gging"),gh=t(", et "),cr=l("code"),vh=t("Face"),bh=t(", nous devrions commencer au caract\xE8re 33 (le d\xE9but de "),dr=l("code"),_h=t("Hu"),jh=t(") et finir avant le caract\xE8re 45 (la fin de "),mr=l("code"),Eh=t("Face"),kh=t(") :"),Go=c(),v(ft.$$.fragment),Vo=c(),v(ht.$$.fragment),Qo=c(),he=l("p"),$h=t("Pour \xE9crire le code qui post-traite les pr\xE9dictions tout en regroupant les entit\xE9s, nous regrouperons les entit\xE9s qui sont cons\xE9cutives et \xE9tiquet\xE9es avec "),fr=l("code"),qh=t("I-XXX"),yh=t(", \xE0 l\u2019exception de la premi\xE8re, qui peut \xEAtre \xE9tiquet\xE9e comme "),hr=l("code"),wh=t("B-XXX"),Ch=t(" ou "),xr=l("code"),Oh=t("I-XXX"),zh=t(" (ainsi, nous arr\xEAtons de regrouper une entit\xE9 lorsque nous obtenons un "),gr=l("code"),Ph=t("O"),Mh=t(", un nouveau type d\u2019entit\xE9, ou un "),vr=l("code"),Ih=t("B-XXX"),Dh=t(" qui nous indique qu\u2019une entit\xE9 du m\xEAme type commence) :"),Jo=c(),v(xt.$$.fragment),Uo=c(),Xt=l("p"),Th=t("Et nous obtenons les m\xEAmes r\xE9sultats qu\u2019avec notre deuxi\xE8me pipeline !"),Yo=c(),v(gt.$$.fragment),Ko=c(),$e=l("p"),Rh=t("Un autre exemple de t\xE2che o\xF9 ces "),br=l("em"),Sh=t("offsets"),Bh=t(" sont extr\xEAmement utiles est la r\xE9ponse aux questions. Plonger dans ce pipeline, ce que nous ferons dans la section suivante, nous permettra de jeter un coup d\u2019\u0153il \xE0 une derni\xE8re caract\xE9ristique des "),_r=l("em"),Ah=t("tokenizers"),Fh=t(" de la biblioth\xE8que \u{1F917} "),jr=l("em"),Nh=t("Transformers"),Lh=t(" : la gestion des "),Er=l("em"),Xh=t("tokens"),Hh=t(" qui d\xE9bordent lorsque nous tronquons une entr\xE9e \xE0 une longueur donn\xE9e."),this.h()},l(s){const i=Zg('[data-svelte="svelte-1phssyn"]',document.head);u=r(i,"META",{name:!0,content:!0}),i.forEach(a),g=d(s),b(m.$$.fragment,s),w=d(s),D=r(s,"H1",{class:!0});var Et=o(D);C=r(Et,"A",{id:!0,class:!0,href:!0});var Ht=o(C);T=r(Ht,"SPAN",{});var kr=o(T);b(F.$$.fragment,kr),kr.forEach(a),Ht.forEach(a),B=d(Et),R=r(Et,"SPAN",{});var kt=o(R);ae=n(kt,"Pouvoirs sp\xE9ciaux des "),X=r(kt,"I",{});var $r=o(X);Q=n($r,"tokenizers"),$r.forEach(a),ue=n(kt," rapides"),kt.forEach(a),Et.forEach(a),P=d(s),A.l(s),W=d(s),k=r(s,"P",{});var K=o(k);le=n(K,"Dans cette section, nous allons examiner de plus pr\xE8s les capacit\xE9s des "),ce=r(K,"EM",{});var Gt=o(ce);be=n(Gt,"tokenizers"),Gt.forEach(a),de=n(K," dans \u{1F917} "),H=r(K,"EM",{});var Vt=o(H);we=n(Vt,"Transformers"),Vt.forEach(a),te=n(K,`.
Jusqu\u2019\xE0 pr\xE9sent, nous ne les avons utilis\xE9s que pour tokeniser les entr\xE9es ou d\xE9coder les identifiants pour revenir \xE0 du texte. Mais les `),ve=r(K,"EM",{});var r2=o(ve);f=n(r2,"tokenizers"),r2.forEach(a),$=n(K,", et surtout ceux soutenus par la biblioth\xE8que \u{1F917} "),Ie=r(K,"EM",{});var o2=o(Ie);Ke=n(o2,"Tokenizers"),o2.forEach(a),We=n(K,", peuvent faire beaucoup plus. Pour illustrer ces fonctionnalit\xE9s suppl\xE9mentaires, nous allons explorer comment reproduire les r\xE9sultats des pipelines "),ne=r(K,"CODE",{});var i2=o(ne);Ze=n(i2,"token-classification"),i2.forEach(a),es=n(K," (que nous avons appel\xE9 "),tn=r(K,"CODE",{});var p2=o(tn);hi=n(p2,"ner"),p2.forEach(a),xi=n(K,") et "),nn=r(K,"CODE",{});var u2=o(nn);gi=n(u2,"question-answering"),u2.forEach(a),vi=n(K," que nous avons rencontr\xE9s pour la premi\xE8re fois dans le "),qt=r(K,"A",{href:!0});var c2=o(qt);bi=n(c2,"chapitre 1"),c2.forEach(a),_i=n(K,"."),K.forEach(a),wr=d(s),b(Cs.$$.fragment,s),Cr=d(s),U=r(s,"P",{});var ie=o(U);ji=n(ie,"Dans la discussion qui suit, nous ferons souvent la distinction entre les "),an=r(ie,"EM",{});var d2=o(an);Ei=n(d2,"tokenizers"),d2.forEach(a),ki=n(ie," \xAB lents \xBB et les \xAB rapides \xBB. Les "),ln=r(ie,"EM",{});var m2=o(ln);$i=n(m2,"tokenizers"),m2.forEach(a),qi=n(ie," lents sont ceux \xE9crits en Python \xE0 l\u2019int\xE9rieur de la biblioth\xE8que \u{1F917} "),rn=r(ie,"EM",{});var f2=o(rn);yi=n(f2,"Transformers"),f2.forEach(a),wi=n(ie,", tandis que les rapides sont ceux fournis par \u{1F917} "),on=r(ie,"EM",{});var h2=o(on);Ci=n(h2,"Tokenizers"),h2.forEach(a),Oi=n(ie," et sont cod\xE9s en Rust. Si vous vous souvenez du tableau du "),yt=r(ie,"A",{href:!0});var x2=o(yt);zi=n(x2,"chapitre 5"),x2.forEach(a),Pi=n(ie," qui indiquait combien de temps il fallait \xE0 un "),pn=r(ie,"EM",{});var g2=o(pn);Mi=n(g2,"tokenizer"),g2.forEach(a),Ii=n(ie," rapide et \xE0 un "),un=r(ie,"EM",{});var v2=o(un);Di=n(v2,"tokenizer"),v2.forEach(a),Ti=n(ie," lent pour tokeniser le jeu de donn\xE9es "),cn=r(ie,"EM",{});var b2=o(cn);Ri=n(b2,"Drug Review"),b2.forEach(a),Si=n(ie,", vous devriez avoir une id\xE9e de la raison pour laquelle nous les appelons rapides et lents :"),ie.forEach(a),Or=d(s),us=r(s,"TABLE",{});var Zo=o(us);dn=r(Zo,"THEAD",{});var _2=o(dn);ss=r(_2,"TR",{});var Qt=o(ss);Os=r(Qt,"TH",{align:!0});var Gh=o(Os);mn=r(Gh,"EM",{});var j2=o(mn);Bi=n(j2,"Tokenizer"),j2.forEach(a),Ai=n(Gh," rapide"),Gh.forEach(a),Fi=d(Qt),zs=r(Qt,"TH",{align:!0});var Vh=o(zs);fn=r(Vh,"EM",{});var E2=o(fn);Ni=n(E2,"Tokenizer"),E2.forEach(a),Li=n(Vh," lent"),Vh.forEach(a),Xi=d(Qt),hn=r(Qt,"TH",{align:!0}),o(hn).forEach(a),Qt.forEach(a),_2.forEach(a),Hi=d(Zo),Ps=r(Zo,"TBODY",{});var ei=o(Ps);ts=r(ei,"TR",{});var Jt=o(ts);wt=r(Jt,"TD",{align:!0});var k2=o(wt);xn=r(k2,"CODE",{});var $2=o(xn);Gi=n($2,"batched=True"),$2.forEach(a),k2.forEach(a),Vi=d(Jt),Ct=r(Jt,"TD",{align:!0});var q2=o(Ct);Qi=n(q2,"10.8s"),q2.forEach(a),Ji=d(Jt),Ot=r(Jt,"TD",{align:!0});var y2=o(Ot);Ui=n(y2,"4min41s"),y2.forEach(a),Jt.forEach(a),Yi=d(ei),ns=r(ei,"TR",{});var Ut=o(ns);zt=r(Ut,"TD",{align:!0});var w2=o(zt);gn=r(w2,"CODE",{});var C2=o(gn);Ki=n(C2,"batched=False"),C2.forEach(a),w2.forEach(a),Wi=d(Ut),Pt=r(Ut,"TD",{align:!0});var O2=o(Pt);Zi=n(O2,"59.2s"),O2.forEach(a),ep=d(Ut),Mt=r(Ut,"TD",{align:!0});var z2=o(Mt);sp=n(z2,"5min3s"),z2.forEach(a),Ut.forEach(a),ei.forEach(a),Zo.forEach(a),zr=d(s),b(cs.$$.fragment,s),Pr=d(s),as=r(s,"H2",{class:!0});var si=o(as);ds=r(si,"A",{id:!0,class:!0,href:!0});var P2=o(ds);vn=r(P2,"SPAN",{});var M2=o(vn);b(Ms.$$.fragment,M2),M2.forEach(a),P2.forEach(a),tp=d(si),It=r(si,"SPAN",{});var Qh=o(It);np=n(Qh,"L'objet "),bn=r(Qh,"I",{});var I2=o(bn);ap=n(I2,"BatchEncoding"),I2.forEach(a),Qh.forEach(a),si.forEach(a),Mr=d(s),b(Is.$$.fragment,s),Ir=d(s),Ce=r(s,"P",{});var ys=o(Ce);lp=n(ys,"La sortie d\u2019un "),_n=r(ys,"EM",{});var D2=o(_n);rp=n(D2,"tokenizer"),D2.forEach(a),op=n(ys," n\u2019est pas un simple dictionnaire Python. Ce que nous obtenons est en fait un objet sp\xE9cial "),jn=r(ys,"CODE",{});var T2=o(jn);ip=n(T2,"BatchEncoding"),T2.forEach(a),pp=n(ys,". C\u2019est une sous-classe d\u2019un dictionnaire (c\u2019est pourquoi nous avons pu indexer ce r\xE9sultat sans probl\xE8me auparavant), mais avec des m\xE9thodes suppl\xE9mentaires qui sont principalement utilis\xE9es par les "),En=r(ys,"EM",{});var R2=o(En);up=n(R2,"tokenizers"),R2.forEach(a),cp=n(ys," rapides."),ys.forEach(a),Dr=d(s),me=r(s,"P",{});var ze=o(me);dp=n(ze,"En plus de leurs capacit\xE9s de parall\xE9lisation, la fonctionnalit\xE9 cl\xE9 des "),kn=r(ze,"EM",{});var S2=o(kn);mp=n(S2,"tokenizers"),S2.forEach(a),fp=n(ze," rapides est qu\u2019ils gardent toujours la trace de l\u2019\xE9tendue originale des textes d\u2019o\xF9 proviennent les "),$n=r(ze,"EM",{});var B2=o($n);hp=n(B2,"tokens"),B2.forEach(a),xp=n(ze," finaux, une fonctionnalit\xE9 que nous appelons "),qn=r(ze,"EM",{});var A2=o(qn);gp=n(A2,"mapping offset"),A2.forEach(a),vp=n(ze,". Cela permet de d\xE9bloquer des fonctionnalit\xE9s telles que le mappage de chaque mot aux "),yn=r(ze,"EM",{});var F2=o(yn);bp=n(F2,"tokens"),F2.forEach(a),_p=n(ze," qu\u2019il a g\xE9n\xE9r\xE9s ou le mappage de chaque caract\xE8re du texte original au "),wn=r(ze,"EM",{});var N2=o(wn);jp=n(N2,"token"),N2.forEach(a),Ep=n(ze," qu\u2019il contient, et vice versa."),ze.forEach(a),Tr=d(s),Dt=r(s,"P",{});var L2=o(Dt);kp=n(L2,"Prenons un exemple :"),L2.forEach(a),Rr=d(s),b(Ds.$$.fragment,s),Sr=d(s),Fe=r(s,"P",{});var Yt=o(Fe);$p=n(Yt,"Comme mentionn\xE9 pr\xE9c\xE9demment, nous obtenons un objet "),Cn=r(Yt,"CODE",{});var X2=o(Cn);qp=n(X2,"BatchEncoding"),X2.forEach(a),yp=n(Yt," dans la sortie du "),On=r(Yt,"EM",{});var H2=o(On);wp=n(H2,"tokenizer"),H2.forEach(a),Cp=n(Yt," :"),Yt.forEach(a),Br=d(s),b(Ts.$$.fragment,s),Ar=d(s),re=r(s,"P",{});var qe=o(re);Op=n(qe,"Puisque la classe "),zn=r(qe,"CODE",{});var G2=o(zn);zp=n(G2,"AutoTokenizer"),G2.forEach(a),Pp=n(qe," choisit un "),Pn=r(qe,"EM",{});var V2=o(Pn);Mp=n(V2,"tokenizer"),V2.forEach(a),Ip=n(qe," rapide par d\xE9faut, nous pouvons utiliser les m\xE9thodes suppl\xE9mentaires que cet objet "),Mn=r(qe,"CODE",{});var Q2=o(Mn);Dp=n(Q2,"BatchEncoding"),Q2.forEach(a),Tp=n(qe," fournit. Nous avons deux fa\xE7ons de v\xE9rifier si notre "),In=r(qe,"EM",{});var J2=o(In);Rp=n(J2,"tokenizer"),J2.forEach(a),Sp=n(qe," est rapide ou lent. Nous pouvons soit v\xE9rifier l\u2019attribut "),Dn=r(qe,"CODE",{});var U2=o(Dn);Bp=n(U2,"is_fast"),U2.forEach(a),Ap=n(qe," du "),Tn=r(qe,"EM",{});var Y2=o(Tn);Fp=n(Y2,"tokenizer"),Y2.forEach(a),Np=n(qe," comme suit :"),qe.forEach(a),Fr=d(s),b(Rs.$$.fragment,s),Nr=d(s),b(Ss.$$.fragment,s),Lr=d(s),ms=r(s,"P",{});var ti=o(ms);Lp=n(ti,"soit v\xE9rifier le m\xEAme attribut mais avec notre "),Rn=r(ti,"CODE",{});var K2=o(Rn);Xp=n(K2,"encoding"),K2.forEach(a),Hp=n(ti," :"),ti.forEach(a),Xr=d(s),b(Bs.$$.fragment,s),Hr=d(s),b(As.$$.fragment,s),Gr=d(s),Oe=r(s,"P",{});var ws=o(Oe);Gp=n(ws,"Voyons ce qu\u2019un "),Sn=r(ws,"EM",{});var W2=o(Sn);Vp=n(W2,"tokenizer"),W2.forEach(a),Qp=n(ws," rapide nous permet de faire. Tout d\u2019abord, nous pouvons acc\xE9der aux "),Bn=r(ws,"EM",{});var Z2=o(Bn);Jp=n(Z2,"tokens"),Z2.forEach(a),Up=n(ws," sans avoir \xE0 reconvertir les identifiants en "),An=r(ws,"EM",{});var ex=o(An);Yp=n(ex,"tokens"),ex.forEach(a),Kp=n(ws," :"),ws.forEach(a),Vr=d(s),b(Fs.$$.fragment,s),Qr=d(s),b(Ns.$$.fragment,s),Jr=d(s),_e=r(s,"P",{});var Ve=o(_e);Wp=n(Ve,"Dans ce cas, le "),Fn=r(Ve,"EM",{});var sx=o(Fn);Zp=n(sx,"token"),sx.forEach(a),eu=n(Ve," \xE0 l\u2019index 5 est "),Nn=r(Ve,"CODE",{});var tx=o(Nn);su=n(tx,"##yl"),tx.forEach(a),tu=n(Ve," et fait partie du mot \xAB Sylvain \xBB dans la phrase originale. Nous pouvons \xE9galement utiliser la m\xE9thode "),Ln=r(Ve,"CODE",{});var nx=o(Ln);nu=n(nx,"word_ids()"),nx.forEach(a),au=n(Ve," pour obtenir l\u2019index du mot dont provient chaque "),Xn=r(Ve,"EM",{});var ax=o(Xn);lu=n(ax,"token"),ax.forEach(a),ru=n(Ve," :"),Ve.forEach(a),Ur=d(s),b(Ls.$$.fragment,s),Yr=d(s),b(Xs.$$.fragment,s),Kr=d(s),O=r(s,"P",{});var M=o(O);ou=n(M,"On peut voir que les "),Hn=r(M,"EM",{});var lx=o(Hn);iu=n(lx,"tokens"),lx.forEach(a),pu=n(M," sp\xE9ciaux du "),Gn=r(M,"EM",{});var rx=o(Gn);uu=n(rx,"tokenizer"),rx.forEach(a),cu=n(M,", "),Vn=r(M,"CODE",{});var ox=o(Vn);du=n(ox,"[CLS]"),ox.forEach(a),mu=n(M," et "),Qn=r(M,"CODE",{});var ix=o(Qn);fu=n(ix,"[SEP]"),ix.forEach(a),hu=n(M,", sont mis en correspondance avec "),Jn=r(M,"CODE",{});var px=o(Jn);xu=n(px,"None"),px.forEach(a),gu=n(M," et que chaque "),Un=r(M,"EM",{});var ux=o(Un);vu=n(ux,"token"),ux.forEach(a),bu=n(M," est mis en correspondance avec le mot dont il provient. Ceci est particuli\xE8rement utile pour d\xE9terminer si un "),Yn=r(M,"EM",{});var cx=o(Yn);_u=n(cx,"token"),cx.forEach(a),ju=n(M," est au d\xE9but d\u2019un mot ou si deux "),Kn=r(M,"EM",{});var dx=o(Kn);Eu=n(dx,"tokens"),dx.forEach(a),ku=n(M," sont dans le m\xEAme mot. Nous pourrions nous appuyer sur le pr\xE9fixe "),Wn=r(M,"CODE",{});var mx=o(Wn);$u=n(mx,"##"),mx.forEach(a),qu=n(M," pour cela, mais il ne fonctionne que pour les "),Zn=r(M,"EM",{});var fx=o(Zn);yu=n(fx,"tokenizers"),fx.forEach(a),wu=n(M," de type BERT. Cette m\xE9thode fonctionne pour n\u2019importe quel type de "),ea=r(M,"EM",{});var hx=o(ea);Cu=n(hx,"tokenizer"),hx.forEach(a),Ou=n(M,", du moment qu\u2019il est rapide. Dans le chapitre suivant, nous verrons comment utiliser cette capacit\xE9 pour appliquer correctement les \xE9tiquettes que nous avons pour chaque mot aux "),sa=r(M,"EM",{});var xx=o(sa);zu=n(xx,"tokens"),xx.forEach(a),Pu=n(M," dans des t\xE2ches comme la reconnaissance d\u2019entit\xE9s nomm\xE9es et le POS ("),ta=r(M,"EM",{});var gx=o(ta);Mu=n(gx,"Part-of-speech"),gx.forEach(a),Iu=n(M,"). Nous pouvons \xE9galement l\u2019utiliser pour masquer tous les "),na=r(M,"EM",{});var vx=o(na);Du=n(vx,"tokens"),vx.forEach(a),Tu=n(M," provenant du m\xEAme mot dans la mod\xE9lisation du langage masqu\xE9 (une technique appel\xE9e "),aa=r(M,"EM",{});var bx=o(aa);Ru=n(bx,"whole word masking"),bx.forEach(a),Su=n(M,")."),M.forEach(a),Wr=d(s),Ne=r(s,"P",{});var Kt=o(Ne);Bu=n(Kt,"La notion de ce qu\u2019est un mot est compliqu\xE9e. Par exemple, est-ce que \xAB I\u2019ll \xBB (contraction de \xAB I will \xBB) compte pour un ou deux mots ? Cela d\xE9pend en fait du "),la=r(Kt,"EM",{});var _x=o(la);Au=n(_x,"tokenizer"),_x.forEach(a),Fu=n(Kt," et de l\u2019op\xE9ration de pr\xE9tok\xE9nisation qu\u2019il applique. Certains "),ra=r(Kt,"EM",{});var jx=o(ra);Nu=n(jx,"tokenizer"),jx.forEach(a),Lu=n(Kt," se contentent de s\xE9parer les espaces et consid\xE8rent donc qu\u2019il s\u2019agit d\u2019un seul mot. D\u2019autres utilisent la ponctuation en plus des espaces et consid\xE8rent donc qu\u2019il s\u2019agit de deux mots."),Kt.forEach(a),Zr=d(s),b(fs.$$.fragment,s),eo=d(s),je=r(s,"P",{});var Qe=o(je);Xu=n(Qe,"De m\xEAme, il existe une m\xE9thode "),oa=r(Qe,"CODE",{});var Ex=o(oa);Hu=n(Ex,"sentence_ids()"),Ex.forEach(a),Gu=n(Qe," que nous pouvons utiliser pour associer un "),ia=r(Qe,"EM",{});var kx=o(ia);Vu=n(kx,"token"),kx.forEach(a),Qu=n(Qe," \xE0 la phrase dont il provient (bien que dans ce cas, le "),pa=r(Qe,"CODE",{});var $x=o(pa);Ju=n($x,"token_type_ids"),$x.forEach(a),Uu=n(Qe," retourn\xE9 par le "),ua=r(Qe,"EM",{});var qx=o(ua);Yu=n(qx,"tokenizer"),qx.forEach(a),Ku=n(Qe," peut nous donner la m\xEAme information)."),Qe.forEach(a),so=d(s),Z=r(s,"P",{});var xe=o(Z);Wu=n(xe,"Enfin, nous pouvons faire correspondre n\u2019importe quel mot ou "),ca=r(xe,"EM",{});var yx=o(ca);Zu=n(yx,"token"),yx.forEach(a),ec=n(xe," aux caract\xE8res du texte d\u2019origine (et vice versa) gr\xE2ce aux m\xE9thodes "),da=r(xe,"CODE",{});var wx=o(da);sc=n(wx,"word_to_chars()"),wx.forEach(a),tc=n(xe," ou "),ma=r(xe,"CODE",{});var Cx=o(ma);nc=n(Cx,"token_to_chars()"),Cx.forEach(a),ac=n(xe," et "),fa=r(xe,"CODE",{});var Ox=o(fa);lc=n(Ox,"char_to_word()"),Ox.forEach(a),rc=n(xe," ou "),ha=r(xe,"CODE",{});var zx=o(ha);oc=n(zx,"char_to_token()"),zx.forEach(a),ic=n(xe,". Par exemple, la m\xE9thode "),xa=r(xe,"CODE",{});var Px=o(xa);pc=n(Px,"word_ids()"),Px.forEach(a),uc=n(xe," nous a dit que "),ga=r(xe,"CODE",{});var Mx=o(ga);cc=n(Mx,"##yl"),Mx.forEach(a),dc=n(xe," fait partie du mot \xE0 l\u2019indice 3, mais de quel mot s\u2019agit-il dans la phrase ? Nous pouvons le d\xE9couvrir comme ceci :"),xe.forEach(a),to=d(s),b(Hs.$$.fragment,s),no=d(s),b(Gs.$$.fragment,s),ao=d(s),Ee=r(s,"P",{});var Je=o(Ee);mc=n(Je,"Comme nous l\u2019avons mentionn\xE9 pr\xE9c\xE9demment, tout ceci est rendu possible par le fait que le "),va=r(Je,"EM",{});var Ix=o(va);fc=n(Ix,"tokenizer"),Ix.forEach(a),hc=n(Je," rapide garde la trace de la partie du texte d\u2019o\xF9 provient chaque "),ba=r(Je,"EM",{});var Dx=o(ba);xc=n(Dx,"token"),Dx.forEach(a),gc=n(Je," dans une liste d\u2019"),_a=r(Je,"EM",{});var Tx=o(_a);vc=n(Tx,"offsets"),Tx.forEach(a),bc=n(Je,". Pour illustrer leur utilisation, nous allons maintenant vous montrer comment reproduire manuellement les r\xE9sultats du pipeline "),ja=r(Je,"CODE",{});var Rx=o(ja);_c=n(Rx,"token-classification"),Rx.forEach(a),jc=n(Je,"."),Je.forEach(a),lo=d(s),b(hs.$$.fragment,s),ro=d(s),ls=r(s,"H2",{class:!0});var ni=o(ls);xs=r(ni,"A",{id:!0,class:!0,href:!0});var Sx=o(xs);Ea=r(Sx,"SPAN",{});var Bx=o(Ea);b(Vs.$$.fragment,Bx),Bx.forEach(a),Sx.forEach(a),Ec=d(ni),Tt=r(ni,"SPAN",{});var Jh=o(Tt);kc=n(Jh,"A l'int\xE9rieur du pipeline "),ka=r(Jh,"CODE",{});var Ax=o(ka);$c=n(Ax,"token-classification"),Ax.forEach(a),Jh.forEach(a),ni.forEach(a),oo=d(s),fe=r(s,"P",{});var Pe=o(fe);qc=n(Pe,"Dans le "),Rt=r(Pe,"A",{href:!0});var Fx=o(Rt);yc=n(Fx,"chapitre 1"),Fx.forEach(a),wc=n(Pe,", nous avons eu un premier aper\xE7u de la NER (o\xF9 la t\xE2che est d\u2019identifier les parties du texte qui correspondent \xE0 des entit\xE9s telles que des personnes, des lieux ou des organisations) avec la fonction "),$a=r(Pe,"CODE",{});var Nx=o($a);Cc=n(Nx,"pipeline()"),Nx.forEach(a),Oc=n(Pe," de \u{1F917} "),qa=r(Pe,"EM",{});var Lx=o(qa);zc=n(Lx,"Transformers"),Lx.forEach(a),Pc=n(Pe,". Puis, dans le "),St=r(Pe,"A",{href:!0});var Xx=o(St);Mc=n(Xx,"chapitre 2"),Xx.forEach(a),Ic=n(Pe,", nous avons vu comment un pipeline regroupe les trois \xE9tapes n\xE9cessaires pour obtenir les pr\xE9dictions \xE0 partir d\u2019un texte brut : la tokenisation, le passage des entr\xE9es dans le mod\xE8le et le post-traitement. Les deux premi\xE8res \xE9tapes du pipeline de "),ya=r(Pe,"CODE",{});var Hx=o(ya);Dc=n(Hx,"token-classification"),Hx.forEach(a),Tc=n(Pe," sont les m\xEAmes que dans tout autre pipeline mais le post-traitement est un peu plus complexe. Voyons comment !"),Pe.forEach(a),io=d(s),Te.l(s),Bt=d(s),rs=r(s,"H3",{class:!0});var ai=o(rs);gs=r(ai,"A",{id:!0,class:!0,href:!0});var Gx=o(gs);wa=r(Gx,"SPAN",{});var Vx=o(wa);b(Qs.$$.fragment,Vx),Vx.forEach(a),Gx.forEach(a),Rc=d(ai),Ca=r(ai,"SPAN",{});var Qx=o(Ca);Sc=n(Qx,"Obtenir les r\xE9sultats de base avec le pipeline"),Qx.forEach(a),ai.forEach(a),po=d(s),Le=r(s,"P",{});var Wt=o(Le);Bc=n(Wt,"Tout d\u2019abord, prenons un pipeline de classification de "),Oa=r(Wt,"EM",{});var Jx=o(Oa);Ac=n(Jx,"tokens"),Jx.forEach(a),Fc=n(Wt," afin d\u2019obtenir des r\xE9sultats \xE0 comparer manuellement. Le mod\xE8le utilis\xE9 par d\xE9faut est "),Js=r(Wt,"A",{href:!0,rel:!0});var Ux=o(Js);za=r(Ux,"CODE",{});var Yx=o(za);Nc=n(Yx,"dbmdz/bert-large-cased-finetuned-conll03-english"),Yx.forEach(a),Ux.forEach(a),Lc=n(Wt,". Il effectue une NER sur les phrases :"),Wt.forEach(a),uo=d(s),b(Us.$$.fragment,s),co=d(s),b(Ys.$$.fragment,s),mo=d(s),ke=r(s,"P",{});var Ue=o(ke);Xc=n(Ue,"Le mod\xE8le a correctement identifi\xE9 chaque "),Pa=r(Ue,"EM",{});var Kx=o(Pa);Hc=n(Kx,"token"),Kx.forEach(a),Gc=n(Ue," g\xE9n\xE9r\xE9 par \xAB Sylvain \xBB comme une personne, chaque "),Ma=r(Ue,"EM",{});var Wx=o(Ma);Vc=n(Wx,"token"),Wx.forEach(a),Qc=n(Ue," g\xE9n\xE9r\xE9 par \xAB Hugging Face \xBB comme une organisation, et le "),Ia=r(Ue,"EM",{});var Zx=o(Ia);Jc=n(Zx,"token"),Zx.forEach(a),Uc=n(Ue," \xAB Brooklyn \xBB comme un lieu. Nous pouvons \xE9galement demander au pipeline de regrouper les "),Da=r(Ue,"EM",{});var e7=o(Da);Yc=n(e7,"tokens"),e7.forEach(a),Kc=n(Ue," qui correspondent \xE0 la m\xEAme entit\xE9 :"),Ue.forEach(a),fo=d(s),b(Ks.$$.fragment,s),ho=d(s),b(Ws.$$.fragment,s),xo=d(s),ee=r(s,"P",{});var ge=o(ee);Wc=n(ge,"La propri\xE9t\xE9 "),Ta=r(ge,"CODE",{});var s7=o(Ta);Zc=n(s7,"aggregation_strategy"),s7.forEach(a),ed=n(ge," choisie va changer les scores calcul\xE9s pour chaque entit\xE9 group\xE9e. Avec "),Ra=r(ge,"CODE",{});var t7=o(Ra);sd=n(t7,'"simple"'),t7.forEach(a),td=n(ge," le score est juste la moyenne des scores de chaque "),Sa=r(ge,"EM",{});var n7=o(Sa);nd=n(n7,"token"),n7.forEach(a),ad=n(ge," dans l\u2019entit\xE9 donn\xE9e. Par exemple, le score de \xAB Sylvain \xBB est la moyenne des scores que nous avons vu dans l\u2019exemple pr\xE9c\xE9dent pour les tokens "),Ba=r(ge,"CODE",{});var a7=o(Ba);ld=n(a7,"S"),a7.forEach(a),rd=n(ge,", "),Aa=r(ge,"CODE",{});var l7=o(Aa);od=n(l7,"##yl"),l7.forEach(a),id=n(ge,", "),Fa=r(ge,"CODE",{});var r7=o(Fa);pd=n(r7,"##va"),r7.forEach(a),ud=n(ge,", et "),Na=r(ge,"CODE",{});var o7=o(Na);cd=n(o7,"##in"),o7.forEach(a),dd=n(ge,". D\u2019autres strat\xE9gies sont disponibles :"),ge.forEach(a),go=d(s),Xe=r(s,"UL",{});var Zt=o(Xe);He=r(Zt,"LI",{});var $t=o(He);La=r($t,"CODE",{});var i7=o(La);md=n(i7,'"first"'),i7.forEach(a),fd=n($t,", o\xF9 le score de chaque entit\xE9 est le score du premier "),Xa=r($t,"EM",{});var p7=o(Xa);hd=n(p7,"token"),p7.forEach(a),xd=n($t," de cette entit\xE9 (donc pour \xAB Sylvain \xBB ce serait 0.993828, le score du token "),Ha=r($t,"CODE",{});var u7=o(Ha);gd=n(u7,"S"),u7.forEach(a),vd=n($t,")"),$t.forEach(a),bd=d(Zt),vs=r(Zt,"LI",{});var qr=o(vs);Ga=r(qr,"CODE",{});var c7=o(Ga);_d=n(c7,'"max"'),c7.forEach(a),jd=n(qr,", o\xF9 le score de chaque entit\xE9 est le score maximal des "),Va=r(qr,"EM",{});var d7=o(Va);Ed=n(d7,"tokens"),d7.forEach(a),kd=n(qr," de cette entit\xE9 (ainsi, pour \xAB Hugging Face \xBB, le score de \xAB Face \xBB serait de 0,98879766)."),qr.forEach(a),$d=d(Zt),bs=r(Zt,"LI",{});var yr=o(bs);Qa=r(yr,"CODE",{});var m7=o(Qa);qd=n(m7,'"average"'),m7.forEach(a),yd=n(yr,", o\xF9 le score de chaque entit\xE9 est la moyenne des scores des mots qui composent cette entit\xE9 (ainsi, pour \xAB Sylvain \xBB, il n\u2019y aurait pas de diff\xE9rence avec la strat\xE9gie "),Ja=r(yr,"CODE",{});var f7=o(Ja);wd=n(f7,'"simple"'),f7.forEach(a),Cd=n(yr,", mais \u201CHugging Face\u201D aurait un score de 0,9819, la moyenne des scores de \xAB Hugging \xBB, 0,975, et \xAB Face \xBB, 0,98879)."),yr.forEach(a),Zt.forEach(a),vo=d(s),_s=r(s,"P",{});var li=o(_s);Od=n(li,"Voyons maintenant comment obtenir ces r\xE9sultats sans utiliser la fonction "),Ua=r(li,"CODE",{});var h7=o(Ua);zd=n(h7,"pipeline()"),h7.forEach(a),Pd=n(li," !"),li.forEach(a),bo=d(s),os=r(s,"H3",{class:!0});var ri=o(os);js=r(ri,"A",{id:!0,class:!0,href:!0});var x7=o(js);Ya=r(x7,"SPAN",{});var g7=o(Ya);b(Zs.$$.fragment,g7),g7.forEach(a),x7.forEach(a),Md=d(ri),Ka=r(ri,"SPAN",{});var v7=o(Ka);Id=n(v7,"Des entr\xE9es aux pr\xE9dictions"),v7.forEach(a),ri.forEach(a),_o=d(s),Se.l(s),At=d(s),Es=r(s,"P",{});var oi=o(Es);Dd=n(oi,"Nous avons un batch avec 1 s\xE9quence de 19 "),Wa=r(oi,"EM",{});var b7=o(Wa);Td=n(b7,"tokens"),b7.forEach(a),Rd=n(oi," et le mod\xE8le a 9 \xE9tiquettes diff\xE9rentes. Ainsi, la sortie du mod\xE8le a une forme de 1 x 19 x 9. Comme pour le pipeline de classification de texte, nous utilisons une fonction softmax pour convertir ces logits en probabilit\xE9s et nous prenons l\u2019argmax pour obtenir des pr\xE9dictions (notez que nous pouvons prendre l\u2019argmax sur les logits car la fonction softmax ne change pas l\u2019ordre) :"),oi.forEach(a),jo=d(s),Ae.l(s),Ft=d(s),b(et.$$.fragment,s),Eo=d(s),ks=r(s,"P",{});var ii=o(ks);Sd=n(ii,"L\u2019attribut "),Za=r(ii,"CODE",{});var _7=o(Za);Bd=n(_7,"model.config.id2label"),_7.forEach(a),Ad=n(ii," contient la correspondance entre les index et les \xE9tiquettes que nous pouvons utiliser pour donner un sens aux pr\xE9dictions :"),ii.forEach(a),ko=d(s),b(st.$$.fragment,s),$o=d(s),b(tt.$$.fragment,s),qo=d(s),q=r(s,"P",{});var z=o(q);Fd=n(z,"Comme nous l\u2019avons vu pr\xE9c\xE9demment, il y a 9 \xE9tiquettes : "),el=r(z,"CODE",{});var j7=o(el);Nd=n(j7,"O"),j7.forEach(a),Ld=n(z," est le label pour les "),sl=r(z,"EM",{});var E7=o(sl);Xd=n(E7,"tokens"),E7.forEach(a),Hd=n(z," qui ne sont dans aucune entit\xE9 nomm\xE9e (il signifie "),tl=r(z,"EM",{});var k7=o(tl);Gd=n(k7,"outside"),k7.forEach(a),Vd=n(z," (en dehors)) et nous avons ensuite deux labels pour chaque type d\u2019entit\xE9 (divers, personne, organisation et lieu). L\u2019\xE9tiquette "),nl=r(z,"CODE",{});var $7=o(nl);Qd=n($7,"B-XXX"),$7.forEach(a),Jd=n(z," indique que le "),al=r(z,"EM",{});var q7=o(al);Ud=n(q7,"token"),q7.forEach(a),Yd=n(z," est au d\xE9but d\u2019une entit\xE9 "),ll=r(z,"CODE",{});var y7=o(ll);Kd=n(y7,"XXX"),y7.forEach(a),Wd=n(z," et l\u2019\xE9tiquette "),rl=r(z,"CODE",{});var w7=o(rl);Zd=n(w7,"I-XXX"),w7.forEach(a),em=n(z," indique que le "),ol=r(z,"EM",{});var C7=o(ol);sm=n(C7,"token"),C7.forEach(a),tm=n(z," est \xE0 l\u2019int\xE9rieur de l\u2019entit\xE9 "),il=r(z,"CODE",{});var O7=o(il);nm=n(O7,"XXX"),O7.forEach(a),am=n(z,". Par exemple, dans l\u2019exemple actuel, nous nous attendons \xE0 ce que notre mod\xE8le classe le "),pl=r(z,"EM",{});var z7=o(pl);lm=n(z7,"token"),z7.forEach(a),rm=d(z),ul=r(z,"CODE",{});var P7=o(ul);om=n(P7,"S"),P7.forEach(a),im=n(z," comme "),cl=r(z,"CODE",{});var M7=o(cl);pm=n(M7,"B-PER"),M7.forEach(a),um=n(z," (d\xE9but d\u2019une entit\xE9 personne) et les "),dl=r(z,"EM",{});var I7=o(dl);cm=n(I7,"tokens"),I7.forEach(a),dm=d(z),ml=r(z,"CODE",{});var D7=o(ml);mm=n(D7,"##yl"),D7.forEach(a),fm=n(z,", "),fl=r(z,"CODE",{});var T7=o(fl);hm=n(T7,"##va"),T7.forEach(a),xm=n(z," et "),hl=r(z,"CODE",{});var R7=o(hl);gm=n(R7,"##in"),R7.forEach(a),vm=n(z," comme "),xl=r(z,"CODE",{});var S7=o(xl);bm=n(S7,"I-PER"),S7.forEach(a),_m=n(z," (\xE0 l\u2019int\xE9rieur d\u2019une entit\xE9 personne)."),z.forEach(a),yo=d(s),S=r(s,"P",{});var V=o(S);jm=n(V,"Vous pourriez penser que le mod\xE8le s\u2019est tromp\xE9 ici car il a attribu\xE9 l\u2019\xE9tiquette "),gl=r(V,"CODE",{});var B7=o(gl);Em=n(B7,"I-PER"),B7.forEach(a),km=n(V," \xE0 ces quatre "),vl=r(V,"EM",{});var A7=o(vl);$m=n(A7,"tokens"),A7.forEach(a),qm=n(V," mais ce n\u2019est pas tout \xE0 fait vrai. Il existe en fait deux formats pour ces \xE9tiquettes "),bl=r(V,"CODE",{});var F7=o(bl);ym=n(F7,"B-"),F7.forEach(a),wm=n(V," et "),_l=r(V,"CODE",{});var N7=o(_l);Cm=n(N7,"I-"),N7.forEach(a),Om=n(V," : "),jl=r(V,"EM",{});var L7=o(jl);zm=n(L7,"IOB1"),L7.forEach(a),Pm=n(V," et "),El=r(V,"EM",{});var X7=o(El);Mm=n(X7,"IOB2"),X7.forEach(a),Im=n(V,". Le format IOB2 (en rose ci-dessous) est celui que nous avons introduit alors que dans le format IOB1 (en bleu), les \xE9tiquettes commen\xE7ant par "),kl=r(V,"CODE",{});var H7=o(kl);Dm=n(H7,"B-"),H7.forEach(a),Tm=n(V," ne sont jamais utilis\xE9es que pour s\xE9parer deux entit\xE9s adjacentes du m\xEAme type. Le mod\xE8le que nous utilisons a \xE9t\xE9 "),$l=r(V,"EM",{});var G7=o($l);Rm=n(G7,"finetun\xE9"),G7.forEach(a),Sm=n(V," sur un jeu de donn\xE9es utilisant ce format, c\u2019est pourquoi il attribue le label "),ql=r(V,"CODE",{});var V7=o(ql);Bm=n(V7,"I-PER"),V7.forEach(a),Am=n(V," au "),yl=r(V,"EM",{});var Q7=o(yl);Fm=n(Q7,"token"),Q7.forEach(a),Nm=d(V),wl=r(V,"CODE",{});var J7=o(wl);Lm=n(J7,"S"),J7.forEach(a),Xm=n(V,"."),V.forEach(a),wo=d(s),is=r(s,"DIV",{class:!0});var pi=o(is);nt=r(pi,"IMG",{class:!0,src:!0,alt:!0}),Hm=d(pi),at=r(pi,"IMG",{class:!0,src:!0,alt:!0}),pi.forEach(a),Co=d(s),Ge=r(s,"P",{});var en=o(Ge);Gm=n(en,"Nous sommes \xE0 pr\xE9sent pr\xEAts \xE0 reproduire (presque enti\xE8rement) les r\xE9sultats du premier pipeline. Nous pouvons simplement r\xE9cup\xE9rer le score et le label de chaque "),Cl=r(en,"EM",{});var U7=o(Cl);Vm=n(U7,"token"),U7.forEach(a),Qm=n(en," qui n\u2019a pas \xE9t\xE9 class\xE9 comme "),Ol=r(en,"CODE",{});var Y7=o(Ol);Jm=n(Y7,"O"),Y7.forEach(a),Um=n(en," :"),en.forEach(a),Oo=d(s),b(lt.$$.fragment,s),zo=d(s),b(rt.$$.fragment,s),Po=d(s),oe=r(s,"P",{});var ye=o(oe);Ym=n(ye,"C\u2019est tr\xE8s similaire \xE0 ce que nous avions avant, \xE0 une exception pr\xE8s : le pipeline nous a aussi donn\xE9 des informations sur le "),zl=r(ye,"CODE",{});var K7=o(zl);Km=n(K7,"d\xE9but"),K7.forEach(a),Wm=n(ye," et la "),Pl=r(ye,"CODE",{});var W7=o(Pl);Zm=n(W7,"fin"),W7.forEach(a),ef=n(ye," de chaque entit\xE9 dans la phrase originale. C\u2019est l\xE0 que notre "),Ml=r(ye,"EM",{});var Z7=o(Ml);sf=n(Z7,"offset mapping"),Z7.forEach(a),tf=n(ye," va entrer en jeu. Pour obtenir les "),Il=r(ye,"EM",{});var eg=o(Il);nf=n(eg,"offsets"),eg.forEach(a),af=n(ye,", il suffit de d\xE9finir "),Dl=r(ye,"CODE",{});var sg=o(Dl);lf=n(sg,"return_offsets_mapping=True"),sg.forEach(a),rf=n(ye," lorsque nous appliquons le "),Tl=r(ye,"EM",{});var tg=o(Tl);of=n(tg,"tokenizer"),tg.forEach(a),pf=n(ye," \xE0 nos entr\xE9es :"),ye.forEach(a),Mo=d(s),b(ot.$$.fragment,s),Io=d(s),b(it.$$.fragment,s),Do=d(s),Y=r(s,"P",{});var pe=o(Y);uf=n(pe,"Chaque "),Rl=r(pe,"EM",{});var ng=o(Rl);cf=n(ng,"tuple"),ng.forEach(a),df=n(pe," est l\u2019\xE9tendue de texte correspondant \xE0 chaque "),Sl=r(pe,"EM",{});var ag=o(Sl);mf=n(ag,"token"),ag.forEach(a),ff=n(pe," o\xF9 "),Bl=r(pe,"CODE",{});var lg=o(Bl);hf=n(lg,"(0, 0)"),lg.forEach(a),xf=n(pe," est r\xE9serv\xE9 aux "),Al=r(pe,"EM",{});var rg=o(Al);gf=n(rg,"tokens"),rg.forEach(a),vf=n(pe," sp\xE9ciaux. Nous avons vu pr\xE9c\xE9demment que le "),Fl=r(pe,"EM",{});var og=o(Fl);bf=n(og,"token"),og.forEach(a),_f=n(pe," \xE0 l\u2019index 5 est "),Nl=r(pe,"CODE",{});var ig=o(Nl);jf=n(ig,"##yl"),ig.forEach(a),Ef=n(pe,", qui a "),Ll=r(pe,"CODE",{});var pg=o(Ll);kf=n(pg,"(12, 14)"),pg.forEach(a),$f=n(pe," comme "),Xl=r(pe,"EM",{});var ug=o(Xl);qf=n(ug,"offsets"),ug.forEach(a),yf=n(pe," ici. Si on prend la tranche correspondante dans notre exemple :"),pe.forEach(a),To=d(s),b(pt.$$.fragment,s),Ro=d(s),$s=r(s,"P",{});var ui=o($s);wf=n(ui,"nous obtenons le bon espace de texte sans le "),Hl=r(ui,"CODE",{});var cg=o(Hl);Cf=n(cg,"##"),cg.forEach(a),Of=n(ui," :"),ui.forEach(a),So=d(s),b(ut.$$.fragment,s),Bo=d(s),Nt=r(s,"P",{});var dg=o(Nt);zf=n(dg,"En utilisant cela, nous pouvons maintenant compl\xE9ter les r\xE9sultats pr\xE9c\xE9dents :"),dg.forEach(a),Ao=d(s),b(ct.$$.fragment,s),Fo=d(s),b(dt.$$.fragment,s),No=d(s),Lt=r(s,"P",{});var mg=o(Lt);Pf=n(mg,"C\u2019est la m\xEAme chose que ce que nous avons obtenu avec le premier pipeline !"),mg.forEach(a),Lo=d(s),ps=r(s,"H3",{class:!0});var ci=o(ps);qs=r(ci,"A",{id:!0,class:!0,href:!0});var fg=o(qs);Gl=r(fg,"SPAN",{});var hg=o(Gl);b(mt.$$.fragment,hg),hg.forEach(a),fg.forEach(a),Mf=d(ci),Vl=r(ci,"SPAN",{});var xg=o(Vl);If=n(xg,"Regroupement des entit\xE9s"),xg.forEach(a),ci.forEach(a),Xo=d(s),I=r(s,"P",{});var N=o(I);Df=n(N,"L\u2019utilisation des "),Ql=r(N,"EM",{});var gg=o(Ql);Tf=n(gg,"offsets"),gg.forEach(a),Rf=n(N," pour d\xE9terminer les cl\xE9s de d\xE9but et de fin pour chaque entit\xE9 est pratique mais cette information n\u2019est pas strictement n\xE9cessaire. Cependant, lorsque nous voulons regrouper les entit\xE9s, les "),Jl=r(N,"EM",{});var vg=o(Jl);Sf=n(vg,"offsets"),vg.forEach(a),Bf=n(N," nous \xE9pargnent un batch de code compliqu\xE9. Par exemple, si nous voulions regrouper les "),Ul=r(N,"EM",{});var bg=o(Ul);Af=n(bg,"tokens"),bg.forEach(a),Ff=d(N),Yl=r(N,"CODE",{});var _g=o(Yl);Nf=n(_g,"Hu"),_g.forEach(a),Lf=n(N,", "),Kl=r(N,"CODE",{});var jg=o(Kl);Xf=n(jg,"##gging"),jg.forEach(a),Hf=n(N,", et "),Wl=r(N,"CODE",{});var Eg=o(Wl);Gf=n(Eg,"Face"),Eg.forEach(a),Vf=n(N,", nous pourrions \xE9tablir des r\xE8gles sp\xE9ciales disant que les deux premiers devraient \xEAtre attach\xE9s tout en enlevant le "),Zl=r(N,"CODE",{});var kg=o(Zl);Qf=n(kg,"##"),kg.forEach(a),Jf=n(N,", et le "),er=r(N,"CODE",{});var $g=o(er);Uf=n($g,"Face"),$g.forEach(a),Yf=n(N," devrait \xEAtre ajout\xE9 avec un espace puisqu\u2019il ne commence pas par "),sr=r(N,"CODE",{});var qg=o(sr);Kf=n(qg,"##"),qg.forEach(a),Wf=n(N," mais cela ne fonctionnerait que pour ce type particulier de "),tr=r(N,"EM",{});var yg=o(tr);Zf=n(yg,"tokenizer"),yg.forEach(a),eh=n(N,". Il faudrait \xE9crire un autre ensemble de r\xE8gles pour un "),nr=r(N,"EM",{});var wg=o(nr);sh=n(wg,"tokenizer"),wg.forEach(a),th=n(N," de type SentencePiece ou "),ar=r(N,"EM",{});var Cg=o(ar);nh=n(Cg,"Byte-Pair-Encoding"),Cg.forEach(a),ah=n(N," (voir plus loin dans ce chapitre)."),N.forEach(a),Ho=d(s),G=r(s,"P",{});var se=o(G);lh=n(se,"Avec les "),lr=r(se,"EM",{});var Og=o(lr);rh=n(Og,"offsets"),Og.forEach(a),oh=n(se,", tout ce code personnalis\xE9 dispara\xEEt : il suffit de prendre l\u2019intervalle du texte original qui commence par le premier "),rr=r(se,"EM",{});var zg=o(rr);ih=n(zg,"token"),zg.forEach(a),ph=n(se," et se termine par le dernier "),or=r(se,"EM",{});var Pg=o(or);uh=n(Pg,"token"),Pg.forEach(a),ch=n(se,". Ainsi, dans le cas des "),ir=r(se,"EM",{});var Mg=o(ir);dh=n(Mg,"tokens"),Mg.forEach(a),mh=d(se),pr=r(se,"CODE",{});var Ig=o(pr);fh=n(Ig,"Hu"),Ig.forEach(a),hh=n(se,", "),ur=r(se,"CODE",{});var Dg=o(ur);xh=n(Dg,"##gging"),Dg.forEach(a),gh=n(se,", et "),cr=r(se,"CODE",{});var Tg=o(cr);vh=n(Tg,"Face"),Tg.forEach(a),bh=n(se,", nous devrions commencer au caract\xE8re 33 (le d\xE9but de "),dr=r(se,"CODE",{});var Rg=o(dr);_h=n(Rg,"Hu"),Rg.forEach(a),jh=n(se,") et finir avant le caract\xE8re 45 (la fin de "),mr=r(se,"CODE",{});var Sg=o(mr);Eh=n(Sg,"Face"),Sg.forEach(a),kh=n(se,") :"),se.forEach(a),Go=d(s),b(ft.$$.fragment,s),Vo=d(s),b(ht.$$.fragment,s),Qo=d(s),he=r(s,"P",{});var Me=o(he);$h=n(Me,"Pour \xE9crire le code qui post-traite les pr\xE9dictions tout en regroupant les entit\xE9s, nous regrouperons les entit\xE9s qui sont cons\xE9cutives et \xE9tiquet\xE9es avec "),fr=r(Me,"CODE",{});var Bg=o(fr);qh=n(Bg,"I-XXX"),Bg.forEach(a),yh=n(Me,", \xE0 l\u2019exception de la premi\xE8re, qui peut \xEAtre \xE9tiquet\xE9e comme "),hr=r(Me,"CODE",{});var Ag=o(hr);wh=n(Ag,"B-XXX"),Ag.forEach(a),Ch=n(Me," ou "),xr=r(Me,"CODE",{});var Fg=o(xr);Oh=n(Fg,"I-XXX"),Fg.forEach(a),zh=n(Me," (ainsi, nous arr\xEAtons de regrouper une entit\xE9 lorsque nous obtenons un "),gr=r(Me,"CODE",{});var Ng=o(gr);Ph=n(Ng,"O"),Ng.forEach(a),Mh=n(Me,", un nouveau type d\u2019entit\xE9, ou un "),vr=r(Me,"CODE",{});var Lg=o(vr);Ih=n(Lg,"B-XXX"),Lg.forEach(a),Dh=n(Me," qui nous indique qu\u2019une entit\xE9 du m\xEAme type commence) :"),Me.forEach(a),Jo=d(s),b(xt.$$.fragment,s),Uo=d(s),Xt=r(s,"P",{});var Xg=o(Xt);Th=n(Xg,"Et nous obtenons les m\xEAmes r\xE9sultats qu\u2019avec notre deuxi\xE8me pipeline !"),Xg.forEach(a),Yo=d(s),b(gt.$$.fragment,s),Ko=d(s),$e=r(s,"P",{});var Ye=o($e);Rh=n(Ye,"Un autre exemple de t\xE2che o\xF9 ces "),br=r(Ye,"EM",{});var Hg=o(br);Sh=n(Hg,"offsets"),Hg.forEach(a),Bh=n(Ye," sont extr\xEAmement utiles est la r\xE9ponse aux questions. Plonger dans ce pipeline, ce que nous ferons dans la section suivante, nous permettra de jeter un coup d\u2019\u0153il \xE0 une derni\xE8re caract\xE9ristique des "),_r=r(Ye,"EM",{});var Gg=o(_r);Ah=n(Gg,"tokenizers"),Gg.forEach(a),Fh=n(Ye," de la biblioth\xE8que \u{1F917} "),jr=r(Ye,"EM",{});var Vg=o(jr);Nh=n(Vg,"Transformers"),Vg.forEach(a),Lh=n(Ye," : la gestion des "),Er=r(Ye,"EM",{});var Qg=o(Er);Xh=n(Qg,"tokens"),Qg.forEach(a),Hh=n(Ye," qui d\xE9bordent lorsque nous tronquons une entr\xE9e \xE0 une longueur donn\xE9e."),Ye.forEach(a),this.h()},h(){E(u,"name","hf:doc:metadata"),E(u,"content",JSON.stringify(fv)),E(C,"id","pouvoirs-spciaux-des-itokenizersi-rapides"),E(C,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),E(C,"href","#pouvoirs-spciaux-des-itokenizersi-rapides"),E(D,"class","relative group"),E(qt,"href","/course/fr/chapter1"),E(yt,"href","/course/fr/chapter5/3"),E(Os,"align","center"),E(zs,"align","center"),E(hn,"align","center"),E(wt,"align","center"),E(Ct,"align","center"),E(Ot,"align","center"),E(zt,"align","center"),E(Pt,"align","center"),E(Mt,"align","center"),E(ds,"id","lobjet-ibatchencodingi"),E(ds,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),E(ds,"href","#lobjet-ibatchencodingi"),E(as,"class","relative group"),E(xs,"id","a-lintrieur-du-pipeline-tokenclassification"),E(xs,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),E(xs,"href","#a-lintrieur-du-pipeline-tokenclassification"),E(ls,"class","relative group"),E(Rt,"href","/course/fr/chapter1"),E(St,"href","/course/fr/chapter2"),E(gs,"id","obtenir-les-rsultats-de-base-avec-le-pipeline"),E(gs,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),E(gs,"href","#obtenir-les-rsultats-de-base-avec-le-pipeline"),E(rs,"class","relative group"),E(Js,"href","https://huggingface.co/dbmdz/bert-large-cased-finetuned-conll03-english"),E(Js,"rel","nofollow"),E(js,"id","des-entres-aux-prdictions"),E(js,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),E(js,"href","#des-entres-aux-prdictions"),E(os,"class","relative group"),E(nt,"class","block dark:hidden"),Jg(nt.src,Yh="https://huggingface.co/datasets/huggingface-course/documentation-images/resolve/main/en/chapter6/IOB_versions.svg")||E(nt,"src",Yh),E(nt,"alt","IOB1 vs IOB2 format"),E(at,"class","hidden dark:block"),Jg(at.src,Kh="https://huggingface.co/datasets/huggingface-course/documentation-images/resolve/main/en/chapter6/IOB_versions-dark.svg")||E(at,"src",Kh),E(at,"alt","IOB1 vs IOB2 format"),E(is,"class","flex justify-center"),E(qs,"id","regroupement-des-entits"),E(qs,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),E(qs,"href","#regroupement-des-entits"),E(ps,"class","relative group")},m(s,i){e(document.head,u),p(s,g,i),_(m,s,i),p(s,w,i),p(s,D,i),e(D,C),e(C,T),_(F,T,null),e(D,B),e(D,R),e(R,ae),e(R,X),e(X,Q),e(R,ue),p(s,P,i),vt[J].m(s,i),p(s,W,i),p(s,k,i),e(k,le),e(k,ce),e(ce,be),e(k,de),e(k,H),e(H,we),e(k,te),e(k,ve),e(ve,f),e(k,$),e(k,Ie),e(Ie,Ke),e(k,We),e(k,ne),e(ne,Ze),e(k,es),e(k,tn),e(tn,hi),e(k,xi),e(k,nn),e(nn,gi),e(k,vi),e(k,qt),e(qt,bi),e(k,_i),p(s,wr,i),_(Cs,s,i),p(s,Cr,i),p(s,U,i),e(U,ji),e(U,an),e(an,Ei),e(U,ki),e(U,ln),e(ln,$i),e(U,qi),e(U,rn),e(rn,yi),e(U,wi),e(U,on),e(on,Ci),e(U,Oi),e(U,yt),e(yt,zi),e(U,Pi),e(U,pn),e(pn,Mi),e(U,Ii),e(U,un),e(un,Di),e(U,Ti),e(U,cn),e(cn,Ri),e(U,Si),p(s,Or,i),p(s,us,i),e(us,dn),e(dn,ss),e(ss,Os),e(Os,mn),e(mn,Bi),e(Os,Ai),e(ss,Fi),e(ss,zs),e(zs,fn),e(fn,Ni),e(zs,Li),e(ss,Xi),e(ss,hn),e(us,Hi),e(us,Ps),e(Ps,ts),e(ts,wt),e(wt,xn),e(xn,Gi),e(ts,Vi),e(ts,Ct),e(Ct,Qi),e(ts,Ji),e(ts,Ot),e(Ot,Ui),e(Ps,Yi),e(Ps,ns),e(ns,zt),e(zt,gn),e(gn,Ki),e(ns,Wi),e(ns,Pt),e(Pt,Zi),e(ns,ep),e(ns,Mt),e(Mt,sp),p(s,zr,i),_(cs,s,i),p(s,Pr,i),p(s,as,i),e(as,ds),e(ds,vn),_(Ms,vn,null),e(as,tp),e(as,It),e(It,np),e(It,bn),e(bn,ap),p(s,Mr,i),_(Is,s,i),p(s,Ir,i),p(s,Ce,i),e(Ce,lp),e(Ce,_n),e(_n,rp),e(Ce,op),e(Ce,jn),e(jn,ip),e(Ce,pp),e(Ce,En),e(En,up),e(Ce,cp),p(s,Dr,i),p(s,me,i),e(me,dp),e(me,kn),e(kn,mp),e(me,fp),e(me,$n),e($n,hp),e(me,xp),e(me,qn),e(qn,gp),e(me,vp),e(me,yn),e(yn,bp),e(me,_p),e(me,wn),e(wn,jp),e(me,Ep),p(s,Tr,i),p(s,Dt,i),e(Dt,kp),p(s,Rr,i),_(Ds,s,i),p(s,Sr,i),p(s,Fe,i),e(Fe,$p),e(Fe,Cn),e(Cn,qp),e(Fe,yp),e(Fe,On),e(On,wp),e(Fe,Cp),p(s,Br,i),_(Ts,s,i),p(s,Ar,i),p(s,re,i),e(re,Op),e(re,zn),e(zn,zp),e(re,Pp),e(re,Pn),e(Pn,Mp),e(re,Ip),e(re,Mn),e(Mn,Dp),e(re,Tp),e(re,In),e(In,Rp),e(re,Sp),e(re,Dn),e(Dn,Bp),e(re,Ap),e(re,Tn),e(Tn,Fp),e(re,Np),p(s,Fr,i),_(Rs,s,i),p(s,Nr,i),_(Ss,s,i),p(s,Lr,i),p(s,ms,i),e(ms,Lp),e(ms,Rn),e(Rn,Xp),e(ms,Hp),p(s,Xr,i),_(Bs,s,i),p(s,Hr,i),_(As,s,i),p(s,Gr,i),p(s,Oe,i),e(Oe,Gp),e(Oe,Sn),e(Sn,Vp),e(Oe,Qp),e(Oe,Bn),e(Bn,Jp),e(Oe,Up),e(Oe,An),e(An,Yp),e(Oe,Kp),p(s,Vr,i),_(Fs,s,i),p(s,Qr,i),_(Ns,s,i),p(s,Jr,i),p(s,_e,i),e(_e,Wp),e(_e,Fn),e(Fn,Zp),e(_e,eu),e(_e,Nn),e(Nn,su),e(_e,tu),e(_e,Ln),e(Ln,nu),e(_e,au),e(_e,Xn),e(Xn,lu),e(_e,ru),p(s,Ur,i),_(Ls,s,i),p(s,Yr,i),_(Xs,s,i),p(s,Kr,i),p(s,O,i),e(O,ou),e(O,Hn),e(Hn,iu),e(O,pu),e(O,Gn),e(Gn,uu),e(O,cu),e(O,Vn),e(Vn,du),e(O,mu),e(O,Qn),e(Qn,fu),e(O,hu),e(O,Jn),e(Jn,xu),e(O,gu),e(O,Un),e(Un,vu),e(O,bu),e(O,Yn),e(Yn,_u),e(O,ju),e(O,Kn),e(Kn,Eu),e(O,ku),e(O,Wn),e(Wn,$u),e(O,qu),e(O,Zn),e(Zn,yu),e(O,wu),e(O,ea),e(ea,Cu),e(O,Ou),e(O,sa),e(sa,zu),e(O,Pu),e(O,ta),e(ta,Mu),e(O,Iu),e(O,na),e(na,Du),e(O,Tu),e(O,aa),e(aa,Ru),e(O,Su),p(s,Wr,i),p(s,Ne,i),e(Ne,Bu),e(Ne,la),e(la,Au),e(Ne,Fu),e(Ne,ra),e(ra,Nu),e(Ne,Lu),p(s,Zr,i),_(fs,s,i),p(s,eo,i),p(s,je,i),e(je,Xu),e(je,oa),e(oa,Hu),e(je,Gu),e(je,ia),e(ia,Vu),e(je,Qu),e(je,pa),e(pa,Ju),e(je,Uu),e(je,ua),e(ua,Yu),e(je,Ku),p(s,so,i),p(s,Z,i),e(Z,Wu),e(Z,ca),e(ca,Zu),e(Z,ec),e(Z,da),e(da,sc),e(Z,tc),e(Z,ma),e(ma,nc),e(Z,ac),e(Z,fa),e(fa,lc),e(Z,rc),e(Z,ha),e(ha,oc),e(Z,ic),e(Z,xa),e(xa,pc),e(Z,uc),e(Z,ga),e(ga,cc),e(Z,dc),p(s,to,i),_(Hs,s,i),p(s,no,i),_(Gs,s,i),p(s,ao,i),p(s,Ee,i),e(Ee,mc),e(Ee,va),e(va,fc),e(Ee,hc),e(Ee,ba),e(ba,xc),e(Ee,gc),e(Ee,_a),e(_a,vc),e(Ee,bc),e(Ee,ja),e(ja,_c),e(Ee,jc),p(s,lo,i),_(hs,s,i),p(s,ro,i),p(s,ls,i),e(ls,xs),e(xs,Ea),_(Vs,Ea,null),e(ls,Ec),e(ls,Tt),e(Tt,kc),e(Tt,ka),e(ka,$c),p(s,oo,i),p(s,fe,i),e(fe,qc),e(fe,Rt),e(Rt,yc),e(fe,wc),e(fe,$a),e($a,Cc),e(fe,Oc),e(fe,qa),e(qa,zc),e(fe,Pc),e(fe,St),e(St,Mc),e(fe,Ic),e(fe,ya),e(ya,Dc),e(fe,Tc),p(s,io,i),bt[De].m(s,i),p(s,Bt,i),p(s,rs,i),e(rs,gs),e(gs,wa),_(Qs,wa,null),e(rs,Rc),e(rs,Ca),e(Ca,Sc),p(s,po,i),p(s,Le,i),e(Le,Bc),e(Le,Oa),e(Oa,Ac),e(Le,Fc),e(Le,Js),e(Js,za),e(za,Nc),e(Le,Lc),p(s,uo,i),_(Us,s,i),p(s,co,i),_(Ys,s,i),p(s,mo,i),p(s,ke,i),e(ke,Xc),e(ke,Pa),e(Pa,Hc),e(ke,Gc),e(ke,Ma),e(Ma,Vc),e(ke,Qc),e(ke,Ia),e(Ia,Jc),e(ke,Uc),e(ke,Da),e(Da,Yc),e(ke,Kc),p(s,fo,i),_(Ks,s,i),p(s,ho,i),_(Ws,s,i),p(s,xo,i),p(s,ee,i),e(ee,Wc),e(ee,Ta),e(Ta,Zc),e(ee,ed),e(ee,Ra),e(Ra,sd),e(ee,td),e(ee,Sa),e(Sa,nd),e(ee,ad),e(ee,Ba),e(Ba,ld),e(ee,rd),e(ee,Aa),e(Aa,od),e(ee,id),e(ee,Fa),e(Fa,pd),e(ee,ud),e(ee,Na),e(Na,cd),e(ee,dd),p(s,go,i),p(s,Xe,i),e(Xe,He),e(He,La),e(La,md),e(He,fd),e(He,Xa),e(Xa,hd),e(He,xd),e(He,Ha),e(Ha,gd),e(He,vd),e(Xe,bd),e(Xe,vs),e(vs,Ga),e(Ga,_d),e(vs,jd),e(vs,Va),e(Va,Ed),e(vs,kd),e(Xe,$d),e(Xe,bs),e(bs,Qa),e(Qa,qd),e(bs,yd),e(bs,Ja),e(Ja,wd),e(bs,Cd),p(s,vo,i),p(s,_s,i),e(_s,Od),e(_s,Ua),e(Ua,zd),e(_s,Pd),p(s,bo,i),p(s,os,i),e(os,js),e(js,Ya),_(Zs,Ya,null),e(os,Md),e(os,Ka),e(Ka,Id),p(s,_o,i),_t[Re].m(s,i),p(s,At,i),p(s,Es,i),e(Es,Dd),e(Es,Wa),e(Wa,Td),e(Es,Rd),p(s,jo,i),jt[Be].m(s,i),p(s,Ft,i),_(et,s,i),p(s,Eo,i),p(s,ks,i),e(ks,Sd),e(ks,Za),e(Za,Bd),e(ks,Ad),p(s,ko,i),_(st,s,i),p(s,$o,i),_(tt,s,i),p(s,qo,i),p(s,q,i),e(q,Fd),e(q,el),e(el,Nd),e(q,Ld),e(q,sl),e(sl,Xd),e(q,Hd),e(q,tl),e(tl,Gd),e(q,Vd),e(q,nl),e(nl,Qd),e(q,Jd),e(q,al),e(al,Ud),e(q,Yd),e(q,ll),e(ll,Kd),e(q,Wd),e(q,rl),e(rl,Zd),e(q,em),e(q,ol),e(ol,sm),e(q,tm),e(q,il),e(il,nm),e(q,am),e(q,pl),e(pl,lm),e(q,rm),e(q,ul),e(ul,om),e(q,im),e(q,cl),e(cl,pm),e(q,um),e(q,dl),e(dl,cm),e(q,dm),e(q,ml),e(ml,mm),e(q,fm),e(q,fl),e(fl,hm),e(q,xm),e(q,hl),e(hl,gm),e(q,vm),e(q,xl),e(xl,bm),e(q,_m),p(s,yo,i),p(s,S,i),e(S,jm),e(S,gl),e(gl,Em),e(S,km),e(S,vl),e(vl,$m),e(S,qm),e(S,bl),e(bl,ym),e(S,wm),e(S,_l),e(_l,Cm),e(S,Om),e(S,jl),e(jl,zm),e(S,Pm),e(S,El),e(El,Mm),e(S,Im),e(S,kl),e(kl,Dm),e(S,Tm),e(S,$l),e($l,Rm),e(S,Sm),e(S,ql),e(ql,Bm),e(S,Am),e(S,yl),e(yl,Fm),e(S,Nm),e(S,wl),e(wl,Lm),e(S,Xm),p(s,wo,i),p(s,is,i),e(is,nt),e(is,Hm),e(is,at),p(s,Co,i),p(s,Ge,i),e(Ge,Gm),e(Ge,Cl),e(Cl,Vm),e(Ge,Qm),e(Ge,Ol),e(Ol,Jm),e(Ge,Um),p(s,Oo,i),_(lt,s,i),p(s,zo,i),_(rt,s,i),p(s,Po,i),p(s,oe,i),e(oe,Ym),e(oe,zl),e(zl,Km),e(oe,Wm),e(oe,Pl),e(Pl,Zm),e(oe,ef),e(oe,Ml),e(Ml,sf),e(oe,tf),e(oe,Il),e(Il,nf),e(oe,af),e(oe,Dl),e(Dl,lf),e(oe,rf),e(oe,Tl),e(Tl,of),e(oe,pf),p(s,Mo,i),_(ot,s,i),p(s,Io,i),_(it,s,i),p(s,Do,i),p(s,Y,i),e(Y,uf),e(Y,Rl),e(Rl,cf),e(Y,df),e(Y,Sl),e(Sl,mf),e(Y,ff),e(Y,Bl),e(Bl,hf),e(Y,xf),e(Y,Al),e(Al,gf),e(Y,vf),e(Y,Fl),e(Fl,bf),e(Y,_f),e(Y,Nl),e(Nl,jf),e(Y,Ef),e(Y,Ll),e(Ll,kf),e(Y,$f),e(Y,Xl),e(Xl,qf),e(Y,yf),p(s,To,i),_(pt,s,i),p(s,Ro,i),p(s,$s,i),e($s,wf),e($s,Hl),e(Hl,Cf),e($s,Of),p(s,So,i),_(ut,s,i),p(s,Bo,i),p(s,Nt,i),e(Nt,zf),p(s,Ao,i),_(ct,s,i),p(s,Fo,i),_(dt,s,i),p(s,No,i),p(s,Lt,i),e(Lt,Pf),p(s,Lo,i),p(s,ps,i),e(ps,qs),e(qs,Gl),_(mt,Gl,null),e(ps,Mf),e(ps,Vl),e(Vl,If),p(s,Xo,i),p(s,I,i),e(I,Df),e(I,Ql),e(Ql,Tf),e(I,Rf),e(I,Jl),e(Jl,Sf),e(I,Bf),e(I,Ul),e(Ul,Af),e(I,Ff),e(I,Yl),e(Yl,Nf),e(I,Lf),e(I,Kl),e(Kl,Xf),e(I,Hf),e(I,Wl),e(Wl,Gf),e(I,Vf),e(I,Zl),e(Zl,Qf),e(I,Jf),e(I,er),e(er,Uf),e(I,Yf),e(I,sr),e(sr,Kf),e(I,Wf),e(I,tr),e(tr,Zf),e(I,eh),e(I,nr),e(nr,sh),e(I,th),e(I,ar),e(ar,nh),e(I,ah),p(s,Ho,i),p(s,G,i),e(G,lh),e(G,lr),e(lr,rh),e(G,oh),e(G,rr),e(rr,ih),e(G,ph),e(G,or),e(or,uh),e(G,ch),e(G,ir),e(ir,dh),e(G,mh),e(G,pr),e(pr,fh),e(G,hh),e(G,ur),e(ur,xh),e(G,gh),e(G,cr),e(cr,vh),e(G,bh),e(G,dr),e(dr,_h),e(G,jh),e(G,mr),e(mr,Eh),e(G,kh),p(s,Go,i),_(ft,s,i),p(s,Vo,i),_(ht,s,i),p(s,Qo,i),p(s,he,i),e(he,$h),e(he,fr),e(fr,qh),e(he,yh),e(he,hr),e(hr,wh),e(he,Ch),e(he,xr),e(xr,Oh),e(he,zh),e(he,gr),e(gr,Ph),e(he,Mh),e(he,vr),e(vr,Ih),e(he,Dh),p(s,Jo,i),_(xt,s,i),p(s,Uo,i),p(s,Xt,i),e(Xt,Th),p(s,Yo,i),_(gt,s,i),p(s,Ko,i),p(s,$e,i),e($e,Rh),e($e,br),e(br,Sh),e($e,Bh),e($e,_r),e(_r,Ah),e($e,Fh),e($e,jr),e(jr,Nh),e($e,Lh),e($e,Er),e(Er,Xh),e($e,Hh),Wo=!0},p(s,[i]){const Et={};i&1&&(Et.fw=s[0]),m.$set(Et);let Ht=J;J=Zh(s),J!==Ht&&(mi(),h(vt[Ht],1,1,()=>{vt[Ht]=null}),di(),A=vt[J],A||(A=vt[J]=Wh[J](s),A.c()),x(A,1),A.m(W.parentNode,W));const kr={};i&2&&(kr.$$scope={dirty:i,ctx:s}),cs.$set(kr);const kt={};i&2&&(kt.$$scope={dirty:i,ctx:s}),fs.$set(kt);const $r={};i&2&&($r.$$scope={dirty:i,ctx:s}),hs.$set($r);let K=De;De=s2(s),De!==K&&(mi(),h(bt[K],1,1,()=>{bt[K]=null}),di(),Te=bt[De],Te||(Te=bt[De]=e2[De](s),Te.c()),x(Te,1),Te.m(Bt.parentNode,Bt));let Gt=Re;Re=n2(s),Re!==Gt&&(mi(),h(_t[Gt],1,1,()=>{_t[Gt]=null}),di(),Se=_t[Re],Se||(Se=_t[Re]=t2[Re](s),Se.c()),x(Se,1),Se.m(At.parentNode,At));let Vt=Be;Be=l2(s),Be!==Vt&&(mi(),h(jt[Vt],1,1,()=>{jt[Vt]=null}),di(),Ae=jt[Be],Ae||(Ae=jt[Be]=a2[Be](s),Ae.c()),x(Ae,1),Ae.m(Ft.parentNode,Ft))},i(s){Wo||(x(m.$$.fragment,s),x(F.$$.fragment,s),x(A),x(Cs.$$.fragment,s),x(cs.$$.fragment,s),x(Ms.$$.fragment,s),x(Is.$$.fragment,s),x(Ds.$$.fragment,s),x(Ts.$$.fragment,s),x(Rs.$$.fragment,s),x(Ss.$$.fragment,s),x(Bs.$$.fragment,s),x(As.$$.fragment,s),x(Fs.$$.fragment,s),x(Ns.$$.fragment,s),x(Ls.$$.fragment,s),x(Xs.$$.fragment,s),x(fs.$$.fragment,s),x(Hs.$$.fragment,s),x(Gs.$$.fragment,s),x(hs.$$.fragment,s),x(Vs.$$.fragment,s),x(Te),x(Qs.$$.fragment,s),x(Us.$$.fragment,s),x(Ys.$$.fragment,s),x(Ks.$$.fragment,s),x(Ws.$$.fragment,s),x(Zs.$$.fragment,s),x(Se),x(Ae),x(et.$$.fragment,s),x(st.$$.fragment,s),x(tt.$$.fragment,s),x(lt.$$.fragment,s),x(rt.$$.fragment,s),x(ot.$$.fragment,s),x(it.$$.fragment,s),x(pt.$$.fragment,s),x(ut.$$.fragment,s),x(ct.$$.fragment,s),x(dt.$$.fragment,s),x(mt.$$.fragment,s),x(ft.$$.fragment,s),x(ht.$$.fragment,s),x(xt.$$.fragment,s),x(gt.$$.fragment,s),Wo=!0)},o(s){h(m.$$.fragment,s),h(F.$$.fragment,s),h(A),h(Cs.$$.fragment,s),h(cs.$$.fragment,s),h(Ms.$$.fragment,s),h(Is.$$.fragment,s),h(Ds.$$.fragment,s),h(Ts.$$.fragment,s),h(Rs.$$.fragment,s),h(Ss.$$.fragment,s),h(Bs.$$.fragment,s),h(As.$$.fragment,s),h(Fs.$$.fragment,s),h(Ns.$$.fragment,s),h(Ls.$$.fragment,s),h(Xs.$$.fragment,s),h(fs.$$.fragment,s),h(Hs.$$.fragment,s),h(Gs.$$.fragment,s),h(hs.$$.fragment,s),h(Vs.$$.fragment,s),h(Te),h(Qs.$$.fragment,s),h(Us.$$.fragment,s),h(Ys.$$.fragment,s),h(Ks.$$.fragment,s),h(Ws.$$.fragment,s),h(Zs.$$.fragment,s),h(Se),h(Ae),h(et.$$.fragment,s),h(st.$$.fragment,s),h(tt.$$.fragment,s),h(lt.$$.fragment,s),h(rt.$$.fragment,s),h(ot.$$.fragment,s),h(it.$$.fragment,s),h(pt.$$.fragment,s),h(ut.$$.fragment,s),h(ct.$$.fragment,s),h(dt.$$.fragment,s),h(mt.$$.fragment,s),h(ft.$$.fragment,s),h(ht.$$.fragment,s),h(xt.$$.fragment,s),h(gt.$$.fragment,s),Wo=!1},d(s){a(u),s&&a(g),j(m,s),s&&a(w),s&&a(D),j(F),s&&a(P),vt[J].d(s),s&&a(W),s&&a(k),s&&a(wr),j(Cs,s),s&&a(Cr),s&&a(U),s&&a(Or),s&&a(us),s&&a(zr),j(cs,s),s&&a(Pr),s&&a(as),j(Ms),s&&a(Mr),j(Is,s),s&&a(Ir),s&&a(Ce),s&&a(Dr),s&&a(me),s&&a(Tr),s&&a(Dt),s&&a(Rr),j(Ds,s),s&&a(Sr),s&&a(Fe),s&&a(Br),j(Ts,s),s&&a(Ar),s&&a(re),s&&a(Fr),j(Rs,s),s&&a(Nr),j(Ss,s),s&&a(Lr),s&&a(ms),s&&a(Xr),j(Bs,s),s&&a(Hr),j(As,s),s&&a(Gr),s&&a(Oe),s&&a(Vr),j(Fs,s),s&&a(Qr),j(Ns,s),s&&a(Jr),s&&a(_e),s&&a(Ur),j(Ls,s),s&&a(Yr),j(Xs,s),s&&a(Kr),s&&a(O),s&&a(Wr),s&&a(Ne),s&&a(Zr),j(fs,s),s&&a(eo),s&&a(je),s&&a(so),s&&a(Z),s&&a(to),j(Hs,s),s&&a(no),j(Gs,s),s&&a(ao),s&&a(Ee),s&&a(lo),j(hs,s),s&&a(ro),s&&a(ls),j(Vs),s&&a(oo),s&&a(fe),s&&a(io),bt[De].d(s),s&&a(Bt),s&&a(rs),j(Qs),s&&a(po),s&&a(Le),s&&a(uo),j(Us,s),s&&a(co),j(Ys,s),s&&a(mo),s&&a(ke),s&&a(fo),j(Ks,s),s&&a(ho),j(Ws,s),s&&a(xo),s&&a(ee),s&&a(go),s&&a(Xe),s&&a(vo),s&&a(_s),s&&a(bo),s&&a(os),j(Zs),s&&a(_o),_t[Re].d(s),s&&a(At),s&&a(Es),s&&a(jo),jt[Be].d(s),s&&a(Ft),j(et,s),s&&a(Eo),s&&a(ks),s&&a(ko),j(st,s),s&&a($o),j(tt,s),s&&a(qo),s&&a(q),s&&a(yo),s&&a(S),s&&a(wo),s&&a(is),s&&a(Co),s&&a(Ge),s&&a(Oo),j(lt,s),s&&a(zo),j(rt,s),s&&a(Po),s&&a(oe),s&&a(Mo),j(ot,s),s&&a(Io),j(it,s),s&&a(Do),s&&a(Y),s&&a(To),j(pt,s),s&&a(Ro),s&&a($s),s&&a(So),j(ut,s),s&&a(Bo),s&&a(Nt),s&&a(Ao),j(ct,s),s&&a(Fo),j(dt,s),s&&a(No),s&&a(Lt),s&&a(Lo),s&&a(ps),j(mt),s&&a(Xo),s&&a(I),s&&a(Ho),s&&a(G),s&&a(Go),j(ft,s),s&&a(Vo),j(ht,s),s&&a(Qo),s&&a(he),s&&a(Jo),j(xt,s),s&&a(Uo),s&&a(Xt),s&&a(Yo),j(gt,s),s&&a(Ko),s&&a($e)}}}const fv={local:"pouvoirs-spciaux-des-itokenizersi-rapides",sections:[{local:"lobjet-ibatchencodingi",title:"L'objet <i>BatchEncoding</i>"},{local:"a-lintrieur-du-pipeline-tokenclassification",sections:[{local:"obtenir-les-rsultats-de-base-avec-le-pipeline",title:"Obtenir les r\xE9sultats de base avec le pipeline"},{local:"des-entres-aux-prdictions",title:"Des entr\xE9es aux pr\xE9dictions"},{local:"regroupement-des-entits",title:"Regroupement des entit\xE9s"}],title:"A l'int\xE9rieur du pipeline `token-classification`"}],title:"Pouvoirs sp\xE9ciaux des <i>tokenizers</i> rapides"};function hv(L,u,g){let m="pt";return ev(()=>{const w=new URLSearchParams(window.location.search);g(0,m=w.get("fw")||"pt")}),[m]}class kv extends Yg{constructor(u){super();Kg(this,u,hv,mv,Wg,{})}}export{kv as default,fv as metadata};
