import{S as Fm,i as Wm,s as Gm,e as o,k as u,w as d,t,U as Ii,M as Ym,c as r,d as a,m,a as p,x as h,h as n,V as Li,b as P,G as e,g as i,y as f,q as g,o as v,B as b,v as Zm}from"../../chunks/vendor-hf-doc-builder.js";import{T as Dt}from"../../chunks/Tip-hf-doc-builder.js";import{Y as Jm}from"../../chunks/Youtube-hf-doc-builder.js";import{I as Nt}from"../../chunks/IconCopyLink-hf-doc-builder.js";import{C as k}from"../../chunks/CodeBlock-hf-doc-builder.js";import{D as Km}from"../../chunks/DocNotebookDropdown-hf-doc-builder.js";function Qm(R){let c,y,q,$,E;return{c(){c=o("p"),y=t("\u{1F4A1} Cette section couvre "),q=o("em"),$=t("Unigram"),E=t(" en profondeur, allant jusqu\u2019\xE0 montrer une impl\xE9mentation compl\xE8te. Vous pouvez passer directement \xE0 la fin si vous souhaitez simplement avoir un aper\xE7u g\xE9n\xE9ral de l\u2019algorithme de tok\xE9nisation.")},l(_){c=r(_,"P",{});var j=p(c);y=n(j,"\u{1F4A1} Cette section couvre "),q=r(j,"EM",{});var x=p(q);$=n(x,"Unigram"),x.forEach(a),E=n(j," en profondeur, allant jusqu\u2019\xE0 montrer une impl\xE9mentation compl\xE8te. Vous pouvez passer directement \xE0 la fin si vous souhaitez simplement avoir un aper\xE7u g\xE9n\xE9ral de l\u2019algorithme de tok\xE9nisation."),j.forEach(a)},m(_,j){i(_,c,j),e(c,y),e(c,q),e(q,$),e(c,E)},d(_){_&&a(c)}}}function sc(R){let c,y,q,$,E;return{c(){c=o("p"),y=t("\u270F\uFE0F "),q=o("strong"),$=t("A votre tour !"),E=t(" Ecrivez le code permettant de calculer les fr\xE9quences ci-dessus et v\xE9rifiez que les r\xE9sultats affich\xE9s sont corrects, de m\xEAme que la somme totale.")},l(_){c=r(_,"P",{});var j=p(c);y=n(j,"\u270F\uFE0F "),q=r(j,"STRONG",{});var x=p(q);$=n(x,"A votre tour !"),x.forEach(a),E=n(j," Ecrivez le code permettant de calculer les fr\xE9quences ci-dessus et v\xE9rifiez que les r\xE9sultats affich\xE9s sont corrects, de m\xEAme que la somme totale."),j.forEach(a)},m(_,j){i(_,c,j),e(c,y),e(c,q),e(q,$),e(c,E)},d(_){_&&a(c)}}}function ec(R){let c,y,q,$,E,_,j,x;return{c(){c=o("p"),y=t("\u270F\uFE0F "),q=o("strong"),$=t("A votre tour !"),E=t(" D\xE9terminer la tokenization du mot "),_=o("code"),j=t('"huggun"'),x=t(" et son score.")},l(C){c=r(C,"P",{});var w=p(c);y=n(w,"\u270F\uFE0F "),q=r(w,"STRONG",{});var B=p(q);$=n(B,"A votre tour !"),B.forEach(a),E=n(w," D\xE9terminer la tokenization du mot "),_=r(w,"CODE",{});var N=p(_);j=n(N,'"huggun"'),N.forEach(a),x=n(w," et son score."),w.forEach(a)},m(C,w){i(C,c,w),e(c,y),e(c,q),e(q,$),e(c,E),e(c,_),e(_,j),e(c,x)},d(C){C&&a(c)}}}function ac(R){let c,y,q,$,E,_,j,x;return{c(){c=o("p"),y=t("\u{1F4A1} "),q=o("em"),$=t("SentencePiece"),E=t(" utilise un algorithme plus efficace appel\xE9 "),_=o("em"),j=t("Enhanced Suffix Array"),x=t(" (ESA) pour cr\xE9er le vocabulaire initial.")},l(C){c=r(C,"P",{});var w=p(c);y=n(w,"\u{1F4A1} "),q=r(w,"EM",{});var B=p(q);$=n(B,"SentencePiece"),B.forEach(a),E=n(w," utilise un algorithme plus efficace appel\xE9 "),_=r(w,"EM",{});var N=p(_);j=n(N,"Enhanced Suffix Array"),N.forEach(a),x=n(w," (ESA) pour cr\xE9er le vocabulaire initial."),w.forEach(a)},m(C,w){i(C,c,w),e(c,y),e(c,q),e(q,$),e(c,E),e(c,_),e(_,j),e(c,x)},d(C){C&&a(c)}}}function tc(R){let c,y,q,$,E,_,j,x,C,w,B;return{c(){c=o("p"),y=t("\u{1F4A1} Cette approche est tr\xE8s inefficace, c\u2019est pourquoi "),q=o("em"),$=t("SentencePiece"),E=t(" utilise une approximation de la perte du mod\xE8le sans le "),_=o("em"),j=t("token"),x=t(" X. Au lieu de partir de z\xE9ro, il remplace simplement le "),C=o("em"),w=t("token"),B=t(" X par sa segmentation dans le vocabulaire restant. De cette fa\xE7on, tous les scores peuvent \xEAtre calcul\xE9s en une seule fois, en m\xEAme temps que la perte du mod\xE8le.")},l(N){c=r(N,"P",{});var A=p(c);y=n(A,"\u{1F4A1} Cette approche est tr\xE8s inefficace, c\u2019est pourquoi "),q=r(A,"EM",{});var Ss=p(q);$=n(Ss,"SentencePiece"),Ss.forEach(a),E=n(A," utilise une approximation de la perte du mod\xE8le sans le "),_=r(A,"EM",{});var V=p(_);j=n(V,"token"),V.forEach(a),x=n(A," X. Au lieu de partir de z\xE9ro, il remplace simplement le "),C=r(A,"EM",{});var _e=p(C);w=n(_e,"token"),_e.forEach(a),B=n(A," X par sa segmentation dans le vocabulaire restant. De cette fa\xE7on, tous les scores peuvent \xEAtre calcul\xE9s en une seule fois, en m\xEAme temps que la perte du mod\xE8le."),A.forEach(a)},m(N,A){i(N,c,A),e(c,y),e(c,q),e(q,$),e(c,E),e(c,_),e(_,j),e(c,x),e(c,C),e(C,w),e(c,B)},d(N){N&&a(c)}}}function nc(R){let c,y,q,$,E,_,j,x,C,w,B,N,A,Ss,V,_e,Ge,Tl,Ul,Ye,Ol,Sl,Tt,Hs,Ut,ds,Ot,ps,hs,Ze,Is,Hl,Je,Il,St,X,Ll,Ke,Bl,Vl,Qe,Rl,Xl,sa,Fl,Wl,Ht,fs,Gl,ea,Yl,Zl,It,gs,Jl,Lt,Vm='<span class="katex"><span class="katex-mathml"><math xmlns="http://www.w3.org/1998/Math/MathML"><semantics><mrow><mi>p</mi></mrow><annotation encoding="application/x-tex">p</annotation></semantics></math></span><span class="katex-html" aria-hidden="true"><span class="base"><span class="strut" style="height:0.625em;vertical-align:-0.1944em;"></span><span class="mord mathnormal">p</span></span></span></span>',Bt,Vt,je,Kl,Rt,K,Ql,aa,so,eo,ta,ao,to,Xt,ke,no,Ft,Ls,Wt,$e,lo,Gt,Bs,Yt,is,vs,na,Vs,oo,la,ro,Zt,z,po,oa,io,uo,ra,mo,co,pa,ho,fo,ia,go,vo,ua,bo,qo,ma,_o,jo,ca,ko,$o,Jt,D,wo,da,Eo,yo,ha,xo,Po,fa,Co,zo,ga,Mo,Ao,va,Do,No,ba,To,Uo,Kt,we,Oo,Qt,Rs,sn,bs,So,qa,Ho,Io,en,qs,an,M,Lo,_a,Bo,Vo,ja,Ro,Xo,ka,Fo,Wo,$a,Go,Yo,wa,Zo,Jo,Ea,Ko,Qo,tn,Rm='<span class="katex-display"><span class="katex"><span class="katex-mathml"><math xmlns="http://www.w3.org/1998/Math/MathML" display="block"><semantics><mrow><mi>P</mi><mo stretchy="false">(</mo><mo stretchy="false">[</mo><mi mathvariant="normal">\u2018</mi><mi mathvariant="normal">\u2018</mi><mi>p</mi><mi mathvariant="normal">&quot;</mi><mo separator="true">,</mo><mi mathvariant="normal">\u2018</mi><mi mathvariant="normal">\u2018</mi><mi>u</mi><mi mathvariant="normal">&quot;</mi><mo separator="true">,</mo><mi mathvariant="normal">\u2018</mi><mi mathvariant="normal">\u2018</mi><mi>g</mi><mi mathvariant="normal">&quot;</mi><mo stretchy="false">]</mo><mo stretchy="false">)</mo><mo>=</mo><mi>P</mi><mo stretchy="false">(</mo><mi mathvariant="normal">\u2018</mi><mi mathvariant="normal">\u2018</mi><mi>p</mi><mi mathvariant="normal">&quot;</mi><mo stretchy="false">)</mo><mo>\xD7</mo><mi>P</mi><mo stretchy="false">(</mo><mi mathvariant="normal">\u2018</mi><mi mathvariant="normal">\u2018</mi><mi>u</mi><mi mathvariant="normal">&quot;</mi><mo stretchy="false">)</mo><mo>\xD7</mo><mi>P</mi><mo stretchy="false">(</mo><mi mathvariant="normal">\u2018</mi><mi mathvariant="normal">\u2018</mi><mi>g</mi><mi mathvariant="normal">&quot;</mi><mo stretchy="false">)</mo><mo>=</mo><mfrac><mn>5</mn><mn>210</mn></mfrac><mo>\xD7</mo><mfrac><mn>36</mn><mn>210</mn></mfrac><mo>\xD7</mo><mfrac><mn>20</mn><mn>210</mn></mfrac><mo>=</mo><mn>0.000389</mn></mrow><annotation encoding="application/x-tex">P([``p&quot;, ``u&quot;, ``g&quot;]) = P(``p&quot;) \\times P(``u&quot;) \\times P(``g&quot;) = \\frac{5}{210} \\times \\frac{36}{210} \\times \\frac{20}{210} = 0.000389</annotation></semantics></math></span><span class="katex-html" aria-hidden="true"><span class="base"><span class="strut" style="height:1em;vertical-align:-0.25em;"></span><span class="mord mathnormal" style="margin-right:0.13889em;">P</span><span class="mopen">([</span><span class="mord">\u2018\u2018</span><span class="mord mathnormal">p</span><span class="mord">&quot;</span><span class="mpunct">,</span><span class="mspace" style="margin-right:0.1667em;"></span><span class="mord">\u2018\u2018</span><span class="mord mathnormal">u</span><span class="mord">&quot;</span><span class="mpunct">,</span><span class="mspace" style="margin-right:0.1667em;"></span><span class="mord">\u2018\u2018</span><span class="mord mathnormal" style="margin-right:0.03588em;">g</span><span class="mord">&quot;</span><span class="mclose">])</span><span class="mspace" style="margin-right:0.2778em;"></span><span class="mrel">=</span><span class="mspace" style="margin-right:0.2778em;"></span></span><span class="base"><span class="strut" style="height:1em;vertical-align:-0.25em;"></span><span class="mord mathnormal" style="margin-right:0.13889em;">P</span><span class="mopen">(</span><span class="mord">\u2018\u2018</span><span class="mord mathnormal">p</span><span class="mord">&quot;</span><span class="mclose">)</span><span class="mspace" style="margin-right:0.2222em;"></span><span class="mbin">\xD7</span><span class="mspace" style="margin-right:0.2222em;"></span></span><span class="base"><span class="strut" style="height:1em;vertical-align:-0.25em;"></span><span class="mord mathnormal" style="margin-right:0.13889em;">P</span><span class="mopen">(</span><span class="mord">\u2018\u2018</span><span class="mord mathnormal">u</span><span class="mord">&quot;</span><span class="mclose">)</span><span class="mspace" style="margin-right:0.2222em;"></span><span class="mbin">\xD7</span><span class="mspace" style="margin-right:0.2222em;"></span></span><span class="base"><span class="strut" style="height:1em;vertical-align:-0.25em;"></span><span class="mord mathnormal" style="margin-right:0.13889em;">P</span><span class="mopen">(</span><span class="mord">\u2018\u2018</span><span class="mord mathnormal" style="margin-right:0.03588em;">g</span><span class="mord">&quot;</span><span class="mclose">)</span><span class="mspace" style="margin-right:0.2778em;"></span><span class="mrel">=</span><span class="mspace" style="margin-right:0.2778em;"></span></span><span class="base"><span class="strut" style="height:2.0074em;vertical-align:-0.686em;"></span><span class="mord"><span class="mopen nulldelimiter"></span><span class="mfrac"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:1.3214em;"><span style="top:-2.314em;"><span class="pstrut" style="height:3em;"></span><span class="mord"><span class="mord">210</span></span></span><span style="top:-3.23em;"><span class="pstrut" style="height:3em;"></span><span class="frac-line" style="border-bottom-width:0.04em;"></span></span><span style="top:-3.677em;"><span class="pstrut" style="height:3em;"></span><span class="mord"><span class="mord">5</span></span></span></span><span class="vlist-s">\u200B</span></span><span class="vlist-r"><span class="vlist" style="height:0.686em;"><span></span></span></span></span></span><span class="mclose nulldelimiter"></span></span><span class="mspace" style="margin-right:0.2222em;"></span><span class="mbin">\xD7</span><span class="mspace" style="margin-right:0.2222em;"></span></span><span class="base"><span class="strut" style="height:2.0074em;vertical-align:-0.686em;"></span><span class="mord"><span class="mopen nulldelimiter"></span><span class="mfrac"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:1.3214em;"><span style="top:-2.314em;"><span class="pstrut" style="height:3em;"></span><span class="mord"><span class="mord">210</span></span></span><span style="top:-3.23em;"><span class="pstrut" style="height:3em;"></span><span class="frac-line" style="border-bottom-width:0.04em;"></span></span><span style="top:-3.677em;"><span class="pstrut" style="height:3em;"></span><span class="mord"><span class="mord">36</span></span></span></span><span class="vlist-s">\u200B</span></span><span class="vlist-r"><span class="vlist" style="height:0.686em;"><span></span></span></span></span></span><span class="mclose nulldelimiter"></span></span><span class="mspace" style="margin-right:0.2222em;"></span><span class="mbin">\xD7</span><span class="mspace" style="margin-right:0.2222em;"></span></span><span class="base"><span class="strut" style="height:2.0074em;vertical-align:-0.686em;"></span><span class="mord"><span class="mopen nulldelimiter"></span><span class="mfrac"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:1.3214em;"><span style="top:-2.314em;"><span class="pstrut" style="height:3em;"></span><span class="mord"><span class="mord">210</span></span></span><span style="top:-3.23em;"><span class="pstrut" style="height:3em;"></span><span class="frac-line" style="border-bottom-width:0.04em;"></span></span><span style="top:-3.677em;"><span class="pstrut" style="height:3em;"></span><span class="mord"><span class="mord">20</span></span></span></span><span class="vlist-s">\u200B</span></span><span class="vlist-r"><span class="vlist" style="height:0.686em;"><span></span></span></span></span></span><span class="mclose nulldelimiter"></span></span><span class="mspace" style="margin-right:0.2778em;"></span><span class="mrel">=</span><span class="mspace" style="margin-right:0.2778em;"></span></span><span class="base"><span class="strut" style="height:0.6444em;"></span><span class="mord">0.000389</span></span></span></span></span>',nn,us,sr,ya,er,ar,ln,Xm='<span class="katex-display"><span class="katex"><span class="katex-mathml"><math xmlns="http://www.w3.org/1998/Math/MathML" display="block"><semantics><mrow><mi>P</mi><mo stretchy="false">(</mo><mo stretchy="false">[</mo><mi mathvariant="normal">\u2018</mi><mi mathvariant="normal">\u2018</mi><mi>p</mi><mi>u</mi><mi mathvariant="normal">&quot;</mi><mo separator="true">,</mo><mi mathvariant="normal">\u2018</mi><mi mathvariant="normal">\u2018</mi><mi>g</mi><mi mathvariant="normal">&quot;</mi><mo stretchy="false">]</mo><mo stretchy="false">)</mo><mo>=</mo><mi>P</mi><mo stretchy="false">(</mo><mi mathvariant="normal">\u2018</mi><mi mathvariant="normal">\u2018</mi><mi>p</mi><mi>u</mi><mi mathvariant="normal">&quot;</mi><mo stretchy="false">)</mo><mo>\xD7</mo><mi>P</mi><mo stretchy="false">(</mo><mi mathvariant="normal">\u2018</mi><mi mathvariant="normal">\u2018</mi><mi>g</mi><mi mathvariant="normal">&quot;</mi><mo stretchy="false">)</mo><mo>=</mo><mfrac><mn>5</mn><mn>210</mn></mfrac><mo>\xD7</mo><mfrac><mn>20</mn><mn>210</mn></mfrac><mo>=</mo><mn>0.0022676</mn></mrow><annotation encoding="application/x-tex">P([``pu&quot;, ``g&quot;]) = P(``pu&quot;) \\times P(``g&quot;) = \\frac{5}{210} \\times \\frac{20}{210} = 0.0022676</annotation></semantics></math></span><span class="katex-html" aria-hidden="true"><span class="base"><span class="strut" style="height:1em;vertical-align:-0.25em;"></span><span class="mord mathnormal" style="margin-right:0.13889em;">P</span><span class="mopen">([</span><span class="mord">\u2018\u2018</span><span class="mord mathnormal">p</span><span class="mord mathnormal">u</span><span class="mord">&quot;</span><span class="mpunct">,</span><span class="mspace" style="margin-right:0.1667em;"></span><span class="mord">\u2018\u2018</span><span class="mord mathnormal" style="margin-right:0.03588em;">g</span><span class="mord">&quot;</span><span class="mclose">])</span><span class="mspace" style="margin-right:0.2778em;"></span><span class="mrel">=</span><span class="mspace" style="margin-right:0.2778em;"></span></span><span class="base"><span class="strut" style="height:1em;vertical-align:-0.25em;"></span><span class="mord mathnormal" style="margin-right:0.13889em;">P</span><span class="mopen">(</span><span class="mord">\u2018\u2018</span><span class="mord mathnormal">p</span><span class="mord mathnormal">u</span><span class="mord">&quot;</span><span class="mclose">)</span><span class="mspace" style="margin-right:0.2222em;"></span><span class="mbin">\xD7</span><span class="mspace" style="margin-right:0.2222em;"></span></span><span class="base"><span class="strut" style="height:1em;vertical-align:-0.25em;"></span><span class="mord mathnormal" style="margin-right:0.13889em;">P</span><span class="mopen">(</span><span class="mord">\u2018\u2018</span><span class="mord mathnormal" style="margin-right:0.03588em;">g</span><span class="mord">&quot;</span><span class="mclose">)</span><span class="mspace" style="margin-right:0.2778em;"></span><span class="mrel">=</span><span class="mspace" style="margin-right:0.2778em;"></span></span><span class="base"><span class="strut" style="height:2.0074em;vertical-align:-0.686em;"></span><span class="mord"><span class="mopen nulldelimiter"></span><span class="mfrac"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:1.3214em;"><span style="top:-2.314em;"><span class="pstrut" style="height:3em;"></span><span class="mord"><span class="mord">210</span></span></span><span style="top:-3.23em;"><span class="pstrut" style="height:3em;"></span><span class="frac-line" style="border-bottom-width:0.04em;"></span></span><span style="top:-3.677em;"><span class="pstrut" style="height:3em;"></span><span class="mord"><span class="mord">5</span></span></span></span><span class="vlist-s">\u200B</span></span><span class="vlist-r"><span class="vlist" style="height:0.686em;"><span></span></span></span></span></span><span class="mclose nulldelimiter"></span></span><span class="mspace" style="margin-right:0.2222em;"></span><span class="mbin">\xD7</span><span class="mspace" style="margin-right:0.2222em;"></span></span><span class="base"><span class="strut" style="height:2.0074em;vertical-align:-0.686em;"></span><span class="mord"><span class="mopen nulldelimiter"></span><span class="mfrac"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:1.3214em;"><span style="top:-2.314em;"><span class="pstrut" style="height:3em;"></span><span class="mord"><span class="mord">210</span></span></span><span style="top:-3.23em;"><span class="pstrut" style="height:3em;"></span><span class="frac-line" style="border-bottom-width:0.04em;"></span></span><span style="top:-3.677em;"><span class="pstrut" style="height:3em;"></span><span class="mord"><span class="mord">20</span></span></span></span><span class="vlist-s">\u200B</span></span><span class="vlist-r"><span class="vlist" style="height:0.686em;"><span></span></span></span></span></span><span class="mclose nulldelimiter"></span></span><span class="mspace" style="margin-right:0.2778em;"></span><span class="mrel">=</span><span class="mspace" style="margin-right:0.2778em;"></span></span><span class="base"><span class="strut" style="height:0.6444em;"></span><span class="mord">0.0022676</span></span></span></span></span>',on,F,tr,xa,nr,lr,Pa,or,rr,Ca,pr,ir,rn,Q,ur,za,mr,cr,Ma,dr,hr,pn,Xs,un,W,fr,Aa,gr,vr,Da,br,qr,Na,_r,jr,mn,T,kr,Ta,$r,wr,Ua,Er,yr,Oa,xr,Pr,Sa,Cr,zr,Ha,Mr,Ar,cn,Ee,Dr,dn,_s,Nr,Ia,Tr,Ur,hn,Fs,fn,ss,Or,La,Sr,Hr,Ba,Ir,Lr,gn,js,vn,ms,ks,Va,Ws,Br,Ra,Vr,bn,es,Rr,Xa,Xr,Fr,Fa,Wr,Gr,qn,$s,Yr,Wa,Zr,Jr,_n,ye,Kr,jn,Gs,kn,xe,Qr,$n,Ys,wn,Pe,sp,En,Zs,yn,H,ep,Ga,ap,tp,Ya,np,lp,Za,op,rp,Ja,pp,ip,xn,G,up,Ka,mp,cp,Qa,dp,hp,st,fp,gp,Pn,Js,Cn,Ce,vp,zn,Ks,Mn,as,bp,et,qp,_p,at,jp,kp,An,cs,ws,tt,Qs,$p,ze,wp,nt,Ep,Dn,ts,yp,lt,xp,Pp,ot,Cp,zp,Nn,Me,Mp,Tn,se,Un,Es,Ap,rt,Dp,Np,On,ee,Sn,ys,Tp,pt,Up,Op,Hn,ae,In,Ae,Sp,Ln,te,Bn,ne,Vn,De,Hp,Rn,le,Xn,xs,Fn,Ne,Ip,Wn,oe,Gn,Y,Lp,it,Bp,Vp,ut,Rp,Xp,mt,Fp,Wp,Yn,Ps,Gp,ct,Yp,Zp,Zn,Cs,Jp,dt,Kp,Qp,Jn,re,Kn,Te,si,Qn,pe,sl,ie,el,Ue,ei,al,ue,tl,Oe,ai,nl,me,ll,ce,ol,ns,ti,ht,ni,li,ft,oi,ri,rl,de,pl,zs,pi,gt,ii,ui,il,he,ul,U,mi,vt,ci,di,bt,hi,fi,qt,gi,vi,_t,bi,qi,jt,_i,ji,ml,fe,cl,Ms,dl,ls,ki,kt,$i,wi,$t,Ei,yi,hl,ge,fl,As,xi,wt,Pi,Ci,gl,ve,vl,be,bl,I,zi,Et,Mi,Ai,yt,Di,Ni,xt,Ti,Ui,Pt,Oi,Si,ql;return _=new Nt({}),A=new Km({props:{classNames:"absolute z-10 right-0 top-0",options:[{label:"Google Colab",value:"https://colab.research.google.com/github/huggingface/notebooks/blob/master/course/chapter6/section7.ipynb"},{label:"Aws Studio",value:"https://studiolab.sagemaker.aws/import/github/huggingface/notebooks/blob/master/course/chapter6/section7.ipynb"}]}}),Hs=new Jm({props:{id:"TGZfZVuF9Yc"}}),ds=new Dt({props:{$$slots:{default:[Qm]},$$scope:{ctx:R}}}),Is=new Nt({}),Ls=new k({props:{code:'("hug", 10), ("pug", 5), ("pun", 12), ("bun", 4), ("hugs", 5) # "c\xE2lin", "carlin", "jeu de mots", "brioche", "c\xE2lins"...',highlighted:'(<span class="hljs-string">&quot;hug&quot;</span>, <span class="hljs-number">10</span>), (<span class="hljs-string">&quot;pug&quot;</span>, <span class="hljs-number">5</span>), (<span class="hljs-string">&quot;pun&quot;</span>, <span class="hljs-number">12</span>), (<span class="hljs-string">&quot;bun&quot;</span>, <span class="hljs-number">4</span>), (<span class="hljs-string">&quot;hugs&quot;</span>, <span class="hljs-number">5</span>) <span class="hljs-meta"># <span class="hljs-string">&quot;c\xE2lin&quot;</span>, <span class="hljs-string">&quot;carlin&quot;</span>, <span class="hljs-string">&quot;jeu de mots&quot;</span>, <span class="hljs-string">&quot;brioche&quot;</span>, <span class="hljs-string">&quot;c\xE2lins&quot;</span>...</span>'}}),Bs=new k({props:{code:'["h", "u", "g", "hu", "ug", "p", "pu", "n", "un", "b", "bu", "s", "hug", "gs", "ugs"]',highlighted:'<span class="hljs-selector-attr">[<span class="hljs-string">&quot;h&quot;</span>, <span class="hljs-string">&quot;u&quot;</span>, <span class="hljs-string">&quot;g&quot;</span>, <span class="hljs-string">&quot;hu&quot;</span>, <span class="hljs-string">&quot;ug&quot;</span>, <span class="hljs-string">&quot;p&quot;</span>, <span class="hljs-string">&quot;pu&quot;</span>, <span class="hljs-string">&quot;n&quot;</span>, <span class="hljs-string">&quot;un&quot;</span>, <span class="hljs-string">&quot;b&quot;</span>, <span class="hljs-string">&quot;bu&quot;</span>, <span class="hljs-string">&quot;s&quot;</span>, <span class="hljs-string">&quot;hug&quot;</span>, <span class="hljs-string">&quot;gs&quot;</span>, <span class="hljs-string">&quot;ugs&quot;</span>]</span>'}}),Vs=new Nt({}),Rs=new k({props:{code:`("h", 15) ("u", 36) ("g", 20) ("hu", 15) ("ug", 20) ("p", 17) ("pu", 17) ("n", 16)
("un", 16) ("b", 4) ("bu", 4) ("s", 5) ("hug", 15) ("gs", 5) ("ugs", 5)`,highlighted:`(<span class="hljs-string">&quot;h&quot;</span><span class="hljs-punctuation">,</span> <span class="hljs-number">15</span>) (<span class="hljs-string">&quot;u&quot;</span><span class="hljs-punctuation">,</span> <span class="hljs-number">36</span>) (<span class="hljs-string">&quot;g&quot;</span><span class="hljs-punctuation">,</span> <span class="hljs-number">20</span>) (<span class="hljs-string">&quot;hu&quot;</span><span class="hljs-punctuation">,</span> <span class="hljs-number">15</span>) (<span class="hljs-string">&quot;ug&quot;</span><span class="hljs-punctuation">,</span> <span class="hljs-number">20</span>) (<span class="hljs-string">&quot;p&quot;</span><span class="hljs-punctuation">,</span> <span class="hljs-number">17</span>) (<span class="hljs-string">&quot;pu&quot;</span><span class="hljs-punctuation">,</span> <span class="hljs-number">17</span>) (<span class="hljs-string">&quot;n&quot;</span><span class="hljs-punctuation">,</span> <span class="hljs-number">16</span>)
(<span class="hljs-string">&quot;un&quot;</span><span class="hljs-punctuation">,</span> <span class="hljs-number">16</span>) (<span class="hljs-string">&quot;b&quot;</span><span class="hljs-punctuation">,</span> <span class="hljs-number">4</span>) (<span class="hljs-string">&quot;bu&quot;</span><span class="hljs-punctuation">,</span> <span class="hljs-number">4</span>) (<span class="hljs-string">&quot;s&quot;</span><span class="hljs-punctuation">,</span> <span class="hljs-number">5</span>) (<span class="hljs-string">&quot;hug&quot;</span><span class="hljs-punctuation">,</span> <span class="hljs-number">15</span>) (<span class="hljs-string">&quot;gs&quot;</span><span class="hljs-punctuation">,</span> <span class="hljs-number">5</span>) (<span class="hljs-string">&quot;ugs&quot;</span><span class="hljs-punctuation">,</span> <span class="hljs-number">5</span>)`}}),qs=new Dt({props:{$$slots:{default:[sc]},$$scope:{ctx:R}}}),Xs=new k({props:{code:`["p", "u", "g"] : 0.000389
["p", "ug"] : 0.0022676
["pu", "g"] : 0.0022676`,highlighted:`[<span class="hljs-string">&quot;p&quot;</span>, <span class="hljs-string">&quot;u&quot;</span>, <span class="hljs-string">&quot;g&quot;</span>] : 0.000389
[<span class="hljs-string">&quot;p&quot;</span>, <span class="hljs-string">&quot;ug&quot;</span>] : 0.0022676
[<span class="hljs-string">&quot;pu&quot;</span>, <span class="hljs-string">&quot;g&quot;</span>] : 0.0022676`}}),Fs=new k({props:{code:`Character 0 (u): "u" (score 0.171429)
Character 1 (n): "un" (score 0.076191)
Character 2 (h): "un" "h" (score 0.005442)
Character 3 (u): "un" "hu" (score 0.005442)
Character 4 (g): "un" "hug" (score 0.005442)`,highlighted:`<span class="hljs-attribute">Character</span> <span class="hljs-number">0</span> (u): <span class="hljs-string">&quot;u&quot;</span> (score <span class="hljs-number">0</span>.<span class="hljs-number">171429</span>)
<span class="hljs-attribute">Character</span> <span class="hljs-number">1</span> (n): <span class="hljs-string">&quot;un&quot;</span> (score <span class="hljs-number">0</span>.<span class="hljs-number">076191</span>)
<span class="hljs-attribute">Character</span> <span class="hljs-number">2</span> (h): <span class="hljs-string">&quot;un&quot;</span> <span class="hljs-string">&quot;h&quot;</span> (score <span class="hljs-number">0</span>.<span class="hljs-number">005442</span>)
<span class="hljs-attribute">Character</span> <span class="hljs-number">3</span> (u): <span class="hljs-string">&quot;un&quot;</span> <span class="hljs-string">&quot;hu&quot;</span> (score <span class="hljs-number">0</span>.<span class="hljs-number">005442</span>)
<span class="hljs-attribute">Character</span> <span class="hljs-number">4</span> (g): <span class="hljs-string">&quot;un&quot;</span> <span class="hljs-string">&quot;hug&quot;</span> (score <span class="hljs-number">0</span>.<span class="hljs-number">005442</span>)`}}),js=new Dt({props:{$$slots:{default:[ec]},$$scope:{ctx:R}}}),Ws=new Nt({}),Gs=new k({props:{code:'("hug", 10), ("pug", 5), ("pun", 12), ("bun", 4), ("hugs", 5)',highlighted:'(<span class="hljs-string">&quot;hug&quot;</span><span class="hljs-punctuation">,</span> <span class="hljs-number">10</span>)<span class="hljs-punctuation">,</span> (<span class="hljs-string">&quot;pug&quot;</span><span class="hljs-punctuation">,</span> <span class="hljs-number">5</span>)<span class="hljs-punctuation">,</span> (<span class="hljs-string">&quot;pun&quot;</span><span class="hljs-punctuation">,</span> <span class="hljs-number">12</span>)<span class="hljs-punctuation">,</span> (<span class="hljs-string">&quot;bun&quot;</span><span class="hljs-punctuation">,</span> <span class="hljs-number">4</span>)<span class="hljs-punctuation">,</span> (<span class="hljs-string">&quot;hugs&quot;</span><span class="hljs-punctuation">,</span> <span class="hljs-number">5</span>)'}}),Ys=new k({props:{code:`"hug": ["hug"] (score 0.071428)
"pug": ["pu", "g"] (score 0.007710)
"pun": ["pu", "n"] (score 0.006168)
"bun": ["bu", "n"] (score 0.001451)
"hugs": ["hug", "s"] (score 0.001701)`,highlighted:`<span class="hljs-string">&quot;hug&quot;</span>: [<span class="hljs-string">&quot;hug&quot;</span>] <span class="hljs-comment">(score 0.071428)</span>
<span class="hljs-string">&quot;pug&quot;</span>: [<span class="hljs-string">&quot;pu&quot;</span>, <span class="hljs-string">&quot;g&quot;</span>] <span class="hljs-comment">(score 0.007710)</span>
<span class="hljs-string">&quot;pun&quot;</span>: [<span class="hljs-string">&quot;pu&quot;</span>, <span class="hljs-string">&quot;n&quot;</span>] <span class="hljs-comment">(score 0.006168)</span>
<span class="hljs-string">&quot;bun&quot;</span>: [<span class="hljs-string">&quot;bu&quot;</span>, <span class="hljs-string">&quot;n&quot;</span>] <span class="hljs-comment">(score 0.001451)</span>
<span class="hljs-string">&quot;hugs&quot;</span>: [<span class="hljs-string">&quot;hug&quot;</span>, <span class="hljs-string">&quot;s&quot;</span>] <span class="hljs-comment">(score 0.001701)</span>`}}),Zs=new k({props:{code:"10 * (-log(0.071428)) + 5 * (-log(0.007710)) + 12 * (-log(0.006168)) + 4 * (-log(0.001451)) + 5 * (-log(0.001701)) = 169.8",highlighted:'<span class="hljs-attribute">10</span> * (-log(<span class="hljs-number">0</span>.<span class="hljs-number">071428</span>)) + <span class="hljs-number">5</span> * (-log(<span class="hljs-number">0</span>.<span class="hljs-number">007710</span>)) + <span class="hljs-number">12</span> * (-log(<span class="hljs-number">0</span>.<span class="hljs-number">006168</span>)) + <span class="hljs-number">4</span> * (-log(<span class="hljs-number">0</span>.<span class="hljs-number">001451</span>)) + <span class="hljs-number">5</span> * (-log(<span class="hljs-number">0</span>.<span class="hljs-number">001701</span>)) = <span class="hljs-number">169</span>.<span class="hljs-number">8</span>'}}),Js=new k({props:{code:`"hug": ["hu", "g"] (score 0.006802)
"hugs": ["hu", "gs"] (score 0.001701)`,highlighted:`<span class="hljs-string">&quot;hug&quot;</span>: [<span class="hljs-string">&quot;hu&quot;</span>, <span class="hljs-string">&quot;g&quot;</span>] <span class="hljs-comment">(score 0.006802)</span>
<span class="hljs-string">&quot;hugs&quot;</span>: [<span class="hljs-string">&quot;hu&quot;</span>, <span class="hljs-string">&quot;gs&quot;</span>] <span class="hljs-comment">(score 0.001701)</span>`}}),Ks=new k({props:{code:"- 10 * (-log(0.071428)) + 10 * (-log(0.006802)) = 23.5",highlighted:'- <span class="hljs-number">10</span> * (<span class="hljs-name">-log</span>(<span class="hljs-number">0.071428</span>)) + <span class="hljs-number">10</span> * (<span class="hljs-name">-log</span>(<span class="hljs-number">0.006802</span>)) = <span class="hljs-number">23.5</span>'}}),Qs=new Nt({}),se=new k({props:{code:`corpus = [
    "This is the Hugging Face Course.",
    # C'est le cours d'Hugging Face.
    "This chapter is about tokenization.",
    # Ce chapitre traite de la tokenisation.
    "This section shows several tokenizer algorithms.",
    # Cette section pr\xE9sente plusieurs algorithmes de *tokenizer*.
    "Hopefully, you will be able to understand how they are trained and generate tokens.",
    # Avec un peu de chance, vous serez en mesure de comprendre comment ils sont entra\xEEn\xE9s et g\xE9n\xE8rent des *tokens*.
]`,highlighted:`corpus = [
    <span class="hljs-string">&quot;This is the Hugging Face Course.&quot;</span>,
    <span class="hljs-comment"># C&#x27;est le cours d&#x27;Hugging Face.</span>
    <span class="hljs-string">&quot;This chapter is about tokenization.&quot;</span>,
    <span class="hljs-comment"># Ce chapitre traite de la tokenisation.</span>
    <span class="hljs-string">&quot;This section shows several tokenizer algorithms.&quot;</span>,
    <span class="hljs-comment"># Cette section pr\xE9sente plusieurs algorithmes de *tokenizer*.</span>
    <span class="hljs-string">&quot;Hopefully, you will be able to understand how they are trained and generate tokens.&quot;</span>,
    <span class="hljs-comment"># Avec un peu de chance, vous serez en mesure de comprendre comment ils sont entra\xEEn\xE9s et g\xE9n\xE8rent des *tokens*.</span>
]`}}),ee=new k({props:{code:`from transformers import AutoTokenizer

tokenizer = AutoTokenizer.from_pretrained("xlnet-base-cased")`,highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoTokenizer

tokenizer = AutoTokenizer.from_pretrained(<span class="hljs-string">&quot;xlnet-base-cased&quot;</span>)`}}),ae=new k({props:{code:`from collections import defaultdict

word_freqs = defaultdict(int)
for text in corpus:
    words_with_offsets = tokenizer.backend_tokenizer.pre_tokenizer.pre_tokenize_str(text)
    new_words = [word for word, offset in words_with_offsets]
    for word in new_words:
        word_freqs[word] += 1

word_freqs`,highlighted:`<span class="hljs-keyword">from</span> collections <span class="hljs-keyword">import</span> defaultdict

word_freqs = defaultdict(<span class="hljs-built_in">int</span>)
<span class="hljs-keyword">for</span> text <span class="hljs-keyword">in</span> corpus:
    words_with_offsets = tokenizer.backend_tokenizer.pre_tokenizer.pre_tokenize_str(text)
    new_words = [word <span class="hljs-keyword">for</span> word, offset <span class="hljs-keyword">in</span> words_with_offsets]
    <span class="hljs-keyword">for</span> word <span class="hljs-keyword">in</span> new_words:
        word_freqs[word] += <span class="hljs-number">1</span>

word_freqs`}}),te=new k({props:{code:`char_freqs = defaultdict(int)
subwords_freqs = defaultdict(int)
for word, freq in word_freqs.items():
    for i in range(len(word)):
        char_freqs[word[i]] += freq
        # Boucle \xE0 travers les sous-mots de longueur au moins \xE9gale \xE0 2
        for j in range(i + 2, len(word) + 1):
            subwords_freqs[word[i:j]] += freq

# Trier les sous-mots par fr\xE9quence
sorted_subwords = sorted(subwords_freqs.items(), key=lambda x: x[1], reverse=True)
sorted_subwords[:10]`,highlighted:`char_freqs = defaultdict(<span class="hljs-built_in">int</span>)
subwords_freqs = defaultdict(<span class="hljs-built_in">int</span>)
<span class="hljs-keyword">for</span> word, freq <span class="hljs-keyword">in</span> word_freqs.items():
    <span class="hljs-keyword">for</span> i <span class="hljs-keyword">in</span> <span class="hljs-built_in">range</span>(<span class="hljs-built_in">len</span>(word)):
        char_freqs[word[i]] += freq
        <span class="hljs-comment"># Boucle \xE0 travers les sous-mots de longueur au moins \xE9gale \xE0 2</span>
        <span class="hljs-keyword">for</span> j <span class="hljs-keyword">in</span> <span class="hljs-built_in">range</span>(i + <span class="hljs-number">2</span>, <span class="hljs-built_in">len</span>(word) + <span class="hljs-number">1</span>):
            subwords_freqs[word[i:j]] += freq

<span class="hljs-comment"># Trier les sous-mots par fr\xE9quence</span>
sorted_subwords = <span class="hljs-built_in">sorted</span>(subwords_freqs.items(), key=<span class="hljs-keyword">lambda</span> x: x[<span class="hljs-number">1</span>], reverse=<span class="hljs-literal">True</span>)
sorted_subwords[:<span class="hljs-number">10</span>]`}}),ne=new k({props:{code:"[('\u2581t', 7), ('is', 5), ('er', 5), ('\u2581a', 5), ('\u2581to', 4), ('to', 4), ('en', 4), ('\u2581T', 3), ('\u2581Th', 3), ('\u2581Thi', 3)]",highlighted:'[(<span class="hljs-string">&#x27;\u2581t&#x27;</span>, <span class="hljs-number">7</span>), (<span class="hljs-string">&#x27;is&#x27;</span>, <span class="hljs-number">5</span>), (<span class="hljs-string">&#x27;er&#x27;</span>, <span class="hljs-number">5</span>), (<span class="hljs-string">&#x27;\u2581a&#x27;</span>, <span class="hljs-number">5</span>), (<span class="hljs-string">&#x27;\u2581to&#x27;</span>, <span class="hljs-number">4</span>), (<span class="hljs-string">&#x27;to&#x27;</span>, <span class="hljs-number">4</span>), (<span class="hljs-string">&#x27;en&#x27;</span>, <span class="hljs-number">4</span>), (<span class="hljs-string">&#x27;\u2581T&#x27;</span>, <span class="hljs-number">3</span>), (<span class="hljs-string">&#x27;\u2581Th&#x27;</span>, <span class="hljs-number">3</span>), (<span class="hljs-string">&#x27;\u2581Thi&#x27;</span>, <span class="hljs-number">3</span>)]'}}),le=new k({props:{code:`token_freqs = list(char_freqs.items()) + sorted_subwords[: 300 - len(char_freqs)]
token_freqs = {token: freq for token, freq in token_freqs}`,highlighted:`token_freqs = <span class="hljs-built_in">list</span>(char_freqs.items()) + sorted_subwords[: <span class="hljs-number">300</span> - <span class="hljs-built_in">len</span>(char_freqs)]
token_freqs = {token: freq <span class="hljs-keyword">for</span> token, freq <span class="hljs-keyword">in</span> token_freqs}`}}),xs=new Dt({props:{$$slots:{default:[ac]},$$scope:{ctx:R}}}),oe=new k({props:{code:`from math import log

total_sum = sum([freq for token, freq in token_freqs.items()])
model = {token: -log(freq / total_sum) for token, freq in token_freqs.items()}`,highlighted:`<span class="hljs-keyword">from</span> math <span class="hljs-keyword">import</span> log

total_sum = <span class="hljs-built_in">sum</span>([freq <span class="hljs-keyword">for</span> token, freq <span class="hljs-keyword">in</span> token_freqs.items()])
model = {token: -log(freq / total_sum) <span class="hljs-keyword">for</span> token, freq <span class="hljs-keyword">in</span> token_freqs.items()}`}}),re=new k({props:{code:`def encode_word(word, model):
    best_segmentations = [{"start": 0, "score": 1}] + [
        {"start": None, "score": None} for _ in range(len(word))
    ]
    for start_idx in range(len(word)):
        # Doit \xEAtre correctement rempli par les \xE9tapes pr\xE9c\xE9dentes de la boucle
        best_score_at_start = best_segmentations[start_idx]["score"]
        for end_idx in range(start_idx + 1, len(word) + 1):
            token = word[start_idx:end_idx]
            if token in model and best_score_at_start is not None:
                score = model[token] + best_score_at_start
                # Si nous avons trouv\xE9 une meilleure segmentation se terminant \xE0 end_idx, nous mettons \xE0 jour
                if (
                    best_segmentations[end_idx]["score"] is None
                    or best_segmentations[end_idx]["score"] > score
                ):
                    best_segmentations[end_idx] = {"start": start_idx, "score": score}

    segmentation = best_segmentations[-1]
    if segmentation["score"] is None:
        # Nous n'avons pas trouv\xE9 de tokenization du mot -> inconnu (<unk>)
        return ["<unk>"], None

    score = segmentation["score"]
    start = segmentation["start"]
    end = len(word)
    tokens = []
    while start != 0:
        tokens.insert(0, word[start:end])
        next_start = best_segmentations[start]["start"]
        end = start
        start = next_start
    tokens.insert(0, word[start:end])
    return tokens, score`,highlighted:`<span class="hljs-keyword">def</span> <span class="hljs-title function_">encode_word</span>(<span class="hljs-params">word, model</span>):
    best_segmentations = [{<span class="hljs-string">&quot;start&quot;</span>: <span class="hljs-number">0</span>, <span class="hljs-string">&quot;score&quot;</span>: <span class="hljs-number">1</span>}] + [
        {<span class="hljs-string">&quot;start&quot;</span>: <span class="hljs-literal">None</span>, <span class="hljs-string">&quot;score&quot;</span>: <span class="hljs-literal">None</span>} <span class="hljs-keyword">for</span> _ <span class="hljs-keyword">in</span> <span class="hljs-built_in">range</span>(<span class="hljs-built_in">len</span>(word))
    ]
    <span class="hljs-keyword">for</span> start_idx <span class="hljs-keyword">in</span> <span class="hljs-built_in">range</span>(<span class="hljs-built_in">len</span>(word)):
        <span class="hljs-comment"># Doit \xEAtre correctement rempli par les \xE9tapes pr\xE9c\xE9dentes de la boucle</span>
        best_score_at_start = best_segmentations[start_idx][<span class="hljs-string">&quot;score&quot;</span>]
        <span class="hljs-keyword">for</span> end_idx <span class="hljs-keyword">in</span> <span class="hljs-built_in">range</span>(start_idx + <span class="hljs-number">1</span>, <span class="hljs-built_in">len</span>(word) + <span class="hljs-number">1</span>):
            token = word[start_idx:end_idx]
            <span class="hljs-keyword">if</span> token <span class="hljs-keyword">in</span> model <span class="hljs-keyword">and</span> best_score_at_start <span class="hljs-keyword">is</span> <span class="hljs-keyword">not</span> <span class="hljs-literal">None</span>:
                score = model[token] + best_score_at_start
                <span class="hljs-comment"># Si nous avons trouv\xE9 une meilleure segmentation se terminant \xE0 end_idx, nous mettons \xE0 jour</span>
                <span class="hljs-keyword">if</span> (
                    best_segmentations[end_idx][<span class="hljs-string">&quot;score&quot;</span>] <span class="hljs-keyword">is</span> <span class="hljs-literal">None</span>
                    <span class="hljs-keyword">or</span> best_segmentations[end_idx][<span class="hljs-string">&quot;score&quot;</span>] &gt; score
                ):
                    best_segmentations[end_idx] = {<span class="hljs-string">&quot;start&quot;</span>: start_idx, <span class="hljs-string">&quot;score&quot;</span>: score}

    segmentation = best_segmentations[-<span class="hljs-number">1</span>]
    <span class="hljs-keyword">if</span> segmentation[<span class="hljs-string">&quot;score&quot;</span>] <span class="hljs-keyword">is</span> <span class="hljs-literal">None</span>:
        <span class="hljs-comment"># Nous n&#x27;avons pas trouv\xE9 de tokenization du mot -&gt; inconnu (&lt;unk&gt;)</span>
        <span class="hljs-keyword">return</span> [<span class="hljs-string">&quot;&lt;unk&gt;&quot;</span>], <span class="hljs-literal">None</span>

    score = segmentation[<span class="hljs-string">&quot;score&quot;</span>]
    start = segmentation[<span class="hljs-string">&quot;start&quot;</span>]
    end = <span class="hljs-built_in">len</span>(word)
    tokens = []
    <span class="hljs-keyword">while</span> start != <span class="hljs-number">0</span>:
        tokens.insert(<span class="hljs-number">0</span>, word[start:end])
        next_start = best_segmentations[start][<span class="hljs-string">&quot;start&quot;</span>]
        end = start
        start = next_start
    tokens.insert(<span class="hljs-number">0</span>, word[start:end])
    <span class="hljs-keyword">return</span> tokens, score`}}),pe=new k({props:{code:`print(encode_word("Hopefully", model))
print(encode_word("This", model))`,highlighted:`<span class="hljs-built_in">print</span>(encode_word(<span class="hljs-string">&quot;Hopefully&quot;</span>, model))
<span class="hljs-built_in">print</span>(encode_word(<span class="hljs-string">&quot;This&quot;</span>, model))`}}),ie=new k({props:{code:`(['H', 'o', 'p', 'e', 'f', 'u', 'll', 'y'], 41.5157494601402)
(['This'], 6.288267030694535)`,highlighted:`([<span class="hljs-string">&#x27;H&#x27;</span>, <span class="hljs-string">&#x27;o&#x27;</span>, <span class="hljs-string">&#x27;p&#x27;</span>, <span class="hljs-string">&#x27;e&#x27;</span>, <span class="hljs-string">&#x27;f&#x27;</span>, <span class="hljs-string">&#x27;u&#x27;</span>, <span class="hljs-string">&#x27;ll&#x27;</span>, <span class="hljs-string">&#x27;y&#x27;</span>], <span class="hljs-number">41.5157494601402</span>)
([<span class="hljs-string">&#x27;This&#x27;</span>], <span class="hljs-number">6.288267030694535</span>)`}}),ue=new k({props:{code:`def compute_loss(model):
    loss = 0
    for word, freq in word_freqs.items():
        _, word_loss = encode_word(word, model)
        loss += freq * word_loss
    return loss`,highlighted:`<span class="hljs-keyword">def</span> <span class="hljs-title function_">compute_loss</span>(<span class="hljs-params">model</span>):
    loss = <span class="hljs-number">0</span>
    <span class="hljs-keyword">for</span> word, freq <span class="hljs-keyword">in</span> word_freqs.items():
        _, word_loss = encode_word(word, model)
        loss += freq * word_loss
    <span class="hljs-keyword">return</span> loss`}}),me=new k({props:{code:"compute_loss(model)",highlighted:"compute_loss(model)"}}),ce=new k({props:{code:"413.10377642940875",highlighted:'<span class="hljs-number">413.10377642940875</span>'}}),de=new k({props:{code:`import copy


def compute_scores(model):
    scores = {}
    model_loss = compute_loss(model)
    for token, score in model.items():
        # Nous gardons toujours les tokens de longueur 1.
        if len(token) == 1:
            continue
        model_without_token = copy.deepcopy(model)
        _ = model_without_token.pop(token)
        scores[token] = compute_loss(model_without_token) - model_loss
    return scores`,highlighted:`<span class="hljs-keyword">import</span> copy


<span class="hljs-keyword">def</span> <span class="hljs-title function_">compute_scores</span>(<span class="hljs-params">model</span>):
    scores = {}
    model_loss = compute_loss(model)
    <span class="hljs-keyword">for</span> token, score <span class="hljs-keyword">in</span> model.items():
        <span class="hljs-comment"># Nous gardons toujours les tokens de longueur 1.</span>
        <span class="hljs-keyword">if</span> <span class="hljs-built_in">len</span>(token) == <span class="hljs-number">1</span>:
            <span class="hljs-keyword">continue</span>
        model_without_token = copy.deepcopy(model)
        _ = model_without_token.pop(token)
        scores[token] = compute_loss(model_without_token) - model_loss
    <span class="hljs-keyword">return</span> scores`}}),he=new k({props:{code:`scores = compute_scores(model)
print(scores["ll"])
print(scores["his"])`,highlighted:`scores = compute_scores(model)
<span class="hljs-built_in">print</span>(scores[<span class="hljs-string">&quot;ll&quot;</span>])
<span class="hljs-built_in">print</span>(scores[<span class="hljs-string">&quot;his&quot;</span>])`}}),fe=new k({props:{code:`6.376412403623874
0.0`,highlighted:`<span class="hljs-number">6.376412403623874</span>
<span class="hljs-number">0.0</span>`}}),Ms=new Dt({props:{$$slots:{default:[tc]},$$scope:{ctx:R}}}),ge=new k({props:{code:`percent_to_remove = 0.1
while len(model) > 100:
    scores = compute_scores(model)
    sorted_scores = sorted(scores.items(), key=lambda x: x[1])
    # Supprime les tokens percent_to_remove ayant les scores les plus bas
    for i in range(int(len(model) * percent_to_remove)):
        _ = token_freqs.pop(sorted_scores[i][0])

    total_sum = sum([freq for token, freq in token_freqs.items()])
    model = {token: -log(freq / total_sum) for token, freq in token_freqs.items()}`,highlighted:`percent_to_remove = <span class="hljs-number">0.1</span>
<span class="hljs-keyword">while</span> <span class="hljs-built_in">len</span>(model) &gt; <span class="hljs-number">100</span>:
    scores = compute_scores(model)
    sorted_scores = <span class="hljs-built_in">sorted</span>(scores.items(), key=<span class="hljs-keyword">lambda</span> x: x[<span class="hljs-number">1</span>])
    <span class="hljs-comment"># Supprime les tokens percent_to_remove ayant les scores les plus bas</span>
    <span class="hljs-keyword">for</span> i <span class="hljs-keyword">in</span> <span class="hljs-built_in">range</span>(<span class="hljs-built_in">int</span>(<span class="hljs-built_in">len</span>(model) * percent_to_remove)):
        _ = token_freqs.pop(sorted_scores[i][<span class="hljs-number">0</span>])

    total_sum = <span class="hljs-built_in">sum</span>([freq <span class="hljs-keyword">for</span> token, freq <span class="hljs-keyword">in</span> token_freqs.items()])
    model = {token: -log(freq / total_sum) <span class="hljs-keyword">for</span> token, freq <span class="hljs-keyword">in</span> token_freqs.items()}`}}),ve=new k({props:{code:`def tokenize(text, model):
    words_with_offsets = tokenizer.backend_tokenizer.pre_tokenizer.pre_tokenize_str(text)
    pre_tokenized_text = [word for word, offset in words_with_offsets]
    encoded_words = [encode_word(word, model)[0] for word in pre_tokenized_text]
    return sum(encoded_words, [])


tokenize("This is the Hugging Face course.", model)`,highlighted:`<span class="hljs-keyword">def</span> <span class="hljs-title function_">tokenize</span>(<span class="hljs-params">text, model</span>):
    words_with_offsets = tokenizer.backend_tokenizer.pre_tokenizer.pre_tokenize_str(text)
    pre_tokenized_text = [word <span class="hljs-keyword">for</span> word, offset <span class="hljs-keyword">in</span> words_with_offsets]
    encoded_words = [encode_word(word, model)[<span class="hljs-number">0</span>] <span class="hljs-keyword">for</span> word <span class="hljs-keyword">in</span> pre_tokenized_text]
    <span class="hljs-keyword">return</span> <span class="hljs-built_in">sum</span>(encoded_words, [])


tokenize(<span class="hljs-string">&quot;This is the Hugging Face course.&quot;</span>, model)`}}),be=new k({props:{code:"['\u2581This', '\u2581is', '\u2581the', '\u2581Hugging', '\u2581Face', '\u2581', 'c', 'ou', 'r', 's', 'e', '.']",highlighted:'[<span class="hljs-string">&#x27;\u2581This&#x27;</span>, <span class="hljs-string">&#x27;\u2581is&#x27;</span>, <span class="hljs-string">&#x27;\u2581the&#x27;</span>, <span class="hljs-string">&#x27;\u2581Hugging&#x27;</span>, <span class="hljs-string">&#x27;\u2581Face&#x27;</span>, <span class="hljs-string">&#x27;\u2581&#x27;</span>, <span class="hljs-string">&#x27;c&#x27;</span>, <span class="hljs-string">&#x27;ou&#x27;</span>, <span class="hljs-string">&#x27;r&#x27;</span>, <span class="hljs-string">&#x27;s&#x27;</span>, <span class="hljs-string">&#x27;e&#x27;</span>, <span class="hljs-string">&#x27;.&#x27;</span>]'}}),{c(){c=o("meta"),y=u(),q=o("h1"),$=o("a"),E=o("span"),d(_.$$.fragment),j=u(),x=o("span"),C=t("Tokenisation "),w=o("i"),B=t("Unigram"),N=u(),d(A.$$.fragment),Ss=u(),V=o("p"),_e=t("L\u2019algorithme "),Ge=o("em"),Tl=t("Unigram"),Ul=t(" est souvent utilis\xE9 dans "),Ye=o("em"),Ol=t("SentencePiece"),Sl=t(", qui est l\u2019algorithme de tokenization utilis\xE9 par des mod\xE8les comme ALBERT, T5, mBART, Big Bird et XLNet."),Tt=u(),d(Hs.$$.fragment),Ut=u(),d(ds.$$.fragment),Ot=u(),ps=o("h2"),hs=o("a"),Ze=o("span"),d(Is.$$.fragment),Hl=u(),Je=o("span"),Il=t("Algorithme d'entra\xEEnement"),St=u(),X=o("p"),Ll=t("Compar\xE9 au BPE et "),Ke=o("em"),Bl=t("WordPiece"),Vl=t(", "),Qe=o("em"),Rl=t("Unigram"),Xl=t(" fonctionne dans l\u2019autre sens : il part d\u2019un grand vocabulaire et enl\xE8ve des "),sa=o("em"),Fl=t("tokens"),Wl=t(" jusqu\u2019\xE0 atteindre la taille de vocabulaire d\xE9sir\xE9e. Il existe plusieurs options pour construire ce vocabulaire de base. Nous pouvons par exemple prendre les sous-cha\xEEnes les plus courantes dans les mots pr\xE9tok\xE9nis\xE9s ou appliquer le BPE sur le corpus initial avec une grande taille de vocabulaire."),Ht=u(),fs=o("p"),Gl=t("\xC0 chaque \xE9tape de l\u2019entra\xEEnement, l\u2019algorithme "),ea=o("em"),Yl=t("Unigram"),Zl=t(" calcule une perte sur le corpus compte tenu du vocabulaire actuel. Ensuite, pour chaque symbole du vocabulaire, l\u2019algorithme calcule de combien la perte globale augmenterait si le symbole \xE9tait supprim\xE9 et recherche les symboles qui l\u2019augmenteraient le moins. Ces symboles ont un effet moindre sur la perte globale du corpus, ils sont donc en quelque sorte \xAB moins n\xE9cessaires \xBB et sont les meilleurs candidats \xE0 la suppression."),It=u(),gs=o("p"),Jl=t("Comme il s\u2019agit d\u2019une op\xE9ration tr\xE8s co\xFBteuse, nous ne nous contentons pas de supprimer le symbole unique associ\xE9 \xE0 la plus faible augmentation de la perte mais le "),Lt=new Ii,Bt=t(" pourcent des symboles associ\xE9s \xE0 la plus faible augmentation de la perte. (p\\) est un hyperparam\xE8tre que vous pouvez contr\xF4ler, valant g\xE9n\xE9ralement 10 ou 20. Ce processus est ensuite r\xE9p\xE9t\xE9 jusqu\u2019\xE0 ce que le vocabulaire ait atteint la taille souhait\xE9e."),Vt=u(),je=o("p"),Kl=t("Notez que nous ne supprimons jamais les caract\xE8res de base, afin de nous assurer que tout mot peut \xEAtre tokenis\xE9."),Rt=u(),K=o("p"),Ql=t("Tout ceci peut para\xEEtre encore un peu vague. En effet, la partie principale de l\u2019algorithme est de calculer une perte sur le corpus et de voir comment elle change lorsque nous supprimons certains "),aa=o("em"),so=t("tokens"),eo=t(" du vocabulaire mais nous n\u2019avons pas encore expliqu\xE9 comment le faire. Cette \xE9tape repose sur l\u2019algorithme de tok\xE9nisation "),ta=o("em"),ao=t("Unigram"),to=t(", nous allons donc l\u2019aborder \xE0 pr\xE9sent."),Xt=u(),ke=o("p"),no=t("Nous allons r\xE9utiliser le corpus des exemples pr\xE9c\xE9dents :"),Ft=u(),d(Ls.$$.fragment),Wt=u(),$e=o("p"),lo=t("et pour cet exemple, nous prendrons toutes les sous-cha\xEEnes strictes pour le vocabulaire initial :"),Gt=u(),d(Bs.$$.fragment),Yt=u(),is=o("h2"),vs=o("a"),na=o("span"),d(Vs.$$.fragment),oo=u(),la=o("span"),ro=t("Algorithme de tokenisation"),Zt=u(),z=o("p"),po=t("Un mod\xE8le "),oa=o("em"),io=t("Unigram"),uo=t(" est un type de mod\xE8le de langage qui consid\xE8re que chaque "),ra=o("em"),mo=t("token"),co=t(" est ind\xE9pendant des "),pa=o("em"),ho=t("tokens"),fo=t(" qui le pr\xE9c\xE8dent. Il s\u2019agit du mod\xE8le de langage le plus simple, dans le sens o\xF9 la probabilit\xE9 du "),ia=o("em"),go=t("token"),vo=t(" X compte tenu du contexte pr\xE9c\xE9dent est simplement la probabilit\xE9 du "),ua=o("em"),bo=t("token"),qo=t(" X. Ainsi, si nous utilisions un mod\xE8le de langage "),ma=o("em"),_o=t("Unigram"),jo=t(" pour g\xE9n\xE9rer du texte, nous pr\xE9dirions toujours le "),ca=o("em"),ko=t("token"),$o=t(" le plus courant."),Jt=u(),D=o("p"),wo=t("La probabilit\xE9 d\u2019un "),da=o("em"),Eo=t("token"),yo=t(" donn\xE9 est sa fr\xE9quence (le nombre de fois que nous le trouvons) dans le corpus original, divis\xE9e par la somme de toutes les fr\xE9quences de tous les "),ha=o("em"),xo=t("tokens"),Po=t(" dans le vocabulaire (pour s\u2019assurer que la somme des probabilit\xE9s est \xE9gale \xE0 1). Par exemple, "),fa=o("code"),Co=t('"ug"'),zo=t(" est pr\xE9sent dans "),ga=o("code"),Mo=t('"hug"'),Ao=t(", "),va=o("code"),Do=t('"pug"'),No=t(", et "),ba=o("code"),To=t('"hugs"'),Uo=t(". Il a donc une fr\xE9quence de 20 dans notre corpus."),Kt=u(),we=o("p"),Oo=t("Voici les fr\xE9quences de tous les sous-mots possibles dans le vocabulaire :"),Qt=u(),d(Rs.$$.fragment),sn=u(),bs=o("p"),So=t("Ainsi, la somme de toutes les fr\xE9quences est de 210 et la probabilit\xE9 du sous-mot "),qa=o("code"),Ho=t('"ug"'),Io=t(" est donc de 20/210."),en=u(),d(qs.$$.fragment),an=u(),M=o("p"),Lo=t("Maintenant, pour tokeniser un mot donn\xE9, nous examinons toutes les segmentations possibles en "),_a=o("em"),Bo=t("tokens"),Vo=t(" et calculons la probabilit\xE9 de chacune d\u2019entre elles selon le mod\xE8le "),ja=o("em"),Ro=t("Unigram"),Xo=t(". Puisque tous les "),ka=o("em"),Fo=t("tokens"),Wo=t(" sont consid\xE9r\xE9s comme ind\xE9pendants, cette probabilit\xE9 est juste le produit de la probabilit\xE9 de chaque "),$a=o("em"),Go=t("token"),Yo=t(". Par exemple, la tokenisation "),wa=o("code"),Zo=t('["p", "u", "g"]'),Jo=t(" de "),Ea=o("code"),Ko=t('"pug"'),Qo=t(` a la probabilit\xE9 :
`),tn=new Ii,nn=u(),us=o("p"),sr=t("Comparativement, la tokenization "),ya=o("code"),er=t('["pu", "g"]'),ar=t(` a la probabilit\xE9 :
`),ln=new Ii,on=u(),F=o("p"),tr=t("donc celle-l\xE0 est beaucoup plus probable. En g\xE9n\xE9ral, les tok\xE9nisations comportant le moins de "),xa=o("em"),nr=t("tokens"),lr=t(" possible auront la probabilit\xE9 la plus \xE9lev\xE9e (en raison de la division par 210 r\xE9p\xE9t\xE9e pour chaque "),Pa=o("em"),or=t("token"),rr=t("), ce qui correspond \xE0 ce que nous voulons intuitivement : diviser un mot en un nombre de "),Ca=o("em"),pr=t("tokens"),ir=t(" le plus faible possible."),rn=u(),Q=o("p"),ur=t("La tokenisation d\u2019un mot avec le mod\xE8le "),za=o("em"),mr=t("Unigram"),cr=t(" est donc la tokenisation avec la plus haute probabilit\xE9. Dans l\u2019exemple de "),Ma=o("code"),dr=t('"pug"'),hr=t(", voici les probabilit\xE9s que nous obtiendrions pour chaque segmentation possible :"),pn=u(),d(Xs.$$.fragment),un=u(),W=o("p"),fr=t("Ainsi, "),Aa=o("code"),gr=t('"pug"'),vr=t(" sera tokenis\xE9 comme "),Da=o("code"),br=t('["p", "ug"]'),qr=t(" ou "),Na=o("code"),_r=t('["pu", "g"]'),jr=t(", selon la segmentation rencontr\xE9e en premier (notez que dans un corpus plus large, les cas d\u2019\xE9galit\xE9 comme celui-ci seront rares)."),mn=u(),T=o("p"),kr=t("Dans ce cas-ci, cela a \xE9t\xE9 facile de trouver toutes les segmentations possibles et de calculer leurs probabilit\xE9s, mais en g\xE9n\xE9ral ce sera un peu plus difficile. Il existe un algorithme classique utilis\xE9 pour cela, appel\xE9 "),Ta=o("em"),$r=t("algorithme de Viterbi"),wr=t(". Essentiellement, on peut construire un graphe pour d\xE9tecter les segmentations possibles d\u2019un mot donn\xE9 en disant qu\u2019il existe une branche du caract\xE8re "),Ua=o("em"),Er=t("a"),yr=t(" au caract\xE8re "),Oa=o("em"),xr=t("b"),Pr=t(" si le sous-mot de "),Sa=o("em"),Cr=t("a"),zr=t(" \xE0 "),Ha=o("em"),Mr=t("b"),Ar=t(" est dans le vocabulaire, et attribuer \xE0 cette branche la probabilit\xE9 du sous-mot."),cn=u(),Ee=o("p"),Dr=t("Pour trouver le chemin qui va avoir le meilleur score dans ce graphe, l\u2019algorithme de Viterbi d\xE9termine, pour chaque position dans le mot, la segmentation avec le meilleur score qui se termine \xE0 cette position. Puisque nous allons du d\xE9but \xE0 la fin, ce meilleur score peut \xEAtre trouv\xE9 en parcourant en boucle tous les sous-mots se terminant \xE0 la position actuelle, puis en utilisant le meilleur score de tokenization de la position \xE0 laquelle ce sous-mot commence. Ensuite, il suffit de d\xE9rouler le chemin emprunt\xE9 pour arriver \xE0 la fin."),dn=u(),_s=o("p"),Nr=t("Prenons un exemple en utilisant notre vocabulaire et le mot "),Ia=o("code"),Tr=t('"unhug"'),Ur=t(". Pour chaque position, les sous-mots avec les meilleurs scores se terminant l\xE0 sont les suivants :"),hn=u(),d(Fs.$$.fragment),fn=u(),ss=o("p"),Or=t("Ainsi, "),La=o("code"),Sr=t('"unhug"'),Hr=t(" serait tokenis\xE9 comme "),Ba=o("code"),Ir=t('["un", "hug"]'),Lr=t("."),gn=u(),d(js.$$.fragment),vn=u(),ms=o("h2"),ks=o("a"),Va=o("span"),d(Ws.$$.fragment),Br=u(),Ra=o("span"),Vr=t("Retour \xE0 l'entra\xEEnement"),bn=u(),es=o("p"),Rr=t("Maintenant que nous avons vu comment fonctionne la tokenisation, nous pouvons nous plonger un peu plus profond\xE9ment dans la perte utilis\xE9e pendant l\u2019entra\xEEnement. \xC0 n\u2019importe quelle \xE9tape, cette perte est calcul\xE9e en tokenisant chaque mot du corpus, en utilisant le vocabulaire courant et le mod\xE8le "),Xa=o("em"),Xr=t("Unigram"),Fr=t(" d\xE9termin\xE9 par les fr\xE9quences de chaque "),Fa=o("em"),Wr=t("token"),Gr=t(" dans le corpus (comme vu pr\xE9c\xE9demment)."),qn=u(),$s=o("p"),Yr=t("Chaque mot du corpus a un score, et la perte est le n\xE9gatif du logarithme de ces scores, c\u2019est-\xE0-dire la somme pour tous les mots du corpus de tous les "),Wa=o("code"),Zr=t("-log(P(word))"),Jr=t("."),_n=u(),ye=o("p"),Kr=t("Revenons \xE0 notre exemple avec le corpus suivant :"),jn=u(),d(Gs.$$.fragment),kn=u(),xe=o("p"),Qr=t("La tokenisation de chaque mot avec leurs scores respectifs est :"),$n=u(),d(Ys.$$.fragment),wn=u(),Pe=o("p"),sp=t("Donc la perte est :"),En=u(),d(Zs.$$.fragment),yn=u(),H=o("p"),ep=t("Maintenant, nous devons calculer comment la suppression de chaque token affecte la perte. C\u2019est plut\xF4t fastidieux, donc nous allons le faire pour deux "),Ga=o("em"),ap=t("tokens"),tp=t(" ici et garder tout le processus pour quand nous aurons du code pour nous aider. Dans ce cas (tr\xE8s) particulier, nous avions deux tokenizations \xE9quivalentes de tous les mots. Par exmeple, comme nous l\u2019avons vu pr\xE9c\xE9demment, "),Ya=o("code"),np=t('"pug"'),lp=t(" pourrait \xEAtre tokenis\xE9 en "),Za=o("code"),op=t('["p", "ug"]'),rp=t(" avec le m\xEAme score. Ainsi, enlever le token "),Ja=o("code"),pp=t('"pu"'),ip=t(" du vocabulaire donnera exactement la m\xEAme perte."),xn=u(),G=o("p"),up=t("D\u2019un autre c\xF4t\xE9, supprimer le mot "),Ka=o("code"),mp=t('"hug"'),cp=t(" aggravera la perte, car la tokenisation de "),Qa=o("code"),dp=t('"hug"'),hp=t(" et "),st=o("code"),fp=t('"hugs"'),gp=t(" deviendra :"),Pn=u(),d(Js.$$.fragment),Cn=u(),Ce=o("p"),vp=t("Ces changements entra\xEEneront une augmentation de la perte de :"),zn=u(),d(Ks.$$.fragment),Mn=u(),as=o("p"),bp=t("Par cons\xE9quent, le token "),et=o("code"),qp=t('"pu"'),_p=t(" sera probablement retir\xE9 du vocabulaire, mais pas "),at=o("code"),jp=t('"hug"'),kp=t("."),An=u(),cs=o("h2"),ws=o("a"),tt=o("span"),d(Qs.$$.fragment),$p=u(),ze=o("span"),wp=t("Impl\xE9mentation d'"),nt=o("i"),Ep=t("Unigram"),Dn=u(),ts=o("p"),yp=t("Maintenant, impl\xE9mentons tout ce que nous avons vu jusqu\u2019\xE0 pr\xE9sent dans le code. Comme pour le BPE et "),lt=o("em"),xp=t("WordPiece"),Pp=t(", ce n\u2019est pas une impl\xE9mentation efficace de l\u2019algorithme "),ot=o("em"),Cp=t("Unigram"),zp=t(" (bien au contraire), mais elle devrait vous aider \xE0 le comprendre un peu mieux."),Nn=u(),Me=o("p"),Mp=t("Nous allons utiliser le m\xEAme corpus que pr\xE9c\xE9demment comme exemple :"),Tn=u(),d(se.$$.fragment),Un=u(),Es=o("p"),Ap=t("Cette fois, nous allons utiliser "),rt=o("code"),Dp=t("xlnet-base-cased"),Np=t(" comme mod\xE8le :"),On=u(),d(ee.$$.fragment),Sn=u(),ys=o("p"),Tp=t("Comme pour le BPE et "),pt=o("em"),Up=t("WordPiece"),Op=t(", nous commen\xE7ons par compter le nombre d\u2019occurrences de chaque mot dans le corpus :"),Hn=u(),d(ae.$$.fragment),In=u(),Ae=o("p"),Sp=t("Ensuite, nous devons initialiser notre vocabulaire \xE0 une taille plus grande que celle du vocabulaire que nous voudrons \xE0 la fin. Nous devons inclure tous les caract\xE8res de base (sinon nous ne serons pas en mesure de tokeniser chaque mot), mais pour les sous-cha\xEEnes plus grandes, nous ne garderons que les plus communs. AInsi nous les trions par fr\xE9quence :"),Ln=u(),d(te.$$.fragment),Bn=u(),d(ne.$$.fragment),Vn=u(),De=o("p"),Hp=t("Nous regroupons les caract\xE8res avec les meilleurs sous-mots pour arriver \xE0 un vocabulaire initial de taille 300 :"),Rn=u(),d(le.$$.fragment),Xn=u(),d(xs.$$.fragment),Fn=u(),Ne=o("p"),Ip=t("Ensuite, nous calculons la somme de toutes les fr\xE9quences, pour convertir les fr\xE9quences en probabilit\xE9s. Pour notre mod\xE8le, nous allons stocker les logarithmes des probabilit\xE9s, car c\u2019est plus stable num\xE9riquement d\u2019additionner des logarithmes que de multiplier des petits nombres. Cela simplifiera aussi le calcul de la perte du mod\xE8le :"),Wn=u(),d(oe.$$.fragment),Gn=u(),Y=o("p"),Lp=t("Maintenant la fonction principale est celle qui tokenise les mots en utilisant l\u2019algorithme de Viterbi. Comme nous l\u2019avons vu pr\xE9c\xE9demment, cet algorithme calcule la meilleure segmentation de chaque sous-cha\xEEne du mot que nous allons stocker dans une variable nomm\xE9e "),it=o("code"),Bp=t("best_segmentations"),Vp=t(". Nous allons stocker un dictionnaire par position dans le mot (de 0 \xE0 sa longueur totale), avec deux cl\xE9s : l\u2019index du d\xE9but du dernier "),ut=o("em"),Rp=t("token"),Xp=t(" dans la meilleure segmentation et le score de la meilleure segmentation. Avec l\u2019index du d\xE9but du dernier "),mt=o("em"),Fp=t("token"),Wp=t(", nous serons en mesure de r\xE9cup\xE9rer la segmentation compl\xE8te une fois que la liste est compl\xE8tement remplie."),Yn=u(),Ps=o("p"),Gp=t("Le remplissage de la liste se fait \xE0 l\u2019aide de deux boucles seulement : la boucle principale passe en revue chaque position de d\xE9part et la seconde boucle essaie toutes les sous-cha\xEEnes commen\xE7ant \xE0 cette position de d\xE9part. Si la sous-cha\xEEne est dans le vocabulaire, nous avons une nouvelle segmentation du mot jusqu\u2019\xE0 cette position finale que nous comparons \xE0 ce qui est dans "),ct=o("code"),Yp=t("best_segmentations"),Zp=t("."),Zn=u(),Cs=o("p"),Jp=t("Une fois que la boucle principale est termin\xE9e, nous commen\xE7ons juste \xE0 la fin et sautons d\u2019une position de d\xE9part \xE0 une autre, en enregistrant les "),dt=o("em"),Kp=t("tokens"),Qp=t(" au fur et \xE0 mesure, jusqu\u2019\xE0 ce que nous atteignions le d\xE9but du mot :"),Jn=u(),d(re.$$.fragment),Kn=u(),Te=o("p"),si=t("Nous pouvons d\xE9j\xE0 essayer notre mod\xE8le initial sur quelques mots :"),Qn=u(),d(pe.$$.fragment),sl=u(),d(ie.$$.fragment),el=u(),Ue=o("p"),ei=t("Il est maintenant facile de calculer la perte du mod\xE8le sur le corpus !"),al=u(),d(ue.$$.fragment),tl=u(),Oe=o("p"),ai=t("Nous pouvons v\xE9rifier que cela fonctionne sur le mod\xE8le que nous avons :"),nl=u(),d(me.$$.fragment),ll=u(),d(ce.$$.fragment),ol=u(),ns=o("p"),ti=t("Le calcul des scores pour chaque "),ht=o("em"),ni=t("token"),li=t(" n\u2019est pas tr\xE8s difficile non plus. Il suffit de calculer la perte pour les mod\xE8les obtenus en supprimant chaque "),ft=o("em"),oi=t("token"),ri=t(" :"),rl=u(),d(de.$$.fragment),pl=u(),zs=o("p"),pi=t("Nous pouvons l\u2019essayer sur un "),gt=o("em"),ii=t("token"),ui=t(" donn\xE9 :"),il=u(),d(he.$$.fragment),ul=u(),U=o("p"),mi=t("Puisque "),vt=o("code"),ci=t('"ll"'),di=t(" est utilis\xE9 dans la tokenisation de "),bt=o("code"),hi=t('"Hopefully"'),fi=t(", et que le supprimer nous fera probablement utiliser le token "),qt=o("code"),gi=t('"l"'),vi=t(" deux fois \xE0 la place, nous nous attendons \xE0 ce qu\u2019il ait une perte positive. "),_t=o("code"),bi=t('"his"'),qi=t(" n\u2019est utilis\xE9 qu\u2019\xE0 l\u2019int\xE9rieur du mot "),jt=o("code"),_i=t('"This"'),ji=t(", qui est tokenis\xE9 comme lui-m\xEAme, donc nous nous attendons \xE0 ce qu\u2019il ait une perte nulle. Voici les r\xE9sultats :"),ml=u(),d(fe.$$.fragment),cl=u(),d(Ms.$$.fragment),dl=u(),ls=o("p"),ki=t("Une fois tout cela en place, la derni\xE8re chose \xE0 faire est d\u2019ajouter les "),kt=o("em"),$i=t("tokens"),wi=t(" sp\xE9ciaux utilis\xE9s par le mod\xE8le au vocabulaire, puis de boucler jusqu\u2019\xE0 ce que nous ayons \xE9lagu\xE9 suffisamment de "),$t=o("em"),Ei=t("tokens"),yi=t(" du vocabulaire pour atteindre la taille souhait\xE9e :"),hl=u(),d(ge.$$.fragment),fl=u(),As=o("p"),xi=t("Ensuite, pour tokeniser un texte, il suffit d\u2019appliquer la pr\xE9tok\xE9nisation et d\u2019utiliser la fonction "),wt=o("code"),Pi=t("encode_word()"),Ci=t(" :"),gl=u(),d(ve.$$.fragment),vl=u(),d(be.$$.fragment),bl=u(),I=o("p"),zi=t("C\u2019est tout pour "),Et=o("em"),Mi=t("Unigram"),Ai=t(" ! Avec un peu de chance, vous vous sentez \xE0 pr\xE9sent \xEAtre un expert des "),yt=o("em"),Di=t("tokenizers"),Ni=t(". Dans la prochaine section, nous allons nous plonger dans les blocs de construction de la biblioth\xE8que \u{1F917} "),xt=o("em"),Ti=t("Tokenizers"),Ui=t(" et allons vous montrer comment vous pouvez les utiliser pour construire votre propre "),Pt=o("em"),Oi=t("tokenizer"),Si=t("."),this.h()},l(s){const l=Ym('[data-svelte="svelte-1phssyn"]',document.head);c=r(l,"META",{name:!0,content:!0}),l.forEach(a),y=m(s),q=r(s,"H1",{class:!0});var qe=p(q);$=r(qe,"A",{id:!0,class:!0,href:!0});var Ct=p($);E=r(Ct,"SPAN",{});var zt=p(E);h(_.$$.fragment,zt),zt.forEach(a),Ct.forEach(a),j=m(qe),x=r(qe,"SPAN",{});var Se=p(x);C=n(Se,"Tokenisation "),w=r(Se,"I",{});var Mt=p(w);B=n(Mt,"Unigram"),Mt.forEach(a),Se.forEach(a),qe.forEach(a),N=m(s),h(A.$$.fragment,s),Ss=m(s),V=r(s,"P",{});var He=p(V);_e=n(He,"L\u2019algorithme "),Ge=r(He,"EM",{});var Bi=p(Ge);Tl=n(Bi,"Unigram"),Bi.forEach(a),Ul=n(He," est souvent utilis\xE9 dans "),Ye=r(He,"EM",{});var Vi=p(Ye);Ol=n(Vi,"SentencePiece"),Vi.forEach(a),Sl=n(He,", qui est l\u2019algorithme de tokenization utilis\xE9 par des mod\xE8les comme ALBERT, T5, mBART, Big Bird et XLNet."),He.forEach(a),Tt=m(s),h(Hs.$$.fragment,s),Ut=m(s),h(ds.$$.fragment,s),Ot=m(s),ps=r(s,"H2",{class:!0});var _l=p(ps);hs=r(_l,"A",{id:!0,class:!0,href:!0});var Ri=p(hs);Ze=r(Ri,"SPAN",{});var Xi=p(Ze);h(Is.$$.fragment,Xi),Xi.forEach(a),Ri.forEach(a),Hl=m(_l),Je=r(_l,"SPAN",{});var Fi=p(Je);Il=n(Fi,"Algorithme d'entra\xEEnement"),Fi.forEach(a),_l.forEach(a),St=m(s),X=r(s,"P",{});var Ds=p(X);Ll=n(Ds,"Compar\xE9 au BPE et "),Ke=r(Ds,"EM",{});var Wi=p(Ke);Bl=n(Wi,"WordPiece"),Wi.forEach(a),Vl=n(Ds,", "),Qe=r(Ds,"EM",{});var Gi=p(Qe);Rl=n(Gi,"Unigram"),Gi.forEach(a),Xl=n(Ds," fonctionne dans l\u2019autre sens : il part d\u2019un grand vocabulaire et enl\xE8ve des "),sa=r(Ds,"EM",{});var Yi=p(sa);Fl=n(Yi,"tokens"),Yi.forEach(a),Wl=n(Ds," jusqu\u2019\xE0 atteindre la taille de vocabulaire d\xE9sir\xE9e. Il existe plusieurs options pour construire ce vocabulaire de base. Nous pouvons par exemple prendre les sous-cha\xEEnes les plus courantes dans les mots pr\xE9tok\xE9nis\xE9s ou appliquer le BPE sur le corpus initial avec une grande taille de vocabulaire."),Ds.forEach(a),Ht=m(s),fs=r(s,"P",{});var jl=p(fs);Gl=n(jl,"\xC0 chaque \xE9tape de l\u2019entra\xEEnement, l\u2019algorithme "),ea=r(jl,"EM",{});var Zi=p(ea);Yl=n(Zi,"Unigram"),Zi.forEach(a),Zl=n(jl," calcule une perte sur le corpus compte tenu du vocabulaire actuel. Ensuite, pour chaque symbole du vocabulaire, l\u2019algorithme calcule de combien la perte globale augmenterait si le symbole \xE9tait supprim\xE9 et recherche les symboles qui l\u2019augmenteraient le moins. Ces symboles ont un effet moindre sur la perte globale du corpus, ils sont donc en quelque sorte \xAB moins n\xE9cessaires \xBB et sont les meilleurs candidats \xE0 la suppression."),jl.forEach(a),It=m(s),gs=r(s,"P",{});var kl=p(gs);Jl=n(kl,"Comme il s\u2019agit d\u2019une op\xE9ration tr\xE8s co\xFBteuse, nous ne nous contentons pas de supprimer le symbole unique associ\xE9 \xE0 la plus faible augmentation de la perte mais le "),Lt=Li(kl),Bt=n(kl," pourcent des symboles associ\xE9s \xE0 la plus faible augmentation de la perte. (p\\) est un hyperparam\xE8tre que vous pouvez contr\xF4ler, valant g\xE9n\xE9ralement 10 ou 20. Ce processus est ensuite r\xE9p\xE9t\xE9 jusqu\u2019\xE0 ce que le vocabulaire ait atteint la taille souhait\xE9e."),kl.forEach(a),Vt=m(s),je=r(s,"P",{});var Ji=p(je);Kl=n(Ji,"Notez que nous ne supprimons jamais les caract\xE8res de base, afin de nous assurer que tout mot peut \xEAtre tokenis\xE9."),Ji.forEach(a),Rt=m(s),K=r(s,"P",{});var Ie=p(K);Ql=n(Ie,"Tout ceci peut para\xEEtre encore un peu vague. En effet, la partie principale de l\u2019algorithme est de calculer une perte sur le corpus et de voir comment elle change lorsque nous supprimons certains "),aa=r(Ie,"EM",{});var Ki=p(aa);so=n(Ki,"tokens"),Ki.forEach(a),eo=n(Ie," du vocabulaire mais nous n\u2019avons pas encore expliqu\xE9 comment le faire. Cette \xE9tape repose sur l\u2019algorithme de tok\xE9nisation "),ta=r(Ie,"EM",{});var Qi=p(ta);ao=n(Qi,"Unigram"),Qi.forEach(a),to=n(Ie,", nous allons donc l\u2019aborder \xE0 pr\xE9sent."),Ie.forEach(a),Xt=m(s),ke=r(s,"P",{});var su=p(ke);no=n(su,"Nous allons r\xE9utiliser le corpus des exemples pr\xE9c\xE9dents :"),su.forEach(a),Ft=m(s),h(Ls.$$.fragment,s),Wt=m(s),$e=r(s,"P",{});var eu=p($e);lo=n(eu,"et pour cet exemple, nous prendrons toutes les sous-cha\xEEnes strictes pour le vocabulaire initial :"),eu.forEach(a),Gt=m(s),h(Bs.$$.fragment,s),Yt=m(s),is=r(s,"H2",{class:!0});var $l=p(is);vs=r($l,"A",{id:!0,class:!0,href:!0});var au=p(vs);na=r(au,"SPAN",{});var tu=p(na);h(Vs.$$.fragment,tu),tu.forEach(a),au.forEach(a),oo=m($l),la=r($l,"SPAN",{});var nu=p(la);ro=n(nu,"Algorithme de tokenisation"),nu.forEach(a),$l.forEach(a),Zt=m(s),z=r(s,"P",{});var O=p(z);po=n(O,"Un mod\xE8le "),oa=r(O,"EM",{});var lu=p(oa);io=n(lu,"Unigram"),lu.forEach(a),uo=n(O," est un type de mod\xE8le de langage qui consid\xE8re que chaque "),ra=r(O,"EM",{});var ou=p(ra);mo=n(ou,"token"),ou.forEach(a),co=n(O," est ind\xE9pendant des "),pa=r(O,"EM",{});var ru=p(pa);ho=n(ru,"tokens"),ru.forEach(a),fo=n(O," qui le pr\xE9c\xE8dent. Il s\u2019agit du mod\xE8le de langage le plus simple, dans le sens o\xF9 la probabilit\xE9 du "),ia=r(O,"EM",{});var pu=p(ia);go=n(pu,"token"),pu.forEach(a),vo=n(O," X compte tenu du contexte pr\xE9c\xE9dent est simplement la probabilit\xE9 du "),ua=r(O,"EM",{});var iu=p(ua);bo=n(iu,"token"),iu.forEach(a),qo=n(O," X. Ainsi, si nous utilisions un mod\xE8le de langage "),ma=r(O,"EM",{});var uu=p(ma);_o=n(uu,"Unigram"),uu.forEach(a),jo=n(O," pour g\xE9n\xE9rer du texte, nous pr\xE9dirions toujours le "),ca=r(O,"EM",{});var mu=p(ca);ko=n(mu,"token"),mu.forEach(a),$o=n(O," le plus courant."),O.forEach(a),Jt=m(s),D=r(s,"P",{});var L=p(D);wo=n(L,"La probabilit\xE9 d\u2019un "),da=r(L,"EM",{});var cu=p(da);Eo=n(cu,"token"),cu.forEach(a),yo=n(L," donn\xE9 est sa fr\xE9quence (le nombre de fois que nous le trouvons) dans le corpus original, divis\xE9e par la somme de toutes les fr\xE9quences de tous les "),ha=r(L,"EM",{});var du=p(ha);xo=n(du,"tokens"),du.forEach(a),Po=n(L," dans le vocabulaire (pour s\u2019assurer que la somme des probabilit\xE9s est \xE9gale \xE0 1). Par exemple, "),fa=r(L,"CODE",{});var hu=p(fa);Co=n(hu,'"ug"'),hu.forEach(a),zo=n(L," est pr\xE9sent dans "),ga=r(L,"CODE",{});var fu=p(ga);Mo=n(fu,'"hug"'),fu.forEach(a),Ao=n(L,", "),va=r(L,"CODE",{});var gu=p(va);Do=n(gu,'"pug"'),gu.forEach(a),No=n(L,", et "),ba=r(L,"CODE",{});var vu=p(ba);To=n(vu,'"hugs"'),vu.forEach(a),Uo=n(L,". Il a donc une fr\xE9quence de 20 dans notre corpus."),L.forEach(a),Kt=m(s),we=r(s,"P",{});var bu=p(we);Oo=n(bu,"Voici les fr\xE9quences de tous les sous-mots possibles dans le vocabulaire :"),bu.forEach(a),Qt=m(s),h(Rs.$$.fragment,s),sn=m(s),bs=r(s,"P",{});var wl=p(bs);So=n(wl,"Ainsi, la somme de toutes les fr\xE9quences est de 210 et la probabilit\xE9 du sous-mot "),qa=r(wl,"CODE",{});var qu=p(qa);Ho=n(qu,'"ug"'),qu.forEach(a),Io=n(wl," est donc de 20/210."),wl.forEach(a),en=m(s),h(qs.$$.fragment,s),an=m(s),M=r(s,"P",{});var S=p(M);Lo=n(S,"Maintenant, pour tokeniser un mot donn\xE9, nous examinons toutes les segmentations possibles en "),_a=r(S,"EM",{});var _u=p(_a);Bo=n(_u,"tokens"),_u.forEach(a),Vo=n(S," et calculons la probabilit\xE9 de chacune d\u2019entre elles selon le mod\xE8le "),ja=r(S,"EM",{});var ju=p(ja);Ro=n(ju,"Unigram"),ju.forEach(a),Xo=n(S,". Puisque tous les "),ka=r(S,"EM",{});var ku=p(ka);Fo=n(ku,"tokens"),ku.forEach(a),Wo=n(S," sont consid\xE9r\xE9s comme ind\xE9pendants, cette probabilit\xE9 est juste le produit de la probabilit\xE9 de chaque "),$a=r(S,"EM",{});var $u=p($a);Go=n($u,"token"),$u.forEach(a),Yo=n(S,". Par exemple, la tokenisation "),wa=r(S,"CODE",{});var wu=p(wa);Zo=n(wu,'["p", "u", "g"]'),wu.forEach(a),Jo=n(S," de "),Ea=r(S,"CODE",{});var Eu=p(Ea);Ko=n(Eu,'"pug"'),Eu.forEach(a),Qo=n(S,` a la probabilit\xE9 :
`),tn=Li(S),S.forEach(a),nn=m(s),us=r(s,"P",{});var At=p(us);sr=n(At,"Comparativement, la tokenization "),ya=r(At,"CODE",{});var yu=p(ya);er=n(yu,'["pu", "g"]'),yu.forEach(a),ar=n(At,` a la probabilit\xE9 :
`),ln=Li(At),At.forEach(a),on=m(s),F=r(s,"P",{});var Ns=p(F);tr=n(Ns,"donc celle-l\xE0 est beaucoup plus probable. En g\xE9n\xE9ral, les tok\xE9nisations comportant le moins de "),xa=r(Ns,"EM",{});var xu=p(xa);nr=n(xu,"tokens"),xu.forEach(a),lr=n(Ns," possible auront la probabilit\xE9 la plus \xE9lev\xE9e (en raison de la division par 210 r\xE9p\xE9t\xE9e pour chaque "),Pa=r(Ns,"EM",{});var Pu=p(Pa);or=n(Pu,"token"),Pu.forEach(a),rr=n(Ns,"), ce qui correspond \xE0 ce que nous voulons intuitivement : diviser un mot en un nombre de "),Ca=r(Ns,"EM",{});var Cu=p(Ca);pr=n(Cu,"tokens"),Cu.forEach(a),ir=n(Ns," le plus faible possible."),Ns.forEach(a),rn=m(s),Q=r(s,"P",{});var Le=p(Q);ur=n(Le,"La tokenisation d\u2019un mot avec le mod\xE8le "),za=r(Le,"EM",{});var zu=p(za);mr=n(zu,"Unigram"),zu.forEach(a),cr=n(Le," est donc la tokenisation avec la plus haute probabilit\xE9. Dans l\u2019exemple de "),Ma=r(Le,"CODE",{});var Mu=p(Ma);dr=n(Mu,'"pug"'),Mu.forEach(a),hr=n(Le,", voici les probabilit\xE9s que nous obtiendrions pour chaque segmentation possible :"),Le.forEach(a),pn=m(s),h(Xs.$$.fragment,s),un=m(s),W=r(s,"P",{});var Ts=p(W);fr=n(Ts,"Ainsi, "),Aa=r(Ts,"CODE",{});var Au=p(Aa);gr=n(Au,'"pug"'),Au.forEach(a),vr=n(Ts," sera tokenis\xE9 comme "),Da=r(Ts,"CODE",{});var Du=p(Da);br=n(Du,'["p", "ug"]'),Du.forEach(a),qr=n(Ts," ou "),Na=r(Ts,"CODE",{});var Nu=p(Na);_r=n(Nu,'["pu", "g"]'),Nu.forEach(a),jr=n(Ts,", selon la segmentation rencontr\xE9e en premier (notez que dans un corpus plus large, les cas d\u2019\xE9galit\xE9 comme celui-ci seront rares)."),Ts.forEach(a),mn=m(s),T=r(s,"P",{});var Z=p(T);kr=n(Z,"Dans ce cas-ci, cela a \xE9t\xE9 facile de trouver toutes les segmentations possibles et de calculer leurs probabilit\xE9s, mais en g\xE9n\xE9ral ce sera un peu plus difficile. Il existe un algorithme classique utilis\xE9 pour cela, appel\xE9 "),Ta=r(Z,"EM",{});var Tu=p(Ta);$r=n(Tu,"algorithme de Viterbi"),Tu.forEach(a),wr=n(Z,". Essentiellement, on peut construire un graphe pour d\xE9tecter les segmentations possibles d\u2019un mot donn\xE9 en disant qu\u2019il existe une branche du caract\xE8re "),Ua=r(Z,"EM",{});var Uu=p(Ua);Er=n(Uu,"a"),Uu.forEach(a),yr=n(Z," au caract\xE8re "),Oa=r(Z,"EM",{});var Ou=p(Oa);xr=n(Ou,"b"),Ou.forEach(a),Pr=n(Z," si le sous-mot de "),Sa=r(Z,"EM",{});var Su=p(Sa);Cr=n(Su,"a"),Su.forEach(a),zr=n(Z," \xE0 "),Ha=r(Z,"EM",{});var Hu=p(Ha);Mr=n(Hu,"b"),Hu.forEach(a),Ar=n(Z," est dans le vocabulaire, et attribuer \xE0 cette branche la probabilit\xE9 du sous-mot."),Z.forEach(a),cn=m(s),Ee=r(s,"P",{});var Iu=p(Ee);Dr=n(Iu,"Pour trouver le chemin qui va avoir le meilleur score dans ce graphe, l\u2019algorithme de Viterbi d\xE9termine, pour chaque position dans le mot, la segmentation avec le meilleur score qui se termine \xE0 cette position. Puisque nous allons du d\xE9but \xE0 la fin, ce meilleur score peut \xEAtre trouv\xE9 en parcourant en boucle tous les sous-mots se terminant \xE0 la position actuelle, puis en utilisant le meilleur score de tokenization de la position \xE0 laquelle ce sous-mot commence. Ensuite, il suffit de d\xE9rouler le chemin emprunt\xE9 pour arriver \xE0 la fin."),Iu.forEach(a),dn=m(s),_s=r(s,"P",{});var El=p(_s);Nr=n(El,"Prenons un exemple en utilisant notre vocabulaire et le mot "),Ia=r(El,"CODE",{});var Lu=p(Ia);Tr=n(Lu,'"unhug"'),Lu.forEach(a),Ur=n(El,". Pour chaque position, les sous-mots avec les meilleurs scores se terminant l\xE0 sont les suivants :"),El.forEach(a),hn=m(s),h(Fs.$$.fragment,s),fn=m(s),ss=r(s,"P",{});var Be=p(ss);Or=n(Be,"Ainsi, "),La=r(Be,"CODE",{});var Bu=p(La);Sr=n(Bu,'"unhug"'),Bu.forEach(a),Hr=n(Be," serait tokenis\xE9 comme "),Ba=r(Be,"CODE",{});var Vu=p(Ba);Ir=n(Vu,'["un", "hug"]'),Vu.forEach(a),Lr=n(Be,"."),Be.forEach(a),gn=m(s),h(js.$$.fragment,s),vn=m(s),ms=r(s,"H2",{class:!0});var yl=p(ms);ks=r(yl,"A",{id:!0,class:!0,href:!0});var Ru=p(ks);Va=r(Ru,"SPAN",{});var Xu=p(Va);h(Ws.$$.fragment,Xu),Xu.forEach(a),Ru.forEach(a),Br=m(yl),Ra=r(yl,"SPAN",{});var Fu=p(Ra);Vr=n(Fu,"Retour \xE0 l'entra\xEEnement"),Fu.forEach(a),yl.forEach(a),bn=m(s),es=r(s,"P",{});var Ve=p(es);Rr=n(Ve,"Maintenant que nous avons vu comment fonctionne la tokenisation, nous pouvons nous plonger un peu plus profond\xE9ment dans la perte utilis\xE9e pendant l\u2019entra\xEEnement. \xC0 n\u2019importe quelle \xE9tape, cette perte est calcul\xE9e en tokenisant chaque mot du corpus, en utilisant le vocabulaire courant et le mod\xE8le "),Xa=r(Ve,"EM",{});var Wu=p(Xa);Xr=n(Wu,"Unigram"),Wu.forEach(a),Fr=n(Ve," d\xE9termin\xE9 par les fr\xE9quences de chaque "),Fa=r(Ve,"EM",{});var Gu=p(Fa);Wr=n(Gu,"token"),Gu.forEach(a),Gr=n(Ve," dans le corpus (comme vu pr\xE9c\xE9demment)."),Ve.forEach(a),qn=m(s),$s=r(s,"P",{});var xl=p($s);Yr=n(xl,"Chaque mot du corpus a un score, et la perte est le n\xE9gatif du logarithme de ces scores, c\u2019est-\xE0-dire la somme pour tous les mots du corpus de tous les "),Wa=r(xl,"CODE",{});var Yu=p(Wa);Zr=n(Yu,"-log(P(word))"),Yu.forEach(a),Jr=n(xl,"."),xl.forEach(a),_n=m(s),ye=r(s,"P",{});var Zu=p(ye);Kr=n(Zu,"Revenons \xE0 notre exemple avec le corpus suivant :"),Zu.forEach(a),jn=m(s),h(Gs.$$.fragment,s),kn=m(s),xe=r(s,"P",{});var Ju=p(xe);Qr=n(Ju,"La tokenisation de chaque mot avec leurs scores respectifs est :"),Ju.forEach(a),$n=m(s),h(Ys.$$.fragment,s),wn=m(s),Pe=r(s,"P",{});var Ku=p(Pe);sp=n(Ku,"Donc la perte est :"),Ku.forEach(a),En=m(s),h(Zs.$$.fragment,s),yn=m(s),H=r(s,"P",{});var os=p(H);ep=n(os,"Maintenant, nous devons calculer comment la suppression de chaque token affecte la perte. C\u2019est plut\xF4t fastidieux, donc nous allons le faire pour deux "),Ga=r(os,"EM",{});var Qu=p(Ga);ap=n(Qu,"tokens"),Qu.forEach(a),tp=n(os," ici et garder tout le processus pour quand nous aurons du code pour nous aider. Dans ce cas (tr\xE8s) particulier, nous avions deux tokenizations \xE9quivalentes de tous les mots. Par exmeple, comme nous l\u2019avons vu pr\xE9c\xE9demment, "),Ya=r(os,"CODE",{});var sm=p(Ya);np=n(sm,'"pug"'),sm.forEach(a),lp=n(os," pourrait \xEAtre tokenis\xE9 en "),Za=r(os,"CODE",{});var em=p(Za);op=n(em,'["p", "ug"]'),em.forEach(a),rp=n(os," avec le m\xEAme score. Ainsi, enlever le token "),Ja=r(os,"CODE",{});var am=p(Ja);pp=n(am,'"pu"'),am.forEach(a),ip=n(os," du vocabulaire donnera exactement la m\xEAme perte."),os.forEach(a),xn=m(s),G=r(s,"P",{});var Us=p(G);up=n(Us,"D\u2019un autre c\xF4t\xE9, supprimer le mot "),Ka=r(Us,"CODE",{});var tm=p(Ka);mp=n(tm,'"hug"'),tm.forEach(a),cp=n(Us," aggravera la perte, car la tokenisation de "),Qa=r(Us,"CODE",{});var nm=p(Qa);dp=n(nm,'"hug"'),nm.forEach(a),hp=n(Us," et "),st=r(Us,"CODE",{});var lm=p(st);fp=n(lm,'"hugs"'),lm.forEach(a),gp=n(Us," deviendra :"),Us.forEach(a),Pn=m(s),h(Js.$$.fragment,s),Cn=m(s),Ce=r(s,"P",{});var om=p(Ce);vp=n(om,"Ces changements entra\xEEneront une augmentation de la perte de :"),om.forEach(a),zn=m(s),h(Ks.$$.fragment,s),Mn=m(s),as=r(s,"P",{});var Re=p(as);bp=n(Re,"Par cons\xE9quent, le token "),et=r(Re,"CODE",{});var rm=p(et);qp=n(rm,'"pu"'),rm.forEach(a),_p=n(Re," sera probablement retir\xE9 du vocabulaire, mais pas "),at=r(Re,"CODE",{});var pm=p(at);jp=n(pm,'"hug"'),pm.forEach(a),kp=n(Re,"."),Re.forEach(a),An=m(s),cs=r(s,"H2",{class:!0});var Pl=p(cs);ws=r(Pl,"A",{id:!0,class:!0,href:!0});var im=p(ws);tt=r(im,"SPAN",{});var um=p(tt);h(Qs.$$.fragment,um),um.forEach(a),im.forEach(a),$p=m(Pl),ze=r(Pl,"SPAN",{});var Hi=p(ze);wp=n(Hi,"Impl\xE9mentation d'"),nt=r(Hi,"I",{});var mm=p(nt);Ep=n(mm,"Unigram"),mm.forEach(a),Hi.forEach(a),Pl.forEach(a),Dn=m(s),ts=r(s,"P",{});var Xe=p(ts);yp=n(Xe,"Maintenant, impl\xE9mentons tout ce que nous avons vu jusqu\u2019\xE0 pr\xE9sent dans le code. Comme pour le BPE et "),lt=r(Xe,"EM",{});var cm=p(lt);xp=n(cm,"WordPiece"),cm.forEach(a),Pp=n(Xe,", ce n\u2019est pas une impl\xE9mentation efficace de l\u2019algorithme "),ot=r(Xe,"EM",{});var dm=p(ot);Cp=n(dm,"Unigram"),dm.forEach(a),zp=n(Xe," (bien au contraire), mais elle devrait vous aider \xE0 le comprendre un peu mieux."),Xe.forEach(a),Nn=m(s),Me=r(s,"P",{});var hm=p(Me);Mp=n(hm,"Nous allons utiliser le m\xEAme corpus que pr\xE9c\xE9demment comme exemple :"),hm.forEach(a),Tn=m(s),h(se.$$.fragment,s),Un=m(s),Es=r(s,"P",{});var Cl=p(Es);Ap=n(Cl,"Cette fois, nous allons utiliser "),rt=r(Cl,"CODE",{});var fm=p(rt);Dp=n(fm,"xlnet-base-cased"),fm.forEach(a),Np=n(Cl," comme mod\xE8le :"),Cl.forEach(a),On=m(s),h(ee.$$.fragment,s),Sn=m(s),ys=r(s,"P",{});var zl=p(ys);Tp=n(zl,"Comme pour le BPE et "),pt=r(zl,"EM",{});var gm=p(pt);Up=n(gm,"WordPiece"),gm.forEach(a),Op=n(zl,", nous commen\xE7ons par compter le nombre d\u2019occurrences de chaque mot dans le corpus :"),zl.forEach(a),Hn=m(s),h(ae.$$.fragment,s),In=m(s),Ae=r(s,"P",{});var vm=p(Ae);Sp=n(vm,"Ensuite, nous devons initialiser notre vocabulaire \xE0 une taille plus grande que celle du vocabulaire que nous voudrons \xE0 la fin. Nous devons inclure tous les caract\xE8res de base (sinon nous ne serons pas en mesure de tokeniser chaque mot), mais pour les sous-cha\xEEnes plus grandes, nous ne garderons que les plus communs. AInsi nous les trions par fr\xE9quence :"),vm.forEach(a),Ln=m(s),h(te.$$.fragment,s),Bn=m(s),h(ne.$$.fragment,s),Vn=m(s),De=r(s,"P",{});var bm=p(De);Hp=n(bm,"Nous regroupons les caract\xE8res avec les meilleurs sous-mots pour arriver \xE0 un vocabulaire initial de taille 300 :"),bm.forEach(a),Rn=m(s),h(le.$$.fragment,s),Xn=m(s),h(xs.$$.fragment,s),Fn=m(s),Ne=r(s,"P",{});var qm=p(Ne);Ip=n(qm,"Ensuite, nous calculons la somme de toutes les fr\xE9quences, pour convertir les fr\xE9quences en probabilit\xE9s. Pour notre mod\xE8le, nous allons stocker les logarithmes des probabilit\xE9s, car c\u2019est plus stable num\xE9riquement d\u2019additionner des logarithmes que de multiplier des petits nombres. Cela simplifiera aussi le calcul de la perte du mod\xE8le :"),qm.forEach(a),Wn=m(s),h(oe.$$.fragment,s),Gn=m(s),Y=r(s,"P",{});var Os=p(Y);Lp=n(Os,"Maintenant la fonction principale est celle qui tokenise les mots en utilisant l\u2019algorithme de Viterbi. Comme nous l\u2019avons vu pr\xE9c\xE9demment, cet algorithme calcule la meilleure segmentation de chaque sous-cha\xEEne du mot que nous allons stocker dans une variable nomm\xE9e "),it=r(Os,"CODE",{});var _m=p(it);Bp=n(_m,"best_segmentations"),_m.forEach(a),Vp=n(Os,". Nous allons stocker un dictionnaire par position dans le mot (de 0 \xE0 sa longueur totale), avec deux cl\xE9s : l\u2019index du d\xE9but du dernier "),ut=r(Os,"EM",{});var jm=p(ut);Rp=n(jm,"token"),jm.forEach(a),Xp=n(Os," dans la meilleure segmentation et le score de la meilleure segmentation. Avec l\u2019index du d\xE9but du dernier "),mt=r(Os,"EM",{});var km=p(mt);Fp=n(km,"token"),km.forEach(a),Wp=n(Os,", nous serons en mesure de r\xE9cup\xE9rer la segmentation compl\xE8te une fois que la liste est compl\xE8tement remplie."),Os.forEach(a),Yn=m(s),Ps=r(s,"P",{});var Ml=p(Ps);Gp=n(Ml,"Le remplissage de la liste se fait \xE0 l\u2019aide de deux boucles seulement : la boucle principale passe en revue chaque position de d\xE9part et la seconde boucle essaie toutes les sous-cha\xEEnes commen\xE7ant \xE0 cette position de d\xE9part. Si la sous-cha\xEEne est dans le vocabulaire, nous avons une nouvelle segmentation du mot jusqu\u2019\xE0 cette position finale que nous comparons \xE0 ce qui est dans "),ct=r(Ml,"CODE",{});var $m=p(ct);Yp=n($m,"best_segmentations"),$m.forEach(a),Zp=n(Ml,"."),Ml.forEach(a),Zn=m(s),Cs=r(s,"P",{});var Al=p(Cs);Jp=n(Al,"Une fois que la boucle principale est termin\xE9e, nous commen\xE7ons juste \xE0 la fin et sautons d\u2019une position de d\xE9part \xE0 une autre, en enregistrant les "),dt=r(Al,"EM",{});var wm=p(dt);Kp=n(wm,"tokens"),wm.forEach(a),Qp=n(Al," au fur et \xE0 mesure, jusqu\u2019\xE0 ce que nous atteignions le d\xE9but du mot :"),Al.forEach(a),Jn=m(s),h(re.$$.fragment,s),Kn=m(s),Te=r(s,"P",{});var Em=p(Te);si=n(Em,"Nous pouvons d\xE9j\xE0 essayer notre mod\xE8le initial sur quelques mots :"),Em.forEach(a),Qn=m(s),h(pe.$$.fragment,s),sl=m(s),h(ie.$$.fragment,s),el=m(s),Ue=r(s,"P",{});var ym=p(Ue);ei=n(ym,"Il est maintenant facile de calculer la perte du mod\xE8le sur le corpus !"),ym.forEach(a),al=m(s),h(ue.$$.fragment,s),tl=m(s),Oe=r(s,"P",{});var xm=p(Oe);ai=n(xm,"Nous pouvons v\xE9rifier que cela fonctionne sur le mod\xE8le que nous avons :"),xm.forEach(a),nl=m(s),h(me.$$.fragment,s),ll=m(s),h(ce.$$.fragment,s),ol=m(s),ns=r(s,"P",{});var Fe=p(ns);ti=n(Fe,"Le calcul des scores pour chaque "),ht=r(Fe,"EM",{});var Pm=p(ht);ni=n(Pm,"token"),Pm.forEach(a),li=n(Fe," n\u2019est pas tr\xE8s difficile non plus. Il suffit de calculer la perte pour les mod\xE8les obtenus en supprimant chaque "),ft=r(Fe,"EM",{});var Cm=p(ft);oi=n(Cm,"token"),Cm.forEach(a),ri=n(Fe," :"),Fe.forEach(a),rl=m(s),h(de.$$.fragment,s),pl=m(s),zs=r(s,"P",{});var Dl=p(zs);pi=n(Dl,"Nous pouvons l\u2019essayer sur un "),gt=r(Dl,"EM",{});var zm=p(gt);ii=n(zm,"token"),zm.forEach(a),ui=n(Dl," donn\xE9 :"),Dl.forEach(a),il=m(s),h(he.$$.fragment,s),ul=m(s),U=r(s,"P",{});var J=p(U);mi=n(J,"Puisque "),vt=r(J,"CODE",{});var Mm=p(vt);ci=n(Mm,'"ll"'),Mm.forEach(a),di=n(J," est utilis\xE9 dans la tokenisation de "),bt=r(J,"CODE",{});var Am=p(bt);hi=n(Am,'"Hopefully"'),Am.forEach(a),fi=n(J,", et que le supprimer nous fera probablement utiliser le token "),qt=r(J,"CODE",{});var Dm=p(qt);gi=n(Dm,'"l"'),Dm.forEach(a),vi=n(J," deux fois \xE0 la place, nous nous attendons \xE0 ce qu\u2019il ait une perte positive. "),_t=r(J,"CODE",{});var Nm=p(_t);bi=n(Nm,'"his"'),Nm.forEach(a),qi=n(J," n\u2019est utilis\xE9 qu\u2019\xE0 l\u2019int\xE9rieur du mot "),jt=r(J,"CODE",{});var Tm=p(jt);_i=n(Tm,'"This"'),Tm.forEach(a),ji=n(J,", qui est tokenis\xE9 comme lui-m\xEAme, donc nous nous attendons \xE0 ce qu\u2019il ait une perte nulle. Voici les r\xE9sultats :"),J.forEach(a),ml=m(s),h(fe.$$.fragment,s),cl=m(s),h(Ms.$$.fragment,s),dl=m(s),ls=r(s,"P",{});var We=p(ls);ki=n(We,"Une fois tout cela en place, la derni\xE8re chose \xE0 faire est d\u2019ajouter les "),kt=r(We,"EM",{});var Um=p(kt);$i=n(Um,"tokens"),Um.forEach(a),wi=n(We," sp\xE9ciaux utilis\xE9s par le mod\xE8le au vocabulaire, puis de boucler jusqu\u2019\xE0 ce que nous ayons \xE9lagu\xE9 suffisamment de "),$t=r(We,"EM",{});var Om=p($t);Ei=n(Om,"tokens"),Om.forEach(a),yi=n(We," du vocabulaire pour atteindre la taille souhait\xE9e :"),We.forEach(a),hl=m(s),h(ge.$$.fragment,s),fl=m(s),As=r(s,"P",{});var Nl=p(As);xi=n(Nl,"Ensuite, pour tokeniser un texte, il suffit d\u2019appliquer la pr\xE9tok\xE9nisation et d\u2019utiliser la fonction "),wt=r(Nl,"CODE",{});var Sm=p(wt);Pi=n(Sm,"encode_word()"),Sm.forEach(a),Ci=n(Nl," :"),Nl.forEach(a),gl=m(s),h(ve.$$.fragment,s),vl=m(s),h(be.$$.fragment,s),bl=m(s),I=r(s,"P",{});var rs=p(I);zi=n(rs,"C\u2019est tout pour "),Et=r(rs,"EM",{});var Hm=p(Et);Mi=n(Hm,"Unigram"),Hm.forEach(a),Ai=n(rs," ! Avec un peu de chance, vous vous sentez \xE0 pr\xE9sent \xEAtre un expert des "),yt=r(rs,"EM",{});var Im=p(yt);Di=n(Im,"tokenizers"),Im.forEach(a),Ni=n(rs,". Dans la prochaine section, nous allons nous plonger dans les blocs de construction de la biblioth\xE8que \u{1F917} "),xt=r(rs,"EM",{});var Lm=p(xt);Ti=n(Lm,"Tokenizers"),Lm.forEach(a),Ui=n(rs," et allons vous montrer comment vous pouvez les utiliser pour construire votre propre "),Pt=r(rs,"EM",{});var Bm=p(Pt);Oi=n(Bm,"tokenizer"),Bm.forEach(a),Si=n(rs,"."),rs.forEach(a),this.h()},h(){P(c,"name","hf:doc:metadata"),P(c,"content",JSON.stringify(lc)),P($,"id","tokenisation-iunigrami"),P($,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),P($,"href","#tokenisation-iunigrami"),P(q,"class","relative group"),P(hs,"id","algorithme-dentranement"),P(hs,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),P(hs,"href","#algorithme-dentranement"),P(ps,"class","relative group"),Lt.a=Bt,P(vs,"id","algorithme-de-tokenisation"),P(vs,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),P(vs,"href","#algorithme-de-tokenisation"),P(is,"class","relative group"),tn.a=null,ln.a=null,P(ks,"id","retour-lentranement"),P(ks,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),P(ks,"href","#retour-lentranement"),P(ms,"class","relative group"),P(ws,"id","implmentation-diunigrami"),P(ws,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),P(ws,"href","#implmentation-diunigrami"),P(cs,"class","relative group")},m(s,l){e(document.head,c),i(s,y,l),i(s,q,l),e(q,$),e($,E),f(_,E,null),e(q,j),e(q,x),e(x,C),e(x,w),e(w,B),i(s,N,l),f(A,s,l),i(s,Ss,l),i(s,V,l),e(V,_e),e(V,Ge),e(Ge,Tl),e(V,Ul),e(V,Ye),e(Ye,Ol),e(V,Sl),i(s,Tt,l),f(Hs,s,l),i(s,Ut,l),f(ds,s,l),i(s,Ot,l),i(s,ps,l),e(ps,hs),e(hs,Ze),f(Is,Ze,null),e(ps,Hl),e(ps,Je),e(Je,Il),i(s,St,l),i(s,X,l),e(X,Ll),e(X,Ke),e(Ke,Bl),e(X,Vl),e(X,Qe),e(Qe,Rl),e(X,Xl),e(X,sa),e(sa,Fl),e(X,Wl),i(s,Ht,l),i(s,fs,l),e(fs,Gl),e(fs,ea),e(ea,Yl),e(fs,Zl),i(s,It,l),i(s,gs,l),e(gs,Jl),Lt.m(Vm,gs),e(gs,Bt),i(s,Vt,l),i(s,je,l),e(je,Kl),i(s,Rt,l),i(s,K,l),e(K,Ql),e(K,aa),e(aa,so),e(K,eo),e(K,ta),e(ta,ao),e(K,to),i(s,Xt,l),i(s,ke,l),e(ke,no),i(s,Ft,l),f(Ls,s,l),i(s,Wt,l),i(s,$e,l),e($e,lo),i(s,Gt,l),f(Bs,s,l),i(s,Yt,l),i(s,is,l),e(is,vs),e(vs,na),f(Vs,na,null),e(is,oo),e(is,la),e(la,ro),i(s,Zt,l),i(s,z,l),e(z,po),e(z,oa),e(oa,io),e(z,uo),e(z,ra),e(ra,mo),e(z,co),e(z,pa),e(pa,ho),e(z,fo),e(z,ia),e(ia,go),e(z,vo),e(z,ua),e(ua,bo),e(z,qo),e(z,ma),e(ma,_o),e(z,jo),e(z,ca),e(ca,ko),e(z,$o),i(s,Jt,l),i(s,D,l),e(D,wo),e(D,da),e(da,Eo),e(D,yo),e(D,ha),e(ha,xo),e(D,Po),e(D,fa),e(fa,Co),e(D,zo),e(D,ga),e(ga,Mo),e(D,Ao),e(D,va),e(va,Do),e(D,No),e(D,ba),e(ba,To),e(D,Uo),i(s,Kt,l),i(s,we,l),e(we,Oo),i(s,Qt,l),f(Rs,s,l),i(s,sn,l),i(s,bs,l),e(bs,So),e(bs,qa),e(qa,Ho),e(bs,Io),i(s,en,l),f(qs,s,l),i(s,an,l),i(s,M,l),e(M,Lo),e(M,_a),e(_a,Bo),e(M,Vo),e(M,ja),e(ja,Ro),e(M,Xo),e(M,ka),e(ka,Fo),e(M,Wo),e(M,$a),e($a,Go),e(M,Yo),e(M,wa),e(wa,Zo),e(M,Jo),e(M,Ea),e(Ea,Ko),e(M,Qo),tn.m(Rm,M),i(s,nn,l),i(s,us,l),e(us,sr),e(us,ya),e(ya,er),e(us,ar),ln.m(Xm,us),i(s,on,l),i(s,F,l),e(F,tr),e(F,xa),e(xa,nr),e(F,lr),e(F,Pa),e(Pa,or),e(F,rr),e(F,Ca),e(Ca,pr),e(F,ir),i(s,rn,l),i(s,Q,l),e(Q,ur),e(Q,za),e(za,mr),e(Q,cr),e(Q,Ma),e(Ma,dr),e(Q,hr),i(s,pn,l),f(Xs,s,l),i(s,un,l),i(s,W,l),e(W,fr),e(W,Aa),e(Aa,gr),e(W,vr),e(W,Da),e(Da,br),e(W,qr),e(W,Na),e(Na,_r),e(W,jr),i(s,mn,l),i(s,T,l),e(T,kr),e(T,Ta),e(Ta,$r),e(T,wr),e(T,Ua),e(Ua,Er),e(T,yr),e(T,Oa),e(Oa,xr),e(T,Pr),e(T,Sa),e(Sa,Cr),e(T,zr),e(T,Ha),e(Ha,Mr),e(T,Ar),i(s,cn,l),i(s,Ee,l),e(Ee,Dr),i(s,dn,l),i(s,_s,l),e(_s,Nr),e(_s,Ia),e(Ia,Tr),e(_s,Ur),i(s,hn,l),f(Fs,s,l),i(s,fn,l),i(s,ss,l),e(ss,Or),e(ss,La),e(La,Sr),e(ss,Hr),e(ss,Ba),e(Ba,Ir),e(ss,Lr),i(s,gn,l),f(js,s,l),i(s,vn,l),i(s,ms,l),e(ms,ks),e(ks,Va),f(Ws,Va,null),e(ms,Br),e(ms,Ra),e(Ra,Vr),i(s,bn,l),i(s,es,l),e(es,Rr),e(es,Xa),e(Xa,Xr),e(es,Fr),e(es,Fa),e(Fa,Wr),e(es,Gr),i(s,qn,l),i(s,$s,l),e($s,Yr),e($s,Wa),e(Wa,Zr),e($s,Jr),i(s,_n,l),i(s,ye,l),e(ye,Kr),i(s,jn,l),f(Gs,s,l),i(s,kn,l),i(s,xe,l),e(xe,Qr),i(s,$n,l),f(Ys,s,l),i(s,wn,l),i(s,Pe,l),e(Pe,sp),i(s,En,l),f(Zs,s,l),i(s,yn,l),i(s,H,l),e(H,ep),e(H,Ga),e(Ga,ap),e(H,tp),e(H,Ya),e(Ya,np),e(H,lp),e(H,Za),e(Za,op),e(H,rp),e(H,Ja),e(Ja,pp),e(H,ip),i(s,xn,l),i(s,G,l),e(G,up),e(G,Ka),e(Ka,mp),e(G,cp),e(G,Qa),e(Qa,dp),e(G,hp),e(G,st),e(st,fp),e(G,gp),i(s,Pn,l),f(Js,s,l),i(s,Cn,l),i(s,Ce,l),e(Ce,vp),i(s,zn,l),f(Ks,s,l),i(s,Mn,l),i(s,as,l),e(as,bp),e(as,et),e(et,qp),e(as,_p),e(as,at),e(at,jp),e(as,kp),i(s,An,l),i(s,cs,l),e(cs,ws),e(ws,tt),f(Qs,tt,null),e(cs,$p),e(cs,ze),e(ze,wp),e(ze,nt),e(nt,Ep),i(s,Dn,l),i(s,ts,l),e(ts,yp),e(ts,lt),e(lt,xp),e(ts,Pp),e(ts,ot),e(ot,Cp),e(ts,zp),i(s,Nn,l),i(s,Me,l),e(Me,Mp),i(s,Tn,l),f(se,s,l),i(s,Un,l),i(s,Es,l),e(Es,Ap),e(Es,rt),e(rt,Dp),e(Es,Np),i(s,On,l),f(ee,s,l),i(s,Sn,l),i(s,ys,l),e(ys,Tp),e(ys,pt),e(pt,Up),e(ys,Op),i(s,Hn,l),f(ae,s,l),i(s,In,l),i(s,Ae,l),e(Ae,Sp),i(s,Ln,l),f(te,s,l),i(s,Bn,l),f(ne,s,l),i(s,Vn,l),i(s,De,l),e(De,Hp),i(s,Rn,l),f(le,s,l),i(s,Xn,l),f(xs,s,l),i(s,Fn,l),i(s,Ne,l),e(Ne,Ip),i(s,Wn,l),f(oe,s,l),i(s,Gn,l),i(s,Y,l),e(Y,Lp),e(Y,it),e(it,Bp),e(Y,Vp),e(Y,ut),e(ut,Rp),e(Y,Xp),e(Y,mt),e(mt,Fp),e(Y,Wp),i(s,Yn,l),i(s,Ps,l),e(Ps,Gp),e(Ps,ct),e(ct,Yp),e(Ps,Zp),i(s,Zn,l),i(s,Cs,l),e(Cs,Jp),e(Cs,dt),e(dt,Kp),e(Cs,Qp),i(s,Jn,l),f(re,s,l),i(s,Kn,l),i(s,Te,l),e(Te,si),i(s,Qn,l),f(pe,s,l),i(s,sl,l),f(ie,s,l),i(s,el,l),i(s,Ue,l),e(Ue,ei),i(s,al,l),f(ue,s,l),i(s,tl,l),i(s,Oe,l),e(Oe,ai),i(s,nl,l),f(me,s,l),i(s,ll,l),f(ce,s,l),i(s,ol,l),i(s,ns,l),e(ns,ti),e(ns,ht),e(ht,ni),e(ns,li),e(ns,ft),e(ft,oi),e(ns,ri),i(s,rl,l),f(de,s,l),i(s,pl,l),i(s,zs,l),e(zs,pi),e(zs,gt),e(gt,ii),e(zs,ui),i(s,il,l),f(he,s,l),i(s,ul,l),i(s,U,l),e(U,mi),e(U,vt),e(vt,ci),e(U,di),e(U,bt),e(bt,hi),e(U,fi),e(U,qt),e(qt,gi),e(U,vi),e(U,_t),e(_t,bi),e(U,qi),e(U,jt),e(jt,_i),e(U,ji),i(s,ml,l),f(fe,s,l),i(s,cl,l),f(Ms,s,l),i(s,dl,l),i(s,ls,l),e(ls,ki),e(ls,kt),e(kt,$i),e(ls,wi),e(ls,$t),e($t,Ei),e(ls,yi),i(s,hl,l),f(ge,s,l),i(s,fl,l),i(s,As,l),e(As,xi),e(As,wt),e(wt,Pi),e(As,Ci),i(s,gl,l),f(ve,s,l),i(s,vl,l),f(be,s,l),i(s,bl,l),i(s,I,l),e(I,zi),e(I,Et),e(Et,Mi),e(I,Ai),e(I,yt),e(yt,Di),e(I,Ni),e(I,xt),e(xt,Ti),e(I,Ui),e(I,Pt),e(Pt,Oi),e(I,Si),ql=!0},p(s,[l]){const qe={};l&2&&(qe.$$scope={dirty:l,ctx:s}),ds.$set(qe);const Ct={};l&2&&(Ct.$$scope={dirty:l,ctx:s}),qs.$set(Ct);const zt={};l&2&&(zt.$$scope={dirty:l,ctx:s}),js.$set(zt);const Se={};l&2&&(Se.$$scope={dirty:l,ctx:s}),xs.$set(Se);const Mt={};l&2&&(Mt.$$scope={dirty:l,ctx:s}),Ms.$set(Mt)},i(s){ql||(g(_.$$.fragment,s),g(A.$$.fragment,s),g(Hs.$$.fragment,s),g(ds.$$.fragment,s),g(Is.$$.fragment,s),g(Ls.$$.fragment,s),g(Bs.$$.fragment,s),g(Vs.$$.fragment,s),g(Rs.$$.fragment,s),g(qs.$$.fragment,s),g(Xs.$$.fragment,s),g(Fs.$$.fragment,s),g(js.$$.fragment,s),g(Ws.$$.fragment,s),g(Gs.$$.fragment,s),g(Ys.$$.fragment,s),g(Zs.$$.fragment,s),g(Js.$$.fragment,s),g(Ks.$$.fragment,s),g(Qs.$$.fragment,s),g(se.$$.fragment,s),g(ee.$$.fragment,s),g(ae.$$.fragment,s),g(te.$$.fragment,s),g(ne.$$.fragment,s),g(le.$$.fragment,s),g(xs.$$.fragment,s),g(oe.$$.fragment,s),g(re.$$.fragment,s),g(pe.$$.fragment,s),g(ie.$$.fragment,s),g(ue.$$.fragment,s),g(me.$$.fragment,s),g(ce.$$.fragment,s),g(de.$$.fragment,s),g(he.$$.fragment,s),g(fe.$$.fragment,s),g(Ms.$$.fragment,s),g(ge.$$.fragment,s),g(ve.$$.fragment,s),g(be.$$.fragment,s),ql=!0)},o(s){v(_.$$.fragment,s),v(A.$$.fragment,s),v(Hs.$$.fragment,s),v(ds.$$.fragment,s),v(Is.$$.fragment,s),v(Ls.$$.fragment,s),v(Bs.$$.fragment,s),v(Vs.$$.fragment,s),v(Rs.$$.fragment,s),v(qs.$$.fragment,s),v(Xs.$$.fragment,s),v(Fs.$$.fragment,s),v(js.$$.fragment,s),v(Ws.$$.fragment,s),v(Gs.$$.fragment,s),v(Ys.$$.fragment,s),v(Zs.$$.fragment,s),v(Js.$$.fragment,s),v(Ks.$$.fragment,s),v(Qs.$$.fragment,s),v(se.$$.fragment,s),v(ee.$$.fragment,s),v(ae.$$.fragment,s),v(te.$$.fragment,s),v(ne.$$.fragment,s),v(le.$$.fragment,s),v(xs.$$.fragment,s),v(oe.$$.fragment,s),v(re.$$.fragment,s),v(pe.$$.fragment,s),v(ie.$$.fragment,s),v(ue.$$.fragment,s),v(me.$$.fragment,s),v(ce.$$.fragment,s),v(de.$$.fragment,s),v(he.$$.fragment,s),v(fe.$$.fragment,s),v(Ms.$$.fragment,s),v(ge.$$.fragment,s),v(ve.$$.fragment,s),v(be.$$.fragment,s),ql=!1},d(s){a(c),s&&a(y),s&&a(q),b(_),s&&a(N),b(A,s),s&&a(Ss),s&&a(V),s&&a(Tt),b(Hs,s),s&&a(Ut),b(ds,s),s&&a(Ot),s&&a(ps),b(Is),s&&a(St),s&&a(X),s&&a(Ht),s&&a(fs),s&&a(It),s&&a(gs),s&&a(Vt),s&&a(je),s&&a(Rt),s&&a(K),s&&a(Xt),s&&a(ke),s&&a(Ft),b(Ls,s),s&&a(Wt),s&&a($e),s&&a(Gt),b(Bs,s),s&&a(Yt),s&&a(is),b(Vs),s&&a(Zt),s&&a(z),s&&a(Jt),s&&a(D),s&&a(Kt),s&&a(we),s&&a(Qt),b(Rs,s),s&&a(sn),s&&a(bs),s&&a(en),b(qs,s),s&&a(an),s&&a(M),s&&a(nn),s&&a(us),s&&a(on),s&&a(F),s&&a(rn),s&&a(Q),s&&a(pn),b(Xs,s),s&&a(un),s&&a(W),s&&a(mn),s&&a(T),s&&a(cn),s&&a(Ee),s&&a(dn),s&&a(_s),s&&a(hn),b(Fs,s),s&&a(fn),s&&a(ss),s&&a(gn),b(js,s),s&&a(vn),s&&a(ms),b(Ws),s&&a(bn),s&&a(es),s&&a(qn),s&&a($s),s&&a(_n),s&&a(ye),s&&a(jn),b(Gs,s),s&&a(kn),s&&a(xe),s&&a($n),b(Ys,s),s&&a(wn),s&&a(Pe),s&&a(En),b(Zs,s),s&&a(yn),s&&a(H),s&&a(xn),s&&a(G),s&&a(Pn),b(Js,s),s&&a(Cn),s&&a(Ce),s&&a(zn),b(Ks,s),s&&a(Mn),s&&a(as),s&&a(An),s&&a(cs),b(Qs),s&&a(Dn),s&&a(ts),s&&a(Nn),s&&a(Me),s&&a(Tn),b(se,s),s&&a(Un),s&&a(Es),s&&a(On),b(ee,s),s&&a(Sn),s&&a(ys),s&&a(Hn),b(ae,s),s&&a(In),s&&a(Ae),s&&a(Ln),b(te,s),s&&a(Bn),b(ne,s),s&&a(Vn),s&&a(De),s&&a(Rn),b(le,s),s&&a(Xn),b(xs,s),s&&a(Fn),s&&a(Ne),s&&a(Wn),b(oe,s),s&&a(Gn),s&&a(Y),s&&a(Yn),s&&a(Ps),s&&a(Zn),s&&a(Cs),s&&a(Jn),b(re,s),s&&a(Kn),s&&a(Te),s&&a(Qn),b(pe,s),s&&a(sl),b(ie,s),s&&a(el),s&&a(Ue),s&&a(al),b(ue,s),s&&a(tl),s&&a(Oe),s&&a(nl),b(me,s),s&&a(ll),b(ce,s),s&&a(ol),s&&a(ns),s&&a(rl),b(de,s),s&&a(pl),s&&a(zs),s&&a(il),b(he,s),s&&a(ul),s&&a(U),s&&a(ml),b(fe,s),s&&a(cl),b(Ms,s),s&&a(dl),s&&a(ls),s&&a(hl),b(ge,s),s&&a(fl),s&&a(As),s&&a(gl),b(ve,s),s&&a(vl),b(be,s),s&&a(bl),s&&a(I)}}}const lc={local:"tokenisation-iunigrami",sections:[{local:"algorithme-dentranement",title:"Algorithme d'entra\xEEnement"},{local:"algorithme-de-tokenisation",title:"Algorithme de tokenisation"},{local:"retour-lentranement",title:"Retour \xE0 l'entra\xEEnement"},{local:"implmentation-diunigrami",title:"Impl\xE9mentation d'<i>Unigram</i>"}],title:"Tokenisation <i>Unigram</i>"};function oc(R){return Zm(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class dc extends Fm{constructor(c){super();Wm(this,c,oc,nc,Gm,{})}}export{dc as default,lc as metadata};
