import{S as Am,i as Dm,s as Tm,e as o,k as u,w as d,t,U as Pi,M as Nm,c as r,d as a,m,a as p,x as h,h as n,V as Ci,b as P,F as e,g as i,y as f,q as g,o as b,B as v,v as Um}from"../../chunks/vendor-1e8b365d.js";import{T as Pt}from"../../chunks/Tip-62b14c6e.js";import{Y as Om}from"../../chunks/Youtube-c2a8cc39.js";import{I as Ct}from"../../chunks/IconCopyLink-483c28ba.js";import{C as k}from"../../chunks/CodeBlock-e5764662.js";import{D as Sm}from"../../chunks/DocNotebookDropdown-37d928d3.js";function Hm(R){let c,y,q,$,E;return{c(){c=o("p"),y=t("\u{1F4A1} Cette section couvre "),q=o("em"),$=t("Unigram"),E=t(" en profondeur, allant jusqu\u2019\xE0 montrer une impl\xE9mentation compl\xE8te. Vous pouvez passer directement \xE0 la fin si vous souhaitez simplement avoir un aper\xE7u g\xE9n\xE9ral de l\u2019algorithme de tok\xE9nisation.")},l(_){c=r(_,"P",{});var j=p(c);y=n(j,"\u{1F4A1} Cette section couvre "),q=r(j,"EM",{});var x=p(q);$=n(x,"Unigram"),x.forEach(a),E=n(j," en profondeur, allant jusqu\u2019\xE0 montrer une impl\xE9mentation compl\xE8te. Vous pouvez passer directement \xE0 la fin si vous souhaitez simplement avoir un aper\xE7u g\xE9n\xE9ral de l\u2019algorithme de tok\xE9nisation."),j.forEach(a)},m(_,j){i(_,c,j),e(c,y),e(c,q),e(q,$),e(c,E)},d(_){_&&a(c)}}}function Lm(R){let c,y,q,$,E;return{c(){c=o("p"),y=t("\u270F\uFE0F "),q=o("strong"),$=t("A votre tour !"),E=t(" Ecrivez le code pour calculer les fr\xE9quences ci-dessus et v\xE9rifiez que les r\xE9sultats affich\xE9s sont corrects, ainsi que la somme totale.")},l(_){c=r(_,"P",{});var j=p(c);y=n(j,"\u270F\uFE0F "),q=r(j,"STRONG",{});var x=p(q);$=n(x,"A votre tour !"),x.forEach(a),E=n(j," Ecrivez le code pour calculer les fr\xE9quences ci-dessus et v\xE9rifiez que les r\xE9sultats affich\xE9s sont corrects, ainsi que la somme totale."),j.forEach(a)},m(_,j){i(_,c,j),e(c,y),e(c,q),e(q,$),e(c,E)},d(_){_&&a(c)}}}function Bm(R){let c,y,q,$,E,_,j,x;return{c(){c=o("p"),y=t("\u270F\uFE0F "),q=o("strong"),$=t("A votre tour !"),E=t(" D\xE9terminer la tokenization du mot "),_=o("code"),j=t('"huggun"'),x=t(" et son score.")},l(C){c=r(C,"P",{});var w=p(c);y=n(w,"\u270F\uFE0F "),q=r(w,"STRONG",{});var D=p(q);$=n(D,"A votre tour !"),D.forEach(a),E=n(w," D\xE9terminer la tokenization du mot "),_=r(w,"CODE",{});var N=p(_);j=n(N,'"huggun"'),N.forEach(a),x=n(w," et son score."),w.forEach(a)},m(C,w){i(C,c,w),e(c,y),e(c,q),e(q,$),e(c,E),e(c,_),e(_,j),e(c,x)},d(C){C&&a(c)}}}function Im(R){let c,y,q,$,E,_,j,x;return{c(){c=o("p"),y=t("\u{1F4A1} "),q=o("em"),$=t("SentencePiece"),E=t(" utilise un algorithme plus efficace appel\xE9 "),_=o("em"),j=t("Enhanced Suffix Array"),x=t(" (ESA) pour cr\xE9er le vocabulaire initial.")},l(C){c=r(C,"P",{});var w=p(c);y=n(w,"\u{1F4A1} "),q=r(w,"EM",{});var D=p(q);$=n(D,"SentencePiece"),D.forEach(a),E=n(w," utilise un algorithme plus efficace appel\xE9 "),_=r(w,"EM",{});var N=p(_);j=n(N,"Enhanced Suffix Array"),N.forEach(a),x=n(w," (ESA) pour cr\xE9er le vocabulaire initial."),w.forEach(a)},m(C,w){i(C,c,w),e(c,y),e(c,q),e(q,$),e(c,E),e(c,_),e(_,j),e(c,x)},d(C){C&&a(c)}}}function Rm(R){let c,y,q,$,E,_,j,x,C,w,D;return{c(){c=o("p"),y=t("\u{1F4A1} Cette approche est tr\xE8s inefficace, c\u2019est pourquoi "),q=o("em"),$=t("SentencePiece"),E=t(" utilise une approximation de la perte du mod\xE8le sans le "),_=o("em"),j=t("token"),x=t(" X : au lieu de partir de z\xE9ro, il remplace simplement le "),C=o("em"),w=t("token"),D=t(" X par sa segmentation dans le vocabulaire restant. De cette fa\xE7on, tous les scores peuvent \xEAtre calcul\xE9s en une seule fois, en m\xEAme temps que la perte du mod\xE8le.")},l(N){c=r(N,"P",{});var T=p(c);y=n(T,"\u{1F4A1} Cette approche est tr\xE8s inefficace, c\u2019est pourquoi "),q=r(T,"EM",{});var qe=p(q);$=n(qe,"SentencePiece"),qe.forEach(a),E=n(T," utilise une approximation de la perte du mod\xE8le sans le "),_=r(T,"EM",{});var Ss=p(_);j=n(Ss,"token"),Ss.forEach(a),x=n(T," X : au lieu de partir de z\xE9ro, il remplace simplement le "),C=r(T,"EM",{});var J=p(C);w=n(J,"token"),J.forEach(a),D=n(T," X par sa segmentation dans le vocabulaire restant. De cette fa\xE7on, tous les scores peuvent \xEAtre calcul\xE9s en une seule fois, en m\xEAme temps que la perte du mod\xE8le."),T.forEach(a)},m(N,T){i(N,c,T),e(c,y),e(c,q),e(q,$),e(c,E),e(c,_),e(_,j),e(c,x),e(c,C),e(C,w),e(c,D)},d(N){N&&a(c)}}}function Vm(R){let c,y,q,$,E,_,j,x,C,w,D,N,T,qe,Ss,J,zt,ds,Mt,ps,hs,Ve,Hs,Cl,Fe,zl,At,V,Ml,We,Al,Dl,Xe,Tl,Nl,Ge,Ul,Ol,Dt,fs,Sl,Ye,Hl,Ll,Tt,gs,Bl,Nt,Cm='<span class="katex"><span class="katex-mathml"><math xmlns="http://www.w3.org/1998/Math/MathML"><semantics><mrow><mi>p</mi></mrow><annotation encoding="application/x-tex">p</annotation></semantics></math></span><span class="katex-html" aria-hidden="true"><span class="base"><span class="strut" style="height:0.625em;vertical-align:-0.1944em;"></span><span class="mord mathnormal">p</span></span></span></span>',Ut,Ot,_e,Il,St,K,Rl,Ze,Vl,Fl,Je,Wl,Xl,Ht,je,Gl,Lt,Ls,Bt,ke,Yl,It,Bs,Rt,is,bs,Ke,Is,Zl,Qe,Jl,Vt,z,Kl,sa,Ql,so,ea,eo,ao,aa,to,no,ta,lo,oo,na,ro,po,la,io,uo,oa,mo,co,Ft,A,ho,ra,fo,go,pa,bo,vo,ia,qo,_o,ua,jo,ko,ma,$o,wo,ca,Eo,yo,Wt,$e,xo,Xt,Rs,Gt,vs,Po,da,Co,zo,Yt,qs,Zt,M,Mo,ha,Ao,Do,fa,To,No,ga,Uo,Oo,ba,So,Ho,va,Lo,Bo,qa,Io,Ro,Jt,zm='<span class="katex-display"><span class="katex"><span class="katex-mathml"><math xmlns="http://www.w3.org/1998/Math/MathML" display="block"><semantics><mrow><mi>P</mi><mo stretchy="false">(</mo><mo stretchy="false">[</mo><mi mathvariant="normal">\u2018</mi><mi mathvariant="normal">\u2018</mi><mi>p</mi><mi mathvariant="normal">&quot;</mi><mo separator="true">,</mo><mi mathvariant="normal">\u2018</mi><mi mathvariant="normal">\u2018</mi><mi>u</mi><mi mathvariant="normal">&quot;</mi><mo separator="true">,</mo><mi mathvariant="normal">\u2018</mi><mi mathvariant="normal">\u2018</mi><mi>g</mi><mi mathvariant="normal">&quot;</mi><mo stretchy="false">]</mo><mo stretchy="false">)</mo><mo>=</mo><mi>P</mi><mo stretchy="false">(</mo><mi mathvariant="normal">\u2018</mi><mi mathvariant="normal">\u2018</mi><mi>p</mi><mi mathvariant="normal">&quot;</mi><mo stretchy="false">)</mo><mo>\xD7</mo><mi>P</mi><mo stretchy="false">(</mo><mi mathvariant="normal">\u2018</mi><mi mathvariant="normal">\u2018</mi><mi>u</mi><mi mathvariant="normal">&quot;</mi><mo stretchy="false">)</mo><mo>\xD7</mo><mi>P</mi><mo stretchy="false">(</mo><mi mathvariant="normal">\u2018</mi><mi mathvariant="normal">\u2018</mi><mi>g</mi><mi mathvariant="normal">&quot;</mi><mo stretchy="false">)</mo><mo>=</mo><mfrac><mn>5</mn><mn>210</mn></mfrac><mo>\xD7</mo><mfrac><mn>36</mn><mn>210</mn></mfrac><mo>\xD7</mo><mfrac><mn>20</mn><mn>210</mn></mfrac><mo>=</mo><mn>0.000389</mn></mrow><annotation encoding="application/x-tex">P([``p&quot;, ``u&quot;, ``g&quot;]) = P(``p&quot;) \\times P(``u&quot;) \\times P(``g&quot;) = \\frac{5}{210} \\times \\frac{36}{210} \\times \\frac{20}{210} = 0.000389</annotation></semantics></math></span><span class="katex-html" aria-hidden="true"><span class="base"><span class="strut" style="height:1em;vertical-align:-0.25em;"></span><span class="mord mathnormal" style="margin-right:0.13889em;">P</span><span class="mopen">([</span><span class="mord">\u2018\u2018</span><span class="mord mathnormal">p</span><span class="mord">&quot;</span><span class="mpunct">,</span><span class="mspace" style="margin-right:0.1667em;"></span><span class="mord">\u2018\u2018</span><span class="mord mathnormal">u</span><span class="mord">&quot;</span><span class="mpunct">,</span><span class="mspace" style="margin-right:0.1667em;"></span><span class="mord">\u2018\u2018</span><span class="mord mathnormal" style="margin-right:0.03588em;">g</span><span class="mord">&quot;</span><span class="mclose">])</span><span class="mspace" style="margin-right:0.2778em;"></span><span class="mrel">=</span><span class="mspace" style="margin-right:0.2778em;"></span></span><span class="base"><span class="strut" style="height:1em;vertical-align:-0.25em;"></span><span class="mord mathnormal" style="margin-right:0.13889em;">P</span><span class="mopen">(</span><span class="mord">\u2018\u2018</span><span class="mord mathnormal">p</span><span class="mord">&quot;</span><span class="mclose">)</span><span class="mspace" style="margin-right:0.2222em;"></span><span class="mbin">\xD7</span><span class="mspace" style="margin-right:0.2222em;"></span></span><span class="base"><span class="strut" style="height:1em;vertical-align:-0.25em;"></span><span class="mord mathnormal" style="margin-right:0.13889em;">P</span><span class="mopen">(</span><span class="mord">\u2018\u2018</span><span class="mord mathnormal">u</span><span class="mord">&quot;</span><span class="mclose">)</span><span class="mspace" style="margin-right:0.2222em;"></span><span class="mbin">\xD7</span><span class="mspace" style="margin-right:0.2222em;"></span></span><span class="base"><span class="strut" style="height:1em;vertical-align:-0.25em;"></span><span class="mord mathnormal" style="margin-right:0.13889em;">P</span><span class="mopen">(</span><span class="mord">\u2018\u2018</span><span class="mord mathnormal" style="margin-right:0.03588em;">g</span><span class="mord">&quot;</span><span class="mclose">)</span><span class="mspace" style="margin-right:0.2778em;"></span><span class="mrel">=</span><span class="mspace" style="margin-right:0.2778em;"></span></span><span class="base"><span class="strut" style="height:2.0074em;vertical-align:-0.686em;"></span><span class="mord"><span class="mopen nulldelimiter"></span><span class="mfrac"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:1.3214em;"><span style="top:-2.314em;"><span class="pstrut" style="height:3em;"></span><span class="mord"><span class="mord">210</span></span></span><span style="top:-3.23em;"><span class="pstrut" style="height:3em;"></span><span class="frac-line" style="border-bottom-width:0.04em;"></span></span><span style="top:-3.677em;"><span class="pstrut" style="height:3em;"></span><span class="mord"><span class="mord">5</span></span></span></span><span class="vlist-s">\u200B</span></span><span class="vlist-r"><span class="vlist" style="height:0.686em;"><span></span></span></span></span></span><span class="mclose nulldelimiter"></span></span><span class="mspace" style="margin-right:0.2222em;"></span><span class="mbin">\xD7</span><span class="mspace" style="margin-right:0.2222em;"></span></span><span class="base"><span class="strut" style="height:2.0074em;vertical-align:-0.686em;"></span><span class="mord"><span class="mopen nulldelimiter"></span><span class="mfrac"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:1.3214em;"><span style="top:-2.314em;"><span class="pstrut" style="height:3em;"></span><span class="mord"><span class="mord">210</span></span></span><span style="top:-3.23em;"><span class="pstrut" style="height:3em;"></span><span class="frac-line" style="border-bottom-width:0.04em;"></span></span><span style="top:-3.677em;"><span class="pstrut" style="height:3em;"></span><span class="mord"><span class="mord">36</span></span></span></span><span class="vlist-s">\u200B</span></span><span class="vlist-r"><span class="vlist" style="height:0.686em;"><span></span></span></span></span></span><span class="mclose nulldelimiter"></span></span><span class="mspace" style="margin-right:0.2222em;"></span><span class="mbin">\xD7</span><span class="mspace" style="margin-right:0.2222em;"></span></span><span class="base"><span class="strut" style="height:2.0074em;vertical-align:-0.686em;"></span><span class="mord"><span class="mopen nulldelimiter"></span><span class="mfrac"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:1.3214em;"><span style="top:-2.314em;"><span class="pstrut" style="height:3em;"></span><span class="mord"><span class="mord">210</span></span></span><span style="top:-3.23em;"><span class="pstrut" style="height:3em;"></span><span class="frac-line" style="border-bottom-width:0.04em;"></span></span><span style="top:-3.677em;"><span class="pstrut" style="height:3em;"></span><span class="mord"><span class="mord">20</span></span></span></span><span class="vlist-s">\u200B</span></span><span class="vlist-r"><span class="vlist" style="height:0.686em;"><span></span></span></span></span></span><span class="mclose nulldelimiter"></span></span><span class="mspace" style="margin-right:0.2778em;"></span><span class="mrel">=</span><span class="mspace" style="margin-right:0.2778em;"></span></span><span class="base"><span class="strut" style="height:0.6444em;"></span><span class="mord">0.000389</span></span></span></span></span>',Kt,us,Vo,_a,Fo,Wo,Qt,Mm='<span class="katex-display"><span class="katex"><span class="katex-mathml"><math xmlns="http://www.w3.org/1998/Math/MathML" display="block"><semantics><mrow><mi>P</mi><mo stretchy="false">(</mo><mo stretchy="false">[</mo><mi mathvariant="normal">\u2018</mi><mi mathvariant="normal">\u2018</mi><mi>p</mi><mi>u</mi><mi mathvariant="normal">&quot;</mi><mo separator="true">,</mo><mi mathvariant="normal">\u2018</mi><mi mathvariant="normal">\u2018</mi><mi>g</mi><mi mathvariant="normal">&quot;</mi><mo stretchy="false">]</mo><mo stretchy="false">)</mo><mo>=</mo><mi>P</mi><mo stretchy="false">(</mo><mi mathvariant="normal">\u2018</mi><mi mathvariant="normal">\u2018</mi><mi>p</mi><mi>u</mi><mi mathvariant="normal">&quot;</mi><mo stretchy="false">)</mo><mo>\xD7</mo><mi>P</mi><mo stretchy="false">(</mo><mi mathvariant="normal">\u2018</mi><mi mathvariant="normal">\u2018</mi><mi>g</mi><mi mathvariant="normal">&quot;</mi><mo stretchy="false">)</mo><mo>=</mo><mfrac><mn>5</mn><mn>210</mn></mfrac><mo>\xD7</mo><mfrac><mn>20</mn><mn>210</mn></mfrac><mo>=</mo><mn>0.0022676</mn></mrow><annotation encoding="application/x-tex">P([``pu&quot;, ``g&quot;]) = P(``pu&quot;) \\times P(``g&quot;) = \\frac{5}{210} \\times \\frac{20}{210} = 0.0022676</annotation></semantics></math></span><span class="katex-html" aria-hidden="true"><span class="base"><span class="strut" style="height:1em;vertical-align:-0.25em;"></span><span class="mord mathnormal" style="margin-right:0.13889em;">P</span><span class="mopen">([</span><span class="mord">\u2018\u2018</span><span class="mord mathnormal">p</span><span class="mord mathnormal">u</span><span class="mord">&quot;</span><span class="mpunct">,</span><span class="mspace" style="margin-right:0.1667em;"></span><span class="mord">\u2018\u2018</span><span class="mord mathnormal" style="margin-right:0.03588em;">g</span><span class="mord">&quot;</span><span class="mclose">])</span><span class="mspace" style="margin-right:0.2778em;"></span><span class="mrel">=</span><span class="mspace" style="margin-right:0.2778em;"></span></span><span class="base"><span class="strut" style="height:1em;vertical-align:-0.25em;"></span><span class="mord mathnormal" style="margin-right:0.13889em;">P</span><span class="mopen">(</span><span class="mord">\u2018\u2018</span><span class="mord mathnormal">p</span><span class="mord mathnormal">u</span><span class="mord">&quot;</span><span class="mclose">)</span><span class="mspace" style="margin-right:0.2222em;"></span><span class="mbin">\xD7</span><span class="mspace" style="margin-right:0.2222em;"></span></span><span class="base"><span class="strut" style="height:1em;vertical-align:-0.25em;"></span><span class="mord mathnormal" style="margin-right:0.13889em;">P</span><span class="mopen">(</span><span class="mord">\u2018\u2018</span><span class="mord mathnormal" style="margin-right:0.03588em;">g</span><span class="mord">&quot;</span><span class="mclose">)</span><span class="mspace" style="margin-right:0.2778em;"></span><span class="mrel">=</span><span class="mspace" style="margin-right:0.2778em;"></span></span><span class="base"><span class="strut" style="height:2.0074em;vertical-align:-0.686em;"></span><span class="mord"><span class="mopen nulldelimiter"></span><span class="mfrac"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:1.3214em;"><span style="top:-2.314em;"><span class="pstrut" style="height:3em;"></span><span class="mord"><span class="mord">210</span></span></span><span style="top:-3.23em;"><span class="pstrut" style="height:3em;"></span><span class="frac-line" style="border-bottom-width:0.04em;"></span></span><span style="top:-3.677em;"><span class="pstrut" style="height:3em;"></span><span class="mord"><span class="mord">5</span></span></span></span><span class="vlist-s">\u200B</span></span><span class="vlist-r"><span class="vlist" style="height:0.686em;"><span></span></span></span></span></span><span class="mclose nulldelimiter"></span></span><span class="mspace" style="margin-right:0.2222em;"></span><span class="mbin">\xD7</span><span class="mspace" style="margin-right:0.2222em;"></span></span><span class="base"><span class="strut" style="height:2.0074em;vertical-align:-0.686em;"></span><span class="mord"><span class="mopen nulldelimiter"></span><span class="mfrac"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:1.3214em;"><span style="top:-2.314em;"><span class="pstrut" style="height:3em;"></span><span class="mord"><span class="mord">210</span></span></span><span style="top:-3.23em;"><span class="pstrut" style="height:3em;"></span><span class="frac-line" style="border-bottom-width:0.04em;"></span></span><span style="top:-3.677em;"><span class="pstrut" style="height:3em;"></span><span class="mord"><span class="mord">20</span></span></span></span><span class="vlist-s">\u200B</span></span><span class="vlist-r"><span class="vlist" style="height:0.686em;"><span></span></span></span></span></span><span class="mclose nulldelimiter"></span></span><span class="mspace" style="margin-right:0.2778em;"></span><span class="mrel">=</span><span class="mspace" style="margin-right:0.2778em;"></span></span><span class="base"><span class="strut" style="height:0.6444em;"></span><span class="mord">0.0022676</span></span></span></span></span>',sn,F,Xo,ja,Go,Yo,ka,Zo,Jo,$a,Ko,Qo,en,Q,sr,wa,er,ar,Ea,tr,nr,an,Vs,tn,W,lr,ya,or,rr,xa,pr,ir,Pa,ur,mr,nn,U,cr,Ca,dr,hr,za,fr,gr,Ma,br,vr,Aa,qr,_r,Da,jr,kr,ln,we,$r,on,_s,wr,Ta,Er,yr,rn,Fs,pn,ss,xr,Na,Pr,Cr,Ua,zr,Mr,un,js,mn,ms,ks,Oa,Ws,Ar,Sa,Dr,cn,es,Tr,Ha,Nr,Ur,La,Or,Sr,dn,$s,Hr,Ba,Lr,Br,hn,Ee,Ir,fn,Xs,gn,ye,Rr,bn,Gs,vn,xe,Vr,qn,Ys,_n,L,Fr,Ia,Wr,Xr,Ra,Gr,Yr,Va,Zr,Jr,Fa,Kr,Qr,jn,X,sp,Wa,ep,ap,Xa,tp,np,Ga,lp,op,kn,Zs,$n,Pe,rp,wn,Js,En,as,pp,Ya,ip,up,Za,mp,cp,yn,cs,ws,Ja,Ks,dp,Ka,hp,xn,ts,fp,Qa,gp,bp,st,vp,qp,Pn,Ce,_p,Cn,Qs,zn,Es,jp,et,kp,$p,Mn,se,An,ys,wp,at,Ep,yp,Dn,ee,Tn,ze,xp,Nn,ae,Un,te,On,Me,Pp,Sn,ne,Hn,xs,Ln,Ae,Cp,Bn,le,In,G,zp,tt,Mp,Ap,nt,Dp,Tp,lt,Np,Up,Rn,Ps,Op,ot,Sp,Hp,Vn,Cs,Lp,rt,Bp,Ip,Fn,oe,Wn,De,Rp,Xn,re,Gn,pe,Yn,Te,Vp,Zn,ie,Jn,Ne,Fp,Kn,ue,Qn,me,sl,ns,Wp,pt,Xp,Gp,it,Yp,Zp,el,ce,al,zs,Jp,ut,Kp,Qp,tl,de,nl,O,si,mt,ei,ai,ct,ti,ni,dt,li,oi,ht,ri,pi,ft,ii,ui,ll,he,ol,Ms,rl,ls,mi,gt,ci,di,bt,hi,fi,pl,fe,il,As,gi,vt,bi,vi,ul,ge,ml,be,cl,B,qi,qt,_i,ji,_t,ki,$i,jt,wi,Ei,kt,yi,xi,dl;return _=new Ct({}),D=new Sm({props:{classNames:"absolute z-10 right-0 top-0",options:[{label:"Google Colab",value:"https://colab.research.google.com/github/huggingface/notebooks/blob/master/course/chapter6/section7.ipynb"},{label:"Aws Studio",value:"https://studiolab.sagemaker.aws/import/github/huggingface/notebooks/blob/master/course/chapter6/section7.ipynb"}]}}),J=new Om({props:{id:"TGZfZVuF9Yc"}}),ds=new Pt({props:{$$slots:{default:[Hm]},$$scope:{ctx:R}}}),Hs=new Ct({}),Ls=new k({props:{code:'("hug", 10), ("pug", 5), ("pun", 12), ("bun", 4), ("hugs", 5)',highlighted:'(<span class="hljs-string">&quot;hug&quot;</span><span class="hljs-punctuation">,</span> <span class="hljs-number">10</span>)<span class="hljs-punctuation">,</span> (<span class="hljs-string">&quot;pug&quot;</span><span class="hljs-punctuation">,</span> <span class="hljs-number">5</span>)<span class="hljs-punctuation">,</span> (<span class="hljs-string">&quot;pun&quot;</span><span class="hljs-punctuation">,</span> <span class="hljs-number">12</span>)<span class="hljs-punctuation">,</span> (<span class="hljs-string">&quot;bun&quot;</span><span class="hljs-punctuation">,</span> <span class="hljs-number">4</span>)<span class="hljs-punctuation">,</span> (<span class="hljs-string">&quot;hugs&quot;</span><span class="hljs-punctuation">,</span> <span class="hljs-number">5</span>)'}}),Bs=new k({props:{code:'["h", "u", "g", "hu", "ug", "p", "pu", "n", "un", "b", "bu", "s", "hug", "gs", "ugs"]',highlighted:'<span class="hljs-selector-attr">[<span class="hljs-string">&quot;h&quot;</span>, <span class="hljs-string">&quot;u&quot;</span>, <span class="hljs-string">&quot;g&quot;</span>, <span class="hljs-string">&quot;hu&quot;</span>, <span class="hljs-string">&quot;ug&quot;</span>, <span class="hljs-string">&quot;p&quot;</span>, <span class="hljs-string">&quot;pu&quot;</span>, <span class="hljs-string">&quot;n&quot;</span>, <span class="hljs-string">&quot;un&quot;</span>, <span class="hljs-string">&quot;b&quot;</span>, <span class="hljs-string">&quot;bu&quot;</span>, <span class="hljs-string">&quot;s&quot;</span>, <span class="hljs-string">&quot;hug&quot;</span>, <span class="hljs-string">&quot;gs&quot;</span>, <span class="hljs-string">&quot;ugs&quot;</span>]</span>'}}),Is=new Ct({}),Rs=new k({props:{code:`("h", 15) ("u", 36) ("g", 20) ("hu", 15) ("ug", 20) ("p", 17) ("pu", 17) ("n", 16)
("un", 16) ("b", 4) ("bu", 4) ("s", 5) ("hug", 15) ("gs", 5) ("ugs", 5)`,highlighted:`(<span class="hljs-string">&quot;h&quot;</span><span class="hljs-punctuation">,</span> <span class="hljs-number">15</span>) (<span class="hljs-string">&quot;u&quot;</span><span class="hljs-punctuation">,</span> <span class="hljs-number">36</span>) (<span class="hljs-string">&quot;g&quot;</span><span class="hljs-punctuation">,</span> <span class="hljs-number">20</span>) (<span class="hljs-string">&quot;hu&quot;</span><span class="hljs-punctuation">,</span> <span class="hljs-number">15</span>) (<span class="hljs-string">&quot;ug&quot;</span><span class="hljs-punctuation">,</span> <span class="hljs-number">20</span>) (<span class="hljs-string">&quot;p&quot;</span><span class="hljs-punctuation">,</span> <span class="hljs-number">17</span>) (<span class="hljs-string">&quot;pu&quot;</span><span class="hljs-punctuation">,</span> <span class="hljs-number">17</span>) (<span class="hljs-string">&quot;n&quot;</span><span class="hljs-punctuation">,</span> <span class="hljs-number">16</span>)
(<span class="hljs-string">&quot;un&quot;</span><span class="hljs-punctuation">,</span> <span class="hljs-number">16</span>) (<span class="hljs-string">&quot;b&quot;</span><span class="hljs-punctuation">,</span> <span class="hljs-number">4</span>) (<span class="hljs-string">&quot;bu&quot;</span><span class="hljs-punctuation">,</span> <span class="hljs-number">4</span>) (<span class="hljs-string">&quot;s&quot;</span><span class="hljs-punctuation">,</span> <span class="hljs-number">5</span>) (<span class="hljs-string">&quot;hug&quot;</span><span class="hljs-punctuation">,</span> <span class="hljs-number">15</span>) (<span class="hljs-string">&quot;gs&quot;</span><span class="hljs-punctuation">,</span> <span class="hljs-number">5</span>) (<span class="hljs-string">&quot;ugs&quot;</span><span class="hljs-punctuation">,</span> <span class="hljs-number">5</span>)`}}),qs=new Pt({props:{$$slots:{default:[Lm]},$$scope:{ctx:R}}}),Vs=new k({props:{code:`["p", "u", "g"] : 0.000389
["p", "ug"] : 0.0022676
["pu", "g"] : 0.0022676`,highlighted:`[<span class="hljs-string">&quot;p&quot;</span>, <span class="hljs-string">&quot;u&quot;</span>, <span class="hljs-string">&quot;g&quot;</span>] : 0.000389
[<span class="hljs-string">&quot;p&quot;</span>, <span class="hljs-string">&quot;ug&quot;</span>] : 0.0022676
[<span class="hljs-string">&quot;pu&quot;</span>, <span class="hljs-string">&quot;g&quot;</span>] : 0.0022676`}}),Fs=new k({props:{code:`Character 0 (u): "u" (score 0.171429)
Character 1 (n): "un" (score 0.076191)
Character 2 (h): "un" "h" (score 0.005442)
Character 3 (u): "un" "hu" (score 0.005442)
Character 4 (g): "un" "hug" (score 0.005442)`,highlighted:`<span class="hljs-attribute">Character</span> <span class="hljs-number">0</span> (u): <span class="hljs-string">&quot;u&quot;</span> (score <span class="hljs-number">0</span>.<span class="hljs-number">171429</span>)
<span class="hljs-attribute">Character</span> <span class="hljs-number">1</span> (n): <span class="hljs-string">&quot;un&quot;</span> (score <span class="hljs-number">0</span>.<span class="hljs-number">076191</span>)
<span class="hljs-attribute">Character</span> <span class="hljs-number">2</span> (h): <span class="hljs-string">&quot;un&quot;</span> <span class="hljs-string">&quot;h&quot;</span> (score <span class="hljs-number">0</span>.<span class="hljs-number">005442</span>)
<span class="hljs-attribute">Character</span> <span class="hljs-number">3</span> (u): <span class="hljs-string">&quot;un&quot;</span> <span class="hljs-string">&quot;hu&quot;</span> (score <span class="hljs-number">0</span>.<span class="hljs-number">005442</span>)
<span class="hljs-attribute">Character</span> <span class="hljs-number">4</span> (g): <span class="hljs-string">&quot;un&quot;</span> <span class="hljs-string">&quot;hug&quot;</span> (score <span class="hljs-number">0</span>.<span class="hljs-number">005442</span>)`}}),js=new Pt({props:{$$slots:{default:[Bm]},$$scope:{ctx:R}}}),Ws=new Ct({}),Xs=new k({props:{code:'("hug", 10), ("pug", 5), ("pun", 12), ("bun", 4), ("hugs", 5)',highlighted:'(<span class="hljs-string">&quot;hug&quot;</span><span class="hljs-punctuation">,</span> <span class="hljs-number">10</span>)<span class="hljs-punctuation">,</span> (<span class="hljs-string">&quot;pug&quot;</span><span class="hljs-punctuation">,</span> <span class="hljs-number">5</span>)<span class="hljs-punctuation">,</span> (<span class="hljs-string">&quot;pun&quot;</span><span class="hljs-punctuation">,</span> <span class="hljs-number">12</span>)<span class="hljs-punctuation">,</span> (<span class="hljs-string">&quot;bun&quot;</span><span class="hljs-punctuation">,</span> <span class="hljs-number">4</span>)<span class="hljs-punctuation">,</span> (<span class="hljs-string">&quot;hugs&quot;</span><span class="hljs-punctuation">,</span> <span class="hljs-number">5</span>)'}}),Gs=new k({props:{code:`"hug": ["hug"] (score 0.071428)
"pug": ["pu", "g"] (score 0.007710)
"pun": ["pu", "n"] (score 0.006168)
"bun": ["bu", "n"] (score 0.001451)
"hugs": ["hug", "s"] (score 0.001701)`,highlighted:`<span class="hljs-string">&quot;hug&quot;</span>: [<span class="hljs-string">&quot;hug&quot;</span>] <span class="hljs-comment">(score 0.071428)</span>
<span class="hljs-string">&quot;pug&quot;</span>: [<span class="hljs-string">&quot;pu&quot;</span>, <span class="hljs-string">&quot;g&quot;</span>] <span class="hljs-comment">(score 0.007710)</span>
<span class="hljs-string">&quot;pun&quot;</span>: [<span class="hljs-string">&quot;pu&quot;</span>, <span class="hljs-string">&quot;n&quot;</span>] <span class="hljs-comment">(score 0.006168)</span>
<span class="hljs-string">&quot;bun&quot;</span>: [<span class="hljs-string">&quot;bu&quot;</span>, <span class="hljs-string">&quot;n&quot;</span>] <span class="hljs-comment">(score 0.001451)</span>
<span class="hljs-string">&quot;hugs&quot;</span>: [<span class="hljs-string">&quot;hug&quot;</span>, <span class="hljs-string">&quot;s&quot;</span>] <span class="hljs-comment">(score 0.001701)</span>`}}),Ys=new k({props:{code:"10 * (-log(0.071428)) + 5 * (-log(0.007710)) + 12 * (-log(0.006168)) + 4 * (-log(0.001451)) + 5 * (-log(0.001701)) = 169.8",highlighted:'<span class="hljs-attribute">10</span> * (-log(<span class="hljs-number">0</span>.<span class="hljs-number">071428</span>)) + <span class="hljs-number">5</span> * (-log(<span class="hljs-number">0</span>.<span class="hljs-number">007710</span>)) + <span class="hljs-number">12</span> * (-log(<span class="hljs-number">0</span>.<span class="hljs-number">006168</span>)) + <span class="hljs-number">4</span> * (-log(<span class="hljs-number">0</span>.<span class="hljs-number">001451</span>)) + <span class="hljs-number">5</span> * (-log(<span class="hljs-number">0</span>.<span class="hljs-number">001701</span>)) = <span class="hljs-number">169</span>.<span class="hljs-number">8</span>'}}),Zs=new k({props:{code:`"hug": ["hu", "g"] (score 0.006802)
"hugs": ["hu", "gs"] (score 0.001701)`,highlighted:`<span class="hljs-string">&quot;hug&quot;</span>: [<span class="hljs-string">&quot;hu&quot;</span>, <span class="hljs-string">&quot;g&quot;</span>] <span class="hljs-comment">(score 0.006802)</span>
<span class="hljs-string">&quot;hugs&quot;</span>: [<span class="hljs-string">&quot;hu&quot;</span>, <span class="hljs-string">&quot;gs&quot;</span>] <span class="hljs-comment">(score 0.001701)</span>`}}),Js=new k({props:{code:"- 10 * (-log(0.071428)) + 10 * (-log(0.006802)) = 23.5",highlighted:'- <span class="hljs-number">10</span> * (<span class="hljs-name">-log</span>(<span class="hljs-number">0.071428</span>)) + <span class="hljs-number">10</span> * (<span class="hljs-name">-log</span>(<span class="hljs-number">0.006802</span>)) = <span class="hljs-number">23.5</span>'}}),Ks=new Ct({}),Qs=new k({props:{code:`corpus = [
    "This is the Hugging Face course.",  # C'est le cours d'Hugging Face.
    "This chapter is about tokenization.",  # This chapter is about tokenization
    "This section shows several tokenizer algorithms.",  # Cette section pr\xE9sente plusieurs algorithmes de *tokenizer*.
    "Hopefully, you will be able to understand how they are trained and generate tokens.",  # Avec un peu de chance, vous serez en mesure de comprendre comment ils sont entra\xEEn\xE9s et g\xE9n\xE8rent des *tokens*.
]`,highlighted:`corpus = [
    <span class="hljs-string">&quot;This is the Hugging Face course.&quot;</span>,  <span class="hljs-comment"># C&#x27;est le cours d&#x27;Hugging Face.</span>
    <span class="hljs-string">&quot;This chapter is about tokenization.&quot;</span>,  <span class="hljs-comment"># This chapter is about tokenization</span>
    <span class="hljs-string">&quot;This section shows several tokenizer algorithms.&quot;</span>,  <span class="hljs-comment"># Cette section pr\xE9sente plusieurs algorithmes de *tokenizer*.</span>
    <span class="hljs-string">&quot;Hopefully, you will be able to understand how they are trained and generate tokens.&quot;</span>,  <span class="hljs-comment"># Avec un peu de chance, vous serez en mesure de comprendre comment ils sont entra\xEEn\xE9s et g\xE9n\xE8rent des *tokens*.</span>
]`}}),se=new k({props:{code:`from transformers import AutoTokenizer

tokenizer = AutoTokenizer.from_pretrained("xlnet-base-cased")`,highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoTokenizer

tokenizer = AutoTokenizer.from_pretrained(<span class="hljs-string">&quot;xlnet-base-cased&quot;</span>)`}}),ee=new k({props:{code:`from collections import defaultdict

word_freqs = defaultdict(int)
for text in corpus:
    words_with_offsets = tokenizer.backend_tokenizer.pre_tokenizer.pre_tokenize_str(text)
    new_words = [word for word, offset in words_with_offsets]
    for word in new_words:
        word_freqs[word] += 1

word_freqs`,highlighted:`<span class="hljs-keyword">from</span> collections <span class="hljs-keyword">import</span> defaultdict

word_freqs = defaultdict(<span class="hljs-built_in">int</span>)
<span class="hljs-keyword">for</span> text <span class="hljs-keyword">in</span> corpus:
    words_with_offsets = tokenizer.backend_tokenizer.pre_tokenizer.pre_tokenize_str(text)
    new_words = [word <span class="hljs-keyword">for</span> word, offset <span class="hljs-keyword">in</span> words_with_offsets]
    <span class="hljs-keyword">for</span> word <span class="hljs-keyword">in</span> new_words:
        word_freqs[word] += <span class="hljs-number">1</span>

word_freqs`}}),ae=new k({props:{code:`char_freqs = defaultdict(int)
subwords_freqs = defaultdict(int)
for word, freq in word_freqs.items():
    for i in range(len(word)):
        char_freqs[word[i]] += freq
        # Loop through the subwords of length at least 2
        for j in range(i + 2, len(word) + 1):
            subwords_freqs[word[i:j]] += freq

# Sort subwords by frequency
sorted_subwords = sorted(subwords_freqs.items(), key=lambda x: x[1], reverse=True)
sorted_subwords[:10]`,highlighted:`char_freqs = defaultdict(<span class="hljs-built_in">int</span>)
subwords_freqs = defaultdict(<span class="hljs-built_in">int</span>)
<span class="hljs-keyword">for</span> word, freq <span class="hljs-keyword">in</span> word_freqs.items():
    <span class="hljs-keyword">for</span> i <span class="hljs-keyword">in</span> <span class="hljs-built_in">range</span>(<span class="hljs-built_in">len</span>(word)):
        char_freqs[word[i]] += freq
        <span class="hljs-comment"># Loop through the subwords of length at least 2</span>
        <span class="hljs-keyword">for</span> j <span class="hljs-keyword">in</span> <span class="hljs-built_in">range</span>(i + <span class="hljs-number">2</span>, <span class="hljs-built_in">len</span>(word) + <span class="hljs-number">1</span>):
            subwords_freqs[word[i:j]] += freq

<span class="hljs-comment"># Sort subwords by frequency</span>
sorted_subwords = <span class="hljs-built_in">sorted</span>(subwords_freqs.items(), key=<span class="hljs-keyword">lambda</span> x: x[<span class="hljs-number">1</span>], reverse=<span class="hljs-literal">True</span>)
sorted_subwords[:<span class="hljs-number">10</span>]`}}),te=new k({props:{code:"[('\u2581t', 7), ('is', 5), ('er', 5), ('\u2581a', 5), ('\u2581to', 4), ('to', 4), ('en', 4), ('\u2581T', 3), ('\u2581Th', 3), ('\u2581Thi', 3)]",highlighted:'[(<span class="hljs-string">&#x27;\u2581t&#x27;</span>, <span class="hljs-number">7</span>), (<span class="hljs-string">&#x27;is&#x27;</span>, <span class="hljs-number">5</span>), (<span class="hljs-string">&#x27;er&#x27;</span>, <span class="hljs-number">5</span>), (<span class="hljs-string">&#x27;\u2581a&#x27;</span>, <span class="hljs-number">5</span>), (<span class="hljs-string">&#x27;\u2581to&#x27;</span>, <span class="hljs-number">4</span>), (<span class="hljs-string">&#x27;to&#x27;</span>, <span class="hljs-number">4</span>), (<span class="hljs-string">&#x27;en&#x27;</span>, <span class="hljs-number">4</span>), (<span class="hljs-string">&#x27;\u2581T&#x27;</span>, <span class="hljs-number">3</span>), (<span class="hljs-string">&#x27;\u2581Th&#x27;</span>, <span class="hljs-number">3</span>), (<span class="hljs-string">&#x27;\u2581Thi&#x27;</span>, <span class="hljs-number">3</span>)]'}}),ne=new k({props:{code:`token_freqs = list(char_freqs.items()) + sorted_subwords[: 300 - len(char_freqs)]
token_freqs = {token: freq for token, freq in token_freqs}`,highlighted:`token_freqs = <span class="hljs-built_in">list</span>(char_freqs.items()) + sorted_subwords[: <span class="hljs-number">300</span> - <span class="hljs-built_in">len</span>(char_freqs)]
token_freqs = {token: freq <span class="hljs-keyword">for</span> token, freq <span class="hljs-keyword">in</span> token_freqs}`}}),xs=new Pt({props:{$$slots:{default:[Im]},$$scope:{ctx:R}}}),le=new k({props:{code:`from math import log

total_sum = sum([freq for token, freq in token_freqs.items()])
model = {token: -log(freq / total_sum) for token, freq in token_freqs.items()}`,highlighted:`<span class="hljs-keyword">from</span> math <span class="hljs-keyword">import</span> log

total_sum = <span class="hljs-built_in">sum</span>([freq <span class="hljs-keyword">for</span> token, freq <span class="hljs-keyword">in</span> token_freqs.items()])
model = {token: -log(freq / total_sum) <span class="hljs-keyword">for</span> token, freq <span class="hljs-keyword">in</span> token_freqs.items()}`}}),oe=new k({props:{code:`def encode_word(word, model):
    best_segmentations = [{"start": 0, "score": 1}] + [
        {"start": None, "score": None} for _ in range(len(word))
    ]
    for start_idx in range(len(word)):
        # This should be properly filled by the previous steps of the loop
        best_score_at_start = best_segmentations[start_idx]["score"]
        for end_idx in range(start_idx + 1, len(word) + 1):
            token = word[start_idx:end_idx]
            if token in model and best_score_at_start is not None:
                score = model[token] + best_score_at_start
                # If we have found a better segmentation ending at end_idx, we update
                if (
                    best_segmentations[end_idx]["score"] is None
                    or best_segmentations[end_idx]["score"] > score
                ):
                    best_segmentations[end_idx] = {"start": start_idx, "score": score}

    segmentation = best_segmentations[-1]
    if segmentation["score"] is None:
        # We did not find a tokenization of the word -> unknown
        return ["<unk>"], None

    score = segmentation["score"]
    start = segmentation["start"]
    end = len(word)
    tokens = []
    while start != 0:
        tokens.insert(0, word[start:end])
        next_start = best_segmentations[start]["start"]
        end = start
        start = next_start
    tokens.insert(0, word[start:end])
    return tokens, score`,highlighted:`<span class="hljs-keyword">def</span> <span class="hljs-title function_">encode_word</span>(<span class="hljs-params">word, model</span>):
    best_segmentations = [{<span class="hljs-string">&quot;start&quot;</span>: <span class="hljs-number">0</span>, <span class="hljs-string">&quot;score&quot;</span>: <span class="hljs-number">1</span>}] + [
        {<span class="hljs-string">&quot;start&quot;</span>: <span class="hljs-literal">None</span>, <span class="hljs-string">&quot;score&quot;</span>: <span class="hljs-literal">None</span>} <span class="hljs-keyword">for</span> _ <span class="hljs-keyword">in</span> <span class="hljs-built_in">range</span>(<span class="hljs-built_in">len</span>(word))
    ]
    <span class="hljs-keyword">for</span> start_idx <span class="hljs-keyword">in</span> <span class="hljs-built_in">range</span>(<span class="hljs-built_in">len</span>(word)):
        <span class="hljs-comment"># This should be properly filled by the previous steps of the loop</span>
        best_score_at_start = best_segmentations[start_idx][<span class="hljs-string">&quot;score&quot;</span>]
        <span class="hljs-keyword">for</span> end_idx <span class="hljs-keyword">in</span> <span class="hljs-built_in">range</span>(start_idx + <span class="hljs-number">1</span>, <span class="hljs-built_in">len</span>(word) + <span class="hljs-number">1</span>):
            token = word[start_idx:end_idx]
            <span class="hljs-keyword">if</span> token <span class="hljs-keyword">in</span> model <span class="hljs-keyword">and</span> best_score_at_start <span class="hljs-keyword">is</span> <span class="hljs-keyword">not</span> <span class="hljs-literal">None</span>:
                score = model[token] + best_score_at_start
                <span class="hljs-comment"># If we have found a better segmentation ending at end_idx, we update</span>
                <span class="hljs-keyword">if</span> (
                    best_segmentations[end_idx][<span class="hljs-string">&quot;score&quot;</span>] <span class="hljs-keyword">is</span> <span class="hljs-literal">None</span>
                    <span class="hljs-keyword">or</span> best_segmentations[end_idx][<span class="hljs-string">&quot;score&quot;</span>] &gt; score
                ):
                    best_segmentations[end_idx] = {<span class="hljs-string">&quot;start&quot;</span>: start_idx, <span class="hljs-string">&quot;score&quot;</span>: score}

    segmentation = best_segmentations[-<span class="hljs-number">1</span>]
    <span class="hljs-keyword">if</span> segmentation[<span class="hljs-string">&quot;score&quot;</span>] <span class="hljs-keyword">is</span> <span class="hljs-literal">None</span>:
        <span class="hljs-comment"># We did not find a tokenization of the word -&gt; unknown</span>
        <span class="hljs-keyword">return</span> [<span class="hljs-string">&quot;&lt;unk&gt;&quot;</span>], <span class="hljs-literal">None</span>

    score = segmentation[<span class="hljs-string">&quot;score&quot;</span>]
    start = segmentation[<span class="hljs-string">&quot;start&quot;</span>]
    end = <span class="hljs-built_in">len</span>(word)
    tokens = []
    <span class="hljs-keyword">while</span> start != <span class="hljs-number">0</span>:
        tokens.insert(<span class="hljs-number">0</span>, word[start:end])
        next_start = best_segmentations[start][<span class="hljs-string">&quot;start&quot;</span>]
        end = start
        start = next_start
    tokens.insert(<span class="hljs-number">0</span>, word[start:end])
    <span class="hljs-keyword">return</span> tokens, score`}}),re=new k({props:{code:`print(encode_word("Hopefully", model))
print(encode_word("This", model))`,highlighted:`<span class="hljs-built_in">print</span>(encode_word(<span class="hljs-string">&quot;Hopefully&quot;</span>, model))
<span class="hljs-built_in">print</span>(encode_word(<span class="hljs-string">&quot;This&quot;</span>, model))`}}),pe=new k({props:{code:`(['H', 'o', 'p', 'e', 'f', 'u', 'll', 'y'], 41.5157494601402)
(['This'], 6.288267030694535)`,highlighted:`([<span class="hljs-string">&#x27;H&#x27;</span>, <span class="hljs-string">&#x27;o&#x27;</span>, <span class="hljs-string">&#x27;p&#x27;</span>, <span class="hljs-string">&#x27;e&#x27;</span>, <span class="hljs-string">&#x27;f&#x27;</span>, <span class="hljs-string">&#x27;u&#x27;</span>, <span class="hljs-string">&#x27;ll&#x27;</span>, <span class="hljs-string">&#x27;y&#x27;</span>], <span class="hljs-number">41.5157494601402</span>)
([<span class="hljs-string">&#x27;This&#x27;</span>], <span class="hljs-number">6.288267030694535</span>)`}}),ie=new k({props:{code:`def compute_loss(model):
    loss = 0
    for word, freq in word_freqs.items():
        _, word_loss = encode_word(word, model)
        loss += freq * word_loss
    return loss`,highlighted:`<span class="hljs-keyword">def</span> <span class="hljs-title function_">compute_loss</span>(<span class="hljs-params">model</span>):
    loss = <span class="hljs-number">0</span>
    <span class="hljs-keyword">for</span> word, freq <span class="hljs-keyword">in</span> word_freqs.items():
        _, word_loss = encode_word(word, model)
        loss += freq * word_loss
    <span class="hljs-keyword">return</span> loss`}}),ue=new k({props:{code:"compute_loss(model)",highlighted:"compute_loss(model)"}}),me=new k({props:{code:"413.10377642940875",highlighted:'<span class="hljs-number">413.10377642940875</span>'}}),ce=new k({props:{code:`import copy


def compute_scores(model):
    scores = {}
    model_loss = compute_loss(model)
    for token, score in model.items():
        # We always keep tokens of length 1
        if len(token) == 1:
            continue
        model_without_token = copy.deepcopy(model)
        _ = model_without_token.pop(token)
        scores[token] = compute_loss(model_without_token) - model_loss
    return scores`,highlighted:`<span class="hljs-keyword">import</span> copy


<span class="hljs-keyword">def</span> <span class="hljs-title function_">compute_scores</span>(<span class="hljs-params">model</span>):
    scores = {}
    model_loss = compute_loss(model)
    <span class="hljs-keyword">for</span> token, score <span class="hljs-keyword">in</span> model.items():
        <span class="hljs-comment"># We always keep tokens of length 1</span>
        <span class="hljs-keyword">if</span> <span class="hljs-built_in">len</span>(token) == <span class="hljs-number">1</span>:
            <span class="hljs-keyword">continue</span>
        model_without_token = copy.deepcopy(model)
        _ = model_without_token.pop(token)
        scores[token] = compute_loss(model_without_token) - model_loss
    <span class="hljs-keyword">return</span> scores`}}),de=new k({props:{code:`scores = compute_scores(model)
print(scores["ll"])
print(scores["his"])`,highlighted:`scores = compute_scores(model)
<span class="hljs-built_in">print</span>(scores[<span class="hljs-string">&quot;ll&quot;</span>])
<span class="hljs-built_in">print</span>(scores[<span class="hljs-string">&quot;his&quot;</span>])`}}),he=new k({props:{code:`6.376412403623874
0.0`,highlighted:`<span class="hljs-number">6.376412403623874</span>
<span class="hljs-number">0.0</span>`}}),Ms=new Pt({props:{$$slots:{default:[Rm]},$$scope:{ctx:R}}}),fe=new k({props:{code:`percent_to_remove = 0.1
while len(model) > 100:
    scores = compute_scores(model)
    sorted_scores = sorted(scores.items(), key=lambda x: x[1])
    # Remove percent_to_remove tokens with the lowest scores.
    for i in range(int(len(model) * percent_to_remove)):
        _ = token_freqs.pop(sorted_scores[i][0])

    total_sum = sum([freq for token, freq in token_freqs.items()])
    model = {token: -log(freq / total_sum) for token, freq in token_freqs.items()}`,highlighted:`percent_to_remove = <span class="hljs-number">0.1</span>
<span class="hljs-keyword">while</span> <span class="hljs-built_in">len</span>(model) &gt; <span class="hljs-number">100</span>:
    scores = compute_scores(model)
    sorted_scores = <span class="hljs-built_in">sorted</span>(scores.items(), key=<span class="hljs-keyword">lambda</span> x: x[<span class="hljs-number">1</span>])
    <span class="hljs-comment"># Remove percent_to_remove tokens with the lowest scores.</span>
    <span class="hljs-keyword">for</span> i <span class="hljs-keyword">in</span> <span class="hljs-built_in">range</span>(<span class="hljs-built_in">int</span>(<span class="hljs-built_in">len</span>(model) * percent_to_remove)):
        _ = token_freqs.pop(sorted_scores[i][<span class="hljs-number">0</span>])

    total_sum = <span class="hljs-built_in">sum</span>([freq <span class="hljs-keyword">for</span> token, freq <span class="hljs-keyword">in</span> token_freqs.items()])
    model = {token: -log(freq / total_sum) <span class="hljs-keyword">for</span> token, freq <span class="hljs-keyword">in</span> token_freqs.items()}`}}),ge=new k({props:{code:`def tokenize(text, model):
    words_with_offsets = tokenizer.backend_tokenizer.pre_tokenizer.pre_tokenize_str(text)
    pre_tokenized_text = [word for word, offset in words_with_offsets]
    encoded_words = [encode_word(word, model)[0] for word in pre_tokenized_text]
    return sum(encoded_words, [])


tokenize("This is the Hugging Face course.", model)`,highlighted:`<span class="hljs-keyword">def</span> <span class="hljs-title function_">tokenize</span>(<span class="hljs-params">text, model</span>):
    words_with_offsets = tokenizer.backend_tokenizer.pre_tokenizer.pre_tokenize_str(text)
    pre_tokenized_text = [word <span class="hljs-keyword">for</span> word, offset <span class="hljs-keyword">in</span> words_with_offsets]
    encoded_words = [encode_word(word, model)[<span class="hljs-number">0</span>] <span class="hljs-keyword">for</span> word <span class="hljs-keyword">in</span> pre_tokenized_text]
    <span class="hljs-keyword">return</span> <span class="hljs-built_in">sum</span>(encoded_words, [])


tokenize(<span class="hljs-string">&quot;This is the Hugging Face course.&quot;</span>, model)`}}),be=new k({props:{code:"['\u2581This', '\u2581is', '\u2581the', '\u2581Hugging', '\u2581Face', '\u2581', 'c', 'ou', 'r', 's', 'e', '.']",highlighted:'[<span class="hljs-string">&#x27;\u2581This&#x27;</span>, <span class="hljs-string">&#x27;\u2581is&#x27;</span>, <span class="hljs-string">&#x27;\u2581the&#x27;</span>, <span class="hljs-string">&#x27;\u2581Hugging&#x27;</span>, <span class="hljs-string">&#x27;\u2581Face&#x27;</span>, <span class="hljs-string">&#x27;\u2581&#x27;</span>, <span class="hljs-string">&#x27;c&#x27;</span>, <span class="hljs-string">&#x27;ou&#x27;</span>, <span class="hljs-string">&#x27;r&#x27;</span>, <span class="hljs-string">&#x27;s&#x27;</span>, <span class="hljs-string">&#x27;e&#x27;</span>, <span class="hljs-string">&#x27;.&#x27;</span>]'}}),{c(){c=o("meta"),y=u(),q=o("h1"),$=o("a"),E=o("span"),d(_.$$.fragment),j=u(),x=o("span"),C=t("Tokenisation *Unigram*"),w=u(),d(D.$$.fragment),N=u(),T=o("p"),qe=t("The Unigram algorithm is often used in SentencePiece, which is the tokenization algorithm used by models like AlBERT, T5, mBART, Big Bird, and XLNet."),Ss=u(),d(J.$$.fragment),zt=u(),d(ds.$$.fragment),Mt=u(),ps=o("h2"),hs=o("a"),Ve=o("span"),d(Hs.$$.fragment),Cl=u(),Fe=o("span"),zl=t("Algorithme d'entra\xEEnement"),At=u(),V=o("p"),Ml=t("Compar\xE9 \xE0 BPE et "),We=o("em"),Al=t("WordPiece"),Dl=t(", "),Xe=o("em"),Tl=t("Unigram"),Nl=t(" fonctionne dans l\u2019autre sens : il part d\u2019un grand vocabulaire et enl\xE8ve des "),Ge=o("em"),Ul=t("tokens"),Ol=t(" jusqu\u2019\xE0 atteindre la taille de vocabulaire d\xE9sir\xE9e. Il existe plusieurs options pour construire ce vocabulaire de base : nous pouvons prendre les sous-cha\xEEnes les plus courantes dans les mots pr\xE9-tok\xE9nis\xE9s, par exemple, ou appliquer BPE sur le corpus initial avec une grande taille de vocabulaire."),Dt=u(),fs=o("p"),Sl=t("\xC0 chaque \xE9tape de l\u2019entra\xEEnement, l\u2019algorithme "),Ye=o("em"),Hl=t("Unigram"),Ll=t(" calcule une perte sur le corpus compte tenu du vocabulaire actuel. Ensuite, pour chaque symbole du vocabulaire, l\u2019algorithme calcule de combien la perte globale augmenterait si le symbole \xE9tait supprim\xE9, et recherche les symboles qui l\u2019augmenteraient le moins. Ces symboles ont un effet moindre sur la perte globale du corpus, ils sont donc en quelque sorte \u201Cmoins n\xE9cessaires\u201D et sont les meilleurs candidats \xE0 la suppression."),Tt=u(),gs=o("p"),Bl=t("Comme il s\u2019agit d\u2019une op\xE9ration tr\xE8s co\xFBteuse, nous ne nous contentons pas de supprimer le symbole unique associ\xE9 \xE0 la plus faible augmentation de la perte, mais le "),Nt=new Pi,Ut=t(" ((p\\) \xE9tant un hyperparam\xE8tre que vous pouvez contr\xF4ler, g\xE9n\xE9ralement 10 ou 20) pour cent des symboles associ\xE9s \xE0 la plus faible augmentation de la perte. Ce processus est ensuite r\xE9p\xE9t\xE9 jusqu\u2019\xE0 ce que le vocabulaire ait atteint la taille souhait\xE9e."),Ot=u(),_e=o("p"),Il=t("Notez que nous ne supprimons jamais les caract\xE8res de base, afin de nous assurer que tout mot peut \xEAtre tokenis\xE9."),St=u(),K=o("p"),Rl=t("Maintenant, c\u2019est encore un peu vague : la partie principale de l\u2019algorithme est de calculer une perte sur le corpus et de voir comment elle change lorsque nous supprimons certains "),Ze=o("em"),Vl=t("tokens"),Fl=t(" du vocabulaire, mais nous n\u2019avons pas encore expliqu\xE9 comment le faire. Cette \xE9tape repose sur l\u2019algorithme de tok\xE9nisation d\u2019un mod\xE8le "),Je=o("em"),Wl=t("Unigram"),Xl=t(", nous allons donc l\u2019aborder maintenant."),Ht=u(),je=o("p"),Gl=t("Nous allons r\xE9utiliser le corpus des exemples pr\xE9c\xE9dents :"),Lt=u(),d(Ls.$$.fragment),Bt=u(),ke=o("p"),Yl=t("et pour cet exemple, nous prendrons toutes les sous-cha\xEEnes strictes pour le vocabulaire initial :"),It=u(),d(Bs.$$.fragment),Rt=u(),is=o("h2"),bs=o("a"),Ke=o("span"),d(Is.$$.fragment),Zl=u(),Qe=o("span"),Jl=t("Algorithme de tokenisation"),Vt=u(),z=o("p"),Kl=t("Un mod\xE8le "),sa=o("em"),Ql=t("Unigram"),so=t(" est un type de mod\xE8le de langage qui consid\xE8re que chaque "),ea=o("em"),eo=t("token"),ao=t(" est ind\xE9pendant des "),aa=o("em"),to=t("tokens"),no=t(" qui le pr\xE9c\xE8dent. Il s\u2019agit du mod\xE8le de langage le plus simple, dans le sens o\xF9 la probabilit\xE9 du "),ta=o("em"),lo=t("token"),oo=t(" X compte tenu du contexte pr\xE9c\xE9dent est simplement la probabilit\xE9 du "),na=o("em"),ro=t("token"),po=t(" X. Ainsi, si nous utilisions un mod\xE8le de langage "),la=o("em"),io=t("Unigram"),uo=t(" pour g\xE9n\xE9rer du texte, nous pr\xE9dirions toujours le "),oa=o("em"),mo=t("token"),co=t(" le plus courant."),Ft=u(),A=o("p"),ho=t("La probabilit\xE9 d\u2019un "),ra=o("em"),fo=t("token"),go=t(" donn\xE9 est sa fr\xE9quence (le nombre de fois que nous le trouvons) dans le corpus original, divis\xE9e par la somme de toutes les fr\xE9quences de tous les "),pa=o("em"),bo=t("tokens"),vo=t(" dans le vocabulaire (pour s\u2019assurer que la somme des probabilit\xE9s est \xE9gale \xE0 1). Par exemple, "),ia=o("code"),qo=t('"ug"'),_o=t(" est pr\xE9sent dans "),ua=o("code"),jo=t('"hug"'),ko=t(", "),ma=o("code"),$o=t('"pug"'),wo=t(", et "),ca=o("code"),Eo=t('"hugs"'),yo=t(", il a donc une fr\xE9quence de 20 dans notre corpus."),Wt=u(),$e=o("p"),xo=t("Voici les fr\xE9quences de tous les sous-mots possibles dans le vocabulaire :"),Xt=u(),d(Rs.$$.fragment),Gt=u(),vs=o("p"),Po=t("Ainsi, la somme de toutes les fr\xE9quences est de 210, et la probabilit\xE9 du sous-mot "),da=o("code"),Co=t('"ug"'),zo=t(" est donc de 20/210."),Yt=u(),d(qs.$$.fragment),Zt=u(),M=o("p"),Mo=t("Maintenant, pour tokeniser un mot donn\xE9, nous examinons toutes les segmentations possibles en "),ha=o("em"),Ao=t("tokens"),Do=t(" et calculons la probabilit\xE9 de chacune d\u2019entre elles selon le mod\xE8le "),fa=o("em"),To=t("Unigram"),No=t(". Puisque tous les "),ga=o("em"),Uo=t("tokens"),Oo=t(" sont consid\xE9r\xE9s comme ind\xE9pendants, cette probabilit\xE9 est juste le produit de la probabilit\xE9 de chaque "),ba=o("em"),So=t("token"),Ho=t(". Par exemple, la tokenisation "),va=o("code"),Lo=t('["p", "u", "g"]'),Bo=t(" de "),qa=o("code"),Io=t('"pug"'),Ro=t(` a la probabilit\xE9 :
`),Jt=new Pi,Kt=u(),us=o("p"),Vo=t("Comparativement, la tokenization "),_a=o("code"),Fo=t('["pu", "g"]'),Wo=t(` a la probabilit\xE9 :
`),Qt=new Pi,sn=u(),F=o("p"),Xo=t("donc celle-l\xE0 est beaucoup plus probable. En g\xE9n\xE9ral, les tok\xE9nisations comportant le moins de "),ja=o("em"),Go=t("tokens"),Yo=t(" possible auront la probabilit\xE9 la plus \xE9lev\xE9e (en raison de la division par 210 r\xE9p\xE9t\xE9e pour chaque "),ka=o("em"),Zo=t("token"),Jo=t("), ce qui correspond \xE0 ce que nous voulons intuitivement : diviser un mot en un nombre de "),$a=o("em"),Ko=t("tokens"),Qo=t(" le plus faible possible."),en=u(),Q=o("p"),sr=t("La tokenisation d\u2019un mot avec le mod\xE8le "),wa=o("em"),er=t("Unigram"),ar=t(" est donc la tokenisation avec la plus haute probabilit\xE9. Dans l\u2019exemple de "),Ea=o("code"),tr=t('"pug"'),nr=t(", voici les probabilit\xE9s que nous obtiendrions pour chaque segmentation possible :"),an=u(),d(Vs.$$.fragment),tn=u(),W=o("p"),lr=t("Ainsi, "),ya=o("code"),or=t('"pug"'),rr=t(" sera tokenis\xE9 comme "),xa=o("code"),pr=t('["p", "ug"]'),ir=t(" ou "),Pa=o("code"),ur=t('["pu", "g"]'),mr=t(", selon la segmentation rencontr\xE9e en premier (notez que dans un corpus plus large, les cas d\u2019\xE9galit\xE9 comme celui-ci seront rares)."),nn=u(),U=o("p"),cr=t("Dans ce cas, il \xE9tait facile de trouver toutes les segmentations possibles et de calculer leurs probabilit\xE9s, mais en g\xE9n\xE9ral, ce sera un peu plus difficile. Il existe un algorithme classique utilis\xE9 pour cela, appel\xE9 "),Ca=o("em"),dr=t("algorithme de Viterbi"),hr=t(". Essentiellement, on peut construire un graphe pour d\xE9tecter les segmentations possibles d\u2019un mot donn\xE9 en disant qu\u2019il existe une branche du caract\xE8re "),za=o("em"),fr=t("a"),gr=t(" au caract\xE8re "),Ma=o("em"),br=t("b"),vr=t(" si le sous-mot de "),Aa=o("em"),qr=t("a"),_r=t(" \xE0 "),Da=o("em"),jr=t("b"),kr=t(" est dans le vocabulaire, et attribuer \xE0 cette branche la probabilit\xE9 du sous-mot."),ln=u(),we=o("p"),$r=t("Pour trouver le chemin dans ce graphe qui va avoir le meilleur score, l\u2019algorithme de Viterbi d\xE9termine, pour chaque position dans le mot, la segmentation avec le meilleur score qui se termine \xE0 cette position. Puisque nous allons du d\xE9but \xE0 la fin, ce meilleur score peut \xEAtre trouv\xE9 en parcourant en boucle tous les sous-mots se terminant \xE0 la position actuelle, puis en utilisant le meilleur score de tokenization de la position \xE0 laquelle ce sous-mot commence. Ensuite, il suffit de d\xE9rouler le chemin emprunt\xE9 pour arriver \xE0 la fin."),on=u(),_s=o("p"),wr=t("Prenons un exemple en utilisant notre vocabulaire et le mot "),Ta=o("code"),Er=t('"unhug"'),yr=t(". Pour chaque position, les sous-mots avec les meilleurs scores se terminant l\xE0 sont les suivants :"),rn=u(),d(Fs.$$.fragment),pn=u(),ss=o("p"),xr=t("Ainsi, "),Na=o("code"),Pr=t('"unhug"'),Cr=t(" serait tokenis\xE9 comme "),Ua=o("code"),zr=t('["un", "hug"]'),Mr=t("."),un=u(),d(js.$$.fragment),mn=u(),ms=o("h2"),ks=o("a"),Oa=o("span"),d(Ws.$$.fragment),Ar=u(),Sa=o("span"),Dr=t("Retour \xE0 l'entra\xEEnement"),cn=u(),es=o("p"),Tr=t("Maintenant que nous avons vu comment fonctionne la tokenisation, nous pouvons nous plonger un peu plus profond\xE9ment dans la perte utilis\xE9e pendant l\u2019entra\xEEnement. \xC0 n\u2019importe quelle \xE9tape, cette perte est calcul\xE9e en tokenisant chaque mot du corpus, en utilisant le vocabulaire courant et le mod\xE8le "),Ha=o("em"),Nr=t("Unigram"),Ur=t(" d\xE9termin\xE9 par les fr\xE9quences de chaque "),La=o("em"),Or=t("token"),Sr=t(" dans le corpus (comme vu pr\xE9c\xE9demment)."),dn=u(),$s=o("p"),Hr=t("Chaque mot du corpus a un score, et la perte est le logarithme n\xE9gatif de ces scores : c\u2019est-\xE0-dire la somme pour tous les mots du corpus de tous les "),Ba=o("code"),Lr=t("-log(P(word))"),Br=t("."),hn=u(),Ee=o("p"),Ir=t("Revenons \xE0 notre exemple avec le corpus suivant :"),fn=u(),d(Xs.$$.fragment),gn=u(),ye=o("p"),Rr=t("La tokenisation de chaque mot avec leurs scores respectifs est :"),bn=u(),d(Gs.$$.fragment),vn=u(),xe=o("p"),Vr=t("Donc la perte est :"),qn=u(),d(Ys.$$.fragment),_n=u(),L=o("p"),Fr=t("Maintenant, nous devons calculer comment la suppression de chaque token affecte la perte. C\u2019est plut\xF4t fastidieux, donc nous allons le faire pour deux "),Ia=o("em"),Wr=t("tokens"),Xr=t(" ici et garder tout le processus pour quand nous aurons du code pour nous aider. Dans ce cas (tr\xE8s) particulier, nous avions deux tokenizations \xE9quivalentes de tous les mots : comme nous l\u2019avons vu pr\xE9c\xE9demment, par exemple, "),Ra=o("code"),Gr=t('"pug"'),Yr=t(" pourrait \xEAtre tokenis\xE9 "),Va=o("code"),Zr=t('["p", "ug"]'),Jr=t(" avec le m\xEAme score. Ainsi, enlever le token "),Fa=o("code"),Kr=t('"pu"'),Qr=t(" du vocabulaire donnera exactement la m\xEAme perte."),jn=u(),X=o("p"),sp=t("D\u2019un autre c\xF4t\xE9, supprimer le mot "),Wa=o("code"),ep=t('"hug"'),ap=t(" aggravera la perte, car la tokenisation de "),Xa=o("code"),tp=t('"hug"'),np=t(" et "),Ga=o("code"),lp=t('"hugs"'),op=t(" deviendra :"),kn=u(),d(Zs.$$.fragment),$n=u(),Pe=o("p"),rp=t("Ces changements entra\xEEneront une augmentation de la perte de :"),wn=u(),d(Js.$$.fragment),En=u(),as=o("p"),pp=t("Par cons\xE9quent, le token "),Ya=o("code"),ip=t('"pu"'),up=t(" sera probablement retir\xE9 du vocabulaire, mais pas "),Za=o("code"),mp=t('"hug"'),cp=t("."),yn=u(),cs=o("h2"),ws=o("a"),Ja=o("span"),d(Ks.$$.fragment),dp=u(),Ka=o("span"),hp=t("Impl\xE9mentation d'*Unigram*"),xn=u(),ts=o("p"),fp=t("Maintenant, impl\xE9mentons tout ce que nous avons vu jusqu\u2019\xE0 pr\xE9sent dans le code. Comme pour BPE et "),Qa=o("em"),gp=t("WordPiece"),bp=t(", ce n\u2019est pas une impl\xE9mentation efficace de l\u2019algorithme "),st=o("em"),vp=t("Unigram"),qp=t(" (bien au contraire), mais cela devrait vous aider \xE0 le comprendre un peu mieux."),Pn=u(),Ce=o("p"),_p=t("Nous allons utiliser le m\xEAme corpus que pr\xE9c\xE9demment comme exemple :"),Cn=u(),d(Qs.$$.fragment),zn=u(),Es=o("p"),jp=t("Cette fois, nous allons utiliser "),et=o("code"),kp=t("xlnet-base-cased"),$p=t(" comme mod\xE8le :"),Mn=u(),d(se.$$.fragment),An=u(),ys=o("p"),wp=t("Comme pour BPE et "),at=o("em"),Ep=t("WordPiece"),yp=t(", nous commen\xE7ons par compter le nombre d\u2019occurrences de chaque mot dans le corpus :"),Dn=u(),d(ee.$$.fragment),Tn=u(),ze=o("p"),xp=t("Ensuite, nous devons initialiser notre vocabulaire \xE0 quelque chose de plus grand que la taille du vocabulaire que nous voudrons \xE0 la fin. Nous devons inclure tous les caract\xE8res de base (sinon nous ne serons pas en mesure de tokeniser chaque mot), mais pour les sous-cha\xEEnes plus grandes, nous ne garderons que les plus communs, donc nous les trions par fr\xE9quence :"),Nn=u(),d(ae.$$.fragment),Un=u(),d(te.$$.fragment),On=u(),Me=o("p"),Pp=t("Nous regroupons les caract\xE8res avec les meilleurs sous-mots pour arriver \xE0 un vocabulaire initial de taille 300 :"),Sn=u(),d(ne.$$.fragment),Hn=u(),d(xs.$$.fragment),Ln=u(),Ae=o("p"),Cp=t("Ensuite, nous calculons la somme de toutes les fr\xE9quences, pour convertir les fr\xE9quences en probabilit\xE9s. Pour notre mod\xE8le, nous allons stocker les logarithmes des probabilit\xE9s, car il est plus stable num\xE9riquement d\u2019additionner des logarithmes que de multiplier des petits nombres, et cela simplifiera le calcul de la perte du mod\xE8le :"),Bn=u(),d(le.$$.fragment),In=u(),G=o("p"),zp=t("Maintenant la fonction principale est celle qui tokenise les mots en utilisant l\u2019algorithme de Viterbi. Comme nous l\u2019avons vu pr\xE9c\xE9demment, cet algorithme calcule la meilleure segmentation de chaque sous-cha\xEEne du mot, que nous allons stocker dans une variable nomm\xE9e "),tt=o("code"),Mp=t("best_segmentations"),Ap=t(". Nous allons stocker un dictionnaire par position dans le mot (de 0 \xE0 sa longueur totale), avec deux cl\xE9s : l\u2019index du d\xE9but du dernier "),nt=o("em"),Dp=t("token"),Tp=t(" dans la meilleure segmentation, et le score de la meilleure segmentation. Avec l\u2019index du d\xE9but du dernier "),lt=o("em"),Np=t("token"),Up=t(", nous serons en mesure de r\xE9cup\xE9rer la segmentation compl\xE8te une fois que la liste est compl\xE8tement remplie."),Rn=u(),Ps=o("p"),Op=t("Le remplissage de la liste se fait \xE0 l\u2019aide de deux boucles seulement : la boucle principale passe en revue chaque position de d\xE9part, et la seconde boucle essaie toutes les sous-cha\xEEnes commen\xE7ant \xE0 cette position de d\xE9part. Si la sous-cha\xEEne est dans le vocabulaire, nous avons une nouvelle segmentation du mot jusqu\u2019\xE0 cette position finale, que nous comparons \xE0 ce qui est dans "),ot=o("code"),Sp=t("best_segmentations"),Hp=t("."),Vn=u(),Cs=o("p"),Lp=t("Une fois que la boucle principale est termin\xE9e, nous commen\xE7ons juste \xE0 la fin et sautons d\u2019une position de d\xE9part \xE0 une autre, en enregistrant les "),rt=o("em"),Bp=t("tokens"),Ip=t(" au fur et \xE0 mesure, jusqu\u2019\xE0 ce que nous atteignions le d\xE9but du mot :"),Fn=u(),d(oe.$$.fragment),Wn=u(),De=o("p"),Rp=t("Nous pouvons d\xE9j\xE0 essayer notre mod\xE8le initial sur quelques mots :"),Xn=u(),d(re.$$.fragment),Gn=u(),d(pe.$$.fragment),Yn=u(),Te=o("p"),Vp=t("Il est maintenant facile de calculer la perte du mod\xE8le sur le corpus !"),Zn=u(),d(ie.$$.fragment),Jn=u(),Ne=o("p"),Fp=t("Nous pouvons v\xE9rifier que cela fonctionne sur le mod\xE8le que nous avons :"),Kn=u(),d(ue.$$.fragment),Qn=u(),d(me.$$.fragment),sl=u(),ns=o("p"),Wp=t("Le calcul des scores pour chaque "),pt=o("em"),Xp=t("token"),Gp=t(" n\u2019est pas tr\xE8s difficile non plus ; il suffit de calculer la perte pour les mod\xE8les obtenus en supprimant chaque "),it=o("em"),Yp=t("token"),Zp=t(" :"),el=u(),d(ce.$$.fragment),al=u(),zs=o("p"),Jp=t("Nous pouvons l\u2019essayer sur un "),ut=o("em"),Kp=t("token"),Qp=t(" donn\xE9 :"),tl=u(),d(de.$$.fragment),nl=u(),O=o("p"),si=t("Puisque "),mt=o("code"),ei=t('"ll"'),ai=t(" est utilis\xE9 dans la tokenisation de "),ct=o("code"),ti=t('"Hopefully"'),ni=t(", et que le supprimer nous fera probablement utiliser le token "),dt=o("code"),li=t('"l"'),oi=t(" deux fois \xE0 la place, nous nous attendons \xE0 ce qu\u2019il ait une perte positive. "),ht=o("code"),ri=t('"his"'),pi=t(" n\u2019est utilis\xE9 qu\u2019\xE0 l\u2019int\xE9rieur du mot "),ft=o("code"),ii=t('"This"'),ui=t(", qui est tokenis\xE9 comme lui-m\xEAme, donc nous nous attendons \xE0 ce qu\u2019il ait une perte nulle. Voici les r\xE9sultats :"),ll=u(),d(he.$$.fragment),ol=u(),d(Ms.$$.fragment),rl=u(),ls=o("p"),mi=t("Une fois tout cela en place, la derni\xE8re chose \xE0 faire est d\u2019ajouter les "),gt=o("em"),ci=t("tokens"),di=t(" sp\xE9ciaux utilis\xE9s par le mod\xE8le au vocabulaire, puis de boucler jusqu\u2019\xE0 ce que nous ayons \xE9lagu\xE9 suffisamment de "),bt=o("em"),hi=t("tokens"),fi=t(" du vocabulaire pour atteindre la taille souhait\xE9e :"),pl=u(),d(fe.$$.fragment),il=u(),As=o("p"),gi=t("Ensuite, pour tokeniser un texte, il suffit d\u2019appliquer la pr\xE9-tok\xE9nisation et d\u2019utiliser la fonction "),vt=o("code"),bi=t("encode_word()"),vi=t(" :"),ul=u(),d(ge.$$.fragment),ml=u(),d(be.$$.fragment),cl=u(),B=o("p"),qi=t("C\u2019est tout pour "),qt=o("em"),_i=t("Unigram"),ji=t(" ! Avec un peu de chance, vous vous sentez maintenant comme un expert en tout ce qui concerne les "),_t=o("em"),ki=t("tokenizers"),$i=t(". Dans la prochaine section, nous allons nous plonger dans les blocs de construction de la biblioth\xE8que \u{1F917} "),jt=o("em"),wi=t("Tokenizers"),Ei=t(", et vous montrer comment vous pouvez les utiliser pour construire votre propre "),kt=o("em"),yi=t("tokenizer"),xi=t("."),this.h()},l(s){const l=Nm('[data-svelte="svelte-1phssyn"]',document.head);c=r(l,"META",{name:!0,content:!0}),l.forEach(a),y=m(s),q=r(s,"H1",{class:!0});var ve=p(q);$=r(ve,"A",{id:!0,class:!0,href:!0});var $t=p($);E=r($t,"SPAN",{});var wt=p(E);h(_.$$.fragment,wt),wt.forEach(a),$t.forEach(a),j=m(ve),x=r(ve,"SPAN",{});var Et=p(x);C=n(Et,"Tokenisation *Unigram*"),Et.forEach(a),ve.forEach(a),w=m(s),h(D.$$.fragment,s),N=m(s),T=r(s,"P",{});var yt=p(T);qe=n(yt,"The Unigram algorithm is often used in SentencePiece, which is the tokenization algorithm used by models like AlBERT, T5, mBART, Big Bird, and XLNet."),yt.forEach(a),Ss=m(s),h(J.$$.fragment,s),zt=m(s),h(ds.$$.fragment,s),Mt=m(s),ps=r(s,"H2",{class:!0});var hl=p(ps);hs=r(hl,"A",{id:!0,class:!0,href:!0});var zi=p(hs);Ve=r(zi,"SPAN",{});var Mi=p(Ve);h(Hs.$$.fragment,Mi),Mi.forEach(a),zi.forEach(a),Cl=m(hl),Fe=r(hl,"SPAN",{});var Ai=p(Fe);zl=n(Ai,"Algorithme d'entra\xEEnement"),Ai.forEach(a),hl.forEach(a),At=m(s),V=r(s,"P",{});var Ds=p(V);Ml=n(Ds,"Compar\xE9 \xE0 BPE et "),We=r(Ds,"EM",{});var Di=p(We);Al=n(Di,"WordPiece"),Di.forEach(a),Dl=n(Ds,", "),Xe=r(Ds,"EM",{});var Ti=p(Xe);Tl=n(Ti,"Unigram"),Ti.forEach(a),Nl=n(Ds," fonctionne dans l\u2019autre sens : il part d\u2019un grand vocabulaire et enl\xE8ve des "),Ge=r(Ds,"EM",{});var Ni=p(Ge);Ul=n(Ni,"tokens"),Ni.forEach(a),Ol=n(Ds," jusqu\u2019\xE0 atteindre la taille de vocabulaire d\xE9sir\xE9e. Il existe plusieurs options pour construire ce vocabulaire de base : nous pouvons prendre les sous-cha\xEEnes les plus courantes dans les mots pr\xE9-tok\xE9nis\xE9s, par exemple, ou appliquer BPE sur le corpus initial avec une grande taille de vocabulaire."),Ds.forEach(a),Dt=m(s),fs=r(s,"P",{});var fl=p(fs);Sl=n(fl,"\xC0 chaque \xE9tape de l\u2019entra\xEEnement, l\u2019algorithme "),Ye=r(fl,"EM",{});var Ui=p(Ye);Hl=n(Ui,"Unigram"),Ui.forEach(a),Ll=n(fl," calcule une perte sur le corpus compte tenu du vocabulaire actuel. Ensuite, pour chaque symbole du vocabulaire, l\u2019algorithme calcule de combien la perte globale augmenterait si le symbole \xE9tait supprim\xE9, et recherche les symboles qui l\u2019augmenteraient le moins. Ces symboles ont un effet moindre sur la perte globale du corpus, ils sont donc en quelque sorte \u201Cmoins n\xE9cessaires\u201D et sont les meilleurs candidats \xE0 la suppression."),fl.forEach(a),Tt=m(s),gs=r(s,"P",{});var gl=p(gs);Bl=n(gl,"Comme il s\u2019agit d\u2019une op\xE9ration tr\xE8s co\xFBteuse, nous ne nous contentons pas de supprimer le symbole unique associ\xE9 \xE0 la plus faible augmentation de la perte, mais le "),Nt=Ci(gl),Ut=n(gl," ((p\\) \xE9tant un hyperparam\xE8tre que vous pouvez contr\xF4ler, g\xE9n\xE9ralement 10 ou 20) pour cent des symboles associ\xE9s \xE0 la plus faible augmentation de la perte. Ce processus est ensuite r\xE9p\xE9t\xE9 jusqu\u2019\xE0 ce que le vocabulaire ait atteint la taille souhait\xE9e."),gl.forEach(a),Ot=m(s),_e=r(s,"P",{});var Oi=p(_e);Il=n(Oi,"Notez que nous ne supprimons jamais les caract\xE8res de base, afin de nous assurer que tout mot peut \xEAtre tokenis\xE9."),Oi.forEach(a),St=m(s),K=r(s,"P",{});var Ue=p(K);Rl=n(Ue,"Maintenant, c\u2019est encore un peu vague : la partie principale de l\u2019algorithme est de calculer une perte sur le corpus et de voir comment elle change lorsque nous supprimons certains "),Ze=r(Ue,"EM",{});var Si=p(Ze);Vl=n(Si,"tokens"),Si.forEach(a),Fl=n(Ue," du vocabulaire, mais nous n\u2019avons pas encore expliqu\xE9 comment le faire. Cette \xE9tape repose sur l\u2019algorithme de tok\xE9nisation d\u2019un mod\xE8le "),Je=r(Ue,"EM",{});var Hi=p(Je);Wl=n(Hi,"Unigram"),Hi.forEach(a),Xl=n(Ue,", nous allons donc l\u2019aborder maintenant."),Ue.forEach(a),Ht=m(s),je=r(s,"P",{});var Li=p(je);Gl=n(Li,"Nous allons r\xE9utiliser le corpus des exemples pr\xE9c\xE9dents :"),Li.forEach(a),Lt=m(s),h(Ls.$$.fragment,s),Bt=m(s),ke=r(s,"P",{});var Bi=p(ke);Yl=n(Bi,"et pour cet exemple, nous prendrons toutes les sous-cha\xEEnes strictes pour le vocabulaire initial :"),Bi.forEach(a),It=m(s),h(Bs.$$.fragment,s),Rt=m(s),is=r(s,"H2",{class:!0});var bl=p(is);bs=r(bl,"A",{id:!0,class:!0,href:!0});var Ii=p(bs);Ke=r(Ii,"SPAN",{});var Ri=p(Ke);h(Is.$$.fragment,Ri),Ri.forEach(a),Ii.forEach(a),Zl=m(bl),Qe=r(bl,"SPAN",{});var Vi=p(Qe);Jl=n(Vi,"Algorithme de tokenisation"),Vi.forEach(a),bl.forEach(a),Vt=m(s),z=r(s,"P",{});var S=p(z);Kl=n(S,"Un mod\xE8le "),sa=r(S,"EM",{});var Fi=p(sa);Ql=n(Fi,"Unigram"),Fi.forEach(a),so=n(S," est un type de mod\xE8le de langage qui consid\xE8re que chaque "),ea=r(S,"EM",{});var Wi=p(ea);eo=n(Wi,"token"),Wi.forEach(a),ao=n(S," est ind\xE9pendant des "),aa=r(S,"EM",{});var Xi=p(aa);to=n(Xi,"tokens"),Xi.forEach(a),no=n(S," qui le pr\xE9c\xE8dent. Il s\u2019agit du mod\xE8le de langage le plus simple, dans le sens o\xF9 la probabilit\xE9 du "),ta=r(S,"EM",{});var Gi=p(ta);lo=n(Gi,"token"),Gi.forEach(a),oo=n(S," X compte tenu du contexte pr\xE9c\xE9dent est simplement la probabilit\xE9 du "),na=r(S,"EM",{});var Yi=p(na);ro=n(Yi,"token"),Yi.forEach(a),po=n(S," X. Ainsi, si nous utilisions un mod\xE8le de langage "),la=r(S,"EM",{});var Zi=p(la);io=n(Zi,"Unigram"),Zi.forEach(a),uo=n(S," pour g\xE9n\xE9rer du texte, nous pr\xE9dirions toujours le "),oa=r(S,"EM",{});var Ji=p(oa);mo=n(Ji,"token"),Ji.forEach(a),co=n(S," le plus courant."),S.forEach(a),Ft=m(s),A=r(s,"P",{});var I=p(A);ho=n(I,"La probabilit\xE9 d\u2019un "),ra=r(I,"EM",{});var Ki=p(ra);fo=n(Ki,"token"),Ki.forEach(a),go=n(I," donn\xE9 est sa fr\xE9quence (le nombre de fois que nous le trouvons) dans le corpus original, divis\xE9e par la somme de toutes les fr\xE9quences de tous les "),pa=r(I,"EM",{});var Qi=p(pa);bo=n(Qi,"tokens"),Qi.forEach(a),vo=n(I," dans le vocabulaire (pour s\u2019assurer que la somme des probabilit\xE9s est \xE9gale \xE0 1). Par exemple, "),ia=r(I,"CODE",{});var su=p(ia);qo=n(su,'"ug"'),su.forEach(a),_o=n(I," est pr\xE9sent dans "),ua=r(I,"CODE",{});var eu=p(ua);jo=n(eu,'"hug"'),eu.forEach(a),ko=n(I,", "),ma=r(I,"CODE",{});var au=p(ma);$o=n(au,'"pug"'),au.forEach(a),wo=n(I,", et "),ca=r(I,"CODE",{});var tu=p(ca);Eo=n(tu,'"hugs"'),tu.forEach(a),yo=n(I,", il a donc une fr\xE9quence de 20 dans notre corpus."),I.forEach(a),Wt=m(s),$e=r(s,"P",{});var nu=p($e);xo=n(nu,"Voici les fr\xE9quences de tous les sous-mots possibles dans le vocabulaire :"),nu.forEach(a),Xt=m(s),h(Rs.$$.fragment,s),Gt=m(s),vs=r(s,"P",{});var vl=p(vs);Po=n(vl,"Ainsi, la somme de toutes les fr\xE9quences est de 210, et la probabilit\xE9 du sous-mot "),da=r(vl,"CODE",{});var lu=p(da);Co=n(lu,'"ug"'),lu.forEach(a),zo=n(vl," est donc de 20/210."),vl.forEach(a),Yt=m(s),h(qs.$$.fragment,s),Zt=m(s),M=r(s,"P",{});var H=p(M);Mo=n(H,"Maintenant, pour tokeniser un mot donn\xE9, nous examinons toutes les segmentations possibles en "),ha=r(H,"EM",{});var ou=p(ha);Ao=n(ou,"tokens"),ou.forEach(a),Do=n(H," et calculons la probabilit\xE9 de chacune d\u2019entre elles selon le mod\xE8le "),fa=r(H,"EM",{});var ru=p(fa);To=n(ru,"Unigram"),ru.forEach(a),No=n(H,". Puisque tous les "),ga=r(H,"EM",{});var pu=p(ga);Uo=n(pu,"tokens"),pu.forEach(a),Oo=n(H," sont consid\xE9r\xE9s comme ind\xE9pendants, cette probabilit\xE9 est juste le produit de la probabilit\xE9 de chaque "),ba=r(H,"EM",{});var iu=p(ba);So=n(iu,"token"),iu.forEach(a),Ho=n(H,". Par exemple, la tokenisation "),va=r(H,"CODE",{});var uu=p(va);Lo=n(uu,'["p", "u", "g"]'),uu.forEach(a),Bo=n(H," de "),qa=r(H,"CODE",{});var mu=p(qa);Io=n(mu,'"pug"'),mu.forEach(a),Ro=n(H,` a la probabilit\xE9 :
`),Jt=Ci(H),H.forEach(a),Kt=m(s),us=r(s,"P",{});var xt=p(us);Vo=n(xt,"Comparativement, la tokenization "),_a=r(xt,"CODE",{});var cu=p(_a);Fo=n(cu,'["pu", "g"]'),cu.forEach(a),Wo=n(xt,` a la probabilit\xE9 :
`),Qt=Ci(xt),xt.forEach(a),sn=m(s),F=r(s,"P",{});var Ts=p(F);Xo=n(Ts,"donc celle-l\xE0 est beaucoup plus probable. En g\xE9n\xE9ral, les tok\xE9nisations comportant le moins de "),ja=r(Ts,"EM",{});var du=p(ja);Go=n(du,"tokens"),du.forEach(a),Yo=n(Ts," possible auront la probabilit\xE9 la plus \xE9lev\xE9e (en raison de la division par 210 r\xE9p\xE9t\xE9e pour chaque "),ka=r(Ts,"EM",{});var hu=p(ka);Zo=n(hu,"token"),hu.forEach(a),Jo=n(Ts,"), ce qui correspond \xE0 ce que nous voulons intuitivement : diviser un mot en un nombre de "),$a=r(Ts,"EM",{});var fu=p($a);Ko=n(fu,"tokens"),fu.forEach(a),Qo=n(Ts," le plus faible possible."),Ts.forEach(a),en=m(s),Q=r(s,"P",{});var Oe=p(Q);sr=n(Oe,"La tokenisation d\u2019un mot avec le mod\xE8le "),wa=r(Oe,"EM",{});var gu=p(wa);er=n(gu,"Unigram"),gu.forEach(a),ar=n(Oe," est donc la tokenisation avec la plus haute probabilit\xE9. Dans l\u2019exemple de "),Ea=r(Oe,"CODE",{});var bu=p(Ea);tr=n(bu,'"pug"'),bu.forEach(a),nr=n(Oe,", voici les probabilit\xE9s que nous obtiendrions pour chaque segmentation possible :"),Oe.forEach(a),an=m(s),h(Vs.$$.fragment,s),tn=m(s),W=r(s,"P",{});var Ns=p(W);lr=n(Ns,"Ainsi, "),ya=r(Ns,"CODE",{});var vu=p(ya);or=n(vu,'"pug"'),vu.forEach(a),rr=n(Ns," sera tokenis\xE9 comme "),xa=r(Ns,"CODE",{});var qu=p(xa);pr=n(qu,'["p", "ug"]'),qu.forEach(a),ir=n(Ns," ou "),Pa=r(Ns,"CODE",{});var _u=p(Pa);ur=n(_u,'["pu", "g"]'),_u.forEach(a),mr=n(Ns,", selon la segmentation rencontr\xE9e en premier (notez que dans un corpus plus large, les cas d\u2019\xE9galit\xE9 comme celui-ci seront rares)."),Ns.forEach(a),nn=m(s),U=r(s,"P",{});var Y=p(U);cr=n(Y,"Dans ce cas, il \xE9tait facile de trouver toutes les segmentations possibles et de calculer leurs probabilit\xE9s, mais en g\xE9n\xE9ral, ce sera un peu plus difficile. Il existe un algorithme classique utilis\xE9 pour cela, appel\xE9 "),Ca=r(Y,"EM",{});var ju=p(Ca);dr=n(ju,"algorithme de Viterbi"),ju.forEach(a),hr=n(Y,". Essentiellement, on peut construire un graphe pour d\xE9tecter les segmentations possibles d\u2019un mot donn\xE9 en disant qu\u2019il existe une branche du caract\xE8re "),za=r(Y,"EM",{});var ku=p(za);fr=n(ku,"a"),ku.forEach(a),gr=n(Y," au caract\xE8re "),Ma=r(Y,"EM",{});var $u=p(Ma);br=n($u,"b"),$u.forEach(a),vr=n(Y," si le sous-mot de "),Aa=r(Y,"EM",{});var wu=p(Aa);qr=n(wu,"a"),wu.forEach(a),_r=n(Y," \xE0 "),Da=r(Y,"EM",{});var Eu=p(Da);jr=n(Eu,"b"),Eu.forEach(a),kr=n(Y," est dans le vocabulaire, et attribuer \xE0 cette branche la probabilit\xE9 du sous-mot."),Y.forEach(a),ln=m(s),we=r(s,"P",{});var yu=p(we);$r=n(yu,"Pour trouver le chemin dans ce graphe qui va avoir le meilleur score, l\u2019algorithme de Viterbi d\xE9termine, pour chaque position dans le mot, la segmentation avec le meilleur score qui se termine \xE0 cette position. Puisque nous allons du d\xE9but \xE0 la fin, ce meilleur score peut \xEAtre trouv\xE9 en parcourant en boucle tous les sous-mots se terminant \xE0 la position actuelle, puis en utilisant le meilleur score de tokenization de la position \xE0 laquelle ce sous-mot commence. Ensuite, il suffit de d\xE9rouler le chemin emprunt\xE9 pour arriver \xE0 la fin."),yu.forEach(a),on=m(s),_s=r(s,"P",{});var ql=p(_s);wr=n(ql,"Prenons un exemple en utilisant notre vocabulaire et le mot "),Ta=r(ql,"CODE",{});var xu=p(Ta);Er=n(xu,'"unhug"'),xu.forEach(a),yr=n(ql,". Pour chaque position, les sous-mots avec les meilleurs scores se terminant l\xE0 sont les suivants :"),ql.forEach(a),rn=m(s),h(Fs.$$.fragment,s),pn=m(s),ss=r(s,"P",{});var Se=p(ss);xr=n(Se,"Ainsi, "),Na=r(Se,"CODE",{});var Pu=p(Na);Pr=n(Pu,'"unhug"'),Pu.forEach(a),Cr=n(Se," serait tokenis\xE9 comme "),Ua=r(Se,"CODE",{});var Cu=p(Ua);zr=n(Cu,'["un", "hug"]'),Cu.forEach(a),Mr=n(Se,"."),Se.forEach(a),un=m(s),h(js.$$.fragment,s),mn=m(s),ms=r(s,"H2",{class:!0});var _l=p(ms);ks=r(_l,"A",{id:!0,class:!0,href:!0});var zu=p(ks);Oa=r(zu,"SPAN",{});var Mu=p(Oa);h(Ws.$$.fragment,Mu),Mu.forEach(a),zu.forEach(a),Ar=m(_l),Sa=r(_l,"SPAN",{});var Au=p(Sa);Dr=n(Au,"Retour \xE0 l'entra\xEEnement"),Au.forEach(a),_l.forEach(a),cn=m(s),es=r(s,"P",{});var He=p(es);Tr=n(He,"Maintenant que nous avons vu comment fonctionne la tokenisation, nous pouvons nous plonger un peu plus profond\xE9ment dans la perte utilis\xE9e pendant l\u2019entra\xEEnement. \xC0 n\u2019importe quelle \xE9tape, cette perte est calcul\xE9e en tokenisant chaque mot du corpus, en utilisant le vocabulaire courant et le mod\xE8le "),Ha=r(He,"EM",{});var Du=p(Ha);Nr=n(Du,"Unigram"),Du.forEach(a),Ur=n(He," d\xE9termin\xE9 par les fr\xE9quences de chaque "),La=r(He,"EM",{});var Tu=p(La);Or=n(Tu,"token"),Tu.forEach(a),Sr=n(He," dans le corpus (comme vu pr\xE9c\xE9demment)."),He.forEach(a),dn=m(s),$s=r(s,"P",{});var jl=p($s);Hr=n(jl,"Chaque mot du corpus a un score, et la perte est le logarithme n\xE9gatif de ces scores : c\u2019est-\xE0-dire la somme pour tous les mots du corpus de tous les "),Ba=r(jl,"CODE",{});var Nu=p(Ba);Lr=n(Nu,"-log(P(word))"),Nu.forEach(a),Br=n(jl,"."),jl.forEach(a),hn=m(s),Ee=r(s,"P",{});var Uu=p(Ee);Ir=n(Uu,"Revenons \xE0 notre exemple avec le corpus suivant :"),Uu.forEach(a),fn=m(s),h(Xs.$$.fragment,s),gn=m(s),ye=r(s,"P",{});var Ou=p(ye);Rr=n(Ou,"La tokenisation de chaque mot avec leurs scores respectifs est :"),Ou.forEach(a),bn=m(s),h(Gs.$$.fragment,s),vn=m(s),xe=r(s,"P",{});var Su=p(xe);Vr=n(Su,"Donc la perte est :"),Su.forEach(a),qn=m(s),h(Ys.$$.fragment,s),_n=m(s),L=r(s,"P",{});var os=p(L);Fr=n(os,"Maintenant, nous devons calculer comment la suppression de chaque token affecte la perte. C\u2019est plut\xF4t fastidieux, donc nous allons le faire pour deux "),Ia=r(os,"EM",{});var Hu=p(Ia);Wr=n(Hu,"tokens"),Hu.forEach(a),Xr=n(os," ici et garder tout le processus pour quand nous aurons du code pour nous aider. Dans ce cas (tr\xE8s) particulier, nous avions deux tokenizations \xE9quivalentes de tous les mots : comme nous l\u2019avons vu pr\xE9c\xE9demment, par exemple, "),Ra=r(os,"CODE",{});var Lu=p(Ra);Gr=n(Lu,'"pug"'),Lu.forEach(a),Yr=n(os," pourrait \xEAtre tokenis\xE9 "),Va=r(os,"CODE",{});var Bu=p(Va);Zr=n(Bu,'["p", "ug"]'),Bu.forEach(a),Jr=n(os," avec le m\xEAme score. Ainsi, enlever le token "),Fa=r(os,"CODE",{});var Iu=p(Fa);Kr=n(Iu,'"pu"'),Iu.forEach(a),Qr=n(os," du vocabulaire donnera exactement la m\xEAme perte."),os.forEach(a),jn=m(s),X=r(s,"P",{});var Us=p(X);sp=n(Us,"D\u2019un autre c\xF4t\xE9, supprimer le mot "),Wa=r(Us,"CODE",{});var Ru=p(Wa);ep=n(Ru,'"hug"'),Ru.forEach(a),ap=n(Us," aggravera la perte, car la tokenisation de "),Xa=r(Us,"CODE",{});var Vu=p(Xa);tp=n(Vu,'"hug"'),Vu.forEach(a),np=n(Us," et "),Ga=r(Us,"CODE",{});var Fu=p(Ga);lp=n(Fu,'"hugs"'),Fu.forEach(a),op=n(Us," deviendra :"),Us.forEach(a),kn=m(s),h(Zs.$$.fragment,s),$n=m(s),Pe=r(s,"P",{});var Wu=p(Pe);rp=n(Wu,"Ces changements entra\xEEneront une augmentation de la perte de :"),Wu.forEach(a),wn=m(s),h(Js.$$.fragment,s),En=m(s),as=r(s,"P",{});var Le=p(as);pp=n(Le,"Par cons\xE9quent, le token "),Ya=r(Le,"CODE",{});var Xu=p(Ya);ip=n(Xu,'"pu"'),Xu.forEach(a),up=n(Le," sera probablement retir\xE9 du vocabulaire, mais pas "),Za=r(Le,"CODE",{});var Gu=p(Za);mp=n(Gu,'"hug"'),Gu.forEach(a),cp=n(Le,"."),Le.forEach(a),yn=m(s),cs=r(s,"H2",{class:!0});var kl=p(cs);ws=r(kl,"A",{id:!0,class:!0,href:!0});var Yu=p(ws);Ja=r(Yu,"SPAN",{});var Zu=p(Ja);h(Ks.$$.fragment,Zu),Zu.forEach(a),Yu.forEach(a),dp=m(kl),Ka=r(kl,"SPAN",{});var Ju=p(Ka);hp=n(Ju,"Impl\xE9mentation d'*Unigram*"),Ju.forEach(a),kl.forEach(a),xn=m(s),ts=r(s,"P",{});var Be=p(ts);fp=n(Be,"Maintenant, impl\xE9mentons tout ce que nous avons vu jusqu\u2019\xE0 pr\xE9sent dans le code. Comme pour BPE et "),Qa=r(Be,"EM",{});var Ku=p(Qa);gp=n(Ku,"WordPiece"),Ku.forEach(a),bp=n(Be,", ce n\u2019est pas une impl\xE9mentation efficace de l\u2019algorithme "),st=r(Be,"EM",{});var Qu=p(st);vp=n(Qu,"Unigram"),Qu.forEach(a),qp=n(Be," (bien au contraire), mais cela devrait vous aider \xE0 le comprendre un peu mieux."),Be.forEach(a),Pn=m(s),Ce=r(s,"P",{});var sm=p(Ce);_p=n(sm,"Nous allons utiliser le m\xEAme corpus que pr\xE9c\xE9demment comme exemple :"),sm.forEach(a),Cn=m(s),h(Qs.$$.fragment,s),zn=m(s),Es=r(s,"P",{});var $l=p(Es);jp=n($l,"Cette fois, nous allons utiliser "),et=r($l,"CODE",{});var em=p(et);kp=n(em,"xlnet-base-cased"),em.forEach(a),$p=n($l," comme mod\xE8le :"),$l.forEach(a),Mn=m(s),h(se.$$.fragment,s),An=m(s),ys=r(s,"P",{});var wl=p(ys);wp=n(wl,"Comme pour BPE et "),at=r(wl,"EM",{});var am=p(at);Ep=n(am,"WordPiece"),am.forEach(a),yp=n(wl,", nous commen\xE7ons par compter le nombre d\u2019occurrences de chaque mot dans le corpus :"),wl.forEach(a),Dn=m(s),h(ee.$$.fragment,s),Tn=m(s),ze=r(s,"P",{});var tm=p(ze);xp=n(tm,"Ensuite, nous devons initialiser notre vocabulaire \xE0 quelque chose de plus grand que la taille du vocabulaire que nous voudrons \xE0 la fin. Nous devons inclure tous les caract\xE8res de base (sinon nous ne serons pas en mesure de tokeniser chaque mot), mais pour les sous-cha\xEEnes plus grandes, nous ne garderons que les plus communs, donc nous les trions par fr\xE9quence :"),tm.forEach(a),Nn=m(s),h(ae.$$.fragment,s),Un=m(s),h(te.$$.fragment,s),On=m(s),Me=r(s,"P",{});var nm=p(Me);Pp=n(nm,"Nous regroupons les caract\xE8res avec les meilleurs sous-mots pour arriver \xE0 un vocabulaire initial de taille 300 :"),nm.forEach(a),Sn=m(s),h(ne.$$.fragment,s),Hn=m(s),h(xs.$$.fragment,s),Ln=m(s),Ae=r(s,"P",{});var lm=p(Ae);Cp=n(lm,"Ensuite, nous calculons la somme de toutes les fr\xE9quences, pour convertir les fr\xE9quences en probabilit\xE9s. Pour notre mod\xE8le, nous allons stocker les logarithmes des probabilit\xE9s, car il est plus stable num\xE9riquement d\u2019additionner des logarithmes que de multiplier des petits nombres, et cela simplifiera le calcul de la perte du mod\xE8le :"),lm.forEach(a),Bn=m(s),h(le.$$.fragment,s),In=m(s),G=r(s,"P",{});var Os=p(G);zp=n(Os,"Maintenant la fonction principale est celle qui tokenise les mots en utilisant l\u2019algorithme de Viterbi. Comme nous l\u2019avons vu pr\xE9c\xE9demment, cet algorithme calcule la meilleure segmentation de chaque sous-cha\xEEne du mot, que nous allons stocker dans une variable nomm\xE9e "),tt=r(Os,"CODE",{});var om=p(tt);Mp=n(om,"best_segmentations"),om.forEach(a),Ap=n(Os,". Nous allons stocker un dictionnaire par position dans le mot (de 0 \xE0 sa longueur totale), avec deux cl\xE9s : l\u2019index du d\xE9but du dernier "),nt=r(Os,"EM",{});var rm=p(nt);Dp=n(rm,"token"),rm.forEach(a),Tp=n(Os," dans la meilleure segmentation, et le score de la meilleure segmentation. Avec l\u2019index du d\xE9but du dernier "),lt=r(Os,"EM",{});var pm=p(lt);Np=n(pm,"token"),pm.forEach(a),Up=n(Os,", nous serons en mesure de r\xE9cup\xE9rer la segmentation compl\xE8te une fois que la liste est compl\xE8tement remplie."),Os.forEach(a),Rn=m(s),Ps=r(s,"P",{});var El=p(Ps);Op=n(El,"Le remplissage de la liste se fait \xE0 l\u2019aide de deux boucles seulement : la boucle principale passe en revue chaque position de d\xE9part, et la seconde boucle essaie toutes les sous-cha\xEEnes commen\xE7ant \xE0 cette position de d\xE9part. Si la sous-cha\xEEne est dans le vocabulaire, nous avons une nouvelle segmentation du mot jusqu\u2019\xE0 cette position finale, que nous comparons \xE0 ce qui est dans "),ot=r(El,"CODE",{});var im=p(ot);Sp=n(im,"best_segmentations"),im.forEach(a),Hp=n(El,"."),El.forEach(a),Vn=m(s),Cs=r(s,"P",{});var yl=p(Cs);Lp=n(yl,"Une fois que la boucle principale est termin\xE9e, nous commen\xE7ons juste \xE0 la fin et sautons d\u2019une position de d\xE9part \xE0 une autre, en enregistrant les "),rt=r(yl,"EM",{});var um=p(rt);Bp=n(um,"tokens"),um.forEach(a),Ip=n(yl," au fur et \xE0 mesure, jusqu\u2019\xE0 ce que nous atteignions le d\xE9but du mot :"),yl.forEach(a),Fn=m(s),h(oe.$$.fragment,s),Wn=m(s),De=r(s,"P",{});var mm=p(De);Rp=n(mm,"Nous pouvons d\xE9j\xE0 essayer notre mod\xE8le initial sur quelques mots :"),mm.forEach(a),Xn=m(s),h(re.$$.fragment,s),Gn=m(s),h(pe.$$.fragment,s),Yn=m(s),Te=r(s,"P",{});var cm=p(Te);Vp=n(cm,"Il est maintenant facile de calculer la perte du mod\xE8le sur le corpus !"),cm.forEach(a),Zn=m(s),h(ie.$$.fragment,s),Jn=m(s),Ne=r(s,"P",{});var dm=p(Ne);Fp=n(dm,"Nous pouvons v\xE9rifier que cela fonctionne sur le mod\xE8le que nous avons :"),dm.forEach(a),Kn=m(s),h(ue.$$.fragment,s),Qn=m(s),h(me.$$.fragment,s),sl=m(s),ns=r(s,"P",{});var Ie=p(ns);Wp=n(Ie,"Le calcul des scores pour chaque "),pt=r(Ie,"EM",{});var hm=p(pt);Xp=n(hm,"token"),hm.forEach(a),Gp=n(Ie," n\u2019est pas tr\xE8s difficile non plus ; il suffit de calculer la perte pour les mod\xE8les obtenus en supprimant chaque "),it=r(Ie,"EM",{});var fm=p(it);Yp=n(fm,"token"),fm.forEach(a),Zp=n(Ie," :"),Ie.forEach(a),el=m(s),h(ce.$$.fragment,s),al=m(s),zs=r(s,"P",{});var xl=p(zs);Jp=n(xl,"Nous pouvons l\u2019essayer sur un "),ut=r(xl,"EM",{});var gm=p(ut);Kp=n(gm,"token"),gm.forEach(a),Qp=n(xl," donn\xE9 :"),xl.forEach(a),tl=m(s),h(de.$$.fragment,s),nl=m(s),O=r(s,"P",{});var Z=p(O);si=n(Z,"Puisque "),mt=r(Z,"CODE",{});var bm=p(mt);ei=n(bm,'"ll"'),bm.forEach(a),ai=n(Z," est utilis\xE9 dans la tokenisation de "),ct=r(Z,"CODE",{});var vm=p(ct);ti=n(vm,'"Hopefully"'),vm.forEach(a),ni=n(Z,", et que le supprimer nous fera probablement utiliser le token "),dt=r(Z,"CODE",{});var qm=p(dt);li=n(qm,'"l"'),qm.forEach(a),oi=n(Z," deux fois \xE0 la place, nous nous attendons \xE0 ce qu\u2019il ait une perte positive. "),ht=r(Z,"CODE",{});var _m=p(ht);ri=n(_m,'"his"'),_m.forEach(a),pi=n(Z," n\u2019est utilis\xE9 qu\u2019\xE0 l\u2019int\xE9rieur du mot "),ft=r(Z,"CODE",{});var jm=p(ft);ii=n(jm,'"This"'),jm.forEach(a),ui=n(Z,", qui est tokenis\xE9 comme lui-m\xEAme, donc nous nous attendons \xE0 ce qu\u2019il ait une perte nulle. Voici les r\xE9sultats :"),Z.forEach(a),ll=m(s),h(he.$$.fragment,s),ol=m(s),h(Ms.$$.fragment,s),rl=m(s),ls=r(s,"P",{});var Re=p(ls);mi=n(Re,"Une fois tout cela en place, la derni\xE8re chose \xE0 faire est d\u2019ajouter les "),gt=r(Re,"EM",{});var km=p(gt);ci=n(km,"tokens"),km.forEach(a),di=n(Re," sp\xE9ciaux utilis\xE9s par le mod\xE8le au vocabulaire, puis de boucler jusqu\u2019\xE0 ce que nous ayons \xE9lagu\xE9 suffisamment de "),bt=r(Re,"EM",{});var $m=p(bt);hi=n($m,"tokens"),$m.forEach(a),fi=n(Re," du vocabulaire pour atteindre la taille souhait\xE9e :"),Re.forEach(a),pl=m(s),h(fe.$$.fragment,s),il=m(s),As=r(s,"P",{});var Pl=p(As);gi=n(Pl,"Ensuite, pour tokeniser un texte, il suffit d\u2019appliquer la pr\xE9-tok\xE9nisation et d\u2019utiliser la fonction "),vt=r(Pl,"CODE",{});var wm=p(vt);bi=n(wm,"encode_word()"),wm.forEach(a),vi=n(Pl," :"),Pl.forEach(a),ul=m(s),h(ge.$$.fragment,s),ml=m(s),h(be.$$.fragment,s),cl=m(s),B=r(s,"P",{});var rs=p(B);qi=n(rs,"C\u2019est tout pour "),qt=r(rs,"EM",{});var Em=p(qt);_i=n(Em,"Unigram"),Em.forEach(a),ji=n(rs," ! Avec un peu de chance, vous vous sentez maintenant comme un expert en tout ce qui concerne les "),_t=r(rs,"EM",{});var ym=p(_t);ki=n(ym,"tokenizers"),ym.forEach(a),$i=n(rs,". Dans la prochaine section, nous allons nous plonger dans les blocs de construction de la biblioth\xE8que \u{1F917} "),jt=r(rs,"EM",{});var xm=p(jt);wi=n(xm,"Tokenizers"),xm.forEach(a),Ei=n(rs,", et vous montrer comment vous pouvez les utiliser pour construire votre propre "),kt=r(rs,"EM",{});var Pm=p(kt);yi=n(Pm,"tokenizer"),Pm.forEach(a),xi=n(rs,"."),rs.forEach(a),this.h()},h(){P(c,"name","hf:doc:metadata"),P(c,"content",JSON.stringify(Fm)),P($,"id","tokenisation-unigram"),P($,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),P($,"href","#tokenisation-unigram"),P(q,"class","relative group"),P(hs,"id","algorithme-dentranement"),P(hs,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),P(hs,"href","#algorithme-dentranement"),P(ps,"class","relative group"),Nt.a=Ut,P(bs,"id","algorithme-de-tokenisation"),P(bs,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),P(bs,"href","#algorithme-de-tokenisation"),P(is,"class","relative group"),Jt.a=null,Qt.a=null,P(ks,"id","retour-lentranement"),P(ks,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),P(ks,"href","#retour-lentranement"),P(ms,"class","relative group"),P(ws,"id","implmentation-dunigram"),P(ws,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),P(ws,"href","#implmentation-dunigram"),P(cs,"class","relative group")},m(s,l){e(document.head,c),i(s,y,l),i(s,q,l),e(q,$),e($,E),f(_,E,null),e(q,j),e(q,x),e(x,C),i(s,w,l),f(D,s,l),i(s,N,l),i(s,T,l),e(T,qe),i(s,Ss,l),f(J,s,l),i(s,zt,l),f(ds,s,l),i(s,Mt,l),i(s,ps,l),e(ps,hs),e(hs,Ve),f(Hs,Ve,null),e(ps,Cl),e(ps,Fe),e(Fe,zl),i(s,At,l),i(s,V,l),e(V,Ml),e(V,We),e(We,Al),e(V,Dl),e(V,Xe),e(Xe,Tl),e(V,Nl),e(V,Ge),e(Ge,Ul),e(V,Ol),i(s,Dt,l),i(s,fs,l),e(fs,Sl),e(fs,Ye),e(Ye,Hl),e(fs,Ll),i(s,Tt,l),i(s,gs,l),e(gs,Bl),Nt.m(Cm,gs),e(gs,Ut),i(s,Ot,l),i(s,_e,l),e(_e,Il),i(s,St,l),i(s,K,l),e(K,Rl),e(K,Ze),e(Ze,Vl),e(K,Fl),e(K,Je),e(Je,Wl),e(K,Xl),i(s,Ht,l),i(s,je,l),e(je,Gl),i(s,Lt,l),f(Ls,s,l),i(s,Bt,l),i(s,ke,l),e(ke,Yl),i(s,It,l),f(Bs,s,l),i(s,Rt,l),i(s,is,l),e(is,bs),e(bs,Ke),f(Is,Ke,null),e(is,Zl),e(is,Qe),e(Qe,Jl),i(s,Vt,l),i(s,z,l),e(z,Kl),e(z,sa),e(sa,Ql),e(z,so),e(z,ea),e(ea,eo),e(z,ao),e(z,aa),e(aa,to),e(z,no),e(z,ta),e(ta,lo),e(z,oo),e(z,na),e(na,ro),e(z,po),e(z,la),e(la,io),e(z,uo),e(z,oa),e(oa,mo),e(z,co),i(s,Ft,l),i(s,A,l),e(A,ho),e(A,ra),e(ra,fo),e(A,go),e(A,pa),e(pa,bo),e(A,vo),e(A,ia),e(ia,qo),e(A,_o),e(A,ua),e(ua,jo),e(A,ko),e(A,ma),e(ma,$o),e(A,wo),e(A,ca),e(ca,Eo),e(A,yo),i(s,Wt,l),i(s,$e,l),e($e,xo),i(s,Xt,l),f(Rs,s,l),i(s,Gt,l),i(s,vs,l),e(vs,Po),e(vs,da),e(da,Co),e(vs,zo),i(s,Yt,l),f(qs,s,l),i(s,Zt,l),i(s,M,l),e(M,Mo),e(M,ha),e(ha,Ao),e(M,Do),e(M,fa),e(fa,To),e(M,No),e(M,ga),e(ga,Uo),e(M,Oo),e(M,ba),e(ba,So),e(M,Ho),e(M,va),e(va,Lo),e(M,Bo),e(M,qa),e(qa,Io),e(M,Ro),Jt.m(zm,M),i(s,Kt,l),i(s,us,l),e(us,Vo),e(us,_a),e(_a,Fo),e(us,Wo),Qt.m(Mm,us),i(s,sn,l),i(s,F,l),e(F,Xo),e(F,ja),e(ja,Go),e(F,Yo),e(F,ka),e(ka,Zo),e(F,Jo),e(F,$a),e($a,Ko),e(F,Qo),i(s,en,l),i(s,Q,l),e(Q,sr),e(Q,wa),e(wa,er),e(Q,ar),e(Q,Ea),e(Ea,tr),e(Q,nr),i(s,an,l),f(Vs,s,l),i(s,tn,l),i(s,W,l),e(W,lr),e(W,ya),e(ya,or),e(W,rr),e(W,xa),e(xa,pr),e(W,ir),e(W,Pa),e(Pa,ur),e(W,mr),i(s,nn,l),i(s,U,l),e(U,cr),e(U,Ca),e(Ca,dr),e(U,hr),e(U,za),e(za,fr),e(U,gr),e(U,Ma),e(Ma,br),e(U,vr),e(U,Aa),e(Aa,qr),e(U,_r),e(U,Da),e(Da,jr),e(U,kr),i(s,ln,l),i(s,we,l),e(we,$r),i(s,on,l),i(s,_s,l),e(_s,wr),e(_s,Ta),e(Ta,Er),e(_s,yr),i(s,rn,l),f(Fs,s,l),i(s,pn,l),i(s,ss,l),e(ss,xr),e(ss,Na),e(Na,Pr),e(ss,Cr),e(ss,Ua),e(Ua,zr),e(ss,Mr),i(s,un,l),f(js,s,l),i(s,mn,l),i(s,ms,l),e(ms,ks),e(ks,Oa),f(Ws,Oa,null),e(ms,Ar),e(ms,Sa),e(Sa,Dr),i(s,cn,l),i(s,es,l),e(es,Tr),e(es,Ha),e(Ha,Nr),e(es,Ur),e(es,La),e(La,Or),e(es,Sr),i(s,dn,l),i(s,$s,l),e($s,Hr),e($s,Ba),e(Ba,Lr),e($s,Br),i(s,hn,l),i(s,Ee,l),e(Ee,Ir),i(s,fn,l),f(Xs,s,l),i(s,gn,l),i(s,ye,l),e(ye,Rr),i(s,bn,l),f(Gs,s,l),i(s,vn,l),i(s,xe,l),e(xe,Vr),i(s,qn,l),f(Ys,s,l),i(s,_n,l),i(s,L,l),e(L,Fr),e(L,Ia),e(Ia,Wr),e(L,Xr),e(L,Ra),e(Ra,Gr),e(L,Yr),e(L,Va),e(Va,Zr),e(L,Jr),e(L,Fa),e(Fa,Kr),e(L,Qr),i(s,jn,l),i(s,X,l),e(X,sp),e(X,Wa),e(Wa,ep),e(X,ap),e(X,Xa),e(Xa,tp),e(X,np),e(X,Ga),e(Ga,lp),e(X,op),i(s,kn,l),f(Zs,s,l),i(s,$n,l),i(s,Pe,l),e(Pe,rp),i(s,wn,l),f(Js,s,l),i(s,En,l),i(s,as,l),e(as,pp),e(as,Ya),e(Ya,ip),e(as,up),e(as,Za),e(Za,mp),e(as,cp),i(s,yn,l),i(s,cs,l),e(cs,ws),e(ws,Ja),f(Ks,Ja,null),e(cs,dp),e(cs,Ka),e(Ka,hp),i(s,xn,l),i(s,ts,l),e(ts,fp),e(ts,Qa),e(Qa,gp),e(ts,bp),e(ts,st),e(st,vp),e(ts,qp),i(s,Pn,l),i(s,Ce,l),e(Ce,_p),i(s,Cn,l),f(Qs,s,l),i(s,zn,l),i(s,Es,l),e(Es,jp),e(Es,et),e(et,kp),e(Es,$p),i(s,Mn,l),f(se,s,l),i(s,An,l),i(s,ys,l),e(ys,wp),e(ys,at),e(at,Ep),e(ys,yp),i(s,Dn,l),f(ee,s,l),i(s,Tn,l),i(s,ze,l),e(ze,xp),i(s,Nn,l),f(ae,s,l),i(s,Un,l),f(te,s,l),i(s,On,l),i(s,Me,l),e(Me,Pp),i(s,Sn,l),f(ne,s,l),i(s,Hn,l),f(xs,s,l),i(s,Ln,l),i(s,Ae,l),e(Ae,Cp),i(s,Bn,l),f(le,s,l),i(s,In,l),i(s,G,l),e(G,zp),e(G,tt),e(tt,Mp),e(G,Ap),e(G,nt),e(nt,Dp),e(G,Tp),e(G,lt),e(lt,Np),e(G,Up),i(s,Rn,l),i(s,Ps,l),e(Ps,Op),e(Ps,ot),e(ot,Sp),e(Ps,Hp),i(s,Vn,l),i(s,Cs,l),e(Cs,Lp),e(Cs,rt),e(rt,Bp),e(Cs,Ip),i(s,Fn,l),f(oe,s,l),i(s,Wn,l),i(s,De,l),e(De,Rp),i(s,Xn,l),f(re,s,l),i(s,Gn,l),f(pe,s,l),i(s,Yn,l),i(s,Te,l),e(Te,Vp),i(s,Zn,l),f(ie,s,l),i(s,Jn,l),i(s,Ne,l),e(Ne,Fp),i(s,Kn,l),f(ue,s,l),i(s,Qn,l),f(me,s,l),i(s,sl,l),i(s,ns,l),e(ns,Wp),e(ns,pt),e(pt,Xp),e(ns,Gp),e(ns,it),e(it,Yp),e(ns,Zp),i(s,el,l),f(ce,s,l),i(s,al,l),i(s,zs,l),e(zs,Jp),e(zs,ut),e(ut,Kp),e(zs,Qp),i(s,tl,l),f(de,s,l),i(s,nl,l),i(s,O,l),e(O,si),e(O,mt),e(mt,ei),e(O,ai),e(O,ct),e(ct,ti),e(O,ni),e(O,dt),e(dt,li),e(O,oi),e(O,ht),e(ht,ri),e(O,pi),e(O,ft),e(ft,ii),e(O,ui),i(s,ll,l),f(he,s,l),i(s,ol,l),f(Ms,s,l),i(s,rl,l),i(s,ls,l),e(ls,mi),e(ls,gt),e(gt,ci),e(ls,di),e(ls,bt),e(bt,hi),e(ls,fi),i(s,pl,l),f(fe,s,l),i(s,il,l),i(s,As,l),e(As,gi),e(As,vt),e(vt,bi),e(As,vi),i(s,ul,l),f(ge,s,l),i(s,ml,l),f(be,s,l),i(s,cl,l),i(s,B,l),e(B,qi),e(B,qt),e(qt,_i),e(B,ji),e(B,_t),e(_t,ki),e(B,$i),e(B,jt),e(jt,wi),e(B,Ei),e(B,kt),e(kt,yi),e(B,xi),dl=!0},p(s,[l]){const ve={};l&2&&(ve.$$scope={dirty:l,ctx:s}),ds.$set(ve);const $t={};l&2&&($t.$$scope={dirty:l,ctx:s}),qs.$set($t);const wt={};l&2&&(wt.$$scope={dirty:l,ctx:s}),js.$set(wt);const Et={};l&2&&(Et.$$scope={dirty:l,ctx:s}),xs.$set(Et);const yt={};l&2&&(yt.$$scope={dirty:l,ctx:s}),Ms.$set(yt)},i(s){dl||(g(_.$$.fragment,s),g(D.$$.fragment,s),g(J.$$.fragment,s),g(ds.$$.fragment,s),g(Hs.$$.fragment,s),g(Ls.$$.fragment,s),g(Bs.$$.fragment,s),g(Is.$$.fragment,s),g(Rs.$$.fragment,s),g(qs.$$.fragment,s),g(Vs.$$.fragment,s),g(Fs.$$.fragment,s),g(js.$$.fragment,s),g(Ws.$$.fragment,s),g(Xs.$$.fragment,s),g(Gs.$$.fragment,s),g(Ys.$$.fragment,s),g(Zs.$$.fragment,s),g(Js.$$.fragment,s),g(Ks.$$.fragment,s),g(Qs.$$.fragment,s),g(se.$$.fragment,s),g(ee.$$.fragment,s),g(ae.$$.fragment,s),g(te.$$.fragment,s),g(ne.$$.fragment,s),g(xs.$$.fragment,s),g(le.$$.fragment,s),g(oe.$$.fragment,s),g(re.$$.fragment,s),g(pe.$$.fragment,s),g(ie.$$.fragment,s),g(ue.$$.fragment,s),g(me.$$.fragment,s),g(ce.$$.fragment,s),g(de.$$.fragment,s),g(he.$$.fragment,s),g(Ms.$$.fragment,s),g(fe.$$.fragment,s),g(ge.$$.fragment,s),g(be.$$.fragment,s),dl=!0)},o(s){b(_.$$.fragment,s),b(D.$$.fragment,s),b(J.$$.fragment,s),b(ds.$$.fragment,s),b(Hs.$$.fragment,s),b(Ls.$$.fragment,s),b(Bs.$$.fragment,s),b(Is.$$.fragment,s),b(Rs.$$.fragment,s),b(qs.$$.fragment,s),b(Vs.$$.fragment,s),b(Fs.$$.fragment,s),b(js.$$.fragment,s),b(Ws.$$.fragment,s),b(Xs.$$.fragment,s),b(Gs.$$.fragment,s),b(Ys.$$.fragment,s),b(Zs.$$.fragment,s),b(Js.$$.fragment,s),b(Ks.$$.fragment,s),b(Qs.$$.fragment,s),b(se.$$.fragment,s),b(ee.$$.fragment,s),b(ae.$$.fragment,s),b(te.$$.fragment,s),b(ne.$$.fragment,s),b(xs.$$.fragment,s),b(le.$$.fragment,s),b(oe.$$.fragment,s),b(re.$$.fragment,s),b(pe.$$.fragment,s),b(ie.$$.fragment,s),b(ue.$$.fragment,s),b(me.$$.fragment,s),b(ce.$$.fragment,s),b(de.$$.fragment,s),b(he.$$.fragment,s),b(Ms.$$.fragment,s),b(fe.$$.fragment,s),b(ge.$$.fragment,s),b(be.$$.fragment,s),dl=!1},d(s){a(c),s&&a(y),s&&a(q),v(_),s&&a(w),v(D,s),s&&a(N),s&&a(T),s&&a(Ss),v(J,s),s&&a(zt),v(ds,s),s&&a(Mt),s&&a(ps),v(Hs),s&&a(At),s&&a(V),s&&a(Dt),s&&a(fs),s&&a(Tt),s&&a(gs),s&&a(Ot),s&&a(_e),s&&a(St),s&&a(K),s&&a(Ht),s&&a(je),s&&a(Lt),v(Ls,s),s&&a(Bt),s&&a(ke),s&&a(It),v(Bs,s),s&&a(Rt),s&&a(is),v(Is),s&&a(Vt),s&&a(z),s&&a(Ft),s&&a(A),s&&a(Wt),s&&a($e),s&&a(Xt),v(Rs,s),s&&a(Gt),s&&a(vs),s&&a(Yt),v(qs,s),s&&a(Zt),s&&a(M),s&&a(Kt),s&&a(us),s&&a(sn),s&&a(F),s&&a(en),s&&a(Q),s&&a(an),v(Vs,s),s&&a(tn),s&&a(W),s&&a(nn),s&&a(U),s&&a(ln),s&&a(we),s&&a(on),s&&a(_s),s&&a(rn),v(Fs,s),s&&a(pn),s&&a(ss),s&&a(un),v(js,s),s&&a(mn),s&&a(ms),v(Ws),s&&a(cn),s&&a(es),s&&a(dn),s&&a($s),s&&a(hn),s&&a(Ee),s&&a(fn),v(Xs,s),s&&a(gn),s&&a(ye),s&&a(bn),v(Gs,s),s&&a(vn),s&&a(xe),s&&a(qn),v(Ys,s),s&&a(_n),s&&a(L),s&&a(jn),s&&a(X),s&&a(kn),v(Zs,s),s&&a($n),s&&a(Pe),s&&a(wn),v(Js,s),s&&a(En),s&&a(as),s&&a(yn),s&&a(cs),v(Ks),s&&a(xn),s&&a(ts),s&&a(Pn),s&&a(Ce),s&&a(Cn),v(Qs,s),s&&a(zn),s&&a(Es),s&&a(Mn),v(se,s),s&&a(An),s&&a(ys),s&&a(Dn),v(ee,s),s&&a(Tn),s&&a(ze),s&&a(Nn),v(ae,s),s&&a(Un),v(te,s),s&&a(On),s&&a(Me),s&&a(Sn),v(ne,s),s&&a(Hn),v(xs,s),s&&a(Ln),s&&a(Ae),s&&a(Bn),v(le,s),s&&a(In),s&&a(G),s&&a(Rn),s&&a(Ps),s&&a(Vn),s&&a(Cs),s&&a(Fn),v(oe,s),s&&a(Wn),s&&a(De),s&&a(Xn),v(re,s),s&&a(Gn),v(pe,s),s&&a(Yn),s&&a(Te),s&&a(Zn),v(ie,s),s&&a(Jn),s&&a(Ne),s&&a(Kn),v(ue,s),s&&a(Qn),v(me,s),s&&a(sl),s&&a(ns),s&&a(el),v(ce,s),s&&a(al),s&&a(zs),s&&a(tl),v(de,s),s&&a(nl),s&&a(O),s&&a(ll),v(he,s),s&&a(ol),v(Ms,s),s&&a(rl),s&&a(ls),s&&a(pl),v(fe,s),s&&a(il),s&&a(As),s&&a(ul),v(ge,s),s&&a(ml),v(be,s),s&&a(cl),s&&a(B)}}}const Fm={local:"tokenisation-unigram",sections:[{local:"algorithme-dentranement",title:"Algorithme d'entra\xEEnement"},{local:"algorithme-de-tokenisation",title:"Algorithme de tokenisation"},{local:"retour-lentranement",title:"Retour \xE0 l'entra\xEEnement"},{local:"implmentation-dunigram",title:"Impl\xE9mentation d'*Unigram*"}],title:"Tokenisation *Unigram*"};function Wm(R){return Um(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class Qm extends Am{constructor(c){super();Dm(this,c,Wm,Vm,Tm,{})}}export{Qm as default,Fm as metadata};
