import{S as Gi,i as Ji,s as Qi,e as r,k as u,w as h,t as n,U as Yi,M as Xi,c as p,d as a,m as c,a as o,x as d,h as t,V as Zi,b as B,G as e,g as i,y as f,q as j,o as g,B as b,v as su}from"../../chunks/vendor-hf-doc-builder.js";import{T as en}from"../../chunks/Tip-hf-doc-builder.js";import{Y as eu}from"../../chunks/Youtube-hf-doc-builder.js";import{I as Bt}from"../../chunks/IconCopyLink-hf-doc-builder.js";import{C as _}from"../../chunks/CodeBlock-hf-doc-builder.js";import{D as au}from"../../chunks/DocNotebookDropdown-hf-doc-builder.js";function nu(F){let m,k,x,$,w;return{c(){m=r("p"),k=n("\u{1F4A1} Cette section couvre le "),x=r("i"),$=n("WordPiece"),w=n(" en profondeur, allant jusqu\u2019\xE0 montrer une impl\xE9mentation compl\xE8te. Vous pouvez passer directement \xE0 la fin si vous souhaitez simplement avoir un aper\xE7u g\xE9n\xE9ral de l\u2019algorithme de tok\xE9nisation.")},l(v){m=p(v,"P",{});var q=o(m);k=t(q,"\u{1F4A1} Cette section couvre le "),x=p(q,"I",{});var y=o(x);$=t(y,"WordPiece"),y.forEach(a),w=t(q," en profondeur, allant jusqu\u2019\xE0 montrer une impl\xE9mentation compl\xE8te. Vous pouvez passer directement \xE0 la fin si vous souhaitez simplement avoir un aper\xE7u g\xE9n\xE9ral de l\u2019algorithme de tok\xE9nisation."),q.forEach(a)},m(v,q){i(v,m,q),e(m,k),e(m,x),e(x,$),e(m,w)},d(v){v&&a(m)}}}function tu(F){let m,k,x,$,w;return{c(){m=r("p"),k=n("\u26A0\uFE0F Google n\u2019a jamais mis en ligne son impl\xE9mentation de l\u2019algorithme d\u2019entra\xEEnement du "),x=r("i"),$=n("WordPiece"),w=n(". Ce qui suit est donc notre meilleure estimation bas\xE9e sur la litt\xE9rature publi\xE9e. Il se peut qu\u2019elle ne soit pas exacte \xE0 100 %.")},l(v){m=p(v,"P",{});var q=o(m);k=t(q,"\u26A0\uFE0F Google n\u2019a jamais mis en ligne son impl\xE9mentation de l\u2019algorithme d\u2019entra\xEEnement du "),x=p(q,"I",{});var y=o(x);$=t(y,"WordPiece"),y.forEach(a),w=t(q,". Ce qui suit est donc notre meilleure estimation bas\xE9e sur la litt\xE9rature publi\xE9e. Il se peut qu\u2019elle ne soit pas exacte \xE0 100 %."),q.forEach(a)},m(v,q){i(v,m,q),e(m,k),e(m,x),e(x,$),e(m,w)},d(v){v&&a(m)}}}function lu(F){let m,k,x,$,w;return{c(){m=r("p"),k=n("\u270F\uFE0F "),x=r("strong"),$=n("A votre tour !"),w=n(" Quelle sera la prochaine r\xE8gle de fusion ?")},l(v){m=p(v,"P",{});var q=o(m);k=t(q,"\u270F\uFE0F "),x=p(q,"STRONG",{});var y=o(x);$=t(y,"A votre tour !"),y.forEach(a),w=t(q," Quelle sera la prochaine r\xE8gle de fusion ?"),q.forEach(a)},m(v,q){i(v,m,q),e(m,k),e(m,x),e(x,$),e(m,w)},d(v){v&&a(m)}}}function ru(F){let m,k,x,$,w,v,q,y;return{c(){m=r("p"),k=n("\u270F\uFE0F "),x=r("strong"),$=n("A votre tour !"),w=n(" Comment le mot "),v=r("code"),q=n('"pugs"'),y=n(" sera-t-il tokenis\xE9 ?")},l(S){m=p(S,"P",{});var T=o(m);k=t(T,"\u270F\uFE0F "),x=p(T,"STRONG",{});var G=o(x);$=t(G,"A votre tour !"),G.forEach(a),w=t(T," Comment le mot "),v=p(T,"CODE",{});var U=o(v);q=t(U,'"pugs"'),U.forEach(a),y=t(T," sera-t-il tokenis\xE9 ?"),T.forEach(a)},m(S,T){i(S,m,T),e(m,k),e(m,x),e(x,$),e(m,w),e(m,v),e(v,q),e(m,y)},d(S){S&&a(m)}}}function pu(F){let m,k,x,$,w,v,q,y,S,T,G;return{c(){m=r("p"),k=n("\u{1F4A1} Utiliser "),x=r("code"),$=n("train_new_from_iterator()"),w=n(" sur le m\xEAme corpus ne donnera pas exactement le m\xEAme vocabulaire. C\u2019est parce que la biblioth\xE8que \u{1F917} "),v=r("em"),q=n("Tokenizers"),y=n(" n\u2019impl\xE9mente pas "),S=r("em"),T=n("WordPiece"),G=n(" pour l\u2019entra\xEEnement (puisque nous ne sommes pas compl\xE8tement s\xFBrs de son fonctionnement interne), mais utilise le BPE \xE0 la place.")},l(U){m=p(U,"P",{});var O=o(m);k=t(O,"\u{1F4A1} Utiliser "),x=p(O,"CODE",{});var _s=o(x);$=t(_s,"train_new_from_iterator()"),_s.forEach(a),w=t(O," sur le m\xEAme corpus ne donnera pas exactement le m\xEAme vocabulaire. C\u2019est parce que la biblioth\xE8que \u{1F917} "),v=p(O,"EM",{});var L=o(v);q=t(L,"Tokenizers"),L.forEach(a),y=t(O," n\u2019impl\xE9mente pas "),S=p(O,"EM",{});var ls=o(S);T=t(ls,"WordPiece"),ls.forEach(a),G=t(O," pour l\u2019entra\xEEnement (puisque nous ne sommes pas compl\xE8tement s\xFBrs de son fonctionnement interne), mais utilise le BPE \xE0 la place."),O.forEach(a)},m(U,O){i(U,m,O),e(m,k),e(m,x),e(x,$),e(m,w),e(m,v),e(v,q),e(m,y),e(m,S),e(S,T),e(m,G)},d(U){U&&a(m)}}}function ou(F){let m,k,x,$,w,v,q,y,S,T,G,U,O,_s,L,ls,Ht,St,ze,Wt,Kt,an,$s,nn,rs,tn,as,ps,De,ws,Ut,Te,Ft,ln,os,rn,W,Lt,Oe,Rt,It,Ae,Vt,Gt,Ne,Jt,Qt,Me,Yt,Xt,pn,ks,on,is,Zt,Be,sl,el,un,V,al,He,nl,tl,Se,ll,rl,cn,Vi='<span class="katex-display"><span class="katex"><span class="katex-mathml"><math xmlns="http://www.w3.org/1998/Math/MathML" display="block"><semantics><mrow><mrow><mi mathvariant="normal">s</mi><mi mathvariant="normal">c</mi><mi mathvariant="normal">o</mi><mi mathvariant="normal">r</mi><mi mathvariant="normal">e</mi></mrow><mo>=</mo><mo stretchy="false">(</mo><mrow><mi mathvariant="normal">f</mi><mi mathvariant="normal">r</mi><mi mathvariant="normal">e</mi><mi mathvariant="normal">q</mi><mi mathvariant="normal">_</mi><mi mathvariant="normal">o</mi><mi mathvariant="normal">f</mi><mi mathvariant="normal">_</mi><mi mathvariant="normal">p</mi><mi mathvariant="normal">a</mi><mi mathvariant="normal">i</mi><mi mathvariant="normal">r</mi></mrow><mo stretchy="false">)</mo><mi mathvariant="normal">/</mi><mo stretchy="false">(</mo><mrow><mi mathvariant="normal">f</mi><mi mathvariant="normal">r</mi><mi mathvariant="normal">e</mi><mi mathvariant="normal">q</mi><mi mathvariant="normal">_</mi><mi mathvariant="normal">o</mi><mi mathvariant="normal">f</mi><mi mathvariant="normal">_</mi><mi mathvariant="normal">f</mi><mi mathvariant="normal">i</mi><mi mathvariant="normal">r</mi><mi mathvariant="normal">s</mi><mi mathvariant="normal">t</mi><mi mathvariant="normal">_</mi><mi mathvariant="normal">e</mi><mi mathvariant="normal">l</mi><mi mathvariant="normal">e</mi><mi mathvariant="normal">m</mi><mi mathvariant="normal">e</mi><mi mathvariant="normal">n</mi><mi mathvariant="normal">t</mi></mrow><mo>\xD7</mo><mrow><mi mathvariant="normal">f</mi><mi mathvariant="normal">r</mi><mi mathvariant="normal">e</mi><mi mathvariant="normal">q</mi><mi mathvariant="normal">_</mi><mi mathvariant="normal">o</mi><mi mathvariant="normal">f</mi><mi mathvariant="normal">_</mi><mi mathvariant="normal">s</mi><mi mathvariant="normal">e</mi><mi mathvariant="normal">c</mi><mi mathvariant="normal">o</mi><mi mathvariant="normal">n</mi><mi mathvariant="normal">d</mi><mi mathvariant="normal">_</mi><mi mathvariant="normal">e</mi><mi mathvariant="normal">l</mi><mi mathvariant="normal">e</mi><mi mathvariant="normal">m</mi><mi mathvariant="normal">e</mi><mi mathvariant="normal">n</mi><mi mathvariant="normal">t</mi></mrow><mo stretchy="false">)</mo></mrow><annotation encoding="application/x-tex">\\mathrm{score} = (\\mathrm{freq\\_of\\_pair}) / (\\mathrm{freq\\_of\\_first\\_element} \\times \\mathrm{freq\\_of\\_second\\_element})</annotation></semantics></math></span><span class="katex-html" aria-hidden="true"><span class="base"><span class="strut" style="height:0.4306em;"></span><span class="mord"><span class="mord mathrm">score</span></span><span class="mspace" style="margin-right:0.2778em;"></span><span class="mrel">=</span><span class="mspace" style="margin-right:0.2778em;"></span></span><span class="base"><span class="strut" style="height:1.06em;vertical-align:-0.31em;"></span><span class="mopen">(</span><span class="mord"><span class="mord mathrm">freq_of_pair</span></span><span class="mclose">)</span><span class="mord">/</span><span class="mopen">(</span><span class="mord"><span class="mord mathrm">freq_of_first_element</span></span><span class="mspace" style="margin-right:0.2222em;"></span><span class="mbin">\xD7</span><span class="mspace" style="margin-right:0.2222em;"></span></span><span class="base"><span class="strut" style="height:1.06em;vertical-align:-0.31em;"></span><span class="mord"><span class="mord mathrm">freq_of_second_element</span></span><span class="mclose">)</span></span></span></span></span>',mn,z,pl,We,ol,il,Ke,ul,cl,Ue,ml,hl,Fe,dl,fl,Le,jl,gl,Re,bl,xl,Ie,vl,ql,hn,pe,_l,dn,Es,fn,oe,$l,jn,ys,gn,P,wl,Ve,kl,El,Ge,yl,Pl,Je,Cl,zl,Qe,Dl,Tl,Ye,Ol,Al,Xe,Nl,Ml,Ze,Bl,Hl,sa,Sl,Wl,bn,R,Kl,ea,Ul,Fl,aa,Ll,Rl,na,Il,Vl,xn,Ps,vn,J,Gl,ta,Jl,Ql,la,Yl,Xl,qn,Cs,_n,Q,Zl,ra,sr,er,pa,ar,nr,$n,zs,wn,ie,tr,kn,us,En,ns,cs,oa,Ds,lr,ia,rr,yn,E,pr,ua,or,ir,ca,ur,cr,ma,mr,hr,ha,dr,fr,da,jr,gr,fa,br,xr,ja,vr,qr,ga,_r,$r,ba,wr,kr,Pn,ms,Er,xa,yr,Pr,Cn,C,Cr,va,zr,Dr,qa,Tr,Or,_a,Ar,Nr,$a,Mr,Br,wa,Hr,Sr,ka,Wr,Kr,Ea,Ur,Fr,ya,Lr,Rr,zn,A,Ir,Pa,Vr,Gr,Ca,Jr,Qr,za,Yr,Xr,Da,Zr,sp,Ta,ep,ap,Oa,np,tp,Dn,hs,Tn,ts,ds,Aa,Ts,lp,ue,rp,Na,pp,On,fs,op,Ma,ip,up,An,ce,cp,Nn,Os,Mn,I,mp,Ba,hp,dp,Ha,fp,jp,Sa,gp,bp,Bn,As,Hn,me,xp,Sn,Ns,Wn,Ms,Kn,js,vp,Wa,qp,_p,Un,Bs,Fn,Hs,Ln,Y,$p,Ka,wp,kp,Ua,Ep,yp,Rn,Ss,In,gs,Pp,Fa,Cp,zp,Vn,Ws,Gn,he,Dp,Jn,Ks,Qn,de,Tp,Yn,Us,Xn,Fs,Zn,fe,Op,st,Ls,et,Rs,at,X,Ap,La,Np,Mp,Ra,Bp,Hp,nt,Is,tt,bs,Sp,Ia,Wp,Kp,lt,Vs,rt,je,Up,pt,Gs,ot,Js,it,ge,Fp,ut,Qs,ct,be,Lp,mt,Ys,ht,Xs,dt,Z,Rp,Va,Ip,Vp,Ga,Gp,Jp,ft,xs,jt,xe,Qp,gt,Zs,bt,ve,Yp,xt,se,vt,ee,qt,qe,Xp,_t,ae,$t,_e,Zp,wt,ne,kt,te,Et,ss,so,Ja,eo,ao,Qa,no,to,yt;return v=new Bt({}),O=new au({props:{classNames:"absolute z-10 right-0 top-0",options:[{label:"Google Colab",value:"https://colab.research.google.com/github/huggingface/notebooks/blob/master/course/chapter6/section6.ipynb"},{label:"Aws Studio",value:"https://studiolab.sagemaker.aws/import/github/huggingface/notebooks/blob/master/course/chapter6/section6.ipynb"}]}}),$s=new eu({props:{id:"qpv6ms_t_1A"}}),rs=new en({props:{$$slots:{default:[nu]},$$scope:{ctx:F}}}),ws=new Bt({}),os=new en({props:{warning:!0,$$slots:{default:[tu]},$$scope:{ctx:F}}}),ks=new _({props:{code:"w ##o ##r ##d",highlighted:"w ##o ##r ##d"}}),Es=new _({props:{code:'("hug", 10), ("pug", 5), ("pun", 12), ("bun", 4), ("hugs", 5)',highlighted:'(<span class="hljs-string">&quot;hug&quot;</span><span class="hljs-punctuation">,</span> <span class="hljs-number">10</span>)<span class="hljs-punctuation">,</span> (<span class="hljs-string">&quot;pug&quot;</span><span class="hljs-punctuation">,</span> <span class="hljs-number">5</span>)<span class="hljs-punctuation">,</span> (<span class="hljs-string">&quot;pun&quot;</span><span class="hljs-punctuation">,</span> <span class="hljs-number">12</span>)<span class="hljs-punctuation">,</span> (<span class="hljs-string">&quot;bun&quot;</span><span class="hljs-punctuation">,</span> <span class="hljs-number">4</span>)<span class="hljs-punctuation">,</span> (<span class="hljs-string">&quot;hugs&quot;</span><span class="hljs-punctuation">,</span> <span class="hljs-number">5</span>)'}}),ys=new _({props:{code:'("h" "##u" "##g", 10), ("p" "##u" "##g", 5), ("p" "##u" "##n", 12), ("b" "##u" "##n", 4), ("h" "##u" "##g" "##s", 5)',highlighted:'(<span class="hljs-string">&quot;h&quot;</span> <span class="hljs-string">&quot;#<span class="hljs-subst">#u</span>&quot;</span> <span class="hljs-string">&quot;#<span class="hljs-subst">#g</span>&quot;</span>, <span class="hljs-number">10</span>), (<span class="hljs-string">&quot;p&quot;</span> <span class="hljs-string">&quot;#<span class="hljs-subst">#u</span>&quot;</span> <span class="hljs-string">&quot;#<span class="hljs-subst">#g</span>&quot;</span>, <span class="hljs-number">5</span>), (<span class="hljs-string">&quot;p&quot;</span> <span class="hljs-string">&quot;#<span class="hljs-subst">#u</span>&quot;</span> <span class="hljs-string">&quot;#<span class="hljs-subst">#n</span>&quot;</span>, <span class="hljs-number">12</span>), (<span class="hljs-string">&quot;b&quot;</span> <span class="hljs-string">&quot;#<span class="hljs-subst">#u</span>&quot;</span> <span class="hljs-string">&quot;#<span class="hljs-subst">#n</span>&quot;</span>, <span class="hljs-number">4</span>), (<span class="hljs-string">&quot;h&quot;</span> <span class="hljs-string">&quot;#<span class="hljs-subst">#u</span>&quot;</span> <span class="hljs-string">&quot;#<span class="hljs-subst">#g</span>&quot;</span> <span class="hljs-string">&quot;#<span class="hljs-subst">#s</span>&quot;</span>, <span class="hljs-number">5</span>)'}}),Ps=new _({props:{code:`Vocabulary: ["b", "h", "p", "##g", "##n", "##s", "##u", "##gs"]
Corpus: ("h" "##u" "##g", 10), ("p" "##u" "##g", 5), ("p" "##u" "##n", 12), ("b" "##u" "##n", 4), ("h" "##u" "##gs", 5)`,highlighted:`Vocabulary: [<span class="hljs-string">&quot;b&quot;</span>, <span class="hljs-string">&quot;h&quot;</span>, <span class="hljs-string">&quot;p&quot;</span>, <span class="hljs-string">&quot;#<span class="hljs-subst">#g</span>&quot;</span>, <span class="hljs-string">&quot;#<span class="hljs-subst">#n</span>&quot;</span>, <span class="hljs-string">&quot;#<span class="hljs-subst">#s</span>&quot;</span>, <span class="hljs-string">&quot;#<span class="hljs-subst">#u</span>&quot;</span>, <span class="hljs-string">&quot;#<span class="hljs-subst">#gs</span>&quot;</span>]
Corpus: (<span class="hljs-string">&quot;h&quot;</span> <span class="hljs-string">&quot;#<span class="hljs-subst">#u</span>&quot;</span> <span class="hljs-string">&quot;#<span class="hljs-subst">#g</span>&quot;</span>, <span class="hljs-number">10</span>), (<span class="hljs-string">&quot;p&quot;</span> <span class="hljs-string">&quot;#<span class="hljs-subst">#u</span>&quot;</span> <span class="hljs-string">&quot;#<span class="hljs-subst">#g</span>&quot;</span>, <span class="hljs-number">5</span>), (<span class="hljs-string">&quot;p&quot;</span> <span class="hljs-string">&quot;#<span class="hljs-subst">#u</span>&quot;</span> <span class="hljs-string">&quot;#<span class="hljs-subst">#n</span>&quot;</span>, <span class="hljs-number">12</span>), (<span class="hljs-string">&quot;b&quot;</span> <span class="hljs-string">&quot;#<span class="hljs-subst">#u</span>&quot;</span> <span class="hljs-string">&quot;#<span class="hljs-subst">#n</span>&quot;</span>, <span class="hljs-number">4</span>), (<span class="hljs-string">&quot;h&quot;</span> <span class="hljs-string">&quot;#<span class="hljs-subst">#u</span>&quot;</span> <span class="hljs-string">&quot;#<span class="hljs-subst">#gs</span>&quot;</span>, <span class="hljs-number">5</span>)`}}),Cs=new _({props:{code:`Vocabulary: ["b", "h", "p", "##g", "##n", "##s", "##u", "##gs", "hu"]
Corpus: ("hu" "##g", 10), ("p" "##u" "##g", 5), ("p" "##u" "##n", 12), ("b" "##u" "##n", 4), ("hu" "##gs", 5)`,highlighted:`Vocabulary: [<span class="hljs-string">&quot;b&quot;</span>, <span class="hljs-string">&quot;h&quot;</span>, <span class="hljs-string">&quot;p&quot;</span>, <span class="hljs-string">&quot;#<span class="hljs-subst">#g</span>&quot;</span>, <span class="hljs-string">&quot;#<span class="hljs-subst">#n</span>&quot;</span>, <span class="hljs-string">&quot;#<span class="hljs-subst">#s</span>&quot;</span>, <span class="hljs-string">&quot;#<span class="hljs-subst">#u</span>&quot;</span>, <span class="hljs-string">&quot;#<span class="hljs-subst">#gs</span>&quot;</span>, <span class="hljs-string">&quot;hu&quot;</span>]
Corpus: (<span class="hljs-string">&quot;hu&quot;</span> <span class="hljs-string">&quot;#<span class="hljs-subst">#g</span>&quot;</span>, <span class="hljs-number">10</span>), (<span class="hljs-string">&quot;p&quot;</span> <span class="hljs-string">&quot;#<span class="hljs-subst">#u</span>&quot;</span> <span class="hljs-string">&quot;#<span class="hljs-subst">#g</span>&quot;</span>, <span class="hljs-number">5</span>), (<span class="hljs-string">&quot;p&quot;</span> <span class="hljs-string">&quot;#<span class="hljs-subst">#u</span>&quot;</span> <span class="hljs-string">&quot;#<span class="hljs-subst">#n</span>&quot;</span>, <span class="hljs-number">12</span>), (<span class="hljs-string">&quot;b&quot;</span> <span class="hljs-string">&quot;#<span class="hljs-subst">#u</span>&quot;</span> <span class="hljs-string">&quot;#<span class="hljs-subst">#n</span>&quot;</span>, <span class="hljs-number">4</span>), (<span class="hljs-string">&quot;hu&quot;</span> <span class="hljs-string">&quot;#<span class="hljs-subst">#gs</span>&quot;</span>, <span class="hljs-number">5</span>)`}}),zs=new _({props:{code:`Vocabulary: ["b", "h", "p", "##g", "##n", "##s", "##u", "##gs", "hu", "hug"]
Corpus: ("hug", 10), ("p" "##u" "##g", 5), ("p" "##u" "##n", 12), ("b" "##u" "##n", 4), ("hu" "##gs", 5)`,highlighted:`Vocabulary: [<span class="hljs-string">&quot;b&quot;</span>, <span class="hljs-string">&quot;h&quot;</span>, <span class="hljs-string">&quot;p&quot;</span>, <span class="hljs-string">&quot;#<span class="hljs-subst">#g</span>&quot;</span>, <span class="hljs-string">&quot;#<span class="hljs-subst">#n</span>&quot;</span>, <span class="hljs-string">&quot;#<span class="hljs-subst">#s</span>&quot;</span>, <span class="hljs-string">&quot;#<span class="hljs-subst">#u</span>&quot;</span>, <span class="hljs-string">&quot;#<span class="hljs-subst">#gs</span>&quot;</span>, <span class="hljs-string">&quot;hu&quot;</span>, <span class="hljs-string">&quot;hug&quot;</span>]
Corpus: (<span class="hljs-string">&quot;hug&quot;</span>, <span class="hljs-number">10</span>), (<span class="hljs-string">&quot;p&quot;</span> <span class="hljs-string">&quot;#<span class="hljs-subst">#u</span>&quot;</span> <span class="hljs-string">&quot;#<span class="hljs-subst">#g</span>&quot;</span>, <span class="hljs-number">5</span>), (<span class="hljs-string">&quot;p&quot;</span> <span class="hljs-string">&quot;#<span class="hljs-subst">#u</span>&quot;</span> <span class="hljs-string">&quot;#<span class="hljs-subst">#n</span>&quot;</span>, <span class="hljs-number">12</span>), (<span class="hljs-string">&quot;b&quot;</span> <span class="hljs-string">&quot;#<span class="hljs-subst">#u</span>&quot;</span> <span class="hljs-string">&quot;#<span class="hljs-subst">#n</span>&quot;</span>, <span class="hljs-number">4</span>), (<span class="hljs-string">&quot;hu&quot;</span> <span class="hljs-string">&quot;#<span class="hljs-subst">#gs</span>&quot;</span>, <span class="hljs-number">5</span>)`}}),us=new en({props:{$$slots:{default:[lu]},$$scope:{ctx:F}}}),Ds=new Bt({}),hs=new en({props:{$$slots:{default:[ru]},$$scope:{ctx:F}}}),Ts=new Bt({}),Os=new _({props:{code:`corpus = [
    "This is the Hugging Face Course.",
    # C'est le cours d'Hugging Face.
    "This chapter is about tokenization.",
    # This chapter is about tokenization
    "This section shows several tokenizer algorithms.",
    # Cette section pr\xE9sente plusieurs algorithmes de tokenizer.
    "Hopefully, you will be able to understand how they are trained and generate tokens.",
    # Avec un peu de chance, vous serez en mesure de comprendre comment ils sont entra\xEEn\xE9s et g\xE9n\xE8rent des tokens.
]`,highlighted:`corpus = [
    <span class="hljs-string">&quot;This is the Hugging Face Course.&quot;</span>,
    <span class="hljs-comment"># C&#x27;est le cours d&#x27;Hugging Face.</span>
    <span class="hljs-string">&quot;This chapter is about tokenization.&quot;</span>,
    <span class="hljs-comment"># This chapter is about tokenization</span>
    <span class="hljs-string">&quot;This section shows several tokenizer algorithms.&quot;</span>,
    <span class="hljs-comment"># Cette section pr\xE9sente plusieurs algorithmes de tokenizer.</span>
    <span class="hljs-string">&quot;Hopefully, you will be able to understand how they are trained and generate tokens.&quot;</span>,
    <span class="hljs-comment"># Avec un peu de chance, vous serez en mesure de comprendre comment ils sont entra\xEEn\xE9s et g\xE9n\xE8rent des tokens.</span>
]`}}),As=new _({props:{code:`from transformers import AutoTokenizer

tokenizer = AutoTokenizer.from_pretrained("bert-base-cased")`,highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoTokenizer

tokenizer = AutoTokenizer.from_pretrained(<span class="hljs-string">&quot;bert-base-cased&quot;</span>)`}}),Ns=new _({props:{code:`from collections import defaultdict

word_freqs = defaultdict(int)
for text in corpus:
    words_with_offsets = tokenizer.backend_tokenizer.pre_tokenizer.pre_tokenize_str(text)
    new_words = [word for word, offset in words_with_offsets]
    for word in new_words:
        word_freqs[word] += 1

word_freqs`,highlighted:`<span class="hljs-keyword">from</span> collections <span class="hljs-keyword">import</span> defaultdict

word_freqs = defaultdict(<span class="hljs-built_in">int</span>)
<span class="hljs-keyword">for</span> text <span class="hljs-keyword">in</span> corpus:
    words_with_offsets = tokenizer.backend_tokenizer.pre_tokenizer.pre_tokenize_str(text)
    new_words = [word <span class="hljs-keyword">for</span> word, offset <span class="hljs-keyword">in</span> words_with_offsets]
    <span class="hljs-keyword">for</span> word <span class="hljs-keyword">in</span> new_words:
        word_freqs[word] += <span class="hljs-number">1</span>

word_freqs`}}),Ms=new _({props:{code:`defaultdict(
    int, {'This': 3, 'is': 2, 'the': 1, 'Hugging': 1, 'Face': 1, 'Course': 1, '.': 4, 'chapter': 1, 'about': 1,
    'tokenization': 1, 'section': 1, 'shows': 1, 'several': 1, 'tokenizer': 1, 'algorithms': 1, 'Hopefully': 1,
    ',': 1, 'you': 1, 'will': 1, 'be': 1, 'able': 1, 'to': 1, 'understand': 1, 'how': 1, 'they': 1, 'are': 1,
    'trained': 1, 'and': 1, 'generate': 1, 'tokens': 1})`,highlighted:`defaultdict(
    <span class="hljs-built_in">int</span>, {<span class="hljs-string">&#x27;This&#x27;</span>: <span class="hljs-number">3</span>, <span class="hljs-string">&#x27;is&#x27;</span>: <span class="hljs-number">2</span>, <span class="hljs-string">&#x27;the&#x27;</span>: <span class="hljs-number">1</span>, <span class="hljs-string">&#x27;Hugging&#x27;</span>: <span class="hljs-number">1</span>, <span class="hljs-string">&#x27;Face&#x27;</span>: <span class="hljs-number">1</span>, <span class="hljs-string">&#x27;Course&#x27;</span>: <span class="hljs-number">1</span>, <span class="hljs-string">&#x27;.&#x27;</span>: <span class="hljs-number">4</span>, <span class="hljs-string">&#x27;chapter&#x27;</span>: <span class="hljs-number">1</span>, <span class="hljs-string">&#x27;about&#x27;</span>: <span class="hljs-number">1</span>,
    <span class="hljs-string">&#x27;tokenization&#x27;</span>: <span class="hljs-number">1</span>, <span class="hljs-string">&#x27;section&#x27;</span>: <span class="hljs-number">1</span>, <span class="hljs-string">&#x27;shows&#x27;</span>: <span class="hljs-number">1</span>, <span class="hljs-string">&#x27;several&#x27;</span>: <span class="hljs-number">1</span>, <span class="hljs-string">&#x27;tokenizer&#x27;</span>: <span class="hljs-number">1</span>, <span class="hljs-string">&#x27;algorithms&#x27;</span>: <span class="hljs-number">1</span>, <span class="hljs-string">&#x27;Hopefully&#x27;</span>: <span class="hljs-number">1</span>,
    <span class="hljs-string">&#x27;,&#x27;</span>: <span class="hljs-number">1</span>, <span class="hljs-string">&#x27;you&#x27;</span>: <span class="hljs-number">1</span>, <span class="hljs-string">&#x27;will&#x27;</span>: <span class="hljs-number">1</span>, <span class="hljs-string">&#x27;be&#x27;</span>: <span class="hljs-number">1</span>, <span class="hljs-string">&#x27;able&#x27;</span>: <span class="hljs-number">1</span>, <span class="hljs-string">&#x27;to&#x27;</span>: <span class="hljs-number">1</span>, <span class="hljs-string">&#x27;understand&#x27;</span>: <span class="hljs-number">1</span>, <span class="hljs-string">&#x27;how&#x27;</span>: <span class="hljs-number">1</span>, <span class="hljs-string">&#x27;they&#x27;</span>: <span class="hljs-number">1</span>, <span class="hljs-string">&#x27;are&#x27;</span>: <span class="hljs-number">1</span>,
    <span class="hljs-string">&#x27;trained&#x27;</span>: <span class="hljs-number">1</span>, <span class="hljs-string">&#x27;and&#x27;</span>: <span class="hljs-number">1</span>, <span class="hljs-string">&#x27;generate&#x27;</span>: <span class="hljs-number">1</span>, <span class="hljs-string">&#x27;tokens&#x27;</span>: <span class="hljs-number">1</span>})`}}),Bs=new _({props:{code:`alphabet = []
for word in word_freqs.keys():
    if word[0] not in alphabet:
        alphabet.append(word[0])
    for letter in word[1:]:
        if f"##{letter}" not in alphabet:
            alphabet.append(f"##{letter}")

alphabet.sort()
alphabet

print(alphabet)`,highlighted:`alphabet = []
<span class="hljs-keyword">for</span> word <span class="hljs-keyword">in</span> word_freqs.keys():
    <span class="hljs-keyword">if</span> word[<span class="hljs-number">0</span>] <span class="hljs-keyword">not</span> <span class="hljs-keyword">in</span> alphabet:
        alphabet.append(word[<span class="hljs-number">0</span>])
    <span class="hljs-keyword">for</span> letter <span class="hljs-keyword">in</span> word[<span class="hljs-number">1</span>:]:
        <span class="hljs-keyword">if</span> <span class="hljs-string">f&quot;##<span class="hljs-subst">{letter}</span>&quot;</span> <span class="hljs-keyword">not</span> <span class="hljs-keyword">in</span> alphabet:
            alphabet.append(<span class="hljs-string">f&quot;##<span class="hljs-subst">{letter}</span>&quot;</span>)

alphabet.sort()
alphabet

<span class="hljs-built_in">print</span>(alphabet)`}}),Hs=new _({props:{code:`['##a', '##b', '##c', '##d', '##e', '##f', '##g', '##h', '##i', '##k', '##l', '##m', '##n', '##o', '##p', '##r', '##s',
 '##t', '##u', '##v', '##w', '##y', '##z', ',', '.', 'C', 'F', 'H', 'T', 'a', 'b', 'c', 'g', 'h', 'i', 's', 't', 'u',
 'w', 'y']`,highlighted:`[<span class="hljs-string">&#x27;##a&#x27;</span>, <span class="hljs-string">&#x27;##b&#x27;</span>, <span class="hljs-string">&#x27;##c&#x27;</span>, <span class="hljs-string">&#x27;##d&#x27;</span>, <span class="hljs-string">&#x27;##e&#x27;</span>, <span class="hljs-string">&#x27;##f&#x27;</span>, <span class="hljs-string">&#x27;##g&#x27;</span>, <span class="hljs-string">&#x27;##h&#x27;</span>, <span class="hljs-string">&#x27;##i&#x27;</span>, <span class="hljs-string">&#x27;##k&#x27;</span>, <span class="hljs-string">&#x27;##l&#x27;</span>, <span class="hljs-string">&#x27;##m&#x27;</span>, <span class="hljs-string">&#x27;##n&#x27;</span>, <span class="hljs-string">&#x27;##o&#x27;</span>, <span class="hljs-string">&#x27;##p&#x27;</span>, <span class="hljs-string">&#x27;##r&#x27;</span>, <span class="hljs-string">&#x27;##s&#x27;</span>,
 <span class="hljs-string">&#x27;##t&#x27;</span>, <span class="hljs-string">&#x27;##u&#x27;</span>, <span class="hljs-string">&#x27;##v&#x27;</span>, <span class="hljs-string">&#x27;##w&#x27;</span>, <span class="hljs-string">&#x27;##y&#x27;</span>, <span class="hljs-string">&#x27;##z&#x27;</span>, <span class="hljs-string">&#x27;,&#x27;</span>, <span class="hljs-string">&#x27;.&#x27;</span>, <span class="hljs-string">&#x27;C&#x27;</span>, <span class="hljs-string">&#x27;F&#x27;</span>, <span class="hljs-string">&#x27;H&#x27;</span>, <span class="hljs-string">&#x27;T&#x27;</span>, <span class="hljs-string">&#x27;a&#x27;</span>, <span class="hljs-string">&#x27;b&#x27;</span>, <span class="hljs-string">&#x27;c&#x27;</span>, <span class="hljs-string">&#x27;g&#x27;</span>, <span class="hljs-string">&#x27;h&#x27;</span>, <span class="hljs-string">&#x27;i&#x27;</span>, <span class="hljs-string">&#x27;s&#x27;</span>, <span class="hljs-string">&#x27;t&#x27;</span>, <span class="hljs-string">&#x27;u&#x27;</span>,
 <span class="hljs-string">&#x27;w&#x27;</span>, <span class="hljs-string">&#x27;y&#x27;</span>]`}}),Ss=new _({props:{code:'vocab = ["[PAD]", "[UNK]", "[CLS]", "[SEP]", "[MASK]"] + alphabet.copy()',highlighted:'vocab = [<span class="hljs-string">&quot;[PAD]&quot;</span>, <span class="hljs-string">&quot;[UNK]&quot;</span>, <span class="hljs-string">&quot;[CLS]&quot;</span>, <span class="hljs-string">&quot;[SEP]&quot;</span>, <span class="hljs-string">&quot;[MASK]&quot;</span>] + alphabet.copy()'}}),Ws=new _({props:{code:`splits = {
    word: [c if i == 0 else f"##{c}" for i, c in enumerate(word)]
    for word in word_freqs.keys()
}`,highlighted:`splits = {
    word: [c <span class="hljs-keyword">if</span> i == <span class="hljs-number">0</span> <span class="hljs-keyword">else</span> <span class="hljs-string">f&quot;##<span class="hljs-subst">{c}</span>&quot;</span> <span class="hljs-keyword">for</span> i, c <span class="hljs-keyword">in</span> <span class="hljs-built_in">enumerate</span>(word)]
    <span class="hljs-keyword">for</span> word <span class="hljs-keyword">in</span> word_freqs.keys()
}`}}),Ks=new _({props:{code:`def compute_pair_scores(splits):
    letter_freqs = defaultdict(int)
    pair_freqs = defaultdict(int)
    for word, freq in word_freqs.items():
        split = splits[word]
        if len(split) == 1:
            letter_freqs[split[0]] += freq
            continue
        for i in range(len(split) - 1):
            pair = (split[i], split[i + 1])
            letter_freqs[split[i]] += freq
            pair_freqs[pair] += freq
        letter_freqs[split[-1]] += freq

    scores = {
        pair: freq / (letter_freqs[pair[0]] * letter_freqs[pair[1]])
        for pair, freq in pair_freqs.items()
    }
    return scores`,highlighted:`<span class="hljs-keyword">def</span> <span class="hljs-title function_">compute_pair_scores</span>(<span class="hljs-params">splits</span>):
    letter_freqs = defaultdict(<span class="hljs-built_in">int</span>)
    pair_freqs = defaultdict(<span class="hljs-built_in">int</span>)
    <span class="hljs-keyword">for</span> word, freq <span class="hljs-keyword">in</span> word_freqs.items():
        split = splits[word]
        <span class="hljs-keyword">if</span> <span class="hljs-built_in">len</span>(split) == <span class="hljs-number">1</span>:
            letter_freqs[split[<span class="hljs-number">0</span>]] += freq
            <span class="hljs-keyword">continue</span>
        <span class="hljs-keyword">for</span> i <span class="hljs-keyword">in</span> <span class="hljs-built_in">range</span>(<span class="hljs-built_in">len</span>(split) - <span class="hljs-number">1</span>):
            pair = (split[i], split[i + <span class="hljs-number">1</span>])
            letter_freqs[split[i]] += freq
            pair_freqs[pair] += freq
        letter_freqs[split[-<span class="hljs-number">1</span>]] += freq

    scores = {
        pair: freq / (letter_freqs[pair[<span class="hljs-number">0</span>]] * letter_freqs[pair[<span class="hljs-number">1</span>]])
        <span class="hljs-keyword">for</span> pair, freq <span class="hljs-keyword">in</span> pair_freqs.items()
    }
    <span class="hljs-keyword">return</span> scores`}}),Us=new _({props:{code:`pair_scores = compute_pair_scores(splits)
for i, key in enumerate(pair_scores.keys()):
    print(f"{key}: {pair_scores[key]}")
    if i >= 5:
        break`,highlighted:`pair_scores = compute_pair_scores(splits)
<span class="hljs-keyword">for</span> i, key <span class="hljs-keyword">in</span> <span class="hljs-built_in">enumerate</span>(pair_scores.keys()):
    <span class="hljs-built_in">print</span>(<span class="hljs-string">f&quot;<span class="hljs-subst">{key}</span>: <span class="hljs-subst">{pair_scores[key]}</span>&quot;</span>)
    <span class="hljs-keyword">if</span> i &gt;= <span class="hljs-number">5</span>:
        <span class="hljs-keyword">break</span>`}}),Fs=new _({props:{code:`('T', '##h'): 0.125
('##h', '##i'): 0.03409090909090909
('##i', '##s'): 0.02727272727272727
('i', '##s'): 0.1
('t', '##h'): 0.03571428571428571
('##h', '##e'): 0.011904761904761904`,highlighted:`(<span class="hljs-string">&#x27;T&#x27;</span>, <span class="hljs-string">&#x27;##h&#x27;</span>): <span class="hljs-number">0.125</span>
(<span class="hljs-string">&#x27;##h&#x27;</span>, <span class="hljs-string">&#x27;##i&#x27;</span>): <span class="hljs-number">0.03409090909090909</span>
(<span class="hljs-string">&#x27;##i&#x27;</span>, <span class="hljs-string">&#x27;##s&#x27;</span>): <span class="hljs-number">0.02727272727272727</span>
(<span class="hljs-string">&#x27;i&#x27;</span>, <span class="hljs-string">&#x27;##s&#x27;</span>): <span class="hljs-number">0.1</span>
(<span class="hljs-string">&#x27;t&#x27;</span>, <span class="hljs-string">&#x27;##h&#x27;</span>): <span class="hljs-number">0.03571428571428571</span>
(<span class="hljs-string">&#x27;##h&#x27;</span>, <span class="hljs-string">&#x27;##e&#x27;</span>): <span class="hljs-number">0.011904761904761904</span>`}}),Ls=new _({props:{code:`best_pair = ""
max_score = None
for pair, score in pair_scores.items():
    if max_score is None or max_score < score:
        best_pair = pair
        max_score = score

print(best_pair, max_score)`,highlighted:`best_pair = <span class="hljs-string">&quot;&quot;</span>
max_score = <span class="hljs-literal">None</span>
<span class="hljs-keyword">for</span> pair, score <span class="hljs-keyword">in</span> pair_scores.items():
    <span class="hljs-keyword">if</span> max_score <span class="hljs-keyword">is</span> <span class="hljs-literal">None</span> <span class="hljs-keyword">or</span> max_score &lt; score:
        best_pair = pair
        max_score = score

<span class="hljs-built_in">print</span>(best_pair, max_score)`}}),Rs=new _({props:{code:"('a', '##b') 0.2",highlighted:'(<span class="hljs-string">&#x27;a&#x27;</span>, <span class="hljs-string">&#x27;##b&#x27;</span>) <span class="hljs-number">0.2</span>'}}),Is=new _({props:{code:'vocab.append("ab")',highlighted:'vocab.append(<span class="hljs-string">&quot;ab&quot;</span>)'}}),Vs=new _({props:{code:`def merge_pair(a, b, splits):
    for word in word_freqs:
        split = splits[word]
        if len(split) == 1:
            continue
        i = 0
        while i < len(split) - 1:
            if split[i] == a and split[i + 1] == b:
                merge = a + b[2:] if b.startswith("##") else a + b
                split = split[:i] + [merge] + split[i + 2 :]
            else:
                i += 1
        splits[word] = split
    return splits`,highlighted:`<span class="hljs-keyword">def</span> <span class="hljs-title function_">merge_pair</span>(<span class="hljs-params">a, b, splits</span>):
    <span class="hljs-keyword">for</span> word <span class="hljs-keyword">in</span> word_freqs:
        split = splits[word]
        <span class="hljs-keyword">if</span> <span class="hljs-built_in">len</span>(split) == <span class="hljs-number">1</span>:
            <span class="hljs-keyword">continue</span>
        i = <span class="hljs-number">0</span>
        <span class="hljs-keyword">while</span> i &lt; <span class="hljs-built_in">len</span>(split) - <span class="hljs-number">1</span>:
            <span class="hljs-keyword">if</span> split[i] == a <span class="hljs-keyword">and</span> split[i + <span class="hljs-number">1</span>] == b:
                merge = a + b[<span class="hljs-number">2</span>:] <span class="hljs-keyword">if</span> b.startswith(<span class="hljs-string">&quot;##&quot;</span>) <span class="hljs-keyword">else</span> a + b
                split = split[:i] + [merge] + split[i + <span class="hljs-number">2</span> :]
            <span class="hljs-keyword">else</span>:
                i += <span class="hljs-number">1</span>
        splits[word] = split
    <span class="hljs-keyword">return</span> splits`}}),Gs=new _({props:{code:`splits = merge_pair("a", "##b", splits)
splits["about"]`,highlighted:`splits = merge_pair(<span class="hljs-string">&quot;a&quot;</span>, <span class="hljs-string">&quot;##b&quot;</span>, splits)
splits[<span class="hljs-string">&quot;about&quot;</span>]`}}),Js=new _({props:{code:"['ab', '##o', '##u', '##t']",highlighted:'[<span class="hljs-string">&#x27;ab&#x27;</span>, <span class="hljs-string">&#x27;##o&#x27;</span>, <span class="hljs-string">&#x27;##u&#x27;</span>, <span class="hljs-string">&#x27;##t&#x27;</span>]'}}),Qs=new _({props:{code:`vocab_size = 70
while len(vocab) < vocab_size:
    scores = compute_pair_scores(splits)
    best_pair, max_score = "", None
    for pair, score in scores.items():
        if max_score is None or max_score < score:
            best_pair = pair
            max_score = score
    splits = merge_pair(*best_pair, splits)
    new_token = (
        best_pair[0] + best_pair[1][2:]
        if best_pair[1].startswith("##")
        else best_pair[0] + best_pair[1]
    )
    vocab.append(new_token)`,highlighted:`vocab_size = <span class="hljs-number">70</span>
<span class="hljs-keyword">while</span> <span class="hljs-built_in">len</span>(vocab) &lt; vocab_size:
    scores = compute_pair_scores(splits)
    best_pair, max_score = <span class="hljs-string">&quot;&quot;</span>, <span class="hljs-literal">None</span>
    <span class="hljs-keyword">for</span> pair, score <span class="hljs-keyword">in</span> scores.items():
        <span class="hljs-keyword">if</span> max_score <span class="hljs-keyword">is</span> <span class="hljs-literal">None</span> <span class="hljs-keyword">or</span> max_score &lt; score:
            best_pair = pair
            max_score = score
    splits = merge_pair(*best_pair, splits)
    new_token = (
        best_pair[<span class="hljs-number">0</span>] + best_pair[<span class="hljs-number">1</span>][<span class="hljs-number">2</span>:]
        <span class="hljs-keyword">if</span> best_pair[<span class="hljs-number">1</span>].startswith(<span class="hljs-string">&quot;##&quot;</span>)
        <span class="hljs-keyword">else</span> best_pair[<span class="hljs-number">0</span>] + best_pair[<span class="hljs-number">1</span>]
    )
    vocab.append(new_token)`}}),Ys=new _({props:{code:"print(vocab)",highlighted:'<span class="hljs-built_in">print</span>(vocab)'}}),Xs=new _({props:{code:`['[PAD]', '[UNK]', '[CLS]', '[SEP]', '[MASK]', '##a', '##b', '##c', '##d', '##e', '##f', '##g', '##h', '##i', '##k',
 '##l', '##m', '##n', '##o', '##p', '##r', '##s', '##t', '##u', '##v', '##w', '##y', '##z', ',', '.', 'C', 'F', 'H',
 'T', 'a', 'b', 'c', 'g', 'h', 'i', 's', 't', 'u', 'w', 'y', '##fu', 'Fa', 'Fac', '##ct', '##ful', '##full', '##fully',
 'Th', 'ch', '##hm', 'cha', 'chap', 'chapt', '##thm', 'Hu', 'Hug', 'Hugg', 'sh', 'th', 'is', '##thms', '##za', '##zat',
 '##ut']`,highlighted:`[<span class="hljs-string">&#x27;[PAD]&#x27;</span>, <span class="hljs-string">&#x27;[UNK]&#x27;</span>, <span class="hljs-string">&#x27;[CLS]&#x27;</span>, <span class="hljs-string">&#x27;[SEP]&#x27;</span>, <span class="hljs-string">&#x27;[MASK]&#x27;</span>, <span class="hljs-string">&#x27;##a&#x27;</span>, <span class="hljs-string">&#x27;##b&#x27;</span>, <span class="hljs-string">&#x27;##c&#x27;</span>, <span class="hljs-string">&#x27;##d&#x27;</span>, <span class="hljs-string">&#x27;##e&#x27;</span>, <span class="hljs-string">&#x27;##f&#x27;</span>, <span class="hljs-string">&#x27;##g&#x27;</span>, <span class="hljs-string">&#x27;##h&#x27;</span>, <span class="hljs-string">&#x27;##i&#x27;</span>, <span class="hljs-string">&#x27;##k&#x27;</span>,
 <span class="hljs-string">&#x27;##l&#x27;</span>, <span class="hljs-string">&#x27;##m&#x27;</span>, <span class="hljs-string">&#x27;##n&#x27;</span>, <span class="hljs-string">&#x27;##o&#x27;</span>, <span class="hljs-string">&#x27;##p&#x27;</span>, <span class="hljs-string">&#x27;##r&#x27;</span>, <span class="hljs-string">&#x27;##s&#x27;</span>, <span class="hljs-string">&#x27;##t&#x27;</span>, <span class="hljs-string">&#x27;##u&#x27;</span>, <span class="hljs-string">&#x27;##v&#x27;</span>, <span class="hljs-string">&#x27;##w&#x27;</span>, <span class="hljs-string">&#x27;##y&#x27;</span>, <span class="hljs-string">&#x27;##z&#x27;</span>, <span class="hljs-string">&#x27;,&#x27;</span>, <span class="hljs-string">&#x27;.&#x27;</span>, <span class="hljs-string">&#x27;C&#x27;</span>, <span class="hljs-string">&#x27;F&#x27;</span>, <span class="hljs-string">&#x27;H&#x27;</span>,
 <span class="hljs-string">&#x27;T&#x27;</span>, <span class="hljs-string">&#x27;a&#x27;</span>, <span class="hljs-string">&#x27;b&#x27;</span>, <span class="hljs-string">&#x27;c&#x27;</span>, <span class="hljs-string">&#x27;g&#x27;</span>, <span class="hljs-string">&#x27;h&#x27;</span>, <span class="hljs-string">&#x27;i&#x27;</span>, <span class="hljs-string">&#x27;s&#x27;</span>, <span class="hljs-string">&#x27;t&#x27;</span>, <span class="hljs-string">&#x27;u&#x27;</span>, <span class="hljs-string">&#x27;w&#x27;</span>, <span class="hljs-string">&#x27;y&#x27;</span>, <span class="hljs-string">&#x27;##fu&#x27;</span>, <span class="hljs-string">&#x27;Fa&#x27;</span>, <span class="hljs-string">&#x27;Fac&#x27;</span>, <span class="hljs-string">&#x27;##ct&#x27;</span>, <span class="hljs-string">&#x27;##ful&#x27;</span>, <span class="hljs-string">&#x27;##full&#x27;</span>, <span class="hljs-string">&#x27;##fully&#x27;</span>,
 <span class="hljs-string">&#x27;Th&#x27;</span>, <span class="hljs-string">&#x27;ch&#x27;</span>, <span class="hljs-string">&#x27;##hm&#x27;</span>, <span class="hljs-string">&#x27;cha&#x27;</span>, <span class="hljs-string">&#x27;chap&#x27;</span>, <span class="hljs-string">&#x27;chapt&#x27;</span>, <span class="hljs-string">&#x27;##thm&#x27;</span>, <span class="hljs-string">&#x27;Hu&#x27;</span>, <span class="hljs-string">&#x27;Hug&#x27;</span>, <span class="hljs-string">&#x27;Hugg&#x27;</span>, <span class="hljs-string">&#x27;sh&#x27;</span>, <span class="hljs-string">&#x27;th&#x27;</span>, <span class="hljs-string">&#x27;is&#x27;</span>, <span class="hljs-string">&#x27;##thms&#x27;</span>, <span class="hljs-string">&#x27;##za&#x27;</span>, <span class="hljs-string">&#x27;##zat&#x27;</span>,
 <span class="hljs-string">&#x27;##ut&#x27;</span>]`}}),xs=new en({props:{$$slots:{default:[pu]},$$scope:{ctx:F}}}),Zs=new _({props:{code:`def encode_word(word):
    tokens = []
    while len(word) > 0:
        i = len(word)
        while i > 0 and word[:i] not in vocab:
            i -= 1
        if i == 0:
            return ["[UNK]"]
        tokens.append(word[:i])
        word = word[i:]
        if len(word) > 0:
            word = f"##{word}"
    return tokens`,highlighted:`<span class="hljs-keyword">def</span> <span class="hljs-title function_">encode_word</span>(<span class="hljs-params">word</span>):
    tokens = []
    <span class="hljs-keyword">while</span> <span class="hljs-built_in">len</span>(word) &gt; <span class="hljs-number">0</span>:
        i = <span class="hljs-built_in">len</span>(word)
        <span class="hljs-keyword">while</span> i &gt; <span class="hljs-number">0</span> <span class="hljs-keyword">and</span> word[:i] <span class="hljs-keyword">not</span> <span class="hljs-keyword">in</span> vocab:
            i -= <span class="hljs-number">1</span>
        <span class="hljs-keyword">if</span> i == <span class="hljs-number">0</span>:
            <span class="hljs-keyword">return</span> [<span class="hljs-string">&quot;[UNK]&quot;</span>]
        tokens.append(word[:i])
        word = word[i:]
        <span class="hljs-keyword">if</span> <span class="hljs-built_in">len</span>(word) &gt; <span class="hljs-number">0</span>:
            word = <span class="hljs-string">f&quot;##<span class="hljs-subst">{word}</span>&quot;</span>
    <span class="hljs-keyword">return</span> tokens`}}),se=new _({props:{code:`print(encode_word("Hugging"))
print(encode_word("HOgging"))`,highlighted:`<span class="hljs-built_in">print</span>(encode_word(<span class="hljs-string">&quot;Hugging&quot;</span>))
<span class="hljs-built_in">print</span>(encode_word(<span class="hljs-string">&quot;HOgging&quot;</span>))`}}),ee=new _({props:{code:`['Hugg', '##i', '##n', '##g']
['[UNK]']`,highlighted:`[<span class="hljs-string">&#x27;Hugg&#x27;</span>, <span class="hljs-string">&#x27;##i&#x27;</span>, <span class="hljs-string">&#x27;##n&#x27;</span>, <span class="hljs-string">&#x27;##g&#x27;</span>]
[<span class="hljs-string">&#x27;[UNK]&#x27;</span>]`}}),ae=new _({props:{code:`def tokenize(text):
    pre_tokenize_result = tokenizer._tokenizer.pre_tokenizer.pre_tokenize_str(text)
    pre_tokenized_text = [word for word, offset in pre_tokenize_result]
    encoded_words = [encode_word(word) for word in pre_tokenized_text]
    return sum(encoded_words, [])`,highlighted:`<span class="hljs-keyword">def</span> <span class="hljs-title function_">tokenize</span>(<span class="hljs-params">text</span>):
    pre_tokenize_result = tokenizer._tokenizer.pre_tokenizer.pre_tokenize_str(text)
    pre_tokenized_text = [word <span class="hljs-keyword">for</span> word, offset <span class="hljs-keyword">in</span> pre_tokenize_result]
    encoded_words = [encode_word(word) <span class="hljs-keyword">for</span> word <span class="hljs-keyword">in</span> pre_tokenized_text]
    <span class="hljs-keyword">return</span> <span class="hljs-built_in">sum</span>(encoded_words, [])`}}),ne=new _({props:{code:`tokenize("This is the Hugging Face Course!")  # C'est le cours d'Hugging Face`,highlighted:'tokenize(<span class="hljs-string">&quot;This is the Hugging Face Course!&quot;</span>)  <span class="hljs-comment"># C&#x27;est le cours d&#x27;Hugging Face</span>'}}),te=new _({props:{code:`['Th', '##i', '##s', 'is', 'th', '##e', 'Hugg', '##i', '##n', '##g', 'Fac', '##e', 'c', '##o', '##u', '##r', '##s',
 '##e', '[UNK]']`,highlighted:`[<span class="hljs-string">&#x27;Th&#x27;</span>, <span class="hljs-string">&#x27;##i&#x27;</span>, <span class="hljs-string">&#x27;##s&#x27;</span>, <span class="hljs-string">&#x27;is&#x27;</span>, <span class="hljs-string">&#x27;th&#x27;</span>, <span class="hljs-string">&#x27;##e&#x27;</span>, <span class="hljs-string">&#x27;Hugg&#x27;</span>, <span class="hljs-string">&#x27;##i&#x27;</span>, <span class="hljs-string">&#x27;##n&#x27;</span>, <span class="hljs-string">&#x27;##g&#x27;</span>, <span class="hljs-string">&#x27;Fac&#x27;</span>, <span class="hljs-string">&#x27;##e&#x27;</span>, <span class="hljs-string">&#x27;c&#x27;</span>, <span class="hljs-string">&#x27;##o&#x27;</span>, <span class="hljs-string">&#x27;##u&#x27;</span>, <span class="hljs-string">&#x27;##r&#x27;</span>, <span class="hljs-string">&#x27;##s&#x27;</span>,
 <span class="hljs-string">&#x27;##e&#x27;</span>, <span class="hljs-string">&#x27;[UNK]&#x27;</span>]`}}),{c(){m=r("meta"),k=u(),x=r("h1"),$=r("a"),w=r("span"),h(v.$$.fragment),q=u(),y=r("span"),S=n("Tok\xE9nisation "),T=r("i"),G=n("WordPiece"),U=u(),h(O.$$.fragment),_s=u(),L=r("p"),ls=r("em"),Ht=n("WordPiece"),St=n(" est l\u2019algorithme de tok\xE9nisation d\xE9velopp\xE9 par Google pour pr\xE9tra\xEEner BERT. Il a depuis \xE9t\xE9 r\xE9utilis\xE9 dans un grand nombre de mod\xE8les de "),ze=r("em"),Wt=n("transformers"),Kt=n(" bas\xE9s sur BERT tels que DistilBERT, MobileBERT, Funnel Transformers et MPNET. Il est tr\xE8s similaire au BPE en termes d\u2019entra\xEEnement mais la tokenisation r\xE9elle est effectu\xE9e diff\xE9remment."),an=u(),h($s.$$.fragment),nn=u(),h(rs.$$.fragment),tn=u(),as=r("h2"),ps=r("a"),De=r("span"),h(ws.$$.fragment),Ut=u(),Te=r("span"),Ft=n("Algorithme d'entra\xEEnement"),ln=u(),h(os.$$.fragment),rn=u(),W=r("p"),Lt=n("Comme le BPE, "),Oe=r("em"),Rt=n("WordPiece"),It=n(" part d\u2019un petit vocabulaire comprenant les "),Ae=r("em"),Vt=n("tokens"),Gt=n(" sp\xE9ciaux utilis\xE9s par le mod\xE8le et l\u2019alphabet initial. Puisqu\u2019il identifie les sous-mots en ajoutant un pr\xE9fixe (comme "),Ne=r("code"),Jt=n("##"),Qt=n(" pour BERT), chaque mot est initialement d\xE9coup\xE9 en ajoutant ce pr\xE9fixe \xE0 tous les caract\xE8res du mot. Ainsi par exemple, "),Me=r("code"),Yt=n('"word"'),Xt=n(" est divis\xE9 comme ceci :"),pn=u(),h(ks.$$.fragment),on=u(),is=r("p"),Zt=n("Ainsi, l\u2019alphabet initial contient tous les caract\xE8res pr\xE9sents au d\xE9but d\u2019un mot et les caract\xE8res pr\xE9sents \xE0 l\u2019int\xE9rieur d\u2019un mot pr\xE9c\xE9d\xE9 du pr\xE9fixe de "),Be=r("em"),sl=n("WordPiece"),el=n("."),un=u(),V=r("p"),al=n("Ensuite, toujours comme le BPE, "),He=r("em"),nl=n("WordPiece"),tl=n(" apprend des r\xE8gles de fusion. La principale diff\xE9rence r\xE9side dans la mani\xE8re dont la paire \xE0 fusionner est s\xE9lectionn\xE9e. Au lieu de s\xE9lectionner la paire la plus fr\xE9quente, "),Se=r("em"),ll=n("WordPiece"),rl=n(` calcule un score pour chaque paire en utilisant la formule suivante :
`),cn=new Yi,mn=u(),z=r("p"),pl=n("En divisant la fr\xE9quence de la paire par le produit des fr\xE9quences de chacune de ses parties, l\u2019algorithme donne la priorit\xE9 \xE0 la fusion des paires dont les parties individuelles sont moins fr\xE9quentes dans le vocabulaire. Par exemple, il ne fusionnera pas n\xE9cessairement "),We=r("code"),ol=n('("un", "##able")'),il=n(" m\xEAme si cette paire appara\xEEt tr\xE8s fr\xE9quemment dans le vocabulaire car les deux paires "),Ke=r("code"),ul=n('"un"'),cl=n("\u201D et "),Ue=r("code"),ml=n('"##able"'),hl=n(" appara\xEEtront probablement chacune dans un batch d\u2019autres mots et auront une fr\xE9quence \xE9lev\xE9e. En revanche, une paire comme "),Fe=r("code"),dl=n('("hu", "##gging")'),fl=n(" sera probablement fusionn\xE9e plus rapidement (en supposant que le mot "),Le=r("code"),jl=n('"hugging"'),gl=n(" apparaisse souvent dans le vocabulaire) puisque "),Re=r("code"),bl=n('"hu"'),xl=n(" et "),Ie=r("code"),vl=n('"##gging"'),ql=n(" sont probablement moins fr\xE9quents individuellement."),hn=u(),pe=r("p"),_l=n("Examinons le m\xEAme vocabulaire que celui utilis\xE9 dans l\u2019exemple d\u2019entra\xEEnement du BPE :"),dn=u(),h(Es.$$.fragment),fn=u(),oe=r("p"),$l=n("Les divisions ici seront :"),jn=u(),h(ys.$$.fragment),gn=u(),P=r("p"),wl=n("Si on oublie les "),Ve=r("em"),kl=n("tokens"),El=n(" sp\xE9ciaux pour l\u2019instant, le vocabulaire initial sera donc "),Ge=r("code"),yl=n('["b", "h", "p", "##g", "##n", "##s", "##u"]'),Pl=n(". La paire la plus fr\xE9quente est "),Je=r("code"),Cl=n('("##u", "##g")'),zl=n(" (pr\xE9sente 20 fois), mais la fr\xE9quence individuelle de "),Qe=r("code"),Dl=n('"##u"'),Tl=n(" est tr\xE8s \xE9lev\xE9e, donc son score n\u2019est pas le plus \xE9lev\xE9 (il est de 1 / 36). Toutes les paires avec un "),Ye=r("code"),Ol=n('"##u"'),Al=n(" ont en fait le m\xEAme score (1 / 36). Ainsi le meilleur score va \xE0 la paire "),Xe=r("code"),Nl=n('("##g", "##s")'),Ml=n(" qui est la seule sans un "),Ze=r("code"),Bl=n('"##u"'),Hl=n(" avec un score 1 / 20. Et la premi\xE8re fusion apprise est "),sa=r("code"),Sl=n('("##g", "##s") -> ("##gs")'),Wl=n("."),bn=u(),R=r("p"),Kl=n("Notez que lorsque nous fusionnons, nous enlevons le "),ea=r("code"),Ul=n("##"),Fl=n(" entre les deux "),aa=r("em"),Ll=n("tokens"),Rl=n(", donc nous ajoutons "),na=r("code"),Il=n('"##gs"'),Vl=n(" au vocabulaire et appliquons la fusion dans les mots du corpus :"),xn=u(),h(Ps.$$.fragment),vn=u(),J=r("p"),Gl=n("\xC0 ce stade, "),ta=r("code"),Jl=n('" ##u "'),Ql=n(" est dans toutes les paires possibles, donc elles finissent toutes par avoir le m\xEAme score. Disons que dans ce cas, la premi\xE8re paire est fusionn\xE9e, donc "),la=r("code"),Yl=n('("h", "##u") -> "hu"'),Xl=n(". Cela nous am\xE8ne \xE0 :"),qn=u(),h(Cs.$$.fragment),_n=u(),Q=r("p"),Zl=n("Ensuite, le meilleur score suivant est partag\xE9 par "),ra=r("code"),sr=n('("hu", "##g")'),er=n(" et "),pa=r("code"),ar=n('("hu", "##gs")'),nr=n(" (avec 1/15, compar\xE9 \xE0 1/21 pour toutes les autres paires). Ainsi la premi\xE8re paire avec le plus grand score est fusionn\xE9e :"),$n=u(),h(zs.$$.fragment),wn=u(),ie=r("p"),tr=n("et nous continuons ainsi jusqu\u2019\xE0 ce que nous atteignions la taille de vocabulaire souhait\xE9e."),kn=u(),h(us.$$.fragment),En=u(),ns=r("h2"),cs=r("a"),oa=r("span"),h(Ds.$$.fragment),lr=u(),ia=r("span"),rr=n("Algorithme de tokenisation"),yn=u(),E=r("p"),pr=n("La tok\xE9nisation diff\xE8re dans "),ua=r("em"),or=n("WordPiece"),ir=n(" et BPE en ce que "),ca=r("em"),ur=n("WordPiece"),cr=n(" ne sauvegarde que le vocabulaire final et non pas les r\xE8gles de fusion apprises. En partant du mot \xE0 tokeniser, "),ma=r("em"),mr=n("WordPiece"),hr=n(" trouve le sous-mot le plus long qui se trouve dans le vocabulaire, puis se s\xE9pare sur celui-ci. Par exemple, si nous utilisons le vocabulaire appris dans l\u2019exemple ci-dessus, pour le mot "),ha=r("code"),dr=n('"hugs"'),fr=n(" le plus long sous-mot en partant du d\xE9but qui est dans le vocabulaire est "),da=r("code"),jr=n('"hug"'),gr=n(". Donc nous le divisons et obtenons "),fa=r("code"),br=n('["hug", "##s"]'),xr=n(". On continue avec "),ja=r("code"),vr=n('"##s"'),qr=n(", qui est dans le vocabulaire, donc la tokenisation de "),ga=r("code"),_r=n('"hugs"'),$r=n(" est "),ba=r("code"),wr=n('["hug", "##s"]'),kr=n("."),Pn=u(),ms=r("p"),Er=n("Avec BPE, nous aurions appliqu\xE9 les fusions apprises dans l\u2019ordre et la tok\xE9nisation aurait \xE9t\xE9 "),xa=r("code"),yr=n('["hu", "##gs"]'),Pr=n(", l\u2019encodage est donc diff\xE9rent."),Cn=u(),C=r("p"),Cr=n("Comme autre exemple, voyons comment le mot "),va=r("code"),zr=n('"bugs"'),Dr=n(" serait tokenis\xE9. "),qa=r("code"),Tr=n('"b"'),Or=n(" est le plus long sous-mot commen\xE7ant au d\xE9but du mot qui est dans le vocabulaire donc on le divise et on obtient "),_a=r("code"),Ar=n('["b", "##ugs"]'),Nr=n(". Ensuite, "),$a=r("code"),Mr=n('"##u"'),Br=n(" est le plus long sous-mot commen\xE7ant au d\xE9but de "),wa=r("code"),Hr=n('"##ugs"'),Sr=n(" qui est dans le vocabulaire, donc on le s\xE9pare et on obtient "),ka=r("code"),Wr=n('["b", "##u, "##gs"]'),Kr=n(". Enfin, "),Ea=r("code"),Ur=n('"##gs"'),Fr=n(" est dans le vocabulaire, donc cette derni\xE8re liste est la tokenization de "),ya=r("code"),Lr=n('"bugs"'),Rr=n("."),zn=u(),A=r("p"),Ir=n("Lorsque la tokenisation arrive \xE0 un stade o\xF9 il n\u2019est pas possible de trouver un sous-mot dans le vocabulaire, le mot entier est tokenis\xE9 comme inconnu. Par exemple, "),Pa=r("code"),Vr=n('"mug"'),Gr=n(" serait tokenis\xE9 comme "),Ca=r("code"),Jr=n('["[UNK]"]'),Qr=n(", tout comme "),za=r("code"),Yr=n('"bum"'),Xr=n(" (m\xEAme si on peut commencer par \u201D b \u201D et \u201D ##u \u201D, \u201D ##m \u201D ne fait pas partie du vocabulaire, et le "),Da=r("em"),Zr=n("tokenizer"),sp=n(" r\xE9sultant sera simplement "),Ta=r("code"),ep=n('["[UNK]"]'),ap=n(" \u201D et non "),Oa=r("code"),np=n('["b", "##u", "[UNK]"]'),tp=n(" \u201D). C\u2019est une autre diff\xE9rence avec le BPE qui classerait seulement les caract\xE8res individuels qui ne sont pas dans le vocabulaire comme inconnus."),Dn=u(),h(hs.$$.fragment),Tn=u(),ts=r("h2"),ds=r("a"),Aa=r("span"),h(Ts.$$.fragment),lp=u(),ue=r("span"),rp=n("Impl\xE9mentation de "),Na=r("i"),pp=n("WordPiece"),On=u(),fs=r("p"),op=n("Voyons maintenant une impl\xE9mentation de l\u2019algorithme "),Ma=r("em"),ip=n("WordPiece"),up=n(". Comme pour le BPE, il s\u2019agit d\u2019un exemple p\xE9dagogique et vous ne pourrez pas l\u2019utiliser sur un grand corpus."),An=u(),ce=r("p"),cp=n("Nous utiliserons le m\xEAme corpus que dans l\u2019exemple BPE :"),Nn=u(),h(Os.$$.fragment),Mn=u(),I=r("p"),mp=n("Tout d\u2019abord, nous devons pr\xE9tok\xE9niser le corpus en mots. Puisque nous r\xE9pliquons un "),Ba=r("em"),hp=n("tokenizer WordPiece"),dp=n(" (comme BERT), nous utiliserons le "),Ha=r("em"),fp=n("tokenizer"),jp=u(),Sa=r("code"),gp=n("bert-base-cased"),bp=n(" pour la pr\xE9tok\xE9nisation :"),Bn=u(),h(As.$$.fragment),Hn=u(),me=r("p"),xp=n("Ensuite, nous calculons les fr\xE9quences de chaque mot dans le corpus comme nous le faisons pour la pr\xE9tok\xE9nisation :"),Sn=u(),h(Ns.$$.fragment),Wn=u(),h(Ms.$$.fragment),Kn=u(),js=r("p"),vp=n("Comme nous l\u2019avons vu pr\xE9c\xE9demment, l\u2019alphabet est l\u2019unique ensemble compos\xE9 de toutes les premi\xE8res lettres des mots, et de toutes les autres lettres qui apparaissent dans les mots pr\xE9fix\xE9s par "),Wa=r("code"),qp=n("##"),_p=n(" :"),Un=u(),h(Bs.$$.fragment),Fn=u(),h(Hs.$$.fragment),Ln=u(),Y=r("p"),$p=n("Nous ajoutons \xE9galement les "),Ka=r("em"),wp=n("tokens"),kp=n(" sp\xE9ciaux utilis\xE9s par le mod\xE8le au d\xE9but de ce vocabulaire. Dans le cas de BERT, il s\u2019agit de la liste "),Ua=r("code"),Ep=n('["[PAD]", "[UNK]", "[CLS]", "[SEP]", "[MASK]"]'),yp=n(" :"),Rn=u(),h(Ss.$$.fragment),In=u(),gs=r("p"),Pp=n("Ensuite, nous devons diviser chaque mot, avec toutes les lettres qui ne sont pas les premi\xE8res pr\xE9fix\xE9es par "),Fa=r("code"),Cp=n("##"),zp=n(" :"),Vn=u(),h(Ws.$$.fragment),Gn=u(),he=r("p"),Dp=n("Maintenant que nous sommes pr\xEAts pour l\u2019entra\xEEnement, \xE9crivons une fonction qui calcule le score de chaque paire. Nous devrons l\u2019utiliser \xE0 chaque \xE9tape de l\u2019entra\xEEnement :"),Jn=u(),h(Ks.$$.fragment),Qn=u(),de=r("p"),Tp=n("Jetons un coup d\u2019\u0153il \xE0 une partie de ce dictionnaire apr\xE8s les premi\xE8res divisions :"),Yn=u(),h(Us.$$.fragment),Xn=u(),h(Fs.$$.fragment),Zn=u(),fe=r("p"),Op=n("Maintenant, trouver la paire avec le meilleur score ne prend qu\u2019une rapide boucle :"),st=u(),h(Ls.$$.fragment),et=u(),h(Rs.$$.fragment),at=u(),X=r("p"),Ap=n("Ainsi, la premi\xE8re fusion \xE0 apprendre est "),La=r("code"),Np=n("('a', '##b') -> 'ab'"),Mp=n(" et nous ajoutons "),Ra=r("code"),Bp=n("'ab'"),Hp=n(" au vocabulaire :"),nt=u(),h(Is.$$.fragment),tt=u(),bs=r("p"),Sp=n("Pour continuer, nous devons appliquer cette fusion dans notre dictionnaire "),Ia=r("code"),Wp=n("splits"),Kp=n(". \xC9crivons une autre fonction pour cela :"),lt=u(),h(Vs.$$.fragment),rt=u(),je=r("p"),Up=n("Et nous pouvons regarder le r\xE9sultat de la premi\xE8re fusion :"),pt=u(),h(Gs.$$.fragment),ot=u(),h(Js.$$.fragment),it=u(),ge=r("p"),Fp=n("Nous avons maintenant tout ce dont nous avons besoin pour boucler jusqu\u2019\xE0 ce que nous ayons appris toutes les fusions que nous voulons. Visons une taille de vocabulaire de 70 :"),ut=u(),h(Qs.$$.fragment),ct=u(),be=r("p"),Lp=n("Nous pouvons ensuite examiner le vocabulaire g\xE9n\xE9r\xE9 :"),mt=u(),h(Ys.$$.fragment),ht=u(),h(Xs.$$.fragment),dt=u(),Z=r("p"),Rp=n("Comme nous pouvons le voir, compar\xE9 \xE0 BPE, ce "),Va=r("em"),Ip=n("tokenizer"),Vp=n(" apprend les parties de mots comme des "),Ga=r("em"),Gp=n("tokens"),Jp=n(" un peu plus rapidement."),ft=u(),h(xs.$$.fragment),jt=u(),xe=r("p"),Qp=n("Pour tokeniser un nouveau texte, on le pr\xE9tokenise, on le divise, puis on applique l\u2019algorithme de tokenisation sur chaque mot. En d\u2019autres termes, nous recherchons le plus grand sous-mot commen\xE7ant au d\xE9but du premier mot et le divisons. Puis nous r\xE9p\xE9tons le processus sur la deuxi\xE8me partie et ainsi de suite pour le reste de ce mot et les mots suivants dans le texte :"),gt=u(),h(Zs.$$.fragment),bt=u(),ve=r("p"),Yp=n("Testons-le sur un mot qui fait partie du vocabulaire, et un autre qui n\u2019en fait pas partie :"),xt=u(),h(se.$$.fragment),vt=u(),h(ee.$$.fragment),qt=u(),qe=r("p"),Xp=n("Maintenant, \xE9crivons une fonction qui permet de tokeniser un texte :"),_t=u(),h(ae.$$.fragment),$t=u(),_e=r("p"),Zp=n("On peut l\u2019essayer sur n\u2019importe quel texte :"),wt=u(),h(ne.$$.fragment),kt=u(),h(te.$$.fragment),Et=u(),ss=r("p"),so=n("C\u2019est tout pour l\u2019algorithme "),Ja=r("em"),eo=n("WordPiece"),ao=n(" ! Maintenant, jetons un coup d\u2019oeil \xE0 "),Qa=r("em"),no=n("Unigram"),to=n("."),this.h()},l(s){const l=Xi('[data-svelte="svelte-1phssyn"]',document.head);m=p(l,"META",{name:!0,content:!0}),l.forEach(a),k=c(s),x=p(s,"H1",{class:!0});var le=o(x);$=p(le,"A",{id:!0,class:!0,href:!0});var Ya=o($);w=p(Ya,"SPAN",{});var Xa=o(w);d(v.$$.fragment,Xa),Xa.forEach(a),Ya.forEach(a),q=c(le),y=p(le,"SPAN",{});var $e=o(y);S=t($e,"Tok\xE9nisation "),T=p($e,"I",{});var Za=o(T);G=t(Za,"WordPiece"),Za.forEach(a),$e.forEach(a),le.forEach(a),U=c(s),d(O.$$.fragment,s),_s=c(s),L=p(s,"P",{});var sn=o(L);ls=p(sn,"EM",{});var ro=o(ls);Ht=t(ro,"WordPiece"),ro.forEach(a),St=t(sn," est l\u2019algorithme de tok\xE9nisation d\xE9velopp\xE9 par Google pour pr\xE9tra\xEEner BERT. Il a depuis \xE9t\xE9 r\xE9utilis\xE9 dans un grand nombre de mod\xE8les de "),ze=p(sn,"EM",{});var po=o(ze);Wt=t(po,"transformers"),po.forEach(a),Kt=t(sn," bas\xE9s sur BERT tels que DistilBERT, MobileBERT, Funnel Transformers et MPNET. Il est tr\xE8s similaire au BPE en termes d\u2019entra\xEEnement mais la tokenisation r\xE9elle est effectu\xE9e diff\xE9remment."),sn.forEach(a),an=c(s),d($s.$$.fragment,s),nn=c(s),d(rs.$$.fragment,s),tn=c(s),as=p(s,"H2",{class:!0});var Pt=o(as);ps=p(Pt,"A",{id:!0,class:!0,href:!0});var oo=o(ps);De=p(oo,"SPAN",{});var io=o(De);d(ws.$$.fragment,io),io.forEach(a),oo.forEach(a),Ut=c(Pt),Te=p(Pt,"SPAN",{});var uo=o(Te);Ft=t(uo,"Algorithme d'entra\xEEnement"),uo.forEach(a),Pt.forEach(a),ln=c(s),d(os.$$.fragment,s),rn=c(s),W=p(s,"P",{});var es=o(W);Lt=t(es,"Comme le BPE, "),Oe=p(es,"EM",{});var co=o(Oe);Rt=t(co,"WordPiece"),co.forEach(a),It=t(es," part d\u2019un petit vocabulaire comprenant les "),Ae=p(es,"EM",{});var mo=o(Ae);Vt=t(mo,"tokens"),mo.forEach(a),Gt=t(es," sp\xE9ciaux utilis\xE9s par le mod\xE8le et l\u2019alphabet initial. Puisqu\u2019il identifie les sous-mots en ajoutant un pr\xE9fixe (comme "),Ne=p(es,"CODE",{});var ho=o(Ne);Jt=t(ho,"##"),ho.forEach(a),Qt=t(es," pour BERT), chaque mot est initialement d\xE9coup\xE9 en ajoutant ce pr\xE9fixe \xE0 tous les caract\xE8res du mot. Ainsi par exemple, "),Me=p(es,"CODE",{});var fo=o(Me);Yt=t(fo,'"word"'),fo.forEach(a),Xt=t(es," est divis\xE9 comme ceci :"),es.forEach(a),pn=c(s),d(ks.$$.fragment,s),on=c(s),is=p(s,"P",{});var Ct=o(is);Zt=t(Ct,"Ainsi, l\u2019alphabet initial contient tous les caract\xE8res pr\xE9sents au d\xE9but d\u2019un mot et les caract\xE8res pr\xE9sents \xE0 l\u2019int\xE9rieur d\u2019un mot pr\xE9c\xE9d\xE9 du pr\xE9fixe de "),Be=p(Ct,"EM",{});var jo=o(Be);sl=t(jo,"WordPiece"),jo.forEach(a),el=t(Ct,"."),Ct.forEach(a),un=c(s),V=p(s,"P",{});var re=o(V);al=t(re,"Ensuite, toujours comme le BPE, "),He=p(re,"EM",{});var go=o(He);nl=t(go,"WordPiece"),go.forEach(a),tl=t(re," apprend des r\xE8gles de fusion. La principale diff\xE9rence r\xE9side dans la mani\xE8re dont la paire \xE0 fusionner est s\xE9lectionn\xE9e. Au lieu de s\xE9lectionner la paire la plus fr\xE9quente, "),Se=p(re,"EM",{});var bo=o(Se);ll=t(bo,"WordPiece"),bo.forEach(a),rl=t(re,` calcule un score pour chaque paire en utilisant la formule suivante :
`),cn=Zi(re),re.forEach(a),mn=c(s),z=p(s,"P",{});var H=o(z);pl=t(H,"En divisant la fr\xE9quence de la paire par le produit des fr\xE9quences de chacune de ses parties, l\u2019algorithme donne la priorit\xE9 \xE0 la fusion des paires dont les parties individuelles sont moins fr\xE9quentes dans le vocabulaire. Par exemple, il ne fusionnera pas n\xE9cessairement "),We=p(H,"CODE",{});var xo=o(We);ol=t(xo,'("un", "##able")'),xo.forEach(a),il=t(H," m\xEAme si cette paire appara\xEEt tr\xE8s fr\xE9quemment dans le vocabulaire car les deux paires "),Ke=p(H,"CODE",{});var vo=o(Ke);ul=t(vo,'"un"'),vo.forEach(a),cl=t(H,"\u201D et "),Ue=p(H,"CODE",{});var qo=o(Ue);ml=t(qo,'"##able"'),qo.forEach(a),hl=t(H," appara\xEEtront probablement chacune dans un batch d\u2019autres mots et auront une fr\xE9quence \xE9lev\xE9e. En revanche, une paire comme "),Fe=p(H,"CODE",{});var _o=o(Fe);dl=t(_o,'("hu", "##gging")'),_o.forEach(a),fl=t(H," sera probablement fusionn\xE9e plus rapidement (en supposant que le mot "),Le=p(H,"CODE",{});var $o=o(Le);jl=t($o,'"hugging"'),$o.forEach(a),gl=t(H," apparaisse souvent dans le vocabulaire) puisque "),Re=p(H,"CODE",{});var wo=o(Re);bl=t(wo,'"hu"'),wo.forEach(a),xl=t(H," et "),Ie=p(H,"CODE",{});var ko=o(Ie);vl=t(ko,'"##gging"'),ko.forEach(a),ql=t(H," sont probablement moins fr\xE9quents individuellement."),H.forEach(a),hn=c(s),pe=p(s,"P",{});var Eo=o(pe);_l=t(Eo,"Examinons le m\xEAme vocabulaire que celui utilis\xE9 dans l\u2019exemple d\u2019entra\xEEnement du BPE :"),Eo.forEach(a),dn=c(s),d(Es.$$.fragment,s),fn=c(s),oe=p(s,"P",{});var yo=o(oe);$l=t(yo,"Les divisions ici seront :"),yo.forEach(a),jn=c(s),d(ys.$$.fragment,s),gn=c(s),P=p(s,"P",{});var N=o(P);wl=t(N,"Si on oublie les "),Ve=p(N,"EM",{});var Po=o(Ve);kl=t(Po,"tokens"),Po.forEach(a),El=t(N," sp\xE9ciaux pour l\u2019instant, le vocabulaire initial sera donc "),Ge=p(N,"CODE",{});var Co=o(Ge);yl=t(Co,'["b", "h", "p", "##g", "##n", "##s", "##u"]'),Co.forEach(a),Pl=t(N,". La paire la plus fr\xE9quente est "),Je=p(N,"CODE",{});var zo=o(Je);Cl=t(zo,'("##u", "##g")'),zo.forEach(a),zl=t(N," (pr\xE9sente 20 fois), mais la fr\xE9quence individuelle de "),Qe=p(N,"CODE",{});var Do=o(Qe);Dl=t(Do,'"##u"'),Do.forEach(a),Tl=t(N," est tr\xE8s \xE9lev\xE9e, donc son score n\u2019est pas le plus \xE9lev\xE9 (il est de 1 / 36). Toutes les paires avec un "),Ye=p(N,"CODE",{});var To=o(Ye);Ol=t(To,'"##u"'),To.forEach(a),Al=t(N," ont en fait le m\xEAme score (1 / 36). Ainsi le meilleur score va \xE0 la paire "),Xe=p(N,"CODE",{});var Oo=o(Xe);Nl=t(Oo,'("##g", "##s")'),Oo.forEach(a),Ml=t(N," qui est la seule sans un "),Ze=p(N,"CODE",{});var Ao=o(Ze);Bl=t(Ao,'"##u"'),Ao.forEach(a),Hl=t(N," avec un score 1 / 20. Et la premi\xE8re fusion apprise est "),sa=p(N,"CODE",{});var No=o(sa);Sl=t(No,'("##g", "##s") -> ("##gs")'),No.forEach(a),Wl=t(N,"."),N.forEach(a),bn=c(s),R=p(s,"P",{});var vs=o(R);Kl=t(vs,"Notez que lorsque nous fusionnons, nous enlevons le "),ea=p(vs,"CODE",{});var Mo=o(ea);Ul=t(Mo,"##"),Mo.forEach(a),Fl=t(vs," entre les deux "),aa=p(vs,"EM",{});var Bo=o(aa);Ll=t(Bo,"tokens"),Bo.forEach(a),Rl=t(vs,", donc nous ajoutons "),na=p(vs,"CODE",{});var Ho=o(na);Il=t(Ho,'"##gs"'),Ho.forEach(a),Vl=t(vs," au vocabulaire et appliquons la fusion dans les mots du corpus :"),vs.forEach(a),xn=c(s),d(Ps.$$.fragment,s),vn=c(s),J=p(s,"P",{});var we=o(J);Gl=t(we,"\xC0 ce stade, "),ta=p(we,"CODE",{});var So=o(ta);Jl=t(So,'" ##u "'),So.forEach(a),Ql=t(we," est dans toutes les paires possibles, donc elles finissent toutes par avoir le m\xEAme score. Disons que dans ce cas, la premi\xE8re paire est fusionn\xE9e, donc "),la=p(we,"CODE",{});var Wo=o(la);Yl=t(Wo,'("h", "##u") -> "hu"'),Wo.forEach(a),Xl=t(we,". Cela nous am\xE8ne \xE0 :"),we.forEach(a),qn=c(s),d(Cs.$$.fragment,s),_n=c(s),Q=p(s,"P",{});var ke=o(Q);Zl=t(ke,"Ensuite, le meilleur score suivant est partag\xE9 par "),ra=p(ke,"CODE",{});var Ko=o(ra);sr=t(Ko,'("hu", "##g")'),Ko.forEach(a),er=t(ke," et "),pa=p(ke,"CODE",{});var Uo=o(pa);ar=t(Uo,'("hu", "##gs")'),Uo.forEach(a),nr=t(ke," (avec 1/15, compar\xE9 \xE0 1/21 pour toutes les autres paires). Ainsi la premi\xE8re paire avec le plus grand score est fusionn\xE9e :"),ke.forEach(a),$n=c(s),d(zs.$$.fragment,s),wn=c(s),ie=p(s,"P",{});var Fo=o(ie);tr=t(Fo,"et nous continuons ainsi jusqu\u2019\xE0 ce que nous atteignions la taille de vocabulaire souhait\xE9e."),Fo.forEach(a),kn=c(s),d(us.$$.fragment,s),En=c(s),ns=p(s,"H2",{class:!0});var zt=o(ns);cs=p(zt,"A",{id:!0,class:!0,href:!0});var Lo=o(cs);oa=p(Lo,"SPAN",{});var Ro=o(oa);d(Ds.$$.fragment,Ro),Ro.forEach(a),Lo.forEach(a),lr=c(zt),ia=p(zt,"SPAN",{});var Io=o(ia);rr=t(Io,"Algorithme de tokenisation"),Io.forEach(a),zt.forEach(a),yn=c(s),E=p(s,"P",{});var D=o(E);pr=t(D,"La tok\xE9nisation diff\xE8re dans "),ua=p(D,"EM",{});var Vo=o(ua);or=t(Vo,"WordPiece"),Vo.forEach(a),ir=t(D," et BPE en ce que "),ca=p(D,"EM",{});var Go=o(ca);ur=t(Go,"WordPiece"),Go.forEach(a),cr=t(D," ne sauvegarde que le vocabulaire final et non pas les r\xE8gles de fusion apprises. En partant du mot \xE0 tokeniser, "),ma=p(D,"EM",{});var Jo=o(ma);mr=t(Jo,"WordPiece"),Jo.forEach(a),hr=t(D," trouve le sous-mot le plus long qui se trouve dans le vocabulaire, puis se s\xE9pare sur celui-ci. Par exemple, si nous utilisons le vocabulaire appris dans l\u2019exemple ci-dessus, pour le mot "),ha=p(D,"CODE",{});var Qo=o(ha);dr=t(Qo,'"hugs"'),Qo.forEach(a),fr=t(D," le plus long sous-mot en partant du d\xE9but qui est dans le vocabulaire est "),da=p(D,"CODE",{});var Yo=o(da);jr=t(Yo,'"hug"'),Yo.forEach(a),gr=t(D,". Donc nous le divisons et obtenons "),fa=p(D,"CODE",{});var Xo=o(fa);br=t(Xo,'["hug", "##s"]'),Xo.forEach(a),xr=t(D,". On continue avec "),ja=p(D,"CODE",{});var Zo=o(ja);vr=t(Zo,'"##s"'),Zo.forEach(a),qr=t(D,", qui est dans le vocabulaire, donc la tokenisation de "),ga=p(D,"CODE",{});var si=o(ga);_r=t(si,'"hugs"'),si.forEach(a),$r=t(D," est "),ba=p(D,"CODE",{});var ei=o(ba);wr=t(ei,'["hug", "##s"]'),ei.forEach(a),kr=t(D,"."),D.forEach(a),Pn=c(s),ms=p(s,"P",{});var Dt=o(ms);Er=t(Dt,"Avec BPE, nous aurions appliqu\xE9 les fusions apprises dans l\u2019ordre et la tok\xE9nisation aurait \xE9t\xE9 "),xa=p(Dt,"CODE",{});var ai=o(xa);yr=t(ai,'["hu", "##gs"]'),ai.forEach(a),Pr=t(Dt,", l\u2019encodage est donc diff\xE9rent."),Dt.forEach(a),Cn=c(s),C=p(s,"P",{});var M=o(C);Cr=t(M,"Comme autre exemple, voyons comment le mot "),va=p(M,"CODE",{});var ni=o(va);zr=t(ni,'"bugs"'),ni.forEach(a),Dr=t(M," serait tokenis\xE9. "),qa=p(M,"CODE",{});var ti=o(qa);Tr=t(ti,'"b"'),ti.forEach(a),Or=t(M," est le plus long sous-mot commen\xE7ant au d\xE9but du mot qui est dans le vocabulaire donc on le divise et on obtient "),_a=p(M,"CODE",{});var li=o(_a);Ar=t(li,'["b", "##ugs"]'),li.forEach(a),Nr=t(M,". Ensuite, "),$a=p(M,"CODE",{});var ri=o($a);Mr=t(ri,'"##u"'),ri.forEach(a),Br=t(M," est le plus long sous-mot commen\xE7ant au d\xE9but de "),wa=p(M,"CODE",{});var pi=o(wa);Hr=t(pi,'"##ugs"'),pi.forEach(a),Sr=t(M," qui est dans le vocabulaire, donc on le s\xE9pare et on obtient "),ka=p(M,"CODE",{});var oi=o(ka);Wr=t(oi,'["b", "##u, "##gs"]'),oi.forEach(a),Kr=t(M,". Enfin, "),Ea=p(M,"CODE",{});var ii=o(Ea);Ur=t(ii,'"##gs"'),ii.forEach(a),Fr=t(M," est dans le vocabulaire, donc cette derni\xE8re liste est la tokenization de "),ya=p(M,"CODE",{});var ui=o(ya);Lr=t(ui,'"bugs"'),ui.forEach(a),Rr=t(M,"."),M.forEach(a),zn=c(s),A=p(s,"P",{});var K=o(A);Ir=t(K,"Lorsque la tokenisation arrive \xE0 un stade o\xF9 il n\u2019est pas possible de trouver un sous-mot dans le vocabulaire, le mot entier est tokenis\xE9 comme inconnu. Par exemple, "),Pa=p(K,"CODE",{});var ci=o(Pa);Vr=t(ci,'"mug"'),ci.forEach(a),Gr=t(K," serait tokenis\xE9 comme "),Ca=p(K,"CODE",{});var mi=o(Ca);Jr=t(mi,'["[UNK]"]'),mi.forEach(a),Qr=t(K,", tout comme "),za=p(K,"CODE",{});var hi=o(za);Yr=t(hi,'"bum"'),hi.forEach(a),Xr=t(K," (m\xEAme si on peut commencer par \u201D b \u201D et \u201D ##u \u201D, \u201D ##m \u201D ne fait pas partie du vocabulaire, et le "),Da=p(K,"EM",{});var di=o(Da);Zr=t(di,"tokenizer"),di.forEach(a),sp=t(K," r\xE9sultant sera simplement "),Ta=p(K,"CODE",{});var fi=o(Ta);ep=t(fi,'["[UNK]"]'),fi.forEach(a),ap=t(K," \u201D et non "),Oa=p(K,"CODE",{});var ji=o(Oa);np=t(ji,'["b", "##u", "[UNK]"]'),ji.forEach(a),tp=t(K," \u201D). C\u2019est une autre diff\xE9rence avec le BPE qui classerait seulement les caract\xE8res individuels qui ne sont pas dans le vocabulaire comme inconnus."),K.forEach(a),Dn=c(s),d(hs.$$.fragment,s),Tn=c(s),ts=p(s,"H2",{class:!0});var Tt=o(ts);ds=p(Tt,"A",{id:!0,class:!0,href:!0});var gi=o(ds);Aa=p(gi,"SPAN",{});var bi=o(Aa);d(Ts.$$.fragment,bi),bi.forEach(a),gi.forEach(a),lp=c(Tt),ue=p(Tt,"SPAN",{});var lo=o(ue);rp=t(lo,"Impl\xE9mentation de "),Na=p(lo,"I",{});var xi=o(Na);pp=t(xi,"WordPiece"),xi.forEach(a),lo.forEach(a),Tt.forEach(a),On=c(s),fs=p(s,"P",{});var Ot=o(fs);op=t(Ot,"Voyons maintenant une impl\xE9mentation de l\u2019algorithme "),Ma=p(Ot,"EM",{});var vi=o(Ma);ip=t(vi,"WordPiece"),vi.forEach(a),up=t(Ot,". Comme pour le BPE, il s\u2019agit d\u2019un exemple p\xE9dagogique et vous ne pourrez pas l\u2019utiliser sur un grand corpus."),Ot.forEach(a),An=c(s),ce=p(s,"P",{});var qi=o(ce);cp=t(qi,"Nous utiliserons le m\xEAme corpus que dans l\u2019exemple BPE :"),qi.forEach(a),Nn=c(s),d(Os.$$.fragment,s),Mn=c(s),I=p(s,"P",{});var qs=o(I);mp=t(qs,"Tout d\u2019abord, nous devons pr\xE9tok\xE9niser le corpus en mots. Puisque nous r\xE9pliquons un "),Ba=p(qs,"EM",{});var _i=o(Ba);hp=t(_i,"tokenizer WordPiece"),_i.forEach(a),dp=t(qs," (comme BERT), nous utiliserons le "),Ha=p(qs,"EM",{});var $i=o(Ha);fp=t($i,"tokenizer"),$i.forEach(a),jp=c(qs),Sa=p(qs,"CODE",{});var wi=o(Sa);gp=t(wi,"bert-base-cased"),wi.forEach(a),bp=t(qs," pour la pr\xE9tok\xE9nisation :"),qs.forEach(a),Bn=c(s),d(As.$$.fragment,s),Hn=c(s),me=p(s,"P",{});var ki=o(me);xp=t(ki,"Ensuite, nous calculons les fr\xE9quences de chaque mot dans le corpus comme nous le faisons pour la pr\xE9tok\xE9nisation :"),ki.forEach(a),Sn=c(s),d(Ns.$$.fragment,s),Wn=c(s),d(Ms.$$.fragment,s),Kn=c(s),js=p(s,"P",{});var At=o(js);vp=t(At,"Comme nous l\u2019avons vu pr\xE9c\xE9demment, l\u2019alphabet est l\u2019unique ensemble compos\xE9 de toutes les premi\xE8res lettres des mots, et de toutes les autres lettres qui apparaissent dans les mots pr\xE9fix\xE9s par "),Wa=p(At,"CODE",{});var Ei=o(Wa);qp=t(Ei,"##"),Ei.forEach(a),_p=t(At," :"),At.forEach(a),Un=c(s),d(Bs.$$.fragment,s),Fn=c(s),d(Hs.$$.fragment,s),Ln=c(s),Y=p(s,"P",{});var Ee=o(Y);$p=t(Ee,"Nous ajoutons \xE9galement les "),Ka=p(Ee,"EM",{});var yi=o(Ka);wp=t(yi,"tokens"),yi.forEach(a),kp=t(Ee," sp\xE9ciaux utilis\xE9s par le mod\xE8le au d\xE9but de ce vocabulaire. Dans le cas de BERT, il s\u2019agit de la liste "),Ua=p(Ee,"CODE",{});var Pi=o(Ua);Ep=t(Pi,'["[PAD]", "[UNK]", "[CLS]", "[SEP]", "[MASK]"]'),Pi.forEach(a),yp=t(Ee," :"),Ee.forEach(a),Rn=c(s),d(Ss.$$.fragment,s),In=c(s),gs=p(s,"P",{});var Nt=o(gs);Pp=t(Nt,"Ensuite, nous devons diviser chaque mot, avec toutes les lettres qui ne sont pas les premi\xE8res pr\xE9fix\xE9es par "),Fa=p(Nt,"CODE",{});var Ci=o(Fa);Cp=t(Ci,"##"),Ci.forEach(a),zp=t(Nt," :"),Nt.forEach(a),Vn=c(s),d(Ws.$$.fragment,s),Gn=c(s),he=p(s,"P",{});var zi=o(he);Dp=t(zi,"Maintenant que nous sommes pr\xEAts pour l\u2019entra\xEEnement, \xE9crivons une fonction qui calcule le score de chaque paire. Nous devrons l\u2019utiliser \xE0 chaque \xE9tape de l\u2019entra\xEEnement :"),zi.forEach(a),Jn=c(s),d(Ks.$$.fragment,s),Qn=c(s),de=p(s,"P",{});var Di=o(de);Tp=t(Di,"Jetons un coup d\u2019\u0153il \xE0 une partie de ce dictionnaire apr\xE8s les premi\xE8res divisions :"),Di.forEach(a),Yn=c(s),d(Us.$$.fragment,s),Xn=c(s),d(Fs.$$.fragment,s),Zn=c(s),fe=p(s,"P",{});var Ti=o(fe);Op=t(Ti,"Maintenant, trouver la paire avec le meilleur score ne prend qu\u2019une rapide boucle :"),Ti.forEach(a),st=c(s),d(Ls.$$.fragment,s),et=c(s),d(Rs.$$.fragment,s),at=c(s),X=p(s,"P",{});var ye=o(X);Ap=t(ye,"Ainsi, la premi\xE8re fusion \xE0 apprendre est "),La=p(ye,"CODE",{});var Oi=o(La);Np=t(Oi,"('a', '##b') -> 'ab'"),Oi.forEach(a),Mp=t(ye," et nous ajoutons "),Ra=p(ye,"CODE",{});var Ai=o(Ra);Bp=t(Ai,"'ab'"),Ai.forEach(a),Hp=t(ye," au vocabulaire :"),ye.forEach(a),nt=c(s),d(Is.$$.fragment,s),tt=c(s),bs=p(s,"P",{});var Mt=o(bs);Sp=t(Mt,"Pour continuer, nous devons appliquer cette fusion dans notre dictionnaire "),Ia=p(Mt,"CODE",{});var Ni=o(Ia);Wp=t(Ni,"splits"),Ni.forEach(a),Kp=t(Mt,". \xC9crivons une autre fonction pour cela :"),Mt.forEach(a),lt=c(s),d(Vs.$$.fragment,s),rt=c(s),je=p(s,"P",{});var Mi=o(je);Up=t(Mi,"Et nous pouvons regarder le r\xE9sultat de la premi\xE8re fusion :"),Mi.forEach(a),pt=c(s),d(Gs.$$.fragment,s),ot=c(s),d(Js.$$.fragment,s),it=c(s),ge=p(s,"P",{});var Bi=o(ge);Fp=t(Bi,"Nous avons maintenant tout ce dont nous avons besoin pour boucler jusqu\u2019\xE0 ce que nous ayons appris toutes les fusions que nous voulons. Visons une taille de vocabulaire de 70 :"),Bi.forEach(a),ut=c(s),d(Qs.$$.fragment,s),ct=c(s),be=p(s,"P",{});var Hi=o(be);Lp=t(Hi,"Nous pouvons ensuite examiner le vocabulaire g\xE9n\xE9r\xE9 :"),Hi.forEach(a),mt=c(s),d(Ys.$$.fragment,s),ht=c(s),d(Xs.$$.fragment,s),dt=c(s),Z=p(s,"P",{});var Pe=o(Z);Rp=t(Pe,"Comme nous pouvons le voir, compar\xE9 \xE0 BPE, ce "),Va=p(Pe,"EM",{});var Si=o(Va);Ip=t(Si,"tokenizer"),Si.forEach(a),Vp=t(Pe," apprend les parties de mots comme des "),Ga=p(Pe,"EM",{});var Wi=o(Ga);Gp=t(Wi,"tokens"),Wi.forEach(a),Jp=t(Pe," un peu plus rapidement."),Pe.forEach(a),ft=c(s),d(xs.$$.fragment,s),jt=c(s),xe=p(s,"P",{});var Ki=o(xe);Qp=t(Ki,"Pour tokeniser un nouveau texte, on le pr\xE9tokenise, on le divise, puis on applique l\u2019algorithme de tokenisation sur chaque mot. En d\u2019autres termes, nous recherchons le plus grand sous-mot commen\xE7ant au d\xE9but du premier mot et le divisons. Puis nous r\xE9p\xE9tons le processus sur la deuxi\xE8me partie et ainsi de suite pour le reste de ce mot et les mots suivants dans le texte :"),Ki.forEach(a),gt=c(s),d(Zs.$$.fragment,s),bt=c(s),ve=p(s,"P",{});var Ui=o(ve);Yp=t(Ui,"Testons-le sur un mot qui fait partie du vocabulaire, et un autre qui n\u2019en fait pas partie :"),Ui.forEach(a),xt=c(s),d(se.$$.fragment,s),vt=c(s),d(ee.$$.fragment,s),qt=c(s),qe=p(s,"P",{});var Fi=o(qe);Xp=t(Fi,"Maintenant, \xE9crivons une fonction qui permet de tokeniser un texte :"),Fi.forEach(a),_t=c(s),d(ae.$$.fragment,s),$t=c(s),_e=p(s,"P",{});var Li=o(_e);Zp=t(Li,"On peut l\u2019essayer sur n\u2019importe quel texte :"),Li.forEach(a),wt=c(s),d(ne.$$.fragment,s),kt=c(s),d(te.$$.fragment,s),Et=c(s),ss=p(s,"P",{});var Ce=o(ss);so=t(Ce,"C\u2019est tout pour l\u2019algorithme "),Ja=p(Ce,"EM",{});var Ri=o(Ja);eo=t(Ri,"WordPiece"),Ri.forEach(a),ao=t(Ce," ! Maintenant, jetons un coup d\u2019oeil \xE0 "),Qa=p(Ce,"EM",{});var Ii=o(Qa);no=t(Ii,"Unigram"),Ii.forEach(a),to=t(Ce,"."),Ce.forEach(a),this.h()},h(){B(m,"name","hf:doc:metadata"),B(m,"content",JSON.stringify(iu)),B($,"id","toknisation-iwordpiecei"),B($,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),B($,"href","#toknisation-iwordpiecei"),B(x,"class","relative group"),B(ps,"id","algorithme-dentranement"),B(ps,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),B(ps,"href","#algorithme-dentranement"),B(as,"class","relative group"),cn.a=null,B(cs,"id","algorithme-de-tokenisation"),B(cs,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),B(cs,"href","#algorithme-de-tokenisation"),B(ns,"class","relative group"),B(ds,"id","implmentation-de-iwordpiecei"),B(ds,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),B(ds,"href","#implmentation-de-iwordpiecei"),B(ts,"class","relative group")},m(s,l){e(document.head,m),i(s,k,l),i(s,x,l),e(x,$),e($,w),f(v,w,null),e(x,q),e(x,y),e(y,S),e(y,T),e(T,G),i(s,U,l),f(O,s,l),i(s,_s,l),i(s,L,l),e(L,ls),e(ls,Ht),e(L,St),e(L,ze),e(ze,Wt),e(L,Kt),i(s,an,l),f($s,s,l),i(s,nn,l),f(rs,s,l),i(s,tn,l),i(s,as,l),e(as,ps),e(ps,De),f(ws,De,null),e(as,Ut),e(as,Te),e(Te,Ft),i(s,ln,l),f(os,s,l),i(s,rn,l),i(s,W,l),e(W,Lt),e(W,Oe),e(Oe,Rt),e(W,It),e(W,Ae),e(Ae,Vt),e(W,Gt),e(W,Ne),e(Ne,Jt),e(W,Qt),e(W,Me),e(Me,Yt),e(W,Xt),i(s,pn,l),f(ks,s,l),i(s,on,l),i(s,is,l),e(is,Zt),e(is,Be),e(Be,sl),e(is,el),i(s,un,l),i(s,V,l),e(V,al),e(V,He),e(He,nl),e(V,tl),e(V,Se),e(Se,ll),e(V,rl),cn.m(Vi,V),i(s,mn,l),i(s,z,l),e(z,pl),e(z,We),e(We,ol),e(z,il),e(z,Ke),e(Ke,ul),e(z,cl),e(z,Ue),e(Ue,ml),e(z,hl),e(z,Fe),e(Fe,dl),e(z,fl),e(z,Le),e(Le,jl),e(z,gl),e(z,Re),e(Re,bl),e(z,xl),e(z,Ie),e(Ie,vl),e(z,ql),i(s,hn,l),i(s,pe,l),e(pe,_l),i(s,dn,l),f(Es,s,l),i(s,fn,l),i(s,oe,l),e(oe,$l),i(s,jn,l),f(ys,s,l),i(s,gn,l),i(s,P,l),e(P,wl),e(P,Ve),e(Ve,kl),e(P,El),e(P,Ge),e(Ge,yl),e(P,Pl),e(P,Je),e(Je,Cl),e(P,zl),e(P,Qe),e(Qe,Dl),e(P,Tl),e(P,Ye),e(Ye,Ol),e(P,Al),e(P,Xe),e(Xe,Nl),e(P,Ml),e(P,Ze),e(Ze,Bl),e(P,Hl),e(P,sa),e(sa,Sl),e(P,Wl),i(s,bn,l),i(s,R,l),e(R,Kl),e(R,ea),e(ea,Ul),e(R,Fl),e(R,aa),e(aa,Ll),e(R,Rl),e(R,na),e(na,Il),e(R,Vl),i(s,xn,l),f(Ps,s,l),i(s,vn,l),i(s,J,l),e(J,Gl),e(J,ta),e(ta,Jl),e(J,Ql),e(J,la),e(la,Yl),e(J,Xl),i(s,qn,l),f(Cs,s,l),i(s,_n,l),i(s,Q,l),e(Q,Zl),e(Q,ra),e(ra,sr),e(Q,er),e(Q,pa),e(pa,ar),e(Q,nr),i(s,$n,l),f(zs,s,l),i(s,wn,l),i(s,ie,l),e(ie,tr),i(s,kn,l),f(us,s,l),i(s,En,l),i(s,ns,l),e(ns,cs),e(cs,oa),f(Ds,oa,null),e(ns,lr),e(ns,ia),e(ia,rr),i(s,yn,l),i(s,E,l),e(E,pr),e(E,ua),e(ua,or),e(E,ir),e(E,ca),e(ca,ur),e(E,cr),e(E,ma),e(ma,mr),e(E,hr),e(E,ha),e(ha,dr),e(E,fr),e(E,da),e(da,jr),e(E,gr),e(E,fa),e(fa,br),e(E,xr),e(E,ja),e(ja,vr),e(E,qr),e(E,ga),e(ga,_r),e(E,$r),e(E,ba),e(ba,wr),e(E,kr),i(s,Pn,l),i(s,ms,l),e(ms,Er),e(ms,xa),e(xa,yr),e(ms,Pr),i(s,Cn,l),i(s,C,l),e(C,Cr),e(C,va),e(va,zr),e(C,Dr),e(C,qa),e(qa,Tr),e(C,Or),e(C,_a),e(_a,Ar),e(C,Nr),e(C,$a),e($a,Mr),e(C,Br),e(C,wa),e(wa,Hr),e(C,Sr),e(C,ka),e(ka,Wr),e(C,Kr),e(C,Ea),e(Ea,Ur),e(C,Fr),e(C,ya),e(ya,Lr),e(C,Rr),i(s,zn,l),i(s,A,l),e(A,Ir),e(A,Pa),e(Pa,Vr),e(A,Gr),e(A,Ca),e(Ca,Jr),e(A,Qr),e(A,za),e(za,Yr),e(A,Xr),e(A,Da),e(Da,Zr),e(A,sp),e(A,Ta),e(Ta,ep),e(A,ap),e(A,Oa),e(Oa,np),e(A,tp),i(s,Dn,l),f(hs,s,l),i(s,Tn,l),i(s,ts,l),e(ts,ds),e(ds,Aa),f(Ts,Aa,null),e(ts,lp),e(ts,ue),e(ue,rp),e(ue,Na),e(Na,pp),i(s,On,l),i(s,fs,l),e(fs,op),e(fs,Ma),e(Ma,ip),e(fs,up),i(s,An,l),i(s,ce,l),e(ce,cp),i(s,Nn,l),f(Os,s,l),i(s,Mn,l),i(s,I,l),e(I,mp),e(I,Ba),e(Ba,hp),e(I,dp),e(I,Ha),e(Ha,fp),e(I,jp),e(I,Sa),e(Sa,gp),e(I,bp),i(s,Bn,l),f(As,s,l),i(s,Hn,l),i(s,me,l),e(me,xp),i(s,Sn,l),f(Ns,s,l),i(s,Wn,l),f(Ms,s,l),i(s,Kn,l),i(s,js,l),e(js,vp),e(js,Wa),e(Wa,qp),e(js,_p),i(s,Un,l),f(Bs,s,l),i(s,Fn,l),f(Hs,s,l),i(s,Ln,l),i(s,Y,l),e(Y,$p),e(Y,Ka),e(Ka,wp),e(Y,kp),e(Y,Ua),e(Ua,Ep),e(Y,yp),i(s,Rn,l),f(Ss,s,l),i(s,In,l),i(s,gs,l),e(gs,Pp),e(gs,Fa),e(Fa,Cp),e(gs,zp),i(s,Vn,l),f(Ws,s,l),i(s,Gn,l),i(s,he,l),e(he,Dp),i(s,Jn,l),f(Ks,s,l),i(s,Qn,l),i(s,de,l),e(de,Tp),i(s,Yn,l),f(Us,s,l),i(s,Xn,l),f(Fs,s,l),i(s,Zn,l),i(s,fe,l),e(fe,Op),i(s,st,l),f(Ls,s,l),i(s,et,l),f(Rs,s,l),i(s,at,l),i(s,X,l),e(X,Ap),e(X,La),e(La,Np),e(X,Mp),e(X,Ra),e(Ra,Bp),e(X,Hp),i(s,nt,l),f(Is,s,l),i(s,tt,l),i(s,bs,l),e(bs,Sp),e(bs,Ia),e(Ia,Wp),e(bs,Kp),i(s,lt,l),f(Vs,s,l),i(s,rt,l),i(s,je,l),e(je,Up),i(s,pt,l),f(Gs,s,l),i(s,ot,l),f(Js,s,l),i(s,it,l),i(s,ge,l),e(ge,Fp),i(s,ut,l),f(Qs,s,l),i(s,ct,l),i(s,be,l),e(be,Lp),i(s,mt,l),f(Ys,s,l),i(s,ht,l),f(Xs,s,l),i(s,dt,l),i(s,Z,l),e(Z,Rp),e(Z,Va),e(Va,Ip),e(Z,Vp),e(Z,Ga),e(Ga,Gp),e(Z,Jp),i(s,ft,l),f(xs,s,l),i(s,jt,l),i(s,xe,l),e(xe,Qp),i(s,gt,l),f(Zs,s,l),i(s,bt,l),i(s,ve,l),e(ve,Yp),i(s,xt,l),f(se,s,l),i(s,vt,l),f(ee,s,l),i(s,qt,l),i(s,qe,l),e(qe,Xp),i(s,_t,l),f(ae,s,l),i(s,$t,l),i(s,_e,l),e(_e,Zp),i(s,wt,l),f(ne,s,l),i(s,kt,l),f(te,s,l),i(s,Et,l),i(s,ss,l),e(ss,so),e(ss,Ja),e(Ja,eo),e(ss,ao),e(ss,Qa),e(Qa,no),e(ss,to),yt=!0},p(s,[l]){const le={};l&2&&(le.$$scope={dirty:l,ctx:s}),rs.$set(le);const Ya={};l&2&&(Ya.$$scope={dirty:l,ctx:s}),os.$set(Ya);const Xa={};l&2&&(Xa.$$scope={dirty:l,ctx:s}),us.$set(Xa);const $e={};l&2&&($e.$$scope={dirty:l,ctx:s}),hs.$set($e);const Za={};l&2&&(Za.$$scope={dirty:l,ctx:s}),xs.$set(Za)},i(s){yt||(j(v.$$.fragment,s),j(O.$$.fragment,s),j($s.$$.fragment,s),j(rs.$$.fragment,s),j(ws.$$.fragment,s),j(os.$$.fragment,s),j(ks.$$.fragment,s),j(Es.$$.fragment,s),j(ys.$$.fragment,s),j(Ps.$$.fragment,s),j(Cs.$$.fragment,s),j(zs.$$.fragment,s),j(us.$$.fragment,s),j(Ds.$$.fragment,s),j(hs.$$.fragment,s),j(Ts.$$.fragment,s),j(Os.$$.fragment,s),j(As.$$.fragment,s),j(Ns.$$.fragment,s),j(Ms.$$.fragment,s),j(Bs.$$.fragment,s),j(Hs.$$.fragment,s),j(Ss.$$.fragment,s),j(Ws.$$.fragment,s),j(Ks.$$.fragment,s),j(Us.$$.fragment,s),j(Fs.$$.fragment,s),j(Ls.$$.fragment,s),j(Rs.$$.fragment,s),j(Is.$$.fragment,s),j(Vs.$$.fragment,s),j(Gs.$$.fragment,s),j(Js.$$.fragment,s),j(Qs.$$.fragment,s),j(Ys.$$.fragment,s),j(Xs.$$.fragment,s),j(xs.$$.fragment,s),j(Zs.$$.fragment,s),j(se.$$.fragment,s),j(ee.$$.fragment,s),j(ae.$$.fragment,s),j(ne.$$.fragment,s),j(te.$$.fragment,s),yt=!0)},o(s){g(v.$$.fragment,s),g(O.$$.fragment,s),g($s.$$.fragment,s),g(rs.$$.fragment,s),g(ws.$$.fragment,s),g(os.$$.fragment,s),g(ks.$$.fragment,s),g(Es.$$.fragment,s),g(ys.$$.fragment,s),g(Ps.$$.fragment,s),g(Cs.$$.fragment,s),g(zs.$$.fragment,s),g(us.$$.fragment,s),g(Ds.$$.fragment,s),g(hs.$$.fragment,s),g(Ts.$$.fragment,s),g(Os.$$.fragment,s),g(As.$$.fragment,s),g(Ns.$$.fragment,s),g(Ms.$$.fragment,s),g(Bs.$$.fragment,s),g(Hs.$$.fragment,s),g(Ss.$$.fragment,s),g(Ws.$$.fragment,s),g(Ks.$$.fragment,s),g(Us.$$.fragment,s),g(Fs.$$.fragment,s),g(Ls.$$.fragment,s),g(Rs.$$.fragment,s),g(Is.$$.fragment,s),g(Vs.$$.fragment,s),g(Gs.$$.fragment,s),g(Js.$$.fragment,s),g(Qs.$$.fragment,s),g(Ys.$$.fragment,s),g(Xs.$$.fragment,s),g(xs.$$.fragment,s),g(Zs.$$.fragment,s),g(se.$$.fragment,s),g(ee.$$.fragment,s),g(ae.$$.fragment,s),g(ne.$$.fragment,s),g(te.$$.fragment,s),yt=!1},d(s){a(m),s&&a(k),s&&a(x),b(v),s&&a(U),b(O,s),s&&a(_s),s&&a(L),s&&a(an),b($s,s),s&&a(nn),b(rs,s),s&&a(tn),s&&a(as),b(ws),s&&a(ln),b(os,s),s&&a(rn),s&&a(W),s&&a(pn),b(ks,s),s&&a(on),s&&a(is),s&&a(un),s&&a(V),s&&a(mn),s&&a(z),s&&a(hn),s&&a(pe),s&&a(dn),b(Es,s),s&&a(fn),s&&a(oe),s&&a(jn),b(ys,s),s&&a(gn),s&&a(P),s&&a(bn),s&&a(R),s&&a(xn),b(Ps,s),s&&a(vn),s&&a(J),s&&a(qn),b(Cs,s),s&&a(_n),s&&a(Q),s&&a($n),b(zs,s),s&&a(wn),s&&a(ie),s&&a(kn),b(us,s),s&&a(En),s&&a(ns),b(Ds),s&&a(yn),s&&a(E),s&&a(Pn),s&&a(ms),s&&a(Cn),s&&a(C),s&&a(zn),s&&a(A),s&&a(Dn),b(hs,s),s&&a(Tn),s&&a(ts),b(Ts),s&&a(On),s&&a(fs),s&&a(An),s&&a(ce),s&&a(Nn),b(Os,s),s&&a(Mn),s&&a(I),s&&a(Bn),b(As,s),s&&a(Hn),s&&a(me),s&&a(Sn),b(Ns,s),s&&a(Wn),b(Ms,s),s&&a(Kn),s&&a(js),s&&a(Un),b(Bs,s),s&&a(Fn),b(Hs,s),s&&a(Ln),s&&a(Y),s&&a(Rn),b(Ss,s),s&&a(In),s&&a(gs),s&&a(Vn),b(Ws,s),s&&a(Gn),s&&a(he),s&&a(Jn),b(Ks,s),s&&a(Qn),s&&a(de),s&&a(Yn),b(Us,s),s&&a(Xn),b(Fs,s),s&&a(Zn),s&&a(fe),s&&a(st),b(Ls,s),s&&a(et),b(Rs,s),s&&a(at),s&&a(X),s&&a(nt),b(Is,s),s&&a(tt),s&&a(bs),s&&a(lt),b(Vs,s),s&&a(rt),s&&a(je),s&&a(pt),b(Gs,s),s&&a(ot),b(Js,s),s&&a(it),s&&a(ge),s&&a(ut),b(Qs,s),s&&a(ct),s&&a(be),s&&a(mt),b(Ys,s),s&&a(ht),b(Xs,s),s&&a(dt),s&&a(Z),s&&a(ft),b(xs,s),s&&a(jt),s&&a(xe),s&&a(gt),b(Zs,s),s&&a(bt),s&&a(ve),s&&a(xt),b(se,s),s&&a(vt),b(ee,s),s&&a(qt),s&&a(qe),s&&a(_t),b(ae,s),s&&a($t),s&&a(_e),s&&a(wt),b(ne,s),s&&a(kt),b(te,s),s&&a(Et),s&&a(ss)}}}const iu={local:"toknisation-iwordpiecei",sections:[{local:"algorithme-dentranement",title:"Algorithme d'entra\xEEnement"},{local:"algorithme-de-tokenisation",title:"Algorithme de tokenisation"},{local:"implmentation-de-iwordpiecei",title:"Impl\xE9mentation de <i>WordPiece</i>"}],title:"Tok\xE9nisation <i>WordPiece</i>"};function uu(F){return su(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class gu extends Gi{constructor(m){super();Ji(this,m,uu,ou,Qi,{})}}export{gu as default,iu as metadata};
