import{S as Ei,i as yi,s as Pi,e as r,k as u,w as h,t as n,U as Ci,M as zi,c as p,d as a,m as c,a as o,x as d,h as t,V as Di,b as N,F as e,g as i,y as f,q as j,o as g,B as b,v as Ti}from"../../chunks/vendor-1e8b365d.js";import{T as Ga}from"../../chunks/Tip-62b14c6e.js";import{Y as Oi}from"../../chunks/Youtube-c2a8cc39.js";import{I as Dt}from"../../chunks/IconCopyLink-483c28ba.js";import{C as _}from"../../chunks/CodeBlock-e5764662.js";import{D as Ai}from"../../chunks/DocNotebookDropdown-37d928d3.js";function Ni(F){let m,k,x,$,w;return{c(){m=r("p"),k=n("\u{1F4A1} Cette section couvre le "),x=r("em"),$=n("WordPiece"),w=n(" en profondeur, allant jusqu\u2019\xE0 montrer une impl\xE9mentation compl\xE8te. Vous pouvez passer directement \xE0 la fin si vous souhaitez simplement avoir un aper\xE7u g\xE9n\xE9ral de l\u2019algorithme de tok\xE9nisation.")},l(v){m=p(v,"P",{});var q=o(m);k=t(q,"\u{1F4A1} Cette section couvre le "),x=p(q,"EM",{});var y=o(x);$=t(y,"WordPiece"),y.forEach(a),w=t(q," en profondeur, allant jusqu\u2019\xE0 montrer une impl\xE9mentation compl\xE8te. Vous pouvez passer directement \xE0 la fin si vous souhaitez simplement avoir un aper\xE7u g\xE9n\xE9ral de l\u2019algorithme de tok\xE9nisation."),q.forEach(a)},m(v,q){i(v,m,q),e(m,k),e(m,x),e(x,$),e(m,w)},d(v){v&&a(m)}}}function Mi(F){let m,k,x,$,w;return{c(){m=r("p"),k=n("\u26A0\uFE0F Google n\u2019a jamais mis en ligne son impl\xE9mentation de l\u2019algorithme d\u2019entra\xEEnement de "),x=r("em"),$=n("WordPiece"),w=n(". Ce qui suit est donc notre meilleure estimation bas\xE9e sur la litt\xE9rature publi\xE9e. Il se peut qu\u2019elle ne soit pas exacte \xE0 100 %.")},l(v){m=p(v,"P",{});var q=o(m);k=t(q,"\u26A0\uFE0F Google n\u2019a jamais mis en ligne son impl\xE9mentation de l\u2019algorithme d\u2019entra\xEEnement de "),x=p(q,"EM",{});var y=o(x);$=t(y,"WordPiece"),y.forEach(a),w=t(q,". Ce qui suit est donc notre meilleure estimation bas\xE9e sur la litt\xE9rature publi\xE9e. Il se peut qu\u2019elle ne soit pas exacte \xE0 100 %."),q.forEach(a)},m(v,q){i(v,m,q),e(m,k),e(m,x),e(x,$),e(m,w)},d(v){v&&a(m)}}}function Bi(F){let m,k,x,$,w;return{c(){m=r("p"),k=n("\u270F\uFE0F "),x=r("strong"),$=n("A votre tour !"),w=n(" Quelle sera la prochaine r\xE8gle de fusion ?")},l(v){m=p(v,"P",{});var q=o(m);k=t(q,"\u270F\uFE0F "),x=p(q,"STRONG",{});var y=o(x);$=t(y,"A votre tour !"),y.forEach(a),w=t(q," Quelle sera la prochaine r\xE8gle de fusion ?"),q.forEach(a)},m(v,q){i(v,m,q),e(m,k),e(m,x),e(x,$),e(m,w)},d(v){v&&a(m)}}}function Hi(F){let m,k,x,$,w,v,q,y;return{c(){m=r("p"),k=n("\u270F\uFE0F "),x=r("strong"),$=n("A votre tour !"),w=n(" Comment le mot "),v=r("code"),q=n('"pugs"'),y=n(" sera-t-il tokenis\xE9 ?")},l(H){m=p(H,"P",{});var M=o(m);k=t(M,"\u270F\uFE0F "),x=p(M,"STRONG",{});var W=o(x);$=t(W,"A votre tour !"),W.forEach(a),w=t(M," Comment le mot "),v=p(M,"CODE",{});var U=o(v);q=t(U,'"pugs"'),U.forEach(a),y=t(M," sera-t-il tokenis\xE9 ?"),M.forEach(a)},m(H,M){i(H,m,M),e(m,k),e(m,x),e(x,$),e(m,w),e(m,v),e(v,q),e(m,y)},d(H){H&&a(m)}}}function Wi(F){let m,k,x,$,w,v,q,y,H,M,W;return{c(){m=r("p"),k=n("\u{1F4A1} Utiliser "),x=r("code"),$=n("train_new_from_iterator()"),w=n(" sur le m\xEAme corpus ne donnera pas exactement le m\xEAme vocabulaire. C\u2019est parce que la biblioth\xE8que \u{1F917} "),v=r("em"),q=n("Tokenizers"),y=n(" n\u2019impl\xE9mente pas "),H=r("em"),M=n("WordPiece"),W=n(" pour l\u2019entra\xEEnement (puisque nous ne sommes pas compl\xE8tement s\xFBrs de ses internes), mais utilise le BPE \xE0 la place.")},l(U){m=p(U,"P",{});var D=o(m);k=t(D,"\u{1F4A1} Utiliser "),x=p(D,"CODE",{});var ns=o(x);$=t(ns,"train_new_from_iterator()"),ns.forEach(a),w=t(D," sur le m\xEAme corpus ne donnera pas exactement le m\xEAme vocabulaire. C\u2019est parce que la biblioth\xE8que \u{1F917} "),v=p(D,"EM",{});var le=o(v);q=t(le,"Tokenizers"),le.forEach(a),y=t(D," n\u2019impl\xE9mente pas "),H=p(D,"EM",{});var re=o(H);M=t(re,"WordPiece"),re.forEach(a),W=t(D," pour l\u2019entra\xEEnement (puisque nous ne sommes pas compl\xE8tement s\xFBrs de ses internes), mais utilise le BPE \xE0 la place."),D.forEach(a)},m(U,D){i(U,m,D),e(m,k),e(m,x),e(x,$),e(m,w),e(m,v),e(v,q),e(m,y),e(m,H),e(H,M),e(m,W)},d(U){U&&a(m)}}}function Si(F){let m,k,x,$,w,v,q,y,H,M,W,U,D,ns,le,re,Ja,qs,Qa,ts,Ya,ss,ls,Pe,_s,Tt,Ce,Ot,Xa,rs,Za,S,At,ze,Nt,Mt,De,Bt,Ht,Te,Wt,St,Oe,Kt,Ut,sn,$s,en,ps,Ft,Ae,Lt,Rt,an,I,Vt,Ne,It,Gt,Me,Jt,Qt,nn,ki='<span class="katex-display"><span class="katex"><span class="katex-mathml"><math xmlns="http://www.w3.org/1998/Math/MathML" display="block"><semantics><mrow><mrow><mi mathvariant="normal">s</mi><mi mathvariant="normal">c</mi><mi mathvariant="normal">o</mi><mi mathvariant="normal">r</mi><mi mathvariant="normal">e</mi></mrow><mo>=</mo><mo stretchy="false">(</mo><mrow><mi mathvariant="normal">f</mi><mi mathvariant="normal">r</mi><mi mathvariant="normal">e</mi><mi mathvariant="normal">q</mi><mi mathvariant="normal">_</mi><mi mathvariant="normal">o</mi><mi mathvariant="normal">f</mi><mi mathvariant="normal">_</mi><mi mathvariant="normal">p</mi><mi mathvariant="normal">a</mi><mi mathvariant="normal">i</mi><mi mathvariant="normal">r</mi></mrow><mo stretchy="false">)</mo><mi mathvariant="normal">/</mi><mo stretchy="false">(</mo><mrow><mi mathvariant="normal">f</mi><mi mathvariant="normal">r</mi><mi mathvariant="normal">e</mi><mi mathvariant="normal">q</mi><mi mathvariant="normal">_</mi><mi mathvariant="normal">o</mi><mi mathvariant="normal">f</mi><mi mathvariant="normal">_</mi><mi mathvariant="normal">f</mi><mi mathvariant="normal">i</mi><mi mathvariant="normal">r</mi><mi mathvariant="normal">s</mi><mi mathvariant="normal">t</mi><mi mathvariant="normal">_</mi><mi mathvariant="normal">e</mi><mi mathvariant="normal">l</mi><mi mathvariant="normal">e</mi><mi mathvariant="normal">m</mi><mi mathvariant="normal">e</mi><mi mathvariant="normal">n</mi><mi mathvariant="normal">t</mi></mrow><mo>\xD7</mo><mrow><mi mathvariant="normal">f</mi><mi mathvariant="normal">r</mi><mi mathvariant="normal">e</mi><mi mathvariant="normal">q</mi><mi mathvariant="normal">_</mi><mi mathvariant="normal">o</mi><mi mathvariant="normal">f</mi><mi mathvariant="normal">_</mi><mi mathvariant="normal">s</mi><mi mathvariant="normal">e</mi><mi mathvariant="normal">c</mi><mi mathvariant="normal">o</mi><mi mathvariant="normal">n</mi><mi mathvariant="normal">d</mi><mi mathvariant="normal">_</mi><mi mathvariant="normal">e</mi><mi mathvariant="normal">l</mi><mi mathvariant="normal">e</mi><mi mathvariant="normal">m</mi><mi mathvariant="normal">e</mi><mi mathvariant="normal">n</mi><mi mathvariant="normal">t</mi></mrow><mo stretchy="false">)</mo></mrow><annotation encoding="application/x-tex">\\mathrm{score} = (\\mathrm{freq\\_of\\_pair}) / (\\mathrm{freq\\_of\\_first\\_element} \\times \\mathrm{freq\\_of\\_second\\_element})</annotation></semantics></math></span><span class="katex-html" aria-hidden="true"><span class="base"><span class="strut" style="height:0.4306em;"></span><span class="mord"><span class="mord mathrm">score</span></span><span class="mspace" style="margin-right:0.2778em;"></span><span class="mrel">=</span><span class="mspace" style="margin-right:0.2778em;"></span></span><span class="base"><span class="strut" style="height:1.06em;vertical-align:-0.31em;"></span><span class="mopen">(</span><span class="mord"><span class="mord mathrm">freq_of_pair</span></span><span class="mclose">)</span><span class="mord">/</span><span class="mopen">(</span><span class="mord"><span class="mord mathrm">freq_of_first_element</span></span><span class="mspace" style="margin-right:0.2222em;"></span><span class="mbin">\xD7</span><span class="mspace" style="margin-right:0.2222em;"></span></span><span class="base"><span class="strut" style="height:1.06em;vertical-align:-0.31em;"></span><span class="mord"><span class="mord mathrm">freq_of_second_element</span></span><span class="mclose">)</span></span></span></span></span>',tn,T,Yt,Be,Xt,Zt,He,sl,el,We,al,nl,Se,tl,ll,Ke,rl,pl,Ue,ol,il,ln,pe,ul,rn,ws,pn,oe,cl,on,ks,un,P,ml,Fe,hl,dl,Le,fl,jl,Re,gl,bl,Ve,xl,vl,Ie,ql,_l,Ge,$l,wl,Je,kl,El,Qe,yl,Pl,cn,L,Cl,Ye,zl,Dl,Xe,Tl,Ol,Ze,Al,Nl,mn,Es,hn,G,Ml,sa,Bl,Hl,ea,Wl,Sl,dn,ys,fn,J,Kl,aa,Ul,Fl,na,Ll,Rl,jn,Ps,gn,ie,Vl,bn,os,xn,es,is,ta,Cs,Il,la,Gl,vn,E,Jl,ra,Ql,Yl,pa,Xl,Zl,oa,sr,er,ia,ar,nr,ua,tr,lr,ca,rr,pr,ma,or,ir,ha,ur,cr,da,mr,hr,qn,us,dr,fa,fr,jr,_n,C,gr,ja,br,xr,ga,vr,qr,ba,_r,$r,xa,wr,kr,va,Er,yr,qa,Pr,Cr,_a,zr,Dr,$a,Tr,Or,$n,B,Ar,wa,Nr,Mr,ka,Br,Hr,Ea,Wr,Sr,ya,Kr,Ur,Pa,Fr,Lr,wn,cs,kn,as,ms,Ca,zs,Rr,za,Vr,En,hs,Ir,Da,Gr,Jr,yn,ue,Qr,Pn,Ds,Cn,R,Yr,Ta,Xr,Zr,Oa,sp,ep,Aa,ap,np,zn,Ts,Dn,ce,tp,Tn,Os,On,As,An,ds,lp,Na,rp,pp,Nn,Ns,Mn,Ms,Bn,fs,op,Ma,ip,up,Hn,Bs,Wn,js,cp,Ba,mp,hp,Sn,Hs,Kn,me,dp,Un,Ws,Fn,he,fp,Ln,Ss,Rn,Ks,Vn,de,jp,In,Us,Gn,Fs,Jn,Q,gp,Ha,bp,xp,Wa,vp,qp,Qn,Ls,Yn,gs,_p,Sa,$p,wp,Xn,Rs,Zn,fe,kp,st,Vs,et,Is,at,je,Ep,nt,Gs,tt,ge,yp,lt,Js,rt,Qs,pt,Y,Pp,Ka,Cp,zp,Ua,Dp,Tp,ot,bs,it,be,Op,ut,Ys,ct,xe,Ap,mt,Xs,ht,Zs,dt,ve,Np,ft,se,jt,qe,Mp,gt,ee,bt,ae,xt,X,Bp,Fa,Hp,Wp,La,Sp,Kp,vt;return v=new Dt({}),W=new Ai({props:{classNames:"absolute z-10 right-0 top-0",options:[{label:"Google Colab",value:"https://colab.research.google.com/github/huggingface/notebooks/blob/master/course/chapter6/section6.ipynb"},{label:"Aws Studio",value:"https://studiolab.sagemaker.aws/import/github/huggingface/notebooks/blob/master/course/chapter6/section6.ipynb"}]}}),qs=new Oi({props:{id:"qpv6ms_t_1A"}}),ts=new Ga({props:{$$slots:{default:[Ni]},$$scope:{ctx:F}}}),_s=new Dt({}),rs=new Ga({props:{warning:!0,$$slots:{default:[Mi]},$$scope:{ctx:F}}}),$s=new _({props:{code:"w ##o ##r ##d",highlighted:"w ##o ##r ##d"}}),ws=new _({props:{code:'("hug", 10), ("pug", 5), ("pun", 12), ("bun", 4), ("hugs", 5)',highlighted:'(<span class="hljs-string">&quot;hug&quot;</span><span class="hljs-punctuation">,</span> <span class="hljs-number">10</span>)<span class="hljs-punctuation">,</span> (<span class="hljs-string">&quot;pug&quot;</span><span class="hljs-punctuation">,</span> <span class="hljs-number">5</span>)<span class="hljs-punctuation">,</span> (<span class="hljs-string">&quot;pun&quot;</span><span class="hljs-punctuation">,</span> <span class="hljs-number">12</span>)<span class="hljs-punctuation">,</span> (<span class="hljs-string">&quot;bun&quot;</span><span class="hljs-punctuation">,</span> <span class="hljs-number">4</span>)<span class="hljs-punctuation">,</span> (<span class="hljs-string">&quot;hugs&quot;</span><span class="hljs-punctuation">,</span> <span class="hljs-number">5</span>)'}}),ks=new _({props:{code:'("h" "##u" "##g", 10), ("p" "##u" "##g", 5), ("p" "##u" "##n", 12), ("b" "##u" "##n", 4), ("h" "##u" "##g" "##s", 5)',highlighted:'(<span class="hljs-string">&quot;h&quot;</span> <span class="hljs-string">&quot;#<span class="hljs-subst">#u</span>&quot;</span> <span class="hljs-string">&quot;#<span class="hljs-subst">#g</span>&quot;</span>, <span class="hljs-number">10</span>), (<span class="hljs-string">&quot;p&quot;</span> <span class="hljs-string">&quot;#<span class="hljs-subst">#u</span>&quot;</span> <span class="hljs-string">&quot;#<span class="hljs-subst">#g</span>&quot;</span>, <span class="hljs-number">5</span>), (<span class="hljs-string">&quot;p&quot;</span> <span class="hljs-string">&quot;#<span class="hljs-subst">#u</span>&quot;</span> <span class="hljs-string">&quot;#<span class="hljs-subst">#n</span>&quot;</span>, <span class="hljs-number">12</span>), (<span class="hljs-string">&quot;b&quot;</span> <span class="hljs-string">&quot;#<span class="hljs-subst">#u</span>&quot;</span> <span class="hljs-string">&quot;#<span class="hljs-subst">#n</span>&quot;</span>, <span class="hljs-number">4</span>), (<span class="hljs-string">&quot;h&quot;</span> <span class="hljs-string">&quot;#<span class="hljs-subst">#u</span>&quot;</span> <span class="hljs-string">&quot;#<span class="hljs-subst">#g</span>&quot;</span> <span class="hljs-string">&quot;#<span class="hljs-subst">#s</span>&quot;</span>, <span class="hljs-number">5</span>)'}}),Es=new _({props:{code:`Vocabulary: ["b", "h", "p", "##g", "##n", "##s", "##u", "##gs"]
Corpus: ("h" "##u" "##g", 10), ("p" "##u" "##g", 5), ("p" "##u" "##n", 12), ("b" "##u" "##n", 4), ("h" "##u" "##gs", 5)`,highlighted:`Vocabulary: [<span class="hljs-string">&quot;b&quot;</span>, <span class="hljs-string">&quot;h&quot;</span>, <span class="hljs-string">&quot;p&quot;</span>, <span class="hljs-string">&quot;#<span class="hljs-subst">#g</span>&quot;</span>, <span class="hljs-string">&quot;#<span class="hljs-subst">#n</span>&quot;</span>, <span class="hljs-string">&quot;#<span class="hljs-subst">#s</span>&quot;</span>, <span class="hljs-string">&quot;#<span class="hljs-subst">#u</span>&quot;</span>, <span class="hljs-string">&quot;#<span class="hljs-subst">#gs</span>&quot;</span>]
Corpus: (<span class="hljs-string">&quot;h&quot;</span> <span class="hljs-string">&quot;#<span class="hljs-subst">#u</span>&quot;</span> <span class="hljs-string">&quot;#<span class="hljs-subst">#g</span>&quot;</span>, <span class="hljs-number">10</span>), (<span class="hljs-string">&quot;p&quot;</span> <span class="hljs-string">&quot;#<span class="hljs-subst">#u</span>&quot;</span> <span class="hljs-string">&quot;#<span class="hljs-subst">#g</span>&quot;</span>, <span class="hljs-number">5</span>), (<span class="hljs-string">&quot;p&quot;</span> <span class="hljs-string">&quot;#<span class="hljs-subst">#u</span>&quot;</span> <span class="hljs-string">&quot;#<span class="hljs-subst">#n</span>&quot;</span>, <span class="hljs-number">12</span>), (<span class="hljs-string">&quot;b&quot;</span> <span class="hljs-string">&quot;#<span class="hljs-subst">#u</span>&quot;</span> <span class="hljs-string">&quot;#<span class="hljs-subst">#n</span>&quot;</span>, <span class="hljs-number">4</span>), (<span class="hljs-string">&quot;h&quot;</span> <span class="hljs-string">&quot;#<span class="hljs-subst">#u</span>&quot;</span> <span class="hljs-string">&quot;#<span class="hljs-subst">#gs</span>&quot;</span>, <span class="hljs-number">5</span>)`}}),ys=new _({props:{code:`Vocabulary: ["b", "h", "p", "##g", "##n", "##s", "##u", "##gs", "hu"]
Corpus: ("hu" "##g", 10), ("p" "##u" "##g", 5), ("p" "##u" "##n", 12), ("b" "##u" "##n", 4), ("hu" "##gs", 5)`,highlighted:`Vocabulary: [<span class="hljs-string">&quot;b&quot;</span>, <span class="hljs-string">&quot;h&quot;</span>, <span class="hljs-string">&quot;p&quot;</span>, <span class="hljs-string">&quot;#<span class="hljs-subst">#g</span>&quot;</span>, <span class="hljs-string">&quot;#<span class="hljs-subst">#n</span>&quot;</span>, <span class="hljs-string">&quot;#<span class="hljs-subst">#s</span>&quot;</span>, <span class="hljs-string">&quot;#<span class="hljs-subst">#u</span>&quot;</span>, <span class="hljs-string">&quot;#<span class="hljs-subst">#gs</span>&quot;</span>, <span class="hljs-string">&quot;hu&quot;</span>]
Corpus: (<span class="hljs-string">&quot;hu&quot;</span> <span class="hljs-string">&quot;#<span class="hljs-subst">#g</span>&quot;</span>, <span class="hljs-number">10</span>), (<span class="hljs-string">&quot;p&quot;</span> <span class="hljs-string">&quot;#<span class="hljs-subst">#u</span>&quot;</span> <span class="hljs-string">&quot;#<span class="hljs-subst">#g</span>&quot;</span>, <span class="hljs-number">5</span>), (<span class="hljs-string">&quot;p&quot;</span> <span class="hljs-string">&quot;#<span class="hljs-subst">#u</span>&quot;</span> <span class="hljs-string">&quot;#<span class="hljs-subst">#n</span>&quot;</span>, <span class="hljs-number">12</span>), (<span class="hljs-string">&quot;b&quot;</span> <span class="hljs-string">&quot;#<span class="hljs-subst">#u</span>&quot;</span> <span class="hljs-string">&quot;#<span class="hljs-subst">#n</span>&quot;</span>, <span class="hljs-number">4</span>), (<span class="hljs-string">&quot;hu&quot;</span> <span class="hljs-string">&quot;#<span class="hljs-subst">#gs</span>&quot;</span>, <span class="hljs-number">5</span>)`}}),Ps=new _({props:{code:`Vocabulary: ["b", "h", "p", "##g", "##n", "##s", "##u", "##gs", "hu", "hug"]
Corpus: ("hug", 10), ("p" "##u" "##g", 5), ("p" "##u" "##n", 12), ("b" "##u" "##n", 4), ("hu" "##gs", 5)`,highlighted:`Vocabulary: [<span class="hljs-string">&quot;b&quot;</span>, <span class="hljs-string">&quot;h&quot;</span>, <span class="hljs-string">&quot;p&quot;</span>, <span class="hljs-string">&quot;#<span class="hljs-subst">#g</span>&quot;</span>, <span class="hljs-string">&quot;#<span class="hljs-subst">#n</span>&quot;</span>, <span class="hljs-string">&quot;#<span class="hljs-subst">#s</span>&quot;</span>, <span class="hljs-string">&quot;#<span class="hljs-subst">#u</span>&quot;</span>, <span class="hljs-string">&quot;#<span class="hljs-subst">#gs</span>&quot;</span>, <span class="hljs-string">&quot;hu&quot;</span>, <span class="hljs-string">&quot;hug&quot;</span>]
Corpus: (<span class="hljs-string">&quot;hug&quot;</span>, <span class="hljs-number">10</span>), (<span class="hljs-string">&quot;p&quot;</span> <span class="hljs-string">&quot;#<span class="hljs-subst">#u</span>&quot;</span> <span class="hljs-string">&quot;#<span class="hljs-subst">#g</span>&quot;</span>, <span class="hljs-number">5</span>), (<span class="hljs-string">&quot;p&quot;</span> <span class="hljs-string">&quot;#<span class="hljs-subst">#u</span>&quot;</span> <span class="hljs-string">&quot;#<span class="hljs-subst">#n</span>&quot;</span>, <span class="hljs-number">12</span>), (<span class="hljs-string">&quot;b&quot;</span> <span class="hljs-string">&quot;#<span class="hljs-subst">#u</span>&quot;</span> <span class="hljs-string">&quot;#<span class="hljs-subst">#n</span>&quot;</span>, <span class="hljs-number">4</span>), (<span class="hljs-string">&quot;hu&quot;</span> <span class="hljs-string">&quot;#<span class="hljs-subst">#gs</span>&quot;</span>, <span class="hljs-number">5</span>)`}}),os=new Ga({props:{$$slots:{default:[Bi]},$$scope:{ctx:F}}}),Cs=new Dt({}),cs=new Ga({props:{$$slots:{default:[Hi]},$$scope:{ctx:F}}}),zs=new Dt({}),Ds=new _({props:{code:`corpus = [
    "This is the Hugging Face course.",  # C'est le cours d'Hugging Face.
    "This chapter is about tokenization.",  # This chapter is about tokenization
    "This section shows several tokenizer algorithms.",  # Cette section pr\xE9sente plusieurs algorithmes de *tokenizer*.
    "Hopefully, you will be able to understand how they are trained and generate tokens.",  # Avec un peu de chance, vous serez en mesure de comprendre comment ils sont entra\xEEn\xE9s et g\xE9n\xE8rent des *tokens*.
]`,highlighted:`corpus = [
    <span class="hljs-string">&quot;This is the Hugging Face course.&quot;</span>,  <span class="hljs-comment"># C&#x27;est le cours d&#x27;Hugging Face.</span>
    <span class="hljs-string">&quot;This chapter is about tokenization.&quot;</span>,  <span class="hljs-comment"># This chapter is about tokenization</span>
    <span class="hljs-string">&quot;This section shows several tokenizer algorithms.&quot;</span>,  <span class="hljs-comment"># Cette section pr\xE9sente plusieurs algorithmes de *tokenizer*.</span>
    <span class="hljs-string">&quot;Hopefully, you will be able to understand how they are trained and generate tokens.&quot;</span>,  <span class="hljs-comment"># Avec un peu de chance, vous serez en mesure de comprendre comment ils sont entra\xEEn\xE9s et g\xE9n\xE8rent des *tokens*.</span>
]`}}),Ts=new _({props:{code:`from transformers import AutoTokenizer

tokenizer = AutoTokenizer.from_pretrained("bert-base-cased")`,highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoTokenizer

tokenizer = AutoTokenizer.from_pretrained(<span class="hljs-string">&quot;bert-base-cased&quot;</span>)`}}),Os=new _({props:{code:`from collections import defaultdict

word_freqs = defaultdict(int)
for text in corpus:
    words_with_offsets = tokenizer.backend_tokenizer.pre_tokenizer.pre_tokenize_str(text)
    new_words = [word for word, offset in words_with_offsets]
    for word in new_words:
        word_freqs[word] += 1

word_freqs`,highlighted:`<span class="hljs-keyword">from</span> collections <span class="hljs-keyword">import</span> defaultdict

word_freqs = defaultdict(<span class="hljs-built_in">int</span>)
<span class="hljs-keyword">for</span> text <span class="hljs-keyword">in</span> corpus:
    words_with_offsets = tokenizer.backend_tokenizer.pre_tokenizer.pre_tokenize_str(text)
    new_words = [word <span class="hljs-keyword">for</span> word, offset <span class="hljs-keyword">in</span> words_with_offsets]
    <span class="hljs-keyword">for</span> word <span class="hljs-keyword">in</span> new_words:
        word_freqs[word] += <span class="hljs-number">1</span>

word_freqs`}}),As=new _({props:{code:`defaultdict(
    int, {'This': 3, 'is': 2, 'the': 1, 'Hugging': 1, 'Face': 1, 'Course': 1, '.': 4, 'chapter': 1, 'about': 1,
    'tokenization': 1, 'section': 1, 'shows': 1, 'several': 1, 'tokenizer': 1, 'algorithms': 1, 'Hopefully': 1,
    ',': 1, 'you': 1, 'will': 1, 'be': 1, 'able': 1, 'to': 1, 'understand': 1, 'how': 1, 'they': 1, 'are': 1,
    'trained': 1, 'and': 1, 'generate': 1, 'tokens': 1})`,highlighted:`defaultdict(
    <span class="hljs-built_in">int</span>, {<span class="hljs-string">&#x27;This&#x27;</span>: <span class="hljs-number">3</span>, <span class="hljs-string">&#x27;is&#x27;</span>: <span class="hljs-number">2</span>, <span class="hljs-string">&#x27;the&#x27;</span>: <span class="hljs-number">1</span>, <span class="hljs-string">&#x27;Hugging&#x27;</span>: <span class="hljs-number">1</span>, <span class="hljs-string">&#x27;Face&#x27;</span>: <span class="hljs-number">1</span>, <span class="hljs-string">&#x27;Course&#x27;</span>: <span class="hljs-number">1</span>, <span class="hljs-string">&#x27;.&#x27;</span>: <span class="hljs-number">4</span>, <span class="hljs-string">&#x27;chapter&#x27;</span>: <span class="hljs-number">1</span>, <span class="hljs-string">&#x27;about&#x27;</span>: <span class="hljs-number">1</span>,
    <span class="hljs-string">&#x27;tokenization&#x27;</span>: <span class="hljs-number">1</span>, <span class="hljs-string">&#x27;section&#x27;</span>: <span class="hljs-number">1</span>, <span class="hljs-string">&#x27;shows&#x27;</span>: <span class="hljs-number">1</span>, <span class="hljs-string">&#x27;several&#x27;</span>: <span class="hljs-number">1</span>, <span class="hljs-string">&#x27;tokenizer&#x27;</span>: <span class="hljs-number">1</span>, <span class="hljs-string">&#x27;algorithms&#x27;</span>: <span class="hljs-number">1</span>, <span class="hljs-string">&#x27;Hopefully&#x27;</span>: <span class="hljs-number">1</span>,
    <span class="hljs-string">&#x27;,&#x27;</span>: <span class="hljs-number">1</span>, <span class="hljs-string">&#x27;you&#x27;</span>: <span class="hljs-number">1</span>, <span class="hljs-string">&#x27;will&#x27;</span>: <span class="hljs-number">1</span>, <span class="hljs-string">&#x27;be&#x27;</span>: <span class="hljs-number">1</span>, <span class="hljs-string">&#x27;able&#x27;</span>: <span class="hljs-number">1</span>, <span class="hljs-string">&#x27;to&#x27;</span>: <span class="hljs-number">1</span>, <span class="hljs-string">&#x27;understand&#x27;</span>: <span class="hljs-number">1</span>, <span class="hljs-string">&#x27;how&#x27;</span>: <span class="hljs-number">1</span>, <span class="hljs-string">&#x27;they&#x27;</span>: <span class="hljs-number">1</span>, <span class="hljs-string">&#x27;are&#x27;</span>: <span class="hljs-number">1</span>,
    <span class="hljs-string">&#x27;trained&#x27;</span>: <span class="hljs-number">1</span>, <span class="hljs-string">&#x27;and&#x27;</span>: <span class="hljs-number">1</span>, <span class="hljs-string">&#x27;generate&#x27;</span>: <span class="hljs-number">1</span>, <span class="hljs-string">&#x27;tokens&#x27;</span>: <span class="hljs-number">1</span>})`}}),Ns=new _({props:{code:`alphabet = []
for word in word_freqs.keys():
    if word[0] not in alphabet:
        alphabet.append(word[0])
    for letter in word[1:]:
        if f"##{letter}" not in alphabet:
            alphabet.append(f"##{letter}")

alphabet.sort()
alphabet

print(alphabet)`,highlighted:`alphabet = []
<span class="hljs-keyword">for</span> word <span class="hljs-keyword">in</span> word_freqs.keys():
    <span class="hljs-keyword">if</span> word[<span class="hljs-number">0</span>] <span class="hljs-keyword">not</span> <span class="hljs-keyword">in</span> alphabet:
        alphabet.append(word[<span class="hljs-number">0</span>])
    <span class="hljs-keyword">for</span> letter <span class="hljs-keyword">in</span> word[<span class="hljs-number">1</span>:]:
        <span class="hljs-keyword">if</span> <span class="hljs-string">f&quot;##<span class="hljs-subst">{letter}</span>&quot;</span> <span class="hljs-keyword">not</span> <span class="hljs-keyword">in</span> alphabet:
            alphabet.append(<span class="hljs-string">f&quot;##<span class="hljs-subst">{letter}</span>&quot;</span>)

alphabet.sort()
alphabet

<span class="hljs-built_in">print</span>(alphabet)`}}),Ms=new _({props:{code:`['##a', '##b', '##c', '##d', '##e', '##f', '##g', '##h', '##i', '##k', '##l', '##m', '##n', '##o', '##p', '##r', '##s',
 '##t', '##u', '##v', '##w', '##y', '##z', ',', '.', 'C', 'F', 'H', 'T', 'a', 'b', 'c', 'g', 'h', 'i', 's', 't', 'u',
 'w', 'y']`,highlighted:`[<span class="hljs-string">&#x27;##a&#x27;</span>, <span class="hljs-string">&#x27;##b&#x27;</span>, <span class="hljs-string">&#x27;##c&#x27;</span>, <span class="hljs-string">&#x27;##d&#x27;</span>, <span class="hljs-string">&#x27;##e&#x27;</span>, <span class="hljs-string">&#x27;##f&#x27;</span>, <span class="hljs-string">&#x27;##g&#x27;</span>, <span class="hljs-string">&#x27;##h&#x27;</span>, <span class="hljs-string">&#x27;##i&#x27;</span>, <span class="hljs-string">&#x27;##k&#x27;</span>, <span class="hljs-string">&#x27;##l&#x27;</span>, <span class="hljs-string">&#x27;##m&#x27;</span>, <span class="hljs-string">&#x27;##n&#x27;</span>, <span class="hljs-string">&#x27;##o&#x27;</span>, <span class="hljs-string">&#x27;##p&#x27;</span>, <span class="hljs-string">&#x27;##r&#x27;</span>, <span class="hljs-string">&#x27;##s&#x27;</span>,
 <span class="hljs-string">&#x27;##t&#x27;</span>, <span class="hljs-string">&#x27;##u&#x27;</span>, <span class="hljs-string">&#x27;##v&#x27;</span>, <span class="hljs-string">&#x27;##w&#x27;</span>, <span class="hljs-string">&#x27;##y&#x27;</span>, <span class="hljs-string">&#x27;##z&#x27;</span>, <span class="hljs-string">&#x27;,&#x27;</span>, <span class="hljs-string">&#x27;.&#x27;</span>, <span class="hljs-string">&#x27;C&#x27;</span>, <span class="hljs-string">&#x27;F&#x27;</span>, <span class="hljs-string">&#x27;H&#x27;</span>, <span class="hljs-string">&#x27;T&#x27;</span>, <span class="hljs-string">&#x27;a&#x27;</span>, <span class="hljs-string">&#x27;b&#x27;</span>, <span class="hljs-string">&#x27;c&#x27;</span>, <span class="hljs-string">&#x27;g&#x27;</span>, <span class="hljs-string">&#x27;h&#x27;</span>, <span class="hljs-string">&#x27;i&#x27;</span>, <span class="hljs-string">&#x27;s&#x27;</span>, <span class="hljs-string">&#x27;t&#x27;</span>, <span class="hljs-string">&#x27;u&#x27;</span>,
 <span class="hljs-string">&#x27;w&#x27;</span>, <span class="hljs-string">&#x27;y&#x27;</span>]`}}),Bs=new _({props:{code:'vocab = ["[PAD]", "[UNK]", "[CLS]", "[SEP]", "[MASK]"] + alphabet.copy()',highlighted:'vocab = [<span class="hljs-string">&quot;[PAD]&quot;</span>, <span class="hljs-string">&quot;[UNK]&quot;</span>, <span class="hljs-string">&quot;[CLS]&quot;</span>, <span class="hljs-string">&quot;[SEP]&quot;</span>, <span class="hljs-string">&quot;[MASK]&quot;</span>] + alphabet.copy()'}}),Hs=new _({props:{code:`splits = {
    word: [c if i == 0 else f"##{c}" for i, c in enumerate(word)]
    for word in word_freqs.keys()
}`,highlighted:`splits = {
    word: [c <span class="hljs-keyword">if</span> i == <span class="hljs-number">0</span> <span class="hljs-keyword">else</span> <span class="hljs-string">f&quot;##<span class="hljs-subst">{c}</span>&quot;</span> <span class="hljs-keyword">for</span> i, c <span class="hljs-keyword">in</span> <span class="hljs-built_in">enumerate</span>(word)]
    <span class="hljs-keyword">for</span> word <span class="hljs-keyword">in</span> word_freqs.keys()
}`}}),Ws=new _({props:{code:`def compute_pair_scores(splits):
    letter_freqs = defaultdict(int)
    pair_freqs = defaultdict(int)
    for word, freq in word_freqs.items():
        split = splits[word]
        if len(split) == 1:
            letter_freqs[split[0]] += freq
            continue
        for i in range(len(split) - 1):
            pair = (split[i], split[i + 1])
            letter_freqs[split[i]] += freq
            pair_freqs[pair] += freq
        letter_freqs[split[-1]] += freq

    scores = {
        pair: freq / (letter_freqs[pair[0]] * letter_freqs[pair[1]])
        for pair, freq in pair_freqs.items()
    }
    return scores`,highlighted:`<span class="hljs-keyword">def</span> <span class="hljs-title function_">compute_pair_scores</span>(<span class="hljs-params">splits</span>):
    letter_freqs = defaultdict(<span class="hljs-built_in">int</span>)
    pair_freqs = defaultdict(<span class="hljs-built_in">int</span>)
    <span class="hljs-keyword">for</span> word, freq <span class="hljs-keyword">in</span> word_freqs.items():
        split = splits[word]
        <span class="hljs-keyword">if</span> <span class="hljs-built_in">len</span>(split) == <span class="hljs-number">1</span>:
            letter_freqs[split[<span class="hljs-number">0</span>]] += freq
            <span class="hljs-keyword">continue</span>
        <span class="hljs-keyword">for</span> i <span class="hljs-keyword">in</span> <span class="hljs-built_in">range</span>(<span class="hljs-built_in">len</span>(split) - <span class="hljs-number">1</span>):
            pair = (split[i], split[i + <span class="hljs-number">1</span>])
            letter_freqs[split[i]] += freq
            pair_freqs[pair] += freq
        letter_freqs[split[-<span class="hljs-number">1</span>]] += freq

    scores = {
        pair: freq / (letter_freqs[pair[<span class="hljs-number">0</span>]] * letter_freqs[pair[<span class="hljs-number">1</span>]])
        <span class="hljs-keyword">for</span> pair, freq <span class="hljs-keyword">in</span> pair_freqs.items()
    }
    <span class="hljs-keyword">return</span> scores`}}),Ss=new _({props:{code:`pair_scores = compute_pair_scores(splits)
for i, key in enumerate(pair_scores.keys()):
    print(f"{key}: {pair_scores[key]}")
    if i >= 5:
        break`,highlighted:`pair_scores = compute_pair_scores(splits)
<span class="hljs-keyword">for</span> i, key <span class="hljs-keyword">in</span> <span class="hljs-built_in">enumerate</span>(pair_scores.keys()):
    <span class="hljs-built_in">print</span>(<span class="hljs-string">f&quot;<span class="hljs-subst">{key}</span>: <span class="hljs-subst">{pair_scores[key]}</span>&quot;</span>)
    <span class="hljs-keyword">if</span> i &gt;= <span class="hljs-number">5</span>:
        <span class="hljs-keyword">break</span>`}}),Ks=new _({props:{code:`('T', '##h'): 0.125
('##h', '##i'): 0.03409090909090909
('##i', '##s'): 0.02727272727272727
('i', '##s'): 0.1
('t', '##h'): 0.03571428571428571
('##h', '##e'): 0.011904761904761904`,highlighted:`(<span class="hljs-string">&#x27;T&#x27;</span>, <span class="hljs-string">&#x27;##h&#x27;</span>): <span class="hljs-number">0.125</span>
(<span class="hljs-string">&#x27;##h&#x27;</span>, <span class="hljs-string">&#x27;##i&#x27;</span>): <span class="hljs-number">0.03409090909090909</span>
(<span class="hljs-string">&#x27;##i&#x27;</span>, <span class="hljs-string">&#x27;##s&#x27;</span>): <span class="hljs-number">0.02727272727272727</span>
(<span class="hljs-string">&#x27;i&#x27;</span>, <span class="hljs-string">&#x27;##s&#x27;</span>): <span class="hljs-number">0.1</span>
(<span class="hljs-string">&#x27;t&#x27;</span>, <span class="hljs-string">&#x27;##h&#x27;</span>): <span class="hljs-number">0.03571428571428571</span>
(<span class="hljs-string">&#x27;##h&#x27;</span>, <span class="hljs-string">&#x27;##e&#x27;</span>): <span class="hljs-number">0.011904761904761904</span>`}}),Us=new _({props:{code:`best_pair = ""
max_score = None
for pair, score in pair_scores.items():
    if max_score is None or max_score < score:
        best_pair = pair
        max_score = score

print(best_pair, max_score)`,highlighted:`best_pair = <span class="hljs-string">&quot;&quot;</span>
max_score = <span class="hljs-literal">None</span>
<span class="hljs-keyword">for</span> pair, score <span class="hljs-keyword">in</span> pair_scores.items():
    <span class="hljs-keyword">if</span> max_score <span class="hljs-keyword">is</span> <span class="hljs-literal">None</span> <span class="hljs-keyword">or</span> max_score &lt; score:
        best_pair = pair
        max_score = score

<span class="hljs-built_in">print</span>(best_pair, max_score)`}}),Fs=new _({props:{code:"('a', '##b') 0.2",highlighted:'(<span class="hljs-string">&#x27;a&#x27;</span>, <span class="hljs-string">&#x27;##b&#x27;</span>) <span class="hljs-number">0.2</span>'}}),Ls=new _({props:{code:'vocab.append("ab")',highlighted:'vocab.append(<span class="hljs-string">&quot;ab&quot;</span>)'}}),Rs=new _({props:{code:`def merge_pair(a, b, splits):
    for word in word_freqs:
        split = splits[word]
        if len(split) == 1:
            continue
        i = 0
        while i < len(split) - 1:
            if split[i] == a and split[i + 1] == b:
                merge = a + b[2:] if b.startswith("##") else a + b
                split = split[:i] + [merge] + split[i + 2 :]
            else:
                i += 1
        splits[word] = split
    return splits`,highlighted:`<span class="hljs-keyword">def</span> <span class="hljs-title function_">merge_pair</span>(<span class="hljs-params">a, b, splits</span>):
    <span class="hljs-keyword">for</span> word <span class="hljs-keyword">in</span> word_freqs:
        split = splits[word]
        <span class="hljs-keyword">if</span> <span class="hljs-built_in">len</span>(split) == <span class="hljs-number">1</span>:
            <span class="hljs-keyword">continue</span>
        i = <span class="hljs-number">0</span>
        <span class="hljs-keyword">while</span> i &lt; <span class="hljs-built_in">len</span>(split) - <span class="hljs-number">1</span>:
            <span class="hljs-keyword">if</span> split[i] == a <span class="hljs-keyword">and</span> split[i + <span class="hljs-number">1</span>] == b:
                merge = a + b[<span class="hljs-number">2</span>:] <span class="hljs-keyword">if</span> b.startswith(<span class="hljs-string">&quot;##&quot;</span>) <span class="hljs-keyword">else</span> a + b
                split = split[:i] + [merge] + split[i + <span class="hljs-number">2</span> :]
            <span class="hljs-keyword">else</span>:
                i += <span class="hljs-number">1</span>
        splits[word] = split
    <span class="hljs-keyword">return</span> splits`}}),Vs=new _({props:{code:`splits = merge_pair("a", "##b", splits)
splits["about"]`,highlighted:`splits = merge_pair(<span class="hljs-string">&quot;a&quot;</span>, <span class="hljs-string">&quot;##b&quot;</span>, splits)
splits[<span class="hljs-string">&quot;about&quot;</span>]`}}),Is=new _({props:{code:"['ab', '##o', '##u', '##t']",highlighted:'[<span class="hljs-string">&#x27;ab&#x27;</span>, <span class="hljs-string">&#x27;##o&#x27;</span>, <span class="hljs-string">&#x27;##u&#x27;</span>, <span class="hljs-string">&#x27;##t&#x27;</span>]'}}),Gs=new _({props:{code:`vocab_size = 70
while len(vocab) < vocab_size:
    scores = compute_pair_scores(splits)
    best_pair, max_score = "", None
    for pair, score in scores.items():
        if max_score is None or max_score < score:
            best_pair = pair
            max_score = score
    splits = merge_pair(*best_pair, splits)
    new_token = (
        best_pair[0] + best_pair[1][2:]
        if best_pair[1].startswith("##")
        else best_pair[0] + best_pair[1]
    )
    vocab.append(new_token)`,highlighted:`vocab_size = <span class="hljs-number">70</span>
<span class="hljs-keyword">while</span> <span class="hljs-built_in">len</span>(vocab) &lt; vocab_size:
    scores = compute_pair_scores(splits)
    best_pair, max_score = <span class="hljs-string">&quot;&quot;</span>, <span class="hljs-literal">None</span>
    <span class="hljs-keyword">for</span> pair, score <span class="hljs-keyword">in</span> scores.items():
        <span class="hljs-keyword">if</span> max_score <span class="hljs-keyword">is</span> <span class="hljs-literal">None</span> <span class="hljs-keyword">or</span> max_score &lt; score:
            best_pair = pair
            max_score = score
    splits = merge_pair(*best_pair, splits)
    new_token = (
        best_pair[<span class="hljs-number">0</span>] + best_pair[<span class="hljs-number">1</span>][<span class="hljs-number">2</span>:]
        <span class="hljs-keyword">if</span> best_pair[<span class="hljs-number">1</span>].startswith(<span class="hljs-string">&quot;##&quot;</span>)
        <span class="hljs-keyword">else</span> best_pair[<span class="hljs-number">0</span>] + best_pair[<span class="hljs-number">1</span>]
    )
    vocab.append(new_token)`}}),Js=new _({props:{code:"print(vocab)",highlighted:'<span class="hljs-built_in">print</span>(vocab)'}}),Qs=new _({props:{code:`['[PAD]', '[UNK]', '[CLS]', '[SEP]', '[MASK]', '##a', '##b', '##c', '##d', '##e', '##f', '##g', '##h', '##i', '##k',
 '##l', '##m', '##n', '##o', '##p', '##r', '##s', '##t', '##u', '##v', '##w', '##y', '##z', ',', '.', 'C', 'F', 'H',
 'T', 'a', 'b', 'c', 'g', 'h', 'i', 's', 't', 'u', 'w', 'y', '##fu', 'Fa', 'Fac', '##ct', '##ful', '##full', '##fully',
 'Th', 'ch', '##hm', 'cha', 'chap', 'chapt', '##thm', 'Hu', 'Hug', 'Hugg', 'sh', 'th', 'is', '##thms', '##za', '##zat',
 '##ut']`,highlighted:`[<span class="hljs-string">&#x27;[PAD]&#x27;</span>, <span class="hljs-string">&#x27;[UNK]&#x27;</span>, <span class="hljs-string">&#x27;[CLS]&#x27;</span>, <span class="hljs-string">&#x27;[SEP]&#x27;</span>, <span class="hljs-string">&#x27;[MASK]&#x27;</span>, <span class="hljs-string">&#x27;##a&#x27;</span>, <span class="hljs-string">&#x27;##b&#x27;</span>, <span class="hljs-string">&#x27;##c&#x27;</span>, <span class="hljs-string">&#x27;##d&#x27;</span>, <span class="hljs-string">&#x27;##e&#x27;</span>, <span class="hljs-string">&#x27;##f&#x27;</span>, <span class="hljs-string">&#x27;##g&#x27;</span>, <span class="hljs-string">&#x27;##h&#x27;</span>, <span class="hljs-string">&#x27;##i&#x27;</span>, <span class="hljs-string">&#x27;##k&#x27;</span>,
 <span class="hljs-string">&#x27;##l&#x27;</span>, <span class="hljs-string">&#x27;##m&#x27;</span>, <span class="hljs-string">&#x27;##n&#x27;</span>, <span class="hljs-string">&#x27;##o&#x27;</span>, <span class="hljs-string">&#x27;##p&#x27;</span>, <span class="hljs-string">&#x27;##r&#x27;</span>, <span class="hljs-string">&#x27;##s&#x27;</span>, <span class="hljs-string">&#x27;##t&#x27;</span>, <span class="hljs-string">&#x27;##u&#x27;</span>, <span class="hljs-string">&#x27;##v&#x27;</span>, <span class="hljs-string">&#x27;##w&#x27;</span>, <span class="hljs-string">&#x27;##y&#x27;</span>, <span class="hljs-string">&#x27;##z&#x27;</span>, <span class="hljs-string">&#x27;,&#x27;</span>, <span class="hljs-string">&#x27;.&#x27;</span>, <span class="hljs-string">&#x27;C&#x27;</span>, <span class="hljs-string">&#x27;F&#x27;</span>, <span class="hljs-string">&#x27;H&#x27;</span>,
 <span class="hljs-string">&#x27;T&#x27;</span>, <span class="hljs-string">&#x27;a&#x27;</span>, <span class="hljs-string">&#x27;b&#x27;</span>, <span class="hljs-string">&#x27;c&#x27;</span>, <span class="hljs-string">&#x27;g&#x27;</span>, <span class="hljs-string">&#x27;h&#x27;</span>, <span class="hljs-string">&#x27;i&#x27;</span>, <span class="hljs-string">&#x27;s&#x27;</span>, <span class="hljs-string">&#x27;t&#x27;</span>, <span class="hljs-string">&#x27;u&#x27;</span>, <span class="hljs-string">&#x27;w&#x27;</span>, <span class="hljs-string">&#x27;y&#x27;</span>, <span class="hljs-string">&#x27;##fu&#x27;</span>, <span class="hljs-string">&#x27;Fa&#x27;</span>, <span class="hljs-string">&#x27;Fac&#x27;</span>, <span class="hljs-string">&#x27;##ct&#x27;</span>, <span class="hljs-string">&#x27;##ful&#x27;</span>, <span class="hljs-string">&#x27;##full&#x27;</span>, <span class="hljs-string">&#x27;##fully&#x27;</span>,
 <span class="hljs-string">&#x27;Th&#x27;</span>, <span class="hljs-string">&#x27;ch&#x27;</span>, <span class="hljs-string">&#x27;##hm&#x27;</span>, <span class="hljs-string">&#x27;cha&#x27;</span>, <span class="hljs-string">&#x27;chap&#x27;</span>, <span class="hljs-string">&#x27;chapt&#x27;</span>, <span class="hljs-string">&#x27;##thm&#x27;</span>, <span class="hljs-string">&#x27;Hu&#x27;</span>, <span class="hljs-string">&#x27;Hug&#x27;</span>, <span class="hljs-string">&#x27;Hugg&#x27;</span>, <span class="hljs-string">&#x27;sh&#x27;</span>, <span class="hljs-string">&#x27;th&#x27;</span>, <span class="hljs-string">&#x27;is&#x27;</span>, <span class="hljs-string">&#x27;##thms&#x27;</span>, <span class="hljs-string">&#x27;##za&#x27;</span>, <span class="hljs-string">&#x27;##zat&#x27;</span>,
 <span class="hljs-string">&#x27;##ut&#x27;</span>]`}}),bs=new Ga({props:{$$slots:{default:[Wi]},$$scope:{ctx:F}}}),Ys=new _({props:{code:`def encode_word(word):
    tokens = []
    while len(word) > 0:
        i = len(word)
        while i > 0 and word[:i] not in vocab:
            i -= 1
        if i == 0:
            return ["[UNK]"]
        tokens.append(word[:i])
        word = word[i:]
        if len(word) > 0:
            word = f"##{word}"
    return tokens`,highlighted:`<span class="hljs-keyword">def</span> <span class="hljs-title function_">encode_word</span>(<span class="hljs-params">word</span>):
    tokens = []
    <span class="hljs-keyword">while</span> <span class="hljs-built_in">len</span>(word) &gt; <span class="hljs-number">0</span>:
        i = <span class="hljs-built_in">len</span>(word)
        <span class="hljs-keyword">while</span> i &gt; <span class="hljs-number">0</span> <span class="hljs-keyword">and</span> word[:i] <span class="hljs-keyword">not</span> <span class="hljs-keyword">in</span> vocab:
            i -= <span class="hljs-number">1</span>
        <span class="hljs-keyword">if</span> i == <span class="hljs-number">0</span>:
            <span class="hljs-keyword">return</span> [<span class="hljs-string">&quot;[UNK]&quot;</span>]
        tokens.append(word[:i])
        word = word[i:]
        <span class="hljs-keyword">if</span> <span class="hljs-built_in">len</span>(word) &gt; <span class="hljs-number">0</span>:
            word = <span class="hljs-string">f&quot;##<span class="hljs-subst">{word}</span>&quot;</span>
    <span class="hljs-keyword">return</span> tokens`}}),Xs=new _({props:{code:`print(encode_word("Hugging"))
print(encode_word("HOgging"))`,highlighted:`<span class="hljs-built_in">print</span>(encode_word(<span class="hljs-string">&quot;Hugging&quot;</span>))
<span class="hljs-built_in">print</span>(encode_word(<span class="hljs-string">&quot;HOgging&quot;</span>))`}}),Zs=new _({props:{code:`['Hugg', '##i', '##n', '##g']
['[UNK]']`,highlighted:`[<span class="hljs-string">&#x27;Hugg&#x27;</span>, <span class="hljs-string">&#x27;##i&#x27;</span>, <span class="hljs-string">&#x27;##n&#x27;</span>, <span class="hljs-string">&#x27;##g&#x27;</span>]
[<span class="hljs-string">&#x27;[UNK]&#x27;</span>]`}}),se=new _({props:{code:`def tokenize(text):
    pre_tokenize_result = tokenizer._tokenizer.pre_tokenizer.pre_tokenize_str(text)
    pre_tokenized_text = [word for word, offset in pre_tokenize_result]
    encoded_words = [encode_word(word) for word in pre_tokenized_text]
    return sum(encoded_words, [])`,highlighted:`<span class="hljs-keyword">def</span> <span class="hljs-title function_">tokenize</span>(<span class="hljs-params">text</span>):
    pre_tokenize_result = tokenizer._tokenizer.pre_tokenizer.pre_tokenize_str(text)
    pre_tokenized_text = [word <span class="hljs-keyword">for</span> word, offset <span class="hljs-keyword">in</span> pre_tokenize_result]
    encoded_words = [encode_word(word) <span class="hljs-keyword">for</span> word <span class="hljs-keyword">in</span> pre_tokenized_text]
    <span class="hljs-keyword">return</span> <span class="hljs-built_in">sum</span>(encoded_words, [])`}}),ee=new _({props:{code:`tokenize("This is the Hugging Face course!")  # C'est le cours d'Hugging Face`,highlighted:'tokenize(<span class="hljs-string">&quot;This is the Hugging Face course!&quot;</span>)  <span class="hljs-comment"># C&#x27;est le cours d&#x27;Hugging Face</span>'}}),ae=new _({props:{code:`['Th', '##i', '##s', 'is', 'th', '##e', 'Hugg', '##i', '##n', '##g', 'Fac', '##e', 'c', '##o', '##u', '##r', '##s',
 '##e', '[UNK]']`,highlighted:`[<span class="hljs-string">&#x27;Th&#x27;</span>, <span class="hljs-string">&#x27;##i&#x27;</span>, <span class="hljs-string">&#x27;##s&#x27;</span>, <span class="hljs-string">&#x27;is&#x27;</span>, <span class="hljs-string">&#x27;th&#x27;</span>, <span class="hljs-string">&#x27;##e&#x27;</span>, <span class="hljs-string">&#x27;Hugg&#x27;</span>, <span class="hljs-string">&#x27;##i&#x27;</span>, <span class="hljs-string">&#x27;##n&#x27;</span>, <span class="hljs-string">&#x27;##g&#x27;</span>, <span class="hljs-string">&#x27;Fac&#x27;</span>, <span class="hljs-string">&#x27;##e&#x27;</span>, <span class="hljs-string">&#x27;c&#x27;</span>, <span class="hljs-string">&#x27;##o&#x27;</span>, <span class="hljs-string">&#x27;##u&#x27;</span>, <span class="hljs-string">&#x27;##r&#x27;</span>, <span class="hljs-string">&#x27;##s&#x27;</span>,
 <span class="hljs-string">&#x27;##e&#x27;</span>, <span class="hljs-string">&#x27;[UNK]&#x27;</span>]`}}),{c(){m=r("meta"),k=u(),x=r("h1"),$=r("a"),w=r("span"),h(v.$$.fragment),q=u(),y=r("span"),H=n("Tok\xE9nisation *WordPiece*"),M=u(),h(W.$$.fragment),U=u(),D=r("p"),ns=r("em"),le=n("WordPiece"),re=n(" est l\u2019algorithme de tok\xE9nisation d\xE9velopp\xE9 par Google pour pr\xE9tra\xEEner BERT. Il a depuis \xE9t\xE9 r\xE9utilis\xE9 dans un grand nombre de mod\xE8les de transformateurs bas\xE9s sur BERT, tels que DistilBERT, MobileBERT, Funnel Transformers et MPNET. Il est tr\xE8s similaire \xE0 BPE en termes d\u2019entra\xEEnement, mais la tokenisation r\xE9elle est effectu\xE9e diff\xE9remment."),Ja=u(),h(qs.$$.fragment),Qa=u(),h(ts.$$.fragment),Ya=u(),ss=r("h2"),ls=r("a"),Pe=r("span"),h(_s.$$.fragment),Tt=u(),Ce=r("span"),Ot=n("Algorithme d'entra\xEEnement"),Xa=u(),h(rs.$$.fragment),Za=u(),S=r("p"),At=n("Comme le BPE, "),ze=r("em"),Nt=n("WordPiece"),Mt=n(" part d\u2019un petit vocabulaire comprenant les "),De=r("em"),Bt=n("tokens"),Ht=n(" sp\xE9ciaux utilis\xE9s par le mod\xE8le et l\u2019alphabet initial. Puisqu\u2019il identifie les sous-mots en ajoutant un pr\xE9fixe (comme "),Te=r("code"),Wt=n("##"),St=n(" pour BERT), chaque mot est initialement d\xE9coup\xE9 en ajoutant ce pr\xE9fixe \xE0 tous les caract\xE8res du mot. Ainsi, par exemple, "),Oe=r("code"),Kt=n('"mot"'),Ut=n(" est divis\xE9 comme ceci :"),sn=u(),h($s.$$.fragment),en=u(),ps=r("p"),Ft=n("Ainsi, l\u2019alphabet initial contient tous les caract\xE8res pr\xE9sents au d\xE9but d\u2019un mot et les caract\xE8res pr\xE9sents \xE0 l\u2019int\xE9rieur d\u2019un mot pr\xE9c\xE9d\xE9 du pr\xE9fixe de "),Ae=r("em"),Lt=n("WordPiece"),Rt=n("."),an=u(),I=r("p"),Vt=n("Ensuite, toujours comme le BPE, "),Ne=r("em"),It=n("WordPiece"),Gt=n(" apprend des r\xE8gles de fusion. La principale diff\xE9rence r\xE9side dans la mani\xE8re dont la paire \xE0 fusionner est s\xE9lectionn\xE9e. Au lieu de s\xE9lectionner la paire la plus fr\xE9quente, "),Me=r("em"),Jt=n("WordPiece"),Qt=n(` calcule un score pour chaque paire, en utilisant la formule suivante :
`),nn=new Ci,tn=u(),T=r("p"),Yt=n("En divisant la fr\xE9quence de la paire par le produit des fr\xE9quences de chacune de ses parties, l\u2019algorithme donne la priorit\xE9 \xE0 la fusion des paires dont les parties individuelles sont moins fr\xE9quentes dans le vocabulaire. Par exemple, il ne fusionnera pas n\xE9cessairement "),Be=r("code"),Xt=n('("un", "##able")'),Zt=n(" m\xEAme si cette paire appara\xEEt tr\xE8s fr\xE9quemment dans le vocabulaire, car les deux paires "),He=r("code"),sl=n('"un"'),el=n("\u201D et "),We=r("code"),al=n('"##able"'),nl=n(" appara\xEEtront probablement chacune dans un lot d\u2019autres mots et auront une fr\xE9quence \xE9lev\xE9e. En revanche, une paire comme "),Se=r("code"),tl=n('("hu", "##gging")'),ll=n(" sera probablement fusionn\xE9e plus rapidement (en supposant que le mot \u201Chugging\u201D apparaisse souvent dans le vocabulaire) puisque "),Ke=r("code"),rl=n('"hu"'),pl=n(" et "),Ue=r("code"),ol=n('"##gging"'),il=n(" sont probablement moins fr\xE9quents individuellement."),ln=u(),pe=r("p"),ul=n("Examinons le m\xEAme vocabulaire que celui utilis\xE9 dans l\u2019exemple d\u2019entra\xEEnement du BPE :"),rn=u(),h(ws.$$.fragment),pn=u(),oe=r("p"),cl=n("Les divisions ici seront :"),on=u(),h(ks.$$.fragment),un=u(),P=r("p"),ml=n("Le vocabulaire initial sera donc "),Fe=r("code"),hl=n('["b", "h", "p", "##g", "##n", "##s", "##u"]'),dl=n(" (si on oublie les "),Le=r("em"),fl=n("tokens"),jl=n(" sp\xE9ciaux pour l\u2019instant). La paire la plus fr\xE9quente est "),Re=r("code"),gl=n('("##u", "##g")'),bl=n(" (pr\xE9sente 20 fois), mais la fr\xE9quence individuelle de "),Ve=r("code"),xl=n('"##u"'),vl=n(" est tr\xE8s \xE9lev\xE9e, donc son score n\u2019est pas le plus \xE9lev\xE9 (il est de 1 / 36). Toutes les paires avec un "),Ie=r("code"),ql=n('"##u"'),_l=n(" ont en fait le m\xEAme score (1 / 36), donc le meilleur score va \xE0 la paire "),Ge=r("code"),$l=n('("##g", "##s")'),wl=n(" \u2014 la seule sans un "),Je=r("code"),kl=n('"##u"'),El=n(" \u2014 \xE0 1 / 20, et la premi\xE8re fusion apprise est "),Qe=r("code"),yl=n('("##g", "##s") -> ("##gs")'),Pl=n("."),cn=u(),L=r("p"),Cl=n("Notez que lorsque nous fusionnons, nous enlevons le "),Ye=r("code"),zl=n("##"),Dl=n(" entre les deux "),Xe=r("em"),Tl=n("tokens"),Ol=n(", donc nous ajoutons "),Ze=r("code"),Al=n('"##gs"'),Nl=n(" au vocabulaire et appliquons la fusion dans les mots du corpus :"),mn=u(),h(Es.$$.fragment),hn=u(),G=r("p"),Ml=n("\xC0 ce stade, "),sa=r("code"),Bl=n('" ##u "'),Hl=n(" est dans toutes les paires possibles, donc elles finissent toutes par avoir le m\xEAme score. Disons que dans ce cas, la premi\xE8re paire est fusionn\xE9e, donc "),ea=r("code"),Wl=n('("h", "##u") -> "hu"'),Sl=n(". Cela nous am\xE8ne \xE0 :"),dn=u(),h(ys.$$.fragment),fn=u(),J=r("p"),Kl=n("Ensuite, le meilleur score suivant est partag\xE9 par "),aa=r("code"),Ul=n('("hu", "##g")'),Fl=n(" et "),na=r("code"),Ll=n('("hu", "##gs")'),Rl=n(" (avec 1/15, compar\xE9 \xE0 1/21 pour toutes les autres paires), donc la premi\xE8re paire avec le plus grand score est fusionn\xE9e :"),jn=u(),h(Ps.$$.fragment),gn=u(),ie=r("p"),Vl=n("et nous continuons ainsi jusqu\u2019\xE0 ce que nous atteignions la taille de vocabulaire souhait\xE9e."),bn=u(),h(os.$$.fragment),xn=u(),es=r("h2"),is=r("a"),ta=r("span"),h(Cs.$$.fragment),Il=u(),la=r("span"),Gl=n("Algorithme de tokenisation"),vn=u(),E=r("p"),Jl=n("La tok\xE9nisation diff\xE8re dans "),ra=r("em"),Ql=n("WordPiece"),Yl=n(" et BPE en ce que "),pa=r("em"),Xl=n("WordPiece"),Zl=n(" ne sauvegarde que le vocabulaire final, pas les r\xE8gles de fusion apprises. En partant du mot \xE0 tokeniser, "),oa=r("em"),sr=n("WordPiece"),er=n(" trouve le sous-mot le plus long qui se trouve dans le vocabulaire, puis se s\xE9pare sur celui-ci. Par exemple, si nous utilisons le vocabulaire appris dans l\u2019exemple ci-dessus, pour le mot "),ia=r("code"),ar=n('"hugs"'),nr=n(" le plus long sous-mot en partant du d\xE9but qui est dans le vocabulaire est "),ua=r("code"),tr=n('"hug"'),lr=n(", donc nous le divisons et obtenons "),ca=r("code"),rr=n('["hug", "##s"]'),pr=n(". On continue avec "),ma=r("code"),or=n('"##s"'),ir=n(", qui est dans le vocabulaire, donc la tokenisation de "),ha=r("code"),ur=n('"hugs"'),cr=n(" est "),da=r("code"),mr=n('["hug", "##s"]'),hr=n("."),qn=u(),us=r("p"),dr=n("Avec BPE, nous aurions appliqu\xE9 les fusions apprises dans l\u2019ordre et la tok\xE9nisation aurait \xE9t\xE9 "),fa=r("code"),fr=n('["hu", "##gs"]'),jr=n(", l\u2019encodage est donc diff\xE9rent."),_n=u(),C=r("p"),gr=n("Comme autre exemple, voyons comment le mot "),ja=r("code"),br=n('"bugs"'),xr=n(" serait tokenis\xE9. "),ga=r("code"),vr=n('"b"'),qr=n(" est le plus long sous-mot commen\xE7ant au d\xE9but du mot qui est dans le vocabulaire, donc on le divise et on obtient "),ba=r("code"),_r=n('["b", "##ugs"]'),$r=n(". Ensuite, "),xa=r("code"),wr=n('"##u"'),kr=n(" est le plus long sous-mot commen\xE7ant au d\xE9but de "),va=r("code"),Er=n('"##ugs"'),yr=n(" qui est dans le vocabulaire, donc on le s\xE9pare et on obtient "),qa=r("code"),Pr=n('["b", "##u, "##gs"]'),Cr=n(". Enfin, "),_a=r("code"),zr=n('"##gs"'),Dr=n(" est dans le vocabulaire, donc cette derni\xE8re liste est la tokenization de "),$a=r("code"),Tr=n('"bugs"'),Or=n("."),$n=u(),B=r("p"),Ar=n("Lorsque la tokenisation arrive \xE0 un stade o\xF9 il n\u2019est pas possible de trouver un sous-mot dans le vocabulaire, le mot entier est tokenis\xE9 comme inconnu \u2014 donc, par exemple, "),wa=r("code"),Nr=n('"mug"'),Mr=n(" serait tokenis\xE9 comme "),ka=r("code"),Br=n('["[UNK]"]'),Hr=n(", tout comme \u201D bum \u201D (m\xEAme si on peut commencer par \u201D b \u201D et \u201D ##u \u201D, \u201D ##m \u201D ne fait pas partie du vocabulaire, et le "),Ea=r("em"),Wr=n("tokenizer"),Sr=n(" r\xE9sultant sera simplement "),ya=r("code"),Kr=n('["[UNK]"]'),Ur=n(" \u201D, et non "),Pa=r("code"),Fr=n('["b", "##u", "[UNK]"]'),Lr=n(" \u201D). C\u2019est une autre diff\xE9rence avec BPE, qui classerait seulement les caract\xE8res individuels qui ne sont pas dans le vocabulaire comme inconnus."),wn=u(),h(cs.$$.fragment),kn=u(),as=r("h2"),ms=r("a"),Ca=r("span"),h(zs.$$.fragment),Rr=u(),za=r("span"),Vr=n("Mise en \u0153uvre de *WordPiece*"),En=u(),hs=r("p"),Ir=n("Voyons maintenant une impl\xE9mentation de l\u2019algorithme "),Da=r("em"),Gr=n("WordPiece"),Jr=n(". Comme pour le BPE, il s\u2019agit d\u2019un exemple p\xE9dagogique, et vous ne pourrez pas l\u2019utiliser sur un grand corpus."),yn=u(),ue=r("p"),Qr=n("Nous utiliserons le m\xEAme corpus que dans l\u2019exemple BPE :"),Pn=u(),h(Ds.$$.fragment),Cn=u(),R=r("p"),Yr=n("Tout d\u2019abord, nous devons pr\xE9-tok\xE9niser le corpus en mots. Puisque nous r\xE9pliquons un "),Ta=r("em"),Xr=n("tokenizer WordPiece"),Zr=n(" (comme BERT), nous utiliserons le "),Oa=r("em"),sp=n("tokenizer"),ep=u(),Aa=r("code"),ap=n("bert-base-cased"),np=n(" pour la pr\xE9-tok\xE9nisation :"),zn=u(),h(Ts.$$.fragment),Dn=u(),ce=r("p"),tp=n("Ensuite, nous calculons les fr\xE9quences de chaque mot dans le corpus comme nous le faisons pour la pr\xE9-tok\xE9nisation :"),Tn=u(),h(Os.$$.fragment),On=u(),h(As.$$.fragment),An=u(),ds=r("p"),lp=n("Comme nous l\u2019avons vu pr\xE9c\xE9demment, l\u2019alphabet est l\u2019ensemble unique compos\xE9 de toutes les premi\xE8res lettres des mots, et de toutes les autres lettres qui apparaissent dans les mots pr\xE9fix\xE9s par "),Na=r("code"),rp=n("##"),pp=n(" :"),Nn=u(),h(Ns.$$.fragment),Mn=u(),h(Ms.$$.fragment),Bn=u(),fs=r("p"),op=n("Nous ajoutons \xE9galement les tokens sp\xE9ciaux utilis\xE9s par le mod\xE8le au d\xE9but de ce vocabulaire. Dans le cas de BERT, il s\u2019agit de la liste "),Ma=r("code"),ip=n('["[PAD]", "[UNK]", "[CLS]", "[SEP]", "[MASK]"]'),up=n(" :"),Hn=u(),h(Bs.$$.fragment),Wn=u(),js=r("p"),cp=n("Ensuite, nous devons diviser chaque mot, avec toutes les lettres qui ne sont pas les premi\xE8res pr\xE9fix\xE9es par "),Ba=r("code"),mp=n("##"),hp=n(" :"),Sn=u(),h(Hs.$$.fragment),Kn=u(),me=r("p"),dp=n("Maintenant que nous sommes pr\xEAts pour l\u2019entra\xEEnement, \xE9crivons une fonction qui calcule le score de chaque paire. Nous devrons l\u2019utiliser \xE0 chaque \xE9tape de l\u2019entra\xEEnement :"),Un=u(),h(Ws.$$.fragment),Fn=u(),he=r("p"),fp=n("Jetons un coup d\u2019\u0153il \xE0 une partie de ce dictionnaire apr\xE8s les premi\xE8res divisions :"),Ln=u(),h(Ss.$$.fragment),Rn=u(),h(Ks.$$.fragment),Vn=u(),de=r("p"),jp=n("Maintenant, trouver la paire avec le meilleur score ne prend qu\u2019une rapide boucle :"),In=u(),h(Us.$$.fragment),Gn=u(),h(Fs.$$.fragment),Jn=u(),Q=r("p"),gp=n("Ainsi, la premi\xE8re fusion \xE0 apprendre est "),Ha=r("code"),bp=n("('a', '##b') -> 'ab'"),xp=n(", et nous ajoutons "),Wa=r("code"),vp=n("'ab'"),qp=n(" au vocabulaire :"),Qn=u(),h(Ls.$$.fragment),Yn=u(),gs=r("p"),_p=n("Pour continuer, nous devons appliquer cette fusion dans notre dictionnaire "),Sa=r("code"),$p=n("splits"),wp=n(". \xC9crivons une autre fonction pour cela :"),Xn=u(),h(Rs.$$.fragment),Zn=u(),fe=r("p"),kp=n("Et nous pouvons regarder le r\xE9sultat de la premi\xE8re fusion :"),st=u(),h(Vs.$$.fragment),et=u(),h(Is.$$.fragment),at=u(),je=r("p"),Ep=n("Nous avons maintenant tout ce dont nous avons besoin pour boucler jusqu\u2019\xE0 ce que nous ayons appris toutes les fusions que nous voulons. Visons une taille de vocabulaire de 70 :"),nt=u(),h(Gs.$$.fragment),tt=u(),ge=r("p"),yp=n("Nous pouvons ensuite examiner le vocabulaire g\xE9n\xE9r\xE9 :"),lt=u(),h(Js.$$.fragment),rt=u(),h(Qs.$$.fragment),pt=u(),Y=r("p"),Pp=n("Comme nous pouvons le voir, compar\xE9 \xE0 BPE, ce "),Ka=r("em"),Cp=n("tokenizer"),zp=n(" apprend les parties de mots comme des "),Ua=r("em"),Dp=n("tokens"),Tp=n(" un peu plus rapidement."),ot=u(),h(bs.$$.fragment),it=u(),be=r("p"),Op=n("Pour tokeniser un nouveau texte, on le pr\xE9-tokenise, on le divise, puis on applique l\u2019algorithme de tokenisation sur chaque mot. En d\u2019autres termes, nous recherchons le plus grand sous-mot commen\xE7ant au d\xE9but du premier mot et le divisons, puis nous r\xE9p\xE9tons le processus sur la deuxi\xE8me partie, et ainsi de suite pour le reste de ce mot et les mots suivants dans le texte :"),ut=u(),h(Ys.$$.fragment),ct=u(),xe=r("p"),Ap=n("Testons-le sur un mot qui fait partie du vocabulaire, et un autre qui n\u2019en fait pas partie :"),mt=u(),h(Xs.$$.fragment),ht=u(),h(Zs.$$.fragment),dt=u(),ve=r("p"),Np=n("Maintenant, \xE9crivons une fonction qui permet de tokeniser un texte :"),ft=u(),h(se.$$.fragment),jt=u(),qe=r("p"),Mp=n("On peut l\u2019essayer sur n\u2019importe quel texte :"),gt=u(),h(ee.$$.fragment),bt=u(),h(ae.$$.fragment),xt=u(),X=r("p"),Bp=n("C\u2019est tout pour l\u2019algorithme "),Fa=r("em"),Hp=n("WordPiece"),Wp=n(" ! Maintenant, jetons un coup d\u2019oeil \xE0 "),La=r("em"),Sp=n("Unigram"),Kp=n("."),this.h()},l(s){const l=zi('[data-svelte="svelte-1phssyn"]',document.head);m=p(l,"META",{name:!0,content:!0}),l.forEach(a),k=c(s),x=p(s,"H1",{class:!0});var ne=o(x);$=p(ne,"A",{id:!0,class:!0,href:!0});var Ra=o($);w=p(Ra,"SPAN",{});var Va=o(w);d(v.$$.fragment,Va),Va.forEach(a),Ra.forEach(a),q=c(ne),y=p(ne,"SPAN",{});var Ia=o(y);H=t(Ia,"Tok\xE9nisation *WordPiece*"),Ia.forEach(a),ne.forEach(a),M=c(s),d(W.$$.fragment,s),U=c(s),D=p(s,"P",{});var _e=o(D);ns=p(_e,"EM",{});var Up=o(ns);le=t(Up,"WordPiece"),Up.forEach(a),re=t(_e," est l\u2019algorithme de tok\xE9nisation d\xE9velopp\xE9 par Google pour pr\xE9tra\xEEner BERT. Il a depuis \xE9t\xE9 r\xE9utilis\xE9 dans un grand nombre de mod\xE8les de transformateurs bas\xE9s sur BERT, tels que DistilBERT, MobileBERT, Funnel Transformers et MPNET. Il est tr\xE8s similaire \xE0 BPE en termes d\u2019entra\xEEnement, mais la tokenisation r\xE9elle est effectu\xE9e diff\xE9remment."),_e.forEach(a),Ja=c(s),d(qs.$$.fragment,s),Qa=c(s),d(ts.$$.fragment,s),Ya=c(s),ss=p(s,"H2",{class:!0});var qt=o(ss);ls=p(qt,"A",{id:!0,class:!0,href:!0});var Fp=o(ls);Pe=p(Fp,"SPAN",{});var Lp=o(Pe);d(_s.$$.fragment,Lp),Lp.forEach(a),Fp.forEach(a),Tt=c(qt),Ce=p(qt,"SPAN",{});var Rp=o(Ce);Ot=t(Rp,"Algorithme d'entra\xEEnement"),Rp.forEach(a),qt.forEach(a),Xa=c(s),d(rs.$$.fragment,s),Za=c(s),S=p(s,"P",{});var Z=o(S);At=t(Z,"Comme le BPE, "),ze=p(Z,"EM",{});var Vp=o(ze);Nt=t(Vp,"WordPiece"),Vp.forEach(a),Mt=t(Z," part d\u2019un petit vocabulaire comprenant les "),De=p(Z,"EM",{});var Ip=o(De);Bt=t(Ip,"tokens"),Ip.forEach(a),Ht=t(Z," sp\xE9ciaux utilis\xE9s par le mod\xE8le et l\u2019alphabet initial. Puisqu\u2019il identifie les sous-mots en ajoutant un pr\xE9fixe (comme "),Te=p(Z,"CODE",{});var Gp=o(Te);Wt=t(Gp,"##"),Gp.forEach(a),St=t(Z," pour BERT), chaque mot est initialement d\xE9coup\xE9 en ajoutant ce pr\xE9fixe \xE0 tous les caract\xE8res du mot. Ainsi, par exemple, "),Oe=p(Z,"CODE",{});var Jp=o(Oe);Kt=t(Jp,'"mot"'),Jp.forEach(a),Ut=t(Z," est divis\xE9 comme ceci :"),Z.forEach(a),sn=c(s),d($s.$$.fragment,s),en=c(s),ps=p(s,"P",{});var _t=o(ps);Ft=t(_t,"Ainsi, l\u2019alphabet initial contient tous les caract\xE8res pr\xE9sents au d\xE9but d\u2019un mot et les caract\xE8res pr\xE9sents \xE0 l\u2019int\xE9rieur d\u2019un mot pr\xE9c\xE9d\xE9 du pr\xE9fixe de "),Ae=p(_t,"EM",{});var Qp=o(Ae);Lt=t(Qp,"WordPiece"),Qp.forEach(a),Rt=t(_t,"."),_t.forEach(a),an=c(s),I=p(s,"P",{});var te=o(I);Vt=t(te,"Ensuite, toujours comme le BPE, "),Ne=p(te,"EM",{});var Yp=o(Ne);It=t(Yp,"WordPiece"),Yp.forEach(a),Gt=t(te," apprend des r\xE8gles de fusion. La principale diff\xE9rence r\xE9side dans la mani\xE8re dont la paire \xE0 fusionner est s\xE9lectionn\xE9e. Au lieu de s\xE9lectionner la paire la plus fr\xE9quente, "),Me=p(te,"EM",{});var Xp=o(Me);Jt=t(Xp,"WordPiece"),Xp.forEach(a),Qt=t(te,` calcule un score pour chaque paire, en utilisant la formule suivante :
`),nn=Di(te),te.forEach(a),tn=c(s),T=p(s,"P",{});var K=o(T);Yt=t(K,"En divisant la fr\xE9quence de la paire par le produit des fr\xE9quences de chacune de ses parties, l\u2019algorithme donne la priorit\xE9 \xE0 la fusion des paires dont les parties individuelles sont moins fr\xE9quentes dans le vocabulaire. Par exemple, il ne fusionnera pas n\xE9cessairement "),Be=p(K,"CODE",{});var Zp=o(Be);Xt=t(Zp,'("un", "##able")'),Zp.forEach(a),Zt=t(K," m\xEAme si cette paire appara\xEEt tr\xE8s fr\xE9quemment dans le vocabulaire, car les deux paires "),He=p(K,"CODE",{});var so=o(He);sl=t(so,'"un"'),so.forEach(a),el=t(K,"\u201D et "),We=p(K,"CODE",{});var eo=o(We);al=t(eo,'"##able"'),eo.forEach(a),nl=t(K," appara\xEEtront probablement chacune dans un lot d\u2019autres mots et auront une fr\xE9quence \xE9lev\xE9e. En revanche, une paire comme "),Se=p(K,"CODE",{});var ao=o(Se);tl=t(ao,'("hu", "##gging")'),ao.forEach(a),ll=t(K," sera probablement fusionn\xE9e plus rapidement (en supposant que le mot \u201Chugging\u201D apparaisse souvent dans le vocabulaire) puisque "),Ke=p(K,"CODE",{});var no=o(Ke);rl=t(no,'"hu"'),no.forEach(a),pl=t(K," et "),Ue=p(K,"CODE",{});var to=o(Ue);ol=t(to,'"##gging"'),to.forEach(a),il=t(K," sont probablement moins fr\xE9quents individuellement."),K.forEach(a),ln=c(s),pe=p(s,"P",{});var lo=o(pe);ul=t(lo,"Examinons le m\xEAme vocabulaire que celui utilis\xE9 dans l\u2019exemple d\u2019entra\xEEnement du BPE :"),lo.forEach(a),rn=c(s),d(ws.$$.fragment,s),pn=c(s),oe=p(s,"P",{});var ro=o(oe);cl=t(ro,"Les divisions ici seront :"),ro.forEach(a),on=c(s),d(ks.$$.fragment,s),un=c(s),P=p(s,"P",{});var O=o(P);ml=t(O,"Le vocabulaire initial sera donc "),Fe=p(O,"CODE",{});var po=o(Fe);hl=t(po,'["b", "h", "p", "##g", "##n", "##s", "##u"]'),po.forEach(a),dl=t(O," (si on oublie les "),Le=p(O,"EM",{});var oo=o(Le);fl=t(oo,"tokens"),oo.forEach(a),jl=t(O," sp\xE9ciaux pour l\u2019instant). La paire la plus fr\xE9quente est "),Re=p(O,"CODE",{});var io=o(Re);gl=t(io,'("##u", "##g")'),io.forEach(a),bl=t(O," (pr\xE9sente 20 fois), mais la fr\xE9quence individuelle de "),Ve=p(O,"CODE",{});var uo=o(Ve);xl=t(uo,'"##u"'),uo.forEach(a),vl=t(O," est tr\xE8s \xE9lev\xE9e, donc son score n\u2019est pas le plus \xE9lev\xE9 (il est de 1 / 36). Toutes les paires avec un "),Ie=p(O,"CODE",{});var co=o(Ie);ql=t(co,'"##u"'),co.forEach(a),_l=t(O," ont en fait le m\xEAme score (1 / 36), donc le meilleur score va \xE0 la paire "),Ge=p(O,"CODE",{});var mo=o(Ge);$l=t(mo,'("##g", "##s")'),mo.forEach(a),wl=t(O," \u2014 la seule sans un "),Je=p(O,"CODE",{});var ho=o(Je);kl=t(ho,'"##u"'),ho.forEach(a),El=t(O," \u2014 \xE0 1 / 20, et la premi\xE8re fusion apprise est "),Qe=p(O,"CODE",{});var fo=o(Qe);yl=t(fo,'("##g", "##s") -> ("##gs")'),fo.forEach(a),Pl=t(O,"."),O.forEach(a),cn=c(s),L=p(s,"P",{});var xs=o(L);Cl=t(xs,"Notez que lorsque nous fusionnons, nous enlevons le "),Ye=p(xs,"CODE",{});var jo=o(Ye);zl=t(jo,"##"),jo.forEach(a),Dl=t(xs," entre les deux "),Xe=p(xs,"EM",{});var go=o(Xe);Tl=t(go,"tokens"),go.forEach(a),Ol=t(xs,", donc nous ajoutons "),Ze=p(xs,"CODE",{});var bo=o(Ze);Al=t(bo,'"##gs"'),bo.forEach(a),Nl=t(xs," au vocabulaire et appliquons la fusion dans les mots du corpus :"),xs.forEach(a),mn=c(s),d(Es.$$.fragment,s),hn=c(s),G=p(s,"P",{});var $e=o(G);Ml=t($e,"\xC0 ce stade, "),sa=p($e,"CODE",{});var xo=o(sa);Bl=t(xo,'" ##u "'),xo.forEach(a),Hl=t($e," est dans toutes les paires possibles, donc elles finissent toutes par avoir le m\xEAme score. Disons que dans ce cas, la premi\xE8re paire est fusionn\xE9e, donc "),ea=p($e,"CODE",{});var vo=o(ea);Wl=t(vo,'("h", "##u") -> "hu"'),vo.forEach(a),Sl=t($e,". Cela nous am\xE8ne \xE0 :"),$e.forEach(a),dn=c(s),d(ys.$$.fragment,s),fn=c(s),J=p(s,"P",{});var we=o(J);Kl=t(we,"Ensuite, le meilleur score suivant est partag\xE9 par "),aa=p(we,"CODE",{});var qo=o(aa);Ul=t(qo,'("hu", "##g")'),qo.forEach(a),Fl=t(we," et "),na=p(we,"CODE",{});var _o=o(na);Ll=t(_o,'("hu", "##gs")'),_o.forEach(a),Rl=t(we," (avec 1/15, compar\xE9 \xE0 1/21 pour toutes les autres paires), donc la premi\xE8re paire avec le plus grand score est fusionn\xE9e :"),we.forEach(a),jn=c(s),d(Ps.$$.fragment,s),gn=c(s),ie=p(s,"P",{});var $o=o(ie);Vl=t($o,"et nous continuons ainsi jusqu\u2019\xE0 ce que nous atteignions la taille de vocabulaire souhait\xE9e."),$o.forEach(a),bn=c(s),d(os.$$.fragment,s),xn=c(s),es=p(s,"H2",{class:!0});var $t=o(es);is=p($t,"A",{id:!0,class:!0,href:!0});var wo=o(is);ta=p(wo,"SPAN",{});var ko=o(ta);d(Cs.$$.fragment,ko),ko.forEach(a),wo.forEach(a),Il=c($t),la=p($t,"SPAN",{});var Eo=o(la);Gl=t(Eo,"Algorithme de tokenisation"),Eo.forEach(a),$t.forEach(a),vn=c(s),E=p(s,"P",{});var z=o(E);Jl=t(z,"La tok\xE9nisation diff\xE8re dans "),ra=p(z,"EM",{});var yo=o(ra);Ql=t(yo,"WordPiece"),yo.forEach(a),Yl=t(z," et BPE en ce que "),pa=p(z,"EM",{});var Po=o(pa);Xl=t(Po,"WordPiece"),Po.forEach(a),Zl=t(z," ne sauvegarde que le vocabulaire final, pas les r\xE8gles de fusion apprises. En partant du mot \xE0 tokeniser, "),oa=p(z,"EM",{});var Co=o(oa);sr=t(Co,"WordPiece"),Co.forEach(a),er=t(z," trouve le sous-mot le plus long qui se trouve dans le vocabulaire, puis se s\xE9pare sur celui-ci. Par exemple, si nous utilisons le vocabulaire appris dans l\u2019exemple ci-dessus, pour le mot "),ia=p(z,"CODE",{});var zo=o(ia);ar=t(zo,'"hugs"'),zo.forEach(a),nr=t(z," le plus long sous-mot en partant du d\xE9but qui est dans le vocabulaire est "),ua=p(z,"CODE",{});var Do=o(ua);tr=t(Do,'"hug"'),Do.forEach(a),lr=t(z,", donc nous le divisons et obtenons "),ca=p(z,"CODE",{});var To=o(ca);rr=t(To,'["hug", "##s"]'),To.forEach(a),pr=t(z,". On continue avec "),ma=p(z,"CODE",{});var Oo=o(ma);or=t(Oo,'"##s"'),Oo.forEach(a),ir=t(z,", qui est dans le vocabulaire, donc la tokenisation de "),ha=p(z,"CODE",{});var Ao=o(ha);ur=t(Ao,'"hugs"'),Ao.forEach(a),cr=t(z," est "),da=p(z,"CODE",{});var No=o(da);mr=t(No,'["hug", "##s"]'),No.forEach(a),hr=t(z,"."),z.forEach(a),qn=c(s),us=p(s,"P",{});var wt=o(us);dr=t(wt,"Avec BPE, nous aurions appliqu\xE9 les fusions apprises dans l\u2019ordre et la tok\xE9nisation aurait \xE9t\xE9 "),fa=p(wt,"CODE",{});var Mo=o(fa);fr=t(Mo,'["hu", "##gs"]'),Mo.forEach(a),jr=t(wt,", l\u2019encodage est donc diff\xE9rent."),wt.forEach(a),_n=c(s),C=p(s,"P",{});var A=o(C);gr=t(A,"Comme autre exemple, voyons comment le mot "),ja=p(A,"CODE",{});var Bo=o(ja);br=t(Bo,'"bugs"'),Bo.forEach(a),xr=t(A," serait tokenis\xE9. "),ga=p(A,"CODE",{});var Ho=o(ga);vr=t(Ho,'"b"'),Ho.forEach(a),qr=t(A," est le plus long sous-mot commen\xE7ant au d\xE9but du mot qui est dans le vocabulaire, donc on le divise et on obtient "),ba=p(A,"CODE",{});var Wo=o(ba);_r=t(Wo,'["b", "##ugs"]'),Wo.forEach(a),$r=t(A,". Ensuite, "),xa=p(A,"CODE",{});var So=o(xa);wr=t(So,'"##u"'),So.forEach(a),kr=t(A," est le plus long sous-mot commen\xE7ant au d\xE9but de "),va=p(A,"CODE",{});var Ko=o(va);Er=t(Ko,'"##ugs"'),Ko.forEach(a),yr=t(A," qui est dans le vocabulaire, donc on le s\xE9pare et on obtient "),qa=p(A,"CODE",{});var Uo=o(qa);Pr=t(Uo,'["b", "##u, "##gs"]'),Uo.forEach(a),Cr=t(A,". Enfin, "),_a=p(A,"CODE",{});var Fo=o(_a);zr=t(Fo,'"##gs"'),Fo.forEach(a),Dr=t(A," est dans le vocabulaire, donc cette derni\xE8re liste est la tokenization de "),$a=p(A,"CODE",{});var Lo=o($a);Tr=t(Lo,'"bugs"'),Lo.forEach(a),Or=t(A,"."),A.forEach(a),$n=c(s),B=p(s,"P",{});var V=o(B);Ar=t(V,"Lorsque la tokenisation arrive \xE0 un stade o\xF9 il n\u2019est pas possible de trouver un sous-mot dans le vocabulaire, le mot entier est tokenis\xE9 comme inconnu \u2014 donc, par exemple, "),wa=p(V,"CODE",{});var Ro=o(wa);Nr=t(Ro,'"mug"'),Ro.forEach(a),Mr=t(V," serait tokenis\xE9 comme "),ka=p(V,"CODE",{});var Vo=o(ka);Br=t(Vo,'["[UNK]"]'),Vo.forEach(a),Hr=t(V,", tout comme \u201D bum \u201D (m\xEAme si on peut commencer par \u201D b \u201D et \u201D ##u \u201D, \u201D ##m \u201D ne fait pas partie du vocabulaire, et le "),Ea=p(V,"EM",{});var Io=o(Ea);Wr=t(Io,"tokenizer"),Io.forEach(a),Sr=t(V," r\xE9sultant sera simplement "),ya=p(V,"CODE",{});var Go=o(ya);Kr=t(Go,'["[UNK]"]'),Go.forEach(a),Ur=t(V," \u201D, et non "),Pa=p(V,"CODE",{});var Jo=o(Pa);Fr=t(Jo,'["b", "##u", "[UNK]"]'),Jo.forEach(a),Lr=t(V," \u201D). C\u2019est une autre diff\xE9rence avec BPE, qui classerait seulement les caract\xE8res individuels qui ne sont pas dans le vocabulaire comme inconnus."),V.forEach(a),wn=c(s),d(cs.$$.fragment,s),kn=c(s),as=p(s,"H2",{class:!0});var kt=o(as);ms=p(kt,"A",{id:!0,class:!0,href:!0});var Qo=o(ms);Ca=p(Qo,"SPAN",{});var Yo=o(Ca);d(zs.$$.fragment,Yo),Yo.forEach(a),Qo.forEach(a),Rr=c(kt),za=p(kt,"SPAN",{});var Xo=o(za);Vr=t(Xo,"Mise en \u0153uvre de *WordPiece*"),Xo.forEach(a),kt.forEach(a),En=c(s),hs=p(s,"P",{});var Et=o(hs);Ir=t(Et,"Voyons maintenant une impl\xE9mentation de l\u2019algorithme "),Da=p(Et,"EM",{});var Zo=o(Da);Gr=t(Zo,"WordPiece"),Zo.forEach(a),Jr=t(Et,". Comme pour le BPE, il s\u2019agit d\u2019un exemple p\xE9dagogique, et vous ne pourrez pas l\u2019utiliser sur un grand corpus."),Et.forEach(a),yn=c(s),ue=p(s,"P",{});var si=o(ue);Qr=t(si,"Nous utiliserons le m\xEAme corpus que dans l\u2019exemple BPE :"),si.forEach(a),Pn=c(s),d(Ds.$$.fragment,s),Cn=c(s),R=p(s,"P",{});var vs=o(R);Yr=t(vs,"Tout d\u2019abord, nous devons pr\xE9-tok\xE9niser le corpus en mots. Puisque nous r\xE9pliquons un "),Ta=p(vs,"EM",{});var ei=o(Ta);Xr=t(ei,"tokenizer WordPiece"),ei.forEach(a),Zr=t(vs," (comme BERT), nous utiliserons le "),Oa=p(vs,"EM",{});var ai=o(Oa);sp=t(ai,"tokenizer"),ai.forEach(a),ep=c(vs),Aa=p(vs,"CODE",{});var ni=o(Aa);ap=t(ni,"bert-base-cased"),ni.forEach(a),np=t(vs," pour la pr\xE9-tok\xE9nisation :"),vs.forEach(a),zn=c(s),d(Ts.$$.fragment,s),Dn=c(s),ce=p(s,"P",{});var ti=o(ce);tp=t(ti,"Ensuite, nous calculons les fr\xE9quences de chaque mot dans le corpus comme nous le faisons pour la pr\xE9-tok\xE9nisation :"),ti.forEach(a),Tn=c(s),d(Os.$$.fragment,s),On=c(s),d(As.$$.fragment,s),An=c(s),ds=p(s,"P",{});var yt=o(ds);lp=t(yt,"Comme nous l\u2019avons vu pr\xE9c\xE9demment, l\u2019alphabet est l\u2019ensemble unique compos\xE9 de toutes les premi\xE8res lettres des mots, et de toutes les autres lettres qui apparaissent dans les mots pr\xE9fix\xE9s par "),Na=p(yt,"CODE",{});var li=o(Na);rp=t(li,"##"),li.forEach(a),pp=t(yt," :"),yt.forEach(a),Nn=c(s),d(Ns.$$.fragment,s),Mn=c(s),d(Ms.$$.fragment,s),Bn=c(s),fs=p(s,"P",{});var Pt=o(fs);op=t(Pt,"Nous ajoutons \xE9galement les tokens sp\xE9ciaux utilis\xE9s par le mod\xE8le au d\xE9but de ce vocabulaire. Dans le cas de BERT, il s\u2019agit de la liste "),Ma=p(Pt,"CODE",{});var ri=o(Ma);ip=t(ri,'["[PAD]", "[UNK]", "[CLS]", "[SEP]", "[MASK]"]'),ri.forEach(a),up=t(Pt," :"),Pt.forEach(a),Hn=c(s),d(Bs.$$.fragment,s),Wn=c(s),js=p(s,"P",{});var Ct=o(js);cp=t(Ct,"Ensuite, nous devons diviser chaque mot, avec toutes les lettres qui ne sont pas les premi\xE8res pr\xE9fix\xE9es par "),Ba=p(Ct,"CODE",{});var pi=o(Ba);mp=t(pi,"##"),pi.forEach(a),hp=t(Ct," :"),Ct.forEach(a),Sn=c(s),d(Hs.$$.fragment,s),Kn=c(s),me=p(s,"P",{});var oi=o(me);dp=t(oi,"Maintenant que nous sommes pr\xEAts pour l\u2019entra\xEEnement, \xE9crivons une fonction qui calcule le score de chaque paire. Nous devrons l\u2019utiliser \xE0 chaque \xE9tape de l\u2019entra\xEEnement :"),oi.forEach(a),Un=c(s),d(Ws.$$.fragment,s),Fn=c(s),he=p(s,"P",{});var ii=o(he);fp=t(ii,"Jetons un coup d\u2019\u0153il \xE0 une partie de ce dictionnaire apr\xE8s les premi\xE8res divisions :"),ii.forEach(a),Ln=c(s),d(Ss.$$.fragment,s),Rn=c(s),d(Ks.$$.fragment,s),Vn=c(s),de=p(s,"P",{});var ui=o(de);jp=t(ui,"Maintenant, trouver la paire avec le meilleur score ne prend qu\u2019une rapide boucle :"),ui.forEach(a),In=c(s),d(Us.$$.fragment,s),Gn=c(s),d(Fs.$$.fragment,s),Jn=c(s),Q=p(s,"P",{});var ke=o(Q);gp=t(ke,"Ainsi, la premi\xE8re fusion \xE0 apprendre est "),Ha=p(ke,"CODE",{});var ci=o(Ha);bp=t(ci,"('a', '##b') -> 'ab'"),ci.forEach(a),xp=t(ke,", et nous ajoutons "),Wa=p(ke,"CODE",{});var mi=o(Wa);vp=t(mi,"'ab'"),mi.forEach(a),qp=t(ke," au vocabulaire :"),ke.forEach(a),Qn=c(s),d(Ls.$$.fragment,s),Yn=c(s),gs=p(s,"P",{});var zt=o(gs);_p=t(zt,"Pour continuer, nous devons appliquer cette fusion dans notre dictionnaire "),Sa=p(zt,"CODE",{});var hi=o(Sa);$p=t(hi,"splits"),hi.forEach(a),wp=t(zt,". \xC9crivons une autre fonction pour cela :"),zt.forEach(a),Xn=c(s),d(Rs.$$.fragment,s),Zn=c(s),fe=p(s,"P",{});var di=o(fe);kp=t(di,"Et nous pouvons regarder le r\xE9sultat de la premi\xE8re fusion :"),di.forEach(a),st=c(s),d(Vs.$$.fragment,s),et=c(s),d(Is.$$.fragment,s),at=c(s),je=p(s,"P",{});var fi=o(je);Ep=t(fi,"Nous avons maintenant tout ce dont nous avons besoin pour boucler jusqu\u2019\xE0 ce que nous ayons appris toutes les fusions que nous voulons. Visons une taille de vocabulaire de 70 :"),fi.forEach(a),nt=c(s),d(Gs.$$.fragment,s),tt=c(s),ge=p(s,"P",{});var ji=o(ge);yp=t(ji,"Nous pouvons ensuite examiner le vocabulaire g\xE9n\xE9r\xE9 :"),ji.forEach(a),lt=c(s),d(Js.$$.fragment,s),rt=c(s),d(Qs.$$.fragment,s),pt=c(s),Y=p(s,"P",{});var Ee=o(Y);Pp=t(Ee,"Comme nous pouvons le voir, compar\xE9 \xE0 BPE, ce "),Ka=p(Ee,"EM",{});var gi=o(Ka);Cp=t(gi,"tokenizer"),gi.forEach(a),zp=t(Ee," apprend les parties de mots comme des "),Ua=p(Ee,"EM",{});var bi=o(Ua);Dp=t(bi,"tokens"),bi.forEach(a),Tp=t(Ee," un peu plus rapidement."),Ee.forEach(a),ot=c(s),d(bs.$$.fragment,s),it=c(s),be=p(s,"P",{});var xi=o(be);Op=t(xi,"Pour tokeniser un nouveau texte, on le pr\xE9-tokenise, on le divise, puis on applique l\u2019algorithme de tokenisation sur chaque mot. En d\u2019autres termes, nous recherchons le plus grand sous-mot commen\xE7ant au d\xE9but du premier mot et le divisons, puis nous r\xE9p\xE9tons le processus sur la deuxi\xE8me partie, et ainsi de suite pour le reste de ce mot et les mots suivants dans le texte :"),xi.forEach(a),ut=c(s),d(Ys.$$.fragment,s),ct=c(s),xe=p(s,"P",{});var vi=o(xe);Ap=t(vi,"Testons-le sur un mot qui fait partie du vocabulaire, et un autre qui n\u2019en fait pas partie :"),vi.forEach(a),mt=c(s),d(Xs.$$.fragment,s),ht=c(s),d(Zs.$$.fragment,s),dt=c(s),ve=p(s,"P",{});var qi=o(ve);Np=t(qi,"Maintenant, \xE9crivons une fonction qui permet de tokeniser un texte :"),qi.forEach(a),ft=c(s),d(se.$$.fragment,s),jt=c(s),qe=p(s,"P",{});var _i=o(qe);Mp=t(_i,"On peut l\u2019essayer sur n\u2019importe quel texte :"),_i.forEach(a),gt=c(s),d(ee.$$.fragment,s),bt=c(s),d(ae.$$.fragment,s),xt=c(s),X=p(s,"P",{});var ye=o(X);Bp=t(ye,"C\u2019est tout pour l\u2019algorithme "),Fa=p(ye,"EM",{});var $i=o(Fa);Hp=t($i,"WordPiece"),$i.forEach(a),Wp=t(ye," ! Maintenant, jetons un coup d\u2019oeil \xE0 "),La=p(ye,"EM",{});var wi=o(La);Sp=t(wi,"Unigram"),wi.forEach(a),Kp=t(ye,"."),ye.forEach(a),this.h()},h(){N(m,"name","hf:doc:metadata"),N(m,"content",JSON.stringify(Ki)),N($,"id","toknisation-wordpiece"),N($,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),N($,"href","#toknisation-wordpiece"),N(x,"class","relative group"),N(ls,"id","algorithme-dentranement"),N(ls,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),N(ls,"href","#algorithme-dentranement"),N(ss,"class","relative group"),nn.a=null,N(is,"id","algorithme-de-tokenisation"),N(is,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),N(is,"href","#algorithme-de-tokenisation"),N(es,"class","relative group"),N(ms,"id","mise-en-uvre-de-wordpiece"),N(ms,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),N(ms,"href","#mise-en-uvre-de-wordpiece"),N(as,"class","relative group")},m(s,l){e(document.head,m),i(s,k,l),i(s,x,l),e(x,$),e($,w),f(v,w,null),e(x,q),e(x,y),e(y,H),i(s,M,l),f(W,s,l),i(s,U,l),i(s,D,l),e(D,ns),e(ns,le),e(D,re),i(s,Ja,l),f(qs,s,l),i(s,Qa,l),f(ts,s,l),i(s,Ya,l),i(s,ss,l),e(ss,ls),e(ls,Pe),f(_s,Pe,null),e(ss,Tt),e(ss,Ce),e(Ce,Ot),i(s,Xa,l),f(rs,s,l),i(s,Za,l),i(s,S,l),e(S,At),e(S,ze),e(ze,Nt),e(S,Mt),e(S,De),e(De,Bt),e(S,Ht),e(S,Te),e(Te,Wt),e(S,St),e(S,Oe),e(Oe,Kt),e(S,Ut),i(s,sn,l),f($s,s,l),i(s,en,l),i(s,ps,l),e(ps,Ft),e(ps,Ae),e(Ae,Lt),e(ps,Rt),i(s,an,l),i(s,I,l),e(I,Vt),e(I,Ne),e(Ne,It),e(I,Gt),e(I,Me),e(Me,Jt),e(I,Qt),nn.m(ki,I),i(s,tn,l),i(s,T,l),e(T,Yt),e(T,Be),e(Be,Xt),e(T,Zt),e(T,He),e(He,sl),e(T,el),e(T,We),e(We,al),e(T,nl),e(T,Se),e(Se,tl),e(T,ll),e(T,Ke),e(Ke,rl),e(T,pl),e(T,Ue),e(Ue,ol),e(T,il),i(s,ln,l),i(s,pe,l),e(pe,ul),i(s,rn,l),f(ws,s,l),i(s,pn,l),i(s,oe,l),e(oe,cl),i(s,on,l),f(ks,s,l),i(s,un,l),i(s,P,l),e(P,ml),e(P,Fe),e(Fe,hl),e(P,dl),e(P,Le),e(Le,fl),e(P,jl),e(P,Re),e(Re,gl),e(P,bl),e(P,Ve),e(Ve,xl),e(P,vl),e(P,Ie),e(Ie,ql),e(P,_l),e(P,Ge),e(Ge,$l),e(P,wl),e(P,Je),e(Je,kl),e(P,El),e(P,Qe),e(Qe,yl),e(P,Pl),i(s,cn,l),i(s,L,l),e(L,Cl),e(L,Ye),e(Ye,zl),e(L,Dl),e(L,Xe),e(Xe,Tl),e(L,Ol),e(L,Ze),e(Ze,Al),e(L,Nl),i(s,mn,l),f(Es,s,l),i(s,hn,l),i(s,G,l),e(G,Ml),e(G,sa),e(sa,Bl),e(G,Hl),e(G,ea),e(ea,Wl),e(G,Sl),i(s,dn,l),f(ys,s,l),i(s,fn,l),i(s,J,l),e(J,Kl),e(J,aa),e(aa,Ul),e(J,Fl),e(J,na),e(na,Ll),e(J,Rl),i(s,jn,l),f(Ps,s,l),i(s,gn,l),i(s,ie,l),e(ie,Vl),i(s,bn,l),f(os,s,l),i(s,xn,l),i(s,es,l),e(es,is),e(is,ta),f(Cs,ta,null),e(es,Il),e(es,la),e(la,Gl),i(s,vn,l),i(s,E,l),e(E,Jl),e(E,ra),e(ra,Ql),e(E,Yl),e(E,pa),e(pa,Xl),e(E,Zl),e(E,oa),e(oa,sr),e(E,er),e(E,ia),e(ia,ar),e(E,nr),e(E,ua),e(ua,tr),e(E,lr),e(E,ca),e(ca,rr),e(E,pr),e(E,ma),e(ma,or),e(E,ir),e(E,ha),e(ha,ur),e(E,cr),e(E,da),e(da,mr),e(E,hr),i(s,qn,l),i(s,us,l),e(us,dr),e(us,fa),e(fa,fr),e(us,jr),i(s,_n,l),i(s,C,l),e(C,gr),e(C,ja),e(ja,br),e(C,xr),e(C,ga),e(ga,vr),e(C,qr),e(C,ba),e(ba,_r),e(C,$r),e(C,xa),e(xa,wr),e(C,kr),e(C,va),e(va,Er),e(C,yr),e(C,qa),e(qa,Pr),e(C,Cr),e(C,_a),e(_a,zr),e(C,Dr),e(C,$a),e($a,Tr),e(C,Or),i(s,$n,l),i(s,B,l),e(B,Ar),e(B,wa),e(wa,Nr),e(B,Mr),e(B,ka),e(ka,Br),e(B,Hr),e(B,Ea),e(Ea,Wr),e(B,Sr),e(B,ya),e(ya,Kr),e(B,Ur),e(B,Pa),e(Pa,Fr),e(B,Lr),i(s,wn,l),f(cs,s,l),i(s,kn,l),i(s,as,l),e(as,ms),e(ms,Ca),f(zs,Ca,null),e(as,Rr),e(as,za),e(za,Vr),i(s,En,l),i(s,hs,l),e(hs,Ir),e(hs,Da),e(Da,Gr),e(hs,Jr),i(s,yn,l),i(s,ue,l),e(ue,Qr),i(s,Pn,l),f(Ds,s,l),i(s,Cn,l),i(s,R,l),e(R,Yr),e(R,Ta),e(Ta,Xr),e(R,Zr),e(R,Oa),e(Oa,sp),e(R,ep),e(R,Aa),e(Aa,ap),e(R,np),i(s,zn,l),f(Ts,s,l),i(s,Dn,l),i(s,ce,l),e(ce,tp),i(s,Tn,l),f(Os,s,l),i(s,On,l),f(As,s,l),i(s,An,l),i(s,ds,l),e(ds,lp),e(ds,Na),e(Na,rp),e(ds,pp),i(s,Nn,l),f(Ns,s,l),i(s,Mn,l),f(Ms,s,l),i(s,Bn,l),i(s,fs,l),e(fs,op),e(fs,Ma),e(Ma,ip),e(fs,up),i(s,Hn,l),f(Bs,s,l),i(s,Wn,l),i(s,js,l),e(js,cp),e(js,Ba),e(Ba,mp),e(js,hp),i(s,Sn,l),f(Hs,s,l),i(s,Kn,l),i(s,me,l),e(me,dp),i(s,Un,l),f(Ws,s,l),i(s,Fn,l),i(s,he,l),e(he,fp),i(s,Ln,l),f(Ss,s,l),i(s,Rn,l),f(Ks,s,l),i(s,Vn,l),i(s,de,l),e(de,jp),i(s,In,l),f(Us,s,l),i(s,Gn,l),f(Fs,s,l),i(s,Jn,l),i(s,Q,l),e(Q,gp),e(Q,Ha),e(Ha,bp),e(Q,xp),e(Q,Wa),e(Wa,vp),e(Q,qp),i(s,Qn,l),f(Ls,s,l),i(s,Yn,l),i(s,gs,l),e(gs,_p),e(gs,Sa),e(Sa,$p),e(gs,wp),i(s,Xn,l),f(Rs,s,l),i(s,Zn,l),i(s,fe,l),e(fe,kp),i(s,st,l),f(Vs,s,l),i(s,et,l),f(Is,s,l),i(s,at,l),i(s,je,l),e(je,Ep),i(s,nt,l),f(Gs,s,l),i(s,tt,l),i(s,ge,l),e(ge,yp),i(s,lt,l),f(Js,s,l),i(s,rt,l),f(Qs,s,l),i(s,pt,l),i(s,Y,l),e(Y,Pp),e(Y,Ka),e(Ka,Cp),e(Y,zp),e(Y,Ua),e(Ua,Dp),e(Y,Tp),i(s,ot,l),f(bs,s,l),i(s,it,l),i(s,be,l),e(be,Op),i(s,ut,l),f(Ys,s,l),i(s,ct,l),i(s,xe,l),e(xe,Ap),i(s,mt,l),f(Xs,s,l),i(s,ht,l),f(Zs,s,l),i(s,dt,l),i(s,ve,l),e(ve,Np),i(s,ft,l),f(se,s,l),i(s,jt,l),i(s,qe,l),e(qe,Mp),i(s,gt,l),f(ee,s,l),i(s,bt,l),f(ae,s,l),i(s,xt,l),i(s,X,l),e(X,Bp),e(X,Fa),e(Fa,Hp),e(X,Wp),e(X,La),e(La,Sp),e(X,Kp),vt=!0},p(s,[l]){const ne={};l&2&&(ne.$$scope={dirty:l,ctx:s}),ts.$set(ne);const Ra={};l&2&&(Ra.$$scope={dirty:l,ctx:s}),rs.$set(Ra);const Va={};l&2&&(Va.$$scope={dirty:l,ctx:s}),os.$set(Va);const Ia={};l&2&&(Ia.$$scope={dirty:l,ctx:s}),cs.$set(Ia);const _e={};l&2&&(_e.$$scope={dirty:l,ctx:s}),bs.$set(_e)},i(s){vt||(j(v.$$.fragment,s),j(W.$$.fragment,s),j(qs.$$.fragment,s),j(ts.$$.fragment,s),j(_s.$$.fragment,s),j(rs.$$.fragment,s),j($s.$$.fragment,s),j(ws.$$.fragment,s),j(ks.$$.fragment,s),j(Es.$$.fragment,s),j(ys.$$.fragment,s),j(Ps.$$.fragment,s),j(os.$$.fragment,s),j(Cs.$$.fragment,s),j(cs.$$.fragment,s),j(zs.$$.fragment,s),j(Ds.$$.fragment,s),j(Ts.$$.fragment,s),j(Os.$$.fragment,s),j(As.$$.fragment,s),j(Ns.$$.fragment,s),j(Ms.$$.fragment,s),j(Bs.$$.fragment,s),j(Hs.$$.fragment,s),j(Ws.$$.fragment,s),j(Ss.$$.fragment,s),j(Ks.$$.fragment,s),j(Us.$$.fragment,s),j(Fs.$$.fragment,s),j(Ls.$$.fragment,s),j(Rs.$$.fragment,s),j(Vs.$$.fragment,s),j(Is.$$.fragment,s),j(Gs.$$.fragment,s),j(Js.$$.fragment,s),j(Qs.$$.fragment,s),j(bs.$$.fragment,s),j(Ys.$$.fragment,s),j(Xs.$$.fragment,s),j(Zs.$$.fragment,s),j(se.$$.fragment,s),j(ee.$$.fragment,s),j(ae.$$.fragment,s),vt=!0)},o(s){g(v.$$.fragment,s),g(W.$$.fragment,s),g(qs.$$.fragment,s),g(ts.$$.fragment,s),g(_s.$$.fragment,s),g(rs.$$.fragment,s),g($s.$$.fragment,s),g(ws.$$.fragment,s),g(ks.$$.fragment,s),g(Es.$$.fragment,s),g(ys.$$.fragment,s),g(Ps.$$.fragment,s),g(os.$$.fragment,s),g(Cs.$$.fragment,s),g(cs.$$.fragment,s),g(zs.$$.fragment,s),g(Ds.$$.fragment,s),g(Ts.$$.fragment,s),g(Os.$$.fragment,s),g(As.$$.fragment,s),g(Ns.$$.fragment,s),g(Ms.$$.fragment,s),g(Bs.$$.fragment,s),g(Hs.$$.fragment,s),g(Ws.$$.fragment,s),g(Ss.$$.fragment,s),g(Ks.$$.fragment,s),g(Us.$$.fragment,s),g(Fs.$$.fragment,s),g(Ls.$$.fragment,s),g(Rs.$$.fragment,s),g(Vs.$$.fragment,s),g(Is.$$.fragment,s),g(Gs.$$.fragment,s),g(Js.$$.fragment,s),g(Qs.$$.fragment,s),g(bs.$$.fragment,s),g(Ys.$$.fragment,s),g(Xs.$$.fragment,s),g(Zs.$$.fragment,s),g(se.$$.fragment,s),g(ee.$$.fragment,s),g(ae.$$.fragment,s),vt=!1},d(s){a(m),s&&a(k),s&&a(x),b(v),s&&a(M),b(W,s),s&&a(U),s&&a(D),s&&a(Ja),b(qs,s),s&&a(Qa),b(ts,s),s&&a(Ya),s&&a(ss),b(_s),s&&a(Xa),b(rs,s),s&&a(Za),s&&a(S),s&&a(sn),b($s,s),s&&a(en),s&&a(ps),s&&a(an),s&&a(I),s&&a(tn),s&&a(T),s&&a(ln),s&&a(pe),s&&a(rn),b(ws,s),s&&a(pn),s&&a(oe),s&&a(on),b(ks,s),s&&a(un),s&&a(P),s&&a(cn),s&&a(L),s&&a(mn),b(Es,s),s&&a(hn),s&&a(G),s&&a(dn),b(ys,s),s&&a(fn),s&&a(J),s&&a(jn),b(Ps,s),s&&a(gn),s&&a(ie),s&&a(bn),b(os,s),s&&a(xn),s&&a(es),b(Cs),s&&a(vn),s&&a(E),s&&a(qn),s&&a(us),s&&a(_n),s&&a(C),s&&a($n),s&&a(B),s&&a(wn),b(cs,s),s&&a(kn),s&&a(as),b(zs),s&&a(En),s&&a(hs),s&&a(yn),s&&a(ue),s&&a(Pn),b(Ds,s),s&&a(Cn),s&&a(R),s&&a(zn),b(Ts,s),s&&a(Dn),s&&a(ce),s&&a(Tn),b(Os,s),s&&a(On),b(As,s),s&&a(An),s&&a(ds),s&&a(Nn),b(Ns,s),s&&a(Mn),b(Ms,s),s&&a(Bn),s&&a(fs),s&&a(Hn),b(Bs,s),s&&a(Wn),s&&a(js),s&&a(Sn),b(Hs,s),s&&a(Kn),s&&a(me),s&&a(Un),b(Ws,s),s&&a(Fn),s&&a(he),s&&a(Ln),b(Ss,s),s&&a(Rn),b(Ks,s),s&&a(Vn),s&&a(de),s&&a(In),b(Us,s),s&&a(Gn),b(Fs,s),s&&a(Jn),s&&a(Q),s&&a(Qn),b(Ls,s),s&&a(Yn),s&&a(gs),s&&a(Xn),b(Rs,s),s&&a(Zn),s&&a(fe),s&&a(st),b(Vs,s),s&&a(et),b(Is,s),s&&a(at),s&&a(je),s&&a(nt),b(Gs,s),s&&a(tt),s&&a(ge),s&&a(lt),b(Js,s),s&&a(rt),b(Qs,s),s&&a(pt),s&&a(Y),s&&a(ot),b(bs,s),s&&a(it),s&&a(be),s&&a(ut),b(Ys,s),s&&a(ct),s&&a(xe),s&&a(mt),b(Xs,s),s&&a(ht),b(Zs,s),s&&a(dt),s&&a(ve),s&&a(ft),b(se,s),s&&a(jt),s&&a(qe),s&&a(gt),b(ee,s),s&&a(bt),b(ae,s),s&&a(xt),s&&a(X)}}}const Ki={local:"toknisation-wordpiece",sections:[{local:"algorithme-dentranement",title:"Algorithme d'entra\xEEnement"},{local:"algorithme-de-tokenisation",title:"Algorithme de tokenisation"},{local:"mise-en-uvre-de-wordpiece",title:"Mise en \u0153uvre de *WordPiece*"}],title:"Tok\xE9nisation *WordPiece*"};function Ui(F){return Ti(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class Ji extends Ei{constructor(m){super();yi(this,m,Ui,Si,Pi,{})}}export{Ji as default,Ki as metadata};
