import{S as Zr,i as Qr,s as Xr,e as r,k as u,w as h,t,M as si,c as i,d as e,m as c,a as o,x as d,h as l,b as T,G as n,g as p,y as j,q as g,o as x,B as q,v as ei}from"../../chunks/vendor-hf-doc-builder.js";import{T as Ee}from"../../chunks/Tip-hf-doc-builder.js";import{Y as ni}from"../../chunks/Youtube-hf-doc-builder.js";import{I as lt}from"../../chunks/IconCopyLink-hf-doc-builder.js";import{C as _}from"../../chunks/CodeBlock-hf-doc-builder.js";import{D as ai}from"../../chunks/DocNotebookDropdown-hf-doc-builder.js";function ti(D){let f,k;return{c(){f=r("p"),k=t("\u{1F4A1} Cette section couvre le BPE en profondeur, allant jusqu\u2019\xE0 montrer une impl\xE9mentation compl\xE8te. Vous pouvez passer directement \xE0 la fin si vous souhaitez simplement avoir un aper\xE7u g\xE9n\xE9ral de l\u2019algorithme de tokenisation.")},l(m){f=i(m,"P",{});var $=o(f);k=l($,"\u{1F4A1} Cette section couvre le BPE en profondeur, allant jusqu\u2019\xE0 montrer une impl\xE9mentation compl\xE8te. Vous pouvez passer directement \xE0 la fin si vous souhaitez simplement avoir un aper\xE7u g\xE9n\xE9ral de l\u2019algorithme de tokenisation."),$.forEach(e)},m(m,$){p(m,f,$),n(f,k)},d(m){m&&e(f)}}}function li(D){let f,k,m,$,w,b,v,E,N,P,U;return{c(){f=r("p"),k=t("Les "),m=r("em"),$=t("tokenizers"),w=t(" du GPT-2 et de RoBERTa (qui sont assez similaires) ont une fa\xE7on intelligente de g\xE9rer ce probl\xE8me : ils ne consid\xE8rent pas les mots comme \xE9tant \xE9crits avec des caract\xE8res Unicode mais avec des octets. De cette fa\xE7on, le vocabulaire de base a une petite taille (256) et tous les caract\xE8res auxquels vous pouvez penser seront inclus dedans et ne finiront pas par \xEAtre convertis en un "),b=r("em"),v=t("token"),E=t(" inconnu. Cette astuce est appel\xE9e "),N=r("em"),P=t("byte-level BPE"),U=t(".")},l(O){f=i(O,"P",{});var z=o(f);k=l(z,"Les "),m=i(z,"EM",{});var xs=o(m);$=l(xs,"tokenizers"),xs.forEach(e),w=l(z," du GPT-2 et de RoBERTa (qui sont assez similaires) ont une fa\xE7on intelligente de g\xE9rer ce probl\xE8me : ils ne consid\xE8rent pas les mots comme \xE9tant \xE9crits avec des caract\xE8res Unicode mais avec des octets. De cette fa\xE7on, le vocabulaire de base a une petite taille (256) et tous les caract\xE8res auxquels vous pouvez penser seront inclus dedans et ne finiront pas par \xEAtre convertis en un "),b=i(z,"EM",{});var L=o(b);v=l(L,"token"),L.forEach(e),E=l(z," inconnu. Cette astuce est appel\xE9e "),N=i(z,"EM",{});var ne=o(N);P=l(ne,"byte-level BPE"),ne.forEach(e),U=l(z,"."),z.forEach(e)},m(O,z){p(O,f,z),n(f,k),n(f,m),n(m,$),n(f,w),n(f,b),n(b,v),n(f,E),n(f,N),n(N,P),n(f,U)},d(O){O&&e(f)}}}function pi(D){let f,k,m,$,w;return{c(){f=r("p"),k=t("\u270F\uFE0F "),m=r("strong"),$=t("A votre tour !"),w=t(" A votre avis, quelle sera la prochaine r\xE8gle de fusion ?")},l(b){f=i(b,"P",{});var v=o(f);k=l(v,"\u270F\uFE0F "),m=i(v,"STRONG",{});var E=o(m);$=l(E,"A votre tour !"),E.forEach(e),w=l(v," A votre avis, quelle sera la prochaine r\xE8gle de fusion ?"),v.forEach(e)},m(b,v){p(b,f,v),n(f,k),n(f,m),n(m,$),n(f,w)},d(b){b&&e(f)}}}function ri(D){let f,k,m,$,w;return{c(){f=r("p"),k=t("\u270F\uFE0F "),m=r("strong"),$=t("A votre tour !"),w=t(" Comment pensez-vous que le mot \xAB unhug \xBB (d\xE9tacher en fran\xE7ais) sera tokenized ?")},l(b){f=i(b,"P",{});var v=o(f);k=l(v,"\u270F\uFE0F "),m=i(v,"STRONG",{});var E=o(m);$=l(E,"A votre tour !"),E.forEach(e),w=l(v," Comment pensez-vous que le mot \xAB unhug \xBB (d\xE9tacher en fran\xE7ais) sera tokenized ?"),v.forEach(e)},m(b,v){p(b,f,v),n(f,k),n(f,m),n(m,$),n(f,w)},d(b){b&&e(f)}}}function ii(D){let f,k,m,$,w,b,v,E;return{c(){f=r("p"),k=t("\u{1F4A1} Utiliser "),m=r("code"),$=t("train_new_from_iterator()"),w=t(" sur le m\xEAme corpus ne donnera pas exactement le m\xEAme vocabulaire. C\u2019est parce que lorsqu\u2019il y a un choix de la paire la plus fr\xE9quente, nous avons s\xE9lectionn\xE9 la premi\xE8re rencontr\xE9e, alors que la biblioth\xE8que \u{1F917} "),b=r("em"),v=t("Tokenizers"),E=t(" s\xE9lectionne la premi\xE8re en fonction de ses identifiants internes.")},l(N){f=i(N,"P",{});var P=o(f);k=l(P,"\u{1F4A1} Utiliser "),m=i(P,"CODE",{});var U=o(m);$=l(U,"train_new_from_iterator()"),U.forEach(e),w=l(P," sur le m\xEAme corpus ne donnera pas exactement le m\xEAme vocabulaire. C\u2019est parce que lorsqu\u2019il y a un choix de la paire la plus fr\xE9quente, nous avons s\xE9lectionn\xE9 la premi\xE8re rencontr\xE9e, alors que la biblioth\xE8que \u{1F917} "),b=i(P,"EM",{});var O=o(b);v=l(O,"Tokenizers"),O.forEach(e),E=l(P," s\xE9lectionne la premi\xE8re en fonction de ses identifiants internes."),P.forEach(e)},m(N,P){p(N,f,P),n(f,k),n(f,m),n(m,$),n(f,w),n(f,b),n(b,v),n(f,E)},d(N){N&&e(f)}}}function oi(D){let f,k,m,$,w;return{c(){f=r("p"),k=t("\u26A0\uFE0F Notre impl\xE9mentation lancera une erreur s\u2019il y a un caract\xE8re inconnu puisque nous n\u2019avons rien fait pour les g\xE9rer. GPT-2 n\u2019a pas r\xE9ellement de "),m=r("i"),$=t("token"),w=t(" inconnu (il est impossible d\u2019obtenir un caract\xE8re inconnu en utilisant le BPE au niveau de l\u2019octet) mais cela pourrait arriver ici car nous n\u2019avons pas inclus tous les octets possibles dans le vocabulaire initial. Cet aspect du BPE d\xE9passe le cadre de cette section, nous avons donc laiss\xE9 ces d\xE9tails de c\xF4t\xE9.")},l(b){f=i(b,"P",{});var v=o(f);k=l(v,"\u26A0\uFE0F Notre impl\xE9mentation lancera une erreur s\u2019il y a un caract\xE8re inconnu puisque nous n\u2019avons rien fait pour les g\xE9rer. GPT-2 n\u2019a pas r\xE9ellement de "),m=i(v,"I",{});var E=o(m);$=l(E,"token"),E.forEach(e),w=l(v," inconnu (il est impossible d\u2019obtenir un caract\xE8re inconnu en utilisant le BPE au niveau de l\u2019octet) mais cela pourrait arriver ici car nous n\u2019avons pas inclus tous les octets possibles dans le vocabulaire initial. Cet aspect du BPE d\xE9passe le cadre de cette section, nous avons donc laiss\xE9 ces d\xE9tails de c\xF4t\xE9."),v.forEach(e)},m(b,v){p(b,f,v),n(f,k),n(f,m),n(m,$),n(f,w)},d(b){b&&e(f)}}}function ui(D){let f,k,m,$,w,b,v,E,N,P,U,O,z,xs,L,ne,ye,pt,rt,Pe,it,ot,Cn,qs,Tn,ss,An,Y,es,ze,bs,ut,Ce,ct,Nn,ae,ft,Dn,$s,Bn,F,mt,Te,ht,dt,Ae,jt,gt,Mn,ns,On,K,xt,Ne,qt,bt,De,$t,vt,Ln,I,_t,Be,kt,wt,Me,Et,yt,Oe,Pt,zt,In,te,Ct,Hn,vs,Sn,C,Tt,Le,At,Nt,Ie,Dt,Bt,He,Mt,Ot,Se,Lt,It,Ge,Ht,St,Re,Gt,Rt,Gn,_s,Rn,y,Vt,Ve,Ut,Ft,Ue,Kt,Jt,Fe,Wt,Yt,Ke,Zt,Qt,Je,Xt,sl,We,el,nl,Ye,al,tl,Vn,H,ll,Ze,pl,rl,Qe,il,ol,Xe,ul,cl,Un,ks,Fn,B,fl,sn,ml,hl,en,dl,jl,nn,gl,xl,an,ql,bl,Kn,ws,Jn,S,$l,tn,vl,_l,ln,kl,wl,pn,El,yl,Wn,Es,Yn,le,Pl,Zn,as,Qn,Z,ts,rn,ys,zl,on,Cl,Xn,pe,Tl,sa,G,un,Al,Nl,cn,Dl,Bl,fn,Ml,Ol,mn,Ll,ea,re,Il,na,Ps,aa,ie,Hl,ta,ls,la,Q,ps,hn,zs,Sl,dn,Gl,pa,oe,Rl,ra,ue,Vl,ia,Cs,oa,R,Ul,jn,Fl,Kl,gn,Jl,Wl,xn,Yl,Zl,ua,Ts,ca,ce,Ql,fa,As,ma,Ns,ha,fe,Xl,da,Ds,ja,Bs,ga,V,sp,qn,ep,np,bn,ap,tp,$n,lp,pp,xa,Ms,qa,me,rp,ba,Os,$a,he,ip,va,Ls,_a,de,op,ka,Is,wa,Hs,Ea,je,up,ya,Ss,Pa,Gs,za,J,cp,vn,fp,mp,_n,hp,dp,Ca,Rs,Ta,rs,jp,kn,gp,xp,Aa,Vs,Na,ge,qp,Da,Us,Ba,Fs,Ma,xe,bp,Oa,Ks,La,is,$p,wn,vp,_p,Ia,Js,Ha,Ws,Sa,os,kp,En,wp,Ep,Ga,Ys,Ra,Zs,Va,us,Ua,qe,yp,Fa,Qs,Ka,be,Pp,Ja,Xs,Wa,se,Ya,cs,Za,$e,zp,Qa;return b=new lt({}),z=new ai({props:{classNames:"absolute z-10 right-0 top-0",options:[{label:"Google Colab",value:"https://colab.research.google.com/github/huggingface/notebooks/blob/master/course/chapter6/section5.ipynb"},{label:"Aws Studio",value:"https://studiolab.sagemaker.aws/import/github/huggingface/notebooks/blob/master/course/chapter6/section5.ipynb"}]}}),qs=new ni({props:{id:"HEikzVL-lZU"}}),ss=new Ee({props:{$$slots:{default:[ti]},$$scope:{ctx:D}}}),bs=new lt({}),$s=new _({props:{code:'"hug", "pug", "pun", "bun", "hugs" # "c\xE2lin", "carlin", "jeu de mots", "brioche", "c\xE2lins"',highlighted:'<span class="hljs-string">&quot;hug&quot;</span>, <span class="hljs-string">&quot;pug&quot;</span>, <span class="hljs-string">&quot;pun&quot;</span>, <span class="hljs-string">&quot;bun&quot;</span>, <span class="hljs-string">&quot;hugs&quot;</span> <span class="hljs-meta"># <span class="hljs-string">&quot;c\xE2lin&quot;</span>, <span class="hljs-string">&quot;carlin&quot;</span>, <span class="hljs-string">&quot;jeu de mots&quot;</span>, <span class="hljs-string">&quot;brioche&quot;</span>, <span class="hljs-string">&quot;c\xE2lins&quot;</span></span>'}}),ns=new Ee({props:{$$slots:{default:[li]},$$scope:{ctx:D}}}),vs=new _({props:{code:'("hug", 10), ("pug", 5), ("pun", 12), ("bun", 4), ("hugs", 5)',highlighted:'(<span class="hljs-string">&quot;hug&quot;</span><span class="hljs-punctuation">,</span> <span class="hljs-number">10</span>)<span class="hljs-punctuation">,</span> (<span class="hljs-string">&quot;pug&quot;</span><span class="hljs-punctuation">,</span> <span class="hljs-number">5</span>)<span class="hljs-punctuation">,</span> (<span class="hljs-string">&quot;pun&quot;</span><span class="hljs-punctuation">,</span> <span class="hljs-number">12</span>)<span class="hljs-punctuation">,</span> (<span class="hljs-string">&quot;bun&quot;</span><span class="hljs-punctuation">,</span> <span class="hljs-number">4</span>)<span class="hljs-punctuation">,</span> (<span class="hljs-string">&quot;hugs&quot;</span><span class="hljs-punctuation">,</span> <span class="hljs-number">5</span>)'}}),_s=new _({props:{code:'("h" "u" "g", 10), ("p" "u" "g", 5), ("p" "u" "n", 12), ("b" "u" "n", 4), ("h" "u" "g" "s", 5)',highlighted:'(<span class="hljs-string">&quot;h&quot;</span> <span class="hljs-string">&quot;u&quot;</span> <span class="hljs-string">&quot;g&quot;</span><span class="hljs-punctuation">,</span> <span class="hljs-number">10</span>)<span class="hljs-punctuation">,</span> (<span class="hljs-string">&quot;p&quot;</span> <span class="hljs-string">&quot;u&quot;</span> <span class="hljs-string">&quot;g&quot;</span><span class="hljs-punctuation">,</span> <span class="hljs-number">5</span>)<span class="hljs-punctuation">,</span> (<span class="hljs-string">&quot;p&quot;</span> <span class="hljs-string">&quot;u&quot;</span> <span class="hljs-string">&quot;n&quot;</span><span class="hljs-punctuation">,</span> <span class="hljs-number">12</span>)<span class="hljs-punctuation">,</span> (<span class="hljs-string">&quot;b&quot;</span> <span class="hljs-string">&quot;u&quot;</span> <span class="hljs-string">&quot;n&quot;</span><span class="hljs-punctuation">,</span> <span class="hljs-number">4</span>)<span class="hljs-punctuation">,</span> (<span class="hljs-string">&quot;h&quot;</span> <span class="hljs-string">&quot;u&quot;</span> <span class="hljs-string">&quot;g&quot;</span> <span class="hljs-string">&quot;s&quot;</span><span class="hljs-punctuation">,</span> <span class="hljs-number">5</span>)'}}),ks=new _({props:{code:`Vocabulary: ["b", "g", "h", "n", "p", "s", "u", "ug"]
Corpus: ("h" "ug", 10), ("p" "ug", 5), ("p" "u" "n", 12), ("b" "u" "n", 4), ("h" "ug" "s", 5)`,highlighted:`<span class="hljs-symbol">Vocabulary:</span> [<span class="hljs-string">&quot;b&quot;</span>, <span class="hljs-string">&quot;g&quot;</span>, <span class="hljs-string">&quot;h&quot;</span>, <span class="hljs-string">&quot;n&quot;</span>, <span class="hljs-string">&quot;p&quot;</span>, <span class="hljs-string">&quot;s&quot;</span>, <span class="hljs-string">&quot;u&quot;</span>, <span class="hljs-string">&quot;ug&quot;</span>]
<span class="hljs-symbol">Corpus:</span> (<span class="hljs-string">&quot;h&quot;</span> <span class="hljs-string">&quot;ug&quot;</span>, <span class="hljs-number">10</span>), (<span class="hljs-string">&quot;p&quot;</span> <span class="hljs-string">&quot;ug&quot;</span>, <span class="hljs-number">5</span>), (<span class="hljs-string">&quot;p&quot;</span> <span class="hljs-string">&quot;u&quot;</span> <span class="hljs-string">&quot;n&quot;</span>, <span class="hljs-number">12</span>), (<span class="hljs-string">&quot;b&quot;</span> <span class="hljs-string">&quot;u&quot;</span> <span class="hljs-string">&quot;n&quot;</span>, <span class="hljs-number">4</span>), (<span class="hljs-string">&quot;h&quot;</span> <span class="hljs-string">&quot;ug&quot;</span> <span class="hljs-string">&quot;s&quot;</span>, <span class="hljs-number">5</span>)`}}),ws=new _({props:{code:`Vocabulary: ["b", "g", "h", "n", "p", "s", "u", "ug", "un"]
Corpus: ("h" "ug", 10), ("p" "ug", 5), ("p" "un", 12), ("b" "un", 4), ("h" "ug" "s", 5)`,highlighted:`<span class="hljs-symbol">Vocabulary:</span> [<span class="hljs-string">&quot;b&quot;</span>, <span class="hljs-string">&quot;g&quot;</span>, <span class="hljs-string">&quot;h&quot;</span>, <span class="hljs-string">&quot;n&quot;</span>, <span class="hljs-string">&quot;p&quot;</span>, <span class="hljs-string">&quot;s&quot;</span>, <span class="hljs-string">&quot;u&quot;</span>, <span class="hljs-string">&quot;ug&quot;</span>, <span class="hljs-string">&quot;un&quot;</span>]
<span class="hljs-symbol">Corpus:</span> (<span class="hljs-string">&quot;h&quot;</span> <span class="hljs-string">&quot;ug&quot;</span>, <span class="hljs-number">10</span>), (<span class="hljs-string">&quot;p&quot;</span> <span class="hljs-string">&quot;ug&quot;</span>, <span class="hljs-number">5</span>), (<span class="hljs-string">&quot;p&quot;</span> <span class="hljs-string">&quot;un&quot;</span>, <span class="hljs-number">12</span>), (<span class="hljs-string">&quot;b&quot;</span> <span class="hljs-string">&quot;un&quot;</span>, <span class="hljs-number">4</span>), (<span class="hljs-string">&quot;h&quot;</span> <span class="hljs-string">&quot;ug&quot;</span> <span class="hljs-string">&quot;s&quot;</span>, <span class="hljs-number">5</span>)`}}),Es=new _({props:{code:`Vocabulary: ["b", "g", "h", "n", "p", "s", "u", "ug", "un", "hug"]
Corpus: ("hug", 10), ("p" "ug", 5), ("p" "un", 12), ("b" "un", 4), ("hug" "s", 5)`,highlighted:`<span class="hljs-symbol">Vocabulary:</span> [<span class="hljs-string">&quot;b&quot;</span>, <span class="hljs-string">&quot;g&quot;</span>, <span class="hljs-string">&quot;h&quot;</span>, <span class="hljs-string">&quot;n&quot;</span>, <span class="hljs-string">&quot;p&quot;</span>, <span class="hljs-string">&quot;s&quot;</span>, <span class="hljs-string">&quot;u&quot;</span>, <span class="hljs-string">&quot;ug&quot;</span>, <span class="hljs-string">&quot;un&quot;</span>, <span class="hljs-string">&quot;hug&quot;</span>]
<span class="hljs-symbol">Corpus:</span> (<span class="hljs-string">&quot;hug&quot;</span>, <span class="hljs-number">10</span>), (<span class="hljs-string">&quot;p&quot;</span> <span class="hljs-string">&quot;ug&quot;</span>, <span class="hljs-number">5</span>), (<span class="hljs-string">&quot;p&quot;</span> <span class="hljs-string">&quot;un&quot;</span>, <span class="hljs-number">12</span>), (<span class="hljs-string">&quot;b&quot;</span> <span class="hljs-string">&quot;un&quot;</span>, <span class="hljs-number">4</span>), (<span class="hljs-string">&quot;hug&quot;</span> <span class="hljs-string">&quot;s&quot;</span>, <span class="hljs-number">5</span>)`}}),as=new Ee({props:{$$slots:{default:[pi]},$$scope:{ctx:D}}}),ys=new lt({}),Ps=new _({props:{code:`("u", "g") -> "ug"
("u", "n") -> "un"
("h", "ug") -> "hug"`,highlighted:`<span class="hljs-function"><span class="hljs-params">(<span class="hljs-string">&quot;u&quot;</span>, <span class="hljs-string">&quot;g&quot;</span>)</span> -&gt;</span> <span class="hljs-string">&quot;ug&quot;</span>
<span class="hljs-function"><span class="hljs-params">(<span class="hljs-string">&quot;u&quot;</span>, <span class="hljs-string">&quot;n&quot;</span>)</span> -&gt;</span> <span class="hljs-string">&quot;un&quot;</span>
<span class="hljs-function"><span class="hljs-params">(<span class="hljs-string">&quot;h&quot;</span>, <span class="hljs-string">&quot;ug&quot;</span>)</span> -&gt;</span> <span class="hljs-string">&quot;hug&quot;</span>`}}),ls=new Ee({props:{$$slots:{default:[ri]},$$scope:{ctx:D}}}),zs=new lt({}),Cs=new _({props:{code:`corpus = [
    "This is the Hugging Face Course.",
    # C'est le cours d'Hugging Face.
    "This chapter is about tokenization.",
    # Ce chapitre traite de la tokenisation.
    "This section shows several tokenizer algorithms.",
    # Cette section pr\xE9sente plusieurs algorithmes de tokenizer.
    "Hopefully, you will be able to understand how they are trained and generate tokens.",
    # Avec un peu de chance, vous serez en mesure de comprendre comment ils sont entra\xEEn\xE9s et g\xE9n\xE8rent des tokens.
]`,highlighted:`corpus = [
    <span class="hljs-string">&quot;This is the Hugging Face Course.&quot;</span>,
    <span class="hljs-comment"># C&#x27;est le cours d&#x27;Hugging Face.</span>
    <span class="hljs-string">&quot;This chapter is about tokenization.&quot;</span>,
    <span class="hljs-comment"># Ce chapitre traite de la tokenisation.</span>
    <span class="hljs-string">&quot;This section shows several tokenizer algorithms.&quot;</span>,
    <span class="hljs-comment"># Cette section pr\xE9sente plusieurs algorithmes de tokenizer.</span>
    <span class="hljs-string">&quot;Hopefully, you will be able to understand how they are trained and generate tokens.&quot;</span>,
    <span class="hljs-comment"># Avec un peu de chance, vous serez en mesure de comprendre comment ils sont entra\xEEn\xE9s et g\xE9n\xE8rent des tokens.</span>
]`}}),Ts=new _({props:{code:`from transformers import AutoTokenizer

tokenizer = AutoTokenizer.from_pretrained("gpt2")`,highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoTokenizer

tokenizer = AutoTokenizer.from_pretrained(<span class="hljs-string">&quot;gpt2&quot;</span>)`}}),As=new _({props:{code:`from collections import defaultdict

word_freqs = defaultdict(int)

for text in corpus:
    words_with_offsets = tokenizer.backend_tokenizer.pre_tokenizer.pre_tokenize_str(text)
    new_words = [word for word, offset in words_with_offsets]
    for word in new_words:
        word_freqs[word] += 1

print(word_freqs)`,highlighted:`<span class="hljs-keyword">from</span> collections <span class="hljs-keyword">import</span> defaultdict

word_freqs = defaultdict(<span class="hljs-built_in">int</span>)

<span class="hljs-keyword">for</span> text <span class="hljs-keyword">in</span> corpus:
    words_with_offsets = tokenizer.backend_tokenizer.pre_tokenizer.pre_tokenize_str(text)
    new_words = [word <span class="hljs-keyword">for</span> word, offset <span class="hljs-keyword">in</span> words_with_offsets]
    <span class="hljs-keyword">for</span> word <span class="hljs-keyword">in</span> new_words:
        word_freqs[word] += <span class="hljs-number">1</span>

<span class="hljs-built_in">print</span>(word_freqs)`}}),Ns=new _({props:{code:`defaultdict(int, {'This': 3, '\u0120is': 2, '\u0120the': 1, '\u0120Hugging': 1, '\u0120Face': 1, '\u0120Course': 1, '.': 4, '\u0120chapter': 1,
    '\u0120about': 1, '\u0120tokenization': 1, '\u0120section': 1, '\u0120shows': 1, '\u0120several': 1, '\u0120tokenizer': 1, '\u0120algorithms': 1,
    'Hopefully': 1, ',': 1, '\u0120you': 1, '\u0120will': 1, '\u0120be': 1, '\u0120able': 1, '\u0120to': 1, '\u0120understand': 1, '\u0120how': 1,
    '\u0120they': 1, '\u0120are': 1, '\u0120trained': 1, '\u0120and': 1, '\u0120generate': 1, '\u0120tokens': 1})`,highlighted:`defaultdict(<span class="hljs-built_in">int</span>, {<span class="hljs-string">&#x27;This&#x27;</span>: <span class="hljs-number">3</span>, <span class="hljs-string">&#x27;\u0120is&#x27;</span>: <span class="hljs-number">2</span>, <span class="hljs-string">&#x27;\u0120the&#x27;</span>: <span class="hljs-number">1</span>, <span class="hljs-string">&#x27;\u0120Hugging&#x27;</span>: <span class="hljs-number">1</span>, <span class="hljs-string">&#x27;\u0120Face&#x27;</span>: <span class="hljs-number">1</span>, <span class="hljs-string">&#x27;\u0120Course&#x27;</span>: <span class="hljs-number">1</span>, <span class="hljs-string">&#x27;.&#x27;</span>: <span class="hljs-number">4</span>, <span class="hljs-string">&#x27;\u0120chapter&#x27;</span>: <span class="hljs-number">1</span>,
    <span class="hljs-string">&#x27;\u0120about&#x27;</span>: <span class="hljs-number">1</span>, <span class="hljs-string">&#x27;\u0120tokenization&#x27;</span>: <span class="hljs-number">1</span>, <span class="hljs-string">&#x27;\u0120section&#x27;</span>: <span class="hljs-number">1</span>, <span class="hljs-string">&#x27;\u0120shows&#x27;</span>: <span class="hljs-number">1</span>, <span class="hljs-string">&#x27;\u0120several&#x27;</span>: <span class="hljs-number">1</span>, <span class="hljs-string">&#x27;\u0120tokenizer&#x27;</span>: <span class="hljs-number">1</span>, <span class="hljs-string">&#x27;\u0120algorithms&#x27;</span>: <span class="hljs-number">1</span>,
    <span class="hljs-string">&#x27;Hopefully&#x27;</span>: <span class="hljs-number">1</span>, <span class="hljs-string">&#x27;,&#x27;</span>: <span class="hljs-number">1</span>, <span class="hljs-string">&#x27;\u0120you&#x27;</span>: <span class="hljs-number">1</span>, <span class="hljs-string">&#x27;\u0120will&#x27;</span>: <span class="hljs-number">1</span>, <span class="hljs-string">&#x27;\u0120be&#x27;</span>: <span class="hljs-number">1</span>, <span class="hljs-string">&#x27;\u0120able&#x27;</span>: <span class="hljs-number">1</span>, <span class="hljs-string">&#x27;\u0120to&#x27;</span>: <span class="hljs-number">1</span>, <span class="hljs-string">&#x27;\u0120understand&#x27;</span>: <span class="hljs-number">1</span>, <span class="hljs-string">&#x27;\u0120how&#x27;</span>: <span class="hljs-number">1</span>,
    <span class="hljs-string">&#x27;\u0120they&#x27;</span>: <span class="hljs-number">1</span>, <span class="hljs-string">&#x27;\u0120are&#x27;</span>: <span class="hljs-number">1</span>, <span class="hljs-string">&#x27;\u0120trained&#x27;</span>: <span class="hljs-number">1</span>, <span class="hljs-string">&#x27;\u0120and&#x27;</span>: <span class="hljs-number">1</span>, <span class="hljs-string">&#x27;\u0120generate&#x27;</span>: <span class="hljs-number">1</span>, <span class="hljs-string">&#x27;\u0120tokens&#x27;</span>: <span class="hljs-number">1</span>})`}}),Ds=new _({props:{code:`alphabet = []

for word in word_freqs.keys():
    for letter in word:
        if letter not in alphabet:
            alphabet.append(letter)
alphabet.sort()

print(alphabet)`,highlighted:`alphabet = []

<span class="hljs-keyword">for</span> word <span class="hljs-keyword">in</span> word_freqs.keys():
    <span class="hljs-keyword">for</span> letter <span class="hljs-keyword">in</span> word:
        <span class="hljs-keyword">if</span> letter <span class="hljs-keyword">not</span> <span class="hljs-keyword">in</span> alphabet:
            alphabet.append(letter)
alphabet.sort()

<span class="hljs-built_in">print</span>(alphabet)`}}),Bs=new _({props:{code:`[ ',', '.', 'C', 'F', 'H', 'T', 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'k', 'l', 'm', 'n', 'o', 'p', 'r', 's',
  't', 'u', 'v', 'w', 'y', 'z', '\u0120']`,highlighted:`[ <span class="hljs-string">&#x27;,&#x27;</span>, <span class="hljs-string">&#x27;.&#x27;</span>, <span class="hljs-string">&#x27;C&#x27;</span>, <span class="hljs-string">&#x27;F&#x27;</span>, <span class="hljs-string">&#x27;H&#x27;</span>, <span class="hljs-string">&#x27;T&#x27;</span>, <span class="hljs-string">&#x27;a&#x27;</span>, <span class="hljs-string">&#x27;b&#x27;</span>, <span class="hljs-string">&#x27;c&#x27;</span>, <span class="hljs-string">&#x27;d&#x27;</span>, <span class="hljs-string">&#x27;e&#x27;</span>, <span class="hljs-string">&#x27;f&#x27;</span>, <span class="hljs-string">&#x27;g&#x27;</span>, <span class="hljs-string">&#x27;h&#x27;</span>, <span class="hljs-string">&#x27;i&#x27;</span>, <span class="hljs-string">&#x27;k&#x27;</span>, <span class="hljs-string">&#x27;l&#x27;</span>, <span class="hljs-string">&#x27;m&#x27;</span>, <span class="hljs-string">&#x27;n&#x27;</span>, <span class="hljs-string">&#x27;o&#x27;</span>, <span class="hljs-string">&#x27;p&#x27;</span>, <span class="hljs-string">&#x27;r&#x27;</span>, <span class="hljs-string">&#x27;s&#x27;</span>,
  <span class="hljs-string">&#x27;t&#x27;</span>, <span class="hljs-string">&#x27;u&#x27;</span>, <span class="hljs-string">&#x27;v&#x27;</span>, <span class="hljs-string">&#x27;w&#x27;</span>, <span class="hljs-string">&#x27;y&#x27;</span>, <span class="hljs-string">&#x27;z&#x27;</span>, <span class="hljs-string">&#x27;\u0120&#x27;</span>]`}}),Ms=new _({props:{code:'vocab = ["<|endoftext|>"] + alphabet.copy()',highlighted:'vocab = [<span class="hljs-string">&quot;&lt;|endoftext|&gt;&quot;</span>] + alphabet.copy()'}}),Os=new _({props:{code:"splits = {word: [c for c in word] for word in word_freqs.keys()}",highlighted:'splits = {word: [c <span class="hljs-keyword">for</span> c <span class="hljs-keyword">in</span> word] <span class="hljs-keyword">for</span> word <span class="hljs-keyword">in</span> word_freqs.keys()}'}}),Ls=new _({props:{code:`def compute_pair_freqs(splits):
    pair_freqs = defaultdict(int)
    for word, freq in word_freqs.items():
        split = splits[word]
        if len(split) == 1:
            continue
        for i in range(len(split) - 1):
            pair = (split[i], split[i + 1])
            pair_freqs[pair] += freq
    return pair_freqs`,highlighted:`<span class="hljs-keyword">def</span> <span class="hljs-title function_">compute_pair_freqs</span>(<span class="hljs-params">splits</span>):
    pair_freqs = defaultdict(<span class="hljs-built_in">int</span>)
    <span class="hljs-keyword">for</span> word, freq <span class="hljs-keyword">in</span> word_freqs.items():
        split = splits[word]
        <span class="hljs-keyword">if</span> <span class="hljs-built_in">len</span>(split) == <span class="hljs-number">1</span>:
            <span class="hljs-keyword">continue</span>
        <span class="hljs-keyword">for</span> i <span class="hljs-keyword">in</span> <span class="hljs-built_in">range</span>(<span class="hljs-built_in">len</span>(split) - <span class="hljs-number">1</span>):
            pair = (split[i], split[i + <span class="hljs-number">1</span>])
            pair_freqs[pair] += freq
    <span class="hljs-keyword">return</span> pair_freqs`}}),Is=new _({props:{code:`pair_freqs = compute_pair_freqs(splits)

for i, key in enumerate(pair_freqs.keys()):
    print(f"{key}: {pair_freqs[key]}")
    if i >= 5:
        break`,highlighted:`pair_freqs = compute_pair_freqs(splits)

<span class="hljs-keyword">for</span> i, key <span class="hljs-keyword">in</span> <span class="hljs-built_in">enumerate</span>(pair_freqs.keys()):
    <span class="hljs-built_in">print</span>(<span class="hljs-string">f&quot;<span class="hljs-subst">{key}</span>: <span class="hljs-subst">{pair_freqs[key]}</span>&quot;</span>)
    <span class="hljs-keyword">if</span> i &gt;= <span class="hljs-number">5</span>:
        <span class="hljs-keyword">break</span>`}}),Hs=new _({props:{code:`('T', 'h'): 3
('h', 'i'): 3
('i', 's'): 5
('\u0120', 'i'): 2
('\u0120', 't'): 7
('t', 'h'): 3`,highlighted:`(<span class="hljs-string">&#x27;T&#x27;</span>, <span class="hljs-string">&#x27;h&#x27;</span>): <span class="hljs-number">3</span>
(<span class="hljs-string">&#x27;h&#x27;</span>, <span class="hljs-string">&#x27;i&#x27;</span>): <span class="hljs-number">3</span>
(<span class="hljs-string">&#x27;i&#x27;</span>, <span class="hljs-string">&#x27;s&#x27;</span>): <span class="hljs-number">5</span>
(<span class="hljs-string">&#x27;\u0120&#x27;</span>, <span class="hljs-string">&#x27;i&#x27;</span>): <span class="hljs-number">2</span>
(<span class="hljs-string">&#x27;\u0120&#x27;</span>, <span class="hljs-string">&#x27;t&#x27;</span>): <span class="hljs-number">7</span>
(<span class="hljs-string">&#x27;t&#x27;</span>, <span class="hljs-string">&#x27;h&#x27;</span>): <span class="hljs-number">3</span>`}}),Ss=new _({props:{code:`best_pair = ""
max_freq = None

for pair, freq in pair_freqs.items():
    if max_freq is None or max_freq < freq:
        best_pair = pair
        max_freq = freq

print(best_pair, max_freq)`,highlighted:`best_pair = <span class="hljs-string">&quot;&quot;</span>
max_freq = <span class="hljs-literal">None</span>

<span class="hljs-keyword">for</span> pair, freq <span class="hljs-keyword">in</span> pair_freqs.items():
    <span class="hljs-keyword">if</span> max_freq <span class="hljs-keyword">is</span> <span class="hljs-literal">None</span> <span class="hljs-keyword">or</span> max_freq &lt; freq:
        best_pair = pair
        max_freq = freq

<span class="hljs-built_in">print</span>(best_pair, max_freq)`}}),Gs=new _({props:{code:"('\u0120', 't') 7",highlighted:'(<span class="hljs-string">&#x27;\u0120&#x27;</span>, <span class="hljs-string">&#x27;t&#x27;</span>) <span class="hljs-number">7</span>'}}),Rs=new _({props:{code:`merges = {("\u0120", "t"): "\u0120t"}
vocab.append("\u0120t")`,highlighted:`merges = {(<span class="hljs-string">&quot;\u0120&quot;</span>, <span class="hljs-string">&quot;t&quot;</span>): <span class="hljs-string">&quot;\u0120t&quot;</span>}
vocab.append(<span class="hljs-string">&quot;\u0120t&quot;</span>)`}}),Vs=new _({props:{code:`def merge_pair(a, b, splits):
    for word in word_freqs:
        split = splits[word]
        if len(split) == 1:
            continue

        i = 0
        while i < len(split) - 1:
            if split[i] == a and split[i + 1] == b:
                split = split[:i] + [a + b] + split[i + 2 :]
            else:
                i += 1
        splits[word] = split
    return splits`,highlighted:`<span class="hljs-keyword">def</span> <span class="hljs-title function_">merge_pair</span>(<span class="hljs-params">a, b, splits</span>):
    <span class="hljs-keyword">for</span> word <span class="hljs-keyword">in</span> word_freqs:
        split = splits[word]
        <span class="hljs-keyword">if</span> <span class="hljs-built_in">len</span>(split) == <span class="hljs-number">1</span>:
            <span class="hljs-keyword">continue</span>

        i = <span class="hljs-number">0</span>
        <span class="hljs-keyword">while</span> i &lt; <span class="hljs-built_in">len</span>(split) - <span class="hljs-number">1</span>:
            <span class="hljs-keyword">if</span> split[i] == a <span class="hljs-keyword">and</span> split[i + <span class="hljs-number">1</span>] == b:
                split = split[:i] + [a + b] + split[i + <span class="hljs-number">2</span> :]
            <span class="hljs-keyword">else</span>:
                i += <span class="hljs-number">1</span>
        splits[word] = split
    <span class="hljs-keyword">return</span> splits`}}),Us=new _({props:{code:`splits = merge_pair("\u0120", "t", splits)
print(splits["\u0120trained"])`,highlighted:`splits = merge_pair(<span class="hljs-string">&quot;\u0120&quot;</span>, <span class="hljs-string">&quot;t&quot;</span>, splits)
<span class="hljs-built_in">print</span>(splits[<span class="hljs-string">&quot;\u0120trained&quot;</span>])`}}),Fs=new _({props:{code:"['\u0120t', 'r', 'a', 'i', 'n', 'e', 'd']",highlighted:'[<span class="hljs-string">&#x27;\u0120t&#x27;</span>, <span class="hljs-string">&#x27;r&#x27;</span>, <span class="hljs-string">&#x27;a&#x27;</span>, <span class="hljs-string">&#x27;i&#x27;</span>, <span class="hljs-string">&#x27;n&#x27;</span>, <span class="hljs-string">&#x27;e&#x27;</span>, <span class="hljs-string">&#x27;d&#x27;</span>]'}}),Ks=new _({props:{code:`vocab_size = 50

while len(vocab) < vocab_size:
    pair_freqs = compute_pair_freqs(splits)
    best_pair = ""
    max_freq = None
    for pair, freq in pair_freqs.items():
        if max_freq is None or max_freq < freq:
            best_pair = pair
            max_freq = freq
    splits = merge_pair(*best_pair, splits)
    merges[best_pair] = best_pair[0] + best_pair[1]
    vocab.append(best_pair[0] + best_pair[1])`,highlighted:`vocab_size = <span class="hljs-number">50</span>

<span class="hljs-keyword">while</span> <span class="hljs-built_in">len</span>(vocab) &lt; vocab_size:
    pair_freqs = compute_pair_freqs(splits)
    best_pair = <span class="hljs-string">&quot;&quot;</span>
    max_freq = <span class="hljs-literal">None</span>
    <span class="hljs-keyword">for</span> pair, freq <span class="hljs-keyword">in</span> pair_freqs.items():
        <span class="hljs-keyword">if</span> max_freq <span class="hljs-keyword">is</span> <span class="hljs-literal">None</span> <span class="hljs-keyword">or</span> max_freq &lt; freq:
            best_pair = pair
            max_freq = freq
    splits = merge_pair(*best_pair, splits)
    merges[best_pair] = best_pair[<span class="hljs-number">0</span>] + best_pair[<span class="hljs-number">1</span>]
    vocab.append(best_pair[<span class="hljs-number">0</span>] + best_pair[<span class="hljs-number">1</span>])`}}),Js=new _({props:{code:"print(merges)",highlighted:'<span class="hljs-built_in">print</span>(merges)'}}),Ws=new _({props:{code:`{('\u0120', 't'): '\u0120t', ('i', 's'): 'is', ('e', 'r'): 'er', ('\u0120', 'a'): '\u0120a', ('\u0120t', 'o'): '\u0120to', ('e', 'n'): 'en',
 ('T', 'h'): 'Th', ('Th', 'is'): 'This', ('o', 'u'): 'ou', ('s', 'e'): 'se', ('\u0120to', 'k'): '\u0120tok',
 ('\u0120tok', 'en'): '\u0120token', ('n', 'd'): 'nd', ('\u0120', 'is'): '\u0120is', ('\u0120t', 'h'): '\u0120th', ('\u0120th', 'e'): '\u0120the',
 ('i', 'n'): 'in', ('\u0120a', 'b'): '\u0120ab', ('\u0120token', 'i'): '\u0120tokeni'}`,highlighted:`{(<span class="hljs-string">&#x27;\u0120&#x27;</span>, <span class="hljs-string">&#x27;t&#x27;</span>): <span class="hljs-string">&#x27;\u0120t&#x27;</span>, (<span class="hljs-string">&#x27;i&#x27;</span>, <span class="hljs-string">&#x27;s&#x27;</span>): <span class="hljs-string">&#x27;is&#x27;</span>, (<span class="hljs-string">&#x27;e&#x27;</span>, <span class="hljs-string">&#x27;r&#x27;</span>): <span class="hljs-string">&#x27;er&#x27;</span>, (<span class="hljs-string">&#x27;\u0120&#x27;</span>, <span class="hljs-string">&#x27;a&#x27;</span>): <span class="hljs-string">&#x27;\u0120a&#x27;</span>, (<span class="hljs-string">&#x27;\u0120t&#x27;</span>, <span class="hljs-string">&#x27;o&#x27;</span>): <span class="hljs-string">&#x27;\u0120to&#x27;</span>, (<span class="hljs-string">&#x27;e&#x27;</span>, <span class="hljs-string">&#x27;n&#x27;</span>): <span class="hljs-string">&#x27;en&#x27;</span>,
 (<span class="hljs-string">&#x27;T&#x27;</span>, <span class="hljs-string">&#x27;h&#x27;</span>): <span class="hljs-string">&#x27;Th&#x27;</span>, (<span class="hljs-string">&#x27;Th&#x27;</span>, <span class="hljs-string">&#x27;is&#x27;</span>): <span class="hljs-string">&#x27;This&#x27;</span>, (<span class="hljs-string">&#x27;o&#x27;</span>, <span class="hljs-string">&#x27;u&#x27;</span>): <span class="hljs-string">&#x27;ou&#x27;</span>, (<span class="hljs-string">&#x27;s&#x27;</span>, <span class="hljs-string">&#x27;e&#x27;</span>): <span class="hljs-string">&#x27;se&#x27;</span>, (<span class="hljs-string">&#x27;\u0120to&#x27;</span>, <span class="hljs-string">&#x27;k&#x27;</span>): <span class="hljs-string">&#x27;\u0120tok&#x27;</span>,
 (<span class="hljs-string">&#x27;\u0120tok&#x27;</span>, <span class="hljs-string">&#x27;en&#x27;</span>): <span class="hljs-string">&#x27;\u0120token&#x27;</span>, (<span class="hljs-string">&#x27;n&#x27;</span>, <span class="hljs-string">&#x27;d&#x27;</span>): <span class="hljs-string">&#x27;nd&#x27;</span>, (<span class="hljs-string">&#x27;\u0120&#x27;</span>, <span class="hljs-string">&#x27;is&#x27;</span>): <span class="hljs-string">&#x27;\u0120is&#x27;</span>, (<span class="hljs-string">&#x27;\u0120t&#x27;</span>, <span class="hljs-string">&#x27;h&#x27;</span>): <span class="hljs-string">&#x27;\u0120th&#x27;</span>, (<span class="hljs-string">&#x27;\u0120th&#x27;</span>, <span class="hljs-string">&#x27;e&#x27;</span>): <span class="hljs-string">&#x27;\u0120the&#x27;</span>,
 (<span class="hljs-string">&#x27;i&#x27;</span>, <span class="hljs-string">&#x27;n&#x27;</span>): <span class="hljs-string">&#x27;in&#x27;</span>, (<span class="hljs-string">&#x27;\u0120a&#x27;</span>, <span class="hljs-string">&#x27;b&#x27;</span>): <span class="hljs-string">&#x27;\u0120ab&#x27;</span>, (<span class="hljs-string">&#x27;\u0120token&#x27;</span>, <span class="hljs-string">&#x27;i&#x27;</span>): <span class="hljs-string">&#x27;\u0120tokeni&#x27;</span>}`}}),Ys=new _({props:{code:"print(vocab)",highlighted:'<span class="hljs-built_in">print</span>(vocab)'}}),Zs=new _({props:{code:`['<|endoftext|>', ',', '.', 'C', 'F', 'H', 'T', 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'k', 'l', 'm', 'n', 'o',
 'p', 'r', 's', 't', 'u', 'v', 'w', 'y', 'z', '\u0120', '\u0120t', 'is', 'er', '\u0120a', '\u0120to', 'en', 'Th', 'This', 'ou', 'se',
 '\u0120tok', '\u0120token', 'nd', '\u0120is', '\u0120th', '\u0120the', 'in', '\u0120ab', '\u0120tokeni']`,highlighted:`[<span class="hljs-string">&#x27;&lt;|endoftext|&gt;&#x27;</span>, <span class="hljs-string">&#x27;,&#x27;</span>, <span class="hljs-string">&#x27;.&#x27;</span>, <span class="hljs-string">&#x27;C&#x27;</span>, <span class="hljs-string">&#x27;F&#x27;</span>, <span class="hljs-string">&#x27;H&#x27;</span>, <span class="hljs-string">&#x27;T&#x27;</span>, <span class="hljs-string">&#x27;a&#x27;</span>, <span class="hljs-string">&#x27;b&#x27;</span>, <span class="hljs-string">&#x27;c&#x27;</span>, <span class="hljs-string">&#x27;d&#x27;</span>, <span class="hljs-string">&#x27;e&#x27;</span>, <span class="hljs-string">&#x27;f&#x27;</span>, <span class="hljs-string">&#x27;g&#x27;</span>, <span class="hljs-string">&#x27;h&#x27;</span>, <span class="hljs-string">&#x27;i&#x27;</span>, <span class="hljs-string">&#x27;k&#x27;</span>, <span class="hljs-string">&#x27;l&#x27;</span>, <span class="hljs-string">&#x27;m&#x27;</span>, <span class="hljs-string">&#x27;n&#x27;</span>, <span class="hljs-string">&#x27;o&#x27;</span>,
 <span class="hljs-string">&#x27;p&#x27;</span>, <span class="hljs-string">&#x27;r&#x27;</span>, <span class="hljs-string">&#x27;s&#x27;</span>, <span class="hljs-string">&#x27;t&#x27;</span>, <span class="hljs-string">&#x27;u&#x27;</span>, <span class="hljs-string">&#x27;v&#x27;</span>, <span class="hljs-string">&#x27;w&#x27;</span>, <span class="hljs-string">&#x27;y&#x27;</span>, <span class="hljs-string">&#x27;z&#x27;</span>, <span class="hljs-string">&#x27;\u0120&#x27;</span>, <span class="hljs-string">&#x27;\u0120t&#x27;</span>, <span class="hljs-string">&#x27;is&#x27;</span>, <span class="hljs-string">&#x27;er&#x27;</span>, <span class="hljs-string">&#x27;\u0120a&#x27;</span>, <span class="hljs-string">&#x27;\u0120to&#x27;</span>, <span class="hljs-string">&#x27;en&#x27;</span>, <span class="hljs-string">&#x27;Th&#x27;</span>, <span class="hljs-string">&#x27;This&#x27;</span>, <span class="hljs-string">&#x27;ou&#x27;</span>, <span class="hljs-string">&#x27;se&#x27;</span>,
 <span class="hljs-string">&#x27;\u0120tok&#x27;</span>, <span class="hljs-string">&#x27;\u0120token&#x27;</span>, <span class="hljs-string">&#x27;nd&#x27;</span>, <span class="hljs-string">&#x27;\u0120is&#x27;</span>, <span class="hljs-string">&#x27;\u0120th&#x27;</span>, <span class="hljs-string">&#x27;\u0120the&#x27;</span>, <span class="hljs-string">&#x27;in&#x27;</span>, <span class="hljs-string">&#x27;\u0120ab&#x27;</span>, <span class="hljs-string">&#x27;\u0120tokeni&#x27;</span>]`}}),us=new Ee({props:{$$slots:{default:[ii]},$$scope:{ctx:D}}}),Qs=new _({props:{code:`def tokenize(text):
    pre_tokenize_result = tokenizer._tokenizer.pre_tokenizer.pre_tokenize_str(text)
    pre_tokenized_text = [word for word, offset in pre_tokenize_result]
    splits = [[l for l in word] for word in pre_tokenized_text]
    for pair, merge in merges.items():
        for idx, split in enumerate(splits):
            i = 0
            while i < len(split) - 1:
                if split[i] == pair[0] and split[i + 1] == pair[1]:
                    split = split[:i] + [merge] + split[i + 2 :]
                else:
                    i += 1
            splits[idx] = split

    return sum(splits, [])`,highlighted:`<span class="hljs-keyword">def</span> <span class="hljs-title function_">tokenize</span>(<span class="hljs-params">text</span>):
    pre_tokenize_result = tokenizer._tokenizer.pre_tokenizer.pre_tokenize_str(text)
    pre_tokenized_text = [word <span class="hljs-keyword">for</span> word, offset <span class="hljs-keyword">in</span> pre_tokenize_result]
    splits = [[l <span class="hljs-keyword">for</span> l <span class="hljs-keyword">in</span> word] <span class="hljs-keyword">for</span> word <span class="hljs-keyword">in</span> pre_tokenized_text]
    <span class="hljs-keyword">for</span> pair, merge <span class="hljs-keyword">in</span> merges.items():
        <span class="hljs-keyword">for</span> idx, split <span class="hljs-keyword">in</span> <span class="hljs-built_in">enumerate</span>(splits):
            i = <span class="hljs-number">0</span>
            <span class="hljs-keyword">while</span> i &lt; <span class="hljs-built_in">len</span>(split) - <span class="hljs-number">1</span>:
                <span class="hljs-keyword">if</span> split[i] == pair[<span class="hljs-number">0</span>] <span class="hljs-keyword">and</span> split[i + <span class="hljs-number">1</span>] == pair[<span class="hljs-number">1</span>]:
                    split = split[:i] + [merge] + split[i + <span class="hljs-number">2</span> :]
                <span class="hljs-keyword">else</span>:
                    i += <span class="hljs-number">1</span>
            splits[idx] = split

    <span class="hljs-keyword">return</span> <span class="hljs-built_in">sum</span>(splits, [])`}}),Xs=new _({props:{code:'tokenize("This is not a token.")',highlighted:'tokenize(<span class="hljs-string">&quot;This is not a token.&quot;</span>)'}}),se=new _({props:{code:"['This', '\u0120is', '\u0120', 'n', 'o', 't', '\u0120a', '\u0120token', '.']",highlighted:'[<span class="hljs-string">&#x27;This&#x27;</span>, <span class="hljs-string">&#x27;\u0120is&#x27;</span>, <span class="hljs-string">&#x27;\u0120&#x27;</span>, <span class="hljs-string">&#x27;n&#x27;</span>, <span class="hljs-string">&#x27;o&#x27;</span>, <span class="hljs-string">&#x27;t&#x27;</span>, <span class="hljs-string">&#x27;\u0120a&#x27;</span>, <span class="hljs-string">&#x27;\u0120token&#x27;</span>, <span class="hljs-string">&#x27;.&#x27;</span>]'}}),cs=new Ee({props:{warning:!0,$$slots:{default:[oi]},$$scope:{ctx:D}}}),{c(){f=r("meta"),k=u(),m=r("h1"),$=r("a"),w=r("span"),h(b.$$.fragment),v=u(),E=r("span"),N=t("Tok\xE9nisation "),P=r("i"),U=t("Byte-Pair Encoding"),O=u(),h(z.$$.fragment),xs=u(),L=r("p"),ne=t("Le "),ye=r("em"),pt=t("Byte-Pair Encoding"),rt=t(" (BPE) a \xE9t\xE9 initialement d\xE9velopp\xE9 en tant qu\u2019algorithme de compression de textes puis utilis\xE9 par OpenAI pour la tokenisation du pr\xE9-entra\xEEnement du mod\xE8le GPT. Il est utilis\xE9 par de nombreux "),Pe=r("em"),it=t("transformers"),ot=t(" dont GPT, GPT-2, RoBERTa, BART et DeBERTa."),Cn=u(),h(qs.$$.fragment),Tn=u(),h(ss.$$.fragment),An=u(),Y=r("h2"),es=r("a"),ze=r("span"),h(bs.$$.fragment),ut=u(),Ce=r("span"),ct=t("Algorithme d'entra\xEEnement"),Nn=u(),ae=r("p"),ft=t("L\u2019entra\xEEnement du BPE commence par le calcul de l\u2019unique ensemble de mots utilis\xE9s dans le corpus (apr\xE8s les \xE9tapes de normalisation et de pr\xE9tok\xE9nisation), puis la construction du vocabulaire en prenant tous les symboles utilis\xE9s pour \xE9crire ces mots. A titre d\u2019exemple, disons que notre corpus utilise ces cinq mots :"),Dn=u(),h($s.$$.fragment),Bn=u(),F=r("p"),mt=t("Le vocabulaire de base sera alors "),Te=r("code"),ht=t('["b", "g", "h", "n", "p", "s", "u"]'),dt=t(". Dans le monde r\xE9el, le vocabulaire de base contient au moins tous les caract\xE8res ASCII et probablement aussi quelques caract\xE8res Unicode. Si un exemple que vous tokenisez utilise un caract\xE8re qui n\u2019est pas dans le corpus d\u2019entra\xEEnement, ce caract\xE8re est converti en "),Ae=r("em"),jt=t("token"),gt=t(" inconnu. C\u2019est l\u2019une des raisons pour lesquelles de nombreux mod\xE8les de NLP sont par exemple tr\xE8s mauvais dans l\u2019analyse de contenus contenant des emojis."),Mn=u(),h(ns.$$.fragment),On=u(),K=r("p"),xt=t("Apr\xE8s avoir obtenu ce vocabulaire de base, nous ajoutons de nouveaux "),Ne=r("em"),qt=t("tokens"),bt=t(" jusqu\u2019\xE0 ce que la taille souhait\xE9e du vocabulaire soit atteinte en apprenant les fusions qui sont des r\xE8gles permettant de fusionner deux \xE9l\xE9ments du vocabulaire existant pour en cr\xE9er un nouveau. Ainsi, au d\xE9but, ces fusions cr\xE9eront des "),De=r("em"),$t=t("tokens"),vt=t(" de deux caract\xE8res, puis au fur et \xE0 mesure de l\u2019entra\xEEnement, des sous-mots plus longs."),Ln=u(),I=r("p"),_t=t("\xC0 chaque \xE9tape de l\u2019entra\xEEnement du "),Be=r("em"),kt=t("tokenizer"),wt=t(", l\u2019algorithme BPE recherche la paire la plus fr\xE9quente de "),Me=r("em"),Et=t("tokens"),yt=t(" existants (par \xAB paire \xBB, nous entendons ici deux "),Oe=r("em"),Pt=t("tokens"),zt=t(" cons\xE9cutifs dans un mot). Cette paire la plus fr\xE9quente est celle qui sera fusionn\xE9e. Nous rin\xE7ons et r\xE9p\xE9tons pour l\u2019\xE9tape suivante."),In=u(),te=r("p"),Ct=t("Pour revenir \xE0 notre exemple pr\xE9c\xE9dent, supposons que les mots ont les fr\xE9quences suivantes :"),Hn=u(),h(vs.$$.fragment),Sn=u(),C=r("p"),Tt=t("ce qui veut dire que "),Le=r("code"),At=t('"hug"'),Nt=t(" \xE9tait pr\xE9sent 10 fois dans le corpus, "),Ie=r("code"),Dt=t('"pug"'),Bt=t(" 5 fois, "),He=r("code"),Mt=t('"pun"'),Ot=t(" 12 fois, "),Se=r("code"),Lt=t('"bun"'),It=t(" 4 fois et "),Ge=r("code"),Ht=t('"hugs"'),St=t("\u201D 5 fois. Nous commen\xE7ons l\u2019entra\xEEnement en divisant chaque mot en caract\xE8res (ceux qui forment notre vocabulaire initial) afin de voir chaque mot comme une liste de "),Re=r("em"),Gt=t("tokens"),Rt=t(" :"),Gn=u(),h(_s.$$.fragment),Rn=u(),y=r("p"),Vt=t("Ensuite, nous regardons les paires. La paire "),Ve=r("code"),Ut=t('("h", "u")'),Ft=t(" est pr\xE9sente dans les mots "),Ue=r("code"),Kt=t('"hug"'),Jt=t(" et "),Fe=r("code"),Wt=t('"hugs"'),Yt=t(", donc 15 fois au total dans le corpus. Ce n\u2019est cependant pas la paire la plus fr\xE9quente. Cet honneur revient \xE0 "),Ke=r("code"),Zt=t('("u", "g")'),Qt=t(" qui est pr\xE9sent dans "),Je=r("code"),Xt=t('"hug"'),sl=t(", "),We=r("code"),el=t('"pug"'),nl=t(", et "),Ye=r("code"),al=t('"hugs"'),tl=t(", pour un total de 20 fois dans le vocabulaire."),Vn=u(),H=r("p"),ll=t("Ainsi, la premi\xE8re r\xE8gle de fusion apprise par le "),Ze=r("em"),pl=t("tokenizer"),rl=t(" est "),Qe=r("code"),il=t('("u", "g") -> "ug"'),ol=t(", ce qui signifie que "),Xe=r("code"),ul=t('"ug"'),cl=t(" est ajout\xE9 au vocabulaire et que la paire doit \xEAtre fusionn\xE9e dans tous les mots du corpus. A la fin de cette \xE9tape, le vocabulaire et le corpus ressemblent \xE0 ceci :"),Un=u(),h(ks.$$.fragment),Fn=u(),B=r("p"),fl=t("Nous avons maintenant quelques paires qui aboutissent \xE0 un "),sn=r("em"),ml=t("token"),hl=t(" de plus de deux caract\xE8res. Par exemple la paire "),en=r("code"),dl=t('("h", "ug")'),jl=t(" pr\xE9sente 15 fois dans le corpus. La paire la plus fr\xE9quente \xE0 ce stade est "),nn=r("code"),gl=t('("u", "n")'),xl=t(", pr\xE9sente 16 fois dans le corpus, donc la deuxi\xE8me r\xE8gle de fusion apprise est "),an=r("code"),ql=t('("u", "n") -> "un"'),bl=t(". En ajoutant cela au vocabulaire et en fusionnant toutes les occurrences existantes, nous obtenons :"),Kn=u(),h(ws.$$.fragment),Jn=u(),S=r("p"),$l=t("Maintenant la paire la plus fr\xE9quente est "),tn=r("code"),vl=t('("h", "ug")'),_l=t(" donc nous apprenons la r\xE8gle de fusion "),ln=r("code"),kl=t('("h", "ug") -> "hug"'),wl=t(". Cela nous donne donc notre premier "),pn=r("em"),El=t("token"),yl=t(" de trois lettres. Apr\xE8s la fusion, le corpus ressemble \xE0 ceci :"),Wn=u(),h(Es.$$.fragment),Yn=u(),le=r("p"),Pl=t("Et nous continuons ainsi jusqu\u2019\xE0 ce que nous atteignions la taille de vocabulaire souhait\xE9e."),Zn=u(),h(as.$$.fragment),Qn=u(),Z=r("h2"),ts=r("a"),rn=r("span"),h(ys.$$.fragment),zl=u(),on=r("span"),Cl=t("Algorithme de tokenisation"),Xn=u(),pe=r("p"),Tl=t("La tokenisation suit de pr\xE8s le processus d\u2019entra\xEEnement, dans le sens o\xF9 les nouvelles entr\xE9es sont tokenis\xE9es en appliquant les \xE9tapes suivantes :"),sa=u(),G=r("ol"),un=r("li"),Al=t("Normalisation"),Nl=u(),cn=r("li"),Dl=t("Pr\xE9tok\xE9nisation"),Bl=u(),fn=r("li"),Ml=t("D\xE9coupage des mots en caract\xE8res individuels"),Ol=u(),mn=r("li"),Ll=t("Application des r\xE8gles de fusion apprises dans l\u2019ordre sur ces divisions."),ea=u(),re=r("p"),Il=t("Prenons l\u2019exemple que nous avons utilis\xE9 pendant l\u2019entra\xEEnement, avec les trois r\xE8gles de fusion apprises :"),na=u(),h(Ps.$$.fragment),aa=u(),ie=r("p"),Hl=t("Le mot \xAB bug \xBB  sera traduit par \xAB [\u201Cb\u201D, \u201Cug\u201D] \xBB. Par contre, le mot \xAB mug \xBB (tasse en fran\xE7ais) sera traduit par \xAB [\u201D[UNK]\u201D, \u201Cug\u201D] \xBB puisque la lettre \xAB m \xBB ne fait pas partie du vocabulaire de base. De la m\xEAme fa\xE7on, le mot \xAB thug \xBB (voyou en fran\xE7ais) sera tokenis\xE9 en \xAB [\u201D[UNK]\u201D, \u201Chug\u201D] \xBB car la lettre \xAB t \xBB n\u2019est pas dans le vocabulaire de base et l\u2019application des r\xE8gles de fusion r\xE9sulte d\u2019abord en la fusion de \xAB u \xBB et \xAB g \xBB et ensuite en la fusion de \xAB hu \xBB et \xAB g \xBB."),ta=u(),h(ls.$$.fragment),la=u(),Q=r("h2"),ps=r("a"),hn=r("span"),h(zs.$$.fragment),Sl=u(),dn=r("span"),Gl=t("Impl\xE9mentation du BPE"),pa=u(),oe=r("p"),Rl=t("Voyons maintenant une impl\xE9mentation de l\u2019algorithme BPE. Il ne s\u2019agira pas d\u2019une version optimis\xE9e que vous pourrez utiliser sur un grand corpus. Nous voulons simplement vous montrer le code afin que vous puissiez comprendre un peu mieux l\u2019algorithme."),ra=u(),ue=r("p"),Vl=t("Tout d\u2019abord, nous avons besoin d\u2019un corpus, alors cr\xE9ons un corpus simple avec quelques phrases :"),ia=u(),h(Cs.$$.fragment),oa=u(),R=r("p"),Ul=t("Ensuite, nous devons pr\xE9tokeniser ce corpus en mots. Puisque nous r\xE9pliquons un "),jn=r("em"),Fl=t("tokenizer"),Kl=t(" BPE (comme celui du GPT-2), nous utiliserons le "),gn=r("em"),Jl=t("tokenizer"),Wl=u(),xn=r("code"),Yl=t("gpt2"),Zl=t(" pour la pr\xE9tok\xE9nisation :"),ua=u(),h(Ts.$$.fragment),ca=u(),ce=r("p"),Ql=t("Ensuite, nous calculons les fr\xE9quences de chaque mot dans le corpus comme nous le faisons pour la pr\xE9tok\xE9nisation :"),fa=u(),h(As.$$.fragment),ma=u(),h(Ns.$$.fragment),ha=u(),fe=r("p"),Xl=t("L\u2019\xE9tape suivante consiste \xE0 calculer le vocabulaire de base, form\xE9 par tous les caract\xE8res utilis\xE9s dans le corpus :"),da=u(),h(Ds.$$.fragment),ja=u(),h(Bs.$$.fragment),ga=u(),V=r("p"),sp=t("Nous ajoutons \xE9galement les "),qn=r("em"),ep=t("tokens"),np=t(" sp\xE9ciaux utilis\xE9s par le mod\xE8le au d\xE9but de ce vocabulaire. Dans le cas du GPT-2, le seul "),bn=r("em"),ap=t("token"),tp=t(" sp\xE9cial est "),$n=r("code"),lp=t('"<|endoftext|>"'),pp=t(" :"),xa=u(),h(Ms.$$.fragment),qa=u(),me=r("p"),rp=t("Nous devons maintenant diviser chaque mot en caract\xE8res individuels pour pouvoir commencer l\u2019entra\xEEnement :"),ba=u(),h(Os.$$.fragment),$a=u(),he=r("p"),ip=t("Maintenant que nous sommes pr\xEAts pour l\u2019entra\xEEnement, \xE9crivons une fonction qui calcule la fr\xE9quence de chaque paire. Nous devrons l\u2019utiliser \xE0 chaque \xE9tape de l\u2019entra\xEEnement :"),va=u(),h(Ls.$$.fragment),_a=u(),de=r("p"),op=t("Jetons un coup d\u2019\u0153il \xE0 une partie de ce dictionnaire apr\xE8s les premi\xE8res divisions :"),ka=u(),h(Is.$$.fragment),wa=u(),h(Hs.$$.fragment),Ea=u(),je=r("p"),up=t("Maintenant, trouver la paire la plus fr\xE9quente ne demande qu\u2019une rapide boucle :"),ya=u(),h(Ss.$$.fragment),Pa=u(),h(Gs.$$.fragment),za=u(),J=r("p"),cp=t("Donc la premi\xE8re fusion \xE0 apprendre est "),vn=r("code"),fp=t("('\u0120', 't') -> '\u0120t'"),mp=t(", et on ajoute "),_n=r("code"),hp=t("'\u0120t'"),dp=t(" au vocabulaire :"),Ca=u(),h(Rs.$$.fragment),Ta=u(),rs=r("p"),jp=t("Pour continuer, nous devons appliquer cette fusion dans notre dictionnaire "),kn=r("code"),gp=t("splits"),xp=t(". \xC9crivons une autre fonction pour cela :"),Aa=u(),h(Vs.$$.fragment),Na=u(),ge=r("p"),qp=t("Et nous pouvons regarder le r\xE9sultat de la premi\xE8re fusion :"),Da=u(),h(Us.$$.fragment),Ba=u(),h(Fs.$$.fragment),Ma=u(),xe=r("p"),bp=t("Maintenant, nous avons tout ce dont nous avons besoin pour boucler jusqu\u2019\xE0 ce que nous ayons appris toutes les fusions que nous voulons. Visons une taille de vocabulaire de 50 :"),Oa=u(),h(Ks.$$.fragment),La=u(),is=r("p"),$p=t("En cons\xE9quence, nous avons appris 19 r\xE8gles de fusion (le vocabulaire initial avait une taille de 31 : 30 caract\xE8res dans l\u2019alphabet plus le "),wn=r("em"),vp=t("token"),_p=t(" sp\xE9cial) :"),Ia=u(),h(Js.$$.fragment),Ha=u(),h(Ws.$$.fragment),Sa=u(),os=r("p"),kp=t("Et le vocabulaire est compos\xE9 du "),En=r("em"),wp=t("token"),Ep=t(" sp\xE9cial, de l\u2019alphabet initial, et de tous les r\xE9sultats des fusions :"),Ga=u(),h(Ys.$$.fragment),Ra=u(),h(Zs.$$.fragment),Va=u(),h(us.$$.fragment),Ua=u(),qe=r("p"),yp=t("Pour tokeniser un nouveau texte, on le pr\xE9tokenise, on le divise, puis on applique toutes les r\xE8gles de fusion apprises :"),Fa=u(),h(Qs.$$.fragment),Ka=u(),be=r("p"),Pp=t("Nous pouvons essayer cela sur n\u2019importe quel texte compos\xE9 de caract\xE8res de l\u2019alphabet :"),Ja=u(),h(Xs.$$.fragment),Wa=u(),h(se.$$.fragment),Ya=u(),h(cs.$$.fragment),Za=u(),$e=r("p"),zp=t("C\u2019est tout pour l\u2019algorithme BPE ! Nous allons nous int\xE9resser \xE0 WordPiece dans la suite."),this.h()},l(s){const a=si('[data-svelte="svelte-1phssyn"]',document.head);f=i(a,"META",{name:!0,content:!0}),a.forEach(e),k=c(s),m=i(s,"H1",{class:!0});var ee=o(m);$=i(ee,"A",{id:!0,class:!0,href:!0});var yn=o($);w=i(yn,"SPAN",{});var Pn=o(w);d(b.$$.fragment,Pn),Pn.forEach(e),yn.forEach(e),v=c(ee),E=i(ee,"SPAN",{});var ve=o(E);N=l(ve,"Tok\xE9nisation "),P=i(ve,"I",{});var zn=o(P);U=l(zn,"Byte-Pair Encoding"),zn.forEach(e),ve.forEach(e),ee.forEach(e),O=c(s),d(z.$$.fragment,s),xs=c(s),L=i(s,"P",{});var X=o(L);ne=l(X,"Le "),ye=i(X,"EM",{});var Cp=o(ye);pt=l(Cp,"Byte-Pair Encoding"),Cp.forEach(e),rt=l(X," (BPE) a \xE9t\xE9 initialement d\xE9velopp\xE9 en tant qu\u2019algorithme de compression de textes puis utilis\xE9 par OpenAI pour la tokenisation du pr\xE9-entra\xEEnement du mod\xE8le GPT. Il est utilis\xE9 par de nombreux "),Pe=i(X,"EM",{});var Tp=o(Pe);it=l(Tp,"transformers"),Tp.forEach(e),ot=l(X," dont GPT, GPT-2, RoBERTa, BART et DeBERTa."),X.forEach(e),Cn=c(s),d(qs.$$.fragment,s),Tn=c(s),d(ss.$$.fragment,s),An=c(s),Y=i(s,"H2",{class:!0});var Xa=o(Y);es=i(Xa,"A",{id:!0,class:!0,href:!0});var Ap=o(es);ze=i(Ap,"SPAN",{});var Np=o(ze);d(bs.$$.fragment,Np),Np.forEach(e),Ap.forEach(e),ut=c(Xa),Ce=i(Xa,"SPAN",{});var Dp=o(Ce);ct=l(Dp,"Algorithme d'entra\xEEnement"),Dp.forEach(e),Xa.forEach(e),Nn=c(s),ae=i(s,"P",{});var Bp=o(ae);ft=l(Bp,"L\u2019entra\xEEnement du BPE commence par le calcul de l\u2019unique ensemble de mots utilis\xE9s dans le corpus (apr\xE8s les \xE9tapes de normalisation et de pr\xE9tok\xE9nisation), puis la construction du vocabulaire en prenant tous les symboles utilis\xE9s pour \xE9crire ces mots. A titre d\u2019exemple, disons que notre corpus utilise ces cinq mots :"),Bp.forEach(e),Dn=c(s),d($s.$$.fragment,s),Bn=c(s),F=i(s,"P",{});var _e=o(F);mt=l(_e,"Le vocabulaire de base sera alors "),Te=i(_e,"CODE",{});var Mp=o(Te);ht=l(Mp,'["b", "g", "h", "n", "p", "s", "u"]'),Mp.forEach(e),dt=l(_e,". Dans le monde r\xE9el, le vocabulaire de base contient au moins tous les caract\xE8res ASCII et probablement aussi quelques caract\xE8res Unicode. Si un exemple que vous tokenisez utilise un caract\xE8re qui n\u2019est pas dans le corpus d\u2019entra\xEEnement, ce caract\xE8re est converti en "),Ae=i(_e,"EM",{});var Op=o(Ae);jt=l(Op,"token"),Op.forEach(e),gt=l(_e," inconnu. C\u2019est l\u2019une des raisons pour lesquelles de nombreux mod\xE8les de NLP sont par exemple tr\xE8s mauvais dans l\u2019analyse de contenus contenant des emojis."),_e.forEach(e),Mn=c(s),d(ns.$$.fragment,s),On=c(s),K=i(s,"P",{});var ke=o(K);xt=l(ke,"Apr\xE8s avoir obtenu ce vocabulaire de base, nous ajoutons de nouveaux "),Ne=i(ke,"EM",{});var Lp=o(Ne);qt=l(Lp,"tokens"),Lp.forEach(e),bt=l(ke," jusqu\u2019\xE0 ce que la taille souhait\xE9e du vocabulaire soit atteinte en apprenant les fusions qui sont des r\xE8gles permettant de fusionner deux \xE9l\xE9ments du vocabulaire existant pour en cr\xE9er un nouveau. Ainsi, au d\xE9but, ces fusions cr\xE9eront des "),De=i(ke,"EM",{});var Ip=o(De);$t=l(Ip,"tokens"),Ip.forEach(e),vt=l(ke," de deux caract\xE8res, puis au fur et \xE0 mesure de l\u2019entra\xEEnement, des sous-mots plus longs."),ke.forEach(e),Ln=c(s),I=i(s,"P",{});var fs=o(I);_t=l(fs,"\xC0 chaque \xE9tape de l\u2019entra\xEEnement du "),Be=i(fs,"EM",{});var Hp=o(Be);kt=l(Hp,"tokenizer"),Hp.forEach(e),wt=l(fs,", l\u2019algorithme BPE recherche la paire la plus fr\xE9quente de "),Me=i(fs,"EM",{});var Sp=o(Me);Et=l(Sp,"tokens"),Sp.forEach(e),yt=l(fs," existants (par \xAB paire \xBB, nous entendons ici deux "),Oe=i(fs,"EM",{});var Gp=o(Oe);Pt=l(Gp,"tokens"),Gp.forEach(e),zt=l(fs," cons\xE9cutifs dans un mot). Cette paire la plus fr\xE9quente est celle qui sera fusionn\xE9e. Nous rin\xE7ons et r\xE9p\xE9tons pour l\u2019\xE9tape suivante."),fs.forEach(e),In=c(s),te=i(s,"P",{});var Rp=o(te);Ct=l(Rp,"Pour revenir \xE0 notre exemple pr\xE9c\xE9dent, supposons que les mots ont les fr\xE9quences suivantes :"),Rp.forEach(e),Hn=c(s),d(vs.$$.fragment,s),Sn=c(s),C=i(s,"P",{});var M=o(C);Tt=l(M,"ce qui veut dire que "),Le=i(M,"CODE",{});var Vp=o(Le);At=l(Vp,'"hug"'),Vp.forEach(e),Nt=l(M," \xE9tait pr\xE9sent 10 fois dans le corpus, "),Ie=i(M,"CODE",{});var Up=o(Ie);Dt=l(Up,'"pug"'),Up.forEach(e),Bt=l(M," 5 fois, "),He=i(M,"CODE",{});var Fp=o(He);Mt=l(Fp,'"pun"'),Fp.forEach(e),Ot=l(M," 12 fois, "),Se=i(M,"CODE",{});var Kp=o(Se);Lt=l(Kp,'"bun"'),Kp.forEach(e),It=l(M," 4 fois et "),Ge=i(M,"CODE",{});var Jp=o(Ge);Ht=l(Jp,'"hugs"'),Jp.forEach(e),St=l(M,"\u201D 5 fois. Nous commen\xE7ons l\u2019entra\xEEnement en divisant chaque mot en caract\xE8res (ceux qui forment notre vocabulaire initial) afin de voir chaque mot comme une liste de "),Re=i(M,"EM",{});var Wp=o(Re);Gt=l(Wp,"tokens"),Wp.forEach(e),Rt=l(M," :"),M.forEach(e),Gn=c(s),d(_s.$$.fragment,s),Rn=c(s),y=i(s,"P",{});var A=o(y);Vt=l(A,"Ensuite, nous regardons les paires. La paire "),Ve=i(A,"CODE",{});var Yp=o(Ve);Ut=l(Yp,'("h", "u")'),Yp.forEach(e),Ft=l(A," est pr\xE9sente dans les mots "),Ue=i(A,"CODE",{});var Zp=o(Ue);Kt=l(Zp,'"hug"'),Zp.forEach(e),Jt=l(A," et "),Fe=i(A,"CODE",{});var Qp=o(Fe);Wt=l(Qp,'"hugs"'),Qp.forEach(e),Yt=l(A,", donc 15 fois au total dans le corpus. Ce n\u2019est cependant pas la paire la plus fr\xE9quente. Cet honneur revient \xE0 "),Ke=i(A,"CODE",{});var Xp=o(Ke);Zt=l(Xp,'("u", "g")'),Xp.forEach(e),Qt=l(A," qui est pr\xE9sent dans "),Je=i(A,"CODE",{});var sr=o(Je);Xt=l(sr,'"hug"'),sr.forEach(e),sl=l(A,", "),We=i(A,"CODE",{});var er=o(We);el=l(er,'"pug"'),er.forEach(e),nl=l(A,", et "),Ye=i(A,"CODE",{});var nr=o(Ye);al=l(nr,'"hugs"'),nr.forEach(e),tl=l(A,", pour un total de 20 fois dans le vocabulaire."),A.forEach(e),Vn=c(s),H=i(s,"P",{});var ms=o(H);ll=l(ms,"Ainsi, la premi\xE8re r\xE8gle de fusion apprise par le "),Ze=i(ms,"EM",{});var ar=o(Ze);pl=l(ar,"tokenizer"),ar.forEach(e),rl=l(ms," est "),Qe=i(ms,"CODE",{});var tr=o(Qe);il=l(tr,'("u", "g") -> "ug"'),tr.forEach(e),ol=l(ms,", ce qui signifie que "),Xe=i(ms,"CODE",{});var lr=o(Xe);ul=l(lr,'"ug"'),lr.forEach(e),cl=l(ms," est ajout\xE9 au vocabulaire et que la paire doit \xEAtre fusionn\xE9e dans tous les mots du corpus. A la fin de cette \xE9tape, le vocabulaire et le corpus ressemblent \xE0 ceci :"),ms.forEach(e),Un=c(s),d(ks.$$.fragment,s),Fn=c(s),B=i(s,"P",{});var W=o(B);fl=l(W,"Nous avons maintenant quelques paires qui aboutissent \xE0 un "),sn=i(W,"EM",{});var pr=o(sn);ml=l(pr,"token"),pr.forEach(e),hl=l(W," de plus de deux caract\xE8res. Par exemple la paire "),en=i(W,"CODE",{});var rr=o(en);dl=l(rr,'("h", "ug")'),rr.forEach(e),jl=l(W," pr\xE9sente 15 fois dans le corpus. La paire la plus fr\xE9quente \xE0 ce stade est "),nn=i(W,"CODE",{});var ir=o(nn);gl=l(ir,'("u", "n")'),ir.forEach(e),xl=l(W,", pr\xE9sente 16 fois dans le corpus, donc la deuxi\xE8me r\xE8gle de fusion apprise est "),an=i(W,"CODE",{});var or=o(an);ql=l(or,'("u", "n") -> "un"'),or.forEach(e),bl=l(W,". En ajoutant cela au vocabulaire et en fusionnant toutes les occurrences existantes, nous obtenons :"),W.forEach(e),Kn=c(s),d(ws.$$.fragment,s),Jn=c(s),S=i(s,"P",{});var hs=o(S);$l=l(hs,"Maintenant la paire la plus fr\xE9quente est "),tn=i(hs,"CODE",{});var ur=o(tn);vl=l(ur,'("h", "ug")'),ur.forEach(e),_l=l(hs," donc nous apprenons la r\xE8gle de fusion "),ln=i(hs,"CODE",{});var cr=o(ln);kl=l(cr,'("h", "ug") -> "hug"'),cr.forEach(e),wl=l(hs,". Cela nous donne donc notre premier "),pn=i(hs,"EM",{});var fr=o(pn);El=l(fr,"token"),fr.forEach(e),yl=l(hs," de trois lettres. Apr\xE8s la fusion, le corpus ressemble \xE0 ceci :"),hs.forEach(e),Wn=c(s),d(Es.$$.fragment,s),Yn=c(s),le=i(s,"P",{});var mr=o(le);Pl=l(mr,"Et nous continuons ainsi jusqu\u2019\xE0 ce que nous atteignions la taille de vocabulaire souhait\xE9e."),mr.forEach(e),Zn=c(s),d(as.$$.fragment,s),Qn=c(s),Z=i(s,"H2",{class:!0});var st=o(Z);ts=i(st,"A",{id:!0,class:!0,href:!0});var hr=o(ts);rn=i(hr,"SPAN",{});var dr=o(rn);d(ys.$$.fragment,dr),dr.forEach(e),hr.forEach(e),zl=c(st),on=i(st,"SPAN",{});var jr=o(on);Cl=l(jr,"Algorithme de tokenisation"),jr.forEach(e),st.forEach(e),Xn=c(s),pe=i(s,"P",{});var gr=o(pe);Tl=l(gr,"La tokenisation suit de pr\xE8s le processus d\u2019entra\xEEnement, dans le sens o\xF9 les nouvelles entr\xE9es sont tokenis\xE9es en appliquant les \xE9tapes suivantes :"),gr.forEach(e),sa=c(s),G=i(s,"OL",{});var ds=o(G);un=i(ds,"LI",{});var xr=o(un);Al=l(xr,"Normalisation"),xr.forEach(e),Nl=c(ds),cn=i(ds,"LI",{});var qr=o(cn);Dl=l(qr,"Pr\xE9tok\xE9nisation"),qr.forEach(e),Bl=c(ds),fn=i(ds,"LI",{});var br=o(fn);Ml=l(br,"D\xE9coupage des mots en caract\xE8res individuels"),br.forEach(e),Ol=c(ds),mn=i(ds,"LI",{});var $r=o(mn);Ll=l($r,"Application des r\xE8gles de fusion apprises dans l\u2019ordre sur ces divisions."),$r.forEach(e),ds.forEach(e),ea=c(s),re=i(s,"P",{});var vr=o(re);Il=l(vr,"Prenons l\u2019exemple que nous avons utilis\xE9 pendant l\u2019entra\xEEnement, avec les trois r\xE8gles de fusion apprises :"),vr.forEach(e),na=c(s),d(Ps.$$.fragment,s),aa=c(s),ie=i(s,"P",{});var _r=o(ie);Hl=l(_r,"Le mot \xAB bug \xBB  sera traduit par \xAB [\u201Cb\u201D, \u201Cug\u201D] \xBB. Par contre, le mot \xAB mug \xBB (tasse en fran\xE7ais) sera traduit par \xAB [\u201D[UNK]\u201D, \u201Cug\u201D] \xBB puisque la lettre \xAB m \xBB ne fait pas partie du vocabulaire de base. De la m\xEAme fa\xE7on, le mot \xAB thug \xBB (voyou en fran\xE7ais) sera tokenis\xE9 en \xAB [\u201D[UNK]\u201D, \u201Chug\u201D] \xBB car la lettre \xAB t \xBB n\u2019est pas dans le vocabulaire de base et l\u2019application des r\xE8gles de fusion r\xE9sulte d\u2019abord en la fusion de \xAB u \xBB et \xAB g \xBB et ensuite en la fusion de \xAB hu \xBB et \xAB g \xBB."),_r.forEach(e),ta=c(s),d(ls.$$.fragment,s),la=c(s),Q=i(s,"H2",{class:!0});var et=o(Q);ps=i(et,"A",{id:!0,class:!0,href:!0});var kr=o(ps);hn=i(kr,"SPAN",{});var wr=o(hn);d(zs.$$.fragment,wr),wr.forEach(e),kr.forEach(e),Sl=c(et),dn=i(et,"SPAN",{});var Er=o(dn);Gl=l(Er,"Impl\xE9mentation du BPE"),Er.forEach(e),et.forEach(e),pa=c(s),oe=i(s,"P",{});var yr=o(oe);Rl=l(yr,"Voyons maintenant une impl\xE9mentation de l\u2019algorithme BPE. Il ne s\u2019agira pas d\u2019une version optimis\xE9e que vous pourrez utiliser sur un grand corpus. Nous voulons simplement vous montrer le code afin que vous puissiez comprendre un peu mieux l\u2019algorithme."),yr.forEach(e),ra=c(s),ue=i(s,"P",{});var Pr=o(ue);Vl=l(Pr,"Tout d\u2019abord, nous avons besoin d\u2019un corpus, alors cr\xE9ons un corpus simple avec quelques phrases :"),Pr.forEach(e),ia=c(s),d(Cs.$$.fragment,s),oa=c(s),R=i(s,"P",{});var js=o(R);Ul=l(js,"Ensuite, nous devons pr\xE9tokeniser ce corpus en mots. Puisque nous r\xE9pliquons un "),jn=i(js,"EM",{});var zr=o(jn);Fl=l(zr,"tokenizer"),zr.forEach(e),Kl=l(js," BPE (comme celui du GPT-2), nous utiliserons le "),gn=i(js,"EM",{});var Cr=o(gn);Jl=l(Cr,"tokenizer"),Cr.forEach(e),Wl=c(js),xn=i(js,"CODE",{});var Tr=o(xn);Yl=l(Tr,"gpt2"),Tr.forEach(e),Zl=l(js," pour la pr\xE9tok\xE9nisation :"),js.forEach(e),ua=c(s),d(Ts.$$.fragment,s),ca=c(s),ce=i(s,"P",{});var Ar=o(ce);Ql=l(Ar,"Ensuite, nous calculons les fr\xE9quences de chaque mot dans le corpus comme nous le faisons pour la pr\xE9tok\xE9nisation :"),Ar.forEach(e),fa=c(s),d(As.$$.fragment,s),ma=c(s),d(Ns.$$.fragment,s),ha=c(s),fe=i(s,"P",{});var Nr=o(fe);Xl=l(Nr,"L\u2019\xE9tape suivante consiste \xE0 calculer le vocabulaire de base, form\xE9 par tous les caract\xE8res utilis\xE9s dans le corpus :"),Nr.forEach(e),da=c(s),d(Ds.$$.fragment,s),ja=c(s),d(Bs.$$.fragment,s),ga=c(s),V=i(s,"P",{});var gs=o(V);sp=l(gs,"Nous ajoutons \xE9galement les "),qn=i(gs,"EM",{});var Dr=o(qn);ep=l(Dr,"tokens"),Dr.forEach(e),np=l(gs," sp\xE9ciaux utilis\xE9s par le mod\xE8le au d\xE9but de ce vocabulaire. Dans le cas du GPT-2, le seul "),bn=i(gs,"EM",{});var Br=o(bn);ap=l(Br,"token"),Br.forEach(e),tp=l(gs," sp\xE9cial est "),$n=i(gs,"CODE",{});var Mr=o($n);lp=l(Mr,'"<|endoftext|>"'),Mr.forEach(e),pp=l(gs," :"),gs.forEach(e),xa=c(s),d(Ms.$$.fragment,s),qa=c(s),me=i(s,"P",{});var Or=o(me);rp=l(Or,"Nous devons maintenant diviser chaque mot en caract\xE8res individuels pour pouvoir commencer l\u2019entra\xEEnement :"),Or.forEach(e),ba=c(s),d(Os.$$.fragment,s),$a=c(s),he=i(s,"P",{});var Lr=o(he);ip=l(Lr,"Maintenant que nous sommes pr\xEAts pour l\u2019entra\xEEnement, \xE9crivons une fonction qui calcule la fr\xE9quence de chaque paire. Nous devrons l\u2019utiliser \xE0 chaque \xE9tape de l\u2019entra\xEEnement :"),Lr.forEach(e),va=c(s),d(Ls.$$.fragment,s),_a=c(s),de=i(s,"P",{});var Ir=o(de);op=l(Ir,"Jetons un coup d\u2019\u0153il \xE0 une partie de ce dictionnaire apr\xE8s les premi\xE8res divisions :"),Ir.forEach(e),ka=c(s),d(Is.$$.fragment,s),wa=c(s),d(Hs.$$.fragment,s),Ea=c(s),je=i(s,"P",{});var Hr=o(je);up=l(Hr,"Maintenant, trouver la paire la plus fr\xE9quente ne demande qu\u2019une rapide boucle :"),Hr.forEach(e),ya=c(s),d(Ss.$$.fragment,s),Pa=c(s),d(Gs.$$.fragment,s),za=c(s),J=i(s,"P",{});var we=o(J);cp=l(we,"Donc la premi\xE8re fusion \xE0 apprendre est "),vn=i(we,"CODE",{});var Sr=o(vn);fp=l(Sr,"('\u0120', 't') -> '\u0120t'"),Sr.forEach(e),mp=l(we,", et on ajoute "),_n=i(we,"CODE",{});var Gr=o(_n);hp=l(Gr,"'\u0120t'"),Gr.forEach(e),dp=l(we," au vocabulaire :"),we.forEach(e),Ca=c(s),d(Rs.$$.fragment,s),Ta=c(s),rs=i(s,"P",{});var nt=o(rs);jp=l(nt,"Pour continuer, nous devons appliquer cette fusion dans notre dictionnaire "),kn=i(nt,"CODE",{});var Rr=o(kn);gp=l(Rr,"splits"),Rr.forEach(e),xp=l(nt,". \xC9crivons une autre fonction pour cela :"),nt.forEach(e),Aa=c(s),d(Vs.$$.fragment,s),Na=c(s),ge=i(s,"P",{});var Vr=o(ge);qp=l(Vr,"Et nous pouvons regarder le r\xE9sultat de la premi\xE8re fusion :"),Vr.forEach(e),Da=c(s),d(Us.$$.fragment,s),Ba=c(s),d(Fs.$$.fragment,s),Ma=c(s),xe=i(s,"P",{});var Ur=o(xe);bp=l(Ur,"Maintenant, nous avons tout ce dont nous avons besoin pour boucler jusqu\u2019\xE0 ce que nous ayons appris toutes les fusions que nous voulons. Visons une taille de vocabulaire de 50 :"),Ur.forEach(e),Oa=c(s),d(Ks.$$.fragment,s),La=c(s),is=i(s,"P",{});var at=o(is);$p=l(at,"En cons\xE9quence, nous avons appris 19 r\xE8gles de fusion (le vocabulaire initial avait une taille de 31 : 30 caract\xE8res dans l\u2019alphabet plus le "),wn=i(at,"EM",{});var Fr=o(wn);vp=l(Fr,"token"),Fr.forEach(e),_p=l(at," sp\xE9cial) :"),at.forEach(e),Ia=c(s),d(Js.$$.fragment,s),Ha=c(s),d(Ws.$$.fragment,s),Sa=c(s),os=i(s,"P",{});var tt=o(os);kp=l(tt,"Et le vocabulaire est compos\xE9 du "),En=i(tt,"EM",{});var Kr=o(En);wp=l(Kr,"token"),Kr.forEach(e),Ep=l(tt," sp\xE9cial, de l\u2019alphabet initial, et de tous les r\xE9sultats des fusions :"),tt.forEach(e),Ga=c(s),d(Ys.$$.fragment,s),Ra=c(s),d(Zs.$$.fragment,s),Va=c(s),d(us.$$.fragment,s),Ua=c(s),qe=i(s,"P",{});var Jr=o(qe);yp=l(Jr,"Pour tokeniser un nouveau texte, on le pr\xE9tokenise, on le divise, puis on applique toutes les r\xE8gles de fusion apprises :"),Jr.forEach(e),Fa=c(s),d(Qs.$$.fragment,s),Ka=c(s),be=i(s,"P",{});var Wr=o(be);Pp=l(Wr,"Nous pouvons essayer cela sur n\u2019importe quel texte compos\xE9 de caract\xE8res de l\u2019alphabet :"),Wr.forEach(e),Ja=c(s),d(Xs.$$.fragment,s),Wa=c(s),d(se.$$.fragment,s),Ya=c(s),d(cs.$$.fragment,s),Za=c(s),$e=i(s,"P",{});var Yr=o($e);zp=l(Yr,"C\u2019est tout pour l\u2019algorithme BPE ! Nous allons nous int\xE9resser \xE0 WordPiece dans la suite."),Yr.forEach(e),this.h()},h(){T(f,"name","hf:doc:metadata"),T(f,"content",JSON.stringify(ci)),T($,"id","toknisation-ibytepair-encodingi"),T($,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),T($,"href","#toknisation-ibytepair-encodingi"),T(m,"class","relative group"),T(es,"id","algorithme-dentranement"),T(es,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),T(es,"href","#algorithme-dentranement"),T(Y,"class","relative group"),T(ts,"id","algorithme-de-tokenisation"),T(ts,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),T(ts,"href","#algorithme-de-tokenisation"),T(Z,"class","relative group"),T(ps,"id","implmentation-du-bpe"),T(ps,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),T(ps,"href","#implmentation-du-bpe"),T(Q,"class","relative group")},m(s,a){n(document.head,f),p(s,k,a),p(s,m,a),n(m,$),n($,w),j(b,w,null),n(m,v),n(m,E),n(E,N),n(E,P),n(P,U),p(s,O,a),j(z,s,a),p(s,xs,a),p(s,L,a),n(L,ne),n(L,ye),n(ye,pt),n(L,rt),n(L,Pe),n(Pe,it),n(L,ot),p(s,Cn,a),j(qs,s,a),p(s,Tn,a),j(ss,s,a),p(s,An,a),p(s,Y,a),n(Y,es),n(es,ze),j(bs,ze,null),n(Y,ut),n(Y,Ce),n(Ce,ct),p(s,Nn,a),p(s,ae,a),n(ae,ft),p(s,Dn,a),j($s,s,a),p(s,Bn,a),p(s,F,a),n(F,mt),n(F,Te),n(Te,ht),n(F,dt),n(F,Ae),n(Ae,jt),n(F,gt),p(s,Mn,a),j(ns,s,a),p(s,On,a),p(s,K,a),n(K,xt),n(K,Ne),n(Ne,qt),n(K,bt),n(K,De),n(De,$t),n(K,vt),p(s,Ln,a),p(s,I,a),n(I,_t),n(I,Be),n(Be,kt),n(I,wt),n(I,Me),n(Me,Et),n(I,yt),n(I,Oe),n(Oe,Pt),n(I,zt),p(s,In,a),p(s,te,a),n(te,Ct),p(s,Hn,a),j(vs,s,a),p(s,Sn,a),p(s,C,a),n(C,Tt),n(C,Le),n(Le,At),n(C,Nt),n(C,Ie),n(Ie,Dt),n(C,Bt),n(C,He),n(He,Mt),n(C,Ot),n(C,Se),n(Se,Lt),n(C,It),n(C,Ge),n(Ge,Ht),n(C,St),n(C,Re),n(Re,Gt),n(C,Rt),p(s,Gn,a),j(_s,s,a),p(s,Rn,a),p(s,y,a),n(y,Vt),n(y,Ve),n(Ve,Ut),n(y,Ft),n(y,Ue),n(Ue,Kt),n(y,Jt),n(y,Fe),n(Fe,Wt),n(y,Yt),n(y,Ke),n(Ke,Zt),n(y,Qt),n(y,Je),n(Je,Xt),n(y,sl),n(y,We),n(We,el),n(y,nl),n(y,Ye),n(Ye,al),n(y,tl),p(s,Vn,a),p(s,H,a),n(H,ll),n(H,Ze),n(Ze,pl),n(H,rl),n(H,Qe),n(Qe,il),n(H,ol),n(H,Xe),n(Xe,ul),n(H,cl),p(s,Un,a),j(ks,s,a),p(s,Fn,a),p(s,B,a),n(B,fl),n(B,sn),n(sn,ml),n(B,hl),n(B,en),n(en,dl),n(B,jl),n(B,nn),n(nn,gl),n(B,xl),n(B,an),n(an,ql),n(B,bl),p(s,Kn,a),j(ws,s,a),p(s,Jn,a),p(s,S,a),n(S,$l),n(S,tn),n(tn,vl),n(S,_l),n(S,ln),n(ln,kl),n(S,wl),n(S,pn),n(pn,El),n(S,yl),p(s,Wn,a),j(Es,s,a),p(s,Yn,a),p(s,le,a),n(le,Pl),p(s,Zn,a),j(as,s,a),p(s,Qn,a),p(s,Z,a),n(Z,ts),n(ts,rn),j(ys,rn,null),n(Z,zl),n(Z,on),n(on,Cl),p(s,Xn,a),p(s,pe,a),n(pe,Tl),p(s,sa,a),p(s,G,a),n(G,un),n(un,Al),n(G,Nl),n(G,cn),n(cn,Dl),n(G,Bl),n(G,fn),n(fn,Ml),n(G,Ol),n(G,mn),n(mn,Ll),p(s,ea,a),p(s,re,a),n(re,Il),p(s,na,a),j(Ps,s,a),p(s,aa,a),p(s,ie,a),n(ie,Hl),p(s,ta,a),j(ls,s,a),p(s,la,a),p(s,Q,a),n(Q,ps),n(ps,hn),j(zs,hn,null),n(Q,Sl),n(Q,dn),n(dn,Gl),p(s,pa,a),p(s,oe,a),n(oe,Rl),p(s,ra,a),p(s,ue,a),n(ue,Vl),p(s,ia,a),j(Cs,s,a),p(s,oa,a),p(s,R,a),n(R,Ul),n(R,jn),n(jn,Fl),n(R,Kl),n(R,gn),n(gn,Jl),n(R,Wl),n(R,xn),n(xn,Yl),n(R,Zl),p(s,ua,a),j(Ts,s,a),p(s,ca,a),p(s,ce,a),n(ce,Ql),p(s,fa,a),j(As,s,a),p(s,ma,a),j(Ns,s,a),p(s,ha,a),p(s,fe,a),n(fe,Xl),p(s,da,a),j(Ds,s,a),p(s,ja,a),j(Bs,s,a),p(s,ga,a),p(s,V,a),n(V,sp),n(V,qn),n(qn,ep),n(V,np),n(V,bn),n(bn,ap),n(V,tp),n(V,$n),n($n,lp),n(V,pp),p(s,xa,a),j(Ms,s,a),p(s,qa,a),p(s,me,a),n(me,rp),p(s,ba,a),j(Os,s,a),p(s,$a,a),p(s,he,a),n(he,ip),p(s,va,a),j(Ls,s,a),p(s,_a,a),p(s,de,a),n(de,op),p(s,ka,a),j(Is,s,a),p(s,wa,a),j(Hs,s,a),p(s,Ea,a),p(s,je,a),n(je,up),p(s,ya,a),j(Ss,s,a),p(s,Pa,a),j(Gs,s,a),p(s,za,a),p(s,J,a),n(J,cp),n(J,vn),n(vn,fp),n(J,mp),n(J,_n),n(_n,hp),n(J,dp),p(s,Ca,a),j(Rs,s,a),p(s,Ta,a),p(s,rs,a),n(rs,jp),n(rs,kn),n(kn,gp),n(rs,xp),p(s,Aa,a),j(Vs,s,a),p(s,Na,a),p(s,ge,a),n(ge,qp),p(s,Da,a),j(Us,s,a),p(s,Ba,a),j(Fs,s,a),p(s,Ma,a),p(s,xe,a),n(xe,bp),p(s,Oa,a),j(Ks,s,a),p(s,La,a),p(s,is,a),n(is,$p),n(is,wn),n(wn,vp),n(is,_p),p(s,Ia,a),j(Js,s,a),p(s,Ha,a),j(Ws,s,a),p(s,Sa,a),p(s,os,a),n(os,kp),n(os,En),n(En,wp),n(os,Ep),p(s,Ga,a),j(Ys,s,a),p(s,Ra,a),j(Zs,s,a),p(s,Va,a),j(us,s,a),p(s,Ua,a),p(s,qe,a),n(qe,yp),p(s,Fa,a),j(Qs,s,a),p(s,Ka,a),p(s,be,a),n(be,Pp),p(s,Ja,a),j(Xs,s,a),p(s,Wa,a),j(se,s,a),p(s,Ya,a),j(cs,s,a),p(s,Za,a),p(s,$e,a),n($e,zp),Qa=!0},p(s,[a]){const ee={};a&2&&(ee.$$scope={dirty:a,ctx:s}),ss.$set(ee);const yn={};a&2&&(yn.$$scope={dirty:a,ctx:s}),ns.$set(yn);const Pn={};a&2&&(Pn.$$scope={dirty:a,ctx:s}),as.$set(Pn);const ve={};a&2&&(ve.$$scope={dirty:a,ctx:s}),ls.$set(ve);const zn={};a&2&&(zn.$$scope={dirty:a,ctx:s}),us.$set(zn);const X={};a&2&&(X.$$scope={dirty:a,ctx:s}),cs.$set(X)},i(s){Qa||(g(b.$$.fragment,s),g(z.$$.fragment,s),g(qs.$$.fragment,s),g(ss.$$.fragment,s),g(bs.$$.fragment,s),g($s.$$.fragment,s),g(ns.$$.fragment,s),g(vs.$$.fragment,s),g(_s.$$.fragment,s),g(ks.$$.fragment,s),g(ws.$$.fragment,s),g(Es.$$.fragment,s),g(as.$$.fragment,s),g(ys.$$.fragment,s),g(Ps.$$.fragment,s),g(ls.$$.fragment,s),g(zs.$$.fragment,s),g(Cs.$$.fragment,s),g(Ts.$$.fragment,s),g(As.$$.fragment,s),g(Ns.$$.fragment,s),g(Ds.$$.fragment,s),g(Bs.$$.fragment,s),g(Ms.$$.fragment,s),g(Os.$$.fragment,s),g(Ls.$$.fragment,s),g(Is.$$.fragment,s),g(Hs.$$.fragment,s),g(Ss.$$.fragment,s),g(Gs.$$.fragment,s),g(Rs.$$.fragment,s),g(Vs.$$.fragment,s),g(Us.$$.fragment,s),g(Fs.$$.fragment,s),g(Ks.$$.fragment,s),g(Js.$$.fragment,s),g(Ws.$$.fragment,s),g(Ys.$$.fragment,s),g(Zs.$$.fragment,s),g(us.$$.fragment,s),g(Qs.$$.fragment,s),g(Xs.$$.fragment,s),g(se.$$.fragment,s),g(cs.$$.fragment,s),Qa=!0)},o(s){x(b.$$.fragment,s),x(z.$$.fragment,s),x(qs.$$.fragment,s),x(ss.$$.fragment,s),x(bs.$$.fragment,s),x($s.$$.fragment,s),x(ns.$$.fragment,s),x(vs.$$.fragment,s),x(_s.$$.fragment,s),x(ks.$$.fragment,s),x(ws.$$.fragment,s),x(Es.$$.fragment,s),x(as.$$.fragment,s),x(ys.$$.fragment,s),x(Ps.$$.fragment,s),x(ls.$$.fragment,s),x(zs.$$.fragment,s),x(Cs.$$.fragment,s),x(Ts.$$.fragment,s),x(As.$$.fragment,s),x(Ns.$$.fragment,s),x(Ds.$$.fragment,s),x(Bs.$$.fragment,s),x(Ms.$$.fragment,s),x(Os.$$.fragment,s),x(Ls.$$.fragment,s),x(Is.$$.fragment,s),x(Hs.$$.fragment,s),x(Ss.$$.fragment,s),x(Gs.$$.fragment,s),x(Rs.$$.fragment,s),x(Vs.$$.fragment,s),x(Us.$$.fragment,s),x(Fs.$$.fragment,s),x(Ks.$$.fragment,s),x(Js.$$.fragment,s),x(Ws.$$.fragment,s),x(Ys.$$.fragment,s),x(Zs.$$.fragment,s),x(us.$$.fragment,s),x(Qs.$$.fragment,s),x(Xs.$$.fragment,s),x(se.$$.fragment,s),x(cs.$$.fragment,s),Qa=!1},d(s){e(f),s&&e(k),s&&e(m),q(b),s&&e(O),q(z,s),s&&e(xs),s&&e(L),s&&e(Cn),q(qs,s),s&&e(Tn),q(ss,s),s&&e(An),s&&e(Y),q(bs),s&&e(Nn),s&&e(ae),s&&e(Dn),q($s,s),s&&e(Bn),s&&e(F),s&&e(Mn),q(ns,s),s&&e(On),s&&e(K),s&&e(Ln),s&&e(I),s&&e(In),s&&e(te),s&&e(Hn),q(vs,s),s&&e(Sn),s&&e(C),s&&e(Gn),q(_s,s),s&&e(Rn),s&&e(y),s&&e(Vn),s&&e(H),s&&e(Un),q(ks,s),s&&e(Fn),s&&e(B),s&&e(Kn),q(ws,s),s&&e(Jn),s&&e(S),s&&e(Wn),q(Es,s),s&&e(Yn),s&&e(le),s&&e(Zn),q(as,s),s&&e(Qn),s&&e(Z),q(ys),s&&e(Xn),s&&e(pe),s&&e(sa),s&&e(G),s&&e(ea),s&&e(re),s&&e(na),q(Ps,s),s&&e(aa),s&&e(ie),s&&e(ta),q(ls,s),s&&e(la),s&&e(Q),q(zs),s&&e(pa),s&&e(oe),s&&e(ra),s&&e(ue),s&&e(ia),q(Cs,s),s&&e(oa),s&&e(R),s&&e(ua),q(Ts,s),s&&e(ca),s&&e(ce),s&&e(fa),q(As,s),s&&e(ma),q(Ns,s),s&&e(ha),s&&e(fe),s&&e(da),q(Ds,s),s&&e(ja),q(Bs,s),s&&e(ga),s&&e(V),s&&e(xa),q(Ms,s),s&&e(qa),s&&e(me),s&&e(ba),q(Os,s),s&&e($a),s&&e(he),s&&e(va),q(Ls,s),s&&e(_a),s&&e(de),s&&e(ka),q(Is,s),s&&e(wa),q(Hs,s),s&&e(Ea),s&&e(je),s&&e(ya),q(Ss,s),s&&e(Pa),q(Gs,s),s&&e(za),s&&e(J),s&&e(Ca),q(Rs,s),s&&e(Ta),s&&e(rs),s&&e(Aa),q(Vs,s),s&&e(Na),s&&e(ge),s&&e(Da),q(Us,s),s&&e(Ba),q(Fs,s),s&&e(Ma),s&&e(xe),s&&e(Oa),q(Ks,s),s&&e(La),s&&e(is),s&&e(Ia),q(Js,s),s&&e(Ha),q(Ws,s),s&&e(Sa),s&&e(os),s&&e(Ga),q(Ys,s),s&&e(Ra),q(Zs,s),s&&e(Va),q(us,s),s&&e(Ua),s&&e(qe),s&&e(Fa),q(Qs,s),s&&e(Ka),s&&e(be),s&&e(Ja),q(Xs,s),s&&e(Wa),q(se,s),s&&e(Ya),q(cs,s),s&&e(Za),s&&e($e)}}}const ci={local:"toknisation-ibytepair-encodingi",sections:[{local:"algorithme-dentranement",title:"Algorithme d'entra\xEEnement"},{local:"algorithme-de-tokenisation",title:"Algorithme de tokenisation"},{local:"implmentation-du-bpe",title:"Impl\xE9mentation du BPE"}],title:"Tok\xE9nisation <i>Byte-Pair Encoding</i>"};function fi(D){return ei(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class qi extends Zr{constructor(f){super();Qr(this,f,fi,ui,Xr,{})}}export{qi as default,ci as metadata};
