import{S as zr,i as Tr,s as Pr,e as r,k as h,w as g,t as n,M as Cr,c as i,d as e,m as u,a as o,x as m,h as l,b as P,G as t,g as p,y as d,q as j,o as w,B as x,v as Nr}from"../../chunks/vendor-hf-doc-builder.js";import{T as we}from"../../chunks/Tip-hf-doc-builder.js";import{Y as Or}from"../../chunks/Youtube-hf-doc-builder.js";import{I as Ma}from"../../chunks/IconCopyLink-hf-doc-builder.js";import{C as $}from"../../chunks/CodeBlock-hf-doc-builder.js";import{D as Ar}from"../../chunks/DocNotebookDropdown-hf-doc-builder.js";function Dr(O){let c,_;return{c(){c=r("p"),_=n("\u{1F4A1} This section covers BPE in depth, going as far as showing a full implementation. You can skip to the end if you just want a general overview of the tokenization algorithm.")},l(f){c=i(f,"P",{});var b=o(c);_=l(b,"\u{1F4A1} This section covers BPE in depth, going as far as showing a full implementation. You can skip to the end if you just want a general overview of the tokenization algorithm."),b.forEach(e)},m(f,b){p(f,c,b),t(c,_)},d(f){f&&e(c)}}}function Br(O){let c,_,f,b,E;return{c(){c=r("p"),_=n("The GPT-2 and RoBERTa tokenizers (which are pretty similar) have a clever way to deal with this: they don\u2019t look at words as being written with Unicode characters, but with bytes. This way the base vocabulary has a small size (256), but every character you can think of will still be included and not end up being converted to the unknown token. This trick is called "),f=r("em"),b=n("byte-level BPE"),E=n(".")},l(k){c=i(k,"P",{});var y=o(c);_=l(y,"The GPT-2 and RoBERTa tokenizers (which are pretty similar) have a clever way to deal with this: they don\u2019t look at words as being written with Unicode characters, but with bytes. This way the base vocabulary has a small size (256), but every character you can think of will still be included and not end up being converted to the unknown token. This trick is called "),f=i(y,"EM",{});var T=o(f);b=l(T,"byte-level BPE"),T.forEach(e),E=l(y,"."),y.forEach(e)},m(k,y){p(k,c,y),t(c,_),t(c,f),t(f,b),t(c,E)},d(k){k&&e(c)}}}function Ir(O){let c,_,f,b,E;return{c(){c=r("p"),_=n("\u270F\uFE0F "),f=r("strong"),b=n("Now your turn!"),E=n(" What do you think the next merge rule will be?")},l(k){c=i(k,"P",{});var y=o(c);_=l(y,"\u270F\uFE0F "),f=i(y,"STRONG",{});var T=o(f);b=l(T,"Now your turn!"),T.forEach(e),E=l(y," What do you think the next merge rule will be?"),y.forEach(e)},m(k,y){p(k,c,y),t(c,_),t(c,f),t(f,b),t(c,E)},d(k){k&&e(c)}}}function Sr(O){let c,_,f,b,E,k,y,T;return{c(){c=r("p"),_=n("\u270F\uFE0F "),f=r("strong"),b=n("Now your turn!"),E=n(" How do you think  the word "),k=r("code"),y=n('"unhug"'),T=n(" will be tokenized?")},l(R){c=i(R,"P",{});var A=o(c);_=l(A,"\u270F\uFE0F "),f=i(A,"STRONG",{});var S=o(f);b=l(S,"Now your turn!"),S.forEach(e),E=l(A," How do you think  the word "),k=i(A,"CODE",{});var is=o(k);y=l(is,'"unhug"'),is.forEach(e),T=l(A," will be tokenized?"),A.forEach(e)},m(R,A){p(R,c,A),t(c,_),t(c,f),t(f,b),t(c,E),t(c,k),t(k,y),t(c,T)},d(R){R&&e(c)}}}function Gr(O){let c,_,f,b,E;return{c(){c=r("p"),_=n("\u{1F4A1} Using "),f=r("code"),b=n("train_new_from_iterator()"),E=n(" on the same corpus won\u2019t result in the exact same vocabulary. This is because when there is a choice of the most frequent pair, we selected the first one encountered, while the \u{1F917} Tokenizers library selects the first one based on its inner IDs.")},l(k){c=i(k,"P",{});var y=o(c);_=l(y,"\u{1F4A1} Using "),f=i(y,"CODE",{});var T=o(f);b=l(T,"train_new_from_iterator()"),T.forEach(e),E=l(y," on the same corpus won\u2019t result in the exact same vocabulary. This is because when there is a choice of the most frequent pair, we selected the first one encountered, while the \u{1F917} Tokenizers library selects the first one based on its inner IDs."),y.forEach(e)},m(k,y){p(k,c,y),t(c,_),t(c,f),t(f,b),t(c,E)},d(k){k&&e(c)}}}function Lr(O){let c,_;return{c(){c=r("p"),_=n("\u26A0\uFE0F Our implementation will throw an error if there is an unknown character since we didn\u2019t do anything to handle them. GPT-2 doesn\u2019t actually have an unknown token (it\u2019s impossible to get an unknown character when using byte-level BPE), but this could happen here because we did not include all the possible bytes in the initial vocabulary. This aspect of BPE is beyond the scope of this section, so we\u2019ve left the details out.")},l(f){c=i(f,"P",{});var b=o(c);_=l(b,"\u26A0\uFE0F Our implementation will throw an error if there is an unknown character since we didn\u2019t do anything to handle them. GPT-2 doesn\u2019t actually have an unknown token (it\u2019s impossible to get an unknown character when using byte-level BPE), but this could happen here because we did not include all the possible bytes in the initial vocabulary. This aspect of BPE is beyond the scope of this section, so we\u2019ve left the details out."),b.forEach(e)},m(f,b){p(f,c,b),t(c,_)},d(f){f&&e(c)}}}function Hr(O){let c,_,f,b,E,k,y,T,R,A,S,is,Ms,Ka,jt,os,wt,V,xt,W,M,xe,hs,Ya,be,Ja,bt,Ks,Za,$t,us,kt,K,Qa,$e,Xa,sn,qt,Y,_t,J,en,ke,tn,an,yt,Ys,nn,vt,Js,ln,Et,cs,zt,C,pn,qe,rn,on,_e,hn,un,ye,cn,fn,ve,gn,mn,Ee,dn,jn,Tt,fs,Pt,z,wn,ze,xn,bn,Te,$n,kn,Pe,qn,_n,Ce,yn,vn,Ne,En,zn,Oe,Tn,Pn,Ae,Cn,Nn,Ct,G,On,De,An,Dn,Be,Bn,In,Nt,gs,Ot,D,Sn,Ie,Gn,Ln,Se,Hn,Rn,Ge,Wn,Fn,At,ms,Dt,L,Un,Le,Vn,Mn,He,Kn,Yn,Bt,ds,It,Zs,Jn,St,Z,Gt,F,Q,Re,js,Zn,We,Qn,Lt,Qs,Xn,Ht,B,Fe,sl,el,Ue,tl,al,Ve,nl,ll,Me,pl,Rt,Xs,rl,Wt,ws,Ft,q,il,Ke,ol,hl,Ye,ul,cl,Je,fl,gl,Ze,ml,dl,Qe,jl,wl,Xe,xl,bl,st,$l,kl,et,ql,_l,tt,yl,vl,at,El,zl,nt,Tl,Pl,lt,Cl,Nl,Ut,X,Vt,U,ss,pt,xs,Ol,rt,Al,Mt,se,Dl,Kt,ee,Bl,Yt,bs,Jt,es,Il,it,Sl,Gl,Zt,$s,Qt,te,Ll,Xt,ks,sa,qs,ea,ae,Hl,ta,_s,aa,ys,na,ts,Rl,ot,Wl,Fl,la,vs,pa,ne,Ul,ra,Es,ia,le,Vl,oa,zs,ha,pe,Ml,ua,Ts,ca,Ps,fa,re,Kl,ga,Cs,ma,Ns,da,H,Yl,ht,Jl,Zl,ut,Ql,Xl,ja,Os,wa,as,sp,ct,ep,tp,xa,As,ba,ie,ap,$a,Ds,ka,Bs,qa,oe,np,_a,Is,ya,he,lp,va,Ss,Ea,Gs,za,ue,pp,Ta,Ls,Pa,Hs,Ca,ns,Na,ce,rp,Oa,Rs,Aa,fe,ip,Da,Ws,Ba,Fs,Ia,ls,Sa,ge,op,Ga;return k=new Ma({}),S=new Ar({props:{classNames:"absolute z-10 right-0 top-0",options:[{label:"Google Colab",value:"https://colab.research.google.com/github/huggingface/notebooks/blob/master/course/chapter6/section5.ipynb"},{label:"Aws Studio",value:"https://studiolab.sagemaker.aws/import/github/huggingface/notebooks/blob/master/course/chapter6/section5.ipynb"}]}}),os=new Or({props:{id:"HEikzVL-lZU"}}),V=new we({props:{$$slots:{default:[Dr]},$$scope:{ctx:O}}}),hs=new Ma({}),us=new $({props:{code:'"hug", "pug", "pun", "bun", "hugs"',highlighted:'<span class="hljs-string">&quot;hug&quot;</span>, <span class="hljs-string">&quot;pug&quot;</span>, <span class="hljs-string">&quot;pun&quot;</span>, <span class="hljs-string">&quot;bun&quot;</span>, <span class="hljs-string">&quot;hugs&quot;</span>'}}),Y=new we({props:{$$slots:{default:[Br]},$$scope:{ctx:O}}}),cs=new $({props:{code:'("hug", 10), ("pug", 5), ("pun", 12), ("bun", 4), ("hugs", 5)',highlighted:'(<span class="hljs-string">&quot;hug&quot;</span><span class="hljs-punctuation">,</span> <span class="hljs-number">10</span>)<span class="hljs-punctuation">,</span> (<span class="hljs-string">&quot;pug&quot;</span><span class="hljs-punctuation">,</span> <span class="hljs-number">5</span>)<span class="hljs-punctuation">,</span> (<span class="hljs-string">&quot;pun&quot;</span><span class="hljs-punctuation">,</span> <span class="hljs-number">12</span>)<span class="hljs-punctuation">,</span> (<span class="hljs-string">&quot;bun&quot;</span><span class="hljs-punctuation">,</span> <span class="hljs-number">4</span>)<span class="hljs-punctuation">,</span> (<span class="hljs-string">&quot;hugs&quot;</span><span class="hljs-punctuation">,</span> <span class="hljs-number">5</span>)'}}),fs=new $({props:{code:'("h" "u" "g", 10), ("p" "u" "g", 5), ("p" "u" "n", 12), ("b" "u" "n", 4), ("h" "u" "g" "s", 5)',highlighted:'(<span class="hljs-string">&quot;h&quot;</span> <span class="hljs-string">&quot;u&quot;</span> <span class="hljs-string">&quot;g&quot;</span><span class="hljs-punctuation">,</span> <span class="hljs-number">10</span>)<span class="hljs-punctuation">,</span> (<span class="hljs-string">&quot;p&quot;</span> <span class="hljs-string">&quot;u&quot;</span> <span class="hljs-string">&quot;g&quot;</span><span class="hljs-punctuation">,</span> <span class="hljs-number">5</span>)<span class="hljs-punctuation">,</span> (<span class="hljs-string">&quot;p&quot;</span> <span class="hljs-string">&quot;u&quot;</span> <span class="hljs-string">&quot;n&quot;</span><span class="hljs-punctuation">,</span> <span class="hljs-number">12</span>)<span class="hljs-punctuation">,</span> (<span class="hljs-string">&quot;b&quot;</span> <span class="hljs-string">&quot;u&quot;</span> <span class="hljs-string">&quot;n&quot;</span><span class="hljs-punctuation">,</span> <span class="hljs-number">4</span>)<span class="hljs-punctuation">,</span> (<span class="hljs-string">&quot;h&quot;</span> <span class="hljs-string">&quot;u&quot;</span> <span class="hljs-string">&quot;g&quot;</span> <span class="hljs-string">&quot;s&quot;</span><span class="hljs-punctuation">,</span> <span class="hljs-number">5</span>)'}}),gs=new $({props:{code:`Vocabulary: ["b", "g", "h", "n", "p", "s", "u", "ug"]
Corpus: ("h" "ug", 10), ("p" "ug", 5), ("p" "u" "n", 12), ("b" "u" "n", 4), ("h" "ug" "s", 5)`,highlighted:`<span class="hljs-symbol">Vocabulary:</span> [<span class="hljs-string">&quot;b&quot;</span>, <span class="hljs-string">&quot;g&quot;</span>, <span class="hljs-string">&quot;h&quot;</span>, <span class="hljs-string">&quot;n&quot;</span>, <span class="hljs-string">&quot;p&quot;</span>, <span class="hljs-string">&quot;s&quot;</span>, <span class="hljs-string">&quot;u&quot;</span>, <span class="hljs-string">&quot;ug&quot;</span>]
<span class="hljs-symbol">Corpus:</span> (<span class="hljs-string">&quot;h&quot;</span> <span class="hljs-string">&quot;ug&quot;</span>, <span class="hljs-number">10</span>), (<span class="hljs-string">&quot;p&quot;</span> <span class="hljs-string">&quot;ug&quot;</span>, <span class="hljs-number">5</span>), (<span class="hljs-string">&quot;p&quot;</span> <span class="hljs-string">&quot;u&quot;</span> <span class="hljs-string">&quot;n&quot;</span>, <span class="hljs-number">12</span>), (<span class="hljs-string">&quot;b&quot;</span> <span class="hljs-string">&quot;u&quot;</span> <span class="hljs-string">&quot;n&quot;</span>, <span class="hljs-number">4</span>), (<span class="hljs-string">&quot;h&quot;</span> <span class="hljs-string">&quot;ug&quot;</span> <span class="hljs-string">&quot;s&quot;</span>, <span class="hljs-number">5</span>)`}}),ms=new $({props:{code:`Vocabulary: ["b", "g", "h", "n", "p", "s", "u", "ug", "un"]
Corpus: ("h" "ug", 10), ("p" "ug", 5), ("p" "un", 12), ("b" "un", 4), ("h" "ug" "s", 5)`,highlighted:`<span class="hljs-symbol">Vocabulary:</span> [<span class="hljs-string">&quot;b&quot;</span>, <span class="hljs-string">&quot;g&quot;</span>, <span class="hljs-string">&quot;h&quot;</span>, <span class="hljs-string">&quot;n&quot;</span>, <span class="hljs-string">&quot;p&quot;</span>, <span class="hljs-string">&quot;s&quot;</span>, <span class="hljs-string">&quot;u&quot;</span>, <span class="hljs-string">&quot;ug&quot;</span>, <span class="hljs-string">&quot;un&quot;</span>]
<span class="hljs-symbol">Corpus:</span> (<span class="hljs-string">&quot;h&quot;</span> <span class="hljs-string">&quot;ug&quot;</span>, <span class="hljs-number">10</span>), (<span class="hljs-string">&quot;p&quot;</span> <span class="hljs-string">&quot;ug&quot;</span>, <span class="hljs-number">5</span>), (<span class="hljs-string">&quot;p&quot;</span> <span class="hljs-string">&quot;un&quot;</span>, <span class="hljs-number">12</span>), (<span class="hljs-string">&quot;b&quot;</span> <span class="hljs-string">&quot;un&quot;</span>, <span class="hljs-number">4</span>), (<span class="hljs-string">&quot;h&quot;</span> <span class="hljs-string">&quot;ug&quot;</span> <span class="hljs-string">&quot;s&quot;</span>, <span class="hljs-number">5</span>)`}}),ds=new $({props:{code:`Vocabulary: ["b", "g", "h", "n", "p", "s", "u", "ug", "un", "hug"]
Corpus: ("hug", 10), ("p" "ug", 5), ("p" "un", 12), ("b" "un", 4), ("hug" "s", 5)`,highlighted:`<span class="hljs-symbol">Vocabulary:</span> [<span class="hljs-string">&quot;b&quot;</span>, <span class="hljs-string">&quot;g&quot;</span>, <span class="hljs-string">&quot;h&quot;</span>, <span class="hljs-string">&quot;n&quot;</span>, <span class="hljs-string">&quot;p&quot;</span>, <span class="hljs-string">&quot;s&quot;</span>, <span class="hljs-string">&quot;u&quot;</span>, <span class="hljs-string">&quot;ug&quot;</span>, <span class="hljs-string">&quot;un&quot;</span>, <span class="hljs-string">&quot;hug&quot;</span>]
<span class="hljs-symbol">Corpus:</span> (<span class="hljs-string">&quot;hug&quot;</span>, <span class="hljs-number">10</span>), (<span class="hljs-string">&quot;p&quot;</span> <span class="hljs-string">&quot;ug&quot;</span>, <span class="hljs-number">5</span>), (<span class="hljs-string">&quot;p&quot;</span> <span class="hljs-string">&quot;un&quot;</span>, <span class="hljs-number">12</span>), (<span class="hljs-string">&quot;b&quot;</span> <span class="hljs-string">&quot;un&quot;</span>, <span class="hljs-number">4</span>), (<span class="hljs-string">&quot;hug&quot;</span> <span class="hljs-string">&quot;s&quot;</span>, <span class="hljs-number">5</span>)`}}),Z=new we({props:{$$slots:{default:[Ir]},$$scope:{ctx:O}}}),js=new Ma({}),ws=new $({props:{code:`("u", "g") -> "ug"
("u", "n") -> "un"
("h", "ug") -> "hug"`,highlighted:`<span class="hljs-function"><span class="hljs-params">(<span class="hljs-string">&quot;u&quot;</span>, <span class="hljs-string">&quot;g&quot;</span>)</span> -&gt;</span> <span class="hljs-string">&quot;ug&quot;</span>
<span class="hljs-function"><span class="hljs-params">(<span class="hljs-string">&quot;u&quot;</span>, <span class="hljs-string">&quot;n&quot;</span>)</span> -&gt;</span> <span class="hljs-string">&quot;un&quot;</span>
<span class="hljs-function"><span class="hljs-params">(<span class="hljs-string">&quot;h&quot;</span>, <span class="hljs-string">&quot;ug&quot;</span>)</span> -&gt;</span> <span class="hljs-string">&quot;hug&quot;</span>`}}),X=new we({props:{$$slots:{default:[Sr]},$$scope:{ctx:O}}}),xs=new Ma({}),bs=new $({props:{code:`corpus = [
    "This is the Hugging Face Course.",
    "This chapter is about tokenization.",
    "This section shows several tokenizer algorithms.",
    "Hopefully, you will be able to understand how they are trained and generate tokens.",
]`,highlighted:`corpus = [
    <span class="hljs-string">&quot;This is the Hugging Face Course.&quot;</span>,
    <span class="hljs-string">&quot;This chapter is about tokenization.&quot;</span>,
    <span class="hljs-string">&quot;This section shows several tokenizer algorithms.&quot;</span>,
    <span class="hljs-string">&quot;Hopefully, you will be able to understand how they are trained and generate tokens.&quot;</span>,
]`}}),$s=new $({props:{code:`from transformers import AutoTokenizer

tokenizer = AutoTokenizer.from_pretrained("gpt2")`,highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoTokenizer

tokenizer = AutoTokenizer.from_pretrained(<span class="hljs-string">&quot;gpt2&quot;</span>)`}}),ks=new $({props:{code:`from collections import defaultdict

word_freqs = defaultdict(int)

for text in corpus:
    words_with_offsets = tokenizer.backend_tokenizer.pre_tokenizer.pre_tokenize_str(text)
    new_words = [word for word, offset in words_with_offsets]
    for word in new_words:
        word_freqs[word] += 1

print(word_freqs)`,highlighted:`<span class="hljs-keyword">from</span> collections <span class="hljs-keyword">import</span> defaultdict

word_freqs = defaultdict(<span class="hljs-built_in">int</span>)

<span class="hljs-keyword">for</span> text <span class="hljs-keyword">in</span> corpus:
    words_with_offsets = tokenizer.backend_tokenizer.pre_tokenizer.pre_tokenize_str(text)
    new_words = [word <span class="hljs-keyword">for</span> word, offset <span class="hljs-keyword">in</span> words_with_offsets]
    <span class="hljs-keyword">for</span> word <span class="hljs-keyword">in</span> new_words:
        word_freqs[word] += <span class="hljs-number">1</span>

<span class="hljs-built_in">print</span>(word_freqs)`}}),qs=new $({props:{code:`defaultdict(int, {'This': 3, '\u0120is': 2, '\u0120the': 1, '\u0120Hugging': 1, '\u0120Face': 1, '\u0120Course': 1, '.': 4, '\u0120chapter': 1,
    '\u0120about': 1, '\u0120tokenization': 1, '\u0120section': 1, '\u0120shows': 1, '\u0120several': 1, '\u0120tokenizer': 1, '\u0120algorithms': 1,
    'Hopefully': 1, ',': 1, '\u0120you': 1, '\u0120will': 1, '\u0120be': 1, '\u0120able': 1, '\u0120to': 1, '\u0120understand': 1, '\u0120how': 1,
    '\u0120they': 1, '\u0120are': 1, '\u0120trained': 1, '\u0120and': 1, '\u0120generate': 1, '\u0120tokens': 1})`,highlighted:`defaultdict(<span class="hljs-built_in">int</span>, {<span class="hljs-string">&#x27;This&#x27;</span>: <span class="hljs-number">3</span>, <span class="hljs-string">&#x27;\u0120is&#x27;</span>: <span class="hljs-number">2</span>, <span class="hljs-string">&#x27;\u0120the&#x27;</span>: <span class="hljs-number">1</span>, <span class="hljs-string">&#x27;\u0120Hugging&#x27;</span>: <span class="hljs-number">1</span>, <span class="hljs-string">&#x27;\u0120Face&#x27;</span>: <span class="hljs-number">1</span>, <span class="hljs-string">&#x27;\u0120Course&#x27;</span>: <span class="hljs-number">1</span>, <span class="hljs-string">&#x27;.&#x27;</span>: <span class="hljs-number">4</span>, <span class="hljs-string">&#x27;\u0120chapter&#x27;</span>: <span class="hljs-number">1</span>,
    <span class="hljs-string">&#x27;\u0120about&#x27;</span>: <span class="hljs-number">1</span>, <span class="hljs-string">&#x27;\u0120tokenization&#x27;</span>: <span class="hljs-number">1</span>, <span class="hljs-string">&#x27;\u0120section&#x27;</span>: <span class="hljs-number">1</span>, <span class="hljs-string">&#x27;\u0120shows&#x27;</span>: <span class="hljs-number">1</span>, <span class="hljs-string">&#x27;\u0120several&#x27;</span>: <span class="hljs-number">1</span>, <span class="hljs-string">&#x27;\u0120tokenizer&#x27;</span>: <span class="hljs-number">1</span>, <span class="hljs-string">&#x27;\u0120algorithms&#x27;</span>: <span class="hljs-number">1</span>,
    <span class="hljs-string">&#x27;Hopefully&#x27;</span>: <span class="hljs-number">1</span>, <span class="hljs-string">&#x27;,&#x27;</span>: <span class="hljs-number">1</span>, <span class="hljs-string">&#x27;\u0120you&#x27;</span>: <span class="hljs-number">1</span>, <span class="hljs-string">&#x27;\u0120will&#x27;</span>: <span class="hljs-number">1</span>, <span class="hljs-string">&#x27;\u0120be&#x27;</span>: <span class="hljs-number">1</span>, <span class="hljs-string">&#x27;\u0120able&#x27;</span>: <span class="hljs-number">1</span>, <span class="hljs-string">&#x27;\u0120to&#x27;</span>: <span class="hljs-number">1</span>, <span class="hljs-string">&#x27;\u0120understand&#x27;</span>: <span class="hljs-number">1</span>, <span class="hljs-string">&#x27;\u0120how&#x27;</span>: <span class="hljs-number">1</span>,
    <span class="hljs-string">&#x27;\u0120they&#x27;</span>: <span class="hljs-number">1</span>, <span class="hljs-string">&#x27;\u0120are&#x27;</span>: <span class="hljs-number">1</span>, <span class="hljs-string">&#x27;\u0120trained&#x27;</span>: <span class="hljs-number">1</span>, <span class="hljs-string">&#x27;\u0120and&#x27;</span>: <span class="hljs-number">1</span>, <span class="hljs-string">&#x27;\u0120generate&#x27;</span>: <span class="hljs-number">1</span>, <span class="hljs-string">&#x27;\u0120tokens&#x27;</span>: <span class="hljs-number">1</span>})`}}),_s=new $({props:{code:`alphabet = []

for word in word_freqs.keys():
    for letter in word:
        if letter not in alphabet:
            alphabet.append(letter)
alphabet.sort()

print(alphabet)`,highlighted:`alphabet = []

<span class="hljs-keyword">for</span> word <span class="hljs-keyword">in</span> word_freqs.keys():
    <span class="hljs-keyword">for</span> letter <span class="hljs-keyword">in</span> word:
        <span class="hljs-keyword">if</span> letter <span class="hljs-keyword">not</span> <span class="hljs-keyword">in</span> alphabet:
            alphabet.append(letter)
alphabet.sort()

<span class="hljs-built_in">print</span>(alphabet)`}}),ys=new $({props:{code:`[ ',', '.', 'C', 'F', 'H', 'T', 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'k', 'l', 'm', 'n', 'o', 'p', 'r', 's',
  't', 'u', 'v', 'w', 'y', 'z', '\u0120']`,highlighted:`[ <span class="hljs-string">&#x27;,&#x27;</span>, <span class="hljs-string">&#x27;.&#x27;</span>, <span class="hljs-string">&#x27;C&#x27;</span>, <span class="hljs-string">&#x27;F&#x27;</span>, <span class="hljs-string">&#x27;H&#x27;</span>, <span class="hljs-string">&#x27;T&#x27;</span>, <span class="hljs-string">&#x27;a&#x27;</span>, <span class="hljs-string">&#x27;b&#x27;</span>, <span class="hljs-string">&#x27;c&#x27;</span>, <span class="hljs-string">&#x27;d&#x27;</span>, <span class="hljs-string">&#x27;e&#x27;</span>, <span class="hljs-string">&#x27;f&#x27;</span>, <span class="hljs-string">&#x27;g&#x27;</span>, <span class="hljs-string">&#x27;h&#x27;</span>, <span class="hljs-string">&#x27;i&#x27;</span>, <span class="hljs-string">&#x27;k&#x27;</span>, <span class="hljs-string">&#x27;l&#x27;</span>, <span class="hljs-string">&#x27;m&#x27;</span>, <span class="hljs-string">&#x27;n&#x27;</span>, <span class="hljs-string">&#x27;o&#x27;</span>, <span class="hljs-string">&#x27;p&#x27;</span>, <span class="hljs-string">&#x27;r&#x27;</span>, <span class="hljs-string">&#x27;s&#x27;</span>,
  <span class="hljs-string">&#x27;t&#x27;</span>, <span class="hljs-string">&#x27;u&#x27;</span>, <span class="hljs-string">&#x27;v&#x27;</span>, <span class="hljs-string">&#x27;w&#x27;</span>, <span class="hljs-string">&#x27;y&#x27;</span>, <span class="hljs-string">&#x27;z&#x27;</span>, <span class="hljs-string">&#x27;\u0120&#x27;</span>]`}}),vs=new $({props:{code:'vocab = ["<|endoftext|>"] + alphabet.copy()',highlighted:'vocab = [<span class="hljs-string">&quot;&lt;|endoftext|&gt;&quot;</span>] + alphabet.copy()'}}),Es=new $({props:{code:"splits = {word: [c for c in word] for word in word_freqs.keys()}",highlighted:'splits = {word: [c <span class="hljs-keyword">for</span> c <span class="hljs-keyword">in</span> word] <span class="hljs-keyword">for</span> word <span class="hljs-keyword">in</span> word_freqs.keys()}'}}),zs=new $({props:{code:`def compute_pair_freqs(splits):
    pair_freqs = defaultdict(int)
    for word, freq in word_freqs.items():
        split = splits[word]
        if len(split) == 1:
            continue
        for i in range(len(split) - 1):
            pair = (split[i], split[i + 1])
            pair_freqs[pair] += freq
    return pair_freqs`,highlighted:`<span class="hljs-keyword">def</span> <span class="hljs-title function_">compute_pair_freqs</span>(<span class="hljs-params">splits</span>):
    pair_freqs = defaultdict(<span class="hljs-built_in">int</span>)
    <span class="hljs-keyword">for</span> word, freq <span class="hljs-keyword">in</span> word_freqs.items():
        split = splits[word]
        <span class="hljs-keyword">if</span> <span class="hljs-built_in">len</span>(split) == <span class="hljs-number">1</span>:
            <span class="hljs-keyword">continue</span>
        <span class="hljs-keyword">for</span> i <span class="hljs-keyword">in</span> <span class="hljs-built_in">range</span>(<span class="hljs-built_in">len</span>(split) - <span class="hljs-number">1</span>):
            pair = (split[i], split[i + <span class="hljs-number">1</span>])
            pair_freqs[pair] += freq
    <span class="hljs-keyword">return</span> pair_freqs`}}),Ts=new $({props:{code:`pair_freqs = compute_pair_freqs(splits)

for i, key in enumerate(pair_freqs.keys()):
    print(f"{key}: {pair_freqs[key]}")
    if i >= 5:
        break`,highlighted:`pair_freqs = compute_pair_freqs(splits)

<span class="hljs-keyword">for</span> i, key <span class="hljs-keyword">in</span> <span class="hljs-built_in">enumerate</span>(pair_freqs.keys()):
    <span class="hljs-built_in">print</span>(<span class="hljs-string">f&quot;<span class="hljs-subst">{key}</span>: <span class="hljs-subst">{pair_freqs[key]}</span>&quot;</span>)
    <span class="hljs-keyword">if</span> i &gt;= <span class="hljs-number">5</span>:
        <span class="hljs-keyword">break</span>`}}),Ps=new $({props:{code:`('T', 'h'): 3
('h', 'i'): 3
('i', 's'): 5
('\u0120', 'i'): 2
('\u0120', 't'): 7
('t', 'h'): 3`,highlighted:`(<span class="hljs-string">&#x27;T&#x27;</span>, <span class="hljs-string">&#x27;h&#x27;</span>): <span class="hljs-number">3</span>
(<span class="hljs-string">&#x27;h&#x27;</span>, <span class="hljs-string">&#x27;i&#x27;</span>): <span class="hljs-number">3</span>
(<span class="hljs-string">&#x27;i&#x27;</span>, <span class="hljs-string">&#x27;s&#x27;</span>): <span class="hljs-number">5</span>
(<span class="hljs-string">&#x27;\u0120&#x27;</span>, <span class="hljs-string">&#x27;i&#x27;</span>): <span class="hljs-number">2</span>
(<span class="hljs-string">&#x27;\u0120&#x27;</span>, <span class="hljs-string">&#x27;t&#x27;</span>): <span class="hljs-number">7</span>
(<span class="hljs-string">&#x27;t&#x27;</span>, <span class="hljs-string">&#x27;h&#x27;</span>): <span class="hljs-number">3</span>`}}),Cs=new $({props:{code:`best_pair = ""
max_freq = None

for pair, freq in pair_freqs.items():
    if max_freq is None or max_freq < freq:
        best_pair = pair
        max_freq = freq

print(best_pair, max_freq)`,highlighted:`best_pair = <span class="hljs-string">&quot;&quot;</span>
max_freq = <span class="hljs-literal">None</span>

<span class="hljs-keyword">for</span> pair, freq <span class="hljs-keyword">in</span> pair_freqs.items():
    <span class="hljs-keyword">if</span> max_freq <span class="hljs-keyword">is</span> <span class="hljs-literal">None</span> <span class="hljs-keyword">or</span> max_freq &lt; freq:
        best_pair = pair
        max_freq = freq

<span class="hljs-built_in">print</span>(best_pair, max_freq)`}}),Ns=new $({props:{code:"('\u0120', 't') 7",highlighted:'(<span class="hljs-string">&#x27;\u0120&#x27;</span>, <span class="hljs-string">&#x27;t&#x27;</span>) <span class="hljs-number">7</span>'}}),Os=new $({props:{code:`merges = {("\u0120", "t"): "\u0120t"}
vocab.append("\u0120t")`,highlighted:`merges = {(<span class="hljs-string">&quot;\u0120&quot;</span>, <span class="hljs-string">&quot;t&quot;</span>): <span class="hljs-string">&quot;\u0120t&quot;</span>}
vocab.append(<span class="hljs-string">&quot;\u0120t&quot;</span>)`}}),As=new $({props:{code:`def merge_pair(a, b, splits):
    for word in word_freqs:
        split = splits[word]
        if len(split) == 1:
            continue

        i = 0
        while i < len(split) - 1:
            if split[i] == a and split[i + 1] == b:
                split = split[:i] + [a + b] + split[i + 2 :]
            else:
                i += 1
        splits[word] = split
    return splits`,highlighted:`<span class="hljs-keyword">def</span> <span class="hljs-title function_">merge_pair</span>(<span class="hljs-params">a, b, splits</span>):
    <span class="hljs-keyword">for</span> word <span class="hljs-keyword">in</span> word_freqs:
        split = splits[word]
        <span class="hljs-keyword">if</span> <span class="hljs-built_in">len</span>(split) == <span class="hljs-number">1</span>:
            <span class="hljs-keyword">continue</span>

        i = <span class="hljs-number">0</span>
        <span class="hljs-keyword">while</span> i &lt; <span class="hljs-built_in">len</span>(split) - <span class="hljs-number">1</span>:
            <span class="hljs-keyword">if</span> split[i] == a <span class="hljs-keyword">and</span> split[i + <span class="hljs-number">1</span>] == b:
                split = split[:i] + [a + b] + split[i + <span class="hljs-number">2</span> :]
            <span class="hljs-keyword">else</span>:
                i += <span class="hljs-number">1</span>
        splits[word] = split
    <span class="hljs-keyword">return</span> splits`}}),Ds=new $({props:{code:`splits = merge_pair("\u0120", "t", splits)
print(splits["\u0120trained"])`,highlighted:`splits = merge_pair(<span class="hljs-string">&quot;\u0120&quot;</span>, <span class="hljs-string">&quot;t&quot;</span>, splits)
<span class="hljs-built_in">print</span>(splits[<span class="hljs-string">&quot;\u0120trained&quot;</span>])`}}),Bs=new $({props:{code:"['\u0120t', 'r', 'a', 'i', 'n', 'e', 'd']",highlighted:'[<span class="hljs-string">&#x27;\u0120t&#x27;</span>, <span class="hljs-string">&#x27;r&#x27;</span>, <span class="hljs-string">&#x27;a&#x27;</span>, <span class="hljs-string">&#x27;i&#x27;</span>, <span class="hljs-string">&#x27;n&#x27;</span>, <span class="hljs-string">&#x27;e&#x27;</span>, <span class="hljs-string">&#x27;d&#x27;</span>]'}}),Is=new $({props:{code:`vocab_size = 50

while len(vocab) < vocab_size:
    pair_freqs = compute_pair_freqs(splits)
    best_pair = ""
    max_freq = None
    for pair, freq in pair_freqs.items():
        if max_freq is None or max_freq < freq:
            best_pair = pair
            max_freq = freq
    splits = merge_pair(*best_pair, splits)
    merges[best_pair] = best_pair[0] + best_pair[1]
    vocab.append(best_pair[0] + best_pair[1])`,highlighted:`vocab_size = <span class="hljs-number">50</span>

<span class="hljs-keyword">while</span> <span class="hljs-built_in">len</span>(vocab) &lt; vocab_size:
    pair_freqs = compute_pair_freqs(splits)
    best_pair = <span class="hljs-string">&quot;&quot;</span>
    max_freq = <span class="hljs-literal">None</span>
    <span class="hljs-keyword">for</span> pair, freq <span class="hljs-keyword">in</span> pair_freqs.items():
        <span class="hljs-keyword">if</span> max_freq <span class="hljs-keyword">is</span> <span class="hljs-literal">None</span> <span class="hljs-keyword">or</span> max_freq &lt; freq:
            best_pair = pair
            max_freq = freq
    splits = merge_pair(*best_pair, splits)
    merges[best_pair] = best_pair[<span class="hljs-number">0</span>] + best_pair[<span class="hljs-number">1</span>]
    vocab.append(best_pair[<span class="hljs-number">0</span>] + best_pair[<span class="hljs-number">1</span>])`}}),Ss=new $({props:{code:"print(merges)",highlighted:'<span class="hljs-built_in">print</span>(merges)'}}),Gs=new $({props:{code:`{('\u0120', 't'): '\u0120t', ('i', 's'): 'is', ('e', 'r'): 'er', ('\u0120', 'a'): '\u0120a', ('\u0120t', 'o'): '\u0120to', ('e', 'n'): 'en',
 ('T', 'h'): 'Th', ('Th', 'is'): 'This', ('o', 'u'): 'ou', ('s', 'e'): 'se', ('\u0120to', 'k'): '\u0120tok',
 ('\u0120tok', 'en'): '\u0120token', ('n', 'd'): 'nd', ('\u0120', 'is'): '\u0120is', ('\u0120t', 'h'): '\u0120th', ('\u0120th', 'e'): '\u0120the',
 ('i', 'n'): 'in', ('\u0120a', 'b'): '\u0120ab', ('\u0120token', 'i'): '\u0120tokeni'}`,highlighted:`{(<span class="hljs-string">&#x27;\u0120&#x27;</span>, <span class="hljs-string">&#x27;t&#x27;</span>): <span class="hljs-string">&#x27;\u0120t&#x27;</span>, (<span class="hljs-string">&#x27;i&#x27;</span>, <span class="hljs-string">&#x27;s&#x27;</span>): <span class="hljs-string">&#x27;is&#x27;</span>, (<span class="hljs-string">&#x27;e&#x27;</span>, <span class="hljs-string">&#x27;r&#x27;</span>): <span class="hljs-string">&#x27;er&#x27;</span>, (<span class="hljs-string">&#x27;\u0120&#x27;</span>, <span class="hljs-string">&#x27;a&#x27;</span>): <span class="hljs-string">&#x27;\u0120a&#x27;</span>, (<span class="hljs-string">&#x27;\u0120t&#x27;</span>, <span class="hljs-string">&#x27;o&#x27;</span>): <span class="hljs-string">&#x27;\u0120to&#x27;</span>, (<span class="hljs-string">&#x27;e&#x27;</span>, <span class="hljs-string">&#x27;n&#x27;</span>): <span class="hljs-string">&#x27;en&#x27;</span>,
 (<span class="hljs-string">&#x27;T&#x27;</span>, <span class="hljs-string">&#x27;h&#x27;</span>): <span class="hljs-string">&#x27;Th&#x27;</span>, (<span class="hljs-string">&#x27;Th&#x27;</span>, <span class="hljs-string">&#x27;is&#x27;</span>): <span class="hljs-string">&#x27;This&#x27;</span>, (<span class="hljs-string">&#x27;o&#x27;</span>, <span class="hljs-string">&#x27;u&#x27;</span>): <span class="hljs-string">&#x27;ou&#x27;</span>, (<span class="hljs-string">&#x27;s&#x27;</span>, <span class="hljs-string">&#x27;e&#x27;</span>): <span class="hljs-string">&#x27;se&#x27;</span>, (<span class="hljs-string">&#x27;\u0120to&#x27;</span>, <span class="hljs-string">&#x27;k&#x27;</span>): <span class="hljs-string">&#x27;\u0120tok&#x27;</span>,
 (<span class="hljs-string">&#x27;\u0120tok&#x27;</span>, <span class="hljs-string">&#x27;en&#x27;</span>): <span class="hljs-string">&#x27;\u0120token&#x27;</span>, (<span class="hljs-string">&#x27;n&#x27;</span>, <span class="hljs-string">&#x27;d&#x27;</span>): <span class="hljs-string">&#x27;nd&#x27;</span>, (<span class="hljs-string">&#x27;\u0120&#x27;</span>, <span class="hljs-string">&#x27;is&#x27;</span>): <span class="hljs-string">&#x27;\u0120is&#x27;</span>, (<span class="hljs-string">&#x27;\u0120t&#x27;</span>, <span class="hljs-string">&#x27;h&#x27;</span>): <span class="hljs-string">&#x27;\u0120th&#x27;</span>, (<span class="hljs-string">&#x27;\u0120th&#x27;</span>, <span class="hljs-string">&#x27;e&#x27;</span>): <span class="hljs-string">&#x27;\u0120the&#x27;</span>,
 (<span class="hljs-string">&#x27;i&#x27;</span>, <span class="hljs-string">&#x27;n&#x27;</span>): <span class="hljs-string">&#x27;in&#x27;</span>, (<span class="hljs-string">&#x27;\u0120a&#x27;</span>, <span class="hljs-string">&#x27;b&#x27;</span>): <span class="hljs-string">&#x27;\u0120ab&#x27;</span>, (<span class="hljs-string">&#x27;\u0120token&#x27;</span>, <span class="hljs-string">&#x27;i&#x27;</span>): <span class="hljs-string">&#x27;\u0120tokeni&#x27;</span>}`}}),Ls=new $({props:{code:"print(vocab)",highlighted:'<span class="hljs-built_in">print</span>(vocab)'}}),Hs=new $({props:{code:`['<|endoftext|>', ',', '.', 'C', 'F', 'H', 'T', 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'k', 'l', 'm', 'n', 'o',
 'p', 'r', 's', 't', 'u', 'v', 'w', 'y', 'z', '\u0120', '\u0120t', 'is', 'er', '\u0120a', '\u0120to', 'en', 'Th', 'This', 'ou', 'se',
 '\u0120tok', '\u0120token', 'nd', '\u0120is', '\u0120th', '\u0120the', 'in', '\u0120ab', '\u0120tokeni']`,highlighted:`[<span class="hljs-string">&#x27;&lt;|endoftext|&gt;&#x27;</span>, <span class="hljs-string">&#x27;,&#x27;</span>, <span class="hljs-string">&#x27;.&#x27;</span>, <span class="hljs-string">&#x27;C&#x27;</span>, <span class="hljs-string">&#x27;F&#x27;</span>, <span class="hljs-string">&#x27;H&#x27;</span>, <span class="hljs-string">&#x27;T&#x27;</span>, <span class="hljs-string">&#x27;a&#x27;</span>, <span class="hljs-string">&#x27;b&#x27;</span>, <span class="hljs-string">&#x27;c&#x27;</span>, <span class="hljs-string">&#x27;d&#x27;</span>, <span class="hljs-string">&#x27;e&#x27;</span>, <span class="hljs-string">&#x27;f&#x27;</span>, <span class="hljs-string">&#x27;g&#x27;</span>, <span class="hljs-string">&#x27;h&#x27;</span>, <span class="hljs-string">&#x27;i&#x27;</span>, <span class="hljs-string">&#x27;k&#x27;</span>, <span class="hljs-string">&#x27;l&#x27;</span>, <span class="hljs-string">&#x27;m&#x27;</span>, <span class="hljs-string">&#x27;n&#x27;</span>, <span class="hljs-string">&#x27;o&#x27;</span>,
 <span class="hljs-string">&#x27;p&#x27;</span>, <span class="hljs-string">&#x27;r&#x27;</span>, <span class="hljs-string">&#x27;s&#x27;</span>, <span class="hljs-string">&#x27;t&#x27;</span>, <span class="hljs-string">&#x27;u&#x27;</span>, <span class="hljs-string">&#x27;v&#x27;</span>, <span class="hljs-string">&#x27;w&#x27;</span>, <span class="hljs-string">&#x27;y&#x27;</span>, <span class="hljs-string">&#x27;z&#x27;</span>, <span class="hljs-string">&#x27;\u0120&#x27;</span>, <span class="hljs-string">&#x27;\u0120t&#x27;</span>, <span class="hljs-string">&#x27;is&#x27;</span>, <span class="hljs-string">&#x27;er&#x27;</span>, <span class="hljs-string">&#x27;\u0120a&#x27;</span>, <span class="hljs-string">&#x27;\u0120to&#x27;</span>, <span class="hljs-string">&#x27;en&#x27;</span>, <span class="hljs-string">&#x27;Th&#x27;</span>, <span class="hljs-string">&#x27;This&#x27;</span>, <span class="hljs-string">&#x27;ou&#x27;</span>, <span class="hljs-string">&#x27;se&#x27;</span>,
 <span class="hljs-string">&#x27;\u0120tok&#x27;</span>, <span class="hljs-string">&#x27;\u0120token&#x27;</span>, <span class="hljs-string">&#x27;nd&#x27;</span>, <span class="hljs-string">&#x27;\u0120is&#x27;</span>, <span class="hljs-string">&#x27;\u0120th&#x27;</span>, <span class="hljs-string">&#x27;\u0120the&#x27;</span>, <span class="hljs-string">&#x27;in&#x27;</span>, <span class="hljs-string">&#x27;\u0120ab&#x27;</span>, <span class="hljs-string">&#x27;\u0120tokeni&#x27;</span>]`}}),ns=new we({props:{$$slots:{default:[Gr]},$$scope:{ctx:O}}}),Rs=new $({props:{code:`def tokenize(text):
    pre_tokenize_result = tokenizer._tokenizer.pre_tokenizer.pre_tokenize_str(text)
    pre_tokenized_text = [word for word, offset in pre_tokenize_result]
    splits = [[l for l in word] for word in pre_tokenized_text]
    for pair, merge in merges.items():
        for idx, split in enumerate(splits):
            i = 0
            while i < len(split) - 1:
                if split[i] == pair[0] and split[i + 1] == pair[1]:
                    split = split[:i] + [merge] + split[i + 2 :]
                else:
                    i += 1
            splits[idx] = split

    return sum(splits, [])`,highlighted:`<span class="hljs-keyword">def</span> <span class="hljs-title function_">tokenize</span>(<span class="hljs-params">text</span>):
    pre_tokenize_result = tokenizer._tokenizer.pre_tokenizer.pre_tokenize_str(text)
    pre_tokenized_text = [word <span class="hljs-keyword">for</span> word, offset <span class="hljs-keyword">in</span> pre_tokenize_result]
    splits = [[l <span class="hljs-keyword">for</span> l <span class="hljs-keyword">in</span> word] <span class="hljs-keyword">for</span> word <span class="hljs-keyword">in</span> pre_tokenized_text]
    <span class="hljs-keyword">for</span> pair, merge <span class="hljs-keyword">in</span> merges.items():
        <span class="hljs-keyword">for</span> idx, split <span class="hljs-keyword">in</span> <span class="hljs-built_in">enumerate</span>(splits):
            i = <span class="hljs-number">0</span>
            <span class="hljs-keyword">while</span> i &lt; <span class="hljs-built_in">len</span>(split) - <span class="hljs-number">1</span>:
                <span class="hljs-keyword">if</span> split[i] == pair[<span class="hljs-number">0</span>] <span class="hljs-keyword">and</span> split[i + <span class="hljs-number">1</span>] == pair[<span class="hljs-number">1</span>]:
                    split = split[:i] + [merge] + split[i + <span class="hljs-number">2</span> :]
                <span class="hljs-keyword">else</span>:
                    i += <span class="hljs-number">1</span>
            splits[idx] = split

    <span class="hljs-keyword">return</span> <span class="hljs-built_in">sum</span>(splits, [])`}}),Ws=new $({props:{code:'tokenize("This is not a token.")',highlighted:'tokenize(<span class="hljs-string">&quot;This is not a token.&quot;</span>)'}}),Fs=new $({props:{code:"['This', '\u0120is', '\u0120', 'n', 'o', 't', '\u0120a', '\u0120token', '.']",highlighted:'[<span class="hljs-string">&#x27;This&#x27;</span>, <span class="hljs-string">&#x27;\u0120is&#x27;</span>, <span class="hljs-string">&#x27;\u0120&#x27;</span>, <span class="hljs-string">&#x27;n&#x27;</span>, <span class="hljs-string">&#x27;o&#x27;</span>, <span class="hljs-string">&#x27;t&#x27;</span>, <span class="hljs-string">&#x27;\u0120a&#x27;</span>, <span class="hljs-string">&#x27;\u0120token&#x27;</span>, <span class="hljs-string">&#x27;.&#x27;</span>]'}}),ls=new we({props:{warning:!0,$$slots:{default:[Lr]},$$scope:{ctx:O}}}),{c(){c=r("meta"),_=h(),f=r("h1"),b=r("a"),E=r("span"),g(k.$$.fragment),y=h(),T=r("span"),R=n("Byte-Pair Encoding tokenization"),A=h(),g(S.$$.fragment),is=h(),Ms=r("p"),Ka=n("Byte-Pair Encoding (BPE) was initially developed as an algorithm to compress texts, and then used by OpenAI for tokenization when pretraining the GPT model. It\u2019s used by a lot of Transformer models, including GPT, GPT-2, RoBERTa, BART, and DeBERTa."),jt=h(),g(os.$$.fragment),wt=h(),g(V.$$.fragment),xt=h(),W=r("h2"),M=r("a"),xe=r("span"),g(hs.$$.fragment),Ya=h(),be=r("span"),Ja=n("Training algorithm"),bt=h(),Ks=r("p"),Za=n("BPE training starts by computing the unique set of words used in the corpus (after the normalization and pre-tokenization steps are completed), then building the vocabulary by taking all the symbols used to write those words. As a very simple example, let\u2019s say our corpus uses these five words:"),$t=h(),g(us.$$.fragment),kt=h(),K=r("p"),Qa=n("The base vocabulary will then be "),$e=r("code"),Xa=n('["b", "g", "h", "n", "p", "s", "u"]'),sn=n(". For real-world cases, that base vocabulary will contain all the ASCII characters, at the very least, and probably some Unicode characters as well. If an example you are tokenizing uses a character that is not in the training corpus, that character will be converted to the unknown token. That\u2019s one reason why lots of NLP models are very bad at analyzing content with emojis, for instance."),qt=h(),g(Y.$$.fragment),_t=h(),J=r("p"),en=n("After getting this base vocabulary, we add new tokens until the desired vocabulary size is reached by learning "),ke=r("em"),tn=n("merges"),an=n(", which are rules to merge two elements of the existing vocabulary together into a new one. So, at the beginning these merges will create tokens with two characters, and then, as training progresses, longer subwords."),yt=h(),Ys=r("p"),nn=n("At any step during the tokenizer training, the BPE algorithm will search for the most frequent pair of existing tokens (by \u201Cpair,\u201D here we mean two consecutive tokens in a word). That most frequent pair is the one that will be merged, and we rinse and repeat for the next step."),vt=h(),Js=r("p"),ln=n("Going back to our previous example, let\u2019s assume the words had the following frequencies:"),Et=h(),g(cs.$$.fragment),zt=h(),C=r("p"),pn=n("meaning "),qe=r("code"),rn=n('"hug"'),on=n(" was present 10 times in the corpus, "),_e=r("code"),hn=n('"pug"'),un=n(" 5 times, "),ye=r("code"),cn=n('"pun"'),fn=n(" 12 times, "),ve=r("code"),gn=n('"bun"'),mn=n(" 4 times, and "),Ee=r("code"),dn=n('"hugs"'),jn=n(" 5 times. We start the training by splitting each word into characters (the ones that form our initial vocabulary) so we can see each word as a list of tokens:"),Tt=h(),g(fs.$$.fragment),Pt=h(),z=r("p"),wn=n("Then we look at pairs. The pair "),ze=r("code"),xn=n('("h", "u")'),bn=n(" is present in the words "),Te=r("code"),$n=n('"hug"'),kn=n(" and "),Pe=r("code"),qn=n('"hugs"'),_n=n(", so 15 times total in the corpus. It\u2019s not the most frequent pair, though: that honor belongs to "),Ce=r("code"),yn=n('("u", "g")'),vn=n(", which is present in "),Ne=r("code"),En=n('"hug"'),zn=n(", "),Oe=r("code"),Tn=n('"pug"'),Pn=n(", and "),Ae=r("code"),Cn=n('"hugs"'),Nn=n(", for a grand total of 20 times in the vocabulary."),Ct=h(),G=r("p"),On=n("Thus, the first merge rule learned by the tokenizer is "),De=r("code"),An=n('("u", "g") -> "ug"'),Dn=n(", which means that "),Be=r("code"),Bn=n('"ug"'),In=n(" will be added to the vocabulary, and the pair should be merged in all the words of the corpus. At the end of this stage, the vocabulary and corpus look like this:"),Nt=h(),g(gs.$$.fragment),Ot=h(),D=r("p"),Sn=n("Now we have some pairs that result in a token longer than two characters: the pair "),Ie=r("code"),Gn=n('("h", "ug")'),Ln=n(", for instance (present 15 times in the corpus). The most frequent pair at this stage is "),Se=r("code"),Hn=n('("u", "n")'),Rn=n(", however, present 16 times in the corpus, so the second merge rule learned is "),Ge=r("code"),Wn=n('("u", "n") -> "un"'),Fn=n(". Adding that to the vocabulary and merging all existing occurrences leads us to:"),At=h(),g(ms.$$.fragment),Dt=h(),L=r("p"),Un=n("Now the most frequent pair is "),Le=r("code"),Vn=n('("h", "ug")'),Mn=n(", so we learn the merge rule "),He=r("code"),Kn=n('("h", "ug") -> "hug"'),Yn=n(", which gives us our first three-letter token. After the merge, the corpus looks like this:"),Bt=h(),g(ds.$$.fragment),It=h(),Zs=r("p"),Jn=n("And we continue like this until we reach the desired vocabulary size."),St=h(),g(Z.$$.fragment),Gt=h(),F=r("h2"),Q=r("a"),Re=r("span"),g(js.$$.fragment),Zn=h(),We=r("span"),Qn=n("Tokenization algorithm"),Lt=h(),Qs=r("p"),Xn=n("Tokenization follows the training process closely, in the sense that new inputs are tokenized by applying the following steps:"),Ht=h(),B=r("ol"),Fe=r("li"),sl=n("Normalization"),el=h(),Ue=r("li"),tl=n("Pre-tokenization"),al=h(),Ve=r("li"),nl=n("Splitting the words into individual characters"),ll=h(),Me=r("li"),pl=n("Applying the merge rules learned in order on those splits"),Rt=h(),Xs=r("p"),rl=n("Let\u2019s take the example we used during training, with the three merge rules learned:"),Wt=h(),g(ws.$$.fragment),Ft=h(),q=r("p"),il=n("The word "),Ke=r("code"),ol=n('"bug"'),hl=n(" will be tokenized as "),Ye=r("code"),ul=n('["b", "ug"]'),cl=n(". "),Je=r("code"),fl=n('"mug"'),gl=n(", however, will be tokenized as "),Ze=r("code"),ml=n('["[UNK]", "ug"]'),dl=n(" since the letter "),Qe=r("code"),jl=n('"m"'),wl=n(" was not in the base vocabulary. Likewise, the word "),Xe=r("code"),xl=n('"thug"'),bl=n(" will be tokenized as "),st=r("code"),$l=n('["[UNK]", "hug"]'),kl=n(": the letter "),et=r("code"),ql=n('"t"'),_l=n(" is not in the base vocabulary, and applying the merge rules results first in "),tt=r("code"),yl=n('"u"'),vl=n(" and "),at=r("code"),El=n('"g"'),zl=n(" being merged and then "),nt=r("code"),Tl=n('"hu"'),Pl=n(" and "),lt=r("code"),Cl=n('"g"'),Nl=n(" being merged."),Ut=h(),g(X.$$.fragment),Vt=h(),U=r("h2"),ss=r("a"),pt=r("span"),g(xs.$$.fragment),Ol=h(),rt=r("span"),Al=n("Implementing BPE"),Mt=h(),se=r("p"),Dl=n("Now let\u2019s take a look at an implementation of the BPE algorithm. This won\u2019t be an optimized version you can actually use on a big corpus; we just want to show you the code so you can understand the algorithm a little bit better."),Kt=h(),ee=r("p"),Bl=n("First we need a corpus, so let\u2019s create a simple one with a few sentences:"),Yt=h(),g(bs.$$.fragment),Jt=h(),es=r("p"),Il=n("Next, we need to pre-tokenize that corpus into words. Since we are replicating a BPE tokenizer (like GPT-2), we will use the "),it=r("code"),Sl=n("gpt2"),Gl=n(" tokenizer for the pre-tokenization:"),Zt=h(),g($s.$$.fragment),Qt=h(),te=r("p"),Ll=n("Then we compute the frequencies of each word in the corpus as we do the pre-tokenization:"),Xt=h(),g(ks.$$.fragment),sa=h(),g(qs.$$.fragment),ea=h(),ae=r("p"),Hl=n("The next step is to compute the base vocabulary, formed by all the characters used in the corpus:"),ta=h(),g(_s.$$.fragment),aa=h(),g(ys.$$.fragment),na=h(),ts=r("p"),Rl=n("We also add the special tokens used by the model at the beginning of that vocabulary. In the case of GPT-2, the only special token is "),ot=r("code"),Wl=n('"<|endoftext|>"'),Fl=n(":"),la=h(),g(vs.$$.fragment),pa=h(),ne=r("p"),Ul=n("We now need to split each word into individual characters, to be able to start training:"),ra=h(),g(Es.$$.fragment),ia=h(),le=r("p"),Vl=n("Now that we are ready for training, let\u2019s write a function that computes the frequency of each pair. We\u2019ll need to use this at each step of the training:"),oa=h(),g(zs.$$.fragment),ha=h(),pe=r("p"),Ml=n("Let\u2019s have a look at a part of this dictionary after the initial splits:"),ua=h(),g(Ts.$$.fragment),ca=h(),g(Ps.$$.fragment),fa=h(),re=r("p"),Kl=n("Now, finding the most frequent pair only takes a quick loop:"),ga=h(),g(Cs.$$.fragment),ma=h(),g(Ns.$$.fragment),da=h(),H=r("p"),Yl=n("So the first merge to learn is "),ht=r("code"),Jl=n("('\u0120', 't') -> '\u0120t'"),Zl=n(", and we add "),ut=r("code"),Ql=n("'\u0120t'"),Xl=n(" to the vocabulary:"),ja=h(),g(Os.$$.fragment),wa=h(),as=r("p"),sp=n("To continue, we need to apply that merge in our "),ct=r("code"),ep=n("splits"),tp=n(" dictionary. Let\u2019s write another function for this:"),xa=h(),g(As.$$.fragment),ba=h(),ie=r("p"),ap=n("And we can have a look at the result of the first merge:"),$a=h(),g(Ds.$$.fragment),ka=h(),g(Bs.$$.fragment),qa=h(),oe=r("p"),np=n("Now we have everything we need to loop until we have learned all the merges we want. Let\u2019s aim for a vocab size of 50:"),_a=h(),g(Is.$$.fragment),ya=h(),he=r("p"),lp=n("As a result, we\u2019ve learned 19 merge rules (the initial vocabulary had a size of 31 \u2014 30 characters in the alphabet, plus the special token):"),va=h(),g(Ss.$$.fragment),Ea=h(),g(Gs.$$.fragment),za=h(),ue=r("p"),pp=n("And the vocabulary is composed of the special token, the initial alphabet, and all the results of the merges:"),Ta=h(),g(Ls.$$.fragment),Pa=h(),g(Hs.$$.fragment),Ca=h(),g(ns.$$.fragment),Na=h(),ce=r("p"),rp=n("To tokenize a new text, we pre-tokenize it, split it, then apply all the merge rules learned:"),Oa=h(),g(Rs.$$.fragment),Aa=h(),fe=r("p"),ip=n("We can try this on any text composed of characters in the alphabet:"),Da=h(),g(Ws.$$.fragment),Ba=h(),g(Fs.$$.fragment),Ia=h(),g(ls.$$.fragment),Sa=h(),ge=r("p"),op=n("That\u2019s it for the BPE algorithm! Next, we\u2019ll have a look at WordPiece."),this.h()},l(s){const a=Cr('[data-svelte="svelte-1phssyn"]',document.head);c=i(a,"META",{name:!0,content:!0}),a.forEach(e),_=u(s),f=i(s,"H1",{class:!0});var Us=o(f);b=i(Us,"A",{id:!0,class:!0,href:!0});var ft=o(b);E=i(ft,"SPAN",{});var gt=o(E);m(k.$$.fragment,gt),gt.forEach(e),ft.forEach(e),y=u(Us),T=i(Us,"SPAN",{});var mt=o(T);R=l(mt,"Byte-Pair Encoding tokenization"),mt.forEach(e),Us.forEach(e),A=u(s),m(S.$$.fragment,s),is=u(s),Ms=i(s,"P",{});var dt=o(Ms);Ka=l(dt,"Byte-Pair Encoding (BPE) was initially developed as an algorithm to compress texts, and then used by OpenAI for tokenization when pretraining the GPT model. It\u2019s used by a lot of Transformer models, including GPT, GPT-2, RoBERTa, BART, and DeBERTa."),dt.forEach(e),jt=u(s),m(os.$$.fragment,s),wt=u(s),m(V.$$.fragment,s),xt=u(s),W=i(s,"H2",{class:!0});var Vs=o(W);M=i(Vs,"A",{id:!0,class:!0,href:!0});var hp=o(M);xe=i(hp,"SPAN",{});var up=o(xe);m(hs.$$.fragment,up),up.forEach(e),hp.forEach(e),Ya=u(Vs),be=i(Vs,"SPAN",{});var cp=o(be);Ja=l(cp,"Training algorithm"),cp.forEach(e),Vs.forEach(e),bt=u(s),Ks=i(s,"P",{});var fp=o(Ks);Za=l(fp,"BPE training starts by computing the unique set of words used in the corpus (after the normalization and pre-tokenization steps are completed), then building the vocabulary by taking all the symbols used to write those words. As a very simple example, let\u2019s say our corpus uses these five words:"),fp.forEach(e),$t=u(s),m(us.$$.fragment,s),kt=u(s),K=i(s,"P",{});var La=o(K);Qa=l(La,"The base vocabulary will then be "),$e=i(La,"CODE",{});var gp=o($e);Xa=l(gp,'["b", "g", "h", "n", "p", "s", "u"]'),gp.forEach(e),sn=l(La,". For real-world cases, that base vocabulary will contain all the ASCII characters, at the very least, and probably some Unicode characters as well. If an example you are tokenizing uses a character that is not in the training corpus, that character will be converted to the unknown token. That\u2019s one reason why lots of NLP models are very bad at analyzing content with emojis, for instance."),La.forEach(e),qt=u(s),m(Y.$$.fragment,s),_t=u(s),J=i(s,"P",{});var Ha=o(J);en=l(Ha,"After getting this base vocabulary, we add new tokens until the desired vocabulary size is reached by learning "),ke=i(Ha,"EM",{});var mp=o(ke);tn=l(mp,"merges"),mp.forEach(e),an=l(Ha,", which are rules to merge two elements of the existing vocabulary together into a new one. So, at the beginning these merges will create tokens with two characters, and then, as training progresses, longer subwords."),Ha.forEach(e),yt=u(s),Ys=i(s,"P",{});var dp=o(Ys);nn=l(dp,"At any step during the tokenizer training, the BPE algorithm will search for the most frequent pair of existing tokens (by \u201Cpair,\u201D here we mean two consecutive tokens in a word). That most frequent pair is the one that will be merged, and we rinse and repeat for the next step."),dp.forEach(e),vt=u(s),Js=i(s,"P",{});var jp=o(Js);ln=l(jp,"Going back to our previous example, let\u2019s assume the words had the following frequencies:"),jp.forEach(e),Et=u(s),m(cs.$$.fragment,s),zt=u(s),C=i(s,"P",{});var I=o(C);pn=l(I,"meaning "),qe=i(I,"CODE",{});var wp=o(qe);rn=l(wp,'"hug"'),wp.forEach(e),on=l(I," was present 10 times in the corpus, "),_e=i(I,"CODE",{});var xp=o(_e);hn=l(xp,'"pug"'),xp.forEach(e),un=l(I," 5 times, "),ye=i(I,"CODE",{});var bp=o(ye);cn=l(bp,'"pun"'),bp.forEach(e),fn=l(I," 12 times, "),ve=i(I,"CODE",{});var $p=o(ve);gn=l($p,'"bun"'),$p.forEach(e),mn=l(I," 4 times, and "),Ee=i(I,"CODE",{});var kp=o(Ee);dn=l(kp,'"hugs"'),kp.forEach(e),jn=l(I," 5 times. We start the training by splitting each word into characters (the ones that form our initial vocabulary) so we can see each word as a list of tokens:"),I.forEach(e),Tt=u(s),m(fs.$$.fragment,s),Pt=u(s),z=i(s,"P",{});var N=o(z);wn=l(N,"Then we look at pairs. The pair "),ze=i(N,"CODE",{});var qp=o(ze);xn=l(qp,'("h", "u")'),qp.forEach(e),bn=l(N," is present in the words "),Te=i(N,"CODE",{});var _p=o(Te);$n=l(_p,'"hug"'),_p.forEach(e),kn=l(N," and "),Pe=i(N,"CODE",{});var yp=o(Pe);qn=l(yp,'"hugs"'),yp.forEach(e),_n=l(N,", so 15 times total in the corpus. It\u2019s not the most frequent pair, though: that honor belongs to "),Ce=i(N,"CODE",{});var vp=o(Ce);yn=l(vp,'("u", "g")'),vp.forEach(e),vn=l(N,", which is present in "),Ne=i(N,"CODE",{});var Ep=o(Ne);En=l(Ep,'"hug"'),Ep.forEach(e),zn=l(N,", "),Oe=i(N,"CODE",{});var zp=o(Oe);Tn=l(zp,'"pug"'),zp.forEach(e),Pn=l(N,", and "),Ae=i(N,"CODE",{});var Tp=o(Ae);Cn=l(Tp,'"hugs"'),Tp.forEach(e),Nn=l(N,", for a grand total of 20 times in the vocabulary."),N.forEach(e),Ct=u(s),G=i(s,"P",{});var me=o(G);On=l(me,"Thus, the first merge rule learned by the tokenizer is "),De=i(me,"CODE",{});var Pp=o(De);An=l(Pp,'("u", "g") -> "ug"'),Pp.forEach(e),Dn=l(me,", which means that "),Be=i(me,"CODE",{});var Cp=o(Be);Bn=l(Cp,'"ug"'),Cp.forEach(e),In=l(me," will be added to the vocabulary, and the pair should be merged in all the words of the corpus. At the end of this stage, the vocabulary and corpus look like this:"),me.forEach(e),Nt=u(s),m(gs.$$.fragment,s),Ot=u(s),D=i(s,"P",{});var ps=o(D);Sn=l(ps,"Now we have some pairs that result in a token longer than two characters: the pair "),Ie=i(ps,"CODE",{});var Np=o(Ie);Gn=l(Np,'("h", "ug")'),Np.forEach(e),Ln=l(ps,", for instance (present 15 times in the corpus). The most frequent pair at this stage is "),Se=i(ps,"CODE",{});var Op=o(Se);Hn=l(Op,'("u", "n")'),Op.forEach(e),Rn=l(ps,", however, present 16 times in the corpus, so the second merge rule learned is "),Ge=i(ps,"CODE",{});var Ap=o(Ge);Wn=l(Ap,'("u", "n") -> "un"'),Ap.forEach(e),Fn=l(ps,". Adding that to the vocabulary and merging all existing occurrences leads us to:"),ps.forEach(e),At=u(s),m(ms.$$.fragment,s),Dt=u(s),L=i(s,"P",{});var de=o(L);Un=l(de,"Now the most frequent pair is "),Le=i(de,"CODE",{});var Dp=o(Le);Vn=l(Dp,'("h", "ug")'),Dp.forEach(e),Mn=l(de,", so we learn the merge rule "),He=i(de,"CODE",{});var Bp=o(He);Kn=l(Bp,'("h", "ug") -> "hug"'),Bp.forEach(e),Yn=l(de,", which gives us our first three-letter token. After the merge, the corpus looks like this:"),de.forEach(e),Bt=u(s),m(ds.$$.fragment,s),It=u(s),Zs=i(s,"P",{});var Ip=o(Zs);Jn=l(Ip,"And we continue like this until we reach the desired vocabulary size."),Ip.forEach(e),St=u(s),m(Z.$$.fragment,s),Gt=u(s),F=i(s,"H2",{class:!0});var Ra=o(F);Q=i(Ra,"A",{id:!0,class:!0,href:!0});var Sp=o(Q);Re=i(Sp,"SPAN",{});var Gp=o(Re);m(js.$$.fragment,Gp),Gp.forEach(e),Sp.forEach(e),Zn=u(Ra),We=i(Ra,"SPAN",{});var Lp=o(We);Qn=l(Lp,"Tokenization algorithm"),Lp.forEach(e),Ra.forEach(e),Lt=u(s),Qs=i(s,"P",{});var Hp=o(Qs);Xn=l(Hp,"Tokenization follows the training process closely, in the sense that new inputs are tokenized by applying the following steps:"),Hp.forEach(e),Ht=u(s),B=i(s,"OL",{});var rs=o(B);Fe=i(rs,"LI",{});var Rp=o(Fe);sl=l(Rp,"Normalization"),Rp.forEach(e),el=u(rs),Ue=i(rs,"LI",{});var Wp=o(Ue);tl=l(Wp,"Pre-tokenization"),Wp.forEach(e),al=u(rs),Ve=i(rs,"LI",{});var Fp=o(Ve);nl=l(Fp,"Splitting the words into individual characters"),Fp.forEach(e),ll=u(rs),Me=i(rs,"LI",{});var Up=o(Me);pl=l(Up,"Applying the merge rules learned in order on those splits"),Up.forEach(e),rs.forEach(e),Rt=u(s),Xs=i(s,"P",{});var Vp=o(Xs);rl=l(Vp,"Let\u2019s take the example we used during training, with the three merge rules learned:"),Vp.forEach(e),Wt=u(s),m(ws.$$.fragment,s),Ft=u(s),q=i(s,"P",{});var v=o(q);il=l(v,"The word "),Ke=i(v,"CODE",{});var Mp=o(Ke);ol=l(Mp,'"bug"'),Mp.forEach(e),hl=l(v," will be tokenized as "),Ye=i(v,"CODE",{});var Kp=o(Ye);ul=l(Kp,'["b", "ug"]'),Kp.forEach(e),cl=l(v,". "),Je=i(v,"CODE",{});var Yp=o(Je);fl=l(Yp,'"mug"'),Yp.forEach(e),gl=l(v,", however, will be tokenized as "),Ze=i(v,"CODE",{});var Jp=o(Ze);ml=l(Jp,'["[UNK]", "ug"]'),Jp.forEach(e),dl=l(v," since the letter "),Qe=i(v,"CODE",{});var Zp=o(Qe);jl=l(Zp,'"m"'),Zp.forEach(e),wl=l(v," was not in the base vocabulary. Likewise, the word "),Xe=i(v,"CODE",{});var Qp=o(Xe);xl=l(Qp,'"thug"'),Qp.forEach(e),bl=l(v," will be tokenized as "),st=i(v,"CODE",{});var Xp=o(st);$l=l(Xp,'["[UNK]", "hug"]'),Xp.forEach(e),kl=l(v,": the letter "),et=i(v,"CODE",{});var sr=o(et);ql=l(sr,'"t"'),sr.forEach(e),_l=l(v," is not in the base vocabulary, and applying the merge rules results first in "),tt=i(v,"CODE",{});var er=o(tt);yl=l(er,'"u"'),er.forEach(e),vl=l(v," and "),at=i(v,"CODE",{});var tr=o(at);El=l(tr,'"g"'),tr.forEach(e),zl=l(v," being merged and then "),nt=i(v,"CODE",{});var ar=o(nt);Tl=l(ar,'"hu"'),ar.forEach(e),Pl=l(v," and "),lt=i(v,"CODE",{});var nr=o(lt);Cl=l(nr,'"g"'),nr.forEach(e),Nl=l(v," being merged."),v.forEach(e),Ut=u(s),m(X.$$.fragment,s),Vt=u(s),U=i(s,"H2",{class:!0});var Wa=o(U);ss=i(Wa,"A",{id:!0,class:!0,href:!0});var lr=o(ss);pt=i(lr,"SPAN",{});var pr=o(pt);m(xs.$$.fragment,pr),pr.forEach(e),lr.forEach(e),Ol=u(Wa),rt=i(Wa,"SPAN",{});var rr=o(rt);Al=l(rr,"Implementing BPE"),rr.forEach(e),Wa.forEach(e),Mt=u(s),se=i(s,"P",{});var ir=o(se);Dl=l(ir,"Now let\u2019s take a look at an implementation of the BPE algorithm. This won\u2019t be an optimized version you can actually use on a big corpus; we just want to show you the code so you can understand the algorithm a little bit better."),ir.forEach(e),Kt=u(s),ee=i(s,"P",{});var or=o(ee);Bl=l(or,"First we need a corpus, so let\u2019s create a simple one with a few sentences:"),or.forEach(e),Yt=u(s),m(bs.$$.fragment,s),Jt=u(s),es=i(s,"P",{});var Fa=o(es);Il=l(Fa,"Next, we need to pre-tokenize that corpus into words. Since we are replicating a BPE tokenizer (like GPT-2), we will use the "),it=i(Fa,"CODE",{});var hr=o(it);Sl=l(hr,"gpt2"),hr.forEach(e),Gl=l(Fa," tokenizer for the pre-tokenization:"),Fa.forEach(e),Zt=u(s),m($s.$$.fragment,s),Qt=u(s),te=i(s,"P",{});var ur=o(te);Ll=l(ur,"Then we compute the frequencies of each word in the corpus as we do the pre-tokenization:"),ur.forEach(e),Xt=u(s),m(ks.$$.fragment,s),sa=u(s),m(qs.$$.fragment,s),ea=u(s),ae=i(s,"P",{});var cr=o(ae);Hl=l(cr,"The next step is to compute the base vocabulary, formed by all the characters used in the corpus:"),cr.forEach(e),ta=u(s),m(_s.$$.fragment,s),aa=u(s),m(ys.$$.fragment,s),na=u(s),ts=i(s,"P",{});var Ua=o(ts);Rl=l(Ua,"We also add the special tokens used by the model at the beginning of that vocabulary. In the case of GPT-2, the only special token is "),ot=i(Ua,"CODE",{});var fr=o(ot);Wl=l(fr,'"<|endoftext|>"'),fr.forEach(e),Fl=l(Ua,":"),Ua.forEach(e),la=u(s),m(vs.$$.fragment,s),pa=u(s),ne=i(s,"P",{});var gr=o(ne);Ul=l(gr,"We now need to split each word into individual characters, to be able to start training:"),gr.forEach(e),ra=u(s),m(Es.$$.fragment,s),ia=u(s),le=i(s,"P",{});var mr=o(le);Vl=l(mr,"Now that we are ready for training, let\u2019s write a function that computes the frequency of each pair. We\u2019ll need to use this at each step of the training:"),mr.forEach(e),oa=u(s),m(zs.$$.fragment,s),ha=u(s),pe=i(s,"P",{});var dr=o(pe);Ml=l(dr,"Let\u2019s have a look at a part of this dictionary after the initial splits:"),dr.forEach(e),ua=u(s),m(Ts.$$.fragment,s),ca=u(s),m(Ps.$$.fragment,s),fa=u(s),re=i(s,"P",{});var jr=o(re);Kl=l(jr,"Now, finding the most frequent pair only takes a quick loop:"),jr.forEach(e),ga=u(s),m(Cs.$$.fragment,s),ma=u(s),m(Ns.$$.fragment,s),da=u(s),H=i(s,"P",{});var je=o(H);Yl=l(je,"So the first merge to learn is "),ht=i(je,"CODE",{});var wr=o(ht);Jl=l(wr,"('\u0120', 't') -> '\u0120t'"),wr.forEach(e),Zl=l(je,", and we add "),ut=i(je,"CODE",{});var xr=o(ut);Ql=l(xr,"'\u0120t'"),xr.forEach(e),Xl=l(je," to the vocabulary:"),je.forEach(e),ja=u(s),m(Os.$$.fragment,s),wa=u(s),as=i(s,"P",{});var Va=o(as);sp=l(Va,"To continue, we need to apply that merge in our "),ct=i(Va,"CODE",{});var br=o(ct);ep=l(br,"splits"),br.forEach(e),tp=l(Va," dictionary. Let\u2019s write another function for this:"),Va.forEach(e),xa=u(s),m(As.$$.fragment,s),ba=u(s),ie=i(s,"P",{});var $r=o(ie);ap=l($r,"And we can have a look at the result of the first merge:"),$r.forEach(e),$a=u(s),m(Ds.$$.fragment,s),ka=u(s),m(Bs.$$.fragment,s),qa=u(s),oe=i(s,"P",{});var kr=o(oe);np=l(kr,"Now we have everything we need to loop until we have learned all the merges we want. Let\u2019s aim for a vocab size of 50:"),kr.forEach(e),_a=u(s),m(Is.$$.fragment,s),ya=u(s),he=i(s,"P",{});var qr=o(he);lp=l(qr,"As a result, we\u2019ve learned 19 merge rules (the initial vocabulary had a size of 31 \u2014 30 characters in the alphabet, plus the special token):"),qr.forEach(e),va=u(s),m(Ss.$$.fragment,s),Ea=u(s),m(Gs.$$.fragment,s),za=u(s),ue=i(s,"P",{});var _r=o(ue);pp=l(_r,"And the vocabulary is composed of the special token, the initial alphabet, and all the results of the merges:"),_r.forEach(e),Ta=u(s),m(Ls.$$.fragment,s),Pa=u(s),m(Hs.$$.fragment,s),Ca=u(s),m(ns.$$.fragment,s),Na=u(s),ce=i(s,"P",{});var yr=o(ce);rp=l(yr,"To tokenize a new text, we pre-tokenize it, split it, then apply all the merge rules learned:"),yr.forEach(e),Oa=u(s),m(Rs.$$.fragment,s),Aa=u(s),fe=i(s,"P",{});var vr=o(fe);ip=l(vr,"We can try this on any text composed of characters in the alphabet:"),vr.forEach(e),Da=u(s),m(Ws.$$.fragment,s),Ba=u(s),m(Fs.$$.fragment,s),Ia=u(s),m(ls.$$.fragment,s),Sa=u(s),ge=i(s,"P",{});var Er=o(ge);op=l(Er,"That\u2019s it for the BPE algorithm! Next, we\u2019ll have a look at WordPiece."),Er.forEach(e),this.h()},h(){P(c,"name","hf:doc:metadata"),P(c,"content",JSON.stringify(Rr)),P(b,"id","bytepair-encoding-tokenization"),P(b,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),P(b,"href","#bytepair-encoding-tokenization"),P(f,"class","relative group"),P(M,"id","training-algorithm"),P(M,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),P(M,"href","#training-algorithm"),P(W,"class","relative group"),P(Q,"id","tokenization-algorithm"),P(Q,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),P(Q,"href","#tokenization-algorithm"),P(F,"class","relative group"),P(ss,"id","implementing-bpe"),P(ss,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),P(ss,"href","#implementing-bpe"),P(U,"class","relative group")},m(s,a){t(document.head,c),p(s,_,a),p(s,f,a),t(f,b),t(b,E),d(k,E,null),t(f,y),t(f,T),t(T,R),p(s,A,a),d(S,s,a),p(s,is,a),p(s,Ms,a),t(Ms,Ka),p(s,jt,a),d(os,s,a),p(s,wt,a),d(V,s,a),p(s,xt,a),p(s,W,a),t(W,M),t(M,xe),d(hs,xe,null),t(W,Ya),t(W,be),t(be,Ja),p(s,bt,a),p(s,Ks,a),t(Ks,Za),p(s,$t,a),d(us,s,a),p(s,kt,a),p(s,K,a),t(K,Qa),t(K,$e),t($e,Xa),t(K,sn),p(s,qt,a),d(Y,s,a),p(s,_t,a),p(s,J,a),t(J,en),t(J,ke),t(ke,tn),t(J,an),p(s,yt,a),p(s,Ys,a),t(Ys,nn),p(s,vt,a),p(s,Js,a),t(Js,ln),p(s,Et,a),d(cs,s,a),p(s,zt,a),p(s,C,a),t(C,pn),t(C,qe),t(qe,rn),t(C,on),t(C,_e),t(_e,hn),t(C,un),t(C,ye),t(ye,cn),t(C,fn),t(C,ve),t(ve,gn),t(C,mn),t(C,Ee),t(Ee,dn),t(C,jn),p(s,Tt,a),d(fs,s,a),p(s,Pt,a),p(s,z,a),t(z,wn),t(z,ze),t(ze,xn),t(z,bn),t(z,Te),t(Te,$n),t(z,kn),t(z,Pe),t(Pe,qn),t(z,_n),t(z,Ce),t(Ce,yn),t(z,vn),t(z,Ne),t(Ne,En),t(z,zn),t(z,Oe),t(Oe,Tn),t(z,Pn),t(z,Ae),t(Ae,Cn),t(z,Nn),p(s,Ct,a),p(s,G,a),t(G,On),t(G,De),t(De,An),t(G,Dn),t(G,Be),t(Be,Bn),t(G,In),p(s,Nt,a),d(gs,s,a),p(s,Ot,a),p(s,D,a),t(D,Sn),t(D,Ie),t(Ie,Gn),t(D,Ln),t(D,Se),t(Se,Hn),t(D,Rn),t(D,Ge),t(Ge,Wn),t(D,Fn),p(s,At,a),d(ms,s,a),p(s,Dt,a),p(s,L,a),t(L,Un),t(L,Le),t(Le,Vn),t(L,Mn),t(L,He),t(He,Kn),t(L,Yn),p(s,Bt,a),d(ds,s,a),p(s,It,a),p(s,Zs,a),t(Zs,Jn),p(s,St,a),d(Z,s,a),p(s,Gt,a),p(s,F,a),t(F,Q),t(Q,Re),d(js,Re,null),t(F,Zn),t(F,We),t(We,Qn),p(s,Lt,a),p(s,Qs,a),t(Qs,Xn),p(s,Ht,a),p(s,B,a),t(B,Fe),t(Fe,sl),t(B,el),t(B,Ue),t(Ue,tl),t(B,al),t(B,Ve),t(Ve,nl),t(B,ll),t(B,Me),t(Me,pl),p(s,Rt,a),p(s,Xs,a),t(Xs,rl),p(s,Wt,a),d(ws,s,a),p(s,Ft,a),p(s,q,a),t(q,il),t(q,Ke),t(Ke,ol),t(q,hl),t(q,Ye),t(Ye,ul),t(q,cl),t(q,Je),t(Je,fl),t(q,gl),t(q,Ze),t(Ze,ml),t(q,dl),t(q,Qe),t(Qe,jl),t(q,wl),t(q,Xe),t(Xe,xl),t(q,bl),t(q,st),t(st,$l),t(q,kl),t(q,et),t(et,ql),t(q,_l),t(q,tt),t(tt,yl),t(q,vl),t(q,at),t(at,El),t(q,zl),t(q,nt),t(nt,Tl),t(q,Pl),t(q,lt),t(lt,Cl),t(q,Nl),p(s,Ut,a),d(X,s,a),p(s,Vt,a),p(s,U,a),t(U,ss),t(ss,pt),d(xs,pt,null),t(U,Ol),t(U,rt),t(rt,Al),p(s,Mt,a),p(s,se,a),t(se,Dl),p(s,Kt,a),p(s,ee,a),t(ee,Bl),p(s,Yt,a),d(bs,s,a),p(s,Jt,a),p(s,es,a),t(es,Il),t(es,it),t(it,Sl),t(es,Gl),p(s,Zt,a),d($s,s,a),p(s,Qt,a),p(s,te,a),t(te,Ll),p(s,Xt,a),d(ks,s,a),p(s,sa,a),d(qs,s,a),p(s,ea,a),p(s,ae,a),t(ae,Hl),p(s,ta,a),d(_s,s,a),p(s,aa,a),d(ys,s,a),p(s,na,a),p(s,ts,a),t(ts,Rl),t(ts,ot),t(ot,Wl),t(ts,Fl),p(s,la,a),d(vs,s,a),p(s,pa,a),p(s,ne,a),t(ne,Ul),p(s,ra,a),d(Es,s,a),p(s,ia,a),p(s,le,a),t(le,Vl),p(s,oa,a),d(zs,s,a),p(s,ha,a),p(s,pe,a),t(pe,Ml),p(s,ua,a),d(Ts,s,a),p(s,ca,a),d(Ps,s,a),p(s,fa,a),p(s,re,a),t(re,Kl),p(s,ga,a),d(Cs,s,a),p(s,ma,a),d(Ns,s,a),p(s,da,a),p(s,H,a),t(H,Yl),t(H,ht),t(ht,Jl),t(H,Zl),t(H,ut),t(ut,Ql),t(H,Xl),p(s,ja,a),d(Os,s,a),p(s,wa,a),p(s,as,a),t(as,sp),t(as,ct),t(ct,ep),t(as,tp),p(s,xa,a),d(As,s,a),p(s,ba,a),p(s,ie,a),t(ie,ap),p(s,$a,a),d(Ds,s,a),p(s,ka,a),d(Bs,s,a),p(s,qa,a),p(s,oe,a),t(oe,np),p(s,_a,a),d(Is,s,a),p(s,ya,a),p(s,he,a),t(he,lp),p(s,va,a),d(Ss,s,a),p(s,Ea,a),d(Gs,s,a),p(s,za,a),p(s,ue,a),t(ue,pp),p(s,Ta,a),d(Ls,s,a),p(s,Pa,a),d(Hs,s,a),p(s,Ca,a),d(ns,s,a),p(s,Na,a),p(s,ce,a),t(ce,rp),p(s,Oa,a),d(Rs,s,a),p(s,Aa,a),p(s,fe,a),t(fe,ip),p(s,Da,a),d(Ws,s,a),p(s,Ba,a),d(Fs,s,a),p(s,Ia,a),d(ls,s,a),p(s,Sa,a),p(s,ge,a),t(ge,op),Ga=!0},p(s,[a]){const Us={};a&2&&(Us.$$scope={dirty:a,ctx:s}),V.$set(Us);const ft={};a&2&&(ft.$$scope={dirty:a,ctx:s}),Y.$set(ft);const gt={};a&2&&(gt.$$scope={dirty:a,ctx:s}),Z.$set(gt);const mt={};a&2&&(mt.$$scope={dirty:a,ctx:s}),X.$set(mt);const dt={};a&2&&(dt.$$scope={dirty:a,ctx:s}),ns.$set(dt);const Vs={};a&2&&(Vs.$$scope={dirty:a,ctx:s}),ls.$set(Vs)},i(s){Ga||(j(k.$$.fragment,s),j(S.$$.fragment,s),j(os.$$.fragment,s),j(V.$$.fragment,s),j(hs.$$.fragment,s),j(us.$$.fragment,s),j(Y.$$.fragment,s),j(cs.$$.fragment,s),j(fs.$$.fragment,s),j(gs.$$.fragment,s),j(ms.$$.fragment,s),j(ds.$$.fragment,s),j(Z.$$.fragment,s),j(js.$$.fragment,s),j(ws.$$.fragment,s),j(X.$$.fragment,s),j(xs.$$.fragment,s),j(bs.$$.fragment,s),j($s.$$.fragment,s),j(ks.$$.fragment,s),j(qs.$$.fragment,s),j(_s.$$.fragment,s),j(ys.$$.fragment,s),j(vs.$$.fragment,s),j(Es.$$.fragment,s),j(zs.$$.fragment,s),j(Ts.$$.fragment,s),j(Ps.$$.fragment,s),j(Cs.$$.fragment,s),j(Ns.$$.fragment,s),j(Os.$$.fragment,s),j(As.$$.fragment,s),j(Ds.$$.fragment,s),j(Bs.$$.fragment,s),j(Is.$$.fragment,s),j(Ss.$$.fragment,s),j(Gs.$$.fragment,s),j(Ls.$$.fragment,s),j(Hs.$$.fragment,s),j(ns.$$.fragment,s),j(Rs.$$.fragment,s),j(Ws.$$.fragment,s),j(Fs.$$.fragment,s),j(ls.$$.fragment,s),Ga=!0)},o(s){w(k.$$.fragment,s),w(S.$$.fragment,s),w(os.$$.fragment,s),w(V.$$.fragment,s),w(hs.$$.fragment,s),w(us.$$.fragment,s),w(Y.$$.fragment,s),w(cs.$$.fragment,s),w(fs.$$.fragment,s),w(gs.$$.fragment,s),w(ms.$$.fragment,s),w(ds.$$.fragment,s),w(Z.$$.fragment,s),w(js.$$.fragment,s),w(ws.$$.fragment,s),w(X.$$.fragment,s),w(xs.$$.fragment,s),w(bs.$$.fragment,s),w($s.$$.fragment,s),w(ks.$$.fragment,s),w(qs.$$.fragment,s),w(_s.$$.fragment,s),w(ys.$$.fragment,s),w(vs.$$.fragment,s),w(Es.$$.fragment,s),w(zs.$$.fragment,s),w(Ts.$$.fragment,s),w(Ps.$$.fragment,s),w(Cs.$$.fragment,s),w(Ns.$$.fragment,s),w(Os.$$.fragment,s),w(As.$$.fragment,s),w(Ds.$$.fragment,s),w(Bs.$$.fragment,s),w(Is.$$.fragment,s),w(Ss.$$.fragment,s),w(Gs.$$.fragment,s),w(Ls.$$.fragment,s),w(Hs.$$.fragment,s),w(ns.$$.fragment,s),w(Rs.$$.fragment,s),w(Ws.$$.fragment,s),w(Fs.$$.fragment,s),w(ls.$$.fragment,s),Ga=!1},d(s){e(c),s&&e(_),s&&e(f),x(k),s&&e(A),x(S,s),s&&e(is),s&&e(Ms),s&&e(jt),x(os,s),s&&e(wt),x(V,s),s&&e(xt),s&&e(W),x(hs),s&&e(bt),s&&e(Ks),s&&e($t),x(us,s),s&&e(kt),s&&e(K),s&&e(qt),x(Y,s),s&&e(_t),s&&e(J),s&&e(yt),s&&e(Ys),s&&e(vt),s&&e(Js),s&&e(Et),x(cs,s),s&&e(zt),s&&e(C),s&&e(Tt),x(fs,s),s&&e(Pt),s&&e(z),s&&e(Ct),s&&e(G),s&&e(Nt),x(gs,s),s&&e(Ot),s&&e(D),s&&e(At),x(ms,s),s&&e(Dt),s&&e(L),s&&e(Bt),x(ds,s),s&&e(It),s&&e(Zs),s&&e(St),x(Z,s),s&&e(Gt),s&&e(F),x(js),s&&e(Lt),s&&e(Qs),s&&e(Ht),s&&e(B),s&&e(Rt),s&&e(Xs),s&&e(Wt),x(ws,s),s&&e(Ft),s&&e(q),s&&e(Ut),x(X,s),s&&e(Vt),s&&e(U),x(xs),s&&e(Mt),s&&e(se),s&&e(Kt),s&&e(ee),s&&e(Yt),x(bs,s),s&&e(Jt),s&&e(es),s&&e(Zt),x($s,s),s&&e(Qt),s&&e(te),s&&e(Xt),x(ks,s),s&&e(sa),x(qs,s),s&&e(ea),s&&e(ae),s&&e(ta),x(_s,s),s&&e(aa),x(ys,s),s&&e(na),s&&e(ts),s&&e(la),x(vs,s),s&&e(pa),s&&e(ne),s&&e(ra),x(Es,s),s&&e(ia),s&&e(le),s&&e(oa),x(zs,s),s&&e(ha),s&&e(pe),s&&e(ua),x(Ts,s),s&&e(ca),x(Ps,s),s&&e(fa),s&&e(re),s&&e(ga),x(Cs,s),s&&e(ma),x(Ns,s),s&&e(da),s&&e(H),s&&e(ja),x(Os,s),s&&e(wa),s&&e(as),s&&e(xa),x(As,s),s&&e(ba),s&&e(ie),s&&e($a),x(Ds,s),s&&e(ka),x(Bs,s),s&&e(qa),s&&e(oe),s&&e(_a),x(Is,s),s&&e(ya),s&&e(he),s&&e(va),x(Ss,s),s&&e(Ea),x(Gs,s),s&&e(za),s&&e(ue),s&&e(Ta),x(Ls,s),s&&e(Pa),x(Hs,s),s&&e(Ca),x(ns,s),s&&e(Na),s&&e(ce),s&&e(Oa),x(Rs,s),s&&e(Aa),s&&e(fe),s&&e(Da),x(Ws,s),s&&e(Ba),x(Fs,s),s&&e(Ia),x(ls,s),s&&e(Sa),s&&e(ge)}}}const Rr={local:"bytepair-encoding-tokenization",sections:[{local:"training-algorithm",title:"Training algorithm"},{local:"tokenization-algorithm",title:"Tokenization algorithm"},{local:"implementing-bpe",title:"Implementing BPE"}],title:"Byte-Pair Encoding tokenization"};function Wr(O){return Nr(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class Jr extends zr{constructor(c){super();Tr(this,c,Wr,Hr,Pr,{})}}export{Jr as default,Rr as metadata};
