import{S as xp,i as $p,s as _p,e as i,k as h,w as m,t as n,Y as vp,M as kp,c as p,d as a,m as c,a as o,x as f,h as l,Z as qp,b as D,G as e,g as r,y as d,q as g,o as b,B as j,v as yp}from"../../chunks/vendor-hf-doc-builder.js";import{T as Ee}from"../../chunks/Tip-hf-doc-builder.js";import{Y as Ep}from"../../chunks/Youtube-hf-doc-builder.js";import{I as on}from"../../chunks/IconCopyLink-hf-doc-builder.js";import{C as x}from"../../chunks/CodeBlock-hf-doc-builder.js";import{C as zp}from"../../chunks/CourseFloatingBanner-hf-doc-builder.js";function Pp(H){let u,v;return{c(){u=i("p"),v=n("\u{1F4A1} This section covers WordPiece in depth, going as far as showing a full implementation. You can skip to the end if you just want a general overview of the tokenization algorithm.")},l(w){u=p(w,"P",{});var $=o(u);v=l($,"\u{1F4A1} This section covers WordPiece in depth, going as far as showing a full implementation. You can skip to the end if you just want a general overview of the tokenization algorithm."),$.forEach(a)},m(w,$){r(w,u,$),e(u,v)},d(w){w&&a(u)}}}function Tp(H){let u,v;return{c(){u=i("p"),v=n("\u26A0\uFE0F Google never open-sourced its implementation of the training algorithm of WordPiece, so what follows is our best guess based on the published literature. It may not be 100% accurate.")},l(w){u=p(w,"P",{});var $=o(u);v=l($,"\u26A0\uFE0F Google never open-sourced its implementation of the training algorithm of WordPiece, so what follows is our best guess based on the published literature. It may not be 100% accurate."),$.forEach(a)},m(w,$){r(w,u,$),e(u,v)},d(w){w&&a(u)}}}function Cp(H){let u,v,w,$,N;return{c(){u=i("p"),v=n("\u270F\uFE0F "),w=i("strong"),$=n("Now your turn!"),N=n(" What will the next merge rule be?")},l(_){u=p(_,"P",{});var y=o(u);v=l(y,"\u270F\uFE0F "),w=p(y,"STRONG",{});var W=o(w);$=l(W,"Now your turn!"),W.forEach(a),N=l(y," What will the next merge rule be?"),y.forEach(a)},m(_,y){r(_,u,y),e(u,v),e(u,w),e(w,$),e(u,N)},d(_){_&&a(u)}}}function Dp(H){let u,v,w,$,N,_,y,W;return{c(){u=i("p"),v=n("\u270F\uFE0F "),w=i("strong"),$=n("Now your turn!"),N=n(" How will the word "),_=i("code"),y=n('"pugs"'),W=n(" be tokenized?")},l(M){u=p(M,"P",{});var B=o(u);v=l(B,"\u270F\uFE0F "),w=p(B,"STRONG",{});var F=o(w);$=l(F,"Now your turn!"),F.forEach(a),N=l(B," How will the word "),_=p(B,"CODE",{});var hs=o(_);y=l(hs,'"pugs"'),hs.forEach(a),W=l(B," be tokenized?"),B.forEach(a)},m(M,B){r(M,u,B),e(u,v),e(u,w),e(w,$),e(u,N),e(u,_),e(_,y),e(u,W)},d(M){M&&a(u)}}}function Np(H){let u,v,w,$,N;return{c(){u=i("p"),v=n("\u{1F4A1} Using "),w=i("code"),$=n("train_new_from_iterator()"),N=n(" on the same corpus won\u2019t result in the exact same vocabulary. This is because the \u{1F917} Tokenizers library does not implement WordPiece for the training (since we are not completely sure of its internals), but uses BPE instead.")},l(_){u=p(_,"P",{});var y=o(u);v=l(y,"\u{1F4A1} Using "),w=p(y,"CODE",{});var W=o(w);$=l(W,"train_new_from_iterator()"),W.forEach(a),N=l(y," on the same corpus won\u2019t result in the exact same vocabulary. This is because the \u{1F917} Tokenizers library does not implement WordPiece for the training (since we are not completely sure of its internals), but uses BPE instead."),y.forEach(a)},m(_,y){r(_,u,y),e(u,v),e(u,w),e(w,$),e(u,N)},d(_){_&&a(u)}}}function Op(H){let u,v,w,$,N,_,y,W,M,B,F,hs,Ys,hn,ze,cs,Pe,J,Te,G,Z,xa,us,cn,$a,un,Ce,Q,De,L,mn,_a,fn,dn,va,gn,bn,Ne,ms,Oe,Js,jn,We,fs,wn,Ae,wp='<span class="katex-display"><span class="katex"><span class="katex-mathml"><math xmlns="http://www.w3.org/1998/Math/MathML" display="block"><semantics><mrow><mrow><mi mathvariant="normal">s</mi><mi mathvariant="normal">c</mi><mi mathvariant="normal">o</mi><mi mathvariant="normal">r</mi><mi mathvariant="normal">e</mi></mrow><mo>=</mo><mo stretchy="false">(</mo><mrow><mi mathvariant="normal">f</mi><mi mathvariant="normal">r</mi><mi mathvariant="normal">e</mi><mi mathvariant="normal">q</mi><mi mathvariant="normal">_</mi><mi mathvariant="normal">o</mi><mi mathvariant="normal">f</mi><mi mathvariant="normal">_</mi><mi mathvariant="normal">p</mi><mi mathvariant="normal">a</mi><mi mathvariant="normal">i</mi><mi mathvariant="normal">r</mi></mrow><mo stretchy="false">)</mo><mi mathvariant="normal">/</mi><mo stretchy="false">(</mo><mrow><mi mathvariant="normal">f</mi><mi mathvariant="normal">r</mi><mi mathvariant="normal">e</mi><mi mathvariant="normal">q</mi><mi mathvariant="normal">_</mi><mi mathvariant="normal">o</mi><mi mathvariant="normal">f</mi><mi mathvariant="normal">_</mi><mi mathvariant="normal">f</mi><mi mathvariant="normal">i</mi><mi mathvariant="normal">r</mi><mi mathvariant="normal">s</mi><mi mathvariant="normal">t</mi><mi mathvariant="normal">_</mi><mi mathvariant="normal">e</mi><mi mathvariant="normal">l</mi><mi mathvariant="normal">e</mi><mi mathvariant="normal">m</mi><mi mathvariant="normal">e</mi><mi mathvariant="normal">n</mi><mi mathvariant="normal">t</mi></mrow><mo>\xD7</mo><mrow><mi mathvariant="normal">f</mi><mi mathvariant="normal">r</mi><mi mathvariant="normal">e</mi><mi mathvariant="normal">q</mi><mi mathvariant="normal">_</mi><mi mathvariant="normal">o</mi><mi mathvariant="normal">f</mi><mi mathvariant="normal">_</mi><mi mathvariant="normal">s</mi><mi mathvariant="normal">e</mi><mi mathvariant="normal">c</mi><mi mathvariant="normal">o</mi><mi mathvariant="normal">n</mi><mi mathvariant="normal">d</mi><mi mathvariant="normal">_</mi><mi mathvariant="normal">e</mi><mi mathvariant="normal">l</mi><mi mathvariant="normal">e</mi><mi mathvariant="normal">m</mi><mi mathvariant="normal">e</mi><mi mathvariant="normal">n</mi><mi mathvariant="normal">t</mi></mrow><mo stretchy="false">)</mo></mrow><annotation encoding="application/x-tex">\\mathrm{score} = (\\mathrm{freq\\_of\\_pair}) / (\\mathrm{freq\\_of\\_first\\_element} \\times \\mathrm{freq\\_of\\_second\\_element})</annotation></semantics></math></span><span class="katex-html" aria-hidden="true"><span class="base"><span class="strut" style="height:0.4306em;"></span><span class="mord"><span class="mord mathrm">score</span></span><span class="mspace" style="margin-right:0.2778em;"></span><span class="mrel">=</span><span class="mspace" style="margin-right:0.2778em;"></span></span><span class="base"><span class="strut" style="height:1.06em;vertical-align:-0.31em;"></span><span class="mopen">(</span><span class="mord"><span class="mord mathrm">freq_of_pair</span></span><span class="mclose">)</span><span class="mord">/</span><span class="mopen">(</span><span class="mord"><span class="mord mathrm">freq_of_first_element</span></span><span class="mspace" style="margin-right:0.2222em;"></span><span class="mbin">\xD7</span><span class="mspace" style="margin-right:0.2222em;"></span></span><span class="base"><span class="strut" style="height:1.06em;vertical-align:-0.31em;"></span><span class="mord"><span class="mord mathrm">freq_of_second_element</span></span><span class="mclose">)</span></span></span></span></span>',Se,z,xn,ka,$n,_n,qa,vn,kn,ya,qn,yn,Ea,En,zn,za,Pn,Tn,Pa,Cn,Dn,Be,Zs,Nn,He,ds,Fe,Qs,On,Le,gs,Ke,E,Wn,Ta,An,Sn,Ca,Bn,Hn,Da,Fn,Ln,Na,Kn,Un,Oa,In,Rn,Wa,Mn,Gn,Aa,Vn,Yn,Ue,K,Jn,Sa,Zn,Qn,Ba,Xn,sl,Ie,bs,Re,U,al,Ha,el,tl,Fa,nl,ll,Me,js,Ge,I,rl,La,il,pl,Ka,ol,hl,Ve,ws,Ye,Xs,cl,Je,X,Ze,V,ss,Ua,xs,ul,Ia,ml,Qe,P,fl,Ra,dl,gl,Ma,bl,jl,Ga,wl,xl,Va,$l,_l,Ya,vl,kl,Ja,ql,yl,Xe,as,El,Za,zl,Pl,st,k,Tl,Qa,Cl,Dl,Xa,Nl,Ol,se,Wl,Al,ae,Sl,Bl,ee,Hl,Fl,te,Ll,Kl,ne,Ul,Il,le,Rl,Ml,at,q,Gl,re,Vl,Yl,ie,Jl,Zl,pe,Ql,Xl,oe,sr,ar,he,er,tr,ce,nr,lr,ue,rr,ir,me,pr,or,et,es,tt,Y,ts,fe,$s,hr,de,cr,nt,sa,ur,lt,aa,mr,rt,_s,it,ns,fr,ge,dr,gr,pt,vs,ot,ea,br,ht,ks,ct,qs,ut,ls,jr,be,wr,xr,mt,ys,ft,Es,dt,rs,$r,je,_r,vr,gt,zs,bt,is,kr,we,qr,yr,jt,Ps,wt,ta,Er,xt,Ts,$t,na,zr,_t,Cs,vt,Ds,kt,la,Pr,qt,Ns,yt,Os,Et,R,Tr,xe,Cr,Dr,$e,Nr,Or,zt,Ws,Pt,ps,Wr,_e,Ar,Sr,Tt,As,Ct,ra,Br,Dt,Ss,Nt,Bs,Ot,ia,Hr,Wt,Hs,At,pa,Fr,St,Fs,Bt,Ls,Ht,oa,Lr,Ft,os,Lt,ha,Kr,Kt,Ks,Ut,ca,Ur,It,Us,Rt,Is,Mt,ua,Ir,Gt,Rs,Vt,ma,Rr,Yt,Ms,Jt,Gs,Zt,fa,Mr,Qt;return _=new on({}),F=new zp({props:{chapter:6,classNames:"absolute z-10 right-0 top-0",notebooks:[{label:"Google Colab",value:"https://colab.research.google.com/github/huggingface/notebooks/blob/master/course/en/chapter6/section6.ipynb"},{label:"Aws Studio",value:"https://studiolab.sagemaker.aws/import/github/huggingface/notebooks/blob/master/course/en/chapter6/section6.ipynb"}]}}),cs=new Ep({props:{id:"qpv6ms_t_1A"}}),J=new Ee({props:{$$slots:{default:[Pp]},$$scope:{ctx:H}}}),us=new on({}),Q=new Ee({props:{warning:!0,$$slots:{default:[Tp]},$$scope:{ctx:H}}}),ms=new x({props:{code:"w ##o ##r ##d",highlighted:"w ##o ##r ##d"}}),ds=new x({props:{code:'("hug", 10), ("pug", 5), ("pun", 12), ("bun", 4), ("hugs", 5)',highlighted:'(<span class="hljs-string">&quot;hug&quot;</span><span class="hljs-punctuation">,</span> <span class="hljs-number">10</span>)<span class="hljs-punctuation">,</span> (<span class="hljs-string">&quot;pug&quot;</span><span class="hljs-punctuation">,</span> <span class="hljs-number">5</span>)<span class="hljs-punctuation">,</span> (<span class="hljs-string">&quot;pun&quot;</span><span class="hljs-punctuation">,</span> <span class="hljs-number">12</span>)<span class="hljs-punctuation">,</span> (<span class="hljs-string">&quot;bun&quot;</span><span class="hljs-punctuation">,</span> <span class="hljs-number">4</span>)<span class="hljs-punctuation">,</span> (<span class="hljs-string">&quot;hugs&quot;</span><span class="hljs-punctuation">,</span> <span class="hljs-number">5</span>)'}}),gs=new x({props:{code:'("h" "##u" "##g", 10), ("p" "##u" "##g", 5), ("p" "##u" "##n", 12), ("b" "##u" "##n", 4), ("h" "##u" "##g" "##s", 5)',highlighted:'(<span class="hljs-string">&quot;h&quot;</span> <span class="hljs-string">&quot;#<span class="hljs-subst">#u</span>&quot;</span> <span class="hljs-string">&quot;#<span class="hljs-subst">#g</span>&quot;</span>, <span class="hljs-number">10</span>), (<span class="hljs-string">&quot;p&quot;</span> <span class="hljs-string">&quot;#<span class="hljs-subst">#u</span>&quot;</span> <span class="hljs-string">&quot;#<span class="hljs-subst">#g</span>&quot;</span>, <span class="hljs-number">5</span>), (<span class="hljs-string">&quot;p&quot;</span> <span class="hljs-string">&quot;#<span class="hljs-subst">#u</span>&quot;</span> <span class="hljs-string">&quot;#<span class="hljs-subst">#n</span>&quot;</span>, <span class="hljs-number">12</span>), (<span class="hljs-string">&quot;b&quot;</span> <span class="hljs-string">&quot;#<span class="hljs-subst">#u</span>&quot;</span> <span class="hljs-string">&quot;#<span class="hljs-subst">#n</span>&quot;</span>, <span class="hljs-number">4</span>), (<span class="hljs-string">&quot;h&quot;</span> <span class="hljs-string">&quot;#<span class="hljs-subst">#u</span>&quot;</span> <span class="hljs-string">&quot;#<span class="hljs-subst">#g</span>&quot;</span> <span class="hljs-string">&quot;#<span class="hljs-subst">#s</span>&quot;</span>, <span class="hljs-number">5</span>)'}}),bs=new x({props:{code:`Vocabulary: ["b", "h", "p", "##g", "##n", "##s", "##u", "##gs"]
Corpus: ("h" "##u" "##g", 10), ("p" "##u" "##g", 5), ("p" "##u" "##n", 12), ("b" "##u" "##n", 4), ("h" "##u" "##gs", 5)`,highlighted:`Vocabulary: [<span class="hljs-string">&quot;b&quot;</span>, <span class="hljs-string">&quot;h&quot;</span>, <span class="hljs-string">&quot;p&quot;</span>, <span class="hljs-string">&quot;#<span class="hljs-subst">#g</span>&quot;</span>, <span class="hljs-string">&quot;#<span class="hljs-subst">#n</span>&quot;</span>, <span class="hljs-string">&quot;#<span class="hljs-subst">#s</span>&quot;</span>, <span class="hljs-string">&quot;#<span class="hljs-subst">#u</span>&quot;</span>, <span class="hljs-string">&quot;#<span class="hljs-subst">#gs</span>&quot;</span>]
Corpus: (<span class="hljs-string">&quot;h&quot;</span> <span class="hljs-string">&quot;#<span class="hljs-subst">#u</span>&quot;</span> <span class="hljs-string">&quot;#<span class="hljs-subst">#g</span>&quot;</span>, <span class="hljs-number">10</span>), (<span class="hljs-string">&quot;p&quot;</span> <span class="hljs-string">&quot;#<span class="hljs-subst">#u</span>&quot;</span> <span class="hljs-string">&quot;#<span class="hljs-subst">#g</span>&quot;</span>, <span class="hljs-number">5</span>), (<span class="hljs-string">&quot;p&quot;</span> <span class="hljs-string">&quot;#<span class="hljs-subst">#u</span>&quot;</span> <span class="hljs-string">&quot;#<span class="hljs-subst">#n</span>&quot;</span>, <span class="hljs-number">12</span>), (<span class="hljs-string">&quot;b&quot;</span> <span class="hljs-string">&quot;#<span class="hljs-subst">#u</span>&quot;</span> <span class="hljs-string">&quot;#<span class="hljs-subst">#n</span>&quot;</span>, <span class="hljs-number">4</span>), (<span class="hljs-string">&quot;h&quot;</span> <span class="hljs-string">&quot;#<span class="hljs-subst">#u</span>&quot;</span> <span class="hljs-string">&quot;#<span class="hljs-subst">#gs</span>&quot;</span>, <span class="hljs-number">5</span>)`}}),js=new x({props:{code:`Vocabulary: ["b", "h", "p", "##g", "##n", "##s", "##u", "##gs", "hu"]
Corpus: ("hu" "##g", 10), ("p" "##u" "##g", 5), ("p" "##u" "##n", 12), ("b" "##u" "##n", 4), ("hu" "##gs", 5)`,highlighted:`Vocabulary: [<span class="hljs-string">&quot;b&quot;</span>, <span class="hljs-string">&quot;h&quot;</span>, <span class="hljs-string">&quot;p&quot;</span>, <span class="hljs-string">&quot;#<span class="hljs-subst">#g</span>&quot;</span>, <span class="hljs-string">&quot;#<span class="hljs-subst">#n</span>&quot;</span>, <span class="hljs-string">&quot;#<span class="hljs-subst">#s</span>&quot;</span>, <span class="hljs-string">&quot;#<span class="hljs-subst">#u</span>&quot;</span>, <span class="hljs-string">&quot;#<span class="hljs-subst">#gs</span>&quot;</span>, <span class="hljs-string">&quot;hu&quot;</span>]
Corpus: (<span class="hljs-string">&quot;hu&quot;</span> <span class="hljs-string">&quot;#<span class="hljs-subst">#g</span>&quot;</span>, <span class="hljs-number">10</span>), (<span class="hljs-string">&quot;p&quot;</span> <span class="hljs-string">&quot;#<span class="hljs-subst">#u</span>&quot;</span> <span class="hljs-string">&quot;#<span class="hljs-subst">#g</span>&quot;</span>, <span class="hljs-number">5</span>), (<span class="hljs-string">&quot;p&quot;</span> <span class="hljs-string">&quot;#<span class="hljs-subst">#u</span>&quot;</span> <span class="hljs-string">&quot;#<span class="hljs-subst">#n</span>&quot;</span>, <span class="hljs-number">12</span>), (<span class="hljs-string">&quot;b&quot;</span> <span class="hljs-string">&quot;#<span class="hljs-subst">#u</span>&quot;</span> <span class="hljs-string">&quot;#<span class="hljs-subst">#n</span>&quot;</span>, <span class="hljs-number">4</span>), (<span class="hljs-string">&quot;hu&quot;</span> <span class="hljs-string">&quot;#<span class="hljs-subst">#gs</span>&quot;</span>, <span class="hljs-number">5</span>)`}}),ws=new x({props:{code:`Vocabulary: ["b", "h", "p", "##g", "##n", "##s", "##u", "##gs", "hu", "hug"]
Corpus: ("hug", 10), ("p" "##u" "##g", 5), ("p" "##u" "##n", 12), ("b" "##u" "##n", 4), ("hu" "##gs", 5)`,highlighted:`Vocabulary: [<span class="hljs-string">&quot;b&quot;</span>, <span class="hljs-string">&quot;h&quot;</span>, <span class="hljs-string">&quot;p&quot;</span>, <span class="hljs-string">&quot;#<span class="hljs-subst">#g</span>&quot;</span>, <span class="hljs-string">&quot;#<span class="hljs-subst">#n</span>&quot;</span>, <span class="hljs-string">&quot;#<span class="hljs-subst">#s</span>&quot;</span>, <span class="hljs-string">&quot;#<span class="hljs-subst">#u</span>&quot;</span>, <span class="hljs-string">&quot;#<span class="hljs-subst">#gs</span>&quot;</span>, <span class="hljs-string">&quot;hu&quot;</span>, <span class="hljs-string">&quot;hug&quot;</span>]
Corpus: (<span class="hljs-string">&quot;hug&quot;</span>, <span class="hljs-number">10</span>), (<span class="hljs-string">&quot;p&quot;</span> <span class="hljs-string">&quot;#<span class="hljs-subst">#u</span>&quot;</span> <span class="hljs-string">&quot;#<span class="hljs-subst">#g</span>&quot;</span>, <span class="hljs-number">5</span>), (<span class="hljs-string">&quot;p&quot;</span> <span class="hljs-string">&quot;#<span class="hljs-subst">#u</span>&quot;</span> <span class="hljs-string">&quot;#<span class="hljs-subst">#n</span>&quot;</span>, <span class="hljs-number">12</span>), (<span class="hljs-string">&quot;b&quot;</span> <span class="hljs-string">&quot;#<span class="hljs-subst">#u</span>&quot;</span> <span class="hljs-string">&quot;#<span class="hljs-subst">#n</span>&quot;</span>, <span class="hljs-number">4</span>), (<span class="hljs-string">&quot;hu&quot;</span> <span class="hljs-string">&quot;#<span class="hljs-subst">#gs</span>&quot;</span>, <span class="hljs-number">5</span>)`}}),X=new Ee({props:{$$slots:{default:[Cp]},$$scope:{ctx:H}}}),xs=new on({}),es=new Ee({props:{$$slots:{default:[Dp]},$$scope:{ctx:H}}}),$s=new on({}),_s=new x({props:{code:`corpus = [
    "This is the Hugging Face Course.",
    "This chapter is about tokenization.",
    "This section shows several tokenizer algorithms.",
    "Hopefully, you will be able to understand how they are trained and generate tokens.",
]`,highlighted:`corpus = [
    <span class="hljs-string">&quot;This is the Hugging Face Course.&quot;</span>,
    <span class="hljs-string">&quot;This chapter is about tokenization.&quot;</span>,
    <span class="hljs-string">&quot;This section shows several tokenizer algorithms.&quot;</span>,
    <span class="hljs-string">&quot;Hopefully, you will be able to understand how they are trained and generate tokens.&quot;</span>,
]`}}),vs=new x({props:{code:`from transformers import AutoTokenizer

tokenizer = AutoTokenizer.from_pretrained("bert-base-cased")`,highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoTokenizer

tokenizer = AutoTokenizer.from_pretrained(<span class="hljs-string">&quot;bert-base-cased&quot;</span>)`}}),ks=new x({props:{code:`from collections import defaultdict

word_freqs = defaultdict(int)
for text in corpus:
    words_with_offsets = tokenizer.backend_tokenizer.pre_tokenizer.pre_tokenize_str(text)
    new_words = [word for word, offset in words_with_offsets]
    for word in new_words:
        word_freqs[word] += 1

word_freqs`,highlighted:`<span class="hljs-keyword">from</span> collections <span class="hljs-keyword">import</span> defaultdict

word_freqs = defaultdict(<span class="hljs-built_in">int</span>)
<span class="hljs-keyword">for</span> text <span class="hljs-keyword">in</span> corpus:
    words_with_offsets = tokenizer.backend_tokenizer.pre_tokenizer.pre_tokenize_str(text)
    new_words = [word <span class="hljs-keyword">for</span> word, offset <span class="hljs-keyword">in</span> words_with_offsets]
    <span class="hljs-keyword">for</span> word <span class="hljs-keyword">in</span> new_words:
        word_freqs[word] += <span class="hljs-number">1</span>

word_freqs`}}),qs=new x({props:{code:`defaultdict(
    int, {'This': 3, 'is': 2, 'the': 1, 'Hugging': 1, 'Face': 1, 'Course': 1, '.': 4, 'chapter': 1, 'about': 1,
    'tokenization': 1, 'section': 1, 'shows': 1, 'several': 1, 'tokenizer': 1, 'algorithms': 1, 'Hopefully': 1,
    ',': 1, 'you': 1, 'will': 1, 'be': 1, 'able': 1, 'to': 1, 'understand': 1, 'how': 1, 'they': 1, 'are': 1,
    'trained': 1, 'and': 1, 'generate': 1, 'tokens': 1})`,highlighted:`defaultdict(
    <span class="hljs-built_in">int</span>, {<span class="hljs-string">&#x27;This&#x27;</span>: <span class="hljs-number">3</span>, <span class="hljs-string">&#x27;is&#x27;</span>: <span class="hljs-number">2</span>, <span class="hljs-string">&#x27;the&#x27;</span>: <span class="hljs-number">1</span>, <span class="hljs-string">&#x27;Hugging&#x27;</span>: <span class="hljs-number">1</span>, <span class="hljs-string">&#x27;Face&#x27;</span>: <span class="hljs-number">1</span>, <span class="hljs-string">&#x27;Course&#x27;</span>: <span class="hljs-number">1</span>, <span class="hljs-string">&#x27;.&#x27;</span>: <span class="hljs-number">4</span>, <span class="hljs-string">&#x27;chapter&#x27;</span>: <span class="hljs-number">1</span>, <span class="hljs-string">&#x27;about&#x27;</span>: <span class="hljs-number">1</span>,
    <span class="hljs-string">&#x27;tokenization&#x27;</span>: <span class="hljs-number">1</span>, <span class="hljs-string">&#x27;section&#x27;</span>: <span class="hljs-number">1</span>, <span class="hljs-string">&#x27;shows&#x27;</span>: <span class="hljs-number">1</span>, <span class="hljs-string">&#x27;several&#x27;</span>: <span class="hljs-number">1</span>, <span class="hljs-string">&#x27;tokenizer&#x27;</span>: <span class="hljs-number">1</span>, <span class="hljs-string">&#x27;algorithms&#x27;</span>: <span class="hljs-number">1</span>, <span class="hljs-string">&#x27;Hopefully&#x27;</span>: <span class="hljs-number">1</span>,
    <span class="hljs-string">&#x27;,&#x27;</span>: <span class="hljs-number">1</span>, <span class="hljs-string">&#x27;you&#x27;</span>: <span class="hljs-number">1</span>, <span class="hljs-string">&#x27;will&#x27;</span>: <span class="hljs-number">1</span>, <span class="hljs-string">&#x27;be&#x27;</span>: <span class="hljs-number">1</span>, <span class="hljs-string">&#x27;able&#x27;</span>: <span class="hljs-number">1</span>, <span class="hljs-string">&#x27;to&#x27;</span>: <span class="hljs-number">1</span>, <span class="hljs-string">&#x27;understand&#x27;</span>: <span class="hljs-number">1</span>, <span class="hljs-string">&#x27;how&#x27;</span>: <span class="hljs-number">1</span>, <span class="hljs-string">&#x27;they&#x27;</span>: <span class="hljs-number">1</span>, <span class="hljs-string">&#x27;are&#x27;</span>: <span class="hljs-number">1</span>,
    <span class="hljs-string">&#x27;trained&#x27;</span>: <span class="hljs-number">1</span>, <span class="hljs-string">&#x27;and&#x27;</span>: <span class="hljs-number">1</span>, <span class="hljs-string">&#x27;generate&#x27;</span>: <span class="hljs-number">1</span>, <span class="hljs-string">&#x27;tokens&#x27;</span>: <span class="hljs-number">1</span>})`}}),ys=new x({props:{code:`alphabet = []
for word in word_freqs.keys():
    if word[0] not in alphabet:
        alphabet.append(word[0])
    for letter in word[1:]:
        if f"##{letter}" not in alphabet:
            alphabet.append(f"##{letter}")

alphabet.sort()
alphabet

print(alphabet)`,highlighted:`alphabet = []
<span class="hljs-keyword">for</span> word <span class="hljs-keyword">in</span> word_freqs.keys():
    <span class="hljs-keyword">if</span> word[<span class="hljs-number">0</span>] <span class="hljs-keyword">not</span> <span class="hljs-keyword">in</span> alphabet:
        alphabet.append(word[<span class="hljs-number">0</span>])
    <span class="hljs-keyword">for</span> letter <span class="hljs-keyword">in</span> word[<span class="hljs-number">1</span>:]:
        <span class="hljs-keyword">if</span> <span class="hljs-string">f&quot;##<span class="hljs-subst">{letter}</span>&quot;</span> <span class="hljs-keyword">not</span> <span class="hljs-keyword">in</span> alphabet:
            alphabet.append(<span class="hljs-string">f&quot;##<span class="hljs-subst">{letter}</span>&quot;</span>)

alphabet.sort()
alphabet

<span class="hljs-built_in">print</span>(alphabet)`}}),Es=new x({props:{code:`['##a', '##b', '##c', '##d', '##e', '##f', '##g', '##h', '##i', '##k', '##l', '##m', '##n', '##o', '##p', '##r', '##s',
 '##t', '##u', '##v', '##w', '##y', '##z', ',', '.', 'C', 'F', 'H', 'T', 'a', 'b', 'c', 'g', 'h', 'i', 's', 't', 'u',
 'w', 'y']`,highlighted:`[<span class="hljs-string">&#x27;##a&#x27;</span>, <span class="hljs-string">&#x27;##b&#x27;</span>, <span class="hljs-string">&#x27;##c&#x27;</span>, <span class="hljs-string">&#x27;##d&#x27;</span>, <span class="hljs-string">&#x27;##e&#x27;</span>, <span class="hljs-string">&#x27;##f&#x27;</span>, <span class="hljs-string">&#x27;##g&#x27;</span>, <span class="hljs-string">&#x27;##h&#x27;</span>, <span class="hljs-string">&#x27;##i&#x27;</span>, <span class="hljs-string">&#x27;##k&#x27;</span>, <span class="hljs-string">&#x27;##l&#x27;</span>, <span class="hljs-string">&#x27;##m&#x27;</span>, <span class="hljs-string">&#x27;##n&#x27;</span>, <span class="hljs-string">&#x27;##o&#x27;</span>, <span class="hljs-string">&#x27;##p&#x27;</span>, <span class="hljs-string">&#x27;##r&#x27;</span>, <span class="hljs-string">&#x27;##s&#x27;</span>,
 <span class="hljs-string">&#x27;##t&#x27;</span>, <span class="hljs-string">&#x27;##u&#x27;</span>, <span class="hljs-string">&#x27;##v&#x27;</span>, <span class="hljs-string">&#x27;##w&#x27;</span>, <span class="hljs-string">&#x27;##y&#x27;</span>, <span class="hljs-string">&#x27;##z&#x27;</span>, <span class="hljs-string">&#x27;,&#x27;</span>, <span class="hljs-string">&#x27;.&#x27;</span>, <span class="hljs-string">&#x27;C&#x27;</span>, <span class="hljs-string">&#x27;F&#x27;</span>, <span class="hljs-string">&#x27;H&#x27;</span>, <span class="hljs-string">&#x27;T&#x27;</span>, <span class="hljs-string">&#x27;a&#x27;</span>, <span class="hljs-string">&#x27;b&#x27;</span>, <span class="hljs-string">&#x27;c&#x27;</span>, <span class="hljs-string">&#x27;g&#x27;</span>, <span class="hljs-string">&#x27;h&#x27;</span>, <span class="hljs-string">&#x27;i&#x27;</span>, <span class="hljs-string">&#x27;s&#x27;</span>, <span class="hljs-string">&#x27;t&#x27;</span>, <span class="hljs-string">&#x27;u&#x27;</span>,
 <span class="hljs-string">&#x27;w&#x27;</span>, <span class="hljs-string">&#x27;y&#x27;</span>]`}}),zs=new x({props:{code:'vocab = ["[PAD]", "[UNK]", "[CLS]", "[SEP]", "[MASK]"] + alphabet.copy()',highlighted:'vocab = [<span class="hljs-string">&quot;[PAD]&quot;</span>, <span class="hljs-string">&quot;[UNK]&quot;</span>, <span class="hljs-string">&quot;[CLS]&quot;</span>, <span class="hljs-string">&quot;[SEP]&quot;</span>, <span class="hljs-string">&quot;[MASK]&quot;</span>] + alphabet.copy()'}}),Ps=new x({props:{code:`splits = {
    word: [c if i == 0 else f"##{c}" for i, c in enumerate(word)]
    for word in word_freqs.keys()
}`,highlighted:`splits = {
    word: [c <span class="hljs-keyword">if</span> i == <span class="hljs-number">0</span> <span class="hljs-keyword">else</span> <span class="hljs-string">f&quot;##<span class="hljs-subst">{c}</span>&quot;</span> <span class="hljs-keyword">for</span> i, c <span class="hljs-keyword">in</span> <span class="hljs-built_in">enumerate</span>(word)]
    <span class="hljs-keyword">for</span> word <span class="hljs-keyword">in</span> word_freqs.keys()
}`}}),Ts=new x({props:{code:`def compute_pair_scores(splits):
    letter_freqs = defaultdict(int)
    pair_freqs = defaultdict(int)
    for word, freq in word_freqs.items():
        split = splits[word]
        if len(split) == 1:
            letter_freqs[split[0]] += freq
            continue
        for i in range(len(split) - 1):
            pair = (split[i], split[i + 1])
            letter_freqs[split[i]] += freq
            pair_freqs[pair] += freq
        letter_freqs[split[-1]] += freq

    scores = {
        pair: freq / (letter_freqs[pair[0]] * letter_freqs[pair[1]])
        for pair, freq in pair_freqs.items()
    }
    return scores`,highlighted:`<span class="hljs-keyword">def</span> <span class="hljs-title function_">compute_pair_scores</span>(<span class="hljs-params">splits</span>):
    letter_freqs = defaultdict(<span class="hljs-built_in">int</span>)
    pair_freqs = defaultdict(<span class="hljs-built_in">int</span>)
    <span class="hljs-keyword">for</span> word, freq <span class="hljs-keyword">in</span> word_freqs.items():
        split = splits[word]
        <span class="hljs-keyword">if</span> <span class="hljs-built_in">len</span>(split) == <span class="hljs-number">1</span>:
            letter_freqs[split[<span class="hljs-number">0</span>]] += freq
            <span class="hljs-keyword">continue</span>
        <span class="hljs-keyword">for</span> i <span class="hljs-keyword">in</span> <span class="hljs-built_in">range</span>(<span class="hljs-built_in">len</span>(split) - <span class="hljs-number">1</span>):
            pair = (split[i], split[i + <span class="hljs-number">1</span>])
            letter_freqs[split[i]] += freq
            pair_freqs[pair] += freq
        letter_freqs[split[-<span class="hljs-number">1</span>]] += freq

    scores = {
        pair: freq / (letter_freqs[pair[<span class="hljs-number">0</span>]] * letter_freqs[pair[<span class="hljs-number">1</span>]])
        <span class="hljs-keyword">for</span> pair, freq <span class="hljs-keyword">in</span> pair_freqs.items()
    }
    <span class="hljs-keyword">return</span> scores`}}),Cs=new x({props:{code:`pair_scores = compute_pair_scores(splits)
for i, key in enumerate(pair_scores.keys()):
    print(f"{key}: {pair_scores[key]}")
    if i >= 5:
        break`,highlighted:`pair_scores = compute_pair_scores(splits)
<span class="hljs-keyword">for</span> i, key <span class="hljs-keyword">in</span> <span class="hljs-built_in">enumerate</span>(pair_scores.keys()):
    <span class="hljs-built_in">print</span>(<span class="hljs-string">f&quot;<span class="hljs-subst">{key}</span>: <span class="hljs-subst">{pair_scores[key]}</span>&quot;</span>)
    <span class="hljs-keyword">if</span> i &gt;= <span class="hljs-number">5</span>:
        <span class="hljs-keyword">break</span>`}}),Ds=new x({props:{code:`('T', '##h'): 0.125
('##h', '##i'): 0.03409090909090909
('##i', '##s'): 0.02727272727272727
('i', '##s'): 0.1
('t', '##h'): 0.03571428571428571
('##h', '##e'): 0.011904761904761904`,highlighted:`(<span class="hljs-string">&#x27;T&#x27;</span>, <span class="hljs-string">&#x27;##h&#x27;</span>): <span class="hljs-number">0.125</span>
(<span class="hljs-string">&#x27;##h&#x27;</span>, <span class="hljs-string">&#x27;##i&#x27;</span>): <span class="hljs-number">0.03409090909090909</span>
(<span class="hljs-string">&#x27;##i&#x27;</span>, <span class="hljs-string">&#x27;##s&#x27;</span>): <span class="hljs-number">0.02727272727272727</span>
(<span class="hljs-string">&#x27;i&#x27;</span>, <span class="hljs-string">&#x27;##s&#x27;</span>): <span class="hljs-number">0.1</span>
(<span class="hljs-string">&#x27;t&#x27;</span>, <span class="hljs-string">&#x27;##h&#x27;</span>): <span class="hljs-number">0.03571428571428571</span>
(<span class="hljs-string">&#x27;##h&#x27;</span>, <span class="hljs-string">&#x27;##e&#x27;</span>): <span class="hljs-number">0.011904761904761904</span>`}}),Ns=new x({props:{code:`best_pair = ""
max_score = None
for pair, score in pair_scores.items():
    if max_score is None or max_score < score:
        best_pair = pair
        max_score = score

print(best_pair, max_score)`,highlighted:`best_pair = <span class="hljs-string">&quot;&quot;</span>
max_score = <span class="hljs-literal">None</span>
<span class="hljs-keyword">for</span> pair, score <span class="hljs-keyword">in</span> pair_scores.items():
    <span class="hljs-keyword">if</span> max_score <span class="hljs-keyword">is</span> <span class="hljs-literal">None</span> <span class="hljs-keyword">or</span> max_score &lt; score:
        best_pair = pair
        max_score = score

<span class="hljs-built_in">print</span>(best_pair, max_score)`}}),Os=new x({props:{code:"('a', '##b') 0.2",highlighted:'(<span class="hljs-string">&#x27;a&#x27;</span>, <span class="hljs-string">&#x27;##b&#x27;</span>) <span class="hljs-number">0.2</span>'}}),Ws=new x({props:{code:'vocab.append("ab")',highlighted:'vocab.append(<span class="hljs-string">&quot;ab&quot;</span>)'}}),As=new x({props:{code:`def merge_pair(a, b, splits):
    for word in word_freqs:
        split = splits[word]
        if len(split) == 1:
            continue
        i = 0
        while i < len(split) - 1:
            if split[i] == a and split[i + 1] == b:
                merge = a + b[2:] if b.startswith("##") else a + b
                split = split[:i] + [merge] + split[i + 2 :]
            else:
                i += 1
        splits[word] = split
    return splits`,highlighted:`<span class="hljs-keyword">def</span> <span class="hljs-title function_">merge_pair</span>(<span class="hljs-params">a, b, splits</span>):
    <span class="hljs-keyword">for</span> word <span class="hljs-keyword">in</span> word_freqs:
        split = splits[word]
        <span class="hljs-keyword">if</span> <span class="hljs-built_in">len</span>(split) == <span class="hljs-number">1</span>:
            <span class="hljs-keyword">continue</span>
        i = <span class="hljs-number">0</span>
        <span class="hljs-keyword">while</span> i &lt; <span class="hljs-built_in">len</span>(split) - <span class="hljs-number">1</span>:
            <span class="hljs-keyword">if</span> split[i] == a <span class="hljs-keyword">and</span> split[i + <span class="hljs-number">1</span>] == b:
                merge = a + b[<span class="hljs-number">2</span>:] <span class="hljs-keyword">if</span> b.startswith(<span class="hljs-string">&quot;##&quot;</span>) <span class="hljs-keyword">else</span> a + b
                split = split[:i] + [merge] + split[i + <span class="hljs-number">2</span> :]
            <span class="hljs-keyword">else</span>:
                i += <span class="hljs-number">1</span>
        splits[word] = split
    <span class="hljs-keyword">return</span> splits`}}),Ss=new x({props:{code:`splits = merge_pair("a", "##b", splits)
splits["about"]`,highlighted:`splits = merge_pair(<span class="hljs-string">&quot;a&quot;</span>, <span class="hljs-string">&quot;##b&quot;</span>, splits)
splits[<span class="hljs-string">&quot;about&quot;</span>]`}}),Bs=new x({props:{code:"['ab', '##o', '##u', '##t']",highlighted:'[<span class="hljs-string">&#x27;ab&#x27;</span>, <span class="hljs-string">&#x27;##o&#x27;</span>, <span class="hljs-string">&#x27;##u&#x27;</span>, <span class="hljs-string">&#x27;##t&#x27;</span>]'}}),Hs=new x({props:{code:`vocab_size = 70
while len(vocab) < vocab_size:
    scores = compute_pair_scores(splits)
    best_pair, max_score = "", None
    for pair, score in scores.items():
        if max_score is None or max_score < score:
            best_pair = pair
            max_score = score
    splits = merge_pair(*best_pair, splits)
    new_token = (
        best_pair[0] + best_pair[1][2:]
        if best_pair[1].startswith("##")
        else best_pair[0] + best_pair[1]
    )
    vocab.append(new_token)`,highlighted:`vocab_size = <span class="hljs-number">70</span>
<span class="hljs-keyword">while</span> <span class="hljs-built_in">len</span>(vocab) &lt; vocab_size:
    scores = compute_pair_scores(splits)
    best_pair, max_score = <span class="hljs-string">&quot;&quot;</span>, <span class="hljs-literal">None</span>
    <span class="hljs-keyword">for</span> pair, score <span class="hljs-keyword">in</span> scores.items():
        <span class="hljs-keyword">if</span> max_score <span class="hljs-keyword">is</span> <span class="hljs-literal">None</span> <span class="hljs-keyword">or</span> max_score &lt; score:
            best_pair = pair
            max_score = score
    splits = merge_pair(*best_pair, splits)
    new_token = (
        best_pair[<span class="hljs-number">0</span>] + best_pair[<span class="hljs-number">1</span>][<span class="hljs-number">2</span>:]
        <span class="hljs-keyword">if</span> best_pair[<span class="hljs-number">1</span>].startswith(<span class="hljs-string">&quot;##&quot;</span>)
        <span class="hljs-keyword">else</span> best_pair[<span class="hljs-number">0</span>] + best_pair[<span class="hljs-number">1</span>]
    )
    vocab.append(new_token)`}}),Fs=new x({props:{code:"print(vocab)",highlighted:'<span class="hljs-built_in">print</span>(vocab)'}}),Ls=new x({props:{code:`['[PAD]', '[UNK]', '[CLS]', '[SEP]', '[MASK]', '##a', '##b', '##c', '##d', '##e', '##f', '##g', '##h', '##i', '##k',
 '##l', '##m', '##n', '##o', '##p', '##r', '##s', '##t', '##u', '##v', '##w', '##y', '##z', ',', '.', 'C', 'F', 'H',
 'T', 'a', 'b', 'c', 'g', 'h', 'i', 's', 't', 'u', 'w', 'y', 'ab', '##fu', 'Fa', 'Fac', '##ct', '##ful', '##full', '##fully',
 'Th', 'ch', '##hm', 'cha', 'chap', 'chapt', '##thm', 'Hu', 'Hug', 'Hugg', 'sh', 'th', 'is', '##thms', '##za', '##zat',
 '##ut']`,highlighted:`[<span class="hljs-string">&#x27;[PAD]&#x27;</span>, <span class="hljs-string">&#x27;[UNK]&#x27;</span>, <span class="hljs-string">&#x27;[CLS]&#x27;</span>, <span class="hljs-string">&#x27;[SEP]&#x27;</span>, <span class="hljs-string">&#x27;[MASK]&#x27;</span>, <span class="hljs-string">&#x27;##a&#x27;</span>, <span class="hljs-string">&#x27;##b&#x27;</span>, <span class="hljs-string">&#x27;##c&#x27;</span>, <span class="hljs-string">&#x27;##d&#x27;</span>, <span class="hljs-string">&#x27;##e&#x27;</span>, <span class="hljs-string">&#x27;##f&#x27;</span>, <span class="hljs-string">&#x27;##g&#x27;</span>, <span class="hljs-string">&#x27;##h&#x27;</span>, <span class="hljs-string">&#x27;##i&#x27;</span>, <span class="hljs-string">&#x27;##k&#x27;</span>,
 <span class="hljs-string">&#x27;##l&#x27;</span>, <span class="hljs-string">&#x27;##m&#x27;</span>, <span class="hljs-string">&#x27;##n&#x27;</span>, <span class="hljs-string">&#x27;##o&#x27;</span>, <span class="hljs-string">&#x27;##p&#x27;</span>, <span class="hljs-string">&#x27;##r&#x27;</span>, <span class="hljs-string">&#x27;##s&#x27;</span>, <span class="hljs-string">&#x27;##t&#x27;</span>, <span class="hljs-string">&#x27;##u&#x27;</span>, <span class="hljs-string">&#x27;##v&#x27;</span>, <span class="hljs-string">&#x27;##w&#x27;</span>, <span class="hljs-string">&#x27;##y&#x27;</span>, <span class="hljs-string">&#x27;##z&#x27;</span>, <span class="hljs-string">&#x27;,&#x27;</span>, <span class="hljs-string">&#x27;.&#x27;</span>, <span class="hljs-string">&#x27;C&#x27;</span>, <span class="hljs-string">&#x27;F&#x27;</span>, <span class="hljs-string">&#x27;H&#x27;</span>,
 <span class="hljs-string">&#x27;T&#x27;</span>, <span class="hljs-string">&#x27;a&#x27;</span>, <span class="hljs-string">&#x27;b&#x27;</span>, <span class="hljs-string">&#x27;c&#x27;</span>, <span class="hljs-string">&#x27;g&#x27;</span>, <span class="hljs-string">&#x27;h&#x27;</span>, <span class="hljs-string">&#x27;i&#x27;</span>, <span class="hljs-string">&#x27;s&#x27;</span>, <span class="hljs-string">&#x27;t&#x27;</span>, <span class="hljs-string">&#x27;u&#x27;</span>, <span class="hljs-string">&#x27;w&#x27;</span>, <span class="hljs-string">&#x27;y&#x27;</span>, <span class="hljs-string">&#x27;ab&#x27;</span>, <span class="hljs-string">&#x27;##fu&#x27;</span>, <span class="hljs-string">&#x27;Fa&#x27;</span>, <span class="hljs-string">&#x27;Fac&#x27;</span>, <span class="hljs-string">&#x27;##ct&#x27;</span>, <span class="hljs-string">&#x27;##ful&#x27;</span>, <span class="hljs-string">&#x27;##full&#x27;</span>, <span class="hljs-string">&#x27;##fully&#x27;</span>,
 <span class="hljs-string">&#x27;Th&#x27;</span>, <span class="hljs-string">&#x27;ch&#x27;</span>, <span class="hljs-string">&#x27;##hm&#x27;</span>, <span class="hljs-string">&#x27;cha&#x27;</span>, <span class="hljs-string">&#x27;chap&#x27;</span>, <span class="hljs-string">&#x27;chapt&#x27;</span>, <span class="hljs-string">&#x27;##thm&#x27;</span>, <span class="hljs-string">&#x27;Hu&#x27;</span>, <span class="hljs-string">&#x27;Hug&#x27;</span>, <span class="hljs-string">&#x27;Hugg&#x27;</span>, <span class="hljs-string">&#x27;sh&#x27;</span>, <span class="hljs-string">&#x27;th&#x27;</span>, <span class="hljs-string">&#x27;is&#x27;</span>, <span class="hljs-string">&#x27;##thms&#x27;</span>, <span class="hljs-string">&#x27;##za&#x27;</span>, <span class="hljs-string">&#x27;##zat&#x27;</span>,
 <span class="hljs-string">&#x27;##ut&#x27;</span>]`}}),os=new Ee({props:{$$slots:{default:[Np]},$$scope:{ctx:H}}}),Ks=new x({props:{code:`def encode_word(word):
    tokens = []
    while len(word) > 0:
        i = len(word)
        while i > 0 and word[:i] not in vocab:
            i -= 1
        if i == 0:
            return ["[UNK]"]
        tokens.append(word[:i])
        word = word[i:]
        if len(word) > 0:
            word = f"##{word}"
    return tokens`,highlighted:`<span class="hljs-keyword">def</span> <span class="hljs-title function_">encode_word</span>(<span class="hljs-params">word</span>):
    tokens = []
    <span class="hljs-keyword">while</span> <span class="hljs-built_in">len</span>(word) &gt; <span class="hljs-number">0</span>:
        i = <span class="hljs-built_in">len</span>(word)
        <span class="hljs-keyword">while</span> i &gt; <span class="hljs-number">0</span> <span class="hljs-keyword">and</span> word[:i] <span class="hljs-keyword">not</span> <span class="hljs-keyword">in</span> vocab:
            i -= <span class="hljs-number">1</span>
        <span class="hljs-keyword">if</span> i == <span class="hljs-number">0</span>:
            <span class="hljs-keyword">return</span> [<span class="hljs-string">&quot;[UNK]&quot;</span>]
        tokens.append(word[:i])
        word = word[i:]
        <span class="hljs-keyword">if</span> <span class="hljs-built_in">len</span>(word) &gt; <span class="hljs-number">0</span>:
            word = <span class="hljs-string">f&quot;##<span class="hljs-subst">{word}</span>&quot;</span>
    <span class="hljs-keyword">return</span> tokens`}}),Us=new x({props:{code:`print(encode_word("Hugging"))
print(encode_word("HOgging"))`,highlighted:`<span class="hljs-built_in">print</span>(encode_word(<span class="hljs-string">&quot;Hugging&quot;</span>))
<span class="hljs-built_in">print</span>(encode_word(<span class="hljs-string">&quot;HOgging&quot;</span>))`}}),Is=new x({props:{code:`['Hugg', '##i', '##n', '##g']
['[UNK]']`,highlighted:`[<span class="hljs-string">&#x27;Hugg&#x27;</span>, <span class="hljs-string">&#x27;##i&#x27;</span>, <span class="hljs-string">&#x27;##n&#x27;</span>, <span class="hljs-string">&#x27;##g&#x27;</span>]
[<span class="hljs-string">&#x27;[UNK]&#x27;</span>]`}}),Rs=new x({props:{code:`def tokenize(text):
    pre_tokenize_result = tokenizer._tokenizer.pre_tokenizer.pre_tokenize_str(text)
    pre_tokenized_text = [word for word, offset in pre_tokenize_result]
    encoded_words = [encode_word(word) for word in pre_tokenized_text]
    return sum(encoded_words, [])`,highlighted:`<span class="hljs-keyword">def</span> <span class="hljs-title function_">tokenize</span>(<span class="hljs-params">text</span>):
    pre_tokenize_result = tokenizer._tokenizer.pre_tokenizer.pre_tokenize_str(text)
    pre_tokenized_text = [word <span class="hljs-keyword">for</span> word, offset <span class="hljs-keyword">in</span> pre_tokenize_result]
    encoded_words = [encode_word(word) <span class="hljs-keyword">for</span> word <span class="hljs-keyword">in</span> pre_tokenized_text]
    <span class="hljs-keyword">return</span> <span class="hljs-built_in">sum</span>(encoded_words, [])`}}),Ms=new x({props:{code:'tokenize("This is the Hugging Face course!")',highlighted:'tokenize(<span class="hljs-string">&quot;This is the Hugging Face course!&quot;</span>)'}}),Gs=new x({props:{code:`['Th', '##i', '##s', 'is', 'th', '##e', 'Hugg', '##i', '##n', '##g', 'Fac', '##e', 'c', '##o', '##u', '##r', '##s',
 '##e', '[UNK]']`,highlighted:`[<span class="hljs-string">&#x27;Th&#x27;</span>, <span class="hljs-string">&#x27;##i&#x27;</span>, <span class="hljs-string">&#x27;##s&#x27;</span>, <span class="hljs-string">&#x27;is&#x27;</span>, <span class="hljs-string">&#x27;th&#x27;</span>, <span class="hljs-string">&#x27;##e&#x27;</span>, <span class="hljs-string">&#x27;Hugg&#x27;</span>, <span class="hljs-string">&#x27;##i&#x27;</span>, <span class="hljs-string">&#x27;##n&#x27;</span>, <span class="hljs-string">&#x27;##g&#x27;</span>, <span class="hljs-string">&#x27;Fac&#x27;</span>, <span class="hljs-string">&#x27;##e&#x27;</span>, <span class="hljs-string">&#x27;c&#x27;</span>, <span class="hljs-string">&#x27;##o&#x27;</span>, <span class="hljs-string">&#x27;##u&#x27;</span>, <span class="hljs-string">&#x27;##r&#x27;</span>, <span class="hljs-string">&#x27;##s&#x27;</span>,
 <span class="hljs-string">&#x27;##e&#x27;</span>, <span class="hljs-string">&#x27;[UNK]&#x27;</span>]`}}),{c(){u=i("meta"),v=h(),w=i("h1"),$=i("a"),N=i("span"),m(_.$$.fragment),y=h(),W=i("span"),M=n("WordPiece tokenization"),B=h(),m(F.$$.fragment),hs=h(),Ys=i("p"),hn=n("WordPiece is the tokenization algorithm Google developed to pretrain BERT. It has since been reused in quite a few Transformer models based on BERT, such as DistilBERT, MobileBERT, Funnel Transformers, and MPNET. It\u2019s very similar to BPE in terms of the training, but the actual tokenization is done differently."),ze=h(),m(cs.$$.fragment),Pe=h(),m(J.$$.fragment),Te=h(),G=i("h2"),Z=i("a"),xa=i("span"),m(us.$$.fragment),cn=h(),$a=i("span"),un=n("Training algorithm"),Ce=h(),m(Q.$$.fragment),De=h(),L=i("p"),mn=n("Like BPE, WordPiece starts from a small vocabulary including the special tokens used by the model and the initial alphabet. Since it identifies subwords by adding a prefix (like "),_a=i("code"),fn=n("##"),dn=n(" for BERT), each word is initially split by adding that prefix to all the characters inside the word. So, for instance, "),va=i("code"),gn=n('"word"'),bn=n(" gets split like this:"),Ne=h(),m(ms.$$.fragment),Oe=h(),Js=i("p"),jn=n("Thus, the initial alphabet contains all the characters present at the beginning of a word and the characters present inside a word preceded by the WordPiece prefix."),We=h(),fs=i("p"),wn=n(`Then, again like BPE, WordPiece learns merge rules. The main difference is the way the pair to be merged is selected. Instead of selecting the most frequent pair, WordPiece computes a score for each pair, using the following formula:
`),Ae=new vp,Se=h(),z=i("p"),xn=n("By dividing the frequency of the pair by the product of the frequencies of each of its parts, the algorithm prioritizes the merging of pairs where the individual parts are less frequent in the vocabulary. For instance, it won\u2019t necessarily merge "),ka=i("code"),$n=n('("un", "##able")'),_n=n(" even if that pair occurs very frequently in the vocabulary, because the two pairs "),qa=i("code"),vn=n('"un"'),kn=n(" and "),ya=i("code"),qn=n('"##able"'),yn=n(" will likely each appear in a lot of other words and have a high frequency. In contrast, a pair like "),Ea=i("code"),En=n('("hu", "##gging")'),zn=n(" will probably be merged faster (assuming the word \u201Chugging\u201D appears often in the vocabulary) since "),za=i("code"),Pn=n('"hu"'),Tn=n(" and "),Pa=i("code"),Cn=n('"##gging"'),Dn=n(" are likely to be less frequent individually."),Be=h(),Zs=i("p"),Nn=n("Let\u2019s look at the same vocabulary we used in the BPE training example:"),He=h(),m(ds.$$.fragment),Fe=h(),Qs=i("p"),On=n("The splits here will be:"),Le=h(),m(gs.$$.fragment),Ke=h(),E=i("p"),Wn=n("so the initial vocabulary will be "),Ta=i("code"),An=n('["b", "h", "p", "##g", "##n", "##s", "##u"]'),Sn=n(" (if we forget about special tokens for now). The most frequent pair is "),Ca=i("code"),Bn=n('("##u", "##g")'),Hn=n(" (present 20 times), but the individual frequency of "),Da=i("code"),Fn=n('"##u"'),Ln=n(" is very high, so its score is not the highest (it\u2019s 1 / 36). All pairs with a "),Na=i("code"),Kn=n('"##u"'),Un=n(" actually have that same score (1 / 36), so the best score goes to the pair "),Oa=i("code"),In=n('("##g", "##s")'),Rn=n(" \u2014 the only one without a "),Wa=i("code"),Mn=n('"##u"'),Gn=n(" \u2014 at 1 / 20, and the first merge learned is "),Aa=i("code"),Vn=n('("##g", "##s") -> ("##gs")'),Yn=n("."),Ue=h(),K=i("p"),Jn=n("Note that when we merge, we remove the "),Sa=i("code"),Zn=n("##"),Qn=n(" between the two tokens, so we add "),Ba=i("code"),Xn=n('"##gs"'),sl=n(" to the vocabulary and apply the merge in the words of the corpus:"),Ie=h(),m(bs.$$.fragment),Re=h(),U=i("p"),al=n("At this point, "),Ha=i("code"),el=n('"##u"'),tl=n(" is in all the possible pairs, so they all end up with the same score. Let\u2019s say that in this case, the first pair is merged, so "),Fa=i("code"),nl=n('("h", "##u") -> "hu"'),ll=n(". This takes us to:"),Me=h(),m(js.$$.fragment),Ge=h(),I=i("p"),rl=n("Then the next best score is shared by "),La=i("code"),il=n('("hu", "##g")'),pl=n(" and "),Ka=i("code"),ol=n('("hu", "##gs")'),hl=n(" (with 1/15, compared to 1/21 for all the other pairs), so the first pair with the biggest score is merged:"),Ve=h(),m(ws.$$.fragment),Ye=h(),Xs=i("p"),cl=n("and we continue like this until we reach the desired vocabulary size."),Je=h(),m(X.$$.fragment),Ze=h(),V=i("h2"),ss=i("a"),Ua=i("span"),m(xs.$$.fragment),ul=h(),Ia=i("span"),ml=n("Tokenization algorithm"),Qe=h(),P=i("p"),fl=n("Tokenization differs in WordPiece and BPE in that WordPiece only saves the final vocabulary, not the merge rules learned. Starting from the word to tokenize, WordPiece finds the longest subword that is in the vocabulary, then splits on it. For instance, if we use the vocabulary learned in the example above, for the word "),Ra=i("code"),dl=n('"hugs"'),gl=n(" the longest subword starting from the beginning that is inside the vocabulary is "),Ma=i("code"),bl=n('"hug"'),jl=n(", so we split there and get "),Ga=i("code"),wl=n('["hug", "##s"]'),xl=n(". We then continue with "),Va=i("code"),$l=n('"##s"'),_l=n(", which is in the vocabulary, so the tokenization of "),Ya=i("code"),vl=n('"hugs"'),kl=n(" is "),Ja=i("code"),ql=n('["hug", "##s"]'),yl=n("."),Xe=h(),as=i("p"),El=n("With BPE, we would have applied the merges learned in order and tokenized this as "),Za=i("code"),zl=n('["hu", "##gs"]'),Pl=n(", so the encoding is different."),st=h(),k=i("p"),Tl=n("As another example, let\u2019s see how the word "),Qa=i("code"),Cl=n('"bugs"'),Dl=n(" would be tokenized. "),Xa=i("code"),Nl=n('"b"'),Ol=n(" is the longest subword starting at the beginning of the word that is in the vocabulary, so we split there and get "),se=i("code"),Wl=n('["b", "##ugs"]'),Al=n(". Then "),ae=i("code"),Sl=n('"##u"'),Bl=n(" is the longest subword starting at the beginning of "),ee=i("code"),Hl=n('"##ugs"'),Fl=n(" that is in the vocabulary, so we split there and get "),te=i("code"),Ll=n('["b", "##u, "##gs"]'),Kl=n(". Finally, "),ne=i("code"),Ul=n('"##gs"'),Il=n(" is in the vocabulary, so this last list is the tokenization of "),le=i("code"),Rl=n('"bugs"'),Ml=n("."),at=h(),q=i("p"),Gl=n("When the tokenization gets to a stage where it\u2019s not possible to find a subword in the vocabulary, the whole word is tokenized as unknown \u2014 so, for instance, "),re=i("code"),Vl=n('"mug"'),Yl=n(" would be tokenized as "),ie=i("code"),Jl=n('["[UNK]"]'),Zl=n(", as would "),pe=i("code"),Ql=n('"bum"'),Xl=n(" (even if we can begin with "),oe=i("code"),sr=n('"b"'),ar=n(" and "),he=i("code"),er=n('"##u"'),tr=n(", "),ce=i("code"),nr=n('"##m"'),lr=n(" is not the vocabulary, and the resulting tokenization will just be "),ue=i("code"),rr=n('["[UNK]"]'),ir=n(", not "),me=i("code"),pr=n('["b", "##u", "[UNK]"]'),or=n("). This is another difference from BPE, which would only classify the individual characters not in the vocabulary as unknown."),et=h(),m(es.$$.fragment),tt=h(),Y=i("h2"),ts=i("a"),fe=i("span"),m($s.$$.fragment),hr=h(),de=i("span"),cr=n("Implementing WordPiece"),nt=h(),sa=i("p"),ur=n("Now let\u2019s take a look at an implementation of the WordPiece algorithm. Like with BPE, this is just pedagogical, and you won\u2019t able to use this on a big corpus."),lt=h(),aa=i("p"),mr=n("We will use the same corpus as in the BPE example:"),rt=h(),m(_s.$$.fragment),it=h(),ns=i("p"),fr=n("First, we need to pre-tokenize the corpus into words. Since we are replicating a WordPiece tokenizer (like BERT), we will use the "),ge=i("code"),dr=n("bert-base-cased"),gr=n(" tokenizer for the pre-tokenization:"),pt=h(),m(vs.$$.fragment),ot=h(),ea=i("p"),br=n("Then we compute the frequencies of each word in the corpus as we do the pre-tokenization:"),ht=h(),m(ks.$$.fragment),ct=h(),m(qs.$$.fragment),ut=h(),ls=i("p"),jr=n("As we saw before, the alphabet is the unique set composed of all the first letters of words, and all the other letters that appear in words prefixed by "),be=i("code"),wr=n("##"),xr=n(":"),mt=h(),m(ys.$$.fragment),ft=h(),m(Es.$$.fragment),dt=h(),rs=i("p"),$r=n("We also add the special tokens used by the model at the beginning of that vocabulary. In the case of BERT, it\u2019s the list "),je=i("code"),_r=n('["[PAD]", "[UNK]", "[CLS]", "[SEP]", "[MASK]"]'),vr=n(":"),gt=h(),m(zs.$$.fragment),bt=h(),is=i("p"),kr=n("Next we need to split each word, with all the letters that are not the first prefixed by "),we=i("code"),qr=n("##"),yr=n(":"),jt=h(),m(Ps.$$.fragment),wt=h(),ta=i("p"),Er=n("Now that we are ready for training, let\u2019s write a function that computes the score of each pair. We\u2019ll need to use this at each step of the training:"),xt=h(),m(Ts.$$.fragment),$t=h(),na=i("p"),zr=n("Let\u2019s have a look at a part of this dictionary after the initial splits:"),_t=h(),m(Cs.$$.fragment),vt=h(),m(Ds.$$.fragment),kt=h(),la=i("p"),Pr=n("Now, finding the pair with the best score only takes a quick loop:"),qt=h(),m(Ns.$$.fragment),yt=h(),m(Os.$$.fragment),Et=h(),R=i("p"),Tr=n("So the first merge to learn is "),xe=i("code"),Cr=n("('a', '##b') -> 'ab'"),Dr=n(", and we add "),$e=i("code"),Nr=n("'ab'"),Or=n(" to the vocabulary:"),zt=h(),m(Ws.$$.fragment),Pt=h(),ps=i("p"),Wr=n("To continue, we need to apply that merge in our "),_e=i("code"),Ar=n("splits"),Sr=n(" dictionary. Let\u2019s write another function for this:"),Tt=h(),m(As.$$.fragment),Ct=h(),ra=i("p"),Br=n("And we can have a look at the result of the first merge:"),Dt=h(),m(Ss.$$.fragment),Nt=h(),m(Bs.$$.fragment),Ot=h(),ia=i("p"),Hr=n("Now we have everything we need to loop until we have learned all the merges we want. Let\u2019s aim for a vocab size of 70:"),Wt=h(),m(Hs.$$.fragment),At=h(),pa=i("p"),Fr=n("We can then look at the generated vocabulary:"),St=h(),m(Fs.$$.fragment),Bt=h(),m(Ls.$$.fragment),Ht=h(),oa=i("p"),Lr=n("As we can see, compared to BPE, this tokenizer learns parts of words as tokens a bit faster."),Ft=h(),m(os.$$.fragment),Lt=h(),ha=i("p"),Kr=n("To tokenize a new text, we pre-tokenize it, split it, then apply the tokenization algorithm on each word. That is, we look for the biggest subword starting at the beginning of the first word and split it, then we repeat the process on the second part, and so on for the rest of that word and the following words in the text:"),Kt=h(),m(Ks.$$.fragment),Ut=h(),ca=i("p"),Ur=n("Let\u2019s test it on one word that\u2019s in the vocabulary, and another that isn\u2019t:"),It=h(),m(Us.$$.fragment),Rt=h(),m(Is.$$.fragment),Mt=h(),ua=i("p"),Ir=n("Now, let\u2019s write a function that tokenizes a text:"),Gt=h(),m(Rs.$$.fragment),Vt=h(),ma=i("p"),Rr=n("We can try it on any text:"),Yt=h(),m(Ms.$$.fragment),Jt=h(),m(Gs.$$.fragment),Zt=h(),fa=i("p"),Mr=n("That\u2019s it for the WordPiece algorithm! Now let\u2019s take a look at Unigram."),this.h()},l(s){const t=kp('[data-svelte="svelte-1phssyn"]',document.head);u=p(t,"META",{name:!0,content:!0}),t.forEach(a),v=c(s),w=p(s,"H1",{class:!0});var Vs=o(w);$=p(Vs,"A",{id:!0,class:!0,href:!0});var ve=o($);N=p(ve,"SPAN",{});var ke=o(N);f(_.$$.fragment,ke),ke.forEach(a),ve.forEach(a),y=c(Vs),W=p(Vs,"SPAN",{});var qe=o(W);M=l(qe,"WordPiece tokenization"),qe.forEach(a),Vs.forEach(a),B=c(s),f(F.$$.fragment,s),hs=c(s),Ys=p(s,"P",{});var ye=o(Ys);hn=l(ye,"WordPiece is the tokenization algorithm Google developed to pretrain BERT. It has since been reused in quite a few Transformer models based on BERT, such as DistilBERT, MobileBERT, Funnel Transformers, and MPNET. It\u2019s very similar to BPE in terms of the training, but the actual tokenization is done differently."),ye.forEach(a),ze=c(s),f(cs.$$.fragment,s),Pe=c(s),f(J.$$.fragment,s),Te=c(s),G=p(s,"H2",{class:!0});var Xt=o(G);Z=p(Xt,"A",{id:!0,class:!0,href:!0});var Vr=o(Z);xa=p(Vr,"SPAN",{});var Yr=o(xa);f(us.$$.fragment,Yr),Yr.forEach(a),Vr.forEach(a),cn=c(Xt),$a=p(Xt,"SPAN",{});var Jr=o($a);un=l(Jr,"Training algorithm"),Jr.forEach(a),Xt.forEach(a),Ce=c(s),f(Q.$$.fragment,s),De=c(s),L=p(s,"P",{});var da=o(L);mn=l(da,"Like BPE, WordPiece starts from a small vocabulary including the special tokens used by the model and the initial alphabet. Since it identifies subwords by adding a prefix (like "),_a=p(da,"CODE",{});var Zr=o(_a);fn=l(Zr,"##"),Zr.forEach(a),dn=l(da," for BERT), each word is initially split by adding that prefix to all the characters inside the word. So, for instance, "),va=p(da,"CODE",{});var Qr=o(va);gn=l(Qr,'"word"'),Qr.forEach(a),bn=l(da," gets split like this:"),da.forEach(a),Ne=c(s),f(ms.$$.fragment,s),Oe=c(s),Js=p(s,"P",{});var Xr=o(Js);jn=l(Xr,"Thus, the initial alphabet contains all the characters present at the beginning of a word and the characters present inside a word preceded by the WordPiece prefix."),Xr.forEach(a),We=c(s),fs=p(s,"P",{});var Gr=o(fs);wn=l(Gr,`Then, again like BPE, WordPiece learns merge rules. The main difference is the way the pair to be merged is selected. Instead of selecting the most frequent pair, WordPiece computes a score for each pair, using the following formula:
`),Ae=qp(Gr),Gr.forEach(a),Se=c(s),z=p(s,"P",{});var A=o(z);xn=l(A,"By dividing the frequency of the pair by the product of the frequencies of each of its parts, the algorithm prioritizes the merging of pairs where the individual parts are less frequent in the vocabulary. For instance, it won\u2019t necessarily merge "),ka=p(A,"CODE",{});var si=o(ka);$n=l(si,'("un", "##able")'),si.forEach(a),_n=l(A," even if that pair occurs very frequently in the vocabulary, because the two pairs "),qa=p(A,"CODE",{});var ai=o(qa);vn=l(ai,'"un"'),ai.forEach(a),kn=l(A," and "),ya=p(A,"CODE",{});var ei=o(ya);qn=l(ei,'"##able"'),ei.forEach(a),yn=l(A," will likely each appear in a lot of other words and have a high frequency. In contrast, a pair like "),Ea=p(A,"CODE",{});var ti=o(Ea);En=l(ti,'("hu", "##gging")'),ti.forEach(a),zn=l(A," will probably be merged faster (assuming the word \u201Chugging\u201D appears often in the vocabulary) since "),za=p(A,"CODE",{});var ni=o(za);Pn=l(ni,'"hu"'),ni.forEach(a),Tn=l(A," and "),Pa=p(A,"CODE",{});var li=o(Pa);Cn=l(li,'"##gging"'),li.forEach(a),Dn=l(A," are likely to be less frequent individually."),A.forEach(a),Be=c(s),Zs=p(s,"P",{});var ri=o(Zs);Nn=l(ri,"Let\u2019s look at the same vocabulary we used in the BPE training example:"),ri.forEach(a),He=c(s),f(ds.$$.fragment,s),Fe=c(s),Qs=p(s,"P",{});var ii=o(Qs);On=l(ii,"The splits here will be:"),ii.forEach(a),Le=c(s),f(gs.$$.fragment,s),Ke=c(s),E=p(s,"P",{});var O=o(E);Wn=l(O,"so the initial vocabulary will be "),Ta=p(O,"CODE",{});var pi=o(Ta);An=l(pi,'["b", "h", "p", "##g", "##n", "##s", "##u"]'),pi.forEach(a),Sn=l(O," (if we forget about special tokens for now). The most frequent pair is "),Ca=p(O,"CODE",{});var oi=o(Ca);Bn=l(oi,'("##u", "##g")'),oi.forEach(a),Hn=l(O," (present 20 times), but the individual frequency of "),Da=p(O,"CODE",{});var hi=o(Da);Fn=l(hi,'"##u"'),hi.forEach(a),Ln=l(O," is very high, so its score is not the highest (it\u2019s 1 / 36). All pairs with a "),Na=p(O,"CODE",{});var ci=o(Na);Kn=l(ci,'"##u"'),ci.forEach(a),Un=l(O," actually have that same score (1 / 36), so the best score goes to the pair "),Oa=p(O,"CODE",{});var ui=o(Oa);In=l(ui,'("##g", "##s")'),ui.forEach(a),Rn=l(O," \u2014 the only one without a "),Wa=p(O,"CODE",{});var mi=o(Wa);Mn=l(mi,'"##u"'),mi.forEach(a),Gn=l(O," \u2014 at 1 / 20, and the first merge learned is "),Aa=p(O,"CODE",{});var fi=o(Aa);Vn=l(fi,'("##g", "##s") -> ("##gs")'),fi.forEach(a),Yn=l(O,"."),O.forEach(a),Ue=c(s),K=p(s,"P",{});var ga=o(K);Jn=l(ga,"Note that when we merge, we remove the "),Sa=p(ga,"CODE",{});var di=o(Sa);Zn=l(di,"##"),di.forEach(a),Qn=l(ga," between the two tokens, so we add "),Ba=p(ga,"CODE",{});var gi=o(Ba);Xn=l(gi,'"##gs"'),gi.forEach(a),sl=l(ga," to the vocabulary and apply the merge in the words of the corpus:"),ga.forEach(a),Ie=c(s),f(bs.$$.fragment,s),Re=c(s),U=p(s,"P",{});var ba=o(U);al=l(ba,"At this point, "),Ha=p(ba,"CODE",{});var bi=o(Ha);el=l(bi,'"##u"'),bi.forEach(a),tl=l(ba," is in all the possible pairs, so they all end up with the same score. Let\u2019s say that in this case, the first pair is merged, so "),Fa=p(ba,"CODE",{});var ji=o(Fa);nl=l(ji,'("h", "##u") -> "hu"'),ji.forEach(a),ll=l(ba,". This takes us to:"),ba.forEach(a),Me=c(s),f(js.$$.fragment,s),Ge=c(s),I=p(s,"P",{});var ja=o(I);rl=l(ja,"Then the next best score is shared by "),La=p(ja,"CODE",{});var wi=o(La);il=l(wi,'("hu", "##g")'),wi.forEach(a),pl=l(ja," and "),Ka=p(ja,"CODE",{});var xi=o(Ka);ol=l(xi,'("hu", "##gs")'),xi.forEach(a),hl=l(ja," (with 1/15, compared to 1/21 for all the other pairs), so the first pair with the biggest score is merged:"),ja.forEach(a),Ve=c(s),f(ws.$$.fragment,s),Ye=c(s),Xs=p(s,"P",{});var $i=o(Xs);cl=l($i,"and we continue like this until we reach the desired vocabulary size."),$i.forEach(a),Je=c(s),f(X.$$.fragment,s),Ze=c(s),V=p(s,"H2",{class:!0});var sn=o(V);ss=p(sn,"A",{id:!0,class:!0,href:!0});var _i=o(ss);Ua=p(_i,"SPAN",{});var vi=o(Ua);f(xs.$$.fragment,vi),vi.forEach(a),_i.forEach(a),ul=c(sn),Ia=p(sn,"SPAN",{});var ki=o(Ia);ml=l(ki,"Tokenization algorithm"),ki.forEach(a),sn.forEach(a),Qe=c(s),P=p(s,"P",{});var S=o(P);fl=l(S,"Tokenization differs in WordPiece and BPE in that WordPiece only saves the final vocabulary, not the merge rules learned. Starting from the word to tokenize, WordPiece finds the longest subword that is in the vocabulary, then splits on it. For instance, if we use the vocabulary learned in the example above, for the word "),Ra=p(S,"CODE",{});var qi=o(Ra);dl=l(qi,'"hugs"'),qi.forEach(a),gl=l(S," the longest subword starting from the beginning that is inside the vocabulary is "),Ma=p(S,"CODE",{});var yi=o(Ma);bl=l(yi,'"hug"'),yi.forEach(a),jl=l(S,", so we split there and get "),Ga=p(S,"CODE",{});var Ei=o(Ga);wl=l(Ei,'["hug", "##s"]'),Ei.forEach(a),xl=l(S,". We then continue with "),Va=p(S,"CODE",{});var zi=o(Va);$l=l(zi,'"##s"'),zi.forEach(a),_l=l(S,", which is in the vocabulary, so the tokenization of "),Ya=p(S,"CODE",{});var Pi=o(Ya);vl=l(Pi,'"hugs"'),Pi.forEach(a),kl=l(S," is "),Ja=p(S,"CODE",{});var Ti=o(Ja);ql=l(Ti,'["hug", "##s"]'),Ti.forEach(a),yl=l(S,"."),S.forEach(a),Xe=c(s),as=p(s,"P",{});var an=o(as);El=l(an,"With BPE, we would have applied the merges learned in order and tokenized this as "),Za=p(an,"CODE",{});var Ci=o(Za);zl=l(Ci,'["hu", "##gs"]'),Ci.forEach(a),Pl=l(an,", so the encoding is different."),an.forEach(a),st=c(s),k=p(s,"P",{});var T=o(k);Tl=l(T,"As another example, let\u2019s see how the word "),Qa=p(T,"CODE",{});var Di=o(Qa);Cl=l(Di,'"bugs"'),Di.forEach(a),Dl=l(T," would be tokenized. "),Xa=p(T,"CODE",{});var Ni=o(Xa);Nl=l(Ni,'"b"'),Ni.forEach(a),Ol=l(T," is the longest subword starting at the beginning of the word that is in the vocabulary, so we split there and get "),se=p(T,"CODE",{});var Oi=o(se);Wl=l(Oi,'["b", "##ugs"]'),Oi.forEach(a),Al=l(T,". Then "),ae=p(T,"CODE",{});var Wi=o(ae);Sl=l(Wi,'"##u"'),Wi.forEach(a),Bl=l(T," is the longest subword starting at the beginning of "),ee=p(T,"CODE",{});var Ai=o(ee);Hl=l(Ai,'"##ugs"'),Ai.forEach(a),Fl=l(T," that is in the vocabulary, so we split there and get "),te=p(T,"CODE",{});var Si=o(te);Ll=l(Si,'["b", "##u, "##gs"]'),Si.forEach(a),Kl=l(T,". Finally, "),ne=p(T,"CODE",{});var Bi=o(ne);Ul=l(Bi,'"##gs"'),Bi.forEach(a),Il=l(T," is in the vocabulary, so this last list is the tokenization of "),le=p(T,"CODE",{});var Hi=o(le);Rl=l(Hi,'"bugs"'),Hi.forEach(a),Ml=l(T,"."),T.forEach(a),at=c(s),q=p(s,"P",{});var C=o(q);Gl=l(C,"When the tokenization gets to a stage where it\u2019s not possible to find a subword in the vocabulary, the whole word is tokenized as unknown \u2014 so, for instance, "),re=p(C,"CODE",{});var Fi=o(re);Vl=l(Fi,'"mug"'),Fi.forEach(a),Yl=l(C," would be tokenized as "),ie=p(C,"CODE",{});var Li=o(ie);Jl=l(Li,'["[UNK]"]'),Li.forEach(a),Zl=l(C,", as would "),pe=p(C,"CODE",{});var Ki=o(pe);Ql=l(Ki,'"bum"'),Ki.forEach(a),Xl=l(C," (even if we can begin with "),oe=p(C,"CODE",{});var Ui=o(oe);sr=l(Ui,'"b"'),Ui.forEach(a),ar=l(C," and "),he=p(C,"CODE",{});var Ii=o(he);er=l(Ii,'"##u"'),Ii.forEach(a),tr=l(C,", "),ce=p(C,"CODE",{});var Ri=o(ce);nr=l(Ri,'"##m"'),Ri.forEach(a),lr=l(C," is not the vocabulary, and the resulting tokenization will just be "),ue=p(C,"CODE",{});var Mi=o(ue);rr=l(Mi,'["[UNK]"]'),Mi.forEach(a),ir=l(C,", not "),me=p(C,"CODE",{});var Gi=o(me);pr=l(Gi,'["b", "##u", "[UNK]"]'),Gi.forEach(a),or=l(C,"). This is another difference from BPE, which would only classify the individual characters not in the vocabulary as unknown."),C.forEach(a),et=c(s),f(es.$$.fragment,s),tt=c(s),Y=p(s,"H2",{class:!0});var en=o(Y);ts=p(en,"A",{id:!0,class:!0,href:!0});var Vi=o(ts);fe=p(Vi,"SPAN",{});var Yi=o(fe);f($s.$$.fragment,Yi),Yi.forEach(a),Vi.forEach(a),hr=c(en),de=p(en,"SPAN",{});var Ji=o(de);cr=l(Ji,"Implementing WordPiece"),Ji.forEach(a),en.forEach(a),nt=c(s),sa=p(s,"P",{});var Zi=o(sa);ur=l(Zi,"Now let\u2019s take a look at an implementation of the WordPiece algorithm. Like with BPE, this is just pedagogical, and you won\u2019t able to use this on a big corpus."),Zi.forEach(a),lt=c(s),aa=p(s,"P",{});var Qi=o(aa);mr=l(Qi,"We will use the same corpus as in the BPE example:"),Qi.forEach(a),rt=c(s),f(_s.$$.fragment,s),it=c(s),ns=p(s,"P",{});var tn=o(ns);fr=l(tn,"First, we need to pre-tokenize the corpus into words. Since we are replicating a WordPiece tokenizer (like BERT), we will use the "),ge=p(tn,"CODE",{});var Xi=o(ge);dr=l(Xi,"bert-base-cased"),Xi.forEach(a),gr=l(tn," tokenizer for the pre-tokenization:"),tn.forEach(a),pt=c(s),f(vs.$$.fragment,s),ot=c(s),ea=p(s,"P",{});var sp=o(ea);br=l(sp,"Then we compute the frequencies of each word in the corpus as we do the pre-tokenization:"),sp.forEach(a),ht=c(s),f(ks.$$.fragment,s),ct=c(s),f(qs.$$.fragment,s),ut=c(s),ls=p(s,"P",{});var nn=o(ls);jr=l(nn,"As we saw before, the alphabet is the unique set composed of all the first letters of words, and all the other letters that appear in words prefixed by "),be=p(nn,"CODE",{});var ap=o(be);wr=l(ap,"##"),ap.forEach(a),xr=l(nn,":"),nn.forEach(a),mt=c(s),f(ys.$$.fragment,s),ft=c(s),f(Es.$$.fragment,s),dt=c(s),rs=p(s,"P",{});var ln=o(rs);$r=l(ln,"We also add the special tokens used by the model at the beginning of that vocabulary. In the case of BERT, it\u2019s the list "),je=p(ln,"CODE",{});var ep=o(je);_r=l(ep,'["[PAD]", "[UNK]", "[CLS]", "[SEP]", "[MASK]"]'),ep.forEach(a),vr=l(ln,":"),ln.forEach(a),gt=c(s),f(zs.$$.fragment,s),bt=c(s),is=p(s,"P",{});var rn=o(is);kr=l(rn,"Next we need to split each word, with all the letters that are not the first prefixed by "),we=p(rn,"CODE",{});var tp=o(we);qr=l(tp,"##"),tp.forEach(a),yr=l(rn,":"),rn.forEach(a),jt=c(s),f(Ps.$$.fragment,s),wt=c(s),ta=p(s,"P",{});var np=o(ta);Er=l(np,"Now that we are ready for training, let\u2019s write a function that computes the score of each pair. We\u2019ll need to use this at each step of the training:"),np.forEach(a),xt=c(s),f(Ts.$$.fragment,s),$t=c(s),na=p(s,"P",{});var lp=o(na);zr=l(lp,"Let\u2019s have a look at a part of this dictionary after the initial splits:"),lp.forEach(a),_t=c(s),f(Cs.$$.fragment,s),vt=c(s),f(Ds.$$.fragment,s),kt=c(s),la=p(s,"P",{});var rp=o(la);Pr=l(rp,"Now, finding the pair with the best score only takes a quick loop:"),rp.forEach(a),qt=c(s),f(Ns.$$.fragment,s),yt=c(s),f(Os.$$.fragment,s),Et=c(s),R=p(s,"P",{});var wa=o(R);Tr=l(wa,"So the first merge to learn is "),xe=p(wa,"CODE",{});var ip=o(xe);Cr=l(ip,"('a', '##b') -> 'ab'"),ip.forEach(a),Dr=l(wa,", and we add "),$e=p(wa,"CODE",{});var pp=o($e);Nr=l(pp,"'ab'"),pp.forEach(a),Or=l(wa," to the vocabulary:"),wa.forEach(a),zt=c(s),f(Ws.$$.fragment,s),Pt=c(s),ps=p(s,"P",{});var pn=o(ps);Wr=l(pn,"To continue, we need to apply that merge in our "),_e=p(pn,"CODE",{});var op=o(_e);Ar=l(op,"splits"),op.forEach(a),Sr=l(pn," dictionary. Let\u2019s write another function for this:"),pn.forEach(a),Tt=c(s),f(As.$$.fragment,s),Ct=c(s),ra=p(s,"P",{});var hp=o(ra);Br=l(hp,"And we can have a look at the result of the first merge:"),hp.forEach(a),Dt=c(s),f(Ss.$$.fragment,s),Nt=c(s),f(Bs.$$.fragment,s),Ot=c(s),ia=p(s,"P",{});var cp=o(ia);Hr=l(cp,"Now we have everything we need to loop until we have learned all the merges we want. Let\u2019s aim for a vocab size of 70:"),cp.forEach(a),Wt=c(s),f(Hs.$$.fragment,s),At=c(s),pa=p(s,"P",{});var up=o(pa);Fr=l(up,"We can then look at the generated vocabulary:"),up.forEach(a),St=c(s),f(Fs.$$.fragment,s),Bt=c(s),f(Ls.$$.fragment,s),Ht=c(s),oa=p(s,"P",{});var mp=o(oa);Lr=l(mp,"As we can see, compared to BPE, this tokenizer learns parts of words as tokens a bit faster."),mp.forEach(a),Ft=c(s),f(os.$$.fragment,s),Lt=c(s),ha=p(s,"P",{});var fp=o(ha);Kr=l(fp,"To tokenize a new text, we pre-tokenize it, split it, then apply the tokenization algorithm on each word. That is, we look for the biggest subword starting at the beginning of the first word and split it, then we repeat the process on the second part, and so on for the rest of that word and the following words in the text:"),fp.forEach(a),Kt=c(s),f(Ks.$$.fragment,s),Ut=c(s),ca=p(s,"P",{});var dp=o(ca);Ur=l(dp,"Let\u2019s test it on one word that\u2019s in the vocabulary, and another that isn\u2019t:"),dp.forEach(a),It=c(s),f(Us.$$.fragment,s),Rt=c(s),f(Is.$$.fragment,s),Mt=c(s),ua=p(s,"P",{});var gp=o(ua);Ir=l(gp,"Now, let\u2019s write a function that tokenizes a text:"),gp.forEach(a),Gt=c(s),f(Rs.$$.fragment,s),Vt=c(s),ma=p(s,"P",{});var bp=o(ma);Rr=l(bp,"We can try it on any text:"),bp.forEach(a),Yt=c(s),f(Ms.$$.fragment,s),Jt=c(s),f(Gs.$$.fragment,s),Zt=c(s),fa=p(s,"P",{});var jp=o(fa);Mr=l(jp,"That\u2019s it for the WordPiece algorithm! Now let\u2019s take a look at Unigram."),jp.forEach(a),this.h()},h(){D(u,"name","hf:doc:metadata"),D(u,"content",JSON.stringify(Wp)),D($,"id","wordpiece-tokenization"),D($,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),D($,"href","#wordpiece-tokenization"),D(w,"class","relative group"),D(Z,"id","training-algorithm"),D(Z,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),D(Z,"href","#training-algorithm"),D(G,"class","relative group"),Ae.a=null,D(ss,"id","tokenization-algorithm"),D(ss,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),D(ss,"href","#tokenization-algorithm"),D(V,"class","relative group"),D(ts,"id","implementing-wordpiece"),D(ts,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),D(ts,"href","#implementing-wordpiece"),D(Y,"class","relative group")},m(s,t){e(document.head,u),r(s,v,t),r(s,w,t),e(w,$),e($,N),d(_,N,null),e(w,y),e(w,W),e(W,M),r(s,B,t),d(F,s,t),r(s,hs,t),r(s,Ys,t),e(Ys,hn),r(s,ze,t),d(cs,s,t),r(s,Pe,t),d(J,s,t),r(s,Te,t),r(s,G,t),e(G,Z),e(Z,xa),d(us,xa,null),e(G,cn),e(G,$a),e($a,un),r(s,Ce,t),d(Q,s,t),r(s,De,t),r(s,L,t),e(L,mn),e(L,_a),e(_a,fn),e(L,dn),e(L,va),e(va,gn),e(L,bn),r(s,Ne,t),d(ms,s,t),r(s,Oe,t),r(s,Js,t),e(Js,jn),r(s,We,t),r(s,fs,t),e(fs,wn),Ae.m(wp,fs),r(s,Se,t),r(s,z,t),e(z,xn),e(z,ka),e(ka,$n),e(z,_n),e(z,qa),e(qa,vn),e(z,kn),e(z,ya),e(ya,qn),e(z,yn),e(z,Ea),e(Ea,En),e(z,zn),e(z,za),e(za,Pn),e(z,Tn),e(z,Pa),e(Pa,Cn),e(z,Dn),r(s,Be,t),r(s,Zs,t),e(Zs,Nn),r(s,He,t),d(ds,s,t),r(s,Fe,t),r(s,Qs,t),e(Qs,On),r(s,Le,t),d(gs,s,t),r(s,Ke,t),r(s,E,t),e(E,Wn),e(E,Ta),e(Ta,An),e(E,Sn),e(E,Ca),e(Ca,Bn),e(E,Hn),e(E,Da),e(Da,Fn),e(E,Ln),e(E,Na),e(Na,Kn),e(E,Un),e(E,Oa),e(Oa,In),e(E,Rn),e(E,Wa),e(Wa,Mn),e(E,Gn),e(E,Aa),e(Aa,Vn),e(E,Yn),r(s,Ue,t),r(s,K,t),e(K,Jn),e(K,Sa),e(Sa,Zn),e(K,Qn),e(K,Ba),e(Ba,Xn),e(K,sl),r(s,Ie,t),d(bs,s,t),r(s,Re,t),r(s,U,t),e(U,al),e(U,Ha),e(Ha,el),e(U,tl),e(U,Fa),e(Fa,nl),e(U,ll),r(s,Me,t),d(js,s,t),r(s,Ge,t),r(s,I,t),e(I,rl),e(I,La),e(La,il),e(I,pl),e(I,Ka),e(Ka,ol),e(I,hl),r(s,Ve,t),d(ws,s,t),r(s,Ye,t),r(s,Xs,t),e(Xs,cl),r(s,Je,t),d(X,s,t),r(s,Ze,t),r(s,V,t),e(V,ss),e(ss,Ua),d(xs,Ua,null),e(V,ul),e(V,Ia),e(Ia,ml),r(s,Qe,t),r(s,P,t),e(P,fl),e(P,Ra),e(Ra,dl),e(P,gl),e(P,Ma),e(Ma,bl),e(P,jl),e(P,Ga),e(Ga,wl),e(P,xl),e(P,Va),e(Va,$l),e(P,_l),e(P,Ya),e(Ya,vl),e(P,kl),e(P,Ja),e(Ja,ql),e(P,yl),r(s,Xe,t),r(s,as,t),e(as,El),e(as,Za),e(Za,zl),e(as,Pl),r(s,st,t),r(s,k,t),e(k,Tl),e(k,Qa),e(Qa,Cl),e(k,Dl),e(k,Xa),e(Xa,Nl),e(k,Ol),e(k,se),e(se,Wl),e(k,Al),e(k,ae),e(ae,Sl),e(k,Bl),e(k,ee),e(ee,Hl),e(k,Fl),e(k,te),e(te,Ll),e(k,Kl),e(k,ne),e(ne,Ul),e(k,Il),e(k,le),e(le,Rl),e(k,Ml),r(s,at,t),r(s,q,t),e(q,Gl),e(q,re),e(re,Vl),e(q,Yl),e(q,ie),e(ie,Jl),e(q,Zl),e(q,pe),e(pe,Ql),e(q,Xl),e(q,oe),e(oe,sr),e(q,ar),e(q,he),e(he,er),e(q,tr),e(q,ce),e(ce,nr),e(q,lr),e(q,ue),e(ue,rr),e(q,ir),e(q,me),e(me,pr),e(q,or),r(s,et,t),d(es,s,t),r(s,tt,t),r(s,Y,t),e(Y,ts),e(ts,fe),d($s,fe,null),e(Y,hr),e(Y,de),e(de,cr),r(s,nt,t),r(s,sa,t),e(sa,ur),r(s,lt,t),r(s,aa,t),e(aa,mr),r(s,rt,t),d(_s,s,t),r(s,it,t),r(s,ns,t),e(ns,fr),e(ns,ge),e(ge,dr),e(ns,gr),r(s,pt,t),d(vs,s,t),r(s,ot,t),r(s,ea,t),e(ea,br),r(s,ht,t),d(ks,s,t),r(s,ct,t),d(qs,s,t),r(s,ut,t),r(s,ls,t),e(ls,jr),e(ls,be),e(be,wr),e(ls,xr),r(s,mt,t),d(ys,s,t),r(s,ft,t),d(Es,s,t),r(s,dt,t),r(s,rs,t),e(rs,$r),e(rs,je),e(je,_r),e(rs,vr),r(s,gt,t),d(zs,s,t),r(s,bt,t),r(s,is,t),e(is,kr),e(is,we),e(we,qr),e(is,yr),r(s,jt,t),d(Ps,s,t),r(s,wt,t),r(s,ta,t),e(ta,Er),r(s,xt,t),d(Ts,s,t),r(s,$t,t),r(s,na,t),e(na,zr),r(s,_t,t),d(Cs,s,t),r(s,vt,t),d(Ds,s,t),r(s,kt,t),r(s,la,t),e(la,Pr),r(s,qt,t),d(Ns,s,t),r(s,yt,t),d(Os,s,t),r(s,Et,t),r(s,R,t),e(R,Tr),e(R,xe),e(xe,Cr),e(R,Dr),e(R,$e),e($e,Nr),e(R,Or),r(s,zt,t),d(Ws,s,t),r(s,Pt,t),r(s,ps,t),e(ps,Wr),e(ps,_e),e(_e,Ar),e(ps,Sr),r(s,Tt,t),d(As,s,t),r(s,Ct,t),r(s,ra,t),e(ra,Br),r(s,Dt,t),d(Ss,s,t),r(s,Nt,t),d(Bs,s,t),r(s,Ot,t),r(s,ia,t),e(ia,Hr),r(s,Wt,t),d(Hs,s,t),r(s,At,t),r(s,pa,t),e(pa,Fr),r(s,St,t),d(Fs,s,t),r(s,Bt,t),d(Ls,s,t),r(s,Ht,t),r(s,oa,t),e(oa,Lr),r(s,Ft,t),d(os,s,t),r(s,Lt,t),r(s,ha,t),e(ha,Kr),r(s,Kt,t),d(Ks,s,t),r(s,Ut,t),r(s,ca,t),e(ca,Ur),r(s,It,t),d(Us,s,t),r(s,Rt,t),d(Is,s,t),r(s,Mt,t),r(s,ua,t),e(ua,Ir),r(s,Gt,t),d(Rs,s,t),r(s,Vt,t),r(s,ma,t),e(ma,Rr),r(s,Yt,t),d(Ms,s,t),r(s,Jt,t),d(Gs,s,t),r(s,Zt,t),r(s,fa,t),e(fa,Mr),Qt=!0},p(s,[t]){const Vs={};t&2&&(Vs.$$scope={dirty:t,ctx:s}),J.$set(Vs);const ve={};t&2&&(ve.$$scope={dirty:t,ctx:s}),Q.$set(ve);const ke={};t&2&&(ke.$$scope={dirty:t,ctx:s}),X.$set(ke);const qe={};t&2&&(qe.$$scope={dirty:t,ctx:s}),es.$set(qe);const ye={};t&2&&(ye.$$scope={dirty:t,ctx:s}),os.$set(ye)},i(s){Qt||(g(_.$$.fragment,s),g(F.$$.fragment,s),g(cs.$$.fragment,s),g(J.$$.fragment,s),g(us.$$.fragment,s),g(Q.$$.fragment,s),g(ms.$$.fragment,s),g(ds.$$.fragment,s),g(gs.$$.fragment,s),g(bs.$$.fragment,s),g(js.$$.fragment,s),g(ws.$$.fragment,s),g(X.$$.fragment,s),g(xs.$$.fragment,s),g(es.$$.fragment,s),g($s.$$.fragment,s),g(_s.$$.fragment,s),g(vs.$$.fragment,s),g(ks.$$.fragment,s),g(qs.$$.fragment,s),g(ys.$$.fragment,s),g(Es.$$.fragment,s),g(zs.$$.fragment,s),g(Ps.$$.fragment,s),g(Ts.$$.fragment,s),g(Cs.$$.fragment,s),g(Ds.$$.fragment,s),g(Ns.$$.fragment,s),g(Os.$$.fragment,s),g(Ws.$$.fragment,s),g(As.$$.fragment,s),g(Ss.$$.fragment,s),g(Bs.$$.fragment,s),g(Hs.$$.fragment,s),g(Fs.$$.fragment,s),g(Ls.$$.fragment,s),g(os.$$.fragment,s),g(Ks.$$.fragment,s),g(Us.$$.fragment,s),g(Is.$$.fragment,s),g(Rs.$$.fragment,s),g(Ms.$$.fragment,s),g(Gs.$$.fragment,s),Qt=!0)},o(s){b(_.$$.fragment,s),b(F.$$.fragment,s),b(cs.$$.fragment,s),b(J.$$.fragment,s),b(us.$$.fragment,s),b(Q.$$.fragment,s),b(ms.$$.fragment,s),b(ds.$$.fragment,s),b(gs.$$.fragment,s),b(bs.$$.fragment,s),b(js.$$.fragment,s),b(ws.$$.fragment,s),b(X.$$.fragment,s),b(xs.$$.fragment,s),b(es.$$.fragment,s),b($s.$$.fragment,s),b(_s.$$.fragment,s),b(vs.$$.fragment,s),b(ks.$$.fragment,s),b(qs.$$.fragment,s),b(ys.$$.fragment,s),b(Es.$$.fragment,s),b(zs.$$.fragment,s),b(Ps.$$.fragment,s),b(Ts.$$.fragment,s),b(Cs.$$.fragment,s),b(Ds.$$.fragment,s),b(Ns.$$.fragment,s),b(Os.$$.fragment,s),b(Ws.$$.fragment,s),b(As.$$.fragment,s),b(Ss.$$.fragment,s),b(Bs.$$.fragment,s),b(Hs.$$.fragment,s),b(Fs.$$.fragment,s),b(Ls.$$.fragment,s),b(os.$$.fragment,s),b(Ks.$$.fragment,s),b(Us.$$.fragment,s),b(Is.$$.fragment,s),b(Rs.$$.fragment,s),b(Ms.$$.fragment,s),b(Gs.$$.fragment,s),Qt=!1},d(s){a(u),s&&a(v),s&&a(w),j(_),s&&a(B),j(F,s),s&&a(hs),s&&a(Ys),s&&a(ze),j(cs,s),s&&a(Pe),j(J,s),s&&a(Te),s&&a(G),j(us),s&&a(Ce),j(Q,s),s&&a(De),s&&a(L),s&&a(Ne),j(ms,s),s&&a(Oe),s&&a(Js),s&&a(We),s&&a(fs),s&&a(Se),s&&a(z),s&&a(Be),s&&a(Zs),s&&a(He),j(ds,s),s&&a(Fe),s&&a(Qs),s&&a(Le),j(gs,s),s&&a(Ke),s&&a(E),s&&a(Ue),s&&a(K),s&&a(Ie),j(bs,s),s&&a(Re),s&&a(U),s&&a(Me),j(js,s),s&&a(Ge),s&&a(I),s&&a(Ve),j(ws,s),s&&a(Ye),s&&a(Xs),s&&a(Je),j(X,s),s&&a(Ze),s&&a(V),j(xs),s&&a(Qe),s&&a(P),s&&a(Xe),s&&a(as),s&&a(st),s&&a(k),s&&a(at),s&&a(q),s&&a(et),j(es,s),s&&a(tt),s&&a(Y),j($s),s&&a(nt),s&&a(sa),s&&a(lt),s&&a(aa),s&&a(rt),j(_s,s),s&&a(it),s&&a(ns),s&&a(pt),j(vs,s),s&&a(ot),s&&a(ea),s&&a(ht),j(ks,s),s&&a(ct),j(qs,s),s&&a(ut),s&&a(ls),s&&a(mt),j(ys,s),s&&a(ft),j(Es,s),s&&a(dt),s&&a(rs),s&&a(gt),j(zs,s),s&&a(bt),s&&a(is),s&&a(jt),j(Ps,s),s&&a(wt),s&&a(ta),s&&a(xt),j(Ts,s),s&&a($t),s&&a(na),s&&a(_t),j(Cs,s),s&&a(vt),j(Ds,s),s&&a(kt),s&&a(la),s&&a(qt),j(Ns,s),s&&a(yt),j(Os,s),s&&a(Et),s&&a(R),s&&a(zt),j(Ws,s),s&&a(Pt),s&&a(ps),s&&a(Tt),j(As,s),s&&a(Ct),s&&a(ra),s&&a(Dt),j(Ss,s),s&&a(Nt),j(Bs,s),s&&a(Ot),s&&a(ia),s&&a(Wt),j(Hs,s),s&&a(At),s&&a(pa),s&&a(St),j(Fs,s),s&&a(Bt),j(Ls,s),s&&a(Ht),s&&a(oa),s&&a(Ft),j(os,s),s&&a(Lt),s&&a(ha),s&&a(Kt),j(Ks,s),s&&a(Ut),s&&a(ca),s&&a(It),j(Us,s),s&&a(Rt),j(Is,s),s&&a(Mt),s&&a(ua),s&&a(Gt),j(Rs,s),s&&a(Vt),s&&a(ma),s&&a(Yt),j(Ms,s),s&&a(Jt),j(Gs,s),s&&a(Zt),s&&a(fa)}}}const Wp={local:"wordpiece-tokenization",sections:[{local:"training-algorithm",title:"Training algorithm"},{local:"tokenization-algorithm",title:"Tokenization algorithm"},{local:"implementing-wordpiece",title:"Implementing WordPiece"}],title:"WordPiece tokenization"};function Ap(H){return yp(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class Up extends xp{constructor(u){super();$p(this,u,Ap,Op,_p,{})}}export{Up as default,Wp as metadata};
