import{S as Sp,i as Op,s as Dp,e as i,k as h,w as u,t as o,U as Tn,M as Ap,c as p,d as e,m,a as r,x as f,h as l,V as Cn,b as $,G as t,g as n,y as d,q as g,o as b,B as w,v as Up}from"../../chunks/vendor-hf-doc-builder.js";import{T as zt}from"../../chunks/Tip-hf-doc-builder.js";import{Y as Wp}from"../../chunks/Youtube-hf-doc-builder.js";import{I as Tt}from"../../chunks/IconCopyLink-hf-doc-builder.js";import{C as j}from"../../chunks/CodeBlock-hf-doc-builder.js";import{D as Hp}from"../../chunks/DocNotebookDropdown-hf-doc-builder.js";function Bp(C){let c,k;return{c(){c=i("p"),k=o("\u{1F4A1} This section covers Unigram in depth, going as far as showing a full implementation. You can skip to the end if you just want a general overview of the tokenization algorithm.")},l(y){c=p(y,"P",{});var v=r(c);k=l(v,"\u{1F4A1} This section covers Unigram in depth, going as far as showing a full implementation. You can skip to the end if you just want a general overview of the tokenization algorithm."),v.forEach(e)},m(y,v){n(y,c,v),t(c,k)},d(y){y&&e(c)}}}function Ip(C){let c,k,y,v,z;return{c(){c=i("p"),k=o("\u270F\uFE0F "),y=i("strong"),v=o("Now your turn!"),z=o(" Write the code to compute the the frequencies above and double-check that the results shown are correct, as well as the total sum.")},l(_){c=p(_,"P",{});var x=r(c);k=l(x,"\u270F\uFE0F "),y=p(x,"STRONG",{});var N=r(y);v=l(N,"Now your turn!"),N.forEach(e),z=l(x," Write the code to compute the the frequencies above and double-check that the results shown are correct, as well as the total sum."),x.forEach(e)},m(_,x){n(_,c,x),t(c,k),t(c,y),t(y,v),t(c,z)},d(_){_&&e(c)}}}function Mp(C){let c,k,y,v,z,_,x,N;return{c(){c=i("p"),k=o("\u270F\uFE0F "),y=i("strong"),v=o("Now your turn!"),z=o(" Determine the tokenization of the word "),_=i("code"),x=o('"huggun"'),N=o(", and its score.")},l(F){c=p(F,"P",{});var T=r(c);k=l(T,"\u270F\uFE0F "),y=p(T,"STRONG",{});var H=r(y);v=l(H,"Now your turn!"),H.forEach(e),z=l(T," Determine the tokenization of the word "),_=p(T,"CODE",{});var gs=r(_);x=l(gs,'"huggun"'),gs.forEach(e),N=l(T,", and its score."),T.forEach(e)},m(F,T){n(F,c,T),t(c,k),t(c,y),t(y,v),t(c,z),t(c,_),t(_,x),t(c,N)},d(F){F&&e(c)}}}function Lp(C){let c,k;return{c(){c=i("p"),k=o("\u{1F4A1} SentencePiece uses a more efficient algorithm called Enhanced Suffix Array (ESA) to create the initial vocabulary.")},l(y){c=p(y,"P",{});var v=r(c);k=l(v,"\u{1F4A1} SentencePiece uses a more efficient algorithm called Enhanced Suffix Array (ESA) to create the initial vocabulary."),v.forEach(e)},m(y,v){n(y,c,v),t(c,k)},d(y){y&&e(c)}}}function Fp(C){let c,k;return{c(){c=i("p"),k=o("\u{1F4A1} This approach is very inefficient, so SentencePiece uses an approximation of the loss of the model without token X: instead of starting from scratch, it just replaces token X by its segmentation in the vocabulary that is left. This way, all the scores can be computed at once at the same time as the model loss.")},l(y){c=p(y,"P",{});var v=r(c);k=l(v,"\u{1F4A1} This approach is very inefficient, so SentencePiece uses an approximation of the loss of the model without token X: instead of starting from scratch, it just replaces token X by its segmentation in the vocabulary that is left. This way, all the scores can be computed at once at the same time as the model loss."),v.forEach(e)},m(y,v){n(y,c,v),t(c,k)},d(y){y&&e(c)}}}function Xp(C){let c,k,y,v,z,_,x,N,F,T,H,gs,Qs,Nn,Ct,bs,Nt,Z,St,X,J,Ne,ws,Sn,Se,On,Ot,se,Dn,Dt,ee,An,At,B,Un,Ut,zp='<span class="katex"><span class="katex-mathml"><math xmlns="http://www.w3.org/1998/Math/MathML"><semantics><mrow><mi>p</mi></mrow><annotation encoding="application/x-tex">p</annotation></semantics></math></span><span class="katex-html" aria-hidden="true"><span class="base"><span class="strut" style="height:0.625em;vertical-align:-0.1944em;"></span><span class="mord mathnormal">p</span></span></span></span>',Wt,Ht,Tp='<span class="katex"><span class="katex-mathml"><math xmlns="http://www.w3.org/1998/Math/MathML"><semantics><mrow><mi>p</mi></mrow><annotation encoding="application/x-tex">p</annotation></semantics></math></span><span class="katex-html" aria-hidden="true"><span class="base"><span class="strut" style="height:0.625em;vertical-align:-0.1944em;"></span><span class="mord mathnormal">p</span></span></span></span>',Bt,It,te,Wn,Mt,ae,Hn,Lt,ne,Bn,Ft,ys,Xt,oe,In,Rt,vs,Vt,R,K,Oe,js,Mn,De,Ln,Gt,le,Fn,Yt,P,Xn,Ae,Rn,Vn,Ue,Gn,Yn,We,Zn,Jn,He,Kn,Qn,Zt,ie,so,Jt,ks,Kt,Q,eo,Be,to,ao,Qt,ss,sa,W,no,Ie,oo,lo,Me,io,po,ea,Cp='<span class="katex-display"><span class="katex"><span class="katex-mathml"><math xmlns="http://www.w3.org/1998/Math/MathML" display="block"><semantics><mrow><mi>P</mi><mo stretchy="false">(</mo><mo stretchy="false">[</mo><mi mathvariant="normal">\u2018</mi><mi mathvariant="normal">\u2018</mi><mi>p</mi><mi mathvariant="normal">&quot;</mi><mo separator="true">,</mo><mi mathvariant="normal">\u2018</mi><mi mathvariant="normal">\u2018</mi><mi>u</mi><mi mathvariant="normal">&quot;</mi><mo separator="true">,</mo><mi mathvariant="normal">\u2018</mi><mi mathvariant="normal">\u2018</mi><mi>g</mi><mi mathvariant="normal">&quot;</mi><mo stretchy="false">]</mo><mo stretchy="false">)</mo><mo>=</mo><mi>P</mi><mo stretchy="false">(</mo><mi mathvariant="normal">\u2018</mi><mi mathvariant="normal">\u2018</mi><mi>p</mi><mi mathvariant="normal">&quot;</mi><mo stretchy="false">)</mo><mo>\xD7</mo><mi>P</mi><mo stretchy="false">(</mo><mi mathvariant="normal">\u2018</mi><mi mathvariant="normal">\u2018</mi><mi>u</mi><mi mathvariant="normal">&quot;</mi><mo stretchy="false">)</mo><mo>\xD7</mo><mi>P</mi><mo stretchy="false">(</mo><mi mathvariant="normal">\u2018</mi><mi mathvariant="normal">\u2018</mi><mi>g</mi><mi mathvariant="normal">&quot;</mi><mo stretchy="false">)</mo><mo>=</mo><mfrac><mn>5</mn><mn>210</mn></mfrac><mo>\xD7</mo><mfrac><mn>36</mn><mn>210</mn></mfrac><mo>\xD7</mo><mfrac><mn>20</mn><mn>210</mn></mfrac><mo>=</mo><mn>0.000389</mn></mrow><annotation encoding="application/x-tex">P([``p&quot;, ``u&quot;, ``g&quot;]) = P(``p&quot;) \\times P(``u&quot;) \\times P(``g&quot;) = \\frac{5}{210} \\times \\frac{36}{210} \\times \\frac{20}{210} = 0.000389</annotation></semantics></math></span><span class="katex-html" aria-hidden="true"><span class="base"><span class="strut" style="height:1em;vertical-align:-0.25em;"></span><span class="mord mathnormal" style="margin-right:0.13889em;">P</span><span class="mopen">([</span><span class="mord">\u2018\u2018</span><span class="mord mathnormal">p</span><span class="mord">&quot;</span><span class="mpunct">,</span><span class="mspace" style="margin-right:0.1667em;"></span><span class="mord">\u2018\u2018</span><span class="mord mathnormal">u</span><span class="mord">&quot;</span><span class="mpunct">,</span><span class="mspace" style="margin-right:0.1667em;"></span><span class="mord">\u2018\u2018</span><span class="mord mathnormal" style="margin-right:0.03588em;">g</span><span class="mord">&quot;</span><span class="mclose">])</span><span class="mspace" style="margin-right:0.2778em;"></span><span class="mrel">=</span><span class="mspace" style="margin-right:0.2778em;"></span></span><span class="base"><span class="strut" style="height:1em;vertical-align:-0.25em;"></span><span class="mord mathnormal" style="margin-right:0.13889em;">P</span><span class="mopen">(</span><span class="mord">\u2018\u2018</span><span class="mord mathnormal">p</span><span class="mord">&quot;</span><span class="mclose">)</span><span class="mspace" style="margin-right:0.2222em;"></span><span class="mbin">\xD7</span><span class="mspace" style="margin-right:0.2222em;"></span></span><span class="base"><span class="strut" style="height:1em;vertical-align:-0.25em;"></span><span class="mord mathnormal" style="margin-right:0.13889em;">P</span><span class="mopen">(</span><span class="mord">\u2018\u2018</span><span class="mord mathnormal">u</span><span class="mord">&quot;</span><span class="mclose">)</span><span class="mspace" style="margin-right:0.2222em;"></span><span class="mbin">\xD7</span><span class="mspace" style="margin-right:0.2222em;"></span></span><span class="base"><span class="strut" style="height:1em;vertical-align:-0.25em;"></span><span class="mord mathnormal" style="margin-right:0.13889em;">P</span><span class="mopen">(</span><span class="mord">\u2018\u2018</span><span class="mord mathnormal" style="margin-right:0.03588em;">g</span><span class="mord">&quot;</span><span class="mclose">)</span><span class="mspace" style="margin-right:0.2778em;"></span><span class="mrel">=</span><span class="mspace" style="margin-right:0.2778em;"></span></span><span class="base"><span class="strut" style="height:2.0074em;vertical-align:-0.686em;"></span><span class="mord"><span class="mopen nulldelimiter"></span><span class="mfrac"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:1.3214em;"><span style="top:-2.314em;"><span class="pstrut" style="height:3em;"></span><span class="mord"><span class="mord">210</span></span></span><span style="top:-3.23em;"><span class="pstrut" style="height:3em;"></span><span class="frac-line" style="border-bottom-width:0.04em;"></span></span><span style="top:-3.677em;"><span class="pstrut" style="height:3em;"></span><span class="mord"><span class="mord">5</span></span></span></span><span class="vlist-s">\u200B</span></span><span class="vlist-r"><span class="vlist" style="height:0.686em;"><span></span></span></span></span></span><span class="mclose nulldelimiter"></span></span><span class="mspace" style="margin-right:0.2222em;"></span><span class="mbin">\xD7</span><span class="mspace" style="margin-right:0.2222em;"></span></span><span class="base"><span class="strut" style="height:2.0074em;vertical-align:-0.686em;"></span><span class="mord"><span class="mopen nulldelimiter"></span><span class="mfrac"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:1.3214em;"><span style="top:-2.314em;"><span class="pstrut" style="height:3em;"></span><span class="mord"><span class="mord">210</span></span></span><span style="top:-3.23em;"><span class="pstrut" style="height:3em;"></span><span class="frac-line" style="border-bottom-width:0.04em;"></span></span><span style="top:-3.677em;"><span class="pstrut" style="height:3em;"></span><span class="mord"><span class="mord">36</span></span></span></span><span class="vlist-s">\u200B</span></span><span class="vlist-r"><span class="vlist" style="height:0.686em;"><span></span></span></span></span></span><span class="mclose nulldelimiter"></span></span><span class="mspace" style="margin-right:0.2222em;"></span><span class="mbin">\xD7</span><span class="mspace" style="margin-right:0.2222em;"></span></span><span class="base"><span class="strut" style="height:2.0074em;vertical-align:-0.686em;"></span><span class="mord"><span class="mopen nulldelimiter"></span><span class="mfrac"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:1.3214em;"><span style="top:-2.314em;"><span class="pstrut" style="height:3em;"></span><span class="mord"><span class="mord">210</span></span></span><span style="top:-3.23em;"><span class="pstrut" style="height:3em;"></span><span class="frac-line" style="border-bottom-width:0.04em;"></span></span><span style="top:-3.677em;"><span class="pstrut" style="height:3em;"></span><span class="mord"><span class="mord">20</span></span></span></span><span class="vlist-s">\u200B</span></span><span class="vlist-r"><span class="vlist" style="height:0.686em;"><span></span></span></span></span></span><span class="mclose nulldelimiter"></span></span><span class="mspace" style="margin-right:0.2778em;"></span><span class="mrel">=</span><span class="mspace" style="margin-right:0.2778em;"></span></span><span class="base"><span class="strut" style="height:0.6444em;"></span><span class="mord">0.000389</span></span></span></span></span>',ta,V,ro,Le,ho,mo,aa,Np='<span class="katex-display"><span class="katex"><span class="katex-mathml"><math xmlns="http://www.w3.org/1998/Math/MathML" display="block"><semantics><mrow><mi>P</mi><mo stretchy="false">(</mo><mo stretchy="false">[</mo><mi mathvariant="normal">\u2018</mi><mi mathvariant="normal">\u2018</mi><mi>p</mi><mi>u</mi><mi mathvariant="normal">&quot;</mi><mo separator="true">,</mo><mi mathvariant="normal">\u2018</mi><mi mathvariant="normal">\u2018</mi><mi>g</mi><mi mathvariant="normal">&quot;</mi><mo stretchy="false">]</mo><mo stretchy="false">)</mo><mo>=</mo><mi>P</mi><mo stretchy="false">(</mo><mi mathvariant="normal">\u2018</mi><mi mathvariant="normal">\u2018</mi><mi>p</mi><mi>u</mi><mi mathvariant="normal">&quot;</mi><mo stretchy="false">)</mo><mo>\xD7</mo><mi>P</mi><mo stretchy="false">(</mo><mi mathvariant="normal">\u2018</mi><mi mathvariant="normal">\u2018</mi><mi>g</mi><mi mathvariant="normal">&quot;</mi><mo stretchy="false">)</mo><mo>=</mo><mfrac><mn>5</mn><mn>210</mn></mfrac><mo>\xD7</mo><mfrac><mn>20</mn><mn>210</mn></mfrac><mo>=</mo><mn>0.0022676</mn></mrow><annotation encoding="application/x-tex">P([``pu&quot;, ``g&quot;]) = P(``pu&quot;) \\times P(``g&quot;) = \\frac{5}{210} \\times \\frac{20}{210} = 0.0022676</annotation></semantics></math></span><span class="katex-html" aria-hidden="true"><span class="base"><span class="strut" style="height:1em;vertical-align:-0.25em;"></span><span class="mord mathnormal" style="margin-right:0.13889em;">P</span><span class="mopen">([</span><span class="mord">\u2018\u2018</span><span class="mord mathnormal">p</span><span class="mord mathnormal">u</span><span class="mord">&quot;</span><span class="mpunct">,</span><span class="mspace" style="margin-right:0.1667em;"></span><span class="mord">\u2018\u2018</span><span class="mord mathnormal" style="margin-right:0.03588em;">g</span><span class="mord">&quot;</span><span class="mclose">])</span><span class="mspace" style="margin-right:0.2778em;"></span><span class="mrel">=</span><span class="mspace" style="margin-right:0.2778em;"></span></span><span class="base"><span class="strut" style="height:1em;vertical-align:-0.25em;"></span><span class="mord mathnormal" style="margin-right:0.13889em;">P</span><span class="mopen">(</span><span class="mord">\u2018\u2018</span><span class="mord mathnormal">p</span><span class="mord mathnormal">u</span><span class="mord">&quot;</span><span class="mclose">)</span><span class="mspace" style="margin-right:0.2222em;"></span><span class="mbin">\xD7</span><span class="mspace" style="margin-right:0.2222em;"></span></span><span class="base"><span class="strut" style="height:1em;vertical-align:-0.25em;"></span><span class="mord mathnormal" style="margin-right:0.13889em;">P</span><span class="mopen">(</span><span class="mord">\u2018\u2018</span><span class="mord mathnormal" style="margin-right:0.03588em;">g</span><span class="mord">&quot;</span><span class="mclose">)</span><span class="mspace" style="margin-right:0.2778em;"></span><span class="mrel">=</span><span class="mspace" style="margin-right:0.2778em;"></span></span><span class="base"><span class="strut" style="height:2.0074em;vertical-align:-0.686em;"></span><span class="mord"><span class="mopen nulldelimiter"></span><span class="mfrac"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:1.3214em;"><span style="top:-2.314em;"><span class="pstrut" style="height:3em;"></span><span class="mord"><span class="mord">210</span></span></span><span style="top:-3.23em;"><span class="pstrut" style="height:3em;"></span><span class="frac-line" style="border-bottom-width:0.04em;"></span></span><span style="top:-3.677em;"><span class="pstrut" style="height:3em;"></span><span class="mord"><span class="mord">5</span></span></span></span><span class="vlist-s">\u200B</span></span><span class="vlist-r"><span class="vlist" style="height:0.686em;"><span></span></span></span></span></span><span class="mclose nulldelimiter"></span></span><span class="mspace" style="margin-right:0.2222em;"></span><span class="mbin">\xD7</span><span class="mspace" style="margin-right:0.2222em;"></span></span><span class="base"><span class="strut" style="height:2.0074em;vertical-align:-0.686em;"></span><span class="mord"><span class="mopen nulldelimiter"></span><span class="mfrac"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:1.3214em;"><span style="top:-2.314em;"><span class="pstrut" style="height:3em;"></span><span class="mord"><span class="mord">210</span></span></span><span style="top:-3.23em;"><span class="pstrut" style="height:3em;"></span><span class="frac-line" style="border-bottom-width:0.04em;"></span></span><span style="top:-3.677em;"><span class="pstrut" style="height:3em;"></span><span class="mord"><span class="mord">20</span></span></span></span><span class="vlist-s">\u200B</span></span><span class="vlist-r"><span class="vlist" style="height:0.686em;"><span></span></span></span></span></span><span class="mclose nulldelimiter"></span></span><span class="mspace" style="margin-right:0.2778em;"></span><span class="mrel">=</span><span class="mspace" style="margin-right:0.2778em;"></span></span><span class="base"><span class="strut" style="height:0.6444em;"></span><span class="mord">0.0022676</span></span></span></span></span>',na,pe,co,oa,es,uo,Fe,fo,go,la,$s,ia,S,bo,Xe,wo,yo,Re,vo,jo,Ve,ko,$o,pa,q,_o,Ge,qo,Eo,Ye,xo,Po,Ze,zo,To,Je,Co,No,Ke,So,Oo,ra,re,Do,ha,ts,Ao,Qe,Uo,Wo,ma,_s,ca,I,Ho,st,Bo,Io,et,Mo,Lo,ua,as,fa,G,ns,tt,qs,Fo,at,Xo,da,he,Ro,ga,os,Vo,nt,Go,Yo,ba,me,Zo,wa,Es,ya,ce,Jo,va,xs,ja,ue,Ko,ka,Ps,$a,O,Qo,ot,sl,el,lt,tl,al,it,nl,ol,_a,D,ll,pt,il,pl,rt,rl,hl,ht,ml,cl,qa,zs,Ea,fe,ul,xa,Ts,Pa,M,fl,mt,dl,gl,ct,bl,wl,za,Y,ls,ut,Cs,yl,ft,vl,Ta,de,jl,Ca,ge,kl,Na,Ns,Sa,is,$l,dt,_l,ql,Oa,Ss,Da,be,El,Aa,Os,Ua,we,xl,Wa,Ds,Ha,As,Ba,ye,Pl,Ia,Us,Ma,ps,La,ve,zl,Fa,Ws,Xa,rs,Tl,gt,Cl,Nl,Ra,hs,Sl,bt,Ol,Dl,Va,je,Al,Ga,Hs,Ya,ke,Ul,Za,Bs,Ja,Is,Ka,$e,Wl,Qa,Ms,sn,_e,Hl,en,Ls,tn,Fs,an,qe,Bl,nn,Xs,on,Ee,Il,ln,Rs,pn,E,Ml,wt,Ll,Fl,yt,Xl,Rl,vt,Vl,Gl,jt,Yl,Zl,kt,Jl,Kl,rn,Vs,hn,ms,mn,xe,Ql,cn,Gs,un,cs,si,$t,ei,ti,fn,Ys,dn,Zs,gn,Pe,ai,bn;return _=new Tt({}),H=new Hp({props:{classNames:"absolute z-10 right-0 top-0",options:[{label:"Google Colab",value:"https://colab.research.google.com/github/huggingface/notebooks/blob/master/course/chapter6/section7.ipynb"},{label:"Aws Studio",value:"https://studiolab.sagemaker.aws/import/github/huggingface/notebooks/blob/master/course/chapter6/section7.ipynb"}]}}),bs=new Wp({props:{id:"TGZfZVuF9Yc"}}),Z=new zt({props:{$$slots:{default:[Bp]},$$scope:{ctx:C}}}),ws=new Tt({}),ys=new j({props:{code:'("hug", 10), ("pug", 5), ("pun", 12), ("bun", 4), ("hugs", 5)',highlighted:'(<span class="hljs-string">&quot;hug&quot;</span><span class="hljs-punctuation">,</span> <span class="hljs-number">10</span>)<span class="hljs-punctuation">,</span> (<span class="hljs-string">&quot;pug&quot;</span><span class="hljs-punctuation">,</span> <span class="hljs-number">5</span>)<span class="hljs-punctuation">,</span> (<span class="hljs-string">&quot;pun&quot;</span><span class="hljs-punctuation">,</span> <span class="hljs-number">12</span>)<span class="hljs-punctuation">,</span> (<span class="hljs-string">&quot;bun&quot;</span><span class="hljs-punctuation">,</span> <span class="hljs-number">4</span>)<span class="hljs-punctuation">,</span> (<span class="hljs-string">&quot;hugs&quot;</span><span class="hljs-punctuation">,</span> <span class="hljs-number">5</span>)'}}),vs=new j({props:{code:'["h", "u", "g", "hu", "ug", "p", "pu", "n", "un", "b", "bu", "s", "hug", "gs", "ugs"]',highlighted:'<span class="hljs-selector-attr">[<span class="hljs-string">&quot;h&quot;</span>, <span class="hljs-string">&quot;u&quot;</span>, <span class="hljs-string">&quot;g&quot;</span>, <span class="hljs-string">&quot;hu&quot;</span>, <span class="hljs-string">&quot;ug&quot;</span>, <span class="hljs-string">&quot;p&quot;</span>, <span class="hljs-string">&quot;pu&quot;</span>, <span class="hljs-string">&quot;n&quot;</span>, <span class="hljs-string">&quot;un&quot;</span>, <span class="hljs-string">&quot;b&quot;</span>, <span class="hljs-string">&quot;bu&quot;</span>, <span class="hljs-string">&quot;s&quot;</span>, <span class="hljs-string">&quot;hug&quot;</span>, <span class="hljs-string">&quot;gs&quot;</span>, <span class="hljs-string">&quot;ugs&quot;</span>]</span>'}}),js=new Tt({}),ks=new j({props:{code:`("h", 15) ("u", 36) ("g", 20) ("hu", 15) ("ug", 20) ("p", 17) ("pu", 17) ("n", 16)
("un", 16) ("b", 4) ("bu", 4) ("s", 5) ("hug", 15) ("gs", 5) ("ugs", 5)`,highlighted:`(<span class="hljs-string">&quot;h&quot;</span><span class="hljs-punctuation">,</span> <span class="hljs-number">15</span>) (<span class="hljs-string">&quot;u&quot;</span><span class="hljs-punctuation">,</span> <span class="hljs-number">36</span>) (<span class="hljs-string">&quot;g&quot;</span><span class="hljs-punctuation">,</span> <span class="hljs-number">20</span>) (<span class="hljs-string">&quot;hu&quot;</span><span class="hljs-punctuation">,</span> <span class="hljs-number">15</span>) (<span class="hljs-string">&quot;ug&quot;</span><span class="hljs-punctuation">,</span> <span class="hljs-number">20</span>) (<span class="hljs-string">&quot;p&quot;</span><span class="hljs-punctuation">,</span> <span class="hljs-number">17</span>) (<span class="hljs-string">&quot;pu&quot;</span><span class="hljs-punctuation">,</span> <span class="hljs-number">17</span>) (<span class="hljs-string">&quot;n&quot;</span><span class="hljs-punctuation">,</span> <span class="hljs-number">16</span>)
(<span class="hljs-string">&quot;un&quot;</span><span class="hljs-punctuation">,</span> <span class="hljs-number">16</span>) (<span class="hljs-string">&quot;b&quot;</span><span class="hljs-punctuation">,</span> <span class="hljs-number">4</span>) (<span class="hljs-string">&quot;bu&quot;</span><span class="hljs-punctuation">,</span> <span class="hljs-number">4</span>) (<span class="hljs-string">&quot;s&quot;</span><span class="hljs-punctuation">,</span> <span class="hljs-number">5</span>) (<span class="hljs-string">&quot;hug&quot;</span><span class="hljs-punctuation">,</span> <span class="hljs-number">15</span>) (<span class="hljs-string">&quot;gs&quot;</span><span class="hljs-punctuation">,</span> <span class="hljs-number">5</span>) (<span class="hljs-string">&quot;ugs&quot;</span><span class="hljs-punctuation">,</span> <span class="hljs-number">5</span>)`}}),ss=new zt({props:{$$slots:{default:[Ip]},$$scope:{ctx:C}}}),$s=new j({props:{code:`["p", "u", "g"] : 0.000389
["p", "ug"] : 0.0022676
["pu", "g"] : 0.0022676`,highlighted:`[<span class="hljs-string">&quot;p&quot;</span>, <span class="hljs-string">&quot;u&quot;</span>, <span class="hljs-string">&quot;g&quot;</span>] : 0.000389
[<span class="hljs-string">&quot;p&quot;</span>, <span class="hljs-string">&quot;ug&quot;</span>] : 0.0022676
[<span class="hljs-string">&quot;pu&quot;</span>, <span class="hljs-string">&quot;g&quot;</span>] : 0.0022676`}}),_s=new j({props:{code:`Character 0 (u): "u" (score 0.171429)
Character 1 (n): "un" (score 0.076191)
Character 2 (h): "un" "h" (score 0.005442)
Character 3 (u): "un" "hu" (score 0.005442)
Character 4 (g): "un" "hug" (score 0.005442)`,highlighted:`<span class="hljs-attribute">Character</span> <span class="hljs-number">0</span> (u): <span class="hljs-string">&quot;u&quot;</span> (score <span class="hljs-number">0</span>.<span class="hljs-number">171429</span>)
<span class="hljs-attribute">Character</span> <span class="hljs-number">1</span> (n): <span class="hljs-string">&quot;un&quot;</span> (score <span class="hljs-number">0</span>.<span class="hljs-number">076191</span>)
<span class="hljs-attribute">Character</span> <span class="hljs-number">2</span> (h): <span class="hljs-string">&quot;un&quot;</span> <span class="hljs-string">&quot;h&quot;</span> (score <span class="hljs-number">0</span>.<span class="hljs-number">005442</span>)
<span class="hljs-attribute">Character</span> <span class="hljs-number">3</span> (u): <span class="hljs-string">&quot;un&quot;</span> <span class="hljs-string">&quot;hu&quot;</span> (score <span class="hljs-number">0</span>.<span class="hljs-number">005442</span>)
<span class="hljs-attribute">Character</span> <span class="hljs-number">4</span> (g): <span class="hljs-string">&quot;un&quot;</span> <span class="hljs-string">&quot;hug&quot;</span> (score <span class="hljs-number">0</span>.<span class="hljs-number">005442</span>)`}}),as=new zt({props:{$$slots:{default:[Mp]},$$scope:{ctx:C}}}),qs=new Tt({}),Es=new j({props:{code:'("hug", 10), ("pug", 5), ("pun", 12), ("bun", 4), ("hugs", 5)',highlighted:'(<span class="hljs-string">&quot;hug&quot;</span><span class="hljs-punctuation">,</span> <span class="hljs-number">10</span>)<span class="hljs-punctuation">,</span> (<span class="hljs-string">&quot;pug&quot;</span><span class="hljs-punctuation">,</span> <span class="hljs-number">5</span>)<span class="hljs-punctuation">,</span> (<span class="hljs-string">&quot;pun&quot;</span><span class="hljs-punctuation">,</span> <span class="hljs-number">12</span>)<span class="hljs-punctuation">,</span> (<span class="hljs-string">&quot;bun&quot;</span><span class="hljs-punctuation">,</span> <span class="hljs-number">4</span>)<span class="hljs-punctuation">,</span> (<span class="hljs-string">&quot;hugs&quot;</span><span class="hljs-punctuation">,</span> <span class="hljs-number">5</span>)'}}),xs=new j({props:{code:`"hug": ["hug"] (score 0.071428)
"pug": ["pu", "g"] (score 0.007710)
"pun": ["pu", "n"] (score 0.006168)
"bun": ["bu", "n"] (score 0.001451)
"hugs": ["hug", "s"] (score 0.001701)`,highlighted:`<span class="hljs-string">&quot;hug&quot;</span>: [<span class="hljs-string">&quot;hug&quot;</span>] <span class="hljs-comment">(score 0.071428)</span>
<span class="hljs-string">&quot;pug&quot;</span>: [<span class="hljs-string">&quot;pu&quot;</span>, <span class="hljs-string">&quot;g&quot;</span>] <span class="hljs-comment">(score 0.007710)</span>
<span class="hljs-string">&quot;pun&quot;</span>: [<span class="hljs-string">&quot;pu&quot;</span>, <span class="hljs-string">&quot;n&quot;</span>] <span class="hljs-comment">(score 0.006168)</span>
<span class="hljs-string">&quot;bun&quot;</span>: [<span class="hljs-string">&quot;bu&quot;</span>, <span class="hljs-string">&quot;n&quot;</span>] <span class="hljs-comment">(score 0.001451)</span>
<span class="hljs-string">&quot;hugs&quot;</span>: [<span class="hljs-string">&quot;hug&quot;</span>, <span class="hljs-string">&quot;s&quot;</span>] <span class="hljs-comment">(score 0.001701)</span>`}}),Ps=new j({props:{code:"10 * (-log(0.071428)) + 5 * (-log(0.007710)) + 12 * (-log(0.006168)) + 4 * (-log(0.001451)) + 5 * (-log(0.001701)) = 169.8",highlighted:'<span class="hljs-attribute">10</span> * (-log(<span class="hljs-number">0</span>.<span class="hljs-number">071428</span>)) + <span class="hljs-number">5</span> * (-log(<span class="hljs-number">0</span>.<span class="hljs-number">007710</span>)) + <span class="hljs-number">12</span> * (-log(<span class="hljs-number">0</span>.<span class="hljs-number">006168</span>)) + <span class="hljs-number">4</span> * (-log(<span class="hljs-number">0</span>.<span class="hljs-number">001451</span>)) + <span class="hljs-number">5</span> * (-log(<span class="hljs-number">0</span>.<span class="hljs-number">001701</span>)) = <span class="hljs-number">169</span>.<span class="hljs-number">8</span>'}}),zs=new j({props:{code:`"hug": ["hu", "g"] (score 0.006802)
"hugs": ["hu", "gs"] (score 0.001701)`,highlighted:`<span class="hljs-string">&quot;hug&quot;</span>: [<span class="hljs-string">&quot;hu&quot;</span>, <span class="hljs-string">&quot;g&quot;</span>] <span class="hljs-comment">(score 0.006802)</span>
<span class="hljs-string">&quot;hugs&quot;</span>: [<span class="hljs-string">&quot;hu&quot;</span>, <span class="hljs-string">&quot;gs&quot;</span>] <span class="hljs-comment">(score 0.001701)</span>`}}),Ts=new j({props:{code:"- 10 * (-log(0.071428)) + 10 * (-log(0.006802)) = 23.5",highlighted:'- <span class="hljs-number">10</span> * (<span class="hljs-name">-log</span>(<span class="hljs-number">0.071428</span>)) + <span class="hljs-number">10</span> * (<span class="hljs-name">-log</span>(<span class="hljs-number">0.006802</span>)) = <span class="hljs-number">23.5</span>'}}),Cs=new Tt({}),Ns=new j({props:{code:`corpus = [
    "This is the Hugging Face Course.",
    "This chapter is about tokenization.",
    "This section shows several tokenizer algorithms.",
    "Hopefully, you will be able to understand how they are trained and generate tokens.",
]`,highlighted:`corpus = [
    <span class="hljs-string">&quot;This is the Hugging Face Course.&quot;</span>,
    <span class="hljs-string">&quot;This chapter is about tokenization.&quot;</span>,
    <span class="hljs-string">&quot;This section shows several tokenizer algorithms.&quot;</span>,
    <span class="hljs-string">&quot;Hopefully, you will be able to understand how they are trained and generate tokens.&quot;</span>,
]`}}),Ss=new j({props:{code:"",highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoTokenizer

tokenizer = AutoTokenizer.from_pretrained(<span class="hljs-string">&quot;xlnet-base-cased&quot;</span>)`}}),Os=new j({props:{code:`
`,highlighted:`<span class="hljs-keyword">from</span> collections <span class="hljs-keyword">import</span> defaultdict

word_freqs = defaultdict(<span class="hljs-built_in">int</span>)
<span class="hljs-keyword">for</span> text <span class="hljs-keyword">in</span> corpus:
    words_with_offsets = tokenizer.backend_tokenizer.pre_tokenizer.pre_tokenize_str(text)
    new_words = [word <span class="hljs-keyword">for</span> word, offset <span class="hljs-keyword">in</span> words_with_offsets]
    <span class="hljs-keyword">for</span> word <span class="hljs-keyword">in</span> new_words:
        word_freqs[word] += <span class="hljs-number">1</span>

word_freqs`}}),Ds=new j({props:{code:"",highlighted:`char_freqs = defaultdict(<span class="hljs-built_in">int</span>)
subwords_freqs = defaultdict(<span class="hljs-built_in">int</span>)
<span class="hljs-keyword">for</span> word, freq <span class="hljs-keyword">in</span> word_freqs.items():
    <span class="hljs-keyword">for</span> i <span class="hljs-keyword">in</span> <span class="hljs-built_in">range</span>(<span class="hljs-built_in">len</span>(word)):
        char_freqs[word[i]] += freq
        <span class="hljs-comment"># Loop through the subwords of length at least 2</span>
        <span class="hljs-keyword">for</span> j <span class="hljs-keyword">in</span> <span class="hljs-built_in">range</span>(i + <span class="hljs-number">2</span>, <span class="hljs-built_in">len</span>(word) + <span class="hljs-number">1</span>):
            subwords_freqs[word[i:j]] += freq

<span class="hljs-comment"># Sort subwords by frequency</span>
sorted_subwords = <span class="hljs-built_in">sorted</span>(subwords_freqs.items(), key=<span class="hljs-keyword">lambda</span> x: x[<span class="hljs-number">1</span>], reverse=<span class="hljs-literal">True</span>)
sorted_subwords[:<span class="hljs-number">10</span>]`}}),As=new j({props:{code:"[('\u2581t', 7), ('is', 5), ('er', 5), ('\u2581a', 5), ('\u2581to', 4), ('to', 4), ('en', 4), ('\u2581T', 3), ('\u2581Th', 3), ('\u2581Thi', 3)]",highlighted:'[(<span class="hljs-string">&#x27;\u2581t&#x27;</span>, <span class="hljs-number">7</span>), (<span class="hljs-string">&#x27;is&#x27;</span>, <span class="hljs-number">5</span>), (<span class="hljs-string">&#x27;er&#x27;</span>, <span class="hljs-number">5</span>), (<span class="hljs-string">&#x27;\u2581a&#x27;</span>, <span class="hljs-number">5</span>), (<span class="hljs-string">&#x27;\u2581to&#x27;</span>, <span class="hljs-number">4</span>), (<span class="hljs-string">&#x27;to&#x27;</span>, <span class="hljs-number">4</span>), (<span class="hljs-string">&#x27;en&#x27;</span>, <span class="hljs-number">4</span>), (<span class="hljs-string">&#x27;\u2581T&#x27;</span>, <span class="hljs-number">3</span>), (<span class="hljs-string">&#x27;\u2581Th&#x27;</span>, <span class="hljs-number">3</span>), (<span class="hljs-string">&#x27;\u2581Thi&#x27;</span>, <span class="hljs-number">3</span>)]'}}),Us=new j({props:{code:`token_freqs = list(char_freqs.items()) + sorted_subwords[: 300 - len(char_freqs)]
token_freqs = {token: freq for token, freq in token_freqs}`,highlighted:`token_freqs = <span class="hljs-built_in">list</span>(char_freqs.items()) + sorted_subwords[: <span class="hljs-number">300</span> - <span class="hljs-built_in">len</span>(char_freqs)]
token_freqs = {token: freq <span class="hljs-keyword">for</span> token, freq <span class="hljs-keyword">in</span> token_freqs}`}}),ps=new zt({props:{$$slots:{default:[Lp]},$$scope:{ctx:C}}}),Ws=new j({props:{code:"",highlighted:`<span class="hljs-keyword">from</span> math <span class="hljs-keyword">import</span> log

total_sum = <span class="hljs-built_in">sum</span>([freq <span class="hljs-keyword">for</span> token, freq <span class="hljs-keyword">in</span> token_freqs.items()])
model = {token: -log(freq / total_sum) <span class="hljs-keyword">for</span> token, freq <span class="hljs-keyword">in</span> token_freqs.items()}`}}),Hs=new j({props:{code:`
`,highlighted:`<span class="hljs-keyword">def</span> <span class="hljs-title function_">encode_word</span>(<span class="hljs-params">word, model</span>):
    best_segmentations = [{<span class="hljs-string">&quot;start&quot;</span>: <span class="hljs-number">0</span>, <span class="hljs-string">&quot;score&quot;</span>: <span class="hljs-number">1</span>}] + [
        {<span class="hljs-string">&quot;start&quot;</span>: <span class="hljs-literal">None</span>, <span class="hljs-string">&quot;score&quot;</span>: <span class="hljs-literal">None</span>} <span class="hljs-keyword">for</span> _ <span class="hljs-keyword">in</span> <span class="hljs-built_in">range</span>(<span class="hljs-built_in">len</span>(word))
    ]
    <span class="hljs-keyword">for</span> start_idx <span class="hljs-keyword">in</span> <span class="hljs-built_in">range</span>(<span class="hljs-built_in">len</span>(word)):
        <span class="hljs-comment"># This should be properly filled by the previous steps of the loop</span>
        best_score_at_start = best_segmentations[start_idx][<span class="hljs-string">&quot;score&quot;</span>]
        <span class="hljs-keyword">for</span> end_idx <span class="hljs-keyword">in</span> <span class="hljs-built_in">range</span>(start_idx + <span class="hljs-number">1</span>, <span class="hljs-built_in">len</span>(word) + <span class="hljs-number">1</span>):
            token = word[start_idx:end_idx]
            <span class="hljs-keyword">if</span> token <span class="hljs-keyword">in</span> model <span class="hljs-keyword">and</span> best_score_at_start <span class="hljs-keyword">is</span> <span class="hljs-keyword">not</span> <span class="hljs-literal">None</span>:
                score = model[token] + best_score_at_start
                <span class="hljs-comment"># If we have found a better segmentation ending at end_idx, we update</span>
                <span class="hljs-keyword">if</span> (
                    best_segmentations[end_idx][<span class="hljs-string">&quot;score&quot;</span>] <span class="hljs-keyword">is</span> <span class="hljs-literal">None</span>
                    <span class="hljs-keyword">or</span> best_segmentations[end_idx][<span class="hljs-string">&quot;score&quot;</span>] &gt; score
                ):
                    best_segmentations[end_idx] = {<span class="hljs-string">&quot;start&quot;</span>: start_idx, <span class="hljs-string">&quot;score&quot;</span>: score}

    segmentation = best_segmentations[-<span class="hljs-number">1</span>]
    <span class="hljs-keyword">if</span> segmentation[<span class="hljs-string">&quot;score&quot;</span>] <span class="hljs-keyword">is</span> <span class="hljs-literal">None</span>:
        <span class="hljs-comment"># We did not find a tokenization of the word -&gt; unknown</span>
        <span class="hljs-keyword">return</span> [<span class="hljs-string">&quot;&lt;unk&gt;&quot;</span>], <span class="hljs-literal">None</span>

    score = segmentation[<span class="hljs-string">&quot;score&quot;</span>]
    start = segmentation[<span class="hljs-string">&quot;start&quot;</span>]
    end = <span class="hljs-built_in">len</span>(word)
    tokens = []
    <span class="hljs-keyword">while</span> start != <span class="hljs-number">0</span>:
        tokens.insert(<span class="hljs-number">0</span>, word[start:end])
        next_start = best_segmentations[start][<span class="hljs-string">&quot;start&quot;</span>]
        end = start
        start = next_start
    tokens.insert(<span class="hljs-number">0</span>, word[start:end])
    <span class="hljs-keyword">return</span> tokens, score`}}),Bs=new j({props:{code:`print(encode_word("Hopefully", model))
print(encode_word("This", model))`,highlighted:`<span class="hljs-built_in">print</span>(encode_word(<span class="hljs-string">&quot;Hopefully&quot;</span>, model))
<span class="hljs-built_in">print</span>(encode_word(<span class="hljs-string">&quot;This&quot;</span>, model))`}}),Is=new j({props:{code:`(['H', 'o', 'p', 'e', 'f', 'u', 'll', 'y'], 41.5157494601402)
(['This'], 6.288267030694535)`,highlighted:`([<span class="hljs-string">&#x27;H&#x27;</span>, <span class="hljs-string">&#x27;o&#x27;</span>, <span class="hljs-string">&#x27;p&#x27;</span>, <span class="hljs-string">&#x27;e&#x27;</span>, <span class="hljs-string">&#x27;f&#x27;</span>, <span class="hljs-string">&#x27;u&#x27;</span>, <span class="hljs-string">&#x27;ll&#x27;</span>, <span class="hljs-string">&#x27;y&#x27;</span>], <span class="hljs-number">41.5157494601402</span>)
([<span class="hljs-string">&#x27;This&#x27;</span>], <span class="hljs-number">6.288267030694535</span>)`}}),Ms=new j({props:{code:`def compute_loss(model):
    loss = 0
    for word, freq in word_freqs.items():
        _, word_loss = encode_word(word, model)
        loss += freq * word_loss
    return loss`,highlighted:`<span class="hljs-keyword">def</span> <span class="hljs-title function_">compute_loss</span>(<span class="hljs-params">model</span>):
    loss = <span class="hljs-number">0</span>
    <span class="hljs-keyword">for</span> word, freq <span class="hljs-keyword">in</span> word_freqs.items():
        _, word_loss = encode_word(word, model)
        loss += freq * word_loss
    <span class="hljs-keyword">return</span> loss`}}),Ls=new j({props:{code:"compute_loss(model)",highlighted:"compute_loss(model)"}}),Fs=new j({props:{code:"413.10377642940875",highlighted:'<span class="hljs-number">413.10377642940875</span>'}}),Xs=new j({props:{code:`
`,highlighted:`<span class="hljs-keyword">import</span> copy


<span class="hljs-keyword">def</span> <span class="hljs-title function_">compute_scores</span>(<span class="hljs-params">model</span>):
    scores = {}
    model_loss = compute_loss(model)
    <span class="hljs-keyword">for</span> token, score <span class="hljs-keyword">in</span> model.items():
        <span class="hljs-comment"># We always keep tokens of length 1</span>
        <span class="hljs-keyword">if</span> <span class="hljs-built_in">len</span>(token) == <span class="hljs-number">1</span>:
            <span class="hljs-keyword">continue</span>
        model_without_token = copy.deepcopy(model)
        _ = model_without_token.pop(token)
        scores[token] = compute_loss(model_without_token) - model_loss
    <span class="hljs-keyword">return</span> scores`}}),Rs=new j({props:{code:`scores = compute_scores(model)
print(scores["ll"])
print(scores["his"])`,highlighted:`scores = compute_scores(model)
<span class="hljs-built_in">print</span>(scores[<span class="hljs-string">&quot;ll&quot;</span>])
<span class="hljs-built_in">print</span>(scores[<span class="hljs-string">&quot;his&quot;</span>])`}}),Vs=new j({props:{code:`6.376412403623874
0.0`,highlighted:`<span class="hljs-number">6.376412403623874</span>
<span class="hljs-number">0.0</span>`}}),ms=new zt({props:{$$slots:{default:[Fp]},$$scope:{ctx:C}}}),Gs=new j({props:{code:"",highlighted:`percent_to_remove = <span class="hljs-number">0.1</span>
<span class="hljs-keyword">while</span> <span class="hljs-built_in">len</span>(model) &gt; <span class="hljs-number">100</span>:
    scores = compute_scores(model)
    sorted_scores = <span class="hljs-built_in">sorted</span>(scores.items(), key=<span class="hljs-keyword">lambda</span> x: x[<span class="hljs-number">1</span>])
    <span class="hljs-comment"># Remove percent_to_remove tokens with the lowest scores.</span>
    <span class="hljs-keyword">for</span> i <span class="hljs-keyword">in</span> <span class="hljs-built_in">range</span>(<span class="hljs-built_in">int</span>(<span class="hljs-built_in">len</span>(model) * percent_to_remove)):
        _ = token_freqs.pop(sorted_scores[i][<span class="hljs-number">0</span>])

    total_sum = <span class="hljs-built_in">sum</span>([freq <span class="hljs-keyword">for</span> token, freq <span class="hljs-keyword">in</span> token_freqs.items()])
    model = {token: -log(freq / total_sum) <span class="hljs-keyword">for</span> token, freq <span class="hljs-keyword">in</span> token_freqs.items()}`}}),Ys=new j({props:{code:`
`,highlighted:`<span class="hljs-keyword">def</span> <span class="hljs-title function_">tokenize</span>(<span class="hljs-params">text, model</span>):
    words_with_offsets = tokenizer.backend_tokenizer.pre_tokenizer.pre_tokenize_str(text)
    pre_tokenized_text = [word <span class="hljs-keyword">for</span> word, offset <span class="hljs-keyword">in</span> words_with_offsets]
    encoded_words = [encode_word(word, model)[<span class="hljs-number">0</span>] <span class="hljs-keyword">for</span> word <span class="hljs-keyword">in</span> pre_tokenized_text]
    <span class="hljs-keyword">return</span> <span class="hljs-built_in">sum</span>(encoded_words, [])


tokenize(<span class="hljs-string">&quot;This is the Hugging Face course.&quot;</span>, model)`}}),Zs=new j({props:{code:"['\u2581This', '\u2581is', '\u2581the', '\u2581Hugging', '\u2581Face', '\u2581', 'c', 'ou', 'r', 's', 'e', '.']",highlighted:'[<span class="hljs-string">&#x27;\u2581This&#x27;</span>, <span class="hljs-string">&#x27;\u2581is&#x27;</span>, <span class="hljs-string">&#x27;\u2581the&#x27;</span>, <span class="hljs-string">&#x27;\u2581Hugging&#x27;</span>, <span class="hljs-string">&#x27;\u2581Face&#x27;</span>, <span class="hljs-string">&#x27;\u2581&#x27;</span>, <span class="hljs-string">&#x27;c&#x27;</span>, <span class="hljs-string">&#x27;ou&#x27;</span>, <span class="hljs-string">&#x27;r&#x27;</span>, <span class="hljs-string">&#x27;s&#x27;</span>, <span class="hljs-string">&#x27;e&#x27;</span>, <span class="hljs-string">&#x27;.&#x27;</span>]'}}),{c(){c=i("meta"),k=h(),y=i("h1"),v=i("a"),z=i("span"),u(_.$$.fragment),x=h(),N=i("span"),F=o("Unigram tokenization"),T=h(),u(H.$$.fragment),gs=h(),Qs=i("p"),Nn=o("The Unigram algorithm is often used in SentencePiece, which is the tokenization algorithm used by models like AlBERT, T5, mBART, Big Bird, and XLNet."),Ct=h(),u(bs.$$.fragment),Nt=h(),u(Z.$$.fragment),St=h(),X=i("h2"),J=i("a"),Ne=i("span"),u(ws.$$.fragment),Sn=h(),Se=i("span"),On=o("Training algorithm"),Ot=h(),se=i("p"),Dn=o("Compared to BPE and WordPiece, Unigram works in the other direction: it starts from a big vocabulary and removes tokens from it until it reaches the desired vocabulary size. There are several options to use to build that base vocabulary: we can take the most common substrings in pre-tokenized words, for instance, or apply BPE on the initial corpus with a large vocabulary size."),Dt=h(),ee=i("p"),An=o("At each step of the training, the Unigram algorithm computes a loss over the corpus given the current vocabulary. Then, for each symbol in the vocabulary, the algorithm computes how much the overall loss would increase if the symbol was removed, and looks for the symbols that would increase it the least. Those symbols have a lower effect on the overall loss over the corpus, so in a sense they are \u201Cless needed\u201D and are the best candidates for removal."),At=h(),B=i("p"),Un=o("This is all a very costly operation, so we don\u2019t just remove the single symbol associated with the lowest loss increase, but the "),Ut=new Tn,Wt=o(" ("),Ht=new Tn,Bt=o(" being a hyperparameter you can control, usually 10 or 20) percent of the symbols associated with the lowest loss increase. This process is then repeated until the vocabulary has reached the desired size."),It=h(),te=i("p"),Wn=o("Note that we never remove the base characters, to make sure any word can be tokenized."),Mt=h(),ae=i("p"),Hn=o("Now, this is still a bit vague: the main part of the algorithm is to compute a loss over the corpus and see how it changes when we remove some tokens from the vocabulary, but we haven\u2019t explained how to do this yet. This step relies on the tokenization algorithm of a Unigram model, so we\u2019ll dive into this next."),Lt=h(),ne=i("p"),Bn=o("We\u2019ll reuse the corpus from the previous examples:"),Ft=h(),u(ys.$$.fragment),Xt=h(),oe=i("p"),In=o("and for this example, we will take all strict substrings for the initial vocabulary :"),Rt=h(),u(vs.$$.fragment),Vt=h(),R=i("h2"),K=i("a"),Oe=i("span"),u(js.$$.fragment),Mn=h(),De=i("span"),Ln=o("Tokenization algorithm"),Gt=h(),le=i("p"),Fn=o("A Unigram model is a type of language model that considers each token to be independent of the tokens before it. It\u2019s the simplest language model, in the sense that the probability of token X given the previous context is just the probability of token X. So, if we used a Unigram language model to generate text, we would always predict the most common token."),Yt=h(),P=i("p"),Xn=o("The probability of a given token is its frequency (the number of times we find it) in the original corpus, divided by the sum of all frequencies of all tokens in the vocabulary (to make sure the probabilities sum up to 1). For instance, "),Ae=i("code"),Rn=o('"ug"'),Vn=o(" is present in "),Ue=i("code"),Gn=o('"hug"'),Yn=o(", "),We=i("code"),Zn=o('"pug"'),Jn=o(", and "),He=i("code"),Kn=o('"hugs"'),Qn=o(", so it has a frequency of 20 in our corpus."),Zt=h(),ie=i("p"),so=o("Here are the frequencies of all the possible subwords in the vocabulary:"),Jt=h(),u(ks.$$.fragment),Kt=h(),Q=i("p"),eo=o("So, the sum of all frequencies is 210, and the probability of the subword "),Be=i("code"),to=o('"ug"'),ao=o(" is thus 20/210."),Qt=h(),u(ss.$$.fragment),sa=h(),W=i("p"),no=o("Now, to tokenize a given word, we look at all the possible segmentations into tokens and compute the probability of each according to the Unigram model. Since all tokens are considered independent, this probability is just the product of the probability of each token. For instance, the tokenization "),Ie=i("code"),oo=o('["p", "u", "g"]'),lo=o(" of "),Me=i("code"),io=o('"pug"'),po=o(` has the probability:
`),ea=new Tn,ta=h(),V=i("p"),ro=o("Comparatively, the tokenization "),Le=i("code"),ho=o('["pu", "g"]'),mo=o(` has the probability:
`),aa=new Tn,na=h(),pe=i("p"),co=o("so that one is way more likely. In general, tokenizations with the least tokens possible will have the highest probability (because of that division by 210 repeated for each token), which corresponds to what we want intuitively: to split a word into the least number of tokens possible."),oa=h(),es=i("p"),uo=o("The tokenization of a word with the Unigram model is then the tokenization with the highest probability. In the example of "),Fe=i("code"),fo=o('"pug"'),go=o(", here are the probabilities we would get for each possible segmentation:"),la=h(),u($s.$$.fragment),ia=h(),S=i("p"),bo=o("So, "),Xe=i("code"),wo=o('"pug"'),yo=o(" would be tokenized as "),Re=i("code"),vo=o('["p", "ug"]'),jo=o(" or "),Ve=i("code"),ko=o('["pu", "g"]'),$o=o(", depending on which of those segmentations is encountered first (note that in a larger corpus, equality cases like this will be rare)."),pa=h(),q=i("p"),_o=o("In this case, it was easy to find all the possible segmentations and compute their probabilities, but in general it\u2019s going to be a bit harder. There is a classic algorithm used for this, called the "),Ge=i("em"),qo=o("Viterbi algorithm"),Eo=o(". Essentially, we can build a graph to detect the possible segmentations of a given word by saying there is a branch from character "),Ye=i("em"),xo=o("a"),Po=o(" to character "),Ze=i("em"),zo=o("b"),To=o(" if the subword from "),Je=i("em"),Co=o("a"),No=o(" to "),Ke=i("em"),So=o("b"),Oo=o(" is in the vocabulary, and attribute to that branch the probability of the subword."),ra=h(),re=i("p"),Do=o("To find the path in that graph that is going to have the best score the Viterbi algorithm determines, for each position in the word, the segmentation with the best score that ends at that position. Since we go from the beginning to the end, that best score can be found by looping through all subwords ending at the current position and then using the best tokenization score from the position this subword begins at. Then, we just have to unroll the path taken to arrive at the end."),ha=h(),ts=i("p"),Ao=o("Let\u2019s take a look at an example using our vocabulary and the word "),Qe=i("code"),Uo=o('"unhug"'),Wo=o(". For each position, the subwords with the best scores ending there are the following:"),ma=h(),u(_s.$$.fragment),ca=h(),I=i("p"),Ho=o("Thus "),st=i("code"),Bo=o('"unhug"'),Io=o(" would be tokenized as "),et=i("code"),Mo=o('["un", "hug"]'),Lo=o("."),ua=h(),u(as.$$.fragment),fa=h(),G=i("h2"),ns=i("a"),tt=i("span"),u(qs.$$.fragment),Fo=h(),at=i("span"),Xo=o("Back to training"),da=h(),he=i("p"),Ro=o("Now that we have seen how the tokenization works, we can dive a little more deeply into the loss used during training. At any given stage, this loss is computed by tokenizing every word in the corpus, using the current vocabulary and the Unigram model determined by the frequencies of each token in the corpus (as seen before)."),ga=h(),os=i("p"),Vo=o("Each word in the corpus has a score, and the loss is the negative log likelihood of those scores \u2014 that is, the sum for all the words in the corpus of all the "),nt=i("code"),Go=o("-log(P(word))"),Yo=o("."),ba=h(),me=i("p"),Zo=o("Let\u2019s go back to our example with the following corpus:"),wa=h(),u(Es.$$.fragment),ya=h(),ce=i("p"),Jo=o("The tokenization of each word with their respective scores is:"),va=h(),u(xs.$$.fragment),ja=h(),ue=i("p"),Ko=o("So the loss is:"),ka=h(),u(Ps.$$.fragment),$a=h(),O=i("p"),Qo=o("Now we need to compute how removing each token affects the loss. This is rather tedious, so we\u2019ll just do it for two tokens here and save the whole process for when we have code to help us. In this (very) particular case, we had two equivalent tokenizations of all the words: as we saw earlier, for example, "),ot=i("code"),sl=o('"pug"'),el=o(" could be tokenized "),lt=i("code"),tl=o('["p", "ug"]'),al=o(" with the same score. Thus, removing the "),it=i("code"),nl=o('"pu"'),ol=o(" token from the vocabulary will give the exact same loss."),_a=h(),D=i("p"),ll=o("On the other hand, removing "),pt=i("code"),il=o('"hug"'),pl=o(" will make the loss worse, because the tokenization of "),rt=i("code"),rl=o('"hug"'),hl=o(" and "),ht=i("code"),ml=o('"hugs"'),cl=o(" will become:"),qa=h(),u(zs.$$.fragment),Ea=h(),fe=i("p"),ul=o("These changes will cause the loss to rise by:"),xa=h(),u(Ts.$$.fragment),Pa=h(),M=i("p"),fl=o("Therefore, the token "),mt=i("code"),dl=o('"pu"'),gl=o(" will probably be removed from the vocabulary, but not "),ct=i("code"),bl=o('"hug"'),wl=o("."),za=h(),Y=i("h2"),ls=i("a"),ut=i("span"),u(Cs.$$.fragment),yl=h(),ft=i("span"),vl=o("Implementing Unigram"),Ta=h(),de=i("p"),jl=o("Now let\u2019s implement everything we\u2019ve seen so far in code. Like with BPE and WordPiece, this is not an efficient implementation of the Unigram algorithm (quite the opposite), but it should help you understand it a bit better."),Ca=h(),ge=i("p"),kl=o("We will use the same corpus as before as an example:"),Na=h(),u(Ns.$$.fragment),Sa=h(),is=i("p"),$l=o("This time, we will use "),dt=i("code"),_l=o("xlnet-base-cased"),ql=o(" as our model:"),Oa=h(),u(Ss.$$.fragment),Da=h(),be=i("p"),El=o("Like for BPE and WordPiece, we begin by counting the number of occurrences of each word in the corpus:"),Aa=h(),u(Os.$$.fragment),Ua=h(),we=i("p"),xl=o("Then, we need to initialize our vocabulary to something larger than the vocab size we will want at the end. We have to include all the basic characters (otherwise we won\u2019t be able to tokenize every word), but for the bigger substrings we\u2019ll only keep the most common ones, so we sort them by frequency:"),Wa=h(),u(Ds.$$.fragment),Ha=h(),u(As.$$.fragment),Ba=h(),ye=i("p"),Pl=o("We group the characters with the best subwords to arrive at an initial vocabulary of size 300:"),Ia=h(),u(Us.$$.fragment),Ma=h(),u(ps.$$.fragment),La=h(),ve=i("p"),zl=o("Next, we compute the sum of all frequencies, to convert the frequencies into probabilities. For our model we will store the logarithms of the probabilities, because it\u2019s more numerically stable to add logarithms than to multiply small numbers, and this will simplify the computation of the loss of the model:"),Fa=h(),u(Ws.$$.fragment),Xa=h(),rs=i("p"),Tl=o("Now the main function is the one that tokenizes words using the Viterbi algorithm. As we saw before, that algorithm computes the best segmentation of each substring of the word, which we will store in a variable named "),gt=i("code"),Cl=o("best_segmentations"),Nl=o(". We will store one dictionary per position in the word (from 0 to its total length), with two keys: the index of the start of the last token in the best segmentation, and the score of the best segmentation. With the index of the start of the last token, we will be able to retrieve the full segmentation once the list is completely populated."),Ra=h(),hs=i("p"),Sl=o("Populating the list is done with just two loops: the main loop goes over each start position, and the second loop tries all substrings beginning at that start position. If the substring is in the vocabulary, we have a new segmentation of the word up until that end position, which we compare to what is in "),bt=i("code"),Ol=o("best_segmentations"),Dl=o("."),Va=h(),je=i("p"),Al=o("Once the main loop is finished, we just start from the end and hop from one start position to the next, recording the tokens as we go, until we reach the start of the word:"),Ga=h(),u(Hs.$$.fragment),Ya=h(),ke=i("p"),Ul=o("We can already try our initial model on some words:"),Za=h(),u(Bs.$$.fragment),Ja=h(),u(Is.$$.fragment),Ka=h(),$e=i("p"),Wl=o("Now it\u2019s easy to compute the loss of the model on the corpus!"),Qa=h(),u(Ms.$$.fragment),sn=h(),_e=i("p"),Hl=o("We can check it works on the model we have:"),en=h(),u(Ls.$$.fragment),tn=h(),u(Fs.$$.fragment),an=h(),qe=i("p"),Bl=o("Computing the scores for each token is not very hard either; we just have to compute the loss for the models obtained by deleting each token:"),nn=h(),u(Xs.$$.fragment),on=h(),Ee=i("p"),Il=o("We can try it on a given token:"),ln=h(),u(Rs.$$.fragment),pn=h(),E=i("p"),Ml=o("Since "),wt=i("code"),Ll=o('"ll"'),Fl=o(" is used in the tokenization of "),yt=i("code"),Xl=o('"Hopefully"'),Rl=o(", and removing it will probably make us use the token "),vt=i("code"),Vl=o('"l"'),Gl=o(" twice instead, we expect it will have a positive loss. "),jt=i("code"),Yl=o('"his"'),Zl=o(" is only used inside the word "),kt=i("code"),Jl=o('"This"'),Kl=o(", which is tokenized as itself, so we expect it to have a zero loss. Here are the results:"),rn=h(),u(Vs.$$.fragment),hn=h(),u(ms.$$.fragment),mn=h(),xe=i("p"),Ql=o("With all of this in place, the last thing we need to do is add the special tokens used by the model to the vocabulary, then loop until we have pruned enough tokens from the vocabulary to reach our desired size:"),cn=h(),u(Gs.$$.fragment),un=h(),cs=i("p"),si=o("Then, to tokenize some text, we just need to apply the pre-tokenization and then use our "),$t=i("code"),ei=o("encode_word()"),ti=o(" function:"),fn=h(),u(Ys.$$.fragment),dn=h(),u(Zs.$$.fragment),gn=h(),Pe=i("p"),ai=o("That\u2019s it for Unigram! Hopefully by now you\u2019re feeling like an expert in all things tokenizer. In the next section, we will delve into the building blocks of the \u{1F917} Tokenizers library, and show you how you can use them to build your own tokenizer."),this.h()},l(s){const a=Ap('[data-svelte="svelte-1phssyn"]',document.head);c=p(a,"META",{name:!0,content:!0}),a.forEach(e),k=m(s),y=p(s,"H1",{class:!0});var Js=r(y);v=p(Js,"A",{id:!0,class:!0,href:!0});var _t=r(v);z=p(_t,"SPAN",{});var qt=r(z);f(_.$$.fragment,qt),qt.forEach(e),_t.forEach(e),x=m(Js),N=p(Js,"SPAN",{});var Et=r(N);F=l(Et,"Unigram tokenization"),Et.forEach(e),Js.forEach(e),T=m(s),f(H.$$.fragment,s),gs=m(s),Qs=p(s,"P",{});var xt=r(Qs);Nn=l(xt,"The Unigram algorithm is often used in SentencePiece, which is the tokenization algorithm used by models like AlBERT, T5, mBART, Big Bird, and XLNet."),xt.forEach(e),Ct=m(s),f(bs.$$.fragment,s),Nt=m(s),f(Z.$$.fragment,s),St=m(s),X=p(s,"H2",{class:!0});var wn=r(X);J=p(wn,"A",{id:!0,class:!0,href:!0});var ni=r(J);Ne=p(ni,"SPAN",{});var oi=r(Ne);f(ws.$$.fragment,oi),oi.forEach(e),ni.forEach(e),Sn=m(wn),Se=p(wn,"SPAN",{});var li=r(Se);On=l(li,"Training algorithm"),li.forEach(e),wn.forEach(e),Ot=m(s),se=p(s,"P",{});var ii=r(se);Dn=l(ii,"Compared to BPE and WordPiece, Unigram works in the other direction: it starts from a big vocabulary and removes tokens from it until it reaches the desired vocabulary size. There are several options to use to build that base vocabulary: we can take the most common substrings in pre-tokenized words, for instance, or apply BPE on the initial corpus with a large vocabulary size."),ii.forEach(e),Dt=m(s),ee=p(s,"P",{});var pi=r(ee);An=l(pi,"At each step of the training, the Unigram algorithm computes a loss over the corpus given the current vocabulary. Then, for each symbol in the vocabulary, the algorithm computes how much the overall loss would increase if the symbol was removed, and looks for the symbols that would increase it the least. Those symbols have a lower effect on the overall loss over the corpus, so in a sense they are \u201Cless needed\u201D and are the best candidates for removal."),pi.forEach(e),At=m(s),B=p(s,"P",{});var ze=r(B);Un=l(ze,"This is all a very costly operation, so we don\u2019t just remove the single symbol associated with the lowest loss increase, but the "),Ut=Cn(ze),Wt=l(ze," ("),Ht=Cn(ze),Bt=l(ze," being a hyperparameter you can control, usually 10 or 20) percent of the symbols associated with the lowest loss increase. This process is then repeated until the vocabulary has reached the desired size."),ze.forEach(e),It=m(s),te=p(s,"P",{});var ri=r(te);Wn=l(ri,"Note that we never remove the base characters, to make sure any word can be tokenized."),ri.forEach(e),Mt=m(s),ae=p(s,"P",{});var hi=r(ae);Hn=l(hi,"Now, this is still a bit vague: the main part of the algorithm is to compute a loss over the corpus and see how it changes when we remove some tokens from the vocabulary, but we haven\u2019t explained how to do this yet. This step relies on the tokenization algorithm of a Unigram model, so we\u2019ll dive into this next."),hi.forEach(e),Lt=m(s),ne=p(s,"P",{});var mi=r(ne);Bn=l(mi,"We\u2019ll reuse the corpus from the previous examples:"),mi.forEach(e),Ft=m(s),f(ys.$$.fragment,s),Xt=m(s),oe=p(s,"P",{});var ci=r(oe);In=l(ci,"and for this example, we will take all strict substrings for the initial vocabulary :"),ci.forEach(e),Rt=m(s),f(vs.$$.fragment,s),Vt=m(s),R=p(s,"H2",{class:!0});var yn=r(R);K=p(yn,"A",{id:!0,class:!0,href:!0});var ui=r(K);Oe=p(ui,"SPAN",{});var fi=r(Oe);f(js.$$.fragment,fi),fi.forEach(e),ui.forEach(e),Mn=m(yn),De=p(yn,"SPAN",{});var di=r(De);Ln=l(di,"Tokenization algorithm"),di.forEach(e),yn.forEach(e),Gt=m(s),le=p(s,"P",{});var gi=r(le);Fn=l(gi,"A Unigram model is a type of language model that considers each token to be independent of the tokens before it. It\u2019s the simplest language model, in the sense that the probability of token X given the previous context is just the probability of token X. So, if we used a Unigram language model to generate text, we would always predict the most common token."),gi.forEach(e),Yt=m(s),P=p(s,"P",{});var L=r(P);Xn=l(L,"The probability of a given token is its frequency (the number of times we find it) in the original corpus, divided by the sum of all frequencies of all tokens in the vocabulary (to make sure the probabilities sum up to 1). For instance, "),Ae=p(L,"CODE",{});var bi=r(Ae);Rn=l(bi,'"ug"'),bi.forEach(e),Vn=l(L," is present in "),Ue=p(L,"CODE",{});var wi=r(Ue);Gn=l(wi,'"hug"'),wi.forEach(e),Yn=l(L,", "),We=p(L,"CODE",{});var yi=r(We);Zn=l(yi,'"pug"'),yi.forEach(e),Jn=l(L,", and "),He=p(L,"CODE",{});var vi=r(He);Kn=l(vi,'"hugs"'),vi.forEach(e),Qn=l(L,", so it has a frequency of 20 in our corpus."),L.forEach(e),Zt=m(s),ie=p(s,"P",{});var ji=r(ie);so=l(ji,"Here are the frequencies of all the possible subwords in the vocabulary:"),ji.forEach(e),Jt=m(s),f(ks.$$.fragment,s),Kt=m(s),Q=p(s,"P",{});var vn=r(Q);eo=l(vn,"So, the sum of all frequencies is 210, and the probability of the subword "),Be=p(vn,"CODE",{});var ki=r(Be);to=l(ki,'"ug"'),ki.forEach(e),ao=l(vn," is thus 20/210."),vn.forEach(e),Qt=m(s),f(ss.$$.fragment,s),sa=m(s),W=p(s,"P",{});var Ks=r(W);no=l(Ks,"Now, to tokenize a given word, we look at all the possible segmentations into tokens and compute the probability of each according to the Unigram model. Since all tokens are considered independent, this probability is just the product of the probability of each token. For instance, the tokenization "),Ie=p(Ks,"CODE",{});var $i=r(Ie);oo=l($i,'["p", "u", "g"]'),$i.forEach(e),lo=l(Ks," of "),Me=p(Ks,"CODE",{});var _i=r(Me);io=l(_i,'"pug"'),_i.forEach(e),po=l(Ks,` has the probability:
`),ea=Cn(Ks),Ks.forEach(e),ta=m(s),V=p(s,"P",{});var Pt=r(V);ro=l(Pt,"Comparatively, the tokenization "),Le=p(Pt,"CODE",{});var qi=r(Le);ho=l(qi,'["pu", "g"]'),qi.forEach(e),mo=l(Pt,` has the probability:
`),aa=Cn(Pt),Pt.forEach(e),na=m(s),pe=p(s,"P",{});var Ei=r(pe);co=l(Ei,"so that one is way more likely. In general, tokenizations with the least tokens possible will have the highest probability (because of that division by 210 repeated for each token), which corresponds to what we want intuitively: to split a word into the least number of tokens possible."),Ei.forEach(e),oa=m(s),es=p(s,"P",{});var jn=r(es);uo=l(jn,"The tokenization of a word with the Unigram model is then the tokenization with the highest probability. In the example of "),Fe=p(jn,"CODE",{});var xi=r(Fe);fo=l(xi,'"pug"'),xi.forEach(e),go=l(jn,", here are the probabilities we would get for each possible segmentation:"),jn.forEach(e),la=m(s),f($s.$$.fragment,s),ia=m(s),S=p(s,"P",{});var us=r(S);bo=l(us,"So, "),Xe=p(us,"CODE",{});var Pi=r(Xe);wo=l(Pi,'"pug"'),Pi.forEach(e),yo=l(us," would be tokenized as "),Re=p(us,"CODE",{});var zi=r(Re);vo=l(zi,'["p", "ug"]'),zi.forEach(e),jo=l(us," or "),Ve=p(us,"CODE",{});var Ti=r(Ve);ko=l(Ti,'["pu", "g"]'),Ti.forEach(e),$o=l(us,", depending on which of those segmentations is encountered first (note that in a larger corpus, equality cases like this will be rare)."),us.forEach(e),pa=m(s),q=p(s,"P",{});var A=r(q);_o=l(A,"In this case, it was easy to find all the possible segmentations and compute their probabilities, but in general it\u2019s going to be a bit harder. There is a classic algorithm used for this, called the "),Ge=p(A,"EM",{});var Ci=r(Ge);qo=l(Ci,"Viterbi algorithm"),Ci.forEach(e),Eo=l(A,". Essentially, we can build a graph to detect the possible segmentations of a given word by saying there is a branch from character "),Ye=p(A,"EM",{});var Ni=r(Ye);xo=l(Ni,"a"),Ni.forEach(e),Po=l(A," to character "),Ze=p(A,"EM",{});var Si=r(Ze);zo=l(Si,"b"),Si.forEach(e),To=l(A," if the subword from "),Je=p(A,"EM",{});var Oi=r(Je);Co=l(Oi,"a"),Oi.forEach(e),No=l(A," to "),Ke=p(A,"EM",{});var Di=r(Ke);So=l(Di,"b"),Di.forEach(e),Oo=l(A," is in the vocabulary, and attribute to that branch the probability of the subword."),A.forEach(e),ra=m(s),re=p(s,"P",{});var Ai=r(re);Do=l(Ai,"To find the path in that graph that is going to have the best score the Viterbi algorithm determines, for each position in the word, the segmentation with the best score that ends at that position. Since we go from the beginning to the end, that best score can be found by looping through all subwords ending at the current position and then using the best tokenization score from the position this subword begins at. Then, we just have to unroll the path taken to arrive at the end."),Ai.forEach(e),ha=m(s),ts=p(s,"P",{});var kn=r(ts);Ao=l(kn,"Let\u2019s take a look at an example using our vocabulary and the word "),Qe=p(kn,"CODE",{});var Ui=r(Qe);Uo=l(Ui,'"unhug"'),Ui.forEach(e),Wo=l(kn,". For each position, the subwords with the best scores ending there are the following:"),kn.forEach(e),ma=m(s),f(_s.$$.fragment,s),ca=m(s),I=p(s,"P",{});var Te=r(I);Ho=l(Te,"Thus "),st=p(Te,"CODE",{});var Wi=r(st);Bo=l(Wi,'"unhug"'),Wi.forEach(e),Io=l(Te," would be tokenized as "),et=p(Te,"CODE",{});var Hi=r(et);Mo=l(Hi,'["un", "hug"]'),Hi.forEach(e),Lo=l(Te,"."),Te.forEach(e),ua=m(s),f(as.$$.fragment,s),fa=m(s),G=p(s,"H2",{class:!0});var $n=r(G);ns=p($n,"A",{id:!0,class:!0,href:!0});var Bi=r(ns);tt=p(Bi,"SPAN",{});var Ii=r(tt);f(qs.$$.fragment,Ii),Ii.forEach(e),Bi.forEach(e),Fo=m($n),at=p($n,"SPAN",{});var Mi=r(at);Xo=l(Mi,"Back to training"),Mi.forEach(e),$n.forEach(e),da=m(s),he=p(s,"P",{});var Li=r(he);Ro=l(Li,"Now that we have seen how the tokenization works, we can dive a little more deeply into the loss used during training. At any given stage, this loss is computed by tokenizing every word in the corpus, using the current vocabulary and the Unigram model determined by the frequencies of each token in the corpus (as seen before)."),Li.forEach(e),ga=m(s),os=p(s,"P",{});var _n=r(os);Vo=l(_n,"Each word in the corpus has a score, and the loss is the negative log likelihood of those scores \u2014 that is, the sum for all the words in the corpus of all the "),nt=p(_n,"CODE",{});var Fi=r(nt);Go=l(Fi,"-log(P(word))"),Fi.forEach(e),Yo=l(_n,"."),_n.forEach(e),ba=m(s),me=p(s,"P",{});var Xi=r(me);Zo=l(Xi,"Let\u2019s go back to our example with the following corpus:"),Xi.forEach(e),wa=m(s),f(Es.$$.fragment,s),ya=m(s),ce=p(s,"P",{});var Ri=r(ce);Jo=l(Ri,"The tokenization of each word with their respective scores is:"),Ri.forEach(e),va=m(s),f(xs.$$.fragment,s),ja=m(s),ue=p(s,"P",{});var Vi=r(ue);Ko=l(Vi,"So the loss is:"),Vi.forEach(e),ka=m(s),f(Ps.$$.fragment,s),$a=m(s),O=p(s,"P",{});var fs=r(O);Qo=l(fs,"Now we need to compute how removing each token affects the loss. This is rather tedious, so we\u2019ll just do it for two tokens here and save the whole process for when we have code to help us. In this (very) particular case, we had two equivalent tokenizations of all the words: as we saw earlier, for example, "),ot=p(fs,"CODE",{});var Gi=r(ot);sl=l(Gi,'"pug"'),Gi.forEach(e),el=l(fs," could be tokenized "),lt=p(fs,"CODE",{});var Yi=r(lt);tl=l(Yi,'["p", "ug"]'),Yi.forEach(e),al=l(fs," with the same score. Thus, removing the "),it=p(fs,"CODE",{});var Zi=r(it);nl=l(Zi,'"pu"'),Zi.forEach(e),ol=l(fs," token from the vocabulary will give the exact same loss."),fs.forEach(e),_a=m(s),D=p(s,"P",{});var ds=r(D);ll=l(ds,"On the other hand, removing "),pt=p(ds,"CODE",{});var Ji=r(pt);il=l(Ji,'"hug"'),Ji.forEach(e),pl=l(ds," will make the loss worse, because the tokenization of "),rt=p(ds,"CODE",{});var Ki=r(rt);rl=l(Ki,'"hug"'),Ki.forEach(e),hl=l(ds," and "),ht=p(ds,"CODE",{});var Qi=r(ht);ml=l(Qi,'"hugs"'),Qi.forEach(e),cl=l(ds," will become:"),ds.forEach(e),qa=m(s),f(zs.$$.fragment,s),Ea=m(s),fe=p(s,"P",{});var sp=r(fe);ul=l(sp,"These changes will cause the loss to rise by:"),sp.forEach(e),xa=m(s),f(Ts.$$.fragment,s),Pa=m(s),M=p(s,"P",{});var Ce=r(M);fl=l(Ce,"Therefore, the token "),mt=p(Ce,"CODE",{});var ep=r(mt);dl=l(ep,'"pu"'),ep.forEach(e),gl=l(Ce," will probably be removed from the vocabulary, but not "),ct=p(Ce,"CODE",{});var tp=r(ct);bl=l(tp,'"hug"'),tp.forEach(e),wl=l(Ce,"."),Ce.forEach(e),za=m(s),Y=p(s,"H2",{class:!0});var qn=r(Y);ls=p(qn,"A",{id:!0,class:!0,href:!0});var ap=r(ls);ut=p(ap,"SPAN",{});var np=r(ut);f(Cs.$$.fragment,np),np.forEach(e),ap.forEach(e),yl=m(qn),ft=p(qn,"SPAN",{});var op=r(ft);vl=l(op,"Implementing Unigram"),op.forEach(e),qn.forEach(e),Ta=m(s),de=p(s,"P",{});var lp=r(de);jl=l(lp,"Now let\u2019s implement everything we\u2019ve seen so far in code. Like with BPE and WordPiece, this is not an efficient implementation of the Unigram algorithm (quite the opposite), but it should help you understand it a bit better."),lp.forEach(e),Ca=m(s),ge=p(s,"P",{});var ip=r(ge);kl=l(ip,"We will use the same corpus as before as an example:"),ip.forEach(e),Na=m(s),f(Ns.$$.fragment,s),Sa=m(s),is=p(s,"P",{});var En=r(is);$l=l(En,"This time, we will use "),dt=p(En,"CODE",{});var pp=r(dt);_l=l(pp,"xlnet-base-cased"),pp.forEach(e),ql=l(En," as our model:"),En.forEach(e),Oa=m(s),f(Ss.$$.fragment,s),Da=m(s),be=p(s,"P",{});var rp=r(be);El=l(rp,"Like for BPE and WordPiece, we begin by counting the number of occurrences of each word in the corpus:"),rp.forEach(e),Aa=m(s),f(Os.$$.fragment,s),Ua=m(s),we=p(s,"P",{});var hp=r(we);xl=l(hp,"Then, we need to initialize our vocabulary to something larger than the vocab size we will want at the end. We have to include all the basic characters (otherwise we won\u2019t be able to tokenize every word), but for the bigger substrings we\u2019ll only keep the most common ones, so we sort them by frequency:"),hp.forEach(e),Wa=m(s),f(Ds.$$.fragment,s),Ha=m(s),f(As.$$.fragment,s),Ba=m(s),ye=p(s,"P",{});var mp=r(ye);Pl=l(mp,"We group the characters with the best subwords to arrive at an initial vocabulary of size 300:"),mp.forEach(e),Ia=m(s),f(Us.$$.fragment,s),Ma=m(s),f(ps.$$.fragment,s),La=m(s),ve=p(s,"P",{});var cp=r(ve);zl=l(cp,"Next, we compute the sum of all frequencies, to convert the frequencies into probabilities. For our model we will store the logarithms of the probabilities, because it\u2019s more numerically stable to add logarithms than to multiply small numbers, and this will simplify the computation of the loss of the model:"),cp.forEach(e),Fa=m(s),f(Ws.$$.fragment,s),Xa=m(s),rs=p(s,"P",{});var xn=r(rs);Tl=l(xn,"Now the main function is the one that tokenizes words using the Viterbi algorithm. As we saw before, that algorithm computes the best segmentation of each substring of the word, which we will store in a variable named "),gt=p(xn,"CODE",{});var up=r(gt);Cl=l(up,"best_segmentations"),up.forEach(e),Nl=l(xn,". We will store one dictionary per position in the word (from 0 to its total length), with two keys: the index of the start of the last token in the best segmentation, and the score of the best segmentation. With the index of the start of the last token, we will be able to retrieve the full segmentation once the list is completely populated."),xn.forEach(e),Ra=m(s),hs=p(s,"P",{});var Pn=r(hs);Sl=l(Pn,"Populating the list is done with just two loops: the main loop goes over each start position, and the second loop tries all substrings beginning at that start position. If the substring is in the vocabulary, we have a new segmentation of the word up until that end position, which we compare to what is in "),bt=p(Pn,"CODE",{});var fp=r(bt);Ol=l(fp,"best_segmentations"),fp.forEach(e),Dl=l(Pn,"."),Pn.forEach(e),Va=m(s),je=p(s,"P",{});var dp=r(je);Al=l(dp,"Once the main loop is finished, we just start from the end and hop from one start position to the next, recording the tokens as we go, until we reach the start of the word:"),dp.forEach(e),Ga=m(s),f(Hs.$$.fragment,s),Ya=m(s),ke=p(s,"P",{});var gp=r(ke);Ul=l(gp,"We can already try our initial model on some words:"),gp.forEach(e),Za=m(s),f(Bs.$$.fragment,s),Ja=m(s),f(Is.$$.fragment,s),Ka=m(s),$e=p(s,"P",{});var bp=r($e);Wl=l(bp,"Now it\u2019s easy to compute the loss of the model on the corpus!"),bp.forEach(e),Qa=m(s),f(Ms.$$.fragment,s),sn=m(s),_e=p(s,"P",{});var wp=r(_e);Hl=l(wp,"We can check it works on the model we have:"),wp.forEach(e),en=m(s),f(Ls.$$.fragment,s),tn=m(s),f(Fs.$$.fragment,s),an=m(s),qe=p(s,"P",{});var yp=r(qe);Bl=l(yp,"Computing the scores for each token is not very hard either; we just have to compute the loss for the models obtained by deleting each token:"),yp.forEach(e),nn=m(s),f(Xs.$$.fragment,s),on=m(s),Ee=p(s,"P",{});var vp=r(Ee);Il=l(vp,"We can try it on a given token:"),vp.forEach(e),ln=m(s),f(Rs.$$.fragment,s),pn=m(s),E=p(s,"P",{});var U=r(E);Ml=l(U,"Since "),wt=p(U,"CODE",{});var jp=r(wt);Ll=l(jp,'"ll"'),jp.forEach(e),Fl=l(U," is used in the tokenization of "),yt=p(U,"CODE",{});var kp=r(yt);Xl=l(kp,'"Hopefully"'),kp.forEach(e),Rl=l(U,", and removing it will probably make us use the token "),vt=p(U,"CODE",{});var $p=r(vt);Vl=l($p,'"l"'),$p.forEach(e),Gl=l(U," twice instead, we expect it will have a positive loss. "),jt=p(U,"CODE",{});var _p=r(jt);Yl=l(_p,'"his"'),_p.forEach(e),Zl=l(U," is only used inside the word "),kt=p(U,"CODE",{});var qp=r(kt);Jl=l(qp,'"This"'),qp.forEach(e),Kl=l(U,", which is tokenized as itself, so we expect it to have a zero loss. Here are the results:"),U.forEach(e),rn=m(s),f(Vs.$$.fragment,s),hn=m(s),f(ms.$$.fragment,s),mn=m(s),xe=p(s,"P",{});var Ep=r(xe);Ql=l(Ep,"With all of this in place, the last thing we need to do is add the special tokens used by the model to the vocabulary, then loop until we have pruned enough tokens from the vocabulary to reach our desired size:"),Ep.forEach(e),cn=m(s),f(Gs.$$.fragment,s),un=m(s),cs=p(s,"P",{});var zn=r(cs);si=l(zn,"Then, to tokenize some text, we just need to apply the pre-tokenization and then use our "),$t=p(zn,"CODE",{});var xp=r($t);ei=l(xp,"encode_word()"),xp.forEach(e),ti=l(zn," function:"),zn.forEach(e),fn=m(s),f(Ys.$$.fragment,s),dn=m(s),f(Zs.$$.fragment,s),gn=m(s),Pe=p(s,"P",{});var Pp=r(Pe);ai=l(Pp,"That\u2019s it for Unigram! Hopefully by now you\u2019re feeling like an expert in all things tokenizer. In the next section, we will delve into the building blocks of the \u{1F917} Tokenizers library, and show you how you can use them to build your own tokenizer."),Pp.forEach(e),this.h()},h(){$(c,"name","hf:doc:metadata"),$(c,"content",JSON.stringify(Rp)),$(v,"id","unigram-tokenization"),$(v,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),$(v,"href","#unigram-tokenization"),$(y,"class","relative group"),$(J,"id","training-algorithm"),$(J,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),$(J,"href","#training-algorithm"),$(X,"class","relative group"),Ut.a=Wt,Ht.a=Bt,$(K,"id","tokenization-algorithm"),$(K,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),$(K,"href","#tokenization-algorithm"),$(R,"class","relative group"),ea.a=null,aa.a=null,$(ns,"id","back-to-training"),$(ns,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),$(ns,"href","#back-to-training"),$(G,"class","relative group"),$(ls,"id","implementing-unigram"),$(ls,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),$(ls,"href","#implementing-unigram"),$(Y,"class","relative group")},m(s,a){t(document.head,c),n(s,k,a),n(s,y,a),t(y,v),t(v,z),d(_,z,null),t(y,x),t(y,N),t(N,F),n(s,T,a),d(H,s,a),n(s,gs,a),n(s,Qs,a),t(Qs,Nn),n(s,Ct,a),d(bs,s,a),n(s,Nt,a),d(Z,s,a),n(s,St,a),n(s,X,a),t(X,J),t(J,Ne),d(ws,Ne,null),t(X,Sn),t(X,Se),t(Se,On),n(s,Ot,a),n(s,se,a),t(se,Dn),n(s,Dt,a),n(s,ee,a),t(ee,An),n(s,At,a),n(s,B,a),t(B,Un),Ut.m(zp,B),t(B,Wt),Ht.m(Tp,B),t(B,Bt),n(s,It,a),n(s,te,a),t(te,Wn),n(s,Mt,a),n(s,ae,a),t(ae,Hn),n(s,Lt,a),n(s,ne,a),t(ne,Bn),n(s,Ft,a),d(ys,s,a),n(s,Xt,a),n(s,oe,a),t(oe,In),n(s,Rt,a),d(vs,s,a),n(s,Vt,a),n(s,R,a),t(R,K),t(K,Oe),d(js,Oe,null),t(R,Mn),t(R,De),t(De,Ln),n(s,Gt,a),n(s,le,a),t(le,Fn),n(s,Yt,a),n(s,P,a),t(P,Xn),t(P,Ae),t(Ae,Rn),t(P,Vn),t(P,Ue),t(Ue,Gn),t(P,Yn),t(P,We),t(We,Zn),t(P,Jn),t(P,He),t(He,Kn),t(P,Qn),n(s,Zt,a),n(s,ie,a),t(ie,so),n(s,Jt,a),d(ks,s,a),n(s,Kt,a),n(s,Q,a),t(Q,eo),t(Q,Be),t(Be,to),t(Q,ao),n(s,Qt,a),d(ss,s,a),n(s,sa,a),n(s,W,a),t(W,no),t(W,Ie),t(Ie,oo),t(W,lo),t(W,Me),t(Me,io),t(W,po),ea.m(Cp,W),n(s,ta,a),n(s,V,a),t(V,ro),t(V,Le),t(Le,ho),t(V,mo),aa.m(Np,V),n(s,na,a),n(s,pe,a),t(pe,co),n(s,oa,a),n(s,es,a),t(es,uo),t(es,Fe),t(Fe,fo),t(es,go),n(s,la,a),d($s,s,a),n(s,ia,a),n(s,S,a),t(S,bo),t(S,Xe),t(Xe,wo),t(S,yo),t(S,Re),t(Re,vo),t(S,jo),t(S,Ve),t(Ve,ko),t(S,$o),n(s,pa,a),n(s,q,a),t(q,_o),t(q,Ge),t(Ge,qo),t(q,Eo),t(q,Ye),t(Ye,xo),t(q,Po),t(q,Ze),t(Ze,zo),t(q,To),t(q,Je),t(Je,Co),t(q,No),t(q,Ke),t(Ke,So),t(q,Oo),n(s,ra,a),n(s,re,a),t(re,Do),n(s,ha,a),n(s,ts,a),t(ts,Ao),t(ts,Qe),t(Qe,Uo),t(ts,Wo),n(s,ma,a),d(_s,s,a),n(s,ca,a),n(s,I,a),t(I,Ho),t(I,st),t(st,Bo),t(I,Io),t(I,et),t(et,Mo),t(I,Lo),n(s,ua,a),d(as,s,a),n(s,fa,a),n(s,G,a),t(G,ns),t(ns,tt),d(qs,tt,null),t(G,Fo),t(G,at),t(at,Xo),n(s,da,a),n(s,he,a),t(he,Ro),n(s,ga,a),n(s,os,a),t(os,Vo),t(os,nt),t(nt,Go),t(os,Yo),n(s,ba,a),n(s,me,a),t(me,Zo),n(s,wa,a),d(Es,s,a),n(s,ya,a),n(s,ce,a),t(ce,Jo),n(s,va,a),d(xs,s,a),n(s,ja,a),n(s,ue,a),t(ue,Ko),n(s,ka,a),d(Ps,s,a),n(s,$a,a),n(s,O,a),t(O,Qo),t(O,ot),t(ot,sl),t(O,el),t(O,lt),t(lt,tl),t(O,al),t(O,it),t(it,nl),t(O,ol),n(s,_a,a),n(s,D,a),t(D,ll),t(D,pt),t(pt,il),t(D,pl),t(D,rt),t(rt,rl),t(D,hl),t(D,ht),t(ht,ml),t(D,cl),n(s,qa,a),d(zs,s,a),n(s,Ea,a),n(s,fe,a),t(fe,ul),n(s,xa,a),d(Ts,s,a),n(s,Pa,a),n(s,M,a),t(M,fl),t(M,mt),t(mt,dl),t(M,gl),t(M,ct),t(ct,bl),t(M,wl),n(s,za,a),n(s,Y,a),t(Y,ls),t(ls,ut),d(Cs,ut,null),t(Y,yl),t(Y,ft),t(ft,vl),n(s,Ta,a),n(s,de,a),t(de,jl),n(s,Ca,a),n(s,ge,a),t(ge,kl),n(s,Na,a),d(Ns,s,a),n(s,Sa,a),n(s,is,a),t(is,$l),t(is,dt),t(dt,_l),t(is,ql),n(s,Oa,a),d(Ss,s,a),n(s,Da,a),n(s,be,a),t(be,El),n(s,Aa,a),d(Os,s,a),n(s,Ua,a),n(s,we,a),t(we,xl),n(s,Wa,a),d(Ds,s,a),n(s,Ha,a),d(As,s,a),n(s,Ba,a),n(s,ye,a),t(ye,Pl),n(s,Ia,a),d(Us,s,a),n(s,Ma,a),d(ps,s,a),n(s,La,a),n(s,ve,a),t(ve,zl),n(s,Fa,a),d(Ws,s,a),n(s,Xa,a),n(s,rs,a),t(rs,Tl),t(rs,gt),t(gt,Cl),t(rs,Nl),n(s,Ra,a),n(s,hs,a),t(hs,Sl),t(hs,bt),t(bt,Ol),t(hs,Dl),n(s,Va,a),n(s,je,a),t(je,Al),n(s,Ga,a),d(Hs,s,a),n(s,Ya,a),n(s,ke,a),t(ke,Ul),n(s,Za,a),d(Bs,s,a),n(s,Ja,a),d(Is,s,a),n(s,Ka,a),n(s,$e,a),t($e,Wl),n(s,Qa,a),d(Ms,s,a),n(s,sn,a),n(s,_e,a),t(_e,Hl),n(s,en,a),d(Ls,s,a),n(s,tn,a),d(Fs,s,a),n(s,an,a),n(s,qe,a),t(qe,Bl),n(s,nn,a),d(Xs,s,a),n(s,on,a),n(s,Ee,a),t(Ee,Il),n(s,ln,a),d(Rs,s,a),n(s,pn,a),n(s,E,a),t(E,Ml),t(E,wt),t(wt,Ll),t(E,Fl),t(E,yt),t(yt,Xl),t(E,Rl),t(E,vt),t(vt,Vl),t(E,Gl),t(E,jt),t(jt,Yl),t(E,Zl),t(E,kt),t(kt,Jl),t(E,Kl),n(s,rn,a),d(Vs,s,a),n(s,hn,a),d(ms,s,a),n(s,mn,a),n(s,xe,a),t(xe,Ql),n(s,cn,a),d(Gs,s,a),n(s,un,a),n(s,cs,a),t(cs,si),t(cs,$t),t($t,ei),t(cs,ti),n(s,fn,a),d(Ys,s,a),n(s,dn,a),d(Zs,s,a),n(s,gn,a),n(s,Pe,a),t(Pe,ai),bn=!0},p(s,[a]){const Js={};a&2&&(Js.$$scope={dirty:a,ctx:s}),Z.$set(Js);const _t={};a&2&&(_t.$$scope={dirty:a,ctx:s}),ss.$set(_t);const qt={};a&2&&(qt.$$scope={dirty:a,ctx:s}),as.$set(qt);const Et={};a&2&&(Et.$$scope={dirty:a,ctx:s}),ps.$set(Et);const xt={};a&2&&(xt.$$scope={dirty:a,ctx:s}),ms.$set(xt)},i(s){bn||(g(_.$$.fragment,s),g(H.$$.fragment,s),g(bs.$$.fragment,s),g(Z.$$.fragment,s),g(ws.$$.fragment,s),g(ys.$$.fragment,s),g(vs.$$.fragment,s),g(js.$$.fragment,s),g(ks.$$.fragment,s),g(ss.$$.fragment,s),g($s.$$.fragment,s),g(_s.$$.fragment,s),g(as.$$.fragment,s),g(qs.$$.fragment,s),g(Es.$$.fragment,s),g(xs.$$.fragment,s),g(Ps.$$.fragment,s),g(zs.$$.fragment,s),g(Ts.$$.fragment,s),g(Cs.$$.fragment,s),g(Ns.$$.fragment,s),g(Ss.$$.fragment,s),g(Os.$$.fragment,s),g(Ds.$$.fragment,s),g(As.$$.fragment,s),g(Us.$$.fragment,s),g(ps.$$.fragment,s),g(Ws.$$.fragment,s),g(Hs.$$.fragment,s),g(Bs.$$.fragment,s),g(Is.$$.fragment,s),g(Ms.$$.fragment,s),g(Ls.$$.fragment,s),g(Fs.$$.fragment,s),g(Xs.$$.fragment,s),g(Rs.$$.fragment,s),g(Vs.$$.fragment,s),g(ms.$$.fragment,s),g(Gs.$$.fragment,s),g(Ys.$$.fragment,s),g(Zs.$$.fragment,s),bn=!0)},o(s){b(_.$$.fragment,s),b(H.$$.fragment,s),b(bs.$$.fragment,s),b(Z.$$.fragment,s),b(ws.$$.fragment,s),b(ys.$$.fragment,s),b(vs.$$.fragment,s),b(js.$$.fragment,s),b(ks.$$.fragment,s),b(ss.$$.fragment,s),b($s.$$.fragment,s),b(_s.$$.fragment,s),b(as.$$.fragment,s),b(qs.$$.fragment,s),b(Es.$$.fragment,s),b(xs.$$.fragment,s),b(Ps.$$.fragment,s),b(zs.$$.fragment,s),b(Ts.$$.fragment,s),b(Cs.$$.fragment,s),b(Ns.$$.fragment,s),b(Ss.$$.fragment,s),b(Os.$$.fragment,s),b(Ds.$$.fragment,s),b(As.$$.fragment,s),b(Us.$$.fragment,s),b(ps.$$.fragment,s),b(Ws.$$.fragment,s),b(Hs.$$.fragment,s),b(Bs.$$.fragment,s),b(Is.$$.fragment,s),b(Ms.$$.fragment,s),b(Ls.$$.fragment,s),b(Fs.$$.fragment,s),b(Xs.$$.fragment,s),b(Rs.$$.fragment,s),b(Vs.$$.fragment,s),b(ms.$$.fragment,s),b(Gs.$$.fragment,s),b(Ys.$$.fragment,s),b(Zs.$$.fragment,s),bn=!1},d(s){e(c),s&&e(k),s&&e(y),w(_),s&&e(T),w(H,s),s&&e(gs),s&&e(Qs),s&&e(Ct),w(bs,s),s&&e(Nt),w(Z,s),s&&e(St),s&&e(X),w(ws),s&&e(Ot),s&&e(se),s&&e(Dt),s&&e(ee),s&&e(At),s&&e(B),s&&e(It),s&&e(te),s&&e(Mt),s&&e(ae),s&&e(Lt),s&&e(ne),s&&e(Ft),w(ys,s),s&&e(Xt),s&&e(oe),s&&e(Rt),w(vs,s),s&&e(Vt),s&&e(R),w(js),s&&e(Gt),s&&e(le),s&&e(Yt),s&&e(P),s&&e(Zt),s&&e(ie),s&&e(Jt),w(ks,s),s&&e(Kt),s&&e(Q),s&&e(Qt),w(ss,s),s&&e(sa),s&&e(W),s&&e(ta),s&&e(V),s&&e(na),s&&e(pe),s&&e(oa),s&&e(es),s&&e(la),w($s,s),s&&e(ia),s&&e(S),s&&e(pa),s&&e(q),s&&e(ra),s&&e(re),s&&e(ha),s&&e(ts),s&&e(ma),w(_s,s),s&&e(ca),s&&e(I),s&&e(ua),w(as,s),s&&e(fa),s&&e(G),w(qs),s&&e(da),s&&e(he),s&&e(ga),s&&e(os),s&&e(ba),s&&e(me),s&&e(wa),w(Es,s),s&&e(ya),s&&e(ce),s&&e(va),w(xs,s),s&&e(ja),s&&e(ue),s&&e(ka),w(Ps,s),s&&e($a),s&&e(O),s&&e(_a),s&&e(D),s&&e(qa),w(zs,s),s&&e(Ea),s&&e(fe),s&&e(xa),w(Ts,s),s&&e(Pa),s&&e(M),s&&e(za),s&&e(Y),w(Cs),s&&e(Ta),s&&e(de),s&&e(Ca),s&&e(ge),s&&e(Na),w(Ns,s),s&&e(Sa),s&&e(is),s&&e(Oa),w(Ss,s),s&&e(Da),s&&e(be),s&&e(Aa),w(Os,s),s&&e(Ua),s&&e(we),s&&e(Wa),w(Ds,s),s&&e(Ha),w(As,s),s&&e(Ba),s&&e(ye),s&&e(Ia),w(Us,s),s&&e(Ma),w(ps,s),s&&e(La),s&&e(ve),s&&e(Fa),w(Ws,s),s&&e(Xa),s&&e(rs),s&&e(Ra),s&&e(hs),s&&e(Va),s&&e(je),s&&e(Ga),w(Hs,s),s&&e(Ya),s&&e(ke),s&&e(Za),w(Bs,s),s&&e(Ja),w(Is,s),s&&e(Ka),s&&e($e),s&&e(Qa),w(Ms,s),s&&e(sn),s&&e(_e),s&&e(en),w(Ls,s),s&&e(tn),w(Fs,s),s&&e(an),s&&e(qe),s&&e(nn),w(Xs,s),s&&e(on),s&&e(Ee),s&&e(ln),w(Rs,s),s&&e(pn),s&&e(E),s&&e(rn),w(Vs,s),s&&e(hn),w(ms,s),s&&e(mn),s&&e(xe),s&&e(cn),w(Gs,s),s&&e(un),s&&e(cs),s&&e(fn),w(Ys,s),s&&e(dn),w(Zs,s),s&&e(gn),s&&e(Pe)}}}const Rp={local:"unigram-tokenization",sections:[{local:"training-algorithm",title:"Training algorithm"},{local:"tokenization-algorithm",title:"Tokenization algorithm"},{local:"back-to-training",title:"Back to training"},{local:"implementing-unigram",title:"Implementing Unigram"}],title:"Unigram tokenization"};function Vp(C){return Up(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class sr extends Sp{constructor(c){super();Op(this,c,Vp,Xp,Dp,{})}}export{sr as default,Rp as metadata};
