import{S as xr,i as kr,s as $r,e as i,k as c,w as m,t as l,U as _r,M as wr,c as r,d as n,m as o,a as h,x as g,h as p,V as qr,b as D,G as t,g as e,y as f,q as j,o as b,B as d,v as yr}from"../../chunks/vendor-hf-doc-builder.js";import{T as Pt}from"../../chunks/Tip-hf-doc-builder.js";import{Y as Er}from"../../chunks/Youtube-hf-doc-builder.js";import{I as hl}from"../../chunks/IconCopyLink-hf-doc-builder.js";import{C as x}from"../../chunks/CodeBlock-hf-doc-builder.js";import{D as Pr}from"../../chunks/DocNotebookDropdown-hf-doc-builder.js";function zr(A){let u,_;return{c(){u=i("p"),_=l("\u{1F4A1} Ph\u1EA7n n\xE0y s\u1EBD \u0111i s\xE2u v\xE0o WordPiece, c\u0169ng nh\u01B0 c\xE1c tri\u1EC3n khai \u0111\u1EA7y \u0111\u1EE7 c\u1EE7a n\xF3. B\u1EA1n c\xF3 th\u1EC3 b\u1ECF qua ph\u1EA7n cu\u1ED1i n\u1EBFu b\u1EA1n ch\u1EC9 mu\u1ED1n c\xF3 m\u1ED9t c\xE1i nh\xECn t\u1ED5ng quan v\u1EC1 thu\u1EADt to\xE1n tokenize n\xE0y.")},l(v){u=r(v,"P",{});var k=h(u);_=p(k,"\u{1F4A1} Ph\u1EA7n n\xE0y s\u1EBD \u0111i s\xE2u v\xE0o WordPiece, c\u0169ng nh\u01B0 c\xE1c tri\u1EC3n khai \u0111\u1EA7y \u0111\u1EE7 c\u1EE7a n\xF3. B\u1EA1n c\xF3 th\u1EC3 b\u1ECF qua ph\u1EA7n cu\u1ED1i n\u1EBFu b\u1EA1n ch\u1EC9 mu\u1ED1n c\xF3 m\u1ED9t c\xE1i nh\xECn t\u1ED5ng quan v\u1EC1 thu\u1EADt to\xE1n tokenize n\xE0y."),k.forEach(n)},m(v,k){e(v,u,k),t(u,_)},d(v){v&&n(u)}}}function Cr(A){let u,_;return{c(){u=i("p"),_=l("\u26A0\uFE0F Google kh\xF4ng bao gi\u1EDD c\xF3 ngu\u1ED3n m\u1EDF v\u1EC1 c\xE1ch tri\u1EC3n khai c\xE1c thu\u1EADt to\xE1n hu\u1EA5n luy\u1EC7n c\u1EE7a WordPiece,v\xEC v\u1EADy nh\u1EEFng g\xEC d\u01B0\u1EDBi \u0111\xE2y l\xE0 ph\u1ECFng \u0111o\xE1n t\u1ED1t nh\u1EA5t c\u1EE7a ch\xFAng t\xF4i d\u1EF1a tr\xEAn c\xE1c t\xE0i li\u1EC7u \u0111\xE3 xu\u1EA5t b\u1EA3n. N\xF3 c\xF3 th\u1EC3 kh\xF4ng ch\xEDnh x\xE1c 100%.")},l(v){u=r(v,"P",{});var k=h(u);_=p(k,"\u26A0\uFE0F Google kh\xF4ng bao gi\u1EDD c\xF3 ngu\u1ED3n m\u1EDF v\u1EC1 c\xE1ch tri\u1EC3n khai c\xE1c thu\u1EADt to\xE1n hu\u1EA5n luy\u1EC7n c\u1EE7a WordPiece,v\xEC v\u1EADy nh\u1EEFng g\xEC d\u01B0\u1EDBi \u0111\xE2y l\xE0 ph\u1ECFng \u0111o\xE1n t\u1ED1t nh\u1EA5t c\u1EE7a ch\xFAng t\xF4i d\u1EF1a tr\xEAn c\xE1c t\xE0i li\u1EC7u \u0111\xE3 xu\u1EA5t b\u1EA3n. N\xF3 c\xF3 th\u1EC3 kh\xF4ng ch\xEDnh x\xE1c 100%."),k.forEach(n)},m(v,k){e(v,u,k),t(u,_)},d(v){v&&n(u)}}}function Tr(A){let u,_,v,k,O;return{c(){u=i("p"),_=l("\u270F\uFE0F "),v=i("strong"),k=l("Gi\u1EDD \u0111\u1EBFn l\u01B0\u1EE3t b\u1EA1n!"),O=l(" B\u1ED9 quy lu\u1EADt h\u1EE3p nh\u1EA5t ti\u1EBFp theo l\xE0 g\xEC?")},l($){u=r($,"P",{});var y=h(u);_=p(y,"\u270F\uFE0F "),v=r(y,"STRONG",{});var B=h(v);k=p(B,"Gi\u1EDD \u0111\u1EBFn l\u01B0\u1EE3t b\u1EA1n!"),B.forEach(n),O=p(y," B\u1ED9 quy lu\u1EADt h\u1EE3p nh\u1EA5t ti\u1EBFp theo l\xE0 g\xEC?"),y.forEach(n)},m($,y){e($,u,y),t(u,_),t(u,v),t(v,k),t(u,O)},d($){$&&n(u)}}}function Dr(A){let u,_,v,k,O,$,y,B;return{c(){u=i("p"),_=l("\u270F\uFE0F "),v=i("strong"),k=l("Gi\u1EDD \u0111\u1EBFn l\u01B0\u1EE3t b\u1EA1n!"),O=c(),$=i("code"),y=l('"pugs"'),B=l(" s\u1EBD \u0111\u01B0\u1EE3c tokenize nh\u01B0 th\u1EBF n\xE0o?")},l(M){u=r(M,"P",{});var W=h(u);_=p(W,"\u270F\uFE0F "),v=r(W,"STRONG",{});var V=h(v);k=p(V,"Gi\u1EDD \u0111\u1EBFn l\u01B0\u1EE3t b\u1EA1n!"),V.forEach(n),O=o(W),$=r(W,"CODE",{});var cs=h($);y=p(cs,'"pugs"'),cs.forEach(n),B=p(W," s\u1EBD \u0111\u01B0\u1EE3c tokenize nh\u01B0 th\u1EBF n\xE0o?"),W.forEach(n)},m(M,W){e(M,u,W),t(u,_),t(u,v),t(v,k),t(u,O),t(u,$),t($,y),t(u,B)},d(M){M&&n(u)}}}function Or(A){let u,_,v,k,O;return{c(){u=i("p"),_=l("\u{1F4A1} S\u1EED d\u1EE5ng "),v=i("code"),k=l("train_new_from_iterator()"),O=l(" tr\xEAn c\xF9ng kho ng\u1EEF li\u1EC7u s\u1EBD kh\xF4ng mang v\u1EC1 k\u1EBFt qu\u1EA3 kho ng\u1EEF li\u1EC7u y h\u1EC7t. \u0110\xF3 l\xE0 b\u1EDFi th\u01B0 vi\u1EC7n \u{1F917} Tokenizers kh\xF4ng tri\u1EC3n khai WordPiece cho hu\u1EA5n luy\u1EC7n (v\xEC ch\xFAng ta kh\xF4ng ho\xE0n to\xE0n n\u1EB1m r\xF5 b\xEAn trong), v\xE0 s\u1EED d\u1EE5ng BPE thay v\xE0o \u0111\xF3.")},l($){u=r($,"P",{});var y=h(u);_=p(y,"\u{1F4A1} S\u1EED d\u1EE5ng "),v=r(y,"CODE",{});var B=h(v);k=p(B,"train_new_from_iterator()"),B.forEach(n),O=p(y," tr\xEAn c\xF9ng kho ng\u1EEF li\u1EC7u s\u1EBD kh\xF4ng mang v\u1EC1 k\u1EBFt qu\u1EA3 kho ng\u1EEF li\u1EC7u y h\u1EC7t. \u0110\xF3 l\xE0 b\u1EDFi th\u01B0 vi\u1EC7n \u{1F917} Tokenizers kh\xF4ng tri\u1EC3n khai WordPiece cho hu\u1EA5n luy\u1EC7n (v\xEC ch\xFAng ta kh\xF4ng ho\xE0n to\xE0n n\u1EB1m r\xF5 b\xEAn trong), v\xE0 s\u1EED d\u1EE5ng BPE thay v\xE0o \u0111\xF3."),y.forEach(n)},m($,y){e($,u,y),t(u,_),t(u,v),t(v,k),t(u,O)},d($){$&&n(u)}}}function Nr(A){let u,_,v,k,O,$,y,B,M,W,V,cs,Ys,cl,zt,os,Ct,J,Tt,L,Q,kn,us,ol,$n,ul,Dt,X,Ot,G,ml,_n,gl,fl,wn,jl,bl,Nt,ms,Bt,Js,dl,St,gs,vl,Ht,vr='<span class="katex-display"><span class="katex"><span class="katex-mathml"><math xmlns="http://www.w3.org/1998/Math/MathML" display="block"><semantics><mrow><mrow><mi mathvariant="normal">s</mi><mi mathvariant="normal">c</mi><mi mathvariant="normal">o</mi><mi mathvariant="normal">r</mi><mi mathvariant="normal">e</mi></mrow><mo>=</mo><mo stretchy="false">(</mo><mrow><mi mathvariant="normal">f</mi><mi mathvariant="normal">r</mi><mi mathvariant="normal">e</mi><mi mathvariant="normal">q</mi><mi mathvariant="normal">_</mi><mi mathvariant="normal">o</mi><mi mathvariant="normal">f</mi><mi mathvariant="normal">_</mi><mi mathvariant="normal">p</mi><mi mathvariant="normal">a</mi><mi mathvariant="normal">i</mi><mi mathvariant="normal">r</mi></mrow><mo stretchy="false">)</mo><mi mathvariant="normal">/</mi><mo stretchy="false">(</mo><mrow><mi mathvariant="normal">f</mi><mi mathvariant="normal">r</mi><mi mathvariant="normal">e</mi><mi mathvariant="normal">q</mi><mi mathvariant="normal">_</mi><mi mathvariant="normal">o</mi><mi mathvariant="normal">f</mi><mi mathvariant="normal">_</mi><mi mathvariant="normal">f</mi><mi mathvariant="normal">i</mi><mi mathvariant="normal">r</mi><mi mathvariant="normal">s</mi><mi mathvariant="normal">t</mi><mi mathvariant="normal">_</mi><mi mathvariant="normal">e</mi><mi mathvariant="normal">l</mi><mi mathvariant="normal">e</mi><mi mathvariant="normal">m</mi><mi mathvariant="normal">e</mi><mi mathvariant="normal">n</mi><mi mathvariant="normal">t</mi></mrow><mo>\xD7</mo><mrow><mi mathvariant="normal">f</mi><mi mathvariant="normal">r</mi><mi mathvariant="normal">e</mi><mi mathvariant="normal">q</mi><mi mathvariant="normal">_</mi><mi mathvariant="normal">o</mi><mi mathvariant="normal">f</mi><mi mathvariant="normal">_</mi><mi mathvariant="normal">s</mi><mi mathvariant="normal">e</mi><mi mathvariant="normal">c</mi><mi mathvariant="normal">o</mi><mi mathvariant="normal">n</mi><mi mathvariant="normal">d</mi><mi mathvariant="normal">_</mi><mi mathvariant="normal">e</mi><mi mathvariant="normal">l</mi><mi mathvariant="normal">e</mi><mi mathvariant="normal">m</mi><mi mathvariant="normal">e</mi><mi mathvariant="normal">n</mi><mi mathvariant="normal">t</mi></mrow><mo stretchy="false">)</mo></mrow><annotation encoding="application/x-tex">\\mathrm{score} = (\\mathrm{freq\\_of\\_pair}) / (\\mathrm{freq\\_of\\_first\\_element} \\times \\mathrm{freq\\_of\\_second\\_element})</annotation></semantics></math></span><span class="katex-html" aria-hidden="true"><span class="base"><span class="strut" style="height:0.4306em;"></span><span class="mord"><span class="mord mathrm">score</span></span><span class="mspace" style="margin-right:0.2778em;"></span><span class="mrel">=</span><span class="mspace" style="margin-right:0.2778em;"></span></span><span class="base"><span class="strut" style="height:1.06em;vertical-align:-0.31em;"></span><span class="mopen">(</span><span class="mord"><span class="mord mathrm">freq_of_pair</span></span><span class="mclose">)</span><span class="mord">/</span><span class="mopen">(</span><span class="mord"><span class="mord mathrm">freq_of_first_element</span></span><span class="mspace" style="margin-right:0.2222em;"></span><span class="mbin">\xD7</span><span class="mspace" style="margin-right:0.2222em;"></span></span><span class="base"><span class="strut" style="height:1.06em;vertical-align:-0.31em;"></span><span class="mord"><span class="mord mathrm">freq_of_second_element</span></span><span class="mclose">)</span></span></span></span></span>',Wt,P,xl,qn,kl,$l,yn,_l,wl,En,ql,yl,Pn,El,Pl,zn,zl,Cl,Cn,Tl,Dl,At,Qs,Ol,Vt,fs,Gt,Xs,Nl,Kt,js,Ut,E,Bl,Tn,Sl,Hl,Dn,Wl,Al,On,Vl,Gl,Nn,Kl,Ul,Bn,Fl,Rl,Sn,Ml,Ll,Hn,Il,Yl,Ft,K,Jl,Wn,Ql,Xl,An,Zl,sp,Rt,bs,Mt,U,np,Vn,tp,ap,Gn,lp,pp,Lt,ds,It,F,ep,Kn,ip,rp,Un,hp,cp,Yt,vs,Jt,Zs,op,Qt,Z,Xt,I,ss,Fn,xs,up,Rn,mp,Zt,z,gp,Mn,fp,jp,Ln,bp,dp,In,vp,xp,Yn,kp,$p,Jn,_p,wp,Qn,qp,yp,sa,ns,Ep,Xn,Pp,zp,na,w,Cp,Zn,Tp,Dp,st,Op,Np,nt,Bp,Sp,tt,Hp,Wp,at,Ap,Vp,lt,Gp,Kp,pt,Up,Fp,et,Rp,Mp,ta,q,Lp,it,Ip,Yp,rt,Jp,Qp,ht,Xp,Zp,ct,se,ne,ot,te,ae,ut,le,pe,mt,ee,ie,gt,re,he,aa,ts,la,Y,as,ft,ks,ce,jt,oe,pa,sn,ue,ea,nn,me,ia,$s,ra,ls,ge,bt,fe,je,ha,_s,ca,tn,be,oa,ws,ua,qs,ma,ps,de,dt,ve,xe,ga,ys,fa,Es,ja,es,ke,vt,$e,_e,ba,Ps,da,is,we,xt,qe,ye,va,zs,xa,an,Ee,ka,Cs,$a,ln,Pe,_a,Ts,wa,Ds,qa,pn,ze,ya,Os,Ea,Ns,Pa,R,Ce,kt,Te,De,$t,Oe,Ne,za,Bs,Ca,rs,Be,_t,Se,He,Ta,Ss,Da,en,We,Oa,Hs,Na,Ws,Ba,rn,Ae,Sa,As,Ha,hn,Ve,Wa,Vs,Aa,Gs,Va,cn,Ge,Ga,hs,Ka,on,Ke,Ua,Ks,Fa,un,Ue,Ra,Us,Ma,Fs,La,mn,Fe,Ia,Rs,Ya,gn,Re,Ja,Ms,Qa,Ls,Xa,fn,Me,Za;return $=new hl({}),V=new Pr({props:{classNames:"absolute z-10 right-0 top-0",options:[{label:"Google Colab",value:"https://colab.research.google.com/github/huggingface/notebooks/blob/master/course/vi/chapter6/section6.ipynb"},{label:"Aws Studio",value:"https://studiolab.sagemaker.aws/import/github/huggingface/notebooks/blob/master/course/vi/chapter6/section6.ipynb"}]}}),os=new Er({props:{id:"qpv6ms_t_1A"}}),J=new Pt({props:{$$slots:{default:[zr]},$$scope:{ctx:A}}}),us=new hl({}),X=new Pt({props:{warning:!0,$$slots:{default:[Cr]},$$scope:{ctx:A}}}),ms=new x({props:{code:"w ##o ##r ##d",highlighted:"w ##o ##r ##d"}}),fs=new x({props:{code:'("hug", 10), ("pug", 5), ("pun", 12), ("bun", 4), ("hugs", 5)',highlighted:'(<span class="hljs-string">&quot;hug&quot;</span><span class="hljs-punctuation">,</span> <span class="hljs-number">10</span>)<span class="hljs-punctuation">,</span> (<span class="hljs-string">&quot;pug&quot;</span><span class="hljs-punctuation">,</span> <span class="hljs-number">5</span>)<span class="hljs-punctuation">,</span> (<span class="hljs-string">&quot;pun&quot;</span><span class="hljs-punctuation">,</span> <span class="hljs-number">12</span>)<span class="hljs-punctuation">,</span> (<span class="hljs-string">&quot;bun&quot;</span><span class="hljs-punctuation">,</span> <span class="hljs-number">4</span>)<span class="hljs-punctuation">,</span> (<span class="hljs-string">&quot;hugs&quot;</span><span class="hljs-punctuation">,</span> <span class="hljs-number">5</span>)'}}),js=new x({props:{code:'("h" "##u" "##g", 10), ("p" "##u" "##g", 5), ("p" "##u" "##n", 12), ("b" "##u" "##n", 4), ("h" "##u" "##g" "##s", 5)',highlighted:'(<span class="hljs-string">&quot;h&quot;</span> <span class="hljs-string">&quot;#<span class="hljs-subst">#u</span>&quot;</span> <span class="hljs-string">&quot;#<span class="hljs-subst">#g</span>&quot;</span>, <span class="hljs-number">10</span>), (<span class="hljs-string">&quot;p&quot;</span> <span class="hljs-string">&quot;#<span class="hljs-subst">#u</span>&quot;</span> <span class="hljs-string">&quot;#<span class="hljs-subst">#g</span>&quot;</span>, <span class="hljs-number">5</span>), (<span class="hljs-string">&quot;p&quot;</span> <span class="hljs-string">&quot;#<span class="hljs-subst">#u</span>&quot;</span> <span class="hljs-string">&quot;#<span class="hljs-subst">#n</span>&quot;</span>, <span class="hljs-number">12</span>), (<span class="hljs-string">&quot;b&quot;</span> <span class="hljs-string">&quot;#<span class="hljs-subst">#u</span>&quot;</span> <span class="hljs-string">&quot;#<span class="hljs-subst">#n</span>&quot;</span>, <span class="hljs-number">4</span>), (<span class="hljs-string">&quot;h&quot;</span> <span class="hljs-string">&quot;#<span class="hljs-subst">#u</span>&quot;</span> <span class="hljs-string">&quot;#<span class="hljs-subst">#g</span>&quot;</span> <span class="hljs-string">&quot;#<span class="hljs-subst">#s</span>&quot;</span>, <span class="hljs-number">5</span>)'}}),bs=new x({props:{code:`Vocabulary: ["b", "h", "p", "##g", "##n", "##s", "##u", "##gs"]
Corpus: ("h" "##u" "##g", 10), ("p" "##u" "##g", 5), ("p" "##u" "##n", 12), ("b" "##u" "##n", 4), ("h" "##u" "##gs", 5)`,highlighted:`Vocabulary: [<span class="hljs-string">&quot;b&quot;</span>, <span class="hljs-string">&quot;h&quot;</span>, <span class="hljs-string">&quot;p&quot;</span>, <span class="hljs-string">&quot;#<span class="hljs-subst">#g</span>&quot;</span>, <span class="hljs-string">&quot;#<span class="hljs-subst">#n</span>&quot;</span>, <span class="hljs-string">&quot;#<span class="hljs-subst">#s</span>&quot;</span>, <span class="hljs-string">&quot;#<span class="hljs-subst">#u</span>&quot;</span>, <span class="hljs-string">&quot;#<span class="hljs-subst">#gs</span>&quot;</span>]
Corpus: (<span class="hljs-string">&quot;h&quot;</span> <span class="hljs-string">&quot;#<span class="hljs-subst">#u</span>&quot;</span> <span class="hljs-string">&quot;#<span class="hljs-subst">#g</span>&quot;</span>, <span class="hljs-number">10</span>), (<span class="hljs-string">&quot;p&quot;</span> <span class="hljs-string">&quot;#<span class="hljs-subst">#u</span>&quot;</span> <span class="hljs-string">&quot;#<span class="hljs-subst">#g</span>&quot;</span>, <span class="hljs-number">5</span>), (<span class="hljs-string">&quot;p&quot;</span> <span class="hljs-string">&quot;#<span class="hljs-subst">#u</span>&quot;</span> <span class="hljs-string">&quot;#<span class="hljs-subst">#n</span>&quot;</span>, <span class="hljs-number">12</span>), (<span class="hljs-string">&quot;b&quot;</span> <span class="hljs-string">&quot;#<span class="hljs-subst">#u</span>&quot;</span> <span class="hljs-string">&quot;#<span class="hljs-subst">#n</span>&quot;</span>, <span class="hljs-number">4</span>), (<span class="hljs-string">&quot;h&quot;</span> <span class="hljs-string">&quot;#<span class="hljs-subst">#u</span>&quot;</span> <span class="hljs-string">&quot;#<span class="hljs-subst">#gs</span>&quot;</span>, <span class="hljs-number">5</span>)`}}),ds=new x({props:{code:`Vocabulary: ["b", "h", "p", "##g", "##n", "##s", "##u", "##gs", "hu"]
Corpus: ("hu" "##g", 10), ("p" "##u" "##g", 5), ("p" "##u" "##n", 12), ("b" "##u" "##n", 4), ("hu" "##gs", 5)`,highlighted:`Vocabulary: [<span class="hljs-string">&quot;b&quot;</span>, <span class="hljs-string">&quot;h&quot;</span>, <span class="hljs-string">&quot;p&quot;</span>, <span class="hljs-string">&quot;#<span class="hljs-subst">#g</span>&quot;</span>, <span class="hljs-string">&quot;#<span class="hljs-subst">#n</span>&quot;</span>, <span class="hljs-string">&quot;#<span class="hljs-subst">#s</span>&quot;</span>, <span class="hljs-string">&quot;#<span class="hljs-subst">#u</span>&quot;</span>, <span class="hljs-string">&quot;#<span class="hljs-subst">#gs</span>&quot;</span>, <span class="hljs-string">&quot;hu&quot;</span>]
Corpus: (<span class="hljs-string">&quot;hu&quot;</span> <span class="hljs-string">&quot;#<span class="hljs-subst">#g</span>&quot;</span>, <span class="hljs-number">10</span>), (<span class="hljs-string">&quot;p&quot;</span> <span class="hljs-string">&quot;#<span class="hljs-subst">#u</span>&quot;</span> <span class="hljs-string">&quot;#<span class="hljs-subst">#g</span>&quot;</span>, <span class="hljs-number">5</span>), (<span class="hljs-string">&quot;p&quot;</span> <span class="hljs-string">&quot;#<span class="hljs-subst">#u</span>&quot;</span> <span class="hljs-string">&quot;#<span class="hljs-subst">#n</span>&quot;</span>, <span class="hljs-number">12</span>), (<span class="hljs-string">&quot;b&quot;</span> <span class="hljs-string">&quot;#<span class="hljs-subst">#u</span>&quot;</span> <span class="hljs-string">&quot;#<span class="hljs-subst">#n</span>&quot;</span>, <span class="hljs-number">4</span>), (<span class="hljs-string">&quot;hu&quot;</span> <span class="hljs-string">&quot;#<span class="hljs-subst">#gs</span>&quot;</span>, <span class="hljs-number">5</span>)`}}),vs=new x({props:{code:`Vocabulary: ["b", "h", "p", "##g", "##n", "##s", "##u", "##gs", "hu", "hug"]
Corpus: ("hug", 10), ("p" "##u" "##g", 5), ("p" "##u" "##n", 12), ("b" "##u" "##n", 4), ("hu" "##gs", 5)`,highlighted:`Vocabulary: [<span class="hljs-string">&quot;b&quot;</span>, <span class="hljs-string">&quot;h&quot;</span>, <span class="hljs-string">&quot;p&quot;</span>, <span class="hljs-string">&quot;#<span class="hljs-subst">#g</span>&quot;</span>, <span class="hljs-string">&quot;#<span class="hljs-subst">#n</span>&quot;</span>, <span class="hljs-string">&quot;#<span class="hljs-subst">#s</span>&quot;</span>, <span class="hljs-string">&quot;#<span class="hljs-subst">#u</span>&quot;</span>, <span class="hljs-string">&quot;#<span class="hljs-subst">#gs</span>&quot;</span>, <span class="hljs-string">&quot;hu&quot;</span>, <span class="hljs-string">&quot;hug&quot;</span>]
Corpus: (<span class="hljs-string">&quot;hug&quot;</span>, <span class="hljs-number">10</span>), (<span class="hljs-string">&quot;p&quot;</span> <span class="hljs-string">&quot;#<span class="hljs-subst">#u</span>&quot;</span> <span class="hljs-string">&quot;#<span class="hljs-subst">#g</span>&quot;</span>, <span class="hljs-number">5</span>), (<span class="hljs-string">&quot;p&quot;</span> <span class="hljs-string">&quot;#<span class="hljs-subst">#u</span>&quot;</span> <span class="hljs-string">&quot;#<span class="hljs-subst">#n</span>&quot;</span>, <span class="hljs-number">12</span>), (<span class="hljs-string">&quot;b&quot;</span> <span class="hljs-string">&quot;#<span class="hljs-subst">#u</span>&quot;</span> <span class="hljs-string">&quot;#<span class="hljs-subst">#n</span>&quot;</span>, <span class="hljs-number">4</span>), (<span class="hljs-string">&quot;hu&quot;</span> <span class="hljs-string">&quot;#<span class="hljs-subst">#gs</span>&quot;</span>, <span class="hljs-number">5</span>)`}}),Z=new Pt({props:{$$slots:{default:[Tr]},$$scope:{ctx:A}}}),xs=new hl({}),ts=new Pt({props:{$$slots:{default:[Dr]},$$scope:{ctx:A}}}),ks=new hl({}),$s=new x({props:{code:`corpus = [
    "This is the Hugging Face Course.",
    "This chapter is about tokenization.",
    "This section shows several tokenizer algorithms.",
    "Hopefully, you will be able to understand how they are trained and generate tokens.",
]`,highlighted:`corpus = [
    <span class="hljs-string">&quot;This is the Hugging Face Course.&quot;</span>,
    <span class="hljs-string">&quot;This chapter is about tokenization.&quot;</span>,
    <span class="hljs-string">&quot;This section shows several tokenizer algorithms.&quot;</span>,
    <span class="hljs-string">&quot;Hopefully, you will be able to understand how they are trained and generate tokens.&quot;</span>,
]`}}),_s=new x({props:{code:`from transformers import AutoTokenizer

tokenizer = AutoTokenizer.from_pretrained("bert-base-cased")`,highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoTokenizer

tokenizer = AutoTokenizer.from_pretrained(<span class="hljs-string">&quot;bert-base-cased&quot;</span>)`}}),ws=new x({props:{code:`from collections import defaultdict

word_freqs = defaultdict(int)
for text in corpus:
    words_with_offsets = tokenizer.backend_tokenizer.pre_tokenizer.pre_tokenize_str(text)
    new_words = [word for word, offset in words_with_offsets]
    for word in new_words:
        word_freqs[word] += 1

word_freqs`,highlighted:`<span class="hljs-keyword">from</span> collections <span class="hljs-keyword">import</span> defaultdict

word_freqs = defaultdict(<span class="hljs-built_in">int</span>)
<span class="hljs-keyword">for</span> text <span class="hljs-keyword">in</span> corpus:
    words_with_offsets = tokenizer.backend_tokenizer.pre_tokenizer.pre_tokenize_str(text)
    new_words = [word <span class="hljs-keyword">for</span> word, offset <span class="hljs-keyword">in</span> words_with_offsets]
    <span class="hljs-keyword">for</span> word <span class="hljs-keyword">in</span> new_words:
        word_freqs[word] += <span class="hljs-number">1</span>

word_freqs`}}),qs=new x({props:{code:`defaultdict(
    int, {'This': 3, 'is': 2, 'the': 1, 'Hugging': 1, 'Face': 1, 'Course': 1, '.': 4, 'chapter': 1, 'about': 1,
    'tokenization': 1, 'section': 1, 'shows': 1, 'several': 1, 'tokenizer': 1, 'algorithms': 1, 'Hopefully': 1,
    ',': 1, 'you': 1, 'will': 1, 'be': 1, 'able': 1, 'to': 1, 'understand': 1, 'how': 1, 'they': 1, 'are': 1,
    'trained': 1, 'and': 1, 'generate': 1, 'tokens': 1})`,highlighted:`defaultdict(
    <span class="hljs-built_in">int</span>, {<span class="hljs-string">&#x27;This&#x27;</span>: <span class="hljs-number">3</span>, <span class="hljs-string">&#x27;is&#x27;</span>: <span class="hljs-number">2</span>, <span class="hljs-string">&#x27;the&#x27;</span>: <span class="hljs-number">1</span>, <span class="hljs-string">&#x27;Hugging&#x27;</span>: <span class="hljs-number">1</span>, <span class="hljs-string">&#x27;Face&#x27;</span>: <span class="hljs-number">1</span>, <span class="hljs-string">&#x27;Course&#x27;</span>: <span class="hljs-number">1</span>, <span class="hljs-string">&#x27;.&#x27;</span>: <span class="hljs-number">4</span>, <span class="hljs-string">&#x27;chapter&#x27;</span>: <span class="hljs-number">1</span>, <span class="hljs-string">&#x27;about&#x27;</span>: <span class="hljs-number">1</span>,
    <span class="hljs-string">&#x27;tokenization&#x27;</span>: <span class="hljs-number">1</span>, <span class="hljs-string">&#x27;section&#x27;</span>: <span class="hljs-number">1</span>, <span class="hljs-string">&#x27;shows&#x27;</span>: <span class="hljs-number">1</span>, <span class="hljs-string">&#x27;several&#x27;</span>: <span class="hljs-number">1</span>, <span class="hljs-string">&#x27;tokenizer&#x27;</span>: <span class="hljs-number">1</span>, <span class="hljs-string">&#x27;algorithms&#x27;</span>: <span class="hljs-number">1</span>, <span class="hljs-string">&#x27;Hopefully&#x27;</span>: <span class="hljs-number">1</span>,
    <span class="hljs-string">&#x27;,&#x27;</span>: <span class="hljs-number">1</span>, <span class="hljs-string">&#x27;you&#x27;</span>: <span class="hljs-number">1</span>, <span class="hljs-string">&#x27;will&#x27;</span>: <span class="hljs-number">1</span>, <span class="hljs-string">&#x27;be&#x27;</span>: <span class="hljs-number">1</span>, <span class="hljs-string">&#x27;able&#x27;</span>: <span class="hljs-number">1</span>, <span class="hljs-string">&#x27;to&#x27;</span>: <span class="hljs-number">1</span>, <span class="hljs-string">&#x27;understand&#x27;</span>: <span class="hljs-number">1</span>, <span class="hljs-string">&#x27;how&#x27;</span>: <span class="hljs-number">1</span>, <span class="hljs-string">&#x27;they&#x27;</span>: <span class="hljs-number">1</span>, <span class="hljs-string">&#x27;are&#x27;</span>: <span class="hljs-number">1</span>,
    <span class="hljs-string">&#x27;trained&#x27;</span>: <span class="hljs-number">1</span>, <span class="hljs-string">&#x27;and&#x27;</span>: <span class="hljs-number">1</span>, <span class="hljs-string">&#x27;generate&#x27;</span>: <span class="hljs-number">1</span>, <span class="hljs-string">&#x27;tokens&#x27;</span>: <span class="hljs-number">1</span>})`}}),ys=new x({props:{code:`alphabet = []
for word in word_freqs.keys():
    if word[0] not in alphabet:
        alphabet.append(word[0])
    for letter in word[1:]:
        if f"##{letter}" not in alphabet:
            alphabet.append(f"##{letter}")

alphabet.sort()
alphabet

print(alphabet)`,highlighted:`alphabet = []
<span class="hljs-keyword">for</span> word <span class="hljs-keyword">in</span> word_freqs.keys():
    <span class="hljs-keyword">if</span> word[<span class="hljs-number">0</span>] <span class="hljs-keyword">not</span> <span class="hljs-keyword">in</span> alphabet:
        alphabet.append(word[<span class="hljs-number">0</span>])
    <span class="hljs-keyword">for</span> letter <span class="hljs-keyword">in</span> word[<span class="hljs-number">1</span>:]:
        <span class="hljs-keyword">if</span> <span class="hljs-string">f&quot;##<span class="hljs-subst">{letter}</span>&quot;</span> <span class="hljs-keyword">not</span> <span class="hljs-keyword">in</span> alphabet:
            alphabet.append(<span class="hljs-string">f&quot;##<span class="hljs-subst">{letter}</span>&quot;</span>)

alphabet.sort()
alphabet

<span class="hljs-built_in">print</span>(alphabet)`}}),Es=new x({props:{code:`['##a', '##b', '##c', '##d', '##e', '##f', '##g', '##h', '##i', '##k', '##l', '##m', '##n', '##o', '##p', '##r', '##s',
 '##t', '##u', '##v', '##w', '##y', '##z', ',', '.', 'C', 'F', 'H', 'T', 'a', 'b', 'c', 'g', 'h', 'i', 's', 't', 'u',
 'w', 'y']`,highlighted:`[<span class="hljs-string">&#x27;##a&#x27;</span>, <span class="hljs-string">&#x27;##b&#x27;</span>, <span class="hljs-string">&#x27;##c&#x27;</span>, <span class="hljs-string">&#x27;##d&#x27;</span>, <span class="hljs-string">&#x27;##e&#x27;</span>, <span class="hljs-string">&#x27;##f&#x27;</span>, <span class="hljs-string">&#x27;##g&#x27;</span>, <span class="hljs-string">&#x27;##h&#x27;</span>, <span class="hljs-string">&#x27;##i&#x27;</span>, <span class="hljs-string">&#x27;##k&#x27;</span>, <span class="hljs-string">&#x27;##l&#x27;</span>, <span class="hljs-string">&#x27;##m&#x27;</span>, <span class="hljs-string">&#x27;##n&#x27;</span>, <span class="hljs-string">&#x27;##o&#x27;</span>, <span class="hljs-string">&#x27;##p&#x27;</span>, <span class="hljs-string">&#x27;##r&#x27;</span>, <span class="hljs-string">&#x27;##s&#x27;</span>,
 <span class="hljs-string">&#x27;##t&#x27;</span>, <span class="hljs-string">&#x27;##u&#x27;</span>, <span class="hljs-string">&#x27;##v&#x27;</span>, <span class="hljs-string">&#x27;##w&#x27;</span>, <span class="hljs-string">&#x27;##y&#x27;</span>, <span class="hljs-string">&#x27;##z&#x27;</span>, <span class="hljs-string">&#x27;,&#x27;</span>, <span class="hljs-string">&#x27;.&#x27;</span>, <span class="hljs-string">&#x27;C&#x27;</span>, <span class="hljs-string">&#x27;F&#x27;</span>, <span class="hljs-string">&#x27;H&#x27;</span>, <span class="hljs-string">&#x27;T&#x27;</span>, <span class="hljs-string">&#x27;a&#x27;</span>, <span class="hljs-string">&#x27;b&#x27;</span>, <span class="hljs-string">&#x27;c&#x27;</span>, <span class="hljs-string">&#x27;g&#x27;</span>, <span class="hljs-string">&#x27;h&#x27;</span>, <span class="hljs-string">&#x27;i&#x27;</span>, <span class="hljs-string">&#x27;s&#x27;</span>, <span class="hljs-string">&#x27;t&#x27;</span>, <span class="hljs-string">&#x27;u&#x27;</span>,
 <span class="hljs-string">&#x27;w&#x27;</span>, <span class="hljs-string">&#x27;y&#x27;</span>]`}}),Ps=new x({props:{code:'vocab = ["[PAD]", "[UNK]", "[CLS]", "[SEP]", "[MASK]"] + alphabet.copy()',highlighted:'vocab = [<span class="hljs-string">&quot;[PAD]&quot;</span>, <span class="hljs-string">&quot;[UNK]&quot;</span>, <span class="hljs-string">&quot;[CLS]&quot;</span>, <span class="hljs-string">&quot;[SEP]&quot;</span>, <span class="hljs-string">&quot;[MASK]&quot;</span>] + alphabet.copy()'}}),zs=new x({props:{code:`splits = {
    word: [c if i == 0 else f"##{c}" for i, c in enumerate(word)]
    for word in word_freqs.keys()
}`,highlighted:`splits = {
    word: [c <span class="hljs-keyword">if</span> i == <span class="hljs-number">0</span> <span class="hljs-keyword">else</span> <span class="hljs-string">f&quot;##<span class="hljs-subst">{c}</span>&quot;</span> <span class="hljs-keyword">for</span> i, c <span class="hljs-keyword">in</span> <span class="hljs-built_in">enumerate</span>(word)]
    <span class="hljs-keyword">for</span> word <span class="hljs-keyword">in</span> word_freqs.keys()
}`}}),Cs=new x({props:{code:`def compute_pair_scores(splits):
    letter_freqs = defaultdict(int)
    pair_freqs = defaultdict(int)
    for word, freq in word_freqs.items():
        split = splits[word]
        if len(split) == 1:
            letter_freqs[split[0]] += freq
            continue
        for i in range(len(split) - 1):
            pair = (split[i], split[i + 1])
            letter_freqs[split[i]] += freq
            pair_freqs[pair] += freq
        letter_freqs[split[-1]] += freq

    scores = {
        pair: freq / (letter_freqs[pair[0]] * letter_freqs[pair[1]])
        for pair, freq in pair_freqs.items()
    }
    return scores`,highlighted:`<span class="hljs-keyword">def</span> <span class="hljs-title function_">compute_pair_scores</span>(<span class="hljs-params">splits</span>):
    letter_freqs = defaultdict(<span class="hljs-built_in">int</span>)
    pair_freqs = defaultdict(<span class="hljs-built_in">int</span>)
    <span class="hljs-keyword">for</span> word, freq <span class="hljs-keyword">in</span> word_freqs.items():
        split = splits[word]
        <span class="hljs-keyword">if</span> <span class="hljs-built_in">len</span>(split) == <span class="hljs-number">1</span>:
            letter_freqs[split[<span class="hljs-number">0</span>]] += freq
            <span class="hljs-keyword">continue</span>
        <span class="hljs-keyword">for</span> i <span class="hljs-keyword">in</span> <span class="hljs-built_in">range</span>(<span class="hljs-built_in">len</span>(split) - <span class="hljs-number">1</span>):
            pair = (split[i], split[i + <span class="hljs-number">1</span>])
            letter_freqs[split[i]] += freq
            pair_freqs[pair] += freq
        letter_freqs[split[-<span class="hljs-number">1</span>]] += freq

    scores = {
        pair: freq / (letter_freqs[pair[<span class="hljs-number">0</span>]] * letter_freqs[pair[<span class="hljs-number">1</span>]])
        <span class="hljs-keyword">for</span> pair, freq <span class="hljs-keyword">in</span> pair_freqs.items()
    }
    <span class="hljs-keyword">return</span> scores`}}),Ts=new x({props:{code:`pair_scores = compute_pair_scores(splits)
for i, key in enumerate(pair_scores.keys()):
    print(f"{key}: {pair_scores[key]}")
    if i >= 5:
        break`,highlighted:`pair_scores = compute_pair_scores(splits)
<span class="hljs-keyword">for</span> i, key <span class="hljs-keyword">in</span> <span class="hljs-built_in">enumerate</span>(pair_scores.keys()):
    <span class="hljs-built_in">print</span>(<span class="hljs-string">f&quot;<span class="hljs-subst">{key}</span>: <span class="hljs-subst">{pair_scores[key]}</span>&quot;</span>)
    <span class="hljs-keyword">if</span> i &gt;= <span class="hljs-number">5</span>:
        <span class="hljs-keyword">break</span>`}}),Ds=new x({props:{code:`('T', '##h'): 0.125
('##h', '##i'): 0.03409090909090909
('##i', '##s'): 0.02727272727272727
('i', '##s'): 0.1
('t', '##h'): 0.03571428571428571
('##h', '##e'): 0.011904761904761904`,highlighted:`(<span class="hljs-string">&#x27;T&#x27;</span>, <span class="hljs-string">&#x27;##h&#x27;</span>): <span class="hljs-number">0.125</span>
(<span class="hljs-string">&#x27;##h&#x27;</span>, <span class="hljs-string">&#x27;##i&#x27;</span>): <span class="hljs-number">0.03409090909090909</span>
(<span class="hljs-string">&#x27;##i&#x27;</span>, <span class="hljs-string">&#x27;##s&#x27;</span>): <span class="hljs-number">0.02727272727272727</span>
(<span class="hljs-string">&#x27;i&#x27;</span>, <span class="hljs-string">&#x27;##s&#x27;</span>): <span class="hljs-number">0.1</span>
(<span class="hljs-string">&#x27;t&#x27;</span>, <span class="hljs-string">&#x27;##h&#x27;</span>): <span class="hljs-number">0.03571428571428571</span>
(<span class="hljs-string">&#x27;##h&#x27;</span>, <span class="hljs-string">&#x27;##e&#x27;</span>): <span class="hljs-number">0.011904761904761904</span>`}}),Os=new x({props:{code:`best_pair = ""
max_score = None
for pair, score in pair_scores.items():
    if max_score is None or max_score < score:
        best_pair = pair
        max_score = score

print(best_pair, max_score)`,highlighted:`best_pair = <span class="hljs-string">&quot;&quot;</span>
max_score = <span class="hljs-literal">None</span>
<span class="hljs-keyword">for</span> pair, score <span class="hljs-keyword">in</span> pair_scores.items():
    <span class="hljs-keyword">if</span> max_score <span class="hljs-keyword">is</span> <span class="hljs-literal">None</span> <span class="hljs-keyword">or</span> max_score &lt; score:
        best_pair = pair
        max_score = score

<span class="hljs-built_in">print</span>(best_pair, max_score)`}}),Ns=new x({props:{code:"('a', '##b') 0.2",highlighted:'(<span class="hljs-string">&#x27;a&#x27;</span>, <span class="hljs-string">&#x27;##b&#x27;</span>) <span class="hljs-number">0.2</span>'}}),Bs=new x({props:{code:'vocab.append("ab")',highlighted:'vocab.append(<span class="hljs-string">&quot;ab&quot;</span>)'}}),Ss=new x({props:{code:`def merge_pair(a, b, splits):
    for word in word_freqs:
        split = splits[word]
        if len(split) == 1:
            continue
        i = 0
        while i < len(split) - 1:
            if split[i] == a and split[i + 1] == b:
                merge = a + b[2:] if b.startswith("##") else a + b
                split = split[:i] + [merge] + split[i + 2 :]
            else:
                i += 1
        splits[word] = split
    return splits`,highlighted:`<span class="hljs-keyword">def</span> <span class="hljs-title function_">merge_pair</span>(<span class="hljs-params">a, b, splits</span>):
    <span class="hljs-keyword">for</span> word <span class="hljs-keyword">in</span> word_freqs:
        split = splits[word]
        <span class="hljs-keyword">if</span> <span class="hljs-built_in">len</span>(split) == <span class="hljs-number">1</span>:
            <span class="hljs-keyword">continue</span>
        i = <span class="hljs-number">0</span>
        <span class="hljs-keyword">while</span> i &lt; <span class="hljs-built_in">len</span>(split) - <span class="hljs-number">1</span>:
            <span class="hljs-keyword">if</span> split[i] == a <span class="hljs-keyword">and</span> split[i + <span class="hljs-number">1</span>] == b:
                merge = a + b[<span class="hljs-number">2</span>:] <span class="hljs-keyword">if</span> b.startswith(<span class="hljs-string">&quot;##&quot;</span>) <span class="hljs-keyword">else</span> a + b
                split = split[:i] + [merge] + split[i + <span class="hljs-number">2</span> :]
            <span class="hljs-keyword">else</span>:
                i += <span class="hljs-number">1</span>
        splits[word] = split
    <span class="hljs-keyword">return</span> splits`}}),Hs=new x({props:{code:`splits = merge_pair("a", "##b", splits)
splits["about"]`,highlighted:`splits = merge_pair(<span class="hljs-string">&quot;a&quot;</span>, <span class="hljs-string">&quot;##b&quot;</span>, splits)
splits[<span class="hljs-string">&quot;about&quot;</span>]`}}),Ws=new x({props:{code:"['ab', '##o', '##u', '##t']",highlighted:'[<span class="hljs-string">&#x27;ab&#x27;</span>, <span class="hljs-string">&#x27;##o&#x27;</span>, <span class="hljs-string">&#x27;##u&#x27;</span>, <span class="hljs-string">&#x27;##t&#x27;</span>]'}}),As=new x({props:{code:`vocab_size = 70
while len(vocab) < vocab_size:
    scores = compute_pair_scores(splits)
    best_pair, max_score = "", None
    for pair, score in scores.items():
        if max_score is None or max_score < score:
            best_pair = pair
            max_score = score
    splits = merge_pair(*best_pair, splits)
    new_token = (
        best_pair[0] + best_pair[1][2:]
        if best_pair[1].startswith("##")
        else best_pair[0] + best_pair[1]
    )
    vocab.append(new_token)`,highlighted:`vocab_size = <span class="hljs-number">70</span>
<span class="hljs-keyword">while</span> <span class="hljs-built_in">len</span>(vocab) &lt; vocab_size:
    scores = compute_pair_scores(splits)
    best_pair, max_score = <span class="hljs-string">&quot;&quot;</span>, <span class="hljs-literal">None</span>
    <span class="hljs-keyword">for</span> pair, score <span class="hljs-keyword">in</span> scores.items():
        <span class="hljs-keyword">if</span> max_score <span class="hljs-keyword">is</span> <span class="hljs-literal">None</span> <span class="hljs-keyword">or</span> max_score &lt; score:
            best_pair = pair
            max_score = score
    splits = merge_pair(*best_pair, splits)
    new_token = (
        best_pair[<span class="hljs-number">0</span>] + best_pair[<span class="hljs-number">1</span>][<span class="hljs-number">2</span>:]
        <span class="hljs-keyword">if</span> best_pair[<span class="hljs-number">1</span>].startswith(<span class="hljs-string">&quot;##&quot;</span>)
        <span class="hljs-keyword">else</span> best_pair[<span class="hljs-number">0</span>] + best_pair[<span class="hljs-number">1</span>]
    )
    vocab.append(new_token)`}}),Vs=new x({props:{code:"print(vocab)",highlighted:'<span class="hljs-built_in">print</span>(vocab)'}}),Gs=new x({props:{code:`['[PAD]', '[UNK]', '[CLS]', '[SEP]', '[MASK]', '##a', '##b', '##c', '##d', '##e', '##f', '##g', '##h', '##i', '##k',
 '##l', '##m', '##n', '##o', '##p', '##r', '##s', '##t', '##u', '##v', '##w', '##y', '##z', ',', '.', 'C', 'F', 'H',
 'T', 'a', 'b', 'c', 'g', 'h', 'i', 's', 't', 'u', 'w', 'y', 'ab', '##fu', 'Fa', 'Fac', '##ct', '##ful', '##full', '##fully',
 'Th', 'ch', '##hm', 'cha', 'chap', 'chapt', '##thm', 'Hu', 'Hug', 'Hugg', 'sh', 'th', 'is', '##thms', '##za', '##zat',
 '##ut']`,highlighted:`[<span class="hljs-string">&#x27;[PAD]&#x27;</span>, <span class="hljs-string">&#x27;[UNK]&#x27;</span>, <span class="hljs-string">&#x27;[CLS]&#x27;</span>, <span class="hljs-string">&#x27;[SEP]&#x27;</span>, <span class="hljs-string">&#x27;[MASK]&#x27;</span>, <span class="hljs-string">&#x27;##a&#x27;</span>, <span class="hljs-string">&#x27;##b&#x27;</span>, <span class="hljs-string">&#x27;##c&#x27;</span>, <span class="hljs-string">&#x27;##d&#x27;</span>, <span class="hljs-string">&#x27;##e&#x27;</span>, <span class="hljs-string">&#x27;##f&#x27;</span>, <span class="hljs-string">&#x27;##g&#x27;</span>, <span class="hljs-string">&#x27;##h&#x27;</span>, <span class="hljs-string">&#x27;##i&#x27;</span>, <span class="hljs-string">&#x27;##k&#x27;</span>,
 <span class="hljs-string">&#x27;##l&#x27;</span>, <span class="hljs-string">&#x27;##m&#x27;</span>, <span class="hljs-string">&#x27;##n&#x27;</span>, <span class="hljs-string">&#x27;##o&#x27;</span>, <span class="hljs-string">&#x27;##p&#x27;</span>, <span class="hljs-string">&#x27;##r&#x27;</span>, <span class="hljs-string">&#x27;##s&#x27;</span>, <span class="hljs-string">&#x27;##t&#x27;</span>, <span class="hljs-string">&#x27;##u&#x27;</span>, <span class="hljs-string">&#x27;##v&#x27;</span>, <span class="hljs-string">&#x27;##w&#x27;</span>, <span class="hljs-string">&#x27;##y&#x27;</span>, <span class="hljs-string">&#x27;##z&#x27;</span>, <span class="hljs-string">&#x27;,&#x27;</span>, <span class="hljs-string">&#x27;.&#x27;</span>, <span class="hljs-string">&#x27;C&#x27;</span>, <span class="hljs-string">&#x27;F&#x27;</span>, <span class="hljs-string">&#x27;H&#x27;</span>,
 <span class="hljs-string">&#x27;T&#x27;</span>, <span class="hljs-string">&#x27;a&#x27;</span>, <span class="hljs-string">&#x27;b&#x27;</span>, <span class="hljs-string">&#x27;c&#x27;</span>, <span class="hljs-string">&#x27;g&#x27;</span>, <span class="hljs-string">&#x27;h&#x27;</span>, <span class="hljs-string">&#x27;i&#x27;</span>, <span class="hljs-string">&#x27;s&#x27;</span>, <span class="hljs-string">&#x27;t&#x27;</span>, <span class="hljs-string">&#x27;u&#x27;</span>, <span class="hljs-string">&#x27;w&#x27;</span>, <span class="hljs-string">&#x27;y&#x27;</span>, <span class="hljs-string">&#x27;ab&#x27;</span>, <span class="hljs-string">&#x27;##fu&#x27;</span>, <span class="hljs-string">&#x27;Fa&#x27;</span>, <span class="hljs-string">&#x27;Fac&#x27;</span>, <span class="hljs-string">&#x27;##ct&#x27;</span>, <span class="hljs-string">&#x27;##ful&#x27;</span>, <span class="hljs-string">&#x27;##full&#x27;</span>, <span class="hljs-string">&#x27;##fully&#x27;</span>,
 <span class="hljs-string">&#x27;Th&#x27;</span>, <span class="hljs-string">&#x27;ch&#x27;</span>, <span class="hljs-string">&#x27;##hm&#x27;</span>, <span class="hljs-string">&#x27;cha&#x27;</span>, <span class="hljs-string">&#x27;chap&#x27;</span>, <span class="hljs-string">&#x27;chapt&#x27;</span>, <span class="hljs-string">&#x27;##thm&#x27;</span>, <span class="hljs-string">&#x27;Hu&#x27;</span>, <span class="hljs-string">&#x27;Hug&#x27;</span>, <span class="hljs-string">&#x27;Hugg&#x27;</span>, <span class="hljs-string">&#x27;sh&#x27;</span>, <span class="hljs-string">&#x27;th&#x27;</span>, <span class="hljs-string">&#x27;is&#x27;</span>, <span class="hljs-string">&#x27;##thms&#x27;</span>, <span class="hljs-string">&#x27;##za&#x27;</span>, <span class="hljs-string">&#x27;##zat&#x27;</span>,
 <span class="hljs-string">&#x27;##ut&#x27;</span>]`}}),hs=new Pt({props:{$$slots:{default:[Or]},$$scope:{ctx:A}}}),Ks=new x({props:{code:`def encode_word(word):
    tokens = []
    while len(word) > 0:
        i = len(word)
        while i > 0 and word[:i] not in vocab:
            i -= 1
        if i == 0:
            return ["[UNK]"]
        tokens.append(word[:i])
        word = word[i:]
        if len(word) > 0:
            word = f"##{word}"
    return tokens`,highlighted:`<span class="hljs-keyword">def</span> <span class="hljs-title function_">encode_word</span>(<span class="hljs-params">word</span>):
    tokens = []
    <span class="hljs-keyword">while</span> <span class="hljs-built_in">len</span>(word) &gt; <span class="hljs-number">0</span>:
        i = <span class="hljs-built_in">len</span>(word)
        <span class="hljs-keyword">while</span> i &gt; <span class="hljs-number">0</span> <span class="hljs-keyword">and</span> word[:i] <span class="hljs-keyword">not</span> <span class="hljs-keyword">in</span> vocab:
            i -= <span class="hljs-number">1</span>
        <span class="hljs-keyword">if</span> i == <span class="hljs-number">0</span>:
            <span class="hljs-keyword">return</span> [<span class="hljs-string">&quot;[UNK]&quot;</span>]
        tokens.append(word[:i])
        word = word[i:]
        <span class="hljs-keyword">if</span> <span class="hljs-built_in">len</span>(word) &gt; <span class="hljs-number">0</span>:
            word = <span class="hljs-string">f&quot;##<span class="hljs-subst">{word}</span>&quot;</span>
    <span class="hljs-keyword">return</span> tokens`}}),Us=new x({props:{code:`print(encode_word("Hugging"))
print(encode_word("HOgging"))`,highlighted:`<span class="hljs-built_in">print</span>(encode_word(<span class="hljs-string">&quot;Hugging&quot;</span>))
<span class="hljs-built_in">print</span>(encode_word(<span class="hljs-string">&quot;HOgging&quot;</span>))`}}),Fs=new x({props:{code:`['Hugg', '##i', '##n', '##g']
['[UNK]']`,highlighted:`[<span class="hljs-string">&#x27;Hugg&#x27;</span>, <span class="hljs-string">&#x27;##i&#x27;</span>, <span class="hljs-string">&#x27;##n&#x27;</span>, <span class="hljs-string">&#x27;##g&#x27;</span>]
[<span class="hljs-string">&#x27;[UNK]&#x27;</span>]`}}),Rs=new x({props:{code:`def tokenize(text):
    pre_tokenize_result = tokenizer._tokenizer.pre_tokenizer.pre_tokenize_str(text)
    pre_tokenized_text = [word for word, offset in pre_tokenize_result]
    encoded_words = [encode_word(word) for word in pre_tokenized_text]
    return sum(encoded_words, [])`,highlighted:`<span class="hljs-keyword">def</span> <span class="hljs-title function_">tokenize</span>(<span class="hljs-params">text</span>):
    pre_tokenize_result = tokenizer._tokenizer.pre_tokenizer.pre_tokenize_str(text)
    pre_tokenized_text = [word <span class="hljs-keyword">for</span> word, offset <span class="hljs-keyword">in</span> pre_tokenize_result]
    encoded_words = [encode_word(word) <span class="hljs-keyword">for</span> word <span class="hljs-keyword">in</span> pre_tokenized_text]
    <span class="hljs-keyword">return</span> <span class="hljs-built_in">sum</span>(encoded_words, [])`}}),Ms=new x({props:{code:'tokenize("This is the Hugging Face course!")',highlighted:'tokenize(<span class="hljs-string">&quot;This is the Hugging Face course!&quot;</span>)'}}),Ls=new x({props:{code:`['Th', '##i', '##s', 'is', 'th', '##e', 'Hugg', '##i', '##n', '##g', 'Fac', '##e', 'c', '##o', '##u', '##r', '##s',
 '##e', '[UNK]']`,highlighted:`[<span class="hljs-string">&#x27;Th&#x27;</span>, <span class="hljs-string">&#x27;##i&#x27;</span>, <span class="hljs-string">&#x27;##s&#x27;</span>, <span class="hljs-string">&#x27;is&#x27;</span>, <span class="hljs-string">&#x27;th&#x27;</span>, <span class="hljs-string">&#x27;##e&#x27;</span>, <span class="hljs-string">&#x27;Hugg&#x27;</span>, <span class="hljs-string">&#x27;##i&#x27;</span>, <span class="hljs-string">&#x27;##n&#x27;</span>, <span class="hljs-string">&#x27;##g&#x27;</span>, <span class="hljs-string">&#x27;Fac&#x27;</span>, <span class="hljs-string">&#x27;##e&#x27;</span>, <span class="hljs-string">&#x27;c&#x27;</span>, <span class="hljs-string">&#x27;##o&#x27;</span>, <span class="hljs-string">&#x27;##u&#x27;</span>, <span class="hljs-string">&#x27;##r&#x27;</span>, <span class="hljs-string">&#x27;##s&#x27;</span>,
 <span class="hljs-string">&#x27;##e&#x27;</span>, <span class="hljs-string">&#x27;[UNK]&#x27;</span>]`}}),{c(){u=i("meta"),_=c(),v=i("h1"),k=i("a"),O=i("span"),m($.$$.fragment),y=c(),B=i("span"),M=l("WordPiece tokenization"),W=c(),m(V.$$.fragment),cs=c(),Ys=i("p"),cl=l("WordPiece l\xE0 m\u1ED9t thu\u1EADt to\xE1n tokenize \u0111\u01B0\u1EE3c Google ph\xE1t tri\u1EC3n \u0111\u1EC3 hu\u1EA5n luy\u1EC7n tr\u01B0\u1EDBc BERT. N\xF3 \u0111\xE3 \u0111\u01B0\u1EE3c t\xE1i s\u1EED d\u1EE5ng trong m\u1ED9t v\xE0i m\xF4 h\xECnh Transformer d\u1EF1a tr\xEAn BERT, nh\u01B0 DistilBERT, MobileBERT, Funnel Transformers, v\xE0 MPNET. N\xF3 kh\xE1 t\u01B0\u01A1ng t\u1EF1 v\u1EDBi BPE v\u1EC1 m\u1EB7t hu\u1EA5n luy\u1EC7n, nh\u01B0ng tokenize th\u1EF1c s\u1EF1 \u0111\u01B0\u1EE3c th\u1EF1c hi\u1EC7n ho\xE0n to\xE0n kh\xE1c."),zt=c(),m(os.$$.fragment),Ct=c(),m(J.$$.fragment),Tt=c(),L=i("h2"),Q=i("a"),kn=i("span"),m(us.$$.fragment),ol=c(),$n=i("span"),ul=l("Thu\u1EADt to\xE1n hu\u1EA5n luy\u1EC7n"),Dt=c(),m(X.$$.fragment),Ot=c(),G=i("p"),ml=l("Gi\u1ED1ng nh\u01B0 BPE, WordPiece b\u1EAFt \u0111\u1EA7u t\u1EEB m\u1ED9t t\u1EEB v\u1EF1ng nh\u1ECF bao g\u1ED3m c\xE1c token \u0111\u1EB7c bi\u1EC7t \u0111\u01B0\u1EE3c s\u1EED d\u1EE5ng b\u1EDFi m\xF4 h\xECnh v\xE0 b\u1EA3ng ch\u1EEF c\xE1i \u0111\u1EA7u ti\xEAn. V\xEC n\xF3 x\xE1c \u0111\u1ECBnh c\xE1c t\u1EEB ph\u1EE5 b\u1EB1ng c\xE1ch th\xEAm ti\u1EC1n t\u1ED1 (nh\u01B0 "),_n=i("code"),gl=l("##"),fl=l(" cho BERT), ban \u0111\u1EA7u m\u1ED7i t\u1EEB \u0111\u01B0\u1EE3c t\xE1ch b\u1EB1ng c\xE1ch th\xEAm ti\u1EC1n t\u1ED1 \u0111\xF3 v\xE0o t\u1EA5t c\u1EA3 c\xE1c k\xFD t\u1EF1 b\xEAn trong t\u1EEB. V\xEC v\u1EADy, v\xED d\u1EE5, "),wn=i("code"),jl=l('"word"'),bl=l(" \u0111\u01B0\u1EE3c chia nh\u01B0 th\u1EBF n\xE0y:"),Nt=c(),m(ms.$$.fragment),Bt=c(),Js=i("p"),dl=l("V\xEC v\u1EADy, b\u1EA3ng ch\u1EEF c\xE1i ch\u1EE9a t\u1EA5t c\u1EA3 c\xE1c k\xED t\u1EF1 xu\u1EA5t hi\u1EC7n \u1EDF \u0111\u1EA7u c\u1EE7a m\u1ED9t t\u1EEB v\xE0 c\xE1c k\xED t\u1EF1 xu\u1EA5t hi\u1EC7n b\xEAn trong c\u1EE7a t\u1EEB \u0111\u01B0\u1EE3c th\xEAm m\u1ED9t ti\u1EC1n t\u1ED1 c\u1EE7a WordPiece ph\xEDa tr\u01B0\u1EDBc."),St=c(),gs=i("p"),vl=l(`Sau \u0111\xF3, m\u1ED9t l\u1EA7n n\u1EEFa, gi\u1ED1ng nh\u01B0 BPE, WordPiece h\u1ECDc c\xE1c quy t\u1EAFc h\u1EE3p nh\u1EA5t. S\u1EF1 kh\xE1c bi\u1EC7t ch\xEDnh l\xE0 c\xE1ch ch\u1ECDn c\u1EB7p \u0111\u01B0\u1EE3c h\u1EE3p nh\u1EA5t. Thay v\xEC ch\u1ECDn c\u1EB7p ph\u1ED5 bi\u1EBFn nh\u1EA5t, WordPiece t\xEDnh \u0111i\u1EC3m cho t\u1EEBng c\u1EB7p, s\u1EED d\u1EE5ng c\xF4ng th\u1EE9c sau:
`),Ht=new _r,Wt=c(),P=i("p"),xl=l("B\u1EB1ng c\xE1ch chia t\u1EA7n su\u1EA5t c\u1EE7a c\u1EB7p cho t\xEDch t\u1EA7n su\u1EA5t c\u1EE7a t\u1EEBng con c\u1EE7a n\xF3, thu\u1EADt to\xE1n \u01B0u ti\xEAn h\u1EE3p nh\u1EA5t c\xE1c c\u1EB7p m\xE0 c\xE1c b\u1ED9 ph\u1EADn ri\xEAng l\u1EBB \xEDt th\u01B0\u1EDDng xuy\xEAn h\u01A1n trong t\u1EEB v\u1EF1ng. V\xED d\u1EE5: n\xF3 s\u1EBD kh\xF4ng nh\u1EA5t thi\u1EBFt ph\u1EA3i h\u1EE3p nh\u1EA5t "),qn=i("code"),kl=l('("un", "##able")'),$l=l(" ngay c\u1EA3 khi c\u1EB7p \u0111\xF3 xu\u1EA5t hi\u1EC7n r\u1EA5t th\u01B0\u1EDDng xuy\xEAn trong t\u1EEB v\u1EF1ng, v\xEC hai c\u1EB7p "),yn=i("code"),_l=l('"un"'),wl=l(" v\xE0 "),En=i("code"),ql=l('"##able"'),yl=l(" m\u1ED7i t\u1EEB c\xF3 th\u1EC3 s\u1EBD xu\u1EA5t hi\u1EC7n b\u1EB1ng nhi\u1EC1u t\u1EEB kh\xE1c v\xE0 c\xF3 t\u1EA7n su\u1EA5t cao. Ng\u01B0\u1EE3c l\u1EA1i, m\u1ED9t c\u1EB7p nh\u01B0 "),Pn=i("code"),El=l('("hu", "##gging")'),Pl=l(" c\xF3 th\u1EC3 s\u1EBD \u0111\u01B0\u1EE3c h\u1EE3p nh\u1EA5t nhanh h\u01A1n (gi\u1EA3 s\u1EED t\u1EEB \u201Chugging\u201D xu\u1EA5t hi\u1EC7n th\u01B0\u1EDDng xuy\xEAn trong t\u1EEB v\u1EF1ng) v\xEC "),zn=i("code"),zl=l('"hu"'),Cl=l(" v\xE0 "),Cn=i("code"),Tl=l('"##gging"'),Dl=l("c\xF3 kh\u1EA3 n\u0103ng \xEDt xu\u1EA5t hi\u1EC7n h\u01A1n v\u1EDBi t\u1EEBng c\xE1 th\u1EC3."),At=c(),Qs=i("p"),Ol=l("H\xE3y c\xF9ng nh\xECn v\xE0o c\xF9ng b\u1ED9 t\u1EF1 v\u1EF1ng ch\xFAng ta s\u1EED d\u1EE5ng cho BPE:"),Vt=c(),m(fs.$$.fragment),Gt=c(),Xs=i("p"),Nl=l("N\xF3 s\u1EBD \u0111\u01B0\u1EE3c chia ra nh\u01B0 sau:"),Kt=c(),m(js.$$.fragment),Ut=c(),E=i("p"),Bl=l("v\xEC v\u1EADy t\u1EEB v\u1EF1ng ban \u0111\u1EA7u s\u1EBD l\xE0 "),Tn=i("code"),Sl=l('["b", "h", "p", "##g", "##n", "##s", "##u"]'),Hl=l(" (n\u1EBFu ta t\u1EA1m qu\xEAn c\xE1c token \u0111\u1EB7c bi\u1EC7t). C\u1EB7p th\u01B0\u1EDDng g\u1EB7p nh\u1EA5t l\xE0 "),Dn=i("code"),Wl=l('("##u", "##g")'),Al=l(" (xu\u1EA5t hi\u1EC7n 20 l\u1EA7n), nh\u01B0ng t\u1EA7n su\u1EA5t xu\u1EA5t hi\u1EC7n ri\xEAng c\u1EE7a "),On=i("code"),Vl=l('"##u"'),Gl=l(" r\u1EA5t cao, v\xEC v\u1EADy \u0111i\u1EC3m c\u1EE7a n\xF3 kh\xF4ng ph\u1EA3i l\xE0 cao nh\u1EA5t (\u0111\xF3 l\xE0 1 / 36). T\u1EA5t c\u1EA3 c\xE1c c\u1EB7p c\xF3 "),Nn=i("code"),Kl=l('"##u"'),Ul=l("th\u1EF1c s\u1EF1 c\xF3 c\xF9ng \u0111i\u1EC3m (1 / 36), v\xEC v\u1EADy \u0111i\u1EC3m t\u1ED1t nh\u1EA5t thu\u1ED9c v\u1EC1 c\u1EB7p "),Bn=i("code"),Fl=l('("##g", "##s")'),Rl=l(" \u2014 c\u1EB7p duy nh\u1EA5t kh\xF4ng c\xF3 "),Sn=i("code"),Ml=l('"##u"'),Ll=l(" \u2014 l\xE0 1 / 20, v\xE0 ph\xE9p h\u1EE3p nh\u1EA5t \u0111\u1EA7u ti\xEAn \u0111\xE3 h\u1ECDc l\xE0 "),Hn=i("code"),Il=l('("##g", "##s") -> ("##gs")'),Yl=l("."),Ft=c(),K=i("p"),Jl=l("L\u01B0u \xFD r\u1EB1ng khi h\u1EE3p nh\u1EA5t, ch\xFAng ta lo\u1EA1i b\u1ECF "),Wn=i("code"),Ql=l("##"),Xl=l(" gi\u1EEFa hai token, v\xEC v\u1EADy ch\xFAng ta th\xEAm "),An=i("code"),Zl=l('"##gs"'),sp=l(" v\xE0o t\u1EEB v\u1EF1ng v\xE0 \xE1p d\u1EE5ng h\u1EE3p nh\u1EA5t trong c\xE1c t\u1EEB c\u1EE7a ng\u1EEF li\u1EC7u:"),Rt=c(),m(bs.$$.fragment),Mt=c(),U=i("p"),np=l("T\u1EA1i th\u1EDDi \u0111i\u1EC3m n\xE0y, "),Vn=i("code"),tp=l('"##u"'),ap=l(" n\u1EB1m trong t\u1EA5t c\u1EA3 c\xE1c c\u1EB7p c\xF3 th\u1EC3 c\xF3, v\xEC v\u1EADy t\u1EA5t c\u1EA3 ch\xFAng \u0111\u1EC1u c\xF3 c\xF9ng \u0111i\u1EC3m. Gi\u1EA3 s\u1EED trong tr\u01B0\u1EDDng h\u1EE3p n\xE0y, c\u1EB7p \u0111\u1EA7u ti\xEAn \u0111\u01B0\u1EE3c h\u1EE3p nh\u1EA5t, v\xEC v\u1EADy "),Gn=i("code"),lp=l('("h", "##u") -> "hu"'),pp=l(". \u0110i\u1EC1u n\xE0y \u0111\u01B0a ch\xFAng ta \u0111\u1EBFn:"),Lt=c(),m(ds.$$.fragment),It=c(),F=i("p"),ep=l("Sau \u0111\xF3, \u0111i\u1EC3m s\u1ED1 t\u1ED1t nh\u1EA5t ti\u1EBFp theo \u0111\u01B0\u1EE3c chia s\u1EBB b\u1EDFi "),Kn=i("code"),ip=l('("hu", "##g")'),rp=l(" and "),Un=i("code"),hp=l('("hu", "##gs")'),cp=l(" (v\u1EDBi 1/15, so v\u1EDBi 1/21 c\u1EE7a c\xE1c c\u1EB7p kh\xE1c), v\xEC v\u1EADy c\u1EB7p \u0111\u1EA7u ti\xEAn c\xF3 \u0111i\u1EC3m l\u1EDBn nh\u1EA5t \u0111\u01B0\u1EE3c h\u1EE3p nh\u1EA5t:"),Yt=c(),m(vs.$$.fragment),Jt=c(),Zs=i("p"),op=l("v\xE0 ti\u1EBFp t\u1EE5c nh\u01B0 v\u1EADy cho \u0111\u1EBFn khi ch\xFAng ta \u0111\u1EA1t \u0111\u01B0\u1EE3c k\xEDch th\u01B0\u1EDBc b\u1ED9 t\u1EEB v\u1EF1ng mong mu\u1ED1n."),Qt=c(),m(Z.$$.fragment),Xt=c(),I=i("h2"),ss=i("a"),Fn=i("span"),m(xs.$$.fragment),up=c(),Rn=i("span"),mp=l("Thu\u1EADt to\xE1n tokenize"),Zt=c(),z=i("p"),gp=l("Tokenize c\u1EE7a WordPiece kh\xE1c BPE \u1EDF ch\u1ED7 WordPiece ch\u1EC9 l\u01B0u t\u1EEB v\u1EF1ng cu\u1ED1i c\xF9ng, kh\xF4ng l\u01B0u c\xE1c quy t\u1EAFc h\u1EE3p nh\u1EA5t \u0111\xE3 h\u1ECDc. B\u1EAFt \u0111\u1EA7u t\u1EEB t\u1EEB c\u1EA7n tokenize, WordPiece t\xECm t\u1EEB con d\xE0i nh\u1EA5t c\xF3 trong t\u1EEB v\u1EF1ng, sau \u0111\xF3 t\xE1ch t\u1EEB \u0111\xF3 ra. V\xED d\u1EE5: n\u1EBFu ch\xFAng ta s\u1EED d\u1EE5ng t\u1EEB v\u1EF1ng \u0111\xE3 h\u1ECDc trong v\xED d\u1EE5 tr\xEAn, \u0111\u1ED1i v\u1EDBi t\u1EEB "),Mn=i("code"),fp=l('"hugs"'),jp=l(" t\u1EEB ph\u1EE5 d\xE0i nh\u1EA5t b\u1EAFt \u0111\u1EA7u t\u1EEB \u0111\u1EA7u m\xE0 b\xEAn trong t\u1EEB v\u1EF1ng l\xE0 "),Ln=i("code"),bp=l('"hug"'),dp=l(", v\xEC v\u1EADy ch\xFAng ta t\xE1ch \u1EDF \u0111\xF3 v\xE0 nh\u1EADn \u0111\u01B0\u1EE3c "),In=i("code"),vp=l('["hug","##s"]'),xp=l(". Sau \u0111\xF3, ch\xFAng ta ti\u1EBFp t\u1EE5c v\u1EDBi "),Yn=i("code"),kp=l('"##s"'),$p=l(", trong t\u1EEB v\u1EF1ng, v\xEC v\u1EADy tokenize c\u1EE7a "),Jn=i("code"),_p=l('"hugs"'),wp=l(" l\xE0  "),Qn=i("code"),qp=l('["hug","##s"]'),yp=l("."),sa=c(),ns=i("p"),Ep=l("V\u1EDBi BPE, ch\xFAng ta s\u1EBD \xE1p d\u1EE5ng c\xE1c ph\xE9p h\u1EE3p nh\u1EA5t \u0111\xE3 h\u1ECDc theo th\u1EE9 t\u1EF1 v\xE0 token \u0111i\u1EC1u n\xE0y th\xE0nh "),Xn=i("code"),Pp=l('["hu", "##gs"]'),zp=l(", do \u0111\xF3, c\xE1ch m\xE3 ho\xE1 s\u1EBD kh\xE1c."),na=c(),w=i("p"),Cp=l("V\xED d\u1EE5 kh\xE1c, h\xE3y xem t\u1EEB "),Zn=i("code"),Tp=l('"bugs"'),Dp=l(" s\u1EBD \u0111\u01B0\u1EE3c tokenize nh\u01B0 th\u1EBF n\xE0o. "),st=i("code"),Op=l('"b"'),Np=l("  l\xE0 t\u1EEB ph\u1EE5 d\xE0i nh\u1EA5t b\u1EAFt \u0111\u1EA7u t\u1EEB \u0111\u1EA7u c\u1EE7a t\u1EEB c\xF3 trong t\u1EEB v\u1EF1ng, v\xEC v\u1EADy ch\xFAng t\xF4i t\xE1ch \u1EDF \u0111\xF3 v\xE0 nh\u1EADn \u0111\u01B0\u1EE3c "),nt=i("code"),Bp=l('["b", "##ugs"]'),Sp=l(". Sau \u0111\xF3, "),tt=i("code"),Hp=l('"##u"'),Wp=l(" l\xE0 t\u1EEB con d\xE0i nh\u1EA5t b\u1EAFt \u0111\u1EA7u \u1EDF \u0111\u1EA7u "),at=i("code"),Ap=l('"##ugs"'),Vp=l(" c\xF3 trong t\u1EEB v\u1EF1ng, v\xEC v\u1EADy ch\xFAng ta t\xE1ch \u1EDF \u0111\xF3 v\xE0 nh\u1EADn \u0111\u01B0\u1EE3c "),lt=i("code"),Gp=l('["b", "##u, "##gs"]'),Kp=l(". Cu\u1ED1i c\xF9ng, "),pt=i("code"),Up=l('"##gs"'),Fp=l(" c\xF3 trong t\u1EEB v\u1EF1ng, v\xEC v\u1EADy danh s\xE1ch cu\u1ED1i c\xF9ng n\xE0y l\xE0 m\xE3 h\xF3a c\u1EE7a "),et=i("code"),Rp=l('"bugs"'),Mp=l("."),ta=c(),q=i("p"),Lp=l("Khi qu\xE1 tr\xECnh tokenize \u0111\u1EBFn giai \u0111o\u1EA1n kh\xF4ng th\u1EC3 t\xECm th\u1EA5y m\u1ED9t t\u1EEB kh\xF3a ph\u1EE5 trong t\u1EEB v\u1EF1ng, to\xE0n b\u1ED9 t\u1EEB \u0111\u01B0\u1EE3c tokenize th\xE0nh kh\xF4ng x\xE1c \u0111\u1ECBnh - v\xEC v\u1EADy, v\xED d\u1EE5: "),it=i("code"),Ip=l('"mug"'),Yp=l(" s\u1EBD \u0111\u01B0\u1EE3c tokenize l\xE0  "),rt=i("code"),Jp=l('["[UNK]"]'),Qp=l(", c\u0169ng nh\u01B0 "),ht=i("code"),Xp=l('"bum"'),Zp=l(" (ngay c\u1EA3 khi ch\xFAng ta c\xF3 th\u1EC3 b\u1EAFt \u0111\u1EA7u b\u1EB1ng "),ct=i("code"),se=l('"b"'),ne=l(" v\xE0 "),ot=i("code"),te=l('"##u"'),ae=l(", "),ut=i("code"),le=l('"##m"'),pe=l(" kh\xF4ng ph\u1EA3i thu\u1ED9c b\u1ED9 t\u1EEB v\u1EF1ng v\xE0 k\u1EBFt qu\u1EA3 tokenize s\u1EBD ch\u1EC9 l\xE0 "),mt=i("code"),ee=l('["[UNK]"]'),ie=l(", kh\xF4ng ph\u1EA3i "),gt=i("code"),re=l('["b", "##u", "[UNK]"]'),he=l("). \u0110\xE2y l\xE0 m\u1ED9t \u0111i\u1EC3m kh\xE1c bi\u1EC7t so v\u1EDBi BPE, ch\u1EC9 ph\xE2n lo\u1EA1i c\xE1c k\xFD t\u1EF1 ri\xEAng l\u1EBB kh\xF4ng c\xF3 trong t\u1EEB v\u1EF1ng l\xE0 kh\xF4ng x\xE1c \u0111\u1ECBnh."),aa=c(),m(ts.$$.fragment),la=c(),Y=i("h2"),as=i("a"),ft=i("span"),m(ks.$$.fragment),ce=c(),jt=i("span"),oe=l("Tri\u1EC3n khai WordPiece"),pa=c(),sn=i("p"),ue=l("B\xE2y gi\u1EDD ch\xFAng ta h\xE3y xem x\xE9t vi\u1EC7c tri\u1EC3n khai thu\u1EADt to\xE1n WordPiece. Gi\u1ED1ng nh\u01B0 v\u1EDBi BPE, \u0111\xE2y ch\u1EC9 l\xE0 ph\u01B0\u01A1ng ph\xE1p s\u01B0 ph\u1EA1m v\xE0 b\u1EA1n s\u1EBD kh\xF4ng th\u1EC3 s\u1EED d\u1EE5ng n\xF3 tr\xEAn m\u1ED9t kho ng\u1EEF li\u1EC7u l\u1EDBn."),ea=c(),nn=i("p"),me=l("Ch\xFAng t\xF4i s\u1EBD s\u1EED d\u1EE5ng c\xF9ng m\u1ED9t kho d\u1EEF li\u1EC7u nh\u01B0 trong v\xED d\u1EE5 BPE:"),ia=c(),m($s.$$.fragment),ra=c(),ls=i("p"),ge=l("\u0110\u1EA7u ti\xEAn, ta c\u1EA7n ti\u1EC1n tokenize kho ng\u1EEF li\u1EC7u th\xE0nh c\xE1c t\u1EEB, V\xEC ta sao ch\xE9p l\u1EA1i WordPiece tokenizer (nh\u01B0 BERT), ta s\u1EBD s\u1EED d\u1EE5ng "),bt=i("code"),fe=l("bert-base-cased"),je=l(" tokenizer cho pre-tokenization:"),ha=c(),m(_s.$$.fragment),ca=c(),tn=i("p"),be=l("sau \u0111\xF3 ta s\u1EBD t\xEDnh t\u1EA7n su\u1EA5t c\u1EE7a m\u1ED7i t\u1EEB trong kho ng\u1EEF li\u1EC7u nh\u01B0 c\xE1ch ta l\xE0m v\u1EDBi pre-tokenization:"),oa=c(),m(ws.$$.fragment),ua=c(),m(qs.$$.fragment),ma=c(),ps=i("p"),de=l("Nh\u01B0 ch\xFAng ta \u0111\xE3 th\u1EA5y tr\u01B0\u1EDBc \u0111\xE2y, b\u1EA3ng ch\u1EEF c\xE1i l\xE0 t\u1EADp h\u1EE3p duy nh\u1EA5t bao g\u1ED3m t\u1EA5t c\u1EA3 c\xE1c ch\u1EEF c\xE1i \u0111\u1EA7u ti\xEAn c\u1EE7a t\u1EEB v\xE0 t\u1EA5t c\u1EA3 c\xE1c ch\u1EEF c\xE1i kh\xE1c xu\u1EA5t hi\u1EC7n trong c\xE1c t\u1EEB c\xF3 ti\u1EC1n t\u1ED1 l\xE0 "),dt=i("code"),ve=l("##"),xe=l(":"),ga=c(),m(ys.$$.fragment),fa=c(),m(Es.$$.fragment),ja=c(),es=i("p"),ke=l("Ta c\u0169ng th\xEAm c\xE1c k\xED t\u1EF1 \u0111\u1EB7c bi\u1EC7t t\u1EEB m\xF4 h\xECnh \u1EDF \u0111\u1EA7u b\u1ED9 t\u1EF1 v\u1EF1ng. Trong tr\u01B0\u1EDDng h\u1EE3p c\u1EE7a BERT, ta c\xF3 danh s\xE1ch "),vt=i("code"),$e=l('["[PAD]", "[UNK]", "[CLS]", "[SEP]", "[MASK]"]'),_e=l(":"),ba=c(),m(Ps.$$.fragment),da=c(),is=i("p"),we=l("Ti\u1EBFp theo, ch\xFAng ta c\u1EA7n t\xE1ch t\u1EEBng t\u1EEB, v\u1EDBi t\u1EA5t c\u1EA3 c\xE1c ch\u1EEF c\xE1i kh\xF4ng ph\u1EA3i l\xE0 ti\u1EC1n t\u1ED1 \u0111\u1EA7u ti\xEAn b\u1EDFi "),xt=i("code"),qe=l("##"),ye=l(":"),va=c(),m(zs.$$.fragment),xa=c(),an=i("p"),Ee=l("B\xE2y gi\u1EDD ch\xFAng ta \u0111\xE3 s\u1EB5n s\xE0ng \u0111\u1EC3 luy\u1EC7n t\u1EADp, h\xE3y vi\u1EBFt m\u1ED9t h\xE0m t\xEDnh \u0111i\u1EC3m c\u1EE7a t\u1EEBng c\u1EB7p. Ch\xFAng ta s\u1EBD c\u1EA7n s\u1EED d\u1EE5ng \u0111i\u1EC1u n\xE0y \u1EDF m\u1ED7i b\u01B0\u1EDBc hu\u1EA5n luy\u1EC7n:"),ka=c(),m(Cs.$$.fragment),$a=c(),ln=i("p"),Pe=l("H\xE3y c\xF9ng nh\xECn v\xE0o m\u1ED9t ph\u1EA7n c\u1EE7a b\u1ED9 t\u1EEB \u0111i\u1EC3n sau l\u1EA7n chia \u0111\u1EA7u:"),_a=c(),m(Ts.$$.fragment),wa=c(),m(Ds.$$.fragment),qa=c(),pn=i("p"),ze=l("Gi\u1EDD th\xEC t\xECm c\u1EB7p c\xF3 \u0111i\u1EC3m cao nh\u1EA5t ch\u1EC9 c\u1EA7n m\u1ED9t v\xF2ng l\u1EAFp nhanh:"),ya=c(),m(Os.$$.fragment),Ea=c(),m(Ns.$$.fragment),Pa=c(),R=i("p"),Ce=l("V\u1EADy quy t\u1EAFc h\u1EE3p nh\u1EA5t \u0111\u1EA7u ti\xEAn l\xE0 "),kt=i("code"),Te=l("('a', '##b') -> 'ab'"),De=l(", v\xE0 ta th\xEAm "),$t=i("code"),Oe=l("'ab'"),Ne=l(" v\xE0o b\u1ED9 t\u1EEB v\u1EF1ng:"),za=c(),m(Bs.$$.fragment),Ca=c(),rs=i("p"),Be=l("Ti\u1EBFp theo, ta c\u1EA7n \xE1p d\u1EE5ng h\u1EE3p nh\u1EA5t trong t\u1EEB \u0111i\u1EC3n "),_t=i("code"),Se=l("splits"),He=l(", H\xE3y c\xF9ng vi\u1EBFt m\u1ED9t h\xE0m cho n\xF3:"),Ta=c(),m(Ss.$$.fragment),Da=c(),en=i("p"),We=l("V\xE0 ta c\xF3 th\u1EC3 th\u1EA5y k\u1EBFt qu\u1EA3 l\u1EA7n h\u1EE3p nh\u1EA5t \u0111\u1EA7u ti\u1EC7n:"),Oa=c(),m(Hs.$$.fragment),Na=c(),m(Ws.$$.fragment),Ba=c(),rn=i("p"),Ae=l("Gi\u1EDD th\xEC ta \u0111\xE3 c\xF3 t\u1EA5t c\u1EA3 nh\u1EEFng g\xEC ta c\u1EA7n \u0111\u1EC3 l\u1EB7p cho \u0111\u1EBFn khi h\u1ECDc h\u1EBFt t\u1EA5t c\u1EA3 c\xE1c h\u1EE3p nh\u1EA5t ta mu\u1ED1n. H\xE3y c\u0169ng h\u01B0\u1EDBng t\u1EDBi b\u1ED9 t\u1EEB v\u1EF1ng c\xF3 k\xEDch th\u01B0\u1EDBc l\xE0 70:"),Sa=c(),m(As.$$.fragment),Ha=c(),hn=i("p"),Ve=l("Gi\u1EDD ta c\xF3 th\u1EC3 nh\xECn v\xE0o b\u1ED9 t\u1EEB \u0111i\u1EC3n \u0111\u01B0\u1EE3c t\u1EA1o ra:"),Wa=c(),m(Vs.$$.fragment),Aa=c(),m(Gs.$$.fragment),Va=c(),cn=i("p"),Ge=l("Nh\u01B0 c\xF3 th\u1EC3 th\u1EA5y, so v\u1EDBi BPE, tokenizer n\xE0y h\u1ECDc c\xE1c ph\u1EA7n c\u1EE7a t\u1EEB nh\u01B0 l\xE0 token nhanh h\u01A1n m\u1ED9t ch\xFAt."),Ga=c(),m(hs.$$.fragment),Ka=c(),on=i("p"),Ke=l("\u0110\u1EC3 tokenize nh\u1EEFng \u0111o\u1EA1n v\u0103n m\u1EDBi, ta ti\u1EC1n tokenize n\xF3, chia nh\u1ECF v\xE0 \xE1p d\u1EE5ng thu\u1EADt to\xE1n tokenize cho t\u1EEBng t\u1EEB. V\u1EADy \u0111\xF3, ch\xFAng ta nh\xECn v\xE0o c\u1EE5m t\u1EEB con d\xE0i nh\u1EA5t b\u1EAFt \u0111\u1EA7u t\u1EEB \u0111\u1EA7u t\u1EEB \u0111\u1EA7u ti\xEAn v\xE0 chia nh\u1ECF n\xF3, sau \u0111\xF3 l\u1EB7p l\u1EA1i quy tr\xECnh v\u1EDBi ph\u1EA7n th\u1EE9 hai, v\xE0 ti\u1EBFp t\u1EE5c cho \u0111\u1EBFn h\u1EBFt t\u1EEB v\xE0 c\xE1c t\u1EEB ti\u1EBFp theo trong v\u0103n b\u1EA3n:"),Ua=c(),m(Ks.$$.fragment),Fa=c(),un=i("p"),Ue=l("H\xE3y c\u0169ng ki\u1EC3m tra tr\xEAn m\u1ED9t t\u1EEB c\xF3 tronng v\xE0 kh\xF4ng c\xF3 trong b\u1ED9 t\u1EEB v\u1EF1ng:"),Ra=c(),m(Us.$$.fragment),Ma=c(),m(Fs.$$.fragment),La=c(),mn=i("p"),Fe=l("Gi\u1EDD h\xE3y c\xF9ng vi\u1EBFt m\u1ED9t h\xE0m tokenize v\u0103n b\u1EA3n:"),Ia=c(),m(Rs.$$.fragment),Ya=c(),gn=i("p"),Re=l("Ta c\xF3 th\u1EC3 th\u1EED tr\xEAn b\u1EA5t k\xEC v\u0103n b\u1EA3n n\xE0o:"),Ja=c(),m(Ms.$$.fragment),Qa=c(),m(Ls.$$.fragment),Xa=c(),fn=i("p"),Me=l("\u0110\xF3 l\xE0 nh\u1EEFng g\xEC ta c\u1EA7n bi\u1EBFt v\u1EC1 thu\u1EADt to\xE1n WordPiece! Ti\u1EBFp theo, ch\xFAng ta s\u1EBD c\xF9ng t\xECm hi\u1EC3u v\u1EC1 Unigram."),this.h()},l(s){const a=wr('[data-svelte="svelte-1phssyn"]',document.head);u=r(a,"META",{name:!0,content:!0}),a.forEach(n),_=o(s),v=r(s,"H1",{class:!0});var Is=h(v);k=r(Is,"A",{id:!0,class:!0,href:!0});var wt=h(k);O=r(wt,"SPAN",{});var qt=h(O);g($.$$.fragment,qt),qt.forEach(n),wt.forEach(n),y=o(Is),B=r(Is,"SPAN",{});var yt=h(B);M=p(yt,"WordPiece tokenization"),yt.forEach(n),Is.forEach(n),W=o(s),g(V.$$.fragment,s),cs=o(s),Ys=r(s,"P",{});var Et=h(Ys);cl=p(Et,"WordPiece l\xE0 m\u1ED9t thu\u1EADt to\xE1n tokenize \u0111\u01B0\u1EE3c Google ph\xE1t tri\u1EC3n \u0111\u1EC3 hu\u1EA5n luy\u1EC7n tr\u01B0\u1EDBc BERT. N\xF3 \u0111\xE3 \u0111\u01B0\u1EE3c t\xE1i s\u1EED d\u1EE5ng trong m\u1ED9t v\xE0i m\xF4 h\xECnh Transformer d\u1EF1a tr\xEAn BERT, nh\u01B0 DistilBERT, MobileBERT, Funnel Transformers, v\xE0 MPNET. N\xF3 kh\xE1 t\u01B0\u01A1ng t\u1EF1 v\u1EDBi BPE v\u1EC1 m\u1EB7t hu\u1EA5n luy\u1EC7n, nh\u01B0ng tokenize th\u1EF1c s\u1EF1 \u0111\u01B0\u1EE3c th\u1EF1c hi\u1EC7n ho\xE0n to\xE0n kh\xE1c."),Et.forEach(n),zt=o(s),g(os.$$.fragment,s),Ct=o(s),g(J.$$.fragment,s),Tt=o(s),L=r(s,"H2",{class:!0});var sl=h(L);Q=r(sl,"A",{id:!0,class:!0,href:!0});var Ie=h(Q);kn=r(Ie,"SPAN",{});var Ye=h(kn);g(us.$$.fragment,Ye),Ye.forEach(n),Ie.forEach(n),ol=o(sl),$n=r(sl,"SPAN",{});var Je=h($n);ul=p(Je,"Thu\u1EADt to\xE1n hu\u1EA5n luy\u1EC7n"),Je.forEach(n),sl.forEach(n),Dt=o(s),g(X.$$.fragment,s),Ot=o(s),G=r(s,"P",{});var jn=h(G);ml=p(jn,"Gi\u1ED1ng nh\u01B0 BPE, WordPiece b\u1EAFt \u0111\u1EA7u t\u1EEB m\u1ED9t t\u1EEB v\u1EF1ng nh\u1ECF bao g\u1ED3m c\xE1c token \u0111\u1EB7c bi\u1EC7t \u0111\u01B0\u1EE3c s\u1EED d\u1EE5ng b\u1EDFi m\xF4 h\xECnh v\xE0 b\u1EA3ng ch\u1EEF c\xE1i \u0111\u1EA7u ti\xEAn. V\xEC n\xF3 x\xE1c \u0111\u1ECBnh c\xE1c t\u1EEB ph\u1EE5 b\u1EB1ng c\xE1ch th\xEAm ti\u1EC1n t\u1ED1 (nh\u01B0 "),_n=r(jn,"CODE",{});var Qe=h(_n);gl=p(Qe,"##"),Qe.forEach(n),fl=p(jn," cho BERT), ban \u0111\u1EA7u m\u1ED7i t\u1EEB \u0111\u01B0\u1EE3c t\xE1ch b\u1EB1ng c\xE1ch th\xEAm ti\u1EC1n t\u1ED1 \u0111\xF3 v\xE0o t\u1EA5t c\u1EA3 c\xE1c k\xFD t\u1EF1 b\xEAn trong t\u1EEB. V\xEC v\u1EADy, v\xED d\u1EE5, "),wn=r(jn,"CODE",{});var Xe=h(wn);jl=p(Xe,'"word"'),Xe.forEach(n),bl=p(jn," \u0111\u01B0\u1EE3c chia nh\u01B0 th\u1EBF n\xE0y:"),jn.forEach(n),Nt=o(s),g(ms.$$.fragment,s),Bt=o(s),Js=r(s,"P",{});var Ze=h(Js);dl=p(Ze,"V\xEC v\u1EADy, b\u1EA3ng ch\u1EEF c\xE1i ch\u1EE9a t\u1EA5t c\u1EA3 c\xE1c k\xED t\u1EF1 xu\u1EA5t hi\u1EC7n \u1EDF \u0111\u1EA7u c\u1EE7a m\u1ED9t t\u1EEB v\xE0 c\xE1c k\xED t\u1EF1 xu\u1EA5t hi\u1EC7n b\xEAn trong c\u1EE7a t\u1EEB \u0111\u01B0\u1EE3c th\xEAm m\u1ED9t ti\u1EC1n t\u1ED1 c\u1EE7a WordPiece ph\xEDa tr\u01B0\u1EDBc."),Ze.forEach(n),St=o(s),gs=r(s,"P",{});var Le=h(gs);vl=p(Le,`Sau \u0111\xF3, m\u1ED9t l\u1EA7n n\u1EEFa, gi\u1ED1ng nh\u01B0 BPE, WordPiece h\u1ECDc c\xE1c quy t\u1EAFc h\u1EE3p nh\u1EA5t. S\u1EF1 kh\xE1c bi\u1EC7t ch\xEDnh l\xE0 c\xE1ch ch\u1ECDn c\u1EB7p \u0111\u01B0\u1EE3c h\u1EE3p nh\u1EA5t. Thay v\xEC ch\u1ECDn c\u1EB7p ph\u1ED5 bi\u1EBFn nh\u1EA5t, WordPiece t\xEDnh \u0111i\u1EC3m cho t\u1EEBng c\u1EB7p, s\u1EED d\u1EE5ng c\xF4ng th\u1EE9c sau:
`),Ht=qr(Le),Le.forEach(n),Wt=o(s),P=r(s,"P",{});var S=h(P);xl=p(S,"B\u1EB1ng c\xE1ch chia t\u1EA7n su\u1EA5t c\u1EE7a c\u1EB7p cho t\xEDch t\u1EA7n su\u1EA5t c\u1EE7a t\u1EEBng con c\u1EE7a n\xF3, thu\u1EADt to\xE1n \u01B0u ti\xEAn h\u1EE3p nh\u1EA5t c\xE1c c\u1EB7p m\xE0 c\xE1c b\u1ED9 ph\u1EADn ri\xEAng l\u1EBB \xEDt th\u01B0\u1EDDng xuy\xEAn h\u01A1n trong t\u1EEB v\u1EF1ng. V\xED d\u1EE5: n\xF3 s\u1EBD kh\xF4ng nh\u1EA5t thi\u1EBFt ph\u1EA3i h\u1EE3p nh\u1EA5t "),qn=r(S,"CODE",{});var si=h(qn);kl=p(si,'("un", "##able")'),si.forEach(n),$l=p(S," ngay c\u1EA3 khi c\u1EB7p \u0111\xF3 xu\u1EA5t hi\u1EC7n r\u1EA5t th\u01B0\u1EDDng xuy\xEAn trong t\u1EEB v\u1EF1ng, v\xEC hai c\u1EB7p "),yn=r(S,"CODE",{});var ni=h(yn);_l=p(ni,'"un"'),ni.forEach(n),wl=p(S," v\xE0 "),En=r(S,"CODE",{});var ti=h(En);ql=p(ti,'"##able"'),ti.forEach(n),yl=p(S," m\u1ED7i t\u1EEB c\xF3 th\u1EC3 s\u1EBD xu\u1EA5t hi\u1EC7n b\u1EB1ng nhi\u1EC1u t\u1EEB kh\xE1c v\xE0 c\xF3 t\u1EA7n su\u1EA5t cao. Ng\u01B0\u1EE3c l\u1EA1i, m\u1ED9t c\u1EB7p nh\u01B0 "),Pn=r(S,"CODE",{});var ai=h(Pn);El=p(ai,'("hu", "##gging")'),ai.forEach(n),Pl=p(S," c\xF3 th\u1EC3 s\u1EBD \u0111\u01B0\u1EE3c h\u1EE3p nh\u1EA5t nhanh h\u01A1n (gi\u1EA3 s\u1EED t\u1EEB \u201Chugging\u201D xu\u1EA5t hi\u1EC7n th\u01B0\u1EDDng xuy\xEAn trong t\u1EEB v\u1EF1ng) v\xEC "),zn=r(S,"CODE",{});var li=h(zn);zl=p(li,'"hu"'),li.forEach(n),Cl=p(S," v\xE0 "),Cn=r(S,"CODE",{});var pi=h(Cn);Tl=p(pi,'"##gging"'),pi.forEach(n),Dl=p(S,"c\xF3 kh\u1EA3 n\u0103ng \xEDt xu\u1EA5t hi\u1EC7n h\u01A1n v\u1EDBi t\u1EEBng c\xE1 th\u1EC3."),S.forEach(n),At=o(s),Qs=r(s,"P",{});var ei=h(Qs);Ol=p(ei,"H\xE3y c\xF9ng nh\xECn v\xE0o c\xF9ng b\u1ED9 t\u1EF1 v\u1EF1ng ch\xFAng ta s\u1EED d\u1EE5ng cho BPE:"),ei.forEach(n),Vt=o(s),g(fs.$$.fragment,s),Gt=o(s),Xs=r(s,"P",{});var ii=h(Xs);Nl=p(ii,"N\xF3 s\u1EBD \u0111\u01B0\u1EE3c chia ra nh\u01B0 sau:"),ii.forEach(n),Kt=o(s),g(js.$$.fragment,s),Ut=o(s),E=r(s,"P",{});var N=h(E);Bl=p(N,"v\xEC v\u1EADy t\u1EEB v\u1EF1ng ban \u0111\u1EA7u s\u1EBD l\xE0 "),Tn=r(N,"CODE",{});var ri=h(Tn);Sl=p(ri,'["b", "h", "p", "##g", "##n", "##s", "##u"]'),ri.forEach(n),Hl=p(N," (n\u1EBFu ta t\u1EA1m qu\xEAn c\xE1c token \u0111\u1EB7c bi\u1EC7t). C\u1EB7p th\u01B0\u1EDDng g\u1EB7p nh\u1EA5t l\xE0 "),Dn=r(N,"CODE",{});var hi=h(Dn);Wl=p(hi,'("##u", "##g")'),hi.forEach(n),Al=p(N," (xu\u1EA5t hi\u1EC7n 20 l\u1EA7n), nh\u01B0ng t\u1EA7n su\u1EA5t xu\u1EA5t hi\u1EC7n ri\xEAng c\u1EE7a "),On=r(N,"CODE",{});var ci=h(On);Vl=p(ci,'"##u"'),ci.forEach(n),Gl=p(N," r\u1EA5t cao, v\xEC v\u1EADy \u0111i\u1EC3m c\u1EE7a n\xF3 kh\xF4ng ph\u1EA3i l\xE0 cao nh\u1EA5t (\u0111\xF3 l\xE0 1 / 36). T\u1EA5t c\u1EA3 c\xE1c c\u1EB7p c\xF3 "),Nn=r(N,"CODE",{});var oi=h(Nn);Kl=p(oi,'"##u"'),oi.forEach(n),Ul=p(N,"th\u1EF1c s\u1EF1 c\xF3 c\xF9ng \u0111i\u1EC3m (1 / 36), v\xEC v\u1EADy \u0111i\u1EC3m t\u1ED1t nh\u1EA5t thu\u1ED9c v\u1EC1 c\u1EB7p "),Bn=r(N,"CODE",{});var ui=h(Bn);Fl=p(ui,'("##g", "##s")'),ui.forEach(n),Rl=p(N," \u2014 c\u1EB7p duy nh\u1EA5t kh\xF4ng c\xF3 "),Sn=r(N,"CODE",{});var mi=h(Sn);Ml=p(mi,'"##u"'),mi.forEach(n),Ll=p(N," \u2014 l\xE0 1 / 20, v\xE0 ph\xE9p h\u1EE3p nh\u1EA5t \u0111\u1EA7u ti\xEAn \u0111\xE3 h\u1ECDc l\xE0 "),Hn=r(N,"CODE",{});var gi=h(Hn);Il=p(gi,'("##g", "##s") -> ("##gs")'),gi.forEach(n),Yl=p(N,"."),N.forEach(n),Ft=o(s),K=r(s,"P",{});var bn=h(K);Jl=p(bn,"L\u01B0u \xFD r\u1EB1ng khi h\u1EE3p nh\u1EA5t, ch\xFAng ta lo\u1EA1i b\u1ECF "),Wn=r(bn,"CODE",{});var fi=h(Wn);Ql=p(fi,"##"),fi.forEach(n),Xl=p(bn," gi\u1EEFa hai token, v\xEC v\u1EADy ch\xFAng ta th\xEAm "),An=r(bn,"CODE",{});var ji=h(An);Zl=p(ji,'"##gs"'),ji.forEach(n),sp=p(bn," v\xE0o t\u1EEB v\u1EF1ng v\xE0 \xE1p d\u1EE5ng h\u1EE3p nh\u1EA5t trong c\xE1c t\u1EEB c\u1EE7a ng\u1EEF li\u1EC7u:"),bn.forEach(n),Rt=o(s),g(bs.$$.fragment,s),Mt=o(s),U=r(s,"P",{});var dn=h(U);np=p(dn,"T\u1EA1i th\u1EDDi \u0111i\u1EC3m n\xE0y, "),Vn=r(dn,"CODE",{});var bi=h(Vn);tp=p(bi,'"##u"'),bi.forEach(n),ap=p(dn," n\u1EB1m trong t\u1EA5t c\u1EA3 c\xE1c c\u1EB7p c\xF3 th\u1EC3 c\xF3, v\xEC v\u1EADy t\u1EA5t c\u1EA3 ch\xFAng \u0111\u1EC1u c\xF3 c\xF9ng \u0111i\u1EC3m. Gi\u1EA3 s\u1EED trong tr\u01B0\u1EDDng h\u1EE3p n\xE0y, c\u1EB7p \u0111\u1EA7u ti\xEAn \u0111\u01B0\u1EE3c h\u1EE3p nh\u1EA5t, v\xEC v\u1EADy "),Gn=r(dn,"CODE",{});var di=h(Gn);lp=p(di,'("h", "##u") -> "hu"'),di.forEach(n),pp=p(dn,". \u0110i\u1EC1u n\xE0y \u0111\u01B0a ch\xFAng ta \u0111\u1EBFn:"),dn.forEach(n),Lt=o(s),g(ds.$$.fragment,s),It=o(s),F=r(s,"P",{});var vn=h(F);ep=p(vn,"Sau \u0111\xF3, \u0111i\u1EC3m s\u1ED1 t\u1ED1t nh\u1EA5t ti\u1EBFp theo \u0111\u01B0\u1EE3c chia s\u1EBB b\u1EDFi "),Kn=r(vn,"CODE",{});var vi=h(Kn);ip=p(vi,'("hu", "##g")'),vi.forEach(n),rp=p(vn," and "),Un=r(vn,"CODE",{});var xi=h(Un);hp=p(xi,'("hu", "##gs")'),xi.forEach(n),cp=p(vn," (v\u1EDBi 1/15, so v\u1EDBi 1/21 c\u1EE7a c\xE1c c\u1EB7p kh\xE1c), v\xEC v\u1EADy c\u1EB7p \u0111\u1EA7u ti\xEAn c\xF3 \u0111i\u1EC3m l\u1EDBn nh\u1EA5t \u0111\u01B0\u1EE3c h\u1EE3p nh\u1EA5t:"),vn.forEach(n),Yt=o(s),g(vs.$$.fragment,s),Jt=o(s),Zs=r(s,"P",{});var ki=h(Zs);op=p(ki,"v\xE0 ti\u1EBFp t\u1EE5c nh\u01B0 v\u1EADy cho \u0111\u1EBFn khi ch\xFAng ta \u0111\u1EA1t \u0111\u01B0\u1EE3c k\xEDch th\u01B0\u1EDBc b\u1ED9 t\u1EEB v\u1EF1ng mong mu\u1ED1n."),ki.forEach(n),Qt=o(s),g(Z.$$.fragment,s),Xt=o(s),I=r(s,"H2",{class:!0});var nl=h(I);ss=r(nl,"A",{id:!0,class:!0,href:!0});var $i=h(ss);Fn=r($i,"SPAN",{});var _i=h(Fn);g(xs.$$.fragment,_i),_i.forEach(n),$i.forEach(n),up=o(nl),Rn=r(nl,"SPAN",{});var wi=h(Rn);mp=p(wi,"Thu\u1EADt to\xE1n tokenize"),wi.forEach(n),nl.forEach(n),Zt=o(s),z=r(s,"P",{});var H=h(z);gp=p(H,"Tokenize c\u1EE7a WordPiece kh\xE1c BPE \u1EDF ch\u1ED7 WordPiece ch\u1EC9 l\u01B0u t\u1EEB v\u1EF1ng cu\u1ED1i c\xF9ng, kh\xF4ng l\u01B0u c\xE1c quy t\u1EAFc h\u1EE3p nh\u1EA5t \u0111\xE3 h\u1ECDc. B\u1EAFt \u0111\u1EA7u t\u1EEB t\u1EEB c\u1EA7n tokenize, WordPiece t\xECm t\u1EEB con d\xE0i nh\u1EA5t c\xF3 trong t\u1EEB v\u1EF1ng, sau \u0111\xF3 t\xE1ch t\u1EEB \u0111\xF3 ra. V\xED d\u1EE5: n\u1EBFu ch\xFAng ta s\u1EED d\u1EE5ng t\u1EEB v\u1EF1ng \u0111\xE3 h\u1ECDc trong v\xED d\u1EE5 tr\xEAn, \u0111\u1ED1i v\u1EDBi t\u1EEB "),Mn=r(H,"CODE",{});var qi=h(Mn);fp=p(qi,'"hugs"'),qi.forEach(n),jp=p(H," t\u1EEB ph\u1EE5 d\xE0i nh\u1EA5t b\u1EAFt \u0111\u1EA7u t\u1EEB \u0111\u1EA7u m\xE0 b\xEAn trong t\u1EEB v\u1EF1ng l\xE0 "),Ln=r(H,"CODE",{});var yi=h(Ln);bp=p(yi,'"hug"'),yi.forEach(n),dp=p(H,", v\xEC v\u1EADy ch\xFAng ta t\xE1ch \u1EDF \u0111\xF3 v\xE0 nh\u1EADn \u0111\u01B0\u1EE3c "),In=r(H,"CODE",{});var Ei=h(In);vp=p(Ei,'["hug","##s"]'),Ei.forEach(n),xp=p(H,". Sau \u0111\xF3, ch\xFAng ta ti\u1EBFp t\u1EE5c v\u1EDBi "),Yn=r(H,"CODE",{});var Pi=h(Yn);kp=p(Pi,'"##s"'),Pi.forEach(n),$p=p(H,", trong t\u1EEB v\u1EF1ng, v\xEC v\u1EADy tokenize c\u1EE7a "),Jn=r(H,"CODE",{});var zi=h(Jn);_p=p(zi,'"hugs"'),zi.forEach(n),wp=p(H," l\xE0  "),Qn=r(H,"CODE",{});var Ci=h(Qn);qp=p(Ci,'["hug","##s"]'),Ci.forEach(n),yp=p(H,"."),H.forEach(n),sa=o(s),ns=r(s,"P",{});var tl=h(ns);Ep=p(tl,"V\u1EDBi BPE, ch\xFAng ta s\u1EBD \xE1p d\u1EE5ng c\xE1c ph\xE9p h\u1EE3p nh\u1EA5t \u0111\xE3 h\u1ECDc theo th\u1EE9 t\u1EF1 v\xE0 token \u0111i\u1EC1u n\xE0y th\xE0nh "),Xn=r(tl,"CODE",{});var Ti=h(Xn);Pp=p(Ti,'["hu", "##gs"]'),Ti.forEach(n),zp=p(tl,", do \u0111\xF3, c\xE1ch m\xE3 ho\xE1 s\u1EBD kh\xE1c."),tl.forEach(n),na=o(s),w=r(s,"P",{});var C=h(w);Cp=p(C,"V\xED d\u1EE5 kh\xE1c, h\xE3y xem t\u1EEB "),Zn=r(C,"CODE",{});var Di=h(Zn);Tp=p(Di,'"bugs"'),Di.forEach(n),Dp=p(C," s\u1EBD \u0111\u01B0\u1EE3c tokenize nh\u01B0 th\u1EBF n\xE0o. "),st=r(C,"CODE",{});var Oi=h(st);Op=p(Oi,'"b"'),Oi.forEach(n),Np=p(C,"  l\xE0 t\u1EEB ph\u1EE5 d\xE0i nh\u1EA5t b\u1EAFt \u0111\u1EA7u t\u1EEB \u0111\u1EA7u c\u1EE7a t\u1EEB c\xF3 trong t\u1EEB v\u1EF1ng, v\xEC v\u1EADy ch\xFAng t\xF4i t\xE1ch \u1EDF \u0111\xF3 v\xE0 nh\u1EADn \u0111\u01B0\u1EE3c "),nt=r(C,"CODE",{});var Ni=h(nt);Bp=p(Ni,'["b", "##ugs"]'),Ni.forEach(n),Sp=p(C,". Sau \u0111\xF3, "),tt=r(C,"CODE",{});var Bi=h(tt);Hp=p(Bi,'"##u"'),Bi.forEach(n),Wp=p(C," l\xE0 t\u1EEB con d\xE0i nh\u1EA5t b\u1EAFt \u0111\u1EA7u \u1EDF \u0111\u1EA7u "),at=r(C,"CODE",{});var Si=h(at);Ap=p(Si,'"##ugs"'),Si.forEach(n),Vp=p(C," c\xF3 trong t\u1EEB v\u1EF1ng, v\xEC v\u1EADy ch\xFAng ta t\xE1ch \u1EDF \u0111\xF3 v\xE0 nh\u1EADn \u0111\u01B0\u1EE3c "),lt=r(C,"CODE",{});var Hi=h(lt);Gp=p(Hi,'["b", "##u, "##gs"]'),Hi.forEach(n),Kp=p(C,". Cu\u1ED1i c\xF9ng, "),pt=r(C,"CODE",{});var Wi=h(pt);Up=p(Wi,'"##gs"'),Wi.forEach(n),Fp=p(C," c\xF3 trong t\u1EEB v\u1EF1ng, v\xEC v\u1EADy danh s\xE1ch cu\u1ED1i c\xF9ng n\xE0y l\xE0 m\xE3 h\xF3a c\u1EE7a "),et=r(C,"CODE",{});var Ai=h(et);Rp=p(Ai,'"bugs"'),Ai.forEach(n),Mp=p(C,"."),C.forEach(n),ta=o(s),q=r(s,"P",{});var T=h(q);Lp=p(T,"Khi qu\xE1 tr\xECnh tokenize \u0111\u1EBFn giai \u0111o\u1EA1n kh\xF4ng th\u1EC3 t\xECm th\u1EA5y m\u1ED9t t\u1EEB kh\xF3a ph\u1EE5 trong t\u1EEB v\u1EF1ng, to\xE0n b\u1ED9 t\u1EEB \u0111\u01B0\u1EE3c tokenize th\xE0nh kh\xF4ng x\xE1c \u0111\u1ECBnh - v\xEC v\u1EADy, v\xED d\u1EE5: "),it=r(T,"CODE",{});var Vi=h(it);Ip=p(Vi,'"mug"'),Vi.forEach(n),Yp=p(T," s\u1EBD \u0111\u01B0\u1EE3c tokenize l\xE0  "),rt=r(T,"CODE",{});var Gi=h(rt);Jp=p(Gi,'["[UNK]"]'),Gi.forEach(n),Qp=p(T,", c\u0169ng nh\u01B0 "),ht=r(T,"CODE",{});var Ki=h(ht);Xp=p(Ki,'"bum"'),Ki.forEach(n),Zp=p(T," (ngay c\u1EA3 khi ch\xFAng ta c\xF3 th\u1EC3 b\u1EAFt \u0111\u1EA7u b\u1EB1ng "),ct=r(T,"CODE",{});var Ui=h(ct);se=p(Ui,'"b"'),Ui.forEach(n),ne=p(T," v\xE0 "),ot=r(T,"CODE",{});var Fi=h(ot);te=p(Fi,'"##u"'),Fi.forEach(n),ae=p(T,", "),ut=r(T,"CODE",{});var Ri=h(ut);le=p(Ri,'"##m"'),Ri.forEach(n),pe=p(T," kh\xF4ng ph\u1EA3i thu\u1ED9c b\u1ED9 t\u1EEB v\u1EF1ng v\xE0 k\u1EBFt qu\u1EA3 tokenize s\u1EBD ch\u1EC9 l\xE0 "),mt=r(T,"CODE",{});var Mi=h(mt);ee=p(Mi,'["[UNK]"]'),Mi.forEach(n),ie=p(T,", kh\xF4ng ph\u1EA3i "),gt=r(T,"CODE",{});var Li=h(gt);re=p(Li,'["b", "##u", "[UNK]"]'),Li.forEach(n),he=p(T,"). \u0110\xE2y l\xE0 m\u1ED9t \u0111i\u1EC3m kh\xE1c bi\u1EC7t so v\u1EDBi BPE, ch\u1EC9 ph\xE2n lo\u1EA1i c\xE1c k\xFD t\u1EF1 ri\xEAng l\u1EBB kh\xF4ng c\xF3 trong t\u1EEB v\u1EF1ng l\xE0 kh\xF4ng x\xE1c \u0111\u1ECBnh."),T.forEach(n),aa=o(s),g(ts.$$.fragment,s),la=o(s),Y=r(s,"H2",{class:!0});var al=h(Y);as=r(al,"A",{id:!0,class:!0,href:!0});var Ii=h(as);ft=r(Ii,"SPAN",{});var Yi=h(ft);g(ks.$$.fragment,Yi),Yi.forEach(n),Ii.forEach(n),ce=o(al),jt=r(al,"SPAN",{});var Ji=h(jt);oe=p(Ji,"Tri\u1EC3n khai WordPiece"),Ji.forEach(n),al.forEach(n),pa=o(s),sn=r(s,"P",{});var Qi=h(sn);ue=p(Qi,"B\xE2y gi\u1EDD ch\xFAng ta h\xE3y xem x\xE9t vi\u1EC7c tri\u1EC3n khai thu\u1EADt to\xE1n WordPiece. Gi\u1ED1ng nh\u01B0 v\u1EDBi BPE, \u0111\xE2y ch\u1EC9 l\xE0 ph\u01B0\u01A1ng ph\xE1p s\u01B0 ph\u1EA1m v\xE0 b\u1EA1n s\u1EBD kh\xF4ng th\u1EC3 s\u1EED d\u1EE5ng n\xF3 tr\xEAn m\u1ED9t kho ng\u1EEF li\u1EC7u l\u1EDBn."),Qi.forEach(n),ea=o(s),nn=r(s,"P",{});var Xi=h(nn);me=p(Xi,"Ch\xFAng t\xF4i s\u1EBD s\u1EED d\u1EE5ng c\xF9ng m\u1ED9t kho d\u1EEF li\u1EC7u nh\u01B0 trong v\xED d\u1EE5 BPE:"),Xi.forEach(n),ia=o(s),g($s.$$.fragment,s),ra=o(s),ls=r(s,"P",{});var ll=h(ls);ge=p(ll,"\u0110\u1EA7u ti\xEAn, ta c\u1EA7n ti\u1EC1n tokenize kho ng\u1EEF li\u1EC7u th\xE0nh c\xE1c t\u1EEB, V\xEC ta sao ch\xE9p l\u1EA1i WordPiece tokenizer (nh\u01B0 BERT), ta s\u1EBD s\u1EED d\u1EE5ng "),bt=r(ll,"CODE",{});var Zi=h(bt);fe=p(Zi,"bert-base-cased"),Zi.forEach(n),je=p(ll," tokenizer cho pre-tokenization:"),ll.forEach(n),ha=o(s),g(_s.$$.fragment,s),ca=o(s),tn=r(s,"P",{});var sr=h(tn);be=p(sr,"sau \u0111\xF3 ta s\u1EBD t\xEDnh t\u1EA7n su\u1EA5t c\u1EE7a m\u1ED7i t\u1EEB trong kho ng\u1EEF li\u1EC7u nh\u01B0 c\xE1ch ta l\xE0m v\u1EDBi pre-tokenization:"),sr.forEach(n),oa=o(s),g(ws.$$.fragment,s),ua=o(s),g(qs.$$.fragment,s),ma=o(s),ps=r(s,"P",{});var pl=h(ps);de=p(pl,"Nh\u01B0 ch\xFAng ta \u0111\xE3 th\u1EA5y tr\u01B0\u1EDBc \u0111\xE2y, b\u1EA3ng ch\u1EEF c\xE1i l\xE0 t\u1EADp h\u1EE3p duy nh\u1EA5t bao g\u1ED3m t\u1EA5t c\u1EA3 c\xE1c ch\u1EEF c\xE1i \u0111\u1EA7u ti\xEAn c\u1EE7a t\u1EEB v\xE0 t\u1EA5t c\u1EA3 c\xE1c ch\u1EEF c\xE1i kh\xE1c xu\u1EA5t hi\u1EC7n trong c\xE1c t\u1EEB c\xF3 ti\u1EC1n t\u1ED1 l\xE0 "),dt=r(pl,"CODE",{});var nr=h(dt);ve=p(nr,"##"),nr.forEach(n),xe=p(pl,":"),pl.forEach(n),ga=o(s),g(ys.$$.fragment,s),fa=o(s),g(Es.$$.fragment,s),ja=o(s),es=r(s,"P",{});var el=h(es);ke=p(el,"Ta c\u0169ng th\xEAm c\xE1c k\xED t\u1EF1 \u0111\u1EB7c bi\u1EC7t t\u1EEB m\xF4 h\xECnh \u1EDF \u0111\u1EA7u b\u1ED9 t\u1EF1 v\u1EF1ng. Trong tr\u01B0\u1EDDng h\u1EE3p c\u1EE7a BERT, ta c\xF3 danh s\xE1ch "),vt=r(el,"CODE",{});var tr=h(vt);$e=p(tr,'["[PAD]", "[UNK]", "[CLS]", "[SEP]", "[MASK]"]'),tr.forEach(n),_e=p(el,":"),el.forEach(n),ba=o(s),g(Ps.$$.fragment,s),da=o(s),is=r(s,"P",{});var il=h(is);we=p(il,"Ti\u1EBFp theo, ch\xFAng ta c\u1EA7n t\xE1ch t\u1EEBng t\u1EEB, v\u1EDBi t\u1EA5t c\u1EA3 c\xE1c ch\u1EEF c\xE1i kh\xF4ng ph\u1EA3i l\xE0 ti\u1EC1n t\u1ED1 \u0111\u1EA7u ti\xEAn b\u1EDFi "),xt=r(il,"CODE",{});var ar=h(xt);qe=p(ar,"##"),ar.forEach(n),ye=p(il,":"),il.forEach(n),va=o(s),g(zs.$$.fragment,s),xa=o(s),an=r(s,"P",{});var lr=h(an);Ee=p(lr,"B\xE2y gi\u1EDD ch\xFAng ta \u0111\xE3 s\u1EB5n s\xE0ng \u0111\u1EC3 luy\u1EC7n t\u1EADp, h\xE3y vi\u1EBFt m\u1ED9t h\xE0m t\xEDnh \u0111i\u1EC3m c\u1EE7a t\u1EEBng c\u1EB7p. Ch\xFAng ta s\u1EBD c\u1EA7n s\u1EED d\u1EE5ng \u0111i\u1EC1u n\xE0y \u1EDF m\u1ED7i b\u01B0\u1EDBc hu\u1EA5n luy\u1EC7n:"),lr.forEach(n),ka=o(s),g(Cs.$$.fragment,s),$a=o(s),ln=r(s,"P",{});var pr=h(ln);Pe=p(pr,"H\xE3y c\xF9ng nh\xECn v\xE0o m\u1ED9t ph\u1EA7n c\u1EE7a b\u1ED9 t\u1EEB \u0111i\u1EC3n sau l\u1EA7n chia \u0111\u1EA7u:"),pr.forEach(n),_a=o(s),g(Ts.$$.fragment,s),wa=o(s),g(Ds.$$.fragment,s),qa=o(s),pn=r(s,"P",{});var er=h(pn);ze=p(er,"Gi\u1EDD th\xEC t\xECm c\u1EB7p c\xF3 \u0111i\u1EC3m cao nh\u1EA5t ch\u1EC9 c\u1EA7n m\u1ED9t v\xF2ng l\u1EAFp nhanh:"),er.forEach(n),ya=o(s),g(Os.$$.fragment,s),Ea=o(s),g(Ns.$$.fragment,s),Pa=o(s),R=r(s,"P",{});var xn=h(R);Ce=p(xn,"V\u1EADy quy t\u1EAFc h\u1EE3p nh\u1EA5t \u0111\u1EA7u ti\xEAn l\xE0 "),kt=r(xn,"CODE",{});var ir=h(kt);Te=p(ir,"('a', '##b') -> 'ab'"),ir.forEach(n),De=p(xn,", v\xE0 ta th\xEAm "),$t=r(xn,"CODE",{});var rr=h($t);Oe=p(rr,"'ab'"),rr.forEach(n),Ne=p(xn," v\xE0o b\u1ED9 t\u1EEB v\u1EF1ng:"),xn.forEach(n),za=o(s),g(Bs.$$.fragment,s),Ca=o(s),rs=r(s,"P",{});var rl=h(rs);Be=p(rl,"Ti\u1EBFp theo, ta c\u1EA7n \xE1p d\u1EE5ng h\u1EE3p nh\u1EA5t trong t\u1EEB \u0111i\u1EC3n "),_t=r(rl,"CODE",{});var hr=h(_t);Se=p(hr,"splits"),hr.forEach(n),He=p(rl,", H\xE3y c\xF9ng vi\u1EBFt m\u1ED9t h\xE0m cho n\xF3:"),rl.forEach(n),Ta=o(s),g(Ss.$$.fragment,s),Da=o(s),en=r(s,"P",{});var cr=h(en);We=p(cr,"V\xE0 ta c\xF3 th\u1EC3 th\u1EA5y k\u1EBFt qu\u1EA3 l\u1EA7n h\u1EE3p nh\u1EA5t \u0111\u1EA7u ti\u1EC7n:"),cr.forEach(n),Oa=o(s),g(Hs.$$.fragment,s),Na=o(s),g(Ws.$$.fragment,s),Ba=o(s),rn=r(s,"P",{});var or=h(rn);Ae=p(or,"Gi\u1EDD th\xEC ta \u0111\xE3 c\xF3 t\u1EA5t c\u1EA3 nh\u1EEFng g\xEC ta c\u1EA7n \u0111\u1EC3 l\u1EB7p cho \u0111\u1EBFn khi h\u1ECDc h\u1EBFt t\u1EA5t c\u1EA3 c\xE1c h\u1EE3p nh\u1EA5t ta mu\u1ED1n. H\xE3y c\u0169ng h\u01B0\u1EDBng t\u1EDBi b\u1ED9 t\u1EEB v\u1EF1ng c\xF3 k\xEDch th\u01B0\u1EDBc l\xE0 70:"),or.forEach(n),Sa=o(s),g(As.$$.fragment,s),Ha=o(s),hn=r(s,"P",{});var ur=h(hn);Ve=p(ur,"Gi\u1EDD ta c\xF3 th\u1EC3 nh\xECn v\xE0o b\u1ED9 t\u1EEB \u0111i\u1EC3n \u0111\u01B0\u1EE3c t\u1EA1o ra:"),ur.forEach(n),Wa=o(s),g(Vs.$$.fragment,s),Aa=o(s),g(Gs.$$.fragment,s),Va=o(s),cn=r(s,"P",{});var mr=h(cn);Ge=p(mr,"Nh\u01B0 c\xF3 th\u1EC3 th\u1EA5y, so v\u1EDBi BPE, tokenizer n\xE0y h\u1ECDc c\xE1c ph\u1EA7n c\u1EE7a t\u1EEB nh\u01B0 l\xE0 token nhanh h\u01A1n m\u1ED9t ch\xFAt."),mr.forEach(n),Ga=o(s),g(hs.$$.fragment,s),Ka=o(s),on=r(s,"P",{});var gr=h(on);Ke=p(gr,"\u0110\u1EC3 tokenize nh\u1EEFng \u0111o\u1EA1n v\u0103n m\u1EDBi, ta ti\u1EC1n tokenize n\xF3, chia nh\u1ECF v\xE0 \xE1p d\u1EE5ng thu\u1EADt to\xE1n tokenize cho t\u1EEBng t\u1EEB. V\u1EADy \u0111\xF3, ch\xFAng ta nh\xECn v\xE0o c\u1EE5m t\u1EEB con d\xE0i nh\u1EA5t b\u1EAFt \u0111\u1EA7u t\u1EEB \u0111\u1EA7u t\u1EEB \u0111\u1EA7u ti\xEAn v\xE0 chia nh\u1ECF n\xF3, sau \u0111\xF3 l\u1EB7p l\u1EA1i quy tr\xECnh v\u1EDBi ph\u1EA7n th\u1EE9 hai, v\xE0 ti\u1EBFp t\u1EE5c cho \u0111\u1EBFn h\u1EBFt t\u1EEB v\xE0 c\xE1c t\u1EEB ti\u1EBFp theo trong v\u0103n b\u1EA3n:"),gr.forEach(n),Ua=o(s),g(Ks.$$.fragment,s),Fa=o(s),un=r(s,"P",{});var fr=h(un);Ue=p(fr,"H\xE3y c\u0169ng ki\u1EC3m tra tr\xEAn m\u1ED9t t\u1EEB c\xF3 tronng v\xE0 kh\xF4ng c\xF3 trong b\u1ED9 t\u1EEB v\u1EF1ng:"),fr.forEach(n),Ra=o(s),g(Us.$$.fragment,s),Ma=o(s),g(Fs.$$.fragment,s),La=o(s),mn=r(s,"P",{});var jr=h(mn);Fe=p(jr,"Gi\u1EDD h\xE3y c\xF9ng vi\u1EBFt m\u1ED9t h\xE0m tokenize v\u0103n b\u1EA3n:"),jr.forEach(n),Ia=o(s),g(Rs.$$.fragment,s),Ya=o(s),gn=r(s,"P",{});var br=h(gn);Re=p(br,"Ta c\xF3 th\u1EC3 th\u1EED tr\xEAn b\u1EA5t k\xEC v\u0103n b\u1EA3n n\xE0o:"),br.forEach(n),Ja=o(s),g(Ms.$$.fragment,s),Qa=o(s),g(Ls.$$.fragment,s),Xa=o(s),fn=r(s,"P",{});var dr=h(fn);Me=p(dr,"\u0110\xF3 l\xE0 nh\u1EEFng g\xEC ta c\u1EA7n bi\u1EBFt v\u1EC1 thu\u1EADt to\xE1n WordPiece! Ti\u1EBFp theo, ch\xFAng ta s\u1EBD c\xF9ng t\xECm hi\u1EC3u v\u1EC1 Unigram."),dr.forEach(n),this.h()},h(){D(u,"name","hf:doc:metadata"),D(u,"content",JSON.stringify(Br)),D(k,"id","wordpiece-tokenization"),D(k,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),D(k,"href","#wordpiece-tokenization"),D(v,"class","relative group"),D(Q,"id","thut-ton-hun-luyn"),D(Q,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),D(Q,"href","#thut-ton-hun-luyn"),D(L,"class","relative group"),Ht.a=null,D(ss,"id","thut-ton-tokenize"),D(ss,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),D(ss,"href","#thut-ton-tokenize"),D(I,"class","relative group"),D(as,"id","trin-khai-wordpiece"),D(as,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),D(as,"href","#trin-khai-wordpiece"),D(Y,"class","relative group")},m(s,a){t(document.head,u),e(s,_,a),e(s,v,a),t(v,k),t(k,O),f($,O,null),t(v,y),t(v,B),t(B,M),e(s,W,a),f(V,s,a),e(s,cs,a),e(s,Ys,a),t(Ys,cl),e(s,zt,a),f(os,s,a),e(s,Ct,a),f(J,s,a),e(s,Tt,a),e(s,L,a),t(L,Q),t(Q,kn),f(us,kn,null),t(L,ol),t(L,$n),t($n,ul),e(s,Dt,a),f(X,s,a),e(s,Ot,a),e(s,G,a),t(G,ml),t(G,_n),t(_n,gl),t(G,fl),t(G,wn),t(wn,jl),t(G,bl),e(s,Nt,a),f(ms,s,a),e(s,Bt,a),e(s,Js,a),t(Js,dl),e(s,St,a),e(s,gs,a),t(gs,vl),Ht.m(vr,gs),e(s,Wt,a),e(s,P,a),t(P,xl),t(P,qn),t(qn,kl),t(P,$l),t(P,yn),t(yn,_l),t(P,wl),t(P,En),t(En,ql),t(P,yl),t(P,Pn),t(Pn,El),t(P,Pl),t(P,zn),t(zn,zl),t(P,Cl),t(P,Cn),t(Cn,Tl),t(P,Dl),e(s,At,a),e(s,Qs,a),t(Qs,Ol),e(s,Vt,a),f(fs,s,a),e(s,Gt,a),e(s,Xs,a),t(Xs,Nl),e(s,Kt,a),f(js,s,a),e(s,Ut,a),e(s,E,a),t(E,Bl),t(E,Tn),t(Tn,Sl),t(E,Hl),t(E,Dn),t(Dn,Wl),t(E,Al),t(E,On),t(On,Vl),t(E,Gl),t(E,Nn),t(Nn,Kl),t(E,Ul),t(E,Bn),t(Bn,Fl),t(E,Rl),t(E,Sn),t(Sn,Ml),t(E,Ll),t(E,Hn),t(Hn,Il),t(E,Yl),e(s,Ft,a),e(s,K,a),t(K,Jl),t(K,Wn),t(Wn,Ql),t(K,Xl),t(K,An),t(An,Zl),t(K,sp),e(s,Rt,a),f(bs,s,a),e(s,Mt,a),e(s,U,a),t(U,np),t(U,Vn),t(Vn,tp),t(U,ap),t(U,Gn),t(Gn,lp),t(U,pp),e(s,Lt,a),f(ds,s,a),e(s,It,a),e(s,F,a),t(F,ep),t(F,Kn),t(Kn,ip),t(F,rp),t(F,Un),t(Un,hp),t(F,cp),e(s,Yt,a),f(vs,s,a),e(s,Jt,a),e(s,Zs,a),t(Zs,op),e(s,Qt,a),f(Z,s,a),e(s,Xt,a),e(s,I,a),t(I,ss),t(ss,Fn),f(xs,Fn,null),t(I,up),t(I,Rn),t(Rn,mp),e(s,Zt,a),e(s,z,a),t(z,gp),t(z,Mn),t(Mn,fp),t(z,jp),t(z,Ln),t(Ln,bp),t(z,dp),t(z,In),t(In,vp),t(z,xp),t(z,Yn),t(Yn,kp),t(z,$p),t(z,Jn),t(Jn,_p),t(z,wp),t(z,Qn),t(Qn,qp),t(z,yp),e(s,sa,a),e(s,ns,a),t(ns,Ep),t(ns,Xn),t(Xn,Pp),t(ns,zp),e(s,na,a),e(s,w,a),t(w,Cp),t(w,Zn),t(Zn,Tp),t(w,Dp),t(w,st),t(st,Op),t(w,Np),t(w,nt),t(nt,Bp),t(w,Sp),t(w,tt),t(tt,Hp),t(w,Wp),t(w,at),t(at,Ap),t(w,Vp),t(w,lt),t(lt,Gp),t(w,Kp),t(w,pt),t(pt,Up),t(w,Fp),t(w,et),t(et,Rp),t(w,Mp),e(s,ta,a),e(s,q,a),t(q,Lp),t(q,it),t(it,Ip),t(q,Yp),t(q,rt),t(rt,Jp),t(q,Qp),t(q,ht),t(ht,Xp),t(q,Zp),t(q,ct),t(ct,se),t(q,ne),t(q,ot),t(ot,te),t(q,ae),t(q,ut),t(ut,le),t(q,pe),t(q,mt),t(mt,ee),t(q,ie),t(q,gt),t(gt,re),t(q,he),e(s,aa,a),f(ts,s,a),e(s,la,a),e(s,Y,a),t(Y,as),t(as,ft),f(ks,ft,null),t(Y,ce),t(Y,jt),t(jt,oe),e(s,pa,a),e(s,sn,a),t(sn,ue),e(s,ea,a),e(s,nn,a),t(nn,me),e(s,ia,a),f($s,s,a),e(s,ra,a),e(s,ls,a),t(ls,ge),t(ls,bt),t(bt,fe),t(ls,je),e(s,ha,a),f(_s,s,a),e(s,ca,a),e(s,tn,a),t(tn,be),e(s,oa,a),f(ws,s,a),e(s,ua,a),f(qs,s,a),e(s,ma,a),e(s,ps,a),t(ps,de),t(ps,dt),t(dt,ve),t(ps,xe),e(s,ga,a),f(ys,s,a),e(s,fa,a),f(Es,s,a),e(s,ja,a),e(s,es,a),t(es,ke),t(es,vt),t(vt,$e),t(es,_e),e(s,ba,a),f(Ps,s,a),e(s,da,a),e(s,is,a),t(is,we),t(is,xt),t(xt,qe),t(is,ye),e(s,va,a),f(zs,s,a),e(s,xa,a),e(s,an,a),t(an,Ee),e(s,ka,a),f(Cs,s,a),e(s,$a,a),e(s,ln,a),t(ln,Pe),e(s,_a,a),f(Ts,s,a),e(s,wa,a),f(Ds,s,a),e(s,qa,a),e(s,pn,a),t(pn,ze),e(s,ya,a),f(Os,s,a),e(s,Ea,a),f(Ns,s,a),e(s,Pa,a),e(s,R,a),t(R,Ce),t(R,kt),t(kt,Te),t(R,De),t(R,$t),t($t,Oe),t(R,Ne),e(s,za,a),f(Bs,s,a),e(s,Ca,a),e(s,rs,a),t(rs,Be),t(rs,_t),t(_t,Se),t(rs,He),e(s,Ta,a),f(Ss,s,a),e(s,Da,a),e(s,en,a),t(en,We),e(s,Oa,a),f(Hs,s,a),e(s,Na,a),f(Ws,s,a),e(s,Ba,a),e(s,rn,a),t(rn,Ae),e(s,Sa,a),f(As,s,a),e(s,Ha,a),e(s,hn,a),t(hn,Ve),e(s,Wa,a),f(Vs,s,a),e(s,Aa,a),f(Gs,s,a),e(s,Va,a),e(s,cn,a),t(cn,Ge),e(s,Ga,a),f(hs,s,a),e(s,Ka,a),e(s,on,a),t(on,Ke),e(s,Ua,a),f(Ks,s,a),e(s,Fa,a),e(s,un,a),t(un,Ue),e(s,Ra,a),f(Us,s,a),e(s,Ma,a),f(Fs,s,a),e(s,La,a),e(s,mn,a),t(mn,Fe),e(s,Ia,a),f(Rs,s,a),e(s,Ya,a),e(s,gn,a),t(gn,Re),e(s,Ja,a),f(Ms,s,a),e(s,Qa,a),f(Ls,s,a),e(s,Xa,a),e(s,fn,a),t(fn,Me),Za=!0},p(s,[a]){const Is={};a&2&&(Is.$$scope={dirty:a,ctx:s}),J.$set(Is);const wt={};a&2&&(wt.$$scope={dirty:a,ctx:s}),X.$set(wt);const qt={};a&2&&(qt.$$scope={dirty:a,ctx:s}),Z.$set(qt);const yt={};a&2&&(yt.$$scope={dirty:a,ctx:s}),ts.$set(yt);const Et={};a&2&&(Et.$$scope={dirty:a,ctx:s}),hs.$set(Et)},i(s){Za||(j($.$$.fragment,s),j(V.$$.fragment,s),j(os.$$.fragment,s),j(J.$$.fragment,s),j(us.$$.fragment,s),j(X.$$.fragment,s),j(ms.$$.fragment,s),j(fs.$$.fragment,s),j(js.$$.fragment,s),j(bs.$$.fragment,s),j(ds.$$.fragment,s),j(vs.$$.fragment,s),j(Z.$$.fragment,s),j(xs.$$.fragment,s),j(ts.$$.fragment,s),j(ks.$$.fragment,s),j($s.$$.fragment,s),j(_s.$$.fragment,s),j(ws.$$.fragment,s),j(qs.$$.fragment,s),j(ys.$$.fragment,s),j(Es.$$.fragment,s),j(Ps.$$.fragment,s),j(zs.$$.fragment,s),j(Cs.$$.fragment,s),j(Ts.$$.fragment,s),j(Ds.$$.fragment,s),j(Os.$$.fragment,s),j(Ns.$$.fragment,s),j(Bs.$$.fragment,s),j(Ss.$$.fragment,s),j(Hs.$$.fragment,s),j(Ws.$$.fragment,s),j(As.$$.fragment,s),j(Vs.$$.fragment,s),j(Gs.$$.fragment,s),j(hs.$$.fragment,s),j(Ks.$$.fragment,s),j(Us.$$.fragment,s),j(Fs.$$.fragment,s),j(Rs.$$.fragment,s),j(Ms.$$.fragment,s),j(Ls.$$.fragment,s),Za=!0)},o(s){b($.$$.fragment,s),b(V.$$.fragment,s),b(os.$$.fragment,s),b(J.$$.fragment,s),b(us.$$.fragment,s),b(X.$$.fragment,s),b(ms.$$.fragment,s),b(fs.$$.fragment,s),b(js.$$.fragment,s),b(bs.$$.fragment,s),b(ds.$$.fragment,s),b(vs.$$.fragment,s),b(Z.$$.fragment,s),b(xs.$$.fragment,s),b(ts.$$.fragment,s),b(ks.$$.fragment,s),b($s.$$.fragment,s),b(_s.$$.fragment,s),b(ws.$$.fragment,s),b(qs.$$.fragment,s),b(ys.$$.fragment,s),b(Es.$$.fragment,s),b(Ps.$$.fragment,s),b(zs.$$.fragment,s),b(Cs.$$.fragment,s),b(Ts.$$.fragment,s),b(Ds.$$.fragment,s),b(Os.$$.fragment,s),b(Ns.$$.fragment,s),b(Bs.$$.fragment,s),b(Ss.$$.fragment,s),b(Hs.$$.fragment,s),b(Ws.$$.fragment,s),b(As.$$.fragment,s),b(Vs.$$.fragment,s),b(Gs.$$.fragment,s),b(hs.$$.fragment,s),b(Ks.$$.fragment,s),b(Us.$$.fragment,s),b(Fs.$$.fragment,s),b(Rs.$$.fragment,s),b(Ms.$$.fragment,s),b(Ls.$$.fragment,s),Za=!1},d(s){n(u),s&&n(_),s&&n(v),d($),s&&n(W),d(V,s),s&&n(cs),s&&n(Ys),s&&n(zt),d(os,s),s&&n(Ct),d(J,s),s&&n(Tt),s&&n(L),d(us),s&&n(Dt),d(X,s),s&&n(Ot),s&&n(G),s&&n(Nt),d(ms,s),s&&n(Bt),s&&n(Js),s&&n(St),s&&n(gs),s&&n(Wt),s&&n(P),s&&n(At),s&&n(Qs),s&&n(Vt),d(fs,s),s&&n(Gt),s&&n(Xs),s&&n(Kt),d(js,s),s&&n(Ut),s&&n(E),s&&n(Ft),s&&n(K),s&&n(Rt),d(bs,s),s&&n(Mt),s&&n(U),s&&n(Lt),d(ds,s),s&&n(It),s&&n(F),s&&n(Yt),d(vs,s),s&&n(Jt),s&&n(Zs),s&&n(Qt),d(Z,s),s&&n(Xt),s&&n(I),d(xs),s&&n(Zt),s&&n(z),s&&n(sa),s&&n(ns),s&&n(na),s&&n(w),s&&n(ta),s&&n(q),s&&n(aa),d(ts,s),s&&n(la),s&&n(Y),d(ks),s&&n(pa),s&&n(sn),s&&n(ea),s&&n(nn),s&&n(ia),d($s,s),s&&n(ra),s&&n(ls),s&&n(ha),d(_s,s),s&&n(ca),s&&n(tn),s&&n(oa),d(ws,s),s&&n(ua),d(qs,s),s&&n(ma),s&&n(ps),s&&n(ga),d(ys,s),s&&n(fa),d(Es,s),s&&n(ja),s&&n(es),s&&n(ba),d(Ps,s),s&&n(da),s&&n(is),s&&n(va),d(zs,s),s&&n(xa),s&&n(an),s&&n(ka),d(Cs,s),s&&n($a),s&&n(ln),s&&n(_a),d(Ts,s),s&&n(wa),d(Ds,s),s&&n(qa),s&&n(pn),s&&n(ya),d(Os,s),s&&n(Ea),d(Ns,s),s&&n(Pa),s&&n(R),s&&n(za),d(Bs,s),s&&n(Ca),s&&n(rs),s&&n(Ta),d(Ss,s),s&&n(Da),s&&n(en),s&&n(Oa),d(Hs,s),s&&n(Na),d(Ws,s),s&&n(Ba),s&&n(rn),s&&n(Sa),d(As,s),s&&n(Ha),s&&n(hn),s&&n(Wa),d(Vs,s),s&&n(Aa),d(Gs,s),s&&n(Va),s&&n(cn),s&&n(Ga),d(hs,s),s&&n(Ka),s&&n(on),s&&n(Ua),d(Ks,s),s&&n(Fa),s&&n(un),s&&n(Ra),d(Us,s),s&&n(Ma),d(Fs,s),s&&n(La),s&&n(mn),s&&n(Ia),d(Rs,s),s&&n(Ya),s&&n(gn),s&&n(Ja),d(Ms,s),s&&n(Qa),d(Ls,s),s&&n(Xa),s&&n(fn)}}}const Br={local:"wordpiece-tokenization",sections:[{local:"thut-ton-hun-luyn",title:"Thu\u1EADt to\xE1n hu\u1EA5n luy\u1EC7n"},{local:"thut-ton-tokenize",title:"Thu\u1EADt to\xE1n tokenize"},{local:"trin-khai-wordpiece",title:"Tri\u1EC3n khai WordPiece"}],title:"WordPiece tokenization"};function Sr(A){return yr(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class Ur extends xr{constructor(u){super();kr(this,u,Sr,Nr,$r,{})}}export{Ur as default,Br as metadata};
