import{S as Th,i as zh,s as Ph,e as h,k as r,w as f,t as l,M as Ch,c as i,d as n,m as o,a as c,x as m,h as p,b as P,G as t,g as e,y as j,q as x,o as k,B as b,v as Dh}from"../../chunks/vendor-hf-doc-builder.js";import{T as bn}from"../../chunks/Tip-hf-doc-builder.js";import{Y as Bh}from"../../chunks/Youtube-hf-doc-builder.js";import{I as Qa}from"../../chunks/IconCopyLink-hf-doc-builder.js";import{C as d}from"../../chunks/CodeBlock-hf-doc-builder.js";import{D as Oh}from"../../chunks/DocNotebookDropdown-hf-doc-builder.js";function Gh(B){let u,_;return{c(){u=h("p"),_=l("\u{1F4A1} Ph\u1EA7n n\xE0y tr\xECnh b\xE0y s\xE2u h\u01A1n v\u1EC1 BPE, \u0111i xa h\u01A1n n\u1EEFa l\xE0 tr\xECnh b\xE0y c\xE1ch tri\u1EC3n khai \u0111\u1EA7y \u0111\u1EE7. B\u1EA1n c\xF3 th\u1EC3 b\u1ECF qua ph\u1EA7n cu\u1ED1i n\u1EBFu b\u1EA1n ch\u1EC9 mu\u1ED1n c\xF3 m\u1ED9t c\xE1i nh\xECn t\u1ED5ng quan chung v\u1EC1 thu\u1EADt to\xE1n tokenize.")},l(g){u=i(g,"P",{});var $=c(u);_=p($,"\u{1F4A1} Ph\u1EA7n n\xE0y tr\xECnh b\xE0y s\xE2u h\u01A1n v\u1EC1 BPE, \u0111i xa h\u01A1n n\u1EEFa l\xE0 tr\xECnh b\xE0y c\xE1ch tri\u1EC3n khai \u0111\u1EA7y \u0111\u1EE7. B\u1EA1n c\xF3 th\u1EC3 b\u1ECF qua ph\u1EA7n cu\u1ED1i n\u1EBFu b\u1EA1n ch\u1EC9 mu\u1ED1n c\xF3 m\u1ED9t c\xE1i nh\xECn t\u1ED5ng quan chung v\u1EC1 thu\u1EADt to\xE1n tokenize."),$.forEach(n)},m(g,$){e(g,u,$),t(u,_)},d(g){g&&n(u)}}}function Nh(B){let u,_,g,$,E;return{c(){u=h("p"),_=l("GPT-2 v\xE0 RoBERTa tokenizer (kh\xE1 gi\u1ED1ng nhau) c\xF3 m\u1ED9t c\xE1ch th\xF4ng minh \u0111\u1EC3 gi\u1EA3i quy\u1EBFt v\u1EA5n \u0111\u1EC1 n\xE0y: ch\xFAng kh\xF4ng xem c\xE1c t\u1EEB \u0111\u01B0\u1EE3c vi\u1EBFt b\u1EB1ng c\xE1c k\xFD t\u1EF1 Unicode m\xE0 l\xE0 c\xE1c byte. B\u1EB1ng c\xE1ch n\xE0y, t\u1EEB v\u1EF1ng c\u01A1 s\u1EDF c\xF3 k\xEDch th\u01B0\u1EDBc nh\u1ECF (256), nh\u01B0ng m\u1ECDi k\xFD t\u1EF1 b\u1EA1n c\xF3 th\u1EC3 ngh\u0129 \u0111\u1EBFn s\u1EBD v\u1EABn \u0111\u01B0\u1EE3c bao g\u1ED3m v\xE0 kh\xF4ng b\u1ECB chuy\u1EC3n \u0111\u1ED5i th\xE0nh token kh\xF4ng x\xE1c \u0111\u1ECBnh. Th\u1EE7 thu\u1EADt n\xE0y \u0111\u01B0\u1EE3c g\u1ECDi l\xE0 "),g=h("em"),$=l("BPE c\u1EA5p byte"),E=l(".")},l(v){u=i(v,"P",{});var y=c(u);_=p(y,"GPT-2 v\xE0 RoBERTa tokenizer (kh\xE1 gi\u1ED1ng nhau) c\xF3 m\u1ED9t c\xE1ch th\xF4ng minh \u0111\u1EC3 gi\u1EA3i quy\u1EBFt v\u1EA5n \u0111\u1EC1 n\xE0y: ch\xFAng kh\xF4ng xem c\xE1c t\u1EEB \u0111\u01B0\u1EE3c vi\u1EBFt b\u1EB1ng c\xE1c k\xFD t\u1EF1 Unicode m\xE0 l\xE0 c\xE1c byte. B\u1EB1ng c\xE1ch n\xE0y, t\u1EEB v\u1EF1ng c\u01A1 s\u1EDF c\xF3 k\xEDch th\u01B0\u1EDBc nh\u1ECF (256), nh\u01B0ng m\u1ECDi k\xFD t\u1EF1 b\u1EA1n c\xF3 th\u1EC3 ngh\u0129 \u0111\u1EBFn s\u1EBD v\u1EABn \u0111\u01B0\u1EE3c bao g\u1ED3m v\xE0 kh\xF4ng b\u1ECB chuy\u1EC3n \u0111\u1ED5i th\xE0nh token kh\xF4ng x\xE1c \u0111\u1ECBnh. Th\u1EE7 thu\u1EADt n\xE0y \u0111\u01B0\u1EE3c g\u1ECDi l\xE0 "),g=i(y,"EM",{});var z=c(g);$=p(z,"BPE c\u1EA5p byte"),z.forEach(n),E=p(y,"."),y.forEach(n)},m(v,y){e(v,u,y),t(u,_),t(u,g),t(g,$),t(u,E)},d(v){v&&n(u)}}}function Hh(B){let u,_,g,$,E;return{c(){u=h("p"),_=l("\u270F\uFE0F "),g=h("strong"),$=l("Gi\u1EDD th\xEC \u0111\u1EBFn l\u01B0\u1EE3t b\u1EA1n!"),E=l(" B\u1EA1n ngh\u0129 b\u01B0\u1EDBc h\u1EE3p nh\u1EA5t ti\u1EBFp theo s\u1EBD l\xE0 g\xEC?")},l(v){u=i(v,"P",{});var y=c(u);_=p(y,"\u270F\uFE0F "),g=i(y,"STRONG",{});var z=c(g);$=p(z,"Gi\u1EDD th\xEC \u0111\u1EBFn l\u01B0\u1EE3t b\u1EA1n!"),z.forEach(n),E=p(y," B\u1EA1n ngh\u0129 b\u01B0\u1EDBc h\u1EE3p nh\u1EA5t ti\u1EBFp theo s\u1EBD l\xE0 g\xEC?"),y.forEach(n)},m(v,y){e(v,u,y),t(u,_),t(u,g),t(g,$),t(u,E)},d(v){v&&n(u)}}}function Sh(B){let u,_,g,$,E,v,y,z;return{c(){u=h("p"),_=l("\u270F\uFE0F "),g=h("strong"),$=l("Gi\u1EDD t\u1EDBi l\u01B0\u1EE3t b\u1EA1n!"),E=l(" B\u1EA1n ngh\u0129 r\u1EB1ng "),v=h("code"),y=l('"unhug"'),z=l(" s\u1EBD \u0111\u01B0\u1EE3c tokenize nh\u01B0 th\u1EBF n\xE0o?")},l(I){u=i(I,"P",{});var O=c(u);_=p(O,"\u270F\uFE0F "),g=i(O,"STRONG",{});var S=c(g);$=p(S,"Gi\u1EDD t\u1EDBi l\u01B0\u1EE3t b\u1EA1n!"),S.forEach(n),E=p(O," B\u1EA1n ngh\u0129 r\u1EB1ng "),v=i(O,"CODE",{});var is=c(v);y=p(is,'"unhug"'),is.forEach(n),z=p(O," s\u1EBD \u0111\u01B0\u1EE3c tokenize nh\u01B0 th\u1EBF n\xE0o?"),O.forEach(n)},m(I,O){e(I,u,O),t(u,_),t(u,g),t(g,$),t(u,E),t(u,v),t(v,y),t(u,z)},d(I){I&&n(u)}}}function Ah(B){let u,_,g,$,E;return{c(){u=h("p"),_=l("\u{1F4A1} S\u1EED d\u1EE5ng "),g=h("code"),$=l("train_new_from_iterator()"),E=l(" tr\xEAn c\xF9ng kho ng\u1EEF li\u1EC7u s\u1EBD kh\xF4ng mang v\u1EC1 k\u1EBFt qu\u1EA3 kho ng\u1EEF li\u1EC7u y h\u1EC7t. \u0110\xF3 l\xE0 b\u1EDFi khi c\xF3 s\u1EF1 l\u1EF1a ch\u1ECDn v\u1EC1 c\u1EB7p c\xF3 t\u1EA7n su\u1EA5t cao nh\u1EA5t, ta \u0111\xE3 ch\u1ECDn c\xE1i \u0111\u1EA7u ti\xEAn xu\u1EA5t hi\u1EC7n, trong khi th\u01B0 vi\u1EC7n \u{1F917} Tokenizers ch\u1ECDn c\xE1i \u0111\u1EA7u ti\xEAn d\u1EF1a tr\xEAn ID b\xEAn trong c\u1EE7a n\xF3.")},l(v){u=i(v,"P",{});var y=c(u);_=p(y,"\u{1F4A1} S\u1EED d\u1EE5ng "),g=i(y,"CODE",{});var z=c(g);$=p(z,"train_new_from_iterator()"),z.forEach(n),E=p(y," tr\xEAn c\xF9ng kho ng\u1EEF li\u1EC7u s\u1EBD kh\xF4ng mang v\u1EC1 k\u1EBFt qu\u1EA3 kho ng\u1EEF li\u1EC7u y h\u1EC7t. \u0110\xF3 l\xE0 b\u1EDFi khi c\xF3 s\u1EF1 l\u1EF1a ch\u1ECDn v\u1EC1 c\u1EB7p c\xF3 t\u1EA7n su\u1EA5t cao nh\u1EA5t, ta \u0111\xE3 ch\u1ECDn c\xE1i \u0111\u1EA7u ti\xEAn xu\u1EA5t hi\u1EC7n, trong khi th\u01B0 vi\u1EC7n \u{1F917} Tokenizers ch\u1ECDn c\xE1i \u0111\u1EA7u ti\xEAn d\u1EF1a tr\xEAn ID b\xEAn trong c\u1EE7a n\xF3."),y.forEach(n)},m(v,y){e(v,u,y),t(u,_),t(u,g),t(g,$),t(u,E)},d(v){v&&n(u)}}}function Vh(B){let u,_;return{c(){u=h("p"),_=l("\u26A0\uFE0F C\xE1c tri\u1EC3n khai c\u1EE7a ch\xFAng ta s\u1EBD g\u1EB7p l\u1ED7i n\u1EBFu c\xF3 nh\u1EEFng k\xED t\u1EF1 v\xF4 danh v\xEC ch\xFAng ta \u0111\xE3 kh\xF4ng l\xE0m g\xEC \u0111\u1EC3 x\u1EED l\xFD ch\xFAng. GPT-2 kh\xF4ng th\u1EF1c s\u1EF1 c\xF3 nh\u1EEFng token v\xF4 danh (kh\xF4ng th\u1EC3 c\xF3 k\xED t\u1EF1 v\xF4 danh khi s\u1EED d\u1EE5ng BPE c\u1EA5p byte), nh\u01B0ng n\xF3 c\xF3 th\u1EC3 x\u1EA3y ra \u1EDF \u0111\xE2y v\xEC ta kh\xF4ng bao g\u1ED3m t\u1EA5t c\u1EA3 c\xE1c byte c\xF3 th\u1EC3 c\xF3 trong b\u1ED9 t\u1EEB v\u1EF1ng g\u1ED1c. Kh\xEDa c\u1EA1nh n\xE0y c\u1EE7a BPE n\u1EB1m ngo\xE0i ph\u1EA1m vi ph\u1EA7n n\xE0y, n\xEAn ch\xFAng t\xF4i s\u1EBD kh\xF4ng \u0111i sau v\xE0o chi ti\u1EBFt.")},l(g){u=i(g,"P",{});var $=c(u);_=p($,"\u26A0\uFE0F C\xE1c tri\u1EC3n khai c\u1EE7a ch\xFAng ta s\u1EBD g\u1EB7p l\u1ED7i n\u1EBFu c\xF3 nh\u1EEFng k\xED t\u1EF1 v\xF4 danh v\xEC ch\xFAng ta \u0111\xE3 kh\xF4ng l\xE0m g\xEC \u0111\u1EC3 x\u1EED l\xFD ch\xFAng. GPT-2 kh\xF4ng th\u1EF1c s\u1EF1 c\xF3 nh\u1EEFng token v\xF4 danh (kh\xF4ng th\u1EC3 c\xF3 k\xED t\u1EF1 v\xF4 danh khi s\u1EED d\u1EE5ng BPE c\u1EA5p byte), nh\u01B0ng n\xF3 c\xF3 th\u1EC3 x\u1EA3y ra \u1EDF \u0111\xE2y v\xEC ta kh\xF4ng bao g\u1ED3m t\u1EA5t c\u1EA3 c\xE1c byte c\xF3 th\u1EC3 c\xF3 trong b\u1ED9 t\u1EEB v\u1EF1ng g\u1ED1c. Kh\xEDa c\u1EA1nh n\xE0y c\u1EE7a BPE n\u1EB1m ngo\xE0i ph\u1EA1m vi ph\u1EA7n n\xE0y, n\xEAn ch\xFAng t\xF4i s\u1EBD kh\xF4ng \u0111i sau v\xE0o chi ti\u1EBFt."),$.forEach(n)},m(g,$){e(g,u,$),t(u,_)},d(g){g&&n(u)}}}function Rh(B){let u,_,g,$,E,v,y,z,I,O,S,is,Ms,Wa,kt,cs,bt,K,$t,L,M,$n,rs,Ya,dn,Ja,dt,Qs,Za,vt,os,qt,Q,Xa,vn,sl,nl,_t,W,yt,Y,tl,qn,al,ll,wt,Ws,pl,Et,Ys,el,Tt,us,zt,C,hl,_n,il,cl,yn,rl,ol,wn,ul,gl,En,fl,ml,Tn,jl,xl,Pt,gs,Ct,T,kl,zn,bl,$l,Pn,dl,vl,Cn,ql,_l,Dn,yl,wl,Bn,El,Tl,On,zl,Pl,Gn,Cl,Dl,Dt,A,Bl,Nn,Ol,Gl,Hn,Nl,Hl,Bt,fs,Ot,G,Sl,Sn,Al,Vl,An,Rl,Il,Vn,Ll,Ul,Gt,ms,Nt,V,Fl,Rn,Kl,Ml,In,Ql,Wl,Ht,js,St,Js,Yl,At,J,Vt,U,Z,Ln,xs,Jl,Un,Zl,Rt,Zs,Xl,It,N,Fn,sp,np,Kn,tp,ap,Mn,lp,pp,Qn,ep,Lt,Xs,hp,Ut,ks,Ft,q,ip,Wn,cp,rp,Yn,op,up,Jn,gp,fp,Zn,mp,jp,Xn,xp,kp,st,bp,$p,nt,dp,vp,tt,qp,_p,at,yp,wp,lt,Ep,Tp,pt,zp,Pp,et,Cp,Dp,Kt,X,Mt,F,ss,ht,bs,Bp,it,Op,Qt,sn,Gp,Wt,nn,Np,Yt,$s,Jt,ns,Hp,ct,Sp,Ap,Zt,ds,Xt,tn,Vp,sa,vs,na,qs,ta,an,Rp,aa,_s,la,ys,pa,ts,Ip,rt,Lp,Up,ea,ws,ha,ln,Fp,ia,Es,ca,pn,Kp,ra,Ts,oa,en,Mp,ua,zs,ga,Ps,fa,hn,Qp,ma,Cs,ja,Ds,xa,R,Wp,ot,Yp,Jp,ut,Zp,Xp,ka,Bs,ba,as,se,gt,ne,te,$a,Os,da,cn,ae,va,Gs,qa,Ns,_a,rn,le,ya,Hs,wa,on,pe,Ea,Ss,Ta,As,za,un,ee,Pa,Vs,Ca,Rs,Da,ls,Ba,gn,he,Oa,Is,Ga,fn,ie,Na,Ls,Ha,Us,Sa,ps,Aa,mn,ce,Va;return v=new Qa({}),S=new Oh({props:{classNames:"absolute z-10 right-0 top-0",options:[{label:"Google Colab",value:"https://colab.research.google.com/github/huggingface/notebooks/blob/master/course/vi/chapter6/section5.ipynb"},{label:"Aws Studio",value:"https://studiolab.sagemaker.aws/import/github/huggingface/notebooks/blob/master/course/vi/chapter6/section5.ipynb"}]}}),cs=new Bh({props:{id:"HEikzVL-lZU"}}),K=new bn({props:{$$slots:{default:[Gh]},$$scope:{ctx:B}}}),rs=new Qa({}),os=new d({props:{code:'"hug", "pug", "pun", "bun", "hugs"',highlighted:'<span class="hljs-string">&quot;hug&quot;</span>, <span class="hljs-string">&quot;pug&quot;</span>, <span class="hljs-string">&quot;pun&quot;</span>, <span class="hljs-string">&quot;bun&quot;</span>, <span class="hljs-string">&quot;hugs&quot;</span>'}}),W=new bn({props:{$$slots:{default:[Nh]},$$scope:{ctx:B}}}),us=new d({props:{code:'("hug", 10), ("pug", 5), ("pun", 12), ("bun", 4), ("hugs", 5)',highlighted:'(<span class="hljs-string">&quot;hug&quot;</span><span class="hljs-punctuation">,</span> <span class="hljs-number">10</span>)<span class="hljs-punctuation">,</span> (<span class="hljs-string">&quot;pug&quot;</span><span class="hljs-punctuation">,</span> <span class="hljs-number">5</span>)<span class="hljs-punctuation">,</span> (<span class="hljs-string">&quot;pun&quot;</span><span class="hljs-punctuation">,</span> <span class="hljs-number">12</span>)<span class="hljs-punctuation">,</span> (<span class="hljs-string">&quot;bun&quot;</span><span class="hljs-punctuation">,</span> <span class="hljs-number">4</span>)<span class="hljs-punctuation">,</span> (<span class="hljs-string">&quot;hugs&quot;</span><span class="hljs-punctuation">,</span> <span class="hljs-number">5</span>)'}}),gs=new d({props:{code:'("h" "u" "g", 10), ("p" "u" "g", 5), ("p" "u" "n", 12), ("b" "u" "n", 4), ("h" "u" "g" "s", 5)',highlighted:'(<span class="hljs-string">&quot;h&quot;</span> <span class="hljs-string">&quot;u&quot;</span> <span class="hljs-string">&quot;g&quot;</span><span class="hljs-punctuation">,</span> <span class="hljs-number">10</span>)<span class="hljs-punctuation">,</span> (<span class="hljs-string">&quot;p&quot;</span> <span class="hljs-string">&quot;u&quot;</span> <span class="hljs-string">&quot;g&quot;</span><span class="hljs-punctuation">,</span> <span class="hljs-number">5</span>)<span class="hljs-punctuation">,</span> (<span class="hljs-string">&quot;p&quot;</span> <span class="hljs-string">&quot;u&quot;</span> <span class="hljs-string">&quot;n&quot;</span><span class="hljs-punctuation">,</span> <span class="hljs-number">12</span>)<span class="hljs-punctuation">,</span> (<span class="hljs-string">&quot;b&quot;</span> <span class="hljs-string">&quot;u&quot;</span> <span class="hljs-string">&quot;n&quot;</span><span class="hljs-punctuation">,</span> <span class="hljs-number">4</span>)<span class="hljs-punctuation">,</span> (<span class="hljs-string">&quot;h&quot;</span> <span class="hljs-string">&quot;u&quot;</span> <span class="hljs-string">&quot;g&quot;</span> <span class="hljs-string">&quot;s&quot;</span><span class="hljs-punctuation">,</span> <span class="hljs-number">5</span>)'}}),fs=new d({props:{code:`Vocabulary: ["b", "g", "h", "n", "p", "s", "u", "ug"]
Corpus: ("h" "ug", 10), ("p" "ug", 5), ("p" "u" "n", 12), ("b" "u" "n", 4), ("h" "ug" "s", 5)`,highlighted:`<span class="hljs-symbol">Vocabulary:</span> [<span class="hljs-string">&quot;b&quot;</span>, <span class="hljs-string">&quot;g&quot;</span>, <span class="hljs-string">&quot;h&quot;</span>, <span class="hljs-string">&quot;n&quot;</span>, <span class="hljs-string">&quot;p&quot;</span>, <span class="hljs-string">&quot;s&quot;</span>, <span class="hljs-string">&quot;u&quot;</span>, <span class="hljs-string">&quot;ug&quot;</span>]
<span class="hljs-symbol">Corpus:</span> (<span class="hljs-string">&quot;h&quot;</span> <span class="hljs-string">&quot;ug&quot;</span>, <span class="hljs-number">10</span>), (<span class="hljs-string">&quot;p&quot;</span> <span class="hljs-string">&quot;ug&quot;</span>, <span class="hljs-number">5</span>), (<span class="hljs-string">&quot;p&quot;</span> <span class="hljs-string">&quot;u&quot;</span> <span class="hljs-string">&quot;n&quot;</span>, <span class="hljs-number">12</span>), (<span class="hljs-string">&quot;b&quot;</span> <span class="hljs-string">&quot;u&quot;</span> <span class="hljs-string">&quot;n&quot;</span>, <span class="hljs-number">4</span>), (<span class="hljs-string">&quot;h&quot;</span> <span class="hljs-string">&quot;ug&quot;</span> <span class="hljs-string">&quot;s&quot;</span>, <span class="hljs-number">5</span>)`}}),ms=new d({props:{code:`Vocabulary: ["b", "g", "h", "n", "p", "s", "u", "ug", "un"]
Corpus: ("h" "ug", 10), ("p" "ug", 5), ("p" "un", 12), ("b" "un", 4), ("h" "ug" "s", 5)`,highlighted:`<span class="hljs-symbol">Vocabulary:</span> [<span class="hljs-string">&quot;b&quot;</span>, <span class="hljs-string">&quot;g&quot;</span>, <span class="hljs-string">&quot;h&quot;</span>, <span class="hljs-string">&quot;n&quot;</span>, <span class="hljs-string">&quot;p&quot;</span>, <span class="hljs-string">&quot;s&quot;</span>, <span class="hljs-string">&quot;u&quot;</span>, <span class="hljs-string">&quot;ug&quot;</span>, <span class="hljs-string">&quot;un&quot;</span>]
<span class="hljs-symbol">Corpus:</span> (<span class="hljs-string">&quot;h&quot;</span> <span class="hljs-string">&quot;ug&quot;</span>, <span class="hljs-number">10</span>), (<span class="hljs-string">&quot;p&quot;</span> <span class="hljs-string">&quot;ug&quot;</span>, <span class="hljs-number">5</span>), (<span class="hljs-string">&quot;p&quot;</span> <span class="hljs-string">&quot;un&quot;</span>, <span class="hljs-number">12</span>), (<span class="hljs-string">&quot;b&quot;</span> <span class="hljs-string">&quot;un&quot;</span>, <span class="hljs-number">4</span>), (<span class="hljs-string">&quot;h&quot;</span> <span class="hljs-string">&quot;ug&quot;</span> <span class="hljs-string">&quot;s&quot;</span>, <span class="hljs-number">5</span>)`}}),js=new d({props:{code:`Vocabulary: ["b", "g", "h", "n", "p", "s", "u", "ug", "un", "hug"]
Corpus: ("hug", 10), ("p" "ug", 5), ("p" "un", 12), ("b" "un", 4), ("hug" "s", 5)`,highlighted:`<span class="hljs-symbol">Vocabulary:</span> [<span class="hljs-string">&quot;b&quot;</span>, <span class="hljs-string">&quot;g&quot;</span>, <span class="hljs-string">&quot;h&quot;</span>, <span class="hljs-string">&quot;n&quot;</span>, <span class="hljs-string">&quot;p&quot;</span>, <span class="hljs-string">&quot;s&quot;</span>, <span class="hljs-string">&quot;u&quot;</span>, <span class="hljs-string">&quot;ug&quot;</span>, <span class="hljs-string">&quot;un&quot;</span>, <span class="hljs-string">&quot;hug&quot;</span>]
<span class="hljs-symbol">Corpus:</span> (<span class="hljs-string">&quot;hug&quot;</span>, <span class="hljs-number">10</span>), (<span class="hljs-string">&quot;p&quot;</span> <span class="hljs-string">&quot;ug&quot;</span>, <span class="hljs-number">5</span>), (<span class="hljs-string">&quot;p&quot;</span> <span class="hljs-string">&quot;un&quot;</span>, <span class="hljs-number">12</span>), (<span class="hljs-string">&quot;b&quot;</span> <span class="hljs-string">&quot;un&quot;</span>, <span class="hljs-number">4</span>), (<span class="hljs-string">&quot;hug&quot;</span> <span class="hljs-string">&quot;s&quot;</span>, <span class="hljs-number">5</span>)`}}),J=new bn({props:{$$slots:{default:[Hh]},$$scope:{ctx:B}}}),xs=new Qa({}),ks=new d({props:{code:`("u", "g") -> "ug"
("u", "n") -> "un"
("h", "ug") -> "hug"`,highlighted:`<span class="hljs-function"><span class="hljs-params">(<span class="hljs-string">&quot;u&quot;</span>, <span class="hljs-string">&quot;g&quot;</span>)</span> -&gt;</span> <span class="hljs-string">&quot;ug&quot;</span>
<span class="hljs-function"><span class="hljs-params">(<span class="hljs-string">&quot;u&quot;</span>, <span class="hljs-string">&quot;n&quot;</span>)</span> -&gt;</span> <span class="hljs-string">&quot;un&quot;</span>
<span class="hljs-function"><span class="hljs-params">(<span class="hljs-string">&quot;h&quot;</span>, <span class="hljs-string">&quot;ug&quot;</span>)</span> -&gt;</span> <span class="hljs-string">&quot;hug&quot;</span>`}}),X=new bn({props:{$$slots:{default:[Sh]},$$scope:{ctx:B}}}),bs=new Qa({}),$s=new d({props:{code:`corpus = [
    "This is the Hugging Face Course.",
    "This chapter is about tokenization.",
    "This section shows several tokenizer algorithms.",
    "Hopefully, you will be able to understand how they are trained and generate tokens.",
]`,highlighted:`corpus = [
    <span class="hljs-string">&quot;This is the Hugging Face Course.&quot;</span>,
    <span class="hljs-string">&quot;This chapter is about tokenization.&quot;</span>,
    <span class="hljs-string">&quot;This section shows several tokenizer algorithms.&quot;</span>,
    <span class="hljs-string">&quot;Hopefully, you will be able to understand how they are trained and generate tokens.&quot;</span>,
]`}}),ds=new d({props:{code:`from transformers import AutoTokenizer

tokenizer = AutoTokenizer.from_pretrained("gpt2")`,highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoTokenizer

tokenizer = AutoTokenizer.from_pretrained(<span class="hljs-string">&quot;gpt2&quot;</span>)`}}),vs=new d({props:{code:`from collections import defaultdict

word_freqs = defaultdict(int)

for text in corpus:
    words_with_offsets = tokenizer.backend_tokenizer.pre_tokenizer.pre_tokenize_str(text)
    new_words = [word for word, offset in words_with_offsets]
    for word in new_words:
        word_freqs[word] += 1

print(word_freqs)`,highlighted:`<span class="hljs-keyword">from</span> collections <span class="hljs-keyword">import</span> defaultdict

word_freqs = defaultdict(<span class="hljs-built_in">int</span>)

<span class="hljs-keyword">for</span> text <span class="hljs-keyword">in</span> corpus:
    words_with_offsets = tokenizer.backend_tokenizer.pre_tokenizer.pre_tokenize_str(text)
    new_words = [word <span class="hljs-keyword">for</span> word, offset <span class="hljs-keyword">in</span> words_with_offsets]
    <span class="hljs-keyword">for</span> word <span class="hljs-keyword">in</span> new_words:
        word_freqs[word] += <span class="hljs-number">1</span>

<span class="hljs-built_in">print</span>(word_freqs)`}}),qs=new d({props:{code:`defaultdict(int, {'This': 3, '\u0120is': 2, '\u0120the': 1, '\u0120Hugging': 1, '\u0120Face': 1, '\u0120Course': 1, '.': 4, '\u0120chapter': 1,
    '\u0120about': 1, '\u0120tokenization': 1, '\u0120section': 1, '\u0120shows': 1, '\u0120several': 1, '\u0120tokenizer': 1, '\u0120algorithms': 1,
    'Hopefully': 1, ',': 1, '\u0120you': 1, '\u0120will': 1, '\u0120be': 1, '\u0120able': 1, '\u0120to': 1, '\u0120understand': 1, '\u0120how': 1,
    '\u0120they': 1, '\u0120are': 1, '\u0120trained': 1, '\u0120and': 1, '\u0120generate': 1, '\u0120tokens': 1})`,highlighted:`defaultdict(<span class="hljs-built_in">int</span>, {<span class="hljs-string">&#x27;This&#x27;</span>: <span class="hljs-number">3</span>, <span class="hljs-string">&#x27;\u0120is&#x27;</span>: <span class="hljs-number">2</span>, <span class="hljs-string">&#x27;\u0120the&#x27;</span>: <span class="hljs-number">1</span>, <span class="hljs-string">&#x27;\u0120Hugging&#x27;</span>: <span class="hljs-number">1</span>, <span class="hljs-string">&#x27;\u0120Face&#x27;</span>: <span class="hljs-number">1</span>, <span class="hljs-string">&#x27;\u0120Course&#x27;</span>: <span class="hljs-number">1</span>, <span class="hljs-string">&#x27;.&#x27;</span>: <span class="hljs-number">4</span>, <span class="hljs-string">&#x27;\u0120chapter&#x27;</span>: <span class="hljs-number">1</span>,
    <span class="hljs-string">&#x27;\u0120about&#x27;</span>: <span class="hljs-number">1</span>, <span class="hljs-string">&#x27;\u0120tokenization&#x27;</span>: <span class="hljs-number">1</span>, <span class="hljs-string">&#x27;\u0120section&#x27;</span>: <span class="hljs-number">1</span>, <span class="hljs-string">&#x27;\u0120shows&#x27;</span>: <span class="hljs-number">1</span>, <span class="hljs-string">&#x27;\u0120several&#x27;</span>: <span class="hljs-number">1</span>, <span class="hljs-string">&#x27;\u0120tokenizer&#x27;</span>: <span class="hljs-number">1</span>, <span class="hljs-string">&#x27;\u0120algorithms&#x27;</span>: <span class="hljs-number">1</span>,
    <span class="hljs-string">&#x27;Hopefully&#x27;</span>: <span class="hljs-number">1</span>, <span class="hljs-string">&#x27;,&#x27;</span>: <span class="hljs-number">1</span>, <span class="hljs-string">&#x27;\u0120you&#x27;</span>: <span class="hljs-number">1</span>, <span class="hljs-string">&#x27;\u0120will&#x27;</span>: <span class="hljs-number">1</span>, <span class="hljs-string">&#x27;\u0120be&#x27;</span>: <span class="hljs-number">1</span>, <span class="hljs-string">&#x27;\u0120able&#x27;</span>: <span class="hljs-number">1</span>, <span class="hljs-string">&#x27;\u0120to&#x27;</span>: <span class="hljs-number">1</span>, <span class="hljs-string">&#x27;\u0120understand&#x27;</span>: <span class="hljs-number">1</span>, <span class="hljs-string">&#x27;\u0120how&#x27;</span>: <span class="hljs-number">1</span>,
    <span class="hljs-string">&#x27;\u0120they&#x27;</span>: <span class="hljs-number">1</span>, <span class="hljs-string">&#x27;\u0120are&#x27;</span>: <span class="hljs-number">1</span>, <span class="hljs-string">&#x27;\u0120trained&#x27;</span>: <span class="hljs-number">1</span>, <span class="hljs-string">&#x27;\u0120and&#x27;</span>: <span class="hljs-number">1</span>, <span class="hljs-string">&#x27;\u0120generate&#x27;</span>: <span class="hljs-number">1</span>, <span class="hljs-string">&#x27;\u0120tokens&#x27;</span>: <span class="hljs-number">1</span>})`}}),_s=new d({props:{code:`alphabet = []

for word in word_freqs.keys():
    for letter in word:
        if letter not in alphabet:
            alphabet.append(letter)
alphabet.sort()

print(alphabet)`,highlighted:`alphabet = []

<span class="hljs-keyword">for</span> word <span class="hljs-keyword">in</span> word_freqs.keys():
    <span class="hljs-keyword">for</span> letter <span class="hljs-keyword">in</span> word:
        <span class="hljs-keyword">if</span> letter <span class="hljs-keyword">not</span> <span class="hljs-keyword">in</span> alphabet:
            alphabet.append(letter)
alphabet.sort()

<span class="hljs-built_in">print</span>(alphabet)`}}),ys=new d({props:{code:`[ ',', '.', 'C', 'F', 'H', 'T', 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'k', 'l', 'm', 'n', 'o', 'p', 'r', 's',
  't', 'u', 'v', 'w', 'y', 'z', '\u0120']`,highlighted:`[ <span class="hljs-string">&#x27;,&#x27;</span>, <span class="hljs-string">&#x27;.&#x27;</span>, <span class="hljs-string">&#x27;C&#x27;</span>, <span class="hljs-string">&#x27;F&#x27;</span>, <span class="hljs-string">&#x27;H&#x27;</span>, <span class="hljs-string">&#x27;T&#x27;</span>, <span class="hljs-string">&#x27;a&#x27;</span>, <span class="hljs-string">&#x27;b&#x27;</span>, <span class="hljs-string">&#x27;c&#x27;</span>, <span class="hljs-string">&#x27;d&#x27;</span>, <span class="hljs-string">&#x27;e&#x27;</span>, <span class="hljs-string">&#x27;f&#x27;</span>, <span class="hljs-string">&#x27;g&#x27;</span>, <span class="hljs-string">&#x27;h&#x27;</span>, <span class="hljs-string">&#x27;i&#x27;</span>, <span class="hljs-string">&#x27;k&#x27;</span>, <span class="hljs-string">&#x27;l&#x27;</span>, <span class="hljs-string">&#x27;m&#x27;</span>, <span class="hljs-string">&#x27;n&#x27;</span>, <span class="hljs-string">&#x27;o&#x27;</span>, <span class="hljs-string">&#x27;p&#x27;</span>, <span class="hljs-string">&#x27;r&#x27;</span>, <span class="hljs-string">&#x27;s&#x27;</span>,
  <span class="hljs-string">&#x27;t&#x27;</span>, <span class="hljs-string">&#x27;u&#x27;</span>, <span class="hljs-string">&#x27;v&#x27;</span>, <span class="hljs-string">&#x27;w&#x27;</span>, <span class="hljs-string">&#x27;y&#x27;</span>, <span class="hljs-string">&#x27;z&#x27;</span>, <span class="hljs-string">&#x27;\u0120&#x27;</span>]`}}),ws=new d({props:{code:'vocab = ["<|endoftext|>"] + alphabet.copy()',highlighted:'vocab = [<span class="hljs-string">&quot;&lt;|endoftext|&gt;&quot;</span>] + alphabet.copy()'}}),Es=new d({props:{code:"splits = {word: [c for c in word] for word in word_freqs.keys()}",highlighted:'splits = {word: [c <span class="hljs-keyword">for</span> c <span class="hljs-keyword">in</span> word] <span class="hljs-keyword">for</span> word <span class="hljs-keyword">in</span> word_freqs.keys()}'}}),Ts=new d({props:{code:`def compute_pair_freqs(splits):
    pair_freqs = defaultdict(int)
    for word, freq in word_freqs.items():
        split = splits[word]
        if len(split) == 1:
            continue
        for i in range(len(split) - 1):
            pair = (split[i], split[i + 1])
            pair_freqs[pair] += freq
    return pair_freqs`,highlighted:`<span class="hljs-keyword">def</span> <span class="hljs-title function_">compute_pair_freqs</span>(<span class="hljs-params">splits</span>):
    pair_freqs = defaultdict(<span class="hljs-built_in">int</span>)
    <span class="hljs-keyword">for</span> word, freq <span class="hljs-keyword">in</span> word_freqs.items():
        split = splits[word]
        <span class="hljs-keyword">if</span> <span class="hljs-built_in">len</span>(split) == <span class="hljs-number">1</span>:
            <span class="hljs-keyword">continue</span>
        <span class="hljs-keyword">for</span> i <span class="hljs-keyword">in</span> <span class="hljs-built_in">range</span>(<span class="hljs-built_in">len</span>(split) - <span class="hljs-number">1</span>):
            pair = (split[i], split[i + <span class="hljs-number">1</span>])
            pair_freqs[pair] += freq
    <span class="hljs-keyword">return</span> pair_freqs`}}),zs=new d({props:{code:`pair_freqs = compute_pair_freqs(splits)

for i, key in enumerate(pair_freqs.keys()):
    print(f"{key}: {pair_freqs[key]}")
    if i >= 5:
        break`,highlighted:`pair_freqs = compute_pair_freqs(splits)

<span class="hljs-keyword">for</span> i, key <span class="hljs-keyword">in</span> <span class="hljs-built_in">enumerate</span>(pair_freqs.keys()):
    <span class="hljs-built_in">print</span>(<span class="hljs-string">f&quot;<span class="hljs-subst">{key}</span>: <span class="hljs-subst">{pair_freqs[key]}</span>&quot;</span>)
    <span class="hljs-keyword">if</span> i &gt;= <span class="hljs-number">5</span>:
        <span class="hljs-keyword">break</span>`}}),Ps=new d({props:{code:`('T', 'h'): 3
('h', 'i'): 3
('i', 's'): 5
('\u0120', 'i'): 2
('\u0120', 't'): 7
('t', 'h'): 3`,highlighted:`(<span class="hljs-string">&#x27;T&#x27;</span>, <span class="hljs-string">&#x27;h&#x27;</span>): <span class="hljs-number">3</span>
(<span class="hljs-string">&#x27;h&#x27;</span>, <span class="hljs-string">&#x27;i&#x27;</span>): <span class="hljs-number">3</span>
(<span class="hljs-string">&#x27;i&#x27;</span>, <span class="hljs-string">&#x27;s&#x27;</span>): <span class="hljs-number">5</span>
(<span class="hljs-string">&#x27;\u0120&#x27;</span>, <span class="hljs-string">&#x27;i&#x27;</span>): <span class="hljs-number">2</span>
(<span class="hljs-string">&#x27;\u0120&#x27;</span>, <span class="hljs-string">&#x27;t&#x27;</span>): <span class="hljs-number">7</span>
(<span class="hljs-string">&#x27;t&#x27;</span>, <span class="hljs-string">&#x27;h&#x27;</span>): <span class="hljs-number">3</span>`}}),Cs=new d({props:{code:`best_pair = ""
max_freq = None

for pair, freq in pair_freqs.items():
    if max_freq is None or max_freq < freq:
        best_pair = pair
        max_freq = freq

print(best_pair, max_freq)`,highlighted:`best_pair = <span class="hljs-string">&quot;&quot;</span>
max_freq = <span class="hljs-literal">None</span>

<span class="hljs-keyword">for</span> pair, freq <span class="hljs-keyword">in</span> pair_freqs.items():
    <span class="hljs-keyword">if</span> max_freq <span class="hljs-keyword">is</span> <span class="hljs-literal">None</span> <span class="hljs-keyword">or</span> max_freq &lt; freq:
        best_pair = pair
        max_freq = freq

<span class="hljs-built_in">print</span>(best_pair, max_freq)`}}),Ds=new d({props:{code:"('\u0120', 't') 7",highlighted:'(<span class="hljs-string">&#x27;\u0120&#x27;</span>, <span class="hljs-string">&#x27;t&#x27;</span>) <span class="hljs-number">7</span>'}}),Bs=new d({props:{code:`merges = {("\u0120", "t"): "\u0120t"}
vocab.append("\u0120t")`,highlighted:`merges = {(<span class="hljs-string">&quot;\u0120&quot;</span>, <span class="hljs-string">&quot;t&quot;</span>): <span class="hljs-string">&quot;\u0120t&quot;</span>}
vocab.append(<span class="hljs-string">&quot;\u0120t&quot;</span>)`}}),Os=new d({props:{code:`def merge_pair(a, b, splits):
    for word in word_freqs:
        split = splits[word]
        if len(split) == 1:
            continue

        i = 0
        while i < len(split) - 1:
            if split[i] == a and split[i + 1] == b:
                split = split[:i] + [a + b] + split[i + 2 :]
            else:
                i += 1
        splits[word] = split
    return splits`,highlighted:`<span class="hljs-keyword">def</span> <span class="hljs-title function_">merge_pair</span>(<span class="hljs-params">a, b, splits</span>):
    <span class="hljs-keyword">for</span> word <span class="hljs-keyword">in</span> word_freqs:
        split = splits[word]
        <span class="hljs-keyword">if</span> <span class="hljs-built_in">len</span>(split) == <span class="hljs-number">1</span>:
            <span class="hljs-keyword">continue</span>

        i = <span class="hljs-number">0</span>
        <span class="hljs-keyword">while</span> i &lt; <span class="hljs-built_in">len</span>(split) - <span class="hljs-number">1</span>:
            <span class="hljs-keyword">if</span> split[i] == a <span class="hljs-keyword">and</span> split[i + <span class="hljs-number">1</span>] == b:
                split = split[:i] + [a + b] + split[i + <span class="hljs-number">2</span> :]
            <span class="hljs-keyword">else</span>:
                i += <span class="hljs-number">1</span>
        splits[word] = split
    <span class="hljs-keyword">return</span> splits`}}),Gs=new d({props:{code:`splits = merge_pair("\u0120", "t", splits)
print(splits["\u0120trained"])`,highlighted:`splits = merge_pair(<span class="hljs-string">&quot;\u0120&quot;</span>, <span class="hljs-string">&quot;t&quot;</span>, splits)
<span class="hljs-built_in">print</span>(splits[<span class="hljs-string">&quot;\u0120trained&quot;</span>])`}}),Ns=new d({props:{code:"['\u0120t', 'r', 'a', 'i', 'n', 'e', 'd']",highlighted:'[<span class="hljs-string">&#x27;\u0120t&#x27;</span>, <span class="hljs-string">&#x27;r&#x27;</span>, <span class="hljs-string">&#x27;a&#x27;</span>, <span class="hljs-string">&#x27;i&#x27;</span>, <span class="hljs-string">&#x27;n&#x27;</span>, <span class="hljs-string">&#x27;e&#x27;</span>, <span class="hljs-string">&#x27;d&#x27;</span>]'}}),Hs=new d({props:{code:`vocab_size = 50

while len(vocab) < vocab_size:
    pair_freqs = compute_pair_freqs(splits)
    best_pair = ""
    max_freq = None
    for pair, freq in pair_freqs.items():
        if max_freq is None or max_freq < freq:
            best_pair = pair
            max_freq = freq
    splits = merge_pair(*best_pair, splits)
    merges[best_pair] = best_pair[0] + best_pair[1]
    vocab.append(best_pair[0] + best_pair[1])`,highlighted:`vocab_size = <span class="hljs-number">50</span>

<span class="hljs-keyword">while</span> <span class="hljs-built_in">len</span>(vocab) &lt; vocab_size:
    pair_freqs = compute_pair_freqs(splits)
    best_pair = <span class="hljs-string">&quot;&quot;</span>
    max_freq = <span class="hljs-literal">None</span>
    <span class="hljs-keyword">for</span> pair, freq <span class="hljs-keyword">in</span> pair_freqs.items():
        <span class="hljs-keyword">if</span> max_freq <span class="hljs-keyword">is</span> <span class="hljs-literal">None</span> <span class="hljs-keyword">or</span> max_freq &lt; freq:
            best_pair = pair
            max_freq = freq
    splits = merge_pair(*best_pair, splits)
    merges[best_pair] = best_pair[<span class="hljs-number">0</span>] + best_pair[<span class="hljs-number">1</span>]
    vocab.append(best_pair[<span class="hljs-number">0</span>] + best_pair[<span class="hljs-number">1</span>])`}}),Ss=new d({props:{code:"print(merges)",highlighted:'<span class="hljs-built_in">print</span>(merges)'}}),As=new d({props:{code:`{('\u0120', 't'): '\u0120t', ('i', 's'): 'is', ('e', 'r'): 'er', ('\u0120', 'a'): '\u0120a', ('\u0120t', 'o'): '\u0120to', ('e', 'n'): 'en',
 ('T', 'h'): 'Th', ('Th', 'is'): 'This', ('o', 'u'): 'ou', ('s', 'e'): 'se', ('\u0120to', 'k'): '\u0120tok',
 ('\u0120tok', 'en'): '\u0120token', ('n', 'd'): 'nd', ('\u0120', 'is'): '\u0120is', ('\u0120t', 'h'): '\u0120th', ('\u0120th', 'e'): '\u0120the',
 ('i', 'n'): 'in', ('\u0120a', 'b'): '\u0120ab', ('\u0120token', 'i'): '\u0120tokeni'}`,highlighted:`{(<span class="hljs-string">&#x27;\u0120&#x27;</span>, <span class="hljs-string">&#x27;t&#x27;</span>): <span class="hljs-string">&#x27;\u0120t&#x27;</span>, (<span class="hljs-string">&#x27;i&#x27;</span>, <span class="hljs-string">&#x27;s&#x27;</span>): <span class="hljs-string">&#x27;is&#x27;</span>, (<span class="hljs-string">&#x27;e&#x27;</span>, <span class="hljs-string">&#x27;r&#x27;</span>): <span class="hljs-string">&#x27;er&#x27;</span>, (<span class="hljs-string">&#x27;\u0120&#x27;</span>, <span class="hljs-string">&#x27;a&#x27;</span>): <span class="hljs-string">&#x27;\u0120a&#x27;</span>, (<span class="hljs-string">&#x27;\u0120t&#x27;</span>, <span class="hljs-string">&#x27;o&#x27;</span>): <span class="hljs-string">&#x27;\u0120to&#x27;</span>, (<span class="hljs-string">&#x27;e&#x27;</span>, <span class="hljs-string">&#x27;n&#x27;</span>): <span class="hljs-string">&#x27;en&#x27;</span>,
 (<span class="hljs-string">&#x27;T&#x27;</span>, <span class="hljs-string">&#x27;h&#x27;</span>): <span class="hljs-string">&#x27;Th&#x27;</span>, (<span class="hljs-string">&#x27;Th&#x27;</span>, <span class="hljs-string">&#x27;is&#x27;</span>): <span class="hljs-string">&#x27;This&#x27;</span>, (<span class="hljs-string">&#x27;o&#x27;</span>, <span class="hljs-string">&#x27;u&#x27;</span>): <span class="hljs-string">&#x27;ou&#x27;</span>, (<span class="hljs-string">&#x27;s&#x27;</span>, <span class="hljs-string">&#x27;e&#x27;</span>): <span class="hljs-string">&#x27;se&#x27;</span>, (<span class="hljs-string">&#x27;\u0120to&#x27;</span>, <span class="hljs-string">&#x27;k&#x27;</span>): <span class="hljs-string">&#x27;\u0120tok&#x27;</span>,
 (<span class="hljs-string">&#x27;\u0120tok&#x27;</span>, <span class="hljs-string">&#x27;en&#x27;</span>): <span class="hljs-string">&#x27;\u0120token&#x27;</span>, (<span class="hljs-string">&#x27;n&#x27;</span>, <span class="hljs-string">&#x27;d&#x27;</span>): <span class="hljs-string">&#x27;nd&#x27;</span>, (<span class="hljs-string">&#x27;\u0120&#x27;</span>, <span class="hljs-string">&#x27;is&#x27;</span>): <span class="hljs-string">&#x27;\u0120is&#x27;</span>, (<span class="hljs-string">&#x27;\u0120t&#x27;</span>, <span class="hljs-string">&#x27;h&#x27;</span>): <span class="hljs-string">&#x27;\u0120th&#x27;</span>, (<span class="hljs-string">&#x27;\u0120th&#x27;</span>, <span class="hljs-string">&#x27;e&#x27;</span>): <span class="hljs-string">&#x27;\u0120the&#x27;</span>,
 (<span class="hljs-string">&#x27;i&#x27;</span>, <span class="hljs-string">&#x27;n&#x27;</span>): <span class="hljs-string">&#x27;in&#x27;</span>, (<span class="hljs-string">&#x27;\u0120a&#x27;</span>, <span class="hljs-string">&#x27;b&#x27;</span>): <span class="hljs-string">&#x27;\u0120ab&#x27;</span>, (<span class="hljs-string">&#x27;\u0120token&#x27;</span>, <span class="hljs-string">&#x27;i&#x27;</span>): <span class="hljs-string">&#x27;\u0120tokeni&#x27;</span>}`}}),Vs=new d({props:{code:"print(vocab)",highlighted:'<span class="hljs-built_in">print</span>(vocab)'}}),Rs=new d({props:{code:`['<|endoftext|>', ',', '.', 'C', 'F', 'H', 'T', 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'k', 'l', 'm', 'n', 'o',
 'p', 'r', 's', 't', 'u', 'v', 'w', 'y', 'z', '\u0120', '\u0120t', 'is', 'er', '\u0120a', '\u0120to', 'en', 'Th', 'This', 'ou', 'se',
 '\u0120tok', '\u0120token', 'nd', '\u0120is', '\u0120th', '\u0120the', 'in', '\u0120ab', '\u0120tokeni']`,highlighted:`[<span class="hljs-string">&#x27;&lt;|endoftext|&gt;&#x27;</span>, <span class="hljs-string">&#x27;,&#x27;</span>, <span class="hljs-string">&#x27;.&#x27;</span>, <span class="hljs-string">&#x27;C&#x27;</span>, <span class="hljs-string">&#x27;F&#x27;</span>, <span class="hljs-string">&#x27;H&#x27;</span>, <span class="hljs-string">&#x27;T&#x27;</span>, <span class="hljs-string">&#x27;a&#x27;</span>, <span class="hljs-string">&#x27;b&#x27;</span>, <span class="hljs-string">&#x27;c&#x27;</span>, <span class="hljs-string">&#x27;d&#x27;</span>, <span class="hljs-string">&#x27;e&#x27;</span>, <span class="hljs-string">&#x27;f&#x27;</span>, <span class="hljs-string">&#x27;g&#x27;</span>, <span class="hljs-string">&#x27;h&#x27;</span>, <span class="hljs-string">&#x27;i&#x27;</span>, <span class="hljs-string">&#x27;k&#x27;</span>, <span class="hljs-string">&#x27;l&#x27;</span>, <span class="hljs-string">&#x27;m&#x27;</span>, <span class="hljs-string">&#x27;n&#x27;</span>, <span class="hljs-string">&#x27;o&#x27;</span>,
 <span class="hljs-string">&#x27;p&#x27;</span>, <span class="hljs-string">&#x27;r&#x27;</span>, <span class="hljs-string">&#x27;s&#x27;</span>, <span class="hljs-string">&#x27;t&#x27;</span>, <span class="hljs-string">&#x27;u&#x27;</span>, <span class="hljs-string">&#x27;v&#x27;</span>, <span class="hljs-string">&#x27;w&#x27;</span>, <span class="hljs-string">&#x27;y&#x27;</span>, <span class="hljs-string">&#x27;z&#x27;</span>, <span class="hljs-string">&#x27;\u0120&#x27;</span>, <span class="hljs-string">&#x27;\u0120t&#x27;</span>, <span class="hljs-string">&#x27;is&#x27;</span>, <span class="hljs-string">&#x27;er&#x27;</span>, <span class="hljs-string">&#x27;\u0120a&#x27;</span>, <span class="hljs-string">&#x27;\u0120to&#x27;</span>, <span class="hljs-string">&#x27;en&#x27;</span>, <span class="hljs-string">&#x27;Th&#x27;</span>, <span class="hljs-string">&#x27;This&#x27;</span>, <span class="hljs-string">&#x27;ou&#x27;</span>, <span class="hljs-string">&#x27;se&#x27;</span>,
 <span class="hljs-string">&#x27;\u0120tok&#x27;</span>, <span class="hljs-string">&#x27;\u0120token&#x27;</span>, <span class="hljs-string">&#x27;nd&#x27;</span>, <span class="hljs-string">&#x27;\u0120is&#x27;</span>, <span class="hljs-string">&#x27;\u0120th&#x27;</span>, <span class="hljs-string">&#x27;\u0120the&#x27;</span>, <span class="hljs-string">&#x27;in&#x27;</span>, <span class="hljs-string">&#x27;\u0120ab&#x27;</span>, <span class="hljs-string">&#x27;\u0120tokeni&#x27;</span>]`}}),ls=new bn({props:{$$slots:{default:[Ah]},$$scope:{ctx:B}}}),Is=new d({props:{code:`def tokenize(text):
    pre_tokenize_result = tokenizer._tokenizer.pre_tokenizer.pre_tokenize_str(text)
    pre_tokenized_text = [word for word, offset in pre_tokenize_result]
    splits = [[l for l in word] for word in pre_tokenized_text]
    for pair, merge in merges.items():
        for idx, split in enumerate(splits):
            i = 0
            while i < len(split) - 1:
                if split[i] == pair[0] and split[i + 1] == pair[1]:
                    split = split[:i] + [merge] + split[i + 2 :]
                else:
                    i += 1
            splits[idx] = split

    return sum(splits, [])`,highlighted:`<span class="hljs-keyword">def</span> <span class="hljs-title function_">tokenize</span>(<span class="hljs-params">text</span>):
    pre_tokenize_result = tokenizer._tokenizer.pre_tokenizer.pre_tokenize_str(text)
    pre_tokenized_text = [word <span class="hljs-keyword">for</span> word, offset <span class="hljs-keyword">in</span> pre_tokenize_result]
    splits = [[l <span class="hljs-keyword">for</span> l <span class="hljs-keyword">in</span> word] <span class="hljs-keyword">for</span> word <span class="hljs-keyword">in</span> pre_tokenized_text]
    <span class="hljs-keyword">for</span> pair, merge <span class="hljs-keyword">in</span> merges.items():
        <span class="hljs-keyword">for</span> idx, split <span class="hljs-keyword">in</span> <span class="hljs-built_in">enumerate</span>(splits):
            i = <span class="hljs-number">0</span>
            <span class="hljs-keyword">while</span> i &lt; <span class="hljs-built_in">len</span>(split) - <span class="hljs-number">1</span>:
                <span class="hljs-keyword">if</span> split[i] == pair[<span class="hljs-number">0</span>] <span class="hljs-keyword">and</span> split[i + <span class="hljs-number">1</span>] == pair[<span class="hljs-number">1</span>]:
                    split = split[:i] + [merge] + split[i + <span class="hljs-number">2</span> :]
                <span class="hljs-keyword">else</span>:
                    i += <span class="hljs-number">1</span>
            splits[idx] = split

    <span class="hljs-keyword">return</span> <span class="hljs-built_in">sum</span>(splits, [])`}}),Ls=new d({props:{code:'tokenize("This is not a token.")',highlighted:'tokenize(<span class="hljs-string">&quot;This is not a token.&quot;</span>)'}}),Us=new d({props:{code:"['This', '\u0120is', '\u0120', 'n', 'o', 't', '\u0120a', '\u0120token', '.']",highlighted:'[<span class="hljs-string">&#x27;This&#x27;</span>, <span class="hljs-string">&#x27;\u0120is&#x27;</span>, <span class="hljs-string">&#x27;\u0120&#x27;</span>, <span class="hljs-string">&#x27;n&#x27;</span>, <span class="hljs-string">&#x27;o&#x27;</span>, <span class="hljs-string">&#x27;t&#x27;</span>, <span class="hljs-string">&#x27;\u0120a&#x27;</span>, <span class="hljs-string">&#x27;\u0120token&#x27;</span>, <span class="hljs-string">&#x27;.&#x27;</span>]'}}),ps=new bn({props:{warning:!0,$$slots:{default:[Vh]},$$scope:{ctx:B}}}),{c(){u=h("meta"),_=r(),g=h("h1"),$=h("a"),E=h("span"),f(v.$$.fragment),y=r(),z=h("span"),I=l("Byte-Pair Encoding tokenization"),O=r(),f(S.$$.fragment),is=r(),Ms=h("p"),Wa=l("M\xE3 h\xF3a theo c\u1EB7p (BPE) ti\u1EC1n th\xE2n \u0111\u01B0\u1EE3c ph\xE1t tri\u1EC3n nh\u01B0 m\u1ED9t thu\u1EADt to\xE1n \u0111\u1EC3 n\xE9n v\u0103n b\u1EA3n, sau \u0111\xF3 \u0111\u01B0\u1EE3c OpenAI s\u1EED d\u1EE5ng \u0111\u1EC3 tokenize khi hu\u1EA5n luy\u1EC7n tr\u01B0\u1EDBc m\xF4 h\xECnh GPT. N\xF3 \u0111\u01B0\u1EE3c s\u1EED d\u1EE5ng b\u1EDFi r\u1EA5t nhi\u1EC1u m\xF4 h\xECnh Transformer, bao g\u1ED3m GPT, GPT-2, RoBERTa, BART v\xE0 DeBERTa."),kt=r(),f(cs.$$.fragment),bt=r(),f(K.$$.fragment),$t=r(),L=h("h2"),M=h("a"),$n=h("span"),f(rs.$$.fragment),Ya=r(),dn=h("span"),Ja=l("Thu\u1EADt to\xE1n hu\u1EA5n luy\u1EC7n"),dt=r(),Qs=h("p"),Za=l("Hu\u1EA5n luy\u1EC7n BPE b\u1EAFt \u0111\u1EA7u b\u1EB1ng c\xE1ch t\xEDnh to\xE1n t\u1EADp h\u1EE3p c\xE1c t\u1EEB duy nh\u1EA5t \u0111\u01B0\u1EE3c s\u1EED d\u1EE5ng trong kho ng\u1EEF li\u1EC7u (sau khi ho\xE0n th\xE0nh c\xE1c b\u01B0\u1EDBc chu\u1EA9n h\xF3a v\xE0 pre-tokenization), sau \u0111\xF3 x\xE2y d\u1EF1ng t\u1EEB v\u1EF1ng b\u1EB1ng c\xE1ch l\u1EA5y t\u1EA5t c\u1EA3 c\xE1c k\xFD hi\u1EC7u \u0111\u01B0\u1EE3c s\u1EED d\u1EE5ng \u0111\u1EC3 vi\u1EBFt nh\u1EEFng t\u1EEB \u0111\xF3. V\xED d\u1EE5 r\u1EA5t \u0111\u01A1n gi\u1EA3n, gi\u1EA3 s\u1EED kho d\u1EEF li\u1EC7u c\u1EE7a ch\xFAng ta s\u1EED d\u1EE5ng n\u0103m t\u1EEB sau:"),vt=r(),f(os.$$.fragment),qt=r(),Q=h("p"),Xa=l("T\u1EEB v\u1EF1ng c\u01A1 s\u1EDF khi \u0111\xF3 s\u1EBD l\xE0 "),vn=h("code"),sl=l('["b", "g", "h", "n", "p", "s", "u"]'),nl=l(". \u0110\u1ED1i v\u1EDBi c\xE1c tr\u01B0\u1EDDng h\u1EE3p trong th\u1EF1c t\u1EBF, t\u1EEB v\u1EF1ng c\u01A1 s\u1EDF \u0111\xF3 s\u1EBD ch\u1EE9a t\u1EA5t c\u1EA3 c\xE1c k\xFD t\u1EF1 ASCII, \xEDt nh\u1EA5t v\xE0 c\xF3 th\u1EC3 l\xE0 m\u1ED9t s\u1ED1 k\xFD t\u1EF1 Unicode. N\u1EBFu m\u1ED9t m\u1EABu b\u1EA1n \u0111ang tokenize s\u1EED d\u1EE5ng m\u1ED9t k\xFD t\u1EF1 kh\xF4ng c\xF3 trong kho d\u1EEF li\u1EC7u hu\u1EA5n luy\u1EC7n, th\xEC k\xFD t\u1EF1 \u0111\xF3 s\u1EBD \u0111\u01B0\u1EE3c chuy\u1EC3n \u0111\u1ED5i th\xE0nh token kh\xF4ng x\xE1c \u0111\u1ECBnh. \u0110\xF3 l\xE0 m\u1ED9t l\xFD do t\u1EA1i sao nhi\u1EC1u m\xF4 h\xECnh NLP r\u1EA5t k\xE9m trong vi\u1EC7c ph\xE2n t\xEDch n\u1ED9i dung b\u1EB1ng bi\u1EC3u t\u01B0\u1EE3ng c\u1EA3m x\xFAc."),_t=r(),f(W.$$.fragment),yt=r(),Y=h("p"),tl=l("Sau khi c\xF3 \u0111\u01B0\u1EE3c b\u1ED9 t\u1EEB v\u1EF1ng c\u01A1 b\u1EA3n n\xE0y, ch\xFAng ta th\xEAm c\xE1c token m\u1EDBi cho \u0111\u1EBFn khi \u0111\u1EA1t \u0111\u01B0\u1EE3c k\xEDch th\u01B0\u1EDBc t\u1EEB v\u1EF1ng mong mu\u1ED1n b\u1EB1ng c\xE1ch h\u1ECDc "),qn=h("em"),al=l("h\u1EE3p nh\u1EA5t"),ll=l(", \u0111\xE2y l\xE0 c\xE1c quy t\u1EAFc \u0111\u1EC3 h\u1EE3p nh\u1EA5t hai y\u1EBFu t\u1ED1 c\u1EE7a t\u1EEB v\u1EF1ng hi\u1EC7n c\xF3 v\u1EDBi nhau th\xE0nh m\u1ED9t t\u1EEB m\u1EDBi. V\xEC v\u1EADy, l\xFAc \u0111\u1EA7u s\u1EF1 h\u1EE3p nh\u1EA5t n\xE0y s\u1EBD t\u1EA1o ra c\xE1c token c\xF3 hai k\xFD t\u1EF1 v\xE0 sau \u0111\xF3, khi qu\xE1 tr\xECnh hu\u1EA5n luy\u1EC7n ti\u1EBFn tri\u1EC3n, c\xE1c t\u1EEB ph\u1EE5 s\u1EBD d\xE0i h\u01A1n."),wt=r(),Ws=h("p"),pl=l("T\u1EA1i b\u1EA5t k\u1EF3 b\u01B0\u1EDBc n\xE0o trong qu\xE1 tr\xECnh hu\u1EA5n luy\u1EC7n token, thu\u1EADt to\xE1n BPE s\u1EBD t\xECm ki\u1EBFm c\u1EB7p token hi\u1EC7n c\xF3 th\u01B0\u1EDDng xuy\xEAn nh\u1EA5t (theo \u201Cc\u1EB7p\u201D, \u1EDF \u0111\xE2y c\xF3 ngh\u0129a l\xE0 hai token li\xEAn ti\u1EBFp trong m\u1ED9t t\u1EEB). C\u1EB7p th\u01B0\u1EDDng xuy\xEAn nh\u1EA5t \u0111\xF3 l\xE0 c\u1EB7p s\u1EBD \u0111\u01B0\u1EE3c h\u1EE3p nh\u1EA5t, v\xE0 ch\xFAng ta x\u1EA3 v\xE0 l\u1EB7p l\u1EA1i cho b\u01B0\u1EDBc ti\u1EBFp theo."),Et=r(),Ys=h("p"),el=l("Quay tr\u1EDF l\u1EA1i v\xED d\u1EE5 tr\u01B0\u1EDBc, gi\u1EA3 s\u1EED c\xE1c t\u1EEB c\xF3 t\u1EA7n s\u1ED1 nh\u01B0 sau:"),Tt=r(),f(us.$$.fragment),zt=r(),C=h("p"),hl=l("ngh\u0129a l\xE0 "),_n=h("code"),il=l('"hug"'),cl=l(" c\xF3 m\u1EB7t 10 l\u1EA7n trong kho ng\u1EEF li\u1EC7u, "),yn=h("code"),rl=l('"pug"'),ol=l(" 5 l\u1EA7n, "),wn=h("code"),ul=l('"pun"'),gl=l(" 12 l\u1EA7n, "),En=h("code"),fl=l('"bun"'),ml=l(" 4 l\u1EA7n v\xE0 "),Tn=h("code"),jl=l('"hug"'),xl=l(" 5 l\u1EA7n. Ch\xFAng ta b\u1EAFt \u0111\u1EA7u hu\u1EA5n luy\u1EC7n b\u1EB1ng c\xE1ch t\xE1ch t\u1EEBng t\u1EEB th\xE0nh c\xE1c k\xFD t\u1EF1 (nh\u1EEFng k\xFD t\u1EF1 h\xECnh th\xE0nh t\u1EEB v\u1EF1ng ban \u0111\u1EA7u c\u1EE7a ch\xFAng ta) \u0111\u1EC3 c\xF3 th\u1EC3 xem m\u1ED7i t\u1EEB nh\u01B0 m\u1ED9t danh s\xE1ch c\xE1c token:"),Pt=r(),f(gs.$$.fragment),Ct=r(),T=h("p"),kl=l("Sau \u0111\xF3, ch\xFAng ta xem x\xE9t c\xE1c c\u1EB7p. C\u1EB7p "),zn=h("code"),bl=l('("h", "u")'),$l=l(" c\xF3 trong c\xE1c t\u1EEB  "),Pn=h("code"),dl=l('"hug"'),vl=l(" v\xE0  "),Cn=h("code"),ql=l('"hugs"'),_l=l(", v\xEC v\u1EADy t\u1ED5ng c\u1ED9ng l\xE0 15 l\u1EA7n trong ng\u1EEF li\u1EC7u. Tuy nhi\xEAn, \u0111\xE2y kh\xF4ng ph\u1EA3i l\xE0 c\u1EB7p th\u01B0\u1EDDng xuy\xEAn nh\u1EA5t: vinh d\u1EF1 \u0111\xF3 thu\u1ED9c v\u1EC1 "),Dn=h("code"),yl=l('("u", "g")'),wl=l(", c\xF3 trong "),Bn=h("code"),El=l('"hug"'),Tl=l(", "),On=h("code"),zl=l('"pug"'),Pl=l(", v\xE0 "),Gn=h("code"),Cl=l('"hugs"'),Dl=l(", v\u1EDBi t\u1ED5ng c\u1ED9ng 20 l\u1EA7n xu\u1EA5t hi\u1EC7n trong b\u1ED9 t\u1EEB v\u1EF1ng."),Dt=r(),A=h("p"),Bl=l("Do \u0111\xF3, quy t\u1EAFc h\u1EE3p nh\u1EA5t \u0111\u1EA7u ti\xEAn \u0111\u01B0\u1EE3c h\u1ECDc b\u1EDFi tokenizer l\xE0 "),Nn=h("code"),Ol=l('("u", "g") -> "ug"'),Gl=l(", c\xF3 ngh\u0129a l\xE0 "),Hn=h("code"),Nl=l('"ug"'),Hl=l(" s\u1EBD \u0111\u01B0\u1EE3c th\xEAm v\xE0o t\u1EEB v\u1EF1ng v\xE0 c\u1EB7p n\xE0y s\u1EBD \u0111\u01B0\u1EE3c h\u1EE3p nh\u1EA5t trong t\u1EA5t c\u1EA3 c\xE1c t\u1EEB c\u1EE7a ng\u1EEF li\u1EC7u. V\xE0o cu\u1ED1i giai \u0111o\u1EA1n n\xE0y, t\u1EEB v\u1EF1ng v\xE0 ng\u1EEF li\u1EC7u s\u1EBD gi\u1ED1ng nh\u01B0 sau:"),Bt=r(),f(fs.$$.fragment),Ot=r(),G=h("p"),Sl=l("B\xE2y gi\u1EDD ch\xFAng ta c\xF3 m\u1ED9t s\u1ED1 c\u1EB7p d\u1EABn \u0111\u1EBFn m\u1ED9t token d\xE0i h\u01A1n hai k\xFD t\u1EF1: v\xED d\u1EE5: c\u1EB7p "),Sn=h("code"),Al=l('("h", "ug")'),Vl=l(", (hi\u1EC7n di\u1EC7n 15 l\u1EA7n trong kho ng\u1EEF li\u1EC7u). C\u1EB7p th\u01B0\u1EDDng g\u1EB7p nh\u1EA5t \u1EDF giai \u0111o\u1EA1n n\xE0y l\xE0 "),An=h("code"),Rl=l('("u", "n")'),Il=l(", xu\u1EA5t hi\u1EC7n 16 l\u1EA7n trong kho ng\u1EEF li\u1EC7u, v\xEC v\u1EADy quy t\u1EAFc h\u1EE3p nh\u1EA5t th\u1EE9 hai \u0111\xE3 h\u1ECDc l\xE0 "),Vn=h("code"),Ll=l('("u", "n") -> "un"'),Ul=l(". Th\xEAm n\xF3 v\xE0o b\u1ED9 t\u1EEB v\u1EF1ng v\xE0 h\u1EE3p nh\u1EA5t t\u1EA5t c\u1EA3 c\xE1c l\u1EA7n xu\u1EA5t hi\u1EC7n hi\u1EC7n c\xF3 s\u1EBD d\u1EABn ch\xFAng ta \u0111\u1EBFn:"),Gt=r(),f(ms.$$.fragment),Nt=r(),V=h("p"),Fl=l("Gi\u1EDD th\xEC c\u1EB7p xu\u1EA5t hi\u1EC7n nhi\u1EC1u nh\u1EA5t l\xE0 "),Rn=h("code"),Kl=l('("h", "ug")'),Ml=l(", n\xEAn ch\xFAng ta h\u1EE3p nh\u1EA5t "),In=h("code"),Ql=l('("h", "ug") -> "hug"'),Wl=l(", tr\u1EA3 v\u1EC1 cho ch\xFAng ta token g\u1ED3n ba k\xED t\u1EF1 \u0111\u1EA7u ti\xEAn. Sau s\u1EF1 h\u1EE3p nh\u1EA5t n\xE0y, kho ng\u1EEF li\u1EC7u s\u1EBD nh\u01B0 sau:"),Ht=r(),f(js.$$.fragment),St=r(),Js=h("p"),Yl=l("V\xE0 ch\xFAng ta ti\u1EBFp t\xFAc l\xE0m v\u1EADy cho \u0111\u1EBFn khi ch\xFAng ta ch\u1EA1m \u0111\u1EBFn k\xEDch th\u01B0\u1EDBc b\u1ED9 t\u1EF1 \u0111i\u1EC3n ta mong mu\u1ED1n."),At=r(),f(J.$$.fragment),Vt=r(),U=h("h2"),Z=h("a"),Ln=h("span"),f(xs.$$.fragment),Jl=r(),Un=h("span"),Zl=l("Thu\u1EADt to\xE1n tokenize"),Rt=r(),Zs=h("p"),Xl=l("Tokenize tu\xE2n th\u1EE7 ch\u1EB7t ch\u1EBD qu\xE1 tr\xECnh hu\u1EA5n luy\u1EC7n, theo ngh\u0129a l\xE0 c\xE1c \u0111\u1EA7u v\xE0o m\u1EDBi \u0111\u01B0\u1EE3c tokenize b\u1EB1ng c\xE1ch \xE1p d\u1EE5ng c\xE1c b\u01B0\u1EDBc sau:"),It=r(),N=h("ol"),Fn=h("li"),sp=l("Chu\u1EA9n ho\xE1"),np=r(),Kn=h("li"),tp=l("Pre-tokenization"),ap=r(),Mn=h("li"),lp=l("T\xE1ch c\xE1c t\u1EEB th\xE0nh c\xE1c k\xFD t\u1EF1 ri\xEAng l\u1EBB"),pp=r(),Qn=h("li"),ep=l("\xC1p d\u1EE5ng c\xE1c quy t\u1EAFc h\u1EE3p nh\u1EA5t \u0111\xE3 h\u1ECDc theo th\u1EE9 t\u1EF1 tr\xEAn c\xE1c ph\u1EA7n t\xE1ch \u0111\xF3"),Lt=r(),Xs=h("p"),hp=l("L\u1EA5y v\xED d\u1EE5 m\xE0 ta \u0111\xE3 s\u1EED d\u1EE5ng trong qu\xE1 tr\xECnh hu\u1EA5n luy\u1EC7n, v\u1EDBi ba quy t\u1EAFc h\u1EE3p nh\u1EA5t \u0111\xE3 h\u1ECDc:"),Ut=r(),f(ks.$$.fragment),Ft=r(),q=h("p"),ip=l("T\u1EEB "),Wn=h("code"),cp=l('"bug"'),rp=l(" s\u1EBD \u0111\u01B0\u1EE3c tokenize th\xE0nh "),Yn=h("code"),op=l('["b", "ug"]'),up=l(". "),Jn=h("code"),gp=l('"mug"'),fp=l(", tuy nhi\xEAn, s\u1EBD tokenize th\xE0nh "),Zn=h("code"),mp=l('["[UNK]", "ug"]'),jp=l(" v\xEC k\xED t\u1EF1 "),Xn=h("code"),xp=l('"m"'),kp=l(" kh\xF4ng c\xF3 trong b\u1ED9 t\u1EF1 v\u1EF1ng g\u1ED1c. T\u01B0\u01A1ng t\u1EF1, t\u1EEB "),st=h("code"),bp=l('"thug"'),$p=l(" s\u1EBD \u0111\u01B0\u1EE3c tokenize th\xE0nh  "),nt=h("code"),dp=l('["[UNK]", "hug"]'),vp=l(": k\xED t\u1EF1 "),tt=h("code"),qp=l('"t"'),_p=l(" kh\xF4ng c\xF3 trong b\u1ED9 t\u1EF1 v\u1EF1ng g\u1ED1c, v\xE0 \xE1p d\u1EE5ng quy t\u1EAFc h\u1EE3p nh\u1EA5t \u1EDF "),at=h("code"),yp=l('"u"'),wp=l(" v\xE0 "),lt=h("code"),Ep=l('"g"'),Tp=l(" v\xE0 sau \u0111\xF3 "),pt=h("code"),zp=l('"hu"'),Pp=l(" v\xE0 "),et=h("code"),Cp=l('"g"'),Dp=l("."),Kt=r(),f(X.$$.fragment),Mt=r(),F=h("h2"),ss=h("a"),ht=h("span"),f(bs.$$.fragment),Bp=r(),it=h("span"),Op=l("Tri\u1EC3n khai BPE"),Qt=r(),sn=h("p"),Gp=l("H\xE3y c\xF9ng xem c\xE1c thu\u1EADt to\xE1n BPE \u0111\u01B0\u1EE3c tri\u1EC3n khai. \u0110\xE2y kh\xF4ng ph\u1EA3i l\xE0 phi\xEAn b\u1EA3n t\u1ED1i \u01B0u m\xE0 b\u1EA1n c\xF3 th\u1EC3 th\u1EF1c s\u1EF1 s\u1EED d\u1EE5ng cho m\u1ED9t kho ng\u1EEF li\u1EC7u l\u1EDBn; ch\xFAng t\xF4i ch\u1EC9 mu\u1ED1n cho b\u1EA1n xem \u0111o\u1EA1n m\xE3 \u0111\u1EC3 b\u1EA1n c\xF3 th\u1EC3 hi\u1EC3u thu\u1EADt to\xE1n n\xE0y t\u1ED1t h\u01A1n."),Wt=r(),nn=h("p"),Np=l("\u0110\u1EA7u ti\xEAn ch\xFAng ta c\u1EA7n m\u1ED9t kho ng\u1EEF li\u1EC7u, v\u1EADy n\xEAn hay t\u1EA1o ra m\u1ED9t b\u1EA3n \u0111\u01A1n gi\u1EA3n v\u1EDBi m\u1ED9t v\xE0i c\xE2u:"),Yt=r(),f($s.$$.fragment),Jt=r(),ns=h("p"),Hp=l("Ti\u1EBFp theo, ta c\u1EA7n ti\u1EC1n tokenize kho ng\u1EEF li\u1EC7u n\xE0y th\xE0nh c\xE1c t\u1EEB. V\xEC ta \u0111ang sao ch\xE9p m\u1ED9t b\u1EA3n BPE tokenizer (nh\u01B0 GPT-2), ta v\u1EABn c\xF3 th\u1EC3 s\u1EED d\u1EE5ng "),ct=h("code"),Sp=l("gpt2"),Ap=l(" tokenize cho b\u01B0\u1EDBc pre-tokenization:"),Zt=r(),f(ds.$$.fragment),Xt=r(),tn=h("p"),Vp=l("Sau \u0111\xF3 ta t\xEDnh t\u1EA7n su\u1EA5t c\u1EE7a t\u1EEBng t\u1EEB trong kho ng\u1EEF li\u1EC7u nh\u01B0 khi l\xE0m v\u1EDBi pre-tokenization:"),sa=r(),f(vs.$$.fragment),na=r(),f(qs.$$.fragment),ta=r(),an=h("p"),Rp=l("Ti\u1EBFp theo ch\xFAng ta s\u1EBD t\xEDnh b\u1ED9 t\u1EEB v\u1EF1ng c\u01A1 s\u1EDF t\u1EEB c\xE1c k\xED t\u1EF1 s\u1EED d\u1EE5ng trong kho ng\u1EEF li\u1EC7u:"),aa=r(),f(_s.$$.fragment),la=r(),f(ys.$$.fragment),pa=r(),ts=h("p"),Ip=l("Ta c\u0169ng c\xF3 th\u1EC3 th\xEAm c\xE1c token \u0111\u1EB7c bi\u1EC7t t\u1EEB m\xF4 h\xECnh \u1EDF \u0111\u1EA7u c\u1EE7a b\u1ED9 t\u1EF1 v\u1EF1ng. Trong tr\u01B0\u1EDDng h\u1EE3p c\u1EE7a GPT-2, token \u0111\u1EB7c bi\u1EC7t duy nh\u1EA5t \u0111\xF3 l\xE0 "),rt=h("code"),Lp=l('"<|endoftext|>"'),Up=l(":"),ea=r(),f(ws.$$.fragment),ha=r(),ln=h("p"),Fp=l("Ta gi\u1EDD c\u1EA7n ph\u1EA3i chia m\u1ED7i t\u1EEB th\xE0nh c\xE1c k\xED t\u1EF1 ri\xEAng l\u1EBB \u0111\u1EC3 c\xF3 th\u1EC3 b\u1EAFt \u0111\u1EA7u hu\u1EA5n luy\u1EC7n"),ia=r(),f(Es.$$.fragment),ca=r(),pn=h("p"),Kp=l("Gi\u1EDD ta \u0111\xE3 s\u1EB5n s\xE0ng \u0111\u1EC3 hu\u1EA5n luy\u1EC7n, h\xE3y c\xF9ng vi\u1EBFt m\u1ED9t h\xE0m t\xEDnh t\u1EA7n su\u1EA5t m\u1ED7i c\u1EB7p. Ta s\u1EBD c\u1EA7n s\u1EED d\u1EE5ng n\xF3 \u1EDF b\u01B0\u1EDBc hu\u1EA5n luy\u1EC7n:"),ra=r(),f(Ts.$$.fragment),oa=r(),en=h("p"),Mp=l("H\xE3y nh\xECn v\xE0o m\u1ED9t ph\u1EA7n t\u1EEB \u0111i\u1EBBn sau khi t\xE1ch:"),ua=r(),f(zs.$$.fragment),ga=r(),f(Ps.$$.fragment),fa=r(),hn=h("p"),Qp=l("Gi\u1EDD th\xEC, t\xECm xem c\u1EB7p xu\u1EA5t hi\u1EC7n nhi\u1EC1u nh\u1EA5t b\u1EB1ng m\u1ED9t v\xF2ng l\u1EB7p nhanh:"),ma=r(),f(Cs.$$.fragment),ja=r(),f(Ds.$$.fragment),xa=r(),R=h("p"),Wp=l("V\u1EADy ph\xE9p h\u1EE3p nh\u1EA5t \u0111\u1EA7u ti\xEAn l\xE0 "),ot=h("code"),Yp=l("('\u0120', 't') -> '\u0120t'"),Jp=l(", v\xE0 ta th\xEAm "),ut=h("code"),Zp=l("'\u0120t'"),Xp=l(" v\xE0o b\u1ED9 t\u1EEB v\u1EF1ng:"),ka=r(),f(Bs.$$.fragment),ba=r(),as=h("p"),se=l("\u0110\u1EC3 ti\u1EBFp t\u1EE5c, ta c\u1EA7n \xE1p d\u1EE5ng s\u1EF1 h\u1EE3p nh\u1EA5t \u1EDF t\u1EEB \u0111i\u1EC3n "),gt=h("code"),ne=l("splits"),te=l(". H\xE3y c\xF9ng vi\u1EBFt m\u1ED9t h\xE0m kh\xE1c cho n\xF3:"),$a=r(),f(Os.$$.fragment),da=r(),cn=h("p"),ae=l("Gi\u1EDD ta c\xF3 th\u1EC3 nh\xECn xem k\u1EBFt qu\u1EA3 c\u1EE7a l\u1EA7n h\u1EE3p nh\u1EA5t \u0111\u1EA7u ti\xEAn:"),va=r(),f(Gs.$$.fragment),qa=r(),f(Ns.$$.fragment),_a=r(),rn=h("p"),le=l("Gi\u1EDD th\xEC ta c\xF3 t\u1EA5t c\u1EA3 nh\u1EEFng g\xEC m\xECnh c\u1EA7n \u0111\u1EC3 l\u1EB7p cho \u0111\u1EBFn khi ta h\u1ECDc t\u1EA5t c\xE1c c\xE1c h\u1EE3p nh\u1EA5t m\xE0 ta mu\u1ED1n. H\xE3y c\u0169ng nh\u1EAFm t\u1EDBi b\u1ED9 t\u1EF1 v\u1EF1ng c\xF3 k\xEDch c\u1EE1 l\xE0 50:"),ya=r(),f(Hs.$$.fragment),wa=r(),on=h("p"),pe=l("K\u1EBFt qu\u1EA3 l\xE0, ch\xFAng ta \u0111\xE3 h\u1ECDc 19 quy t\u1EAFc h\u1EE3p nh\u1EA5t (b\u1ED9 t\u1EEB \u0111i\u1EC3n g\u1ED1c c\xF3 k\xEDch c\u1EE1 l\xE0 31 t\u01B0\u01A1ng \u1EE9ng 30 k\xED t\u1EF1 trong b\u1EA3ng ch\u1EEF c\xE1i c\xF9ng m\u1ED9t token \u0111\u1EB7t bi\u1EC7t):"),Ea=r(),f(Ss.$$.fragment),Ta=r(),f(As.$$.fragment),za=r(),un=h("p"),ee=l("V\xE0 b\u1ED9 t\u1EF1 v\u1EF1ng c\u1EA5u th\xE0nh b\u1EDFi token \u0111\u1EB7c bi\u1EBFt, c\xE1c k\xED t\u1EF1 trong b\u1EA3ng ch\u1EEF c\xE1i, v\xE0 t\u1EA5t c\u1EA3 k\u1EBFt qu\u1EA3 t\u1EEB c\xE1c quy t\u1EAFc h\u1EE3p nh\u1EA5t:"),Pa=r(),f(Vs.$$.fragment),Ca=r(),f(Rs.$$.fragment),Da=r(),f(ls.$$.fragment),Ba=r(),gn=h("p"),he=l("\u0110\u1EC3 tokenize v\u0103n b\u1EA3n m\u1EDBi, ch\xFAng ta ti\u1EC1n tokenize n\xF3, t\xE1ch ra, r\u1ED3i \xE1p d\u1EE5ng quy t\u1EAFc h\u1EE3p nh\u1EA5t \u0111\u01B0\u1EE3c h\u1ECDc:"),Oa=r(),f(Is.$$.fragment),Ga=r(),fn=h("p"),ie=l(`t
Ta c\xF3 th\u1EC3 th\u1EED c\xE1c n\xE0y v\u1EDBi b\u1EA5t k\xEC \u0111o\u1EA1n v\u0103n n\xE0o kh\xE1c \u0111\u01B0\u1EE3c t\u1EA1o th\xE0nh t\u1EEB c\xE1c k\xED t\u1EF1 trong b\u1EA3ng ch\u1EEF c\xE1i:`),Na=r(),f(Ls.$$.fragment),Ha=r(),f(Us.$$.fragment),Sa=r(),f(ps.$$.fragment),Aa=r(),mn=h("p"),ce=l("\u0110\xF3 l\xE0 nh\u1EEFng g\xEC ta c\u1EA7n bi\u1EBFt v\u1EC1 thu\u1EADt to\xE1n BPE! Ti\u1EBFp theo, ch\xFAng ta s\u1EBD c\xF9ng t\xECm hi\u1EC3u v\u1EC1 WordPiece."),this.h()},l(s){const a=Ch('[data-svelte="svelte-1phssyn"]',document.head);u=i(a,"META",{name:!0,content:!0}),a.forEach(n),_=o(s),g=i(s,"H1",{class:!0});var Fs=c(g);$=i(Fs,"A",{id:!0,class:!0,href:!0});var ft=c($);E=i(ft,"SPAN",{});var mt=c(E);m(v.$$.fragment,mt),mt.forEach(n),ft.forEach(n),y=o(Fs),z=i(Fs,"SPAN",{});var jt=c(z);I=p(jt,"Byte-Pair Encoding tokenization"),jt.forEach(n),Fs.forEach(n),O=o(s),m(S.$$.fragment,s),is=o(s),Ms=i(s,"P",{});var xt=c(Ms);Wa=p(xt,"M\xE3 h\xF3a theo c\u1EB7p (BPE) ti\u1EC1n th\xE2n \u0111\u01B0\u1EE3c ph\xE1t tri\u1EC3n nh\u01B0 m\u1ED9t thu\u1EADt to\xE1n \u0111\u1EC3 n\xE9n v\u0103n b\u1EA3n, sau \u0111\xF3 \u0111\u01B0\u1EE3c OpenAI s\u1EED d\u1EE5ng \u0111\u1EC3 tokenize khi hu\u1EA5n luy\u1EC7n tr\u01B0\u1EDBc m\xF4 h\xECnh GPT. N\xF3 \u0111\u01B0\u1EE3c s\u1EED d\u1EE5ng b\u1EDFi r\u1EA5t nhi\u1EC1u m\xF4 h\xECnh Transformer, bao g\u1ED3m GPT, GPT-2, RoBERTa, BART v\xE0 DeBERTa."),xt.forEach(n),kt=o(s),m(cs.$$.fragment,s),bt=o(s),m(K.$$.fragment,s),$t=o(s),L=i(s,"H2",{class:!0});var Ks=c(L);M=i(Ks,"A",{id:!0,class:!0,href:!0});var re=c(M);$n=i(re,"SPAN",{});var oe=c($n);m(rs.$$.fragment,oe),oe.forEach(n),re.forEach(n),Ya=o(Ks),dn=i(Ks,"SPAN",{});var ue=c(dn);Ja=p(ue,"Thu\u1EADt to\xE1n hu\u1EA5n luy\u1EC7n"),ue.forEach(n),Ks.forEach(n),dt=o(s),Qs=i(s,"P",{});var ge=c(Qs);Za=p(ge,"Hu\u1EA5n luy\u1EC7n BPE b\u1EAFt \u0111\u1EA7u b\u1EB1ng c\xE1ch t\xEDnh to\xE1n t\u1EADp h\u1EE3p c\xE1c t\u1EEB duy nh\u1EA5t \u0111\u01B0\u1EE3c s\u1EED d\u1EE5ng trong kho ng\u1EEF li\u1EC7u (sau khi ho\xE0n th\xE0nh c\xE1c b\u01B0\u1EDBc chu\u1EA9n h\xF3a v\xE0 pre-tokenization), sau \u0111\xF3 x\xE2y d\u1EF1ng t\u1EEB v\u1EF1ng b\u1EB1ng c\xE1ch l\u1EA5y t\u1EA5t c\u1EA3 c\xE1c k\xFD hi\u1EC7u \u0111\u01B0\u1EE3c s\u1EED d\u1EE5ng \u0111\u1EC3 vi\u1EBFt nh\u1EEFng t\u1EEB \u0111\xF3. V\xED d\u1EE5 r\u1EA5t \u0111\u01A1n gi\u1EA3n, gi\u1EA3 s\u1EED kho d\u1EEF li\u1EC7u c\u1EE7a ch\xFAng ta s\u1EED d\u1EE5ng n\u0103m t\u1EEB sau:"),ge.forEach(n),vt=o(s),m(os.$$.fragment,s),qt=o(s),Q=i(s,"P",{});var Ra=c(Q);Xa=p(Ra,"T\u1EEB v\u1EF1ng c\u01A1 s\u1EDF khi \u0111\xF3 s\u1EBD l\xE0 "),vn=i(Ra,"CODE",{});var fe=c(vn);sl=p(fe,'["b", "g", "h", "n", "p", "s", "u"]'),fe.forEach(n),nl=p(Ra,". \u0110\u1ED1i v\u1EDBi c\xE1c tr\u01B0\u1EDDng h\u1EE3p trong th\u1EF1c t\u1EBF, t\u1EEB v\u1EF1ng c\u01A1 s\u1EDF \u0111\xF3 s\u1EBD ch\u1EE9a t\u1EA5t c\u1EA3 c\xE1c k\xFD t\u1EF1 ASCII, \xEDt nh\u1EA5t v\xE0 c\xF3 th\u1EC3 l\xE0 m\u1ED9t s\u1ED1 k\xFD t\u1EF1 Unicode. N\u1EBFu m\u1ED9t m\u1EABu b\u1EA1n \u0111ang tokenize s\u1EED d\u1EE5ng m\u1ED9t k\xFD t\u1EF1 kh\xF4ng c\xF3 trong kho d\u1EEF li\u1EC7u hu\u1EA5n luy\u1EC7n, th\xEC k\xFD t\u1EF1 \u0111\xF3 s\u1EBD \u0111\u01B0\u1EE3c chuy\u1EC3n \u0111\u1ED5i th\xE0nh token kh\xF4ng x\xE1c \u0111\u1ECBnh. \u0110\xF3 l\xE0 m\u1ED9t l\xFD do t\u1EA1i sao nhi\u1EC1u m\xF4 h\xECnh NLP r\u1EA5t k\xE9m trong vi\u1EC7c ph\xE2n t\xEDch n\u1ED9i dung b\u1EB1ng bi\u1EC3u t\u01B0\u1EE3ng c\u1EA3m x\xFAc."),Ra.forEach(n),_t=o(s),m(W.$$.fragment,s),yt=o(s),Y=i(s,"P",{});var Ia=c(Y);tl=p(Ia,"Sau khi c\xF3 \u0111\u01B0\u1EE3c b\u1ED9 t\u1EEB v\u1EF1ng c\u01A1 b\u1EA3n n\xE0y, ch\xFAng ta th\xEAm c\xE1c token m\u1EDBi cho \u0111\u1EBFn khi \u0111\u1EA1t \u0111\u01B0\u1EE3c k\xEDch th\u01B0\u1EDBc t\u1EEB v\u1EF1ng mong mu\u1ED1n b\u1EB1ng c\xE1ch h\u1ECDc "),qn=i(Ia,"EM",{});var me=c(qn);al=p(me,"h\u1EE3p nh\u1EA5t"),me.forEach(n),ll=p(Ia,", \u0111\xE2y l\xE0 c\xE1c quy t\u1EAFc \u0111\u1EC3 h\u1EE3p nh\u1EA5t hai y\u1EBFu t\u1ED1 c\u1EE7a t\u1EEB v\u1EF1ng hi\u1EC7n c\xF3 v\u1EDBi nhau th\xE0nh m\u1ED9t t\u1EEB m\u1EDBi. V\xEC v\u1EADy, l\xFAc \u0111\u1EA7u s\u1EF1 h\u1EE3p nh\u1EA5t n\xE0y s\u1EBD t\u1EA1o ra c\xE1c token c\xF3 hai k\xFD t\u1EF1 v\xE0 sau \u0111\xF3, khi qu\xE1 tr\xECnh hu\u1EA5n luy\u1EC7n ti\u1EBFn tri\u1EC3n, c\xE1c t\u1EEB ph\u1EE5 s\u1EBD d\xE0i h\u01A1n."),Ia.forEach(n),wt=o(s),Ws=i(s,"P",{});var je=c(Ws);pl=p(je,"T\u1EA1i b\u1EA5t k\u1EF3 b\u01B0\u1EDBc n\xE0o trong qu\xE1 tr\xECnh hu\u1EA5n luy\u1EC7n token, thu\u1EADt to\xE1n BPE s\u1EBD t\xECm ki\u1EBFm c\u1EB7p token hi\u1EC7n c\xF3 th\u01B0\u1EDDng xuy\xEAn nh\u1EA5t (theo \u201Cc\u1EB7p\u201D, \u1EDF \u0111\xE2y c\xF3 ngh\u0129a l\xE0 hai token li\xEAn ti\u1EBFp trong m\u1ED9t t\u1EEB). C\u1EB7p th\u01B0\u1EDDng xuy\xEAn nh\u1EA5t \u0111\xF3 l\xE0 c\u1EB7p s\u1EBD \u0111\u01B0\u1EE3c h\u1EE3p nh\u1EA5t, v\xE0 ch\xFAng ta x\u1EA3 v\xE0 l\u1EB7p l\u1EA1i cho b\u01B0\u1EDBc ti\u1EBFp theo."),je.forEach(n),Et=o(s),Ys=i(s,"P",{});var xe=c(Ys);el=p(xe,"Quay tr\u1EDF l\u1EA1i v\xED d\u1EE5 tr\u01B0\u1EDBc, gi\u1EA3 s\u1EED c\xE1c t\u1EEB c\xF3 t\u1EA7n s\u1ED1 nh\u01B0 sau:"),xe.forEach(n),Tt=o(s),m(us.$$.fragment,s),zt=o(s),C=i(s,"P",{});var H=c(C);hl=p(H,"ngh\u0129a l\xE0 "),_n=i(H,"CODE",{});var ke=c(_n);il=p(ke,'"hug"'),ke.forEach(n),cl=p(H," c\xF3 m\u1EB7t 10 l\u1EA7n trong kho ng\u1EEF li\u1EC7u, "),yn=i(H,"CODE",{});var be=c(yn);rl=p(be,'"pug"'),be.forEach(n),ol=p(H," 5 l\u1EA7n, "),wn=i(H,"CODE",{});var $e=c(wn);ul=p($e,'"pun"'),$e.forEach(n),gl=p(H," 12 l\u1EA7n, "),En=i(H,"CODE",{});var de=c(En);fl=p(de,'"bun"'),de.forEach(n),ml=p(H," 4 l\u1EA7n v\xE0 "),Tn=i(H,"CODE",{});var ve=c(Tn);jl=p(ve,'"hug"'),ve.forEach(n),xl=p(H," 5 l\u1EA7n. Ch\xFAng ta b\u1EAFt \u0111\u1EA7u hu\u1EA5n luy\u1EC7n b\u1EB1ng c\xE1ch t\xE1ch t\u1EEBng t\u1EEB th\xE0nh c\xE1c k\xFD t\u1EF1 (nh\u1EEFng k\xFD t\u1EF1 h\xECnh th\xE0nh t\u1EEB v\u1EF1ng ban \u0111\u1EA7u c\u1EE7a ch\xFAng ta) \u0111\u1EC3 c\xF3 th\u1EC3 xem m\u1ED7i t\u1EEB nh\u01B0 m\u1ED9t danh s\xE1ch c\xE1c token:"),H.forEach(n),Pt=o(s),m(gs.$$.fragment,s),Ct=o(s),T=i(s,"P",{});var D=c(T);kl=p(D,"Sau \u0111\xF3, ch\xFAng ta xem x\xE9t c\xE1c c\u1EB7p. C\u1EB7p "),zn=i(D,"CODE",{});var qe=c(zn);bl=p(qe,'("h", "u")'),qe.forEach(n),$l=p(D," c\xF3 trong c\xE1c t\u1EEB  "),Pn=i(D,"CODE",{});var _e=c(Pn);dl=p(_e,'"hug"'),_e.forEach(n),vl=p(D," v\xE0  "),Cn=i(D,"CODE",{});var ye=c(Cn);ql=p(ye,'"hugs"'),ye.forEach(n),_l=p(D,", v\xEC v\u1EADy t\u1ED5ng c\u1ED9ng l\xE0 15 l\u1EA7n trong ng\u1EEF li\u1EC7u. Tuy nhi\xEAn, \u0111\xE2y kh\xF4ng ph\u1EA3i l\xE0 c\u1EB7p th\u01B0\u1EDDng xuy\xEAn nh\u1EA5t: vinh d\u1EF1 \u0111\xF3 thu\u1ED9c v\u1EC1 "),Dn=i(D,"CODE",{});var we=c(Dn);yl=p(we,'("u", "g")'),we.forEach(n),wl=p(D,", c\xF3 trong "),Bn=i(D,"CODE",{});var Ee=c(Bn);El=p(Ee,'"hug"'),Ee.forEach(n),Tl=p(D,", "),On=i(D,"CODE",{});var Te=c(On);zl=p(Te,'"pug"'),Te.forEach(n),Pl=p(D,", v\xE0 "),Gn=i(D,"CODE",{});var ze=c(Gn);Cl=p(ze,'"hugs"'),ze.forEach(n),Dl=p(D,", v\u1EDBi t\u1ED5ng c\u1ED9ng 20 l\u1EA7n xu\u1EA5t hi\u1EC7n trong b\u1ED9 t\u1EEB v\u1EF1ng."),D.forEach(n),Dt=o(s),A=i(s,"P",{});var jn=c(A);Bl=p(jn,"Do \u0111\xF3, quy t\u1EAFc h\u1EE3p nh\u1EA5t \u0111\u1EA7u ti\xEAn \u0111\u01B0\u1EE3c h\u1ECDc b\u1EDFi tokenizer l\xE0 "),Nn=i(jn,"CODE",{});var Pe=c(Nn);Ol=p(Pe,'("u", "g") -> "ug"'),Pe.forEach(n),Gl=p(jn,", c\xF3 ngh\u0129a l\xE0 "),Hn=i(jn,"CODE",{});var Ce=c(Hn);Nl=p(Ce,'"ug"'),Ce.forEach(n),Hl=p(jn," s\u1EBD \u0111\u01B0\u1EE3c th\xEAm v\xE0o t\u1EEB v\u1EF1ng v\xE0 c\u1EB7p n\xE0y s\u1EBD \u0111\u01B0\u1EE3c h\u1EE3p nh\u1EA5t trong t\u1EA5t c\u1EA3 c\xE1c t\u1EEB c\u1EE7a ng\u1EEF li\u1EC7u. V\xE0o cu\u1ED1i giai \u0111o\u1EA1n n\xE0y, t\u1EEB v\u1EF1ng v\xE0 ng\u1EEF li\u1EC7u s\u1EBD gi\u1ED1ng nh\u01B0 sau:"),jn.forEach(n),Bt=o(s),m(fs.$$.fragment,s),Ot=o(s),G=i(s,"P",{});var es=c(G);Sl=p(es,"B\xE2y gi\u1EDD ch\xFAng ta c\xF3 m\u1ED9t s\u1ED1 c\u1EB7p d\u1EABn \u0111\u1EBFn m\u1ED9t token d\xE0i h\u01A1n hai k\xFD t\u1EF1: v\xED d\u1EE5: c\u1EB7p "),Sn=i(es,"CODE",{});var De=c(Sn);Al=p(De,'("h", "ug")'),De.forEach(n),Vl=p(es,", (hi\u1EC7n di\u1EC7n 15 l\u1EA7n trong kho ng\u1EEF li\u1EC7u). C\u1EB7p th\u01B0\u1EDDng g\u1EB7p nh\u1EA5t \u1EDF giai \u0111o\u1EA1n n\xE0y l\xE0 "),An=i(es,"CODE",{});var Be=c(An);Rl=p(Be,'("u", "n")'),Be.forEach(n),Il=p(es,", xu\u1EA5t hi\u1EC7n 16 l\u1EA7n trong kho ng\u1EEF li\u1EC7u, v\xEC v\u1EADy quy t\u1EAFc h\u1EE3p nh\u1EA5t th\u1EE9 hai \u0111\xE3 h\u1ECDc l\xE0 "),Vn=i(es,"CODE",{});var Oe=c(Vn);Ll=p(Oe,'("u", "n") -> "un"'),Oe.forEach(n),Ul=p(es,". Th\xEAm n\xF3 v\xE0o b\u1ED9 t\u1EEB v\u1EF1ng v\xE0 h\u1EE3p nh\u1EA5t t\u1EA5t c\u1EA3 c\xE1c l\u1EA7n xu\u1EA5t hi\u1EC7n hi\u1EC7n c\xF3 s\u1EBD d\u1EABn ch\xFAng ta \u0111\u1EBFn:"),es.forEach(n),Gt=o(s),m(ms.$$.fragment,s),Nt=o(s),V=i(s,"P",{});var xn=c(V);Fl=p(xn,"Gi\u1EDD th\xEC c\u1EB7p xu\u1EA5t hi\u1EC7n nhi\u1EC1u nh\u1EA5t l\xE0 "),Rn=i(xn,"CODE",{});var Ge=c(Rn);Kl=p(Ge,'("h", "ug")'),Ge.forEach(n),Ml=p(xn,", n\xEAn ch\xFAng ta h\u1EE3p nh\u1EA5t "),In=i(xn,"CODE",{});var Ne=c(In);Ql=p(Ne,'("h", "ug") -> "hug"'),Ne.forEach(n),Wl=p(xn,", tr\u1EA3 v\u1EC1 cho ch\xFAng ta token g\u1ED3n ba k\xED t\u1EF1 \u0111\u1EA7u ti\xEAn. Sau s\u1EF1 h\u1EE3p nh\u1EA5t n\xE0y, kho ng\u1EEF li\u1EC7u s\u1EBD nh\u01B0 sau:"),xn.forEach(n),Ht=o(s),m(js.$$.fragment,s),St=o(s),Js=i(s,"P",{});var He=c(Js);Yl=p(He,"V\xE0 ch\xFAng ta ti\u1EBFp t\xFAc l\xE0m v\u1EADy cho \u0111\u1EBFn khi ch\xFAng ta ch\u1EA1m \u0111\u1EBFn k\xEDch th\u01B0\u1EDBc b\u1ED9 t\u1EF1 \u0111i\u1EC3n ta mong mu\u1ED1n."),He.forEach(n),At=o(s),m(J.$$.fragment,s),Vt=o(s),U=i(s,"H2",{class:!0});var La=c(U);Z=i(La,"A",{id:!0,class:!0,href:!0});var Se=c(Z);Ln=i(Se,"SPAN",{});var Ae=c(Ln);m(xs.$$.fragment,Ae),Ae.forEach(n),Se.forEach(n),Jl=o(La),Un=i(La,"SPAN",{});var Ve=c(Un);Zl=p(Ve,"Thu\u1EADt to\xE1n tokenize"),Ve.forEach(n),La.forEach(n),Rt=o(s),Zs=i(s,"P",{});var Re=c(Zs);Xl=p(Re,"Tokenize tu\xE2n th\u1EE7 ch\u1EB7t ch\u1EBD qu\xE1 tr\xECnh hu\u1EA5n luy\u1EC7n, theo ngh\u0129a l\xE0 c\xE1c \u0111\u1EA7u v\xE0o m\u1EDBi \u0111\u01B0\u1EE3c tokenize b\u1EB1ng c\xE1ch \xE1p d\u1EE5ng c\xE1c b\u01B0\u1EDBc sau:"),Re.forEach(n),It=o(s),N=i(s,"OL",{});var hs=c(N);Fn=i(hs,"LI",{});var Ie=c(Fn);sp=p(Ie,"Chu\u1EA9n ho\xE1"),Ie.forEach(n),np=o(hs),Kn=i(hs,"LI",{});var Le=c(Kn);tp=p(Le,"Pre-tokenization"),Le.forEach(n),ap=o(hs),Mn=i(hs,"LI",{});var Ue=c(Mn);lp=p(Ue,"T\xE1ch c\xE1c t\u1EEB th\xE0nh c\xE1c k\xFD t\u1EF1 ri\xEAng l\u1EBB"),Ue.forEach(n),pp=o(hs),Qn=i(hs,"LI",{});var Fe=c(Qn);ep=p(Fe,"\xC1p d\u1EE5ng c\xE1c quy t\u1EAFc h\u1EE3p nh\u1EA5t \u0111\xE3 h\u1ECDc theo th\u1EE9 t\u1EF1 tr\xEAn c\xE1c ph\u1EA7n t\xE1ch \u0111\xF3"),Fe.forEach(n),hs.forEach(n),Lt=o(s),Xs=i(s,"P",{});var Ke=c(Xs);hp=p(Ke,"L\u1EA5y v\xED d\u1EE5 m\xE0 ta \u0111\xE3 s\u1EED d\u1EE5ng trong qu\xE1 tr\xECnh hu\u1EA5n luy\u1EC7n, v\u1EDBi ba quy t\u1EAFc h\u1EE3p nh\u1EA5t \u0111\xE3 h\u1ECDc:"),Ke.forEach(n),Ut=o(s),m(ks.$$.fragment,s),Ft=o(s),q=i(s,"P",{});var w=c(q);ip=p(w,"T\u1EEB "),Wn=i(w,"CODE",{});var Me=c(Wn);cp=p(Me,'"bug"'),Me.forEach(n),rp=p(w," s\u1EBD \u0111\u01B0\u1EE3c tokenize th\xE0nh "),Yn=i(w,"CODE",{});var Qe=c(Yn);op=p(Qe,'["b", "ug"]'),Qe.forEach(n),up=p(w,". "),Jn=i(w,"CODE",{});var We=c(Jn);gp=p(We,'"mug"'),We.forEach(n),fp=p(w,", tuy nhi\xEAn, s\u1EBD tokenize th\xE0nh "),Zn=i(w,"CODE",{});var Ye=c(Zn);mp=p(Ye,'["[UNK]", "ug"]'),Ye.forEach(n),jp=p(w," v\xEC k\xED t\u1EF1 "),Xn=i(w,"CODE",{});var Je=c(Xn);xp=p(Je,'"m"'),Je.forEach(n),kp=p(w," kh\xF4ng c\xF3 trong b\u1ED9 t\u1EF1 v\u1EF1ng g\u1ED1c. T\u01B0\u01A1ng t\u1EF1, t\u1EEB "),st=i(w,"CODE",{});var Ze=c(st);bp=p(Ze,'"thug"'),Ze.forEach(n),$p=p(w," s\u1EBD \u0111\u01B0\u1EE3c tokenize th\xE0nh  "),nt=i(w,"CODE",{});var Xe=c(nt);dp=p(Xe,'["[UNK]", "hug"]'),Xe.forEach(n),vp=p(w,": k\xED t\u1EF1 "),tt=i(w,"CODE",{});var sh=c(tt);qp=p(sh,'"t"'),sh.forEach(n),_p=p(w," kh\xF4ng c\xF3 trong b\u1ED9 t\u1EF1 v\u1EF1ng g\u1ED1c, v\xE0 \xE1p d\u1EE5ng quy t\u1EAFc h\u1EE3p nh\u1EA5t \u1EDF "),at=i(w,"CODE",{});var nh=c(at);yp=p(nh,'"u"'),nh.forEach(n),wp=p(w," v\xE0 "),lt=i(w,"CODE",{});var th=c(lt);Ep=p(th,'"g"'),th.forEach(n),Tp=p(w," v\xE0 sau \u0111\xF3 "),pt=i(w,"CODE",{});var ah=c(pt);zp=p(ah,'"hu"'),ah.forEach(n),Pp=p(w," v\xE0 "),et=i(w,"CODE",{});var lh=c(et);Cp=p(lh,'"g"'),lh.forEach(n),Dp=p(w,"."),w.forEach(n),Kt=o(s),m(X.$$.fragment,s),Mt=o(s),F=i(s,"H2",{class:!0});var Ua=c(F);ss=i(Ua,"A",{id:!0,class:!0,href:!0});var ph=c(ss);ht=i(ph,"SPAN",{});var eh=c(ht);m(bs.$$.fragment,eh),eh.forEach(n),ph.forEach(n),Bp=o(Ua),it=i(Ua,"SPAN",{});var hh=c(it);Op=p(hh,"Tri\u1EC3n khai BPE"),hh.forEach(n),Ua.forEach(n),Qt=o(s),sn=i(s,"P",{});var ih=c(sn);Gp=p(ih,"H\xE3y c\xF9ng xem c\xE1c thu\u1EADt to\xE1n BPE \u0111\u01B0\u1EE3c tri\u1EC3n khai. \u0110\xE2y kh\xF4ng ph\u1EA3i l\xE0 phi\xEAn b\u1EA3n t\u1ED1i \u01B0u m\xE0 b\u1EA1n c\xF3 th\u1EC3 th\u1EF1c s\u1EF1 s\u1EED d\u1EE5ng cho m\u1ED9t kho ng\u1EEF li\u1EC7u l\u1EDBn; ch\xFAng t\xF4i ch\u1EC9 mu\u1ED1n cho b\u1EA1n xem \u0111o\u1EA1n m\xE3 \u0111\u1EC3 b\u1EA1n c\xF3 th\u1EC3 hi\u1EC3u thu\u1EADt to\xE1n n\xE0y t\u1ED1t h\u01A1n."),ih.forEach(n),Wt=o(s),nn=i(s,"P",{});var ch=c(nn);Np=p(ch,"\u0110\u1EA7u ti\xEAn ch\xFAng ta c\u1EA7n m\u1ED9t kho ng\u1EEF li\u1EC7u, v\u1EADy n\xEAn hay t\u1EA1o ra m\u1ED9t b\u1EA3n \u0111\u01A1n gi\u1EA3n v\u1EDBi m\u1ED9t v\xE0i c\xE2u:"),ch.forEach(n),Yt=o(s),m($s.$$.fragment,s),Jt=o(s),ns=i(s,"P",{});var Fa=c(ns);Hp=p(Fa,"Ti\u1EBFp theo, ta c\u1EA7n ti\u1EC1n tokenize kho ng\u1EEF li\u1EC7u n\xE0y th\xE0nh c\xE1c t\u1EEB. V\xEC ta \u0111ang sao ch\xE9p m\u1ED9t b\u1EA3n BPE tokenizer (nh\u01B0 GPT-2), ta v\u1EABn c\xF3 th\u1EC3 s\u1EED d\u1EE5ng "),ct=i(Fa,"CODE",{});var rh=c(ct);Sp=p(rh,"gpt2"),rh.forEach(n),Ap=p(Fa," tokenize cho b\u01B0\u1EDBc pre-tokenization:"),Fa.forEach(n),Zt=o(s),m(ds.$$.fragment,s),Xt=o(s),tn=i(s,"P",{});var oh=c(tn);Vp=p(oh,"Sau \u0111\xF3 ta t\xEDnh t\u1EA7n su\u1EA5t c\u1EE7a t\u1EEBng t\u1EEB trong kho ng\u1EEF li\u1EC7u nh\u01B0 khi l\xE0m v\u1EDBi pre-tokenization:"),oh.forEach(n),sa=o(s),m(vs.$$.fragment,s),na=o(s),m(qs.$$.fragment,s),ta=o(s),an=i(s,"P",{});var uh=c(an);Rp=p(uh,"Ti\u1EBFp theo ch\xFAng ta s\u1EBD t\xEDnh b\u1ED9 t\u1EEB v\u1EF1ng c\u01A1 s\u1EDF t\u1EEB c\xE1c k\xED t\u1EF1 s\u1EED d\u1EE5ng trong kho ng\u1EEF li\u1EC7u:"),uh.forEach(n),aa=o(s),m(_s.$$.fragment,s),la=o(s),m(ys.$$.fragment,s),pa=o(s),ts=i(s,"P",{});var Ka=c(ts);Ip=p(Ka,"Ta c\u0169ng c\xF3 th\u1EC3 th\xEAm c\xE1c token \u0111\u1EB7c bi\u1EC7t t\u1EEB m\xF4 h\xECnh \u1EDF \u0111\u1EA7u c\u1EE7a b\u1ED9 t\u1EF1 v\u1EF1ng. Trong tr\u01B0\u1EDDng h\u1EE3p c\u1EE7a GPT-2, token \u0111\u1EB7c bi\u1EC7t duy nh\u1EA5t \u0111\xF3 l\xE0 "),rt=i(Ka,"CODE",{});var gh=c(rt);Lp=p(gh,'"<|endoftext|>"'),gh.forEach(n),Up=p(Ka,":"),Ka.forEach(n),ea=o(s),m(ws.$$.fragment,s),ha=o(s),ln=i(s,"P",{});var fh=c(ln);Fp=p(fh,"Ta gi\u1EDD c\u1EA7n ph\u1EA3i chia m\u1ED7i t\u1EEB th\xE0nh c\xE1c k\xED t\u1EF1 ri\xEAng l\u1EBB \u0111\u1EC3 c\xF3 th\u1EC3 b\u1EAFt \u0111\u1EA7u hu\u1EA5n luy\u1EC7n"),fh.forEach(n),ia=o(s),m(Es.$$.fragment,s),ca=o(s),pn=i(s,"P",{});var mh=c(pn);Kp=p(mh,"Gi\u1EDD ta \u0111\xE3 s\u1EB5n s\xE0ng \u0111\u1EC3 hu\u1EA5n luy\u1EC7n, h\xE3y c\xF9ng vi\u1EBFt m\u1ED9t h\xE0m t\xEDnh t\u1EA7n su\u1EA5t m\u1ED7i c\u1EB7p. Ta s\u1EBD c\u1EA7n s\u1EED d\u1EE5ng n\xF3 \u1EDF b\u01B0\u1EDBc hu\u1EA5n luy\u1EC7n:"),mh.forEach(n),ra=o(s),m(Ts.$$.fragment,s),oa=o(s),en=i(s,"P",{});var jh=c(en);Mp=p(jh,"H\xE3y nh\xECn v\xE0o m\u1ED9t ph\u1EA7n t\u1EEB \u0111i\u1EBBn sau khi t\xE1ch:"),jh.forEach(n),ua=o(s),m(zs.$$.fragment,s),ga=o(s),m(Ps.$$.fragment,s),fa=o(s),hn=i(s,"P",{});var xh=c(hn);Qp=p(xh,"Gi\u1EDD th\xEC, t\xECm xem c\u1EB7p xu\u1EA5t hi\u1EC7n nhi\u1EC1u nh\u1EA5t b\u1EB1ng m\u1ED9t v\xF2ng l\u1EB7p nhanh:"),xh.forEach(n),ma=o(s),m(Cs.$$.fragment,s),ja=o(s),m(Ds.$$.fragment,s),xa=o(s),R=i(s,"P",{});var kn=c(R);Wp=p(kn,"V\u1EADy ph\xE9p h\u1EE3p nh\u1EA5t \u0111\u1EA7u ti\xEAn l\xE0 "),ot=i(kn,"CODE",{});var kh=c(ot);Yp=p(kh,"('\u0120', 't') -> '\u0120t'"),kh.forEach(n),Jp=p(kn,", v\xE0 ta th\xEAm "),ut=i(kn,"CODE",{});var bh=c(ut);Zp=p(bh,"'\u0120t'"),bh.forEach(n),Xp=p(kn," v\xE0o b\u1ED9 t\u1EEB v\u1EF1ng:"),kn.forEach(n),ka=o(s),m(Bs.$$.fragment,s),ba=o(s),as=i(s,"P",{});var Ma=c(as);se=p(Ma,"\u0110\u1EC3 ti\u1EBFp t\u1EE5c, ta c\u1EA7n \xE1p d\u1EE5ng s\u1EF1 h\u1EE3p nh\u1EA5t \u1EDF t\u1EEB \u0111i\u1EC3n "),gt=i(Ma,"CODE",{});var $h=c(gt);ne=p($h,"splits"),$h.forEach(n),te=p(Ma,". H\xE3y c\xF9ng vi\u1EBFt m\u1ED9t h\xE0m kh\xE1c cho n\xF3:"),Ma.forEach(n),$a=o(s),m(Os.$$.fragment,s),da=o(s),cn=i(s,"P",{});var dh=c(cn);ae=p(dh,"Gi\u1EDD ta c\xF3 th\u1EC3 nh\xECn xem k\u1EBFt qu\u1EA3 c\u1EE7a l\u1EA7n h\u1EE3p nh\u1EA5t \u0111\u1EA7u ti\xEAn:"),dh.forEach(n),va=o(s),m(Gs.$$.fragment,s),qa=o(s),m(Ns.$$.fragment,s),_a=o(s),rn=i(s,"P",{});var vh=c(rn);le=p(vh,"Gi\u1EDD th\xEC ta c\xF3 t\u1EA5t c\u1EA3 nh\u1EEFng g\xEC m\xECnh c\u1EA7n \u0111\u1EC3 l\u1EB7p cho \u0111\u1EBFn khi ta h\u1ECDc t\u1EA5t c\xE1c c\xE1c h\u1EE3p nh\u1EA5t m\xE0 ta mu\u1ED1n. H\xE3y c\u0169ng nh\u1EAFm t\u1EDBi b\u1ED9 t\u1EF1 v\u1EF1ng c\xF3 k\xEDch c\u1EE1 l\xE0 50:"),vh.forEach(n),ya=o(s),m(Hs.$$.fragment,s),wa=o(s),on=i(s,"P",{});var qh=c(on);pe=p(qh,"K\u1EBFt qu\u1EA3 l\xE0, ch\xFAng ta \u0111\xE3 h\u1ECDc 19 quy t\u1EAFc h\u1EE3p nh\u1EA5t (b\u1ED9 t\u1EEB \u0111i\u1EC3n g\u1ED1c c\xF3 k\xEDch c\u1EE1 l\xE0 31 t\u01B0\u01A1ng \u1EE9ng 30 k\xED t\u1EF1 trong b\u1EA3ng ch\u1EEF c\xE1i c\xF9ng m\u1ED9t token \u0111\u1EB7t bi\u1EC7t):"),qh.forEach(n),Ea=o(s),m(Ss.$$.fragment,s),Ta=o(s),m(As.$$.fragment,s),za=o(s),un=i(s,"P",{});var _h=c(un);ee=p(_h,"V\xE0 b\u1ED9 t\u1EF1 v\u1EF1ng c\u1EA5u th\xE0nh b\u1EDFi token \u0111\u1EB7c bi\u1EBFt, c\xE1c k\xED t\u1EF1 trong b\u1EA3ng ch\u1EEF c\xE1i, v\xE0 t\u1EA5t c\u1EA3 k\u1EBFt qu\u1EA3 t\u1EEB c\xE1c quy t\u1EAFc h\u1EE3p nh\u1EA5t:"),_h.forEach(n),Pa=o(s),m(Vs.$$.fragment,s),Ca=o(s),m(Rs.$$.fragment,s),Da=o(s),m(ls.$$.fragment,s),Ba=o(s),gn=i(s,"P",{});var yh=c(gn);he=p(yh,"\u0110\u1EC3 tokenize v\u0103n b\u1EA3n m\u1EDBi, ch\xFAng ta ti\u1EC1n tokenize n\xF3, t\xE1ch ra, r\u1ED3i \xE1p d\u1EE5ng quy t\u1EAFc h\u1EE3p nh\u1EA5t \u0111\u01B0\u1EE3c h\u1ECDc:"),yh.forEach(n),Oa=o(s),m(Is.$$.fragment,s),Ga=o(s),fn=i(s,"P",{});var wh=c(fn);ie=p(wh,`t
Ta c\xF3 th\u1EC3 th\u1EED c\xE1c n\xE0y v\u1EDBi b\u1EA5t k\xEC \u0111o\u1EA1n v\u0103n n\xE0o kh\xE1c \u0111\u01B0\u1EE3c t\u1EA1o th\xE0nh t\u1EEB c\xE1c k\xED t\u1EF1 trong b\u1EA3ng ch\u1EEF c\xE1i:`),wh.forEach(n),Na=o(s),m(Ls.$$.fragment,s),Ha=o(s),m(Us.$$.fragment,s),Sa=o(s),m(ps.$$.fragment,s),Aa=o(s),mn=i(s,"P",{});var Eh=c(mn);ce=p(Eh,"\u0110\xF3 l\xE0 nh\u1EEFng g\xEC ta c\u1EA7n bi\u1EBFt v\u1EC1 thu\u1EADt to\xE1n BPE! Ti\u1EBFp theo, ch\xFAng ta s\u1EBD c\xF9ng t\xECm hi\u1EC3u v\u1EC1 WordPiece."),Eh.forEach(n),this.h()},h(){P(u,"name","hf:doc:metadata"),P(u,"content",JSON.stringify(Ih)),P($,"id","bytepair-encoding-tokenization"),P($,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),P($,"href","#bytepair-encoding-tokenization"),P(g,"class","relative group"),P(M,"id","thut-ton-hun-luyn"),P(M,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),P(M,"href","#thut-ton-hun-luyn"),P(L,"class","relative group"),P(Z,"id","thut-ton-tokenize"),P(Z,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),P(Z,"href","#thut-ton-tokenize"),P(U,"class","relative group"),P(ss,"id","trin-khai-bpe"),P(ss,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),P(ss,"href","#trin-khai-bpe"),P(F,"class","relative group")},m(s,a){t(document.head,u),e(s,_,a),e(s,g,a),t(g,$),t($,E),j(v,E,null),t(g,y),t(g,z),t(z,I),e(s,O,a),j(S,s,a),e(s,is,a),e(s,Ms,a),t(Ms,Wa),e(s,kt,a),j(cs,s,a),e(s,bt,a),j(K,s,a),e(s,$t,a),e(s,L,a),t(L,M),t(M,$n),j(rs,$n,null),t(L,Ya),t(L,dn),t(dn,Ja),e(s,dt,a),e(s,Qs,a),t(Qs,Za),e(s,vt,a),j(os,s,a),e(s,qt,a),e(s,Q,a),t(Q,Xa),t(Q,vn),t(vn,sl),t(Q,nl),e(s,_t,a),j(W,s,a),e(s,yt,a),e(s,Y,a),t(Y,tl),t(Y,qn),t(qn,al),t(Y,ll),e(s,wt,a),e(s,Ws,a),t(Ws,pl),e(s,Et,a),e(s,Ys,a),t(Ys,el),e(s,Tt,a),j(us,s,a),e(s,zt,a),e(s,C,a),t(C,hl),t(C,_n),t(_n,il),t(C,cl),t(C,yn),t(yn,rl),t(C,ol),t(C,wn),t(wn,ul),t(C,gl),t(C,En),t(En,fl),t(C,ml),t(C,Tn),t(Tn,jl),t(C,xl),e(s,Pt,a),j(gs,s,a),e(s,Ct,a),e(s,T,a),t(T,kl),t(T,zn),t(zn,bl),t(T,$l),t(T,Pn),t(Pn,dl),t(T,vl),t(T,Cn),t(Cn,ql),t(T,_l),t(T,Dn),t(Dn,yl),t(T,wl),t(T,Bn),t(Bn,El),t(T,Tl),t(T,On),t(On,zl),t(T,Pl),t(T,Gn),t(Gn,Cl),t(T,Dl),e(s,Dt,a),e(s,A,a),t(A,Bl),t(A,Nn),t(Nn,Ol),t(A,Gl),t(A,Hn),t(Hn,Nl),t(A,Hl),e(s,Bt,a),j(fs,s,a),e(s,Ot,a),e(s,G,a),t(G,Sl),t(G,Sn),t(Sn,Al),t(G,Vl),t(G,An),t(An,Rl),t(G,Il),t(G,Vn),t(Vn,Ll),t(G,Ul),e(s,Gt,a),j(ms,s,a),e(s,Nt,a),e(s,V,a),t(V,Fl),t(V,Rn),t(Rn,Kl),t(V,Ml),t(V,In),t(In,Ql),t(V,Wl),e(s,Ht,a),j(js,s,a),e(s,St,a),e(s,Js,a),t(Js,Yl),e(s,At,a),j(J,s,a),e(s,Vt,a),e(s,U,a),t(U,Z),t(Z,Ln),j(xs,Ln,null),t(U,Jl),t(U,Un),t(Un,Zl),e(s,Rt,a),e(s,Zs,a),t(Zs,Xl),e(s,It,a),e(s,N,a),t(N,Fn),t(Fn,sp),t(N,np),t(N,Kn),t(Kn,tp),t(N,ap),t(N,Mn),t(Mn,lp),t(N,pp),t(N,Qn),t(Qn,ep),e(s,Lt,a),e(s,Xs,a),t(Xs,hp),e(s,Ut,a),j(ks,s,a),e(s,Ft,a),e(s,q,a),t(q,ip),t(q,Wn),t(Wn,cp),t(q,rp),t(q,Yn),t(Yn,op),t(q,up),t(q,Jn),t(Jn,gp),t(q,fp),t(q,Zn),t(Zn,mp),t(q,jp),t(q,Xn),t(Xn,xp),t(q,kp),t(q,st),t(st,bp),t(q,$p),t(q,nt),t(nt,dp),t(q,vp),t(q,tt),t(tt,qp),t(q,_p),t(q,at),t(at,yp),t(q,wp),t(q,lt),t(lt,Ep),t(q,Tp),t(q,pt),t(pt,zp),t(q,Pp),t(q,et),t(et,Cp),t(q,Dp),e(s,Kt,a),j(X,s,a),e(s,Mt,a),e(s,F,a),t(F,ss),t(ss,ht),j(bs,ht,null),t(F,Bp),t(F,it),t(it,Op),e(s,Qt,a),e(s,sn,a),t(sn,Gp),e(s,Wt,a),e(s,nn,a),t(nn,Np),e(s,Yt,a),j($s,s,a),e(s,Jt,a),e(s,ns,a),t(ns,Hp),t(ns,ct),t(ct,Sp),t(ns,Ap),e(s,Zt,a),j(ds,s,a),e(s,Xt,a),e(s,tn,a),t(tn,Vp),e(s,sa,a),j(vs,s,a),e(s,na,a),j(qs,s,a),e(s,ta,a),e(s,an,a),t(an,Rp),e(s,aa,a),j(_s,s,a),e(s,la,a),j(ys,s,a),e(s,pa,a),e(s,ts,a),t(ts,Ip),t(ts,rt),t(rt,Lp),t(ts,Up),e(s,ea,a),j(ws,s,a),e(s,ha,a),e(s,ln,a),t(ln,Fp),e(s,ia,a),j(Es,s,a),e(s,ca,a),e(s,pn,a),t(pn,Kp),e(s,ra,a),j(Ts,s,a),e(s,oa,a),e(s,en,a),t(en,Mp),e(s,ua,a),j(zs,s,a),e(s,ga,a),j(Ps,s,a),e(s,fa,a),e(s,hn,a),t(hn,Qp),e(s,ma,a),j(Cs,s,a),e(s,ja,a),j(Ds,s,a),e(s,xa,a),e(s,R,a),t(R,Wp),t(R,ot),t(ot,Yp),t(R,Jp),t(R,ut),t(ut,Zp),t(R,Xp),e(s,ka,a),j(Bs,s,a),e(s,ba,a),e(s,as,a),t(as,se),t(as,gt),t(gt,ne),t(as,te),e(s,$a,a),j(Os,s,a),e(s,da,a),e(s,cn,a),t(cn,ae),e(s,va,a),j(Gs,s,a),e(s,qa,a),j(Ns,s,a),e(s,_a,a),e(s,rn,a),t(rn,le),e(s,ya,a),j(Hs,s,a),e(s,wa,a),e(s,on,a),t(on,pe),e(s,Ea,a),j(Ss,s,a),e(s,Ta,a),j(As,s,a),e(s,za,a),e(s,un,a),t(un,ee),e(s,Pa,a),j(Vs,s,a),e(s,Ca,a),j(Rs,s,a),e(s,Da,a),j(ls,s,a),e(s,Ba,a),e(s,gn,a),t(gn,he),e(s,Oa,a),j(Is,s,a),e(s,Ga,a),e(s,fn,a),t(fn,ie),e(s,Na,a),j(Ls,s,a),e(s,Ha,a),j(Us,s,a),e(s,Sa,a),j(ps,s,a),e(s,Aa,a),e(s,mn,a),t(mn,ce),Va=!0},p(s,[a]){const Fs={};a&2&&(Fs.$$scope={dirty:a,ctx:s}),K.$set(Fs);const ft={};a&2&&(ft.$$scope={dirty:a,ctx:s}),W.$set(ft);const mt={};a&2&&(mt.$$scope={dirty:a,ctx:s}),J.$set(mt);const jt={};a&2&&(jt.$$scope={dirty:a,ctx:s}),X.$set(jt);const xt={};a&2&&(xt.$$scope={dirty:a,ctx:s}),ls.$set(xt);const Ks={};a&2&&(Ks.$$scope={dirty:a,ctx:s}),ps.$set(Ks)},i(s){Va||(x(v.$$.fragment,s),x(S.$$.fragment,s),x(cs.$$.fragment,s),x(K.$$.fragment,s),x(rs.$$.fragment,s),x(os.$$.fragment,s),x(W.$$.fragment,s),x(us.$$.fragment,s),x(gs.$$.fragment,s),x(fs.$$.fragment,s),x(ms.$$.fragment,s),x(js.$$.fragment,s),x(J.$$.fragment,s),x(xs.$$.fragment,s),x(ks.$$.fragment,s),x(X.$$.fragment,s),x(bs.$$.fragment,s),x($s.$$.fragment,s),x(ds.$$.fragment,s),x(vs.$$.fragment,s),x(qs.$$.fragment,s),x(_s.$$.fragment,s),x(ys.$$.fragment,s),x(ws.$$.fragment,s),x(Es.$$.fragment,s),x(Ts.$$.fragment,s),x(zs.$$.fragment,s),x(Ps.$$.fragment,s),x(Cs.$$.fragment,s),x(Ds.$$.fragment,s),x(Bs.$$.fragment,s),x(Os.$$.fragment,s),x(Gs.$$.fragment,s),x(Ns.$$.fragment,s),x(Hs.$$.fragment,s),x(Ss.$$.fragment,s),x(As.$$.fragment,s),x(Vs.$$.fragment,s),x(Rs.$$.fragment,s),x(ls.$$.fragment,s),x(Is.$$.fragment,s),x(Ls.$$.fragment,s),x(Us.$$.fragment,s),x(ps.$$.fragment,s),Va=!0)},o(s){k(v.$$.fragment,s),k(S.$$.fragment,s),k(cs.$$.fragment,s),k(K.$$.fragment,s),k(rs.$$.fragment,s),k(os.$$.fragment,s),k(W.$$.fragment,s),k(us.$$.fragment,s),k(gs.$$.fragment,s),k(fs.$$.fragment,s),k(ms.$$.fragment,s),k(js.$$.fragment,s),k(J.$$.fragment,s),k(xs.$$.fragment,s),k(ks.$$.fragment,s),k(X.$$.fragment,s),k(bs.$$.fragment,s),k($s.$$.fragment,s),k(ds.$$.fragment,s),k(vs.$$.fragment,s),k(qs.$$.fragment,s),k(_s.$$.fragment,s),k(ys.$$.fragment,s),k(ws.$$.fragment,s),k(Es.$$.fragment,s),k(Ts.$$.fragment,s),k(zs.$$.fragment,s),k(Ps.$$.fragment,s),k(Cs.$$.fragment,s),k(Ds.$$.fragment,s),k(Bs.$$.fragment,s),k(Os.$$.fragment,s),k(Gs.$$.fragment,s),k(Ns.$$.fragment,s),k(Hs.$$.fragment,s),k(Ss.$$.fragment,s),k(As.$$.fragment,s),k(Vs.$$.fragment,s),k(Rs.$$.fragment,s),k(ls.$$.fragment,s),k(Is.$$.fragment,s),k(Ls.$$.fragment,s),k(Us.$$.fragment,s),k(ps.$$.fragment,s),Va=!1},d(s){n(u),s&&n(_),s&&n(g),b(v),s&&n(O),b(S,s),s&&n(is),s&&n(Ms),s&&n(kt),b(cs,s),s&&n(bt),b(K,s),s&&n($t),s&&n(L),b(rs),s&&n(dt),s&&n(Qs),s&&n(vt),b(os,s),s&&n(qt),s&&n(Q),s&&n(_t),b(W,s),s&&n(yt),s&&n(Y),s&&n(wt),s&&n(Ws),s&&n(Et),s&&n(Ys),s&&n(Tt),b(us,s),s&&n(zt),s&&n(C),s&&n(Pt),b(gs,s),s&&n(Ct),s&&n(T),s&&n(Dt),s&&n(A),s&&n(Bt),b(fs,s),s&&n(Ot),s&&n(G),s&&n(Gt),b(ms,s),s&&n(Nt),s&&n(V),s&&n(Ht),b(js,s),s&&n(St),s&&n(Js),s&&n(At),b(J,s),s&&n(Vt),s&&n(U),b(xs),s&&n(Rt),s&&n(Zs),s&&n(It),s&&n(N),s&&n(Lt),s&&n(Xs),s&&n(Ut),b(ks,s),s&&n(Ft),s&&n(q),s&&n(Kt),b(X,s),s&&n(Mt),s&&n(F),b(bs),s&&n(Qt),s&&n(sn),s&&n(Wt),s&&n(nn),s&&n(Yt),b($s,s),s&&n(Jt),s&&n(ns),s&&n(Zt),b(ds,s),s&&n(Xt),s&&n(tn),s&&n(sa),b(vs,s),s&&n(na),b(qs,s),s&&n(ta),s&&n(an),s&&n(aa),b(_s,s),s&&n(la),b(ys,s),s&&n(pa),s&&n(ts),s&&n(ea),b(ws,s),s&&n(ha),s&&n(ln),s&&n(ia),b(Es,s),s&&n(ca),s&&n(pn),s&&n(ra),b(Ts,s),s&&n(oa),s&&n(en),s&&n(ua),b(zs,s),s&&n(ga),b(Ps,s),s&&n(fa),s&&n(hn),s&&n(ma),b(Cs,s),s&&n(ja),b(Ds,s),s&&n(xa),s&&n(R),s&&n(ka),b(Bs,s),s&&n(ba),s&&n(as),s&&n($a),b(Os,s),s&&n(da),s&&n(cn),s&&n(va),b(Gs,s),s&&n(qa),b(Ns,s),s&&n(_a),s&&n(rn),s&&n(ya),b(Hs,s),s&&n(wa),s&&n(on),s&&n(Ea),b(Ss,s),s&&n(Ta),b(As,s),s&&n(za),s&&n(un),s&&n(Pa),b(Vs,s),s&&n(Ca),b(Rs,s),s&&n(Da),b(ls,s),s&&n(Ba),s&&n(gn),s&&n(Oa),b(Is,s),s&&n(Ga),s&&n(fn),s&&n(Na),b(Ls,s),s&&n(Ha),b(Us,s),s&&n(Sa),b(ps,s),s&&n(Aa),s&&n(mn)}}}const Ih={local:"bytepair-encoding-tokenization",sections:[{local:"thut-ton-hun-luyn",title:"Thu\u1EADt to\xE1n hu\u1EA5n luy\u1EC7n"},{local:"thut-ton-tokenize",title:"Thu\u1EADt to\xE1n tokenize"},{local:"trin-khai-bpe",title:"Tri\u1EC3n khai BPE"}],title:"Byte-Pair Encoding tokenization"};function Lh(B){return Dh(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class Yh extends Th{constructor(u){super();zh(this,u,Lh,Rh,Ph,{})}}export{Yh as default,Ih as metadata};
