import{S as _r,i as $r,s as zr,e as c,k as l,w as m,t as o,M as yr,c as h,d as e,m as p,a,x as f,h as r,b as u,N as dr,G as n,g as s,y as v,q as k,o as d,B as b,v as Er}from"../../chunks/vendor-hf-doc-builder.js";import{T as Tr}from"../../chunks/Tip-hf-doc-builder.js";import{Y as br}from"../../chunks/Youtube-hf-doc-builder.js";import{I as Kn}from"../../chunks/IconCopyLink-hf-doc-builder.js";import{C as G}from"../../chunks/CodeBlock-hf-doc-builder.js";import{D as wr}from"../../chunks/DocNotebookDropdown-hf-doc-builder.js";function xr(Qn){let g,K,_,E,M,z,ft,R;return{c(){g=c("p"),K=o("\u270F\uFE0F "),_=c("strong"),E=o("Try it out!"),M=o(" T\u1EA3i tokenizer t\u1EEB checkpoint "),z=c("code"),ft=o("bert-base-cased"),R=o(" v\xE0 truy\u1EC1n v\xE0o c\xF9ng m\u1ED9t v\xED d\u1EE5 v\xE0o.S\u1EF1 kh\xE1c bi\u1EC7t ch\xEDnh m\xE0 b\u1EA1n c\xF3 th\u1EC3 th\u1EA5y gi\u1EEFa c\xE1c phi\xEAn b\u1EA3n c\xF3 d\u1EA5u v\xE0 kh\xF4ng d\u1EA5u c\u1EE7a tokenizer l\xE0 g\xEC?")},l(U){g=h(U,"P",{});var y=a(g);K=r(y,"\u270F\uFE0F "),_=h(y,"STRONG",{});var A=a(_);E=r(A,"Try it out!"),A.forEach(e),M=r(y," T\u1EA3i tokenizer t\u1EEB checkpoint "),z=h(y,"CODE",{});var vt=a(z);ft=r(vt,"bert-base-cased"),vt.forEach(e),R=r(y," v\xE0 truy\u1EC1n v\xE0o c\xF9ng m\u1ED9t v\xED d\u1EE5 v\xE0o.S\u1EF1 kh\xE1c bi\u1EC7t ch\xEDnh m\xE0 b\u1EA1n c\xF3 th\u1EC3 th\u1EA5y gi\u1EEFa c\xE1c phi\xEAn b\u1EA3n c\xF3 d\u1EA5u v\xE0 kh\xF4ng d\u1EA5u c\u1EE7a tokenizer l\xE0 g\xEC?"),y.forEach(e)},m(U,y){s(U,g,y),n(g,K),n(g,_),n(_,E),n(g,M),n(g,z),n(z,ft),n(g,R)},d(U){U&&e(g)}}}function jr(Qn){let g,K,_,E,M,z,ft,R,U,y,A,vt,Gt,Ve,Jn,I,kt,to,We,dt,no,Xn,H,Fe,kn,Ye,Ke,dn,Qe,Je,Zn,L,Q,bn,bt,Xe,_n,Ze,te,_t,ne,J,tc,$t,nc,ec,ee,V,$n,cc,hc,zn,oc,rc,ce,zt,he,yt,oe,T,ac,yn,ic,sc,En,lc,pc,Tn,uc,gc,re,Et,ae,Tt,ie,X,mc,wn,fc,vc,se,Z,le,W,tt,xn,wt,kc,jn,dc,pe,xt,ue,nt,bc,Mt,_c,$c,ge,w,zc,Pn,yc,Ec,qn,Tc,wc,Cn,xc,jc,me,jt,fe,Pt,ve,S,Pc,Dn,qc,Cc,Bn,Dc,Bc,ke,Rt,Nc,de,qt,be,et,Ac,Nn,Hc,Sc,_e,Ct,$e,Ut,Oc,ze,It,Gc,ye,Dt,Ee,Bt,Te,$,Mc,An,Rc,Uc,Hn,Ic,Lc,Sn,Vc,Wc,On,Fc,Yc,we,Lt,Kc,xe,F,ct,Gn,Nt,Qc,Mn,Jc,je,j,At,Xc,Zc,Rn,th,nh,Vt,eh,ch,Pe,x,hh,Un,oh,rh,In,ah,ih,Ln,sh,lh,qe,Y,ht,Vn,Ht,ph,Wn,uh,Ce,Wt,gh,De,ot,Fn,P,Ft,mh,fh,Yt,vh,kh,Kt,dh,bh,Qt,_h,$h,q,C,Jt,zh,yh,Xt,Eh,Th,Zt,wh,xh,tn,jh,Ph,D,nn,qh,Ch,en,Dh,Bh,cn,Nh,Ah,hn,Hh,Sh,B,on,Oh,Gh,rn,Mh,Rh,an,Uh,Ih,sn,Lh,Vh,N,ln,Wh,Fh,pn,Yh,Kh,un,Qh,Jh,gn,Xh,Be,mn,Zh,Ne;return z=new Kn({}),A=new wr({props:{classNames:"absolute z-10 right-0 top-0",options:[{label:"Google Colab",value:"https://colab.research.google.com/github/huggingface/notebooks/blob/master/course/vi/chapter6/section4.ipynb"},{label:"Aws Studio",value:"https://studiolab.sagemaker.aws/import/github/huggingface/notebooks/blob/master/course/vi/chapter6/section4.ipynb"}]}}),bt=new Kn({}),_t=new br({props:{id:"4IIC2jI9CaU"}}),zt=new G({props:{code:`from transformers import AutoTokenizer

tokenizer = AutoTokenizer.from_pretrained("bert-base-uncased")
print(type(tokenizer.backend_tokenizer))`,highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoTokenizer

tokenizer = AutoTokenizer.from_pretrained(<span class="hljs-string">&quot;bert-base-uncased&quot;</span>)
<span class="hljs-built_in">print</span>(<span class="hljs-built_in">type</span>(tokenizer.backend_tokenizer))`}}),yt=new G({props:{code:"<class 'tokenizers.Tokenizer'>",highlighted:'&lt;<span class="hljs-keyword">class</span> <span class="hljs-string">&#x27;tokenizers.Tokenizer&#x27;</span>&gt;'}}),Et=new G({props:{code:'print(tokenizer.backend_tokenizer.normalizer.normalize_str("H\xE9ll\xF2 h\xF4w are \xFC?"))',highlighted:'<span class="hljs-built_in">print</span>(tokenizer.backend_tokenizer.normalizer.normalize_str(<span class="hljs-string">&quot;H\xE9ll\xF2 h\xF4w are \xFC?&quot;</span>))'}}),Tt=new G({props:{code:"'hello how are u?'",highlighted:'<span class="hljs-string">&#x27;hello how are u?&#x27;</span>'}}),Z=new Tr({props:{$$slots:{default:[xr]},$$scope:{ctx:Qn}}}),wt=new Kn({}),xt=new br({props:{id:"grlLV8AIXug"}}),jt=new G({props:{code:'tokenizer.backend_tokenizer.pre_tokenizer.pre_tokenize_str("Hello, how are  you?")',highlighted:'tokenizer.backend_tokenizer.pre_tokenizer.pre_tokenize_str(<span class="hljs-string">&quot;Hello, how are  you?&quot;</span>)'}}),Pt=new G({props:{code:"[('Hello', (0, 5)), (',', (5, 6)), ('how', (7, 10)), ('are', (11, 14)), ('you', (16, 19)), ('?', (19, 20))]",highlighted:'[(<span class="hljs-string">&#x27;Hello&#x27;</span>, (<span class="hljs-number">0</span>, <span class="hljs-number">5</span>)), (<span class="hljs-string">&#x27;,&#x27;</span>, (<span class="hljs-number">5</span>, <span class="hljs-number">6</span>)), (<span class="hljs-string">&#x27;how&#x27;</span>, (<span class="hljs-number">7</span>, <span class="hljs-number">10</span>)), (<span class="hljs-string">&#x27;are&#x27;</span>, (<span class="hljs-number">11</span>, <span class="hljs-number">14</span>)), (<span class="hljs-string">&#x27;you&#x27;</span>, (<span class="hljs-number">16</span>, <span class="hljs-number">19</span>)), (<span class="hljs-string">&#x27;?&#x27;</span>, (<span class="hljs-number">19</span>, <span class="hljs-number">20</span>))]'}}),qt=new G({props:{code:`tokenizer = AutoTokenizer.from_pretrained("gpt2")
tokenizer.backend_tokenizer.pre_tokenizer.pre_tokenize_str("Hello, how are  you?")`,highlighted:`tokenizer = AutoTokenizer.from_pretrained(<span class="hljs-string">&quot;gpt2&quot;</span>)
tokenizer.backend_tokenizer.pre_tokenizer.pre_tokenize_str(<span class="hljs-string">&quot;Hello, how are  you?&quot;</span>)`}}),Ct=new G({props:{code:`[('Hello', (0, 5)), (',', (5, 6)), ('\u0120how', (6, 10)), ('\u0120are', (10, 14)), ('\u0120', (14, 15)), ('\u0120you', (15, 19)),
 ('?', (19, 20))]`,highlighted:`[(<span class="hljs-string">&#x27;Hello&#x27;</span>, (<span class="hljs-number">0</span>, <span class="hljs-number">5</span>)), (<span class="hljs-string">&#x27;,&#x27;</span>, (<span class="hljs-number">5</span>, <span class="hljs-number">6</span>)), (<span class="hljs-string">&#x27;\u0120how&#x27;</span>, (<span class="hljs-number">6</span>, <span class="hljs-number">10</span>)), (<span class="hljs-string">&#x27;\u0120are&#x27;</span>, (<span class="hljs-number">10</span>, <span class="hljs-number">14</span>)), (<span class="hljs-string">&#x27;\u0120&#x27;</span>, (<span class="hljs-number">14</span>, <span class="hljs-number">15</span>)), (<span class="hljs-string">&#x27;\u0120you&#x27;</span>, (<span class="hljs-number">15</span>, <span class="hljs-number">19</span>)),
 (<span class="hljs-string">&#x27;?&#x27;</span>, (<span class="hljs-number">19</span>, <span class="hljs-number">20</span>))]`}}),Dt=new G({props:{code:`tokenizer = AutoTokenizer.from_pretrained("t5-small")
tokenizer.backend_tokenizer.pre_tokenizer.pre_tokenize_str("Hello, how are  you?")`,highlighted:`tokenizer = AutoTokenizer.from_pretrained(<span class="hljs-string">&quot;t5-small&quot;</span>)
tokenizer.backend_tokenizer.pre_tokenizer.pre_tokenize_str(<span class="hljs-string">&quot;Hello, how are  you?&quot;</span>)`}}),Bt=new G({props:{code:"[('\u2581Hello,', (0, 6)), ('\u2581how', (7, 10)), ('\u2581are', (11, 14)), ('\u2581you?', (16, 20))]",highlighted:'[(<span class="hljs-string">&#x27;\u2581Hello,&#x27;</span>, (<span class="hljs-number">0</span>, <span class="hljs-number">6</span>)), (<span class="hljs-string">&#x27;\u2581how&#x27;</span>, (<span class="hljs-number">7</span>, <span class="hljs-number">10</span>)), (<span class="hljs-string">&#x27;\u2581are&#x27;</span>, (<span class="hljs-number">11</span>, <span class="hljs-number">14</span>)), (<span class="hljs-string">&#x27;\u2581you?&#x27;</span>, (<span class="hljs-number">16</span>, <span class="hljs-number">20</span>))]'}}),Nt=new Kn({}),Ht=new Kn({}),{c(){g=c("meta"),K=l(),_=c("h1"),E=c("a"),M=c("span"),m(z.$$.fragment),ft=l(),R=c("span"),U=o("Chu\u1EA9n ho\xE1 v\xE0 ti\u1EC1n tokenize"),y=l(),m(A.$$.fragment),vt=l(),Gt=c("p"),Ve=o("Tr\u01B0\u1EDBc khi \u0111i s\xE2u h\u01A1n v\xE0o ba thu\u1EADt to\xE1n tokenize t\u1EEB ph\u1EE5 ph\u1ED5 bi\u1EBFn nh\u1EA5t \u0111\u01B0\u1EE3c s\u1EED d\u1EE5ng v\u1EDBi c\xE1c m\xF4 h\xECnh Transformer (M\xE3 h\xF3a theo c\u1EB7p [BPE], WordPiece v\xE0 Unigram), tr\u01B0\u1EDBc ti\xEAn ch\xFAng ta s\u1EBD xem x\xE9t ti\u1EC1n x\u1EED l\xFD m\xE0 m\u1ED7i tr\xECnh tokenize \xE1p d\u1EE5ng cho v\u0103n b\u1EA3n. D\u01B0\u1EDBi \u0111\xE2y l\xE0 t\u1ED5ng quan c\u1EA5p cao v\u1EC1 c\xE1c b\u01B0\u1EDBc trong pipeline tokenize:"),Jn=l(),I=c("div"),kt=c("img"),We=l(),dt=c("img"),Xn=l(),H=c("p"),Fe=o("Tr\u01B0\u1EDBc khi t\xE1ch m\u1ED9t \u0111o\u1EA1n v\u0103n b\u1EA3n th\xE0nh c\xE1c token ph\u1EE5 (d\u1EF1a theo m\xF4 h\xECnh)tokenizer s\u1EBD th\u1EF1c hi\u1EC7n 2 b\u01B0\u1EDBc: "),kn=c("em"),Ye=o("normalization"),Ke=o(" (chu\u1EA9n ho\xE1) v\xE0 "),dn=c("em"),Qe=o("pre-tokenization"),Je=o(" (ti\u1EC1n tokenize)."),Zn=l(),L=c("h2"),Q=c("a"),bn=c("span"),m(bt.$$.fragment),Xe=l(),_n=c("span"),Ze=o("Chu\u1EA9n ho\xE1"),te=l(),m(_t.$$.fragment),ne=l(),J=c("p"),tc=o("B\u01B0\u1EDBc chu\u1EA9n h\xF3a bao g\u1ED3m m\u1ED9t s\u1ED1 thao t\xE1c d\u1ECDn d\u1EB9p, ch\u1EB3ng h\u1EA1n nh\u01B0 lo\u1EA1i b\u1ECF kho\u1EA3ng tr\u1EAFng kh\xF4ng c\u1EA7n thi\u1EBFt, vi\u1EBFt th\u01B0\u1EDDng t\u1EA5t c\u1EA3 c\xE1c ch\u1EEF, v\xE0/ho\u1EB7c x\xF3a d\u1EA5u. N\u1EBFu b\u1EA1n \u0111\xE3 quen v\u1EDBi "),$t=c("a"),nc=o("chu\u1EA9n h\xF3a Unicode"),ec=o(" (ch\u1EB3ng h\u1EA1n nh\u01B0 NFC ho\u1EB7c NFKC), th\xEC \u0111\xE2y c\u0169ng l\xE0 \u0111i\u1EC1u m\xE0 tokenizer c\xF3 th\u1EC3 \xE1p d\u1EE5ng."),ee=l(),V=c("p"),$n=c("code"),cc=o("tokenizer"),hc=o(" c\u1EE7a \u{1F917} Transformers c\xF3 m\u1ED9t thu\u1ED9c t\xEDnh g\u1ECDi l\xE0 "),zn=c("code"),oc=o("backend_tokenizer"),rc=o(" cung c\u1EA5p quy\u1EC1n truy c\u1EADp v\xE0o tokenizer b\xEAn d\u01B0\u1EDBi t\u1EEB th\u01B0 vi\u1EC7n \u{1F917} Tokenizers:"),ce=l(),m(zt.$$.fragment),he=l(),m(yt.$$.fragment),oe=l(),T=c("p"),ac=o("Thu\u1ED9c t\xEDnh "),yn=c("code"),ic=o("normalizer"),sc=o(" c\u1EE7a \u0111\u1ED1i t\u01B0\u1EE3ng "),En=c("code"),lc=o("tokenizer"),pc=o(" c\xF3 ph\u01B0\u01A1ng th\u1EE9c "),Tn=c("code"),uc=o("normalize_str()"),gc=o(" m\xE0 ta c\xF3 th\u1EC3 d\xF9ng \u0111\u1EC3 th\u1EA5y c\xE1ch b\u01B0\u1EDBc chu\u1EA9n ho\xE1 \u0111\u01B0\u1EE3c th\u1EF1c hi\u1EC7n:"),re=l(),m(Et.$$.fragment),ae=l(),m(Tt.$$.fragment),ie=l(),X=c("p"),mc=o("Trong v\xED d\u1EE5 n\xE0y, v\xEC ch\xFAng ta ch\u1ECDn checkpoint "),wn=c("code"),fc=o("bert-base-uncased"),vc=o(", b\u01B0\u1EDBc chu\u1EA9n ho\xE1 s\u1EBD th\u1EF1c hi\u1EC7n vi\u1EBFt th\u01B0\u1EDDng v\xE0 lo\u1EA1i b\u1ECF c\xE1c d\u1EA5u."),se=l(),m(Z.$$.fragment),le=l(),W=c("h2"),tt=c("a"),xn=c("span"),m(wt.$$.fragment),kc=l(),jn=c("span"),dc=o("Pre-tokenization"),pe=l(),m(xt.$$.fragment),ue=l(),nt=c("p"),bc=o("Nh\u01B0 ch\xFAng ta s\u1EBD th\u1EA5y trong c\xE1c ph\u1EA7n ti\u1EBFp theo, m\u1ED9t tokenizer kh\xF4ng th\u1EC3 \u0111\u01B0\u1EE3c hu\u1EA5n luy\u1EC7n tr\xEAn v\u0103n b\u1EA3n th\xF4. Thay v\xE0o \u0111\xF3, tr\u01B0\u1EDBc ti\xEAn ch\xFAng ta c\u1EA7n chia c\xE1c v\u0103n b\u1EA3n th\xE0nh c\xE1c th\u1EF1c th\u1EC3 nh\u1ECF, nh\u01B0 c\xE1c t\u1EEB. \u0110\xF3 l\xE0 khi b\u01B0\u1EDBc pre-tokenization b\u1EAFt \u0111\u1EA7u. Nh\u01B0 ch\xFAng ta \u0111\xE3 th\u1EA5y trong "),Mt=c("a"),_c=o("Ch\u01B0\u01A1ng 2"),$c=o(", tr\xECnh tokenize d\u1EF1a tr\xEAn t\u1EEB c\xF3 th\u1EC3 ch\u1EC9 c\u1EA7n t\xE1ch m\u1ED9t v\u0103n b\u1EA3n th\xF4 th\xE0nh c\xE1c t\u1EEB d\u1EF1a tr\xEAn kho\u1EA3ng tr\u1EAFng v\xE0 d\u1EA5u c\xE2u. Nh\u1EEFng t\u1EEB \u0111\xF3 s\u1EBD l\xE0 ranh gi\u1EDBi c\u1EE7a c\xE1c token con m\xE0 tokenizer c\xF3 th\u1EC3 h\u1ECDc \u0111\u01B0\u1EE3c trong qu\xE1 tr\xECnh hu\u1EA5n luy\u1EC7n c\u1EE7a n\xF3."),ge=l(),w=c("p"),zc=o("\u0110\u1EC3 xem c\xE1ch m\u1ED9t tokenizer nhanh th\u1EF1c hi\u1EC7n pre-tokenization, ch\xFAng ta c\xF3 th\u1EC3 s\u1EED d\u1EE5ng ph\u01B0\u01A1ng th\u1EE9c "),Pn=c("code"),yc=o("pre_tokenize_str()"),Ec=o(" c\u1EE7a thu\u1ED9c t\xEDnh "),qn=c("code"),Tc=o("pre_tokenizer"),wc=o(" c\u1EE7a \u0111\u1ED1i t\u01B0\u1EE3ng "),Cn=c("code"),xc=o("tokenizer"),jc=o(":"),me=l(),m(jt.$$.fragment),fe=l(),m(Pt.$$.fragment),ve=l(),S=c("p"),Pc=o("L\u01B0u \xFD c\xE1ch tokenizer \u0111\xE3 theo d\xF5i c\xE1c offset, \u0111\xF3 l\xE0 c\xE1ch n\xF3 c\xF3 th\u1EC3 cung c\u1EA5p cho ch\xFAng ta \xE1nh x\u1EA1 offset m\xE0 ta \u0111\xE3 s\u1EED d\u1EE5ng trong ph\u1EA7n tr\u01B0\u1EDBc. \u1EDE \u0111\xE2y tokenizer b\u1ECF qua hai kho\u1EA3ng tr\u1EAFng v\xE0 thay th\u1EBF ch\xFAng b\u1EB1ng ch\u1EC9 m\u1ED9t, nh\u01B0ng c\xE1c offset xen gi\u1EEFa "),Dn=c("code"),qc=o("are"),Cc=o(" v\xE0 "),Bn=c("code"),Dc=o("you"),Bc=o(" \u0111\u1EC3 gi\u1EA3i th\xEDch \u0111i\u1EC1u \u0111\xF3."),ke=l(),Rt=c("p"),Nc=o("V\xEC ch\xFAng ta \u0111ang s\u1EED d\u1EE5ng BERT tokenizer, pre-tokenization li\xEAn quan \u0111\u1EBFn vi\u1EC7c ph\xE2n t\xE1ch d\u1EF1a tr\xEAn kho\u1EA3ng tr\u1EAFng v\xE0 d\u1EA5u ch\u1EA5m c\xE2u. C\xE1c tokenizer kh\xE1c c\xF3 th\u1EC3 c\xF3 c\xE1c quy t\u1EAFc kh\xE1c nhau cho b\u01B0\u1EDBc n\xE0y. V\xED d\u1EE5: n\u1EBFu s\u1EED d\u1EE5ng GPT-2 tokenizer:"),de=l(),m(qt.$$.fragment),be=l(),et=c("p"),Ac=o("n\xF3 s\u1EBD t\xE1ch d\u1EF1a tr\xEAn d\u1EA5u c\xE1ch v\xE0 d\u1EA5u c\xE2u, nh\u01B0ng s\u1EBD gi\u1EEFa d\u1EA5u c\xE1ch v\xE0 thay th\u1EBF ch\xFAng b\u1EDFi k\xED hi\u1EC7u "),Nn=c("code"),Hc=o("\u0120"),Sc=o(", cho ph\xE9p n\xF3 kh\xF4i ph\u1EE5c kh\xF4ng gian ban \u0111\u1EA7u n\u1EBFu ch\xFAng t\xF4i gi\u1EA3i m\xE3 c\xE1c token:"),_e=l(),m(Ct.$$.fragment),$e=l(),Ut=c("p"),Oc=o("C\u1EA7n l\u01B0u \xFD th\xEAm r\u1EB1ng kh\xF4ng nh\u01B0 BERT tokenizer, tokenizer n\xE0y b\u1ECF qua d\u1EA5u c\xE1ch k\xE9p."),ze=l(),It=c("p"),Gc=o("\u1EDE v\xED d\u1EE5 cu\u1ED1i, h\xE3y c\xF9ng xem T5 tokenizer d\u1EF1a tr\xEAn thu\u1EADt to\xE1n SentencePiece:"),ye=l(),m(Dt.$$.fragment),Ee=l(),m(Bt.$$.fragment),Te=l(),$=c("p"),Mc=o("Gi\u1ED1ng nh\u01B0 GPT-2 tokenizer, ph\u01B0\u01A1ng ph\xE1p n\xE0y gi\u1EEF c\xE1c d\u1EA5u c\xE1ch v\xE0 thay th\u1EBF ch\xFAng b\u1EDFi m\u1ED9t t\xED t\u1EF1 \u0111\u1EB7c bi\u1EC7t ("),An=c("code"),Rc=o("_"),Uc=o("), nh\u01B0ng T5 tokenizer ch\u1EC9 t\xE1ch d\u1EF1a theo d\u1EA5u c\xE1ch, kh\xF4ng d\u1EF1a theo d\u1EA5u c\xE2u. M\u1ED9t l\u01B0u \xFD n\u1EEFa \u0111\xF3 l\xE0 n\xF3 c\u0169ng m\u1EB7c \u0111\u1ECBnh th\xEAm d\u1EA5u c\xE1ch \u1EDF ph\xEDa \u0111\u1EA7u c\xE2u (tr\u01B0\u1EDBc "),Hn=c("code"),Ic=o("Hello"),Lc=o(") v\xE0 b\u1ECF qua nh\u1EEFng d\u1EA5u c\xE1ch k\u1EB9p \u1EDF gi\u1EEFa "),Sn=c("code"),Vc=o("are"),Wc=o(" v\xE0 "),On=c("code"),Fc=o("you"),Yc=o("."),we=l(),Lt=c("p"),Kc=o("B\xE2y gi\u1EDD ch\xFAng ta \u0111\xE3 bi\u1EBFt m\u1ED9t ch\xFAt v\u1EC1 c\xE1ch m\u1ED9t s\u1ED1 lo\u1EA1i tokenizers kh\xE1c nhau \u0111\u1EC3 x\u1EED l\xFD v\u0103n b\u1EA3n, ch\xFAng ta c\xF3 th\u1EC3 b\u1EAFt \u0111\u1EA7u t\u1EF1 kh\xE1m ph\xE1 c\xE1c thu\u1EADt to\xE1n c\u01A1 b\u1EA3n. Ch\xFAng ta s\u1EBD b\u1EAFt \u0111\u1EA7u b\u1EB1ng m\u1ED9t c\xE1i nh\xECn nhanh v\u1EC1 SentencePiece \u0111\u01B0\u1EE3c \xE1p d\u1EE5ng r\u1ED9ng r\xE3i; sau \u0111\xF3, trong ba ph\u1EA7n ti\u1EBFp theo, ch\xFAng ta s\u1EBD xem x\xE9t c\xE1ch th\u1EE9c ho\u1EA1t \u0111\u1ED9ng c\u1EE7a ba thu\u1EADt to\xE1n ch\xEDnh \u0111\u01B0\u1EE3c s\u1EED d\u1EE5ng \u0111\u1EC3 m\xE3 h\xF3a t\u1EEB ph\u1EE5."),xe=l(),F=c("h2"),ct=c("a"),Gn=c("span"),m(Nt.$$.fragment),Qc=l(),Mn=c("span"),Jc=o("SentencePiece"),je=l(),j=c("p"),At=c("a"),Xc=o("SentencePiece"),Zc=o(" l\xE0 m\u1ED9t thu\u1EADt to\xE1n tokenize \u0111\u1EC3 ti\u1EC1n x\u1EED l\xFD v\u0103n b\u1EA3n m\xE0 b\u1EA1n c\xF3 th\u1EC3 s\u1EED d\u1EE5ng v\u1EDBi b\u1EA5t k\u1EF3 m\xF4 h\xECnh n\xE0o ch\xFAng ta s\u1EBD th\u1EA5y trong ba ph\u1EA7n ti\u1EBFp theo. N\xF3 coi v\u0103n b\u1EA3n l\xE0 m\u1ED9t chu\u1ED7i c\xE1c k\xFD t\u1EF1 Unicode v\xE0 thay th\u1EBF d\u1EA5u c\xE1ch b\u1EB1ng m\u1ED9t k\xFD t\u1EF1 \u0111\u1EB7c bi\u1EC7t, "),Rn=c("code"),th=o("\u2581"),nh=o(". \u0110\u01B0\u1EE3c s\u1EED d\u1EE5ng c\xF9ng v\u1EDBi thu\u1EADt to\xE1n Unigram (xem "),Vt=c("a"),eh=o("ph\u1EA7n 7"),ch=o("), n\xF3 th\u1EADm ch\xED kh\xF4ng y\xEAu c\u1EA7u b\u01B0\u1EDBc pre-tokenization, r\u1EA5t h\u1EEFu \xEDch cho c\xE1c ng\xF4n ng\u1EEF kh\xF4ng s\u1EED d\u1EE5ng d\u1EA5u c\xE1ch (nh\u01B0 Trung Qu\u1ED1c ho\u1EB7c Nh\u1EADt B\u1EA3n)."),Pe=l(),x=c("p"),hh=o("T\xEDnh n\u0103ng ch\xEDnh kh\xE1c c\u1EE7a SentencePiece l\xE0 "),Un=c("em"),oh=o("reversible tokenization"),rh=o(" hay "),In=c("em"),ah=o("tokenize c\xF3 th\u1EC3 \u0111\u1EA3o ng\u01B0\u1EE3c"),ih=o(": v\xEC kh\xF4ng c\xF3 c\xE1ch x\u1EED l\xFD \u0111\u1EB7c bi\u1EC7t n\xE0o cho d\u1EA5u c\xE1ch, n\xEAn vi\u1EC7c gi\u1EA3i m\xE3 c\xE1c token \u0111\u01B0\u1EE3c th\u1EF1c hi\u1EC7n \u0111\u01A1n gi\u1EA3n b\u1EB1ng c\xE1ch n\u1ED1i ch\xFAng v\xE0 thay th\u1EBF c\xE1c d\u1EA5u "),Ln=c("code"),sh=o("_"),lh=o(" b\u1EB1ng d\u1EA5u c\xE1ch - \u0111i\u1EC1u n\xE0y gi\xFAp v\u0103n b\u1EA3n \u0111\u01B0\u1EE3c chu\u1EA9n h\xF3a. Nh\u01B0 ch\xFAng ta \u0111\xE3 th\u1EA5y tr\u01B0\u1EDBc \u0111\xF3, BERT tokenizer lo\u1EA1i b\u1ECF c\xE1c d\u1EA5u c\xE1ch l\u1EB7p l\u1EA1i, v\xEC v\u1EADy token c\u1EE7a n\xF3 kh\xF4ng th\u1EC3 \u0111\u1EA3o ng\u01B0\u1EE3c."),qe=l(),Y=c("h2"),ht=c("a"),Vn=c("span"),m(Ht.$$.fragment),ph=l(),Wn=c("span"),uh=o("T\u1ED5ng quan thu\u1EADt to\xE1n"),Ce=l(),Wt=c("p"),gh=o("Trong c\xE1c ph\u1EA7n ti\u1EBFp theo, ch\xFAng ta s\u1EBD \u0111i s\xE2u v\xE0o ba thu\u1EADt to\xE1n tokenize t\u1EEB ph\u1EE5 ti\xEAu bi\u1EC3u: BPE (\u0111\u01B0\u1EE3c s\u1EED d\u1EE5ng b\u1EDFi GPT-2 v\xE0 c\xE1c thu\u1EADt to\xE1n kh\xE1c), WordPiece (\u0111\u01B0\u1EE3c s\u1EED d\u1EE5ng b\u1EDFi BERT) v\xE0 Unigram (\u0111\u01B0\u1EE3c s\u1EED d\u1EE5ng b\u1EDFi T5 v\xE0 c\xE1c thu\u1EADt to\xE1n kh\xE1c). Tr\u01B0\u1EDBc khi ch\xFAng ta b\u1EAFt \u0111\u1EA7u, \u0111\xE2y l\xE0 t\u1ED5ng quan nhanh v\u1EC1 c\xE1ch ho\u1EA1t \u0111\u1ED9ng c\u1EE7a t\u1EEBng lo\u1EA1i. \u0110\u1EEBng ng\u1EA7n ng\u1EA1i quay l\u1EA1i b\u1EA3ng n\xE0y sau khi \u0111\u1ECDc t\u1EEBng ph\u1EA7n ti\u1EBFp theo n\u1EBFu b\u1EA1n ch\u01B0a hi\u1EC3u h\u1EBFt."),De=l(),ot=c("table"),Fn=c("thead"),P=c("tr"),Ft=c("th"),mh=o("M\xF4 h\xECnh"),fh=l(),Yt=c("th"),vh=o("BPE"),kh=l(),Kt=c("th"),dh=o("WordPiece"),bh=l(),Qt=c("th"),_h=o("Unigram"),$h=l(),q=c("tbody"),C=c("tr"),Jt=c("td"),zh=o("Hu\u1EA5n luy\u1EC7n"),yh=l(),Xt=c("td"),Eh=o("B\u1EAFt \u0111\u1EA7u v\u1EDBi m\u1ED9t b\u1ED9 t\u1EEB v\u1EF1ng nh\u1ECF v\xE0 h\u1ECDc b\u1ED9 quy t\u1EAFc h\u1EE3p nh\u1EA5t token"),Th=l(),Zt=c("td"),wh=o("B\u1EAFt \u0111\u1EA7u v\u1EDBi m\u1ED9t b\u1ED9 t\u1EEB v\u1EF1ng nh\u1ECF v\xE0 h\u1ECDc b\u1ED9 quy t\u1EAFc h\u1EE3p nh\u1EA5t token"),xh=l(),tn=c("td"),jh=o("B\u1EAFt \u0111\u1EA7u v\u1EDBi m\u1ED9t b\u1ED9 t\u1EEB v\u1EF1ng l\u1EDBn v\xE0 h\u1ECDc b\u1ED9 quy t\u1EAFc \u0111\u1EC3 lo\u1EA1i b\u1ECF token"),Ph=l(),D=c("tr"),nn=c("td"),qh=o("B\u01B0\u1EDBc hu\u1EA5n luy\u1EC7n"),Ch=l(),en=c("td"),Dh=o("G\u1ED9p c\xE1c token li\xEAn quan \u0111\u1EBFn c\u1EB7p ph\u1ED5 bi\u1EBFn nh\u1EA5t"),Bh=l(),cn=c("td"),Nh=o("G\u1ED9p c\xE1c token li\xEAn quan \u0111\u1EBFn c\u1EB7p c\xF3 \u0111i\u1EC3m cao nh\u1EA5t d\u1EF1a tr\xEAn t\u1EA7n su\u1EA5t c\u1EE7a c\u1EB7p, with the best score based on the frequency of the pair,  \u01B0u ti\xEAn c\xE1c c\u1EB7p m\xE0 m\u1ED7i token c\xE1 nh\xE2n t\u1EA7n su\u1EA5t th\u1EA5p h\u01A1n"),Ah=l(),hn=c("td"),Hh=o("Lo\u1EA1i b\u1ECF t\u1EA5t c\u1EA3 c\xE1c token trong b\u1ED9 t\u1EEB \u0111i\u1EC3n gi\u1EA3m thi\u1EC3u t\u1ED1i \u0111a \u0111\u1ED9 m\u1EA5t m\xE1t \u0111\u01B0\u1EE3c t\xEDnh tr\xEAn to\xE0n b\u1ED9 kho ng\u1EEF li\u1EC7u"),Sh=l(),B=c("tr"),on=c("td"),Oh=o("H\u1ECDc"),Gh=l(),rn=c("td"),Mh=o("G\u1ED9p b\u1ED9 quy t\u1EAFc v\xE0 b\u1ED9 t\u1EEB v\u1EF1ng"),Rh=l(),an=c("td"),Uh=o("Ch\u1EC9 b\u1ED9 t\u1EEB v\u1EF1ng"),Ih=l(),sn=c("td"),Lh=o("M\u1ED9t b\u1ED9 t\u1EF1 v\u1EF1ng v\u1EDBi \u0111i\u1EC3m cho m\u1ED7i token"),Vh=l(),N=c("tr"),ln=c("td"),Wh=o("M\xE3 ho\xE1"),Fh=l(),pn=c("td"),Yh=o("Chia t\u1EEB th\xE0nh c\xE1c k\xED t\u1EF1 v\xE0 \xE1p d\u1EE5ng b\u01B0\u1EDBc g\u1ED9p t\u1EEB qu\xE1 tr\xECnh hu\u1EA5n luy\u1EC7n"),Kh=l(),un=c("td"),Qh=o("T\xECm ra chu\u1ED7i t\u1EEB ph\u1EE5 d\xE0i nh\u1EA5t b\u1EAFt \u0111\u1EA7u t\u1EEB ph\u1EA7n b\u1EAFt \u0111\u1EA7u c\xF3 trong b\u1ED9 t\u1EEB v\u1EF1ng, sau \u0111\xF3 l\xE0m t\u01B0\u01A1ng t\u1EF1 v\u1EDBi c\xE1c ph\u1EA7n c\xF2n l\u1EA1i c\u1EE7a t\u1EEB"),Jh=l(),gn=c("td"),Xh=o("T\xECm t\u1EEB c\xF3 kh\u1EA3 n\u0103ng chia th\xE0nh token cao nh\u1EA5t s\u1EED d\u1EE5ng \u0111i\u1EC3m c\xF3 \u0111\u01B0\u1EE3c t\u1EEB qu\xE1 tr\xECnh hu\u1EA5n luy\u1EC7n"),Be=l(),mn=c("p"),Zh=o("Gi\u1EDD ch\xFAng ta h\xE3y \u0111i s\xE2u v\xE0o BPE th\xF4i!"),this.h()},l(t){const i=yr('[data-svelte="svelte-1phssyn"]',document.head);g=h(i,"META",{name:!0,content:!0}),i.forEach(e),K=p(t),_=h(t,"H1",{class:!0});var St=a(_);E=h(St,"A",{id:!0,class:!0,href:!0});var eo=a(E);M=h(eo,"SPAN",{});var co=a(M);f(z.$$.fragment,co),co.forEach(e),eo.forEach(e),ft=p(St),R=h(St,"SPAN",{});var ho=a(R);U=r(ho,"Chu\u1EA9n ho\xE1 v\xE0 ti\u1EC1n tokenize"),ho.forEach(e),St.forEach(e),y=p(t),f(A.$$.fragment,t),vt=p(t),Gt=h(t,"P",{});var oo=a(Gt);Ve=r(oo,"Tr\u01B0\u1EDBc khi \u0111i s\xE2u h\u01A1n v\xE0o ba thu\u1EADt to\xE1n tokenize t\u1EEB ph\u1EE5 ph\u1ED5 bi\u1EBFn nh\u1EA5t \u0111\u01B0\u1EE3c s\u1EED d\u1EE5ng v\u1EDBi c\xE1c m\xF4 h\xECnh Transformer (M\xE3 h\xF3a theo c\u1EB7p [BPE], WordPiece v\xE0 Unigram), tr\u01B0\u1EDBc ti\xEAn ch\xFAng ta s\u1EBD xem x\xE9t ti\u1EC1n x\u1EED l\xFD m\xE0 m\u1ED7i tr\xECnh tokenize \xE1p d\u1EE5ng cho v\u0103n b\u1EA3n. D\u01B0\u1EDBi \u0111\xE2y l\xE0 t\u1ED5ng quan c\u1EA5p cao v\u1EC1 c\xE1c b\u01B0\u1EDBc trong pipeline tokenize:"),oo.forEach(e),Jn=p(t),I=h(t,"DIV",{class:!0});var Ae=a(I);kt=h(Ae,"IMG",{class:!0,src:!0,alt:!0}),We=p(Ae),dt=h(Ae,"IMG",{class:!0,src:!0,alt:!0}),Ae.forEach(e),Xn=p(t),H=h(t,"P",{});var fn=a(H);Fe=r(fn,"Tr\u01B0\u1EDBc khi t\xE1ch m\u1ED9t \u0111o\u1EA1n v\u0103n b\u1EA3n th\xE0nh c\xE1c token ph\u1EE5 (d\u1EF1a theo m\xF4 h\xECnh)tokenizer s\u1EBD th\u1EF1c hi\u1EC7n 2 b\u01B0\u1EDBc: "),kn=h(fn,"EM",{});var ro=a(kn);Ye=r(ro,"normalization"),ro.forEach(e),Ke=r(fn," (chu\u1EA9n ho\xE1) v\xE0 "),dn=h(fn,"EM",{});var ao=a(dn);Qe=r(ao,"pre-tokenization"),ao.forEach(e),Je=r(fn," (ti\u1EC1n tokenize)."),fn.forEach(e),Zn=p(t),L=h(t,"H2",{class:!0});var He=a(L);Q=h(He,"A",{id:!0,class:!0,href:!0});var io=a(Q);bn=h(io,"SPAN",{});var so=a(bn);f(bt.$$.fragment,so),so.forEach(e),io.forEach(e),Xe=p(He),_n=h(He,"SPAN",{});var lo=a(_n);Ze=r(lo,"Chu\u1EA9n ho\xE1"),lo.forEach(e),He.forEach(e),te=p(t),f(_t.$$.fragment,t),ne=p(t),J=h(t,"P",{});var Se=a(J);tc=r(Se,"B\u01B0\u1EDBc chu\u1EA9n h\xF3a bao g\u1ED3m m\u1ED9t s\u1ED1 thao t\xE1c d\u1ECDn d\u1EB9p, ch\u1EB3ng h\u1EA1n nh\u01B0 lo\u1EA1i b\u1ECF kho\u1EA3ng tr\u1EAFng kh\xF4ng c\u1EA7n thi\u1EBFt, vi\u1EBFt th\u01B0\u1EDDng t\u1EA5t c\u1EA3 c\xE1c ch\u1EEF, v\xE0/ho\u1EB7c x\xF3a d\u1EA5u. N\u1EBFu b\u1EA1n \u0111\xE3 quen v\u1EDBi "),$t=h(Se,"A",{href:!0,rel:!0});var po=a($t);nc=r(po,"chu\u1EA9n h\xF3a Unicode"),po.forEach(e),ec=r(Se," (ch\u1EB3ng h\u1EA1n nh\u01B0 NFC ho\u1EB7c NFKC), th\xEC \u0111\xE2y c\u0169ng l\xE0 \u0111i\u1EC1u m\xE0 tokenizer c\xF3 th\u1EC3 \xE1p d\u1EE5ng."),Se.forEach(e),ee=p(t),V=h(t,"P",{});var Yn=a(V);$n=h(Yn,"CODE",{});var uo=a($n);cc=r(uo,"tokenizer"),uo.forEach(e),hc=r(Yn," c\u1EE7a \u{1F917} Transformers c\xF3 m\u1ED9t thu\u1ED9c t\xEDnh g\u1ECDi l\xE0 "),zn=h(Yn,"CODE",{});var go=a(zn);oc=r(go,"backend_tokenizer"),go.forEach(e),rc=r(Yn," cung c\u1EA5p quy\u1EC1n truy c\u1EADp v\xE0o tokenizer b\xEAn d\u01B0\u1EDBi t\u1EEB th\u01B0 vi\u1EC7n \u{1F917} Tokenizers:"),Yn.forEach(e),ce=p(t),f(zt.$$.fragment,t),he=p(t),f(yt.$$.fragment,t),oe=p(t),T=h(t,"P",{});var rt=a(T);ac=r(rt,"Thu\u1ED9c t\xEDnh "),yn=h(rt,"CODE",{});var mo=a(yn);ic=r(mo,"normalizer"),mo.forEach(e),sc=r(rt," c\u1EE7a \u0111\u1ED1i t\u01B0\u1EE3ng "),En=h(rt,"CODE",{});var fo=a(En);lc=r(fo,"tokenizer"),fo.forEach(e),pc=r(rt," c\xF3 ph\u01B0\u01A1ng th\u1EE9c "),Tn=h(rt,"CODE",{});var vo=a(Tn);uc=r(vo,"normalize_str()"),vo.forEach(e),gc=r(rt," m\xE0 ta c\xF3 th\u1EC3 d\xF9ng \u0111\u1EC3 th\u1EA5y c\xE1ch b\u01B0\u1EDBc chu\u1EA9n ho\xE1 \u0111\u01B0\u1EE3c th\u1EF1c hi\u1EC7n:"),rt.forEach(e),re=p(t),f(Et.$$.fragment,t),ae=p(t),f(Tt.$$.fragment,t),ie=p(t),X=h(t,"P",{});var Oe=a(X);mc=r(Oe,"Trong v\xED d\u1EE5 n\xE0y, v\xEC ch\xFAng ta ch\u1ECDn checkpoint "),wn=h(Oe,"CODE",{});var ko=a(wn);fc=r(ko,"bert-base-uncased"),ko.forEach(e),vc=r(Oe,", b\u01B0\u1EDBc chu\u1EA9n ho\xE1 s\u1EBD th\u1EF1c hi\u1EC7n vi\u1EBFt th\u01B0\u1EDDng v\xE0 lo\u1EA1i b\u1ECF c\xE1c d\u1EA5u."),Oe.forEach(e),se=p(t),f(Z.$$.fragment,t),le=p(t),W=h(t,"H2",{class:!0});var Ge=a(W);tt=h(Ge,"A",{id:!0,class:!0,href:!0});var bo=a(tt);xn=h(bo,"SPAN",{});var _o=a(xn);f(wt.$$.fragment,_o),_o.forEach(e),bo.forEach(e),kc=p(Ge),jn=h(Ge,"SPAN",{});var $o=a(jn);dc=r($o,"Pre-tokenization"),$o.forEach(e),Ge.forEach(e),pe=p(t),f(xt.$$.fragment,t),ue=p(t),nt=h(t,"P",{});var Me=a(nt);bc=r(Me,"Nh\u01B0 ch\xFAng ta s\u1EBD th\u1EA5y trong c\xE1c ph\u1EA7n ti\u1EBFp theo, m\u1ED9t tokenizer kh\xF4ng th\u1EC3 \u0111\u01B0\u1EE3c hu\u1EA5n luy\u1EC7n tr\xEAn v\u0103n b\u1EA3n th\xF4. Thay v\xE0o \u0111\xF3, tr\u01B0\u1EDBc ti\xEAn ch\xFAng ta c\u1EA7n chia c\xE1c v\u0103n b\u1EA3n th\xE0nh c\xE1c th\u1EF1c th\u1EC3 nh\u1ECF, nh\u01B0 c\xE1c t\u1EEB. \u0110\xF3 l\xE0 khi b\u01B0\u1EDBc pre-tokenization b\u1EAFt \u0111\u1EA7u. Nh\u01B0 ch\xFAng ta \u0111\xE3 th\u1EA5y trong "),Mt=h(Me,"A",{href:!0});var zo=a(Mt);_c=r(zo,"Ch\u01B0\u01A1ng 2"),zo.forEach(e),$c=r(Me,", tr\xECnh tokenize d\u1EF1a tr\xEAn t\u1EEB c\xF3 th\u1EC3 ch\u1EC9 c\u1EA7n t\xE1ch m\u1ED9t v\u0103n b\u1EA3n th\xF4 th\xE0nh c\xE1c t\u1EEB d\u1EF1a tr\xEAn kho\u1EA3ng tr\u1EAFng v\xE0 d\u1EA5u c\xE2u. Nh\u1EEFng t\u1EEB \u0111\xF3 s\u1EBD l\xE0 ranh gi\u1EDBi c\u1EE7a c\xE1c token con m\xE0 tokenizer c\xF3 th\u1EC3 h\u1ECDc \u0111\u01B0\u1EE3c trong qu\xE1 tr\xECnh hu\u1EA5n luy\u1EC7n c\u1EE7a n\xF3."),Me.forEach(e),ge=p(t),w=h(t,"P",{});var at=a(w);zc=r(at,"\u0110\u1EC3 xem c\xE1ch m\u1ED9t tokenizer nhanh th\u1EF1c hi\u1EC7n pre-tokenization, ch\xFAng ta c\xF3 th\u1EC3 s\u1EED d\u1EE5ng ph\u01B0\u01A1ng th\u1EE9c "),Pn=h(at,"CODE",{});var yo=a(Pn);yc=r(yo,"pre_tokenize_str()"),yo.forEach(e),Ec=r(at," c\u1EE7a thu\u1ED9c t\xEDnh "),qn=h(at,"CODE",{});var Eo=a(qn);Tc=r(Eo,"pre_tokenizer"),Eo.forEach(e),wc=r(at," c\u1EE7a \u0111\u1ED1i t\u01B0\u1EE3ng "),Cn=h(at,"CODE",{});var To=a(Cn);xc=r(To,"tokenizer"),To.forEach(e),jc=r(at,":"),at.forEach(e),me=p(t),f(jt.$$.fragment,t),fe=p(t),f(Pt.$$.fragment,t),ve=p(t),S=h(t,"P",{});var vn=a(S);Pc=r(vn,"L\u01B0u \xFD c\xE1ch tokenizer \u0111\xE3 theo d\xF5i c\xE1c offset, \u0111\xF3 l\xE0 c\xE1ch n\xF3 c\xF3 th\u1EC3 cung c\u1EA5p cho ch\xFAng ta \xE1nh x\u1EA1 offset m\xE0 ta \u0111\xE3 s\u1EED d\u1EE5ng trong ph\u1EA7n tr\u01B0\u1EDBc. \u1EDE \u0111\xE2y tokenizer b\u1ECF qua hai kho\u1EA3ng tr\u1EAFng v\xE0 thay th\u1EBF ch\xFAng b\u1EB1ng ch\u1EC9 m\u1ED9t, nh\u01B0ng c\xE1c offset xen gi\u1EEFa "),Dn=h(vn,"CODE",{});var wo=a(Dn);qc=r(wo,"are"),wo.forEach(e),Cc=r(vn," v\xE0 "),Bn=h(vn,"CODE",{});var xo=a(Bn);Dc=r(xo,"you"),xo.forEach(e),Bc=r(vn," \u0111\u1EC3 gi\u1EA3i th\xEDch \u0111i\u1EC1u \u0111\xF3."),vn.forEach(e),ke=p(t),Rt=h(t,"P",{});var jo=a(Rt);Nc=r(jo,"V\xEC ch\xFAng ta \u0111ang s\u1EED d\u1EE5ng BERT tokenizer, pre-tokenization li\xEAn quan \u0111\u1EBFn vi\u1EC7c ph\xE2n t\xE1ch d\u1EF1a tr\xEAn kho\u1EA3ng tr\u1EAFng v\xE0 d\u1EA5u ch\u1EA5m c\xE2u. C\xE1c tokenizer kh\xE1c c\xF3 th\u1EC3 c\xF3 c\xE1c quy t\u1EAFc kh\xE1c nhau cho b\u01B0\u1EDBc n\xE0y. V\xED d\u1EE5: n\u1EBFu s\u1EED d\u1EE5ng GPT-2 tokenizer:"),jo.forEach(e),de=p(t),f(qt.$$.fragment,t),be=p(t),et=h(t,"P",{});var Re=a(et);Ac=r(Re,"n\xF3 s\u1EBD t\xE1ch d\u1EF1a tr\xEAn d\u1EA5u c\xE1ch v\xE0 d\u1EA5u c\xE2u, nh\u01B0ng s\u1EBD gi\u1EEFa d\u1EA5u c\xE1ch v\xE0 thay th\u1EBF ch\xFAng b\u1EDFi k\xED hi\u1EC7u "),Nn=h(Re,"CODE",{});var Po=a(Nn);Hc=r(Po,"\u0120"),Po.forEach(e),Sc=r(Re,", cho ph\xE9p n\xF3 kh\xF4i ph\u1EE5c kh\xF4ng gian ban \u0111\u1EA7u n\u1EBFu ch\xFAng t\xF4i gi\u1EA3i m\xE3 c\xE1c token:"),Re.forEach(e),_e=p(t),f(Ct.$$.fragment,t),$e=p(t),Ut=h(t,"P",{});var qo=a(Ut);Oc=r(qo,"C\u1EA7n l\u01B0u \xFD th\xEAm r\u1EB1ng kh\xF4ng nh\u01B0 BERT tokenizer, tokenizer n\xE0y b\u1ECF qua d\u1EA5u c\xE1ch k\xE9p."),qo.forEach(e),ze=p(t),It=h(t,"P",{});var Co=a(It);Gc=r(Co,"\u1EDE v\xED d\u1EE5 cu\u1ED1i, h\xE3y c\xF9ng xem T5 tokenizer d\u1EF1a tr\xEAn thu\u1EADt to\xE1n SentencePiece:"),Co.forEach(e),ye=p(t),f(Dt.$$.fragment,t),Ee=p(t),f(Bt.$$.fragment,t),Te=p(t),$=h(t,"P",{});var O=a($);Mc=r(O,"Gi\u1ED1ng nh\u01B0 GPT-2 tokenizer, ph\u01B0\u01A1ng ph\xE1p n\xE0y gi\u1EEF c\xE1c d\u1EA5u c\xE1ch v\xE0 thay th\u1EBF ch\xFAng b\u1EDFi m\u1ED9t t\xED t\u1EF1 \u0111\u1EB7c bi\u1EC7t ("),An=h(O,"CODE",{});var Do=a(An);Rc=r(Do,"_"),Do.forEach(e),Uc=r(O,"), nh\u01B0ng T5 tokenizer ch\u1EC9 t\xE1ch d\u1EF1a theo d\u1EA5u c\xE1ch, kh\xF4ng d\u1EF1a theo d\u1EA5u c\xE2u. M\u1ED9t l\u01B0u \xFD n\u1EEFa \u0111\xF3 l\xE0 n\xF3 c\u0169ng m\u1EB7c \u0111\u1ECBnh th\xEAm d\u1EA5u c\xE1ch \u1EDF ph\xEDa \u0111\u1EA7u c\xE2u (tr\u01B0\u1EDBc "),Hn=h(O,"CODE",{});var Bo=a(Hn);Ic=r(Bo,"Hello"),Bo.forEach(e),Lc=r(O,") v\xE0 b\u1ECF qua nh\u1EEFng d\u1EA5u c\xE1ch k\u1EB9p \u1EDF gi\u1EEFa "),Sn=h(O,"CODE",{});var No=a(Sn);Vc=r(No,"are"),No.forEach(e),Wc=r(O," v\xE0 "),On=h(O,"CODE",{});var Ao=a(On);Fc=r(Ao,"you"),Ao.forEach(e),Yc=r(O,"."),O.forEach(e),we=p(t),Lt=h(t,"P",{});var Ho=a(Lt);Kc=r(Ho,"B\xE2y gi\u1EDD ch\xFAng ta \u0111\xE3 bi\u1EBFt m\u1ED9t ch\xFAt v\u1EC1 c\xE1ch m\u1ED9t s\u1ED1 lo\u1EA1i tokenizers kh\xE1c nhau \u0111\u1EC3 x\u1EED l\xFD v\u0103n b\u1EA3n, ch\xFAng ta c\xF3 th\u1EC3 b\u1EAFt \u0111\u1EA7u t\u1EF1 kh\xE1m ph\xE1 c\xE1c thu\u1EADt to\xE1n c\u01A1 b\u1EA3n. Ch\xFAng ta s\u1EBD b\u1EAFt \u0111\u1EA7u b\u1EB1ng m\u1ED9t c\xE1i nh\xECn nhanh v\u1EC1 SentencePiece \u0111\u01B0\u1EE3c \xE1p d\u1EE5ng r\u1ED9ng r\xE3i; sau \u0111\xF3, trong ba ph\u1EA7n ti\u1EBFp theo, ch\xFAng ta s\u1EBD xem x\xE9t c\xE1ch th\u1EE9c ho\u1EA1t \u0111\u1ED9ng c\u1EE7a ba thu\u1EADt to\xE1n ch\xEDnh \u0111\u01B0\u1EE3c s\u1EED d\u1EE5ng \u0111\u1EC3 m\xE3 h\xF3a t\u1EEB ph\u1EE5."),Ho.forEach(e),xe=p(t),F=h(t,"H2",{class:!0});var Ue=a(F);ct=h(Ue,"A",{id:!0,class:!0,href:!0});var So=a(ct);Gn=h(So,"SPAN",{});var Oo=a(Gn);f(Nt.$$.fragment,Oo),Oo.forEach(e),So.forEach(e),Qc=p(Ue),Mn=h(Ue,"SPAN",{});var Go=a(Mn);Jc=r(Go,"SentencePiece"),Go.forEach(e),Ue.forEach(e),je=p(t),j=h(t,"P",{});var Ot=a(j);At=h(Ot,"A",{href:!0,rel:!0});var Mo=a(At);Xc=r(Mo,"SentencePiece"),Mo.forEach(e),Zc=r(Ot," l\xE0 m\u1ED9t thu\u1EADt to\xE1n tokenize \u0111\u1EC3 ti\u1EC1n x\u1EED l\xFD v\u0103n b\u1EA3n m\xE0 b\u1EA1n c\xF3 th\u1EC3 s\u1EED d\u1EE5ng v\u1EDBi b\u1EA5t k\u1EF3 m\xF4 h\xECnh n\xE0o ch\xFAng ta s\u1EBD th\u1EA5y trong ba ph\u1EA7n ti\u1EBFp theo. N\xF3 coi v\u0103n b\u1EA3n l\xE0 m\u1ED9t chu\u1ED7i c\xE1c k\xFD t\u1EF1 Unicode v\xE0 thay th\u1EBF d\u1EA5u c\xE1ch b\u1EB1ng m\u1ED9t k\xFD t\u1EF1 \u0111\u1EB7c bi\u1EC7t, "),Rn=h(Ot,"CODE",{});var Ro=a(Rn);th=r(Ro,"\u2581"),Ro.forEach(e),nh=r(Ot,". \u0110\u01B0\u1EE3c s\u1EED d\u1EE5ng c\xF9ng v\u1EDBi thu\u1EADt to\xE1n Unigram (xem "),Vt=h(Ot,"A",{href:!0});var Uo=a(Vt);eh=r(Uo,"ph\u1EA7n 7"),Uo.forEach(e),ch=r(Ot,"), n\xF3 th\u1EADm ch\xED kh\xF4ng y\xEAu c\u1EA7u b\u01B0\u1EDBc pre-tokenization, r\u1EA5t h\u1EEFu \xEDch cho c\xE1c ng\xF4n ng\u1EEF kh\xF4ng s\u1EED d\u1EE5ng d\u1EA5u c\xE1ch (nh\u01B0 Trung Qu\u1ED1c ho\u1EB7c Nh\u1EADt B\u1EA3n)."),Ot.forEach(e),Pe=p(t),x=h(t,"P",{});var it=a(x);hh=r(it,"T\xEDnh n\u0103ng ch\xEDnh kh\xE1c c\u1EE7a SentencePiece l\xE0 "),Un=h(it,"EM",{});var Io=a(Un);oh=r(Io,"reversible tokenization"),Io.forEach(e),rh=r(it," hay "),In=h(it,"EM",{});var Lo=a(In);ah=r(Lo,"tokenize c\xF3 th\u1EC3 \u0111\u1EA3o ng\u01B0\u1EE3c"),Lo.forEach(e),ih=r(it,": v\xEC kh\xF4ng c\xF3 c\xE1ch x\u1EED l\xFD \u0111\u1EB7c bi\u1EC7t n\xE0o cho d\u1EA5u c\xE1ch, n\xEAn vi\u1EC7c gi\u1EA3i m\xE3 c\xE1c token \u0111\u01B0\u1EE3c th\u1EF1c hi\u1EC7n \u0111\u01A1n gi\u1EA3n b\u1EB1ng c\xE1ch n\u1ED1i ch\xFAng v\xE0 thay th\u1EBF c\xE1c d\u1EA5u "),Ln=h(it,"CODE",{});var Vo=a(Ln);sh=r(Vo,"_"),Vo.forEach(e),lh=r(it," b\u1EB1ng d\u1EA5u c\xE1ch - \u0111i\u1EC1u n\xE0y gi\xFAp v\u0103n b\u1EA3n \u0111\u01B0\u1EE3c chu\u1EA9n h\xF3a. Nh\u01B0 ch\xFAng ta \u0111\xE3 th\u1EA5y tr\u01B0\u1EDBc \u0111\xF3, BERT tokenizer lo\u1EA1i b\u1ECF c\xE1c d\u1EA5u c\xE1ch l\u1EB7p l\u1EA1i, v\xEC v\u1EADy token c\u1EE7a n\xF3 kh\xF4ng th\u1EC3 \u0111\u1EA3o ng\u01B0\u1EE3c."),it.forEach(e),qe=p(t),Y=h(t,"H2",{class:!0});var Ie=a(Y);ht=h(Ie,"A",{id:!0,class:!0,href:!0});var Wo=a(ht);Vn=h(Wo,"SPAN",{});var Fo=a(Vn);f(Ht.$$.fragment,Fo),Fo.forEach(e),Wo.forEach(e),ph=p(Ie),Wn=h(Ie,"SPAN",{});var Yo=a(Wn);uh=r(Yo,"T\u1ED5ng quan thu\u1EADt to\xE1n"),Yo.forEach(e),Ie.forEach(e),Ce=p(t),Wt=h(t,"P",{});var Ko=a(Wt);gh=r(Ko,"Trong c\xE1c ph\u1EA7n ti\u1EBFp theo, ch\xFAng ta s\u1EBD \u0111i s\xE2u v\xE0o ba thu\u1EADt to\xE1n tokenize t\u1EEB ph\u1EE5 ti\xEAu bi\u1EC3u: BPE (\u0111\u01B0\u1EE3c s\u1EED d\u1EE5ng b\u1EDFi GPT-2 v\xE0 c\xE1c thu\u1EADt to\xE1n kh\xE1c), WordPiece (\u0111\u01B0\u1EE3c s\u1EED d\u1EE5ng b\u1EDFi BERT) v\xE0 Unigram (\u0111\u01B0\u1EE3c s\u1EED d\u1EE5ng b\u1EDFi T5 v\xE0 c\xE1c thu\u1EADt to\xE1n kh\xE1c). Tr\u01B0\u1EDBc khi ch\xFAng ta b\u1EAFt \u0111\u1EA7u, \u0111\xE2y l\xE0 t\u1ED5ng quan nhanh v\u1EC1 c\xE1ch ho\u1EA1t \u0111\u1ED9ng c\u1EE7a t\u1EEBng lo\u1EA1i. \u0110\u1EEBng ng\u1EA7n ng\u1EA1i quay l\u1EA1i b\u1EA3ng n\xE0y sau khi \u0111\u1ECDc t\u1EEBng ph\u1EA7n ti\u1EBFp theo n\u1EBFu b\u1EA1n ch\u01B0a hi\u1EC3u h\u1EBFt."),Ko.forEach(e),De=p(t),ot=h(t,"TABLE",{});var Le=a(ot);Fn=h(Le,"THEAD",{});var Qo=a(Fn);P=h(Qo,"TR",{});var st=a(P);Ft=h(st,"TH",{align:!0});var Jo=a(Ft);mh=r(Jo,"M\xF4 h\xECnh"),Jo.forEach(e),fh=p(st),Yt=h(st,"TH",{align:!0});var Xo=a(Yt);vh=r(Xo,"BPE"),Xo.forEach(e),kh=p(st),Kt=h(st,"TH",{align:!0});var Zo=a(Kt);dh=r(Zo,"WordPiece"),Zo.forEach(e),bh=p(st),Qt=h(st,"TH",{align:!0});var tr=a(Qt);_h=r(tr,"Unigram"),tr.forEach(e),st.forEach(e),Qo.forEach(e),$h=p(Le),q=h(Le,"TBODY",{});var lt=a(q);C=h(lt,"TR",{});var pt=a(C);Jt=h(pt,"TD",{align:!0});var nr=a(Jt);zh=r(nr,"Hu\u1EA5n luy\u1EC7n"),nr.forEach(e),yh=p(pt),Xt=h(pt,"TD",{align:!0});var er=a(Xt);Eh=r(er,"B\u1EAFt \u0111\u1EA7u v\u1EDBi m\u1ED9t b\u1ED9 t\u1EEB v\u1EF1ng nh\u1ECF v\xE0 h\u1ECDc b\u1ED9 quy t\u1EAFc h\u1EE3p nh\u1EA5t token"),er.forEach(e),Th=p(pt),Zt=h(pt,"TD",{align:!0});var cr=a(Zt);wh=r(cr,"B\u1EAFt \u0111\u1EA7u v\u1EDBi m\u1ED9t b\u1ED9 t\u1EEB v\u1EF1ng nh\u1ECF v\xE0 h\u1ECDc b\u1ED9 quy t\u1EAFc h\u1EE3p nh\u1EA5t token"),cr.forEach(e),xh=p(pt),tn=h(pt,"TD",{align:!0});var hr=a(tn);jh=r(hr,"B\u1EAFt \u0111\u1EA7u v\u1EDBi m\u1ED9t b\u1ED9 t\u1EEB v\u1EF1ng l\u1EDBn v\xE0 h\u1ECDc b\u1ED9 quy t\u1EAFc \u0111\u1EC3 lo\u1EA1i b\u1ECF token"),hr.forEach(e),pt.forEach(e),Ph=p(lt),D=h(lt,"TR",{});var ut=a(D);nn=h(ut,"TD",{align:!0});var or=a(nn);qh=r(or,"B\u01B0\u1EDBc hu\u1EA5n luy\u1EC7n"),or.forEach(e),Ch=p(ut),en=h(ut,"TD",{align:!0});var rr=a(en);Dh=r(rr,"G\u1ED9p c\xE1c token li\xEAn quan \u0111\u1EBFn c\u1EB7p ph\u1ED5 bi\u1EBFn nh\u1EA5t"),rr.forEach(e),Bh=p(ut),cn=h(ut,"TD",{align:!0});var ar=a(cn);Nh=r(ar,"G\u1ED9p c\xE1c token li\xEAn quan \u0111\u1EBFn c\u1EB7p c\xF3 \u0111i\u1EC3m cao nh\u1EA5t d\u1EF1a tr\xEAn t\u1EA7n su\u1EA5t c\u1EE7a c\u1EB7p, with the best score based on the frequency of the pair,  \u01B0u ti\xEAn c\xE1c c\u1EB7p m\xE0 m\u1ED7i token c\xE1 nh\xE2n t\u1EA7n su\u1EA5t th\u1EA5p h\u01A1n"),ar.forEach(e),Ah=p(ut),hn=h(ut,"TD",{align:!0});var ir=a(hn);Hh=r(ir,"Lo\u1EA1i b\u1ECF t\u1EA5t c\u1EA3 c\xE1c token trong b\u1ED9 t\u1EEB \u0111i\u1EC3n gi\u1EA3m thi\u1EC3u t\u1ED1i \u0111a \u0111\u1ED9 m\u1EA5t m\xE1t \u0111\u01B0\u1EE3c t\xEDnh tr\xEAn to\xE0n b\u1ED9 kho ng\u1EEF li\u1EC7u"),ir.forEach(e),ut.forEach(e),Sh=p(lt),B=h(lt,"TR",{});var gt=a(B);on=h(gt,"TD",{align:!0});var sr=a(on);Oh=r(sr,"H\u1ECDc"),sr.forEach(e),Gh=p(gt),rn=h(gt,"TD",{align:!0});var lr=a(rn);Mh=r(lr,"G\u1ED9p b\u1ED9 quy t\u1EAFc v\xE0 b\u1ED9 t\u1EEB v\u1EF1ng"),lr.forEach(e),Rh=p(gt),an=h(gt,"TD",{align:!0});var pr=a(an);Uh=r(pr,"Ch\u1EC9 b\u1ED9 t\u1EEB v\u1EF1ng"),pr.forEach(e),Ih=p(gt),sn=h(gt,"TD",{align:!0});var ur=a(sn);Lh=r(ur,"M\u1ED9t b\u1ED9 t\u1EF1 v\u1EF1ng v\u1EDBi \u0111i\u1EC3m cho m\u1ED7i token"),ur.forEach(e),gt.forEach(e),Vh=p(lt),N=h(lt,"TR",{});var mt=a(N);ln=h(mt,"TD",{align:!0});var gr=a(ln);Wh=r(gr,"M\xE3 ho\xE1"),gr.forEach(e),Fh=p(mt),pn=h(mt,"TD",{align:!0});var mr=a(pn);Yh=r(mr,"Chia t\u1EEB th\xE0nh c\xE1c k\xED t\u1EF1 v\xE0 \xE1p d\u1EE5ng b\u01B0\u1EDBc g\u1ED9p t\u1EEB qu\xE1 tr\xECnh hu\u1EA5n luy\u1EC7n"),mr.forEach(e),Kh=p(mt),un=h(mt,"TD",{align:!0});var fr=a(un);Qh=r(fr,"T\xECm ra chu\u1ED7i t\u1EEB ph\u1EE5 d\xE0i nh\u1EA5t b\u1EAFt \u0111\u1EA7u t\u1EEB ph\u1EA7n b\u1EAFt \u0111\u1EA7u c\xF3 trong b\u1ED9 t\u1EEB v\u1EF1ng, sau \u0111\xF3 l\xE0m t\u01B0\u01A1ng t\u1EF1 v\u1EDBi c\xE1c ph\u1EA7n c\xF2n l\u1EA1i c\u1EE7a t\u1EEB"),fr.forEach(e),Jh=p(mt),gn=h(mt,"TD",{align:!0});var vr=a(gn);Xh=r(vr,"T\xECm t\u1EEB c\xF3 kh\u1EA3 n\u0103ng chia th\xE0nh token cao nh\u1EA5t s\u1EED d\u1EE5ng \u0111i\u1EC3m c\xF3 \u0111\u01B0\u1EE3c t\u1EEB qu\xE1 tr\xECnh hu\u1EA5n luy\u1EC7n"),vr.forEach(e),mt.forEach(e),lt.forEach(e),Le.forEach(e),Be=p(t),mn=h(t,"P",{});var kr=a(mn);Zh=r(kr,"Gi\u1EDD ch\xFAng ta h\xE3y \u0111i s\xE2u v\xE0o BPE th\xF4i!"),kr.forEach(e),this.h()},h(){u(g,"name","hf:doc:metadata"),u(g,"content",JSON.stringify(Pr)),u(E,"id","chun-ho-v-tin-tokenize"),u(E,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),u(E,"href","#chun-ho-v-tin-tokenize"),u(_,"class","relative group"),u(kt,"class","block dark:hidden"),dr(kt.src,to="https://huggingface.co/datasets/huggingface-course/documentation-images/resolve/main/en/chapter6/tokenization_pipeline.svg")||u(kt,"src",to),u(kt,"alt","The tokenization pipeline."),u(dt,"class","hidden dark:block"),dr(dt.src,no="https://huggingface.co/datasets/huggingface-course/documentation-images/resolve/main/en/chapter6/tokenization_pipeline-dark.svg")||u(dt,"src",no),u(dt,"alt","The tokenization pipeline."),u(I,"class","flex justify-center"),u(Q,"id","chun-ho"),u(Q,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),u(Q,"href","#chun-ho"),u(L,"class","relative group"),u($t,"href","http://www.unicode.org/reports/tr15/"),u($t,"rel","nofollow"),u(tt,"id","pretokenization"),u(tt,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),u(tt,"href","#pretokenization"),u(W,"class","relative group"),u(Mt,"href","/course/chapter2"),u(ct,"id","sentencepiece"),u(ct,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),u(ct,"href","#sentencepiece"),u(F,"class","relative group"),u(At,"href","https://github.com/google/sentencepiece"),u(At,"rel","nofollow"),u(Vt,"href","/course/chapter7/7"),u(ht,"id","tng-quan-thut-ton"),u(ht,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),u(ht,"href","#tng-quan-thut-ton"),u(Y,"class","relative group"),u(Ft,"align","center"),u(Yt,"align","center"),u(Kt,"align","center"),u(Qt,"align","center"),u(Jt,"align","center"),u(Xt,"align","center"),u(Zt,"align","center"),u(tn,"align","center"),u(nn,"align","center"),u(en,"align","center"),u(cn,"align","center"),u(hn,"align","center"),u(on,"align","center"),u(rn,"align","center"),u(an,"align","center"),u(sn,"align","center"),u(ln,"align","center"),u(pn,"align","center"),u(un,"align","center"),u(gn,"align","center")},m(t,i){n(document.head,g),s(t,K,i),s(t,_,i),n(_,E),n(E,M),v(z,M,null),n(_,ft),n(_,R),n(R,U),s(t,y,i),v(A,t,i),s(t,vt,i),s(t,Gt,i),n(Gt,Ve),s(t,Jn,i),s(t,I,i),n(I,kt),n(I,We),n(I,dt),s(t,Xn,i),s(t,H,i),n(H,Fe),n(H,kn),n(kn,Ye),n(H,Ke),n(H,dn),n(dn,Qe),n(H,Je),s(t,Zn,i),s(t,L,i),n(L,Q),n(Q,bn),v(bt,bn,null),n(L,Xe),n(L,_n),n(_n,Ze),s(t,te,i),v(_t,t,i),s(t,ne,i),s(t,J,i),n(J,tc),n(J,$t),n($t,nc),n(J,ec),s(t,ee,i),s(t,V,i),n(V,$n),n($n,cc),n(V,hc),n(V,zn),n(zn,oc),n(V,rc),s(t,ce,i),v(zt,t,i),s(t,he,i),v(yt,t,i),s(t,oe,i),s(t,T,i),n(T,ac),n(T,yn),n(yn,ic),n(T,sc),n(T,En),n(En,lc),n(T,pc),n(T,Tn),n(Tn,uc),n(T,gc),s(t,re,i),v(Et,t,i),s(t,ae,i),v(Tt,t,i),s(t,ie,i),s(t,X,i),n(X,mc),n(X,wn),n(wn,fc),n(X,vc),s(t,se,i),v(Z,t,i),s(t,le,i),s(t,W,i),n(W,tt),n(tt,xn),v(wt,xn,null),n(W,kc),n(W,jn),n(jn,dc),s(t,pe,i),v(xt,t,i),s(t,ue,i),s(t,nt,i),n(nt,bc),n(nt,Mt),n(Mt,_c),n(nt,$c),s(t,ge,i),s(t,w,i),n(w,zc),n(w,Pn),n(Pn,yc),n(w,Ec),n(w,qn),n(qn,Tc),n(w,wc),n(w,Cn),n(Cn,xc),n(w,jc),s(t,me,i),v(jt,t,i),s(t,fe,i),v(Pt,t,i),s(t,ve,i),s(t,S,i),n(S,Pc),n(S,Dn),n(Dn,qc),n(S,Cc),n(S,Bn),n(Bn,Dc),n(S,Bc),s(t,ke,i),s(t,Rt,i),n(Rt,Nc),s(t,de,i),v(qt,t,i),s(t,be,i),s(t,et,i),n(et,Ac),n(et,Nn),n(Nn,Hc),n(et,Sc),s(t,_e,i),v(Ct,t,i),s(t,$e,i),s(t,Ut,i),n(Ut,Oc),s(t,ze,i),s(t,It,i),n(It,Gc),s(t,ye,i),v(Dt,t,i),s(t,Ee,i),v(Bt,t,i),s(t,Te,i),s(t,$,i),n($,Mc),n($,An),n(An,Rc),n($,Uc),n($,Hn),n(Hn,Ic),n($,Lc),n($,Sn),n(Sn,Vc),n($,Wc),n($,On),n(On,Fc),n($,Yc),s(t,we,i),s(t,Lt,i),n(Lt,Kc),s(t,xe,i),s(t,F,i),n(F,ct),n(ct,Gn),v(Nt,Gn,null),n(F,Qc),n(F,Mn),n(Mn,Jc),s(t,je,i),s(t,j,i),n(j,At),n(At,Xc),n(j,Zc),n(j,Rn),n(Rn,th),n(j,nh),n(j,Vt),n(Vt,eh),n(j,ch),s(t,Pe,i),s(t,x,i),n(x,hh),n(x,Un),n(Un,oh),n(x,rh),n(x,In),n(In,ah),n(x,ih),n(x,Ln),n(Ln,sh),n(x,lh),s(t,qe,i),s(t,Y,i),n(Y,ht),n(ht,Vn),v(Ht,Vn,null),n(Y,ph),n(Y,Wn),n(Wn,uh),s(t,Ce,i),s(t,Wt,i),n(Wt,gh),s(t,De,i),s(t,ot,i),n(ot,Fn),n(Fn,P),n(P,Ft),n(Ft,mh),n(P,fh),n(P,Yt),n(Yt,vh),n(P,kh),n(P,Kt),n(Kt,dh),n(P,bh),n(P,Qt),n(Qt,_h),n(ot,$h),n(ot,q),n(q,C),n(C,Jt),n(Jt,zh),n(C,yh),n(C,Xt),n(Xt,Eh),n(C,Th),n(C,Zt),n(Zt,wh),n(C,xh),n(C,tn),n(tn,jh),n(q,Ph),n(q,D),n(D,nn),n(nn,qh),n(D,Ch),n(D,en),n(en,Dh),n(D,Bh),n(D,cn),n(cn,Nh),n(D,Ah),n(D,hn),n(hn,Hh),n(q,Sh),n(q,B),n(B,on),n(on,Oh),n(B,Gh),n(B,rn),n(rn,Mh),n(B,Rh),n(B,an),n(an,Uh),n(B,Ih),n(B,sn),n(sn,Lh),n(q,Vh),n(q,N),n(N,ln),n(ln,Wh),n(N,Fh),n(N,pn),n(pn,Yh),n(N,Kh),n(N,un),n(un,Qh),n(N,Jh),n(N,gn),n(gn,Xh),s(t,Be,i),s(t,mn,i),n(mn,Zh),Ne=!0},p(t,[i]){const St={};i&2&&(St.$$scope={dirty:i,ctx:t}),Z.$set(St)},i(t){Ne||(k(z.$$.fragment,t),k(A.$$.fragment,t),k(bt.$$.fragment,t),k(_t.$$.fragment,t),k(zt.$$.fragment,t),k(yt.$$.fragment,t),k(Et.$$.fragment,t),k(Tt.$$.fragment,t),k(Z.$$.fragment,t),k(wt.$$.fragment,t),k(xt.$$.fragment,t),k(jt.$$.fragment,t),k(Pt.$$.fragment,t),k(qt.$$.fragment,t),k(Ct.$$.fragment,t),k(Dt.$$.fragment,t),k(Bt.$$.fragment,t),k(Nt.$$.fragment,t),k(Ht.$$.fragment,t),Ne=!0)},o(t){d(z.$$.fragment,t),d(A.$$.fragment,t),d(bt.$$.fragment,t),d(_t.$$.fragment,t),d(zt.$$.fragment,t),d(yt.$$.fragment,t),d(Et.$$.fragment,t),d(Tt.$$.fragment,t),d(Z.$$.fragment,t),d(wt.$$.fragment,t),d(xt.$$.fragment,t),d(jt.$$.fragment,t),d(Pt.$$.fragment,t),d(qt.$$.fragment,t),d(Ct.$$.fragment,t),d(Dt.$$.fragment,t),d(Bt.$$.fragment,t),d(Nt.$$.fragment,t),d(Ht.$$.fragment,t),Ne=!1},d(t){e(g),t&&e(K),t&&e(_),b(z),t&&e(y),b(A,t),t&&e(vt),t&&e(Gt),t&&e(Jn),t&&e(I),t&&e(Xn),t&&e(H),t&&e(Zn),t&&e(L),b(bt),t&&e(te),b(_t,t),t&&e(ne),t&&e(J),t&&e(ee),t&&e(V),t&&e(ce),b(zt,t),t&&e(he),b(yt,t),t&&e(oe),t&&e(T),t&&e(re),b(Et,t),t&&e(ae),b(Tt,t),t&&e(ie),t&&e(X),t&&e(se),b(Z,t),t&&e(le),t&&e(W),b(wt),t&&e(pe),b(xt,t),t&&e(ue),t&&e(nt),t&&e(ge),t&&e(w),t&&e(me),b(jt,t),t&&e(fe),b(Pt,t),t&&e(ve),t&&e(S),t&&e(ke),t&&e(Rt),t&&e(de),b(qt,t),t&&e(be),t&&e(et),t&&e(_e),b(Ct,t),t&&e($e),t&&e(Ut),t&&e(ze),t&&e(It),t&&e(ye),b(Dt,t),t&&e(Ee),b(Bt,t),t&&e(Te),t&&e($),t&&e(we),t&&e(Lt),t&&e(xe),t&&e(F),b(Nt),t&&e(je),t&&e(j),t&&e(Pe),t&&e(x),t&&e(qe),t&&e(Y),b(Ht),t&&e(Ce),t&&e(Wt),t&&e(De),t&&e(ot),t&&e(Be),t&&e(mn)}}}const Pr={local:"chun-ho-v-tin-tokenize",sections:[{local:"chun-ho",title:"Chu\u1EA9n ho\xE1"},{local:"pretokenization",title:"Pre-tokenization"},{local:"sentencepiece",title:"SentencePiece"},{local:"tng-quan-thut-ton",title:"T\u1ED5ng quan thu\u1EADt to\xE1n"}],title:"Chu\u1EA9n ho\xE1 v\xE0 ti\u1EC1n tokenize"};function qr(Qn){return Er(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class Sr extends _r{constructor(g){super();$r(this,g,qr,jr,zr,{})}}export{Sr as default,Pr as metadata};
