import{S as jr,i as Ar,s as Dr,e as i,k as a,w as k,t as r,M as Cr,c as s,d as n,m as p,x as d,a as o,h as l,b as u,N as Ln,G as e,g as c,y as b,o as f,p as qr,q as v,B as $,v as Nr,n as Ir}from"../../chunks/vendor-hf-doc-builder.js";import{T as Mr}from"../../chunks/Tip-hf-doc-builder.js";import{Y as Ve}from"../../chunks/Youtube-hf-doc-builder.js";import{I as G}from"../../chunks/IconCopyLink-hf-doc-builder.js";import{C as D}from"../../chunks/CodeBlock-hf-doc-builder.js";import{C as Tr}from"../../chunks/CourseFloatingBanner-hf-doc-builder.js";import{F as Sr}from"../../chunks/FrameworkSwitchCourse-hf-doc-builder.js";import"../../chunks/DocNotebookDropdown-hf-doc-builder.js";function Vr(C){let g,y;return g=new Tr({props:{chapter:2,classNames:"absolute z-10 right-0 top-0",notebooks:[{label:"Google Colab",value:"https://colab.research.google.com/github/huggingface/notebooks/blob/master/course/vi/chapter2/section4_tf.ipynb"},{label:"Aws Studio",value:"https://studiolab.sagemaker.aws/import/github/huggingface/notebooks/blob/master/course/vi/chapter2/section4_tf.ipynb"}]}}),{c(){k(g.$$.fragment)},l(m){d(g.$$.fragment,m)},m(m,z){b(g,m,z),y=!0},i(m){y||(v(g.$$.fragment,m),y=!0)},o(m){f(g.$$.fragment,m),y=!1},d(m){$(g,m)}}}function Hr(C){let g,y;return g=new Tr({props:{chapter:2,classNames:"absolute z-10 right-0 top-0",notebooks:[{label:"Google Colab",value:"https://colab.research.google.com/github/huggingface/notebooks/blob/master/course/vi/chapter2/section4_pt.ipynb"},{label:"Aws Studio",value:"https://studiolab.sagemaker.aws/import/github/huggingface/notebooks/blob/master/course/vi/chapter2/section4_pt.ipynb"}]}}),{c(){k(g.$$.fragment)},l(m){d(g.$$.fragment,m)},m(m,z){b(g,m,z),y=!0},i(m){y||(v(g.$$.fragment,m),y=!0)},o(m){f(g.$$.fragment,m),y=!1},d(m){$(g,m)}}}function Lr(C){let g,y,m,z,w,_,x,P;return{c(){g=i("p"),y=r("T\u01B0\u01A1ng t\u1EF1 "),m=i("code"),z=r("TFAutoModel"),w=r(", l\u1EDBp "),_=i("code"),x=r("AutoTokenizer"),P=r(" s\u1EBD l\u1EA5y l\u1EDBp tokenizer th\xEDch h\u1EE3p trong th\u01B0 vi\u1EC7n d\u1EF1a tr\xEAn t\xEAn checkpoint v\xE0 c\xF3 th\u1EC3 \u0111\u01B0\u1EE3c s\u1EED d\u1EE5ng tr\u1EF1c ti\u1EBFp v\u1EDBi b\u1EA5t k\u1EF3 checkpoint n\xE0o:")},l(T){g=s(T,"P",{});var E=o(g);y=l(E,"T\u01B0\u01A1ng t\u1EF1 "),m=s(E,"CODE",{});var U=o(m);z=l(U,"TFAutoModel"),U.forEach(n),w=l(E,", l\u1EDBp "),_=s(E,"CODE",{});var S=o(_);x=l(S,"AutoTokenizer"),S.forEach(n),P=l(E," s\u1EBD l\u1EA5y l\u1EDBp tokenizer th\xEDch h\u1EE3p trong th\u01B0 vi\u1EC7n d\u1EF1a tr\xEAn t\xEAn checkpoint v\xE0 c\xF3 th\u1EC3 \u0111\u01B0\u1EE3c s\u1EED d\u1EE5ng tr\u1EF1c ti\u1EBFp v\u1EDBi b\u1EA5t k\u1EF3 checkpoint n\xE0o:"),E.forEach(n)},m(T,E){c(T,g,E),e(g,y),e(g,m),e(m,z),e(g,w),e(g,_),e(_,x),e(g,P)},d(T){T&&n(g)}}}function Br(C){let g,y,m,z,w,_,x,P;return{c(){g=i("p"),y=r("T\u01B0\u01A1ng t\u1EF1 "),m=i("code"),z=r("AutoModel"),w=r(", l\u1EDBp "),_=i("code"),x=r("AutoTokenizer"),P=r(" s\u1EBD l\u1EA5y l\u1EDBp tokenizer th\xEDch h\u1EE3p trong th\u01B0 vi\u1EC7n d\u1EF1a tr\xEAn t\xEAn checkpoint v\xE0 c\xF3 th\u1EC3 \u0111\u01B0\u1EE3c s\u1EED d\u1EE5ng tr\u1EF1c ti\u1EBFp v\u1EDBi b\u1EA5t k\u1EF3 checkpoint n\xE0o:")},l(T){g=s(T,"P",{});var E=o(g);y=l(E,"T\u01B0\u01A1ng t\u1EF1 "),m=s(E,"CODE",{});var U=o(m);z=l(U,"AutoModel"),U.forEach(n),w=l(E,", l\u1EDBp "),_=s(E,"CODE",{});var S=o(_);x=l(S,"AutoTokenizer"),S.forEach(n),P=l(E," s\u1EBD l\u1EA5y l\u1EDBp tokenizer th\xEDch h\u1EE3p trong th\u01B0 vi\u1EC7n d\u1EF1a tr\xEAn t\xEAn checkpoint v\xE0 c\xF3 th\u1EC3 \u0111\u01B0\u1EE3c s\u1EED d\u1EE5ng tr\u1EF1c ti\u1EBFp v\u1EDBi b\u1EA5t k\u1EF3 checkpoint n\xE0o:"),E.forEach(n)},m(T,E){c(T,g,E),e(g,y),e(g,m),e(m,z),e(g,w),e(g,_),e(_,x),e(g,P)},d(T){T&&n(g)}}}function Or(C){let g,y,m,z,w;return{c(){g=i("p"),y=r("\u270F\uFE0F "),m=i("strong"),z=r("Th\u1EED nghi\u1EC7m th\xF4i!"),w=r(" Sao ch\xE9p hai b\u01B0\u1EDBc cu\u1ED1i c\xF9ng (tokenize v\xE0 chuy\u1EC3n \u0111\u1ED5i sang ID \u0111\u1EA7u v\xE0o) tr\xEAn c\xE1c c\xE2u \u0111\u1EA7u v\xE0o m\xE0 ch\xFAng ta \u0111\xE3 s\u1EED d\u1EE5ng trong ph\u1EA7n 2 (\u201CI\u2019ve been waiting for a HuggingFace course my whole life.\u201D v\xE0 \u201CI hate this so much!\u201D). Ki\u1EC3m tra xem b\u1EA1n c\xF3 nh\u1EADn \u0111\u01B0\u1EE3c c\xE1c ID \u0111\u1EA7u v\xE0o gi\u1ED1ng nh\u01B0 ch\xFAng t\xF4i \u0111\xE3 nh\u1EADn tr\u01B0\u1EDBc \u0111\xF3 kh\xF4ng!")},l(_){g=s(_,"P",{});var x=o(g);y=l(x,"\u270F\uFE0F "),m=s(x,"STRONG",{});var P=o(m);z=l(P,"Th\u1EED nghi\u1EC7m th\xF4i!"),P.forEach(n),w=l(x," Sao ch\xE9p hai b\u01B0\u1EDBc cu\u1ED1i c\xF9ng (tokenize v\xE0 chuy\u1EC3n \u0111\u1ED5i sang ID \u0111\u1EA7u v\xE0o) tr\xEAn c\xE1c c\xE2u \u0111\u1EA7u v\xE0o m\xE0 ch\xFAng ta \u0111\xE3 s\u1EED d\u1EE5ng trong ph\u1EA7n 2 (\u201CI\u2019ve been waiting for a HuggingFace course my whole life.\u201D v\xE0 \u201CI hate this so much!\u201D). Ki\u1EC3m tra xem b\u1EA1n c\xF3 nh\u1EADn \u0111\u01B0\u1EE3c c\xE1c ID \u0111\u1EA7u v\xE0o gi\u1ED1ng nh\u01B0 ch\xFAng t\xF4i \u0111\xE3 nh\u1EADn tr\u01B0\u1EDBc \u0111\xF3 kh\xF4ng!"),x.forEach(n)},m(_,x){c(_,g,x),e(g,y),e(g,m),e(m,z),e(g,w)},d(_){_&&n(g)}}}function Gr(C){let g,y,m,z,w,_,x,P,T,E,U,S,N,I,an,wt,He,pn,Pc,Le,un,Tc,Be,xt,Oe,gn,jc,Ge,mn,Ac,Ue,K,ct,Bn,Pt,Dc,On,Cc,Ke,Tt,Fe,it,qc,Gn,Nc,Ic,Je,F,jt,Os,Mc,At,Gs,Re,st,Sc,Un,Vc,Hc,Qe,Dt,Ye,Ct,We,fn,Lc,Xe,vn,Bc,Ze,kn,Oc,th,qt,Gc,Kn,Uc,nh,ot,Kc,Fn,Fc,Jc,eh,J,rt,Jn,Nt,Rc,Rn,Qc,hh,It,ch,lt,Qn,Yc,Wc,Yn,Xc,ih,dn,Zc,sh,R,Mt,Us,ti,St,Ks,oh,bn,ni,rh,$n,ei,lh,at,hi,Wn,ci,ii,ah,Q,pt,Xn,Vt,si,Zn,oi,ph,Ht,uh,yn,ri,gh,_n,li,mh,En,ai,fh,Y,Lt,Fs,pi,Bt,Js,vh,zn,ui,kh,wn,gi,dh,W,ut,te,Ot,mi,ne,fi,bh,xn,vi,$h,V,ee,ki,di,he,bi,$i,ce,yi,yh,Pn,_i,_h,X,gt,ie,Gt,Ei,se,zi,Eh,j,wi,oe,xi,Pi,re,Ti,ji,le,Ai,Di,ae,Ci,qi,zh,mt,Ni,pe,Ii,Mi,wh,Ut,xh,Tn,Kt,Ph,jn,Si,Th,Ft,jh,Jt,Ah,An,Vi,Dh,Rt,Ch,A,Hi,ue,Li,Bi,Dn,Oi,Gi,ge,Ui,Ki,me,Fi,Ji,qh,Z,ft,fe,Qt,Ri,ve,Qi,Nh,Yt,Ih,H,Yi,ke,Wi,Xi,de,Zi,ts,Mh,vt,ns,be,es,hs,Sh,L,cs,$e,is,ss,ye,os,rs,Vh,Cn,ls,Hh,tt,kt,_e,Wt,as,Ee,ps,Lh,dt,us,ze,gs,ms,Bh,Xt,Oh,qn,fs,Gh,Zt,Uh,q,vs,we,ks,ds,xe,bs,$s,Pe,ys,_s,Kh,nt,bt,Te,tn,Es,je,zs,Fh,$t,ws,Ae,xs,Ps,Jh,nn,Rh,en,Qh,Nn,Ts,Yh,yt,Wh,et,_t,De,hn,js,Ce,As,Xh,M,qe,Ds,Cs,Ne,qs,Ns,Ie,Is,Ms,Zh,cn,tc,sn,nc,Et,Ss,Me,Vs,Hs,ec,In,Ls,hc;m=new Sr({props:{fw:C[0]}}),P=new G({});const Rs=[Hr,Vr],on=[];function Qs(t,h){return t[0]==="pt"?0:1}N=Qs(C),I=on[N]=Rs[N](C),wt=new Ve({props:{id:"VFp38yj8h3A"}}),xt=new D({props:{code:"Jim Henson was a puppeteer",highlighted:'<span class="hljs-comment">Jim Henson was a puppeteer</span>'}}),Pt=new G({}),Tt=new Ve({props:{id:"nhJxYji1aho"}}),Dt=new D({props:{code:`tokenized_text = "Jim Henson was a puppeteer".split()
print(tokenized_text)`,highlighted:`tokenized_text = <span class="hljs-string">&quot;Jim Henson was a puppeteer&quot;</span>.split()
<span class="hljs-built_in">print</span>(tokenized_text)`}}),Ct=new D({props:{code:"['Jim', 'Henson', 'was', 'a', 'puppeteer']",highlighted:'[<span class="hljs-string">&#x27;Jim&#x27;</span>, <span class="hljs-string">&#x27;Henson&#x27;</span>, <span class="hljs-string">&#x27;was&#x27;</span>, <span class="hljs-string">&#x27;a&#x27;</span>, <span class="hljs-string">&#x27;puppeteer&#x27;</span>]'}}),Nt=new G({}),It=new Ve({props:{id:"ssLq_EK2jLE"}}),Vt=new G({}),Ht=new Ve({props:{id:"zHvTiHr506c"}}),Ot=new G({}),Gt=new G({}),Ut=new D({props:{code:`from transformers import BertTokenizer

tokenizer = BertTokenizer.from_pretrained("bert-base-cased")`,highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> BertTokenizer

tokenizer = BertTokenizer.from_pretrained(<span class="hljs-string">&quot;bert-base-cased&quot;</span>)`}});function Ys(t,h){return t[0]==="pt"?Br:Lr}let cc=Ys(C),ht=cc(C);return Kt=new D({props:{code:`from transformers import AutoTokenizer

tokenizer = AutoTokenizer.from_pretrained("bert-base-cased")`,highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoTokenizer

tokenizer = AutoTokenizer.from_pretrained(<span class="hljs-string">&quot;bert-base-cased&quot;</span>)`}}),Ft=new D({props:{code:'tokenizer("Using a Transformer network is simple")',highlighted:'tokenizer(<span class="hljs-string">&quot;Using a Transformer network is simple&quot;</span>)'}}),Jt=new D({props:{code:`{'input_ids': [101, 7993, 170, 11303, 1200, 2443, 1110, 3014, 102],
 'token_type_ids': [0, 0, 0, 0, 0, 0, 0, 0, 0],
 'attention_mask': [1, 1, 1, 1, 1, 1, 1, 1, 1]}`,highlighted:`{<span class="hljs-string">&#x27;input_ids&#x27;</span>: [<span class="hljs-number">101</span>, <span class="hljs-number">7993</span>, <span class="hljs-number">170</span>, <span class="hljs-number">11303</span>, <span class="hljs-number">1200</span>, <span class="hljs-number">2443</span>, <span class="hljs-number">1110</span>, <span class="hljs-number">3014</span>, <span class="hljs-number">102</span>],
 <span class="hljs-string">&#x27;token_type_ids&#x27;</span>: [<span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>],
 <span class="hljs-string">&#x27;attention_mask&#x27;</span>: [<span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>]}`}}),Rt=new D({props:{code:'tokenizer.save_pretrained("directory_on_my_computer")',highlighted:'tokenizer.save_pretrained(<span class="hljs-string">&quot;directory_on_my_computer&quot;</span>)'}}),Qt=new G({}),Yt=new Ve({props:{id:"Yffk5aydLzg"}}),Wt=new G({}),Xt=new D({props:{code:`from transformers import AutoTokenizer

tokenizer = AutoTokenizer.from_pretrained("bert-base-cased")

sequence = "Using a Transformer network is simple"
tokens = tokenizer.tokenize(sequence)

print(tokens)`,highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoTokenizer

tokenizer = AutoTokenizer.from_pretrained(<span class="hljs-string">&quot;bert-base-cased&quot;</span>)

sequence = <span class="hljs-string">&quot;Using a Transformer network is simple&quot;</span>
tokens = tokenizer.tokenize(sequence)

<span class="hljs-built_in">print</span>(tokens)`}}),Zt=new D({props:{code:"['Using', 'a', 'transform', '##er', 'network', 'is', 'simple']",highlighted:'[<span class="hljs-string">&#x27;Using&#x27;</span>, <span class="hljs-string">&#x27;a&#x27;</span>, <span class="hljs-string">&#x27;transform&#x27;</span>, <span class="hljs-string">&#x27;##er&#x27;</span>, <span class="hljs-string">&#x27;network&#x27;</span>, <span class="hljs-string">&#x27;is&#x27;</span>, <span class="hljs-string">&#x27;simple&#x27;</span>]'}}),tn=new G({}),nn=new D({props:{code:`ids = tokenizer.convert_tokens_to_ids(tokens)

print(ids)`,highlighted:`ids = tokenizer.convert_tokens_to_ids(tokens)

<span class="hljs-built_in">print</span>(ids)`}}),en=new D({props:{code:"[7993, 170, 11303, 1200, 2443, 1110, 3014]",highlighted:'[<span class="hljs-number">7993</span>, <span class="hljs-number">170</span>, <span class="hljs-number">11303</span>, <span class="hljs-number">1200</span>, <span class="hljs-number">2443</span>, <span class="hljs-number">1110</span>, <span class="hljs-number">3014</span>]'}}),yt=new Mr({props:{$$slots:{default:[Or]},$$scope:{ctx:C}}}),hn=new G({}),cn=new D({props:{code:`decoded_string = tokenizer.decode([7993, 170, 11303, 1200, 2443, 1110, 3014])
print(decoded_string)`,highlighted:`decoded_string = tokenizer.decode([<span class="hljs-number">7993</span>, <span class="hljs-number">170</span>, <span class="hljs-number">11303</span>, <span class="hljs-number">1200</span>, <span class="hljs-number">2443</span>, <span class="hljs-number">1110</span>, <span class="hljs-number">3014</span>])
<span class="hljs-built_in">print</span>(decoded_string)`}}),sn=new D({props:{code:"'Using a Transformer network is simple'",highlighted:'<span class="hljs-string">&#x27;Using a Transformer network is simple&#x27;</span>'}}),{c(){g=i("meta"),y=a(),k(m.$$.fragment),z=a(),w=i("h1"),_=i("a"),x=i("span"),k(P.$$.fragment),T=a(),E=i("span"),U=r("Tokenizers"),S=a(),I.c(),an=a(),k(wt.$$.fragment),He=a(),pn=i("p"),Pc=r("Tokenizer l\xE0 m\u1ED9t trong nh\u1EEFng th\xE0nh ph\u1EA7n c\u1ED1t l\xF5i c\u1EE7a pipeline NLP. Ch\xFAng ph\u1EE5c v\u1EE5 m\u1ED9t m\u1EE5c \u0111\xEDch: d\u1ECBch v\u0103n b\u1EA3n th\xE0nh d\u1EEF li\u1EC7u c\xF3 th\u1EC3 \u0111\u01B0\u1EE3c x\u1EED l\xFD b\u1EDFi m\xF4 h\xECnh. M\xF4 h\xECnh ch\u1EC9 c\xF3 th\u1EC3 x\u1EED l\xFD d\u1EA1ng s\u1ED1, do \u0111\xF3, c\xE1c tokenizer c\u1EA7n ph\u1EA3i chuy\u1EC3n \u0111\u1ED5i \u0111\u1EA7u v\xE0o v\u0103n b\u1EA3n c\u1EE7a ch\xFAng ta th\xE0nh d\u1EEF li\u1EC7u s\u1ED1. Trong ph\u1EA7n n\xE0y, ch\xFAng ta s\u1EBD kh\xE1m ph\xE1 ch\xEDnh x\xE1c nh\u1EEFng g\xEC x\u1EA3y ra trong \u0111\u01B0\u1EDDng d\u1EABn m\xE3 h\xF3a."),Le=a(),un=i("p"),Tc=r("Trong c\xE1c t\xE1c v\u1EE5 NLP, d\u1EEF li\u1EC7u th\u01B0\u1EDDng \u0111\u01B0\u1EE3c x\u1EED l\xFD l\xE0 v\u0103n b\u1EA3n th\xF4. \u0110\xE2y l\xE0 m\u1ED9t v\xED d\u1EE5 v\u1EC1 v\u0103n b\u1EA3n nh\u01B0 v\u1EADy:"),Be=a(),k(xt.$$.fragment),Oe=a(),gn=i("p"),jc=r("Tuy nhi\xEAn, c\xE1c m\xF4 h\xECnh ch\u1EC9 c\xF3 th\u1EC3 x\u1EED l\xFD s\u1ED1, v\xEC v\u1EADy ch\xFAng ta c\u1EA7n t\xECm c\xE1ch chuy\u1EC3n v\u0103n b\u1EA3n th\xF4 th\xE0nh s\u1ED1. \u0110\xF3 l\xE0 nh\u1EEFng g\xEC m\xE0 tokenizer l\xE0m, v\xE0 c\xF3 r\u1EA5t nhi\u1EC1u c\xE1ch \u0111\u1EC3 th\u1EF1c hi\u1EC7n \u0111i\u1EC1u n\xE0y. M\u1EE5c ti\xEAu \u0111\u1EC1 ra l\xE0 t\xECm ra c\xE1ch bi\u1EC3u di\u1EC5n c\xF3 \xFD ngh\u0129a nh\u1EA5t - ngh\u0129a l\xE0 c\xE1i c\xF3 \xFD ngh\u0129a nh\u1EA5t \u0111\u1ED1i v\u1EDBi m\xF4 h\xECnh - v\xE0, n\u1EBFu c\xF3 th\u1EC3, l\xE0 c\xE1ch bi\u1EC3u di\u1EC5n nh\u1ECF nh\u1EA5t."),Ge=a(),mn=i("p"),Ac=r("H\xE3y c\xF9ng xem m\u1ED9t s\u1ED1 v\xED d\u1EE5 v\u1EC1 thu\u1EADt to\xE1n tokenize v\xE0 c\u1ED1 g\u1EAFng tr\u1EA3 l\u1EDDi m\u1ED9t s\u1ED1 c\xE2u h\u1ECFi b\u1EA1n c\xF3 th\u1EC3 c\xF3 v\u1EC1 tokenize."),Ue=a(),K=i("h2"),ct=i("a"),Bn=i("span"),k(Pt.$$.fragment),Dc=a(),On=i("span"),Cc=r("D\u1EF1a tr\xEAn t\u1EEB"),Ke=a(),k(Tt.$$.fragment),Fe=a(),it=i("p"),qc=r("Lo\u1EA1i tokenizer \u0111\u1EA7u ti\xEAn ta ngh\u0129 \u0111\u1EBFn \u0111\xF3 l\xE0 "),Gn=i("em"),Nc=r("d\u1EF1a tr\xEAn t\u1EEB v\u1EF1ng"),Ic=r(". N\xF3 th\u01B0\u1EDDng r\u1EA5t d\u1EC5 thi\u1EBFt l\u1EADp v\xE0 s\u1EED d\u1EE5ng ch\u1EC9 v\u1EDBi m\u1ED9t s\u1ED1 quy t\u1EAFc v\xE0 n\xF3 th\u01B0\u1EDDng mang l\u1EA1i k\u1EBFt qu\u1EA3 t\u1ED1t. V\xED d\u1EE5: trong h\xECnh \u1EA3nh b\xEAn d\u01B0\u1EDBi, m\u1EE5c ti\xEAu l\xE0 t\xE1ch v\u0103n b\u1EA3n th\xF4 th\xE0nh c\xE1c t\u1EEB v\xE0 t\xECm bi\u1EC3u di\u1EC5n s\u1ED1 cho m\u1ED7i t\u1EEB:"),Je=a(),F=i("div"),jt=i("img"),Mc=a(),At=i("img"),Re=a(),st=i("p"),Sc=r("C\xF3 nhi\u1EC1u c\xE1ch kh\xE1c nhau \u0111\u1EC3 t\xE1ch v\u0103n b\u1EA3n. V\xED d\u1EE5: ch\xFAng ta c\xF3 th\u1EC3 s\u1EED d\u1EE5ng kho\u1EA3ng tr\u1EAFng \u0111\u1EC3 tokenize v\u0103n b\u1EA3n th\xE0nh c\xE1c t\u1EEB b\u1EB1ng c\xE1ch \xE1p d\u1EE5ng h\xE0m "),Un=i("code"),Vc=r("split()"),Hc=r(" c\u1EE7a Python:"),Qe=a(),k(Dt.$$.fragment),Ye=a(),k(Ct.$$.fragment),We=a(),fn=i("p"),Lc=r("Ngo\xE0i ra c\xF2n c\xF3 c\xE1c bi\u1EBFn th\u1EC3 c\u1EE7a tokenize m\u1EE9c t\u1EEB v\u1EDBi c\xE1c quy t\u1EAFc b\u1ED5 sung cho d\u1EA5u c\xE2u. V\u1EDBi lo\u1EA1i tokenizer n\xE0y, ch\xFAng ta c\xF3 th\u1EC3 \u0111\xFAc k\u1EBFt v\u1EDBi m\u1ED9t b\u1ED9 \u201Ct\u1EEB v\u1EF1ng\u201D kh\xE1 l\u1EDBn, trong \u0111\xF3 t\u1EEB v\u1EF1ng \u0111\u01B0\u1EE3c x\xE1c \u0111\u1ECBnh b\u1EB1ng t\u1ED5ng s\u1ED1 token \u0111\u1ED9c l\u1EADp m\xE0 ch\xFAng ta c\xF3 trong corpus (kho ng\u1EEF li\u1EC7u) c\u1EE7a m\xECnh."),Xe=a(),vn=i("p"),Bc=r("M\u1ED7i t\u1EEB \u0111\u01B0\u1EE3c g\xE1n m\u1ED9t ID, b\u1EAFt \u0111\u1EA7u t\u1EEB 0 v\xE0 t\u0103ng d\u1EA7n theo k\xEDch th\u01B0\u1EDBc c\u1EE7a b\u1ED9 t\u1EEB v\u1EF1ng. M\xF4 h\xECnh s\u1EED d\u1EE5ng c\xE1c ID n\xE0y \u0111\u1EC3 x\xE1c \u0111\u1ECBnh t\u1EEBng t\u1EEB."),Ze=a(),kn=i("p"),Oc=r("N\u1EBFu ch\xFAng ta mu\u1ED1n bao ph\u1EE7 ho\xE0n to\xE0n m\u1ED9t ng\xF4n ng\u1EEF b\u1EB1ng tokenize m\u1EE9c t\u1EEB, ch\xFAng ta s\u1EBD c\u1EA7n ph\u1EA3i c\xF3 m\u1ED9t ch\u1EC9 s\u1ED1 nh\u1EADn d\u1EA1ng cho m\u1ED7i t\u1EEB trong ng\xF4n ng\u1EEF, \u0111i\u1EC1u n\xE0y s\u1EBD t\u1EA1o ra m\u1ED9t l\u01B0\u1EE3ng l\u1EDBn token. V\xED d\u1EE5: c\xF3 h\u01A1n 500,000 t\u1EEB trong ti\u1EBFng Anh, v\xEC v\u1EADy \u0111\u1EC3 x\xE2y d\u1EF1ng b\u1EA3n \u0111\u1ED3 n\u1ED1i m\u1ED7i t\u1EEB \u0111\u1EBFn m\u1ED9t ID \u0111\u1EA7u v\xE0o, ch\xFAng ta c\u1EA7n theo d\xF5i ng\u1EA7n \u0111\xF3 ID. H\u01A1n n\u1EEFa, c\xE1c t\u1EEB nh\u01B0 \u201Cdog\u201D \u0111\u01B0\u1EE3c bi\u1EC3u di\u1EC5n kh\xE1c v\u1EDBi c\xE1c t\u1EEB nh\u01B0 \u201Cdogs\u201D, v\xE0 ban \u0111\u1EA7u m\xF4 h\xECnh s\u1EBD kh\xF4ng c\xF3 c\xE1ch n\xE0o \u0111\u1EC3 bi\u1EBFt r\u1EB1ng \u201Cdog\u201D (ch\xF3) v\xE0 \u201Cdogs\u201D l\xE0 t\u01B0\u01A1ng t\u1EF1 nhau: n\xF3 s\u1EBD x\xE1c \u0111\u1ECBnh hai t\u1EEB n\xE0y kh\xF4ng li\xEAn quan. \u0110i\u1EC1u n\xE0y c\u0169ng \xE1p d\u1EE5ng cho c\xE1c t\u1EEB t\u01B0\u01A1ng t\u1EF1 kh\xE1c, nh\u01B0 \u201Crun\u201D (ch\u1EA1y) v\xE0 \u201Crunning\u201D, m\xE0 ban \u0111\u1EA7u m\xF4 h\xECnh s\u1EBD kh\xF4ng th\u1EA5y l\xE0 t\u01B0\u01A1ng t\u1EF1."),th=a(),qt=i("p"),Gc=r("Cu\u1ED1i c\xF9ng, ch\xFAng ta c\u1EA7n m\u1ED9t token t\xF9y ch\u1EC9nh \u0111\u1EC3 \u0111\u1EA1i di\u1EC7n cho c\xE1c t\u1EEB kh\xF4ng c\xF3 trong v\u1ED1n t\u1EEB v\u1EF1ng c\u1EE7a ch\xFAng ta. M\xE3 n\xE0y \u0111\u01B0\u1EE3c g\u1ECDi l\xE0 token \u201Ckh\xF4ng x\xE1c \u0111\u1ECBnh\u201D, th\u01B0\u1EDDng \u0111\u01B0\u1EE3c bi\u1EC3u th\u1ECB l\xE0 \u201D[UNK]\u201D ho\u1EB7c \u201D"),Kn=i("unk"),Uc=r("\u201D. N\xF3i chung, \u0111\xF3 l\xE0 m\u1ED9t d\u1EA5u hi\u1EC7u x\u1EA5u n\u1EBFu b\u1EA1n th\u1EA5y tr\xECnh tokenize \u0111ang t\u1EA1o ra r\u1EA5t nhi\u1EC1u token n\xE0y, v\xEC n\xF3 kh\xF4ng th\u1EC3 truy xu\u1EA5t m\u1ED9t bi\u1EC3u hi\u1EC7n h\u1EE3p l\xFD c\u1EE7a m\u1ED9t t\u1EEB v\xE0 b\u1EA1n \u0111ang m\u1EA5t th\xF4ng tin trong su\u1ED1t qu\xE1 tr\xECnh. M\u1EE5c ti\xEAu khi t\u1EA1o t\u1EEB v\u1EF1ng l\xE0 l\xE0m sao cho tr\xECnh tokenize m\xE3 h\xF3a c\xE0ng \xEDt t\u1EEB th\xE0nh token kh\xF4ng x\xE1c \u0111\u1ECBnh c\xE0ng t\u1ED1t."),nh=a(),ot=i("p"),Kc=r("M\u1ED9t c\xE1ch \u0111\u1EC3 gi\u1EA3m s\u1ED1 l\u01B0\u1EE3ng m\xE3 th\xF4ng b\xE1o kh\xF4ng x\xE1c \u0111\u1ECBnh l\xE0 \u0111i s\xE2u h\u01A1n xu\u1ED1ng m\u1ED9t c\u1EA5p, s\u1EED d\u1EE5ng tokenize "),Fn=i("em"),Fc=r("m\u1EE9c k\xED t\u1EF1"),Jc=r("."),eh=a(),J=i("h2"),rt=i("a"),Jn=i("span"),k(Nt.$$.fragment),Rc=a(),Rn=i("span"),Qc=r("D\u1EF1a tr\xEAn k\xED t\u1EF1"),hh=a(),k(It.$$.fragment),ch=a(),lt=i("ul"),Qn=i("li"),Yc=r("V\u1ED1n t\u1EEB v\u1EF1ng \xEDt h\u01A1n nhi\u1EC1u."),Wc=a(),Yn=i("li"),Xc=r("C\xF3 \xEDt token ngo\xE0i b\u1ED9 t\u1EEB v\u1EF1ng (kh\xF4ng x\xE1c \u0111\u1ECBnh) h\u01A1n nhi\u1EC1u, v\xEC m\u1ECDi t\u1EEB \u0111\u1EC1u c\xF3 th\u1EC3 \u0111\u01B0\u1EE3c x\xE2y d\u1EF1ng t\u1EEB c\xE1c k\xFD t\u1EF1."),ih=a(),dn=i("p"),Zc=r("Nh\u01B0ng \u1EDF \u0111\xE2y c\u0169ng c\xF3 m\u1ED9t s\u1ED1 c\xE2u h\u1ECFi n\u1EA3y sinh li\xEAn quan \u0111\u1EBFn d\u1EA5u c\xE1ch v\xE0 c\xE1c d\u1EA5u c\xE2u:"),sh=a(),R=i("div"),Mt=i("img"),ti=a(),St=i("img"),oh=a(),bn=i("p"),ni=r("C\xE1ch ti\u1EBFp c\u1EADn n\xE0y c\u0169ng kh\xF4ng ho\xE0n h\u1EA3o. V\xEC bi\u1EC3u di\u1EC5n b\xE2y gi\u1EDD d\u1EF1a tr\xEAn c\xE1c k\xFD t\u1EF1 ch\u1EE9 kh\xF4ng ph\u1EA3i t\u1EEB, ng\u01B0\u1EDDi ta c\xF3 th\u1EC3 l\u1EADp lu\u1EADn r\u1EB1ng, theo tr\u1EF1c gi\xE1c, n\xF3 \xEDt \xFD ngh\u0129a h\u01A1n: m\u1ED7i k\xFD t\u1EF1 kh\xF4ng c\xF3 nhi\u1EC1u \xFD ngh\u0129a ri\xEAng so v\u1EDBi tr\u01B0\u1EDDng h\u1EE3p c\u1EE7a c\xE1c t\u1EEB. Tuy nhi\xEAn, \u0111i\u1EC1u n\xE0y l\u1EA1i kh\xE1c nhau t\xF9y theo ng\xF4n ng\u1EEF; trong ti\u1EBFng Trung, ch\u1EB3ng h\u1EA1n, m\u1ED7i k\xFD t\u1EF1 mang nhi\u1EC1u th\xF4ng tin h\u01A1n m\u1ED9t k\xFD t\u1EF1 trong ng\xF4n ng\u1EEF Latinh."),rh=a(),$n=i("p"),ei=r("M\u1ED9t \u0111i\u1EC1u kh\xE1c c\u1EA7n xem x\xE9t l\xE0 ch\xFAng ta s\u1EBD c\xF3 m\u1ED9t l\u01B0\u1EE3ng r\u1EA5t l\u1EDBn token s\u1EBD \u0111\u01B0\u1EE3c x\u1EED l\xFD b\u1EDFi m\xF4 h\xECnh c\u1EE7a ch\xFAng ta: trong khi m\u1ED9t t\u1EEB ch\u1EC9 l\xE0 m\u1ED9t token duy nh\u1EA5t khi tokenize d\u1EF1a tr\xEAn t\u1EEB, n\xF3 c\xF3 th\u1EC3 d\u1EC5 d\xE0ng chuy\u1EC3n th\xE0nh 10 token tr\u1EDF l\xEAn khi chuy\u1EC3n \u0111\u1ED5i th\xE0nh c\xE1c k\xFD t\u1EF1."),lh=a(),at=i("p"),hi=r("\u0110\u1EC3 t\u1EADn d\u1EE5ng t\u1ED1i \u0111a c\u1EA3 hai, ch\xFAng ta c\xF3 th\u1EC3 s\u1EED d\u1EE5ng k\u1EF9 thu\u1EADt th\u1EE9 ba k\u1EBFt h\u1EE3p hai c\xE1ch ti\u1EBFp c\u1EADn: "),Wn=i("em"),ci=r("tokenize theo t\u1EEB ph\u1EE5"),ii=r("."),ah=a(),Q=i("h2"),pt=i("a"),Xn=i("span"),k(Vt.$$.fragment),si=a(),Zn=i("span"),oi=r("Tokenize theo t\u1EEB ph\u1EE5"),ph=a(),k(Ht.$$.fragment),uh=a(),yn=i("p"),ri=r("C\xE1c thu\u1EADt to\xE1n token theo t\u1EEB ph\u1EE5 d\u1EF1a tr\xEAn nguy\xEAn t\u1EAFc r\u1EB1ng c\xE1c t\u1EEB \u0111\u01B0\u1EE3c s\u1EED d\u1EE5ng th\u01B0\u1EDDng xuy\xEAn kh\xF4ng \u0111\u01B0\u1EE3c chia th\xE0nh c\xE1c t\u1EEB ph\u1EE5 nh\u1ECF h\u01A1n, nh\u01B0ng c\xE1c t\u1EEB hi\u1EBFm ph\u1EA3i \u0111\u01B0\u1EE3c ph\xE2n t\xE1ch th\xE0nh c\xE1c t\u1EEB ph\u1EE5 c\xF3 \xFD ngh\u0129a."),gh=a(),_n=i("p"),li=r("V\xED d\u1EE5: \u201Cannoyingly\u201D (kh\xF3 ch\u1ECBu) c\xF3 th\u1EC3 \u0111\u01B0\u1EE3c coi l\xE0 m\u1ED9t t\u1EEB hi\u1EBFm v\xE0 c\xF3 th\u1EC3 \u0111\u01B0\u1EE3c chuy\u1EC3n th\xE0nh \u201Cannoying\u201D v\xE0 \u201Cly\u201D. C\u1EA3 hai \u0111\u1EC1u c\xF3 kh\u1EA3 n\u0103ng xu\u1EA5t hi\u1EC7n th\u01B0\u1EDDng xuy\xEAn h\u01A1n d\u01B0\u1EDBi d\u1EA1ng c\xE1c t\u1EEB ph\u1EE5 \u0111\u1ED9c l\u1EADp, \u0111\u1ED3ng th\u1EDDi ngh\u0129a c\u1EE7a \u201Cannoying\u201D \u0111\u01B0\u1EE3c gi\u1EEF nguy\xEAn b\u1EDFi ngh\u0129a k\u1EBFt h\u1EE3p c\u1EE7a \u201Cannoying\u201D v\xE0 \u201Cly\u201D."),mh=a(),En=i("p"),ai=r("D\u01B0\u1EDBi \u0111\xE2y l\xE0 m\u1ED9t v\xED d\u1EE5 cho th\u1EA5y c\xE1ch m\u1ED9t thu\u1EADt to\xE1n tokenize theo t\u1EEB ph\u1EE5 s\u1EBD tokenize chu\u1ED7i \u201CLet\u2019s do tokenization!\u201D (H\xE3y th\u1EF1c hi\u1EC7n tokenize!):"),fh=a(),Y=i("div"),Lt=i("img"),pi=a(),Bt=i("img"),vh=a(),zn=i("p"),ui=r("Nh\u1EEFng t\u1EEB ph\u1EE5 n\xE0y cung c\u1EA5p r\u1EA5t nhi\u1EC1u \xFD ngh\u0129a v\u1EC1 m\u1EB7t ng\u1EEF ngh\u0129a: v\xED d\u1EE5: trong v\xED d\u1EE5 \u1EDF tr\xEAn \u201Ctokenization\u201D \u0111\u01B0\u1EE3c chia th\xE0nh \u201Ctoken\u201D v\xE0 \u201Cization\u201D, hai token \u0111\u1EC1u c\xF3 \xFD ngh\u0129a v\u1EC1 m\u1EB7t ng\u1EEF ngh\u0129a \u0111\u1ED3ng th\u1EDDi ti\u1EBFt ki\u1EC7m kh\xF4ng gian (ch\u1EC9 c\u1EA7n hai token \u0111\u1EC3 bi\u1EC3u th\u1ECB m\u1ED9t t\u1EEB d\xE0i). \u0110i\u1EC1u n\xE0y cho ph\xE9p ch\xFAng ta c\xF3 th\u1EC3 bao qu\xE1t t\u01B0\u01A1ng \u0111\u1ED1i t\u1ED1t v\u1EDBi c\xE1c t\u1EEB v\u1EF1ng nh\u1ECF v\xE0 g\u1EA7n nh\u01B0 kh\xF4ng c\xF3 token n\xE0o kh\xF4ng x\xE1c \u0111\u1ECBnh."),kh=a(),wn=i("p"),gi=r("C\xE1ch ti\u1EBFp c\u1EADn n\xE0y \u0111\u1EB7c bi\u1EC7t h\u1EEFu \xEDch trong c\xE1c ng\xF4n ng\u1EEF t\u1ED5ng h\u1EE3p nh\u01B0 ti\u1EBFng Th\u1ED5 Nh\u0129 K\u1EF3, n\u01A1i b\u1EA1n c\xF3 th\u1EC3 t\u1EA1o (g\u1EA7n nh\u01B0) c\xE1c t\u1EEB ph\u1EE9c d\xE0i t\xF9y \xFD b\u1EB1ng c\xE1ch x\xE2u chu\u1ED7i c\xE1c t\u1EEB ph\u1EE5 l\u1EA1i v\u1EDBi nhau."),dh=a(),W=i("h3"),ut=i("a"),te=i("span"),k(Ot.$$.fragment),mi=a(),ne=i("span"),fi=r("V\xE0 h\u01A1n th\u1EBF n\u1EEFa!"),bh=a(),xn=i("p"),vi=r("Kh\xF4ng c\xF3 g\xEC \u0111\xE1ng ng\u1EA1c nhi\xEAn, c\xF3 r\u1EA5t nhi\u1EC1u k\u1EF9 thu\u1EADt kh\xE1c, c\xF3 th\u1EC3 k\u1EC3 \u0111\u1EBFn:"),$h=a(),V=i("ul"),ee=i("li"),ki=r("Byte-level BPE (BPE c\u1EA5p byte), nh\u01B0 \u0111\u01B0\u1EE3c s\u1EED d\u1EE5ng trong GPT-2"),di=a(),he=i("li"),bi=r("WordPiece, nh\u01B0 \u0111\u01B0\u1EE3c s\u1EED d\u1EE5ng trong BERT"),$i=a(),ce=i("li"),yi=r("SentencePiece ho\u1EB7c Unigram, nh\u01B0 \u0111\u01B0\u1EE3c s\u1EED d\u1EE5ng trong m\u1ED9t s\u1ED1 m\xF4 h\xECnh \u0111a ng\xF4n ng\u1EEF"),yh=a(),Pn=i("p"),_i=r("B\xE2y gi\u1EDD, b\u1EA1n \u0111\xE3 c\xF3 \u0111\u1EE7 ki\u1EBFn th\u1EE9c v\u1EC1 c\xE1ch th\u1EE9c ho\u1EA1t \u0111\u1ED9ng c\u1EE7a tokenize \u0111\u1EC3 b\u1EAFt \u0111\u1EA7u v\u1EDBi API."),_h=a(),X=i("h2"),gt=i("a"),ie=i("span"),k(Gt.$$.fragment),Ei=a(),se=i("span"),zi=r("T\u1EA3i v\xE0 l\u01B0u"),Eh=a(),j=i("p"),wi=r("Vi\u1EC7c t\u1EA3i v\xE0 l\u01B0u tokenizer c\u0169ng \u0111\u01A1n gi\u1EA3n nh\u01B0 v\u1EDBi c\xE1c m\xF4 h\xECnh. Tr\xEAn th\u1EF1c t\u1EBF, n\xF3 d\u1EF1a tr\xEAn hai ph\u01B0\u01A1ng th\u1EE9c gi\u1ED1ng nhau: "),oe=i("code"),xi=r("from_pretrained()"),Pi=r(" v\xE0 "),re=i("code"),Ti=r("save_pretrained()"),ji=r(". C\xE1c ph\u01B0\u01A1ng th\u1EE9c n\xE0y s\u1EBD t\u1EA3i ho\u1EB7c l\u01B0u thu\u1EADt to\xE1n \u0111\u01B0\u1EE3c s\u1EED d\u1EE5ng b\u1EDFi tokenizer (h\u01A1i gi\u1ED1ng v\u1EDBi "),le=i("em"),Ai=r("ki\u1EBFn tr\xFAc"),Di=r(" c\u1EE7a m\xF4 h\xECnh) c\u0169ng nh\u01B0 t\u1EEB v\u1EF1ng c\u1EE7a n\xF3 (h\u01A1i gi\u1ED1ng v\u1EDBi "),ae=i("em"),Ci=r("tr\u1ECDng s\u1ED1"),qi=r(" c\u1EE7a m\xF4 h\xECnh)."),zh=a(),mt=i("p"),Ni=r("Vi\u1EC7c t\u1EA3i BERT tokenizer \u0111\u01B0\u1EE3c hu\u1EA5n luy\u1EC7n v\u1EDBi c\xF9ng m\u1ED9t checkpoint v\u1EDBi BERT \u0111\u01B0\u1EE3c th\u1EF1c hi\u1EC7n gi\u1ED1ng nh\u01B0 c\xE1ch t\u1EA3i m\xF4 h\xECnh, ngo\u1EA1i tr\u1EEB vi\u1EC7c ch\xFAng ta s\u1EED d\u1EE5ng l\u1EDBp "),pe=i("code"),Ii=r("BertTokenizer"),Mi=r(":"),wh=a(),k(Ut.$$.fragment),xh=a(),ht.c(),Tn=a(),k(Kt.$$.fragment),Ph=a(),jn=i("p"),Si=r("Gi\u1EDD ch\xFAng ta c\xF3 th\u1EC3 s\u1EED d\u1EE5ng tokenizer nh\u01B0 trong \u0111o\u1EA1n d\u01B0\u1EDBi \u0111\xE2y:"),Th=a(),k(Ft.$$.fragment),jh=a(),k(Jt.$$.fragment),Ah=a(),An=i("p"),Vi=r("L\u01B0u m\u1ED9t tokenizer gi\u1ED1ng nh\u01B0 khi l\u01B0u m\u1ED9t m\xF4 h\xECnh v\u1EADy:"),Dh=a(),k(Rt.$$.fragment),Ch=a(),A=i("p"),Hi=r("Ch\xFAng ta s\u1EBD trao \u0111\u1ED5i th\xEAm v\u1EC1 "),ue=i("code"),Li=r("token_type_ids"),Bi=r(" trong "),Dn=i("a"),Oi=r("Ch\u01B0\u01A1ng 3"),Gi=r(", v\xE0 ch\xFAng ta s\u1EBD gi\u1EA3i th\xEDch c\u01A1 ch\u1EBF c\u1EE7a t\u1EEB kho\xE1 "),ge=i("code"),Ui=r("attention_mask"),Ki=r(" sau \u0111\xF3. \u0110\u1EA7u ti\xEAnm h\xE3y c\xF9ng xem c\xE1ch "),me=i("code"),Fi=r("input_ids"),Ji=r(" \u0111\u01B0\u1EE3c t\u1EA1o ra. \u0110\u1EC3 l\xE0m \u0111i\u1EC1u n\xE0y, ch\xFAng ta s\u1EBD c\u1EA7n xem x\xE9t c\xE1c ph\u01B0\u01A1ng th\u1EE9c trung gian c\u1EE7a tokenizer."),qh=a(),Z=i("h2"),ft=i("a"),fe=i("span"),k(Qt.$$.fragment),Ri=a(),ve=i("span"),Qi=r("M\xE3 ho\xE1"),Nh=a(),k(Yt.$$.fragment),Ih=a(),H=i("p"),Yi=r("D\u1ECBch v\u0103n b\u1EA3n sang s\u1ED1 \u0111\u01B0\u1EE3c g\u1ECDi l\xE0 "),ke=i("em"),Wi=r("encoding"),Xi=r(" hay "),de=i("em"),Zi=r("m\xE3 ho\xE1"),ts=r(". Vi\u1EC7c m\xE3 h\xF3a \u0111\u01B0\u1EE3c th\u1EF1c hi\u1EC7n theo quy tr\xECnh g\u1ED3m hai b\u01B0\u1EDBc: tokenize, ti\u1EBFp theo l\xE0 chuy\u1EC3n \u0111\u1ED5i sang ID \u0111\u1EA7u v\xE0o."),Mh=a(),vt=i("p"),ns=r("Nh\u01B0 ch\xFAng ta \u0111\xE3 th\u1EA5y, b\u01B0\u1EDBc \u0111\u1EA7u ti\xEAn l\xE0 chia v\u0103n b\u1EA3n th\xE0nh c\xE1c t\u1EEB (ho\u1EB7c c\xE1c ph\u1EA7n c\u1EE7a t\u1EEB,theo k\xFD hi\u1EC7u d\u1EA5u c\xE2u, v.v.), th\u01B0\u1EDDng \u0111\u01B0\u1EE3c g\u1ECDi l\xE0 "),be=i("em"),es=r("token"),hs=r(". C\xF3 nhi\u1EC1u quy t\u1EAFc c\xF3 th\u1EC3 chi ph\u1ED1i qu\xE1 tr\xECnh \u0111\xF3, \u0111\xF3 l\xE0 l\xFD do t\u1EA1i sao ch\xFAng ta c\u1EA7n kh\u1EDFi t\u1EA1o tr\xECnh token b\u1EB1ng c\xE1ch s\u1EED d\u1EE5ng t\xEAn c\u1EE7a m\xF4 h\xECnh, \u0111\u1EC3 \u0111\u1EA3m b\u1EA3o r\u1EB1ng ch\xFAng t\xF4i s\u1EED d\u1EE5ng c\xF9ng c\xE1c quy t\u1EAFc \u0111\xE3 \u0111\u01B0\u1EE3c s\u1EED d\u1EE5ng khi m\xF4 h\xECnh \u0111\u01B0\u1EE3c hu\u1EA5n luy\u1EC7n tr\u01B0\u1EDBc."),Sh=a(),L=i("p"),cs=r("B\u01B0\u1EDBc th\u1EE9 hai l\xE0 chuy\u1EC3n \u0111\u1ED5i c\xE1c token \u0111\xF3 th\xE0nh s\u1ED1 \u0111\u1EC3 ch\xFAng ta c\xF3 th\u1EC3 x\xE2y d\u1EF1ng m\u1ED9t tensor t\u1EEB ch\xFAng v\xE0 \u0111\u01B0a ch\xFAng v\xE0o m\xF4 h\xECnh. \u0110\u1EC3 l\xE0m \u0111i\u1EC1u n\xE0y, tokenizer c\xF3 "),$e=i("em"),is=r("t\u1EEB v\u1EF1ng"),ss=r(", l\xE0 ph\u1EA7n ch\xFAng ta t\u1EA3i xu\u1ED1ng khi kh\u1EDFi t\u1EA1o n\xF3 b\u1EB1ng ph\u01B0\u01A1ng th\u1EE9c "),ye=i("code"),os=r("from_pretrained()"),rs=r(". M\u1ED9t l\u1EA7n n\u1EEFa, ch\xFAng ta c\u1EA7n s\u1EED d\u1EE5ng c\xF9ng m\u1ED9t b\u1ED9 t\u1EEB v\u1EF1ng \u0111\u01B0\u1EE3c s\u1EED d\u1EE5ng khi m\xF4 h\xECnh \u0111\u01B0\u1EE3c hu\u1EA5n luy\u1EC7n tr\u01B0\u1EDBc."),Vh=a(),Cn=i("p"),ls=r("\u0110\u1EC3 hi\u1EC3u r\xF5 h\u01A1n v\u1EC1 hai b\u01B0\u1EDBc, ch\xFAng ta s\u1EBD kh\xE1m ph\xE1 ch\xFAng m\u1ED9t c\xE1ch ri\xEAng bi\u1EC7t. L\u01B0u \xFD r\u1EB1ng ch\xFAng t\xF4i s\u1EBD s\u1EED d\u1EE5ng m\u1ED9t s\u1ED1 ph\u01B0\u01A1ng ph\xE1p th\u1EF1c hi\u1EC7n c\xE1c ph\u1EA7n c\u1EE7a pipeline tokenize ri\xEAng bi\u1EC7t \u0111\u1EC3 hi\u1EC3n th\u1ECB cho b\u1EA1n k\u1EBFt qu\u1EA3 trung gian c\u1EE7a c\xE1c b\u01B0\u1EDBc \u0111\xF3, nh\u01B0ng tr\xEAn th\u1EF1c t\u1EBF, b\u1EA1n n\xEAn g\u1ECDi tokenize tr\u1EF1c ti\u1EBFp tr\xEAn \u0111\u1EA7u v\xE0o c\u1EE7a m\xECnh (nh\u01B0 \u0111\u01B0\u1EE3c hi\u1EC3n th\u1ECB trong ph\u1EA7n 2)."),Hh=a(),tt=i("h3"),kt=i("a"),_e=i("span"),k(Wt.$$.fragment),as=a(),Ee=i("span"),ps=r("Tokenize"),Lh=a(),dt=i("p"),us=r("Qu\xE1 tr\xECnh tokenize \u0111\u01B0\u1EE3c th\u1EF1c hi\u1EC7n b\u1EDFi ph\u01B0\u01A1ng th\u1EE9c "),ze=i("code"),gs=r("tokenize()"),ms=r(" c\u1EE7a tokenizer:"),Bh=a(),k(Xt.$$.fragment),Oh=a(),qn=i("p"),fs=r("K\u1EBFt qu\u1EA3 c\u1EE7a ph\u01B0\u01A1ng th\u1EE9c n\xE0y l\xE0 m\u1ED9t danh s\xE1ch c\xE1c chu\u1ED7i v\u0103n b\u1EA3n ho\u1EB7c tokens:"),Gh=a(),k(Zt.$$.fragment),Uh=a(),q=i("p"),vs=r("Tokenizer n\xE0y l\xE0 m\u1ED9t tokenizer d\u1EF1 theo t\u1EEB ph\u1EE5: n\xF3 chia c\xE1c t\u1EEB cho \u0111\u1EBFn khi l\u1EA5y \u0111\u01B0\u1EE3c c\xE1c tokens \u0111\u01B0\u1EE3c bi\u1EC3u di\u1EC5n b\u1EDFi b\u1ED9 t\u1EEB v\u1EF1ng c\u1EE7a n\xF3. V\xED d\u1EE5, "),we=i("code"),ks=r("transformer"),ds=r(" s\u1EBD \u0111\u01B0\u1EE3c chia th\xE0nh hai token: "),xe=i("code"),bs=r("transform"),$s=r(" v\xE0 "),Pe=i("code"),ys=r("##er"),_s=r("."),Kh=a(),nt=i("h3"),bt=i("a"),Te=i("span"),k(tn.$$.fragment),Es=a(),je=i("span"),zs=r("T\u1EEB token t\u1EDBi ID \u0111\u1EA7u v\xE0o"),Fh=a(),$t=i("p"),ws=r("Qu\xE1 t\xECnh chuy\u1EC3n \u0111\u1ED5i sang ID \u0111\u1EA7u v\xE0o \u0111\u01B0\u1EE3c th\u1EF1c hi\u1EC7n b\u1EDFi "),Ae=i("code"),xs=r("convert_tokens_to_ids()"),Ps=r(" c\u1EE7a tokenizer:"),Jh=a(),k(nn.$$.fragment),Rh=a(),k(en.$$.fragment),Qh=a(),Nn=i("p"),Ts=r("C\xE1c \u0111\u1EA7u ra n\xE0y, sau khi \u0111\u01B0\u1EE3c chuy\u1EC3n \u0111\u1ED5i sang khung tensor th\xEDch h\u1EE3p, c\xF3 th\u1EC3 \u0111\u01B0\u1EE3c s\u1EED d\u1EE5ng l\xE0m \u0111\u1EA7u v\xE0o cho m\u1ED9t m\xF4 h\xECnh nh\u01B0 \u0111\xE3 th\u1EA5y \u1EDF ph\u1EA7n tr\u01B0\u1EDBc trong ch\u01B0\u01A1ng n\xE0y."),Yh=a(),k(yt.$$.fragment),Wh=a(),et=i("h2"),_t=i("a"),De=i("span"),k(hn.$$.fragment),js=a(),Ce=i("span"),As=r("Gi\u1EA3i m\xE3"),Xh=a(),M=i("p"),qe=i("em"),Ds=r("Decoding"),Cs=r(" hay "),Ne=i("em"),qs=r("gi\u1EA3i m\xE3"),Ns=r(" th\xEC ng\u01B0\u1EE3c l\u1EA1i: t\u1EEB c\xE1c ch\u1EC9 s\u1ED1 t\u1EEB v\u1EF1ng, ta mu\u1ED1n tr\u1EA3 v\u1EC1 m\u1ED9t chu\u1ED7i v\u0103n b\u1EA3n. \u0110i\u1EC1u n\xE0y c\xF3 th\u1EC3 \u0111\u01B0\u1EE3c th\u1EF1c hi\u1EC7n v\u1EDBi ph\u01B0\u01A1ng th\u1EE9c "),Ie=i("code"),Is=r("decode()"),Ms=r(" nh\u01B0 sau:"),Zh=a(),k(cn.$$.fragment),tc=a(),k(sn.$$.fragment),nc=a(),Et=i("p"),Ss=r("L\u01B0u \xFD r\u1EB1ng ph\u01B0\u01A1ng ph\xE1p "),Me=i("code"),Vs=r("gi\u1EA3i m\xE3"),Hs=r(" kh\xF4ng ch\u1EC9 chuy\u1EC3n \u0111\u1ED5i c\xE1c ch\u1EC9 s\u1ED1 tr\u1EDF l\u1EA1i th\xE0nh token, m\xE0 c\xF2n nh\xF3m c\xE1c token l\xE0 m\u1ED9t ph\u1EA7n c\u1EE7a c\xF9ng m\u1ED9t t\u1EEB l\u1EA1i v\u1EDBi nhau \u0111\u1EC3 t\u1EA1o ra m\u1ED9t c\xE2u c\xF3 th\u1EC3 \u0111\u1ECDc \u0111\u01B0\u1EE3c. H\xE0nh vi n\xE0y s\u1EBD c\u1EF1c k\u1EF3 h\u1EEFu \xEDch khi ch\xFAng ta s\u1EED d\u1EE5ng c\xE1c m\xF4 h\xECnh d\u1EF1 \u0111o\xE1n v\u0103n b\u1EA3n m\u1EDBi (v\u0103n b\u1EA3n \u0111\u01B0\u1EE3c t\u1EA1o t\u1EEB l\u1EDDi nh\u1EAFc ho\u1EB7c \u0111\u1ED1i v\u1EDBi c\xE1c b\xE0i to\xE1n chu\u1ED7i-sang-chu\u1ED7i nh\u01B0 d\u1ECBch ho\u1EB7c t\xF3m t\u1EAFt v\u0103n b\u1EA3n)."),ec=a(),In=i("p"),Ls=r("B\xE2y gi\u1EDD b\u1EA1n \u0111\xE3 hi\u1EC3u c\xE1c ho\u1EA1t \u0111\u1ED9ng nguy\xEAn t\u1EED m\xE0 m\u1ED9t tokenizer c\xF3 th\u1EC3 x\u1EED l\xFD: tokenize, chuy\u1EC3n \u0111\u1ED5i sang ID v\xE0 chuy\u1EC3n \u0111\u1ED5i ID tr\u1EDF l\u1EA1i m\u1ED9t chu\u1ED7i. Tuy nhi\xEAn, t\u1EA5t c\u1EA3 ch\u1EC9 m\u1EDBi l\xE0 s\u1EF1 b\u1EAFt \u0111\u1EA7u. Trong ph\u1EA7n sau, ch\xFAng ta s\u1EBD ti\u1EBFp c\u1EADn c\xE1c gi\u1EDBi h\u1EA1n c\u1EE7a n\xF3 v\xE0 xem c\xE1ch v\u01B0\u1EE3t qua ch\xFAng."),this.h()},l(t){const h=Cr('[data-svelte="svelte-1phssyn"]',document.head);g=s(h,"META",{name:!0,content:!0}),h.forEach(n),y=p(t),d(m.$$.fragment,t),z=p(t),w=s(t,"H1",{class:!0});var rn=o(w);_=s(rn,"A",{id:!0,class:!0,href:!0});var Mn=o(_);x=s(Mn,"SPAN",{});var Se=o(x);d(P.$$.fragment,Se),Se.forEach(n),Mn.forEach(n),T=p(rn),E=s(rn,"SPAN",{});var Ws=o(E);U=l(Ws,"Tokenizers"),Ws.forEach(n),rn.forEach(n),S=p(t),I.l(t),an=p(t),d(wt.$$.fragment,t),He=p(t),pn=s(t,"P",{});var Xs=o(pn);Pc=l(Xs,"Tokenizer l\xE0 m\u1ED9t trong nh\u1EEFng th\xE0nh ph\u1EA7n c\u1ED1t l\xF5i c\u1EE7a pipeline NLP. Ch\xFAng ph\u1EE5c v\u1EE5 m\u1ED9t m\u1EE5c \u0111\xEDch: d\u1ECBch v\u0103n b\u1EA3n th\xE0nh d\u1EEF li\u1EC7u c\xF3 th\u1EC3 \u0111\u01B0\u1EE3c x\u1EED l\xFD b\u1EDFi m\xF4 h\xECnh. M\xF4 h\xECnh ch\u1EC9 c\xF3 th\u1EC3 x\u1EED l\xFD d\u1EA1ng s\u1ED1, do \u0111\xF3, c\xE1c tokenizer c\u1EA7n ph\u1EA3i chuy\u1EC3n \u0111\u1ED5i \u0111\u1EA7u v\xE0o v\u0103n b\u1EA3n c\u1EE7a ch\xFAng ta th\xE0nh d\u1EEF li\u1EC7u s\u1ED1. Trong ph\u1EA7n n\xE0y, ch\xFAng ta s\u1EBD kh\xE1m ph\xE1 ch\xEDnh x\xE1c nh\u1EEFng g\xEC x\u1EA3y ra trong \u0111\u01B0\u1EDDng d\u1EABn m\xE3 h\xF3a."),Xs.forEach(n),Le=p(t),un=s(t,"P",{});var Zs=o(un);Tc=l(Zs,"Trong c\xE1c t\xE1c v\u1EE5 NLP, d\u1EEF li\u1EC7u th\u01B0\u1EDDng \u0111\u01B0\u1EE3c x\u1EED l\xFD l\xE0 v\u0103n b\u1EA3n th\xF4. \u0110\xE2y l\xE0 m\u1ED9t v\xED d\u1EE5 v\u1EC1 v\u0103n b\u1EA3n nh\u01B0 v\u1EADy:"),Zs.forEach(n),Be=p(t),d(xt.$$.fragment,t),Oe=p(t),gn=s(t,"P",{});var to=o(gn);jc=l(to,"Tuy nhi\xEAn, c\xE1c m\xF4 h\xECnh ch\u1EC9 c\xF3 th\u1EC3 x\u1EED l\xFD s\u1ED1, v\xEC v\u1EADy ch\xFAng ta c\u1EA7n t\xECm c\xE1ch chuy\u1EC3n v\u0103n b\u1EA3n th\xF4 th\xE0nh s\u1ED1. \u0110\xF3 l\xE0 nh\u1EEFng g\xEC m\xE0 tokenizer l\xE0m, v\xE0 c\xF3 r\u1EA5t nhi\u1EC1u c\xE1ch \u0111\u1EC3 th\u1EF1c hi\u1EC7n \u0111i\u1EC1u n\xE0y. M\u1EE5c ti\xEAu \u0111\u1EC1 ra l\xE0 t\xECm ra c\xE1ch bi\u1EC3u di\u1EC5n c\xF3 \xFD ngh\u0129a nh\u1EA5t - ngh\u0129a l\xE0 c\xE1i c\xF3 \xFD ngh\u0129a nh\u1EA5t \u0111\u1ED1i v\u1EDBi m\xF4 h\xECnh - v\xE0, n\u1EBFu c\xF3 th\u1EC3, l\xE0 c\xE1ch bi\u1EC3u di\u1EC5n nh\u1ECF nh\u1EA5t."),to.forEach(n),Ge=p(t),mn=s(t,"P",{});var no=o(mn);Ac=l(no,"H\xE3y c\xF9ng xem m\u1ED9t s\u1ED1 v\xED d\u1EE5 v\u1EC1 thu\u1EADt to\xE1n tokenize v\xE0 c\u1ED1 g\u1EAFng tr\u1EA3 l\u1EDDi m\u1ED9t s\u1ED1 c\xE2u h\u1ECFi b\u1EA1n c\xF3 th\u1EC3 c\xF3 v\u1EC1 tokenize."),no.forEach(n),Ue=p(t),K=s(t,"H2",{class:!0});var ic=o(K);ct=s(ic,"A",{id:!0,class:!0,href:!0});var eo=o(ct);Bn=s(eo,"SPAN",{});var ho=o(Bn);d(Pt.$$.fragment,ho),ho.forEach(n),eo.forEach(n),Dc=p(ic),On=s(ic,"SPAN",{});var co=o(On);Cc=l(co,"D\u1EF1a tr\xEAn t\u1EEB"),co.forEach(n),ic.forEach(n),Ke=p(t),d(Tt.$$.fragment,t),Fe=p(t),it=s(t,"P",{});var sc=o(it);qc=l(sc,"Lo\u1EA1i tokenizer \u0111\u1EA7u ti\xEAn ta ngh\u0129 \u0111\u1EBFn \u0111\xF3 l\xE0 "),Gn=s(sc,"EM",{});var io=o(Gn);Nc=l(io,"d\u1EF1a tr\xEAn t\u1EEB v\u1EF1ng"),io.forEach(n),Ic=l(sc,". N\xF3 th\u01B0\u1EDDng r\u1EA5t d\u1EC5 thi\u1EBFt l\u1EADp v\xE0 s\u1EED d\u1EE5ng ch\u1EC9 v\u1EDBi m\u1ED9t s\u1ED1 quy t\u1EAFc v\xE0 n\xF3 th\u01B0\u1EDDng mang l\u1EA1i k\u1EBFt qu\u1EA3 t\u1ED1t. V\xED d\u1EE5: trong h\xECnh \u1EA3nh b\xEAn d\u01B0\u1EDBi, m\u1EE5c ti\xEAu l\xE0 t\xE1ch v\u0103n b\u1EA3n th\xF4 th\xE0nh c\xE1c t\u1EEB v\xE0 t\xECm bi\u1EC3u di\u1EC5n s\u1ED1 cho m\u1ED7i t\u1EEB:"),sc.forEach(n),Je=p(t),F=s(t,"DIV",{class:!0});var oc=o(F);jt=s(oc,"IMG",{class:!0,src:!0,alt:!0}),Mc=p(oc),At=s(oc,"IMG",{class:!0,src:!0,alt:!0}),oc.forEach(n),Re=p(t),st=s(t,"P",{});var rc=o(st);Sc=l(rc,"C\xF3 nhi\u1EC1u c\xE1ch kh\xE1c nhau \u0111\u1EC3 t\xE1ch v\u0103n b\u1EA3n. V\xED d\u1EE5: ch\xFAng ta c\xF3 th\u1EC3 s\u1EED d\u1EE5ng kho\u1EA3ng tr\u1EAFng \u0111\u1EC3 tokenize v\u0103n b\u1EA3n th\xE0nh c\xE1c t\u1EEB b\u1EB1ng c\xE1ch \xE1p d\u1EE5ng h\xE0m "),Un=s(rc,"CODE",{});var so=o(Un);Vc=l(so,"split()"),so.forEach(n),Hc=l(rc," c\u1EE7a Python:"),rc.forEach(n),Qe=p(t),d(Dt.$$.fragment,t),Ye=p(t),d(Ct.$$.fragment,t),We=p(t),fn=s(t,"P",{});var oo=o(fn);Lc=l(oo,"Ngo\xE0i ra c\xF2n c\xF3 c\xE1c bi\u1EBFn th\u1EC3 c\u1EE7a tokenize m\u1EE9c t\u1EEB v\u1EDBi c\xE1c quy t\u1EAFc b\u1ED5 sung cho d\u1EA5u c\xE2u. V\u1EDBi lo\u1EA1i tokenizer n\xE0y, ch\xFAng ta c\xF3 th\u1EC3 \u0111\xFAc k\u1EBFt v\u1EDBi m\u1ED9t b\u1ED9 \u201Ct\u1EEB v\u1EF1ng\u201D kh\xE1 l\u1EDBn, trong \u0111\xF3 t\u1EEB v\u1EF1ng \u0111\u01B0\u1EE3c x\xE1c \u0111\u1ECBnh b\u1EB1ng t\u1ED5ng s\u1ED1 token \u0111\u1ED9c l\u1EADp m\xE0 ch\xFAng ta c\xF3 trong corpus (kho ng\u1EEF li\u1EC7u) c\u1EE7a m\xECnh."),oo.forEach(n),Xe=p(t),vn=s(t,"P",{});var ro=o(vn);Bc=l(ro,"M\u1ED7i t\u1EEB \u0111\u01B0\u1EE3c g\xE1n m\u1ED9t ID, b\u1EAFt \u0111\u1EA7u t\u1EEB 0 v\xE0 t\u0103ng d\u1EA7n theo k\xEDch th\u01B0\u1EDBc c\u1EE7a b\u1ED9 t\u1EEB v\u1EF1ng. M\xF4 h\xECnh s\u1EED d\u1EE5ng c\xE1c ID n\xE0y \u0111\u1EC3 x\xE1c \u0111\u1ECBnh t\u1EEBng t\u1EEB."),ro.forEach(n),Ze=p(t),kn=s(t,"P",{});var lo=o(kn);Oc=l(lo,"N\u1EBFu ch\xFAng ta mu\u1ED1n bao ph\u1EE7 ho\xE0n to\xE0n m\u1ED9t ng\xF4n ng\u1EEF b\u1EB1ng tokenize m\u1EE9c t\u1EEB, ch\xFAng ta s\u1EBD c\u1EA7n ph\u1EA3i c\xF3 m\u1ED9t ch\u1EC9 s\u1ED1 nh\u1EADn d\u1EA1ng cho m\u1ED7i t\u1EEB trong ng\xF4n ng\u1EEF, \u0111i\u1EC1u n\xE0y s\u1EBD t\u1EA1o ra m\u1ED9t l\u01B0\u1EE3ng l\u1EDBn token. V\xED d\u1EE5: c\xF3 h\u01A1n 500,000 t\u1EEB trong ti\u1EBFng Anh, v\xEC v\u1EADy \u0111\u1EC3 x\xE2y d\u1EF1ng b\u1EA3n \u0111\u1ED3 n\u1ED1i m\u1ED7i t\u1EEB \u0111\u1EBFn m\u1ED9t ID \u0111\u1EA7u v\xE0o, ch\xFAng ta c\u1EA7n theo d\xF5i ng\u1EA7n \u0111\xF3 ID. H\u01A1n n\u1EEFa, c\xE1c t\u1EEB nh\u01B0 \u201Cdog\u201D \u0111\u01B0\u1EE3c bi\u1EC3u di\u1EC5n kh\xE1c v\u1EDBi c\xE1c t\u1EEB nh\u01B0 \u201Cdogs\u201D, v\xE0 ban \u0111\u1EA7u m\xF4 h\xECnh s\u1EBD kh\xF4ng c\xF3 c\xE1ch n\xE0o \u0111\u1EC3 bi\u1EBFt r\u1EB1ng \u201Cdog\u201D (ch\xF3) v\xE0 \u201Cdogs\u201D l\xE0 t\u01B0\u01A1ng t\u1EF1 nhau: n\xF3 s\u1EBD x\xE1c \u0111\u1ECBnh hai t\u1EEB n\xE0y kh\xF4ng li\xEAn quan. \u0110i\u1EC1u n\xE0y c\u0169ng \xE1p d\u1EE5ng cho c\xE1c t\u1EEB t\u01B0\u01A1ng t\u1EF1 kh\xE1c, nh\u01B0 \u201Crun\u201D (ch\u1EA1y) v\xE0 \u201Crunning\u201D, m\xE0 ban \u0111\u1EA7u m\xF4 h\xECnh s\u1EBD kh\xF4ng th\u1EA5y l\xE0 t\u01B0\u01A1ng t\u1EF1."),lo.forEach(n),th=p(t),qt=s(t,"P",{});var Bs=o(qt);Gc=l(Bs,"Cu\u1ED1i c\xF9ng, ch\xFAng ta c\u1EA7n m\u1ED9t token t\xF9y ch\u1EC9nh \u0111\u1EC3 \u0111\u1EA1i di\u1EC7n cho c\xE1c t\u1EEB kh\xF4ng c\xF3 trong v\u1ED1n t\u1EEB v\u1EF1ng c\u1EE7a ch\xFAng ta. M\xE3 n\xE0y \u0111\u01B0\u1EE3c g\u1ECDi l\xE0 token \u201Ckh\xF4ng x\xE1c \u0111\u1ECBnh\u201D, th\u01B0\u1EDDng \u0111\u01B0\u1EE3c bi\u1EC3u th\u1ECB l\xE0 \u201D[UNK]\u201D ho\u1EB7c \u201D"),Kn=s(Bs,"UNK",{});var ao=o(Kn);Uc=l(ao,"\u201D. N\xF3i chung, \u0111\xF3 l\xE0 m\u1ED9t d\u1EA5u hi\u1EC7u x\u1EA5u n\u1EBFu b\u1EA1n th\u1EA5y tr\xECnh tokenize \u0111ang t\u1EA1o ra r\u1EA5t nhi\u1EC1u token n\xE0y, v\xEC n\xF3 kh\xF4ng th\u1EC3 truy xu\u1EA5t m\u1ED9t bi\u1EC3u hi\u1EC7n h\u1EE3p l\xFD c\u1EE7a m\u1ED9t t\u1EEB v\xE0 b\u1EA1n \u0111ang m\u1EA5t th\xF4ng tin trong su\u1ED1t qu\xE1 tr\xECnh. M\u1EE5c ti\xEAu khi t\u1EA1o t\u1EEB v\u1EF1ng l\xE0 l\xE0m sao cho tr\xECnh tokenize m\xE3 h\xF3a c\xE0ng \xEDt t\u1EEB th\xE0nh token kh\xF4ng x\xE1c \u0111\u1ECBnh c\xE0ng t\u1ED1t."),ao.forEach(n),Bs.forEach(n),nh=p(t),ot=s(t,"P",{});var lc=o(ot);Kc=l(lc,"M\u1ED9t c\xE1ch \u0111\u1EC3 gi\u1EA3m s\u1ED1 l\u01B0\u1EE3ng m\xE3 th\xF4ng b\xE1o kh\xF4ng x\xE1c \u0111\u1ECBnh l\xE0 \u0111i s\xE2u h\u01A1n xu\u1ED1ng m\u1ED9t c\u1EA5p, s\u1EED d\u1EE5ng tokenize "),Fn=s(lc,"EM",{});var po=o(Fn);Fc=l(po,"m\u1EE9c k\xED t\u1EF1"),po.forEach(n),Jc=l(lc,"."),lc.forEach(n),eh=p(t),J=s(t,"H2",{class:!0});var ac=o(J);rt=s(ac,"A",{id:!0,class:!0,href:!0});var uo=o(rt);Jn=s(uo,"SPAN",{});var go=o(Jn);d(Nt.$$.fragment,go),go.forEach(n),uo.forEach(n),Rc=p(ac),Rn=s(ac,"SPAN",{});var mo=o(Rn);Qc=l(mo,"D\u1EF1a tr\xEAn k\xED t\u1EF1"),mo.forEach(n),ac.forEach(n),hh=p(t),d(It.$$.fragment,t),ch=p(t),lt=s(t,"UL",{});var pc=o(lt);Qn=s(pc,"LI",{});var fo=o(Qn);Yc=l(fo,"V\u1ED1n t\u1EEB v\u1EF1ng \xEDt h\u01A1n nhi\u1EC1u."),fo.forEach(n),Wc=p(pc),Yn=s(pc,"LI",{});var vo=o(Yn);Xc=l(vo,"C\xF3 \xEDt token ngo\xE0i b\u1ED9 t\u1EEB v\u1EF1ng (kh\xF4ng x\xE1c \u0111\u1ECBnh) h\u01A1n nhi\u1EC1u, v\xEC m\u1ECDi t\u1EEB \u0111\u1EC1u c\xF3 th\u1EC3 \u0111\u01B0\u1EE3c x\xE2y d\u1EF1ng t\u1EEB c\xE1c k\xFD t\u1EF1."),vo.forEach(n),pc.forEach(n),ih=p(t),dn=s(t,"P",{});var ko=o(dn);Zc=l(ko,"Nh\u01B0ng \u1EDF \u0111\xE2y c\u0169ng c\xF3 m\u1ED9t s\u1ED1 c\xE2u h\u1ECFi n\u1EA3y sinh li\xEAn quan \u0111\u1EBFn d\u1EA5u c\xE1ch v\xE0 c\xE1c d\u1EA5u c\xE2u:"),ko.forEach(n),sh=p(t),R=s(t,"DIV",{class:!0});var uc=o(R);Mt=s(uc,"IMG",{class:!0,src:!0,alt:!0}),ti=p(uc),St=s(uc,"IMG",{class:!0,src:!0,alt:!0}),uc.forEach(n),oh=p(t),bn=s(t,"P",{});var bo=o(bn);ni=l(bo,"C\xE1ch ti\u1EBFp c\u1EADn n\xE0y c\u0169ng kh\xF4ng ho\xE0n h\u1EA3o. V\xEC bi\u1EC3u di\u1EC5n b\xE2y gi\u1EDD d\u1EF1a tr\xEAn c\xE1c k\xFD t\u1EF1 ch\u1EE9 kh\xF4ng ph\u1EA3i t\u1EEB, ng\u01B0\u1EDDi ta c\xF3 th\u1EC3 l\u1EADp lu\u1EADn r\u1EB1ng, theo tr\u1EF1c gi\xE1c, n\xF3 \xEDt \xFD ngh\u0129a h\u01A1n: m\u1ED7i k\xFD t\u1EF1 kh\xF4ng c\xF3 nhi\u1EC1u \xFD ngh\u0129a ri\xEAng so v\u1EDBi tr\u01B0\u1EDDng h\u1EE3p c\u1EE7a c\xE1c t\u1EEB. Tuy nhi\xEAn, \u0111i\u1EC1u n\xE0y l\u1EA1i kh\xE1c nhau t\xF9y theo ng\xF4n ng\u1EEF; trong ti\u1EBFng Trung, ch\u1EB3ng h\u1EA1n, m\u1ED7i k\xFD t\u1EF1 mang nhi\u1EC1u th\xF4ng tin h\u01A1n m\u1ED9t k\xFD t\u1EF1 trong ng\xF4n ng\u1EEF Latinh."),bo.forEach(n),rh=p(t),$n=s(t,"P",{});var $o=o($n);ei=l($o,"M\u1ED9t \u0111i\u1EC1u kh\xE1c c\u1EA7n xem x\xE9t l\xE0 ch\xFAng ta s\u1EBD c\xF3 m\u1ED9t l\u01B0\u1EE3ng r\u1EA5t l\u1EDBn token s\u1EBD \u0111\u01B0\u1EE3c x\u1EED l\xFD b\u1EDFi m\xF4 h\xECnh c\u1EE7a ch\xFAng ta: trong khi m\u1ED9t t\u1EEB ch\u1EC9 l\xE0 m\u1ED9t token duy nh\u1EA5t khi tokenize d\u1EF1a tr\xEAn t\u1EEB, n\xF3 c\xF3 th\u1EC3 d\u1EC5 d\xE0ng chuy\u1EC3n th\xE0nh 10 token tr\u1EDF l\xEAn khi chuy\u1EC3n \u0111\u1ED5i th\xE0nh c\xE1c k\xFD t\u1EF1."),$o.forEach(n),lh=p(t),at=s(t,"P",{});var gc=o(at);hi=l(gc,"\u0110\u1EC3 t\u1EADn d\u1EE5ng t\u1ED1i \u0111a c\u1EA3 hai, ch\xFAng ta c\xF3 th\u1EC3 s\u1EED d\u1EE5ng k\u1EF9 thu\u1EADt th\u1EE9 ba k\u1EBFt h\u1EE3p hai c\xE1ch ti\u1EBFp c\u1EADn: "),Wn=s(gc,"EM",{});var yo=o(Wn);ci=l(yo,"tokenize theo t\u1EEB ph\u1EE5"),yo.forEach(n),ii=l(gc,"."),gc.forEach(n),ah=p(t),Q=s(t,"H2",{class:!0});var mc=o(Q);pt=s(mc,"A",{id:!0,class:!0,href:!0});var _o=o(pt);Xn=s(_o,"SPAN",{});var Eo=o(Xn);d(Vt.$$.fragment,Eo),Eo.forEach(n),_o.forEach(n),si=p(mc),Zn=s(mc,"SPAN",{});var zo=o(Zn);oi=l(zo,"Tokenize theo t\u1EEB ph\u1EE5"),zo.forEach(n),mc.forEach(n),ph=p(t),d(Ht.$$.fragment,t),uh=p(t),yn=s(t,"P",{});var wo=o(yn);ri=l(wo,"C\xE1c thu\u1EADt to\xE1n token theo t\u1EEB ph\u1EE5 d\u1EF1a tr\xEAn nguy\xEAn t\u1EAFc r\u1EB1ng c\xE1c t\u1EEB \u0111\u01B0\u1EE3c s\u1EED d\u1EE5ng th\u01B0\u1EDDng xuy\xEAn kh\xF4ng \u0111\u01B0\u1EE3c chia th\xE0nh c\xE1c t\u1EEB ph\u1EE5 nh\u1ECF h\u01A1n, nh\u01B0ng c\xE1c t\u1EEB hi\u1EBFm ph\u1EA3i \u0111\u01B0\u1EE3c ph\xE2n t\xE1ch th\xE0nh c\xE1c t\u1EEB ph\u1EE5 c\xF3 \xFD ngh\u0129a."),wo.forEach(n),gh=p(t),_n=s(t,"P",{});var xo=o(_n);li=l(xo,"V\xED d\u1EE5: \u201Cannoyingly\u201D (kh\xF3 ch\u1ECBu) c\xF3 th\u1EC3 \u0111\u01B0\u1EE3c coi l\xE0 m\u1ED9t t\u1EEB hi\u1EBFm v\xE0 c\xF3 th\u1EC3 \u0111\u01B0\u1EE3c chuy\u1EC3n th\xE0nh \u201Cannoying\u201D v\xE0 \u201Cly\u201D. C\u1EA3 hai \u0111\u1EC1u c\xF3 kh\u1EA3 n\u0103ng xu\u1EA5t hi\u1EC7n th\u01B0\u1EDDng xuy\xEAn h\u01A1n d\u01B0\u1EDBi d\u1EA1ng c\xE1c t\u1EEB ph\u1EE5 \u0111\u1ED9c l\u1EADp, \u0111\u1ED3ng th\u1EDDi ngh\u0129a c\u1EE7a \u201Cannoying\u201D \u0111\u01B0\u1EE3c gi\u1EEF nguy\xEAn b\u1EDFi ngh\u0129a k\u1EBFt h\u1EE3p c\u1EE7a \u201Cannoying\u201D v\xE0 \u201Cly\u201D."),xo.forEach(n),mh=p(t),En=s(t,"P",{});var Po=o(En);ai=l(Po,"D\u01B0\u1EDBi \u0111\xE2y l\xE0 m\u1ED9t v\xED d\u1EE5 cho th\u1EA5y c\xE1ch m\u1ED9t thu\u1EADt to\xE1n tokenize theo t\u1EEB ph\u1EE5 s\u1EBD tokenize chu\u1ED7i \u201CLet\u2019s do tokenization!\u201D (H\xE3y th\u1EF1c hi\u1EC7n tokenize!):"),Po.forEach(n),fh=p(t),Y=s(t,"DIV",{class:!0});var fc=o(Y);Lt=s(fc,"IMG",{class:!0,src:!0,alt:!0}),pi=p(fc),Bt=s(fc,"IMG",{class:!0,src:!0,alt:!0}),fc.forEach(n),vh=p(t),zn=s(t,"P",{});var To=o(zn);ui=l(To,"Nh\u1EEFng t\u1EEB ph\u1EE5 n\xE0y cung c\u1EA5p r\u1EA5t nhi\u1EC1u \xFD ngh\u0129a v\u1EC1 m\u1EB7t ng\u1EEF ngh\u0129a: v\xED d\u1EE5: trong v\xED d\u1EE5 \u1EDF tr\xEAn \u201Ctokenization\u201D \u0111\u01B0\u1EE3c chia th\xE0nh \u201Ctoken\u201D v\xE0 \u201Cization\u201D, hai token \u0111\u1EC1u c\xF3 \xFD ngh\u0129a v\u1EC1 m\u1EB7t ng\u1EEF ngh\u0129a \u0111\u1ED3ng th\u1EDDi ti\u1EBFt ki\u1EC7m kh\xF4ng gian (ch\u1EC9 c\u1EA7n hai token \u0111\u1EC3 bi\u1EC3u th\u1ECB m\u1ED9t t\u1EEB d\xE0i). \u0110i\u1EC1u n\xE0y cho ph\xE9p ch\xFAng ta c\xF3 th\u1EC3 bao qu\xE1t t\u01B0\u01A1ng \u0111\u1ED1i t\u1ED1t v\u1EDBi c\xE1c t\u1EEB v\u1EF1ng nh\u1ECF v\xE0 g\u1EA7n nh\u01B0 kh\xF4ng c\xF3 token n\xE0o kh\xF4ng x\xE1c \u0111\u1ECBnh."),To.forEach(n),kh=p(t),wn=s(t,"P",{});var jo=o(wn);gi=l(jo,"C\xE1ch ti\u1EBFp c\u1EADn n\xE0y \u0111\u1EB7c bi\u1EC7t h\u1EEFu \xEDch trong c\xE1c ng\xF4n ng\u1EEF t\u1ED5ng h\u1EE3p nh\u01B0 ti\u1EBFng Th\u1ED5 Nh\u0129 K\u1EF3, n\u01A1i b\u1EA1n c\xF3 th\u1EC3 t\u1EA1o (g\u1EA7n nh\u01B0) c\xE1c t\u1EEB ph\u1EE9c d\xE0i t\xF9y \xFD b\u1EB1ng c\xE1ch x\xE2u chu\u1ED7i c\xE1c t\u1EEB ph\u1EE5 l\u1EA1i v\u1EDBi nhau."),jo.forEach(n),dh=p(t),W=s(t,"H3",{class:!0});var vc=o(W);ut=s(vc,"A",{id:!0,class:!0,href:!0});var Ao=o(ut);te=s(Ao,"SPAN",{});var Do=o(te);d(Ot.$$.fragment,Do),Do.forEach(n),Ao.forEach(n),mi=p(vc),ne=s(vc,"SPAN",{});var Co=o(ne);fi=l(Co,"V\xE0 h\u01A1n th\u1EBF n\u1EEFa!"),Co.forEach(n),vc.forEach(n),bh=p(t),xn=s(t,"P",{});var qo=o(xn);vi=l(qo,"Kh\xF4ng c\xF3 g\xEC \u0111\xE1ng ng\u1EA1c nhi\xEAn, c\xF3 r\u1EA5t nhi\u1EC1u k\u1EF9 thu\u1EADt kh\xE1c, c\xF3 th\u1EC3 k\u1EC3 \u0111\u1EBFn:"),qo.forEach(n),$h=p(t),V=s(t,"UL",{});var Sn=o(V);ee=s(Sn,"LI",{});var No=o(ee);ki=l(No,"Byte-level BPE (BPE c\u1EA5p byte), nh\u01B0 \u0111\u01B0\u1EE3c s\u1EED d\u1EE5ng trong GPT-2"),No.forEach(n),di=p(Sn),he=s(Sn,"LI",{});var Io=o(he);bi=l(Io,"WordPiece, nh\u01B0 \u0111\u01B0\u1EE3c s\u1EED d\u1EE5ng trong BERT"),Io.forEach(n),$i=p(Sn),ce=s(Sn,"LI",{});var Mo=o(ce);yi=l(Mo,"SentencePiece ho\u1EB7c Unigram, nh\u01B0 \u0111\u01B0\u1EE3c s\u1EED d\u1EE5ng trong m\u1ED9t s\u1ED1 m\xF4 h\xECnh \u0111a ng\xF4n ng\u1EEF"),Mo.forEach(n),Sn.forEach(n),yh=p(t),Pn=s(t,"P",{});var So=o(Pn);_i=l(So,"B\xE2y gi\u1EDD, b\u1EA1n \u0111\xE3 c\xF3 \u0111\u1EE7 ki\u1EBFn th\u1EE9c v\u1EC1 c\xE1ch th\u1EE9c ho\u1EA1t \u0111\u1ED9ng c\u1EE7a tokenize \u0111\u1EC3 b\u1EAFt \u0111\u1EA7u v\u1EDBi API."),So.forEach(n),_h=p(t),X=s(t,"H2",{class:!0});var kc=o(X);gt=s(kc,"A",{id:!0,class:!0,href:!0});var Vo=o(gt);ie=s(Vo,"SPAN",{});var Ho=o(ie);d(Gt.$$.fragment,Ho),Ho.forEach(n),Vo.forEach(n),Ei=p(kc),se=s(kc,"SPAN",{});var Lo=o(se);zi=l(Lo,"T\u1EA3i v\xE0 l\u01B0u"),Lo.forEach(n),kc.forEach(n),Eh=p(t),j=s(t,"P",{});var B=o(j);wi=l(B,"Vi\u1EC7c t\u1EA3i v\xE0 l\u01B0u tokenizer c\u0169ng \u0111\u01A1n gi\u1EA3n nh\u01B0 v\u1EDBi c\xE1c m\xF4 h\xECnh. Tr\xEAn th\u1EF1c t\u1EBF, n\xF3 d\u1EF1a tr\xEAn hai ph\u01B0\u01A1ng th\u1EE9c gi\u1ED1ng nhau: "),oe=s(B,"CODE",{});var Bo=o(oe);xi=l(Bo,"from_pretrained()"),Bo.forEach(n),Pi=l(B," v\xE0 "),re=s(B,"CODE",{});var Oo=o(re);Ti=l(Oo,"save_pretrained()"),Oo.forEach(n),ji=l(B,". C\xE1c ph\u01B0\u01A1ng th\u1EE9c n\xE0y s\u1EBD t\u1EA3i ho\u1EB7c l\u01B0u thu\u1EADt to\xE1n \u0111\u01B0\u1EE3c s\u1EED d\u1EE5ng b\u1EDFi tokenizer (h\u01A1i gi\u1ED1ng v\u1EDBi "),le=s(B,"EM",{});var Go=o(le);Ai=l(Go,"ki\u1EBFn tr\xFAc"),Go.forEach(n),Di=l(B," c\u1EE7a m\xF4 h\xECnh) c\u0169ng nh\u01B0 t\u1EEB v\u1EF1ng c\u1EE7a n\xF3 (h\u01A1i gi\u1ED1ng v\u1EDBi "),ae=s(B,"EM",{});var Uo=o(ae);Ci=l(Uo,"tr\u1ECDng s\u1ED1"),Uo.forEach(n),qi=l(B," c\u1EE7a m\xF4 h\xECnh)."),B.forEach(n),zh=p(t),mt=s(t,"P",{});var dc=o(mt);Ni=l(dc,"Vi\u1EC7c t\u1EA3i BERT tokenizer \u0111\u01B0\u1EE3c hu\u1EA5n luy\u1EC7n v\u1EDBi c\xF9ng m\u1ED9t checkpoint v\u1EDBi BERT \u0111\u01B0\u1EE3c th\u1EF1c hi\u1EC7n gi\u1ED1ng nh\u01B0 c\xE1ch t\u1EA3i m\xF4 h\xECnh, ngo\u1EA1i tr\u1EEB vi\u1EC7c ch\xFAng ta s\u1EED d\u1EE5ng l\u1EDBp "),pe=s(dc,"CODE",{});var Ko=o(pe);Ii=l(Ko,"BertTokenizer"),Ko.forEach(n),Mi=l(dc,":"),dc.forEach(n),wh=p(t),d(Ut.$$.fragment,t),xh=p(t),ht.l(t),Tn=p(t),d(Kt.$$.fragment,t),Ph=p(t),jn=s(t,"P",{});var Fo=o(jn);Si=l(Fo,"Gi\u1EDD ch\xFAng ta c\xF3 th\u1EC3 s\u1EED d\u1EE5ng tokenizer nh\u01B0 trong \u0111o\u1EA1n d\u01B0\u1EDBi \u0111\xE2y:"),Fo.forEach(n),Th=p(t),d(Ft.$$.fragment,t),jh=p(t),d(Jt.$$.fragment,t),Ah=p(t),An=s(t,"P",{});var Jo=o(An);Vi=l(Jo,"L\u01B0u m\u1ED9t tokenizer gi\u1ED1ng nh\u01B0 khi l\u01B0u m\u1ED9t m\xF4 h\xECnh v\u1EADy:"),Jo.forEach(n),Dh=p(t),d(Rt.$$.fragment,t),Ch=p(t),A=s(t,"P",{});var O=o(A);Hi=l(O,"Ch\xFAng ta s\u1EBD trao \u0111\u1ED5i th\xEAm v\u1EC1 "),ue=s(O,"CODE",{});var Ro=o(ue);Li=l(Ro,"token_type_ids"),Ro.forEach(n),Bi=l(O," trong "),Dn=s(O,"A",{href:!0});var Qo=o(Dn);Oi=l(Qo,"Ch\u01B0\u01A1ng 3"),Qo.forEach(n),Gi=l(O,", v\xE0 ch\xFAng ta s\u1EBD gi\u1EA3i th\xEDch c\u01A1 ch\u1EBF c\u1EE7a t\u1EEB kho\xE1 "),ge=s(O,"CODE",{});var Yo=o(ge);Ui=l(Yo,"attention_mask"),Yo.forEach(n),Ki=l(O," sau \u0111\xF3. \u0110\u1EA7u ti\xEAnm h\xE3y c\xF9ng xem c\xE1ch "),me=s(O,"CODE",{});var Wo=o(me);Fi=l(Wo,"input_ids"),Wo.forEach(n),Ji=l(O," \u0111\u01B0\u1EE3c t\u1EA1o ra. \u0110\u1EC3 l\xE0m \u0111i\u1EC1u n\xE0y, ch\xFAng ta s\u1EBD c\u1EA7n xem x\xE9t c\xE1c ph\u01B0\u01A1ng th\u1EE9c trung gian c\u1EE7a tokenizer."),O.forEach(n),qh=p(t),Z=s(t,"H2",{class:!0});var bc=o(Z);ft=s(bc,"A",{id:!0,class:!0,href:!0});var Xo=o(ft);fe=s(Xo,"SPAN",{});var Zo=o(fe);d(Qt.$$.fragment,Zo),Zo.forEach(n),Xo.forEach(n),Ri=p(bc),ve=s(bc,"SPAN",{});var tr=o(ve);Qi=l(tr,"M\xE3 ho\xE1"),tr.forEach(n),bc.forEach(n),Nh=p(t),d(Yt.$$.fragment,t),Ih=p(t),H=s(t,"P",{});var Vn=o(H);Yi=l(Vn,"D\u1ECBch v\u0103n b\u1EA3n sang s\u1ED1 \u0111\u01B0\u1EE3c g\u1ECDi l\xE0 "),ke=s(Vn,"EM",{});var nr=o(ke);Wi=l(nr,"encoding"),nr.forEach(n),Xi=l(Vn," hay "),de=s(Vn,"EM",{});var er=o(de);Zi=l(er,"m\xE3 ho\xE1"),er.forEach(n),ts=l(Vn,". Vi\u1EC7c m\xE3 h\xF3a \u0111\u01B0\u1EE3c th\u1EF1c hi\u1EC7n theo quy tr\xECnh g\u1ED3m hai b\u01B0\u1EDBc: tokenize, ti\u1EBFp theo l\xE0 chuy\u1EC3n \u0111\u1ED5i sang ID \u0111\u1EA7u v\xE0o."),Vn.forEach(n),Mh=p(t),vt=s(t,"P",{});var $c=o(vt);ns=l($c,"Nh\u01B0 ch\xFAng ta \u0111\xE3 th\u1EA5y, b\u01B0\u1EDBc \u0111\u1EA7u ti\xEAn l\xE0 chia v\u0103n b\u1EA3n th\xE0nh c\xE1c t\u1EEB (ho\u1EB7c c\xE1c ph\u1EA7n c\u1EE7a t\u1EEB,theo k\xFD hi\u1EC7u d\u1EA5u c\xE2u, v.v.), th\u01B0\u1EDDng \u0111\u01B0\u1EE3c g\u1ECDi l\xE0 "),be=s($c,"EM",{});var hr=o(be);es=l(hr,"token"),hr.forEach(n),hs=l($c,". C\xF3 nhi\u1EC1u quy t\u1EAFc c\xF3 th\u1EC3 chi ph\u1ED1i qu\xE1 tr\xECnh \u0111\xF3, \u0111\xF3 l\xE0 l\xFD do t\u1EA1i sao ch\xFAng ta c\u1EA7n kh\u1EDFi t\u1EA1o tr\xECnh token b\u1EB1ng c\xE1ch s\u1EED d\u1EE5ng t\xEAn c\u1EE7a m\xF4 h\xECnh, \u0111\u1EC3 \u0111\u1EA3m b\u1EA3o r\u1EB1ng ch\xFAng t\xF4i s\u1EED d\u1EE5ng c\xF9ng c\xE1c quy t\u1EAFc \u0111\xE3 \u0111\u01B0\u1EE3c s\u1EED d\u1EE5ng khi m\xF4 h\xECnh \u0111\u01B0\u1EE3c hu\u1EA5n luy\u1EC7n tr\u01B0\u1EDBc."),$c.forEach(n),Sh=p(t),L=s(t,"P",{});var Hn=o(L);cs=l(Hn,"B\u01B0\u1EDBc th\u1EE9 hai l\xE0 chuy\u1EC3n \u0111\u1ED5i c\xE1c token \u0111\xF3 th\xE0nh s\u1ED1 \u0111\u1EC3 ch\xFAng ta c\xF3 th\u1EC3 x\xE2y d\u1EF1ng m\u1ED9t tensor t\u1EEB ch\xFAng v\xE0 \u0111\u01B0a ch\xFAng v\xE0o m\xF4 h\xECnh. \u0110\u1EC3 l\xE0m \u0111i\u1EC1u n\xE0y, tokenizer c\xF3 "),$e=s(Hn,"EM",{});var cr=o($e);is=l(cr,"t\u1EEB v\u1EF1ng"),cr.forEach(n),ss=l(Hn,", l\xE0 ph\u1EA7n ch\xFAng ta t\u1EA3i xu\u1ED1ng khi kh\u1EDFi t\u1EA1o n\xF3 b\u1EB1ng ph\u01B0\u01A1ng th\u1EE9c "),ye=s(Hn,"CODE",{});var ir=o(ye);os=l(ir,"from_pretrained()"),ir.forEach(n),rs=l(Hn,". M\u1ED9t l\u1EA7n n\u1EEFa, ch\xFAng ta c\u1EA7n s\u1EED d\u1EE5ng c\xF9ng m\u1ED9t b\u1ED9 t\u1EEB v\u1EF1ng \u0111\u01B0\u1EE3c s\u1EED d\u1EE5ng khi m\xF4 h\xECnh \u0111\u01B0\u1EE3c hu\u1EA5n luy\u1EC7n tr\u01B0\u1EDBc."),Hn.forEach(n),Vh=p(t),Cn=s(t,"P",{});var sr=o(Cn);ls=l(sr,"\u0110\u1EC3 hi\u1EC3u r\xF5 h\u01A1n v\u1EC1 hai b\u01B0\u1EDBc, ch\xFAng ta s\u1EBD kh\xE1m ph\xE1 ch\xFAng m\u1ED9t c\xE1ch ri\xEAng bi\u1EC7t. L\u01B0u \xFD r\u1EB1ng ch\xFAng t\xF4i s\u1EBD s\u1EED d\u1EE5ng m\u1ED9t s\u1ED1 ph\u01B0\u01A1ng ph\xE1p th\u1EF1c hi\u1EC7n c\xE1c ph\u1EA7n c\u1EE7a pipeline tokenize ri\xEAng bi\u1EC7t \u0111\u1EC3 hi\u1EC3n th\u1ECB cho b\u1EA1n k\u1EBFt qu\u1EA3 trung gian c\u1EE7a c\xE1c b\u01B0\u1EDBc \u0111\xF3, nh\u01B0ng tr\xEAn th\u1EF1c t\u1EBF, b\u1EA1n n\xEAn g\u1ECDi tokenize tr\u1EF1c ti\u1EBFp tr\xEAn \u0111\u1EA7u v\xE0o c\u1EE7a m\xECnh (nh\u01B0 \u0111\u01B0\u1EE3c hi\u1EC3n th\u1ECB trong ph\u1EA7n 2)."),sr.forEach(n),Hh=p(t),tt=s(t,"H3",{class:!0});var yc=o(tt);kt=s(yc,"A",{id:!0,class:!0,href:!0});var or=o(kt);_e=s(or,"SPAN",{});var rr=o(_e);d(Wt.$$.fragment,rr),rr.forEach(n),or.forEach(n),as=p(yc),Ee=s(yc,"SPAN",{});var lr=o(Ee);ps=l(lr,"Tokenize"),lr.forEach(n),yc.forEach(n),Lh=p(t),dt=s(t,"P",{});var _c=o(dt);us=l(_c,"Qu\xE1 tr\xECnh tokenize \u0111\u01B0\u1EE3c th\u1EF1c hi\u1EC7n b\u1EDFi ph\u01B0\u01A1ng th\u1EE9c "),ze=s(_c,"CODE",{});var ar=o(ze);gs=l(ar,"tokenize()"),ar.forEach(n),ms=l(_c," c\u1EE7a tokenizer:"),_c.forEach(n),Bh=p(t),d(Xt.$$.fragment,t),Oh=p(t),qn=s(t,"P",{});var pr=o(qn);fs=l(pr,"K\u1EBFt qu\u1EA3 c\u1EE7a ph\u01B0\u01A1ng th\u1EE9c n\xE0y l\xE0 m\u1ED9t danh s\xE1ch c\xE1c chu\u1ED7i v\u0103n b\u1EA3n ho\u1EB7c tokens:"),pr.forEach(n),Gh=p(t),d(Zt.$$.fragment,t),Uh=p(t),q=s(t,"P",{});var zt=o(q);vs=l(zt,"Tokenizer n\xE0y l\xE0 m\u1ED9t tokenizer d\u1EF1 theo t\u1EEB ph\u1EE5: n\xF3 chia c\xE1c t\u1EEB cho \u0111\u1EBFn khi l\u1EA5y \u0111\u01B0\u1EE3c c\xE1c tokens \u0111\u01B0\u1EE3c bi\u1EC3u di\u1EC5n b\u1EDFi b\u1ED9 t\u1EEB v\u1EF1ng c\u1EE7a n\xF3. V\xED d\u1EE5, "),we=s(zt,"CODE",{});var ur=o(we);ks=l(ur,"transformer"),ur.forEach(n),ds=l(zt," s\u1EBD \u0111\u01B0\u1EE3c chia th\xE0nh hai token: "),xe=s(zt,"CODE",{});var gr=o(xe);bs=l(gr,"transform"),gr.forEach(n),$s=l(zt," v\xE0 "),Pe=s(zt,"CODE",{});var mr=o(Pe);ys=l(mr,"##er"),mr.forEach(n),_s=l(zt,"."),zt.forEach(n),Kh=p(t),nt=s(t,"H3",{class:!0});var Ec=o(nt);bt=s(Ec,"A",{id:!0,class:!0,href:!0});var fr=o(bt);Te=s(fr,"SPAN",{});var vr=o(Te);d(tn.$$.fragment,vr),vr.forEach(n),fr.forEach(n),Es=p(Ec),je=s(Ec,"SPAN",{});var kr=o(je);zs=l(kr,"T\u1EEB token t\u1EDBi ID \u0111\u1EA7u v\xE0o"),kr.forEach(n),Ec.forEach(n),Fh=p(t),$t=s(t,"P",{});var zc=o($t);ws=l(zc,"Qu\xE1 t\xECnh chuy\u1EC3n \u0111\u1ED5i sang ID \u0111\u1EA7u v\xE0o \u0111\u01B0\u1EE3c th\u1EF1c hi\u1EC7n b\u1EDFi "),Ae=s(zc,"CODE",{});var dr=o(Ae);xs=l(dr,"convert_tokens_to_ids()"),dr.forEach(n),Ps=l(zc," c\u1EE7a tokenizer:"),zc.forEach(n),Jh=p(t),d(nn.$$.fragment,t),Rh=p(t),d(en.$$.fragment,t),Qh=p(t),Nn=s(t,"P",{});var br=o(Nn);Ts=l(br,"C\xE1c \u0111\u1EA7u ra n\xE0y, sau khi \u0111\u01B0\u1EE3c chuy\u1EC3n \u0111\u1ED5i sang khung tensor th\xEDch h\u1EE3p, c\xF3 th\u1EC3 \u0111\u01B0\u1EE3c s\u1EED d\u1EE5ng l\xE0m \u0111\u1EA7u v\xE0o cho m\u1ED9t m\xF4 h\xECnh nh\u01B0 \u0111\xE3 th\u1EA5y \u1EDF ph\u1EA7n tr\u01B0\u1EDBc trong ch\u01B0\u01A1ng n\xE0y."),br.forEach(n),Yh=p(t),d(yt.$$.fragment,t),Wh=p(t),et=s(t,"H2",{class:!0});var wc=o(et);_t=s(wc,"A",{id:!0,class:!0,href:!0});var $r=o(_t);De=s($r,"SPAN",{});var yr=o(De);d(hn.$$.fragment,yr),yr.forEach(n),$r.forEach(n),js=p(wc),Ce=s(wc,"SPAN",{});var _r=o(Ce);As=l(_r,"Gi\u1EA3i m\xE3"),_r.forEach(n),wc.forEach(n),Xh=p(t),M=s(t,"P",{});var ln=o(M);qe=s(ln,"EM",{});var Er=o(qe);Ds=l(Er,"Decoding"),Er.forEach(n),Cs=l(ln," hay "),Ne=s(ln,"EM",{});var zr=o(Ne);qs=l(zr,"gi\u1EA3i m\xE3"),zr.forEach(n),Ns=l(ln," th\xEC ng\u01B0\u1EE3c l\u1EA1i: t\u1EEB c\xE1c ch\u1EC9 s\u1ED1 t\u1EEB v\u1EF1ng, ta mu\u1ED1n tr\u1EA3 v\u1EC1 m\u1ED9t chu\u1ED7i v\u0103n b\u1EA3n. \u0110i\u1EC1u n\xE0y c\xF3 th\u1EC3 \u0111\u01B0\u1EE3c th\u1EF1c hi\u1EC7n v\u1EDBi ph\u01B0\u01A1ng th\u1EE9c "),Ie=s(ln,"CODE",{});var wr=o(Ie);Is=l(wr,"decode()"),wr.forEach(n),Ms=l(ln," nh\u01B0 sau:"),ln.forEach(n),Zh=p(t),d(cn.$$.fragment,t),tc=p(t),d(sn.$$.fragment,t),nc=p(t),Et=s(t,"P",{});var xc=o(Et);Ss=l(xc,"L\u01B0u \xFD r\u1EB1ng ph\u01B0\u01A1ng ph\xE1p "),Me=s(xc,"CODE",{});var xr=o(Me);Vs=l(xr,"gi\u1EA3i m\xE3"),xr.forEach(n),Hs=l(xc," kh\xF4ng ch\u1EC9 chuy\u1EC3n \u0111\u1ED5i c\xE1c ch\u1EC9 s\u1ED1 tr\u1EDF l\u1EA1i th\xE0nh token, m\xE0 c\xF2n nh\xF3m c\xE1c token l\xE0 m\u1ED9t ph\u1EA7n c\u1EE7a c\xF9ng m\u1ED9t t\u1EEB l\u1EA1i v\u1EDBi nhau \u0111\u1EC3 t\u1EA1o ra m\u1ED9t c\xE2u c\xF3 th\u1EC3 \u0111\u1ECDc \u0111\u01B0\u1EE3c. H\xE0nh vi n\xE0y s\u1EBD c\u1EF1c k\u1EF3 h\u1EEFu \xEDch khi ch\xFAng ta s\u1EED d\u1EE5ng c\xE1c m\xF4 h\xECnh d\u1EF1 \u0111o\xE1n v\u0103n b\u1EA3n m\u1EDBi (v\u0103n b\u1EA3n \u0111\u01B0\u1EE3c t\u1EA1o t\u1EEB l\u1EDDi nh\u1EAFc ho\u1EB7c \u0111\u1ED1i v\u1EDBi c\xE1c b\xE0i to\xE1n chu\u1ED7i-sang-chu\u1ED7i nh\u01B0 d\u1ECBch ho\u1EB7c t\xF3m t\u1EAFt v\u0103n b\u1EA3n)."),xc.forEach(n),ec=p(t),In=s(t,"P",{});var Pr=o(In);Ls=l(Pr,"B\xE2y gi\u1EDD b\u1EA1n \u0111\xE3 hi\u1EC3u c\xE1c ho\u1EA1t \u0111\u1ED9ng nguy\xEAn t\u1EED m\xE0 m\u1ED9t tokenizer c\xF3 th\u1EC3 x\u1EED l\xFD: tokenize, chuy\u1EC3n \u0111\u1ED5i sang ID v\xE0 chuy\u1EC3n \u0111\u1ED5i ID tr\u1EDF l\u1EA1i m\u1ED9t chu\u1ED7i. Tuy nhi\xEAn, t\u1EA5t c\u1EA3 ch\u1EC9 m\u1EDBi l\xE0 s\u1EF1 b\u1EAFt \u0111\u1EA7u. Trong ph\u1EA7n sau, ch\xFAng ta s\u1EBD ti\u1EBFp c\u1EADn c\xE1c gi\u1EDBi h\u1EA1n c\u1EE7a n\xF3 v\xE0 xem c\xE1ch v\u01B0\u1EE3t qua ch\xFAng."),Pr.forEach(n),this.h()},h(){u(g,"name","hf:doc:metadata"),u(g,"content",JSON.stringify(Ur)),u(_,"id","tokenizers"),u(_,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),u(_,"href","#tokenizers"),u(w,"class","relative group"),u(ct,"id","da-trn-t"),u(ct,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),u(ct,"href","#da-trn-t"),u(K,"class","relative group"),u(jt,"class","block dark:hidden"),Ln(jt.src,Os="https://huggingface.co/datasets/huggingface-course/documentation-images/resolve/main/en/chapter2/word_based_tokenization.svg")||u(jt,"src",Os),u(jt,"alt","An example of word-based tokenization."),u(At,"class","hidden dark:block"),Ln(At.src,Gs="https://huggingface.co/datasets/huggingface-course/documentation-images/resolve/main/en/chapter2/word_based_tokenization-dark.svg")||u(At,"src",Gs),u(At,"alt","An example of word-based tokenization."),u(F,"class","flex justify-center"),u(rt,"id","da-trn-k-t"),u(rt,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),u(rt,"href","#da-trn-k-t"),u(J,"class","relative group"),u(Mt,"class","block dark:hidden"),Ln(Mt.src,Us="https://huggingface.co/datasets/huggingface-course/documentation-images/resolve/main/en/chapter2/character_based_tokenization.svg")||u(Mt,"src",Us),u(Mt,"alt","An example of character-based tokenization."),u(St,"class","hidden dark:block"),Ln(St.src,Ks="https://huggingface.co/datasets/huggingface-course/documentation-images/resolve/main/en/chapter2/character_based_tokenization-dark.svg")||u(St,"src",Ks),u(St,"alt","An example of character-based tokenization."),u(R,"class","flex justify-center"),u(pt,"id","tokenize-theo-t-ph"),u(pt,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),u(pt,"href","#tokenize-theo-t-ph"),u(Q,"class","relative group"),u(Lt,"class","block dark:hidden"),Ln(Lt.src,Fs="https://huggingface.co/datasets/huggingface-course/documentation-images/resolve/main/en/chapter2/bpe_subword.svg")||u(Lt,"src",Fs),u(Lt,"alt","A subword tokenization algorithm."),u(Bt,"class","hidden dark:block"),Ln(Bt.src,Js="https://huggingface.co/datasets/huggingface-course/documentation-images/resolve/main/en/chapter2/bpe_subword-dark.svg")||u(Bt,"src",Js),u(Bt,"alt","A subword tokenization algorithm."),u(Y,"class","flex justify-center"),u(ut,"id","v-hn-th-na"),u(ut,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),u(ut,"href","#v-hn-th-na"),u(W,"class","relative group"),u(gt,"id","ti-v-lu"),u(gt,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),u(gt,"href","#ti-v-lu"),u(X,"class","relative group"),u(Dn,"href","/course/chapter3"),u(ft,"id","m-ho"),u(ft,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),u(ft,"href","#m-ho"),u(Z,"class","relative group"),u(kt,"id","tokenize"),u(kt,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),u(kt,"href","#tokenize"),u(tt,"class","relative group"),u(bt,"id","t-token-ti-id-u-vo"),u(bt,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),u(bt,"href","#t-token-ti-id-u-vo"),u(nt,"class","relative group"),u(_t,"id","gii-m"),u(_t,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),u(_t,"href","#gii-m"),u(et,"class","relative group")},m(t,h){e(document.head,g),c(t,y,h),b(m,t,h),c(t,z,h),c(t,w,h),e(w,_),e(_,x),b(P,x,null),e(w,T),e(w,E),e(E,U),c(t,S,h),on[N].m(t,h),c(t,an,h),b(wt,t,h),c(t,He,h),c(t,pn,h),e(pn,Pc),c(t,Le,h),c(t,un,h),e(un,Tc),c(t,Be,h),b(xt,t,h),c(t,Oe,h),c(t,gn,h),e(gn,jc),c(t,Ge,h),c(t,mn,h),e(mn,Ac),c(t,Ue,h),c(t,K,h),e(K,ct),e(ct,Bn),b(Pt,Bn,null),e(K,Dc),e(K,On),e(On,Cc),c(t,Ke,h),b(Tt,t,h),c(t,Fe,h),c(t,it,h),e(it,qc),e(it,Gn),e(Gn,Nc),e(it,Ic),c(t,Je,h),c(t,F,h),e(F,jt),e(F,Mc),e(F,At),c(t,Re,h),c(t,st,h),e(st,Sc),e(st,Un),e(Un,Vc),e(st,Hc),c(t,Qe,h),b(Dt,t,h),c(t,Ye,h),b(Ct,t,h),c(t,We,h),c(t,fn,h),e(fn,Lc),c(t,Xe,h),c(t,vn,h),e(vn,Bc),c(t,Ze,h),c(t,kn,h),e(kn,Oc),c(t,th,h),c(t,qt,h),e(qt,Gc),e(qt,Kn),e(Kn,Uc),c(t,nh,h),c(t,ot,h),e(ot,Kc),e(ot,Fn),e(Fn,Fc),e(ot,Jc),c(t,eh,h),c(t,J,h),e(J,rt),e(rt,Jn),b(Nt,Jn,null),e(J,Rc),e(J,Rn),e(Rn,Qc),c(t,hh,h),b(It,t,h),c(t,ch,h),c(t,lt,h),e(lt,Qn),e(Qn,Yc),e(lt,Wc),e(lt,Yn),e(Yn,Xc),c(t,ih,h),c(t,dn,h),e(dn,Zc),c(t,sh,h),c(t,R,h),e(R,Mt),e(R,ti),e(R,St),c(t,oh,h),c(t,bn,h),e(bn,ni),c(t,rh,h),c(t,$n,h),e($n,ei),c(t,lh,h),c(t,at,h),e(at,hi),e(at,Wn),e(Wn,ci),e(at,ii),c(t,ah,h),c(t,Q,h),e(Q,pt),e(pt,Xn),b(Vt,Xn,null),e(Q,si),e(Q,Zn),e(Zn,oi),c(t,ph,h),b(Ht,t,h),c(t,uh,h),c(t,yn,h),e(yn,ri),c(t,gh,h),c(t,_n,h),e(_n,li),c(t,mh,h),c(t,En,h),e(En,ai),c(t,fh,h),c(t,Y,h),e(Y,Lt),e(Y,pi),e(Y,Bt),c(t,vh,h),c(t,zn,h),e(zn,ui),c(t,kh,h),c(t,wn,h),e(wn,gi),c(t,dh,h),c(t,W,h),e(W,ut),e(ut,te),b(Ot,te,null),e(W,mi),e(W,ne),e(ne,fi),c(t,bh,h),c(t,xn,h),e(xn,vi),c(t,$h,h),c(t,V,h),e(V,ee),e(ee,ki),e(V,di),e(V,he),e(he,bi),e(V,$i),e(V,ce),e(ce,yi),c(t,yh,h),c(t,Pn,h),e(Pn,_i),c(t,_h,h),c(t,X,h),e(X,gt),e(gt,ie),b(Gt,ie,null),e(X,Ei),e(X,se),e(se,zi),c(t,Eh,h),c(t,j,h),e(j,wi),e(j,oe),e(oe,xi),e(j,Pi),e(j,re),e(re,Ti),e(j,ji),e(j,le),e(le,Ai),e(j,Di),e(j,ae),e(ae,Ci),e(j,qi),c(t,zh,h),c(t,mt,h),e(mt,Ni),e(mt,pe),e(pe,Ii),e(mt,Mi),c(t,wh,h),b(Ut,t,h),c(t,xh,h),ht.m(t,h),c(t,Tn,h),b(Kt,t,h),c(t,Ph,h),c(t,jn,h),e(jn,Si),c(t,Th,h),b(Ft,t,h),c(t,jh,h),b(Jt,t,h),c(t,Ah,h),c(t,An,h),e(An,Vi),c(t,Dh,h),b(Rt,t,h),c(t,Ch,h),c(t,A,h),e(A,Hi),e(A,ue),e(ue,Li),e(A,Bi),e(A,Dn),e(Dn,Oi),e(A,Gi),e(A,ge),e(ge,Ui),e(A,Ki),e(A,me),e(me,Fi),e(A,Ji),c(t,qh,h),c(t,Z,h),e(Z,ft),e(ft,fe),b(Qt,fe,null),e(Z,Ri),e(Z,ve),e(ve,Qi),c(t,Nh,h),b(Yt,t,h),c(t,Ih,h),c(t,H,h),e(H,Yi),e(H,ke),e(ke,Wi),e(H,Xi),e(H,de),e(de,Zi),e(H,ts),c(t,Mh,h),c(t,vt,h),e(vt,ns),e(vt,be),e(be,es),e(vt,hs),c(t,Sh,h),c(t,L,h),e(L,cs),e(L,$e),e($e,is),e(L,ss),e(L,ye),e(ye,os),e(L,rs),c(t,Vh,h),c(t,Cn,h),e(Cn,ls),c(t,Hh,h),c(t,tt,h),e(tt,kt),e(kt,_e),b(Wt,_e,null),e(tt,as),e(tt,Ee),e(Ee,ps),c(t,Lh,h),c(t,dt,h),e(dt,us),e(dt,ze),e(ze,gs),e(dt,ms),c(t,Bh,h),b(Xt,t,h),c(t,Oh,h),c(t,qn,h),e(qn,fs),c(t,Gh,h),b(Zt,t,h),c(t,Uh,h),c(t,q,h),e(q,vs),e(q,we),e(we,ks),e(q,ds),e(q,xe),e(xe,bs),e(q,$s),e(q,Pe),e(Pe,ys),e(q,_s),c(t,Kh,h),c(t,nt,h),e(nt,bt),e(bt,Te),b(tn,Te,null),e(nt,Es),e(nt,je),e(je,zs),c(t,Fh,h),c(t,$t,h),e($t,ws),e($t,Ae),e(Ae,xs),e($t,Ps),c(t,Jh,h),b(nn,t,h),c(t,Rh,h),b(en,t,h),c(t,Qh,h),c(t,Nn,h),e(Nn,Ts),c(t,Yh,h),b(yt,t,h),c(t,Wh,h),c(t,et,h),e(et,_t),e(_t,De),b(hn,De,null),e(et,js),e(et,Ce),e(Ce,As),c(t,Xh,h),c(t,M,h),e(M,qe),e(qe,Ds),e(M,Cs),e(M,Ne),e(Ne,qs),e(M,Ns),e(M,Ie),e(Ie,Is),e(M,Ms),c(t,Zh,h),b(cn,t,h),c(t,tc,h),b(sn,t,h),c(t,nc,h),c(t,Et,h),e(Et,Ss),e(Et,Me),e(Me,Vs),e(Et,Hs),c(t,ec,h),c(t,In,h),e(In,Ls),hc=!0},p(t,[h]){const rn={};h&1&&(rn.fw=t[0]),m.$set(rn);let Mn=N;N=Qs(t),N!==Mn&&(Ir(),f(on[Mn],1,1,()=>{on[Mn]=null}),qr(),I=on[N],I||(I=on[N]=Rs[N](t),I.c()),v(I,1),I.m(an.parentNode,an)),cc!==(cc=Ys(t))&&(ht.d(1),ht=cc(t),ht&&(ht.c(),ht.m(Tn.parentNode,Tn)));const Se={};h&2&&(Se.$$scope={dirty:h,ctx:t}),yt.$set(Se)},i(t){hc||(v(m.$$.fragment,t),v(P.$$.fragment,t),v(I),v(wt.$$.fragment,t),v(xt.$$.fragment,t),v(Pt.$$.fragment,t),v(Tt.$$.fragment,t),v(Dt.$$.fragment,t),v(Ct.$$.fragment,t),v(Nt.$$.fragment,t),v(It.$$.fragment,t),v(Vt.$$.fragment,t),v(Ht.$$.fragment,t),v(Ot.$$.fragment,t),v(Gt.$$.fragment,t),v(Ut.$$.fragment,t),v(Kt.$$.fragment,t),v(Ft.$$.fragment,t),v(Jt.$$.fragment,t),v(Rt.$$.fragment,t),v(Qt.$$.fragment,t),v(Yt.$$.fragment,t),v(Wt.$$.fragment,t),v(Xt.$$.fragment,t),v(Zt.$$.fragment,t),v(tn.$$.fragment,t),v(nn.$$.fragment,t),v(en.$$.fragment,t),v(yt.$$.fragment,t),v(hn.$$.fragment,t),v(cn.$$.fragment,t),v(sn.$$.fragment,t),hc=!0)},o(t){f(m.$$.fragment,t),f(P.$$.fragment,t),f(I),f(wt.$$.fragment,t),f(xt.$$.fragment,t),f(Pt.$$.fragment,t),f(Tt.$$.fragment,t),f(Dt.$$.fragment,t),f(Ct.$$.fragment,t),f(Nt.$$.fragment,t),f(It.$$.fragment,t),f(Vt.$$.fragment,t),f(Ht.$$.fragment,t),f(Ot.$$.fragment,t),f(Gt.$$.fragment,t),f(Ut.$$.fragment,t),f(Kt.$$.fragment,t),f(Ft.$$.fragment,t),f(Jt.$$.fragment,t),f(Rt.$$.fragment,t),f(Qt.$$.fragment,t),f(Yt.$$.fragment,t),f(Wt.$$.fragment,t),f(Xt.$$.fragment,t),f(Zt.$$.fragment,t),f(tn.$$.fragment,t),f(nn.$$.fragment,t),f(en.$$.fragment,t),f(yt.$$.fragment,t),f(hn.$$.fragment,t),f(cn.$$.fragment,t),f(sn.$$.fragment,t),hc=!1},d(t){n(g),t&&n(y),$(m,t),t&&n(z),t&&n(w),$(P),t&&n(S),on[N].d(t),t&&n(an),$(wt,t),t&&n(He),t&&n(pn),t&&n(Le),t&&n(un),t&&n(Be),$(xt,t),t&&n(Oe),t&&n(gn),t&&n(Ge),t&&n(mn),t&&n(Ue),t&&n(K),$(Pt),t&&n(Ke),$(Tt,t),t&&n(Fe),t&&n(it),t&&n(Je),t&&n(F),t&&n(Re),t&&n(st),t&&n(Qe),$(Dt,t),t&&n(Ye),$(Ct,t),t&&n(We),t&&n(fn),t&&n(Xe),t&&n(vn),t&&n(Ze),t&&n(kn),t&&n(th),t&&n(qt),t&&n(nh),t&&n(ot),t&&n(eh),t&&n(J),$(Nt),t&&n(hh),$(It,t),t&&n(ch),t&&n(lt),t&&n(ih),t&&n(dn),t&&n(sh),t&&n(R),t&&n(oh),t&&n(bn),t&&n(rh),t&&n($n),t&&n(lh),t&&n(at),t&&n(ah),t&&n(Q),$(Vt),t&&n(ph),$(Ht,t),t&&n(uh),t&&n(yn),t&&n(gh),t&&n(_n),t&&n(mh),t&&n(En),t&&n(fh),t&&n(Y),t&&n(vh),t&&n(zn),t&&n(kh),t&&n(wn),t&&n(dh),t&&n(W),$(Ot),t&&n(bh),t&&n(xn),t&&n($h),t&&n(V),t&&n(yh),t&&n(Pn),t&&n(_h),t&&n(X),$(Gt),t&&n(Eh),t&&n(j),t&&n(zh),t&&n(mt),t&&n(wh),$(Ut,t),t&&n(xh),ht.d(t),t&&n(Tn),$(Kt,t),t&&n(Ph),t&&n(jn),t&&n(Th),$(Ft,t),t&&n(jh),$(Jt,t),t&&n(Ah),t&&n(An),t&&n(Dh),$(Rt,t),t&&n(Ch),t&&n(A),t&&n(qh),t&&n(Z),$(Qt),t&&n(Nh),$(Yt,t),t&&n(Ih),t&&n(H),t&&n(Mh),t&&n(vt),t&&n(Sh),t&&n(L),t&&n(Vh),t&&n(Cn),t&&n(Hh),t&&n(tt),$(Wt),t&&n(Lh),t&&n(dt),t&&n(Bh),$(Xt,t),t&&n(Oh),t&&n(qn),t&&n(Gh),$(Zt,t),t&&n(Uh),t&&n(q),t&&n(Kh),t&&n(nt),$(tn),t&&n(Fh),t&&n($t),t&&n(Jh),$(nn,t),t&&n(Rh),$(en,t),t&&n(Qh),t&&n(Nn),t&&n(Yh),$(yt,t),t&&n(Wh),t&&n(et),$(hn),t&&n(Xh),t&&n(M),t&&n(Zh),$(cn,t),t&&n(tc),$(sn,t),t&&n(nc),t&&n(Et),t&&n(ec),t&&n(In)}}}const Ur={local:"tokenizers",sections:[{local:"da-trn-t",title:"D\u1EF1a tr\xEAn t\u1EEB"},{local:"da-trn-k-t",title:"D\u1EF1a tr\xEAn k\xED t\u1EF1"},{local:"tokenize-theo-t-ph",sections:[{local:"v-hn-th-na",title:"V\xE0 h\u01A1n th\u1EBF n\u1EEFa!"}],title:"Tokenize theo t\u1EEB ph\u1EE5"},{local:"ti-v-lu",title:"T\u1EA3i v\xE0 l\u01B0u"},{local:"m-ho",sections:[{local:"tokenize",title:"Tokenize"},{local:"t-token-ti-id-u-vo",title:"T\u1EEB token t\u1EDBi ID \u0111\u1EA7u v\xE0o"}],title:"M\xE3 ho\xE1"},{local:"gii-m",title:"Gi\u1EA3i m\xE3"}],title:"Tokenizers"};function Kr(C,g,y){let m="pt";return Nr(()=>{const z=new URLSearchParams(window.location.search);y(0,m=z.get("fw")||"pt")}),[m]}class tl extends jr{constructor(g){super();Ar(this,g,Kr,Gr,Dr,{})}}export{tl as default,Ur as metadata};
