import{S as Cv,i as Dv,s as Ov,e as i,k as h,w as m,t as s,M as Lv,c,d as n,m as p,a as l,x as u,h as o,b as _,N as wv,G as e,g as a,y as k,q as f,o as g,B as d,v as Sv}from"../../chunks/vendor-hf-doc-builder.js";import{T as Bv}from"../../chunks/Tip-hf-doc-builder.js";import{Y as Nv}from"../../chunks/Youtube-hf-doc-builder.js";import{I as ni}from"../../chunks/IconCopyLink-hf-doc-builder.js";import{C as v}from"../../chunks/CodeBlock-hf-doc-builder.js";import{D as Fv}from"../../chunks/DocNotebookDropdown-hf-doc-builder.js";function Av(si){let $,it,N,W,Q,tt,he,et,pe,Pt,F,wt,ct,xt,Ct,T,nt,me,ue,st,ke,fe,ot,ge,de;return{c(){$=i("p"),it=i("strong"),N=s("\u0110\xE0o s\xE2u h\u01A1n"),W=s(" N\u1EBFu b\u1EA1n mu\u1ED1n ki\u1EC3m tra xem hai phi\xEAn b\u1EA3n chu\u1EA9n ho\xE1 tr\u01B0\u1EDBc \u0111\xF3 tr\xEAn c\u0169ng m\u1ED9t chu\u1ED7i ch\u1EE9a k\xED t\u1EF1 unicode "),Q=i("code"),tt=s('u"\\u0085"'),he=s(`, b\u1EA1n ch\u1EAFc ch\u1EAFn s\u1EBD nh\u1EADn th\u1EA5y r\u1EB1ng hai c\xE1ch chu\u1EA9n ho\xE1 n\xE0y kh\xF4ng ho\xE0n to\xE0n gi\u1ED1ng nhau.
\u0110\u1EC3 tr\xE1nh ph\u1EE9c t\u1EA1p ho\xE1 phi\xEAn b\u1EA3n v\u1EDBi `),et=i("code"),pe=s("normalizers.Sequence"),Pt=s(" qu\xE1 nhi\u1EC1u, ch\xFAng t\xF4i s\u1EBD kh\xF4ng bao g\u1ED3m c\xE1c s\u1EF1 thay th\u1EBF theo Regex m\xE0 "),F=i("code"),wt=s("BertNormalizer"),ct=s(" y\xEAu c\u1EA7u khi tham s\u1ED1 "),xt=i("code"),Ct=s("clean_text"),T=s(" \u0111\u01B0\u1EE3c thi\u1EBFt l\u1EADp l\xE0 "),nt=i("code"),me=s("True"),ue=s(" - \u0111\xE2y c\u0169ng l\xE0 gi\xE1 tr\u1ECB m\u1EB7c \u0111\u1ECBnh. Nh\u01B0ng \u0111\u1EEBng lo: c\xF3 kh\u1EA3 n\u0103ng ta s\u1EBD nh\u1EADn \u0111\u01B0\u1EE3c k\u1EBFt qu\u1EA3 chu\u1EA9n ho\xE1 gi\u1ED1ng nhau m\xE0 kh\xF4ng c\u1EA7n s\u1EED d\u1EE5ng "),st=i("code"),ke=s("BertNormalizer"),fe=s(" th\u1EE7 c\xF4ng b\u1EB1ng c\xE1ch th\xEAm hai "),ot=i("code"),ge=s("normalizers.Replace"),de=s(" v\xE0o chu\u1ED7i chu\u1EA9n ho\xE1.")},l(rt){$=c(rt,"P",{});var z=l($);it=c(z,"STRONG",{});var ve=l(it);N=o(ve,"\u0110\xE0o s\xE2u h\u01A1n"),ve.forEach(n),W=o(z," N\u1EBFu b\u1EA1n mu\u1ED1n ki\u1EC3m tra xem hai phi\xEAn b\u1EA3n chu\u1EA9n ho\xE1 tr\u01B0\u1EDBc \u0111\xF3 tr\xEAn c\u0169ng m\u1ED9t chu\u1ED7i ch\u1EE9a k\xED t\u1EF1 unicode "),Q=c(z,"CODE",{});var U=l(Q);tt=o(U,'u"\\u0085"'),U.forEach(n),he=o(z,`, b\u1EA1n ch\u1EAFc ch\u1EAFn s\u1EBD nh\u1EADn th\u1EA5y r\u1EB1ng hai c\xE1ch chu\u1EA9n ho\xE1 n\xE0y kh\xF4ng ho\xE0n to\xE0n gi\u1ED1ng nhau.
\u0110\u1EC3 tr\xE1nh ph\u1EE9c t\u1EA1p ho\xE1 phi\xEAn b\u1EA3n v\u1EDBi `),et=c(z,"CODE",{});var lt=l(et);pe=o(lt,"normalizers.Sequence"),lt.forEach(n),Pt=o(z," qu\xE1 nhi\u1EC1u, ch\xFAng t\xF4i s\u1EBD kh\xF4ng bao g\u1ED3m c\xE1c s\u1EF1 thay th\u1EBF theo Regex m\xE0 "),F=c(z,"CODE",{});var Ws=l(F);wt=o(Ws,"BertNormalizer"),Ws.forEach(n),ct=o(z," y\xEAu c\u1EA7u khi tham s\u1ED1 "),xt=c(z,"CODE",{});var Qn=l(xt);Ct=o(Qn,"clean_text"),Qn.forEach(n),T=o(z," \u0111\u01B0\u1EE3c thi\u1EBFt l\u1EADp l\xE0 "),nt=c(z,"CODE",{});var at=l(nt);me=o(at,"True"),at.forEach(n),ue=o(z," - \u0111\xE2y c\u0169ng l\xE0 gi\xE1 tr\u1ECB m\u1EB7c \u0111\u1ECBnh. Nh\u01B0ng \u0111\u1EEBng lo: c\xF3 kh\u1EA3 n\u0103ng ta s\u1EBD nh\u1EADn \u0111\u01B0\u1EE3c k\u1EBFt qu\u1EA3 chu\u1EA9n ho\xE1 gi\u1ED1ng nhau m\xE0 kh\xF4ng c\u1EA7n s\u1EED d\u1EE5ng "),st=c(z,"CODE",{});var Us=l(st);ke=o(Us,"BertNormalizer"),Us.forEach(n),fe=o(z," th\u1EE7 c\xF4ng b\u1EB1ng c\xE1ch th\xEAm hai "),ot=c(z,"CODE",{});var _e=l(ot);ge=o(_e,"normalizers.Replace"),_e.forEach(n),de=o(z," v\xE0o chu\u1ED7i chu\u1EA9n ho\xE1."),z.forEach(n)},m(rt,z){a(rt,$,z),e($,it),e(it,N),e($,W),e($,Q),e(Q,tt),e($,he),e($,et),e(et,pe),e($,Pt),e($,F),e(F,wt),e($,ct),e($,xt),e(xt,Ct),e($,T),e($,nt),e(nt,me),e($,ue),e($,st),e(st,ke),e($,fe),e($,ot),e(ot,ge),e($,de)},d(rt){rt&&n($)}}}function Wv(si){let $,it,N,W,Q,tt,he,et,pe,Pt,F,wt,ct,xt,Ct,T,nt,me,ue,st,ke,fe,ot,ge,de,rt,z,ve,U,lt,Ws,Qn,at,Us,_e,Dt,ma,ts,ua,ka,oi,$e,ri,Ot,fa,Rs,ga,da,ii,q,ht,Gs,va,_a,Ms,$a,za,ze,ba,Ea,xa,pt,Is,ja,ya,Hs,Ta,qa,be,Pa,wa,Ca,P,Xs,Da,Oa,Vs,La,Sa,Ks,Ba,Na,Js,Fa,Aa,Ys,Wa,Ua,Ee,Ra,Ga,Ma,mt,Zs,Ia,Ha,Qs,Xa,Va,xe,Ka,Ja,Ya,ut,to,Za,Qa,eo,th,eh,je,nh,sh,oh,kt,no,rh,ih,so,ch,lh,ye,ah,hh,ci,Lt,ph,Te,mh,uh,li,jt,St,oo,qe,kh,ro,fh,ai,ft,gh,es,dh,vh,Pe,_h,$h,hi,we,pi,Bt,zh,io,bh,Eh,mi,ns,xh,ui,Ce,ki,ss,jh,fi,yt,Nt,co,De,yh,lo,Th,gi,x,qh,ao,Ph,wh,ho,Ch,Dh,po,Oh,Lh,mo,Sh,Bh,uo,Nh,Fh,ko,Ah,Wh,di,Ft,Uh,fo,Rh,Gh,vi,Oe,_i,R,Mh,go,Ih,Hh,vo,Xh,Vh,_o,Kh,Jh,$i,j,Yh,$o,Zh,Qh,zo,tp,ep,bo,np,sp,Eo,op,rp,xo,ip,cp,jo,lp,ap,zi,Le,bi,G,hp,yo,pp,mp,To,up,kp,qo,fp,gp,Ei,Se,xi,gt,dp,Po,vp,_p,wo,$p,zp,ji,dt,bp,Co,Ep,xp,Do,jp,yp,yi,Be,Ti,Ne,qi,At,Pi,Wt,Tp,Oo,qp,Pp,wi,Fe,Ci,os,wp,Di,Ae,Oi,Ut,Cp,Lo,Dp,Op,Li,We,Si,Ue,Bi,Rt,Lp,So,Sp,Bp,Ni,Re,Fi,Ge,Ai,Gt,Np,Bo,Fp,Ap,Wi,Me,Ui,Ie,Ri,Mt,Wp,No,Up,Rp,Gi,He,Mi,w,Gp,Fo,Mp,Ip,Ao,Hp,Xp,Wo,Vp,Kp,Uo,Jp,Yp,Ro,Zp,Qp,Ii,rs,tm,Hi,Xe,Xi,It,em,Go,nm,sm,Vi,Ve,Ki,Ht,om,Mo,rm,im,Ji,Ke,Yi,Je,Zi,b,Io,cm,lm,Ho,am,hm,Xo,pm,mm,Vo,um,km,Ko,fm,gm,Jo,dm,vm,Yo,_m,$m,Zo,zm,bm,Qo,Em,xm,Qi,C,jm,tr,ym,Tm,er,qm,Pm,nr,wm,Cm,sr,Dm,Om,or,Lm,Sm,tc,Ye,ec,Ze,nc,M,Bm,rr,Nm,Fm,ir,Am,Wm,cr,Um,Rm,sc,is,Gm,oc,Qe,rc,cs,Mm,ic,ls,Im,cc,tn,lc,en,ac,as,Hm,hc,nn,pc,sn,mc,hs,Xm,uc,on,kc,Xt,Vm,lr,Km,Jm,fc,rn,gc,cn,dc,ps,Ym,vc,ln,_c,vt,Zm,ar,Qm,tu,hr,eu,nu,$c,an,zc,_t,su,pr,ou,ru,mr,iu,cu,bc,D,lu,ur,au,hu,kr,pu,mu,fr,uu,ku,gr,fu,gu,dr,du,vu,Ec,hn,xc,Vt,_u,vr,$u,zu,jc,pn,yc,$t,bu,_r,Eu,xu,$r,ju,yu,Tc,ms,Tu,qc,Tt,Kt,zr,mn,qu,br,Pu,Pc,Jt,wu,Er,Cu,Du,wc,un,Cc,I,Ou,xr,Lu,Su,jr,Bu,Nu,yr,Fu,Au,Dc,us,Wu,Oc,kn,Lc,Yt,Uu,Tr,Ru,Gu,Sc,fn,Bc,gn,Nc,ks,Mu,Fc,dn,Ac,y,Iu,qr,Hu,Xu,Pr,Vu,Ku,wr,Ju,Yu,Cr,Zu,Qu,Dr,tk,ek,Or,nk,sk,Wc,fs,ok,Uc,vn,Rc,gs,rk,Gc,_n,Mc,$n,Ic,ds,ik,Hc,zn,Xc,zt,ck,Lr,lk,ak,Sr,hk,pk,Vc,bn,Kc,En,Jc,vs,mk,Yc,xn,Zc,_s,uk,Qc,jn,tl,yn,el,bt,kk,Br,fk,gk,Nr,dk,vk,nl,Tn,sl,$s,_k,ol,qn,rl,zs,$k,il,qt,Zt,Fr,Pn,zk,Ar,bk,cl,Qt,Ek,Wr,xk,jk,ll,wn,al,bs,yk,hl,Es,Tk,pl,Cn,ml,H,qk,Ur,Pk,wk,Rr,Ck,Dk,Gr,Ok,Lk,ul,te,Sk,Mr,Bk,Nk,kl,Dn,fl,xs,Fk,gl,On,dl,Ln,vl,js,Ak,_l,Sn,$l,O,Wk,Ir,Uk,Rk,Hr,Gk,Mk,Xr,Ik,Hk,Vr,Xk,Vk,zl,ys,Kk,bl,Bn,El,Ts,Jk,xl,Nn,jl,Fn,yl,X,Yk,Kr,Zk,Qk,Jr,tf,ef,Yr,nf,sf,Tl,An,ql,Wn,Pl,qs,of,wl,Un,Cl,Ps,rf,Dl,Rn,Ol,Gn,Ll,ee,cf,Zr,lf,af,Sl,Mn,Bl,V,hf,Qr,pf,mf,ti,uf,kf,ei,ff,gf,Nl,In,Fl,ws,df,Al,Hn,Wl,Cs,vf,Ul;return tt=new ni({}),F=new Fv({props:{classNames:"absolute z-10 right-0 top-0",options:[{label:"Google Colab",value:"https://colab.research.google.com/github/huggingface/notebooks/blob/master/course/vi/chapter6/section8.ipynb"},{label:"Aws Studio",value:"https://studiolab.sagemaker.aws/import/github/huggingface/notebooks/blob/master/course/vi/chapter6/section8.ipynb"}]}}),$e=new Nv({props:{id:"MR8tZm5ViWU"}}),qe=new ni({}),we=new v({props:{code:`from datasets import load_dataset

dataset = load_dataset("wikitext", name="wikitext-2-raw-v1", split="train")


def get_training_corpus():
    for i in range(0, len(dataset), 1000):
        yield dataset[i : i + 1000]["text"]`,highlighted:`<span class="hljs-keyword">from</span> datasets <span class="hljs-keyword">import</span> load_dataset

dataset = load_dataset(<span class="hljs-string">&quot;wikitext&quot;</span>, name=<span class="hljs-string">&quot;wikitext-2-raw-v1&quot;</span>, split=<span class="hljs-string">&quot;train&quot;</span>)


<span class="hljs-keyword">def</span> <span class="hljs-title function_">get_training_corpus</span>():
    <span class="hljs-keyword">for</span> i <span class="hljs-keyword">in</span> <span class="hljs-built_in">range</span>(<span class="hljs-number">0</span>, <span class="hljs-built_in">len</span>(dataset), <span class="hljs-number">1000</span>):
        <span class="hljs-keyword">yield</span> dataset[i : i + <span class="hljs-number">1000</span>][<span class="hljs-string">&quot;text&quot;</span>]`}}),Ce=new v({props:{code:`with open("wikitext-2.txt", "w", encoding="utf-8") as f:
    for i in range(len(dataset)):
        f.write(dataset[i]["text"] + "\\n")`,highlighted:`<span class="hljs-keyword">with</span> <span class="hljs-built_in">open</span>(<span class="hljs-string">&quot;wikitext-2.txt&quot;</span>, <span class="hljs-string">&quot;w&quot;</span>, encoding=<span class="hljs-string">&quot;utf-8&quot;</span>) <span class="hljs-keyword">as</span> f:
    <span class="hljs-keyword">for</span> i <span class="hljs-keyword">in</span> <span class="hljs-built_in">range</span>(<span class="hljs-built_in">len</span>(dataset)):
        f.write(dataset[i][<span class="hljs-string">&quot;text&quot;</span>] + <span class="hljs-string">&quot;\\n&quot;</span>)`}}),De=new ni({}),Oe=new v({props:{code:`from tokenizers import (
    decoders,
    models,
    normalizers,
    pre_tokenizers,
    processors,
    trainers,
    Tokenizer,
)

tokenizer = Tokenizer(models.WordPiece(unk_token="[UNK]"))`,highlighted:`<span class="hljs-keyword">from</span> tokenizers <span class="hljs-keyword">import</span> (
    decoders,
    models,
    normalizers,
    pre_tokenizers,
    processors,
    trainers,
    Tokenizer,
)

tokenizer = Tokenizer(models.WordPiece(unk_token=<span class="hljs-string">&quot;[UNK]&quot;</span>))`}}),Le=new v({props:{code:"tokenizer.normalizer = normalizers.BertNormalizer(lowercase=True)",highlighted:'tokenizer.normalizer = normalizers.BertNormalizer(lowercase=<span class="hljs-literal">True</span>)'}}),Se=new v({props:{code:`tokenizer.normalizer = normalizers.Sequence(
    [normalizers.NFD(), normalizers.Lowercase(), normalizers.StripAccents()]
)`,highlighted:`tokenizer.normalizer = normalizers.<span class="hljs-type">Sequence</span>(
    [normalizers.NFD(), normalizers.Lowercase(), normalizers.StripAccents()]
)`}}),Be=new v({props:{code:'print(tokenizer.normalizer.normalize_str("H\xE9ll\xF2 h\xF4w are \xFC?"))',highlighted:'<span class="hljs-built_in">print</span>(tokenizer.normalizer.normalize_str(<span class="hljs-string">&quot;H\xE9ll\xF2 h\xF4w are \xFC?&quot;</span>))'}}),Ne=new v({props:{code:"hello how are u?",highlighted:"hello how are u?"}}),At=new Bv({props:{$$slots:{default:[Av]},$$scope:{ctx:si}}}),Fe=new v({props:{code:"tokenizer.pre_tokenizer = pre_tokenizers.BertPreTokenizer()",highlighted:"tokenizer.pre_tokenizer = pre_tokenizers.BertPreTokenizer()"}}),Ae=new v({props:{code:"tokenizer.pre_tokenizer = pre_tokenizers.Whitespace()",highlighted:"tokenizer.pre_tokenizer = pre_tokenizers.Whitespace()"}}),We=new v({props:{code:`tokenizer.pre_tokenizer.pre_tokenize_str("Let's test my pre-tokenizer.")`,highlighted:'tokenizer.pre_tokenizer.pre_tokenize_str(<span class="hljs-string">&quot;Let&#x27;s test my pre-tokenizer.&quot;</span>)'}}),Ue=new v({props:{code:`[('Let', (0, 3)), ("'", (3, 4)), ('s', (4, 5)), ('test', (6, 10)), ('my', (11, 13)), ('pre', (14, 17)),
 ('-', (17, 18)), ('tokenizer', (18, 27)), ('.', (27, 28))]`,highlighted:`[(<span class="hljs-string">&#x27;Let&#x27;</span>, (<span class="hljs-number">0</span>, <span class="hljs-number">3</span>)), (<span class="hljs-string">&quot;&#x27;&quot;</span>, (<span class="hljs-number">3</span>, <span class="hljs-number">4</span>)), (<span class="hljs-string">&#x27;s&#x27;</span>, (<span class="hljs-number">4</span>, <span class="hljs-number">5</span>)), (<span class="hljs-string">&#x27;test&#x27;</span>, (<span class="hljs-number">6</span>, <span class="hljs-number">10</span>)), (<span class="hljs-string">&#x27;my&#x27;</span>, (<span class="hljs-number">11</span>, <span class="hljs-number">13</span>)), (<span class="hljs-string">&#x27;pre&#x27;</span>, (<span class="hljs-number">14</span>, <span class="hljs-number">17</span>)),
 (<span class="hljs-string">&#x27;-&#x27;</span>, (<span class="hljs-number">17</span>, <span class="hljs-number">18</span>)), (<span class="hljs-string">&#x27;tokenizer&#x27;</span>, (<span class="hljs-number">18</span>, <span class="hljs-number">27</span>)), (<span class="hljs-string">&#x27;.&#x27;</span>, (<span class="hljs-number">27</span>, <span class="hljs-number">28</span>))]`}}),Re=new v({props:{code:`pre_tokenizer = pre_tokenizers.WhitespaceSplit()
pre_tokenizer.pre_tokenize_str("Let's test my pre-tokenizer.")`,highlighted:`pre_tokenizer = pre_tokenizers.WhitespaceSplit()
pre_tokenizer.pre_tokenize_str(<span class="hljs-string">&quot;Let&#x27;s test my pre-tokenizer.&quot;</span>)`}}),Ge=new v({props:{code:`[("Let's", (0, 5)), ('test', (6, 10)), ('my', (11, 13)), ('pre-tokenizer.', (14, 28))]`,highlighted:'[(<span class="hljs-string">&quot;Let&#x27;s&quot;</span>, (<span class="hljs-number">0</span>, <span class="hljs-number">5</span>)), (<span class="hljs-string">&#x27;test&#x27;</span>, (<span class="hljs-number">6</span>, <span class="hljs-number">10</span>)), (<span class="hljs-string">&#x27;my&#x27;</span>, (<span class="hljs-number">11</span>, <span class="hljs-number">13</span>)), (<span class="hljs-string">&#x27;pre-tokenizer.&#x27;</span>, (<span class="hljs-number">14</span>, <span class="hljs-number">28</span>))]'}}),Me=new v({props:{code:`pre_tokenizer = pre_tokenizers.Sequence(
    [pre_tokenizers.WhitespaceSplit(), pre_tokenizers.Punctuation()]
)
pre_tokenizer.pre_tokenize_str("Let's test my pre-tokenizer.")`,highlighted:`pre_tokenizer = pre_tokenizers.<span class="hljs-type">Sequence</span>(
    [pre_tokenizers.WhitespaceSplit(), pre_tokenizers.Punctuation()]
)
pre_tokenizer.pre_tokenize_str(<span class="hljs-string">&quot;Let&#x27;s test my pre-tokenizer.&quot;</span>)`}}),Ie=new v({props:{code:`[('Let', (0, 3)), ("'", (3, 4)), ('s', (4, 5)), ('test', (6, 10)), ('my', (11, 13)), ('pre', (14, 17)),
 ('-', (17, 18)), ('tokenizer', (18, 27)), ('.', (27, 28))]`,highlighted:`[(<span class="hljs-string">&#x27;Let&#x27;</span>, (<span class="hljs-number">0</span>, <span class="hljs-number">3</span>)), (<span class="hljs-string">&quot;&#x27;&quot;</span>, (<span class="hljs-number">3</span>, <span class="hljs-number">4</span>)), (<span class="hljs-string">&#x27;s&#x27;</span>, (<span class="hljs-number">4</span>, <span class="hljs-number">5</span>)), (<span class="hljs-string">&#x27;test&#x27;</span>, (<span class="hljs-number">6</span>, <span class="hljs-number">10</span>)), (<span class="hljs-string">&#x27;my&#x27;</span>, (<span class="hljs-number">11</span>, <span class="hljs-number">13</span>)), (<span class="hljs-string">&#x27;pre&#x27;</span>, (<span class="hljs-number">14</span>, <span class="hljs-number">17</span>)),
 (<span class="hljs-string">&#x27;-&#x27;</span>, (<span class="hljs-number">17</span>, <span class="hljs-number">18</span>)), (<span class="hljs-string">&#x27;tokenizer&#x27;</span>, (<span class="hljs-number">18</span>, <span class="hljs-number">27</span>)), (<span class="hljs-string">&#x27;.&#x27;</span>, (<span class="hljs-number">27</span>, <span class="hljs-number">28</span>))]`}}),He=new v({props:{code:`special_tokens = ["[UNK]", "[PAD]", "[CLS]", "[SEP]", "[MASK]"]
trainer = trainers.WordPieceTrainer(vocab_size=25000, special_tokens=special_tokens)`,highlighted:`special_tokens = [<span class="hljs-string">&quot;[UNK]&quot;</span>, <span class="hljs-string">&quot;[PAD]&quot;</span>, <span class="hljs-string">&quot;[CLS]&quot;</span>, <span class="hljs-string">&quot;[SEP]&quot;</span>, <span class="hljs-string">&quot;[MASK]&quot;</span>]
trainer = trainers.WordPieceTrainer(vocab_size=<span class="hljs-number">25000</span>, special_tokens=special_tokens)`}}),Xe=new v({props:{code:"tokenizer.train_from_iterator(get_training_corpus(), trainer=trainer)",highlighted:"tokenizer.train_from_iterator(get_training_corpus(), trainer=trainer)"}}),Ve=new v({props:{code:`tokenizer.model = models.WordPiece(unk_token="[UNK]")
tokenizer.train(["wikitext-2.txt"], trainer=trainer)`,highlighted:`tokenizer.model = models.WordPiece(unk_token=<span class="hljs-string">&quot;[UNK]&quot;</span>)
tokenizer.train([<span class="hljs-string">&quot;wikitext-2.txt&quot;</span>], trainer=trainer)`}}),Ke=new v({props:{code:`encoding = tokenizer.encode("Let's test this tokenizer.")
print(encoding.tokens)`,highlighted:`encoding = tokenizer.encode(<span class="hljs-string">&quot;Let&#x27;s test this tokenizer.&quot;</span>)
<span class="hljs-built_in">print</span>(encoding.tokens)`}}),Je=new v({props:{code:`['let', "'", 's', 'test', 'this', 'tok', '##eni', '##zer', '.']`,highlighted:'[<span class="hljs-string">&#x27;let&#x27;</span>, <span class="hljs-string">&quot;&#x27;&quot;</span>, <span class="hljs-string">&#x27;s&#x27;</span>, <span class="hljs-string">&#x27;test&#x27;</span>, <span class="hljs-string">&#x27;this&#x27;</span>, <span class="hljs-string">&#x27;tok&#x27;</span>, <span class="hljs-string">&#x27;##eni&#x27;</span>, <span class="hljs-string">&#x27;##zer&#x27;</span>, <span class="hljs-string">&#x27;.&#x27;</span>]'}}),Ye=new v({props:{code:`cls_token_id = tokenizer.token_to_id("[CLS]")
sep_token_id = tokenizer.token_to_id("[SEP]")
print(cls_token_id, sep_token_id)`,highlighted:`cls_token_id = tokenizer.token_to_id(<span class="hljs-string">&quot;[CLS]&quot;</span>)
sep_token_id = tokenizer.token_to_id(<span class="hljs-string">&quot;[SEP]&quot;</span>)
<span class="hljs-built_in">print</span>(cls_token_id, sep_token_id)`}}),Ze=new v({props:{code:"(2, 3)",highlighted:'(<span class="hljs-number">2</span>, <span class="hljs-number">3</span>)'}}),Qe=new v({props:{code:`tokenizer.post_processor = processors.TemplateProcessing(
    single=f"[CLS]:0 $A:0 [SEP]:0",
    pair=f"[CLS]:0 $A:0 [SEP]:0 $B:1 [SEP]:1",
    special_tokens=[("[CLS]", cls_token_id), ("[SEP]", sep_token_id)],
)`,highlighted:`tokenizer.post_processor = processors.TemplateProcessing(
    single=<span class="hljs-string">f&quot;[CLS]:0 $A:0 [SEP]:0&quot;</span>,
    pair=<span class="hljs-string">f&quot;[CLS]:0 $A:0 [SEP]:0 $B:1 [SEP]:1&quot;</span>,
    special_tokens=[(<span class="hljs-string">&quot;[CLS]&quot;</span>, cls_token_id), (<span class="hljs-string">&quot;[SEP]&quot;</span>, sep_token_id)],
)`}}),tn=new v({props:{code:`encoding = tokenizer.encode("Let's test this tokenizer.")
print(encoding.tokens)`,highlighted:`encoding = tokenizer.encode(<span class="hljs-string">&quot;Let&#x27;s test this tokenizer.&quot;</span>)
<span class="hljs-built_in">print</span>(encoding.tokens)`}}),en=new v({props:{code:`['[CLS]', 'let', "'", 's', 'test', 'this', 'tok', '##eni', '##zer', '.', '[SEP]']`,highlighted:'[<span class="hljs-string">&#x27;[CLS]&#x27;</span>, <span class="hljs-string">&#x27;let&#x27;</span>, <span class="hljs-string">&quot;&#x27;&quot;</span>, <span class="hljs-string">&#x27;s&#x27;</span>, <span class="hljs-string">&#x27;test&#x27;</span>, <span class="hljs-string">&#x27;this&#x27;</span>, <span class="hljs-string">&#x27;tok&#x27;</span>, <span class="hljs-string">&#x27;##eni&#x27;</span>, <span class="hljs-string">&#x27;##zer&#x27;</span>, <span class="hljs-string">&#x27;.&#x27;</span>, <span class="hljs-string">&#x27;[SEP]&#x27;</span>]'}}),nn=new v({props:{code:`encoding = tokenizer.encode("Let's test this tokenizer...", "on a pair of sentences.")
print(encoding.tokens)
print(encoding.type_ids)`,highlighted:`encoding = tokenizer.encode(<span class="hljs-string">&quot;Let&#x27;s test this tokenizer...&quot;</span>, <span class="hljs-string">&quot;on a pair of sentences.&quot;</span>)
<span class="hljs-built_in">print</span>(encoding.tokens)
<span class="hljs-built_in">print</span>(encoding.type_ids)`}}),sn=new v({props:{code:`['[CLS]', 'let', "'", 's', 'test', 'this', 'tok', '##eni', '##zer', '...', '[SEP]', 'on', 'a', 'pair', 'of', 'sentences', '.', '[SEP]']
[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1]`,highlighted:`[<span class="hljs-string">&#x27;[CLS]&#x27;</span>, <span class="hljs-string">&#x27;let&#x27;</span>, <span class="hljs-string">&quot;&#x27;&quot;</span>, <span class="hljs-string">&#x27;s&#x27;</span>, <span class="hljs-string">&#x27;test&#x27;</span>, <span class="hljs-string">&#x27;this&#x27;</span>, <span class="hljs-string">&#x27;tok&#x27;</span>, <span class="hljs-string">&#x27;##eni&#x27;</span>, <span class="hljs-string">&#x27;##zer&#x27;</span>, <span class="hljs-string">&#x27;...&#x27;</span>, <span class="hljs-string">&#x27;[SEP]&#x27;</span>, <span class="hljs-string">&#x27;on&#x27;</span>, <span class="hljs-string">&#x27;a&#x27;</span>, <span class="hljs-string">&#x27;pair&#x27;</span>, <span class="hljs-string">&#x27;of&#x27;</span>, <span class="hljs-string">&#x27;sentences&#x27;</span>, <span class="hljs-string">&#x27;.&#x27;</span>, <span class="hljs-string">&#x27;[SEP]&#x27;</span>]
[<span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>]`}}),on=new v({props:{code:'tokenizer.decoder = decoders.WordPiece(prefix="##")',highlighted:'tokenizer.decoder = decoders.WordPiece(prefix=<span class="hljs-string">&quot;##&quot;</span>)'}}),rn=new v({props:{code:"tokenizer.decode(encoding.ids)",highlighted:"tokenizer.decode(encoding.ids)"}}),cn=new v({props:{code:`"let's test this tokenizer... on a pair of sentences."`,highlighted:'<span class="hljs-string">&quot;let&#x27;s test this tokenizer... on a pair of sentences.&quot;</span>'}}),ln=new v({props:{code:'tokenizer.save("tokenizer.json")',highlighted:'tokenizer.save(<span class="hljs-string">&quot;tokenizer.json&quot;</span>)'}}),an=new v({props:{code:'new_tokenizer = Tokenizer.from_file("tokenizer.json")',highlighted:'new_tokenizer = Tokenizer.from_file(<span class="hljs-string">&quot;tokenizer.json&quot;</span>)'}}),hn=new v({props:{code:`from transformers import PreTrainedTokenizerFast

wrapped_tokenizer = PreTrainedTokenizerFast(
    tokenizer_object=tokenizer,
    # tokenizer_file="tokenizer.json", # B\u1EA1n c\xF3 th\u1EC3 t\u1EA3i t\u1EEB t\u1EC7p tokenizer
    unk_token="[UNK]",
    pad_token="[PAD]",
    cls_token="[CLS]",
    sep_token="[SEP]",
    mask_token="[MASK]",
)`,highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> PreTrainedTokenizerFast

wrapped_tokenizer = PreTrainedTokenizerFast(
    tokenizer_object=tokenizer,
    <span class="hljs-comment"># tokenizer_file=&quot;tokenizer.json&quot;, # B\u1EA1n c\xF3 th\u1EC3 t\u1EA3i t\u1EEB t\u1EC7p tokenizer</span>
    unk_token=<span class="hljs-string">&quot;[UNK]&quot;</span>,
    pad_token=<span class="hljs-string">&quot;[PAD]&quot;</span>,
    cls_token=<span class="hljs-string">&quot;[CLS]&quot;</span>,
    sep_token=<span class="hljs-string">&quot;[SEP]&quot;</span>,
    mask_token=<span class="hljs-string">&quot;[MASK]&quot;</span>,
)`}}),pn=new v({props:{code:`from transformers import BertTokenizerFast

wrapped_tokenizer = BertTokenizerFast(tokenizer_object=tokenizer)`,highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> BertTokenizerFast

wrapped_tokenizer = BertTokenizerFast(tokenizer_object=tokenizer)`}}),mn=new ni({}),un=new v({props:{code:"tokenizer = Tokenizer(models.BPE())",highlighted:"tokenizer = Tokenizer(models.BPE())"}}),kn=new v({props:{code:"tokenizer.pre_tokenizer = pre_tokenizers.ByteLevel(add_prefix_space=False)",highlighted:'tokenizer.pre_tokenizer = pre_tokenizers.ByteLevel(add_prefix_space=<span class="hljs-literal">False</span>)'}}),fn=new v({props:{code:`tokenizer.pre_tokenizer.pre_tokenize_str("Let's test pre-tokenization!")`,highlighted:'tokenizer.pre_tokenizer.pre_tokenize_str(<span class="hljs-string">&quot;Let&#x27;s test pre-tokenization!&quot;</span>)'}}),gn=new v({props:{code:`[('Let', (0, 3)), ("'s", (3, 5)), ('\u0120test', (5, 10)), ('\u0120pre', (10, 14)), ('-', (14, 15)),
 ('tokenization', (15, 27)), ('!', (27, 28))]`,highlighted:`[(<span class="hljs-string">&#x27;Let&#x27;</span>, (<span class="hljs-number">0</span>, <span class="hljs-number">3</span>)), (<span class="hljs-string">&quot;&#x27;s&quot;</span>, (<span class="hljs-number">3</span>, <span class="hljs-number">5</span>)), (<span class="hljs-string">&#x27;\u0120test&#x27;</span>, (<span class="hljs-number">5</span>, <span class="hljs-number">10</span>)), (<span class="hljs-string">&#x27;\u0120pre&#x27;</span>, (<span class="hljs-number">10</span>, <span class="hljs-number">14</span>)), (<span class="hljs-string">&#x27;-&#x27;</span>, (<span class="hljs-number">14</span>, <span class="hljs-number">15</span>)),
 (<span class="hljs-string">&#x27;tokenization&#x27;</span>, (<span class="hljs-number">15</span>, <span class="hljs-number">27</span>)), (<span class="hljs-string">&#x27;!&#x27;</span>, (<span class="hljs-number">27</span>, <span class="hljs-number">28</span>))]`}}),dn=new v({props:{code:`trainer = trainers.BpeTrainer(vocab_size=25000, special_tokens=["<|endoftext|>"])
tokenizer.train_from_iterator(get_training_corpus(), trainer=trainer)`,highlighted:`trainer = trainers.BpeTrainer(vocab_size=<span class="hljs-number">25000</span>, special_tokens=[<span class="hljs-string">&quot;&lt;|endoftext|&gt;&quot;</span>])
tokenizer.train_from_iterator(get_training_corpus(), trainer=trainer)`}}),vn=new v({props:{code:`tokenizer.model = models.BPE()
tokenizer.train(["wikitext-2.txt"], trainer=trainer)`,highlighted:`tokenizer.model = models.BPE()
tokenizer.train([<span class="hljs-string">&quot;wikitext-2.txt&quot;</span>], trainer=trainer)`}}),_n=new v({props:{code:`encoding = tokenizer.encode("Let's test this tokenizer.")
print(encoding.tokens)`,highlighted:`encoding = tokenizer.encode(<span class="hljs-string">&quot;Let&#x27;s test this tokenizer.&quot;</span>)
<span class="hljs-built_in">print</span>(encoding.tokens)`}}),$n=new v({props:{code:`['L', 'et', "'", 's', '\u0120test', '\u0120this', '\u0120to', 'ken', 'izer', '.']`,highlighted:'[<span class="hljs-string">&#x27;L&#x27;</span>, <span class="hljs-string">&#x27;et&#x27;</span>, <span class="hljs-string">&quot;&#x27;&quot;</span>, <span class="hljs-string">&#x27;s&#x27;</span>, <span class="hljs-string">&#x27;\u0120test&#x27;</span>, <span class="hljs-string">&#x27;\u0120this&#x27;</span>, <span class="hljs-string">&#x27;\u0120to&#x27;</span>, <span class="hljs-string">&#x27;ken&#x27;</span>, <span class="hljs-string">&#x27;izer&#x27;</span>, <span class="hljs-string">&#x27;.&#x27;</span>]'}}),zn=new v({props:{code:"tokenizer.post_processor = processors.ByteLevel(trim_offsets=False)",highlighted:'tokenizer.post_processor = processors.ByteLevel(trim_offsets=<span class="hljs-literal">False</span>)'}}),bn=new v({props:{code:`sentence = "Let's test this tokenizer."
encoding = tokenizer.encode(sentence)
start, end = encoding.offsets[4]
sentence[start:end]`,highlighted:`sentence = <span class="hljs-string">&quot;Let&#x27;s test this tokenizer.&quot;</span>
encoding = tokenizer.encode(sentence)
start, end = encoding.offsets[<span class="hljs-number">4</span>]
sentence[start:end]`}}),En=new v({props:{code:"' test'",highlighted:'<span class="hljs-string">&#x27; test&#x27;</span>'}}),xn=new v({props:{code:"tokenizer.decoder = decoders.ByteLevel()",highlighted:"tokenizer.decoder = decoders.ByteLevel()"}}),jn=new v({props:{code:"tokenizer.decode(encoding.ids)",highlighted:"tokenizer.decode(encoding.ids)"}}),yn=new v({props:{code:`"Let's test this tokenizer."`,highlighted:'<span class="hljs-string">&quot;Let&#x27;s test this tokenizer.&quot;</span>'}}),Tn=new v({props:{code:`from transformers import PreTrainedTokenizerFast

wrapped_tokenizer = PreTrainedTokenizerFast(
    tokenizer_object=tokenizer,
    bos_token="<|endoftext|>",
    eos_token="<|endoftext|>",
)`,highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> PreTrainedTokenizerFast

wrapped_tokenizer = PreTrainedTokenizerFast(
    tokenizer_object=tokenizer,
    bos_token=<span class="hljs-string">&quot;&lt;|endoftext|&gt;&quot;</span>,
    eos_token=<span class="hljs-string">&quot;&lt;|endoftext|&gt;&quot;</span>,
)`}}),qn=new v({props:{code:`from transformers import GPT2TokenizerFast

wrapped_tokenizer = GPT2TokenizerFast(tokenizer_object=tokenizer)`,highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> GPT2TokenizerFast

wrapped_tokenizer = GPT2TokenizerFast(tokenizer_object=tokenizer)`}}),Pn=new ni({}),wn=new v({props:{code:"tokenizer = Tokenizer(models.Unigram())",highlighted:"tokenizer = Tokenizer(models.Unigram())"}}),Cn=new v({props:{code:`from tokenizers import Regex

tokenizer.normalizer = normalizers.Sequence(
    [
        normalizers.Replace("\`\`", '"'),
        normalizers.Replace("''", '"'),
        normalizers.NFKD(),
        normalizers.StripAccents(),
        normalizers.Replace(Regex(" {2,}"), " "),
    ]
)`,highlighted:`<span class="hljs-keyword">from</span> tokenizers <span class="hljs-keyword">import</span> Regex

tokenizer.normalizer = normalizers.<span class="hljs-type">Sequence</span>(
    [
        normalizers.Replace(<span class="hljs-string">&quot;\`\`&quot;</span>, <span class="hljs-string">&#x27;&quot;&#x27;</span>),
        normalizers.Replace(<span class="hljs-string">&quot;&#x27;&#x27;&quot;</span>, <span class="hljs-string">&#x27;&quot;&#x27;</span>),
        normalizers.NFKD(),
        normalizers.StripAccents(),
        normalizers.Replace(Regex(<span class="hljs-string">&quot; {2,}&quot;</span>), <span class="hljs-string">&quot; &quot;</span>),
    ]
)`}}),Dn=new v({props:{code:"tokenizer.pre_tokenizer = pre_tokenizers.Metaspace()",highlighted:"tokenizer.pre_tokenizer = pre_tokenizers.Metaspace()"}}),On=new v({props:{code:`tokenizer.pre_tokenizer.pre_tokenize_str("Let's test the pre-tokenizer!")`,highlighted:'tokenizer.pre_tokenizer.pre_tokenize_str(<span class="hljs-string">&quot;Let&#x27;s test the pre-tokenizer!&quot;</span>)'}}),Ln=new v({props:{code:`[("\u2581Let's", (0, 5)), ('\u2581test', (5, 10)), ('\u2581the', (10, 14)), ('\u2581pre-tokenizer!', (14, 29))]`,highlighted:'[(<span class="hljs-string">&quot;\u2581Let&#x27;s&quot;</span>, (<span class="hljs-number">0</span>, <span class="hljs-number">5</span>)), (<span class="hljs-string">&#x27;\u2581test&#x27;</span>, (<span class="hljs-number">5</span>, <span class="hljs-number">10</span>)), (<span class="hljs-string">&#x27;\u2581the&#x27;</span>, (<span class="hljs-number">10</span>, <span class="hljs-number">14</span>)), (<span class="hljs-string">&#x27;\u2581pre-tokenizer!&#x27;</span>, (<span class="hljs-number">14</span>, <span class="hljs-number">29</span>))]'}}),Sn=new v({props:{code:`special_tokens = ["<cls>", "<sep>", "<unk>", "<pad>", "<mask>", "<s>", "</s>"]
trainer = trainers.UnigramTrainer(
    vocab_size=25000, special_tokens=special_tokens, unk_token="<unk>"
)
tokenizer.train_from_iterator(get_training_corpus(), trainer=trainer)`,highlighted:`special_tokens = [<span class="hljs-string">&quot;&lt;cls&gt;&quot;</span>, <span class="hljs-string">&quot;&lt;sep&gt;&quot;</span>, <span class="hljs-string">&quot;&lt;unk&gt;&quot;</span>, <span class="hljs-string">&quot;&lt;pad&gt;&quot;</span>, <span class="hljs-string">&quot;&lt;mask&gt;&quot;</span>, <span class="hljs-string">&quot;&lt;s&gt;&quot;</span>, <span class="hljs-string">&quot;&lt;/s&gt;&quot;</span>]
trainer = trainers.UnigramTrainer(
    vocab_size=<span class="hljs-number">25000</span>, special_tokens=special_tokens, unk_token=<span class="hljs-string">&quot;&lt;unk&gt;&quot;</span>
)
tokenizer.train_from_iterator(get_training_corpus(), trainer=trainer)`}}),Bn=new v({props:{code:`tokenizer.model = models.Unigram()
tokenizer.train(["wikitext-2.txt"], trainer=trainer)`,highlighted:`tokenizer.model = models.Unigram()
tokenizer.train([<span class="hljs-string">&quot;wikitext-2.txt&quot;</span>], trainer=trainer)`}}),Nn=new v({props:{code:`encoding = tokenizer.encode("Let's test this tokenizer.")
print(encoding.tokens)`,highlighted:`encoding = tokenizer.encode(<span class="hljs-string">&quot;Let&#x27;s test this tokenizer.&quot;</span>)
<span class="hljs-built_in">print</span>(encoding.tokens)`}}),Fn=new v({props:{code:`['\u2581Let', "'", 's', '\u2581test', '\u2581this', '\u2581to', 'ken', 'izer', '.']`,highlighted:'[<span class="hljs-string">&#x27;\u2581Let&#x27;</span>, <span class="hljs-string">&quot;&#x27;&quot;</span>, <span class="hljs-string">&#x27;s&#x27;</span>, <span class="hljs-string">&#x27;\u2581test&#x27;</span>, <span class="hljs-string">&#x27;\u2581this&#x27;</span>, <span class="hljs-string">&#x27;\u2581to&#x27;</span>, <span class="hljs-string">&#x27;ken&#x27;</span>, <span class="hljs-string">&#x27;izer&#x27;</span>, <span class="hljs-string">&#x27;.&#x27;</span>]'}}),An=new v({props:{code:`cls_token_id = tokenizer.token_to_id("<cls>")
sep_token_id = tokenizer.token_to_id("<sep>")
print(cls_token_id, sep_token_id)`,highlighted:`cls_token_id = tokenizer.token_to_id(<span class="hljs-string">&quot;&lt;cls&gt;&quot;</span>)
sep_token_id = tokenizer.token_to_id(<span class="hljs-string">&quot;&lt;sep&gt;&quot;</span>)
<span class="hljs-built_in">print</span>(cls_token_id, sep_token_id)`}}),Wn=new v({props:{code:"0 1",highlighted:'<span class="hljs-number">0</span> <span class="hljs-number">1</span>'}}),Un=new v({props:{code:`tokenizer.post_processor = processors.TemplateProcessing(
    single="$A:0 <sep>:0 <cls>:2",
    pair="$A:0 <sep>:0 $B:1 <sep>:1 <cls>:2",
    special_tokens=[("<sep>", sep_token_id), ("<cls>", cls_token_id)],
)`,highlighted:`tokenizer.post_processor = processors.TemplateProcessing(
    single=<span class="hljs-string">&quot;$A:0 &lt;sep&gt;:0 &lt;cls&gt;:2&quot;</span>,
    pair=<span class="hljs-string">&quot;$A:0 &lt;sep&gt;:0 $B:1 &lt;sep&gt;:1 &lt;cls&gt;:2&quot;</span>,
    special_tokens=[(<span class="hljs-string">&quot;&lt;sep&gt;&quot;</span>, sep_token_id), (<span class="hljs-string">&quot;&lt;cls&gt;&quot;</span>, cls_token_id)],
)`}}),Rn=new v({props:{code:`encoding = tokenizer.encode("Let's test this tokenizer...", "on a pair of sentences!")
print(encoding.tokens)
print(encoding.type_ids)`,highlighted:`encoding = tokenizer.encode(<span class="hljs-string">&quot;Let&#x27;s test this tokenizer...&quot;</span>, <span class="hljs-string">&quot;on a pair of sentences!&quot;</span>)
<span class="hljs-built_in">print</span>(encoding.tokens)
<span class="hljs-built_in">print</span>(encoding.type_ids)`}}),Gn=new v({props:{code:`['\u2581Let', "'", 's', '\u2581test', '\u2581this', '\u2581to', 'ken', 'izer', '.', '.', '.', '<sep>', '\u2581', 'on', '\u2581', 'a', '\u2581pair', 
  '\u2581of', '\u2581sentence', 's', '!', '<sep>', '<cls>']
[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2]`,highlighted:`[<span class="hljs-string">&#x27;\u2581Let&#x27;</span>, <span class="hljs-string">&quot;&#x27;&quot;</span>, <span class="hljs-string">&#x27;s&#x27;</span>, <span class="hljs-string">&#x27;\u2581test&#x27;</span>, <span class="hljs-string">&#x27;\u2581this&#x27;</span>, <span class="hljs-string">&#x27;\u2581to&#x27;</span>, <span class="hljs-string">&#x27;ken&#x27;</span>, <span class="hljs-string">&#x27;izer&#x27;</span>, <span class="hljs-string">&#x27;.&#x27;</span>, <span class="hljs-string">&#x27;.&#x27;</span>, <span class="hljs-string">&#x27;.&#x27;</span>, <span class="hljs-string">&#x27;&lt;sep&gt;&#x27;</span>, <span class="hljs-string">&#x27;\u2581&#x27;</span>, <span class="hljs-string">&#x27;on&#x27;</span>, <span class="hljs-string">&#x27;\u2581&#x27;</span>, <span class="hljs-string">&#x27;a&#x27;</span>, <span class="hljs-string">&#x27;\u2581pair&#x27;</span>, 
  <span class="hljs-string">&#x27;\u2581of&#x27;</span>, <span class="hljs-string">&#x27;\u2581sentence&#x27;</span>, <span class="hljs-string">&#x27;s&#x27;</span>, <span class="hljs-string">&#x27;!&#x27;</span>, <span class="hljs-string">&#x27;&lt;sep&gt;&#x27;</span>, <span class="hljs-string">&#x27;&lt;cls&gt;&#x27;</span>]
[<span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">2</span>]`}}),Mn=new v({props:{code:"tokenizer.decoder = decoders.Metaspace()",highlighted:"tokenizer.decoder = decoders.Metaspace()"}}),In=new v({props:{code:`from transformers import PreTrainedTokenizerFast

wrapped_tokenizer = PreTrainedTokenizerFast(
    tokenizer_object=tokenizer,
    bos_token="<s>",
    eos_token="</s>",
    unk_token="<unk>",
    pad_token="<pad>",
    cls_token="<cls>",
    sep_token="<sep>",
    mask_token="<mask>",
    padding_side="left",
)`,highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> PreTrainedTokenizerFast

wrapped_tokenizer = PreTrainedTokenizerFast(
    tokenizer_object=tokenizer,
    bos_token=<span class="hljs-string">&quot;&lt;s&gt;&quot;</span>,
    eos_token=<span class="hljs-string">&quot;&lt;/s&gt;&quot;</span>,
    unk_token=<span class="hljs-string">&quot;&lt;unk&gt;&quot;</span>,
    pad_token=<span class="hljs-string">&quot;&lt;pad&gt;&quot;</span>,
    cls_token=<span class="hljs-string">&quot;&lt;cls&gt;&quot;</span>,
    sep_token=<span class="hljs-string">&quot;&lt;sep&gt;&quot;</span>,
    mask_token=<span class="hljs-string">&quot;&lt;mask&gt;&quot;</span>,
    padding_side=<span class="hljs-string">&quot;left&quot;</span>,
)`}}),Hn=new v({props:{code:`from transformers import XLNetTokenizerFast

wrapped_tokenizer = XLNetTokenizerFast(tokenizer_object=tokenizer)`,highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> XLNetTokenizerFast

wrapped_tokenizer = XLNetTokenizerFast(tokenizer_object=tokenizer)`}}),{c(){$=i("meta"),it=h(),N=i("h1"),W=i("a"),Q=i("span"),m(tt.$$.fragment),he=h(),et=i("span"),pe=s("X\xE2y d\u1EF1ng t\u1EEBng kh\u1ED1i tokenizer"),Pt=h(),m(F.$$.fragment),wt=h(),ct=i("p"),xt=s("Nh\u01B0 ch\xFAng ta \u0111\xE3 th\u1EA5y trong c\xE1c ph\u1EA7n tr\u01B0\u1EDBc, tokenize bao g\u1ED3m m\u1ED9t s\u1ED1 b\u01B0\u1EDBc:"),Ct=h(),T=i("ul"),nt=i("li"),me=s("Chu\u1EA9n h\xF3a (m\u1ECDi thao t\xE1c d\u1ECDn d\u1EB9p v\u0103n b\u1EA3n \u0111\u01B0\u1EE3c cho l\xE0 c\u1EA7n thi\u1EBFt, ch\u1EB3ng h\u1EA1n nh\u01B0 x\xF3a d\u1EA5u c\xE1ch ho\u1EB7c d\u1EA5u, chu\u1EA9n h\xF3a Unicode, v.v.)"),ue=h(),st=i("li"),ke=s("Ti\u1EC1n tokenize (chia nh\u1ECF \u0111\u1EA7u v\xE0o th\xE0nh c\xE1c t\u1EEB)"),fe=h(),ot=i("li"),ge=s("\u0110\u01B0a \u0111\u1EA7u v\xE0o th\xF4ng qua m\xF4 h\xECnh (s\u1EED d\u1EE5ng c\xE1c t\u1EEB \u0111\u01B0\u1EE3c ti\u1EC1n tokenize \u0111\u1EC3 t\u1EA1o ra m\u1ED9t chu\u1ED7i token)"),de=h(),rt=i("li"),z=s("H\u1EADu x\u1EED l\xFD (th\xEAm token \u0111\u1EB7c bi\u1EC7t c\u1EE7a tr\xECnh tokenize, t\u1EA1o attention mask v\xE0 ID token)"),ve=h(),U=i("div"),lt=i("img"),Qn=h(),at=i("img"),_e=h(),Dt=i("p"),ma=s("Th\u01B0 vi\u1EC7n \u{1F917} Tokenizers \u0111\xE3 \u0111\u01B0\u1EE3c x\xE2y d\u1EF1ng \u0111\u1EC3 cung c\u1EA5p nhi\u1EC1u s\u1EF1 l\u1EF1a ch\u1ECDn cho c\xE1c b\u01B0\u1EDBc n\xE0y, v\xE0 ta c\xF3 th\u1EC3 k\u1EBFt h\u1EE3p v\xE0 n\u1ED1i ch\xFAng v\u1EDBi nhau. Trong ph\u1EA7n n\xE0y, ch\xFAng ta s\u1EBD xem c\xE1c c\xF3 th\u1EC3 x\xE2y m\u1ED9t tokenizer t\u1EEB \u0111\u1EA7u, tr\xE1i ng\u01B0\u1EE3c v\u1EDBi c\xE1ch hu\u1EA5n luy\u1EC7n m\u1ED9t tokenizer m\u1EDBi t\u1EEB c\xE1i c\u0169 nh\u01B0 ta \u0111\xE3 l\xE0m \u1EDF "),ts=i("a"),ua=s("ph\u1EA7n 2"),ka=s(". Ch\xFAng ta s\u1EBD c\xF3 th\u1EC3 x\xE2y b\u1EA5t k\xEC ki\u1EC3u tokenizer n\xE0o ta c\xF3 th\u1EC3 ngh\u0129 ra!"),oi=h(),m($e.$$.fragment),ri=h(),Ot=i("p"),fa=s("Ch\xEDnh x\xE1c h\u01A1n, th\u01B0 vi\u1EC7n \u0111\u01B0\u1EE3c x\xE2y d\u1EF1ng t\u1EADp trung v\xE0o l\u1EDBp "),Rs=i("code"),ga=s("Tokenizer"),da=s(" v\u1EDBi c\xE1c kh\u1ED1i \u0111\u01B0\u1EE3c t\u1EADp h\u1EE3p l\u1EA1i trong c\xE1c m\xF4-\u0111un con:"),ii=h(),q=i("ul"),ht=i("li"),Gs=i("code"),va=s("normalizers"),_a=s(" ch\u1EE9a t\u1EA5t c\u1EA3 c\xE1c ki\u1EC3u "),Ms=i("code"),$a=s("Normalizer"),za=s(" b\u1EA1n c\xF3 th\u1EC3 s\u1EED d\u1EE5ng (ho\xE0n thi\u1EC7n danh s\xE1ch t\u1EA1i "),ze=i("a"),ba=s("\u0111\xE2y"),Ea=s(")."),xa=h(),pt=i("li"),Is=i("code"),ja=s("pre_tokenizers"),ya=s(" ch\u1EE9a t\u1EA5t c\u1EA3 c\xE1c ki\u1EC3u "),Hs=i("code"),Ta=s("PreTokenizer"),qa=s(" b\u1EA1n c\xF3 th\u1EC3 s\u1EED d\u1EE5ng (ho\xE0n thi\u1EC7n danh s\xE1ch t\u1EA1i "),be=i("a"),Pa=s("\u0111\xE2y"),wa=s(")."),Ca=h(),P=i("li"),Xs=i("code"),Da=s("models"),Oa=s(" ch\u1EE9a t\u1EA5t c\u1EA3 c\xE1c ki\u1EC3u "),Vs=i("code"),La=s("Model"),Sa=s(" b\u1EA1n c\xF3 th\u1EC3 s\u1EED d\u1EE5ng, nh\u01B0 "),Ks=i("code"),Ba=s("BPE"),Na=s(", "),Js=i("code"),Fa=s("WordPiece"),Aa=s(", and "),Ys=i("code"),Wa=s("Unigram"),Ua=s(" (ho\xE0n thi\u1EC7n danh s\xE1ch t\u1EA1i "),Ee=i("a"),Ra=s("\u0111\xE2y"),Ga=s(")."),Ma=h(),mt=i("li"),Zs=i("code"),Ia=s("trainers"),Ha=s(" ch\u1EE9a t\u1EA5t c\u1EA3 c\xE1c ki\u1EC3u "),Qs=i("code"),Xa=s("Trainer"),Va=s(" kh\xE1c nhau b\u1EA1n c\xF3 th\u1EC3 s\u1EED d\u1EE5ng \u0111\u1EC3 hu\u1EA5n luy\u1EC7n m\xF4 h\xECnh c\u1EE7a b\u1EA1n tr\xEAn kho ng\u1EEF li\u1EC7u (m\u1ED9t cho m\u1ED7i lo\u1EA1i m\xF4 h\xECnh; ho\xE0n thi\u1EC7n danh s\xE1ch t\u1EA1i "),xe=i("a"),Ka=s("\u0111\xE2y"),Ja=s(")."),Ya=h(),ut=i("li"),to=i("code"),Za=s("post_processors"),Qa=s(" ch\u1EE9a t\u1EA5t c\u1EA3 c\xE1c ki\u1EC3u "),eo=i("code"),th=s("PostProcessor"),eh=s(" b\u1EA1n c\xF3 th\u1EC3 s\u1EED d\u1EE5ng (ho\xE0n thi\u1EC7n danh s\xE1ch t\u1EA1i "),je=i("a"),nh=s("\u0111\xE2y"),sh=s(")."),oh=h(),kt=i("li"),no=i("code"),rh=s("decoders"),ih=s(" ch\u1EE9a t\u1EA5t c\u1EA3 c\xE1c ki\u1EC3u "),so=i("code"),ch=s("Decoder"),lh=s(" \u0111a d\u1EA1ng b\u1EA1n c\xF3 th\u1EC3 s\u1EED d\u1EE5ng \u0111\u1EC3 gi\u1EA3i m\xE3 \u0111\u1EA7u ra c\u1EE7a tokenize (ho\xE0n thi\u1EC7n danh s\xE1ch t\u1EA1i "),ye=i("a"),ah=s("\u0111\xE2y"),hh=s(")."),ci=h(),Lt=i("p"),ph=s("B\u1EA1n c\xF3 th\u1EC3 t\xECm \u0111\u01B0\u1EE3c to\xE0n b\u1ED9 danh s\xE1ch c\xE1c kh\u1ED1i t\u1EA1i "),Te=i("a"),mh=s("\u0111\xE2y"),uh=s("."),li=h(),jt=i("h2"),St=i("a"),oo=i("span"),m(qe.$$.fragment),kh=h(),ro=i("span"),fh=s("Thu th\u1EADp m\u1ED9t kho ng\u1EEF li\u1EC7u"),ai=h(),ft=i("p"),gh=s("\u0110\u1EC3 hu\u1EA5n luy\u1EC7n tokenizer m\u1EDBi c\u1EE7a m\xECnh, ch\xFAng ta s\u1EBD s\u1EED d\u1EE5ng m\u1ED9t kho ng\u1EEF li\u1EC7u nh\u1ECF ch\u1EE9a c\xE1c \u0111o\u1EA1n v\u0103n (cho nhanh). C\xE1c b\u01B0\u1EDBc \u0111\u1EC3 c\xF3 \u0111\u01B0\u1EE3c kho ng\u1EEF li\u1EC7u t\u01B0\u01A1ng t\u1EF1 nh\u01B0 ch\xFAng ta \u0111\xE3 l\xE0m \u1EDF "),es=i("a"),dh=s("ph\u1EA7n \u0111\u1EA7u c\u1EE7a ch\u01B0\u01A1ng n\xE0y"),vh=s(", nh\u01B0ng l\u1EA7n n\xE0y ch\xFAng ta s\u1EBD s\u1EED d\u1EE5ng "),Pe=i("a"),_h=s("WikiText-2"),$h=s(":"),hi=h(),m(we.$$.fragment),pi=h(),Bt=i("p"),zh=s("H\xE0m "),io=i("code"),bh=s("get_training_corpus()"),Eh=s(" l\xE0 m\u1ED9t h\xE0m t\u1EA1o c\xF3 th\u1EC3 tr\u1EA3 v\u1EC1 c\xE1c l\xF4 v\u1EDBi m\u1ED7i l\xF4 l\xE0 1,000 \u0111o\u1EA1n v\u0103n, c\xE1i m\xE0 ta s\u1EBD s\u1EED d\u1EE5ng \u0111\u1EC3 hu\u1EA5n luy\u1EC7n  tokenizer."),mi=h(),ns=i("p"),xh=s("\u{1F917} Tokenizers c\u0169ng c\xF3 th\u1EC3 \u0111\u01B0\u1EE3c hu\u1EA5n luy\u1EC7n tr\u1EF1c ti\u1EBFp tr\xEAn c\xE1c t\u1EC7p v\u0103n b\u1EA3n. \u0110\xE2y l\xE0 c\xE1ch ch\xFAng ta t\u1EA1o ra m\u1ED9t t\u1EC7p v\u0103n b\u1EA3n bao g\u1ED3m c\xE1c \u0111o\u1EA1n v\u0103n/\u0111\u1EA7u v\xE0o t\u1EEB WikiText-2 m\xE0 ta c\xF3 th\u1EC3 s\u1EED d\u1EE5ng c\u1EE5c b\u1ED9:"),ui=h(),m(Ce.$$.fragment),ki=h(),ss=i("p"),jh=s("Ti\u1EBFp theo ch\xFAng t\xF4i s\u1EBD h\u01B0\u1EDBng d\u1EABn b\u1EA1n c\xE1ch t\u1EF1 x\xE2y d\u1EF1ng t\u1EEBng kh\u1ED1i BERT, GPT-2, v\xE0 XLNet tokenizer c\u1EE7a ri\xEAng m\xECnh. \u0110i\u1EC1u n\xE0y s\u1EBD cung c\u1EA5p cho ch\xFAng ta m\u1ED9t v\xED d\u1EE5 v\u1EC1 t\u1EEBng thu\u1EADt to\xE1n trong s\u1ED1 ba thu\u1EADt to\xE1n tokenize ch\xEDnh: WordPiece, BPE, v\xE0 Unigram. H\xE3y c\u0169ng b\u1EAFt \u0111\u1EA7u v\u1EDBi BERT!"),fi=h(),yt=i("h2"),Nt=i("a"),co=i("span"),m(De.$$.fragment),yh=h(),lo=i("span"),Th=s("X\xE2y d\u1EF1ng m\u1ED9t WordPiece tokenizer t\u1EEB \u0111\u1EA7u"),gi=h(),x=i("p"),qh=s("\u0110\u1EC3 x\xE2y d\u1EF1ng m\u1ED9t tokenizer v\u1EDBi th\u01B0 vi\u1EC7n \u{1F917} Tokenizers, ch\xFAng ta s\u1EBD b\u1EAFt \u0111\u1EA7u v\u1EDBi vi\u1EC7c kh\u1EDFi t\u1EA1o m\u1ED9t \u0111\u1ED1i t\u01B0\u1EE3ng "),ao=i("code"),Ph=s("Tokenizer"),wh=s(" v\u1EDBi "),ho=i("code"),Ch=s("model"),Dh=s(", sau \u0111\xF3 thi\u1EBFt l\u1EADp "),po=i("code"),Oh=s("normalizer"),Lh=s(", "),mo=i("code"),Sh=s("pre_tokenizer"),Bh=s(", "),uo=i("code"),Nh=s("post_processor"),Fh=s(", v\xE0 "),ko=i("code"),Ah=s("decoder"),Wh=s(" t\u1EDBi c\xE1c gi\xE1 tr\u1ECB ta mu\u1ED1n."),di=h(),Ft=i("p"),Uh=s("V\u1EDBi v\xED d\u1EE5 n\xE0y, ta s\u1EBD t\u1EA1o ra m\u1ED9t "),fo=i("code"),Rh=s("Tokenizer"),Gh=s(" v\u1EDBi m\u1ED9t m\xF4 h\xECnh WordPiece:"),vi=h(),m(Oe.$$.fragment),_i=h(),R=i("p"),Mh=s("Ch\xFAng ta ph\u1EA3i ch\u1EC9 r\xF5 "),go=i("code"),Ih=s("unk_token"),Hh=s(" \u0111\u1EC3 m\xF4 h\xECnh bi\u1EBFt ph\u1EA3i tr\u1EA3 v\u1EC1 g\xEC khi g\u1EB7p c\xE1c k\xED t\u1EF1 ch\u01B0a t\u1EEBng g\u1EB7p tr\u01B0\u1EDBc \u0111\xF3. C\xE1c tham s\u1ED1 kh\xE1c ch\xFAng ta c\xF3 th\u1EC3 c\xE0i \u0111\u1EB7t g\u1ED3m "),vo=i("code"),Xh=s("vocab"),Vh=s(" c\u1EE7a m\xF4 h\xECnh (ta s\u1EBD hu\u1EA5n luy\u1EC7n m\xF4 h\xECnh n\xEAn kh\xF4ng c\u1EA7n thi\u1EBFt l\u1EADp n\xF3) v\xE0 "),_o=i("code"),Kh=s("max_input_chars_per_word"),Jh=s(", t\u01B0\u01A1ng \u1EE9ng \u0111\u1ED9 d\xE0i t\u1ED1i \u0111a cho m\u1ED9t t\u1EEB (t\u1EEB d\xE0i h\u01A1n gi\xE1 tr\u1ECB n\xE0y s\u1EBD b\u1ECB chia nh\u1ECF)"),$i=h(),j=i("p"),Yh=s("B\u01B0\u1EDBc \u0111\u1EA7u ti\xEAn \u0111\u1EC3 tokenize \u0111\xF3 l\xE0 chu\u1EA9n ho\xE1, v\xEC v\u1EADy h\xE3y c\u0169ng b\u1EAFt \u0111\u1EA7u v\u1EDBi b\u01B0\u1EDBc n\xE0y. V\xEC BERT \u0111\u01B0\u1EE3c s\u1EED d\u1EE5ng r\u1ED9ng t\xE3i, ta c\xF3 th\u1EC3 s\u1EED d\u1EE5ng "),$o=i("code"),Zh=s("BertNormalizer"),Qh=s(" v\u1EDBi tu\u1EF3 ch\u1ECDn kinh \u0111i\u1EC3n \u0111\u1EC3 thi\u1EBFt l\u1EADp cho BERT: "),zo=i("code"),tp=s("lowercase"),ep=s(" v\xE0 "),bo=i("code"),np=s("strip_accents"),sp=s(", t\u1EF1 c\xE1i t\xEAn \u0111\xE3 gi\u1EA3i th\xEDch m\u1EE5c \u0111\xEDch c\u1EE7a ch\xFAng; "),Eo=i("code"),op=s("clean_text"),rp=s(" \u0111\u1EC3 lo\u1EA1i b\u1ECF t\u1EA5t c\u1EA3 c\xE1c k\xED t\u1EF1 ki\u1EC3m so\xE1t v\xE0 d\u1EA5u c\xE1ch l\u1EB7p l\u1EA1i th\xE0nh m\u1ED9t; v\xE0 "),xo=i("code"),ip=s("handle_chinese_chars"),cp=s(" th\xEAm d\u1EA5u c\xE1ch gi\u1EEFa c\xE1c k\xED t\u1EF1 ti\u1EBFng Trung. \u0110\u1EC3 , which places spaces around Chinese characters. To t\xE1i t\u1EA1o tokenizer "),jo=i("code"),lp=s("bert-base-uncased"),ap=s(", ta c\xF3 th\u1EC3 thi\u1EBFt l\u1EADp chu\u1EA9n ho\xE1 sau:"),zi=h(),m(Le.$$.fragment),bi=h(),G=i("p"),hp=s("Th\xF4ng th\u01B0\u1EDDng, khi x\xE2y d\u1EF1ng m\u1ED9t tokenizer, b\u1EA1n kh\xF4ng c\u1EA7n ph\u1EA3i truy c\u1EADp v\xE0o m\u1ED9t h\xE0m chu\u1EA9n ho\xE1 th\u1EE7 c\xF4ng v\xEC n\xF3 \u0111\xE3 c\xF3 s\u1EB5n trong th\u01B0 vi\u1EC7n \u{1F917} Tokenizers library \u2014 tuy nhi\xEAn, h\xE3y c\xF9ng t\u1EA1o ra chu\u1EA9n ho\xE1 BERT th\u1EE7 c\xF4ng. Th\u01B0 vi\u1EC7n cung c\xE2p tr\xECnh chu\u1EA9n ho\xE1 "),yo=i("code"),pp=s("Lowercase"),mp=s(" v\xE0 "),To=i("code"),up=s("StripAccents"),kp=s(", b\u1EA1n ho\xE0n to\xE0n c\xF3 th\u1EC3 k\u1EBFt h\u1EE3p nhi\u1EC1u tr\xECnh chu\u1EA9n ho\xE1 v\u1EDBi nhau th\xF4ng qua "),qo=i("code"),fp=s("Sequence"),gp=s(":"),Ei=h(),m(Se.$$.fragment),xi=h(),gt=i("p"),dp=s("Ta c\u0169ng c\xF3 th\u1EC3 s\u1EED d\u1EE5ng chu\u1EA9n ho\xE1 Unicode "),Po=i("code"),vp=s("NFD"),_p=s(" Unicode normalizer, v\xEC n\u1EBFu kh\xF4ng chu\u1EA9n ho\xE1 "),wo=i("code"),$p=s("StripAccents"),zp=s(" s\u1EBD kh\xF4ng nh\u1EADn di\u1EC7n \u0111\u01B0\u1EE3c nh\u1EEFng k\xED t\u1EF1 c\xF3 d\u1EA5u v\xE0 kh\xF4ng th\u1EC3 t\xE1ch n\xF3 \u0111\xFAng nh\u01B0 ta mu\u1ED1n."),ji=h(),dt=i("p"),bp=s("Nh\u01B0 \u0111\xE3 th\u1EA5y \u1EDF tr\xEAn, ta c\xF3 th\u1EC3 s\u1EED d\u1EE5ng ph\u01B0\u01A1ng th\u1EE9c "),Co=i("code"),Ep=s("normalize_str()"),xp=s(" c\u1EE7a "),Do=i("code"),jp=s("normalizer"),yp=s(" \u0111\u1EC3 ki\u1EC3m tra t\xE1c \u0111\u1ED9ng c\u1EE7a n\xF3 l\xEAn m\u1ED9t chu\u1ED7i v\u0103n b\u1EA3n:"),yi=h(),m(Be.$$.fragment),Ti=h(),m(Ne.$$.fragment),qi=h(),m(At.$$.fragment),Pi=h(),Wt=i("p"),Tp=s("Ti\u1EBFp theo l\xE0 b\u01B0\u1EDBc pre-tokenization. M\u1ED9t l\u1EA7n n\u1EEFa, ta c\xF3 "),Oo=i("code"),qp=s("BertPreTokenizer"),Pp=s(" \u0111\u01B0\u1EE3c x\xE2y d\u1EF1ng s\u1EB5n \u0111\u1EC3 d\xF9ng:"),wi=h(),m(Fe.$$.fragment),Ci=h(),os=i("p"),wp=s("Ho\u1EB7c ta c\xF3 th\u1EC3 x\xE2y t\u1EEB \u0111\u1EA7u:"),Di=h(),m(Ae.$$.fragment),Oi=h(),Ut=i("p"),Cp=s("L\u01B0u \xFD r\u1EB1ng "),Lo=i("code"),Dp=s("Whitespace"),Op=s(" s\u1EBD t\xE1ch theo d\u1EA5u c\xE1ch v\xE0 c\xE1c k\xED t\u1EF1 kh\xF4ng ph\u1EA3i ch\u1EEF c\xE1i, s\u1ED1, ho\u1EB7c d\u1EA5u g\u1EA1ch d\u01B0\u1EDBi, n\xEAn v\u1EC1 m\u1EB7t k\u1EF9 thu\u1EADt n\xF3 s\u1EBD t\xE1ch theo d\u1EA5u c\xE1ch v\xE0 d\u1EA5u c\xE2u:"),Li=h(),m(We.$$.fragment),Si=h(),m(Ue.$$.fragment),Bi=h(),Rt=i("p"),Lp=s("N\xEAu sbanj ch\u1EC9 mu\u1ED1n t\xE1ch theo d\u1EA5u c\xE1ch, b\u1EA1n c\xF3 th\u1EC3 s\u1EED d\u1EE5ng "),So=i("code"),Sp=s("WhitespaceSplit"),Bp=s(" thay th\u1EBF:"),Ni=h(),m(Re.$$.fragment),Fi=h(),m(Ge.$$.fragment),Ai=h(),Gt=i("p"),Np=s("Gi\u1ED1ng nh\u01B0 chu\u1EA9n ho\xE1, b\u1EA3n c\xF3 th\u1EC3 s\u1EED d\u1EE5ng "),Bo=i("code"),Fp=s("Sequence"),Ap=s(" \u0111\u1EC3 k\u1EBFt h\u1EE3p c\xE1c ti\u1EC1n tokenizer v\u1EDBi nhau:"),Wi=h(),m(Me.$$.fragment),Ui=h(),m(Ie.$$.fragment),Ri=h(),Mt=i("p"),Wp=s("B\u01B0\u1EDBc ti\u1EBFp theo trong pipeline tokenize l\xE0 \u0111\u01B0a \u0111\u1EA7u v\xE0o qua m\xF4 h\xECnh. Ta \u0111\xE3 ch\u1EC9 \u0111\u1ECBnh m\xF4 h\xECnh c\u1EE7a m\xECnh khi kh\u1EDFi t\u1EA1o, nh\u01B0ng ta v\u1EABn c\u1EA7n hu\u1EA5n luy\u1EC7n n\xF3, \u0111i\u1EC1u n\xE0y c\u1EA7n t\u1EDBi "),No=i("code"),Up=s("WordPieceTrainer"),Rp=s(".  V\u1EA5n \u0111\u1EC1 ch\xEDnh \u1EDF \u0111\xE2y l\xE0 khi kh\u1EDFi t\u1EA1o m\u1ED9t tr\xECnh hu\u1EA5n luy\u1EC7n trong \u{1F917} Tokenizers th\xEC b\u1EA1n c\u1EA7n ph\u1EA3i truy\u1EC1n t\u1EA5t c\u1EA3 c\xE1c token \u0111\u1EB7c bi\u1EC7t b\u1EA1n c\xF3 \xFD \u0111\u1ECBnh s\u1EED d\u1EE5ng, n\u1EBFu kh\xF4ng n\xF3 s\u1EBD kh\xF4ng th\xEAm v\xE0o b\u1ED9 t\u1EEB v\u1EF1ng, v\xEC ch\xFAng kh\xF4ng c\xF3 trong kho ng\u1EEF li\u1EC7u hu\u1EA5n luy\u1EC7n:"),Gi=h(),m(He.$$.fragment),Mi=h(),w=i("p"),Gp=s("C\u0169ng nh\u01B0 vi\u1EC7c ch\u1EC9 \u0111\u1ECBnh "),Fo=i("code"),Mp=s("vocab_size"),Ip=s(" v\xE0 "),Ao=i("code"),Hp=s("special_tokens"),Xp=s(", ta c\u1EA7n thi\u1EBFt l\u1EADp "),Wo=i("code"),Vp=s("min_frequency"),Kp=s(" (s\u1ED1 l\u1EA7n m\u1ED9t token ph\u1EA3i xu\u1EA5t hi\u1EC7n \u0111\u1EC3 \u0111\u01B0\u1EE3c th\xEAm v\xE0o b\u1ED9 t\u1EEB v\u1EF1ng) ho\u1EB7c thay \u0111\u1ED5i "),Uo=i("code"),Jp=s("continuing_subword_prefix"),Yp=s(" (n\u1EBFu ta mu\u1ED1n s\u1EED d\u1EE5ng th\u1EE9 g\xEC kh\xE1c ngo\xE0i "),Ro=i("code"),Zp=s("##"),Qp=s(")."),Ii=h(),rs=i("p"),tm=s("\u0110\u1EC3 hu\u1EA5n luy\u1EC7n m\u1ED9t m\xF4 h\xECnh s\u1EED d\u1EE5ng tr\xECnh l\u1EB7p ta \u0111\u1ECBnh ngh\u0129a tr\u01B0\u1EDBc \u0111\xF3, ta ch\u1EC9 c\u1EA7n th\u1EF1c hi\u1EC7n l\u1EC7nh n\xE0y:"),Hi=h(),m(Xe.$$.fragment),Xi=h(),It=i("p"),em=s("Ch\xFAng ta c\u0169ng c\xF3 th\u1EC3 s\u1EED d\u1EE5ng c\xE1c t\u1EC7p v\u0103n b\u1EA3n \u0111\u1EC3 hu\u1EA5n luy\u1EC7n tokenizer c\u1EE7a m\xECnh nh\u01B0 sau (ta t\xE1i kh\u1EDFi t\u1EA1o m\xF4 h\xECnh v\u1EDBi m\u1ED9t "),Go=i("code"),nm=s("WordPiece"),sm=s(" r\u1ED7ng):"),Vi=h(),m(Ve.$$.fragment),Ki=h(),Ht=i("p"),om=s("Trong c\u1EA3 hai tr\u01B0\u1EDDng h\u1EE3p, ta c\xF3 th\u1EC3 ki\u1EC3m tra xem tokenizer tr\xEAn m\u1ED9t \u0111o\u1EA1n v\u0103n b\u1EA3n b\u1EB1ng c\xE1ch s\u1EED d\u1EE5ng ph\u01B0\u01A1ng th\u1EE9c "),Mo=i("code"),rm=s("encode()"),im=s(":"),Ji=h(),m(Ke.$$.fragment),Yi=h(),m(Je.$$.fragment),Zi=h(),b=i("p"),Io=i("code"),cm=s("encoding"),lm=s(" thu \u0111\u01B0\u1EE3c l\xE0 m\u1ED9t "),Ho=i("code"),am=s("Encoding"),hm=s(" g\u1ED3m t\u1EA5t c\u1EA3 c\xE1c \u0111\u1EA7u ra c\u1EA7n thi\u1EBFt c\u1EE7a m\u1ED9t tokenizer trong t\u1EA5t c\u1EA3 c\xE1c th\xF4ng s\u1ED1 \u0111a d\u1EA1ng c\u1EE7a n\xF3: "),Xo=i("code"),pm=s("ids"),mm=s(", "),Vo=i("code"),um=s("type_ids"),km=s(", "),Ko=i("code"),fm=s("tokens"),gm=s(", "),Jo=i("code"),dm=s("offsets"),vm=s(", "),Yo=i("code"),_m=s("attention_mask"),$m=s(", "),Zo=i("code"),zm=s("special_tokens_mask"),bm=s(", v\xE0 "),Qo=i("code"),Em=s("overflowing"),xm=s("."),Qi=h(),C=i("p"),jm=s("B\u01B0\u1EDBc cu\u1ED1i c\u1EE7a quy tr\xECnh \u0111\xF3 l\xE0 h\u1EADu x\u1EED l\xFD. Ta c\u1EA7n th\xEAm token "),tr=i("code"),ym=s("[CLS]"),Tm=s(" token at the beginning and the "),er=i("code"),qm=s("[SEP]"),Pm=s(" \u1EDF cu\u1ED1i (ho\u1EB7c sau m\u1ED7i c\xE2u, n\u1EBFu ta c\xF3 c\u1EB7p c\xE2u). Ch\xFAng ta s\u1EBD s\u1EED d\u1EE5ng "),nr=i("code"),wm=s("TemplateProcessor"),Cm=s(" \u0111\u1EC3 th\u1EF1c hi\u1EC7n \u0111i\u1EC1u n\xE0y, nh\u01B0ng tr\u01B0\u1EDBc h\u1EBFt ta c\u1EA7n bi\u1EBFt c\xE1c ID c\u1EE7a token "),sr=i("code"),Dm=s("[CLS]"),Om=s(" v\xE0 "),or=i("code"),Lm=s("[SEP]"),Sm=s(" trong b\u1ED9 t\u1EEB v\u1EF1ng."),tc=h(),m(Ye.$$.fragment),ec=h(),m(Ze.$$.fragment),nc=h(),M=i("p"),Bm=s("\u0110\u1EC3 vi\u1EBFt b\u1EA3n m\u1EABu cho "),rr=i("code"),Nm=s("TemplateProcessor"),Fm=s(", ch\xFAng ta ph\u1EA3i ch\u1EC9 \u0111\u1ECBnh c\xE1ch x\u1EED l\xFD m\u1ED9t c\xE2u \u0111\u01A1n v\xE0 m\u1ED9t c\u1EB7p c\xE2u. \u0110\u1ED1i v\u1EDBi c\u1EA3 hai, ch\xFAng t\xF4i vi\u1EBFt c\xE1c token \u0111\u1EB7c bi\u1EC7t mu\u1ED1n s\u1EED d\u1EE5ng; c\xE2u \u0111\u1EA7u ti\xEAn (ho\u1EB7c c\xE2u \u0111\u01A1n) \u0111\u01B0\u1EE3c bi\u1EC3u th\u1ECB b\u1EB1ng "),ir=i("code"),Am=s("$A"),Wm=s(", trong khi c\xE2u th\u1EE9 hai (n\u1EBFu token m\u1ED9t c\u1EB7p) \u0111\u01B0\u1EE3c bi\u1EC3u th\u1ECB b\u1EB1ng "),cr=i("code"),Um=s("$B"),Rm=s(". \u0110\u1ED1i v\u1EDBi m\u1ED7i lo\u1EA1i trong s\u1ED1 n\xE0y (token v\xE0 c\xE2u \u0111\u1EB7c bi\u1EC7t), ch\xFAng ta c\u0169ng ch\u1EC9 \u0111\u1ECBnh lo\u1EA1i token ID t\u01B0\u01A1ng \u1EE9ng sau d\u1EA5u hai ch\u1EA5m."),sc=h(),is=i("p"),Gm=s("Do \u0111\xF3, b\u1EA3n m\u1EABu BERT c\u1ED5 \u0111i\u1EC3n \u0111\u01B0\u1EE3c \u0111\u1ECBnh ngh\u0129a nh\u01B0 sau:"),oc=h(),m(Qe.$$.fragment),rc=h(),cs=i("p"),Mm=s("L\u01B0u \xFD r\u1EB1ng ch\xFAng ta c\u1EA7n truy\u1EC1n v\xE0o t\u1EA5t c\u1EA3 c\xE1c IDs c\u1EE7a c\xE1c k\xED t\u1EF1 \u0111\u1EB7c bi\u1EC7t, n\xEAn c\xE1c tokenize c\xF3 th\u1EC3 chuy\u1EC3n \u0111\u1ED5i ch\xFAng th\xE0nh c\xE1c c\u1EB7p ID."),ic=h(),ls=i("p"),Im=s("M\u1ED9t khi \u0111\xE3 \u0111\u01B0\u1EE3c th\xEAm v\xE0o, ch\xFAng ta c\xF3 th\u1EC3 quay l\u1EA1i v\xED d\u1EE5 tr\u01B0\u1EDBc \u0111\xF3 v\xE0 s\u1EBD nh\u1EADn \u0111\u01B0\u1EE3c:"),cc=h(),m(tn.$$.fragment),lc=h(),m(en.$$.fragment),ac=h(),as=i("p"),Hm=s("V\xE0 tr\xEAn m\u1ED9t c\u1EB7p c\xE2u, ch\xFAng ta c\xF3 th\u1EC3 c\xF3 \u0111\u01B0\u1EE3c k\u1EBFt qu\u1EA3 sau:"),hc=h(),m(nn.$$.fragment),pc=h(),m(sn.$$.fragment),mc=h(),hs=i("p"),Xm=s("Ch\xFAng ta \u0111\xE3 g\u1EA7n nh\u01B0 ho\xE0n th\xE0nh vi\u1EC7c x\xE2y d\u1EF1ng tokenizer n\xE0y t\u1EEB \u0111\u1EA7u \u2014 b\u01B0\u1EDBc cu\u1ED1i c\xF9ng l\xE0 th\xEAm v\xE0o m\u1ED9t tr\xECnh gi\u1EA3i m\xE3:"),uc=h(),m(on.$$.fragment),kc=h(),Xt=i("p"),Vm=s("H\xE3y c\u0169ng ki\u1EC3m th\u1EED v\u1EDBi "),lr=i("code"),Km=s("encoding"),Jm=s(":"),fc=h(),m(rn.$$.fragment),gc=h(),m(cn.$$.fragment),dc=h(),ps=i("p"),Ym=s("Tuy\u1EC7t v\u1EDDi! Ta c\xF3 th\u1EC3 l\u01B0u tokenizer c\u1EE7a m\xECnh v\xE0o trong m\u1ED9t t\u1EC7p JSON nh\u01B0 d\u01B0\u1EDBi \u0111\xE2y:"),vc=h(),m(ln.$$.fragment),_c=h(),vt=i("p"),Zm=s("Ta sau \u0111\xF3 c\xF3 th\u1EC3 t\u1EA3i l\u1EA1i t\u1EC7p n\xE0y trong \u0111\u1ED1i t\u01B0\u1EE3ng "),ar=i("code"),Qm=s("Tokenizer"),tu=s(" v\u1EDBi ph\u01B0\u01A1ng th\u1EE9c "),hr=i("code"),eu=s("from_file()"),nu=s(":"),$c=h(),m(an.$$.fragment),zc=h(),_t=i("p"),su=s("\u0110\u1EC3 s\u1EED d\u1EE5ng tokenizer n\xE0y trong \u{1F917} Transformers, ch\xFAng ta ph\u1EA3i b\u1ECDc n\xF3 trong "),pr=i("code"),ou=s("PreTrainedTokenizerFast"),ru=s(". Ch\xFAng ta c\xF3 th\u1EC3 s\u1EED d\u1EE5ng l\u1EDBp chung ho\u1EB7c, n\u1EBFu tokenizer c\u1EE7a ch\xFAng ta t\u01B0\u01A1ng \u1EE9ng v\u1EDBi m\u1ED9t m\xF4 h\xECnh hi\u1EC7n c\xF3, h\xE3y s\u1EED d\u1EE5ng l\u1EDBp \u0111\xF3 (\u1EDF \u0111\xE2y l\xE0 "),mr=i("code"),iu=s("BertTokenizerFast"),cu=s("). N\u1EBFu b\u1EA1n \xE1p d\u1EE5ng b\xE0i h\u1ECDc n\xE0y \u0111\u1EC3 x\xE2y d\u1EF1ng m\u1ED9t tokenizer ho\xE0n to\xE0n m\u1EDBi, b\u1EA1n s\u1EBD ph\u1EA3i s\u1EED d\u1EE5ng t\xF9y ch\u1ECDn \u0111\u1EA7u ti\xEAn."),bc=h(),D=i("p"),lu=s("\u0110\u1EC3 b\u1ECDc tokenizer trong m\u1ED9t "),ur=i("code"),au=s("PreTrainedTokenizerFast"),hu=s(", ch\xFAng ta c\xF3 th\u1EC3 chuy\u1EC3n tokenizer m\xE0 ch\xFAng ta \u0111\xE3 x\xE2y d\u1EF1ng d\u01B0\u1EDBi d\u1EA1ng "),kr=i("code"),pu=s("tokenizer_object"),mu=s(" ho\u1EB7c truy\u1EC1n t\u1EC7p tokenizer m\xE0 ch\xFAng ta \u0111\xE3 l\u01B0u d\u01B0\u1EDBi d\u1EA1ng "),fr=i("code"),uu=s("tokenizer_file"),ku=s(". \u0110i\u1EC1u quan tr\u1ECDng c\u1EA7n nh\u1EDB l\xE0 ch\xFAng ta ph\u1EA3i \u0111\u1EB7t th\u1EE7 c\xF4ng t\u1EA5t c\u1EA3 c\xE1c token \u0111\u1EB7c bi\u1EC7t, v\xEC l\u1EDBp \u0111\xF3 kh\xF4ng th\u1EC3 suy ra t\u1EEB \u0111\u1ED1i t\u01B0\u1EE3ng "),gr=i("code"),fu=s("tokenizer"),gu=s(" token n\xE0o l\xE0 token b\u1ECB che, "),dr=i("code"),du=s("[CLS]"),vu=s(", v.v.:"),Ec=h(),m(hn.$$.fragment),xc=h(),Vt=i("p"),_u=s("N\u1EBFu b\u1EA1n \u0111ang s\u1EF1 d\u1EE5ng m\u1ED9t l\u1EDBp tokenizer \u0111\u1EB7c bi\u1EC7t (nh\u01B0 "),vr=i("code"),$u=s("BertTokenizerFast"),zu=s("), b\u1EA1n ch\u1EC9 c\u1EA7n ch\u1EC9 \u0111\u1ECBnh m\u1ED9t token \u0111\u1EB7c bi\u1EBFt kh\xE1c so v\u1EDBi m\u1EB7c \u0111\u1ECBnh (\u1EDF \u0111\xE2y l\xE0 kh\xF4ng x\xE1c \u0111\u1ECBnh):"),jc=h(),m(pn.$$.fragment),yc=h(),$t=i("p"),bu=s("B\u1EA1n c\xF3 th\u1EC3 s\u1EED d\u1EE5ng tokenizer nh\u01B0 b\u1EA5t k\u1EF3 tokenizer n\xE0o kh\xE1c c\u1EE7a \u{1F917} Transformers. B\u1EA1n c\xF3 th\u1EC3 l\u01B0u n\xF3 v\u1EDBi ph\u01B0\u01A1ng th\u1EE9c "),_r=i("code"),Eu=s("save_pretrained()"),xu=s(", ho\u1EB7c l\u1EA1i n\xF3 l\xEAn Hub s\u1EED d\u1EE5ng ph\u01B0\u01A1ng th\u1EE9c "),$r=i("code"),ju=s("push_to_hub()"),yu=s("."),Tc=h(),ms=i("p"),Tu=s("Gi\u1EDD ch\xFAng ta \u0111\xE3 th\u1EA5y c\xE1ch x\xE2y d\u1EF1ng b\u1ED9 WordPiece tokenizer, h\xE3y l\xE0m t\u01B0\u01A1ng t\u1EF1 \u0111\u1ED1i v\u1EDBi BPE tokenizer. Ch\xFAng ta s\u1EBD ti\u1EBFn h\xE0nh nhanh h\u01A1n m\u1ED9t ch\xFAt v\xEC b\u1EA1n \u0111\xE3 bi\u1EBFt t\u1EA5t c\u1EA3 c\xE1c b\u01B0\u1EDBc v\xE0 ch\u1EC9 l\xE0m n\u1ED5i b\u1EADt nh\u1EEFng \u0111i\u1EC3m kh\xE1c bi\u1EC7t."),qc=h(),Tt=i("h2"),Kt=i("a"),zr=i("span"),m(mn.$$.fragment),qu=h(),br=i("span"),Pu=s("X\xE2y d\u1EF1ng m\u1ED9t BPE tokenizer t\u1EEB \u0111\u1EA7u"),Pc=h(),Jt=i("p"),wu=s("Gi\u1EDD h\xE3y c\u0169ng nhau x\xE2y d\u1EF1ng GPT-2 tokenizer. Gi\u1ED1ng nh\u01B0 BERT tokenizer, ch\xFAng ta b\u1EAFt \u0111\u1EA7u b\u1EB1ng vi\u1EC7c kh\u1EDFi t\u1EA1o "),Er=i("code"),Cu=s("Tokenizer"),Du=s(" v\u1EDBi m\xF4 h\xECnh BPE:"),wc=h(),m(un.$$.fragment),Cc=h(),I=i("p"),Ou=s("C\u0169ng gi\u1ED1ng nh\u01B0 BERT, ch\xFAng ta c\xF3 th\u1EC3 kh\u1EDFi t\u1EA1o m\xF4 h\xECnh n\xE0y v\u1EDBi m\u1ED9t b\u1ED9 t\u1EEB v\u1EF1ng n\u1EBFu ta \u0111\xE3 c\xF3 (ta s\u1EBD c\u1EA7n truy\u1EC1n v\xE0o "),xr=i("code"),Lu=s("vocab"),Su=s(" v\xE0 "),jr=i("code"),Bu=s("merges"),Nu=s(" trong tr\u01B0\u1EDDng h\u1EE3p n\xE0y), nh\u01B0ng v\xEC ta s\u1EBD hu\u1EA5n luy\u1EC7n t\u1EEB \u0111\u1EA7u, ch\xFAng ta kh\xF4ng c\u1EA7n l\xE0m v\u1EADy. Ta c\u0169ng kh\xF4ng c\u1EA7n ch\u1EC9 \u0111\u1ECBnh "),yr=i("code"),Fu=s("unk_token"),Au=s(" v\xEC GPT-2 s\u1EED d\u1EE5ng BPE c\u1EA5p byte, ph\u01B0\u01A1ng ph\xE1p kh\xF4ng c\u1EA7n \u0111\u1EBFn n\xF3."),Dc=h(),us=i("p"),Wu=s("GPT-2 kh\xF4ng s\u1EED d\u1EE5ng m\u1ED9t tr\xECnh chu\u1EA9n ho\xE1, n\xEAn ta c\xF3 th\u1EC3 b\u1ECF qua b\u01B0\u1EDBc n\xE0y v\xE0 \u0111i tr\u1EF1c ti\u1EBFp v\xE0o b\u01B0\u1EDBc pre-tokenization:"),Oc=h(),m(kn.$$.fragment),Lc=h(),Yt=i("p"),Uu=s("Tu\u1EF3 ch\u1ECDn "),Tr=i("code"),Ru=s("ByteLevel"),Gu=s(" ch\xFAng ta th\xEAm v\xE0o \u1EDF \u0111\xE2y kh\xF4ng th\xEAm d\u1EA5u c\xE1ch v\xE0o \u0111\u1EA7u c\u1EE7a m\u1ED9t c\xE2u (th\u01B0\u1EDDng n\xF3 l\xE0 m\u1EB7c \u0111\u1ECBnh). Ta c\xF3 th\u1EC3 nh\xECn c\xE1c pre-tokenization t\u1EEB v\xED d\u1EE5 t\u01B0\u01A1ng t\u1EF1 \u1EDF tr\xEAn:"),Sc=h(),m(fn.$$.fragment),Bc=h(),m(gn.$$.fragment),Nc=h(),ks=i("p"),Mu=s("Ti\u1EBFp theo l\xE0 m\xF4 h\xECnh m\xE0 ta c\u1EA7n hu\u1EA5n luy\u1EC7n. V\u1EDBi GPT-2, token \u0111\u1EB7c bi\u1EC7t duy nh\u1EA5t ta c\u1EA7n l\xE0 token k\u1EBFt th\xFAc v\u0103n b\u1EA3n:"),Fc=h(),m(dn.$$.fragment),Ac=h(),y=i("p"),Iu=s("Nh\u01B0 v\u1EDBi "),qr=i("code"),Hu=s("WordPieceTrainer"),Xu=s(", c\u0169ng nh\u01B0 "),Pr=i("code"),Vu=s("vocab_size"),Ku=s(" v\xE0 "),wr=i("code"),Ju=s("special_tokens"),Yu=s(", ta c\xF3 th\u1EC3 ch\u1EC9 \u0111\u1ECBnh "),Cr=i("code"),Zu=s("min_frequency"),Qu=s(" n\u1EBFu mu\u1ED1n, ho\u1EB7c n\u1EBFu ta c\xF3 h\u1EADu t\u1ED1 k\u1EBFt th\xFAc t\u1EEB (nh\u01B0 "),Dr=i("code"),tk=s("</w>"),ek=s("), ta c\xF3 th\u1EC3 thi\u1EBFt l\u1EADp n\xF3 v\u1EDBi "),Or=i("code"),nk=s("end_of_word_suffix"),sk=s("."),Wc=h(),fs=i("p"),ok=s("Tokenizer n\xE0y c\u0169ng c\xF3 th\u1EC3 \u0111\u01B0\u1EE3c hu\u1EA5n luy\u1EC7n tr\xEAn c\xE1c t\u1EC7p v\u0103n b\u1EA3n:"),Uc=h(),m(vn.$$.fragment),Rc=h(),gs=i("p"),rk=s("H\xE3y c\u0169ng xem k\u1EBFt qu\u1EA3 tokenize tr\xEAn m\u1ED9t v\u0103n b\u1EA3n m\u1EABu:"),Gc=h(),m(_n.$$.fragment),Mc=h(),m($n.$$.fragment),Ic=h(),ds=i("p"),ik=s("Ta \xE1p d\u1EE5ng h\u1EADu x\u1EED l\xFD c\u1EA5p byte cho GPT-2 tokenizer nh\u01B0 sau:"),Hc=h(),m(zn.$$.fragment),Xc=h(),zt=i("p"),ck=s("Tu\u1EF3 ch\u1ECDn "),Lr=i("code"),lk=s("trim_offsets = False"),ak=s(" ch\u1EC9 cho tr\xECnh h\u1EADu x\u1EED l\xFD bi\u1EBFt r\u1EB1ng ta c\u1EA7n b\u1ECF m\u1ED1t s\u1ED1 offset token b\u1EAFt \u0111\u1EA7u v\u1EDBi \u2018\u0120\u2019: theo c\xE1ch n\xE0y, \u0111i\u1EC3m b\u1EAFt \u0111\u1EA7u c\u1EE7a offset s\u1EBD tr\u1ECF v\xE0o v\xF9ng kh\xF4ng gian ph\xEDa tr\u01B0\u1EDBc c\u1EE7a t\u1EEB, kh\xF4ng ph\u1EA3i k\xED t\u1EF1 \u0111\u1EA7u ti\xEAn c\u1EE7a t\u1EEB (v\xEC kh\xF4ng gian n\xE0y v\u1EC1 m\u1EB7t k\u1EF9 thu\u1EADt l\xE0 m\u1ED9t ph\u1EA7n c\u1EE7a t\u1EEB). H\xE3y c\xF9ng nh\xECn xem k\u1EBFt qu\u1EA3 v\u1EDBi chu\u1ED7i v\u0103n b\u1EA3n ta v\u1EEBa m\xE3 ho\xE1 v\u1EDBi "),Sr=i("code"),hk=s("'\u0120test'"),pk=s(" l\xE0 token \u1EDF ch\u1EC9 m\u1EE5c 4:"),Vc=h(),m(bn.$$.fragment),Kc=h(),m(En.$$.fragment),Jc=h(),vs=i("p"),mk=s("Cu\u1ED1i c\xF9ng, ta th\xEAm m\u1ED9t tr\xECnh gi\u1EA3i m\xE3i c\u1EA5p byte:"),Yc=h(),m(xn.$$.fragment),Zc=h(),_s=i("p"),uk=s("v\xE0 ta ki\u1EC3m tra l\u1EA1i xem n\xF3 ho\u1EA1t \u0111\u1ED9ng \u0111\xFAng ch\u01B0a:"),Qc=h(),m(jn.$$.fragment),tl=h(),m(yn.$$.fragment),el=h(),bt=i("p"),kk=s("Tuy\u1EC7t v\u1EDDi! Gi\u1EDD ta \u0111\xE3 xong r\u1ED3i, ta c\xF3 th\u1EC3 l\u01B0u tokenizer nh\u01B0 tr\xEAn, v\xE0 bao n\xF3 l\u1EA1i trong "),Br=i("code"),fk=s("PreTrainedTokenizerFast"),gk=s(" ho\u1EB7c  "),Nr=i("code"),dk=s("GPT2TokenizerFast"),vk=s(" n\u1EBFu ta mu\u1ED1n n\xF3 trong \u{1F917} Transformers:"),nl=h(),m(Tn.$$.fragment),sl=h(),$s=i("p"),_k=s("or:"),ol=h(),m(qn.$$.fragment),rl=h(),zs=i("p"),$k=s("Nh\u01B0 m\u1ED9t v\xED d\u1EE5 cu\u1ED1i, ch\xFAng t\xF4i s\u1EBD ch\u1EC9 b\u1EA1n c\xE1ch x\xE2y d\u1EF1ng m\u1ED9t Unigram tokenizer t\u1EEB \u0111\u1EA7u."),il=h(),qt=i("h2"),Zt=i("a"),Fr=i("span"),m(Pn.$$.fragment),zk=h(),Ar=i("span"),bk=s("X\xE2y d\u1EF1ng m\u1ED9t Unigram tokenizer t\u1EEB \u0111\u1EA7u"),cl=h(),Qt=i("p"),Ek=s("H\xE3y c\xF9ng nhau x\xE2y d\u1EF1ng m\u1ED9t XLNet tokenizer. C\u0169ng gi\u1ED1ng nh\u01B0 c\xE1c tokenizer tr\u01B0\u1EDBc \u0111\xF3, ta c\xF3 th\u1EC3 b\u1EAFt \u0111\u1EA7u kh\u1EDFi t\u1EA1o "),Wr=i("code"),xk=s("Tokenizer"),jk=s(" v\u1EDBi m\u1ED9t m\xF4 h\xECnh Unigram:"),ll=h(),m(wn.$$.fragment),al=h(),bs=i("p"),yk=s("M\u1ED9t l\u1EA7n n\u1EEFa, ch\xFAng ta c\xF3 th\u1EC3 kh\u1EDFi t\u1EA1o m\xF4 h\xECnh n\xE0y v\u1EDBi m\u1ED9t t\u1EEB v\u1EF1ng n\u1EBFu c\xF3."),hl=h(),Es=i("p"),Tk=s("V\u1EDBi s\u1EF1 chu\u1EA9n ho\xE1 n\xE0y, XLNet s\u1EED d\u1EE5ng m\u1ED9t v\xE0i ph\u01B0\u01A1ng ph\xE1p thay th\u1EBF (\u0111\u1EBFn t\u1EEB SentencePiece):"),pl=h(),m(Cn.$$.fragment),ml=h(),H=i("p"),qk=s("\u0110i\u1EC1u n\xE0y thay th\u1EBF "),Ur=i("code"),Pk=s("\u201C"),wk=s(" and "),Rr=i("code"),Ck=s("\u201D"),Dk=s(" b\u1EB1ng "),Gr=i("code"),Ok=s("\u201D"),Lk=s(" v\xE0 thay th\u1EBF b\u1EA5t k\xEC chu\u1ED7i n\xE0o ch\u1EE9a hai ho\u1EB7c nhi\u1EC1u h\u01A1n d\u1EA5u c\xE1ch li\u1EC1n nhau th\xE0nh m\u1ED9t d\u1EA5u duy nh\u1EA5t, c\u0169ng nh\u01B0 lo\u1EA1i b\u1ECF c\xE1c d\u1EA5u c\xF3 trong v\u0103n b\u1EA3n \u0111\u1EC3 tokenize."),ul=h(),te=i("p"),Sk=s("Ti\u1EC1n tokenizer \u0111\u01B0\u1EE3c s\u1EED d\u1EE5ng cho b\u1EA5t k\u1EF3 SentencePiece tokenizer l\xE0 "),Mr=i("code"),Bk=s("Metaspace"),Nk=s(":"),kl=h(),m(Dn.$$.fragment),fl=h(),xs=i("p"),Fk=s("Ta c\xF3 th\u1EC3 nh\xECn v\xE0o \u0111\u1EA7u ra quy tr\xECnh ti\u1EC1n tokenize qua v\xED d\u1EE5 v\u0103n b\u1EA3n \u1EDF d\u01B0\u1EDBi:"),gl=h(),m(On.$$.fragment),dl=h(),m(Ln.$$.fragment),vl=h(),js=i("p"),Ak=s("Ti\u1EBFp theo l\xE0 m\xF4 h\xECnh ta c\u1EA7n hu\u1EA5n luy\u1EC7n. XLNet c\xF3 m\u1ED9t s\u1ED1 token \u0111\u1EB7c bi\u1EC7t:"),_l=h(),m(Sn.$$.fragment),$l=h(),O=i("p"),Wk=s("M\u1ED9t tham s\u1ED1 v\xF4 c\xF9ng quan trong m\xE0 ta kh\xF4ng th\u1EC3 qu\xEAn c\u1EE7a "),Ir=i("code"),Uk=s("UnigramTrainer"),Rk=s(" l\xE0 "),Hr=i("code"),Gk=s("unk_token"),Mk=s(". Ta c\xF3 th\u1EC3 truy\u1EC1n v\xE0o c\xE1c tham s\u1ED1 c\u1EE5 th\u1EC3 kh\xE1c t\u1EDBi thu\u1EADt to\xE1n Unigram, v\xED d\u1EE5 "),Xr=i("code"),Ik=s("shrinking_factor"),Hk=s(" cho c\xE1c b\u01B0\u1EDBc m\xE0 ta xo\xE1 token (m\u1EB7c \u0111\u1ECBnh l\xE0 0.75) ho\u1EB7c "),Vr=i("code"),Xk=s("max_piece_length"),Vk=s(" \u0111\u1EC3 ch\u1EC9 \u0111\u1ECBnh \u0111\u1ED9 d\xE0i t\u1ED1i \u0111a c\u1EE7a m\u1ED9t token (m\u1EB7c \u0111\u1ECBnh l\xE0 16)."),zl=h(),ys=i("p"),Kk=s("Tokenizer n\xE0y c\xF3 th\u1EC3 \u0111\u01B0\u1EE3c hu\u1EA5n luy\u1EC7n tr\xEAn c\xE1c t\u1EC7p v\u0103n b\u1EA3n:"),bl=h(),m(Bn.$$.fragment),El=h(),Ts=i("p"),Jk=s("H\xE3y c\xF9ng nh\xECn xem k\u1EBFt qu\u1EA3 tokenize tr\xEAn t\u1EADp m\u1EABu:"),xl=h(),m(Nn.$$.fragment),jl=h(),m(Fn.$$.fragment),yl=h(),X=i("p"),Yk=s("M\u1ED9t \u0111i\u1EC3m \u0111\u1EB7c bi\u1EC7t c\u1EE7a XLNet \u0111\xF3 l\xE0 n\xF3 th\xEAm token "),Kr=i("code"),Zk=s("<cls>"),Qk=s(" \u1EDF cu\u1ED1i m\u1ED7i c\xE2u, v\u1EDBi ki\u1EC3u ID l\xE0 2 (\u0111\u1EC3 ph\xE2n bi\u1EBFt v\u1EDBi c\xE1c token kh\xE1c). N\xF3 \u0111\u1EC7m th\xEAm v\xE0o ph\xEDa b\xEAn tay tr\xE1i gi\u1ED1ng nh\u01B0 k\u1EBFt qu\u1EA3 \u1EDF tr\xEAn. Ta c\xF3 th\u1EC3 x\u1EED l\xFD t\u1EA5t c\u1EA3 c\xE1c token \u0111\u1EB7c bi\u1EC7t v\xE0 c\xE1c token ki\u1EC3u ID v\u1EDBi c\xF9ng m\u1ED9t b\u1EA3n m\u1EABu, nh\u01B0 BERT, nh\u01B0ng \u0111\u1EA7u ti\xEAn ta ph\u1EA3i l\u1EA5y c\xE1c ID c\u1EE7a token "),Jr=i("code"),tf=s("<cls>"),ef=s(" v\xE0 "),Yr=i("code"),nf=s("<sep>"),sf=s(":"),Tl=h(),m(An.$$.fragment),ql=h(),m(Wn.$$.fragment),Pl=h(),qs=i("p"),of=s("B\u1EA3n m\u1EABu s\u1EBD tr\xF4ng nh\u01B0 sau:"),wl=h(),m(Un.$$.fragment),Cl=h(),Ps=i("p"),rf=s("V\xE0 ta c\xF3 th\u1EC3 ki\u1EC3m tra xem n\xF3 ho\u1EA1t \u0111\u1ED9ng kh\xF4ng b\u1EB1ng c\xE1ch m\xE3 ho\xE1 c\u1EB7p c\xE2u:"),Dl=h(),m(Rn.$$.fragment),Ol=h(),m(Gn.$$.fragment),Ll=h(),ee=i("p"),cf=s("Cu\u1ED1i c\xF9ng, ta s\u1EBD th\xEAm tr\xECnh gi\u1EA3i m\xE3 "),Zr=i("code"),lf=s("Metaspace"),af=s(":"),Sl=h(),m(Mn.$$.fragment),Bl=h(),V=i("p"),hf=s("v\xE0 ta \u0111\xE3 xong v\u1EDBi tokenizer n\xE0y! Ta c\xF3 th\u1EC3 l\u01B0u tokenizer nh\u01B0 tr\xEAn, v\xE0 bao n\xF3 l\u1EA1i trong "),Qr=i("code"),pf=s("PreTrainedTokenizerFast"),mf=s(" ho\u1EB7c  "),ti=i("code"),uf=s("XLNetTokenizerFast"),kf=s(" n\u1EBFu ta mu\u1ED1n n\xF3 trong \u{1F917} Transformers. M\u1ED9t \u0111i\u1EC3m c\u1EA7n l\u01B0u \xFD l\xE0 khi s\u1EED d\u1EE5ng "),ei=i("code"),ff=s("PreTrainedTokenizerFast"),gf=s(" th\xEC tr\xEAn \u0111\u1EA7u c\u1EE7a c\xE1c token \u0111\u1EB7c bi\u1EC7t ta c\u1EA7n n\xF3i cho th\u01B0 vi\u1EC7n \u{1F917} Transformers vi\u1EBFt ta c\u1EA7n \u0111\u1EC7m v\xE0o ph\xEDa b\xEAn tr\xE1i:"),Nl=h(),m(In.$$.fragment),Fl=h(),ws=i("p"),df=s("Ho\u1EB7c m\u1ED9t c\xE1ch kh\xE1c:"),Al=h(),m(Hn.$$.fragment),Wl=h(),Cs=i("p"),vf=s("B\xE2y gi\u1EDD b\u1EA1n \u0111\xE3 th\u1EA5y c\xE1ch c\xE1c kh\u1ED1i kh\xE1c nhau \u0111\u01B0\u1EE3c s\u1EED d\u1EE5ng \u0111\u1EC3 x\xE2y d\u1EF1ng c\xE1c tokenizer hi\u1EC7n nay, b\u1EA1n s\u1EBD c\xF3 th\u1EC3 vi\u1EBFt b\u1EA5t k\u1EF3 tr\xECnh tokenize n\xE0o m\xE0 b\u1EA1n mu\u1ED1n v\u1EDBi th\u01B0 vi\u1EC7n \u{1F917} Tokenizers v\xE0 c\xF3 th\u1EC3 s\u1EED d\u1EE5ng n\xF3 trong \u{1F917} Transformers."),this.h()},l(t){const r=Lv('[data-svelte="svelte-1phssyn"]',document.head);$=c(r,"META",{name:!0,content:!0}),r.forEach(n),it=p(t),N=c(t,"H1",{class:!0});var Xn=l(N);W=c(Xn,"A",{id:!0,class:!0,href:!0});var _f=l(W);Q=c(_f,"SPAN",{});var $f=l(Q);u(tt.$$.fragment,$f),$f.forEach(n),_f.forEach(n),he=p(Xn),et=c(Xn,"SPAN",{});var zf=l(et);pe=o(zf,"X\xE2y d\u1EF1ng t\u1EEBng kh\u1ED1i tokenizer"),zf.forEach(n),Xn.forEach(n),Pt=p(t),u(F.$$.fragment,t),wt=p(t),ct=c(t,"P",{});var bf=l(ct);xt=o(bf,"Nh\u01B0 ch\xFAng ta \u0111\xE3 th\u1EA5y trong c\xE1c ph\u1EA7n tr\u01B0\u1EDBc, tokenize bao g\u1ED3m m\u1ED9t s\u1ED1 b\u01B0\u1EDBc:"),bf.forEach(n),Ct=p(t),T=c(t,"UL",{});var ne=l(T);nt=c(ne,"LI",{});var Ef=l(nt);me=o(Ef,"Chu\u1EA9n h\xF3a (m\u1ECDi thao t\xE1c d\u1ECDn d\u1EB9p v\u0103n b\u1EA3n \u0111\u01B0\u1EE3c cho l\xE0 c\u1EA7n thi\u1EBFt, ch\u1EB3ng h\u1EA1n nh\u01B0 x\xF3a d\u1EA5u c\xE1ch ho\u1EB7c d\u1EA5u, chu\u1EA9n h\xF3a Unicode, v.v.)"),Ef.forEach(n),ue=p(ne),st=c(ne,"LI",{});var xf=l(st);ke=o(xf,"Ti\u1EC1n tokenize (chia nh\u1ECF \u0111\u1EA7u v\xE0o th\xE0nh c\xE1c t\u1EEB)"),xf.forEach(n),fe=p(ne),ot=c(ne,"LI",{});var jf=l(ot);ge=o(jf,"\u0110\u01B0a \u0111\u1EA7u v\xE0o th\xF4ng qua m\xF4 h\xECnh (s\u1EED d\u1EE5ng c\xE1c t\u1EEB \u0111\u01B0\u1EE3c ti\u1EC1n tokenize \u0111\u1EC3 t\u1EA1o ra m\u1ED9t chu\u1ED7i token)"),jf.forEach(n),de=p(ne),rt=c(ne,"LI",{});var yf=l(rt);z=o(yf,"H\u1EADu x\u1EED l\xFD (th\xEAm token \u0111\u1EB7c bi\u1EC7t c\u1EE7a tr\xECnh tokenize, t\u1EA1o attention mask v\xE0 ID token)"),yf.forEach(n),ne.forEach(n),ve=p(t),U=c(t,"DIV",{class:!0});var Rl=l(U);lt=c(Rl,"IMG",{class:!0,src:!0,alt:!0}),Qn=p(Rl),at=c(Rl,"IMG",{class:!0,src:!0,alt:!0}),Rl.forEach(n),_e=p(t),Dt=c(t,"P",{});var Gl=l(Dt);ma=o(Gl,"Th\u01B0 vi\u1EC7n \u{1F917} Tokenizers \u0111\xE3 \u0111\u01B0\u1EE3c x\xE2y d\u1EF1ng \u0111\u1EC3 cung c\u1EA5p nhi\u1EC1u s\u1EF1 l\u1EF1a ch\u1ECDn cho c\xE1c b\u01B0\u1EDBc n\xE0y, v\xE0 ta c\xF3 th\u1EC3 k\u1EBFt h\u1EE3p v\xE0 n\u1ED1i ch\xFAng v\u1EDBi nhau. Trong ph\u1EA7n n\xE0y, ch\xFAng ta s\u1EBD xem c\xE1c c\xF3 th\u1EC3 x\xE2y m\u1ED9t tokenizer t\u1EEB \u0111\u1EA7u, tr\xE1i ng\u01B0\u1EE3c v\u1EDBi c\xE1ch hu\u1EA5n luy\u1EC7n m\u1ED9t tokenizer m\u1EDBi t\u1EEB c\xE1i c\u0169 nh\u01B0 ta \u0111\xE3 l\xE0m \u1EDF "),ts=c(Gl,"A",{href:!0});var Tf=l(ts);ua=o(Tf,"ph\u1EA7n 2"),Tf.forEach(n),ka=o(Gl,". Ch\xFAng ta s\u1EBD c\xF3 th\u1EC3 x\xE2y b\u1EA5t k\xEC ki\u1EC3u tokenizer n\xE0o ta c\xF3 th\u1EC3 ngh\u0129 ra!"),Gl.forEach(n),oi=p(t),u($e.$$.fragment,t),ri=p(t),Ot=c(t,"P",{});var Ml=l(Ot);fa=o(Ml,"Ch\xEDnh x\xE1c h\u01A1n, th\u01B0 vi\u1EC7n \u0111\u01B0\u1EE3c x\xE2y d\u1EF1ng t\u1EADp trung v\xE0o l\u1EDBp "),Rs=c(Ml,"CODE",{});var qf=l(Rs);ga=o(qf,"Tokenizer"),qf.forEach(n),da=o(Ml," v\u1EDBi c\xE1c kh\u1ED1i \u0111\u01B0\u1EE3c t\u1EADp h\u1EE3p l\u1EA1i trong c\xE1c m\xF4-\u0111un con:"),Ml.forEach(n),ii=p(t),q=c(t,"UL",{});var K=l(q);ht=c(K,"LI",{});var Vn=l(ht);Gs=c(Vn,"CODE",{});var Pf=l(Gs);va=o(Pf,"normalizers"),Pf.forEach(n),_a=o(Vn," ch\u1EE9a t\u1EA5t c\u1EA3 c\xE1c ki\u1EC3u "),Ms=c(Vn,"CODE",{});var wf=l(Ms);$a=o(wf,"Normalizer"),wf.forEach(n),za=o(Vn," b\u1EA1n c\xF3 th\u1EC3 s\u1EED d\u1EE5ng (ho\xE0n thi\u1EC7n danh s\xE1ch t\u1EA1i "),ze=c(Vn,"A",{href:!0,rel:!0});var Cf=l(ze);ba=o(Cf,"\u0111\xE2y"),Cf.forEach(n),Ea=o(Vn,")."),Vn.forEach(n),xa=p(K),pt=c(K,"LI",{});var Kn=l(pt);Is=c(Kn,"CODE",{});var Df=l(Is);ja=o(Df,"pre_tokenizers"),Df.forEach(n),ya=o(Kn," ch\u1EE9a t\u1EA5t c\u1EA3 c\xE1c ki\u1EC3u "),Hs=c(Kn,"CODE",{});var Of=l(Hs);Ta=o(Of,"PreTokenizer"),Of.forEach(n),qa=o(Kn," b\u1EA1n c\xF3 th\u1EC3 s\u1EED d\u1EE5ng (ho\xE0n thi\u1EC7n danh s\xE1ch t\u1EA1i "),be=c(Kn,"A",{href:!0,rel:!0});var Lf=l(be);Pa=o(Lf,"\u0111\xE2y"),Lf.forEach(n),wa=o(Kn,")."),Kn.forEach(n),Ca=p(K),P=c(K,"LI",{});var A=l(P);Xs=c(A,"CODE",{});var Sf=l(Xs);Da=o(Sf,"models"),Sf.forEach(n),Oa=o(A," ch\u1EE9a t\u1EA5t c\u1EA3 c\xE1c ki\u1EC3u "),Vs=c(A,"CODE",{});var Bf=l(Vs);La=o(Bf,"Model"),Bf.forEach(n),Sa=o(A," b\u1EA1n c\xF3 th\u1EC3 s\u1EED d\u1EE5ng, nh\u01B0 "),Ks=c(A,"CODE",{});var Nf=l(Ks);Ba=o(Nf,"BPE"),Nf.forEach(n),Na=o(A,", "),Js=c(A,"CODE",{});var Ff=l(Js);Fa=o(Ff,"WordPiece"),Ff.forEach(n),Aa=o(A,", and "),Ys=c(A,"CODE",{});var Af=l(Ys);Wa=o(Af,"Unigram"),Af.forEach(n),Ua=o(A," (ho\xE0n thi\u1EC7n danh s\xE1ch t\u1EA1i "),Ee=c(A,"A",{href:!0,rel:!0});var Wf=l(Ee);Ra=o(Wf,"\u0111\xE2y"),Wf.forEach(n),Ga=o(A,")."),A.forEach(n),Ma=p(K),mt=c(K,"LI",{});var Jn=l(mt);Zs=c(Jn,"CODE",{});var Uf=l(Zs);Ia=o(Uf,"trainers"),Uf.forEach(n),Ha=o(Jn," ch\u1EE9a t\u1EA5t c\u1EA3 c\xE1c ki\u1EC3u "),Qs=c(Jn,"CODE",{});var Rf=l(Qs);Xa=o(Rf,"Trainer"),Rf.forEach(n),Va=o(Jn," kh\xE1c nhau b\u1EA1n c\xF3 th\u1EC3 s\u1EED d\u1EE5ng \u0111\u1EC3 hu\u1EA5n luy\u1EC7n m\xF4 h\xECnh c\u1EE7a b\u1EA1n tr\xEAn kho ng\u1EEF li\u1EC7u (m\u1ED9t cho m\u1ED7i lo\u1EA1i m\xF4 h\xECnh; ho\xE0n thi\u1EC7n danh s\xE1ch t\u1EA1i "),xe=c(Jn,"A",{href:!0,rel:!0});var Gf=l(xe);Ka=o(Gf,"\u0111\xE2y"),Gf.forEach(n),Ja=o(Jn,")."),Jn.forEach(n),Ya=p(K),ut=c(K,"LI",{});var Yn=l(ut);to=c(Yn,"CODE",{});var Mf=l(to);Za=o(Mf,"post_processors"),Mf.forEach(n),Qa=o(Yn," ch\u1EE9a t\u1EA5t c\u1EA3 c\xE1c ki\u1EC3u "),eo=c(Yn,"CODE",{});var If=l(eo);th=o(If,"PostProcessor"),If.forEach(n),eh=o(Yn," b\u1EA1n c\xF3 th\u1EC3 s\u1EED d\u1EE5ng (ho\xE0n thi\u1EC7n danh s\xE1ch t\u1EA1i "),je=c(Yn,"A",{href:!0,rel:!0});var Hf=l(je);nh=o(Hf,"\u0111\xE2y"),Hf.forEach(n),sh=o(Yn,")."),Yn.forEach(n),oh=p(K),kt=c(K,"LI",{});var Zn=l(kt);no=c(Zn,"CODE",{});var Xf=l(no);rh=o(Xf,"decoders"),Xf.forEach(n),ih=o(Zn," ch\u1EE9a t\u1EA5t c\u1EA3 c\xE1c ki\u1EC3u "),so=c(Zn,"CODE",{});var Vf=l(so);ch=o(Vf,"Decoder"),Vf.forEach(n),lh=o(Zn," \u0111a d\u1EA1ng b\u1EA1n c\xF3 th\u1EC3 s\u1EED d\u1EE5ng \u0111\u1EC3 gi\u1EA3i m\xE3 \u0111\u1EA7u ra c\u1EE7a tokenize (ho\xE0n thi\u1EC7n danh s\xE1ch t\u1EA1i "),ye=c(Zn,"A",{href:!0,rel:!0});var Kf=l(ye);ah=o(Kf,"\u0111\xE2y"),Kf.forEach(n),hh=o(Zn,")."),Zn.forEach(n),K.forEach(n),ci=p(t),Lt=c(t,"P",{});var Il=l(Lt);ph=o(Il,"B\u1EA1n c\xF3 th\u1EC3 t\xECm \u0111\u01B0\u1EE3c to\xE0n b\u1ED9 danh s\xE1ch c\xE1c kh\u1ED1i t\u1EA1i "),Te=c(Il,"A",{href:!0,rel:!0});var Jf=l(Te);mh=o(Jf,"\u0111\xE2y"),Jf.forEach(n),uh=o(Il,"."),Il.forEach(n),li=p(t),jt=c(t,"H2",{class:!0});var Hl=l(jt);St=c(Hl,"A",{id:!0,class:!0,href:!0});var Yf=l(St);oo=c(Yf,"SPAN",{});var Zf=l(oo);u(qe.$$.fragment,Zf),Zf.forEach(n),Yf.forEach(n),kh=p(Hl),ro=c(Hl,"SPAN",{});var Qf=l(ro);fh=o(Qf,"Thu th\u1EADp m\u1ED9t kho ng\u1EEF li\u1EC7u"),Qf.forEach(n),Hl.forEach(n),ai=p(t),ft=c(t,"P",{});var Ds=l(ft);gh=o(Ds,"\u0110\u1EC3 hu\u1EA5n luy\u1EC7n tokenizer m\u1EDBi c\u1EE7a m\xECnh, ch\xFAng ta s\u1EBD s\u1EED d\u1EE5ng m\u1ED9t kho ng\u1EEF li\u1EC7u nh\u1ECF ch\u1EE9a c\xE1c \u0111o\u1EA1n v\u0103n (cho nhanh). C\xE1c b\u01B0\u1EDBc \u0111\u1EC3 c\xF3 \u0111\u01B0\u1EE3c kho ng\u1EEF li\u1EC7u t\u01B0\u01A1ng t\u1EF1 nh\u01B0 ch\xFAng ta \u0111\xE3 l\xE0m \u1EDF "),es=c(Ds,"A",{href:!0});var tg=l(es);dh=o(tg,"ph\u1EA7n \u0111\u1EA7u c\u1EE7a ch\u01B0\u01A1ng n\xE0y"),tg.forEach(n),vh=o(Ds,", nh\u01B0ng l\u1EA7n n\xE0y ch\xFAng ta s\u1EBD s\u1EED d\u1EE5ng "),Pe=c(Ds,"A",{href:!0,rel:!0});var eg=l(Pe);_h=o(eg,"WikiText-2"),eg.forEach(n),$h=o(Ds,":"),Ds.forEach(n),hi=p(t),u(we.$$.fragment,t),pi=p(t),Bt=c(t,"P",{});var Xl=l(Bt);zh=o(Xl,"H\xE0m "),io=c(Xl,"CODE",{});var ng=l(io);bh=o(ng,"get_training_corpus()"),ng.forEach(n),Eh=o(Xl," l\xE0 m\u1ED9t h\xE0m t\u1EA1o c\xF3 th\u1EC3 tr\u1EA3 v\u1EC1 c\xE1c l\xF4 v\u1EDBi m\u1ED7i l\xF4 l\xE0 1,000 \u0111o\u1EA1n v\u0103n, c\xE1i m\xE0 ta s\u1EBD s\u1EED d\u1EE5ng \u0111\u1EC3 hu\u1EA5n luy\u1EC7n  tokenizer."),Xl.forEach(n),mi=p(t),ns=c(t,"P",{});var sg=l(ns);xh=o(sg,"\u{1F917} Tokenizers c\u0169ng c\xF3 th\u1EC3 \u0111\u01B0\u1EE3c hu\u1EA5n luy\u1EC7n tr\u1EF1c ti\u1EBFp tr\xEAn c\xE1c t\u1EC7p v\u0103n b\u1EA3n. \u0110\xE2y l\xE0 c\xE1ch ch\xFAng ta t\u1EA1o ra m\u1ED9t t\u1EC7p v\u0103n b\u1EA3n bao g\u1ED3m c\xE1c \u0111o\u1EA1n v\u0103n/\u0111\u1EA7u v\xE0o t\u1EEB WikiText-2 m\xE0 ta c\xF3 th\u1EC3 s\u1EED d\u1EE5ng c\u1EE5c b\u1ED9:"),sg.forEach(n),ui=p(t),u(Ce.$$.fragment,t),ki=p(t),ss=c(t,"P",{});var og=l(ss);jh=o(og,"Ti\u1EBFp theo ch\xFAng t\xF4i s\u1EBD h\u01B0\u1EDBng d\u1EABn b\u1EA1n c\xE1ch t\u1EF1 x\xE2y d\u1EF1ng t\u1EEBng kh\u1ED1i BERT, GPT-2, v\xE0 XLNet tokenizer c\u1EE7a ri\xEAng m\xECnh. \u0110i\u1EC1u n\xE0y s\u1EBD cung c\u1EA5p cho ch\xFAng ta m\u1ED9t v\xED d\u1EE5 v\u1EC1 t\u1EEBng thu\u1EADt to\xE1n trong s\u1ED1 ba thu\u1EADt to\xE1n tokenize ch\xEDnh: WordPiece, BPE, v\xE0 Unigram. H\xE3y c\u0169ng b\u1EAFt \u0111\u1EA7u v\u1EDBi BERT!"),og.forEach(n),fi=p(t),yt=c(t,"H2",{class:!0});var Vl=l(yt);Nt=c(Vl,"A",{id:!0,class:!0,href:!0});var rg=l(Nt);co=c(rg,"SPAN",{});var ig=l(co);u(De.$$.fragment,ig),ig.forEach(n),rg.forEach(n),yh=p(Vl),lo=c(Vl,"SPAN",{});var cg=l(lo);Th=o(cg,"X\xE2y d\u1EF1ng m\u1ED9t WordPiece tokenizer t\u1EEB \u0111\u1EA7u"),cg.forEach(n),Vl.forEach(n),gi=p(t),x=c(t,"P",{});var L=l(x);qh=o(L,"\u0110\u1EC3 x\xE2y d\u1EF1ng m\u1ED9t tokenizer v\u1EDBi th\u01B0 vi\u1EC7n \u{1F917} Tokenizers, ch\xFAng ta s\u1EBD b\u1EAFt \u0111\u1EA7u v\u1EDBi vi\u1EC7c kh\u1EDFi t\u1EA1o m\u1ED9t \u0111\u1ED1i t\u01B0\u1EE3ng "),ao=c(L,"CODE",{});var lg=l(ao);Ph=o(lg,"Tokenizer"),lg.forEach(n),wh=o(L," v\u1EDBi "),ho=c(L,"CODE",{});var ag=l(ho);Ch=o(ag,"model"),ag.forEach(n),Dh=o(L,", sau \u0111\xF3 thi\u1EBFt l\u1EADp "),po=c(L,"CODE",{});var hg=l(po);Oh=o(hg,"normalizer"),hg.forEach(n),Lh=o(L,", "),mo=c(L,"CODE",{});var pg=l(mo);Sh=o(pg,"pre_tokenizer"),pg.forEach(n),Bh=o(L,", "),uo=c(L,"CODE",{});var mg=l(uo);Nh=o(mg,"post_processor"),mg.forEach(n),Fh=o(L,", v\xE0 "),ko=c(L,"CODE",{});var ug=l(ko);Ah=o(ug,"decoder"),ug.forEach(n),Wh=o(L," t\u1EDBi c\xE1c gi\xE1 tr\u1ECB ta mu\u1ED1n."),L.forEach(n),di=p(t),Ft=c(t,"P",{});var Kl=l(Ft);Uh=o(Kl,"V\u1EDBi v\xED d\u1EE5 n\xE0y, ta s\u1EBD t\u1EA1o ra m\u1ED9t "),fo=c(Kl,"CODE",{});var kg=l(fo);Rh=o(kg,"Tokenizer"),kg.forEach(n),Gh=o(Kl," v\u1EDBi m\u1ED9t m\xF4 h\xECnh WordPiece:"),Kl.forEach(n),vi=p(t),u(Oe.$$.fragment,t),_i=p(t),R=c(t,"P",{});var se=l(R);Mh=o(se,"Ch\xFAng ta ph\u1EA3i ch\u1EC9 r\xF5 "),go=c(se,"CODE",{});var fg=l(go);Ih=o(fg,"unk_token"),fg.forEach(n),Hh=o(se," \u0111\u1EC3 m\xF4 h\xECnh bi\u1EBFt ph\u1EA3i tr\u1EA3 v\u1EC1 g\xEC khi g\u1EB7p c\xE1c k\xED t\u1EF1 ch\u01B0a t\u1EEBng g\u1EB7p tr\u01B0\u1EDBc \u0111\xF3. C\xE1c tham s\u1ED1 kh\xE1c ch\xFAng ta c\xF3 th\u1EC3 c\xE0i \u0111\u1EB7t g\u1ED3m "),vo=c(se,"CODE",{});var gg=l(vo);Xh=o(gg,"vocab"),gg.forEach(n),Vh=o(se," c\u1EE7a m\xF4 h\xECnh (ta s\u1EBD hu\u1EA5n luy\u1EC7n m\xF4 h\xECnh n\xEAn kh\xF4ng c\u1EA7n thi\u1EBFt l\u1EADp n\xF3) v\xE0 "),_o=c(se,"CODE",{});var dg=l(_o);Kh=o(dg,"max_input_chars_per_word"),dg.forEach(n),Jh=o(se,", t\u01B0\u01A1ng \u1EE9ng \u0111\u1ED9 d\xE0i t\u1ED1i \u0111a cho m\u1ED9t t\u1EEB (t\u1EEB d\xE0i h\u01A1n gi\xE1 tr\u1ECB n\xE0y s\u1EBD b\u1ECB chia nh\u1ECF)"),se.forEach(n),$i=p(t),j=c(t,"P",{});var S=l(j);Yh=o(S,"B\u01B0\u1EDBc \u0111\u1EA7u ti\xEAn \u0111\u1EC3 tokenize \u0111\xF3 l\xE0 chu\u1EA9n ho\xE1, v\xEC v\u1EADy h\xE3y c\u0169ng b\u1EAFt \u0111\u1EA7u v\u1EDBi b\u01B0\u1EDBc n\xE0y. V\xEC BERT \u0111\u01B0\u1EE3c s\u1EED d\u1EE5ng r\u1ED9ng t\xE3i, ta c\xF3 th\u1EC3 s\u1EED d\u1EE5ng "),$o=c(S,"CODE",{});var vg=l($o);Zh=o(vg,"BertNormalizer"),vg.forEach(n),Qh=o(S," v\u1EDBi tu\u1EF3 ch\u1ECDn kinh \u0111i\u1EC3n \u0111\u1EC3 thi\u1EBFt l\u1EADp cho BERT: "),zo=c(S,"CODE",{});var _g=l(zo);tp=o(_g,"lowercase"),_g.forEach(n),ep=o(S," v\xE0 "),bo=c(S,"CODE",{});var $g=l(bo);np=o($g,"strip_accents"),$g.forEach(n),sp=o(S,", t\u1EF1 c\xE1i t\xEAn \u0111\xE3 gi\u1EA3i th\xEDch m\u1EE5c \u0111\xEDch c\u1EE7a ch\xFAng; "),Eo=c(S,"CODE",{});var zg=l(Eo);op=o(zg,"clean_text"),zg.forEach(n),rp=o(S," \u0111\u1EC3 lo\u1EA1i b\u1ECF t\u1EA5t c\u1EA3 c\xE1c k\xED t\u1EF1 ki\u1EC3m so\xE1t v\xE0 d\u1EA5u c\xE1ch l\u1EB7p l\u1EA1i th\xE0nh m\u1ED9t; v\xE0 "),xo=c(S,"CODE",{});var bg=l(xo);ip=o(bg,"handle_chinese_chars"),bg.forEach(n),cp=o(S," th\xEAm d\u1EA5u c\xE1ch gi\u1EEFa c\xE1c k\xED t\u1EF1 ti\u1EBFng Trung. \u0110\u1EC3 , which places spaces around Chinese characters. To t\xE1i t\u1EA1o tokenizer "),jo=c(S,"CODE",{});var Eg=l(jo);lp=o(Eg,"bert-base-uncased"),Eg.forEach(n),ap=o(S,", ta c\xF3 th\u1EC3 thi\u1EBFt l\u1EADp chu\u1EA9n ho\xE1 sau:"),S.forEach(n),zi=p(t),u(Le.$$.fragment,t),bi=p(t),G=c(t,"P",{});var oe=l(G);hp=o(oe,"Th\xF4ng th\u01B0\u1EDDng, khi x\xE2y d\u1EF1ng m\u1ED9t tokenizer, b\u1EA1n kh\xF4ng c\u1EA7n ph\u1EA3i truy c\u1EADp v\xE0o m\u1ED9t h\xE0m chu\u1EA9n ho\xE1 th\u1EE7 c\xF4ng v\xEC n\xF3 \u0111\xE3 c\xF3 s\u1EB5n trong th\u01B0 vi\u1EC7n \u{1F917} Tokenizers library \u2014 tuy nhi\xEAn, h\xE3y c\xF9ng t\u1EA1o ra chu\u1EA9n ho\xE1 BERT th\u1EE7 c\xF4ng. Th\u01B0 vi\u1EC7n cung c\xE2p tr\xECnh chu\u1EA9n ho\xE1 "),yo=c(oe,"CODE",{});var xg=l(yo);pp=o(xg,"Lowercase"),xg.forEach(n),mp=o(oe," v\xE0 "),To=c(oe,"CODE",{});var jg=l(To);up=o(jg,"StripAccents"),jg.forEach(n),kp=o(oe,", b\u1EA1n ho\xE0n to\xE0n c\xF3 th\u1EC3 k\u1EBFt h\u1EE3p nhi\u1EC1u tr\xECnh chu\u1EA9n ho\xE1 v\u1EDBi nhau th\xF4ng qua "),qo=c(oe,"CODE",{});var yg=l(qo);fp=o(yg,"Sequence"),yg.forEach(n),gp=o(oe,":"),oe.forEach(n),Ei=p(t),u(Se.$$.fragment,t),xi=p(t),gt=c(t,"P",{});var Os=l(gt);dp=o(Os,"Ta c\u0169ng c\xF3 th\u1EC3 s\u1EED d\u1EE5ng chu\u1EA9n ho\xE1 Unicode "),Po=c(Os,"CODE",{});var Tg=l(Po);vp=o(Tg,"NFD"),Tg.forEach(n),_p=o(Os," Unicode normalizer, v\xEC n\u1EBFu kh\xF4ng chu\u1EA9n ho\xE1 "),wo=c(Os,"CODE",{});var qg=l(wo);$p=o(qg,"StripAccents"),qg.forEach(n),zp=o(Os," s\u1EBD kh\xF4ng nh\u1EADn di\u1EC7n \u0111\u01B0\u1EE3c nh\u1EEFng k\xED t\u1EF1 c\xF3 d\u1EA5u v\xE0 kh\xF4ng th\u1EC3 t\xE1ch n\xF3 \u0111\xFAng nh\u01B0 ta mu\u1ED1n."),Os.forEach(n),ji=p(t),dt=c(t,"P",{});var Ls=l(dt);bp=o(Ls,"Nh\u01B0 \u0111\xE3 th\u1EA5y \u1EDF tr\xEAn, ta c\xF3 th\u1EC3 s\u1EED d\u1EE5ng ph\u01B0\u01A1ng th\u1EE9c "),Co=c(Ls,"CODE",{});var Pg=l(Co);Ep=o(Pg,"normalize_str()"),Pg.forEach(n),xp=o(Ls," c\u1EE7a "),Do=c(Ls,"CODE",{});var wg=l(Do);jp=o(wg,"normalizer"),wg.forEach(n),yp=o(Ls," \u0111\u1EC3 ki\u1EC3m tra t\xE1c \u0111\u1ED9ng c\u1EE7a n\xF3 l\xEAn m\u1ED9t chu\u1ED7i v\u0103n b\u1EA3n:"),Ls.forEach(n),yi=p(t),u(Be.$$.fragment,t),Ti=p(t),u(Ne.$$.fragment,t),qi=p(t),u(At.$$.fragment,t),Pi=p(t),Wt=c(t,"P",{});var Jl=l(Wt);Tp=o(Jl,"Ti\u1EBFp theo l\xE0 b\u01B0\u1EDBc pre-tokenization. M\u1ED9t l\u1EA7n n\u1EEFa, ta c\xF3 "),Oo=c(Jl,"CODE",{});var Cg=l(Oo);qp=o(Cg,"BertPreTokenizer"),Cg.forEach(n),Pp=o(Jl," \u0111\u01B0\u1EE3c x\xE2y d\u1EF1ng s\u1EB5n \u0111\u1EC3 d\xF9ng:"),Jl.forEach(n),wi=p(t),u(Fe.$$.fragment,t),Ci=p(t),os=c(t,"P",{});var Dg=l(os);wp=o(Dg,"Ho\u1EB7c ta c\xF3 th\u1EC3 x\xE2y t\u1EEB \u0111\u1EA7u:"),Dg.forEach(n),Di=p(t),u(Ae.$$.fragment,t),Oi=p(t),Ut=c(t,"P",{});var Yl=l(Ut);Cp=o(Yl,"L\u01B0u \xFD r\u1EB1ng "),Lo=c(Yl,"CODE",{});var Og=l(Lo);Dp=o(Og,"Whitespace"),Og.forEach(n),Op=o(Yl," s\u1EBD t\xE1ch theo d\u1EA5u c\xE1ch v\xE0 c\xE1c k\xED t\u1EF1 kh\xF4ng ph\u1EA3i ch\u1EEF c\xE1i, s\u1ED1, ho\u1EB7c d\u1EA5u g\u1EA1ch d\u01B0\u1EDBi, n\xEAn v\u1EC1 m\u1EB7t k\u1EF9 thu\u1EADt n\xF3 s\u1EBD t\xE1ch theo d\u1EA5u c\xE1ch v\xE0 d\u1EA5u c\xE2u:"),Yl.forEach(n),Li=p(t),u(We.$$.fragment,t),Si=p(t),u(Ue.$$.fragment,t),Bi=p(t),Rt=c(t,"P",{});var Zl=l(Rt);Lp=o(Zl,"N\xEAu sbanj ch\u1EC9 mu\u1ED1n t\xE1ch theo d\u1EA5u c\xE1ch, b\u1EA1n c\xF3 th\u1EC3 s\u1EED d\u1EE5ng "),So=c(Zl,"CODE",{});var Lg=l(So);Sp=o(Lg,"WhitespaceSplit"),Lg.forEach(n),Bp=o(Zl," thay th\u1EBF:"),Zl.forEach(n),Ni=p(t),u(Re.$$.fragment,t),Fi=p(t),u(Ge.$$.fragment,t),Ai=p(t),Gt=c(t,"P",{});var Ql=l(Gt);Np=o(Ql,"Gi\u1ED1ng nh\u01B0 chu\u1EA9n ho\xE1, b\u1EA3n c\xF3 th\u1EC3 s\u1EED d\u1EE5ng "),Bo=c(Ql,"CODE",{});var Sg=l(Bo);Fp=o(Sg,"Sequence"),Sg.forEach(n),Ap=o(Ql," \u0111\u1EC3 k\u1EBFt h\u1EE3p c\xE1c ti\u1EC1n tokenizer v\u1EDBi nhau:"),Ql.forEach(n),Wi=p(t),u(Me.$$.fragment,t),Ui=p(t),u(Ie.$$.fragment,t),Ri=p(t),Mt=c(t,"P",{});var ta=l(Mt);Wp=o(ta,"B\u01B0\u1EDBc ti\u1EBFp theo trong pipeline tokenize l\xE0 \u0111\u01B0a \u0111\u1EA7u v\xE0o qua m\xF4 h\xECnh. Ta \u0111\xE3 ch\u1EC9 \u0111\u1ECBnh m\xF4 h\xECnh c\u1EE7a m\xECnh khi kh\u1EDFi t\u1EA1o, nh\u01B0ng ta v\u1EABn c\u1EA7n hu\u1EA5n luy\u1EC7n n\xF3, \u0111i\u1EC1u n\xE0y c\u1EA7n t\u1EDBi "),No=c(ta,"CODE",{});var Bg=l(No);Up=o(Bg,"WordPieceTrainer"),Bg.forEach(n),Rp=o(ta,".  V\u1EA5n \u0111\u1EC1 ch\xEDnh \u1EDF \u0111\xE2y l\xE0 khi kh\u1EDFi t\u1EA1o m\u1ED9t tr\xECnh hu\u1EA5n luy\u1EC7n trong \u{1F917} Tokenizers th\xEC b\u1EA1n c\u1EA7n ph\u1EA3i truy\u1EC1n t\u1EA5t c\u1EA3 c\xE1c token \u0111\u1EB7c bi\u1EC7t b\u1EA1n c\xF3 \xFD \u0111\u1ECBnh s\u1EED d\u1EE5ng, n\u1EBFu kh\xF4ng n\xF3 s\u1EBD kh\xF4ng th\xEAm v\xE0o b\u1ED9 t\u1EEB v\u1EF1ng, v\xEC ch\xFAng kh\xF4ng c\xF3 trong kho ng\u1EEF li\u1EC7u hu\u1EA5n luy\u1EC7n:"),ta.forEach(n),Gi=p(t),u(He.$$.fragment,t),Mi=p(t),w=c(t,"P",{});var J=l(w);Gp=o(J,"C\u0169ng nh\u01B0 vi\u1EC7c ch\u1EC9 \u0111\u1ECBnh "),Fo=c(J,"CODE",{});var Ng=l(Fo);Mp=o(Ng,"vocab_size"),Ng.forEach(n),Ip=o(J," v\xE0 "),Ao=c(J,"CODE",{});var Fg=l(Ao);Hp=o(Fg,"special_tokens"),Fg.forEach(n),Xp=o(J,", ta c\u1EA7n thi\u1EBFt l\u1EADp "),Wo=c(J,"CODE",{});var Ag=l(Wo);Vp=o(Ag,"min_frequency"),Ag.forEach(n),Kp=o(J," (s\u1ED1 l\u1EA7n m\u1ED9t token ph\u1EA3i xu\u1EA5t hi\u1EC7n \u0111\u1EC3 \u0111\u01B0\u1EE3c th\xEAm v\xE0o b\u1ED9 t\u1EEB v\u1EF1ng) ho\u1EB7c thay \u0111\u1ED5i "),Uo=c(J,"CODE",{});var Wg=l(Uo);Jp=o(Wg,"continuing_subword_prefix"),Wg.forEach(n),Yp=o(J," (n\u1EBFu ta mu\u1ED1n s\u1EED d\u1EE5ng th\u1EE9 g\xEC kh\xE1c ngo\xE0i "),Ro=c(J,"CODE",{});var Ug=l(Ro);Zp=o(Ug,"##"),Ug.forEach(n),Qp=o(J,")."),J.forEach(n),Ii=p(t),rs=c(t,"P",{});var Rg=l(rs);tm=o(Rg,"\u0110\u1EC3 hu\u1EA5n luy\u1EC7n m\u1ED9t m\xF4 h\xECnh s\u1EED d\u1EE5ng tr\xECnh l\u1EB7p ta \u0111\u1ECBnh ngh\u0129a tr\u01B0\u1EDBc \u0111\xF3, ta ch\u1EC9 c\u1EA7n th\u1EF1c hi\u1EC7n l\u1EC7nh n\xE0y:"),Rg.forEach(n),Hi=p(t),u(Xe.$$.fragment,t),Xi=p(t),It=c(t,"P",{});var ea=l(It);em=o(ea,"Ch\xFAng ta c\u0169ng c\xF3 th\u1EC3 s\u1EED d\u1EE5ng c\xE1c t\u1EC7p v\u0103n b\u1EA3n \u0111\u1EC3 hu\u1EA5n luy\u1EC7n tokenizer c\u1EE7a m\xECnh nh\u01B0 sau (ta t\xE1i kh\u1EDFi t\u1EA1o m\xF4 h\xECnh v\u1EDBi m\u1ED9t "),Go=c(ea,"CODE",{});var Gg=l(Go);nm=o(Gg,"WordPiece"),Gg.forEach(n),sm=o(ea," r\u1ED7ng):"),ea.forEach(n),Vi=p(t),u(Ve.$$.fragment,t),Ki=p(t),Ht=c(t,"P",{});var na=l(Ht);om=o(na,"Trong c\u1EA3 hai tr\u01B0\u1EDDng h\u1EE3p, ta c\xF3 th\u1EC3 ki\u1EC3m tra xem tokenizer tr\xEAn m\u1ED9t \u0111o\u1EA1n v\u0103n b\u1EA3n b\u1EB1ng c\xE1ch s\u1EED d\u1EE5ng ph\u01B0\u01A1ng th\u1EE9c "),Mo=c(na,"CODE",{});var Mg=l(Mo);rm=o(Mg,"encode()"),Mg.forEach(n),im=o(na,":"),na.forEach(n),Ji=p(t),u(Ke.$$.fragment,t),Yi=p(t),u(Je.$$.fragment,t),Zi=p(t),b=c(t,"P",{});var E=l(b);Io=c(E,"CODE",{});var Ig=l(Io);cm=o(Ig,"encoding"),Ig.forEach(n),lm=o(E," thu \u0111\u01B0\u1EE3c l\xE0 m\u1ED9t "),Ho=c(E,"CODE",{});var Hg=l(Ho);am=o(Hg,"Encoding"),Hg.forEach(n),hm=o(E," g\u1ED3m t\u1EA5t c\u1EA3 c\xE1c \u0111\u1EA7u ra c\u1EA7n thi\u1EBFt c\u1EE7a m\u1ED9t tokenizer trong t\u1EA5t c\u1EA3 c\xE1c th\xF4ng s\u1ED1 \u0111a d\u1EA1ng c\u1EE7a n\xF3: "),Xo=c(E,"CODE",{});var Xg=l(Xo);pm=o(Xg,"ids"),Xg.forEach(n),mm=o(E,", "),Vo=c(E,"CODE",{});var Vg=l(Vo);um=o(Vg,"type_ids"),Vg.forEach(n),km=o(E,", "),Ko=c(E,"CODE",{});var Kg=l(Ko);fm=o(Kg,"tokens"),Kg.forEach(n),gm=o(E,", "),Jo=c(E,"CODE",{});var Jg=l(Jo);dm=o(Jg,"offsets"),Jg.forEach(n),vm=o(E,", "),Yo=c(E,"CODE",{});var Yg=l(Yo);_m=o(Yg,"attention_mask"),Yg.forEach(n),$m=o(E,", "),Zo=c(E,"CODE",{});var Zg=l(Zo);zm=o(Zg,"special_tokens_mask"),Zg.forEach(n),bm=o(E,", v\xE0 "),Qo=c(E,"CODE",{});var Qg=l(Qo);Em=o(Qg,"overflowing"),Qg.forEach(n),xm=o(E,"."),E.forEach(n),Qi=p(t),C=c(t,"P",{});var Y=l(C);jm=o(Y,"B\u01B0\u1EDBc cu\u1ED1i c\u1EE7a quy tr\xECnh \u0111\xF3 l\xE0 h\u1EADu x\u1EED l\xFD. Ta c\u1EA7n th\xEAm token "),tr=c(Y,"CODE",{});var td=l(tr);ym=o(td,"[CLS]"),td.forEach(n),Tm=o(Y," token at the beginning and the "),er=c(Y,"CODE",{});var ed=l(er);qm=o(ed,"[SEP]"),ed.forEach(n),Pm=o(Y," \u1EDF cu\u1ED1i (ho\u1EB7c sau m\u1ED7i c\xE2u, n\u1EBFu ta c\xF3 c\u1EB7p c\xE2u). Ch\xFAng ta s\u1EBD s\u1EED d\u1EE5ng "),nr=c(Y,"CODE",{});var nd=l(nr);wm=o(nd,"TemplateProcessor"),nd.forEach(n),Cm=o(Y," \u0111\u1EC3 th\u1EF1c hi\u1EC7n \u0111i\u1EC1u n\xE0y, nh\u01B0ng tr\u01B0\u1EDBc h\u1EBFt ta c\u1EA7n bi\u1EBFt c\xE1c ID c\u1EE7a token "),sr=c(Y,"CODE",{});var sd=l(sr);Dm=o(sd,"[CLS]"),sd.forEach(n),Om=o(Y," v\xE0 "),or=c(Y,"CODE",{});var od=l(or);Lm=o(od,"[SEP]"),od.forEach(n),Sm=o(Y," trong b\u1ED9 t\u1EEB v\u1EF1ng."),Y.forEach(n),tc=p(t),u(Ye.$$.fragment,t),ec=p(t),u(Ze.$$.fragment,t),nc=p(t),M=c(t,"P",{});var re=l(M);Bm=o(re,"\u0110\u1EC3 vi\u1EBFt b\u1EA3n m\u1EABu cho "),rr=c(re,"CODE",{});var rd=l(rr);Nm=o(rd,"TemplateProcessor"),rd.forEach(n),Fm=o(re,", ch\xFAng ta ph\u1EA3i ch\u1EC9 \u0111\u1ECBnh c\xE1ch x\u1EED l\xFD m\u1ED9t c\xE2u \u0111\u01A1n v\xE0 m\u1ED9t c\u1EB7p c\xE2u. \u0110\u1ED1i v\u1EDBi c\u1EA3 hai, ch\xFAng t\xF4i vi\u1EBFt c\xE1c token \u0111\u1EB7c bi\u1EC7t mu\u1ED1n s\u1EED d\u1EE5ng; c\xE2u \u0111\u1EA7u ti\xEAn (ho\u1EB7c c\xE2u \u0111\u01A1n) \u0111\u01B0\u1EE3c bi\u1EC3u th\u1ECB b\u1EB1ng "),ir=c(re,"CODE",{});var id=l(ir);Am=o(id,"$A"),id.forEach(n),Wm=o(re,", trong khi c\xE2u th\u1EE9 hai (n\u1EBFu token m\u1ED9t c\u1EB7p) \u0111\u01B0\u1EE3c bi\u1EC3u th\u1ECB b\u1EB1ng "),cr=c(re,"CODE",{});var cd=l(cr);Um=o(cd,"$B"),cd.forEach(n),Rm=o(re,". \u0110\u1ED1i v\u1EDBi m\u1ED7i lo\u1EA1i trong s\u1ED1 n\xE0y (token v\xE0 c\xE2u \u0111\u1EB7c bi\u1EC7t), ch\xFAng ta c\u0169ng ch\u1EC9 \u0111\u1ECBnh lo\u1EA1i token ID t\u01B0\u01A1ng \u1EE9ng sau d\u1EA5u hai ch\u1EA5m."),re.forEach(n),sc=p(t),is=c(t,"P",{});var ld=l(is);Gm=o(ld,"Do \u0111\xF3, b\u1EA3n m\u1EABu BERT c\u1ED5 \u0111i\u1EC3n \u0111\u01B0\u1EE3c \u0111\u1ECBnh ngh\u0129a nh\u01B0 sau:"),ld.forEach(n),oc=p(t),u(Qe.$$.fragment,t),rc=p(t),cs=c(t,"P",{});var ad=l(cs);Mm=o(ad,"L\u01B0u \xFD r\u1EB1ng ch\xFAng ta c\u1EA7n truy\u1EC1n v\xE0o t\u1EA5t c\u1EA3 c\xE1c IDs c\u1EE7a c\xE1c k\xED t\u1EF1 \u0111\u1EB7c bi\u1EC7t, n\xEAn c\xE1c tokenize c\xF3 th\u1EC3 chuy\u1EC3n \u0111\u1ED5i ch\xFAng th\xE0nh c\xE1c c\u1EB7p ID."),ad.forEach(n),ic=p(t),ls=c(t,"P",{});var hd=l(ls);Im=o(hd,"M\u1ED9t khi \u0111\xE3 \u0111\u01B0\u1EE3c th\xEAm v\xE0o, ch\xFAng ta c\xF3 th\u1EC3 quay l\u1EA1i v\xED d\u1EE5 tr\u01B0\u1EDBc \u0111\xF3 v\xE0 s\u1EBD nh\u1EADn \u0111\u01B0\u1EE3c:"),hd.forEach(n),cc=p(t),u(tn.$$.fragment,t),lc=p(t),u(en.$$.fragment,t),ac=p(t),as=c(t,"P",{});var pd=l(as);Hm=o(pd,"V\xE0 tr\xEAn m\u1ED9t c\u1EB7p c\xE2u, ch\xFAng ta c\xF3 th\u1EC3 c\xF3 \u0111\u01B0\u1EE3c k\u1EBFt qu\u1EA3 sau:"),pd.forEach(n),hc=p(t),u(nn.$$.fragment,t),pc=p(t),u(sn.$$.fragment,t),mc=p(t),hs=c(t,"P",{});var md=l(hs);Xm=o(md,"Ch\xFAng ta \u0111\xE3 g\u1EA7n nh\u01B0 ho\xE0n th\xE0nh vi\u1EC7c x\xE2y d\u1EF1ng tokenizer n\xE0y t\u1EEB \u0111\u1EA7u \u2014 b\u01B0\u1EDBc cu\u1ED1i c\xF9ng l\xE0 th\xEAm v\xE0o m\u1ED9t tr\xECnh gi\u1EA3i m\xE3:"),md.forEach(n),uc=p(t),u(on.$$.fragment,t),kc=p(t),Xt=c(t,"P",{});var sa=l(Xt);Vm=o(sa,"H\xE3y c\u0169ng ki\u1EC3m th\u1EED v\u1EDBi "),lr=c(sa,"CODE",{});var ud=l(lr);Km=o(ud,"encoding"),ud.forEach(n),Jm=o(sa,":"),sa.forEach(n),fc=p(t),u(rn.$$.fragment,t),gc=p(t),u(cn.$$.fragment,t),dc=p(t),ps=c(t,"P",{});var kd=l(ps);Ym=o(kd,"Tuy\u1EC7t v\u1EDDi! Ta c\xF3 th\u1EC3 l\u01B0u tokenizer c\u1EE7a m\xECnh v\xE0o trong m\u1ED9t t\u1EC7p JSON nh\u01B0 d\u01B0\u1EDBi \u0111\xE2y:"),kd.forEach(n),vc=p(t),u(ln.$$.fragment,t),_c=p(t),vt=c(t,"P",{});var Ss=l(vt);Zm=o(Ss,"Ta sau \u0111\xF3 c\xF3 th\u1EC3 t\u1EA3i l\u1EA1i t\u1EC7p n\xE0y trong \u0111\u1ED1i t\u01B0\u1EE3ng "),ar=c(Ss,"CODE",{});var fd=l(ar);Qm=o(fd,"Tokenizer"),fd.forEach(n),tu=o(Ss," v\u1EDBi ph\u01B0\u01A1ng th\u1EE9c "),hr=c(Ss,"CODE",{});var gd=l(hr);eu=o(gd,"from_file()"),gd.forEach(n),nu=o(Ss,":"),Ss.forEach(n),$c=p(t),u(an.$$.fragment,t),zc=p(t),_t=c(t,"P",{});var Bs=l(_t);su=o(Bs,"\u0110\u1EC3 s\u1EED d\u1EE5ng tokenizer n\xE0y trong \u{1F917} Transformers, ch\xFAng ta ph\u1EA3i b\u1ECDc n\xF3 trong "),pr=c(Bs,"CODE",{});var dd=l(pr);ou=o(dd,"PreTrainedTokenizerFast"),dd.forEach(n),ru=o(Bs,". Ch\xFAng ta c\xF3 th\u1EC3 s\u1EED d\u1EE5ng l\u1EDBp chung ho\u1EB7c, n\u1EBFu tokenizer c\u1EE7a ch\xFAng ta t\u01B0\u01A1ng \u1EE9ng v\u1EDBi m\u1ED9t m\xF4 h\xECnh hi\u1EC7n c\xF3, h\xE3y s\u1EED d\u1EE5ng l\u1EDBp \u0111\xF3 (\u1EDF \u0111\xE2y l\xE0 "),mr=c(Bs,"CODE",{});var vd=l(mr);iu=o(vd,"BertTokenizerFast"),vd.forEach(n),cu=o(Bs,"). N\u1EBFu b\u1EA1n \xE1p d\u1EE5ng b\xE0i h\u1ECDc n\xE0y \u0111\u1EC3 x\xE2y d\u1EF1ng m\u1ED9t tokenizer ho\xE0n to\xE0n m\u1EDBi, b\u1EA1n s\u1EBD ph\u1EA3i s\u1EED d\u1EE5ng t\xF9y ch\u1ECDn \u0111\u1EA7u ti\xEAn."),Bs.forEach(n),bc=p(t),D=c(t,"P",{});var Z=l(D);lu=o(Z,"\u0110\u1EC3 b\u1ECDc tokenizer trong m\u1ED9t "),ur=c(Z,"CODE",{});var _d=l(ur);au=o(_d,"PreTrainedTokenizerFast"),_d.forEach(n),hu=o(Z,", ch\xFAng ta c\xF3 th\u1EC3 chuy\u1EC3n tokenizer m\xE0 ch\xFAng ta \u0111\xE3 x\xE2y d\u1EF1ng d\u01B0\u1EDBi d\u1EA1ng "),kr=c(Z,"CODE",{});var $d=l(kr);pu=o($d,"tokenizer_object"),$d.forEach(n),mu=o(Z," ho\u1EB7c truy\u1EC1n t\u1EC7p tokenizer m\xE0 ch\xFAng ta \u0111\xE3 l\u01B0u d\u01B0\u1EDBi d\u1EA1ng "),fr=c(Z,"CODE",{});var zd=l(fr);uu=o(zd,"tokenizer_file"),zd.forEach(n),ku=o(Z,". \u0110i\u1EC1u quan tr\u1ECDng c\u1EA7n nh\u1EDB l\xE0 ch\xFAng ta ph\u1EA3i \u0111\u1EB7t th\u1EE7 c\xF4ng t\u1EA5t c\u1EA3 c\xE1c token \u0111\u1EB7c bi\u1EC7t, v\xEC l\u1EDBp \u0111\xF3 kh\xF4ng th\u1EC3 suy ra t\u1EEB \u0111\u1ED1i t\u01B0\u1EE3ng "),gr=c(Z,"CODE",{});var bd=l(gr);fu=o(bd,"tokenizer"),bd.forEach(n),gu=o(Z," token n\xE0o l\xE0 token b\u1ECB che, "),dr=c(Z,"CODE",{});var Ed=l(dr);du=o(Ed,"[CLS]"),Ed.forEach(n),vu=o(Z,", v.v.:"),Z.forEach(n),Ec=p(t),u(hn.$$.fragment,t),xc=p(t),Vt=c(t,"P",{});var oa=l(Vt);_u=o(oa,"N\u1EBFu b\u1EA1n \u0111ang s\u1EF1 d\u1EE5ng m\u1ED9t l\u1EDBp tokenizer \u0111\u1EB7c bi\u1EC7t (nh\u01B0 "),vr=c(oa,"CODE",{});var xd=l(vr);$u=o(xd,"BertTokenizerFast"),xd.forEach(n),zu=o(oa,"), b\u1EA1n ch\u1EC9 c\u1EA7n ch\u1EC9 \u0111\u1ECBnh m\u1ED9t token \u0111\u1EB7c bi\u1EBFt kh\xE1c so v\u1EDBi m\u1EB7c \u0111\u1ECBnh (\u1EDF \u0111\xE2y l\xE0 kh\xF4ng x\xE1c \u0111\u1ECBnh):"),oa.forEach(n),jc=p(t),u(pn.$$.fragment,t),yc=p(t),$t=c(t,"P",{});var Ns=l($t);bu=o(Ns,"B\u1EA1n c\xF3 th\u1EC3 s\u1EED d\u1EE5ng tokenizer nh\u01B0 b\u1EA5t k\u1EF3 tokenizer n\xE0o kh\xE1c c\u1EE7a \u{1F917} Transformers. B\u1EA1n c\xF3 th\u1EC3 l\u01B0u n\xF3 v\u1EDBi ph\u01B0\u01A1ng th\u1EE9c "),_r=c(Ns,"CODE",{});var jd=l(_r);Eu=o(jd,"save_pretrained()"),jd.forEach(n),xu=o(Ns,", ho\u1EB7c l\u1EA1i n\xF3 l\xEAn Hub s\u1EED d\u1EE5ng ph\u01B0\u01A1ng th\u1EE9c "),$r=c(Ns,"CODE",{});var yd=l($r);ju=o(yd,"push_to_hub()"),yd.forEach(n),yu=o(Ns,"."),Ns.forEach(n),Tc=p(t),ms=c(t,"P",{});var Td=l(ms);Tu=o(Td,"Gi\u1EDD ch\xFAng ta \u0111\xE3 th\u1EA5y c\xE1ch x\xE2y d\u1EF1ng b\u1ED9 WordPiece tokenizer, h\xE3y l\xE0m t\u01B0\u01A1ng t\u1EF1 \u0111\u1ED1i v\u1EDBi BPE tokenizer. Ch\xFAng ta s\u1EBD ti\u1EBFn h\xE0nh nhanh h\u01A1n m\u1ED9t ch\xFAt v\xEC b\u1EA1n \u0111\xE3 bi\u1EBFt t\u1EA5t c\u1EA3 c\xE1c b\u01B0\u1EDBc v\xE0 ch\u1EC9 l\xE0m n\u1ED5i b\u1EADt nh\u1EEFng \u0111i\u1EC3m kh\xE1c bi\u1EC7t."),Td.forEach(n),qc=p(t),Tt=c(t,"H2",{class:!0});var ra=l(Tt);Kt=c(ra,"A",{id:!0,class:!0,href:!0});var qd=l(Kt);zr=c(qd,"SPAN",{});var Pd=l(zr);u(mn.$$.fragment,Pd),Pd.forEach(n),qd.forEach(n),qu=p(ra),br=c(ra,"SPAN",{});var wd=l(br);Pu=o(wd,"X\xE2y d\u1EF1ng m\u1ED9t BPE tokenizer t\u1EEB \u0111\u1EA7u"),wd.forEach(n),ra.forEach(n),Pc=p(t),Jt=c(t,"P",{});var ia=l(Jt);wu=o(ia,"Gi\u1EDD h\xE3y c\u0169ng nhau x\xE2y d\u1EF1ng GPT-2 tokenizer. Gi\u1ED1ng nh\u01B0 BERT tokenizer, ch\xFAng ta b\u1EAFt \u0111\u1EA7u b\u1EB1ng vi\u1EC7c kh\u1EDFi t\u1EA1o "),Er=c(ia,"CODE",{});var Cd=l(Er);Cu=o(Cd,"Tokenizer"),Cd.forEach(n),Du=o(ia," v\u1EDBi m\xF4 h\xECnh BPE:"),ia.forEach(n),wc=p(t),u(un.$$.fragment,t),Cc=p(t),I=c(t,"P",{});var ie=l(I);Ou=o(ie,"C\u0169ng gi\u1ED1ng nh\u01B0 BERT, ch\xFAng ta c\xF3 th\u1EC3 kh\u1EDFi t\u1EA1o m\xF4 h\xECnh n\xE0y v\u1EDBi m\u1ED9t b\u1ED9 t\u1EEB v\u1EF1ng n\u1EBFu ta \u0111\xE3 c\xF3 (ta s\u1EBD c\u1EA7n truy\u1EC1n v\xE0o "),xr=c(ie,"CODE",{});var Dd=l(xr);Lu=o(Dd,"vocab"),Dd.forEach(n),Su=o(ie," v\xE0 "),jr=c(ie,"CODE",{});var Od=l(jr);Bu=o(Od,"merges"),Od.forEach(n),Nu=o(ie," trong tr\u01B0\u1EDDng h\u1EE3p n\xE0y), nh\u01B0ng v\xEC ta s\u1EBD hu\u1EA5n luy\u1EC7n t\u1EEB \u0111\u1EA7u, ch\xFAng ta kh\xF4ng c\u1EA7n l\xE0m v\u1EADy. Ta c\u0169ng kh\xF4ng c\u1EA7n ch\u1EC9 \u0111\u1ECBnh "),yr=c(ie,"CODE",{});var Ld=l(yr);Fu=o(Ld,"unk_token"),Ld.forEach(n),Au=o(ie," v\xEC GPT-2 s\u1EED d\u1EE5ng BPE c\u1EA5p byte, ph\u01B0\u01A1ng ph\xE1p kh\xF4ng c\u1EA7n \u0111\u1EBFn n\xF3."),ie.forEach(n),Dc=p(t),us=c(t,"P",{});var Sd=l(us);Wu=o(Sd,"GPT-2 kh\xF4ng s\u1EED d\u1EE5ng m\u1ED9t tr\xECnh chu\u1EA9n ho\xE1, n\xEAn ta c\xF3 th\u1EC3 b\u1ECF qua b\u01B0\u1EDBc n\xE0y v\xE0 \u0111i tr\u1EF1c ti\u1EBFp v\xE0o b\u01B0\u1EDBc pre-tokenization:"),Sd.forEach(n),Oc=p(t),u(kn.$$.fragment,t),Lc=p(t),Yt=c(t,"P",{});var ca=l(Yt);Uu=o(ca,"Tu\u1EF3 ch\u1ECDn "),Tr=c(ca,"CODE",{});var Bd=l(Tr);Ru=o(Bd,"ByteLevel"),Bd.forEach(n),Gu=o(ca," ch\xFAng ta th\xEAm v\xE0o \u1EDF \u0111\xE2y kh\xF4ng th\xEAm d\u1EA5u c\xE1ch v\xE0o \u0111\u1EA7u c\u1EE7a m\u1ED9t c\xE2u (th\u01B0\u1EDDng n\xF3 l\xE0 m\u1EB7c \u0111\u1ECBnh). Ta c\xF3 th\u1EC3 nh\xECn c\xE1c pre-tokenization t\u1EEB v\xED d\u1EE5 t\u01B0\u01A1ng t\u1EF1 \u1EDF tr\xEAn:"),ca.forEach(n),Sc=p(t),u(fn.$$.fragment,t),Bc=p(t),u(gn.$$.fragment,t),Nc=p(t),ks=c(t,"P",{});var Nd=l(ks);Mu=o(Nd,"Ti\u1EBFp theo l\xE0 m\xF4 h\xECnh m\xE0 ta c\u1EA7n hu\u1EA5n luy\u1EC7n. V\u1EDBi GPT-2, token \u0111\u1EB7c bi\u1EC7t duy nh\u1EA5t ta c\u1EA7n l\xE0 token k\u1EBFt th\xFAc v\u0103n b\u1EA3n:"),Nd.forEach(n),Fc=p(t),u(dn.$$.fragment,t),Ac=p(t),y=c(t,"P",{});var B=l(y);Iu=o(B,"Nh\u01B0 v\u1EDBi "),qr=c(B,"CODE",{});var Fd=l(qr);Hu=o(Fd,"WordPieceTrainer"),Fd.forEach(n),Xu=o(B,", c\u0169ng nh\u01B0 "),Pr=c(B,"CODE",{});var Ad=l(Pr);Vu=o(Ad,"vocab_size"),Ad.forEach(n),Ku=o(B," v\xE0 "),wr=c(B,"CODE",{});var Wd=l(wr);Ju=o(Wd,"special_tokens"),Wd.forEach(n),Yu=o(B,", ta c\xF3 th\u1EC3 ch\u1EC9 \u0111\u1ECBnh "),Cr=c(B,"CODE",{});var Ud=l(Cr);Zu=o(Ud,"min_frequency"),Ud.forEach(n),Qu=o(B," n\u1EBFu mu\u1ED1n, ho\u1EB7c n\u1EBFu ta c\xF3 h\u1EADu t\u1ED1 k\u1EBFt th\xFAc t\u1EEB (nh\u01B0 "),Dr=c(B,"CODE",{});var Rd=l(Dr);tk=o(Rd,"</w>"),Rd.forEach(n),ek=o(B,"), ta c\xF3 th\u1EC3 thi\u1EBFt l\u1EADp n\xF3 v\u1EDBi "),Or=c(B,"CODE",{});var Gd=l(Or);nk=o(Gd,"end_of_word_suffix"),Gd.forEach(n),sk=o(B,"."),B.forEach(n),Wc=p(t),fs=c(t,"P",{});var Md=l(fs);ok=o(Md,"Tokenizer n\xE0y c\u0169ng c\xF3 th\u1EC3 \u0111\u01B0\u1EE3c hu\u1EA5n luy\u1EC7n tr\xEAn c\xE1c t\u1EC7p v\u0103n b\u1EA3n:"),Md.forEach(n),Uc=p(t),u(vn.$$.fragment,t),Rc=p(t),gs=c(t,"P",{});var Id=l(gs);rk=o(Id,"H\xE3y c\u0169ng xem k\u1EBFt qu\u1EA3 tokenize tr\xEAn m\u1ED9t v\u0103n b\u1EA3n m\u1EABu:"),Id.forEach(n),Gc=p(t),u(_n.$$.fragment,t),Mc=p(t),u($n.$$.fragment,t),Ic=p(t),ds=c(t,"P",{});var Hd=l(ds);ik=o(Hd,"Ta \xE1p d\u1EE5ng h\u1EADu x\u1EED l\xFD c\u1EA5p byte cho GPT-2 tokenizer nh\u01B0 sau:"),Hd.forEach(n),Hc=p(t),u(zn.$$.fragment,t),Xc=p(t),zt=c(t,"P",{});var Fs=l(zt);ck=o(Fs,"Tu\u1EF3 ch\u1ECDn "),Lr=c(Fs,"CODE",{});var Xd=l(Lr);lk=o(Xd,"trim_offsets = False"),Xd.forEach(n),ak=o(Fs," ch\u1EC9 cho tr\xECnh h\u1EADu x\u1EED l\xFD bi\u1EBFt r\u1EB1ng ta c\u1EA7n b\u1ECF m\u1ED1t s\u1ED1 offset token b\u1EAFt \u0111\u1EA7u v\u1EDBi \u2018\u0120\u2019: theo c\xE1ch n\xE0y, \u0111i\u1EC3m b\u1EAFt \u0111\u1EA7u c\u1EE7a offset s\u1EBD tr\u1ECF v\xE0o v\xF9ng kh\xF4ng gian ph\xEDa tr\u01B0\u1EDBc c\u1EE7a t\u1EEB, kh\xF4ng ph\u1EA3i k\xED t\u1EF1 \u0111\u1EA7u ti\xEAn c\u1EE7a t\u1EEB (v\xEC kh\xF4ng gian n\xE0y v\u1EC1 m\u1EB7t k\u1EF9 thu\u1EADt l\xE0 m\u1ED9t ph\u1EA7n c\u1EE7a t\u1EEB). H\xE3y c\xF9ng nh\xECn xem k\u1EBFt qu\u1EA3 v\u1EDBi chu\u1ED7i v\u0103n b\u1EA3n ta v\u1EEBa m\xE3 ho\xE1 v\u1EDBi "),Sr=c(Fs,"CODE",{});var Vd=l(Sr);hk=o(Vd,"'\u0120test'"),Vd.forEach(n),pk=o(Fs," l\xE0 token \u1EDF ch\u1EC9 m\u1EE5c 4:"),Fs.forEach(n),Vc=p(t),u(bn.$$.fragment,t),Kc=p(t),u(En.$$.fragment,t),Jc=p(t),vs=c(t,"P",{});var Kd=l(vs);mk=o(Kd,"Cu\u1ED1i c\xF9ng, ta th\xEAm m\u1ED9t tr\xECnh gi\u1EA3i m\xE3i c\u1EA5p byte:"),Kd.forEach(n),Yc=p(t),u(xn.$$.fragment,t),Zc=p(t),_s=c(t,"P",{});var Jd=l(_s);uk=o(Jd,"v\xE0 ta ki\u1EC3m tra l\u1EA1i xem n\xF3 ho\u1EA1t \u0111\u1ED9ng \u0111\xFAng ch\u01B0a:"),Jd.forEach(n),Qc=p(t),u(jn.$$.fragment,t),tl=p(t),u(yn.$$.fragment,t),el=p(t),bt=c(t,"P",{});var As=l(bt);kk=o(As,"Tuy\u1EC7t v\u1EDDi! Gi\u1EDD ta \u0111\xE3 xong r\u1ED3i, ta c\xF3 th\u1EC3 l\u01B0u tokenizer nh\u01B0 tr\xEAn, v\xE0 bao n\xF3 l\u1EA1i trong "),Br=c(As,"CODE",{});var Yd=l(Br);fk=o(Yd,"PreTrainedTokenizerFast"),Yd.forEach(n),gk=o(As," ho\u1EB7c  "),Nr=c(As,"CODE",{});var Zd=l(Nr);dk=o(Zd,"GPT2TokenizerFast"),Zd.forEach(n),vk=o(As," n\u1EBFu ta mu\u1ED1n n\xF3 trong \u{1F917} Transformers:"),As.forEach(n),nl=p(t),u(Tn.$$.fragment,t),sl=p(t),$s=c(t,"P",{});var Qd=l($s);_k=o(Qd,"or:"),Qd.forEach(n),ol=p(t),u(qn.$$.fragment,t),rl=p(t),zs=c(t,"P",{});var tv=l(zs);$k=o(tv,"Nh\u01B0 m\u1ED9t v\xED d\u1EE5 cu\u1ED1i, ch\xFAng t\xF4i s\u1EBD ch\u1EC9 b\u1EA1n c\xE1ch x\xE2y d\u1EF1ng m\u1ED9t Unigram tokenizer t\u1EEB \u0111\u1EA7u."),tv.forEach(n),il=p(t),qt=c(t,"H2",{class:!0});var la=l(qt);Zt=c(la,"A",{id:!0,class:!0,href:!0});var ev=l(Zt);Fr=c(ev,"SPAN",{});var nv=l(Fr);u(Pn.$$.fragment,nv),nv.forEach(n),ev.forEach(n),zk=p(la),Ar=c(la,"SPAN",{});var sv=l(Ar);bk=o(sv,"X\xE2y d\u1EF1ng m\u1ED9t Unigram tokenizer t\u1EEB \u0111\u1EA7u"),sv.forEach(n),la.forEach(n),cl=p(t),Qt=c(t,"P",{});var aa=l(Qt);Ek=o(aa,"H\xE3y c\xF9ng nhau x\xE2y d\u1EF1ng m\u1ED9t XLNet tokenizer. C\u0169ng gi\u1ED1ng nh\u01B0 c\xE1c tokenizer tr\u01B0\u1EDBc \u0111\xF3, ta c\xF3 th\u1EC3 b\u1EAFt \u0111\u1EA7u kh\u1EDFi t\u1EA1o "),Wr=c(aa,"CODE",{});var ov=l(Wr);xk=o(ov,"Tokenizer"),ov.forEach(n),jk=o(aa," v\u1EDBi m\u1ED9t m\xF4 h\xECnh Unigram:"),aa.forEach(n),ll=p(t),u(wn.$$.fragment,t),al=p(t),bs=c(t,"P",{});var rv=l(bs);yk=o(rv,"M\u1ED9t l\u1EA7n n\u1EEFa, ch\xFAng ta c\xF3 th\u1EC3 kh\u1EDFi t\u1EA1o m\xF4 h\xECnh n\xE0y v\u1EDBi m\u1ED9t t\u1EEB v\u1EF1ng n\u1EBFu c\xF3."),rv.forEach(n),hl=p(t),Es=c(t,"P",{});var iv=l(Es);Tk=o(iv,"V\u1EDBi s\u1EF1 chu\u1EA9n ho\xE1 n\xE0y, XLNet s\u1EED d\u1EE5ng m\u1ED9t v\xE0i ph\u01B0\u01A1ng ph\xE1p thay th\u1EBF (\u0111\u1EBFn t\u1EEB SentencePiece):"),iv.forEach(n),pl=p(t),u(Cn.$$.fragment,t),ml=p(t),H=c(t,"P",{});var ce=l(H);qk=o(ce,"\u0110i\u1EC1u n\xE0y thay th\u1EBF "),Ur=c(ce,"CODE",{});var cv=l(Ur);Pk=o(cv,"\u201C"),cv.forEach(n),wk=o(ce," and "),Rr=c(ce,"CODE",{});var lv=l(Rr);Ck=o(lv,"\u201D"),lv.forEach(n),Dk=o(ce," b\u1EB1ng "),Gr=c(ce,"CODE",{});var av=l(Gr);Ok=o(av,"\u201D"),av.forEach(n),Lk=o(ce," v\xE0 thay th\u1EBF b\u1EA5t k\xEC chu\u1ED7i n\xE0o ch\u1EE9a hai ho\u1EB7c nhi\u1EC1u h\u01A1n d\u1EA5u c\xE1ch li\u1EC1n nhau th\xE0nh m\u1ED9t d\u1EA5u duy nh\u1EA5t, c\u0169ng nh\u01B0 lo\u1EA1i b\u1ECF c\xE1c d\u1EA5u c\xF3 trong v\u0103n b\u1EA3n \u0111\u1EC3 tokenize."),ce.forEach(n),ul=p(t),te=c(t,"P",{});var ha=l(te);Sk=o(ha,"Ti\u1EC1n tokenizer \u0111\u01B0\u1EE3c s\u1EED d\u1EE5ng cho b\u1EA5t k\u1EF3 SentencePiece tokenizer l\xE0 "),Mr=c(ha,"CODE",{});var hv=l(Mr);Bk=o(hv,"Metaspace"),hv.forEach(n),Nk=o(ha,":"),ha.forEach(n),kl=p(t),u(Dn.$$.fragment,t),fl=p(t),xs=c(t,"P",{});var pv=l(xs);Fk=o(pv,"Ta c\xF3 th\u1EC3 nh\xECn v\xE0o \u0111\u1EA7u ra quy tr\xECnh ti\u1EC1n tokenize qua v\xED d\u1EE5 v\u0103n b\u1EA3n \u1EDF d\u01B0\u1EDBi:"),pv.forEach(n),gl=p(t),u(On.$$.fragment,t),dl=p(t),u(Ln.$$.fragment,t),vl=p(t),js=c(t,"P",{});var mv=l(js);Ak=o(mv,"Ti\u1EBFp theo l\xE0 m\xF4 h\xECnh ta c\u1EA7n hu\u1EA5n luy\u1EC7n. XLNet c\xF3 m\u1ED9t s\u1ED1 token \u0111\u1EB7c bi\u1EC7t:"),mv.forEach(n),_l=p(t),u(Sn.$$.fragment,t),$l=p(t),O=c(t,"P",{});var Et=l(O);Wk=o(Et,"M\u1ED9t tham s\u1ED1 v\xF4 c\xF9ng quan trong m\xE0 ta kh\xF4ng th\u1EC3 qu\xEAn c\u1EE7a "),Ir=c(Et,"CODE",{});var uv=l(Ir);Uk=o(uv,"UnigramTrainer"),uv.forEach(n),Rk=o(Et," l\xE0 "),Hr=c(Et,"CODE",{});var kv=l(Hr);Gk=o(kv,"unk_token"),kv.forEach(n),Mk=o(Et,". Ta c\xF3 th\u1EC3 truy\u1EC1n v\xE0o c\xE1c tham s\u1ED1 c\u1EE5 th\u1EC3 kh\xE1c t\u1EDBi thu\u1EADt to\xE1n Unigram, v\xED d\u1EE5 "),Xr=c(Et,"CODE",{});var fv=l(Xr);Ik=o(fv,"shrinking_factor"),fv.forEach(n),Hk=o(Et," cho c\xE1c b\u01B0\u1EDBc m\xE0 ta xo\xE1 token (m\u1EB7c \u0111\u1ECBnh l\xE0 0.75) ho\u1EB7c "),Vr=c(Et,"CODE",{});var gv=l(Vr);Xk=o(gv,"max_piece_length"),gv.forEach(n),Vk=o(Et," \u0111\u1EC3 ch\u1EC9 \u0111\u1ECBnh \u0111\u1ED9 d\xE0i t\u1ED1i \u0111a c\u1EE7a m\u1ED9t token (m\u1EB7c \u0111\u1ECBnh l\xE0 16)."),Et.forEach(n),zl=p(t),ys=c(t,"P",{});var dv=l(ys);Kk=o(dv,"Tokenizer n\xE0y c\xF3 th\u1EC3 \u0111\u01B0\u1EE3c hu\u1EA5n luy\u1EC7n tr\xEAn c\xE1c t\u1EC7p v\u0103n b\u1EA3n:"),dv.forEach(n),bl=p(t),u(Bn.$$.fragment,t),El=p(t),Ts=c(t,"P",{});var vv=l(Ts);Jk=o(vv,"H\xE3y c\xF9ng nh\xECn xem k\u1EBFt qu\u1EA3 tokenize tr\xEAn t\u1EADp m\u1EABu:"),vv.forEach(n),xl=p(t),u(Nn.$$.fragment,t),jl=p(t),u(Fn.$$.fragment,t),yl=p(t),X=c(t,"P",{});var le=l(X);Yk=o(le,"M\u1ED9t \u0111i\u1EC3m \u0111\u1EB7c bi\u1EC7t c\u1EE7a XLNet \u0111\xF3 l\xE0 n\xF3 th\xEAm token "),Kr=c(le,"CODE",{});var _v=l(Kr);Zk=o(_v,"<cls>"),_v.forEach(n),Qk=o(le," \u1EDF cu\u1ED1i m\u1ED7i c\xE2u, v\u1EDBi ki\u1EC3u ID l\xE0 2 (\u0111\u1EC3 ph\xE2n bi\u1EBFt v\u1EDBi c\xE1c token kh\xE1c). N\xF3 \u0111\u1EC7m th\xEAm v\xE0o ph\xEDa b\xEAn tay tr\xE1i gi\u1ED1ng nh\u01B0 k\u1EBFt qu\u1EA3 \u1EDF tr\xEAn. Ta c\xF3 th\u1EC3 x\u1EED l\xFD t\u1EA5t c\u1EA3 c\xE1c token \u0111\u1EB7c bi\u1EC7t v\xE0 c\xE1c token ki\u1EC3u ID v\u1EDBi c\xF9ng m\u1ED9t b\u1EA3n m\u1EABu, nh\u01B0 BERT, nh\u01B0ng \u0111\u1EA7u ti\xEAn ta ph\u1EA3i l\u1EA5y c\xE1c ID c\u1EE7a token "),Jr=c(le,"CODE",{});var $v=l(Jr);tf=o($v,"<cls>"),$v.forEach(n),ef=o(le," v\xE0 "),Yr=c(le,"CODE",{});var zv=l(Yr);nf=o(zv,"<sep>"),zv.forEach(n),sf=o(le,":"),le.forEach(n),Tl=p(t),u(An.$$.fragment,t),ql=p(t),u(Wn.$$.fragment,t),Pl=p(t),qs=c(t,"P",{});var bv=l(qs);of=o(bv,"B\u1EA3n m\u1EABu s\u1EBD tr\xF4ng nh\u01B0 sau:"),bv.forEach(n),wl=p(t),u(Un.$$.fragment,t),Cl=p(t),Ps=c(t,"P",{});var Ev=l(Ps);rf=o(Ev,"V\xE0 ta c\xF3 th\u1EC3 ki\u1EC3m tra xem n\xF3 ho\u1EA1t \u0111\u1ED9ng kh\xF4ng b\u1EB1ng c\xE1ch m\xE3 ho\xE1 c\u1EB7p c\xE2u:"),Ev.forEach(n),Dl=p(t),u(Rn.$$.fragment,t),Ol=p(t),u(Gn.$$.fragment,t),Ll=p(t),ee=c(t,"P",{});var pa=l(ee);cf=o(pa,"Cu\u1ED1i c\xF9ng, ta s\u1EBD th\xEAm tr\xECnh gi\u1EA3i m\xE3 "),Zr=c(pa,"CODE",{});var xv=l(Zr);lf=o(xv,"Metaspace"),xv.forEach(n),af=o(pa,":"),pa.forEach(n),Sl=p(t),u(Mn.$$.fragment,t),Bl=p(t),V=c(t,"P",{});var ae=l(V);hf=o(ae,"v\xE0 ta \u0111\xE3 xong v\u1EDBi tokenizer n\xE0y! Ta c\xF3 th\u1EC3 l\u01B0u tokenizer nh\u01B0 tr\xEAn, v\xE0 bao n\xF3 l\u1EA1i trong "),Qr=c(ae,"CODE",{});var jv=l(Qr);pf=o(jv,"PreTrainedTokenizerFast"),jv.forEach(n),mf=o(ae," ho\u1EB7c  "),ti=c(ae,"CODE",{});var yv=l(ti);uf=o(yv,"XLNetTokenizerFast"),yv.forEach(n),kf=o(ae," n\u1EBFu ta mu\u1ED1n n\xF3 trong \u{1F917} Transformers. M\u1ED9t \u0111i\u1EC3m c\u1EA7n l\u01B0u \xFD l\xE0 khi s\u1EED d\u1EE5ng "),ei=c(ae,"CODE",{});var Tv=l(ei);ff=o(Tv,"PreTrainedTokenizerFast"),Tv.forEach(n),gf=o(ae," th\xEC tr\xEAn \u0111\u1EA7u c\u1EE7a c\xE1c token \u0111\u1EB7c bi\u1EC7t ta c\u1EA7n n\xF3i cho th\u01B0 vi\u1EC7n \u{1F917} Transformers vi\u1EBFt ta c\u1EA7n \u0111\u1EC7m v\xE0o ph\xEDa b\xEAn tr\xE1i:"),ae.forEach(n),Nl=p(t),u(In.$$.fragment,t),Fl=p(t),ws=c(t,"P",{});var qv=l(ws);df=o(qv,"Ho\u1EB7c m\u1ED9t c\xE1ch kh\xE1c:"),qv.forEach(n),Al=p(t),u(Hn.$$.fragment,t),Wl=p(t),Cs=c(t,"P",{});var Pv=l(Cs);vf=o(Pv,"B\xE2y gi\u1EDD b\u1EA1n \u0111\xE3 th\u1EA5y c\xE1ch c\xE1c kh\u1ED1i kh\xE1c nhau \u0111\u01B0\u1EE3c s\u1EED d\u1EE5ng \u0111\u1EC3 x\xE2y d\u1EF1ng c\xE1c tokenizer hi\u1EC7n nay, b\u1EA1n s\u1EBD c\xF3 th\u1EC3 vi\u1EBFt b\u1EA5t k\u1EF3 tr\xECnh tokenize n\xE0o m\xE0 b\u1EA1n mu\u1ED1n v\u1EDBi th\u01B0 vi\u1EC7n \u{1F917} Tokenizers v\xE0 c\xF3 th\u1EC3 s\u1EED d\u1EE5ng n\xF3 trong \u{1F917} Transformers."),Pv.forEach(n),this.h()},h(){_($,"name","hf:doc:metadata"),_($,"content",JSON.stringify(Uv)),_(W,"id","xy-dng-tng-khi-tokenizer"),_(W,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),_(W,"href","#xy-dng-tng-khi-tokenizer"),_(N,"class","relative group"),_(lt,"class","block dark:hidden"),wv(lt.src,Ws="https://huggingface.co/datasets/huggingface-course/documentation-images/resolve/main/en/chapter6/tokenization_pipeline.svg")||_(lt,"src",Ws),_(lt,"alt","The tokenization pipeline."),_(at,"class","hidden dark:block"),wv(at.src,Us="https://huggingface.co/datasets/huggingface-course/documentation-images/resolve/main/en/chapter6/tokenization_pipeline-dark.svg")||_(at,"src",Us),_(at,"alt","The tokenization pipeline."),_(U,"class","flex justify-center"),_(ts,"href","/course/chapter6/2"),_(ze,"href","https://huggingface.co/docs/tokenizers/python/latest/api/reference.html#module-tokenizers.normalizers"),_(ze,"rel","nofollow"),_(be,"href","https://huggingface.co/docs/tokenizers/python/latest/api/reference.html#module-tokenizers.pre_tokenizers"),_(be,"rel","nofollow"),_(Ee,"href","https://huggingface.co/docs/tokenizers/python/latest/api/reference.html#module-tokenizers.models"),_(Ee,"rel","nofollow"),_(xe,"href","https://huggingface.co/docs/tokenizers/python/latest/api/reference.html#module-tokenizers.trainers"),_(xe,"rel","nofollow"),_(je,"href","https://huggingface.co/docs/tokenizers/python/latest/api/reference.html#module-tokenizers.processors"),_(je,"rel","nofollow"),_(ye,"href","https://huggingface.co/docs/tokenizers/python/latest/components.html#decoders"),_(ye,"rel","nofollow"),_(Te,"href","https://huggingface.co/docs/tokenizers/python/latest/components.html"),_(Te,"rel","nofollow"),_(St,"id","thu-thp-mt-kho-ng-liu"),_(St,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),_(St,"href","#thu-thp-mt-kho-ng-liu"),_(jt,"class","relative group"),_(es,"href","/course/chapter6/2"),_(Pe,"href","https://huggingface.co/datasets/wikitext"),_(Pe,"rel","nofollow"),_(Nt,"id","xy-dng-mt-wordpiece-tokenizer-t-u"),_(Nt,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),_(Nt,"href","#xy-dng-mt-wordpiece-tokenizer-t-u"),_(yt,"class","relative group"),_(Kt,"id","xy-dng-mt-bpe-tokenizer-t-u"),_(Kt,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),_(Kt,"href","#xy-dng-mt-bpe-tokenizer-t-u"),_(Tt,"class","relative group"),_(Zt,"id","xy-dng-mt-unigram-tokenizer-t-u"),_(Zt,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),_(Zt,"href","#xy-dng-mt-unigram-tokenizer-t-u"),_(qt,"class","relative group")},m(t,r){e(document.head,$),a(t,it,r),a(t,N,r),e(N,W),e(W,Q),k(tt,Q,null),e(N,he),e(N,et),e(et,pe),a(t,Pt,r),k(F,t,r),a(t,wt,r),a(t,ct,r),e(ct,xt),a(t,Ct,r),a(t,T,r),e(T,nt),e(nt,me),e(T,ue),e(T,st),e(st,ke),e(T,fe),e(T,ot),e(ot,ge),e(T,de),e(T,rt),e(rt,z),a(t,ve,r),a(t,U,r),e(U,lt),e(U,Qn),e(U,at),a(t,_e,r),a(t,Dt,r),e(Dt,ma),e(Dt,ts),e(ts,ua),e(Dt,ka),a(t,oi,r),k($e,t,r),a(t,ri,r),a(t,Ot,r),e(Ot,fa),e(Ot,Rs),e(Rs,ga),e(Ot,da),a(t,ii,r),a(t,q,r),e(q,ht),e(ht,Gs),e(Gs,va),e(ht,_a),e(ht,Ms),e(Ms,$a),e(ht,za),e(ht,ze),e(ze,ba),e(ht,Ea),e(q,xa),e(q,pt),e(pt,Is),e(Is,ja),e(pt,ya),e(pt,Hs),e(Hs,Ta),e(pt,qa),e(pt,be),e(be,Pa),e(pt,wa),e(q,Ca),e(q,P),e(P,Xs),e(Xs,Da),e(P,Oa),e(P,Vs),e(Vs,La),e(P,Sa),e(P,Ks),e(Ks,Ba),e(P,Na),e(P,Js),e(Js,Fa),e(P,Aa),e(P,Ys),e(Ys,Wa),e(P,Ua),e(P,Ee),e(Ee,Ra),e(P,Ga),e(q,Ma),e(q,mt),e(mt,Zs),e(Zs,Ia),e(mt,Ha),e(mt,Qs),e(Qs,Xa),e(mt,Va),e(mt,xe),e(xe,Ka),e(mt,Ja),e(q,Ya),e(q,ut),e(ut,to),e(to,Za),e(ut,Qa),e(ut,eo),e(eo,th),e(ut,eh),e(ut,je),e(je,nh),e(ut,sh),e(q,oh),e(q,kt),e(kt,no),e(no,rh),e(kt,ih),e(kt,so),e(so,ch),e(kt,lh),e(kt,ye),e(ye,ah),e(kt,hh),a(t,ci,r),a(t,Lt,r),e(Lt,ph),e(Lt,Te),e(Te,mh),e(Lt,uh),a(t,li,r),a(t,jt,r),e(jt,St),e(St,oo),k(qe,oo,null),e(jt,kh),e(jt,ro),e(ro,fh),a(t,ai,r),a(t,ft,r),e(ft,gh),e(ft,es),e(es,dh),e(ft,vh),e(ft,Pe),e(Pe,_h),e(ft,$h),a(t,hi,r),k(we,t,r),a(t,pi,r),a(t,Bt,r),e(Bt,zh),e(Bt,io),e(io,bh),e(Bt,Eh),a(t,mi,r),a(t,ns,r),e(ns,xh),a(t,ui,r),k(Ce,t,r),a(t,ki,r),a(t,ss,r),e(ss,jh),a(t,fi,r),a(t,yt,r),e(yt,Nt),e(Nt,co),k(De,co,null),e(yt,yh),e(yt,lo),e(lo,Th),a(t,gi,r),a(t,x,r),e(x,qh),e(x,ao),e(ao,Ph),e(x,wh),e(x,ho),e(ho,Ch),e(x,Dh),e(x,po),e(po,Oh),e(x,Lh),e(x,mo),e(mo,Sh),e(x,Bh),e(x,uo),e(uo,Nh),e(x,Fh),e(x,ko),e(ko,Ah),e(x,Wh),a(t,di,r),a(t,Ft,r),e(Ft,Uh),e(Ft,fo),e(fo,Rh),e(Ft,Gh),a(t,vi,r),k(Oe,t,r),a(t,_i,r),a(t,R,r),e(R,Mh),e(R,go),e(go,Ih),e(R,Hh),e(R,vo),e(vo,Xh),e(R,Vh),e(R,_o),e(_o,Kh),e(R,Jh),a(t,$i,r),a(t,j,r),e(j,Yh),e(j,$o),e($o,Zh),e(j,Qh),e(j,zo),e(zo,tp),e(j,ep),e(j,bo),e(bo,np),e(j,sp),e(j,Eo),e(Eo,op),e(j,rp),e(j,xo),e(xo,ip),e(j,cp),e(j,jo),e(jo,lp),e(j,ap),a(t,zi,r),k(Le,t,r),a(t,bi,r),a(t,G,r),e(G,hp),e(G,yo),e(yo,pp),e(G,mp),e(G,To),e(To,up),e(G,kp),e(G,qo),e(qo,fp),e(G,gp),a(t,Ei,r),k(Se,t,r),a(t,xi,r),a(t,gt,r),e(gt,dp),e(gt,Po),e(Po,vp),e(gt,_p),e(gt,wo),e(wo,$p),e(gt,zp),a(t,ji,r),a(t,dt,r),e(dt,bp),e(dt,Co),e(Co,Ep),e(dt,xp),e(dt,Do),e(Do,jp),e(dt,yp),a(t,yi,r),k(Be,t,r),a(t,Ti,r),k(Ne,t,r),a(t,qi,r),k(At,t,r),a(t,Pi,r),a(t,Wt,r),e(Wt,Tp),e(Wt,Oo),e(Oo,qp),e(Wt,Pp),a(t,wi,r),k(Fe,t,r),a(t,Ci,r),a(t,os,r),e(os,wp),a(t,Di,r),k(Ae,t,r),a(t,Oi,r),a(t,Ut,r),e(Ut,Cp),e(Ut,Lo),e(Lo,Dp),e(Ut,Op),a(t,Li,r),k(We,t,r),a(t,Si,r),k(Ue,t,r),a(t,Bi,r),a(t,Rt,r),e(Rt,Lp),e(Rt,So),e(So,Sp),e(Rt,Bp),a(t,Ni,r),k(Re,t,r),a(t,Fi,r),k(Ge,t,r),a(t,Ai,r),a(t,Gt,r),e(Gt,Np),e(Gt,Bo),e(Bo,Fp),e(Gt,Ap),a(t,Wi,r),k(Me,t,r),a(t,Ui,r),k(Ie,t,r),a(t,Ri,r),a(t,Mt,r),e(Mt,Wp),e(Mt,No),e(No,Up),e(Mt,Rp),a(t,Gi,r),k(He,t,r),a(t,Mi,r),a(t,w,r),e(w,Gp),e(w,Fo),e(Fo,Mp),e(w,Ip),e(w,Ao),e(Ao,Hp),e(w,Xp),e(w,Wo),e(Wo,Vp),e(w,Kp),e(w,Uo),e(Uo,Jp),e(w,Yp),e(w,Ro),e(Ro,Zp),e(w,Qp),a(t,Ii,r),a(t,rs,r),e(rs,tm),a(t,Hi,r),k(Xe,t,r),a(t,Xi,r),a(t,It,r),e(It,em),e(It,Go),e(Go,nm),e(It,sm),a(t,Vi,r),k(Ve,t,r),a(t,Ki,r),a(t,Ht,r),e(Ht,om),e(Ht,Mo),e(Mo,rm),e(Ht,im),a(t,Ji,r),k(Ke,t,r),a(t,Yi,r),k(Je,t,r),a(t,Zi,r),a(t,b,r),e(b,Io),e(Io,cm),e(b,lm),e(b,Ho),e(Ho,am),e(b,hm),e(b,Xo),e(Xo,pm),e(b,mm),e(b,Vo),e(Vo,um),e(b,km),e(b,Ko),e(Ko,fm),e(b,gm),e(b,Jo),e(Jo,dm),e(b,vm),e(b,Yo),e(Yo,_m),e(b,$m),e(b,Zo),e(Zo,zm),e(b,bm),e(b,Qo),e(Qo,Em),e(b,xm),a(t,Qi,r),a(t,C,r),e(C,jm),e(C,tr),e(tr,ym),e(C,Tm),e(C,er),e(er,qm),e(C,Pm),e(C,nr),e(nr,wm),e(C,Cm),e(C,sr),e(sr,Dm),e(C,Om),e(C,or),e(or,Lm),e(C,Sm),a(t,tc,r),k(Ye,t,r),a(t,ec,r),k(Ze,t,r),a(t,nc,r),a(t,M,r),e(M,Bm),e(M,rr),e(rr,Nm),e(M,Fm),e(M,ir),e(ir,Am),e(M,Wm),e(M,cr),e(cr,Um),e(M,Rm),a(t,sc,r),a(t,is,r),e(is,Gm),a(t,oc,r),k(Qe,t,r),a(t,rc,r),a(t,cs,r),e(cs,Mm),a(t,ic,r),a(t,ls,r),e(ls,Im),a(t,cc,r),k(tn,t,r),a(t,lc,r),k(en,t,r),a(t,ac,r),a(t,as,r),e(as,Hm),a(t,hc,r),k(nn,t,r),a(t,pc,r),k(sn,t,r),a(t,mc,r),a(t,hs,r),e(hs,Xm),a(t,uc,r),k(on,t,r),a(t,kc,r),a(t,Xt,r),e(Xt,Vm),e(Xt,lr),e(lr,Km),e(Xt,Jm),a(t,fc,r),k(rn,t,r),a(t,gc,r),k(cn,t,r),a(t,dc,r),a(t,ps,r),e(ps,Ym),a(t,vc,r),k(ln,t,r),a(t,_c,r),a(t,vt,r),e(vt,Zm),e(vt,ar),e(ar,Qm),e(vt,tu),e(vt,hr),e(hr,eu),e(vt,nu),a(t,$c,r),k(an,t,r),a(t,zc,r),a(t,_t,r),e(_t,su),e(_t,pr),e(pr,ou),e(_t,ru),e(_t,mr),e(mr,iu),e(_t,cu),a(t,bc,r),a(t,D,r),e(D,lu),e(D,ur),e(ur,au),e(D,hu),e(D,kr),e(kr,pu),e(D,mu),e(D,fr),e(fr,uu),e(D,ku),e(D,gr),e(gr,fu),e(D,gu),e(D,dr),e(dr,du),e(D,vu),a(t,Ec,r),k(hn,t,r),a(t,xc,r),a(t,Vt,r),e(Vt,_u),e(Vt,vr),e(vr,$u),e(Vt,zu),a(t,jc,r),k(pn,t,r),a(t,yc,r),a(t,$t,r),e($t,bu),e($t,_r),e(_r,Eu),e($t,xu),e($t,$r),e($r,ju),e($t,yu),a(t,Tc,r),a(t,ms,r),e(ms,Tu),a(t,qc,r),a(t,Tt,r),e(Tt,Kt),e(Kt,zr),k(mn,zr,null),e(Tt,qu),e(Tt,br),e(br,Pu),a(t,Pc,r),a(t,Jt,r),e(Jt,wu),e(Jt,Er),e(Er,Cu),e(Jt,Du),a(t,wc,r),k(un,t,r),a(t,Cc,r),a(t,I,r),e(I,Ou),e(I,xr),e(xr,Lu),e(I,Su),e(I,jr),e(jr,Bu),e(I,Nu),e(I,yr),e(yr,Fu),e(I,Au),a(t,Dc,r),a(t,us,r),e(us,Wu),a(t,Oc,r),k(kn,t,r),a(t,Lc,r),a(t,Yt,r),e(Yt,Uu),e(Yt,Tr),e(Tr,Ru),e(Yt,Gu),a(t,Sc,r),k(fn,t,r),a(t,Bc,r),k(gn,t,r),a(t,Nc,r),a(t,ks,r),e(ks,Mu),a(t,Fc,r),k(dn,t,r),a(t,Ac,r),a(t,y,r),e(y,Iu),e(y,qr),e(qr,Hu),e(y,Xu),e(y,Pr),e(Pr,Vu),e(y,Ku),e(y,wr),e(wr,Ju),e(y,Yu),e(y,Cr),e(Cr,Zu),e(y,Qu),e(y,Dr),e(Dr,tk),e(y,ek),e(y,Or),e(Or,nk),e(y,sk),a(t,Wc,r),a(t,fs,r),e(fs,ok),a(t,Uc,r),k(vn,t,r),a(t,Rc,r),a(t,gs,r),e(gs,rk),a(t,Gc,r),k(_n,t,r),a(t,Mc,r),k($n,t,r),a(t,Ic,r),a(t,ds,r),e(ds,ik),a(t,Hc,r),k(zn,t,r),a(t,Xc,r),a(t,zt,r),e(zt,ck),e(zt,Lr),e(Lr,lk),e(zt,ak),e(zt,Sr),e(Sr,hk),e(zt,pk),a(t,Vc,r),k(bn,t,r),a(t,Kc,r),k(En,t,r),a(t,Jc,r),a(t,vs,r),e(vs,mk),a(t,Yc,r),k(xn,t,r),a(t,Zc,r),a(t,_s,r),e(_s,uk),a(t,Qc,r),k(jn,t,r),a(t,tl,r),k(yn,t,r),a(t,el,r),a(t,bt,r),e(bt,kk),e(bt,Br),e(Br,fk),e(bt,gk),e(bt,Nr),e(Nr,dk),e(bt,vk),a(t,nl,r),k(Tn,t,r),a(t,sl,r),a(t,$s,r),e($s,_k),a(t,ol,r),k(qn,t,r),a(t,rl,r),a(t,zs,r),e(zs,$k),a(t,il,r),a(t,qt,r),e(qt,Zt),e(Zt,Fr),k(Pn,Fr,null),e(qt,zk),e(qt,Ar),e(Ar,bk),a(t,cl,r),a(t,Qt,r),e(Qt,Ek),e(Qt,Wr),e(Wr,xk),e(Qt,jk),a(t,ll,r),k(wn,t,r),a(t,al,r),a(t,bs,r),e(bs,yk),a(t,hl,r),a(t,Es,r),e(Es,Tk),a(t,pl,r),k(Cn,t,r),a(t,ml,r),a(t,H,r),e(H,qk),e(H,Ur),e(Ur,Pk),e(H,wk),e(H,Rr),e(Rr,Ck),e(H,Dk),e(H,Gr),e(Gr,Ok),e(H,Lk),a(t,ul,r),a(t,te,r),e(te,Sk),e(te,Mr),e(Mr,Bk),e(te,Nk),a(t,kl,r),k(Dn,t,r),a(t,fl,r),a(t,xs,r),e(xs,Fk),a(t,gl,r),k(On,t,r),a(t,dl,r),k(Ln,t,r),a(t,vl,r),a(t,js,r),e(js,Ak),a(t,_l,r),k(Sn,t,r),a(t,$l,r),a(t,O,r),e(O,Wk),e(O,Ir),e(Ir,Uk),e(O,Rk),e(O,Hr),e(Hr,Gk),e(O,Mk),e(O,Xr),e(Xr,Ik),e(O,Hk),e(O,Vr),e(Vr,Xk),e(O,Vk),a(t,zl,r),a(t,ys,r),e(ys,Kk),a(t,bl,r),k(Bn,t,r),a(t,El,r),a(t,Ts,r),e(Ts,Jk),a(t,xl,r),k(Nn,t,r),a(t,jl,r),k(Fn,t,r),a(t,yl,r),a(t,X,r),e(X,Yk),e(X,Kr),e(Kr,Zk),e(X,Qk),e(X,Jr),e(Jr,tf),e(X,ef),e(X,Yr),e(Yr,nf),e(X,sf),a(t,Tl,r),k(An,t,r),a(t,ql,r),k(Wn,t,r),a(t,Pl,r),a(t,qs,r),e(qs,of),a(t,wl,r),k(Un,t,r),a(t,Cl,r),a(t,Ps,r),e(Ps,rf),a(t,Dl,r),k(Rn,t,r),a(t,Ol,r),k(Gn,t,r),a(t,Ll,r),a(t,ee,r),e(ee,cf),e(ee,Zr),e(Zr,lf),e(ee,af),a(t,Sl,r),k(Mn,t,r),a(t,Bl,r),a(t,V,r),e(V,hf),e(V,Qr),e(Qr,pf),e(V,mf),e(V,ti),e(ti,uf),e(V,kf),e(V,ei),e(ei,ff),e(V,gf),a(t,Nl,r),k(In,t,r),a(t,Fl,r),a(t,ws,r),e(ws,df),a(t,Al,r),k(Hn,t,r),a(t,Wl,r),a(t,Cs,r),e(Cs,vf),Ul=!0},p(t,[r]){const Xn={};r&2&&(Xn.$$scope={dirty:r,ctx:t}),At.$set(Xn)},i(t){Ul||(f(tt.$$.fragment,t),f(F.$$.fragment,t),f($e.$$.fragment,t),f(qe.$$.fragment,t),f(we.$$.fragment,t),f(Ce.$$.fragment,t),f(De.$$.fragment,t),f(Oe.$$.fragment,t),f(Le.$$.fragment,t),f(Se.$$.fragment,t),f(Be.$$.fragment,t),f(Ne.$$.fragment,t),f(At.$$.fragment,t),f(Fe.$$.fragment,t),f(Ae.$$.fragment,t),f(We.$$.fragment,t),f(Ue.$$.fragment,t),f(Re.$$.fragment,t),f(Ge.$$.fragment,t),f(Me.$$.fragment,t),f(Ie.$$.fragment,t),f(He.$$.fragment,t),f(Xe.$$.fragment,t),f(Ve.$$.fragment,t),f(Ke.$$.fragment,t),f(Je.$$.fragment,t),f(Ye.$$.fragment,t),f(Ze.$$.fragment,t),f(Qe.$$.fragment,t),f(tn.$$.fragment,t),f(en.$$.fragment,t),f(nn.$$.fragment,t),f(sn.$$.fragment,t),f(on.$$.fragment,t),f(rn.$$.fragment,t),f(cn.$$.fragment,t),f(ln.$$.fragment,t),f(an.$$.fragment,t),f(hn.$$.fragment,t),f(pn.$$.fragment,t),f(mn.$$.fragment,t),f(un.$$.fragment,t),f(kn.$$.fragment,t),f(fn.$$.fragment,t),f(gn.$$.fragment,t),f(dn.$$.fragment,t),f(vn.$$.fragment,t),f(_n.$$.fragment,t),f($n.$$.fragment,t),f(zn.$$.fragment,t),f(bn.$$.fragment,t),f(En.$$.fragment,t),f(xn.$$.fragment,t),f(jn.$$.fragment,t),f(yn.$$.fragment,t),f(Tn.$$.fragment,t),f(qn.$$.fragment,t),f(Pn.$$.fragment,t),f(wn.$$.fragment,t),f(Cn.$$.fragment,t),f(Dn.$$.fragment,t),f(On.$$.fragment,t),f(Ln.$$.fragment,t),f(Sn.$$.fragment,t),f(Bn.$$.fragment,t),f(Nn.$$.fragment,t),f(Fn.$$.fragment,t),f(An.$$.fragment,t),f(Wn.$$.fragment,t),f(Un.$$.fragment,t),f(Rn.$$.fragment,t),f(Gn.$$.fragment,t),f(Mn.$$.fragment,t),f(In.$$.fragment,t),f(Hn.$$.fragment,t),Ul=!0)},o(t){g(tt.$$.fragment,t),g(F.$$.fragment,t),g($e.$$.fragment,t),g(qe.$$.fragment,t),g(we.$$.fragment,t),g(Ce.$$.fragment,t),g(De.$$.fragment,t),g(Oe.$$.fragment,t),g(Le.$$.fragment,t),g(Se.$$.fragment,t),g(Be.$$.fragment,t),g(Ne.$$.fragment,t),g(At.$$.fragment,t),g(Fe.$$.fragment,t),g(Ae.$$.fragment,t),g(We.$$.fragment,t),g(Ue.$$.fragment,t),g(Re.$$.fragment,t),g(Ge.$$.fragment,t),g(Me.$$.fragment,t),g(Ie.$$.fragment,t),g(He.$$.fragment,t),g(Xe.$$.fragment,t),g(Ve.$$.fragment,t),g(Ke.$$.fragment,t),g(Je.$$.fragment,t),g(Ye.$$.fragment,t),g(Ze.$$.fragment,t),g(Qe.$$.fragment,t),g(tn.$$.fragment,t),g(en.$$.fragment,t),g(nn.$$.fragment,t),g(sn.$$.fragment,t),g(on.$$.fragment,t),g(rn.$$.fragment,t),g(cn.$$.fragment,t),g(ln.$$.fragment,t),g(an.$$.fragment,t),g(hn.$$.fragment,t),g(pn.$$.fragment,t),g(mn.$$.fragment,t),g(un.$$.fragment,t),g(kn.$$.fragment,t),g(fn.$$.fragment,t),g(gn.$$.fragment,t),g(dn.$$.fragment,t),g(vn.$$.fragment,t),g(_n.$$.fragment,t),g($n.$$.fragment,t),g(zn.$$.fragment,t),g(bn.$$.fragment,t),g(En.$$.fragment,t),g(xn.$$.fragment,t),g(jn.$$.fragment,t),g(yn.$$.fragment,t),g(Tn.$$.fragment,t),g(qn.$$.fragment,t),g(Pn.$$.fragment,t),g(wn.$$.fragment,t),g(Cn.$$.fragment,t),g(Dn.$$.fragment,t),g(On.$$.fragment,t),g(Ln.$$.fragment,t),g(Sn.$$.fragment,t),g(Bn.$$.fragment,t),g(Nn.$$.fragment,t),g(Fn.$$.fragment,t),g(An.$$.fragment,t),g(Wn.$$.fragment,t),g(Un.$$.fragment,t),g(Rn.$$.fragment,t),g(Gn.$$.fragment,t),g(Mn.$$.fragment,t),g(In.$$.fragment,t),g(Hn.$$.fragment,t),Ul=!1},d(t){n($),t&&n(it),t&&n(N),d(tt),t&&n(Pt),d(F,t),t&&n(wt),t&&n(ct),t&&n(Ct),t&&n(T),t&&n(ve),t&&n(U),t&&n(_e),t&&n(Dt),t&&n(oi),d($e,t),t&&n(ri),t&&n(Ot),t&&n(ii),t&&n(q),t&&n(ci),t&&n(Lt),t&&n(li),t&&n(jt),d(qe),t&&n(ai),t&&n(ft),t&&n(hi),d(we,t),t&&n(pi),t&&n(Bt),t&&n(mi),t&&n(ns),t&&n(ui),d(Ce,t),t&&n(ki),t&&n(ss),t&&n(fi),t&&n(yt),d(De),t&&n(gi),t&&n(x),t&&n(di),t&&n(Ft),t&&n(vi),d(Oe,t),t&&n(_i),t&&n(R),t&&n($i),t&&n(j),t&&n(zi),d(Le,t),t&&n(bi),t&&n(G),t&&n(Ei),d(Se,t),t&&n(xi),t&&n(gt),t&&n(ji),t&&n(dt),t&&n(yi),d(Be,t),t&&n(Ti),d(Ne,t),t&&n(qi),d(At,t),t&&n(Pi),t&&n(Wt),t&&n(wi),d(Fe,t),t&&n(Ci),t&&n(os),t&&n(Di),d(Ae,t),t&&n(Oi),t&&n(Ut),t&&n(Li),d(We,t),t&&n(Si),d(Ue,t),t&&n(Bi),t&&n(Rt),t&&n(Ni),d(Re,t),t&&n(Fi),d(Ge,t),t&&n(Ai),t&&n(Gt),t&&n(Wi),d(Me,t),t&&n(Ui),d(Ie,t),t&&n(Ri),t&&n(Mt),t&&n(Gi),d(He,t),t&&n(Mi),t&&n(w),t&&n(Ii),t&&n(rs),t&&n(Hi),d(Xe,t),t&&n(Xi),t&&n(It),t&&n(Vi),d(Ve,t),t&&n(Ki),t&&n(Ht),t&&n(Ji),d(Ke,t),t&&n(Yi),d(Je,t),t&&n(Zi),t&&n(b),t&&n(Qi),t&&n(C),t&&n(tc),d(Ye,t),t&&n(ec),d(Ze,t),t&&n(nc),t&&n(M),t&&n(sc),t&&n(is),t&&n(oc),d(Qe,t),t&&n(rc),t&&n(cs),t&&n(ic),t&&n(ls),t&&n(cc),d(tn,t),t&&n(lc),d(en,t),t&&n(ac),t&&n(as),t&&n(hc),d(nn,t),t&&n(pc),d(sn,t),t&&n(mc),t&&n(hs),t&&n(uc),d(on,t),t&&n(kc),t&&n(Xt),t&&n(fc),d(rn,t),t&&n(gc),d(cn,t),t&&n(dc),t&&n(ps),t&&n(vc),d(ln,t),t&&n(_c),t&&n(vt),t&&n($c),d(an,t),t&&n(zc),t&&n(_t),t&&n(bc),t&&n(D),t&&n(Ec),d(hn,t),t&&n(xc),t&&n(Vt),t&&n(jc),d(pn,t),t&&n(yc),t&&n($t),t&&n(Tc),t&&n(ms),t&&n(qc),t&&n(Tt),d(mn),t&&n(Pc),t&&n(Jt),t&&n(wc),d(un,t),t&&n(Cc),t&&n(I),t&&n(Dc),t&&n(us),t&&n(Oc),d(kn,t),t&&n(Lc),t&&n(Yt),t&&n(Sc),d(fn,t),t&&n(Bc),d(gn,t),t&&n(Nc),t&&n(ks),t&&n(Fc),d(dn,t),t&&n(Ac),t&&n(y),t&&n(Wc),t&&n(fs),t&&n(Uc),d(vn,t),t&&n(Rc),t&&n(gs),t&&n(Gc),d(_n,t),t&&n(Mc),d($n,t),t&&n(Ic),t&&n(ds),t&&n(Hc),d(zn,t),t&&n(Xc),t&&n(zt),t&&n(Vc),d(bn,t),t&&n(Kc),d(En,t),t&&n(Jc),t&&n(vs),t&&n(Yc),d(xn,t),t&&n(Zc),t&&n(_s),t&&n(Qc),d(jn,t),t&&n(tl),d(yn,t),t&&n(el),t&&n(bt),t&&n(nl),d(Tn,t),t&&n(sl),t&&n($s),t&&n(ol),d(qn,t),t&&n(rl),t&&n(zs),t&&n(il),t&&n(qt),d(Pn),t&&n(cl),t&&n(Qt),t&&n(ll),d(wn,t),t&&n(al),t&&n(bs),t&&n(hl),t&&n(Es),t&&n(pl),d(Cn,t),t&&n(ml),t&&n(H),t&&n(ul),t&&n(te),t&&n(kl),d(Dn,t),t&&n(fl),t&&n(xs),t&&n(gl),d(On,t),t&&n(dl),d(Ln,t),t&&n(vl),t&&n(js),t&&n(_l),d(Sn,t),t&&n($l),t&&n(O),t&&n(zl),t&&n(ys),t&&n(bl),d(Bn,t),t&&n(El),t&&n(Ts),t&&n(xl),d(Nn,t),t&&n(jl),d(Fn,t),t&&n(yl),t&&n(X),t&&n(Tl),d(An,t),t&&n(ql),d(Wn,t),t&&n(Pl),t&&n(qs),t&&n(wl),d(Un,t),t&&n(Cl),t&&n(Ps),t&&n(Dl),d(Rn,t),t&&n(Ol),d(Gn,t),t&&n(Ll),t&&n(ee),t&&n(Sl),d(Mn,t),t&&n(Bl),t&&n(V),t&&n(Nl),d(In,t),t&&n(Fl),t&&n(ws),t&&n(Al),d(Hn,t),t&&n(Wl),t&&n(Cs)}}}const Uv={local:"xy-dng-tng-khi-tokenizer",sections:[{local:"thu-thp-mt-kho-ng-liu",title:"Thu th\u1EADp m\u1ED9t kho ng\u1EEF li\u1EC7u"},{local:"xy-dng-mt-wordpiece-tokenizer-t-u",title:"X\xE2y d\u1EF1ng m\u1ED9t WordPiece tokenizer t\u1EEB \u0111\u1EA7u"},{local:"xy-dng-mt-bpe-tokenizer-t-u",title:"X\xE2y d\u1EF1ng m\u1ED9t BPE tokenizer t\u1EEB \u0111\u1EA7u"},{local:"xy-dng-mt-unigram-tokenizer-t-u",title:"X\xE2y d\u1EF1ng m\u1ED9t Unigram tokenizer t\u1EEB \u0111\u1EA7u"}],title:"X\xE2y d\u1EF1ng t\u1EEBng kh\u1ED1i tokenizer"};function Rv(si){return Sv(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class Kv extends Cv{constructor($){super();Dv(this,$,Rv,Wv,Ov,{})}}export{Kv as default,Uv as metadata};
