import{S as cu,i as ku,s as _u,e as a,k as f,t as n,w as c,M as $u,c as p,d as t,m,a as i,h as r,x as k,b as g,N as mu,G as s,g as o,y as _,q as $,o as u,B as z,v as uu}from"../../chunks/vendor-hf-doc-builder.js";import{T as zu}from"../../chunks/Tip-hf-doc-builder.js";import{Y as hu}from"../../chunks/Youtube-hf-doc-builder.js";import{I as _k}from"../../chunks/IconCopyLink-hf-doc-builder.js";import{C as h}from"../../chunks/CodeBlock-hf-doc-builder.js";import{C as gu}from"../../chunks/CourseFloatingBanner-hf-doc-builder.js";import"../../chunks/DocNotebookDropdown-hf-doc-builder.js";function ju(Sl){let j,Y,K,Qe,J,H,ge,G,es,je,v,pe,ss,ie,fe,ts,me,ce,ns,ke,_e,rs;return{c(){j=a("p"),Y=a("strong"),K=n("\u66F4\u8FDB\u4E00\u6B65"),Qe=n(`\u5982\u679C\u60A8\u5728\u5305\u542B unicode \u5B57\u7B26\u7684\u5B57\u7B26\u4E32\u4E0A\u6D4B\u8BD5\u5148\u524Dnormalizers\u7684\u4E24\u4E2A\u7248\u672C\uFF0C\u60A8\u80AF\u5B9A\u4F1A\u6CE8\u610F\u5230\u8FD9\u4E24\u4E2Anormalizers\u5E76\u4E0D\u5B8C\u5168\u7B49\u6548\u3002
\u4E3A\u4E86\u4E0D\u8FC7\u5EA6\u4F7F\u7528 `),J=a("code"),H=n("normalizers.Sequence"),ge=n(" \u4F7F\u7248\u672C\u8FC7\u4E8E\u590D\u6742\uFF0C\u6211\u4EEC\u6CA1\u6709\u5305\u542B\u5F53 "),G=a("code"),es=n("clean_text"),je=n(" \u53C2\u6570\u8BBE\u7F6E\u4E3A "),v=a("code"),pe=n("True"),ss=n(" \u65F6 "),ie=a("code"),fe=n("BertNormalizer"),ts=n(" \u9700\u8981\u7684\u6B63\u5219\u8868\u8FBE\u5F0F\u66FF\u6362 - \u8FD9\u662F\u9ED8\u8BA4\u884C\u4E3A\u3002 \u4F46\u4E0D\u8981\u62C5\u5FC3\uFF1A\u901A\u8FC7\u5728normalizer\u5E8F\u5217\u4E2D\u6DFB\u52A0\u4E24\u4E2A "),me=a("code"),ce=n("normalizers.Replace"),ns=n(" \u53EF\u4EE5\u5728\u4E0D\u4F7F\u7528\u65B9\u4FBF\u7684 "),ke=a("code"),_e=n("BertNormalizer"),rs=n(" \u7684\u60C5\u51B5\u4E0B\u83B7\u5F97\u5B8C\u5168\u76F8\u540C\u7684\u89C4\u8303\u5316\u3002")},l(V){j=p(V,"P",{});var d=i(j);Y=p(d,"STRONG",{});var Lt=i(Y);K=r(Lt,"\u66F4\u8FDB\u4E00\u6B65"),Lt.forEach(t),Qe=r(d,`\u5982\u679C\u60A8\u5728\u5305\u542B unicode \u5B57\u7B26\u7684\u5B57\u7B26\u4E32\u4E0A\u6D4B\u8BD5\u5148\u524Dnormalizers\u7684\u4E24\u4E2A\u7248\u672C\uFF0C\u60A8\u80AF\u5B9A\u4F1A\u6CE8\u610F\u5230\u8FD9\u4E24\u4E2Anormalizers\u5E76\u4E0D\u5B8C\u5168\u7B49\u6548\u3002
\u4E3A\u4E86\u4E0D\u8FC7\u5EA6\u4F7F\u7528 `),J=p(d,"CODE",{});var ls=i(J);H=r(ls,"normalizers.Sequence"),ls.forEach(t),ge=r(d," \u4F7F\u7248\u672C\u8FC7\u4E8E\u590D\u6742\uFF0C\u6211\u4EEC\u6CA1\u6709\u5305\u542B\u5F53 "),G=p(d,"CODE",{});var C=i(G);es=r(C,"clean_text"),C.forEach(t),je=r(d," \u53C2\u6570\u8BBE\u7F6E\u4E3A "),v=p(d,"CODE",{});var Z=i(v);pe=r(Z,"True"),Z.forEach(t),ss=r(d," \u65F6 "),ie=p(d,"CODE",{});var vn=i(ie);fe=r(vn,"BertNormalizer"),vn.forEach(t),ts=r(d," \u9700\u8981\u7684\u6B63\u5219\u8868\u8FBE\u5F0F\u66FF\u6362 - \u8FD9\u662F\u9ED8\u8BA4\u884C\u4E3A\u3002 \u4F46\u4E0D\u8981\u62C5\u5FC3\uFF1A\u901A\u8FC7\u5728normalizer\u5E8F\u5217\u4E2D\u6DFB\u52A0\u4E24\u4E2A "),me=p(d,"CODE",{});var yt=i(me);ce=r(yt,"normalizers.Replace"),yt.forEach(t),ns=r(d," \u53EF\u4EE5\u5728\u4E0D\u4F7F\u7528\u65B9\u4FBF\u7684 "),ke=p(d,"CODE",{});var Q=i(ke);_e=r(Q,"BertNormalizer"),Q.forEach(t),rs=r(d," \u7684\u60C5\u51B5\u4E0B\u83B7\u5F97\u5B8C\u5168\u76F8\u540C\u7684\u89C4\u8303\u5316\u3002"),d.forEach(t)},m(V,d){o(V,j,d),s(j,Y),s(Y,K),s(j,Qe),s(j,J),s(J,H),s(j,ge),s(j,G),s(G,es),s(j,je),s(j,v),s(v,pe),s(j,ss),s(j,ie),s(ie,fe),s(j,ts),s(j,me),s(me,ce),s(j,ns),s(j,ke),s(ke,_e),s(j,rs)},d(V){V&&t(j)}}}function du(Sl){let j,Y,K,Qe,J,H,ge,G,es,je,v,pe,ss,ie,fe,ts,me,ce,ns,ke,_e,rs,V,d,Lt,ls,C,Z,vn,yt,Q,$k,Nl,de,Cp,Rt,Bp,Fp,Ol,os,Ll,Dt,Ap,yl,w,Ee,bn,Wp,Ip,as,Up,Mp,Xp,xe,Tn,Kp,Hp,ps,Yp,Jp,Vp,ve,Pn,Zp,Qp,is,ei,si,ti,be,qn,ni,ri,fs,li,oi,ai,Te,wn,pi,ii,ms,fi,mi,ci,Pe,Sn,ki,_i,cs,$i,ui,Rl,qe,zi,ks,hi,gi,Dl,_s,ji,Gl,we,di,$s,Ei,xi,Cl,us,Bl,zs,Nn,vi,bi,Fl,Gt,Ti,Al,hs,Wl,Ct,Pi,Il,$e,Se,On,gs,qi,Ln,wi,Ul,b,Si,yn,Ni,Oi,Rn,Li,yi,Dn,Ri,Di,Gn,Gi,Ci,Cn,Bi,Fi,Bn,Ai,Wi,Ml,Ne,Ii,Fn,Ui,Mi,Xl,js,Kl,B,Xi,An,Ki,Hi,Wn,Yi,Ji,In,Vi,Zi,Hl,T,Qi,Un,ef,sf,Mn,tf,nf,Xn,rf,lf,Kn,of,af,Hn,pf,ff,Yn,mf,cf,Yl,ds,Jl,Bt,kf,Vl,Es,Zl,ee,_f,Jn,$f,uf,Vn,zf,hf,Ql,se,gf,Zn,jf,df,Qn,Ef,xf,eo,xs,so,vs,to,Oe,no,Ft,vf,ro,bs,lo,At,bf,oo,Ts,ao,Le,Tf,er,Pf,qf,po,Ps,io,qs,fo,ye,wf,sr,Sf,Nf,mo,ws,co,Ss,ko,Re,Of,tr,Lf,yf,_o,Ns,$o,Os,uo,De,Rf,nr,Df,Gf,zo,Ls,ho,S,Cf,rr,Bf,Ff,lr,Af,Wf,or,If,Uf,ar,Mf,Xf,pr,Kf,Hf,go,Wt,Yf,jo,ys,Eo,Ge,Jf,ir,Vf,Zf,xo,Rs,vo,Ce,Qf,fr,em,sm,bo,Ds,To,Gs,Po,E,tm,mr,nm,rm,cr,lm,om,kr,am,pm,_r,im,fm,$r,mm,cm,ur,km,_m,zr,$m,um,hr,zm,hm,gr,gm,jm,qo,N,dm,jr,Em,xm,dr,vm,bm,Er,Tm,Pm,xr,qm,wm,vr,Sm,Nm,wo,Cs,So,Bs,No,F,Om,br,Lm,ym,Tr,Rm,Dm,Pr,Gm,Cm,Oo,It,Bm,Lo,Fs,yo,Ut,Fm,Ro,Mt,Am,Do,As,Go,Ws,Co,Xt,Wm,Bo,Is,Fo,Us,Ao,Kt,Im,Wo,Ms,Io,Be,Um,qr,Mm,Xm,Uo,Xs,Mo,Ks,Xo,Ht,Km,Ko,Hs,Ho,te,Hm,wr,Ym,Jm,Sr,Vm,Zm,Yo,Ys,Jo,ne,Qm,Nr,ec,sc,Or,tc,nc,Vo,O,rc,Lr,lc,oc,yr,ac,pc,Rr,ic,fc,Dr,mc,cc,Gr,kc,_c,Zo,Js,Qo,Fe,$c,Cr,uc,zc,ea,Vs,sa,re,hc,Br,gc,jc,Fr,dc,Ec,ta,Yt,xc,na,ue,Ae,Ar,Zs,vc,Wr,bc,ra,We,Tc,Ir,Pc,qc,la,Qs,oa,le,wc,Ur,Sc,Nc,Mr,Oc,Lc,aa,Jt,yc,pa,et,ia,Ie,Rc,Xr,Dc,Gc,fa,st,ma,tt,ca,Vt,Cc,ka,nt,_a,P,Bc,Kr,Fc,Ac,Hr,Wc,Ic,Yr,Uc,Mc,Jr,Xc,Kc,Vr,Hc,Yc,Zr,Jc,Vc,$a,Zt,Zc,ua,rt,za,Qt,Qc,ha,lt,ga,ot,ja,en,e2,da,at,Ea,ze,Qr,s2,t2,el,n2,r2,xa,pt,va,it,ba,sn,l2,Ta,ft,Pa,tn,o2,qa,mt,wa,ct,Sa,oe,a2,sl,p2,i2,tl,f2,m2,Na,kt,Oa,nn,c2,La,_t,ya,rn,k2,Ra,he,Ue,nl,$t,_2,rl,$2,Da,Me,u2,ll,z2,h2,Ga,ut,Ca,ln,g2,Ba,on,j2,Fa,zt,Aa,A,d2,ol,E2,x2,al,v2,b2,pl,T2,P2,Wa,Xe,q2,il,w2,S2,Ia,ht,Ua,an,N2,Ma,gt,Xa,jt,Ka,pn,O2,Ha,dt,Ya,ae,L2,fl,y2,R2,ml,D2,G2,Ja,fn,C2,Va,Et,Za,mn,B2,Qa,xt,ep,vt,sp,q,F2,cl,A2,W2,kl,I2,U2,_l,M2,X2,$l,K2,H2,ul,Y2,J2,zl,V2,Z2,tp,bt,np,Tt,rp,cn,Q2,lp,Pt,op,kn,ek,ap,qt,pp,wt,ip,Ke,sk,hl,tk,nk,fp,St,mp,W,rk,gl,lk,ok,jl,ak,pk,dl,ik,fk,cp,Nt,kp,_n,mk,_p,Ot,$p,$n,ck,up;return H=new gu({props:{chapter:6,classNames:"absolute z-10 right-0 top-0",notebooks:[{label:"Google Colab",value:"https://colab.research.google.com/github/huggingface/notebooks/blob/master/course/zh-CN/chapter6/section8.ipynb"},{label:"Aws Studio",value:"https://studiolab.sagemaker.aws/import/github/huggingface/notebooks/blob/master/course/zh-CN/chapter6/section8.ipynb"}]}}),os=new hu({props:{id:"MR8tZm5ViWU"}}),us=new h({props:{code:`from datasets import load_dataset

dataset = load_dataset("wikitext", name="wikitext-2-raw-v1", split="train")


def get_training_corpus():
    for i in range(0, len(dataset), 1000):
        yield dataset[i : i + 1000]["text"]`,highlighted:`<span class="hljs-keyword">from</span> datasets <span class="hljs-keyword">import</span> load_dataset

dataset = load_dataset(<span class="hljs-string">&quot;wikitext&quot;</span>, name=<span class="hljs-string">&quot;wikitext-2-raw-v1&quot;</span>, split=<span class="hljs-string">&quot;train&quot;</span>)


<span class="hljs-keyword">def</span> <span class="hljs-title function_">get_training_corpus</span>():
    <span class="hljs-keyword">for</span> i <span class="hljs-keyword">in</span> <span class="hljs-built_in">range</span>(<span class="hljs-number">0</span>, <span class="hljs-built_in">len</span>(dataset), <span class="hljs-number">1000</span>):
        <span class="hljs-keyword">yield</span> dataset[i : i + <span class="hljs-number">1000</span>][<span class="hljs-string">&quot;text&quot;</span>]`}}),hs=new h({props:{code:`with open("wikitext-2.txt", "w", encoding="utf-8") as f:
    for i in range(len(dataset)):
        f.write(dataset[i]["text"] + "\\n")`,highlighted:`<span class="hljs-keyword">with</span> <span class="hljs-built_in">open</span>(<span class="hljs-string">&quot;wikitext-2.txt&quot;</span>, <span class="hljs-string">&quot;w&quot;</span>, encoding=<span class="hljs-string">&quot;utf-8&quot;</span>) <span class="hljs-keyword">as</span> f:
    <span class="hljs-keyword">for</span> i <span class="hljs-keyword">in</span> <span class="hljs-built_in">range</span>(<span class="hljs-built_in">len</span>(dataset)):
        f.write(dataset[i][<span class="hljs-string">&quot;text&quot;</span>] + <span class="hljs-string">&quot;\\n&quot;</span>)`}}),gs=new _k({}),js=new h({props:{code:`from tokenizers import (
    decoders,
    models,
    normalizers,
    pre_tokenizers,
    processors,
    trainers,
    Tokenizer,
)

tokenizer = Tokenizer(models.WordPiece(unk_token="[UNK]"))`,highlighted:`<span class="hljs-keyword">from</span> tokenizers <span class="hljs-keyword">import</span> (
    decoders,
    models,
    normalizers,
    pre_tokenizers,
    processors,
    trainers,
    Tokenizer,
)

tokenizer = Tokenizer(models.WordPiece(unk_token=<span class="hljs-string">&quot;[UNK]&quot;</span>))`}}),ds=new h({props:{code:"tokenizer.normalizer = normalizers.BertNormalizer(lowercase=True)",highlighted:'tokenizer.normalizer = normalizers.BertNormalizer(lowercase=<span class="hljs-literal">True</span>)'}}),Es=new h({props:{code:`tokenizer.normalizer = normalizers.Sequence(
    [normalizers.NFD(), normalizers.Lowercase(), normalizers.StripAccents()]
)`,highlighted:`tokenizer.normalizer = normalizers.<span class="hljs-type">Sequence</span>(
    [normalizers.NFD(), normalizers.Lowercase(), normalizers.StripAccents()]
)`}}),xs=new h({props:{code:'print(tokenizer.normalizer.normalize_str("H\xE9ll\xF2 h\xF4w are \xFC?"))',highlighted:'<span class="hljs-built_in">print</span>(tokenizer.normalizer.normalize_str(<span class="hljs-string">&quot;H\xE9ll\xF2 h\xF4w are \xFC?&quot;</span>))'}}),vs=new h({props:{code:"hello how are u?",highlighted:"hello how are u?"}}),Oe=new zu({props:{$$slots:{default:[ju]},$$scope:{ctx:Sl}}}),bs=new h({props:{code:"tokenizer.pre_tokenizer = pre_tokenizers.BertPreTokenizer()",highlighted:"tokenizer.pre_tokenizer = pre_tokenizers.BertPreTokenizer()"}}),Ts=new h({props:{code:"tokenizer.pre_tokenizer = pre_tokenizers.Whitespace()",highlighted:"tokenizer.pre_tokenizer = pre_tokenizers.Whitespace()"}}),Ps=new h({props:{code:`tokenizer.pre_tokenizer.pre_tokenize_str("Let's test my pre-tokenizer.")`,highlighted:'tokenizer.pre_tokenizer.pre_tokenize_str(<span class="hljs-string">&quot;Let&#x27;s test my pre-tokenizer.&quot;</span>)'}}),qs=new h({props:{code:`[('Let', (0, 3)), ("'", (3, 4)), ('s', (4, 5)), ('test', (6, 10)), ('my', (11, 13)), ('pre', (14, 17)),
 ('-', (17, 18)), ('tokenizer', (18, 27)), ('.', (27, 28))]`,highlighted:`[(<span class="hljs-string">&#x27;Let&#x27;</span>, (<span class="hljs-number">0</span>, <span class="hljs-number">3</span>)), (<span class="hljs-string">&quot;&#x27;&quot;</span>, (<span class="hljs-number">3</span>, <span class="hljs-number">4</span>)), (<span class="hljs-string">&#x27;s&#x27;</span>, (<span class="hljs-number">4</span>, <span class="hljs-number">5</span>)), (<span class="hljs-string">&#x27;test&#x27;</span>, (<span class="hljs-number">6</span>, <span class="hljs-number">10</span>)), (<span class="hljs-string">&#x27;my&#x27;</span>, (<span class="hljs-number">11</span>, <span class="hljs-number">13</span>)), (<span class="hljs-string">&#x27;pre&#x27;</span>, (<span class="hljs-number">14</span>, <span class="hljs-number">17</span>)),
 (<span class="hljs-string">&#x27;-&#x27;</span>, (<span class="hljs-number">17</span>, <span class="hljs-number">18</span>)), (<span class="hljs-string">&#x27;tokenizer&#x27;</span>, (<span class="hljs-number">18</span>, <span class="hljs-number">27</span>)), (<span class="hljs-string">&#x27;.&#x27;</span>, (<span class="hljs-number">27</span>, <span class="hljs-number">28</span>))]`}}),ws=new h({props:{code:`pre_tokenizer = pre_tokenizers.WhitespaceSplit()
pre_tokenizer.pre_tokenize_str("Let's test my pre-tokenizer.")`,highlighted:`pre_tokenizer = pre_tokenizers.WhitespaceSplit()
pre_tokenizer.pre_tokenize_str(<span class="hljs-string">&quot;Let&#x27;s test my pre-tokenizer.&quot;</span>)`}}),Ss=new h({props:{code:`[("Let's", (0, 5)), ('test', (6, 10)), ('my', (11, 13)), ('pre-tokenizer.', (14, 28))]`,highlighted:'[(<span class="hljs-string">&quot;Let&#x27;s&quot;</span>, (<span class="hljs-number">0</span>, <span class="hljs-number">5</span>)), (<span class="hljs-string">&#x27;test&#x27;</span>, (<span class="hljs-number">6</span>, <span class="hljs-number">10</span>)), (<span class="hljs-string">&#x27;my&#x27;</span>, (<span class="hljs-number">11</span>, <span class="hljs-number">13</span>)), (<span class="hljs-string">&#x27;pre-tokenizer.&#x27;</span>, (<span class="hljs-number">14</span>, <span class="hljs-number">28</span>))]'}}),Ns=new h({props:{code:`pre_tokenizer = pre_tokenizers.Sequence(
    [pre_tokenizers.WhitespaceSplit(), pre_tokenizers.Punctuation()]
)
pre_tokenizer.pre_tokenize_str("Let's test my pre-tokenizer.")`,highlighted:`pre_tokenizer = pre_tokenizers.<span class="hljs-type">Sequence</span>(
    [pre_tokenizers.WhitespaceSplit(), pre_tokenizers.Punctuation()]
)
pre_tokenizer.pre_tokenize_str(<span class="hljs-string">&quot;Let&#x27;s test my pre-tokenizer.&quot;</span>)`}}),Os=new h({props:{code:`[('Let', (0, 3)), ("'", (3, 4)), ('s', (4, 5)), ('test', (6, 10)), ('my', (11, 13)), ('pre', (14, 17)),
 ('-', (17, 18)), ('tokenizer', (18, 27)), ('.', (27, 28))]`,highlighted:`[(<span class="hljs-string">&#x27;Let&#x27;</span>, (<span class="hljs-number">0</span>, <span class="hljs-number">3</span>)), (<span class="hljs-string">&quot;&#x27;&quot;</span>, (<span class="hljs-number">3</span>, <span class="hljs-number">4</span>)), (<span class="hljs-string">&#x27;s&#x27;</span>, (<span class="hljs-number">4</span>, <span class="hljs-number">5</span>)), (<span class="hljs-string">&#x27;test&#x27;</span>, (<span class="hljs-number">6</span>, <span class="hljs-number">10</span>)), (<span class="hljs-string">&#x27;my&#x27;</span>, (<span class="hljs-number">11</span>, <span class="hljs-number">13</span>)), (<span class="hljs-string">&#x27;pre&#x27;</span>, (<span class="hljs-number">14</span>, <span class="hljs-number">17</span>)),
 (<span class="hljs-string">&#x27;-&#x27;</span>, (<span class="hljs-number">17</span>, <span class="hljs-number">18</span>)), (<span class="hljs-string">&#x27;tokenizer&#x27;</span>, (<span class="hljs-number">18</span>, <span class="hljs-number">27</span>)), (<span class="hljs-string">&#x27;.&#x27;</span>, (<span class="hljs-number">27</span>, <span class="hljs-number">28</span>))]`}}),Ls=new h({props:{code:`special_tokens = ["[UNK]", "[PAD]", "[CLS]", "[SEP]", "[MASK]"]
trainer = trainers.WordPieceTrainer(vocab_size=25000, special_tokens=special_tokens)`,highlighted:`special_tokens = [<span class="hljs-string">&quot;[UNK]&quot;</span>, <span class="hljs-string">&quot;[PAD]&quot;</span>, <span class="hljs-string">&quot;[CLS]&quot;</span>, <span class="hljs-string">&quot;[SEP]&quot;</span>, <span class="hljs-string">&quot;[MASK]&quot;</span>]
trainer = trainers.WordPieceTrainer(vocab_size=<span class="hljs-number">25000</span>, special_tokens=special_tokens)`}}),ys=new h({props:{code:"tokenizer.train_from_iterator(get_training_corpus(), trainer=trainer)",highlighted:"tokenizer.train_from_iterator(get_training_corpus(), trainer=trainer)"}}),Rs=new h({props:{code:`tokenizer.model = models.WordPiece(unk_token="[UNK]")
tokenizer.train(["wikitext-2.txt"], trainer=trainer)`,highlighted:`tokenizer.model = models.WordPiece(unk_token=<span class="hljs-string">&quot;[UNK]&quot;</span>)
tokenizer.train([<span class="hljs-string">&quot;wikitext-2.txt&quot;</span>], trainer=trainer)`}}),Ds=new h({props:{code:`encoding = tokenizer.encode("Let's test this tokenizer.")
print(encoding.tokens)`,highlighted:`encoding = tokenizer.encode(<span class="hljs-string">&quot;Let&#x27;s test this tokenizer.&quot;</span>)
<span class="hljs-built_in">print</span>(encoding.tokens)`}}),Gs=new h({props:{code:`['let', "'", 's', 'test', 'this', 'tok', '##eni', '##zer', '.']`,highlighted:'[<span class="hljs-string">&#x27;let&#x27;</span>, <span class="hljs-string">&quot;&#x27;&quot;</span>, <span class="hljs-string">&#x27;s&#x27;</span>, <span class="hljs-string">&#x27;test&#x27;</span>, <span class="hljs-string">&#x27;this&#x27;</span>, <span class="hljs-string">&#x27;tok&#x27;</span>, <span class="hljs-string">&#x27;##eni&#x27;</span>, <span class="hljs-string">&#x27;##zer&#x27;</span>, <span class="hljs-string">&#x27;.&#x27;</span>]'}}),Cs=new h({props:{code:`cls_token_id = tokenizer.token_to_id("[CLS]")
sep_token_id = tokenizer.token_to_id("[SEP]")
print(cls_token_id, sep_token_id)`,highlighted:`cls_token_id = tokenizer.token_to_id(<span class="hljs-string">&quot;[CLS]&quot;</span>)
sep_token_id = tokenizer.token_to_id(<span class="hljs-string">&quot;[SEP]&quot;</span>)
<span class="hljs-built_in">print</span>(cls_token_id, sep_token_id)`}}),Bs=new h({props:{code:"(2, 3)",highlighted:'(<span class="hljs-number">2</span>, <span class="hljs-number">3</span>)'}}),Fs=new h({props:{code:`tokenizer.post_processor = processors.TemplateProcessing(
    single=f"[CLS]:0 $A:0 [SEP]:0",
    pair=f"[CLS]:0 $A:0 [SEP]:0 $B:1 [SEP]:1",
    special_tokens=[("[CLS]", cls_token_id), ("[SEP]", sep_token_id)],
)`,highlighted:`tokenizer.post_processor = processors.TemplateProcessing(
    single=<span class="hljs-string">f&quot;[CLS]:0 $A:0 [SEP]:0&quot;</span>,
    pair=<span class="hljs-string">f&quot;[CLS]:0 $A:0 [SEP]:0 $B:1 [SEP]:1&quot;</span>,
    special_tokens=[(<span class="hljs-string">&quot;[CLS]&quot;</span>, cls_token_id), (<span class="hljs-string">&quot;[SEP]&quot;</span>, sep_token_id)],
)`}}),As=new h({props:{code:`encoding = tokenizer.encode("Let's test this tokenizer.")
print(encoding.tokens)`,highlighted:`encoding = tokenizer.encode(<span class="hljs-string">&quot;Let&#x27;s test this tokenizer.&quot;</span>)
<span class="hljs-built_in">print</span>(encoding.tokens)`}}),Ws=new h({props:{code:`['[CLS]', 'let', "'", 's', 'test', 'this', 'tok', '##eni', '##zer', '.', '[SEP]']`,highlighted:'[<span class="hljs-string">&#x27;[CLS]&#x27;</span>, <span class="hljs-string">&#x27;let&#x27;</span>, <span class="hljs-string">&quot;&#x27;&quot;</span>, <span class="hljs-string">&#x27;s&#x27;</span>, <span class="hljs-string">&#x27;test&#x27;</span>, <span class="hljs-string">&#x27;this&#x27;</span>, <span class="hljs-string">&#x27;tok&#x27;</span>, <span class="hljs-string">&#x27;##eni&#x27;</span>, <span class="hljs-string">&#x27;##zer&#x27;</span>, <span class="hljs-string">&#x27;.&#x27;</span>, <span class="hljs-string">&#x27;[SEP]&#x27;</span>]'}}),Is=new h({props:{code:`encoding = tokenizer.encode("Let's test this tokenizer...", "on a pair of sentences.")
print(encoding.tokens)
print(encoding.type_ids)`,highlighted:`encoding = tokenizer.encode(<span class="hljs-string">&quot;Let&#x27;s test this tokenizer...&quot;</span>, <span class="hljs-string">&quot;on a pair of sentences.&quot;</span>)
<span class="hljs-built_in">print</span>(encoding.tokens)
<span class="hljs-built_in">print</span>(encoding.type_ids)`}}),Us=new h({props:{code:`['[CLS]', 'let', "'", 's', 'test', 'this', 'tok', '##eni', '##zer', '...', '[SEP]', 'on', 'a', 'pair', 'of', 'sentences', '.', '[SEP]']
[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1]`,highlighted:`[<span class="hljs-string">&#x27;[CLS]&#x27;</span>, <span class="hljs-string">&#x27;let&#x27;</span>, <span class="hljs-string">&quot;&#x27;&quot;</span>, <span class="hljs-string">&#x27;s&#x27;</span>, <span class="hljs-string">&#x27;test&#x27;</span>, <span class="hljs-string">&#x27;this&#x27;</span>, <span class="hljs-string">&#x27;tok&#x27;</span>, <span class="hljs-string">&#x27;##eni&#x27;</span>, <span class="hljs-string">&#x27;##zer&#x27;</span>, <span class="hljs-string">&#x27;...&#x27;</span>, <span class="hljs-string">&#x27;[SEP]&#x27;</span>, <span class="hljs-string">&#x27;on&#x27;</span>, <span class="hljs-string">&#x27;a&#x27;</span>, <span class="hljs-string">&#x27;pair&#x27;</span>, <span class="hljs-string">&#x27;of&#x27;</span>, <span class="hljs-string">&#x27;sentences&#x27;</span>, <span class="hljs-string">&#x27;.&#x27;</span>, <span class="hljs-string">&#x27;[SEP]&#x27;</span>]
[<span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>]`}}),Ms=new h({props:{code:'tokenizer.decoder = decoders.WordPiece(prefix="##")',highlighted:'tokenizer.decoder = decoders.WordPiece(prefix=<span class="hljs-string">&quot;##&quot;</span>)'}}),Xs=new h({props:{code:"tokenizer.decode(encoding.ids)",highlighted:"tokenizer.decode(encoding.ids)"}}),Ks=new h({props:{code:`"let's test this tokenizer... on a pair of sentences."`,highlighted:'<span class="hljs-string">&quot;let&#x27;s test this tokenizer... on a pair of sentences.&quot;</span>'}}),Hs=new h({props:{code:'tokenizer.save("tokenizer.json")',highlighted:'tokenizer.save(<span class="hljs-string">&quot;tokenizer.json&quot;</span>)'}}),Ys=new h({props:{code:'new_tokenizer = Tokenizer.from_file("tokenizer.json")',highlighted:'new_tokenizer = Tokenizer.from_file(<span class="hljs-string">&quot;tokenizer.json&quot;</span>)'}}),Js=new h({props:{code:`from transformers import PreTrainedTokenizerFast

wrapped_tokenizer = PreTrainedTokenizerFast(
    tokenizer_object=tokenizer,
    # tokenizer_file="tokenizer.json", # You can load from the tokenizer file, alternatively
    unk_token="[UNK]",
    pad_token="[PAD]",
    cls_token="[CLS]",
    sep_token="[SEP]",
    mask_token="[MASK]",
)`,highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> PreTrainedTokenizerFast

wrapped_tokenizer = PreTrainedTokenizerFast(
    tokenizer_object=tokenizer,
    <span class="hljs-comment"># tokenizer_file=&quot;tokenizer.json&quot;, # You can load from the tokenizer file, alternatively</span>
    unk_token=<span class="hljs-string">&quot;[UNK]&quot;</span>,
    pad_token=<span class="hljs-string">&quot;[PAD]&quot;</span>,
    cls_token=<span class="hljs-string">&quot;[CLS]&quot;</span>,
    sep_token=<span class="hljs-string">&quot;[SEP]&quot;</span>,
    mask_token=<span class="hljs-string">&quot;[MASK]&quot;</span>,
)`}}),Vs=new h({props:{code:`from transformers import BertTokenizerFast

wrapped_tokenizer = BertTokenizerFast(tokenizer_object=tokenizer)`,highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> BertTokenizerFast

wrapped_tokenizer = BertTokenizerFast(tokenizer_object=tokenizer)`}}),Zs=new _k({}),Qs=new h({props:{code:"tokenizer = Tokenizer(models.BPE())",highlighted:"tokenizer = Tokenizer(models.BPE())"}}),et=new h({props:{code:"tokenizer.pre_tokenizer = pre_tokenizers.ByteLevel(add_prefix_space=False)",highlighted:'tokenizer.pre_tokenizer = pre_tokenizers.ByteLevel(add_prefix_space=<span class="hljs-literal">False</span>)'}}),st=new h({props:{code:`tokenizer.pre_tokenizer.pre_tokenize_str("Let's test pre-tokenization!")`,highlighted:'tokenizer.pre_tokenizer.pre_tokenize_str(<span class="hljs-string">&quot;Let&#x27;s test pre-tokenization!&quot;</span>)'}}),tt=new h({props:{code:`[('Let', (0, 3)), ("'s", (3, 5)), ('\u0120test', (5, 10)), ('\u0120pre', (10, 14)), ('-', (14, 15)),
 ('tokenization', (15, 27)), ('!', (27, 28))]`,highlighted:`[(<span class="hljs-string">&#x27;Let&#x27;</span>, (<span class="hljs-number">0</span>, <span class="hljs-number">3</span>)), (<span class="hljs-string">&quot;&#x27;s&quot;</span>, (<span class="hljs-number">3</span>, <span class="hljs-number">5</span>)), (<span class="hljs-string">&#x27;\u0120test&#x27;</span>, (<span class="hljs-number">5</span>, <span class="hljs-number">10</span>)), (<span class="hljs-string">&#x27;\u0120pre&#x27;</span>, (<span class="hljs-number">10</span>, <span class="hljs-number">14</span>)), (<span class="hljs-string">&#x27;-&#x27;</span>, (<span class="hljs-number">14</span>, <span class="hljs-number">15</span>)),
 (<span class="hljs-string">&#x27;tokenization&#x27;</span>, (<span class="hljs-number">15</span>, <span class="hljs-number">27</span>)), (<span class="hljs-string">&#x27;!&#x27;</span>, (<span class="hljs-number">27</span>, <span class="hljs-number">28</span>))]`}}),nt=new h({props:{code:`trainer = trainers.BpeTrainer(vocab_size=25000, special_tokens=["<|endoftext|>"])
tokenizer.train_from_iterator(get_training_corpus(), trainer=trainer)`,highlighted:`trainer = trainers.BpeTrainer(vocab_size=<span class="hljs-number">25000</span>, special_tokens=[<span class="hljs-string">&quot;&lt;|endoftext|&gt;&quot;</span>])
tokenizer.train_from_iterator(get_training_corpus(), trainer=trainer)`}}),rt=new h({props:{code:`tokenizer.model = models.BPE()
tokenizer.train(["wikitext-2.txt"], trainer=trainer)`,highlighted:`tokenizer.model = models.BPE()
tokenizer.train([<span class="hljs-string">&quot;wikitext-2.txt&quot;</span>], trainer=trainer)`}}),lt=new h({props:{code:`encoding = tokenizer.encode("Let's test this tokenizer.")
print(encoding.tokens)`,highlighted:`encoding = tokenizer.encode(<span class="hljs-string">&quot;Let&#x27;s test this tokenizer.&quot;</span>)
<span class="hljs-built_in">print</span>(encoding.tokens)`}}),ot=new h({props:{code:`['L', 'et', "'", 's', '\u0120test', '\u0120this', '\u0120to', 'ken', 'izer', '.']`,highlighted:'[<span class="hljs-string">&#x27;L&#x27;</span>, <span class="hljs-string">&#x27;et&#x27;</span>, <span class="hljs-string">&quot;&#x27;&quot;</span>, <span class="hljs-string">&#x27;s&#x27;</span>, <span class="hljs-string">&#x27;\u0120test&#x27;</span>, <span class="hljs-string">&#x27;\u0120this&#x27;</span>, <span class="hljs-string">&#x27;\u0120to&#x27;</span>, <span class="hljs-string">&#x27;ken&#x27;</span>, <span class="hljs-string">&#x27;izer&#x27;</span>, <span class="hljs-string">&#x27;.&#x27;</span>]'}}),at=new h({props:{code:"tokenizer.post_processor = processors.ByteLevel(trim_offsets=False)",highlighted:'tokenizer.post_processor = processors.ByteLevel(trim_offsets=<span class="hljs-literal">False</span>)'}}),pt=new h({props:{code:`sentence = "Let's test this tokenizer."
encoding = tokenizer.encode(sentence)
start, end = encoding.offsets[4]
sentence[start:end]`,highlighted:`sentence = <span class="hljs-string">&quot;Let&#x27;s test this tokenizer.&quot;</span>
encoding = tokenizer.encode(sentence)
start, end = encoding.offsets[<span class="hljs-number">4</span>]
sentence[start:end]`}}),it=new h({props:{code:"' test'",highlighted:'<span class="hljs-string">&#x27; test&#x27;</span>'}}),ft=new h({props:{code:"tokenizer.decoder = decoders.ByteLevel()",highlighted:"tokenizer.decoder = decoders.ByteLevel()"}}),mt=new h({props:{code:"tokenizer.decode(encoding.ids)",highlighted:"tokenizer.decode(encoding.ids)"}}),ct=new h({props:{code:`"Let's test this tokenizer."`,highlighted:'<span class="hljs-string">&quot;Let&#x27;s test this tokenizer.&quot;</span>'}}),kt=new h({props:{code:`from transformers import PreTrainedTokenizerFast

wrapped_tokenizer = PreTrainedTokenizerFast(
    tokenizer_object=tokenizer,
    bos_token="<|endoftext|>",
    eos_token="<|endoftext|>",
)`,highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> PreTrainedTokenizerFast

wrapped_tokenizer = PreTrainedTokenizerFast(
    tokenizer_object=tokenizer,
    bos_token=<span class="hljs-string">&quot;&lt;|endoftext|&gt;&quot;</span>,
    eos_token=<span class="hljs-string">&quot;&lt;|endoftext|&gt;&quot;</span>,
)`}}),_t=new h({props:{code:`from transformers import GPT2TokenizerFast

wrapped_tokenizer = GPT2TokenizerFast(tokenizer_object=tokenizer)`,highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> GPT2TokenizerFast

wrapped_tokenizer = GPT2TokenizerFast(tokenizer_object=tokenizer)`}}),$t=new _k({}),ut=new h({props:{code:"tokenizer = Tokenizer(models.Unigram())",highlighted:"tokenizer = Tokenizer(models.Unigram())"}}),zt=new h({props:{code:`from tokenizers import Regex

tokenizer.normalizer = normalizers.Sequence(
    [
        normalizers.Replace("\`\`", '"'),
        normalizers.Replace("''", '"'),
        normalizers.NFKD(),
        normalizers.StripAccents(),
        normalizers.Replace(Regex(" {2,}"), " "),
    ]
)`,highlighted:`<span class="hljs-keyword">from</span> tokenizers <span class="hljs-keyword">import</span> Regex

tokenizer.normalizer = normalizers.<span class="hljs-type">Sequence</span>(
    [
        normalizers.Replace(<span class="hljs-string">&quot;\`\`&quot;</span>, <span class="hljs-string">&#x27;&quot;&#x27;</span>),
        normalizers.Replace(<span class="hljs-string">&quot;&#x27;&#x27;&quot;</span>, <span class="hljs-string">&#x27;&quot;&#x27;</span>),
        normalizers.NFKD(),
        normalizers.StripAccents(),
        normalizers.Replace(Regex(<span class="hljs-string">&quot; {2,}&quot;</span>), <span class="hljs-string">&quot; &quot;</span>),
    ]
)`}}),ht=new h({props:{code:"tokenizer.pre_tokenizer = pre_tokenizers.Metaspace()",highlighted:"tokenizer.pre_tokenizer = pre_tokenizers.Metaspace()"}}),gt=new h({props:{code:`tokenizer.pre_tokenizer.pre_tokenize_str("Let's test the pre-tokenizer!")`,highlighted:'tokenizer.pre_tokenizer.pre_tokenize_str(<span class="hljs-string">&quot;Let&#x27;s test the pre-tokenizer!&quot;</span>)'}}),jt=new h({props:{code:`[("\u2581Let's", (0, 5)), ('\u2581test', (5, 10)), ('\u2581the', (10, 14)), ('\u2581pre-tokenizer!', (14, 29))]`,highlighted:'[(<span class="hljs-string">&quot;\u2581Let&#x27;s&quot;</span>, (<span class="hljs-number">0</span>, <span class="hljs-number">5</span>)), (<span class="hljs-string">&#x27;\u2581test&#x27;</span>, (<span class="hljs-number">5</span>, <span class="hljs-number">10</span>)), (<span class="hljs-string">&#x27;\u2581the&#x27;</span>, (<span class="hljs-number">10</span>, <span class="hljs-number">14</span>)), (<span class="hljs-string">&#x27;\u2581pre-tokenizer!&#x27;</span>, (<span class="hljs-number">14</span>, <span class="hljs-number">29</span>))]'}}),dt=new h({props:{code:`special_tokens = ["<cls>", "<sep>", "<unk>", "<pad>", "<mask>", "<s>", "</s>"]
trainer = trainers.UnigramTrainer(
    vocab_size=25000, special_tokens=special_tokens, unk_token="<unk>"
)
tokenizer.train_from_iterator(get_training_corpus(), trainer=trainer)`,highlighted:`special_tokens = [<span class="hljs-string">&quot;&lt;cls&gt;&quot;</span>, <span class="hljs-string">&quot;&lt;sep&gt;&quot;</span>, <span class="hljs-string">&quot;&lt;unk&gt;&quot;</span>, <span class="hljs-string">&quot;&lt;pad&gt;&quot;</span>, <span class="hljs-string">&quot;&lt;mask&gt;&quot;</span>, <span class="hljs-string">&quot;&lt;s&gt;&quot;</span>, <span class="hljs-string">&quot;&lt;/s&gt;&quot;</span>]
trainer = trainers.UnigramTrainer(
    vocab_size=<span class="hljs-number">25000</span>, special_tokens=special_tokens, unk_token=<span class="hljs-string">&quot;&lt;unk&gt;&quot;</span>
)
tokenizer.train_from_iterator(get_training_corpus(), trainer=trainer)`}}),Et=new h({props:{code:`tokenizer.model = models.Unigram()
tokenizer.train(["wikitext-2.txt"], trainer=trainer)`,highlighted:`tokenizer.model = models.Unigram()
tokenizer.train([<span class="hljs-string">&quot;wikitext-2.txt&quot;</span>], trainer=trainer)`}}),xt=new h({props:{code:`encoding = tokenizer.encode("Let's test this tokenizer.")
print(encoding.tokens)`,highlighted:`encoding = tokenizer.encode(<span class="hljs-string">&quot;Let&#x27;s test this tokenizer.&quot;</span>)
<span class="hljs-built_in">print</span>(encoding.tokens)`}}),vt=new h({props:{code:`['\u2581Let', "'", 's', '\u2581test', '\u2581this', '\u2581to', 'ken', 'izer', '.']`,highlighted:'[<span class="hljs-string">&#x27;\u2581Let&#x27;</span>, <span class="hljs-string">&quot;&#x27;&quot;</span>, <span class="hljs-string">&#x27;s&#x27;</span>, <span class="hljs-string">&#x27;\u2581test&#x27;</span>, <span class="hljs-string">&#x27;\u2581this&#x27;</span>, <span class="hljs-string">&#x27;\u2581to&#x27;</span>, <span class="hljs-string">&#x27;ken&#x27;</span>, <span class="hljs-string">&#x27;izer&#x27;</span>, <span class="hljs-string">&#x27;.&#x27;</span>]'}}),bt=new h({props:{code:`cls_token_id = tokenizer.token_to_id("<cls>")
sep_token_id = tokenizer.token_to_id("<sep>")
print(cls_token_id, sep_token_id)`,highlighted:`cls_token_id = tokenizer.token_to_id(<span class="hljs-string">&quot;&lt;cls&gt;&quot;</span>)
sep_token_id = tokenizer.token_to_id(<span class="hljs-string">&quot;&lt;sep&gt;&quot;</span>)
<span class="hljs-built_in">print</span>(cls_token_id, sep_token_id)`}}),Tt=new h({props:{code:"0 1",highlighted:'<span class="hljs-number">0</span> <span class="hljs-number">1</span>'}}),Pt=new h({props:{code:`tokenizer.post_processor = processors.TemplateProcessing(
    single="$A:0 <sep>:0 <cls>:2",
    pair="$A:0 <sep>:0 $B:1 <sep>:1 <cls>:2",
    special_tokens=[("<sep>", sep_token_id), ("<cls>", cls_token_id)],
)`,highlighted:`tokenizer.post_processor = processors.TemplateProcessing(
    single=<span class="hljs-string">&quot;$A:0 &lt;sep&gt;:0 &lt;cls&gt;:2&quot;</span>,
    pair=<span class="hljs-string">&quot;$A:0 &lt;sep&gt;:0 $B:1 &lt;sep&gt;:1 &lt;cls&gt;:2&quot;</span>,
    special_tokens=[(<span class="hljs-string">&quot;&lt;sep&gt;&quot;</span>, sep_token_id), (<span class="hljs-string">&quot;&lt;cls&gt;&quot;</span>, cls_token_id)],
)`}}),qt=new h({props:{code:`encoding = tokenizer.encode("Let's test this tokenizer...", "on a pair of sentences!")
print(encoding.tokens)
print(encoding.type_ids)`,highlighted:`encoding = tokenizer.encode(<span class="hljs-string">&quot;Let&#x27;s test this tokenizer...&quot;</span>, <span class="hljs-string">&quot;on a pair of sentences!&quot;</span>)
<span class="hljs-built_in">print</span>(encoding.tokens)
<span class="hljs-built_in">print</span>(encoding.type_ids)`}}),wt=new h({props:{code:`['\u2581Let', "'", 's', '\u2581test', '\u2581this', '\u2581to', 'ken', 'izer', '.', '.', '.', '<sep>', '\u2581', 'on', '\u2581', 'a', '\u2581pair', 
  '\u2581of', '\u2581sentence', 's', '!', '<sep>', '<cls>']
[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2]`,highlighted:`[<span class="hljs-string">&#x27;\u2581Let&#x27;</span>, <span class="hljs-string">&quot;&#x27;&quot;</span>, <span class="hljs-string">&#x27;s&#x27;</span>, <span class="hljs-string">&#x27;\u2581test&#x27;</span>, <span class="hljs-string">&#x27;\u2581this&#x27;</span>, <span class="hljs-string">&#x27;\u2581to&#x27;</span>, <span class="hljs-string">&#x27;ken&#x27;</span>, <span class="hljs-string">&#x27;izer&#x27;</span>, <span class="hljs-string">&#x27;.&#x27;</span>, <span class="hljs-string">&#x27;.&#x27;</span>, <span class="hljs-string">&#x27;.&#x27;</span>, <span class="hljs-string">&#x27;&lt;sep&gt;&#x27;</span>, <span class="hljs-string">&#x27;\u2581&#x27;</span>, <span class="hljs-string">&#x27;on&#x27;</span>, <span class="hljs-string">&#x27;\u2581&#x27;</span>, <span class="hljs-string">&#x27;a&#x27;</span>, <span class="hljs-string">&#x27;\u2581pair&#x27;</span>, 
  <span class="hljs-string">&#x27;\u2581of&#x27;</span>, <span class="hljs-string">&#x27;\u2581sentence&#x27;</span>, <span class="hljs-string">&#x27;s&#x27;</span>, <span class="hljs-string">&#x27;!&#x27;</span>, <span class="hljs-string">&#x27;&lt;sep&gt;&#x27;</span>, <span class="hljs-string">&#x27;&lt;cls&gt;&#x27;</span>]
[<span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">2</span>]`}}),St=new h({props:{code:"tokenizer.decoder = decoders.Metaspace()",highlighted:"tokenizer.decoder = decoders.Metaspace()"}}),Nt=new h({props:{code:`from transformers import PreTrainedTokenizerFast

wrapped_tokenizer = PreTrainedTokenizerFast(
    tokenizer_object=tokenizer,
    bos_token="<s>",
    eos_token="</s>",
    unk_token="<unk>",
    pad_token="<pad>",
    cls_token="<cls>",
    sep_token="<sep>",
    mask_token="<mask>",
    padding_side="left",
)`,highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> PreTrainedTokenizerFast

wrapped_tokenizer = PreTrainedTokenizerFast(
    tokenizer_object=tokenizer,
    bos_token=<span class="hljs-string">&quot;&lt;s&gt;&quot;</span>,
    eos_token=<span class="hljs-string">&quot;&lt;/s&gt;&quot;</span>,
    unk_token=<span class="hljs-string">&quot;&lt;unk&gt;&quot;</span>,
    pad_token=<span class="hljs-string">&quot;&lt;pad&gt;&quot;</span>,
    cls_token=<span class="hljs-string">&quot;&lt;cls&gt;&quot;</span>,
    sep_token=<span class="hljs-string">&quot;&lt;sep&gt;&quot;</span>,
    mask_token=<span class="hljs-string">&quot;&lt;mask&gt;&quot;</span>,
    padding_side=<span class="hljs-string">&quot;left&quot;</span>,
)`}}),Ot=new h({props:{code:`from transformers import XLNetTokenizerFast

wrapped_tokenizer = XLNetTokenizerFast(tokenizer_object=tokenizer)`,highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> XLNetTokenizerFast

wrapped_tokenizer = XLNetTokenizerFast(tokenizer_object=tokenizer)`}}),{c(){j=a("meta"),Y=f(),K=a("h1"),Qe=n("\u9010\u5757\u5730\u6784\u5EFA\u6807\u8BB0\u5668"),J=f(),c(H.$$.fragment),ge=f(),G=a("p"),es=n("\u6B63\u5982\u6211\u4EEC\u5728\u524D\u51E0\u8282\u4E2D\u770B\u5230\u7684\uFF0C\u6807\u8BB0\u5316\u5305\u62EC\u51E0\u4E2A\u6B65\u9AA4\uFF1A"),je=f(),v=a("ul"),pe=a("li"),ss=n("\u89C4\u8303\u5316\uFF08\u4EFB\u4F55\u8BA4\u4E3A\u5FC5\u8981\u7684\u6587\u672C\u6E05\u7406\uFF0C\u4F8B\u5982\u5220\u9664\u7A7A\u683C\u6216\u91CD\u97F3\u7B26\u53F7\u3001Unicode \u89C4\u8303\u5316\u7B49\uFF09"),ie=f(),fe=a("li"),ts=n("\u9884\u6807\u8BB0\u5316\uFF08\u5C06\u8F93\u5165\u62C6\u5206\u4E3A\u5355\u8BCD\uFF09"),me=f(),ce=a("li"),ns=n("\u901A\u8FC7\u6A21\u578B\u5904\u7406\u8F93\u5165\uFF08\u4F7F\u7528\u9884\u5148\u62C6\u5206\u7684\u8BCD\u6765\u751F\u6210\u4E00\u7CFB\u5217\u6807\u8BB0\uFF09"),ke=f(),_e=a("li"),rs=n("\u540E\u5904\u7406\uFF08\u6DFB\u52A0\u6807\u8BB0\u5668\u7684\u7279\u6B8A\u6807\u8BB0\uFF0C\u751F\u6210\u6CE8\u610F\u529B\u63A9\u7801\u548C\u6807\u8BB0\u7C7B\u578B ID\uFF09"),V=f(),d=a("p"),Lt=n("\u63D0\u9192\u4E00\u4E0B\uFF0C\u8FD9\u91CC\u518D\u770B\u4E00\u4E0B\u6574\u4E2A\u8FC7\u7A0B"),ls=f(),C=a("div"),Z=a("img"),yt=f(),Q=a("img"),Nl=f(),de=a("p"),Cp=n("\u{1F917} Tokenizers \u5E93\u65E8\u5728\u4E3A\u6BCF\u4E2A\u6B65\u9AA4\u63D0\u4F9B\u591A\u4E2A\u9009\u9879\uFF0C\u60A8\u53EF\u4EE5\u5C06\u5B83\u4EEC\u6DF7\u5408\u548C\u5339\u914D\u5728\u4E00\u8D77\u3002\u5728\u672C\u8282\u4E2D\uFF0C\u6211\u4EEC\u5C06\u770B\u5230\u5982\u4F55\u4ECE\u5934\u5F00\u59CB\u6784\u5EFA\u6807\u8BB0\u5668\uFF0C\u800C\u4E0D\u662F\u50CF\u6211\u4EEC"),Rt=a("a"),Bp=n("\u7B2C\u4E8C\u8282 2"),Fp=n("\u90A3\u6837\u4ECE\u65E7\u7684\u6807\u8BB0\u5668\u8BAD\u7EC3\u65B0\u7684\u6807\u8BB0\u5668.\u7136\u540E\uFF0C\u60A8\u5C06\u80FD\u591F\u6784\u5EFA\u60A8\u80FD\u60F3\u5230\u7684\u4EFB\u4F55\u7C7B\u578B\u7684\u6807\u8BB0\u5668\uFF01"),Ol=f(),c(os.$$.fragment),Ll=f(),Dt=a("p"),Ap=n("\u66F4\u51C6\u786E\u5730\u8BF4\uFF0C\u8BE5\u5E93\u662F\u56F4\u7ED5\u4E00\u4E2A\u4E2D\u592E\u201CTokenizer\u201D\u7C7B\u6784\u5EFA\u7684\uFF0C\u6784\u5EFA\u8FD9\u4E2A\u7C7B\u7684\u6BCF\u4E00\u90E8\u5206\u53EF\u4EE5\u5728\u5B50\u6A21\u5757\u7684\u5217\u8868\u4E2D\u91CD\u65B0\u7EC4\u5408\uFF1A"),yl=f(),w=a("ul"),Ee=a("li"),bn=a("code"),Wp=n("normalizers"),Ip=n(" \u5305\u542B\u4F60\u53EF\u4EE5\u4F7F\u7528\u7684\u6240\u6709\u53EF\u80FD\u7684Normalizer\u7C7B\u578B\uFF08\u5B8C\u6574\u5217\u8868"),as=a("a"),Up=n("\u5728\u8FD9\u91CC"),Mp=n("\uFF09\u3002"),Xp=f(),xe=a("li"),Tn=a("code"),Kp=n("pre_tokenizesr"),Hp=n(" \u5305\u542B\u60A8\u53EF\u4EE5\u4F7F\u7528\u7684\u6240\u6709\u53EF\u80FD\u7684PreTokenizer\u7C7B\u578B\uFF08\u5B8C\u6574\u5217\u8868"),ps=a("a"),Yp=n("\u5728\u8FD9\u91CC"),Jp=n("\uFF09\u3002"),Vp=f(),ve=a("li"),Pn=a("code"),Zp=n("models"),Qp=n(" \u5305\u542B\u60A8\u53EF\u4EE5\u4F7F\u7528\u7684\u5404\u79CD\u7C7B\u578B\u7684Model\uFF0C\u5982BPE\u3001WordPiece\u548CUnigram\uFF08\u5B8C\u6574\u5217\u8868"),is=a("a"),ei=n("\u5728\u8FD9\u91CC"),si=n("\uFF09\u3002"),ti=f(),be=a("li"),qn=a("code"),ni=n("trainers"),ri=n(" \u5305\u542B\u6240\u6709\u4E0D\u540C\u7C7B\u578B\u7684 trainer\uFF0C\u4F60\u53EF\u4EE5\u4F7F\u7528\u4E00\u4E2A\u8BED\u6599\u5E93\u8BAD\u7EC3\u4F60\u7684\u6A21\u578B\uFF08\u6BCF\u79CD\u6A21\u578B\u4E00\u4E2A\uFF1B\u5B8C\u6574\u5217\u8868"),fs=a("a"),li=n("\u5728\u8FD9\u91CC"),oi=n("\uFF09\u3002"),ai=f(),Te=a("li"),wn=a("code"),pi=n("post_processors"),ii=n(" \u5305\u542B\u4F60\u53EF\u4EE5\u4F7F\u7528\u7684\u5404\u79CD\u7C7B\u578B\u7684PostProcessor\uFF08\u5B8C\u6574\u5217\u8868"),ms=a("a"),fi=n("\u5728\u8FD9\u91CC"),mi=n("\uFF09\u3002"),ci=f(),Pe=a("li"),Sn=a("code"),ki=n("decoders"),_i=n(" \u5305\u542B\u5404\u79CD\u7C7B\u578B\u7684Decoder\uFF0C\u53EF\u4EE5\u7528\u6765\u89E3\u7801\u6807\u8BB0\u5316\u7684\u8F93\u51FA\uFF08\u5B8C\u6574\u5217\u8868"),cs=a("a"),$i=n("\u5728\u8FD9\u91CC"),ui=n("\uFF09\u3002"),Rl=f(),qe=a("p"),zi=n("\u60A8\u53EF\u4EE5"),ks=a("a"),hi=n("\u5728\u8FD9\u91CC"),gi=n("\u627E\u5230\u5B8C\u6574\u7684\u6A21\u5757\u5217\u8868\u3002"),Dl=f(),_s=a("h2"),ji=n("\u83B7\u53D6\u8BED\u200B\u200B\u6599\u5E93"),Gl=f(),we=a("p"),di=n("\u4E3A\u4E86\u8BAD\u7EC3\u6211\u4EEC\u7684\u65B0\u6807\u8BB0\u5668\uFF0C\u6211\u4EEC\u5C06\u4F7F\u7528\u4E00\u4E2A\u5C0F\u7684\u6587\u672C\u8BED\u6599\u5E93\uFF08\u56E0\u6B64\u793A\u4F8B\u8FD0\u884C\u5F97\u5F88\u5FEB\uFF09\u3002\u83B7\u53D6\u8BED\u200B\u200B\u6599\u5E93\u7684\u6B65\u9AA4\u4E0E\u6211\u4EEC\u5728[\u5728\u8FD9\u7AE0\u7684\u5F00\u59CB]((/course/chapter6/2)\u90A3\u4E00\u5C0F\u8282\uFF0C\u4F46\u8FD9\u6B21\u6211\u4EEC\u5C06\u4F7F\u7528"),$s=a("a"),Ei=n("WikiText-2"),xi=n("\u6570\u636E\u96C6\uFF1A"),Cl=f(),c(us.$$.fragment),Bl=f(),zs=a("p"),Nn=a("strong"),vi=n("get_training_corpus()"),bi=n(" \u51FD\u6570\u662F\u4E00\u4E2A\u751F\u6210\u5668\uFF0C\u6BCF\u6B21\u8C03\u7528\u7684\u65F6\u5019\u5C06\u4EA7\u751F 1,000 \u4E2A\u6587\u672C\uFF0C\u6211\u4EEC\u5C06\u7528\u5B83\u6765\u8BAD\u7EC3\u6807\u8BB0\u5668\u3002"),Fl=f(),Gt=a("p"),Ti=n("\u{1F917} Tokenizers \u4E5F\u53EF\u4EE5\u76F4\u63A5\u5728\u6587\u672C\u6587\u4EF6\u4E0A\u8FDB\u884C\u8BAD\u7EC3\u3002\u4EE5\u4E0B\u662F\u6211\u4EEC\u5982\u4F55\u751F\u6210\u4E00\u4E2A\u6587\u672C\u6587\u4EF6\uFF0C\u5176\u4E2D\u5305\u542B\u6211\u4EEC\u53EF\u4EE5\u5728\u672C\u5730\u4F7F\u7528\u7684\u6765\u81EA WikiText-2 \u7684\u6240\u6709\u6587\u672C/\u8F93\u5165\uFF1A"),Al=f(),c(hs.$$.fragment),Wl=f(),Ct=a("p"),Pi=n("\u63A5\u4E0B\u6765\uFF0C\u6211\u4EEC\u5C06\u5411\u60A8\u5C55\u793A\u5982\u4F55\u9010\u5757\u6784\u5EFA\u60A8\u81EA\u5DF1\u7684 BERT\u3001GPT-2 \u548C XLNet \u6807\u8BB0\u5668\u3002\u8FD9\u5C06\u4E3A\u6211\u4EEC\u63D0\u4F9B\u4E09\u4E2A\u4E3B\u8981\u6807\u8BB0\u5316\u7B97\u6CD5\u7684\u793A\u4F8B\uFF1AWordPiece\u3001BPE \u548C Unigram\u3002\u8BA9\u6211\u4EEC\u4ECE BERT \u5F00\u59CB\u5427\uFF01"),Il=f(),$e=a("h2"),Se=a("a"),On=a("span"),c(gs.$$.fragment),qi=f(),Ln=a("span"),wi=n("\u4ECE\u5934\u5F00\u59CB\u6784\u5EFA WordPiece \u6807\u8BB0\u5668"),Ul=f(),b=a("p"),Si=n("\u8981\u4F7F\u7528 \u{1F917} Tokenizers \u5E93\u6784\u5EFA\u6807\u8BB0\u5668\uFF0C\u6211\u4EEC\u9996\u5148\u4F7F\u7528"),yn=a("strong"),Ni=n("model"),Oi=n("\u5B9E\u4F8B\u5316\u4E00\u4E2A "),Rn=a("strong"),Li=n("Tokenizer"),yi=n(" \u5BF9\u8C61\u4E0E  \uFF0C\u7136\u540E\u5C06 "),Dn=a("strong"),Ri=n("normalizer"),Di=n(" , "),Gn=a("strong"),Gi=n("pre_tokenizer"),Ci=n(" , "),Cn=a("strong"),Bi=n("post_processor"),Fi=n(" \uFF0C \u548C "),Bn=a("strong"),Ai=n("decoder"),Wi=n(" \u5C5E\u6027\u8BBE\u7F6E\u6210\u6211\u4EEC\u60F3\u8981\u7684\u503C\u3002"),Ml=f(),Ne=a("p"),Ii=n("\u5BF9\u4E8E\u8FD9\u4E2A\u4F8B\u5B50\uFF0C\u6211\u4EEC\u5C06\u521B\u5EFA\u4E00\u4E2A "),Fn=a("strong"),Ui=n("Tokenizer"),Mi=n(" \u4F7F\u7528 WordPiece \u6A21\u578B\uFF1A"),Xl=f(),c(js.$$.fragment),Kl=f(),B=a("p"),Xi=n("\u6211\u4EEC\u5FC5\u987B\u6307\u5B9A "),An=a("strong"),Ki=n("unk_token"),Hi=n(" \u8FD9\u6837\u6A21\u578B\u624D\u77E5\u9053\u5F53\u5B83\u9047\u5230\u4EE5\u524D\u6CA1\u6709\u89C1\u8FC7\u7684\u5B57\u7B26\u65F6\u8981\u8FD4\u56DE\u4EC0\u4E48\u3002\u6211\u4EEC\u53EF\u4EE5\u5728\u6B64\u5904\u8BBE\u7F6E\u7684\u5176\u4ED6\u53C2\u6570\u5305\u62EC\u6211\u4EEC\u6A21\u578B\u7684"),Wn=a("strong"),Yi=n("vocab\uFF08\u5B57\u5178\uFF09"),Ji=n("\uFF08\u6211\u4EEC\u5C06\u8BAD\u7EC3\u6A21\u578B\uFF0C\u6240\u4EE5\u6211\u4EEC\u4E0D\u9700\u8981\u8BBE\u7F6E\u5B83\uFF09\u548C "),In=a("strong"),Vi=n("max_input_chars_per_word"),Zi=n(" \u5373\u6BCF\u4E2A\u5355\u8BCD\u7684\u6700\u5927\u957F\u5EA6\uFF08\u6BD4\u4F20\u9012\u7684\u503C\u957F\u7684\u5355\u8BCD\u5C06\u88AB\u62C6\u5206\uFF09"),Hl=f(),T=a("p"),Qi=n("\u6807\u8BB0\u5316\u7684\u7B2C\u4E00\u6B65\u662F\u89C4\u8303\u5316\uFF0C\u6240\u4EE5\u8BA9\u6211\u4EEC\u4ECE\u5B83\u5F00\u59CB\u3002 \u7531\u4E8E BERT \u88AB\u5E7F\u6CDB\u4F7F\u7528\uFF0C\u6240\u4EE5\u6709\u4E00\u4E2A\u53EF\u4EE5\u4F7F\u7528\u7684 "),Un=a("code"),ef=n("BertNormalizer"),sf=n("\uFF0C\u6211\u4EEC\u53EF\u4EE5\u4E3A BERT \u8BBE\u7F6E\u7ECF\u5178\u7684\u9009\u9879\uFF1A"),Mn=a("code"),tf=n("lowercase\uFF08\u5C0F\u5199\uFF09"),nf=n(" \u548C "),Xn=a("code"),rf=n("strip_accents\uFF08\u53BB\u9664\u97F3\u8C03\uFF09"),lf=n("\uFF0C\u4E0D\u8A00\u81EA\u660E\uFF1B "),Kn=a("code"),of=n("clean_text"),af=n(" \u5220\u9664\u6240\u6709\u63A7\u5236\u5B57\u7B26\u5E76\u5C06\u91CD\u590D\u7684\u7A7A\u683C\u66FF\u6362\u4E3A\u4E00\u4E2A\uFF1B \u548C "),Hn=a("code"),pf=n("handle_chinese_chars"),ff=n("\uFF0C\u5728\u6C49\u5B57\u5468\u56F4\u653E\u7F6E\u7A7A\u683C\u3002 \u8981\u5B9E\u73B0 "),Yn=a("code"),mf=n("bert-base-uncased"),cf=n(" \uFF0C\u6211\u4EEC\u53EF\u4EE5\u8FD9\u6837\u8BBE\u7F6E\u8FD9\u4E2A\u89C4\u8303\u5668\uFF1A"),Yl=f(),c(ds.$$.fragment),Jl=f(),Bt=a("p"),kf=n("\u7136\u800C\uFF0C\u4E00\u822C\u6765\u8BF4\uFF0C\u5728\u6784\u5EFA\u65B0\u7684\u6807\u8BB0\u5668\u65F6\uFF0C\u60A8\u53EF\u4EE5\u4F7F\u7528\u5DF2\u7ECF\u5728 \u{1F917} Tokenizers\u5E93\u4E2D\u5B9E\u73B0\u7684\u975E\u5E38\u65B9\u4FBF\u7684normalizer\u2014\u2014\u6240\u4EE5\u8BA9\u6211\u4EEC\u770B\u770B\u5982\u4F55\u624B\u52A8\u521B\u5EFA BERT normalizer\u3002 \u8BE5\u5E93\u63D0\u4F9B\u4E86\u4E00\u4E2A\u201CLowercase\uFF08\u5C0F\u5199\uFF09\u201D\u7684normalizer\u548C\u4E00\u4E2A\u201CStripAccents\u201D\u7684normalizer\uFF0C\u60A8\u53EF\u4EE5\u4F7F\u7528\u201C\u5E8F\u5217\u201D\u7EC4\u5408\u591A\u4E2Anormalizer\uFF1A"),Vl=f(),c(Es.$$.fragment),Zl=f(),ee=a("p"),_f=n("\u6211\u4EEC\u4E5F\u5728\u4F7F\u7528 "),Jn=a("strong"),$f=n("NFD"),uf=n(" Unicode normalizer\uFF0C\u5426\u5219 "),Vn=a("strong"),zf=n("StripAccents"),hf=n(" normalizer \u65E0\u6CD5\u6B63\u786E\u8BC6\u522B\u5E26\u91CD\u97F3\u7684\u5B57\u7B26\uFF0C\u56E0\u6B64\u6CA1\u529E\u6CD5\u5220\u9664\u5B83\u4EEC\u3002"),Ql=f(),se=a("p"),gf=n("\u6B63\u5982\u6211\u4EEC\u4E4B\u524D\u770B\u5230\u7684\uFF0C\u6211\u4EEC\u53EF\u4EE5\u4F7F\u7528 "),Zn=a("strong"),jf=n("normalize"),df=n(" \u7684 "),Qn=a("strong"),Ef=n("normalize_str()"),xf=n(" \u65B9\u6CD5\u67E5\u770B\u5B83\u5BF9\u7ED9\u5B9A\u6587\u672C\u7684\u5F71\u54CD\uFF1A"),eo=f(),c(xs.$$.fragment),so=f(),c(vs.$$.fragment),to=f(),c(Oe.$$.fragment),no=f(),Ft=a("p"),vf=n("\u63A5\u4E0B\u6765\u662F\u9884\u6807\u8BB0\u6B65\u9AA4\u3002 \u540C\u6837\uFF0C\u6211\u4EEC\u53EF\u4EE5\u4F7F\u7528\u4E00\u4E2A\u9884\u6784\u5EFA\u7684\u201CBertPreTokenizer\u201D\uFF1A"),ro=f(),c(bs.$$.fragment),lo=f(),At=a("p"),bf=n("\u6216\u8005\u6211\u4EEC\u53EF\u4EE5\u4ECE\u5934\u5F00\u59CB\u6784\u5EFA\u5B83\uFF1A"),oo=f(),c(Ts.$$.fragment),ao=f(),Le=a("p"),Tf=n("\u8BF7\u6CE8\u610F\uFF0C"),er=a("code"),Pf=n("Whitespace"),qf=n(" \u9884\u6807\u8BB0\u5668\u4F1A\u5728\u7A7A\u683C\u548C\u6240\u6709\u975E\u5B57\u6BCD\u3001\u6570\u5B57\u6216\u4E0B\u5212\u7EBF\u5B57\u7B26\u7684\u5B57\u7B26\u4E0A\u8FDB\u884C\u62C6\u5206\uFF0C\u56E0\u6B64\u5728\u672C\u6B21\u7684\u4F8B\u5B50\u4E2D\u4E0A\u4F1A\u6839\u636E\u7A7A\u683C\u548C\u6807\u70B9\u7B26\u53F7\u8FDB\u884C\u62C6\u5206\uFF1A"),po=f(),c(Ps.$$.fragment),io=f(),c(qs.$$.fragment),fo=f(),ye=a("p"),wf=n("\u5982\u679C\u60A8\u53EA\u60F3\u5728\u7A7A\u767D\u5904\u8FDB\u884C\u62C6\u5206\uFF0C\u5219\u5E94\u4F7F\u7528 "),sr=a("strong"),Sf=n("WhitespaceSplit"),Nf=n(" \u4EE3\u66FF\u9884\u6807\u8BB0\u5668\uFF1A"),mo=f(),c(ws.$$.fragment),co=f(),c(Ss.$$.fragment),ko=f(),Re=a("p"),Of=n("\u50CFnormalizers\u4E00\u6837\uFF0C\u60A8\u53EF\u4EE5\u4F7F\u7528 "),tr=a("strong"),Lf=n("Sequence"),yf=n(" \u7EC4\u6210\u51E0\u4E2A\u9884\u6807\u8BB0\u5668\uFF1A"),_o=f(),c(Ns.$$.fragment),$o=f(),c(Os.$$.fragment),uo=f(),De=a("p"),Rf=n("\u6807\u8BB0\u5316\u7BA1\u9053\u7684\u4E0B\u4E00\u6B65\u662F\u8F93\u5165\u7ED9\u6A21\u578B\u3002\u6211\u4EEC\u5DF2\u7ECF\u5728\u521D\u59CB\u5316\u4E2D\u6307\u5B9A\u4E86\u6211\u4EEC\u7684\u6A21\u578B\uFF0C\u4F46\u6211\u4EEC\u4ECD\u7136\u9700\u8981\u8BAD\u7EC3\u5B83\uFF0C\u8FD9\u5C06\u9700\u8981\u4E00\u4E2A "),nr=a("strong"),Df=n("WordPieceTrainer"),Gf=n(" .\u5728 \u{1F917} Tokenizers \u4E2D\u5B9E\u4F8B\u5316\u8BAD\u7EC3\u5668\u65F6\u8981\u8BB0\u4F4F\u7684\u4E3B\u8981\u4E8B\u60C5\u662F\uFF0C\u60A8\u9700\u8981\u5C06\u60A8\u6253\u7B97\u4F7F\u7528\u7684\u6240\u6709\u7279\u6B8A\u6807\u8BB0\u4F20\u9012\u7ED9\u5B83 - \u5426\u5219\u5B83\u4E0D\u4F1A\u5C06\u5B83\u4EEC\u6DFB\u52A0\u5230\u8BCD\u6C47\u8868\u4E2D\uFF0C\u56E0\u4E3A\u5B83\u4EEC\u4E0D\u5728\u8BAD\u7EC3\u8BED\u6599\u5E93\u4E2D\uFF1A"),zo=f(),c(Ls.$$.fragment),ho=f(),S=a("p"),Cf=n("\u4EE5\u53CA\u6307\u5B9A "),rr=a("strong"),Bf=n("vocab_size\uFF08\u8BCD\u5178\u5927\u5C0F\uFF09"),Ff=n(" \u548C "),lr=a("strong"),Af=n("special_tokens\uFF08\u7279\u6B8A\u7684\u6807\u8BB0\uFF09"),Wf=n(" \uFF0C\u6211\u4EEC\u53EF\u4EE5\u8BBE\u7F6E "),or=a("strong"),If=n("min_frequency"),Uf=n(" \uFF08\u8BB0\u53F7\u5FC5\u987B\u51FA\u73B0\u5728\u8BCD\u6C47\u8868\u4E2D\u7684\u6B21\u6570\uFF09\u6216\u66F4\u6539 "),ar=a("strong"),Mf=n("continuing_subword_prefix"),Xf=n(" \uFF08\u5982\u679C\u6211\u4EEC\u60F3\u4F7F\u7528\u4E0E "),pr=a("strong"),Kf=n("##"),Hf=n("\u6307\u4EE3\u5B58\u5728\u4E0E\u5B57\u8BCD\u76F8\u540C\u7684\u524D\u7F00 \uFF09\u3002"),go=f(),Wt=a("p"),Yf=n("\u8981\u4F7F\u7528\u6211\u4EEC\u4E4B\u524D\u5B9A\u4E49\u7684\u8FED\u4EE3\u5668\u8BAD\u7EC3\u6211\u4EEC\u7684\u6A21\u578B\uFF0C\u6211\u4EEC\u53EA\u9700\u8981\u6267\u884C\u4EE5\u4E0B\u547D\u4EE4\uFF1A"),jo=f(),c(ys.$$.fragment),Eo=f(),Ge=a("p"),Jf=n("\u6211\u4EEC\u8FD8\u53EF\u4EE5\u4F7F\u7528\u6587\u672C\u6587\u4EF6\u6765\u8BAD\u7EC3\u6211\u4EEC\u7684\u6807\u8BB0\u5668\uFF0C\u5B83\u770B\u8D77\u6765\u50CF\u8FD9\u6837\uFF08\u6211\u4EEC\u9700\u8981\u5148\u521D\u59CB\u5316\u4E00\u4E2A\u7A7A\u7684 "),ir=a("strong"),Vf=n("WordPiece"),Zf=n(" \uFF09\uFF1A"),xo=f(),c(Rs.$$.fragment),vo=f(),Ce=a("p"),Qf=n("\u5728\u8FD9\u4E24\u79CD\u60C5\u51B5\u4E0B\uFF0C\u6211\u4EEC\u90FD\u53EF\u4EE5\u901A\u8FC7\u8C03\u7528\u6587\u672C\u6765\u6D4B\u8BD5\u6807\u8BB0\u5668 "),fr=a("strong"),em=n("encode()"),sm=n(" \u65B9\u6CD5\uFF1A"),bo=f(),c(Ds.$$.fragment),To=f(),c(Gs.$$.fragment),Po=f(),E=a("p"),tm=n("\u8FD9\u4E2A "),mr=a("strong"),nm=n("encoding"),rm=n(" \u83B7\u5F97\u7684\u662F\u4E00\u4E2A "),cr=a("strong"),lm=n("Encoding"),om=n("\u5BF9\u8C61 \uFF0C\u5B83\u7684\u5C5E\u6027\u4E2D\u5305\u542B\u6807\u8BB0\u5668\u7684\u6240\u6709\u5FC5\u8981\u8F93\u51FA\uFF1A "),kr=a("strong"),am=n("ids"),pm=n(" , "),_r=a("strong"),im=n("type_ids"),fm=n(" , "),$r=a("strong"),mm=n("tokens"),cm=n(" , "),ur=a("strong"),km=n("offsets"),_m=n(" , "),zr=a("strong"),$m=n("attention_mask"),um=n(" , "),hr=a("strong"),zm=n("special_tokens_mask"),hm=n(" \uFF0C \u548C "),gr=a("strong"),gm=n("overflowing"),jm=n(" ."),qo=f(),N=a("p"),dm=n("\u6807\u8BB0\u5316\u7BA1\u9053\u7684\u6700\u540E\u4E00\u6B65\u662F\u540E\u5904\u7406\u3002\u6211\u4EEC\u9700\u8981\u6DFB\u52A0 "),jr=a("strong"),Em=n("[CLS]"),xm=n(" \u5F00\u5934\u7684\u6807\u8BB0\u548C "),dr=a("strong"),vm=n("[SEP]"),bm=n(" \u6807\u8BB0\u5728\u672B\u5C3E\uFF08\u6216\u5728\u6BCF\u4E2A\u53E5\u5B50\u4E4B\u540E\uFF0C\u5982\u679C\u6211\u4EEC\u6709\u4E00\u5BF9\u53E5\u5B50\uFF09\u3002\u6211\u4EEC\u5C06\u4F7F\u7528\u4E00\u4E2A "),Er=a("strong"),Tm=n("TemplateProcessor"),Pm=n(" \u4E3A\u6B64\uFF0C\u4F46\u9996\u5148\u6211\u4EEC\u9700\u8981\u77E5\u9053 "),xr=a("strong"),qm=n("[CLS]"),wm=n(" \u548C "),vr=a("strong"),Sm=n("[SEP]"),Nm=n(" \u5728\u8BCD\u6C47\u8868\u4E2D\u7684ID\uFF1A"),wo=f(),c(Cs.$$.fragment),So=f(),c(Bs.$$.fragment),No=f(),F=a("p"),Om=n("\u4E3A\u4E86\u7ED9 "),br=a("strong"),Lm=n("TemplateProcessor"),ym=n(" \u7F16\u5199\u6A21\u677F\uFF0C\u6211\u4EEC\u5FC5\u987B\u6307\u5B9A\u5982\u4F55\u5904\u7406\u5355\u4E2A\u53E5\u5B50\u548C\u4E00\u5BF9\u53E5\u5B50\u3002\u5BF9\u4E8E\u4E24\u8005\uFF0C\u6211\u4EEC\u90FD\u7F16\u5199\u4E86\u6211\u4EEC\u60F3\u8981\u4F7F\u7528\u7684\u7279\u6B8A\u6807\u8BB0\uFF1B\u7B2C\u4E00\u4E2A\uFF08\u6216\u5355\u4E2A\uFF09\u53E5\u5B50\u8868\u793A\u4E3A "),Tr=a("strong"),Rm=n("$A"),Dm=n(" \uFF0C\u800C\u7B2C\u4E8C\u4E2A\u53E5\u5B50\uFF08\u5982\u679C\u5BF9\u4E00\u5BF9\u8FDB\u884C\u7F16\u7801\uFF09\u8868\u793A\u4E3A "),Pr=a("strong"),Gm=n("$B"),Cm=n(" .\u5BF9\u4E8E\u8FD9\u4E9B\u7279\u6B8A\u6807\u8BB0\u548C\u53E5\u5B50\uFF0C\u6211\u4EEC\u8FD8\u9700\u8981\u4F7F\u7528\u5728\u5192\u53F7\u540E\u6307\u5B9A\u76F8\u5E94\u7684\u6807\u8BB0\u7C7B\u578B ID\u3002"),Oo=f(),It=a("p"),Bm=n("\u56E0\u6B64\u7ECF\u5178\u7684 BERT \u6A21\u677F\u5B9A\u4E49\u5982\u4E0B\uFF1A"),Lo=f(),c(Fs.$$.fragment),yo=f(),Ut=a("p"),Fm=n("\u8BF7\u6CE8\u610F\uFF0C\u6211\u4EEC\u9700\u8981\u4F20\u9012\u7279\u6B8A\u6807\u8BB0\u7684 ID\uFF0C\u4EE5\u4FBF\u6807\u8BB0\u5668\u53EF\u4EE5\u6B63\u786E\u5730\u5C06\u7279\u6B8A\u6807\u8BB0\u8F6C\u6362\u4E3A\u5B83\u4EEC\u7684 ID\u3002"),Ro=f(),Mt=a("p"),Am=n("\u6DFB\u52A0\u540E\uFF0C\u6211\u4EEC\u4E4B\u524D\u7684\u793A\u4F8B\u5C06\u8F93\u51FA\u51FA\uFF1A"),Do=f(),c(As.$$.fragment),Go=f(),c(Ws.$$.fragment),Co=f(),Xt=a("p"),Wm=n("\u5728\u4E00\u5BF9\u53E5\u5B50\u4E2D\uFF0C\u6211\u4EEC\u5F97\u5230\u4E86\u6B63\u786E\u7684\u7ED3\u679C\uFF1A"),Bo=f(),c(Is.$$.fragment),Fo=f(),c(Us.$$.fragment),Ao=f(),Kt=a("p"),Im=n("\u6211\u4EEC\u51E0\u4E4E\u4ECE\u5934\u5F00\u59CB\u6784\u5EFA\u4E86\u8FD9\u4E2A\u6807\u8BB0\u5668\u2014\u2014\u4F46\u662F\u8FD8\u6709\u6700\u540E\u4E00\u6B65\u662F\u6307\u5B9A\u4E00\u4E2A\u89E3\u7801\u5668\uFF1A"),Wo=f(),c(Ms.$$.fragment),Io=f(),Be=a("p"),Um=n("\u8BA9\u6211\u4EEC\u6D4B\u8BD5\u4E00\u4E0B\u6211\u4EEC\u4E4B\u524D\u7684 "),qr=a("strong"),Mm=n("encoding"),Xm=n(" \uFF1A"),Uo=f(),c(Xs.$$.fragment),Mo=f(),c(Ks.$$.fragment),Xo=f(),Ht=a("p"),Km=n("\u5F88\u597D\uFF01\u6211\u4EEC\u53EF\u4EE5\u5C06\u6807\u8BB0\u5668\u4FDD\u5B58\u5728\u4E00\u4E2A JSON \u6587\u4EF6\u4E2D\uFF0C\u5982\u4E0B\u6240\u793A\uFF1A"),Ko=f(),c(Hs.$$.fragment),Ho=f(),te=a("p"),Hm=n("\u7136\u540E\u6211\u4EEC\u53EF\u4EE5\u4F7F\u7528"),wr=a("strong"),Ym=n("from_file()"),Jm=n(" \u65B9\u6CD5\u4ECE\u8BE5\u6587\u4EF6\u91CC\u91CD\u65B0\u52A0\u8F7D "),Sr=a("strong"),Vm=n("Tokenizer"),Zm=n(" \u5BF9\u8C61\uFF1A"),Yo=f(),c(Ys.$$.fragment),Jo=f(),ne=a("p"),Qm=n("\u8981\u5728 \u{1F917} Transformers \u4E2D\u4F7F\u7528\u8FD9\u4E2A\u6807\u8BB0\u5668\uFF0C\u6211\u4EEC\u5FC5\u987B\u5C06\u5B83\u5305\u88F9\u5728\u4E00\u4E2A "),Nr=a("strong"),ec=n("PreTrainedTokenizerFast"),sc=n(" \u7C7B\u4E2D\u3002\u6211\u4EEC\u53EF\u4EE5\u4F7F\u7528\u6CDB\u578B\u7C7B\uFF0C\u6216\u8005\uFF0C\u5982\u679C\u6211\u4EEC\u7684\u6807\u8BB0\u5668\u5BF9\u5E94\u4E8E\u73B0\u6709\u6A21\u578B\uFF0C\u5219\u4F7F\u7528\u8BE5\u7C7B\uFF08\u4F8B\u5982\u8FD9\u91CC\u7684 "),Or=a("strong"),tc=n("BertTokenizerFast"),nc=n(" \uFF09\u3002\u5982\u679C\u60A8\u5E94\u7528\u672C\u8BFE\u6765\u6784\u5EFA\u5168\u65B0\u7684\u6807\u8BB0\u5668\uFF0C\u5219\u5FC5\u987B\u4F7F\u7528\u7B2C\u4E00\u4E2A\u9009\u9879\u3002"),Vo=f(),O=a("p"),rc=n("\u8981\u5C06\u6807\u8BB0\u5668\u5305\u88C5\u5728 "),Lr=a("code"),lc=n("PreTrainedTokenizerFast"),oc=n(" \u7C7B\u4E2D\uFF0C\u6211\u4EEC\u53EF\u4EE5\u5C06\u6211\u4EEC\u6784\u5EFA\u7684\u6807\u8BB0\u5668\u4F5C\u4E3A"),yr=a("code"),ac=n("tokenizer_object"),pc=n(" \u4F20\u9012\uFF0C\u6216\u8005\u5C06\u6211\u4EEC\u4FDD\u5B58\u4E3A"),Rr=a("code"),ic=n("tokenizer_file"),fc=n(" \u7684\u6807\u8BB0\u5668\u6587\u4EF6\u4F20\u9012\u3002 \u8981\u8BB0\u4F4F\u7684\u5173\u952E\u662F\u6211\u4EEC\u5FC5\u987B\u624B\u52A8\u8BBE\u7F6E\u6240\u6709\u7279\u6B8A\u6807\u8BB0\uFF0C\u56E0\u4E3A\u8BE5\u7C7B\u65E0\u6CD5\u4ECE "),Dr=a("code"),mc=n("tokenizer"),cc=n(" \u5BF9\u8C61\u63A8\u65AD\u51FA\u54EA\u4E2A\u6807\u8BB0\u662F\u63A9\u7801\u6807\u8BB0\u3001"),Gr=a("code"),kc=n("[CLS]"),_c=n(" \u6807\u8BB0\u7B49\uFF1A"),Zo=f(),c(Js.$$.fragment),Qo=f(),Fe=a("p"),$c=n("\u5982\u679C\u60A8\u4F7F\u7528\u7279\u5B9A\u7684\u6807\u8BB0\u5668\u7C7B\uFF08\u4F8B\u5982 "),Cr=a("strong"),uc=n("BertTokenizerFast"),zc=n(" )\uFF0C\u60A8\u53EA\u9700\u8981\u6307\u5B9A\u4E0E\u9ED8\u8BA4\u6807\u8BB0\u4E0D\u540C\u7684\u7279\u6B8A\u6807\u8BB0\uFF08\u6B64\u5904\u6CA1\u6709\uFF09\uFF1A"),ea=f(),c(Vs.$$.fragment),sa=f(),re=a("p"),hc=n("\u7136\u540E\uFF0C\u60A8\u53EF\u4EE5\u50CF\u4F7F\u7528\u4EFB\u4F55\u5176\u4ED6 \u{1F917} Transformers \u6807\u8BB0\u5668\u4E00\u6837\u4F7F\u7528\u6B64\u6807\u8BB0\u5668\u3002\u4F60\u53EF\u4EE5\u7528 "),Br=a("strong"),gc=n("save_pretrained()"),jc=n(" \u65B9\u6CD5\uFF0C\u6216\u4F7F\u7528 "),Fr=a("strong"),dc=n("push_to_hub()"),Ec=n(" \u65B9\u6CD5\u3002"),ta=f(),Yt=a("p"),xc=n("\u73B0\u5728\u6211\u4EEC\u5DF2\u7ECF\u4E86\u89E3\u4E86\u5982\u4F55\u6784\u5EFA WordPiece \u6807\u8BB0\u5668\uFF0C\u8BA9\u6211\u4EEC\u5BF9 BPE \u6807\u8BB0\u5668\u8FDB\u884C\u540C\u6837\u7684\u64CD\u4F5C\u3002\u56E0\u4E3A\u60A8\u5DF2\u7ECF\u77E5\u9053\u4E86\u6240\u6709\u6B65\u9AA4\uFF0C\u6240\u4EE5\u6211\u4EEC\u4F1A\u8FDB\u884C\u5730\u66F4\u5FEB\u4E00\u70B9\uFF0C\u5E76\u4E14\u53EA\u7A81\u51FA\u5C55\u793A\u4E24\u8005\u4E0D\u4E00\u6837\u7684\u5730\u65B9\u3002"),na=f(),ue=a("h2"),Ae=a("a"),Ar=a("span"),c(Zs.$$.fragment),vc=f(),Wr=a("span"),bc=n("\u4ECE\u5934\u5F00\u59CB\u6784\u5EFA BPE \u6807\u8BB0\u5668"),ra=f(),We=a("p"),Tc=n("\u73B0\u5728\u8BA9\u6211\u4EEC\u6784\u5EFA\u4E00\u4E2A GPT-2 \u6807\u8BB0\u5668\u3002\u4E0E BERT \u6807\u8BB0\u5668\u4E00\u6837\uFF0C\u6211\u4EEC\u9996\u5148\u4F7F\u7528 "),Ir=a("strong"),Pc=n("Tokenizer"),qc=n(" \u521D\u59CB\u5316\u4E00\u4E2ABPE \u6A21\u578B\uFF1A"),la=f(),c(Qs.$$.fragment),oa=f(),le=a("p"),wc=n("\u548C BERT \u4E00\u6837\uFF0C\u5982\u679C\u6211\u4EEC\u6709\u4E00\u4E2A\u8BCD\u6C47\u8868\uFF0C\u6211\u4EEC\u53EF\u4EE5\u7528\u4E00\u4E2A\u8BCD\u6C47\u8868\u6765\u521D\u59CB\u5316\u8FD9\u4E2A\u6A21\u578B\uFF08\u5728\u8FD9\u79CD\u60C5\u51B5\u4E0B\uFF0C\u6211\u4EEC\u9700\u8981\u4F20\u9012 "),Ur=a("code"),Sc=n("vocab"),Nc=n(" \u548C "),Mr=a("code"),Oc=n("merges"),Lc=n("\uFF09\uFF0C\u4F46\u662F\u7531\u4E8E\u6211\u4EEC\u5C06\u4ECE\u5934\u5F00\u59CB\u8BAD\u7EC3\uFF0C\u6240\u4EE5\u6211\u4EEC\u4E0D\u9700\u8981\u8FD9\u6837\u53BB\u505A\u3002 \u6211\u4EEC\u4E5F\u4E0D\u9700\u8981\u6307\u5B9A\u201Cunk_token\u201D\uFF0C\u56E0\u4E3A GPT-2 \u4F7F\u7528\u7684\u5B57\u8282\u7EA7 BPE\uFF0C\u4E0D\u9700\u8981\u201Cunk_token\u201D\u3002"),aa=f(),Jt=a("p"),yc=n("GPT-2 \u4E0D\u4F7F\u7528\u5F52\u4E00\u5316\u5668\uFF0C\u56E0\u6B64\u6211\u4EEC\u8DF3\u8FC7\u8BE5\u6B65\u9AA4\u5E76\u76F4\u63A5\u8FDB\u5165\u9884\u6807\u8BB0\u5316\uFF1A"),pa=f(),c(et.$$.fragment),ia=f(),Ie=a("p"),Rc=n("\u6211\u4EEC\u5728\u6B64\u5904\u6DFB\u52A0\u5230 "),Xr=a("code"),Dc=n("ByteLevel"),Gc=n(" \u7684\u9009\u9879\u662F\u4E0D\u5728\u53E5\u5B50\u5F00\u5934\u6DFB\u52A0\u7A7A\u683C\uFF08\u9ED8\u8BA4\u4E3Ature\uFF09\u3002 \u6211\u4EEC\u53EF\u4EE5\u770B\u4E00\u4E0B\u4F7F\u7528\u8FD9\u4E2A\u6807\u8BB0\u5668\u5BF9\u4E4B\u524D\u793A\u4F8B\u6587\u672C\u7684\u9884\u6807\u8BB0\uFF1A"),fa=f(),c(st.$$.fragment),ma=f(),c(tt.$$.fragment),ca=f(),Vt=a("p"),Cc=n("\u63A5\u4E0B\u6765\u662F\u9700\u8981\u8BAD\u7EC3\u7684\u6A21\u578B\u3002\u5BF9\u4E8E GPT-2\uFF0C\u552F\u4E00\u7684\u7279\u6B8A\u6807\u8BB0\u662F\u6587\u672C\u7ED3\u675F\u6807\u8BB0\uFF1A"),ka=f(),c(nt.$$.fragment),_a=f(),P=a("p"),Bc=n("\u4E0E "),Kr=a("code"),Fc=n("WordPieceTrainer"),Ac=n(" \u4EE5\u53CA "),Hr=a("code"),Wc=n("vocab_size"),Ic=n(" \u548C "),Yr=a("code"),Uc=n("special_tokens"),Mc=n(" \u4E00\u6837\uFF0C\u6211\u4EEC\u53EF\u4EE5\u6307\u5B9A "),Jr=a("code"),Xc=n("min_frequency"),Kc=n(" \u5982\u679C\u6211\u4EEC\u613F\u610F\uFF0C\u6216\u8005\u5982\u679C\u6211\u4EEC\u6709\u4E00\u4E2A\u8BCD\u5C3E\u540E\u7F00\uFF08\u5982 "),Vr=a("code"),Hc=n("</w>"),Yc=n(" )\uFF0C\u6211\u4EEC\u53EF\u4EE5\u4F7F\u7528 "),Zr=a("code"),Jc=n("end_of_word_suffix"),Vc=n(" \u8BBE\u7F6E\u5B83\u3002"),$a=f(),Zt=a("p"),Zc=n("\u8FD9\u4E2A\u6807\u8BB0\u5668\u4E5F\u53EF\u4EE5\u5728\u6587\u672C\u6587\u4EF6\u4E0A\u8BAD\u7EC3\uFF1A"),ua=f(),c(rt.$$.fragment),za=f(),Qt=a("p"),Qc=n("\u8BA9\u6211\u4EEC\u770B\u4E00\u4E0B\u793A\u4F8B\u6587\u672C\u7684\u6807\u8BB0\u5316\u540E\u7684\u7ED3\u679C\uFF1A"),ha=f(),c(lt.$$.fragment),ga=f(),c(ot.$$.fragment),ja=f(),en=a("p"),e2=n("\u6211\u4EEC\u5BF9 GPT-2 \u6807\u8BB0\u5668\u6DFB\u52A0\u5B57\u8282\u7EA7\u540E\u5904\u7406\uFF0C\u5982\u4E0B\u6240\u793A\uFF1A"),da=f(),c(at.$$.fragment),Ea=f(),ze=a("p"),Qr=a("code"),s2=n("trim_offsets = False"),t2=n(" \u9009\u9879\u6307\u793A\u6211\u4EEC\u5E94\u8BE5\u4FDD\u7559\u4EE5 \u2018\u0120\u2019 \u5F00\u5934\u7684\u6807\u8BB0\u7684\u504F\u79FB\u91CF\uFF1A\u8FD9\u6837\u504F\u79FB\u91CF\u7684\u5F00\u5934\u5C06\u6307\u5411\u5355\u8BCD\u4E4B\u524D\u7684\u7A7A\u683C\uFF0C\u800C\u4E0D\u662F\u7B2C\u4E00\u4E2A\u5355\u8BCD\u7684\u5B57\u7B26\uFF08\u56E0\u4E3A\u7A7A\u683C\u5728\u6280\u672F\u4E0A\u662F\u6807\u8BB0\u7684\u4E00\u90E8\u5206\uFF09\u3002 \u8BA9\u6211\u4EEC\u770B\u770B\u6211\u4EEC\u521A\u521A\u7F16\u7801\u7684\u6587\u672C\u7684\u7ED3\u679C\uFF0C\u5176\u4E2D "),el=a("code"),n2=n("'\u0120test'"),r2=n(" \u662F\u7D22\u5F15\u7B2C 4 \u5904\u7684\u6807\u8BB0\uFF1A"),xa=f(),c(pt.$$.fragment),va=f(),c(it.$$.fragment),ba=f(),sn=a("p"),l2=n("\u6700\u540E\uFF0C\u6211\u4EEC\u6DFB\u52A0\u4E00\u4E2A\u5B57\u8282\u7EA7\u89E3\u7801\u5668\uFF1A"),Ta=f(),c(ft.$$.fragment),Pa=f(),tn=a("p"),o2=n("\u6211\u4EEC\u53EF\u4EE5\u4ED4\u7EC6\u68C0\u67E5\u5B83\u662F\u5426\u6B63\u5E38\u5DE5\u4F5C\uFF1A"),qa=f(),c(mt.$$.fragment),wa=f(),c(ct.$$.fragment),Sa=f(),oe=a("p"),a2=n("\u5F88\u597D\uFF01\u73B0\u5728\u6211\u4EEC\u5B8C\u6210\u4E86\uFF0C\u6211\u4EEC\u53EF\u4EE5\u50CF\u4EE5\u524D\u4E00\u6837\u4FDD\u5B58\u6807\u8BB0\u5668\uFF0C\u5E76\u5C06\u5B83\u5305\u88C5\u5728\u4E00\u4E2A "),sl=a("strong"),p2=n("PreTrainedTokenizerFast"),i2=n(" \u6216\u8005 "),tl=a("strong"),f2=n("GPT2TokenizerFast"),m2=n(" \u5982\u679C\u6211\u4EEC\u60F3\u5728 \u{1F917} Transformers\u4E2D\u4F7F\u7528\u5B83\uFF1A"),Na=f(),c(kt.$$.fragment),Oa=f(),nn=a("p"),c2=n("\u6216\u8005\uFF1A"),La=f(),c(_t.$$.fragment),ya=f(),rn=a("p"),k2=n("\u4F5C\u4E3A\u6700\u540E\u4E00\u4E2A\u793A\u4F8B\uFF0C\u6211\u4EEC\u5C06\u5411\u60A8\u5C55\u793A\u5982\u4F55\u4ECE\u5934\u5F00\u59CB\u6784\u5EFA Unigram \u6807\u8BB0\u5668\u3002"),Ra=f(),he=a("h2"),Ue=a("a"),nl=a("span"),c($t.$$.fragment),_2=f(),rl=a("span"),$2=n("\u4ECE\u5934\u5F00\u59CB\u6784\u5EFA Unigram \u6807\u8BB0\u5668"),Da=f(),Me=a("p"),u2=n("\u73B0\u5728\u8BA9\u6211\u4EEC\u6784\u5EFA\u4E00\u4E2A XLNet \u6807\u8BB0\u5668\u3002\u4E0E\u4E4B\u524D\u7684\u6807\u8BB0\u5668\u4E00\u6837\uFF0C\u6211\u4EEC\u9996\u5148\u4F7F\u7528 Unigram \u6A21\u578B\u521D\u59CB\u5316\u4E00\u4E2A "),ll=a("strong"),z2=n("Tokenizer"),h2=n(" \uFF1A"),Ga=f(),c(ut.$$.fragment),Ca=f(),ln=a("p"),g2=n("\u540C\u6837\uFF0C\u5982\u679C\u6211\u4EEC\u6709\u8BCD\u6C47\u8868\uFF0C\u6211\u4EEC\u53EF\u4EE5\u7528\u8BCD\u6C47\u8868\u521D\u59CB\u5316\u8FD9\u4E2A\u6A21\u578B\u3002"),Ba=f(),on=a("p"),j2=n("\u5BF9\u4E8E\u6807\u51C6\u5316\uFF0CXLNet \u4F7F\u7528\u4E86\u4E00\u4E9B\u66FF\u6362\u7684\u65B9\u6CD5\uFF08\u6765\u81EA SentencePiece\uFF09\uFF1A"),Fa=f(),c(zt.$$.fragment),Aa=f(),A=a("p"),d2=n("\u8FD9\u4F1A\u53D6\u4EE3 "),ol=a("strong"),E2=n("\u201C"),x2=n(" \u548C "),al=a("strong"),v2=n("\u201D"),b2=n(" \u548C "),pl=a("strong"),T2=n("\u201D"),P2=n(" \u4EE5\u53CA\u4EFB\u4F55\u4E24\u4E2A\u6216\u591A\u4E2A\u7A7A\u683C\u4E0E\u5355\u4E2A\u7A7A\u683C\u7684\u5E8F\u5217\uFF0C\u4EE5\u53CA\u5220\u9664\u6587\u672C\u4E2D\u7684\u91CD\u97F3\u4EE5\u8FDB\u884C\u6807\u8BB0\u3002"),Wa=f(),Xe=a("p"),q2=n("\u7528\u4E8E\u4EFB\u4F55 SentencePiece \u6807\u8BB0\u5668\u7684\u9884\u6807\u8BB0\u5668\u662F "),il=a("code"),w2=n("Metaspace"),S2=n("\uFF1A"),Ia=f(),c(ht.$$.fragment),Ua=f(),an=a("p"),N2=n("\u6211\u4EEC\u53EF\u4EE5\u50CF\u4EE5\u524D\u4E00\u6837\u67E5\u770B\u793A\u4F8B\u6587\u672C\u7684\u9884\u6807\u8BB0\u5316\uFF1A"),Ma=f(),c(gt.$$.fragment),Xa=f(),c(jt.$$.fragment),Ka=f(),pn=a("p"),O2=n("\u63A5\u4E0B\u6765\u662F\u9700\u8981\u8BAD\u7EC3\u7684\u6A21\u578B\u3002 XLNet \u6709\u4E0D\u5C11\u7279\u6B8A\u7684\u6807\u8BB0\uFF1A"),Ha=f(),c(dt.$$.fragment),Ya=f(),ae=a("p"),L2=n("\u4E0D\u8981\u5FD8\u8BB0"),fl=a("code"),y2=n("UnigramTrainer"),R2=n(" \u7684\u4E00\u4E2A\u975E\u5E38\u91CD\u8981\u7684\u53C2\u6570\u662F"),ml=a("code"),D2=n("unk_token"),G2=n("\u3002 \u6211\u4EEC\u8FD8\u53EF\u4EE5\u4F20\u9012\u7279\u5B9A\u4E8E Unigram \u7B97\u6CD5\u7684\u5176\u4ED6\u53C2\u6570\uFF0C\u4F8B\u5982\u5220\u9664\u6807\u8BB0\u7684\u6BCF\u4E2A\u6B65\u9AA4\u7684\u201Cshrinking_factor\uFF08\u6536\u7F29\u56E0\u5B50\uFF09\u201D\uFF08\u9ED8\u8BA4\u4E3A 0.75\uFF09\u6216\u6307\u5B9A\u7ED9\u5B9A\u6807\u8BB0\u7684\u6700\u5927\u957F\u5EA6\u7684\u201Cmax_piece_length\u201D\uFF08\u9ED8\u8BA4\u4E3A 16\uFF09 ."),Ja=f(),fn=a("p"),C2=n("\u8FD9\u4E2A\u6807\u8BB0\u5668\u4E5F\u53EF\u4EE5\u5728\u6587\u672C\u6587\u4EF6\u4E0A\u8BAD\u7EC3\uFF1A"),Va=f(),c(Et.$$.fragment),Za=f(),mn=a("p"),B2=n("\u8BA9\u6211\u4EEC\u770B\u4E00\u4E0B\u793A\u4F8B\u6587\u672C\u7684\u6807\u8BB0\u5316\u540E\u7684\u7ED3\u679C\uFF1A"),Qa=f(),c(xt.$$.fragment),ep=f(),c(vt.$$.fragment),sp=f(),q=a("p"),F2=n("A peculiarity of XLNet is that it puts the "),cl=a("code"),A2=n("<cls>"),W2=n(" token at the end of the sentence, with a type ID of 2 (to distinguish it from the other tokens). It\u2019s padding on the left, as a result. We can deal with all the special tokens and token type IDs with a template, like for BERT, but first we have to get the IDs of the "),kl=a("code"),I2=n("<cls>"),U2=n(" and "),_l=a("code"),M2=n("<sep>"),X2=n(` tokens:
XLNet \u7684\u4E00\u4E2A\u7279\u70B9\u662F\u5B83\u5C06`),$l=a("code"),K2=n("<cls>"),H2=n(" \u6807\u8BB0\u653E\u5728\u53E5\u5B50\u7684\u672B\u5C3E\uFF0C\u7C7B\u578BID \u4E3A2\uFF08\u4EE5\u5C06\u5176\u4E0E\u5176\u4ED6\u6807\u8BB0\u533A\u5206\u5F00\u6765\uFF09\u3002\u5B83\u4F1A\u5C06\u7ED3\u679C\u586B\u5145\u5728\u5DE6\u4FA7\u3002 \u6211\u4EEC\u53EF\u4EE5\u4F7F\u7528\u6A21\u677F\u5904\u7406\u6240\u6709\u7279\u6B8A\u6807\u8BB0\u548C\u6807\u8BB0\u7C7B\u578B ID\uFF0C\u4F8B\u5982 BERT\uFF0C\u4F46\u9996\u5148\u6211\u4EEC\u5FC5\u987B\u83B7\u53D6 "),ul=a("code"),Y2=n("<cls>"),J2=n(" \u548C "),zl=a("code"),V2=n("<sep>"),Z2=n(" \u6807\u8BB0\u7684 ID\uFF1A"),tp=f(),c(bt.$$.fragment),np=f(),c(Tt.$$.fragment),rp=f(),cn=a("p"),Q2=n("\u6A21\u677F\u5982\u4E0B\u6240\u793A\uFF1A"),lp=f(),c(Pt.$$.fragment),op=f(),kn=a("p"),ek=n("\u6211\u4EEC\u53EF\u4EE5\u901A\u8FC7\u7F16\u7801\u4E00\u5BF9\u53E5\u5B50\u6765\u6D4B\u8BD5\u5B83\u7684\u5DE5\u4F5C\u539F\u7406\uFF1A"),ap=f(),c(qt.$$.fragment),pp=f(),c(wt.$$.fragment),ip=f(),Ke=a("p"),sk=n("\u6700\u540E\uFF0C\u6211\u4EEC\u6DFB\u52A0\u4E00\u4E2A "),hl=a("strong"),tk=n("Metaspace"),nk=n(" \u89E3\u7801\u5668\uFF1A"),fp=f(),c(St.$$.fragment),mp=f(),W=a("p"),rk=n("\u6211\u4EEC\u5B8C\u6210\u4E86\u8FD9\u4E2A\u6807\u8BB0\u5668\uFF01 \u6211\u4EEC\u53EF\u4EE5\u50CF\u4EE5\u524D\u4E00\u6837\u4FDD\u5B58\u6807\u8BB0\u5668\uFF0C\u5982\u679C\u6211\u4EEC\u60F3\u5728 \u{1F917} Transformers \u4E2D\u4F7F\u7528\u5B83\uFF0C\u53EF\u4EE5\u5C06\u5B83\u5305\u88C5\u5728 "),gl=a("code"),lk=n("PreTrainedTokenizerFast"),ok=n(" \u6216 "),jl=a("code"),ak=n("XLNetTokenizerFast"),pk=n(" \u4E2D\u3002 \u4F7F\u7528 "),dl=a("code"),ik=n("PreTrainedTokenizerFast"),fk=n(" \u65F6\u8981\u6CE8\u610F\u7684\u4E00\u4EF6\u4E8B\u662F\uFF0C\u6211\u4EEC\u9700\u8981\u544A\u8BC9\u{1F917} Transformers \u5E93\u5E94\u8BE5\u5728\u5DE6\u4FA7\u586B\u5145\u7279\u6B8A\u6807\u8BB0\uFF1A"),cp=f(),c(Nt.$$.fragment),kp=f(),_n=a("p"),mk=n("\u6216\u8005\uFF1A"),_p=f(),c(Ot.$$.fragment),$p=f(),$n=a("p"),ck=n("\u73B0\u5728\u60A8\u5DF2\u7ECF\u4E86\u89E3\u4E86\u5982\u4F55\u4F7F\u7528\u5404\u79CD\u6784\u5EFA\u5757\u6765\u6784\u5EFA\u73B0\u6709\u7684\u6807\u8BB0\u5668\uFF0C\u60A8\u5E94\u8BE5\u80FD\u591F\u4F7F\u7528 \u{1F917} tokenizer\u5E93\u7F16\u5199\u60A8\u60F3\u8981\u7684\u4EFB\u4F55\u6807\u8BB0\u5668\uFF0C\u5E76\u80FD\u591F\u5728 \u{1F917} Transformers\u4E2D\u4F7F\u7528\u5B83\u3002"),this.h()},l(e){const l=$u('[data-svelte="svelte-1phssyn"]',document.head);j=p(l,"META",{name:!0,content:!0}),l.forEach(t),Y=m(e),K=p(e,"H1",{id:!0});var El=i(K);Qe=r(El,"\u9010\u5757\u5730\u6784\u5EFA\u6807\u8BB0\u5668"),El.forEach(t),J=m(e),k(H.$$.fragment,e),ge=m(e),G=p(e,"P",{});var uk=i(G);es=r(uk,"\u6B63\u5982\u6211\u4EEC\u5728\u524D\u51E0\u8282\u4E2D\u770B\u5230\u7684\uFF0C\u6807\u8BB0\u5316\u5305\u62EC\u51E0\u4E2A\u6B65\u9AA4\uFF1A"),uk.forEach(t),je=m(e),v=p(e,"UL",{});var He=i(v);pe=p(He,"LI",{});var zk=i(pe);ss=r(zk,"\u89C4\u8303\u5316\uFF08\u4EFB\u4F55\u8BA4\u4E3A\u5FC5\u8981\u7684\u6587\u672C\u6E05\u7406\uFF0C\u4F8B\u5982\u5220\u9664\u7A7A\u683C\u6216\u91CD\u97F3\u7B26\u53F7\u3001Unicode \u89C4\u8303\u5316\u7B49\uFF09"),zk.forEach(t),ie=m(He),fe=p(He,"LI",{});var hk=i(fe);ts=r(hk,"\u9884\u6807\u8BB0\u5316\uFF08\u5C06\u8F93\u5165\u62C6\u5206\u4E3A\u5355\u8BCD\uFF09"),hk.forEach(t),me=m(He),ce=p(He,"LI",{});var gk=i(ce);ns=r(gk,"\u901A\u8FC7\u6A21\u578B\u5904\u7406\u8F93\u5165\uFF08\u4F7F\u7528\u9884\u5148\u62C6\u5206\u7684\u8BCD\u6765\u751F\u6210\u4E00\u7CFB\u5217\u6807\u8BB0\uFF09"),gk.forEach(t),ke=m(He),_e=p(He,"LI",{});var jk=i(_e);rs=r(jk,"\u540E\u5904\u7406\uFF08\u6DFB\u52A0\u6807\u8BB0\u5668\u7684\u7279\u6B8A\u6807\u8BB0\uFF0C\u751F\u6210\u6CE8\u610F\u529B\u63A9\u7801\u548C\u6807\u8BB0\u7C7B\u578B ID\uFF09"),jk.forEach(t),He.forEach(t),V=m(e),d=p(e,"P",{});var dk=i(d);Lt=r(dk,"\u63D0\u9192\u4E00\u4E0B\uFF0C\u8FD9\u91CC\u518D\u770B\u4E00\u4E0B\u6574\u4E2A\u8FC7\u7A0B"),dk.forEach(t),ls=m(e),C=p(e,"DIV",{class:!0});var zp=i(C);Z=p(zp,"IMG",{class:!0,src:!0,alt:!0}),yt=m(zp),Q=p(zp,"IMG",{class:!0,src:!0,alt:!0}),zp.forEach(t),Nl=m(e),de=p(e,"P",{});var hp=i(de);Cp=r(hp,"\u{1F917} Tokenizers \u5E93\u65E8\u5728\u4E3A\u6BCF\u4E2A\u6B65\u9AA4\u63D0\u4F9B\u591A\u4E2A\u9009\u9879\uFF0C\u60A8\u53EF\u4EE5\u5C06\u5B83\u4EEC\u6DF7\u5408\u548C\u5339\u914D\u5728\u4E00\u8D77\u3002\u5728\u672C\u8282\u4E2D\uFF0C\u6211\u4EEC\u5C06\u770B\u5230\u5982\u4F55\u4ECE\u5934\u5F00\u59CB\u6784\u5EFA\u6807\u8BB0\u5668\uFF0C\u800C\u4E0D\u662F\u50CF\u6211\u4EEC"),Rt=p(hp,"A",{href:!0});var Ek=i(Rt);Bp=r(Ek,"\u7B2C\u4E8C\u8282 2"),Ek.forEach(t),Fp=r(hp,"\u90A3\u6837\u4ECE\u65E7\u7684\u6807\u8BB0\u5668\u8BAD\u7EC3\u65B0\u7684\u6807\u8BB0\u5668.\u7136\u540E\uFF0C\u60A8\u5C06\u80FD\u591F\u6784\u5EFA\u60A8\u80FD\u60F3\u5230\u7684\u4EFB\u4F55\u7C7B\u578B\u7684\u6807\u8BB0\u5668\uFF01"),hp.forEach(t),Ol=m(e),k(os.$$.fragment,e),Ll=m(e),Dt=p(e,"P",{});var xk=i(Dt);Ap=r(xk,"\u66F4\u51C6\u786E\u5730\u8BF4\uFF0C\u8BE5\u5E93\u662F\u56F4\u7ED5\u4E00\u4E2A\u4E2D\u592E\u201CTokenizer\u201D\u7C7B\u6784\u5EFA\u7684\uFF0C\u6784\u5EFA\u8FD9\u4E2A\u7C7B\u7684\u6BCF\u4E00\u90E8\u5206\u53EF\u4EE5\u5728\u5B50\u6A21\u5757\u7684\u5217\u8868\u4E2D\u91CD\u65B0\u7EC4\u5408\uFF1A"),xk.forEach(t),yl=m(e),w=p(e,"UL",{});var I=i(w);Ee=p(I,"LI",{});var xl=i(Ee);bn=p(xl,"CODE",{});var vk=i(bn);Wp=r(vk,"normalizers"),vk.forEach(t),Ip=r(xl," \u5305\u542B\u4F60\u53EF\u4EE5\u4F7F\u7528\u7684\u6240\u6709\u53EF\u80FD\u7684Normalizer\u7C7B\u578B\uFF08\u5B8C\u6574\u5217\u8868"),as=p(xl,"A",{href:!0,rel:!0});var bk=i(as);Up=r(bk,"\u5728\u8FD9\u91CC"),bk.forEach(t),Mp=r(xl,"\uFF09\u3002"),xl.forEach(t),Xp=m(I),xe=p(I,"LI",{});var vl=i(xe);Tn=p(vl,"CODE",{});var Tk=i(Tn);Kp=r(Tk,"pre_tokenizesr"),Tk.forEach(t),Hp=r(vl," \u5305\u542B\u60A8\u53EF\u4EE5\u4F7F\u7528\u7684\u6240\u6709\u53EF\u80FD\u7684PreTokenizer\u7C7B\u578B\uFF08\u5B8C\u6574\u5217\u8868"),ps=p(vl,"A",{href:!0,rel:!0});var Pk=i(ps);Yp=r(Pk,"\u5728\u8FD9\u91CC"),Pk.forEach(t),Jp=r(vl,"\uFF09\u3002"),vl.forEach(t),Vp=m(I),ve=p(I,"LI",{});var bl=i(ve);Pn=p(bl,"CODE",{});var qk=i(Pn);Zp=r(qk,"models"),qk.forEach(t),Qp=r(bl," \u5305\u542B\u60A8\u53EF\u4EE5\u4F7F\u7528\u7684\u5404\u79CD\u7C7B\u578B\u7684Model\uFF0C\u5982BPE\u3001WordPiece\u548CUnigram\uFF08\u5B8C\u6574\u5217\u8868"),is=p(bl,"A",{href:!0,rel:!0});var wk=i(is);ei=r(wk,"\u5728\u8FD9\u91CC"),wk.forEach(t),si=r(bl,"\uFF09\u3002"),bl.forEach(t),ti=m(I),be=p(I,"LI",{});var Tl=i(be);qn=p(Tl,"CODE",{});var Sk=i(qn);ni=r(Sk,"trainers"),Sk.forEach(t),ri=r(Tl," \u5305\u542B\u6240\u6709\u4E0D\u540C\u7C7B\u578B\u7684 trainer\uFF0C\u4F60\u53EF\u4EE5\u4F7F\u7528\u4E00\u4E2A\u8BED\u6599\u5E93\u8BAD\u7EC3\u4F60\u7684\u6A21\u578B\uFF08\u6BCF\u79CD\u6A21\u578B\u4E00\u4E2A\uFF1B\u5B8C\u6574\u5217\u8868"),fs=p(Tl,"A",{href:!0,rel:!0});var Nk=i(fs);li=r(Nk,"\u5728\u8FD9\u91CC"),Nk.forEach(t),oi=r(Tl,"\uFF09\u3002"),Tl.forEach(t),ai=m(I),Te=p(I,"LI",{});var Pl=i(Te);wn=p(Pl,"CODE",{});var Ok=i(wn);pi=r(Ok,"post_processors"),Ok.forEach(t),ii=r(Pl," \u5305\u542B\u4F60\u53EF\u4EE5\u4F7F\u7528\u7684\u5404\u79CD\u7C7B\u578B\u7684PostProcessor\uFF08\u5B8C\u6574\u5217\u8868"),ms=p(Pl,"A",{href:!0,rel:!0});var Lk=i(ms);fi=r(Lk,"\u5728\u8FD9\u91CC"),Lk.forEach(t),mi=r(Pl,"\uFF09\u3002"),Pl.forEach(t),ci=m(I),Pe=p(I,"LI",{});var ql=i(Pe);Sn=p(ql,"CODE",{});var yk=i(Sn);ki=r(yk,"decoders"),yk.forEach(t),_i=r(ql," \u5305\u542B\u5404\u79CD\u7C7B\u578B\u7684Decoder\uFF0C\u53EF\u4EE5\u7528\u6765\u89E3\u7801\u6807\u8BB0\u5316\u7684\u8F93\u51FA\uFF08\u5B8C\u6574\u5217\u8868"),cs=p(ql,"A",{href:!0,rel:!0});var Rk=i(cs);$i=r(Rk,"\u5728\u8FD9\u91CC"),Rk.forEach(t),ui=r(ql,"\uFF09\u3002"),ql.forEach(t),I.forEach(t),Rl=m(e),qe=p(e,"P",{});var gp=i(qe);zi=r(gp,"\u60A8\u53EF\u4EE5"),ks=p(gp,"A",{href:!0,rel:!0});var Dk=i(ks);hi=r(Dk,"\u5728\u8FD9\u91CC"),Dk.forEach(t),gi=r(gp,"\u627E\u5230\u5B8C\u6574\u7684\u6A21\u5757\u5217\u8868\u3002"),gp.forEach(t),Dl=m(e),_s=p(e,"H2",{id:!0});var Gk=i(_s);ji=r(Gk,"\u83B7\u53D6\u8BED\u200B\u200B\u6599\u5E93"),Gk.forEach(t),Gl=m(e),we=p(e,"P",{});var jp=i(we);di=r(jp,"\u4E3A\u4E86\u8BAD\u7EC3\u6211\u4EEC\u7684\u65B0\u6807\u8BB0\u5668\uFF0C\u6211\u4EEC\u5C06\u4F7F\u7528\u4E00\u4E2A\u5C0F\u7684\u6587\u672C\u8BED\u6599\u5E93\uFF08\u56E0\u6B64\u793A\u4F8B\u8FD0\u884C\u5F97\u5F88\u5FEB\uFF09\u3002\u83B7\u53D6\u8BED\u200B\u200B\u6599\u5E93\u7684\u6B65\u9AA4\u4E0E\u6211\u4EEC\u5728[\u5728\u8FD9\u7AE0\u7684\u5F00\u59CB]((/course/chapter6/2)\u90A3\u4E00\u5C0F\u8282\uFF0C\u4F46\u8FD9\u6B21\u6211\u4EEC\u5C06\u4F7F\u7528"),$s=p(jp,"A",{href:!0,rel:!0});var Ck=i($s);Ei=r(Ck,"WikiText-2"),Ck.forEach(t),xi=r(jp,"\u6570\u636E\u96C6\uFF1A"),jp.forEach(t),Cl=m(e),k(us.$$.fragment,e),Bl=m(e),zs=p(e,"P",{});var kk=i(zs);Nn=p(kk,"STRONG",{});var Bk=i(Nn);vi=r(Bk,"get_training_corpus()"),Bk.forEach(t),bi=r(kk," \u51FD\u6570\u662F\u4E00\u4E2A\u751F\u6210\u5668\uFF0C\u6BCF\u6B21\u8C03\u7528\u7684\u65F6\u5019\u5C06\u4EA7\u751F 1,000 \u4E2A\u6587\u672C\uFF0C\u6211\u4EEC\u5C06\u7528\u5B83\u6765\u8BAD\u7EC3\u6807\u8BB0\u5668\u3002"),kk.forEach(t),Fl=m(e),Gt=p(e,"P",{});var Fk=i(Gt);Ti=r(Fk,"\u{1F917} Tokenizers \u4E5F\u53EF\u4EE5\u76F4\u63A5\u5728\u6587\u672C\u6587\u4EF6\u4E0A\u8FDB\u884C\u8BAD\u7EC3\u3002\u4EE5\u4E0B\u662F\u6211\u4EEC\u5982\u4F55\u751F\u6210\u4E00\u4E2A\u6587\u672C\u6587\u4EF6\uFF0C\u5176\u4E2D\u5305\u542B\u6211\u4EEC\u53EF\u4EE5\u5728\u672C\u5730\u4F7F\u7528\u7684\u6765\u81EA WikiText-2 \u7684\u6240\u6709\u6587\u672C/\u8F93\u5165\uFF1A"),Fk.forEach(t),Al=m(e),k(hs.$$.fragment,e),Wl=m(e),Ct=p(e,"P",{});var Ak=i(Ct);Pi=r(Ak,"\u63A5\u4E0B\u6765\uFF0C\u6211\u4EEC\u5C06\u5411\u60A8\u5C55\u793A\u5982\u4F55\u9010\u5757\u6784\u5EFA\u60A8\u81EA\u5DF1\u7684 BERT\u3001GPT-2 \u548C XLNet \u6807\u8BB0\u5668\u3002\u8FD9\u5C06\u4E3A\u6211\u4EEC\u63D0\u4F9B\u4E09\u4E2A\u4E3B\u8981\u6807\u8BB0\u5316\u7B97\u6CD5\u7684\u793A\u4F8B\uFF1AWordPiece\u3001BPE \u548C Unigram\u3002\u8BA9\u6211\u4EEC\u4ECE BERT \u5F00\u59CB\u5427\uFF01"),Ak.forEach(t),Il=m(e),$e=p(e,"H2",{class:!0});var dp=i($e);Se=p(dp,"A",{id:!0,class:!0,href:!0});var Wk=i(Se);On=p(Wk,"SPAN",{});var Ik=i(On);k(gs.$$.fragment,Ik),Ik.forEach(t),Wk.forEach(t),qi=m(dp),Ln=p(dp,"SPAN",{});var Uk=i(Ln);wi=r(Uk,"\u4ECE\u5934\u5F00\u59CB\u6784\u5EFA WordPiece \u6807\u8BB0\u5668"),Uk.forEach(t),dp.forEach(t),Ul=m(e),b=p(e,"P",{});var L=i(b);Si=r(L,"\u8981\u4F7F\u7528 \u{1F917} Tokenizers \u5E93\u6784\u5EFA\u6807\u8BB0\u5668\uFF0C\u6211\u4EEC\u9996\u5148\u4F7F\u7528"),yn=p(L,"STRONG",{});var Mk=i(yn);Ni=r(Mk,"model"),Mk.forEach(t),Oi=r(L,"\u5B9E\u4F8B\u5316\u4E00\u4E2A "),Rn=p(L,"STRONG",{});var Xk=i(Rn);Li=r(Xk,"Tokenizer"),Xk.forEach(t),yi=r(L," \u5BF9\u8C61\u4E0E  \uFF0C\u7136\u540E\u5C06 "),Dn=p(L,"STRONG",{});var Kk=i(Dn);Ri=r(Kk,"normalizer"),Kk.forEach(t),Di=r(L," , "),Gn=p(L,"STRONG",{});var Hk=i(Gn);Gi=r(Hk,"pre_tokenizer"),Hk.forEach(t),Ci=r(L," , "),Cn=p(L,"STRONG",{});var Yk=i(Cn);Bi=r(Yk,"post_processor"),Yk.forEach(t),Fi=r(L," \uFF0C \u548C "),Bn=p(L,"STRONG",{});var Jk=i(Bn);Ai=r(Jk,"decoder"),Jk.forEach(t),Wi=r(L," \u5C5E\u6027\u8BBE\u7F6E\u6210\u6211\u4EEC\u60F3\u8981\u7684\u503C\u3002"),L.forEach(t),Ml=m(e),Ne=p(e,"P",{});var Ep=i(Ne);Ii=r(Ep,"\u5BF9\u4E8E\u8FD9\u4E2A\u4F8B\u5B50\uFF0C\u6211\u4EEC\u5C06\u521B\u5EFA\u4E00\u4E2A "),Fn=p(Ep,"STRONG",{});var Vk=i(Fn);Ui=r(Vk,"Tokenizer"),Vk.forEach(t),Mi=r(Ep," \u4F7F\u7528 WordPiece \u6A21\u578B\uFF1A"),Ep.forEach(t),Xl=m(e),k(js.$$.fragment,e),Kl=m(e),B=p(e,"P",{});var Ye=i(B);Xi=r(Ye,"\u6211\u4EEC\u5FC5\u987B\u6307\u5B9A "),An=p(Ye,"STRONG",{});var Zk=i(An);Ki=r(Zk,"unk_token"),Zk.forEach(t),Hi=r(Ye," \u8FD9\u6837\u6A21\u578B\u624D\u77E5\u9053\u5F53\u5B83\u9047\u5230\u4EE5\u524D\u6CA1\u6709\u89C1\u8FC7\u7684\u5B57\u7B26\u65F6\u8981\u8FD4\u56DE\u4EC0\u4E48\u3002\u6211\u4EEC\u53EF\u4EE5\u5728\u6B64\u5904\u8BBE\u7F6E\u7684\u5176\u4ED6\u53C2\u6570\u5305\u62EC\u6211\u4EEC\u6A21\u578B\u7684"),Wn=p(Ye,"STRONG",{});var Qk=i(Wn);Yi=r(Qk,"vocab\uFF08\u5B57\u5178\uFF09"),Qk.forEach(t),Ji=r(Ye,"\uFF08\u6211\u4EEC\u5C06\u8BAD\u7EC3\u6A21\u578B\uFF0C\u6240\u4EE5\u6211\u4EEC\u4E0D\u9700\u8981\u8BBE\u7F6E\u5B83\uFF09\u548C "),In=p(Ye,"STRONG",{});var e_=i(In);Vi=r(e_,"max_input_chars_per_word"),e_.forEach(t),Zi=r(Ye," \u5373\u6BCF\u4E2A\u5355\u8BCD\u7684\u6700\u5927\u957F\u5EA6\uFF08\u6BD4\u4F20\u9012\u7684\u503C\u957F\u7684\u5355\u8BCD\u5C06\u88AB\u62C6\u5206\uFF09"),Ye.forEach(t),Hl=m(e),T=p(e,"P",{});var y=i(T);Qi=r(y,"\u6807\u8BB0\u5316\u7684\u7B2C\u4E00\u6B65\u662F\u89C4\u8303\u5316\uFF0C\u6240\u4EE5\u8BA9\u6211\u4EEC\u4ECE\u5B83\u5F00\u59CB\u3002 \u7531\u4E8E BERT \u88AB\u5E7F\u6CDB\u4F7F\u7528\uFF0C\u6240\u4EE5\u6709\u4E00\u4E2A\u53EF\u4EE5\u4F7F\u7528\u7684 "),Un=p(y,"CODE",{});var s_=i(Un);ef=r(s_,"BertNormalizer"),s_.forEach(t),sf=r(y,"\uFF0C\u6211\u4EEC\u53EF\u4EE5\u4E3A BERT \u8BBE\u7F6E\u7ECF\u5178\u7684\u9009\u9879\uFF1A"),Mn=p(y,"CODE",{});var t_=i(Mn);tf=r(t_,"lowercase\uFF08\u5C0F\u5199\uFF09"),t_.forEach(t),nf=r(y," \u548C "),Xn=p(y,"CODE",{});var n_=i(Xn);rf=r(n_,"strip_accents\uFF08\u53BB\u9664\u97F3\u8C03\uFF09"),n_.forEach(t),lf=r(y,"\uFF0C\u4E0D\u8A00\u81EA\u660E\uFF1B "),Kn=p(y,"CODE",{});var r_=i(Kn);of=r(r_,"clean_text"),r_.forEach(t),af=r(y," \u5220\u9664\u6240\u6709\u63A7\u5236\u5B57\u7B26\u5E76\u5C06\u91CD\u590D\u7684\u7A7A\u683C\u66FF\u6362\u4E3A\u4E00\u4E2A\uFF1B \u548C "),Hn=p(y,"CODE",{});var l_=i(Hn);pf=r(l_,"handle_chinese_chars"),l_.forEach(t),ff=r(y,"\uFF0C\u5728\u6C49\u5B57\u5468\u56F4\u653E\u7F6E\u7A7A\u683C\u3002 \u8981\u5B9E\u73B0 "),Yn=p(y,"CODE",{});var o_=i(Yn);mf=r(o_,"bert-base-uncased"),o_.forEach(t),cf=r(y," \uFF0C\u6211\u4EEC\u53EF\u4EE5\u8FD9\u6837\u8BBE\u7F6E\u8FD9\u4E2A\u89C4\u8303\u5668\uFF1A"),y.forEach(t),Yl=m(e),k(ds.$$.fragment,e),Jl=m(e),Bt=p(e,"P",{});var a_=i(Bt);kf=r(a_,"\u7136\u800C\uFF0C\u4E00\u822C\u6765\u8BF4\uFF0C\u5728\u6784\u5EFA\u65B0\u7684\u6807\u8BB0\u5668\u65F6\uFF0C\u60A8\u53EF\u4EE5\u4F7F\u7528\u5DF2\u7ECF\u5728 \u{1F917} Tokenizers\u5E93\u4E2D\u5B9E\u73B0\u7684\u975E\u5E38\u65B9\u4FBF\u7684normalizer\u2014\u2014\u6240\u4EE5\u8BA9\u6211\u4EEC\u770B\u770B\u5982\u4F55\u624B\u52A8\u521B\u5EFA BERT normalizer\u3002 \u8BE5\u5E93\u63D0\u4F9B\u4E86\u4E00\u4E2A\u201CLowercase\uFF08\u5C0F\u5199\uFF09\u201D\u7684normalizer\u548C\u4E00\u4E2A\u201CStripAccents\u201D\u7684normalizer\uFF0C\u60A8\u53EF\u4EE5\u4F7F\u7528\u201C\u5E8F\u5217\u201D\u7EC4\u5408\u591A\u4E2Anormalizer\uFF1A"),a_.forEach(t),Vl=m(e),k(Es.$$.fragment,e),Zl=m(e),ee=p(e,"P",{});var un=i(ee);_f=r(un,"\u6211\u4EEC\u4E5F\u5728\u4F7F\u7528 "),Jn=p(un,"STRONG",{});var p_=i(Jn);$f=r(p_,"NFD"),p_.forEach(t),uf=r(un," Unicode normalizer\uFF0C\u5426\u5219 "),Vn=p(un,"STRONG",{});var i_=i(Vn);zf=r(i_,"StripAccents"),i_.forEach(t),hf=r(un," normalizer \u65E0\u6CD5\u6B63\u786E\u8BC6\u522B\u5E26\u91CD\u97F3\u7684\u5B57\u7B26\uFF0C\u56E0\u6B64\u6CA1\u529E\u6CD5\u5220\u9664\u5B83\u4EEC\u3002"),un.forEach(t),Ql=m(e),se=p(e,"P",{});var zn=i(se);gf=r(zn,"\u6B63\u5982\u6211\u4EEC\u4E4B\u524D\u770B\u5230\u7684\uFF0C\u6211\u4EEC\u53EF\u4EE5\u4F7F\u7528 "),Zn=p(zn,"STRONG",{});var f_=i(Zn);jf=r(f_,"normalize"),f_.forEach(t),df=r(zn," \u7684 "),Qn=p(zn,"STRONG",{});var m_=i(Qn);Ef=r(m_,"normalize_str()"),m_.forEach(t),xf=r(zn," \u65B9\u6CD5\u67E5\u770B\u5B83\u5BF9\u7ED9\u5B9A\u6587\u672C\u7684\u5F71\u54CD\uFF1A"),zn.forEach(t),eo=m(e),k(xs.$$.fragment,e),so=m(e),k(vs.$$.fragment,e),to=m(e),k(Oe.$$.fragment,e),no=m(e),Ft=p(e,"P",{});var c_=i(Ft);vf=r(c_,"\u63A5\u4E0B\u6765\u662F\u9884\u6807\u8BB0\u6B65\u9AA4\u3002 \u540C\u6837\uFF0C\u6211\u4EEC\u53EF\u4EE5\u4F7F\u7528\u4E00\u4E2A\u9884\u6784\u5EFA\u7684\u201CBertPreTokenizer\u201D\uFF1A"),c_.forEach(t),ro=m(e),k(bs.$$.fragment,e),lo=m(e),At=p(e,"P",{});var k_=i(At);bf=r(k_,"\u6216\u8005\u6211\u4EEC\u53EF\u4EE5\u4ECE\u5934\u5F00\u59CB\u6784\u5EFA\u5B83\uFF1A"),k_.forEach(t),oo=m(e),k(Ts.$$.fragment,e),ao=m(e),Le=p(e,"P",{});var xp=i(Le);Tf=r(xp,"\u8BF7\u6CE8\u610F\uFF0C"),er=p(xp,"CODE",{});var __=i(er);Pf=r(__,"Whitespace"),__.forEach(t),qf=r(xp," \u9884\u6807\u8BB0\u5668\u4F1A\u5728\u7A7A\u683C\u548C\u6240\u6709\u975E\u5B57\u6BCD\u3001\u6570\u5B57\u6216\u4E0B\u5212\u7EBF\u5B57\u7B26\u7684\u5B57\u7B26\u4E0A\u8FDB\u884C\u62C6\u5206\uFF0C\u56E0\u6B64\u5728\u672C\u6B21\u7684\u4F8B\u5B50\u4E2D\u4E0A\u4F1A\u6839\u636E\u7A7A\u683C\u548C\u6807\u70B9\u7B26\u53F7\u8FDB\u884C\u62C6\u5206\uFF1A"),xp.forEach(t),po=m(e),k(Ps.$$.fragment,e),io=m(e),k(qs.$$.fragment,e),fo=m(e),ye=p(e,"P",{});var vp=i(ye);wf=r(vp,"\u5982\u679C\u60A8\u53EA\u60F3\u5728\u7A7A\u767D\u5904\u8FDB\u884C\u62C6\u5206\uFF0C\u5219\u5E94\u4F7F\u7528 "),sr=p(vp,"STRONG",{});var $_=i(sr);Sf=r($_,"WhitespaceSplit"),$_.forEach(t),Nf=r(vp," \u4EE3\u66FF\u9884\u6807\u8BB0\u5668\uFF1A"),vp.forEach(t),mo=m(e),k(ws.$$.fragment,e),co=m(e),k(Ss.$$.fragment,e),ko=m(e),Re=p(e,"P",{});var bp=i(Re);Of=r(bp,"\u50CFnormalizers\u4E00\u6837\uFF0C\u60A8\u53EF\u4EE5\u4F7F\u7528 "),tr=p(bp,"STRONG",{});var u_=i(tr);Lf=r(u_,"Sequence"),u_.forEach(t),yf=r(bp," \u7EC4\u6210\u51E0\u4E2A\u9884\u6807\u8BB0\u5668\uFF1A"),bp.forEach(t),_o=m(e),k(Ns.$$.fragment,e),$o=m(e),k(Os.$$.fragment,e),uo=m(e),De=p(e,"P",{});var Tp=i(De);Rf=r(Tp,"\u6807\u8BB0\u5316\u7BA1\u9053\u7684\u4E0B\u4E00\u6B65\u662F\u8F93\u5165\u7ED9\u6A21\u578B\u3002\u6211\u4EEC\u5DF2\u7ECF\u5728\u521D\u59CB\u5316\u4E2D\u6307\u5B9A\u4E86\u6211\u4EEC\u7684\u6A21\u578B\uFF0C\u4F46\u6211\u4EEC\u4ECD\u7136\u9700\u8981\u8BAD\u7EC3\u5B83\uFF0C\u8FD9\u5C06\u9700\u8981\u4E00\u4E2A "),nr=p(Tp,"STRONG",{});var z_=i(nr);Df=r(z_,"WordPieceTrainer"),z_.forEach(t),Gf=r(Tp," .\u5728 \u{1F917} Tokenizers \u4E2D\u5B9E\u4F8B\u5316\u8BAD\u7EC3\u5668\u65F6\u8981\u8BB0\u4F4F\u7684\u4E3B\u8981\u4E8B\u60C5\u662F\uFF0C\u60A8\u9700\u8981\u5C06\u60A8\u6253\u7B97\u4F7F\u7528\u7684\u6240\u6709\u7279\u6B8A\u6807\u8BB0\u4F20\u9012\u7ED9\u5B83 - \u5426\u5219\u5B83\u4E0D\u4F1A\u5C06\u5B83\u4EEC\u6DFB\u52A0\u5230\u8BCD\u6C47\u8868\u4E2D\uFF0C\u56E0\u4E3A\u5B83\u4EEC\u4E0D\u5728\u8BAD\u7EC3\u8BED\u6599\u5E93\u4E2D\uFF1A"),Tp.forEach(t),zo=m(e),k(Ls.$$.fragment,e),ho=m(e),S=p(e,"P",{});var U=i(S);Cf=r(U,"\u4EE5\u53CA\u6307\u5B9A "),rr=p(U,"STRONG",{});var h_=i(rr);Bf=r(h_,"vocab_size\uFF08\u8BCD\u5178\u5927\u5C0F\uFF09"),h_.forEach(t),Ff=r(U," \u548C "),lr=p(U,"STRONG",{});var g_=i(lr);Af=r(g_,"special_tokens\uFF08\u7279\u6B8A\u7684\u6807\u8BB0\uFF09"),g_.forEach(t),Wf=r(U," \uFF0C\u6211\u4EEC\u53EF\u4EE5\u8BBE\u7F6E "),or=p(U,"STRONG",{});var j_=i(or);If=r(j_,"min_frequency"),j_.forEach(t),Uf=r(U," \uFF08\u8BB0\u53F7\u5FC5\u987B\u51FA\u73B0\u5728\u8BCD\u6C47\u8868\u4E2D\u7684\u6B21\u6570\uFF09\u6216\u66F4\u6539 "),ar=p(U,"STRONG",{});var d_=i(ar);Mf=r(d_,"continuing_subword_prefix"),d_.forEach(t),Xf=r(U," \uFF08\u5982\u679C\u6211\u4EEC\u60F3\u4F7F\u7528\u4E0E "),pr=p(U,"STRONG",{});var E_=i(pr);Kf=r(E_,"##"),E_.forEach(t),Hf=r(U,"\u6307\u4EE3\u5B58\u5728\u4E0E\u5B57\u8BCD\u76F8\u540C\u7684\u524D\u7F00 \uFF09\u3002"),U.forEach(t),go=m(e),Wt=p(e,"P",{});var x_=i(Wt);Yf=r(x_,"\u8981\u4F7F\u7528\u6211\u4EEC\u4E4B\u524D\u5B9A\u4E49\u7684\u8FED\u4EE3\u5668\u8BAD\u7EC3\u6211\u4EEC\u7684\u6A21\u578B\uFF0C\u6211\u4EEC\u53EA\u9700\u8981\u6267\u884C\u4EE5\u4E0B\u547D\u4EE4\uFF1A"),x_.forEach(t),jo=m(e),k(ys.$$.fragment,e),Eo=m(e),Ge=p(e,"P",{});var Pp=i(Ge);Jf=r(Pp,"\u6211\u4EEC\u8FD8\u53EF\u4EE5\u4F7F\u7528\u6587\u672C\u6587\u4EF6\u6765\u8BAD\u7EC3\u6211\u4EEC\u7684\u6807\u8BB0\u5668\uFF0C\u5B83\u770B\u8D77\u6765\u50CF\u8FD9\u6837\uFF08\u6211\u4EEC\u9700\u8981\u5148\u521D\u59CB\u5316\u4E00\u4E2A\u7A7A\u7684 "),ir=p(Pp,"STRONG",{});var v_=i(ir);Vf=r(v_,"WordPiece"),v_.forEach(t),Zf=r(Pp," \uFF09\uFF1A"),Pp.forEach(t),xo=m(e),k(Rs.$$.fragment,e),vo=m(e),Ce=p(e,"P",{});var qp=i(Ce);Qf=r(qp,"\u5728\u8FD9\u4E24\u79CD\u60C5\u51B5\u4E0B\uFF0C\u6211\u4EEC\u90FD\u53EF\u4EE5\u901A\u8FC7\u8C03\u7528\u6587\u672C\u6765\u6D4B\u8BD5\u6807\u8BB0\u5668 "),fr=p(qp,"STRONG",{});var b_=i(fr);em=r(b_,"encode()"),b_.forEach(t),sm=r(qp," \u65B9\u6CD5\uFF1A"),qp.forEach(t),bo=m(e),k(Ds.$$.fragment,e),To=m(e),k(Gs.$$.fragment,e),Po=m(e),E=p(e,"P",{});var x=i(E);tm=r(x,"\u8FD9\u4E2A "),mr=p(x,"STRONG",{});var T_=i(mr);nm=r(T_,"encoding"),T_.forEach(t),rm=r(x," \u83B7\u5F97\u7684\u662F\u4E00\u4E2A "),cr=p(x,"STRONG",{});var P_=i(cr);lm=r(P_,"Encoding"),P_.forEach(t),om=r(x,"\u5BF9\u8C61 \uFF0C\u5B83\u7684\u5C5E\u6027\u4E2D\u5305\u542B\u6807\u8BB0\u5668\u7684\u6240\u6709\u5FC5\u8981\u8F93\u51FA\uFF1A "),kr=p(x,"STRONG",{});var q_=i(kr);am=r(q_,"ids"),q_.forEach(t),pm=r(x," , "),_r=p(x,"STRONG",{});var w_=i(_r);im=r(w_,"type_ids"),w_.forEach(t),fm=r(x," , "),$r=p(x,"STRONG",{});var S_=i($r);mm=r(S_,"tokens"),S_.forEach(t),cm=r(x," , "),ur=p(x,"STRONG",{});var N_=i(ur);km=r(N_,"offsets"),N_.forEach(t),_m=r(x," , "),zr=p(x,"STRONG",{});var O_=i(zr);$m=r(O_,"attention_mask"),O_.forEach(t),um=r(x," , "),hr=p(x,"STRONG",{});var L_=i(hr);zm=r(L_,"special_tokens_mask"),L_.forEach(t),hm=r(x," \uFF0C \u548C "),gr=p(x,"STRONG",{});var y_=i(gr);gm=r(y_,"overflowing"),y_.forEach(t),jm=r(x," ."),x.forEach(t),qo=m(e),N=p(e,"P",{});var M=i(N);dm=r(M,"\u6807\u8BB0\u5316\u7BA1\u9053\u7684\u6700\u540E\u4E00\u6B65\u662F\u540E\u5904\u7406\u3002\u6211\u4EEC\u9700\u8981\u6DFB\u52A0 "),jr=p(M,"STRONG",{});var R_=i(jr);Em=r(R_,"[CLS]"),R_.forEach(t),xm=r(M," \u5F00\u5934\u7684\u6807\u8BB0\u548C "),dr=p(M,"STRONG",{});var D_=i(dr);vm=r(D_,"[SEP]"),D_.forEach(t),bm=r(M," \u6807\u8BB0\u5728\u672B\u5C3E\uFF08\u6216\u5728\u6BCF\u4E2A\u53E5\u5B50\u4E4B\u540E\uFF0C\u5982\u679C\u6211\u4EEC\u6709\u4E00\u5BF9\u53E5\u5B50\uFF09\u3002\u6211\u4EEC\u5C06\u4F7F\u7528\u4E00\u4E2A "),Er=p(M,"STRONG",{});var G_=i(Er);Tm=r(G_,"TemplateProcessor"),G_.forEach(t),Pm=r(M," \u4E3A\u6B64\uFF0C\u4F46\u9996\u5148\u6211\u4EEC\u9700\u8981\u77E5\u9053 "),xr=p(M,"STRONG",{});var C_=i(xr);qm=r(C_,"[CLS]"),C_.forEach(t),wm=r(M," \u548C "),vr=p(M,"STRONG",{});var B_=i(vr);Sm=r(B_,"[SEP]"),B_.forEach(t),Nm=r(M," \u5728\u8BCD\u6C47\u8868\u4E2D\u7684ID\uFF1A"),M.forEach(t),wo=m(e),k(Cs.$$.fragment,e),So=m(e),k(Bs.$$.fragment,e),No=m(e),F=p(e,"P",{});var Je=i(F);Om=r(Je,"\u4E3A\u4E86\u7ED9 "),br=p(Je,"STRONG",{});var F_=i(br);Lm=r(F_,"TemplateProcessor"),F_.forEach(t),ym=r(Je," \u7F16\u5199\u6A21\u677F\uFF0C\u6211\u4EEC\u5FC5\u987B\u6307\u5B9A\u5982\u4F55\u5904\u7406\u5355\u4E2A\u53E5\u5B50\u548C\u4E00\u5BF9\u53E5\u5B50\u3002\u5BF9\u4E8E\u4E24\u8005\uFF0C\u6211\u4EEC\u90FD\u7F16\u5199\u4E86\u6211\u4EEC\u60F3\u8981\u4F7F\u7528\u7684\u7279\u6B8A\u6807\u8BB0\uFF1B\u7B2C\u4E00\u4E2A\uFF08\u6216\u5355\u4E2A\uFF09\u53E5\u5B50\u8868\u793A\u4E3A "),Tr=p(Je,"STRONG",{});var A_=i(Tr);Rm=r(A_,"$A"),A_.forEach(t),Dm=r(Je," \uFF0C\u800C\u7B2C\u4E8C\u4E2A\u53E5\u5B50\uFF08\u5982\u679C\u5BF9\u4E00\u5BF9\u8FDB\u884C\u7F16\u7801\uFF09\u8868\u793A\u4E3A "),Pr=p(Je,"STRONG",{});var W_=i(Pr);Gm=r(W_,"$B"),W_.forEach(t),Cm=r(Je," .\u5BF9\u4E8E\u8FD9\u4E9B\u7279\u6B8A\u6807\u8BB0\u548C\u53E5\u5B50\uFF0C\u6211\u4EEC\u8FD8\u9700\u8981\u4F7F\u7528\u5728\u5192\u53F7\u540E\u6307\u5B9A\u76F8\u5E94\u7684\u6807\u8BB0\u7C7B\u578B ID\u3002"),Je.forEach(t),Oo=m(e),It=p(e,"P",{});var I_=i(It);Bm=r(I_,"\u56E0\u6B64\u7ECF\u5178\u7684 BERT \u6A21\u677F\u5B9A\u4E49\u5982\u4E0B\uFF1A"),I_.forEach(t),Lo=m(e),k(Fs.$$.fragment,e),yo=m(e),Ut=p(e,"P",{});var U_=i(Ut);Fm=r(U_,"\u8BF7\u6CE8\u610F\uFF0C\u6211\u4EEC\u9700\u8981\u4F20\u9012\u7279\u6B8A\u6807\u8BB0\u7684 ID\uFF0C\u4EE5\u4FBF\u6807\u8BB0\u5668\u53EF\u4EE5\u6B63\u786E\u5730\u5C06\u7279\u6B8A\u6807\u8BB0\u8F6C\u6362\u4E3A\u5B83\u4EEC\u7684 ID\u3002"),U_.forEach(t),Ro=m(e),Mt=p(e,"P",{});var M_=i(Mt);Am=r(M_,"\u6DFB\u52A0\u540E\uFF0C\u6211\u4EEC\u4E4B\u524D\u7684\u793A\u4F8B\u5C06\u8F93\u51FA\u51FA\uFF1A"),M_.forEach(t),Do=m(e),k(As.$$.fragment,e),Go=m(e),k(Ws.$$.fragment,e),Co=m(e),Xt=p(e,"P",{});var X_=i(Xt);Wm=r(X_,"\u5728\u4E00\u5BF9\u53E5\u5B50\u4E2D\uFF0C\u6211\u4EEC\u5F97\u5230\u4E86\u6B63\u786E\u7684\u7ED3\u679C\uFF1A"),X_.forEach(t),Bo=m(e),k(Is.$$.fragment,e),Fo=m(e),k(Us.$$.fragment,e),Ao=m(e),Kt=p(e,"P",{});var K_=i(Kt);Im=r(K_,"\u6211\u4EEC\u51E0\u4E4E\u4ECE\u5934\u5F00\u59CB\u6784\u5EFA\u4E86\u8FD9\u4E2A\u6807\u8BB0\u5668\u2014\u2014\u4F46\u662F\u8FD8\u6709\u6700\u540E\u4E00\u6B65\u662F\u6307\u5B9A\u4E00\u4E2A\u89E3\u7801\u5668\uFF1A"),K_.forEach(t),Wo=m(e),k(Ms.$$.fragment,e),Io=m(e),Be=p(e,"P",{});var wp=i(Be);Um=r(wp,"\u8BA9\u6211\u4EEC\u6D4B\u8BD5\u4E00\u4E0B\u6211\u4EEC\u4E4B\u524D\u7684 "),qr=p(wp,"STRONG",{});var H_=i(qr);Mm=r(H_,"encoding"),H_.forEach(t),Xm=r(wp," \uFF1A"),wp.forEach(t),Uo=m(e),k(Xs.$$.fragment,e),Mo=m(e),k(Ks.$$.fragment,e),Xo=m(e),Ht=p(e,"P",{});var Y_=i(Ht);Km=r(Y_,"\u5F88\u597D\uFF01\u6211\u4EEC\u53EF\u4EE5\u5C06\u6807\u8BB0\u5668\u4FDD\u5B58\u5728\u4E00\u4E2A JSON \u6587\u4EF6\u4E2D\uFF0C\u5982\u4E0B\u6240\u793A\uFF1A"),Y_.forEach(t),Ko=m(e),k(Hs.$$.fragment,e),Ho=m(e),te=p(e,"P",{});var hn=i(te);Hm=r(hn,"\u7136\u540E\u6211\u4EEC\u53EF\u4EE5\u4F7F\u7528"),wr=p(hn,"STRONG",{});var J_=i(wr);Ym=r(J_,"from_file()"),J_.forEach(t),Jm=r(hn," \u65B9\u6CD5\u4ECE\u8BE5\u6587\u4EF6\u91CC\u91CD\u65B0\u52A0\u8F7D "),Sr=p(hn,"STRONG",{});var V_=i(Sr);Vm=r(V_,"Tokenizer"),V_.forEach(t),Zm=r(hn," \u5BF9\u8C61\uFF1A"),hn.forEach(t),Yo=m(e),k(Ys.$$.fragment,e),Jo=m(e),ne=p(e,"P",{});var gn=i(ne);Qm=r(gn,"\u8981\u5728 \u{1F917} Transformers \u4E2D\u4F7F\u7528\u8FD9\u4E2A\u6807\u8BB0\u5668\uFF0C\u6211\u4EEC\u5FC5\u987B\u5C06\u5B83\u5305\u88F9\u5728\u4E00\u4E2A "),Nr=p(gn,"STRONG",{});var Z_=i(Nr);ec=r(Z_,"PreTrainedTokenizerFast"),Z_.forEach(t),sc=r(gn," \u7C7B\u4E2D\u3002\u6211\u4EEC\u53EF\u4EE5\u4F7F\u7528\u6CDB\u578B\u7C7B\uFF0C\u6216\u8005\uFF0C\u5982\u679C\u6211\u4EEC\u7684\u6807\u8BB0\u5668\u5BF9\u5E94\u4E8E\u73B0\u6709\u6A21\u578B\uFF0C\u5219\u4F7F\u7528\u8BE5\u7C7B\uFF08\u4F8B\u5982\u8FD9\u91CC\u7684 "),Or=p(gn,"STRONG",{});var Q_=i(Or);tc=r(Q_,"BertTokenizerFast"),Q_.forEach(t),nc=r(gn," \uFF09\u3002\u5982\u679C\u60A8\u5E94\u7528\u672C\u8BFE\u6765\u6784\u5EFA\u5168\u65B0\u7684\u6807\u8BB0\u5668\uFF0C\u5219\u5FC5\u987B\u4F7F\u7528\u7B2C\u4E00\u4E2A\u9009\u9879\u3002"),gn.forEach(t),Vo=m(e),O=p(e,"P",{});var X=i(O);rc=r(X,"\u8981\u5C06\u6807\u8BB0\u5668\u5305\u88C5\u5728 "),Lr=p(X,"CODE",{});var e$=i(Lr);lc=r(e$,"PreTrainedTokenizerFast"),e$.forEach(t),oc=r(X," \u7C7B\u4E2D\uFF0C\u6211\u4EEC\u53EF\u4EE5\u5C06\u6211\u4EEC\u6784\u5EFA\u7684\u6807\u8BB0\u5668\u4F5C\u4E3A"),yr=p(X,"CODE",{});var s$=i(yr);ac=r(s$,"tokenizer_object"),s$.forEach(t),pc=r(X," \u4F20\u9012\uFF0C\u6216\u8005\u5C06\u6211\u4EEC\u4FDD\u5B58\u4E3A"),Rr=p(X,"CODE",{});var t$=i(Rr);ic=r(t$,"tokenizer_file"),t$.forEach(t),fc=r(X," \u7684\u6807\u8BB0\u5668\u6587\u4EF6\u4F20\u9012\u3002 \u8981\u8BB0\u4F4F\u7684\u5173\u952E\u662F\u6211\u4EEC\u5FC5\u987B\u624B\u52A8\u8BBE\u7F6E\u6240\u6709\u7279\u6B8A\u6807\u8BB0\uFF0C\u56E0\u4E3A\u8BE5\u7C7B\u65E0\u6CD5\u4ECE "),Dr=p(X,"CODE",{});var n$=i(Dr);mc=r(n$,"tokenizer"),n$.forEach(t),cc=r(X," \u5BF9\u8C61\u63A8\u65AD\u51FA\u54EA\u4E2A\u6807\u8BB0\u662F\u63A9\u7801\u6807\u8BB0\u3001"),Gr=p(X,"CODE",{});var r$=i(Gr);kc=r(r$,"[CLS]"),r$.forEach(t),_c=r(X," \u6807\u8BB0\u7B49\uFF1A"),X.forEach(t),Zo=m(e),k(Js.$$.fragment,e),Qo=m(e),Fe=p(e,"P",{});var Sp=i(Fe);$c=r(Sp,"\u5982\u679C\u60A8\u4F7F\u7528\u7279\u5B9A\u7684\u6807\u8BB0\u5668\u7C7B\uFF08\u4F8B\u5982 "),Cr=p(Sp,"STRONG",{});var l$=i(Cr);uc=r(l$,"BertTokenizerFast"),l$.forEach(t),zc=r(Sp," )\uFF0C\u60A8\u53EA\u9700\u8981\u6307\u5B9A\u4E0E\u9ED8\u8BA4\u6807\u8BB0\u4E0D\u540C\u7684\u7279\u6B8A\u6807\u8BB0\uFF08\u6B64\u5904\u6CA1\u6709\uFF09\uFF1A"),Sp.forEach(t),ea=m(e),k(Vs.$$.fragment,e),sa=m(e),re=p(e,"P",{});var jn=i(re);hc=r(jn,"\u7136\u540E\uFF0C\u60A8\u53EF\u4EE5\u50CF\u4F7F\u7528\u4EFB\u4F55\u5176\u4ED6 \u{1F917} Transformers \u6807\u8BB0\u5668\u4E00\u6837\u4F7F\u7528\u6B64\u6807\u8BB0\u5668\u3002\u4F60\u53EF\u4EE5\u7528 "),Br=p(jn,"STRONG",{});var o$=i(Br);gc=r(o$,"save_pretrained()"),o$.forEach(t),jc=r(jn," \u65B9\u6CD5\uFF0C\u6216\u4F7F\u7528 "),Fr=p(jn,"STRONG",{});var a$=i(Fr);dc=r(a$,"push_to_hub()"),a$.forEach(t),Ec=r(jn," \u65B9\u6CD5\u3002"),jn.forEach(t),ta=m(e),Yt=p(e,"P",{});var p$=i(Yt);xc=r(p$,"\u73B0\u5728\u6211\u4EEC\u5DF2\u7ECF\u4E86\u89E3\u4E86\u5982\u4F55\u6784\u5EFA WordPiece \u6807\u8BB0\u5668\uFF0C\u8BA9\u6211\u4EEC\u5BF9 BPE \u6807\u8BB0\u5668\u8FDB\u884C\u540C\u6837\u7684\u64CD\u4F5C\u3002\u56E0\u4E3A\u60A8\u5DF2\u7ECF\u77E5\u9053\u4E86\u6240\u6709\u6B65\u9AA4\uFF0C\u6240\u4EE5\u6211\u4EEC\u4F1A\u8FDB\u884C\u5730\u66F4\u5FEB\u4E00\u70B9\uFF0C\u5E76\u4E14\u53EA\u7A81\u51FA\u5C55\u793A\u4E24\u8005\u4E0D\u4E00\u6837\u7684\u5730\u65B9\u3002"),p$.forEach(t),na=m(e),ue=p(e,"H2",{class:!0});var Np=i(ue);Ae=p(Np,"A",{id:!0,class:!0,href:!0});var i$=i(Ae);Ar=p(i$,"SPAN",{});var f$=i(Ar);k(Zs.$$.fragment,f$),f$.forEach(t),i$.forEach(t),vc=m(Np),Wr=p(Np,"SPAN",{});var m$=i(Wr);bc=r(m$,"\u4ECE\u5934\u5F00\u59CB\u6784\u5EFA BPE \u6807\u8BB0\u5668"),m$.forEach(t),Np.forEach(t),ra=m(e),We=p(e,"P",{});var Op=i(We);Tc=r(Op,"\u73B0\u5728\u8BA9\u6211\u4EEC\u6784\u5EFA\u4E00\u4E2A GPT-2 \u6807\u8BB0\u5668\u3002\u4E0E BERT \u6807\u8BB0\u5668\u4E00\u6837\uFF0C\u6211\u4EEC\u9996\u5148\u4F7F\u7528 "),Ir=p(Op,"STRONG",{});var c$=i(Ir);Pc=r(c$,"Tokenizer"),c$.forEach(t),qc=r(Op," \u521D\u59CB\u5316\u4E00\u4E2ABPE \u6A21\u578B\uFF1A"),Op.forEach(t),la=m(e),k(Qs.$$.fragment,e),oa=m(e),le=p(e,"P",{});var dn=i(le);wc=r(dn,"\u548C BERT \u4E00\u6837\uFF0C\u5982\u679C\u6211\u4EEC\u6709\u4E00\u4E2A\u8BCD\u6C47\u8868\uFF0C\u6211\u4EEC\u53EF\u4EE5\u7528\u4E00\u4E2A\u8BCD\u6C47\u8868\u6765\u521D\u59CB\u5316\u8FD9\u4E2A\u6A21\u578B\uFF08\u5728\u8FD9\u79CD\u60C5\u51B5\u4E0B\uFF0C\u6211\u4EEC\u9700\u8981\u4F20\u9012 "),Ur=p(dn,"CODE",{});var k$=i(Ur);Sc=r(k$,"vocab"),k$.forEach(t),Nc=r(dn," \u548C "),Mr=p(dn,"CODE",{});var _$=i(Mr);Oc=r(_$,"merges"),_$.forEach(t),Lc=r(dn,"\uFF09\uFF0C\u4F46\u662F\u7531\u4E8E\u6211\u4EEC\u5C06\u4ECE\u5934\u5F00\u59CB\u8BAD\u7EC3\uFF0C\u6240\u4EE5\u6211\u4EEC\u4E0D\u9700\u8981\u8FD9\u6837\u53BB\u505A\u3002 \u6211\u4EEC\u4E5F\u4E0D\u9700\u8981\u6307\u5B9A\u201Cunk_token\u201D\uFF0C\u56E0\u4E3A GPT-2 \u4F7F\u7528\u7684\u5B57\u8282\u7EA7 BPE\uFF0C\u4E0D\u9700\u8981\u201Cunk_token\u201D\u3002"),dn.forEach(t),aa=m(e),Jt=p(e,"P",{});var $$=i(Jt);yc=r($$,"GPT-2 \u4E0D\u4F7F\u7528\u5F52\u4E00\u5316\u5668\uFF0C\u56E0\u6B64\u6211\u4EEC\u8DF3\u8FC7\u8BE5\u6B65\u9AA4\u5E76\u76F4\u63A5\u8FDB\u5165\u9884\u6807\u8BB0\u5316\uFF1A"),$$.forEach(t),pa=m(e),k(et.$$.fragment,e),ia=m(e),Ie=p(e,"P",{});var Lp=i(Ie);Rc=r(Lp,"\u6211\u4EEC\u5728\u6B64\u5904\u6DFB\u52A0\u5230 "),Xr=p(Lp,"CODE",{});var u$=i(Xr);Dc=r(u$,"ByteLevel"),u$.forEach(t),Gc=r(Lp," \u7684\u9009\u9879\u662F\u4E0D\u5728\u53E5\u5B50\u5F00\u5934\u6DFB\u52A0\u7A7A\u683C\uFF08\u9ED8\u8BA4\u4E3Ature\uFF09\u3002 \u6211\u4EEC\u53EF\u4EE5\u770B\u4E00\u4E0B\u4F7F\u7528\u8FD9\u4E2A\u6807\u8BB0\u5668\u5BF9\u4E4B\u524D\u793A\u4F8B\u6587\u672C\u7684\u9884\u6807\u8BB0\uFF1A"),Lp.forEach(t),fa=m(e),k(st.$$.fragment,e),ma=m(e),k(tt.$$.fragment,e),ca=m(e),Vt=p(e,"P",{});var z$=i(Vt);Cc=r(z$,"\u63A5\u4E0B\u6765\u662F\u9700\u8981\u8BAD\u7EC3\u7684\u6A21\u578B\u3002\u5BF9\u4E8E GPT-2\uFF0C\u552F\u4E00\u7684\u7279\u6B8A\u6807\u8BB0\u662F\u6587\u672C\u7ED3\u675F\u6807\u8BB0\uFF1A"),z$.forEach(t),ka=m(e),k(nt.$$.fragment,e),_a=m(e),P=p(e,"P",{});var R=i(P);Bc=r(R,"\u4E0E "),Kr=p(R,"CODE",{});var h$=i(Kr);Fc=r(h$,"WordPieceTrainer"),h$.forEach(t),Ac=r(R," \u4EE5\u53CA "),Hr=p(R,"CODE",{});var g$=i(Hr);Wc=r(g$,"vocab_size"),g$.forEach(t),Ic=r(R," \u548C "),Yr=p(R,"CODE",{});var j$=i(Yr);Uc=r(j$,"special_tokens"),j$.forEach(t),Mc=r(R," \u4E00\u6837\uFF0C\u6211\u4EEC\u53EF\u4EE5\u6307\u5B9A "),Jr=p(R,"CODE",{});var d$=i(Jr);Xc=r(d$,"min_frequency"),d$.forEach(t),Kc=r(R," \u5982\u679C\u6211\u4EEC\u613F\u610F\uFF0C\u6216\u8005\u5982\u679C\u6211\u4EEC\u6709\u4E00\u4E2A\u8BCD\u5C3E\u540E\u7F00\uFF08\u5982 "),Vr=p(R,"CODE",{});var E$=i(Vr);Hc=r(E$,"</w>"),E$.forEach(t),Yc=r(R," )\uFF0C\u6211\u4EEC\u53EF\u4EE5\u4F7F\u7528 "),Zr=p(R,"CODE",{});var x$=i(Zr);Jc=r(x$,"end_of_word_suffix"),x$.forEach(t),Vc=r(R," \u8BBE\u7F6E\u5B83\u3002"),R.forEach(t),$a=m(e),Zt=p(e,"P",{});var v$=i(Zt);Zc=r(v$,"\u8FD9\u4E2A\u6807\u8BB0\u5668\u4E5F\u53EF\u4EE5\u5728\u6587\u672C\u6587\u4EF6\u4E0A\u8BAD\u7EC3\uFF1A"),v$.forEach(t),ua=m(e),k(rt.$$.fragment,e),za=m(e),Qt=p(e,"P",{});var b$=i(Qt);Qc=r(b$,"\u8BA9\u6211\u4EEC\u770B\u4E00\u4E0B\u793A\u4F8B\u6587\u672C\u7684\u6807\u8BB0\u5316\u540E\u7684\u7ED3\u679C\uFF1A"),b$.forEach(t),ha=m(e),k(lt.$$.fragment,e),ga=m(e),k(ot.$$.fragment,e),ja=m(e),en=p(e,"P",{});var T$=i(en);e2=r(T$,"\u6211\u4EEC\u5BF9 GPT-2 \u6807\u8BB0\u5668\u6DFB\u52A0\u5B57\u8282\u7EA7\u540E\u5904\u7406\uFF0C\u5982\u4E0B\u6240\u793A\uFF1A"),T$.forEach(t),da=m(e),k(at.$$.fragment,e),Ea=m(e),ze=p(e,"P",{});var wl=i(ze);Qr=p(wl,"CODE",{});var P$=i(Qr);s2=r(P$,"trim_offsets = False"),P$.forEach(t),t2=r(wl," \u9009\u9879\u6307\u793A\u6211\u4EEC\u5E94\u8BE5\u4FDD\u7559\u4EE5 \u2018\u0120\u2019 \u5F00\u5934\u7684\u6807\u8BB0\u7684\u504F\u79FB\u91CF\uFF1A\u8FD9\u6837\u504F\u79FB\u91CF\u7684\u5F00\u5934\u5C06\u6307\u5411\u5355\u8BCD\u4E4B\u524D\u7684\u7A7A\u683C\uFF0C\u800C\u4E0D\u662F\u7B2C\u4E00\u4E2A\u5355\u8BCD\u7684\u5B57\u7B26\uFF08\u56E0\u4E3A\u7A7A\u683C\u5728\u6280\u672F\u4E0A\u662F\u6807\u8BB0\u7684\u4E00\u90E8\u5206\uFF09\u3002 \u8BA9\u6211\u4EEC\u770B\u770B\u6211\u4EEC\u521A\u521A\u7F16\u7801\u7684\u6587\u672C\u7684\u7ED3\u679C\uFF0C\u5176\u4E2D "),el=p(wl,"CODE",{});var q$=i(el);n2=r(q$,"'\u0120test'"),q$.forEach(t),r2=r(wl," \u662F\u7D22\u5F15\u7B2C 4 \u5904\u7684\u6807\u8BB0\uFF1A"),wl.forEach(t),xa=m(e),k(pt.$$.fragment,e),va=m(e),k(it.$$.fragment,e),ba=m(e),sn=p(e,"P",{});var w$=i(sn);l2=r(w$,"\u6700\u540E\uFF0C\u6211\u4EEC\u6DFB\u52A0\u4E00\u4E2A\u5B57\u8282\u7EA7\u89E3\u7801\u5668\uFF1A"),w$.forEach(t),Ta=m(e),k(ft.$$.fragment,e),Pa=m(e),tn=p(e,"P",{});var S$=i(tn);o2=r(S$,"\u6211\u4EEC\u53EF\u4EE5\u4ED4\u7EC6\u68C0\u67E5\u5B83\u662F\u5426\u6B63\u5E38\u5DE5\u4F5C\uFF1A"),S$.forEach(t),qa=m(e),k(mt.$$.fragment,e),wa=m(e),k(ct.$$.fragment,e),Sa=m(e),oe=p(e,"P",{});var En=i(oe);a2=r(En,"\u5F88\u597D\uFF01\u73B0\u5728\u6211\u4EEC\u5B8C\u6210\u4E86\uFF0C\u6211\u4EEC\u53EF\u4EE5\u50CF\u4EE5\u524D\u4E00\u6837\u4FDD\u5B58\u6807\u8BB0\u5668\uFF0C\u5E76\u5C06\u5B83\u5305\u88C5\u5728\u4E00\u4E2A "),sl=p(En,"STRONG",{});var N$=i(sl);p2=r(N$,"PreTrainedTokenizerFast"),N$.forEach(t),i2=r(En," \u6216\u8005 "),tl=p(En,"STRONG",{});var O$=i(tl);f2=r(O$,"GPT2TokenizerFast"),O$.forEach(t),m2=r(En," \u5982\u679C\u6211\u4EEC\u60F3\u5728 \u{1F917} Transformers\u4E2D\u4F7F\u7528\u5B83\uFF1A"),En.forEach(t),Na=m(e),k(kt.$$.fragment,e),Oa=m(e),nn=p(e,"P",{});var L$=i(nn);c2=r(L$,"\u6216\u8005\uFF1A"),L$.forEach(t),La=m(e),k(_t.$$.fragment,e),ya=m(e),rn=p(e,"P",{});var y$=i(rn);k2=r(y$,"\u4F5C\u4E3A\u6700\u540E\u4E00\u4E2A\u793A\u4F8B\uFF0C\u6211\u4EEC\u5C06\u5411\u60A8\u5C55\u793A\u5982\u4F55\u4ECE\u5934\u5F00\u59CB\u6784\u5EFA Unigram \u6807\u8BB0\u5668\u3002"),y$.forEach(t),Ra=m(e),he=p(e,"H2",{class:!0});var yp=i(he);Ue=p(yp,"A",{id:!0,class:!0,href:!0});var R$=i(Ue);nl=p(R$,"SPAN",{});var D$=i(nl);k($t.$$.fragment,D$),D$.forEach(t),R$.forEach(t),_2=m(yp),rl=p(yp,"SPAN",{});var G$=i(rl);$2=r(G$,"\u4ECE\u5934\u5F00\u59CB\u6784\u5EFA Unigram \u6807\u8BB0\u5668"),G$.forEach(t),yp.forEach(t),Da=m(e),Me=p(e,"P",{});var Rp=i(Me);u2=r(Rp,"\u73B0\u5728\u8BA9\u6211\u4EEC\u6784\u5EFA\u4E00\u4E2A XLNet \u6807\u8BB0\u5668\u3002\u4E0E\u4E4B\u524D\u7684\u6807\u8BB0\u5668\u4E00\u6837\uFF0C\u6211\u4EEC\u9996\u5148\u4F7F\u7528 Unigram \u6A21\u578B\u521D\u59CB\u5316\u4E00\u4E2A "),ll=p(Rp,"STRONG",{});var C$=i(ll);z2=r(C$,"Tokenizer"),C$.forEach(t),h2=r(Rp," \uFF1A"),Rp.forEach(t),Ga=m(e),k(ut.$$.fragment,e),Ca=m(e),ln=p(e,"P",{});var B$=i(ln);g2=r(B$,"\u540C\u6837\uFF0C\u5982\u679C\u6211\u4EEC\u6709\u8BCD\u6C47\u8868\uFF0C\u6211\u4EEC\u53EF\u4EE5\u7528\u8BCD\u6C47\u8868\u521D\u59CB\u5316\u8FD9\u4E2A\u6A21\u578B\u3002"),B$.forEach(t),Ba=m(e),on=p(e,"P",{});var F$=i(on);j2=r(F$,"\u5BF9\u4E8E\u6807\u51C6\u5316\uFF0CXLNet \u4F7F\u7528\u4E86\u4E00\u4E9B\u66FF\u6362\u7684\u65B9\u6CD5\uFF08\u6765\u81EA SentencePiece\uFF09\uFF1A"),F$.forEach(t),Fa=m(e),k(zt.$$.fragment,e),Aa=m(e),A=p(e,"P",{});var Ve=i(A);d2=r(Ve,"\u8FD9\u4F1A\u53D6\u4EE3 "),ol=p(Ve,"STRONG",{});var A$=i(ol);E2=r(A$,"\u201C"),A$.forEach(t),x2=r(Ve," \u548C "),al=p(Ve,"STRONG",{});var W$=i(al);v2=r(W$,"\u201D"),W$.forEach(t),b2=r(Ve," \u548C "),pl=p(Ve,"STRONG",{});var I$=i(pl);T2=r(I$,"\u201D"),I$.forEach(t),P2=r(Ve," \u4EE5\u53CA\u4EFB\u4F55\u4E24\u4E2A\u6216\u591A\u4E2A\u7A7A\u683C\u4E0E\u5355\u4E2A\u7A7A\u683C\u7684\u5E8F\u5217\uFF0C\u4EE5\u53CA\u5220\u9664\u6587\u672C\u4E2D\u7684\u91CD\u97F3\u4EE5\u8FDB\u884C\u6807\u8BB0\u3002"),Ve.forEach(t),Wa=m(e),Xe=p(e,"P",{});var Dp=i(Xe);q2=r(Dp,"\u7528\u4E8E\u4EFB\u4F55 SentencePiece \u6807\u8BB0\u5668\u7684\u9884\u6807\u8BB0\u5668\u662F "),il=p(Dp,"CODE",{});var U$=i(il);w2=r(U$,"Metaspace"),U$.forEach(t),S2=r(Dp,"\uFF1A"),Dp.forEach(t),Ia=m(e),k(ht.$$.fragment,e),Ua=m(e),an=p(e,"P",{});var M$=i(an);N2=r(M$,"\u6211\u4EEC\u53EF\u4EE5\u50CF\u4EE5\u524D\u4E00\u6837\u67E5\u770B\u793A\u4F8B\u6587\u672C\u7684\u9884\u6807\u8BB0\u5316\uFF1A"),M$.forEach(t),Ma=m(e),k(gt.$$.fragment,e),Xa=m(e),k(jt.$$.fragment,e),Ka=m(e),pn=p(e,"P",{});var X$=i(pn);O2=r(X$,"\u63A5\u4E0B\u6765\u662F\u9700\u8981\u8BAD\u7EC3\u7684\u6A21\u578B\u3002 XLNet \u6709\u4E0D\u5C11\u7279\u6B8A\u7684\u6807\u8BB0\uFF1A"),X$.forEach(t),Ha=m(e),k(dt.$$.fragment,e),Ya=m(e),ae=p(e,"P",{});var xn=i(ae);L2=r(xn,"\u4E0D\u8981\u5FD8\u8BB0"),fl=p(xn,"CODE",{});var K$=i(fl);y2=r(K$,"UnigramTrainer"),K$.forEach(t),R2=r(xn," \u7684\u4E00\u4E2A\u975E\u5E38\u91CD\u8981\u7684\u53C2\u6570\u662F"),ml=p(xn,"CODE",{});var H$=i(ml);D2=r(H$,"unk_token"),H$.forEach(t),G2=r(xn,"\u3002 \u6211\u4EEC\u8FD8\u53EF\u4EE5\u4F20\u9012\u7279\u5B9A\u4E8E Unigram \u7B97\u6CD5\u7684\u5176\u4ED6\u53C2\u6570\uFF0C\u4F8B\u5982\u5220\u9664\u6807\u8BB0\u7684\u6BCF\u4E2A\u6B65\u9AA4\u7684\u201Cshrinking_factor\uFF08\u6536\u7F29\u56E0\u5B50\uFF09\u201D\uFF08\u9ED8\u8BA4\u4E3A 0.75\uFF09\u6216\u6307\u5B9A\u7ED9\u5B9A\u6807\u8BB0\u7684\u6700\u5927\u957F\u5EA6\u7684\u201Cmax_piece_length\u201D\uFF08\u9ED8\u8BA4\u4E3A 16\uFF09 ."),xn.forEach(t),Ja=m(e),fn=p(e,"P",{});var Y$=i(fn);C2=r(Y$,"\u8FD9\u4E2A\u6807\u8BB0\u5668\u4E5F\u53EF\u4EE5\u5728\u6587\u672C\u6587\u4EF6\u4E0A\u8BAD\u7EC3\uFF1A"),Y$.forEach(t),Va=m(e),k(Et.$$.fragment,e),Za=m(e),mn=p(e,"P",{});var J$=i(mn);B2=r(J$,"\u8BA9\u6211\u4EEC\u770B\u4E00\u4E0B\u793A\u4F8B\u6587\u672C\u7684\u6807\u8BB0\u5316\u540E\u7684\u7ED3\u679C\uFF1A"),J$.forEach(t),Qa=m(e),k(xt.$$.fragment,e),ep=m(e),k(vt.$$.fragment,e),sp=m(e),q=p(e,"P",{});var D=i(q);F2=r(D,"A peculiarity of XLNet is that it puts the "),cl=p(D,"CODE",{});var V$=i(cl);A2=r(V$,"<cls>"),V$.forEach(t),W2=r(D," token at the end of the sentence, with a type ID of 2 (to distinguish it from the other tokens). It\u2019s padding on the left, as a result. We can deal with all the special tokens and token type IDs with a template, like for BERT, but first we have to get the IDs of the "),kl=p(D,"CODE",{});var Z$=i(kl);I2=r(Z$,"<cls>"),Z$.forEach(t),U2=r(D," and "),_l=p(D,"CODE",{});var Q$=i(_l);M2=r(Q$,"<sep>"),Q$.forEach(t),X2=r(D,` tokens:
XLNet \u7684\u4E00\u4E2A\u7279\u70B9\u662F\u5B83\u5C06`),$l=p(D,"CODE",{});var eu=i($l);K2=r(eu,"<cls>"),eu.forEach(t),H2=r(D," \u6807\u8BB0\u653E\u5728\u53E5\u5B50\u7684\u672B\u5C3E\uFF0C\u7C7B\u578BID \u4E3A2\uFF08\u4EE5\u5C06\u5176\u4E0E\u5176\u4ED6\u6807\u8BB0\u533A\u5206\u5F00\u6765\uFF09\u3002\u5B83\u4F1A\u5C06\u7ED3\u679C\u586B\u5145\u5728\u5DE6\u4FA7\u3002 \u6211\u4EEC\u53EF\u4EE5\u4F7F\u7528\u6A21\u677F\u5904\u7406\u6240\u6709\u7279\u6B8A\u6807\u8BB0\u548C\u6807\u8BB0\u7C7B\u578B ID\uFF0C\u4F8B\u5982 BERT\uFF0C\u4F46\u9996\u5148\u6211\u4EEC\u5FC5\u987B\u83B7\u53D6 "),ul=p(D,"CODE",{});var su=i(ul);Y2=r(su,"<cls>"),su.forEach(t),J2=r(D," \u548C "),zl=p(D,"CODE",{});var tu=i(zl);V2=r(tu,"<sep>"),tu.forEach(t),Z2=r(D," \u6807\u8BB0\u7684 ID\uFF1A"),D.forEach(t),tp=m(e),k(bt.$$.fragment,e),np=m(e),k(Tt.$$.fragment,e),rp=m(e),cn=p(e,"P",{});var nu=i(cn);Q2=r(nu,"\u6A21\u677F\u5982\u4E0B\u6240\u793A\uFF1A"),nu.forEach(t),lp=m(e),k(Pt.$$.fragment,e),op=m(e),kn=p(e,"P",{});var ru=i(kn);ek=r(ru,"\u6211\u4EEC\u53EF\u4EE5\u901A\u8FC7\u7F16\u7801\u4E00\u5BF9\u53E5\u5B50\u6765\u6D4B\u8BD5\u5B83\u7684\u5DE5\u4F5C\u539F\u7406\uFF1A"),ru.forEach(t),ap=m(e),k(qt.$$.fragment,e),pp=m(e),k(wt.$$.fragment,e),ip=m(e),Ke=p(e,"P",{});var Gp=i(Ke);sk=r(Gp,"\u6700\u540E\uFF0C\u6211\u4EEC\u6DFB\u52A0\u4E00\u4E2A "),hl=p(Gp,"STRONG",{});var lu=i(hl);tk=r(lu,"Metaspace"),lu.forEach(t),nk=r(Gp," \u89E3\u7801\u5668\uFF1A"),Gp.forEach(t),fp=m(e),k(St.$$.fragment,e),mp=m(e),W=p(e,"P",{});var Ze=i(W);rk=r(Ze,"\u6211\u4EEC\u5B8C\u6210\u4E86\u8FD9\u4E2A\u6807\u8BB0\u5668\uFF01 \u6211\u4EEC\u53EF\u4EE5\u50CF\u4EE5\u524D\u4E00\u6837\u4FDD\u5B58\u6807\u8BB0\u5668\uFF0C\u5982\u679C\u6211\u4EEC\u60F3\u5728 \u{1F917} Transformers \u4E2D\u4F7F\u7528\u5B83\uFF0C\u53EF\u4EE5\u5C06\u5B83\u5305\u88C5\u5728 "),gl=p(Ze,"CODE",{});var ou=i(gl);lk=r(ou,"PreTrainedTokenizerFast"),ou.forEach(t),ok=r(Ze," \u6216 "),jl=p(Ze,"CODE",{});var au=i(jl);ak=r(au,"XLNetTokenizerFast"),au.forEach(t),pk=r(Ze," \u4E2D\u3002 \u4F7F\u7528 "),dl=p(Ze,"CODE",{});var pu=i(dl);ik=r(pu,"PreTrainedTokenizerFast"),pu.forEach(t),fk=r(Ze," \u65F6\u8981\u6CE8\u610F\u7684\u4E00\u4EF6\u4E8B\u662F\uFF0C\u6211\u4EEC\u9700\u8981\u544A\u8BC9\u{1F917} Transformers \u5E93\u5E94\u8BE5\u5728\u5DE6\u4FA7\u586B\u5145\u7279\u6B8A\u6807\u8BB0\uFF1A"),Ze.forEach(t),cp=m(e),k(Nt.$$.fragment,e),kp=m(e),_n=p(e,"P",{});var iu=i(_n);mk=r(iu,"\u6216\u8005\uFF1A"),iu.forEach(t),_p=m(e),k(Ot.$$.fragment,e),$p=m(e),$n=p(e,"P",{});var fu=i($n);ck=r(fu,"\u73B0\u5728\u60A8\u5DF2\u7ECF\u4E86\u89E3\u4E86\u5982\u4F55\u4F7F\u7528\u5404\u79CD\u6784\u5EFA\u5757\u6765\u6784\u5EFA\u73B0\u6709\u7684\u6807\u8BB0\u5668\uFF0C\u60A8\u5E94\u8BE5\u80FD\u591F\u4F7F\u7528 \u{1F917} tokenizer\u5E93\u7F16\u5199\u60A8\u60F3\u8981\u7684\u4EFB\u4F55\u6807\u8BB0\u5668\uFF0C\u5E76\u80FD\u591F\u5728 \u{1F917} Transformers\u4E2D\u4F7F\u7528\u5B83\u3002"),fu.forEach(t),this.h()},h(){g(j,"name","hf:doc:metadata"),g(j,"content",JSON.stringify(Eu)),g(K,"id",""),g(Z,"class","block dark:hidden"),mu(Z.src,vn="https://huggingface.co/datasets/huggingface-course/documentation-images/resolve/main/en/chapter6/tokenization_pipeline.svg")||g(Z,"src",vn),g(Z,"alt","The tokenization pipeline."),g(Q,"class","hidden dark:block"),mu(Q.src,$k="https://huggingface.co/datasets/huggingface-course/documentation-images/resolve/main/en/chapter6/tokenization_pipeline-dark.svg")||g(Q,"src",$k),g(Q,"alt","The tokenization pipeline."),g(C,"class","flex justify-center"),g(Rt,"href","/course/chapter6/2"),g(as,"href","https://huggingface.co/docs/tokenizers/python/latest/api/reference.html#module-tokenizers.normalizers"),g(as,"rel","nofollow"),g(ps,"href","https://huggingface.co/docs/tokenizers/python/latest/api/reference.html#module-tokenizers.pre_tokenizers"),g(ps,"rel","nofollow"),g(is,"href","https://huggingface.co/docs/tokenizers/python/latest/api/reference.html#module-tokenizers.models"),g(is,"rel","nofollow"),g(fs,"href","https://huggingface.co/docs/tokenizers/python/latest/api/reference.html#module-tokenizers.trainers"),g(fs,"rel","nofollow"),g(ms,"href","https://huggingface.co/docs/tokenizers/python/latest/api/reference.html#module-tokenizers.processors"),g(ms,"rel","nofollow"),g(cs,"href","https://huggingface.co/docs/tokenizers/python/latest/components.html#decoders"),g(cs,"rel","nofollow"),g(ks,"href","https://huggingface.co/docs/tokenizers/python/latest/components.html"),g(ks,"rel","nofollow"),g(_s,"id",""),g($s,"href","https://huggingface.co/datasets/wikitext"),g($s,"rel","nofollow"),g(Se,"id","wordpiece"),g(Se,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),g(Se,"href","#wordpiece"),g($e,"class","relative group"),g(Ae,"id","bpe"),g(Ae,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),g(Ae,"href","#bpe"),g(ue,"class","relative group"),g(Ue,"id","unigram"),g(Ue,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),g(Ue,"href","#unigram"),g(he,"class","relative group")},m(e,l){s(document.head,j),o(e,Y,l),o(e,K,l),s(K,Qe),o(e,J,l),_(H,e,l),o(e,ge,l),o(e,G,l),s(G,es),o(e,je,l),o(e,v,l),s(v,pe),s(pe,ss),s(v,ie),s(v,fe),s(fe,ts),s(v,me),s(v,ce),s(ce,ns),s(v,ke),s(v,_e),s(_e,rs),o(e,V,l),o(e,d,l),s(d,Lt),o(e,ls,l),o(e,C,l),s(C,Z),s(C,yt),s(C,Q),o(e,Nl,l),o(e,de,l),s(de,Cp),s(de,Rt),s(Rt,Bp),s(de,Fp),o(e,Ol,l),_(os,e,l),o(e,Ll,l),o(e,Dt,l),s(Dt,Ap),o(e,yl,l),o(e,w,l),s(w,Ee),s(Ee,bn),s(bn,Wp),s(Ee,Ip),s(Ee,as),s(as,Up),s(Ee,Mp),s(w,Xp),s(w,xe),s(xe,Tn),s(Tn,Kp),s(xe,Hp),s(xe,ps),s(ps,Yp),s(xe,Jp),s(w,Vp),s(w,ve),s(ve,Pn),s(Pn,Zp),s(ve,Qp),s(ve,is),s(is,ei),s(ve,si),s(w,ti),s(w,be),s(be,qn),s(qn,ni),s(be,ri),s(be,fs),s(fs,li),s(be,oi),s(w,ai),s(w,Te),s(Te,wn),s(wn,pi),s(Te,ii),s(Te,ms),s(ms,fi),s(Te,mi),s(w,ci),s(w,Pe),s(Pe,Sn),s(Sn,ki),s(Pe,_i),s(Pe,cs),s(cs,$i),s(Pe,ui),o(e,Rl,l),o(e,qe,l),s(qe,zi),s(qe,ks),s(ks,hi),s(qe,gi),o(e,Dl,l),o(e,_s,l),s(_s,ji),o(e,Gl,l),o(e,we,l),s(we,di),s(we,$s),s($s,Ei),s(we,xi),o(e,Cl,l),_(us,e,l),o(e,Bl,l),o(e,zs,l),s(zs,Nn),s(Nn,vi),s(zs,bi),o(e,Fl,l),o(e,Gt,l),s(Gt,Ti),o(e,Al,l),_(hs,e,l),o(e,Wl,l),o(e,Ct,l),s(Ct,Pi),o(e,Il,l),o(e,$e,l),s($e,Se),s(Se,On),_(gs,On,null),s($e,qi),s($e,Ln),s(Ln,wi),o(e,Ul,l),o(e,b,l),s(b,Si),s(b,yn),s(yn,Ni),s(b,Oi),s(b,Rn),s(Rn,Li),s(b,yi),s(b,Dn),s(Dn,Ri),s(b,Di),s(b,Gn),s(Gn,Gi),s(b,Ci),s(b,Cn),s(Cn,Bi),s(b,Fi),s(b,Bn),s(Bn,Ai),s(b,Wi),o(e,Ml,l),o(e,Ne,l),s(Ne,Ii),s(Ne,Fn),s(Fn,Ui),s(Ne,Mi),o(e,Xl,l),_(js,e,l),o(e,Kl,l),o(e,B,l),s(B,Xi),s(B,An),s(An,Ki),s(B,Hi),s(B,Wn),s(Wn,Yi),s(B,Ji),s(B,In),s(In,Vi),s(B,Zi),o(e,Hl,l),o(e,T,l),s(T,Qi),s(T,Un),s(Un,ef),s(T,sf),s(T,Mn),s(Mn,tf),s(T,nf),s(T,Xn),s(Xn,rf),s(T,lf),s(T,Kn),s(Kn,of),s(T,af),s(T,Hn),s(Hn,pf),s(T,ff),s(T,Yn),s(Yn,mf),s(T,cf),o(e,Yl,l),_(ds,e,l),o(e,Jl,l),o(e,Bt,l),s(Bt,kf),o(e,Vl,l),_(Es,e,l),o(e,Zl,l),o(e,ee,l),s(ee,_f),s(ee,Jn),s(Jn,$f),s(ee,uf),s(ee,Vn),s(Vn,zf),s(ee,hf),o(e,Ql,l),o(e,se,l),s(se,gf),s(se,Zn),s(Zn,jf),s(se,df),s(se,Qn),s(Qn,Ef),s(se,xf),o(e,eo,l),_(xs,e,l),o(e,so,l),_(vs,e,l),o(e,to,l),_(Oe,e,l),o(e,no,l),o(e,Ft,l),s(Ft,vf),o(e,ro,l),_(bs,e,l),o(e,lo,l),o(e,At,l),s(At,bf),o(e,oo,l),_(Ts,e,l),o(e,ao,l),o(e,Le,l),s(Le,Tf),s(Le,er),s(er,Pf),s(Le,qf),o(e,po,l),_(Ps,e,l),o(e,io,l),_(qs,e,l),o(e,fo,l),o(e,ye,l),s(ye,wf),s(ye,sr),s(sr,Sf),s(ye,Nf),o(e,mo,l),_(ws,e,l),o(e,co,l),_(Ss,e,l),o(e,ko,l),o(e,Re,l),s(Re,Of),s(Re,tr),s(tr,Lf),s(Re,yf),o(e,_o,l),_(Ns,e,l),o(e,$o,l),_(Os,e,l),o(e,uo,l),o(e,De,l),s(De,Rf),s(De,nr),s(nr,Df),s(De,Gf),o(e,zo,l),_(Ls,e,l),o(e,ho,l),o(e,S,l),s(S,Cf),s(S,rr),s(rr,Bf),s(S,Ff),s(S,lr),s(lr,Af),s(S,Wf),s(S,or),s(or,If),s(S,Uf),s(S,ar),s(ar,Mf),s(S,Xf),s(S,pr),s(pr,Kf),s(S,Hf),o(e,go,l),o(e,Wt,l),s(Wt,Yf),o(e,jo,l),_(ys,e,l),o(e,Eo,l),o(e,Ge,l),s(Ge,Jf),s(Ge,ir),s(ir,Vf),s(Ge,Zf),o(e,xo,l),_(Rs,e,l),o(e,vo,l),o(e,Ce,l),s(Ce,Qf),s(Ce,fr),s(fr,em),s(Ce,sm),o(e,bo,l),_(Ds,e,l),o(e,To,l),_(Gs,e,l),o(e,Po,l),o(e,E,l),s(E,tm),s(E,mr),s(mr,nm),s(E,rm),s(E,cr),s(cr,lm),s(E,om),s(E,kr),s(kr,am),s(E,pm),s(E,_r),s(_r,im),s(E,fm),s(E,$r),s($r,mm),s(E,cm),s(E,ur),s(ur,km),s(E,_m),s(E,zr),s(zr,$m),s(E,um),s(E,hr),s(hr,zm),s(E,hm),s(E,gr),s(gr,gm),s(E,jm),o(e,qo,l),o(e,N,l),s(N,dm),s(N,jr),s(jr,Em),s(N,xm),s(N,dr),s(dr,vm),s(N,bm),s(N,Er),s(Er,Tm),s(N,Pm),s(N,xr),s(xr,qm),s(N,wm),s(N,vr),s(vr,Sm),s(N,Nm),o(e,wo,l),_(Cs,e,l),o(e,So,l),_(Bs,e,l),o(e,No,l),o(e,F,l),s(F,Om),s(F,br),s(br,Lm),s(F,ym),s(F,Tr),s(Tr,Rm),s(F,Dm),s(F,Pr),s(Pr,Gm),s(F,Cm),o(e,Oo,l),o(e,It,l),s(It,Bm),o(e,Lo,l),_(Fs,e,l),o(e,yo,l),o(e,Ut,l),s(Ut,Fm),o(e,Ro,l),o(e,Mt,l),s(Mt,Am),o(e,Do,l),_(As,e,l),o(e,Go,l),_(Ws,e,l),o(e,Co,l),o(e,Xt,l),s(Xt,Wm),o(e,Bo,l),_(Is,e,l),o(e,Fo,l),_(Us,e,l),o(e,Ao,l),o(e,Kt,l),s(Kt,Im),o(e,Wo,l),_(Ms,e,l),o(e,Io,l),o(e,Be,l),s(Be,Um),s(Be,qr),s(qr,Mm),s(Be,Xm),o(e,Uo,l),_(Xs,e,l),o(e,Mo,l),_(Ks,e,l),o(e,Xo,l),o(e,Ht,l),s(Ht,Km),o(e,Ko,l),_(Hs,e,l),o(e,Ho,l),o(e,te,l),s(te,Hm),s(te,wr),s(wr,Ym),s(te,Jm),s(te,Sr),s(Sr,Vm),s(te,Zm),o(e,Yo,l),_(Ys,e,l),o(e,Jo,l),o(e,ne,l),s(ne,Qm),s(ne,Nr),s(Nr,ec),s(ne,sc),s(ne,Or),s(Or,tc),s(ne,nc),o(e,Vo,l),o(e,O,l),s(O,rc),s(O,Lr),s(Lr,lc),s(O,oc),s(O,yr),s(yr,ac),s(O,pc),s(O,Rr),s(Rr,ic),s(O,fc),s(O,Dr),s(Dr,mc),s(O,cc),s(O,Gr),s(Gr,kc),s(O,_c),o(e,Zo,l),_(Js,e,l),o(e,Qo,l),o(e,Fe,l),s(Fe,$c),s(Fe,Cr),s(Cr,uc),s(Fe,zc),o(e,ea,l),_(Vs,e,l),o(e,sa,l),o(e,re,l),s(re,hc),s(re,Br),s(Br,gc),s(re,jc),s(re,Fr),s(Fr,dc),s(re,Ec),o(e,ta,l),o(e,Yt,l),s(Yt,xc),o(e,na,l),o(e,ue,l),s(ue,Ae),s(Ae,Ar),_(Zs,Ar,null),s(ue,vc),s(ue,Wr),s(Wr,bc),o(e,ra,l),o(e,We,l),s(We,Tc),s(We,Ir),s(Ir,Pc),s(We,qc),o(e,la,l),_(Qs,e,l),o(e,oa,l),o(e,le,l),s(le,wc),s(le,Ur),s(Ur,Sc),s(le,Nc),s(le,Mr),s(Mr,Oc),s(le,Lc),o(e,aa,l),o(e,Jt,l),s(Jt,yc),o(e,pa,l),_(et,e,l),o(e,ia,l),o(e,Ie,l),s(Ie,Rc),s(Ie,Xr),s(Xr,Dc),s(Ie,Gc),o(e,fa,l),_(st,e,l),o(e,ma,l),_(tt,e,l),o(e,ca,l),o(e,Vt,l),s(Vt,Cc),o(e,ka,l),_(nt,e,l),o(e,_a,l),o(e,P,l),s(P,Bc),s(P,Kr),s(Kr,Fc),s(P,Ac),s(P,Hr),s(Hr,Wc),s(P,Ic),s(P,Yr),s(Yr,Uc),s(P,Mc),s(P,Jr),s(Jr,Xc),s(P,Kc),s(P,Vr),s(Vr,Hc),s(P,Yc),s(P,Zr),s(Zr,Jc),s(P,Vc),o(e,$a,l),o(e,Zt,l),s(Zt,Zc),o(e,ua,l),_(rt,e,l),o(e,za,l),o(e,Qt,l),s(Qt,Qc),o(e,ha,l),_(lt,e,l),o(e,ga,l),_(ot,e,l),o(e,ja,l),o(e,en,l),s(en,e2),o(e,da,l),_(at,e,l),o(e,Ea,l),o(e,ze,l),s(ze,Qr),s(Qr,s2),s(ze,t2),s(ze,el),s(el,n2),s(ze,r2),o(e,xa,l),_(pt,e,l),o(e,va,l),_(it,e,l),o(e,ba,l),o(e,sn,l),s(sn,l2),o(e,Ta,l),_(ft,e,l),o(e,Pa,l),o(e,tn,l),s(tn,o2),o(e,qa,l),_(mt,e,l),o(e,wa,l),_(ct,e,l),o(e,Sa,l),o(e,oe,l),s(oe,a2),s(oe,sl),s(sl,p2),s(oe,i2),s(oe,tl),s(tl,f2),s(oe,m2),o(e,Na,l),_(kt,e,l),o(e,Oa,l),o(e,nn,l),s(nn,c2),o(e,La,l),_(_t,e,l),o(e,ya,l),o(e,rn,l),s(rn,k2),o(e,Ra,l),o(e,he,l),s(he,Ue),s(Ue,nl),_($t,nl,null),s(he,_2),s(he,rl),s(rl,$2),o(e,Da,l),o(e,Me,l),s(Me,u2),s(Me,ll),s(ll,z2),s(Me,h2),o(e,Ga,l),_(ut,e,l),o(e,Ca,l),o(e,ln,l),s(ln,g2),o(e,Ba,l),o(e,on,l),s(on,j2),o(e,Fa,l),_(zt,e,l),o(e,Aa,l),o(e,A,l),s(A,d2),s(A,ol),s(ol,E2),s(A,x2),s(A,al),s(al,v2),s(A,b2),s(A,pl),s(pl,T2),s(A,P2),o(e,Wa,l),o(e,Xe,l),s(Xe,q2),s(Xe,il),s(il,w2),s(Xe,S2),o(e,Ia,l),_(ht,e,l),o(e,Ua,l),o(e,an,l),s(an,N2),o(e,Ma,l),_(gt,e,l),o(e,Xa,l),_(jt,e,l),o(e,Ka,l),o(e,pn,l),s(pn,O2),o(e,Ha,l),_(dt,e,l),o(e,Ya,l),o(e,ae,l),s(ae,L2),s(ae,fl),s(fl,y2),s(ae,R2),s(ae,ml),s(ml,D2),s(ae,G2),o(e,Ja,l),o(e,fn,l),s(fn,C2),o(e,Va,l),_(Et,e,l),o(e,Za,l),o(e,mn,l),s(mn,B2),o(e,Qa,l),_(xt,e,l),o(e,ep,l),_(vt,e,l),o(e,sp,l),o(e,q,l),s(q,F2),s(q,cl),s(cl,A2),s(q,W2),s(q,kl),s(kl,I2),s(q,U2),s(q,_l),s(_l,M2),s(q,X2),s(q,$l),s($l,K2),s(q,H2),s(q,ul),s(ul,Y2),s(q,J2),s(q,zl),s(zl,V2),s(q,Z2),o(e,tp,l),_(bt,e,l),o(e,np,l),_(Tt,e,l),o(e,rp,l),o(e,cn,l),s(cn,Q2),o(e,lp,l),_(Pt,e,l),o(e,op,l),o(e,kn,l),s(kn,ek),o(e,ap,l),_(qt,e,l),o(e,pp,l),_(wt,e,l),o(e,ip,l),o(e,Ke,l),s(Ke,sk),s(Ke,hl),s(hl,tk),s(Ke,nk),o(e,fp,l),_(St,e,l),o(e,mp,l),o(e,W,l),s(W,rk),s(W,gl),s(gl,lk),s(W,ok),s(W,jl),s(jl,ak),s(W,pk),s(W,dl),s(dl,ik),s(W,fk),o(e,cp,l),_(Nt,e,l),o(e,kp,l),o(e,_n,l),s(_n,mk),o(e,_p,l),_(Ot,e,l),o(e,$p,l),o(e,$n,l),s($n,ck),up=!0},p(e,[l]){const El={};l&2&&(El.$$scope={dirty:l,ctx:e}),Oe.$set(El)},i(e){up||($(H.$$.fragment,e),$(os.$$.fragment,e),$(us.$$.fragment,e),$(hs.$$.fragment,e),$(gs.$$.fragment,e),$(js.$$.fragment,e),$(ds.$$.fragment,e),$(Es.$$.fragment,e),$(xs.$$.fragment,e),$(vs.$$.fragment,e),$(Oe.$$.fragment,e),$(bs.$$.fragment,e),$(Ts.$$.fragment,e),$(Ps.$$.fragment,e),$(qs.$$.fragment,e),$(ws.$$.fragment,e),$(Ss.$$.fragment,e),$(Ns.$$.fragment,e),$(Os.$$.fragment,e),$(Ls.$$.fragment,e),$(ys.$$.fragment,e),$(Rs.$$.fragment,e),$(Ds.$$.fragment,e),$(Gs.$$.fragment,e),$(Cs.$$.fragment,e),$(Bs.$$.fragment,e),$(Fs.$$.fragment,e),$(As.$$.fragment,e),$(Ws.$$.fragment,e),$(Is.$$.fragment,e),$(Us.$$.fragment,e),$(Ms.$$.fragment,e),$(Xs.$$.fragment,e),$(Ks.$$.fragment,e),$(Hs.$$.fragment,e),$(Ys.$$.fragment,e),$(Js.$$.fragment,e),$(Vs.$$.fragment,e),$(Zs.$$.fragment,e),$(Qs.$$.fragment,e),$(et.$$.fragment,e),$(st.$$.fragment,e),$(tt.$$.fragment,e),$(nt.$$.fragment,e),$(rt.$$.fragment,e),$(lt.$$.fragment,e),$(ot.$$.fragment,e),$(at.$$.fragment,e),$(pt.$$.fragment,e),$(it.$$.fragment,e),$(ft.$$.fragment,e),$(mt.$$.fragment,e),$(ct.$$.fragment,e),$(kt.$$.fragment,e),$(_t.$$.fragment,e),$($t.$$.fragment,e),$(ut.$$.fragment,e),$(zt.$$.fragment,e),$(ht.$$.fragment,e),$(gt.$$.fragment,e),$(jt.$$.fragment,e),$(dt.$$.fragment,e),$(Et.$$.fragment,e),$(xt.$$.fragment,e),$(vt.$$.fragment,e),$(bt.$$.fragment,e),$(Tt.$$.fragment,e),$(Pt.$$.fragment,e),$(qt.$$.fragment,e),$(wt.$$.fragment,e),$(St.$$.fragment,e),$(Nt.$$.fragment,e),$(Ot.$$.fragment,e),up=!0)},o(e){u(H.$$.fragment,e),u(os.$$.fragment,e),u(us.$$.fragment,e),u(hs.$$.fragment,e),u(gs.$$.fragment,e),u(js.$$.fragment,e),u(ds.$$.fragment,e),u(Es.$$.fragment,e),u(xs.$$.fragment,e),u(vs.$$.fragment,e),u(Oe.$$.fragment,e),u(bs.$$.fragment,e),u(Ts.$$.fragment,e),u(Ps.$$.fragment,e),u(qs.$$.fragment,e),u(ws.$$.fragment,e),u(Ss.$$.fragment,e),u(Ns.$$.fragment,e),u(Os.$$.fragment,e),u(Ls.$$.fragment,e),u(ys.$$.fragment,e),u(Rs.$$.fragment,e),u(Ds.$$.fragment,e),u(Gs.$$.fragment,e),u(Cs.$$.fragment,e),u(Bs.$$.fragment,e),u(Fs.$$.fragment,e),u(As.$$.fragment,e),u(Ws.$$.fragment,e),u(Is.$$.fragment,e),u(Us.$$.fragment,e),u(Ms.$$.fragment,e),u(Xs.$$.fragment,e),u(Ks.$$.fragment,e),u(Hs.$$.fragment,e),u(Ys.$$.fragment,e),u(Js.$$.fragment,e),u(Vs.$$.fragment,e),u(Zs.$$.fragment,e),u(Qs.$$.fragment,e),u(et.$$.fragment,e),u(st.$$.fragment,e),u(tt.$$.fragment,e),u(nt.$$.fragment,e),u(rt.$$.fragment,e),u(lt.$$.fragment,e),u(ot.$$.fragment,e),u(at.$$.fragment,e),u(pt.$$.fragment,e),u(it.$$.fragment,e),u(ft.$$.fragment,e),u(mt.$$.fragment,e),u(ct.$$.fragment,e),u(kt.$$.fragment,e),u(_t.$$.fragment,e),u($t.$$.fragment,e),u(ut.$$.fragment,e),u(zt.$$.fragment,e),u(ht.$$.fragment,e),u(gt.$$.fragment,e),u(jt.$$.fragment,e),u(dt.$$.fragment,e),u(Et.$$.fragment,e),u(xt.$$.fragment,e),u(vt.$$.fragment,e),u(bt.$$.fragment,e),u(Tt.$$.fragment,e),u(Pt.$$.fragment,e),u(qt.$$.fragment,e),u(wt.$$.fragment,e),u(St.$$.fragment,e),u(Nt.$$.fragment,e),u(Ot.$$.fragment,e),up=!1},d(e){t(j),e&&t(Y),e&&t(K),e&&t(J),z(H,e),e&&t(ge),e&&t(G),e&&t(je),e&&t(v),e&&t(V),e&&t(d),e&&t(ls),e&&t(C),e&&t(Nl),e&&t(de),e&&t(Ol),z(os,e),e&&t(Ll),e&&t(Dt),e&&t(yl),e&&t(w),e&&t(Rl),e&&t(qe),e&&t(Dl),e&&t(_s),e&&t(Gl),e&&t(we),e&&t(Cl),z(us,e),e&&t(Bl),e&&t(zs),e&&t(Fl),e&&t(Gt),e&&t(Al),z(hs,e),e&&t(Wl),e&&t(Ct),e&&t(Il),e&&t($e),z(gs),e&&t(Ul),e&&t(b),e&&t(Ml),e&&t(Ne),e&&t(Xl),z(js,e),e&&t(Kl),e&&t(B),e&&t(Hl),e&&t(T),e&&t(Yl),z(ds,e),e&&t(Jl),e&&t(Bt),e&&t(Vl),z(Es,e),e&&t(Zl),e&&t(ee),e&&t(Ql),e&&t(se),e&&t(eo),z(xs,e),e&&t(so),z(vs,e),e&&t(to),z(Oe,e),e&&t(no),e&&t(Ft),e&&t(ro),z(bs,e),e&&t(lo),e&&t(At),e&&t(oo),z(Ts,e),e&&t(ao),e&&t(Le),e&&t(po),z(Ps,e),e&&t(io),z(qs,e),e&&t(fo),e&&t(ye),e&&t(mo),z(ws,e),e&&t(co),z(Ss,e),e&&t(ko),e&&t(Re),e&&t(_o),z(Ns,e),e&&t($o),z(Os,e),e&&t(uo),e&&t(De),e&&t(zo),z(Ls,e),e&&t(ho),e&&t(S),e&&t(go),e&&t(Wt),e&&t(jo),z(ys,e),e&&t(Eo),e&&t(Ge),e&&t(xo),z(Rs,e),e&&t(vo),e&&t(Ce),e&&t(bo),z(Ds,e),e&&t(To),z(Gs,e),e&&t(Po),e&&t(E),e&&t(qo),e&&t(N),e&&t(wo),z(Cs,e),e&&t(So),z(Bs,e),e&&t(No),e&&t(F),e&&t(Oo),e&&t(It),e&&t(Lo),z(Fs,e),e&&t(yo),e&&t(Ut),e&&t(Ro),e&&t(Mt),e&&t(Do),z(As,e),e&&t(Go),z(Ws,e),e&&t(Co),e&&t(Xt),e&&t(Bo),z(Is,e),e&&t(Fo),z(Us,e),e&&t(Ao),e&&t(Kt),e&&t(Wo),z(Ms,e),e&&t(Io),e&&t(Be),e&&t(Uo),z(Xs,e),e&&t(Mo),z(Ks,e),e&&t(Xo),e&&t(Ht),e&&t(Ko),z(Hs,e),e&&t(Ho),e&&t(te),e&&t(Yo),z(Ys,e),e&&t(Jo),e&&t(ne),e&&t(Vo),e&&t(O),e&&t(Zo),z(Js,e),e&&t(Qo),e&&t(Fe),e&&t(ea),z(Vs,e),e&&t(sa),e&&t(re),e&&t(ta),e&&t(Yt),e&&t(na),e&&t(ue),z(Zs),e&&t(ra),e&&t(We),e&&t(la),z(Qs,e),e&&t(oa),e&&t(le),e&&t(aa),e&&t(Jt),e&&t(pa),z(et,e),e&&t(ia),e&&t(Ie),e&&t(fa),z(st,e),e&&t(ma),z(tt,e),e&&t(ca),e&&t(Vt),e&&t(ka),z(nt,e),e&&t(_a),e&&t(P),e&&t($a),e&&t(Zt),e&&t(ua),z(rt,e),e&&t(za),e&&t(Qt),e&&t(ha),z(lt,e),e&&t(ga),z(ot,e),e&&t(ja),e&&t(en),e&&t(da),z(at,e),e&&t(Ea),e&&t(ze),e&&t(xa),z(pt,e),e&&t(va),z(it,e),e&&t(ba),e&&t(sn),e&&t(Ta),z(ft,e),e&&t(Pa),e&&t(tn),e&&t(qa),z(mt,e),e&&t(wa),z(ct,e),e&&t(Sa),e&&t(oe),e&&t(Na),z(kt,e),e&&t(Oa),e&&t(nn),e&&t(La),z(_t,e),e&&t(ya),e&&t(rn),e&&t(Ra),e&&t(he),z($t),e&&t(Da),e&&t(Me),e&&t(Ga),z(ut,e),e&&t(Ca),e&&t(ln),e&&t(Ba),e&&t(on),e&&t(Fa),z(zt,e),e&&t(Aa),e&&t(A),e&&t(Wa),e&&t(Xe),e&&t(Ia),z(ht,e),e&&t(Ua),e&&t(an),e&&t(Ma),z(gt,e),e&&t(Xa),z(jt,e),e&&t(Ka),e&&t(pn),e&&t(Ha),z(dt,e),e&&t(Ya),e&&t(ae),e&&t(Ja),e&&t(fn),e&&t(Va),z(Et,e),e&&t(Za),e&&t(mn),e&&t(Qa),z(xt,e),e&&t(ep),z(vt,e),e&&t(sp),e&&t(q),e&&t(tp),z(bt,e),e&&t(np),z(Tt,e),e&&t(rp),e&&t(cn),e&&t(lp),z(Pt,e),e&&t(op),e&&t(kn),e&&t(ap),z(qt,e),e&&t(pp),z(wt,e),e&&t(ip),e&&t(Ke),e&&t(fp),z(St,e),e&&t(mp),e&&t(W),e&&t(cp),z(Nt,e),e&&t(kp),e&&t(_n),e&&t(_p),z(Ot,e),e&&t($p),e&&t($n)}}}const Eu={local:"",sections:[{local:"",title:"\u83B7\u53D6\u8BED\u200B\u200B\u6599\u5E93"},{local:"wordpiece",title:"\u4ECE\u5934\u5F00\u59CB\u6784\u5EFA WordPiece \u6807\u8BB0\u5668"},{local:"bpe",title:"\u4ECE\u5934\u5F00\u59CB\u6784\u5EFA BPE \u6807\u8BB0\u5668"},{local:"unigram",title:"\u4ECE\u5934\u5F00\u59CB\u6784\u5EFA Unigram \u6807\u8BB0\u5668"}],title:"\u9010\u5757\u5730\u6784\u5EFA\u6807\u8BB0\u5668"};function xu(Sl){return uu(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class Nu extends cu{constructor(j){super();ku(this,j,xu,du,_u,{})}}export{Nu as default,Eu as metadata};
