import{S as wn,i as jn,s as yn,e as n,k as f,w as b,t as o,M as Pn,c as a,d as t,m,x as d,a as p,h as i,b as c,N as jt,G as s,g as r,y as E,o as _,p as xn,q as $,B as h,v as Dn,n as Tn}from"../../chunks/vendor-hf-doc-builder.js";import{T as In}from"../../chunks/Tip-hf-doc-builder.js";import{Y as as}from"../../chunks/Youtube-hf-doc-builder.js";import{I as Jl}from"../../chunks/IconCopyLink-hf-doc-builder.js";import{C as A}from"../../chunks/CodeBlock-hf-doc-builder.js";import{C as zn}from"../../chunks/CourseFloatingBanner-hf-doc-builder.js";import{F as An}from"../../chunks/FrameworkSwitchCourse-hf-doc-builder.js";import"../../chunks/DocNotebookDropdown-hf-doc-builder.js";function Cn(C){let u,v;return u=new zn({props:{chapter:2,classNames:"absolute z-10 right-0 top-0",notebooks:[{label:"Google Colab",value:"https://colab.research.google.com/github/huggingface/notebooks/blob/master/course/zh-CN/chapter2/section4_tf.ipynb"},{label:"Aws Studio",value:"https://studiolab.sagemaker.aws/import/github/huggingface/notebooks/blob/master/course/zh-CN/chapter2/section4_tf.ipynb"}]}}),{c(){b(u.$$.fragment)},l(k){d(u.$$.fragment,k)},m(k,j){E(u,k,j),v=!0},i(k){v||($(u.$$.fragment,k),v=!0)},o(k){_(u.$$.fragment,k),v=!1},d(k){h(u,k)}}}function gn(C){let u,v;return u=new zn({props:{chapter:2,classNames:"absolute z-10 right-0 top-0",notebooks:[{label:"Google Colab",value:"https://colab.research.google.com/github/huggingface/notebooks/blob/master/course/zh-CN/chapter2/section4_pt.ipynb"},{label:"Aws Studio",value:"https://studiolab.sagemaker.aws/import/github/huggingface/notebooks/blob/master/course/zh-CN/chapter2/section4_pt.ipynb"}]}}),{c(){b(u.$$.fragment)},l(k){d(u.$$.fragment,k)},m(k,j){E(u,k,j),v=!0},i(k){v||($(u.$$.fragment,k),v=!0)},o(k){_(u.$$.fragment,k),v=!1},d(k){h(u,k)}}}function Nn(C){let u,v,k,j,y,z,P,x;return{c(){u=n("p"),v=o("\u5982\u540C "),k=n("code"),j=o("TFAutoModel"),y=o("\uFF0C  "),z=n("code"),P=o("AutoTokenizer"),x=o(" \u7C7B\u5C06\u6839\u636E\u68C0\u67E5\u70B9\u540D\u79F0\u5728\u5E93\u4E2D\u83B7\u53D6\u6B63\u786E\u7684\u6807\u8BB0\u5668(tokenizer)\u7C7B\uFF0C\u5E76\u4E14\u53EF\u4EE5\u76F4\u63A5\u4E0E\u4EFB\u4F55\u68C0\u67E5\u70B9\u4E00\u8D77\u4F7F\u7528\uFF1A")},l(D){u=a(D,"P",{});var w=p(u);v=i(w,"\u5982\u540C "),k=a(w,"CODE",{});var S=p(k);j=i(S,"TFAutoModel"),S.forEach(t),y=i(w,"\uFF0C  "),z=a(w,"CODE",{});var M=p(z);P=i(M,"AutoTokenizer"),M.forEach(t),x=i(w," \u7C7B\u5C06\u6839\u636E\u68C0\u67E5\u70B9\u540D\u79F0\u5728\u5E93\u4E2D\u83B7\u53D6\u6B63\u786E\u7684\u6807\u8BB0\u5668(tokenizer)\u7C7B\uFF0C\u5E76\u4E14\u53EF\u4EE5\u76F4\u63A5\u4E0E\u4EFB\u4F55\u68C0\u67E5\u70B9\u4E00\u8D77\u4F7F\u7528\uFF1A"),w.forEach(t)},m(D,w){r(D,u,w),s(u,v),s(u,k),s(k,j),s(u,y),s(u,z),s(z,P),s(u,x)},d(D){D&&t(u)}}}function qn(C){let u,v,k,j,y,z,P,x;return{c(){u=n("p"),v=o("\u5982\u540C "),k=n("code"),j=o("AutoModel"),y=o("\uFF0C"),z=n("code"),P=o("AutoTokenizer"),x=o(" \u7C7B\u5C06\u6839\u636E\u68C0\u67E5\u70B9\u540D\u79F0\u5728\u5E93\u4E2D\u83B7\u53D6\u6B63\u786E\u7684\u6807\u8BB0\u5668(tokenizer)\u7C7B\uFF0C\u5E76\u4E14\u53EF\u4EE5\u76F4\u63A5\u4E0E\u4EFB\u4F55\u68C0\u67E5\u70B9\u4E00\u8D77\u4F7F\u7528\uFF1A")},l(D){u=a(D,"P",{});var w=p(u);v=i(w,"\u5982\u540C "),k=a(w,"CODE",{});var S=p(k);j=i(S,"AutoModel"),S.forEach(t),y=i(w,"\uFF0C"),z=a(w,"CODE",{});var M=p(z);P=i(M,"AutoTokenizer"),M.forEach(t),x=i(w," \u7C7B\u5C06\u6839\u636E\u68C0\u67E5\u70B9\u540D\u79F0\u5728\u5E93\u4E2D\u83B7\u53D6\u6B63\u786E\u7684\u6807\u8BB0\u5668(tokenizer)\u7C7B\uFF0C\u5E76\u4E14\u53EF\u4EE5\u76F4\u63A5\u4E0E\u4EFB\u4F55\u68C0\u67E5\u70B9\u4E00\u8D77\u4F7F\u7528\uFF1A"),w.forEach(t)},m(D,w){r(D,u,w),s(u,v),s(u,k),s(k,j),s(u,y),s(u,z),s(z,P),s(u,x)},d(D){D&&t(u)}}}function Mn(C){let u,v,k,j,y;return{c(){u=n("p"),v=o("\u270F\uFE0F "),k=n("strong"),j=o("\u8BD5\u8BD5\u770B\uFF01"),y=o(" \u5728\u6211\u4EEC\u5728\u7B2C 2 \u8282\u4E2D\u4F7F\u7528\u7684\u8F93\u5165\u53E5\u5B50\uFF08\u201CI\u2019ve been waiting for a HuggingFace course my whole life.\u201D\u548C\u201CI hate this so much!\u201D\uFF09\u590D\u5236\u6700\u540E\u4E24\u4E2A\u6B65\u9AA4\uFF08\u6807\u8BB0\u5316\u548C\u8F6C\u6362\u4E3A\u8F93\u5165 ID\uFF09\u3002\u68C0\u67E5\u60A8\u83B7\u5F97\u7684\u8F93\u5165 ID \u662F\u5426\u4E0E\u6211\u4EEC\u4E4B\u524D\u83B7\u5F97\u7684\u76F8\u540C\uFF01")},l(z){u=a(z,"P",{});var P=p(u);v=i(P,"\u270F\uFE0F "),k=a(P,"STRONG",{});var x=p(k);j=i(x,"\u8BD5\u8BD5\u770B\uFF01"),x.forEach(t),y=i(P," \u5728\u6211\u4EEC\u5728\u7B2C 2 \u8282\u4E2D\u4F7F\u7528\u7684\u8F93\u5165\u53E5\u5B50\uFF08\u201CI\u2019ve been waiting for a HuggingFace course my whole life.\u201D\u548C\u201CI hate this so much!\u201D\uFF09\u590D\u5236\u6700\u540E\u4E24\u4E2A\u6B65\u9AA4\uFF08\u6807\u8BB0\u5316\u548C\u8F6C\u6362\u4E3A\u8F93\u5165 ID\uFF09\u3002\u68C0\u67E5\u60A8\u83B7\u5F97\u7684\u8F93\u5165 ID \u662F\u5426\u4E0E\u6211\u4EEC\u4E4B\u524D\u83B7\u5F97\u7684\u76F8\u540C\uFF01"),P.forEach(t)},m(z,P){r(z,u,P),s(u,v),s(u,k),s(k,j),s(u,y)},d(z){z&&t(u)}}}function Hn(C){let u,v,k,j,y,z,P,x,D,w,S,M,N,q,Ke,me,ps,Ve,Rl,fs,Ye,Wl,ms,ue,us,Qe,Kl,ks,Xe,Vl,cs,U,Y,yt,ke,Yl,Pt,Ql,_s,ce,$s,Q,Xl,xt,Zl,er,vs,G,_e,Yo,tr,$e,Qo,bs,X,sr,Dt,lr,rr,ds,ve,Es,be,hs,Ze,or,zs,et,ir,ws,tt,nr,js,de,ar,Tt,pr,ys,Z,fr,It,mr,ur,Ps,F,ee,At,Ee,kr,Ct,cr,xs,he,Ds,st,_r,Ts,te,gt,$r,vr,Nt,br,Is,lt,dr,As,J,ze,Xo,Er,we,Zo,Cs,rt,hr,gs,ot,zr,Ns,se,wr,qt,jr,yr,qs,je,Pr,Ms,ye,Hs,it,xr,Os,nt,Dr,Bs,at,Tr,Ls,R,Pe,ei,Ir,xe,ti,Ss,pt,Ar,Us,ft,Cr,Gs,De,gr,Fs,mt,Nr,Js,H,Mt,qr,Mr,Ht,Hr,Or,Ot,Br,Rs,ut,Lr,Ws,Te,Sr,Ks,T,Ur,Bt,Gr,Fr,Lt,Jr,Rr,St,Wr,Kr,Ut,Vr,Yr,Vs,le,Qr,Gt,Xr,Zr,Ys,Ie,Qs,kt,Ae,Xs,ct,eo,Zs,Ce,el,ge,tl,_t,to,sl,Ne,ll,I,so,$t,lo,ro,Ft,oo,io,Jt,no,ao,Rt,po,fo,rl,qe,mo,ol,Me,il,re,uo,Wt,ko,co,nl,oe,_o,Kt,$o,vo,al,O,bo,Vt,Eo,ho,Yt,zo,wo,pl,vt,jo,fl,He,yo,ml,ie,Po,Qt,xo,Do,ul,Oe,kl,bt,To,cl,Be,_l,g,Io,Xt,Ao,Co,Zt,go,No,es,qo,Mo,$l,W,ne,ts,Le,Ho,ss,Oo,vl,Se,bl,Ue,dl,dt,Bo,El,ae,hl,Ge,Lo,zl,K,ls,So,Uo,rs,Go,Fo,wl,Fe,jl,Je,yl,pe,Jo,os,Ro,Wo,Pl,Et,Ko,xl;k=new An({props:{fw:C[0]}}),x=new Jl({});const si=[gn,Cn],Re=[];function li(e,l){return e[0]==="pt"?0:1}N=li(C),q=Re[N]=si[N](C),me=new as({props:{id:"VFp38yj8h3A"}}),ue=new A({props:{code:"Jim Henson was a puppeteer",highlighted:'<span class="hljs-comment">Jim Henson was a puppeteer</span>'}}),ke=new Jl({}),ce=new as({props:{id:"nhJxYji1aho"}}),ve=new A({props:{code:`tokenized_text = "Jim Henson was a puppeteer".split()
print(tokenized_text)`,highlighted:`tokenized_text = <span class="hljs-string">&quot;Jim Henson was a puppeteer&quot;</span>.split()
<span class="hljs-built_in">print</span>(tokenized_text)`}}),be=new A({props:{code:"['Jim', 'Henson', 'was', 'a', 'puppeteer']",highlighted:'[<span class="hljs-string">&#x27;Jim&#x27;</span>, <span class="hljs-string">&#x27;Henson&#x27;</span>, <span class="hljs-string">&#x27;was&#x27;</span>, <span class="hljs-string">&#x27;a&#x27;</span>, <span class="hljs-string">&#x27;puppeteer&#x27;</span>]'}}),Ee=new Jl({}),he=new as({props:{id:"ssLq_EK2jLE"}}),ye=new as({props:{id:"zHvTiHr506c"}}),Ie=new A({props:{code:`from transformers import BertTokenizer

tokenizer = BertTokenizer.from_pretrained("bert-base-cased")`,highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> BertTokenizer

tokenizer = BertTokenizer.from_pretrained(<span class="hljs-string">&quot;bert-base-cased&quot;</span>)`}});function ri(e,l){return e[0]==="pt"?qn:Nn}let Dl=ri(C),V=Dl(C);return Ae=new A({props:{code:`from transformers import AutoTokenizer

tokenizer = AutoTokenizer.from_pretrained("bert-base-cased")`,highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoTokenizer

tokenizer = AutoTokenizer.from_pretrained(<span class="hljs-string">&quot;bert-base-cased&quot;</span>)`}}),Ce=new A({props:{code:'tokenizer("Using a Transformer network is simple")',highlighted:'tokenizer(<span class="hljs-string">&quot;Using a Transformer network is simple&quot;</span>)'}}),ge=new A({props:{code:`{'input_ids': [101, 7993, 170, 11303, 1200, 2443, 1110, 3014, 102],
 'token_type_ids': [0, 0, 0, 0, 0, 0, 0, 0, 0],
 'attention_mask': [1, 1, 1, 1, 1, 1, 1, 1, 1]}`,highlighted:`{<span class="hljs-string">&#x27;input_ids&#x27;</span>: [<span class="hljs-number">101</span>, <span class="hljs-number">7993</span>, <span class="hljs-number">170</span>, <span class="hljs-number">11303</span>, <span class="hljs-number">1200</span>, <span class="hljs-number">2443</span>, <span class="hljs-number">1110</span>, <span class="hljs-number">3014</span>, <span class="hljs-number">102</span>],
 <span class="hljs-string">&#x27;token_type_ids&#x27;</span>: [<span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>],
 <span class="hljs-string">&#x27;attention_mask&#x27;</span>: [<span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>]}`}}),Ne=new A({props:{code:'tokenizer.save_pretrained("directory_on_my_computer")',highlighted:'tokenizer.save_pretrained(<span class="hljs-string">&quot;directory_on_my_computer&quot;</span>)'}}),Me=new as({props:{id:"Yffk5aydLzg"}}),Oe=new A({props:{code:`from transformers import AutoTokenizer

tokenizer = AutoTokenizer.from_pretrained("bert-base-cased")

sequence = "Using a Transformer network is simple"
tokens = tokenizer.tokenize(sequence)

print(tokens)`,highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoTokenizer

tokenizer = AutoTokenizer.from_pretrained(<span class="hljs-string">&quot;bert-base-cased&quot;</span>)

sequence = <span class="hljs-string">&quot;Using a Transformer network is simple&quot;</span>
tokens = tokenizer.tokenize(sequence)

<span class="hljs-built_in">print</span>(tokens)`}}),Be=new A({props:{code:"['Using', 'a', 'transform', '##er', 'network', 'is', 'simple']",highlighted:'[<span class="hljs-string">&#x27;Using&#x27;</span>, <span class="hljs-string">&#x27;a&#x27;</span>, <span class="hljs-string">&#x27;transform&#x27;</span>, <span class="hljs-string">&#x27;##er&#x27;</span>, <span class="hljs-string">&#x27;network&#x27;</span>, <span class="hljs-string">&#x27;is&#x27;</span>, <span class="hljs-string">&#x27;simple&#x27;</span>]'}}),Le=new Jl({}),Se=new A({props:{code:`ids = tokenizer.convert_tokens_to_ids(tokens)

print(ids)`,highlighted:`ids = tokenizer.convert_tokens_to_ids(tokens)

<span class="hljs-built_in">print</span>(ids)`}}),Ue=new A({props:{code:"[7993, 170, 11303, 1200, 2443, 1110, 3014]",highlighted:'[<span class="hljs-number">7993</span>, <span class="hljs-number">170</span>, <span class="hljs-number">11303</span>, <span class="hljs-number">1200</span>, <span class="hljs-number">2443</span>, <span class="hljs-number">1110</span>, <span class="hljs-number">3014</span>]'}}),ae=new In({props:{$$slots:{default:[Mn]},$$scope:{ctx:C}}}),Fe=new A({props:{code:`decoded_string = tokenizer.decode([7993, 170, 11303, 1200, 2443, 1110, 3014])
print(decoded_string)`,highlighted:`decoded_string = tokenizer.decode([<span class="hljs-number">7993</span>, <span class="hljs-number">170</span>, <span class="hljs-number">11303</span>, <span class="hljs-number">1200</span>, <span class="hljs-number">2443</span>, <span class="hljs-number">1110</span>, <span class="hljs-number">3014</span>])
<span class="hljs-built_in">print</span>(decoded_string)`}}),Je=new A({props:{code:"'Using a Transformer network is simple'",highlighted:'<span class="hljs-string">&#x27;Using a Transformer network is simple&#x27;</span>'}}),{c(){u=n("meta"),v=f(),b(k.$$.fragment),j=f(),y=n("h1"),z=n("a"),P=n("span"),b(x.$$.fragment),D=f(),w=n("span"),S=o("\u6807\u8BB0\u5668\uFF08Tokenizer\uFF09"),M=f(),q.c(),Ke=f(),b(me.$$.fragment),ps=f(),Ve=n("p"),Rl=o("\u6807\u8BB0\u5668(Tokenizer)\u662F NLP \u7BA1\u9053\u7684\u6838\u5FC3\u7EC4\u4EF6\u4E4B\u4E00\u3002\u5B83\u4EEC\u6709\u4E00\u4E2A\u76EE\u7684\uFF1A\u5C06\u6587\u672C\u8F6C\u6362\u4E3A\u6A21\u578B\u53EF\u4EE5\u5904\u7406\u7684\u6570\u636E\u3002\u6A21\u578B\u53EA\u80FD\u5904\u7406\u6570\u5B57\uFF0C\u56E0\u6B64\u6807\u8BB0\u5668(Tokenizer)\u9700\u8981\u5C06\u6211\u4EEC\u7684\u6587\u672C\u8F93\u5165\u8F6C\u6362\u4E3A\u6570\u5B57\u6570\u636E\u3002\u5728\u672C\u8282\u4E2D\uFF0C\u6211\u4EEC\u5C06\u786E\u5207\u5730\u63A2\u8BA8\u6807\u8BB0\u5316\u7BA1\u9053\u4E2D\u53D1\u751F\u7684\u4E8B\u60C5\u3002"),fs=f(),Ye=n("p"),Wl=o("\u5728 NLP \u4EFB\u52A1\u4E2D\uFF0C\u901A\u5E38\u5904\u7406\u7684\u6570\u636E\u662F\u539F\u59CB\u6587\u672C\u3002\u8FD9\u662F\u6B64\u7C7B\u6587\u672C\u7684\u793A\u4F8B"),ms=f(),b(ue.$$.fragment),us=f(),Qe=n("p"),Kl=o("\u4F46\u662F\uFF0C\u6A21\u578B\u53EA\u80FD\u5904\u7406\u6570\u5B57\uFF0C\u56E0\u6B64\u6211\u4EEC\u9700\u8981\u627E\u5230\u4E00\u79CD\u5C06\u539F\u59CB\u6587\u672C\u8F6C\u6362\u4E3A\u6570\u5B57\u7684\u65B9\u6CD5\u3002\u8FD9\u5C31\u662F\u6807\u8BB0\u5668\uFF08tokenizer\uFF09\u6240\u505A\u7684\uFF0C\u5E76\u4E14\u6709\u5F88\u591A\u65B9\u6CD5\u53EF\u4EE5\u89E3\u51B3\u8FD9\u4E2A\u95EE\u9898\u3002\u76EE\u6807\u662F\u627E\u5230\u6700\u6709\u610F\u4E49\u7684\u8868\u793A\u2014\u2014\u5373\u5BF9\u6A21\u578B\u6700\u6709\u610F\u4E49\u7684\u8868\u793A\u2014\u2014\u5E76\u4E14\u5982\u679C\u53EF\u80FD\u7684\u8BDD\uFF0C\u627E\u5230\u6700\u5C0F\u7684\u8868\u793A\u3002"),ks=f(),Xe=n("p"),Vl=o("\u8BA9\u6211\u4EEC\u770B\u4E00\u4E0B\u6807\u8BB0\u5316\u7B97\u6CD5\u7684\u4E00\u4E9B\u793A\u4F8B\uFF0C\u5E76\u5C1D\u8BD5\u56DE\u7B54\u60A8\u53EF\u80FD\u5BF9\u6807\u8BB0\u5316\u63D0\u51FA\u7684\u4E00\u4E9B\u95EE\u9898\u3002"),cs=f(),U=n("h2"),Y=n("a"),yt=n("span"),b(ke.$$.fragment),Yl=f(),Pt=n("span"),Ql=o("\u57FA\u4E8E\u8BCD\u7684(Word-based)"),_s=f(),b(ce.$$.fragment),$s=f(),Q=n("p"),Xl=o("\u60F3\u5230\u7684\u7B2C\u4E00\u79CD\u6807\u8BB0\u5668\u662F\u57FA\u4E8E\u8BCD\u7684("),xt=n("em"),Zl=o("word-based"),er=o(").\u5B83\u901A\u5E38\u5F88\u5BB9\u6613\u8BBE\u7F6E\u548C\u4F7F\u7528\uFF0C\u53EA\u9700\u51E0\u6761\u89C4\u5219\uFF0C\u5E76\u4E14\u901A\u5E38\u4F1A\u4EA7\u751F\u4E0D\u9519\u7684\u7ED3\u679C\u3002\u4F8B\u5982\uFF0C\u5728\u4E0B\u56FE\u4E2D\uFF0C\u76EE\u6807\u662F\u5C06\u539F\u59CB\u6587\u672C\u62C6\u5206\u4E3A\u5355\u8BCD\u5E76\u4E3A\u6BCF\u4E2A\u5355\u8BCD\u627E\u5230\u4E00\u4E2A\u6570\u5B57\u8868\u793A\uFF1A"),vs=f(),G=n("div"),_e=n("img"),tr=f(),$e=n("img"),bs=f(),X=n("p"),sr=o("\u6709\u591A\u79CD\u65B9\u6CD5\u53EF\u4EE5\u62C6\u5206\u6587\u672C\u3002\u4F8B\u5982\uFF0C\u6211\u4EEC\u53EF\u4EE5\u901A\u8FC7\u5E94\u7528Python\u7684"),Dt=n("code"),lr=o("split()"),rr=o("\u51FD\u6570\uFF0C\u4F7F\u7528\u7A7A\u683C\u5C06\u6587\u672C\u6807\u8BB0\u4E3A\u5355\u8BCD\uFF1A"),ds=f(),b(ve.$$.fragment),Es=f(),b(be.$$.fragment),hs=f(),Ze=n("p"),or=o("\u8FD8\u6709\u4E00\u4E9B\u5355\u8BCD\u6807\u8BB0\u5668\u7684\u53D8\u4F53\uFF0C\u5B83\u4EEC\u5177\u6709\u989D\u5916\u7684\u6807\u70B9\u7B26\u53F7\u89C4\u5219\u3002\u4F7F\u7528\u8FD9\u79CD\u6807\u8BB0\u5668\uFF0C\u6211\u4EEC\u6700\u7EC8\u53EF\u4EE5\u5F97\u5230\u4E00\u4E9B\u975E\u5E38\u5927\u7684\u201C\u8BCD\u6C47\u8868\u201D\uFF0C\u5176\u4E2D\u8BCD\u6C47\u8868\u7531\u6211\u4EEC\u5728\u8BED\u6599\u5E93\u4E2D\u62E5\u6709\u7684\u72EC\u7ACB\u6807\u8BB0\u7684\u603B\u6570\u5B9A\u4E49\u3002"),zs=f(),et=n("p"),ir=o("\u6BCF\u4E2A\u5355\u8BCD\u90FD\u5206\u914D\u4E86\u4E00\u4E2A ID\uFF0C\u4ECE 0 \u5F00\u59CB\u4E00\u76F4\u5230\u8BCD\u6C47\u8868\u7684\u5927\u5C0F\u3002\u8BE5\u6A21\u578B\u4F7F\u7528\u8FD9\u4E9B ID \u6765\u8BC6\u522B\u6BCF\u4E2A\u5355\u8BCD\u3002"),ws=f(),tt=n("p"),nr=o("\u5982\u679C\u6211\u4EEC\u60F3\u7528\u57FA\u4E8E\u5355\u8BCD\u7684\u6807\u8BB0\u5668(tokenizer)\u5B8C\u5168\u8986\u76D6\u4E00\u79CD\u8BED\u8A00\uFF0C\u6211\u4EEC\u9700\u8981\u4E3A\u8BED\u8A00\u4E2D\u7684\u6BCF\u4E2A\u5355\u8BCD\u90FD\u6709\u4E00\u4E2A\u6807\u8BC6\u7B26\uFF0C\u8FD9\u5C06\u751F\u6210\u5927\u91CF\u7684\u6807\u8BB0\u3002\u4F8B\u5982\uFF0C\u82F1\u8BED\u4E2D\u6709\u8D85\u8FC7 500,000 \u4E2A\u5355\u8BCD\uFF0C\u56E0\u6B64\u8981\u6784\u5EFA\u4ECE\u6BCF\u4E2A\u5355\u8BCD\u5230\u8F93\u5165 ID \u7684\u6620\u5C04\uFF0C\u6211\u4EEC\u9700\u8981\u8DDF\u8E2A\u8FD9\u4E48\u591A ID\u3002\u6B64\u5916\uFF0C\u50CF\u201Cdog\u201D\u8FD9\u6837\u7684\u8BCD\u4E0E\u201Cdogs\u201D\u8FD9\u6837\u7684\u8BCD\u7684\u8868\u793A\u65B9\u5F0F\u4E0D\u540C\uFF0C\u6A21\u578B\u6700\u521D\u65E0\u6CD5\u77E5\u9053\u201Cdog\u201D\u548C\u201Cdogs\u201D\u662F\u76F8\u4F3C\u7684\uFF1A\u5B83\u4F1A\u5C06\u8FD9\u4E24\u4E2A\u8BCD\u8BC6\u522B\u4E3A\u4E0D\u76F8\u5173\u3002\u8FD9\u540C\u6837\u9002\u7528\u4E8E\u5176\u4ED6\u76F8\u4F3C\u7684\u8BCD\uFF0C\u4F8B\u5982\u201Crun\u201D\u548C\u201Crunning\u201D\uFF0C\u6A21\u578B\u6700\u521D\u4E0D\u4F1A\u8BA4\u4E3A\u5B83\u4EEC\u662F\u76F8\u4F3C\u7684\u3002"),js=f(),de=n("p"),ar=o("\u6700\u540E\uFF0C\u6211\u4EEC\u9700\u8981\u4E00\u4E2A\u81EA\u5B9A\u4E49\u6807\u8BB0(token)\u6765\u8868\u793A\u4E0D\u5728\u6211\u4EEC\u8BCD\u6C47\u8868\u4E2D\u7684\u5355\u8BCD\u3002\u8FD9\u88AB\u79F0\u4E3A\u201C\u672A\u77E5\u201D\u6807\u8BB0(token)\uFF0C\u901A\u5E38\u8868\u793A\u4E3A\u201C[UNK]\u201D\u6216\u201D"),Tt=n("unk"),pr=o("\u201C\u3002\u5982\u679C\u4F60\u770B\u5230\u6807\u8BB0\u5668\u4EA7\u751F\u4E86\u5F88\u591A\u8FD9\u6837\u7684\u6807\u8BB0\uFF0C\u8FD9\u901A\u5E38\u662F\u4E00\u4E2A\u4E0D\u597D\u7684\u8FF9\u8C61\uFF0C\u56E0\u4E3A\u5B83\u65E0\u6CD5\u68C0\u7D22\u5230\u4E00\u4E2A\u8BCD\u7684\u5408\u7406\u8868\u793A\uFF0C\u5E76\u4E14\u4F60\u4F1A\u5728\u8FD9\u4E2A\u8FC7\u7A0B\u4E2D\u4E22\u5931\u4FE1\u606F\u3002\u5236\u4F5C\u8BCD\u6C47\u8868\u65F6\u7684\u76EE\u6807\u662F\u4EE5\u8FD9\u6837\u4E00\u79CD\u65B9\u5F0F\u8FDB\u884C\uFF0C\u5373\u6807\u8BB0\u5668\u5C06\u5C3D\u53EF\u80FD\u5C11\u7684\u5355\u8BCD\u6807\u8BB0\u4E3A\u672A\u77E5\u6807\u8BB0\u3002"),ys=f(),Z=n("p"),fr=o("\u51CF\u5C11\u672A\u77E5\u6807\u8BB0\u6570\u91CF\u7684\u4E00\u79CD\u65B9\u6CD5\u662F\u4F7F\u7528\u66F4\u6DF1\u4E00\u5C42\u7684\u6807\u8BB0\u5668(tokenizer)\uFF0C\u5373\u57FA\u4E8E\u5B57\u7B26\u7684("),It=n("em"),mr=o("character-based"),ur=o(")\u6807\u8BB0\u5668(tokenizer)\u3002"),Ps=f(),F=n("h2"),ee=n("a"),At=n("span"),b(Ee.$$.fragment),kr=f(),Ct=n("span"),cr=o("\u57FA\u4E8E\u5B57\u7B26(Character-based)"),xs=f(),b(he.$$.fragment),Ds=f(),st=n("p"),_r=o("\u57FA\u4E8E\u5B57\u7B26\u7684\u6807\u8BB0\u5668(tokenizer)\u5C06\u6587\u672C\u62C6\u5206\u4E3A\u5B57\u7B26\uFF0C\u800C\u4E0D\u662F\u5355\u8BCD\u3002\u8FD9\u6709\u4E24\u4E2A\u4E3B\u8981\u597D\u5904\uFF1A"),Ts=f(),te=n("ul"),gt=n("li"),$r=o("\u8BCD\u6C47\u91CF\u8981\u5C0F\u5F97\u591A\u3002"),vr=f(),Nt=n("li"),br=o("\u8BCD\u6C47\u5916\uFF08\u672A\u77E5\uFF09\u6807\u8BB0(token)\u8981\u5C11\u5F97\u591A\uFF0C\u56E0\u4E3A\u6BCF\u4E2A\u5355\u8BCD\u90FD\u53EF\u4EE5\u4ECE\u5B57\u7B26\u6784\u5EFA\u3002"),Is=f(),lt=n("p"),dr=o("\u4F46\u662F\u8FD9\u91CC\u4E5F\u51FA\u73B0\u4E86\u4E00\u4E9B\u5173\u4E8E\u7A7A\u683C\u548C\u6807\u70B9\u7B26\u53F7\u7684\u95EE\u9898\uFF1A"),As=f(),J=n("div"),ze=n("img"),Er=f(),we=n("img"),Cs=f(),rt=n("p"),hr=o("\u8FD9\u79CD\u65B9\u6CD5\u4E5F\u4E0D\u662F\u5B8C\u7F8E\u7684\u3002\u7531\u4E8E\u73B0\u5728\u8868\u793A\u662F\u57FA\u4E8E\u5B57\u7B26\u800C\u4E0D\u662F\u5355\u8BCD\uFF0C\u56E0\u6B64\u4EBA\u4EEC\u53EF\u80FD\u4F1A\u4E89\u8FA9\u8BF4\uFF0C\u4ECE\u76F4\u89C9\u4E0A\u8BB2\uFF0C\u5B83\u7684\u610F\u4E49\u4E0D\u5927\uFF1A\u6BCF\u4E2A\u5B57\u7B26\u672C\u8EAB\u5E76\u6CA1\u6709\u591A\u5927\u610F\u4E49\uFF0C\u800C\u5355\u8BCD\u5C31\u662F\u8FD9\u79CD\u60C5\u51B5\u3002\u7136\u800C\uFF0C\u8FD9\u53C8\u56E0\u8BED\u8A00\u800C\u5F02\uFF1B\u4F8B\u5982\uFF0C\u5728\u4E2D\u6587\u4E2D\uFF0C\u6BCF\u4E2A\u5B57\u7B26\u6BD4\u62C9\u4E01\u8BED\u8A00\u4E2D\u7684\u5B57\u7B26\u5305\u542B\u66F4\u591A\u7684\u4FE1\u606F\u3002"),gs=f(),ot=n("p"),zr=o("\u53E6\u4E00\u4EF6\u8981\u8003\u8651\u7684\u4E8B\u60C5\u662F\uFF0C\u6211\u4EEC\u7684\u6A21\u578B\u6700\u7EC8\u4F1A\u5904\u7406\u5927\u91CF\u7684\u8BCD\u7B26(token)\uFF1A\u867D\u7136\u4F7F\u7528\u57FA\u4E8E\u5355\u8BCD\u7684\u6807\u8BB0\u5668(tokenizer)\uFF0C\u5355\u8BCD\u53EA\u4F1A\u662F\u5355\u4E2A\u6807\u8BB0\uFF0C\u4F46\u5F53\u8F6C\u6362\u4E3A\u5B57\u7B26\u65F6\uFF0C\u5B83\u5F88\u5BB9\u6613\u53D8\u6210 10 \u4E2A\u6216\u66F4\u591A\u7684\u8BCD\u7B26(token)\u3002"),Ns=f(),se=n("p"),wr=o("\u4E3A\u4E86\u4E24\u5168\u5176\u7F8E\uFF0C\u6211\u4EEC\u53EF\u4EE5\u4F7F\u7528\u7ED3\u5408\u8FD9\u4E24\u79CD\u65B9\u6CD5\u7684\u7B2C\u4E09\u79CD\u6280\u672F\uFF1A"),qt=n("em"),jr=o("\u5B50\u8BCD\u6807\u8BB0\u5316(subword tokenization)"),yr=o("\u3002"),qs=f(),je=n("h2"),Pr=o("\u5B50\u8BCD\u6807\u8BB0\u5316"),Ms=f(),b(ye.$$.fragment),Hs=f(),it=n("p"),xr=o("\u5B50\u8BCD\u5206\u8BCD\u7B97\u6CD5\u4F9D\u8D56\u4E8E\u8FD9\u6837\u4E00\u4E2A\u539F\u5219\uFF0C\u5373\u4E0D\u5E94\u5C06\u5E38\u7528\u8BCD\u62C6\u5206\u4E3A\u66F4\u5C0F\u7684\u5B50\u8BCD\uFF0C\u800C\u5E94\u5C06\u7A00\u6709\u8BCD\u5206\u89E3\u4E3A\u6709\u610F\u4E49\u7684\u5B50\u8BCD\u3002"),Os=f(),nt=n("p"),Dr=o("\u4F8B\u5982\uFF0C\u201Cannoyingly\u201D\u53EF\u80FD\u88AB\u8BA4\u4E3A\u662F\u4E00\u4E2A\u7F55\u89C1\u7684\u8BCD\uFF0C\u53EF\u4EE5\u5206\u89E3\u4E3A\u201Cannoying\u201D\u548C\u201Cly\u201D\u3002\u8FD9\u4E24\u8005\u90FD\u53EF\u80FD\u4F5C\u4E3A\u72EC\u7ACB\u7684\u5B50\u8BCD\u51FA\u73B0\u5F97\u66F4\u9891\u7E41\uFF0C\u540C\u65F6\u201Cannoyingly\u201D\u7684\u542B\u4E49\u7531\u201Cannoying\u201D\u548C\u201Cly\u201D\u7684\u590D\u5408\u542B\u4E49\u4FDD\u6301\u3002"),Bs=f(),at=n("p"),Tr=o("\u8FD9\u662F\u4E00\u4E2A\u793A\u4F8B\uFF0C\u5C55\u793A\u4E86\u5B50\u8BCD\u6807\u8BB0\u5316\u7B97\u6CD5\u5982\u4F55\u6807\u8BB0\u5E8F\u5217\u201CLet\u2019s do tokenization!\u201D\uFF1A"),Ls=f(),R=n("div"),Pe=n("img"),Ir=f(),xe=n("img"),Ss=f(),pt=n("p"),Ar=o("\u8FD9\u4E9B\u5B50\u8BCD\u6700\u7EC8\u63D0\u4F9B\u4E86\u5F88\u591A\u8BED\u4E49\u542B\u4E49\uFF1A\u4F8B\u5982\uFF0C\u5728\u4E0A\u9762\u7684\u793A\u4F8B\u4E2D\uFF0C\u201Ctokenization\u201D\u88AB\u62C6\u5206\u4E3A\u201Ctoken\u201D\u548C\u201Cization\u201D\uFF0C\u8FD9\u4E24\u4E2A\u5177\u6709\u8BED\u4E49\u610F\u4E49\u540C\u65F6\u8282\u7701\u7A7A\u95F4\u7684\u8BCD\u7B26(token)\uFF08\u53EA\u9700\u8981\u4E24\u4E2A\u6807\u8BB0(token)\u4EE3\u8868\u4E00\u4E2A\u957F\u8BCD\uFF09\u3002\u8FD9\u4F7F\u6211\u4EEC\u80FD\u591F\u5BF9\u8F83\u5C0F\u7684\u8BCD\u6C47\u8868\u8FDB\u884C\u76F8\u5BF9\u8F83\u597D\u7684\u8986\u76D6\uFF0C\u5E76\u4E14\u51E0\u4E4E\u6CA1\u6709\u672A\u77E5\u7684\u6807\u8BB0"),Us=f(),ft=n("p"),Cr=o("\u8FD9\u79CD\u65B9\u6CD5\u5728\u571F\u8033\u5176\u8BED\u7B49\u7C98\u7740\u578B\u8BED\u8A00(agglutinative languages)\u4E2D\u7279\u522B\u6709\u7528\uFF0C\u60A8\u53EF\u4EE5\u901A\u8FC7\u5C06\u5B50\u8BCD\u4E32\u5728\u4E00\u8D77\u6765\u5F62\u6210\uFF08\u51E0\u4E4E\uFF09\u4EFB\u610F\u957F\u7684\u590D\u6742\u8BCD\u3002"),Gs=f(),De=n("h3"),gr=o("\u8FD8\u6709\u66F4\u591A\uFF01"),Fs=f(),mt=n("p"),Nr=o("\u4E0D\u51FA\u6240\u6599\uFF0C\u8FD8\u6709\u66F4\u591A\u7684\u6280\u672F\u3002\u4EC5\u4E3E\u51E0\u4F8B\uFF1A"),Js=f(),H=n("ul"),Mt=n("li"),qr=o("Byte-level BPE, \u7528\u4E8E GPT-2"),Mr=f(),Ht=n("li"),Hr=o("WordPiece, \u7528\u4E8E BERT"),Or=f(),Ot=n("li"),Br=o("SentencePiece or Unigram, \u7528\u4E8E\u591A\u4E2A\u591A\u8BED\u8A00\u6A21\u578B"),Rs=f(),ut=n("p"),Lr=o("\u60A8\u73B0\u5728\u5E94\u8BE5\u5BF9\u6807\u8BB0\u5668(tokenizers)\u7684\u5DE5\u4F5C\u539F\u7406\u6709\u8DB3\u591F\u7684\u4E86\u89E3\uFF0C\u4EE5\u4FBF\u5F00\u59CB\u4F7F\u7528 API\u3002"),Ws=f(),Te=n("h2"),Sr=o("\u52A0\u8F7D\u548C\u4FDD\u5B58"),Ks=f(),T=n("p"),Ur=o("\u52A0\u8F7D\u548C\u4FDD\u5B58\u6807\u8BB0\u5668(tokenizer)\u5C31\u50CF\u4F7F\u7528\u6A21\u578B\u4E00\u6837\u7B80\u5355\u3002\u5B9E\u9645\u4E0A\uFF0C\u5B83\u57FA\u4E8E\u76F8\u540C\u7684\u4E24\u79CD\u65B9\u6CD5\uFF1A "),Bt=n("code"),Gr=o("from_pretrained()"),Fr=o(" \u548C "),Lt=n("code"),Jr=o("save_pretrained()"),Rr=o(" \u3002\u8FD9\u4E9B\u65B9\u6CD5\u5C06\u52A0\u8F7D\u6216\u4FDD\u5B58\u6807\u8BB0\u5668(tokenizer)\u4F7F\u7528\u7684\u7B97\u6CD5\uFF08\u6709\u70B9\u50CF"),St=n("em"),Wr=o("\u5EFA\u7B51\u5B66(architecture)"),Kr=o("\u7684\u6A21\u578B\uFF09\u4EE5\u53CA\u5B83\u7684\u8BCD\u6C47\uFF08\u6709\u70B9\u50CF"),Ut=n("em"),Vr=o("\u6743\u91CD(weights)"),Yr=o("\u6A21\u578B\uFF09\u3002"),Vs=f(),le=n("p"),Qr=o("\u52A0\u8F7D\u4F7F\u7528\u4E0E BERT \u76F8\u540C\u7684\u68C0\u67E5\u70B9\u8BAD\u7EC3\u7684 BERT \u6807\u8BB0\u5668(tokenizer)\u4E0E\u52A0\u8F7D\u6A21\u578B\u7684\u65B9\u5F0F\u76F8\u540C\uFF0C\u9664\u4E86\u6211\u4EEC\u4F7F\u7528  "),Gt=n("code"),Xr=o("BertTokenizer"),Zr=o(" \u7C7B\uFF1A"),Ys=f(),b(Ie.$$.fragment),Qs=f(),V.c(),kt=f(),b(Ae.$$.fragment),Xs=f(),ct=n("p"),eo=o("\u6211\u4EEC\u73B0\u5728\u53EF\u4EE5\u4F7F\u7528\u6807\u8BB0\u5668(tokenizer)\uFF0C\u5982\u4E0A\u4E00\u8282\u6240\u793A\uFF1A"),Zs=f(),b(Ce.$$.fragment),el=f(),b(ge.$$.fragment),tl=f(),_t=n("p"),to=o("\u4FDD\u5B58\u6807\u8BB0\u5668(tokenizer)\u4E0E\u4FDD\u5B58\u6A21\u578B\u76F8\u540C:"),sl=f(),b(Ne.$$.fragment),ll=f(),I=n("p"),so=o("\u6211\u4EEC\u5728"),$t=n("a"),lo=o("Chapter 3"),ro=o("\u4E2D\u5C06\u66F4\u591A\u5730\u8C08\u8BBA"),Ft=n("code"),oo=o("token_type_ids"),io=o("\uFF0C\u7A0D\u540E\u6211\u4EEC\u5C06\u89E3\u91CA "),Jt=n("code"),no=o("attention_mask"),ao=o(" \u952E\u3002\u9996\u5148\uFF0C\u8BA9\u6211\u4EEC\u770B\u770B "),Rt=n("code"),po=o("input_ids"),fo=o(" \u5982\u4F55\u751F\u6210\u3002\u4E3A\u6B64\uFF0C\u6211\u4EEC\u9700\u8981\u67E5\u770B\u6807\u8BB0\u5668(tokenizer)\u7684\u4E2D\u95F4\u65B9\u6CD5\u3002"),rl=f(),qe=n("h2"),mo=o("\u7F16\u7801"),ol=f(),b(Me.$$.fragment),il=f(),re=n("p"),uo=o("\u5C06\u6587\u672C\u7FFB\u8BD1\u6210\u6570\u5B57\u88AB\u79F0\u4E3A\u7F16\u7801("),Wt=n("em"),ko=o("encoding"),co=o(").\u7F16\u7801\u5206\u4E24\u6B65\u5B8C\u6210\uFF1A\u6807\u8BB0\u5316\uFF0C\u7136\u540E\u8F6C\u6362\u4E3A\u8F93\u5165 ID\u3002"),nl=f(),oe=n("p"),_o=o("\u6B63\u5982\u6211\u4EEC\u6240\u89C1\uFF0C\u7B2C\u4E00\u6B65\u662F\u5C06\u6587\u672C\u62C6\u5206\u4E3A\u5355\u8BCD\uFF08\u6216\u5355\u8BCD\u7684\u4E00\u90E8\u5206\u3001\u6807\u70B9\u7B26\u53F7\u7B49\uFF09\uFF0C\u901A\u5E38\u79F0\u4E3A"),Kt=n("em"),$o=o("\u6807\u8BB0(token)"),vo=o("\u3002\u6709\u591A\u4E2A\u89C4\u5219\u53EF\u4EE5\u7BA1\u7406\u8BE5\u8FC7\u7A0B\uFF0C\u8FD9\u5C31\u662F\u4E3A\u4EC0\u4E48\u6211\u4EEC\u9700\u8981\u4F7F\u7528\u6A21\u578B\u540D\u79F0\u6765\u5B9E\u4F8B\u5316\u6807\u8BB0\u5668(tokenizer)\uFF0C\u4EE5\u786E\u4FDD\u6211\u4EEC\u4F7F\u7528\u6A21\u578B\u9884\u8BAD\u7EC3\u65F6\u4F7F\u7528\u7684\u76F8\u540C\u89C4\u5219\u3002"),al=f(),O=n("p"),bo=o("\u7B2C\u4E8C\u6B65\u662F\u5C06\u8FD9\u4E9B\u6807\u8BB0\u8F6C\u6362\u4E3A\u6570\u5B57\uFF0C\u8FD9\u6837\u6211\u4EEC\u5C31\u53EF\u4EE5\u7528\u5B83\u4EEC\u6784\u5EFA\u4E00\u4E2A\u5F20\u91CF\u5E76\u5C06\u5B83\u4EEC\u63D0\u4F9B\u7ED9\u6A21\u578B\u3002\u4E3A\u6B64\uFF0C\u6807\u8BB0\u5668(tokenizer)\u6709\u4E00\u4E2A"),Vt=n("em"),Eo=o("\u8BCD\u6C47(vocabulary)"),ho=o("\uFF0C\u8FD9\u662F\u6211\u4EEC\u5728\u5B9E\u4F8B\u5316\u5B83\u65F6\u4E0B\u8F7D\u7684\u90E8\u5206 "),Yt=n("code"),zo=o("from_pretrained()"),wo=o(" \u65B9\u6CD5\u3002\u540C\u6837\uFF0C\u6211\u4EEC\u9700\u8981\u4F7F\u7528\u6A21\u578B\u9884\u8BAD\u7EC3\u65F6\u4F7F\u7528\u7684\u76F8\u540C\u8BCD\u6C47\u3002"),pl=f(),vt=n("p"),jo=o("\u4E3A\u4E86\u66F4\u597D\u5730\u7406\u89E3\u8FD9\u4E24\u4E2A\u6B65\u9AA4\uFF0C\u6211\u4EEC\u5C06\u5206\u522B\u63A2\u8BA8\u5B83\u4EEC\u3002\u8BF7\u6CE8\u610F\uFF0C\u6211\u4EEC\u5C06\u4F7F\u7528\u4E00\u4E9B\u5355\u72EC\u6267\u884C\u90E8\u5206\u6807\u8BB0\u5316\u7BA1\u9053\u7684\u65B9\u6CD5\u6765\u5411\u60A8\u5C55\u793A\u8FD9\u4E9B\u6B65\u9AA4\u7684\u4E2D\u95F4\u7ED3\u679C\uFF0C\u4F46\u5B9E\u9645\u4E0A\uFF0C\u60A8\u5E94\u8BE5\u76F4\u63A5\u5728\u60A8\u7684\u8F93\u5165\u4E0A\u8C03\u7528\u6807\u8BB0\u5668(tokenizer)\uFF08\u5982\u7B2C 2 \u90E8\u5206\u6240\u793A\uFF09\u3002"),fl=f(),He=n("h3"),yo=o("\u6807\u8BB0\u5316"),ml=f(),ie=n("p"),Po=o("\u6807\u8BB0\u5316\u8FC7\u7A0B\u7531\u6807\u8BB0\u5668(tokenizer)\u7684"),Qt=n("code"),xo=o("tokenize()"),Do=o(" \u65B9\u6CD5\u5B9E\u73B0\uFF1A"),ul=f(),b(Oe.$$.fragment),kl=f(),bt=n("p"),To=o("\u6B64\u65B9\u6CD5\u7684\u8F93\u51FA\u662F\u4E00\u4E2A\u5B57\u7B26\u4E32\u5217\u8868\u6216\u6807\u8BB0(token)\uFF1A"),cl=f(),b(Be.$$.fragment),_l=f(),g=n("p"),Io=o("\u8FD9\u4E2A\u6807\u8BB0\u5668(tokenizer)\u662F\u4E00\u4E2A\u5B50\u8BCD\u6807\u8BB0\u5668(tokenizer)\uFF1A\u5B83\u5BF9\u8BCD\u8FDB\u884C\u62C6\u5206\uFF0C\u76F4\u5230\u83B7\u5F97\u53EF\u4EE5\u7528\u5176\u8BCD\u6C47\u8868\u8868\u793A\u7684\u6807\u8BB0(token)\u3002"),Xt=n("code"),Ao=o("transformer"),Co=o(" \u5C31\u662F\u8FD9\u79CD\u60C5\u51B5\uFF0C\u5B83\u5206\u4E3A\u4E24\u4E2A\u6807\u8BB0\uFF1A"),Zt=n("code"),go=o("transform"),No=o(" \u548C "),es=n("code"),qo=o("##er"),Mo=o("\u3002"),$l=f(),W=n("h3"),ne=n("a"),ts=n("span"),b(Le.$$.fragment),Ho=f(),ss=n("span"),Oo=o("\u4ECE\u8BCD\u7B26(token)\u5230\u8F93\u5165 ID"),vl=o(`

\u8F93\u5165 ID \u7684\u8F6C\u6362\u7531\u6807\u8BB0\u5668(tokenizer)\u7684\`convert_tokens_to_ids()\`\u65B9\u6CD5\u5B9E\u73B0\uFF1A

	`),b(Se.$$.fragment),bl=f(),b(Ue.$$.fragment),dl=f(),dt=n("p"),Bo=o("\u8FD9\u4E9B\u8F93\u51FA\u4E00\u65E6\u8F6C\u6362\u4E3A\u9002\u5F53\u7684\u6846\u67B6\u5F20\u91CF\uFF0C\u5C31\u53EF\u4EE5\u7528\u4F5C\u6A21\u578B\u7684\u8F93\u5165\uFF0C\u5982\u672C\u7AE0\u524D\u9762\u6240\u89C1\u3002"),El=f(),b(ae.$$.fragment),hl=f(),Ge=n("h2"),Lo=o("\u89E3\u7801"),zl=f(),K=n("p"),ls=n("em"),So=o("\u89E3\u7801(Decoding)"),Uo=o(" \u6B63\u597D\u76F8\u53CD\uFF1A\u4ECE\u8BCD\u6C47\u7D22\u5F15\u4E2D\uFF0C\u6211\u4EEC\u60F3\u8981\u5F97\u5230\u4E00\u4E2A\u5B57\u7B26\u4E32\u3002\u8FD9\u53EF\u4EE5\u901A\u8FC7  "),rs=n("code"),Go=o("decode()"),Fo=o(" \u65B9\u6CD5\u5B9E\u73B0\uFF0C\u5982\u4E0B\uFF1A"),wl=f(),b(Fe.$$.fragment),jl=f(),b(Je.$$.fragment),yl=f(),pe=n("p"),Jo=o("\u8BF7\u6CE8\u610F\uFF0C "),os=n("code"),Ro=o("decode"),Wo=o(" \u65B9\u6CD5\u4E0D\u4EC5\u5C06\u7D22\u5F15\u8F6C\u6362\u56DE\u6807\u8BB0(token)\uFF0C\u8FD8\u5C06\u5C5E\u4E8E\u76F8\u540C\u5355\u8BCD\u7684\u6807\u8BB0(token)\u7EC4\u5408\u5728\u4E00\u8D77\u4EE5\u751F\u6210\u53EF\u8BFB\u7684\u53E5\u5B50\u3002\u5F53\u6211\u4EEC\u4F7F\u7528\u9884\u6D4B\u65B0\u6587\u672C\u7684\u6A21\u578B\uFF08\u6839\u636E\u63D0\u793A\u751F\u6210\u7684\u6587\u672C\uFF0C\u6216\u5E8F\u5217\u5230\u5E8F\u5217\u95EE\u9898\uFF08\u5982\u7FFB\u8BD1\u6216\u6458\u8981\uFF09\uFF09\u65F6\uFF0C\u8FD9\u79CD\u884C\u4E3A\u5C06\u975E\u5E38\u6709\u7528\u3002"),Pl=f(),Et=n("p"),Ko=o("\u5230\u73B0\u5728\u4E3A\u6B62\uFF0C\u60A8\u5E94\u8BE5\u4E86\u89E3\u6807\u8BB0\u5668(tokenizer)\u53EF\u4EE5\u5904\u7406\u7684\u539F\u5B50\u64CD\u4F5C\uFF1A\u6807\u8BB0\u5316\u3001\u8F6C\u6362\u4E3A ID \u4EE5\u53CA\u5C06 ID \u8F6C\u6362\u56DE\u5B57\u7B26\u4E32\u3002\u7136\u800C\uFF0C\u6211\u4EEC\u53EA\u662F\u522E\u5230\u4E86\u51B0\u5C71\u4E00\u89D2\u3002\u5728\u4E0B\u4E00\u8282\u4E2D\uFF0C\u6211\u4EEC\u5C06\u91C7\u7528\u6211\u4EEC\u7684\u65B9\u6CD5\u6765\u514B\u670D\u5B83\u7684\u9650\u5236\uFF0C\u5E76\u770B\u770B\u5982\u4F55\u514B\u670D\u5B83\u4EEC\u3002"),this.h()},l(e){const l=Pn('[data-svelte="svelte-1phssyn"]',document.head);u=a(l,"META",{name:!0,content:!0}),l.forEach(t),v=m(e),d(k.$$.fragment,e),j=m(e),y=a(e,"H1",{class:!0});var We=p(y);z=a(We,"A",{id:!0,class:!0,href:!0});var ht=p(z);P=a(ht,"SPAN",{});var is=p(P);d(x.$$.fragment,is),is.forEach(t),ht.forEach(t),D=m(We),w=a(We,"SPAN",{});var oi=p(w);S=i(oi,"\u6807\u8BB0\u5668\uFF08Tokenizer\uFF09"),oi.forEach(t),We.forEach(t),M=m(e),q.l(e),Ke=m(e),d(me.$$.fragment,e),ps=m(e),Ve=a(e,"P",{});var ii=p(Ve);Rl=i(ii,"\u6807\u8BB0\u5668(Tokenizer)\u662F NLP \u7BA1\u9053\u7684\u6838\u5FC3\u7EC4\u4EF6\u4E4B\u4E00\u3002\u5B83\u4EEC\u6709\u4E00\u4E2A\u76EE\u7684\uFF1A\u5C06\u6587\u672C\u8F6C\u6362\u4E3A\u6A21\u578B\u53EF\u4EE5\u5904\u7406\u7684\u6570\u636E\u3002\u6A21\u578B\u53EA\u80FD\u5904\u7406\u6570\u5B57\uFF0C\u56E0\u6B64\u6807\u8BB0\u5668(Tokenizer)\u9700\u8981\u5C06\u6211\u4EEC\u7684\u6587\u672C\u8F93\u5165\u8F6C\u6362\u4E3A\u6570\u5B57\u6570\u636E\u3002\u5728\u672C\u8282\u4E2D\uFF0C\u6211\u4EEC\u5C06\u786E\u5207\u5730\u63A2\u8BA8\u6807\u8BB0\u5316\u7BA1\u9053\u4E2D\u53D1\u751F\u7684\u4E8B\u60C5\u3002"),ii.forEach(t),fs=m(e),Ye=a(e,"P",{});var ni=p(Ye);Wl=i(ni,"\u5728 NLP \u4EFB\u52A1\u4E2D\uFF0C\u901A\u5E38\u5904\u7406\u7684\u6570\u636E\u662F\u539F\u59CB\u6587\u672C\u3002\u8FD9\u662F\u6B64\u7C7B\u6587\u672C\u7684\u793A\u4F8B"),ni.forEach(t),ms=m(e),d(ue.$$.fragment,e),us=m(e),Qe=a(e,"P",{});var ai=p(Qe);Kl=i(ai,"\u4F46\u662F\uFF0C\u6A21\u578B\u53EA\u80FD\u5904\u7406\u6570\u5B57\uFF0C\u56E0\u6B64\u6211\u4EEC\u9700\u8981\u627E\u5230\u4E00\u79CD\u5C06\u539F\u59CB\u6587\u672C\u8F6C\u6362\u4E3A\u6570\u5B57\u7684\u65B9\u6CD5\u3002\u8FD9\u5C31\u662F\u6807\u8BB0\u5668\uFF08tokenizer\uFF09\u6240\u505A\u7684\uFF0C\u5E76\u4E14\u6709\u5F88\u591A\u65B9\u6CD5\u53EF\u4EE5\u89E3\u51B3\u8FD9\u4E2A\u95EE\u9898\u3002\u76EE\u6807\u662F\u627E\u5230\u6700\u6709\u610F\u4E49\u7684\u8868\u793A\u2014\u2014\u5373\u5BF9\u6A21\u578B\u6700\u6709\u610F\u4E49\u7684\u8868\u793A\u2014\u2014\u5E76\u4E14\u5982\u679C\u53EF\u80FD\u7684\u8BDD\uFF0C\u627E\u5230\u6700\u5C0F\u7684\u8868\u793A\u3002"),ai.forEach(t),ks=m(e),Xe=a(e,"P",{});var pi=p(Xe);Vl=i(pi,"\u8BA9\u6211\u4EEC\u770B\u4E00\u4E0B\u6807\u8BB0\u5316\u7B97\u6CD5\u7684\u4E00\u4E9B\u793A\u4F8B\uFF0C\u5E76\u5C1D\u8BD5\u56DE\u7B54\u60A8\u53EF\u80FD\u5BF9\u6807\u8BB0\u5316\u63D0\u51FA\u7684\u4E00\u4E9B\u95EE\u9898\u3002"),pi.forEach(t),cs=m(e),U=a(e,"H2",{class:!0});var Tl=p(U);Y=a(Tl,"A",{id:!0,class:!0,href:!0});var fi=p(Y);yt=a(fi,"SPAN",{});var mi=p(yt);d(ke.$$.fragment,mi),mi.forEach(t),fi.forEach(t),Yl=m(Tl),Pt=a(Tl,"SPAN",{});var ui=p(Pt);Ql=i(ui,"\u57FA\u4E8E\u8BCD\u7684(Word-based)"),ui.forEach(t),Tl.forEach(t),_s=m(e),d(ce.$$.fragment,e),$s=m(e),Q=a(e,"P",{});var Il=p(Q);Xl=i(Il,"\u60F3\u5230\u7684\u7B2C\u4E00\u79CD\u6807\u8BB0\u5668\u662F\u57FA\u4E8E\u8BCD\u7684("),xt=a(Il,"EM",{});var ki=p(xt);Zl=i(ki,"word-based"),ki.forEach(t),er=i(Il,").\u5B83\u901A\u5E38\u5F88\u5BB9\u6613\u8BBE\u7F6E\u548C\u4F7F\u7528\uFF0C\u53EA\u9700\u51E0\u6761\u89C4\u5219\uFF0C\u5E76\u4E14\u901A\u5E38\u4F1A\u4EA7\u751F\u4E0D\u9519\u7684\u7ED3\u679C\u3002\u4F8B\u5982\uFF0C\u5728\u4E0B\u56FE\u4E2D\uFF0C\u76EE\u6807\u662F\u5C06\u539F\u59CB\u6587\u672C\u62C6\u5206\u4E3A\u5355\u8BCD\u5E76\u4E3A\u6BCF\u4E2A\u5355\u8BCD\u627E\u5230\u4E00\u4E2A\u6570\u5B57\u8868\u793A\uFF1A"),Il.forEach(t),vs=m(e),G=a(e,"DIV",{class:!0});var Al=p(G);_e=a(Al,"IMG",{class:!0,src:!0,alt:!0}),tr=m(Al),$e=a(Al,"IMG",{class:!0,src:!0,alt:!0}),Al.forEach(t),bs=m(e),X=a(e,"P",{});var Cl=p(X);sr=i(Cl,"\u6709\u591A\u79CD\u65B9\u6CD5\u53EF\u4EE5\u62C6\u5206\u6587\u672C\u3002\u4F8B\u5982\uFF0C\u6211\u4EEC\u53EF\u4EE5\u901A\u8FC7\u5E94\u7528Python\u7684"),Dt=a(Cl,"CODE",{});var ci=p(Dt);lr=i(ci,"split()"),ci.forEach(t),rr=i(Cl,"\u51FD\u6570\uFF0C\u4F7F\u7528\u7A7A\u683C\u5C06\u6587\u672C\u6807\u8BB0\u4E3A\u5355\u8BCD\uFF1A"),Cl.forEach(t),ds=m(e),d(ve.$$.fragment,e),Es=m(e),d(be.$$.fragment,e),hs=m(e),Ze=a(e,"P",{});var _i=p(Ze);or=i(_i,"\u8FD8\u6709\u4E00\u4E9B\u5355\u8BCD\u6807\u8BB0\u5668\u7684\u53D8\u4F53\uFF0C\u5B83\u4EEC\u5177\u6709\u989D\u5916\u7684\u6807\u70B9\u7B26\u53F7\u89C4\u5219\u3002\u4F7F\u7528\u8FD9\u79CD\u6807\u8BB0\u5668\uFF0C\u6211\u4EEC\u6700\u7EC8\u53EF\u4EE5\u5F97\u5230\u4E00\u4E9B\u975E\u5E38\u5927\u7684\u201C\u8BCD\u6C47\u8868\u201D\uFF0C\u5176\u4E2D\u8BCD\u6C47\u8868\u7531\u6211\u4EEC\u5728\u8BED\u6599\u5E93\u4E2D\u62E5\u6709\u7684\u72EC\u7ACB\u6807\u8BB0\u7684\u603B\u6570\u5B9A\u4E49\u3002"),_i.forEach(t),zs=m(e),et=a(e,"P",{});var $i=p(et);ir=i($i,"\u6BCF\u4E2A\u5355\u8BCD\u90FD\u5206\u914D\u4E86\u4E00\u4E2A ID\uFF0C\u4ECE 0 \u5F00\u59CB\u4E00\u76F4\u5230\u8BCD\u6C47\u8868\u7684\u5927\u5C0F\u3002\u8BE5\u6A21\u578B\u4F7F\u7528\u8FD9\u4E9B ID \u6765\u8BC6\u522B\u6BCF\u4E2A\u5355\u8BCD\u3002"),$i.forEach(t),ws=m(e),tt=a(e,"P",{});var vi=p(tt);nr=i(vi,"\u5982\u679C\u6211\u4EEC\u60F3\u7528\u57FA\u4E8E\u5355\u8BCD\u7684\u6807\u8BB0\u5668(tokenizer)\u5B8C\u5168\u8986\u76D6\u4E00\u79CD\u8BED\u8A00\uFF0C\u6211\u4EEC\u9700\u8981\u4E3A\u8BED\u8A00\u4E2D\u7684\u6BCF\u4E2A\u5355\u8BCD\u90FD\u6709\u4E00\u4E2A\u6807\u8BC6\u7B26\uFF0C\u8FD9\u5C06\u751F\u6210\u5927\u91CF\u7684\u6807\u8BB0\u3002\u4F8B\u5982\uFF0C\u82F1\u8BED\u4E2D\u6709\u8D85\u8FC7 500,000 \u4E2A\u5355\u8BCD\uFF0C\u56E0\u6B64\u8981\u6784\u5EFA\u4ECE\u6BCF\u4E2A\u5355\u8BCD\u5230\u8F93\u5165 ID \u7684\u6620\u5C04\uFF0C\u6211\u4EEC\u9700\u8981\u8DDF\u8E2A\u8FD9\u4E48\u591A ID\u3002\u6B64\u5916\uFF0C\u50CF\u201Cdog\u201D\u8FD9\u6837\u7684\u8BCD\u4E0E\u201Cdogs\u201D\u8FD9\u6837\u7684\u8BCD\u7684\u8868\u793A\u65B9\u5F0F\u4E0D\u540C\uFF0C\u6A21\u578B\u6700\u521D\u65E0\u6CD5\u77E5\u9053\u201Cdog\u201D\u548C\u201Cdogs\u201D\u662F\u76F8\u4F3C\u7684\uFF1A\u5B83\u4F1A\u5C06\u8FD9\u4E24\u4E2A\u8BCD\u8BC6\u522B\u4E3A\u4E0D\u76F8\u5173\u3002\u8FD9\u540C\u6837\u9002\u7528\u4E8E\u5176\u4ED6\u76F8\u4F3C\u7684\u8BCD\uFF0C\u4F8B\u5982\u201Crun\u201D\u548C\u201Crunning\u201D\uFF0C\u6A21\u578B\u6700\u521D\u4E0D\u4F1A\u8BA4\u4E3A\u5B83\u4EEC\u662F\u76F8\u4F3C\u7684\u3002"),vi.forEach(t),js=m(e),de=a(e,"P",{});var Vo=p(de);ar=i(Vo,"\u6700\u540E\uFF0C\u6211\u4EEC\u9700\u8981\u4E00\u4E2A\u81EA\u5B9A\u4E49\u6807\u8BB0(token)\u6765\u8868\u793A\u4E0D\u5728\u6211\u4EEC\u8BCD\u6C47\u8868\u4E2D\u7684\u5355\u8BCD\u3002\u8FD9\u88AB\u79F0\u4E3A\u201C\u672A\u77E5\u201D\u6807\u8BB0(token)\uFF0C\u901A\u5E38\u8868\u793A\u4E3A\u201C[UNK]\u201D\u6216\u201D"),Tt=a(Vo,"UNK",{});var bi=p(Tt);pr=i(bi,"\u201C\u3002\u5982\u679C\u4F60\u770B\u5230\u6807\u8BB0\u5668\u4EA7\u751F\u4E86\u5F88\u591A\u8FD9\u6837\u7684\u6807\u8BB0\uFF0C\u8FD9\u901A\u5E38\u662F\u4E00\u4E2A\u4E0D\u597D\u7684\u8FF9\u8C61\uFF0C\u56E0\u4E3A\u5B83\u65E0\u6CD5\u68C0\u7D22\u5230\u4E00\u4E2A\u8BCD\u7684\u5408\u7406\u8868\u793A\uFF0C\u5E76\u4E14\u4F60\u4F1A\u5728\u8FD9\u4E2A\u8FC7\u7A0B\u4E2D\u4E22\u5931\u4FE1\u606F\u3002\u5236\u4F5C\u8BCD\u6C47\u8868\u65F6\u7684\u76EE\u6807\u662F\u4EE5\u8FD9\u6837\u4E00\u79CD\u65B9\u5F0F\u8FDB\u884C\uFF0C\u5373\u6807\u8BB0\u5668\u5C06\u5C3D\u53EF\u80FD\u5C11\u7684\u5355\u8BCD\u6807\u8BB0\u4E3A\u672A\u77E5\u6807\u8BB0\u3002"),bi.forEach(t),Vo.forEach(t),ys=m(e),Z=a(e,"P",{});var gl=p(Z);fr=i(gl,"\u51CF\u5C11\u672A\u77E5\u6807\u8BB0\u6570\u91CF\u7684\u4E00\u79CD\u65B9\u6CD5\u662F\u4F7F\u7528\u66F4\u6DF1\u4E00\u5C42\u7684\u6807\u8BB0\u5668(tokenizer)\uFF0C\u5373\u57FA\u4E8E\u5B57\u7B26\u7684("),It=a(gl,"EM",{});var di=p(It);mr=i(di,"character-based"),di.forEach(t),ur=i(gl,")\u6807\u8BB0\u5668(tokenizer)\u3002"),gl.forEach(t),Ps=m(e),F=a(e,"H2",{class:!0});var Nl=p(F);ee=a(Nl,"A",{id:!0,class:!0,href:!0});var Ei=p(ee);At=a(Ei,"SPAN",{});var hi=p(At);d(Ee.$$.fragment,hi),hi.forEach(t),Ei.forEach(t),kr=m(Nl),Ct=a(Nl,"SPAN",{});var zi=p(Ct);cr=i(zi,"\u57FA\u4E8E\u5B57\u7B26(Character-based)"),zi.forEach(t),Nl.forEach(t),xs=m(e),d(he.$$.fragment,e),Ds=m(e),st=a(e,"P",{});var wi=p(st);_r=i(wi,"\u57FA\u4E8E\u5B57\u7B26\u7684\u6807\u8BB0\u5668(tokenizer)\u5C06\u6587\u672C\u62C6\u5206\u4E3A\u5B57\u7B26\uFF0C\u800C\u4E0D\u662F\u5355\u8BCD\u3002\u8FD9\u6709\u4E24\u4E2A\u4E3B\u8981\u597D\u5904\uFF1A"),wi.forEach(t),Ts=m(e),te=a(e,"UL",{});var ql=p(te);gt=a(ql,"LI",{});var ji=p(gt);$r=i(ji,"\u8BCD\u6C47\u91CF\u8981\u5C0F\u5F97\u591A\u3002"),ji.forEach(t),vr=m(ql),Nt=a(ql,"LI",{});var yi=p(Nt);br=i(yi,"\u8BCD\u6C47\u5916\uFF08\u672A\u77E5\uFF09\u6807\u8BB0(token)\u8981\u5C11\u5F97\u591A\uFF0C\u56E0\u4E3A\u6BCF\u4E2A\u5355\u8BCD\u90FD\u53EF\u4EE5\u4ECE\u5B57\u7B26\u6784\u5EFA\u3002"),yi.forEach(t),ql.forEach(t),Is=m(e),lt=a(e,"P",{});var Pi=p(lt);dr=i(Pi,"\u4F46\u662F\u8FD9\u91CC\u4E5F\u51FA\u73B0\u4E86\u4E00\u4E9B\u5173\u4E8E\u7A7A\u683C\u548C\u6807\u70B9\u7B26\u53F7\u7684\u95EE\u9898\uFF1A"),Pi.forEach(t),As=m(e),J=a(e,"DIV",{class:!0});var Ml=p(J);ze=a(Ml,"IMG",{class:!0,src:!0,alt:!0}),Er=m(Ml),we=a(Ml,"IMG",{class:!0,src:!0,alt:!0}),Ml.forEach(t),Cs=m(e),rt=a(e,"P",{});var xi=p(rt);hr=i(xi,"\u8FD9\u79CD\u65B9\u6CD5\u4E5F\u4E0D\u662F\u5B8C\u7F8E\u7684\u3002\u7531\u4E8E\u73B0\u5728\u8868\u793A\u662F\u57FA\u4E8E\u5B57\u7B26\u800C\u4E0D\u662F\u5355\u8BCD\uFF0C\u56E0\u6B64\u4EBA\u4EEC\u53EF\u80FD\u4F1A\u4E89\u8FA9\u8BF4\uFF0C\u4ECE\u76F4\u89C9\u4E0A\u8BB2\uFF0C\u5B83\u7684\u610F\u4E49\u4E0D\u5927\uFF1A\u6BCF\u4E2A\u5B57\u7B26\u672C\u8EAB\u5E76\u6CA1\u6709\u591A\u5927\u610F\u4E49\uFF0C\u800C\u5355\u8BCD\u5C31\u662F\u8FD9\u79CD\u60C5\u51B5\u3002\u7136\u800C\uFF0C\u8FD9\u53C8\u56E0\u8BED\u8A00\u800C\u5F02\uFF1B\u4F8B\u5982\uFF0C\u5728\u4E2D\u6587\u4E2D\uFF0C\u6BCF\u4E2A\u5B57\u7B26\u6BD4\u62C9\u4E01\u8BED\u8A00\u4E2D\u7684\u5B57\u7B26\u5305\u542B\u66F4\u591A\u7684\u4FE1\u606F\u3002"),xi.forEach(t),gs=m(e),ot=a(e,"P",{});var Di=p(ot);zr=i(Di,"\u53E6\u4E00\u4EF6\u8981\u8003\u8651\u7684\u4E8B\u60C5\u662F\uFF0C\u6211\u4EEC\u7684\u6A21\u578B\u6700\u7EC8\u4F1A\u5904\u7406\u5927\u91CF\u7684\u8BCD\u7B26(token)\uFF1A\u867D\u7136\u4F7F\u7528\u57FA\u4E8E\u5355\u8BCD\u7684\u6807\u8BB0\u5668(tokenizer)\uFF0C\u5355\u8BCD\u53EA\u4F1A\u662F\u5355\u4E2A\u6807\u8BB0\uFF0C\u4F46\u5F53\u8F6C\u6362\u4E3A\u5B57\u7B26\u65F6\uFF0C\u5B83\u5F88\u5BB9\u6613\u53D8\u6210 10 \u4E2A\u6216\u66F4\u591A\u7684\u8BCD\u7B26(token)\u3002"),Di.forEach(t),Ns=m(e),se=a(e,"P",{});var Hl=p(se);wr=i(Hl,"\u4E3A\u4E86\u4E24\u5168\u5176\u7F8E\uFF0C\u6211\u4EEC\u53EF\u4EE5\u4F7F\u7528\u7ED3\u5408\u8FD9\u4E24\u79CD\u65B9\u6CD5\u7684\u7B2C\u4E09\u79CD\u6280\u672F\uFF1A"),qt=a(Hl,"EM",{});var Ti=p(qt);jr=i(Ti,"\u5B50\u8BCD\u6807\u8BB0\u5316(subword tokenization)"),Ti.forEach(t),yr=i(Hl,"\u3002"),Hl.forEach(t),qs=m(e),je=a(e,"H2",{id:!0});var Ii=p(je);Pr=i(Ii,"\u5B50\u8BCD\u6807\u8BB0\u5316"),Ii.forEach(t),Ms=m(e),d(ye.$$.fragment,e),Hs=m(e),it=a(e,"P",{});var Ai=p(it);xr=i(Ai,"\u5B50\u8BCD\u5206\u8BCD\u7B97\u6CD5\u4F9D\u8D56\u4E8E\u8FD9\u6837\u4E00\u4E2A\u539F\u5219\uFF0C\u5373\u4E0D\u5E94\u5C06\u5E38\u7528\u8BCD\u62C6\u5206\u4E3A\u66F4\u5C0F\u7684\u5B50\u8BCD\uFF0C\u800C\u5E94\u5C06\u7A00\u6709\u8BCD\u5206\u89E3\u4E3A\u6709\u610F\u4E49\u7684\u5B50\u8BCD\u3002"),Ai.forEach(t),Os=m(e),nt=a(e,"P",{});var Ci=p(nt);Dr=i(Ci,"\u4F8B\u5982\uFF0C\u201Cannoyingly\u201D\u53EF\u80FD\u88AB\u8BA4\u4E3A\u662F\u4E00\u4E2A\u7F55\u89C1\u7684\u8BCD\uFF0C\u53EF\u4EE5\u5206\u89E3\u4E3A\u201Cannoying\u201D\u548C\u201Cly\u201D\u3002\u8FD9\u4E24\u8005\u90FD\u53EF\u80FD\u4F5C\u4E3A\u72EC\u7ACB\u7684\u5B50\u8BCD\u51FA\u73B0\u5F97\u66F4\u9891\u7E41\uFF0C\u540C\u65F6\u201Cannoyingly\u201D\u7684\u542B\u4E49\u7531\u201Cannoying\u201D\u548C\u201Cly\u201D\u7684\u590D\u5408\u542B\u4E49\u4FDD\u6301\u3002"),Ci.forEach(t),Bs=m(e),at=a(e,"P",{});var gi=p(at);Tr=i(gi,"\u8FD9\u662F\u4E00\u4E2A\u793A\u4F8B\uFF0C\u5C55\u793A\u4E86\u5B50\u8BCD\u6807\u8BB0\u5316\u7B97\u6CD5\u5982\u4F55\u6807\u8BB0\u5E8F\u5217\u201CLet\u2019s do tokenization!\u201D\uFF1A"),gi.forEach(t),Ls=m(e),R=a(e,"DIV",{class:!0});var Ol=p(R);Pe=a(Ol,"IMG",{class:!0,src:!0,alt:!0}),Ir=m(Ol),xe=a(Ol,"IMG",{class:!0,src:!0,alt:!0}),Ol.forEach(t),Ss=m(e),pt=a(e,"P",{});var Ni=p(pt);Ar=i(Ni,"\u8FD9\u4E9B\u5B50\u8BCD\u6700\u7EC8\u63D0\u4F9B\u4E86\u5F88\u591A\u8BED\u4E49\u542B\u4E49\uFF1A\u4F8B\u5982\uFF0C\u5728\u4E0A\u9762\u7684\u793A\u4F8B\u4E2D\uFF0C\u201Ctokenization\u201D\u88AB\u62C6\u5206\u4E3A\u201Ctoken\u201D\u548C\u201Cization\u201D\uFF0C\u8FD9\u4E24\u4E2A\u5177\u6709\u8BED\u4E49\u610F\u4E49\u540C\u65F6\u8282\u7701\u7A7A\u95F4\u7684\u8BCD\u7B26(token)\uFF08\u53EA\u9700\u8981\u4E24\u4E2A\u6807\u8BB0(token)\u4EE3\u8868\u4E00\u4E2A\u957F\u8BCD\uFF09\u3002\u8FD9\u4F7F\u6211\u4EEC\u80FD\u591F\u5BF9\u8F83\u5C0F\u7684\u8BCD\u6C47\u8868\u8FDB\u884C\u76F8\u5BF9\u8F83\u597D\u7684\u8986\u76D6\uFF0C\u5E76\u4E14\u51E0\u4E4E\u6CA1\u6709\u672A\u77E5\u7684\u6807\u8BB0"),Ni.forEach(t),Us=m(e),ft=a(e,"P",{});var qi=p(ft);Cr=i(qi,"\u8FD9\u79CD\u65B9\u6CD5\u5728\u571F\u8033\u5176\u8BED\u7B49\u7C98\u7740\u578B\u8BED\u8A00(agglutinative languages)\u4E2D\u7279\u522B\u6709\u7528\uFF0C\u60A8\u53EF\u4EE5\u901A\u8FC7\u5C06\u5B50\u8BCD\u4E32\u5728\u4E00\u8D77\u6765\u5F62\u6210\uFF08\u51E0\u4E4E\uFF09\u4EFB\u610F\u957F\u7684\u590D\u6742\u8BCD\u3002"),qi.forEach(t),Gs=m(e),De=a(e,"H3",{id:!0});var Mi=p(De);gr=i(Mi,"\u8FD8\u6709\u66F4\u591A\uFF01"),Mi.forEach(t),Fs=m(e),mt=a(e,"P",{});var Hi=p(mt);Nr=i(Hi,"\u4E0D\u51FA\u6240\u6599\uFF0C\u8FD8\u6709\u66F4\u591A\u7684\u6280\u672F\u3002\u4EC5\u4E3E\u51E0\u4F8B\uFF1A"),Hi.forEach(t),Js=m(e),H=a(e,"UL",{});var zt=p(H);Mt=a(zt,"LI",{});var Oi=p(Mt);qr=i(Oi,"Byte-level BPE, \u7528\u4E8E GPT-2"),Oi.forEach(t),Mr=m(zt),Ht=a(zt,"LI",{});var Bi=p(Ht);Hr=i(Bi,"WordPiece, \u7528\u4E8E BERT"),Bi.forEach(t),Or=m(zt),Ot=a(zt,"LI",{});var Li=p(Ot);Br=i(Li,"SentencePiece or Unigram, \u7528\u4E8E\u591A\u4E2A\u591A\u8BED\u8A00\u6A21\u578B"),Li.forEach(t),zt.forEach(t),Rs=m(e),ut=a(e,"P",{});var Si=p(ut);Lr=i(Si,"\u60A8\u73B0\u5728\u5E94\u8BE5\u5BF9\u6807\u8BB0\u5668(tokenizers)\u7684\u5DE5\u4F5C\u539F\u7406\u6709\u8DB3\u591F\u7684\u4E86\u89E3\uFF0C\u4EE5\u4FBF\u5F00\u59CB\u4F7F\u7528 API\u3002"),Si.forEach(t),Ws=m(e),Te=a(e,"H2",{id:!0});var Ui=p(Te);Sr=i(Ui,"\u52A0\u8F7D\u548C\u4FDD\u5B58"),Ui.forEach(t),Ks=m(e),T=a(e,"P",{});var B=p(T);Ur=i(B,"\u52A0\u8F7D\u548C\u4FDD\u5B58\u6807\u8BB0\u5668(tokenizer)\u5C31\u50CF\u4F7F\u7528\u6A21\u578B\u4E00\u6837\u7B80\u5355\u3002\u5B9E\u9645\u4E0A\uFF0C\u5B83\u57FA\u4E8E\u76F8\u540C\u7684\u4E24\u79CD\u65B9\u6CD5\uFF1A "),Bt=a(B,"CODE",{});var Gi=p(Bt);Gr=i(Gi,"from_pretrained()"),Gi.forEach(t),Fr=i(B," \u548C "),Lt=a(B,"CODE",{});var Fi=p(Lt);Jr=i(Fi,"save_pretrained()"),Fi.forEach(t),Rr=i(B," \u3002\u8FD9\u4E9B\u65B9\u6CD5\u5C06\u52A0\u8F7D\u6216\u4FDD\u5B58\u6807\u8BB0\u5668(tokenizer)\u4F7F\u7528\u7684\u7B97\u6CD5\uFF08\u6709\u70B9\u50CF"),St=a(B,"EM",{});var Ji=p(St);Wr=i(Ji,"\u5EFA\u7B51\u5B66(architecture)"),Ji.forEach(t),Kr=i(B,"\u7684\u6A21\u578B\uFF09\u4EE5\u53CA\u5B83\u7684\u8BCD\u6C47\uFF08\u6709\u70B9\u50CF"),Ut=a(B,"EM",{});var Ri=p(Ut);Vr=i(Ri,"\u6743\u91CD(weights)"),Ri.forEach(t),Yr=i(B,"\u6A21\u578B\uFF09\u3002"),B.forEach(t),Vs=m(e),le=a(e,"P",{});var Bl=p(le);Qr=i(Bl,"\u52A0\u8F7D\u4F7F\u7528\u4E0E BERT \u76F8\u540C\u7684\u68C0\u67E5\u70B9\u8BAD\u7EC3\u7684 BERT \u6807\u8BB0\u5668(tokenizer)\u4E0E\u52A0\u8F7D\u6A21\u578B\u7684\u65B9\u5F0F\u76F8\u540C\uFF0C\u9664\u4E86\u6211\u4EEC\u4F7F\u7528  "),Gt=a(Bl,"CODE",{});var Wi=p(Gt);Xr=i(Wi,"BertTokenizer"),Wi.forEach(t),Zr=i(Bl," \u7C7B\uFF1A"),Bl.forEach(t),Ys=m(e),d(Ie.$$.fragment,e),Qs=m(e),V.l(e),kt=m(e),d(Ae.$$.fragment,e),Xs=m(e),ct=a(e,"P",{});var Ki=p(ct);eo=i(Ki,"\u6211\u4EEC\u73B0\u5728\u53EF\u4EE5\u4F7F\u7528\u6807\u8BB0\u5668(tokenizer)\uFF0C\u5982\u4E0A\u4E00\u8282\u6240\u793A\uFF1A"),Ki.forEach(t),Zs=m(e),d(Ce.$$.fragment,e),el=m(e),d(ge.$$.fragment,e),tl=m(e),_t=a(e,"P",{});var Vi=p(_t);to=i(Vi,"\u4FDD\u5B58\u6807\u8BB0\u5668(tokenizer)\u4E0E\u4FDD\u5B58\u6A21\u578B\u76F8\u540C:"),Vi.forEach(t),sl=m(e),d(Ne.$$.fragment,e),ll=m(e),I=a(e,"P",{});var L=p(I);so=i(L,"\u6211\u4EEC\u5728"),$t=a(L,"A",{href:!0});var Yi=p($t);lo=i(Yi,"Chapter 3"),Yi.forEach(t),ro=i(L,"\u4E2D\u5C06\u66F4\u591A\u5730\u8C08\u8BBA"),Ft=a(L,"CODE",{});var Qi=p(Ft);oo=i(Qi,"token_type_ids"),Qi.forEach(t),io=i(L,"\uFF0C\u7A0D\u540E\u6211\u4EEC\u5C06\u89E3\u91CA "),Jt=a(L,"CODE",{});var Xi=p(Jt);no=i(Xi,"attention_mask"),Xi.forEach(t),ao=i(L," \u952E\u3002\u9996\u5148\uFF0C\u8BA9\u6211\u4EEC\u770B\u770B "),Rt=a(L,"CODE",{});var Zi=p(Rt);po=i(Zi,"input_ids"),Zi.forEach(t),fo=i(L," \u5982\u4F55\u751F\u6210\u3002\u4E3A\u6B64\uFF0C\u6211\u4EEC\u9700\u8981\u67E5\u770B\u6807\u8BB0\u5668(tokenizer)\u7684\u4E2D\u95F4\u65B9\u6CD5\u3002"),L.forEach(t),rl=m(e),qe=a(e,"H2",{id:!0});var en=p(qe);mo=i(en,"\u7F16\u7801"),en.forEach(t),ol=m(e),d(Me.$$.fragment,e),il=m(e),re=a(e,"P",{});var Ll=p(re);uo=i(Ll,"\u5C06\u6587\u672C\u7FFB\u8BD1\u6210\u6570\u5B57\u88AB\u79F0\u4E3A\u7F16\u7801("),Wt=a(Ll,"EM",{});var tn=p(Wt);ko=i(tn,"encoding"),tn.forEach(t),co=i(Ll,").\u7F16\u7801\u5206\u4E24\u6B65\u5B8C\u6210\uFF1A\u6807\u8BB0\u5316\uFF0C\u7136\u540E\u8F6C\u6362\u4E3A\u8F93\u5165 ID\u3002"),Ll.forEach(t),nl=m(e),oe=a(e,"P",{});var Sl=p(oe);_o=i(Sl,"\u6B63\u5982\u6211\u4EEC\u6240\u89C1\uFF0C\u7B2C\u4E00\u6B65\u662F\u5C06\u6587\u672C\u62C6\u5206\u4E3A\u5355\u8BCD\uFF08\u6216\u5355\u8BCD\u7684\u4E00\u90E8\u5206\u3001\u6807\u70B9\u7B26\u53F7\u7B49\uFF09\uFF0C\u901A\u5E38\u79F0\u4E3A"),Kt=a(Sl,"EM",{});var sn=p(Kt);$o=i(sn,"\u6807\u8BB0(token)"),sn.forEach(t),vo=i(Sl,"\u3002\u6709\u591A\u4E2A\u89C4\u5219\u53EF\u4EE5\u7BA1\u7406\u8BE5\u8FC7\u7A0B\uFF0C\u8FD9\u5C31\u662F\u4E3A\u4EC0\u4E48\u6211\u4EEC\u9700\u8981\u4F7F\u7528\u6A21\u578B\u540D\u79F0\u6765\u5B9E\u4F8B\u5316\u6807\u8BB0\u5668(tokenizer)\uFF0C\u4EE5\u786E\u4FDD\u6211\u4EEC\u4F7F\u7528\u6A21\u578B\u9884\u8BAD\u7EC3\u65F6\u4F7F\u7528\u7684\u76F8\u540C\u89C4\u5219\u3002"),Sl.forEach(t),al=m(e),O=a(e,"P",{});var wt=p(O);bo=i(wt,"\u7B2C\u4E8C\u6B65\u662F\u5C06\u8FD9\u4E9B\u6807\u8BB0\u8F6C\u6362\u4E3A\u6570\u5B57\uFF0C\u8FD9\u6837\u6211\u4EEC\u5C31\u53EF\u4EE5\u7528\u5B83\u4EEC\u6784\u5EFA\u4E00\u4E2A\u5F20\u91CF\u5E76\u5C06\u5B83\u4EEC\u63D0\u4F9B\u7ED9\u6A21\u578B\u3002\u4E3A\u6B64\uFF0C\u6807\u8BB0\u5668(tokenizer)\u6709\u4E00\u4E2A"),Vt=a(wt,"EM",{});var ln=p(Vt);Eo=i(ln,"\u8BCD\u6C47(vocabulary)"),ln.forEach(t),ho=i(wt,"\uFF0C\u8FD9\u662F\u6211\u4EEC\u5728\u5B9E\u4F8B\u5316\u5B83\u65F6\u4E0B\u8F7D\u7684\u90E8\u5206 "),Yt=a(wt,"CODE",{});var rn=p(Yt);zo=i(rn,"from_pretrained()"),rn.forEach(t),wo=i(wt," \u65B9\u6CD5\u3002\u540C\u6837\uFF0C\u6211\u4EEC\u9700\u8981\u4F7F\u7528\u6A21\u578B\u9884\u8BAD\u7EC3\u65F6\u4F7F\u7528\u7684\u76F8\u540C\u8BCD\u6C47\u3002"),wt.forEach(t),pl=m(e),vt=a(e,"P",{});var on=p(vt);jo=i(on,"\u4E3A\u4E86\u66F4\u597D\u5730\u7406\u89E3\u8FD9\u4E24\u4E2A\u6B65\u9AA4\uFF0C\u6211\u4EEC\u5C06\u5206\u522B\u63A2\u8BA8\u5B83\u4EEC\u3002\u8BF7\u6CE8\u610F\uFF0C\u6211\u4EEC\u5C06\u4F7F\u7528\u4E00\u4E9B\u5355\u72EC\u6267\u884C\u90E8\u5206\u6807\u8BB0\u5316\u7BA1\u9053\u7684\u65B9\u6CD5\u6765\u5411\u60A8\u5C55\u793A\u8FD9\u4E9B\u6B65\u9AA4\u7684\u4E2D\u95F4\u7ED3\u679C\uFF0C\u4F46\u5B9E\u9645\u4E0A\uFF0C\u60A8\u5E94\u8BE5\u76F4\u63A5\u5728\u60A8\u7684\u8F93\u5165\u4E0A\u8C03\u7528\u6807\u8BB0\u5668(tokenizer)\uFF08\u5982\u7B2C 2 \u90E8\u5206\u6240\u793A\uFF09\u3002"),on.forEach(t),fl=m(e),He=a(e,"H3",{id:!0});var nn=p(He);yo=i(nn,"\u6807\u8BB0\u5316"),nn.forEach(t),ml=m(e),ie=a(e,"P",{});var Ul=p(ie);Po=i(Ul,"\u6807\u8BB0\u5316\u8FC7\u7A0B\u7531\u6807\u8BB0\u5668(tokenizer)\u7684"),Qt=a(Ul,"CODE",{});var an=p(Qt);xo=i(an,"tokenize()"),an.forEach(t),Do=i(Ul," \u65B9\u6CD5\u5B9E\u73B0\uFF1A"),Ul.forEach(t),ul=m(e),d(Oe.$$.fragment,e),kl=m(e),bt=a(e,"P",{});var pn=p(bt);To=i(pn,"\u6B64\u65B9\u6CD5\u7684\u8F93\u51FA\u662F\u4E00\u4E2A\u5B57\u7B26\u4E32\u5217\u8868\u6216\u6807\u8BB0(token)\uFF1A"),pn.forEach(t),cl=m(e),d(Be.$$.fragment,e),_l=m(e),g=a(e,"P",{});var fe=p(g);Io=i(fe,"\u8FD9\u4E2A\u6807\u8BB0\u5668(tokenizer)\u662F\u4E00\u4E2A\u5B50\u8BCD\u6807\u8BB0\u5668(tokenizer)\uFF1A\u5B83\u5BF9\u8BCD\u8FDB\u884C\u62C6\u5206\uFF0C\u76F4\u5230\u83B7\u5F97\u53EF\u4EE5\u7528\u5176\u8BCD\u6C47\u8868\u8868\u793A\u7684\u6807\u8BB0(token)\u3002"),Xt=a(fe,"CODE",{});var fn=p(Xt);Ao=i(fn,"transformer"),fn.forEach(t),Co=i(fe," \u5C31\u662F\u8FD9\u79CD\u60C5\u51B5\uFF0C\u5B83\u5206\u4E3A\u4E24\u4E2A\u6807\u8BB0\uFF1A"),Zt=a(fe,"CODE",{});var mn=p(Zt);go=i(mn,"transform"),mn.forEach(t),No=i(fe," \u548C "),es=a(fe,"CODE",{});var un=p(es);qo=i(un,"##er"),un.forEach(t),Mo=i(fe,"\u3002"),fe.forEach(t),$l=m(e),W=a(e,"H3",{class:!0});var Gl=p(W);ne=a(Gl,"A",{id:!0,class:!0,href:!0});var kn=p(ne);ts=a(kn,"SPAN",{});var cn=p(ts);d(Le.$$.fragment,cn),cn.forEach(t),kn.forEach(t),Ho=m(Gl),ss=a(Gl,"SPAN",{});var _n=p(ss);Oo=i(_n,"\u4ECE\u8BCD\u7B26(token)\u5230\u8F93\u5165 ID"),_n.forEach(t),Gl.forEach(t),vl=i(e,`

\u8F93\u5165 ID \u7684\u8F6C\u6362\u7531\u6807\u8BB0\u5668(tokenizer)\u7684\`convert_tokens_to_ids()\`\u65B9\u6CD5\u5B9E\u73B0\uFF1A

	`),d(Se.$$.fragment,e),bl=m(e),d(Ue.$$.fragment,e),dl=m(e),dt=a(e,"P",{});var $n=p(dt);Bo=i($n,"\u8FD9\u4E9B\u8F93\u51FA\u4E00\u65E6\u8F6C\u6362\u4E3A\u9002\u5F53\u7684\u6846\u67B6\u5F20\u91CF\uFF0C\u5C31\u53EF\u4EE5\u7528\u4F5C\u6A21\u578B\u7684\u8F93\u5165\uFF0C\u5982\u672C\u7AE0\u524D\u9762\u6240\u89C1\u3002"),$n.forEach(t),El=m(e),d(ae.$$.fragment,e),hl=m(e),Ge=a(e,"H2",{id:!0});var vn=p(Ge);Lo=i(vn,"\u89E3\u7801"),vn.forEach(t),zl=m(e),K=a(e,"P",{});var ns=p(K);ls=a(ns,"EM",{});var bn=p(ls);So=i(bn,"\u89E3\u7801(Decoding)"),bn.forEach(t),Uo=i(ns," \u6B63\u597D\u76F8\u53CD\uFF1A\u4ECE\u8BCD\u6C47\u7D22\u5F15\u4E2D\uFF0C\u6211\u4EEC\u60F3\u8981\u5F97\u5230\u4E00\u4E2A\u5B57\u7B26\u4E32\u3002\u8FD9\u53EF\u4EE5\u901A\u8FC7  "),rs=a(ns,"CODE",{});var dn=p(rs);Go=i(dn,"decode()"),dn.forEach(t),Fo=i(ns," \u65B9\u6CD5\u5B9E\u73B0\uFF0C\u5982\u4E0B\uFF1A"),ns.forEach(t),wl=m(e),d(Fe.$$.fragment,e),jl=m(e),d(Je.$$.fragment,e),yl=m(e),pe=a(e,"P",{});var Fl=p(pe);Jo=i(Fl,"\u8BF7\u6CE8\u610F\uFF0C "),os=a(Fl,"CODE",{});var En=p(os);Ro=i(En,"decode"),En.forEach(t),Wo=i(Fl," \u65B9\u6CD5\u4E0D\u4EC5\u5C06\u7D22\u5F15\u8F6C\u6362\u56DE\u6807\u8BB0(token)\uFF0C\u8FD8\u5C06\u5C5E\u4E8E\u76F8\u540C\u5355\u8BCD\u7684\u6807\u8BB0(token)\u7EC4\u5408\u5728\u4E00\u8D77\u4EE5\u751F\u6210\u53EF\u8BFB\u7684\u53E5\u5B50\u3002\u5F53\u6211\u4EEC\u4F7F\u7528\u9884\u6D4B\u65B0\u6587\u672C\u7684\u6A21\u578B\uFF08\u6839\u636E\u63D0\u793A\u751F\u6210\u7684\u6587\u672C\uFF0C\u6216\u5E8F\u5217\u5230\u5E8F\u5217\u95EE\u9898\uFF08\u5982\u7FFB\u8BD1\u6216\u6458\u8981\uFF09\uFF09\u65F6\uFF0C\u8FD9\u79CD\u884C\u4E3A\u5C06\u975E\u5E38\u6709\u7528\u3002"),Fl.forEach(t),Pl=m(e),Et=a(e,"P",{});var hn=p(Et);Ko=i(hn,"\u5230\u73B0\u5728\u4E3A\u6B62\uFF0C\u60A8\u5E94\u8BE5\u4E86\u89E3\u6807\u8BB0\u5668(tokenizer)\u53EF\u4EE5\u5904\u7406\u7684\u539F\u5B50\u64CD\u4F5C\uFF1A\u6807\u8BB0\u5316\u3001\u8F6C\u6362\u4E3A ID \u4EE5\u53CA\u5C06 ID \u8F6C\u6362\u56DE\u5B57\u7B26\u4E32\u3002\u7136\u800C\uFF0C\u6211\u4EEC\u53EA\u662F\u522E\u5230\u4E86\u51B0\u5C71\u4E00\u89D2\u3002\u5728\u4E0B\u4E00\u8282\u4E2D\uFF0C\u6211\u4EEC\u5C06\u91C7\u7528\u6211\u4EEC\u7684\u65B9\u6CD5\u6765\u514B\u670D\u5B83\u7684\u9650\u5236\uFF0C\u5E76\u770B\u770B\u5982\u4F55\u514B\u670D\u5B83\u4EEC\u3002"),hn.forEach(t),this.h()},h(){c(u,"name","hf:doc:metadata"),c(u,"content",JSON.stringify(On)),c(z,"id","tokenizer"),c(z,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),c(z,"href","#tokenizer"),c(y,"class","relative group"),c(Y,"id","wordbased"),c(Y,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),c(Y,"href","#wordbased"),c(U,"class","relative group"),c(_e,"class","block dark:hidden"),jt(_e.src,Yo="https://huggingface.co/datasets/huggingface-course/documentation-images/resolve/main/en/chapter2/word_based_tokenization.svg")||c(_e,"src",Yo),c(_e,"alt","An example of word-based tokenization."),c($e,"class","hidden dark:block"),jt($e.src,Qo="https://huggingface.co/datasets/huggingface-course/documentation-images/resolve/main/en/chapter2/word_based_tokenization-dark.svg")||c($e,"src",Qo),c($e,"alt","An example of word-based tokenization."),c(G,"class","flex justify-center"),c(ee,"id","characterbased"),c(ee,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),c(ee,"href","#characterbased"),c(F,"class","relative group"),c(ze,"class","block dark:hidden"),jt(ze.src,Xo="https://huggingface.co/datasets/huggingface-course/documentation-images/resolve/main/en/chapter2/character_based_tokenization.svg")||c(ze,"src",Xo),c(ze,"alt","An example of character-based tokenization."),c(we,"class","hidden dark:block"),jt(we.src,Zo="https://huggingface.co/datasets/huggingface-course/documentation-images/resolve/main/en/chapter2/character_based_tokenization-dark.svg")||c(we,"src",Zo),c(we,"alt","An example of character-based tokenization."),c(J,"class","flex justify-center"),c(je,"id",""),c(Pe,"class","block dark:hidden"),jt(Pe.src,ei="https://huggingface.co/datasets/huggingface-course/documentation-images/resolve/main/en/chapter2/bpe_subword.svg")||c(Pe,"src",ei),c(Pe,"alt","A subword tokenization algorithm."),c(xe,"class","hidden dark:block"),jt(xe.src,ti="https://huggingface.co/datasets/huggingface-course/documentation-images/resolve/main/en/chapter2/bpe_subword-dark.svg")||c(xe,"src",ti),c(xe,"alt","A subword tokenization algorithm."),c(R,"class","flex justify-center"),c(De,"id",""),c(Te,"id",""),c($t,"href","/Couse/chapter3"),c(qe,"id",""),c(He,"id",""),c(ne,"id","token-id"),c(ne,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),c(ne,"href","#token-id"),c(W,"class","relative group"),c(Ge,"id","")},m(e,l){s(document.head,u),r(e,v,l),E(k,e,l),r(e,j,l),r(e,y,l),s(y,z),s(z,P),E(x,P,null),s(y,D),s(y,w),s(w,S),r(e,M,l),Re[N].m(e,l),r(e,Ke,l),E(me,e,l),r(e,ps,l),r(e,Ve,l),s(Ve,Rl),r(e,fs,l),r(e,Ye,l),s(Ye,Wl),r(e,ms,l),E(ue,e,l),r(e,us,l),r(e,Qe,l),s(Qe,Kl),r(e,ks,l),r(e,Xe,l),s(Xe,Vl),r(e,cs,l),r(e,U,l),s(U,Y),s(Y,yt),E(ke,yt,null),s(U,Yl),s(U,Pt),s(Pt,Ql),r(e,_s,l),E(ce,e,l),r(e,$s,l),r(e,Q,l),s(Q,Xl),s(Q,xt),s(xt,Zl),s(Q,er),r(e,vs,l),r(e,G,l),s(G,_e),s(G,tr),s(G,$e),r(e,bs,l),r(e,X,l),s(X,sr),s(X,Dt),s(Dt,lr),s(X,rr),r(e,ds,l),E(ve,e,l),r(e,Es,l),E(be,e,l),r(e,hs,l),r(e,Ze,l),s(Ze,or),r(e,zs,l),r(e,et,l),s(et,ir),r(e,ws,l),r(e,tt,l),s(tt,nr),r(e,js,l),r(e,de,l),s(de,ar),s(de,Tt),s(Tt,pr),r(e,ys,l),r(e,Z,l),s(Z,fr),s(Z,It),s(It,mr),s(Z,ur),r(e,Ps,l),r(e,F,l),s(F,ee),s(ee,At),E(Ee,At,null),s(F,kr),s(F,Ct),s(Ct,cr),r(e,xs,l),E(he,e,l),r(e,Ds,l),r(e,st,l),s(st,_r),r(e,Ts,l),r(e,te,l),s(te,gt),s(gt,$r),s(te,vr),s(te,Nt),s(Nt,br),r(e,Is,l),r(e,lt,l),s(lt,dr),r(e,As,l),r(e,J,l),s(J,ze),s(J,Er),s(J,we),r(e,Cs,l),r(e,rt,l),s(rt,hr),r(e,gs,l),r(e,ot,l),s(ot,zr),r(e,Ns,l),r(e,se,l),s(se,wr),s(se,qt),s(qt,jr),s(se,yr),r(e,qs,l),r(e,je,l),s(je,Pr),r(e,Ms,l),E(ye,e,l),r(e,Hs,l),r(e,it,l),s(it,xr),r(e,Os,l),r(e,nt,l),s(nt,Dr),r(e,Bs,l),r(e,at,l),s(at,Tr),r(e,Ls,l),r(e,R,l),s(R,Pe),s(R,Ir),s(R,xe),r(e,Ss,l),r(e,pt,l),s(pt,Ar),r(e,Us,l),r(e,ft,l),s(ft,Cr),r(e,Gs,l),r(e,De,l),s(De,gr),r(e,Fs,l),r(e,mt,l),s(mt,Nr),r(e,Js,l),r(e,H,l),s(H,Mt),s(Mt,qr),s(H,Mr),s(H,Ht),s(Ht,Hr),s(H,Or),s(H,Ot),s(Ot,Br),r(e,Rs,l),r(e,ut,l),s(ut,Lr),r(e,Ws,l),r(e,Te,l),s(Te,Sr),r(e,Ks,l),r(e,T,l),s(T,Ur),s(T,Bt),s(Bt,Gr),s(T,Fr),s(T,Lt),s(Lt,Jr),s(T,Rr),s(T,St),s(St,Wr),s(T,Kr),s(T,Ut),s(Ut,Vr),s(T,Yr),r(e,Vs,l),r(e,le,l),s(le,Qr),s(le,Gt),s(Gt,Xr),s(le,Zr),r(e,Ys,l),E(Ie,e,l),r(e,Qs,l),V.m(e,l),r(e,kt,l),E(Ae,e,l),r(e,Xs,l),r(e,ct,l),s(ct,eo),r(e,Zs,l),E(Ce,e,l),r(e,el,l),E(ge,e,l),r(e,tl,l),r(e,_t,l),s(_t,to),r(e,sl,l),E(Ne,e,l),r(e,ll,l),r(e,I,l),s(I,so),s(I,$t),s($t,lo),s(I,ro),s(I,Ft),s(Ft,oo),s(I,io),s(I,Jt),s(Jt,no),s(I,ao),s(I,Rt),s(Rt,po),s(I,fo),r(e,rl,l),r(e,qe,l),s(qe,mo),r(e,ol,l),E(Me,e,l),r(e,il,l),r(e,re,l),s(re,uo),s(re,Wt),s(Wt,ko),s(re,co),r(e,nl,l),r(e,oe,l),s(oe,_o),s(oe,Kt),s(Kt,$o),s(oe,vo),r(e,al,l),r(e,O,l),s(O,bo),s(O,Vt),s(Vt,Eo),s(O,ho),s(O,Yt),s(Yt,zo),s(O,wo),r(e,pl,l),r(e,vt,l),s(vt,jo),r(e,fl,l),r(e,He,l),s(He,yo),r(e,ml,l),r(e,ie,l),s(ie,Po),s(ie,Qt),s(Qt,xo),s(ie,Do),r(e,ul,l),E(Oe,e,l),r(e,kl,l),r(e,bt,l),s(bt,To),r(e,cl,l),E(Be,e,l),r(e,_l,l),r(e,g,l),s(g,Io),s(g,Xt),s(Xt,Ao),s(g,Co),s(g,Zt),s(Zt,go),s(g,No),s(g,es),s(es,qo),s(g,Mo),r(e,$l,l),r(e,W,l),s(W,ne),s(ne,ts),E(Le,ts,null),s(W,Ho),s(W,ss),s(ss,Oo),r(e,vl,l),E(Se,e,l),r(e,bl,l),E(Ue,e,l),r(e,dl,l),r(e,dt,l),s(dt,Bo),r(e,El,l),E(ae,e,l),r(e,hl,l),r(e,Ge,l),s(Ge,Lo),r(e,zl,l),r(e,K,l),s(K,ls),s(ls,So),s(K,Uo),s(K,rs),s(rs,Go),s(K,Fo),r(e,wl,l),E(Fe,e,l),r(e,jl,l),E(Je,e,l),r(e,yl,l),r(e,pe,l),s(pe,Jo),s(pe,os),s(os,Ro),s(pe,Wo),r(e,Pl,l),r(e,Et,l),s(Et,Ko),xl=!0},p(e,[l]){const We={};l&1&&(We.fw=e[0]),k.$set(We);let ht=N;N=li(e),N!==ht&&(Tn(),_(Re[ht],1,1,()=>{Re[ht]=null}),xn(),q=Re[N],q||(q=Re[N]=si[N](e),q.c()),$(q,1),q.m(Ke.parentNode,Ke)),Dl!==(Dl=ri(e))&&(V.d(1),V=Dl(e),V&&(V.c(),V.m(kt.parentNode,kt)));const is={};l&2&&(is.$$scope={dirty:l,ctx:e}),ae.$set(is)},i(e){xl||($(k.$$.fragment,e),$(x.$$.fragment,e),$(q),$(me.$$.fragment,e),$(ue.$$.fragment,e),$(ke.$$.fragment,e),$(ce.$$.fragment,e),$(ve.$$.fragment,e),$(be.$$.fragment,e),$(Ee.$$.fragment,e),$(he.$$.fragment,e),$(ye.$$.fragment,e),$(Ie.$$.fragment,e),$(Ae.$$.fragment,e),$(Ce.$$.fragment,e),$(ge.$$.fragment,e),$(Ne.$$.fragment,e),$(Me.$$.fragment,e),$(Oe.$$.fragment,e),$(Be.$$.fragment,e),$(Le.$$.fragment,e),$(Se.$$.fragment,e),$(Ue.$$.fragment,e),$(ae.$$.fragment,e),$(Fe.$$.fragment,e),$(Je.$$.fragment,e),xl=!0)},o(e){_(k.$$.fragment,e),_(x.$$.fragment,e),_(q),_(me.$$.fragment,e),_(ue.$$.fragment,e),_(ke.$$.fragment,e),_(ce.$$.fragment,e),_(ve.$$.fragment,e),_(be.$$.fragment,e),_(Ee.$$.fragment,e),_(he.$$.fragment,e),_(ye.$$.fragment,e),_(Ie.$$.fragment,e),_(Ae.$$.fragment,e),_(Ce.$$.fragment,e),_(ge.$$.fragment,e),_(Ne.$$.fragment,e),_(Me.$$.fragment,e),_(Oe.$$.fragment,e),_(Be.$$.fragment,e),_(Le.$$.fragment,e),_(Se.$$.fragment,e),_(Ue.$$.fragment,e),_(ae.$$.fragment,e),_(Fe.$$.fragment,e),_(Je.$$.fragment,e),xl=!1},d(e){t(u),e&&t(v),h(k,e),e&&t(j),e&&t(y),h(x),e&&t(M),Re[N].d(e),e&&t(Ke),h(me,e),e&&t(ps),e&&t(Ve),e&&t(fs),e&&t(Ye),e&&t(ms),h(ue,e),e&&t(us),e&&t(Qe),e&&t(ks),e&&t(Xe),e&&t(cs),e&&t(U),h(ke),e&&t(_s),h(ce,e),e&&t($s),e&&t(Q),e&&t(vs),e&&t(G),e&&t(bs),e&&t(X),e&&t(ds),h(ve,e),e&&t(Es),h(be,e),e&&t(hs),e&&t(Ze),e&&t(zs),e&&t(et),e&&t(ws),e&&t(tt),e&&t(js),e&&t(de),e&&t(ys),e&&t(Z),e&&t(Ps),e&&t(F),h(Ee),e&&t(xs),h(he,e),e&&t(Ds),e&&t(st),e&&t(Ts),e&&t(te),e&&t(Is),e&&t(lt),e&&t(As),e&&t(J),e&&t(Cs),e&&t(rt),e&&t(gs),e&&t(ot),e&&t(Ns),e&&t(se),e&&t(qs),e&&t(je),e&&t(Ms),h(ye,e),e&&t(Hs),e&&t(it),e&&t(Os),e&&t(nt),e&&t(Bs),e&&t(at),e&&t(Ls),e&&t(R),e&&t(Ss),e&&t(pt),e&&t(Us),e&&t(ft),e&&t(Gs),e&&t(De),e&&t(Fs),e&&t(mt),e&&t(Js),e&&t(H),e&&t(Rs),e&&t(ut),e&&t(Ws),e&&t(Te),e&&t(Ks),e&&t(T),e&&t(Vs),e&&t(le),e&&t(Ys),h(Ie,e),e&&t(Qs),V.d(e),e&&t(kt),h(Ae,e),e&&t(Xs),e&&t(ct),e&&t(Zs),h(Ce,e),e&&t(el),h(ge,e),e&&t(tl),e&&t(_t),e&&t(sl),h(Ne,e),e&&t(ll),e&&t(I),e&&t(rl),e&&t(qe),e&&t(ol),h(Me,e),e&&t(il),e&&t(re),e&&t(nl),e&&t(oe),e&&t(al),e&&t(O),e&&t(pl),e&&t(vt),e&&t(fl),e&&t(He),e&&t(ml),e&&t(ie),e&&t(ul),h(Oe,e),e&&t(kl),e&&t(bt),e&&t(cl),h(Be,e),e&&t(_l),e&&t(g),e&&t($l),e&&t(W),h(Le),e&&t(vl),h(Se,e),e&&t(bl),h(Ue,e),e&&t(dl),e&&t(dt),e&&t(El),h(ae,e),e&&t(hl),e&&t(Ge),e&&t(zl),e&&t(K),e&&t(wl),h(Fe,e),e&&t(jl),h(Je,e),e&&t(yl),e&&t(pe),e&&t(Pl),e&&t(Et)}}}const On={local:"tokenizer",sections:[{local:"wordbased",title:"\u57FA\u4E8E\u8BCD\u7684(Word-based)"},{local:"characterbased",title:"\u57FA\u4E8E\u5B57\u7B26(Character-based)"},{local:"",sections:[{local:"",title:"\u8FD8\u6709\u66F4\u591A\uFF01"}],title:"\u5B50\u8BCD\u6807\u8BB0\u5316"},{local:"",title:"\u52A0\u8F7D\u548C\u4FDD\u5B58"},{local:"",sections:[{local:"",title:"\u6807\u8BB0\u5316"},{local:"token-id",title:"\u4ECE\u8BCD\u7B26(token)\u5230\u8F93\u5165 ID"}],title:"\u7F16\u7801"},{local:"",title:"\u89E3\u7801"}],title:"\u6807\u8BB0\u5668\uFF08Tokenizer\uFF09"};function Bn(C,u,v){let k="pt";return Dn(()=>{const j=new URLSearchParams(window.location.search);v(0,k=j.get("fw")||"pt")}),[k]}class Kn extends wn{constructor(u){super();jn(this,u,Bn,Hn,yn,{})}}export{Kn as default,On as metadata};
