import{S as zn,i as jn,s as wn,e as a,k as f,t as r,w as _,M as yn,c as o,d as s,m,a as i,h as l,x as g,b as u,N as En,G as t,g as p,y as d,q as k,o as b,B as $,v as Pn}from"../../chunks/vendor-hf-doc-builder.js";import{T as xn}from"../../chunks/Tip-hf-doc-builder.js";import{Y as Tn}from"../../chunks/Youtube-hf-doc-builder.js";import{C as O}from"../../chunks/CodeBlock-hf-doc-builder.js";import{C as Sn}from"../../chunks/CourseFloatingBanner-hf-doc-builder.js";import"../../chunks/DocNotebookDropdown-hf-doc-builder.js";function Rn(zt){let c,C,v,se,M;return{c(){c=a("p"),C=r("\u270F\uFE0F "),v=a("strong"),se=r("\u8BD5\u8BD5\u770B!"),M=r(" \u4ECE\u68C0\u67E5\u70B9\u52A0\u8F7D\u6807\u8BB0\u5668\u5E76\u5C06\u76F8\u540C\u7684\u793A\u4F8B\u4F20\u9012\u7ED9\u5B83\u3002\u60A8\u53EF\u4EE5\u770B\u5230\u5206\u8BCD\u5668\u7684\u5E26\u58F3\u548C\u65E0\u58F3\u7248\u672C\u4E4B\u95F4\u7684\u4E3B\u8981\u533A\u522B\u662F\u4EC0\u4E48\uFF1F")},l(E){c=o(E,"P",{});var z=i(c);C=l(z,"\u270F\uFE0F "),v=o(z,"STRONG",{});var A=i(v);se=l(A,"\u8BD5\u8BD5\u770B!"),A.forEach(s),M=l(z," \u4ECE\u68C0\u67E5\u70B9\u52A0\u8F7D\u6807\u8BB0\u5668\u5E76\u5C06\u76F8\u540C\u7684\u793A\u4F8B\u4F20\u9012\u7ED9\u5B83\u3002\u60A8\u53EF\u4EE5\u770B\u5230\u5206\u8BCD\u5668\u7684\u5E26\u58F3\u548C\u65E0\u58F3\u7248\u672C\u4E4B\u95F4\u7684\u4E3B\u8981\u533A\u522B\u662F\u4EC0\u4E48\uFF1F"),z.forEach(s)},m(E,z){p(E,c,z),t(c,C),t(c,v),t(v,se),t(c,M)},d(E){E&&s(c)}}}function Gn(zt){let c,C,v,se,M,E,z,A,gs,jt,D,re,dl,ds,le,kl,wt,N,ks,st,bs,$s,rt,vs,Es,yt,ne,Ts,Pt,ae,xt,U,zs,oe,js,ws,St,H,ys,lt,Ps,xs,nt,Ss,Rs,Rt,ie,Gt,pe,Nt,j,at,Gs,Ns,ot,Hs,qs,it,Bs,Os,Ht,fe,qt,me,Bt,F,As,pt,Ds,Cs,Ot,I,At,ue,Ms,Dt,ce,Ct,W,Us,je,Fs,Is,Mt,T,Ws,ft,Ls,Js,mt,Ys,Ks,ut,Vs,Xs,Ut,he,Ft,_e,It,q,Qs,ct,Zs,er,ht,tr,sr,Wt,we,rr,Lt,ge,Jt,L,lr,_t,nr,ar,Yt,de,Kt,ye,or,Vt,Pe,ir,Xt,ke,Qt,be,Zt,h,pr,gt,fr,mr,dt,ur,cr,kt,hr,_r,bt,gr,dr,es,xe,kr,ts,$e,br,ss,w,ve,$r,vr,$t,Er,Tr,Se,zr,jr,rs,J,wr,vt,yr,Pr,ls,Ee,xr,ns,Re,Sr,as,Y,Et,y,Ge,Rr,Gr,Ne,Nr,Hr,He,qr,Br,qe,Or,Ar,P,x,Be,Dr,Cr,Oe,Mr,Ur,Ae,Fr,Ir,De,Wr,Lr,S,Ce,Jr,Yr,Me,Kr,Vr,Ue,Xr,Qr,Fe,Zr,el,R,Ie,tl,sl,We,rl,ll,Le,nl,al,Je,ol,il,G,Ye,pl,fl,Ke,ml,ul,Ve,cl,hl,Xe,_l,os,Qe,gl,is;return E=new Sn({props:{chapter:6,classNames:"absolute z-10 right-0 top-0",notebooks:[{label:"Google Colab",value:"https://colab.research.google.com/github/huggingface/notebooks/blob/master/course/zh-CN/chapter6/section4.ipynb"},{label:"Aws Studio",value:"https://studiolab.sagemaker.aws/import/github/huggingface/notebooks/blob/master/course/zh-CN/chapter6/section4.ipynb"}]}}),ae=new Tn({props:{id:"4IIC2jI9CaU"}}),ie=new O({props:{code:`from transformers import AutoTokenizer

tokenizer = AutoTokenizer.from_pretrained("bert-base-uncased")
print(type(tokenizer.backend_tokenizer))`,highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoTokenizer

tokenizer = AutoTokenizer.from_pretrained(<span class="hljs-string">&quot;bert-base-uncased&quot;</span>)
<span class="hljs-built_in">print</span>(<span class="hljs-built_in">type</span>(tokenizer.backend_tokenizer))`}}),pe=new O({props:{code:"<class 'tokenizers.Tokenizer'>",highlighted:'&lt;<span class="hljs-keyword">class</span> <span class="hljs-string">&#x27;tokenizers.Tokenizer&#x27;</span>&gt;'}}),fe=new O({props:{code:'print(tokenizer.backend_tokenizer.normalizer.normalize_str("H\xE9ll\xF2 h\xF4w are \xFC?"))',highlighted:'<span class="hljs-built_in">print</span>(tokenizer.backend_tokenizer.normalizer.normalize_str(<span class="hljs-string">&quot;H\xE9ll\xF2 h\xF4w are \xFC?&quot;</span>))'}}),me=new O({props:{code:"'hello how are u?'",highlighted:'<span class="hljs-string">&#x27;hello how are u?&#x27;</span>'}}),I=new xn({props:{$$slots:{default:[Rn]},$$scope:{ctx:zt}}}),ce=new Tn({props:{id:"grlLV8AIXug"}}),he=new O({props:{code:'tokenizer.backend_tokenizer.pre_tokenizer.pre_tokenize_str("Hello, how are  you?")',highlighted:'tokenizer.backend_tokenizer.pre_tokenizer.pre_tokenize_str(<span class="hljs-string">&quot;Hello, how are  you?&quot;</span>)'}}),_e=new O({props:{code:"[('Hello', (0, 5)), (',', (5, 6)), ('how', (7, 10)), ('are', (11, 14)), ('you', (16, 19)), ('?', (19, 20))]",highlighted:'[(<span class="hljs-string">&#x27;Hello&#x27;</span>, (<span class="hljs-number">0</span>, <span class="hljs-number">5</span>)), (<span class="hljs-string">&#x27;,&#x27;</span>, (<span class="hljs-number">5</span>, <span class="hljs-number">6</span>)), (<span class="hljs-string">&#x27;how&#x27;</span>, (<span class="hljs-number">7</span>, <span class="hljs-number">10</span>)), (<span class="hljs-string">&#x27;are&#x27;</span>, (<span class="hljs-number">11</span>, <span class="hljs-number">14</span>)), (<span class="hljs-string">&#x27;you&#x27;</span>, (<span class="hljs-number">16</span>, <span class="hljs-number">19</span>)), (<span class="hljs-string">&#x27;?&#x27;</span>, (<span class="hljs-number">19</span>, <span class="hljs-number">20</span>))]'}}),ge=new O({props:{code:`tokenizer = AutoTokenizer.from_pretrained("gpt2")
tokenizer.backend_tokenizer.pre_tokenizer.pre_tokenize_str("Hello, how are  you?")`,highlighted:`tokenizer = AutoTokenizer.from_pretrained(<span class="hljs-string">&quot;gpt2&quot;</span>)
tokenizer.backend_tokenizer.pre_tokenizer.pre_tokenize_str(<span class="hljs-string">&quot;Hello, how are  you?&quot;</span>)`}}),de=new O({props:{code:`[('Hello', (0, 5)), (',', (5, 6)), ('\u0120how', (6, 10)), ('\u0120are', (10, 14)), ('\u0120', (14, 15)), ('\u0120you', (15, 19)),
 ('?', (19, 20))]`,highlighted:`[(<span class="hljs-string">&#x27;Hello&#x27;</span>, (<span class="hljs-number">0</span>, <span class="hljs-number">5</span>)), (<span class="hljs-string">&#x27;,&#x27;</span>, (<span class="hljs-number">5</span>, <span class="hljs-number">6</span>)), (<span class="hljs-string">&#x27;\u0120how&#x27;</span>, (<span class="hljs-number">6</span>, <span class="hljs-number">10</span>)), (<span class="hljs-string">&#x27;\u0120are&#x27;</span>, (<span class="hljs-number">10</span>, <span class="hljs-number">14</span>)), (<span class="hljs-string">&#x27;\u0120&#x27;</span>, (<span class="hljs-number">14</span>, <span class="hljs-number">15</span>)), (<span class="hljs-string">&#x27;\u0120you&#x27;</span>, (<span class="hljs-number">15</span>, <span class="hljs-number">19</span>)),
 (<span class="hljs-string">&#x27;?&#x27;</span>, (<span class="hljs-number">19</span>, <span class="hljs-number">20</span>))]`}}),ke=new O({props:{code:`tokenizer = AutoTokenizer.from_pretrained("t5-small")
tokenizer.backend_tokenizer.pre_tokenizer.pre_tokenize_str("Hello, how are  you?")`,highlighted:`tokenizer = AutoTokenizer.from_pretrained(<span class="hljs-string">&quot;t5-small&quot;</span>)
tokenizer.backend_tokenizer.pre_tokenizer.pre_tokenize_str(<span class="hljs-string">&quot;Hello, how are  you?&quot;</span>)`}}),be=new O({props:{code:"[('\u2581Hello,', (0, 6)), ('\u2581how', (7, 10)), ('\u2581are', (11, 14)), ('\u2581you?', (16, 20))]",highlighted:'[(<span class="hljs-string">&#x27;\u2581Hello,&#x27;</span>, (<span class="hljs-number">0</span>, <span class="hljs-number">6</span>)), (<span class="hljs-string">&#x27;\u2581how&#x27;</span>, (<span class="hljs-number">7</span>, <span class="hljs-number">10</span>)), (<span class="hljs-string">&#x27;\u2581are&#x27;</span>, (<span class="hljs-number">11</span>, <span class="hljs-number">14</span>)), (<span class="hljs-string">&#x27;\u2581you?&#x27;</span>, (<span class="hljs-number">16</span>, <span class="hljs-number">20</span>))]'}}),{c(){c=a("meta"),C=f(),v=a("h1"),se=r("\u6807\u51C6\u5316\u548C\u9884\u6807\u8BB0\u5316"),M=f(),_(E.$$.fragment),z=f(),A=a("p"),gs=r("\u5728\u6211\u4EEC\u66F4\u6DF1\u5165\u5730\u7814\u7A76\u4E0E Transformer \u6A21\u578B\uFF08\u5B57\u8282\u5BF9\u7F16\u7801 [BPE]\u3001WordPiece \u548C Unigram\uFF09\u4E00\u8D77\u4F7F\u7528\u7684\u4E09\u79CD\u6700\u5E38\u89C1\u7684\u5B50\u8BCD\u6807\u8BB0\u5316\u7B97\u6CD5\u4E4B\u524D\uFF0C\u6211\u4EEC\u5C06\u9996\u5148\u770B\u4E00\u4E0B\u6BCF\u4E2A\u6807\u8BB0\u5668\u5E94\u7528\u4E8E\u6587\u672C\u7684\u9884\u5904\u7406\u3002\u4EE5\u4E0B\u662F\u6807\u8BB0\u5316\u7BA1\u9053\u4E2D\u6B65\u9AA4\u7684\u9AD8\u7EA7\u6982\u8FF0\uFF1A"),jt=f(),D=a("div"),re=a("img"),ds=f(),le=a("img"),wt=f(),N=a("p"),ks=r("\u5728\u5C06\u6587\u672C\u62C6\u5206\u4E3A\u5B50\u6807\u8BB0\u4E4B\u524D\uFF08\u6839\u636E\u5176\u6A21\u578B\uFF09\uFF0C\u5206\u8BCD\u5668\u6267\u884C\u4E24\u4E2A\u6B65\u9AA4\uFF1A "),st=a("em"),bs=r("normalization"),$s=r(" \u548C "),rt=a("em"),vs=r("pre-tokenization"),Es=r("."),yt=f(),ne=a("h2"),Ts=r("\u6B63\u5E38\u5316"),Pt=f(),_(ae.$$.fragment),xt=f(),U=a("p"),zs=r("\u6807\u51C6\u5316\u6B65\u9AA4\u6D89\u53CA\u4E00\u4E9B\u5E38\u89C4\u6E05\u7406\uFF0C\u4F8B\u5982\u5220\u9664\u4E0D\u5FC5\u8981\u7684\u7A7A\u683C\u3001\u5C0F\u5199\u548C/\u6216\u5220\u9664\u91CD\u97F3\u7B26\u53F7\u3002\u5982\u679C\u4F60\u719F\u6089"),oe=a("a"),js=r("Unicode normalization"),ws=r("\uFF08\u4F8B\u5982 NFC \u6216 NFKC\uFF09\uFF0C\u8FD9\u4E5F\u662F tokenizer \u53EF\u80FD\u5E94\u7528\u7684\u4E1C\u897F\u3002"),St=f(),H=a("p"),ys=r("\u{1F917}Transformers "),lt=a("strong"),Ps=r("tokenizer"),xs=r(" \u6709\u4E00\u4E2A\u5C5E\u6027\u53EB\u505A "),nt=a("strong"),Ss=r("backend_tokenizer"),Rs=r(" \u5B83\u63D0\u4F9B\u4E86\u5BF9 \u{1F917} Tokenizers \u5E93\u4E2D\u5E95\u5C42\u6807\u8BB0\u5668\u7684\u8BBF\u95EE\uFF1A"),Rt=f(),_(ie.$$.fragment),Gt=f(),_(pe.$$.fragment),Nt=f(),j=a("p"),at=a("strong"),Gs=r("normalizer"),Ns=r(" \u7684\u5C5E\u6027 "),ot=a("strong"),Hs=r("tokenizer"),qs=r(" \u5BF9\u8C61\u6709\u4E00\u4E2A "),it=a("strong"),Bs=r("normalize_str()"),Os=r(" \u6211\u4EEC\u53EF\u4EE5\u7528\u6765\u67E5\u770B\u6807\u51C6\u5316\u662F\u5982\u4F55\u6267\u884C\u7684\u65B9\u6CD5\uFF1A"),Ht=f(),_(fe.$$.fragment),qt=f(),_(me.$$.fragment),Bt=f(),F=a("p"),As=r("\u5728\u8FD9\u4E2A\u4F8B\u5B50\u4E2D\uFF0C\u56E0\u4E3A\u6211\u4EEC\u9009\u62E9\u4E86 "),pt=a("strong"),Ds=r("bert-base-uncased"),Cs=r(" \u68C0\u67E5\u70B9\uFF0C\u6807\u51C6\u5316\u5E94\u7528\u5C0F\u5199\u5E76\u5220\u9664\u91CD\u97F3\u3002"),Ot=f(),_(I.$$.fragment),At=f(),ue=a("h2"),Ms=r("\u9884\u6807\u8BB0\u5316"),Dt=f(),_(ce.$$.fragment),Ct=f(),W=a("p"),Us=r("\u6B63\u5982\u6211\u4EEC\u5C06\u5728\u4E0B\u4E00\u8282\u4E2D\u770B\u5230\u7684\uFF0C\u5206\u8BCD\u5668\u4E0D\u80FD\u5355\u72EC\u5728\u539F\u59CB\u6587\u672C\u4E0A\u8FDB\u884C\u8BAD\u7EC3\u3002\u76F8\u53CD\uFF0C\u6211\u4EEC\u9996\u5148\u9700\u8981\u5C06\u6587\u672C\u62C6\u5206\u4E3A\u5C0F\u5B9E\u4F53\uFF0C\u4F8B\u5982\u5355\u8BCD\u3002\u8FD9\u5C31\u662F\u9884\u6807\u8BB0\u5316\u6B65\u9AA4\u7684\u7528\u6B66\u4E4B\u5730\u3002 \u6B63\u5982\u6211\u4EEC\u5728"),je=a("a"),Fs=r("Chapter 2"),Is=r(", \u57FA\u4E8E\u5355\u8BCD\u7684\u6807\u8BB0\u5668\u53EF\u4EE5\u7B80\u5355\u5730\u5C06\u539F\u59CB\u6587\u672C\u62C6\u5206\u4E3A\u7A7A\u767D\u548C\u6807\u70B9\u7B26\u53F7\u7684\u5355\u8BCD\u3002\u8FD9\u4E9B\u8BCD\u5C06\u662F\u5206\u8BCD\u5668\u5728\u8BAD\u7EC3\u671F\u95F4\u53EF\u4EE5\u5B66\u4E60\u7684\u5B50\u6807\u8BB0\u7684\u8FB9\u754C\u3002"),Mt=f(),T=a("p"),Ws=r("\u8981\u67E5\u770B\u5FEB\u901F\u5206\u8BCD\u5668\u5982\u4F55\u6267\u884C\u9884\u5206\u8BCD\uFF0C\u6211\u4EEC\u53EF\u4EE5\u4F7F\u7528 "),ft=a("strong"),Ls=r("pre_tokenize_str()"),Js=r(" \u7684\u65B9\u6CD5 "),mt=a("strong"),Ys=r("pre_tokenizer"),Ks=r(" \u7684\u5C5E\u6027 "),ut=a("strong"),Vs=r("tokenizer"),Xs=r(" \u76EE\u7684\uFF1A"),Ut=f(),_(he.$$.fragment),Ft=f(),_(_e.$$.fragment),It=f(),q=a("p"),Qs=r("\u8BF7\u6CE8\u610F\u5206\u8BCD\u5668\u5982\u4F55\u5DF2\u7ECF\u8DDF\u8E2A\u504F\u79FB\u91CF\uFF0C\u8FD9\u5C31\u662F\u5B83\u5982\u4F55\u4E3A\u6211\u4EEC\u63D0\u4F9B\u4E0A\u4E00\u8282\u4E2D\u4F7F\u7528\u7684\u504F\u79FB\u91CF\u6620\u5C04\u3002\u8FD9\u91CC\u5206\u8BCD\u5668\u5FFD\u7565\u4E86\u8FD9\u4E24\u4E2A\u7A7A\u683C\uFF0C\u53EA\u7528\u4E00\u4E2A\u66FF\u6362\u5B83\u4EEC\uFF0C\u4F46\u504F\u79FB\u91CF\u5728 "),ct=a("strong"),Zs=r("are"),er=r(" \u548C "),ht=a("strong"),tr=r("you"),sr=r(" \u8003\u8651\u5230\u8FD9\u4E00\u70B9\u3002"),Wt=f(),we=a("p"),rr=r("\u7531\u4E8E\u6211\u4EEC\u4F7F\u7528\u7684\u662F BERT \u5206\u8BCD\u5668\uFF0C\u9884\u5206\u8BCD\u6D89\u53CA\u5BF9\u7A7A\u683C\u548C\u6807\u70B9\u7B26\u53F7\u8FDB\u884C\u62C6\u5206\u3002\u5BF9\u4E8E\u8FD9\u4E00\u6B65\uFF0C\u5176\u4ED6\u6807\u8BB0\u5668\u53EF\u4EE5\u6709\u4E0D\u540C\u7684\u89C4\u5219\u3002\u4F8B\u5982\uFF0C\u5982\u679C\u6211\u4EEC\u4F7F\u7528 GPT-2 \u6807\u8BB0\u5668\uFF1A"),Lt=f(),_(ge.$$.fragment),Jt=f(),L=a("p"),lr=r("\u5B83\u4E5F\u4F1A\u5728\u7A7A\u683C\u548C\u6807\u70B9\u7B26\u53F7\u4E0A\u62C6\u5206\uFF0C\u4F46\u5B83\u4F1A\u4FDD\u7559\u7A7A\u683C\u5E76\u5C06\u5B83\u4EEC\u66FF\u6362\u4E3A "),_t=a("strong"),nr=r("\u0120"),ar=r(" \u7B26\u53F7\uFF0C\u5982\u679C\u6211\u4EEC\u89E3\u7801\u4EE4\u724C\uFF0C\u5219\u4F7F\u5176\u80FD\u591F\u6062\u590D\u539F\u59CB\u7A7A\u683C\uFF1A"),Yt=f(),_(de.$$.fragment),Kt=f(),ye=a("p"),or=r("\u53E6\u8BF7\u6CE8\u610F\uFF0C\u4E0E BERT \u5206\u8BCD\u5668\u4E0D\u540C\uFF0C\u6B64\u5206\u8BCD\u5668\u4E0D\u4F1A\u5FFD\u7565\u53CC\u7A7A\u683C"),Vt=f(),Pe=a("p"),ir=r("\u6700\u540E\u4E00\u4E2A\u4F8B\u5B50\uFF0C\u8BA9\u6211\u4EEC\u770B\u4E00\u4E0B\u57FA\u4E8E SentencePiece \u7B97\u6CD5\u7684 T5 \u5206\u8BCD\u5668\uFF1A"),Xt=f(),_(ke.$$.fragment),Qt=f(),_(be.$$.fragment),Zt=f(),h=a("p"),pr=r("\u4E0E GPT-2 \u6807\u8BB0\u5668\u4E00\u6837\uFF0C\u8FD9\u4E2A\u6807\u8BB0\u5668\u4FDD\u7559\u7A7A\u683C\u5E76\u7528\u7279\u5B9A\u6807\u8BB0\u66FF\u6362\u5B83\u4EEC\uFF08 "),gt=a("strong"),fr=r("_"),mr=r(" )\uFF0C\u4F46 T5 \u5206\u8BCD\u5668\u53EA\u5728\u7A7A\u683C\u4E0A\u62C6\u5206\uFF0C\u800C\u4E0D\u662F\u6807\u70B9\u7B26\u53F7\u3002\u8FD8\u8981\u6CE8\u610F\uFF0C\u5B83\u9ED8\u8BA4\u5728\u53E5\u5B50\u7684\u5F00\u5934\u6DFB\u52A0\u4E86\u4E00\u4E2A\u7A7A\u683C\uFF08\u4E4B\u524D "),dt=a("strong"),ur=r("Hello"),cr=r(" ) \u5E76\u5FFD\u7565\u4E86\u4E4B\u95F4\u7684\u53CC\u7A7A\u683C "),kt=a("strong"),hr=r("are"),_r=r(" \u548C "),bt=a("strong"),gr=r("you"),dr=r(" ."),es=f(),xe=a("p"),kr=r("\u73B0\u5728\u6211\u4EEC\u5DF2\u7ECF\u4E86\u89E3\u4E86\u4E00\u4E9B\u4E0D\u540C\u7684\u6807\u8BB0\u5668\u5982\u4F55\u5904\u7406\u6587\u672C\uFF0C\u6211\u4EEC\u53EF\u4EE5\u5F00\u59CB\u63A2\u7D22\u5E95\u5C42\u7B97\u6CD5\u672C\u8EAB\u3002\u6211\u4EEC\u9996\u5148\u5FEB\u901F\u6D4F\u89C8\u4E00\u4E0B\u5E7F\u6CDB\u9002\u7528\u7684 SentencePiece\uFF1B\u7136\u540E\uFF0C\u5728\u63A5\u4E0B\u6765\u7684\u4E09\u4E2A\u90E8\u5206\u4E2D\uFF0C\u6211\u4EEC\u5C06\u7814\u7A76\u7528\u4E8E\u5B50\u8BCD\u6807\u8BB0\u5316\u7684\u4E09\u79CD\u4E3B\u8981\u7B97\u6CD5\u662F\u5982\u4F55\u5DE5\u4F5C\u7684\u3002"),ts=f(),$e=a("h2"),br=r("\u53E5\u5B50"),ss=f(),w=a("p"),ve=a("a"),$r=r("SentencePiece"),vr=r(" \u662F\u4E00\u79CD\u7528\u4E8E\u6587\u672C\u9884\u5904\u7406\u7684\u6807\u8BB0\u5316\u7B97\u6CD5\uFF0C\u60A8\u53EF\u4EE5\u5C06\u5176\u4E0E\u6211\u4EEC\u5C06\u5728\u63A5\u4E0B\u6765\u7684\u4E09\u4E2A\u90E8\u5206\u4E2D\u770B\u5230\u7684\u4EFB\u4F55\u6A21\u578B\u4E00\u8D77\u4F7F\u7528\u3002\u5B83\u5C06\u6587\u672C\u89C6\u4E3A Unicode \u5B57\u7B26\u5E8F\u5217\uFF0C\u5E76\u7528\u7279\u6B8A\u5B57\u7B26\u66FF\u6362\u7A7A\u683C\uFF0C "),$t=a("strong"),Er=r("\u2581"),Tr=r(" .\u4E0E Unigram \u7B97\u6CD5\u7ED3\u5408\u4F7F\u7528\uFF08\u53C2\u89C1"),Se=a("a"),zr=r("section 7"),jr=r("), \u5B83\u751A\u81F3\u4E0D\u9700\u8981\u9884\u6807\u8BB0\u5316\u6B65\u9AA4\uFF0C\u8FD9\u5BF9\u4E8E\u4E0D\u4F7F\u7528\u7A7A\u683C\u5B57\u7B26\u7684\u8BED\u8A00\uFF08\u5982\u4E2D\u6587\u6216\u65E5\u8BED\uFF09\u975E\u5E38\u6709\u7528\u3002"),rs=f(),J=a("p"),wr=r("SentencePiece \u7684\u53E6\u4E00\u4E2A\u4E3B\u8981\u7279\u70B9\u662F\u53EF\u9006\u6807\u8BB0\u5316\uFF1A\u7531\u4E8E\u6CA1\u6709\u5BF9\u7A7A\u683C\u8FDB\u884C\u7279\u6B8A\u5904\u7406\uFF0C\u56E0\u6B64\u53EA\u9700\u901A\u8FC7\u5C06\u5B83\u4EEC\u8FDE\u63A5\u8D77\u6765\u5E76\u66FF\u6362 "),vt=a("strong"),yr=r("_"),Pr=r(" s \u5E26\u7A7A\u683C\u2014\u2014\u8FD9\u4F1A\u5BFC\u81F4\u6807\u51C6\u5316\u7684\u6587\u672C\u3002\u6B63\u5982\u6211\u4EEC\u4E4B\u524D\u770B\u5230\u7684\uFF0CBERT \u5206\u8BCD\u5668\u5220\u9664\u4E86\u91CD\u590D\u7684\u7A7A\u683C\uFF0C\u56E0\u6B64\u5B83\u7684\u5206\u8BCD\u662F\u4E0D\u53EF\u9006\u7684\u3002"),ls=f(),Ee=a("h2"),xr=r("\u7B97\u6CD5\u6982\u8FF0"),ns=f(),Re=a("p"),Sr=r("\u5728\u4E0B\u9762\u7684\u90E8\u5206\u4E2D\uFF0C\u6211\u4EEC\u5C06\u6DF1\u5165\u7814\u7A76\u4E09\u79CD\u4E3B\u8981\u7684\u5B50\u8BCD\u6807\u8BB0\u5316\u7B97\u6CD5\uFF1ABPE\uFF08\u7531 GPT-2 \u548C\u5176\u4ED6\u4EBA\u4F7F\u7528\uFF09\u3001WordPiece\uFF08\u4F8B\u5982\u7531 BERT \u4F7F\u7528\uFF09\u548C Unigram\uFF08\u7531 T5 \u548C\u5176\u4ED6\u4EBA\u4F7F\u7528\uFF09\u3002\u5728\u6211\u4EEC\u5F00\u59CB\u4E4B\u524D\uFF0C\u8FD9\u91CC\u662F\u5B83\u4EEC\u5404\u81EA\u5DE5\u4F5C\u539F\u7406\u7684\u5FEB\u901F\u6982\u8FF0\u3002\u5982\u679C\u60A8\u8FD8\u6CA1\u6709\u7406\u89E3\uFF0C\u8BF7\u5728\u9605\u8BFB\u4E0B\u4E00\u8282\u540E\u7ACB\u5373\u56DE\u5230\u6B64\u8868\u3002"),as=f(),Y=a("table"),Et=a("thead"),y=a("tr"),Ge=a("th"),Rr=r("Model"),Gr=f(),Ne=a("th"),Nr=r("BPE"),Hr=f(),He=a("th"),qr=r("WordPiece"),Br=f(),qe=a("th"),Or=r("Unigram"),Ar=f(),P=a("tbody"),x=a("tr"),Be=a("td"),Dr=r("Training"),Cr=f(),Oe=a("td"),Mr=r("Starts from a small vocabulary and learns rules to merge tokens"),Ur=f(),Ae=a("td"),Fr=r("Starts from a small vocabulary and learns rules to merge tokens"),Ir=f(),De=a("td"),Wr=r("Starts from a large vocabulary and learns rules to remove tokens"),Lr=f(),S=a("tr"),Ce=a("td"),Jr=r("Training step"),Yr=f(),Me=a("td"),Kr=r("Merges the tokens corresponding to the most common pair"),Vr=f(),Ue=a("td"),Xr=r("Merges the tokens corresponding to the pair with the best score based on the frequency of the pair, privileging pairs where each individual token is less frequent"),Qr=f(),Fe=a("td"),Zr=r("Removes all the tokens in the vocabulary that will minimize the loss computed on the whole corpus"),el=f(),R=a("tr"),Ie=a("td"),tl=r("Learns"),sl=f(),We=a("td"),rl=r("Merge rules and a vocabulary"),ll=f(),Le=a("td"),nl=r("Just a vocabulary"),al=f(),Je=a("td"),ol=r("A vocabulary with a score for each token"),il=f(),G=a("tr"),Ye=a("td"),pl=r("Encoding"),fl=f(),Ke=a("td"),ml=r("Splits a word into characters and applies the merges learned during training"),ul=f(),Ve=a("td"),cl=r("Finds the longest subword starting from the beginning that is in the vocabulary, then does the same for the rest of the word"),hl=f(),Xe=a("td"),_l=r("Finds the most likely split into tokens, using the scores learned during training"),os=f(),Qe=a("p"),gl=r("\u73B0\u5728\u8BA9\u6211\u4EEC\u6DF1\u5165\u4E86\u89E3 BPE\uFF01"),this.h()},l(e){const n=yn('[data-svelte="svelte-1phssyn"]',document.head);c=o(n,"META",{name:!0,content:!0}),n.forEach(s),C=m(e),v=o(e,"H1",{id:!0});var Tt=i(v);se=l(Tt,"\u6807\u51C6\u5316\u548C\u9884\u6807\u8BB0\u5316"),Tt.forEach(s),M=m(e),g(E.$$.fragment,e),z=m(e),A=o(e,"P",{});var bl=i(A);gs=l(bl,"\u5728\u6211\u4EEC\u66F4\u6DF1\u5165\u5730\u7814\u7A76\u4E0E Transformer \u6A21\u578B\uFF08\u5B57\u8282\u5BF9\u7F16\u7801 [BPE]\u3001WordPiece \u548C Unigram\uFF09\u4E00\u8D77\u4F7F\u7528\u7684\u4E09\u79CD\u6700\u5E38\u89C1\u7684\u5B50\u8BCD\u6807\u8BB0\u5316\u7B97\u6CD5\u4E4B\u524D\uFF0C\u6211\u4EEC\u5C06\u9996\u5148\u770B\u4E00\u4E0B\u6BCF\u4E2A\u6807\u8BB0\u5668\u5E94\u7528\u4E8E\u6587\u672C\u7684\u9884\u5904\u7406\u3002\u4EE5\u4E0B\u662F\u6807\u8BB0\u5316\u7BA1\u9053\u4E2D\u6B65\u9AA4\u7684\u9AD8\u7EA7\u6982\u8FF0\uFF1A"),bl.forEach(s),jt=m(e),D=o(e,"DIV",{class:!0});var ps=i(D);re=o(ps,"IMG",{class:!0,src:!0,alt:!0}),ds=m(ps),le=o(ps,"IMG",{class:!0,src:!0,alt:!0}),ps.forEach(s),wt=m(e),N=o(e,"P",{});var Ze=i(N);ks=l(Ze,"\u5728\u5C06\u6587\u672C\u62C6\u5206\u4E3A\u5B50\u6807\u8BB0\u4E4B\u524D\uFF08\u6839\u636E\u5176\u6A21\u578B\uFF09\uFF0C\u5206\u8BCD\u5668\u6267\u884C\u4E24\u4E2A\u6B65\u9AA4\uFF1A "),st=o(Ze,"EM",{});var $l=i(st);bs=l($l,"normalization"),$l.forEach(s),$s=l(Ze," \u548C "),rt=o(Ze,"EM",{});var vl=i(rt);vs=l(vl,"pre-tokenization"),vl.forEach(s),Es=l(Ze,"."),Ze.forEach(s),yt=m(e),ne=o(e,"H2",{id:!0});var El=i(ne);Ts=l(El,"\u6B63\u5E38\u5316"),El.forEach(s),Pt=m(e),g(ae.$$.fragment,e),xt=m(e),U=o(e,"P",{});var fs=i(U);zs=l(fs,"\u6807\u51C6\u5316\u6B65\u9AA4\u6D89\u53CA\u4E00\u4E9B\u5E38\u89C4\u6E05\u7406\uFF0C\u4F8B\u5982\u5220\u9664\u4E0D\u5FC5\u8981\u7684\u7A7A\u683C\u3001\u5C0F\u5199\u548C/\u6216\u5220\u9664\u91CD\u97F3\u7B26\u53F7\u3002\u5982\u679C\u4F60\u719F\u6089"),oe=o(fs,"A",{href:!0,rel:!0});var Tl=i(oe);js=l(Tl,"Unicode normalization"),Tl.forEach(s),ws=l(fs,"\uFF08\u4F8B\u5982 NFC \u6216 NFKC\uFF09\uFF0C\u8FD9\u4E5F\u662F tokenizer \u53EF\u80FD\u5E94\u7528\u7684\u4E1C\u897F\u3002"),fs.forEach(s),St=m(e),H=o(e,"P",{});var et=i(H);ys=l(et,"\u{1F917}Transformers "),lt=o(et,"STRONG",{});var zl=i(lt);Ps=l(zl,"tokenizer"),zl.forEach(s),xs=l(et," \u6709\u4E00\u4E2A\u5C5E\u6027\u53EB\u505A "),nt=o(et,"STRONG",{});var jl=i(nt);Ss=l(jl,"backend_tokenizer"),jl.forEach(s),Rs=l(et," \u5B83\u63D0\u4F9B\u4E86\u5BF9 \u{1F917} Tokenizers \u5E93\u4E2D\u5E95\u5C42\u6807\u8BB0\u5668\u7684\u8BBF\u95EE\uFF1A"),et.forEach(s),Rt=m(e),g(ie.$$.fragment,e),Gt=m(e),g(pe.$$.fragment,e),Nt=m(e),j=o(e,"P",{});var Te=i(j);at=o(Te,"STRONG",{});var wl=i(at);Gs=l(wl,"normalizer"),wl.forEach(s),Ns=l(Te," \u7684\u5C5E\u6027 "),ot=o(Te,"STRONG",{});var yl=i(ot);Hs=l(yl,"tokenizer"),yl.forEach(s),qs=l(Te," \u5BF9\u8C61\u6709\u4E00\u4E2A "),it=o(Te,"STRONG",{});var Pl=i(it);Bs=l(Pl,"normalize_str()"),Pl.forEach(s),Os=l(Te," \u6211\u4EEC\u53EF\u4EE5\u7528\u6765\u67E5\u770B\u6807\u51C6\u5316\u662F\u5982\u4F55\u6267\u884C\u7684\u65B9\u6CD5\uFF1A"),Te.forEach(s),Ht=m(e),g(fe.$$.fragment,e),qt=m(e),g(me.$$.fragment,e),Bt=m(e),F=o(e,"P",{});var ms=i(F);As=l(ms,"\u5728\u8FD9\u4E2A\u4F8B\u5B50\u4E2D\uFF0C\u56E0\u4E3A\u6211\u4EEC\u9009\u62E9\u4E86 "),pt=o(ms,"STRONG",{});var xl=i(pt);Ds=l(xl,"bert-base-uncased"),xl.forEach(s),Cs=l(ms," \u68C0\u67E5\u70B9\uFF0C\u6807\u51C6\u5316\u5E94\u7528\u5C0F\u5199\u5E76\u5220\u9664\u91CD\u97F3\u3002"),ms.forEach(s),Ot=m(e),g(I.$$.fragment,e),At=m(e),ue=o(e,"H2",{id:!0});var Sl=i(ue);Ms=l(Sl,"\u9884\u6807\u8BB0\u5316"),Sl.forEach(s),Dt=m(e),g(ce.$$.fragment,e),Ct=m(e),W=o(e,"P",{});var us=i(W);Us=l(us,"\u6B63\u5982\u6211\u4EEC\u5C06\u5728\u4E0B\u4E00\u8282\u4E2D\u770B\u5230\u7684\uFF0C\u5206\u8BCD\u5668\u4E0D\u80FD\u5355\u72EC\u5728\u539F\u59CB\u6587\u672C\u4E0A\u8FDB\u884C\u8BAD\u7EC3\u3002\u76F8\u53CD\uFF0C\u6211\u4EEC\u9996\u5148\u9700\u8981\u5C06\u6587\u672C\u62C6\u5206\u4E3A\u5C0F\u5B9E\u4F53\uFF0C\u4F8B\u5982\u5355\u8BCD\u3002\u8FD9\u5C31\u662F\u9884\u6807\u8BB0\u5316\u6B65\u9AA4\u7684\u7528\u6B66\u4E4B\u5730\u3002 \u6B63\u5982\u6211\u4EEC\u5728"),je=o(us,"A",{href:!0});var Rl=i(je);Fs=l(Rl,"Chapter 2"),Rl.forEach(s),Is=l(us,", \u57FA\u4E8E\u5355\u8BCD\u7684\u6807\u8BB0\u5668\u53EF\u4EE5\u7B80\u5355\u5730\u5C06\u539F\u59CB\u6587\u672C\u62C6\u5206\u4E3A\u7A7A\u767D\u548C\u6807\u70B9\u7B26\u53F7\u7684\u5355\u8BCD\u3002\u8FD9\u4E9B\u8BCD\u5C06\u662F\u5206\u8BCD\u5668\u5728\u8BAD\u7EC3\u671F\u95F4\u53EF\u4EE5\u5B66\u4E60\u7684\u5B50\u6807\u8BB0\u7684\u8FB9\u754C\u3002"),us.forEach(s),Mt=m(e),T=o(e,"P",{});var K=i(T);Ws=l(K,"\u8981\u67E5\u770B\u5FEB\u901F\u5206\u8BCD\u5668\u5982\u4F55\u6267\u884C\u9884\u5206\u8BCD\uFF0C\u6211\u4EEC\u53EF\u4EE5\u4F7F\u7528 "),ft=o(K,"STRONG",{});var Gl=i(ft);Ls=l(Gl,"pre_tokenize_str()"),Gl.forEach(s),Js=l(K," \u7684\u65B9\u6CD5 "),mt=o(K,"STRONG",{});var Nl=i(mt);Ys=l(Nl,"pre_tokenizer"),Nl.forEach(s),Ks=l(K," \u7684\u5C5E\u6027 "),ut=o(K,"STRONG",{});var Hl=i(ut);Vs=l(Hl,"tokenizer"),Hl.forEach(s),Xs=l(K," \u76EE\u7684\uFF1A"),K.forEach(s),Ut=m(e),g(he.$$.fragment,e),Ft=m(e),g(_e.$$.fragment,e),It=m(e),q=o(e,"P",{});var tt=i(q);Qs=l(tt,"\u8BF7\u6CE8\u610F\u5206\u8BCD\u5668\u5982\u4F55\u5DF2\u7ECF\u8DDF\u8E2A\u504F\u79FB\u91CF\uFF0C\u8FD9\u5C31\u662F\u5B83\u5982\u4F55\u4E3A\u6211\u4EEC\u63D0\u4F9B\u4E0A\u4E00\u8282\u4E2D\u4F7F\u7528\u7684\u504F\u79FB\u91CF\u6620\u5C04\u3002\u8FD9\u91CC\u5206\u8BCD\u5668\u5FFD\u7565\u4E86\u8FD9\u4E24\u4E2A\u7A7A\u683C\uFF0C\u53EA\u7528\u4E00\u4E2A\u66FF\u6362\u5B83\u4EEC\uFF0C\u4F46\u504F\u79FB\u91CF\u5728 "),ct=o(tt,"STRONG",{});var ql=i(ct);Zs=l(ql,"are"),ql.forEach(s),er=l(tt," \u548C "),ht=o(tt,"STRONG",{});var Bl=i(ht);tr=l(Bl,"you"),Bl.forEach(s),sr=l(tt," \u8003\u8651\u5230\u8FD9\u4E00\u70B9\u3002"),tt.forEach(s),Wt=m(e),we=o(e,"P",{});var Ol=i(we);rr=l(Ol,"\u7531\u4E8E\u6211\u4EEC\u4F7F\u7528\u7684\u662F BERT \u5206\u8BCD\u5668\uFF0C\u9884\u5206\u8BCD\u6D89\u53CA\u5BF9\u7A7A\u683C\u548C\u6807\u70B9\u7B26\u53F7\u8FDB\u884C\u62C6\u5206\u3002\u5BF9\u4E8E\u8FD9\u4E00\u6B65\uFF0C\u5176\u4ED6\u6807\u8BB0\u5668\u53EF\u4EE5\u6709\u4E0D\u540C\u7684\u89C4\u5219\u3002\u4F8B\u5982\uFF0C\u5982\u679C\u6211\u4EEC\u4F7F\u7528 GPT-2 \u6807\u8BB0\u5668\uFF1A"),Ol.forEach(s),Lt=m(e),g(ge.$$.fragment,e),Jt=m(e),L=o(e,"P",{});var cs=i(L);lr=l(cs,"\u5B83\u4E5F\u4F1A\u5728\u7A7A\u683C\u548C\u6807\u70B9\u7B26\u53F7\u4E0A\u62C6\u5206\uFF0C\u4F46\u5B83\u4F1A\u4FDD\u7559\u7A7A\u683C\u5E76\u5C06\u5B83\u4EEC\u66FF\u6362\u4E3A "),_t=o(cs,"STRONG",{});var Al=i(_t);nr=l(Al,"\u0120"),Al.forEach(s),ar=l(cs," \u7B26\u53F7\uFF0C\u5982\u679C\u6211\u4EEC\u89E3\u7801\u4EE4\u724C\uFF0C\u5219\u4F7F\u5176\u80FD\u591F\u6062\u590D\u539F\u59CB\u7A7A\u683C\uFF1A"),cs.forEach(s),Yt=m(e),g(de.$$.fragment,e),Kt=m(e),ye=o(e,"P",{});var Dl=i(ye);or=l(Dl,"\u53E6\u8BF7\u6CE8\u610F\uFF0C\u4E0E BERT \u5206\u8BCD\u5668\u4E0D\u540C\uFF0C\u6B64\u5206\u8BCD\u5668\u4E0D\u4F1A\u5FFD\u7565\u53CC\u7A7A\u683C"),Dl.forEach(s),Vt=m(e),Pe=o(e,"P",{});var Cl=i(Pe);ir=l(Cl,"\u6700\u540E\u4E00\u4E2A\u4F8B\u5B50\uFF0C\u8BA9\u6211\u4EEC\u770B\u4E00\u4E0B\u57FA\u4E8E SentencePiece \u7B97\u6CD5\u7684 T5 \u5206\u8BCD\u5668\uFF1A"),Cl.forEach(s),Xt=m(e),g(ke.$$.fragment,e),Qt=m(e),g(be.$$.fragment,e),Zt=m(e),h=o(e,"P",{});var B=i(h);pr=l(B,"\u4E0E GPT-2 \u6807\u8BB0\u5668\u4E00\u6837\uFF0C\u8FD9\u4E2A\u6807\u8BB0\u5668\u4FDD\u7559\u7A7A\u683C\u5E76\u7528\u7279\u5B9A\u6807\u8BB0\u66FF\u6362\u5B83\u4EEC\uFF08 "),gt=o(B,"STRONG",{});var Ml=i(gt);fr=l(Ml,"_"),Ml.forEach(s),mr=l(B," )\uFF0C\u4F46 T5 \u5206\u8BCD\u5668\u53EA\u5728\u7A7A\u683C\u4E0A\u62C6\u5206\uFF0C\u800C\u4E0D\u662F\u6807\u70B9\u7B26\u53F7\u3002\u8FD8\u8981\u6CE8\u610F\uFF0C\u5B83\u9ED8\u8BA4\u5728\u53E5\u5B50\u7684\u5F00\u5934\u6DFB\u52A0\u4E86\u4E00\u4E2A\u7A7A\u683C\uFF08\u4E4B\u524D "),dt=o(B,"STRONG",{});var Ul=i(dt);ur=l(Ul,"Hello"),Ul.forEach(s),cr=l(B," ) \u5E76\u5FFD\u7565\u4E86\u4E4B\u95F4\u7684\u53CC\u7A7A\u683C "),kt=o(B,"STRONG",{});var Fl=i(kt);hr=l(Fl,"are"),Fl.forEach(s),_r=l(B," \u548C "),bt=o(B,"STRONG",{});var Il=i(bt);gr=l(Il,"you"),Il.forEach(s),dr=l(B," ."),B.forEach(s),es=m(e),xe=o(e,"P",{});var Wl=i(xe);kr=l(Wl,"\u73B0\u5728\u6211\u4EEC\u5DF2\u7ECF\u4E86\u89E3\u4E86\u4E00\u4E9B\u4E0D\u540C\u7684\u6807\u8BB0\u5668\u5982\u4F55\u5904\u7406\u6587\u672C\uFF0C\u6211\u4EEC\u53EF\u4EE5\u5F00\u59CB\u63A2\u7D22\u5E95\u5C42\u7B97\u6CD5\u672C\u8EAB\u3002\u6211\u4EEC\u9996\u5148\u5FEB\u901F\u6D4F\u89C8\u4E00\u4E0B\u5E7F\u6CDB\u9002\u7528\u7684 SentencePiece\uFF1B\u7136\u540E\uFF0C\u5728\u63A5\u4E0B\u6765\u7684\u4E09\u4E2A\u90E8\u5206\u4E2D\uFF0C\u6211\u4EEC\u5C06\u7814\u7A76\u7528\u4E8E\u5B50\u8BCD\u6807\u8BB0\u5316\u7684\u4E09\u79CD\u4E3B\u8981\u7B97\u6CD5\u662F\u5982\u4F55\u5DE5\u4F5C\u7684\u3002"),Wl.forEach(s),ts=m(e),$e=o(e,"H2",{id:!0});var Ll=i($e);br=l(Ll,"\u53E5\u5B50"),Ll.forEach(s),ss=m(e),w=o(e,"P",{});var ze=i(w);ve=o(ze,"A",{href:!0,rel:!0});var Jl=i(ve);$r=l(Jl,"SentencePiece"),Jl.forEach(s),vr=l(ze," \u662F\u4E00\u79CD\u7528\u4E8E\u6587\u672C\u9884\u5904\u7406\u7684\u6807\u8BB0\u5316\u7B97\u6CD5\uFF0C\u60A8\u53EF\u4EE5\u5C06\u5176\u4E0E\u6211\u4EEC\u5C06\u5728\u63A5\u4E0B\u6765\u7684\u4E09\u4E2A\u90E8\u5206\u4E2D\u770B\u5230\u7684\u4EFB\u4F55\u6A21\u578B\u4E00\u8D77\u4F7F\u7528\u3002\u5B83\u5C06\u6587\u672C\u89C6\u4E3A Unicode \u5B57\u7B26\u5E8F\u5217\uFF0C\u5E76\u7528\u7279\u6B8A\u5B57\u7B26\u66FF\u6362\u7A7A\u683C\uFF0C "),$t=o(ze,"STRONG",{});var Yl=i($t);Er=l(Yl,"\u2581"),Yl.forEach(s),Tr=l(ze," .\u4E0E Unigram \u7B97\u6CD5\u7ED3\u5408\u4F7F\u7528\uFF08\u53C2\u89C1"),Se=o(ze,"A",{href:!0});var Kl=i(Se);zr=l(Kl,"section 7"),Kl.forEach(s),jr=l(ze,"), \u5B83\u751A\u81F3\u4E0D\u9700\u8981\u9884\u6807\u8BB0\u5316\u6B65\u9AA4\uFF0C\u8FD9\u5BF9\u4E8E\u4E0D\u4F7F\u7528\u7A7A\u683C\u5B57\u7B26\u7684\u8BED\u8A00\uFF08\u5982\u4E2D\u6587\u6216\u65E5\u8BED\uFF09\u975E\u5E38\u6709\u7528\u3002"),ze.forEach(s),rs=m(e),J=o(e,"P",{});var hs=i(J);wr=l(hs,"SentencePiece \u7684\u53E6\u4E00\u4E2A\u4E3B\u8981\u7279\u70B9\u662F\u53EF\u9006\u6807\u8BB0\u5316\uFF1A\u7531\u4E8E\u6CA1\u6709\u5BF9\u7A7A\u683C\u8FDB\u884C\u7279\u6B8A\u5904\u7406\uFF0C\u56E0\u6B64\u53EA\u9700\u901A\u8FC7\u5C06\u5B83\u4EEC\u8FDE\u63A5\u8D77\u6765\u5E76\u66FF\u6362 "),vt=o(hs,"STRONG",{});var Vl=i(vt);yr=l(Vl,"_"),Vl.forEach(s),Pr=l(hs," s \u5E26\u7A7A\u683C\u2014\u2014\u8FD9\u4F1A\u5BFC\u81F4\u6807\u51C6\u5316\u7684\u6587\u672C\u3002\u6B63\u5982\u6211\u4EEC\u4E4B\u524D\u770B\u5230\u7684\uFF0CBERT \u5206\u8BCD\u5668\u5220\u9664\u4E86\u91CD\u590D\u7684\u7A7A\u683C\uFF0C\u56E0\u6B64\u5B83\u7684\u5206\u8BCD\u662F\u4E0D\u53EF\u9006\u7684\u3002"),hs.forEach(s),ls=m(e),Ee=o(e,"H2",{id:!0});var Xl=i(Ee);xr=l(Xl,"\u7B97\u6CD5\u6982\u8FF0"),Xl.forEach(s),ns=m(e),Re=o(e,"P",{});var Ql=i(Re);Sr=l(Ql,"\u5728\u4E0B\u9762\u7684\u90E8\u5206\u4E2D\uFF0C\u6211\u4EEC\u5C06\u6DF1\u5165\u7814\u7A76\u4E09\u79CD\u4E3B\u8981\u7684\u5B50\u8BCD\u6807\u8BB0\u5316\u7B97\u6CD5\uFF1ABPE\uFF08\u7531 GPT-2 \u548C\u5176\u4ED6\u4EBA\u4F7F\u7528\uFF09\u3001WordPiece\uFF08\u4F8B\u5982\u7531 BERT \u4F7F\u7528\uFF09\u548C Unigram\uFF08\u7531 T5 \u548C\u5176\u4ED6\u4EBA\u4F7F\u7528\uFF09\u3002\u5728\u6211\u4EEC\u5F00\u59CB\u4E4B\u524D\uFF0C\u8FD9\u91CC\u662F\u5B83\u4EEC\u5404\u81EA\u5DE5\u4F5C\u539F\u7406\u7684\u5FEB\u901F\u6982\u8FF0\u3002\u5982\u679C\u60A8\u8FD8\u6CA1\u6709\u7406\u89E3\uFF0C\u8BF7\u5728\u9605\u8BFB\u4E0B\u4E00\u8282\u540E\u7ACB\u5373\u56DE\u5230\u6B64\u8868\u3002"),Ql.forEach(s),as=m(e),Y=o(e,"TABLE",{});var _s=i(Y);Et=o(_s,"THEAD",{});var Zl=i(Et);y=o(Zl,"TR",{});var V=i(y);Ge=o(V,"TH",{align:!0});var en=i(Ge);Rr=l(en,"Model"),en.forEach(s),Gr=m(V),Ne=o(V,"TH",{align:!0});var tn=i(Ne);Nr=l(tn,"BPE"),tn.forEach(s),Hr=m(V),He=o(V,"TH",{align:!0});var sn=i(He);qr=l(sn,"WordPiece"),sn.forEach(s),Br=m(V),qe=o(V,"TH",{align:!0});var rn=i(qe);Or=l(rn,"Unigram"),rn.forEach(s),V.forEach(s),Zl.forEach(s),Ar=m(_s),P=o(_s,"TBODY",{});var X=i(P);x=o(X,"TR",{});var Q=i(x);Be=o(Q,"TD",{align:!0});var ln=i(Be);Dr=l(ln,"Training"),ln.forEach(s),Cr=m(Q),Oe=o(Q,"TD",{align:!0});var nn=i(Oe);Mr=l(nn,"Starts from a small vocabulary and learns rules to merge tokens"),nn.forEach(s),Ur=m(Q),Ae=o(Q,"TD",{align:!0});var an=i(Ae);Fr=l(an,"Starts from a small vocabulary and learns rules to merge tokens"),an.forEach(s),Ir=m(Q),De=o(Q,"TD",{align:!0});var on=i(De);Wr=l(on,"Starts from a large vocabulary and learns rules to remove tokens"),on.forEach(s),Q.forEach(s),Lr=m(X),S=o(X,"TR",{});var Z=i(S);Ce=o(Z,"TD",{align:!0});var pn=i(Ce);Jr=l(pn,"Training step"),pn.forEach(s),Yr=m(Z),Me=o(Z,"TD",{align:!0});var fn=i(Me);Kr=l(fn,"Merges the tokens corresponding to the most common pair"),fn.forEach(s),Vr=m(Z),Ue=o(Z,"TD",{align:!0});var mn=i(Ue);Xr=l(mn,"Merges the tokens corresponding to the pair with the best score based on the frequency of the pair, privileging pairs where each individual token is less frequent"),mn.forEach(s),Qr=m(Z),Fe=o(Z,"TD",{align:!0});var un=i(Fe);Zr=l(un,"Removes all the tokens in the vocabulary that will minimize the loss computed on the whole corpus"),un.forEach(s),Z.forEach(s),el=m(X),R=o(X,"TR",{});var ee=i(R);Ie=o(ee,"TD",{align:!0});var cn=i(Ie);tl=l(cn,"Learns"),cn.forEach(s),sl=m(ee),We=o(ee,"TD",{align:!0});var hn=i(We);rl=l(hn,"Merge rules and a vocabulary"),hn.forEach(s),ll=m(ee),Le=o(ee,"TD",{align:!0});var _n=i(Le);nl=l(_n,"Just a vocabulary"),_n.forEach(s),al=m(ee),Je=o(ee,"TD",{align:!0});var gn=i(Je);ol=l(gn,"A vocabulary with a score for each token"),gn.forEach(s),ee.forEach(s),il=m(X),G=o(X,"TR",{});var te=i(G);Ye=o(te,"TD",{align:!0});var dn=i(Ye);pl=l(dn,"Encoding"),dn.forEach(s),fl=m(te),Ke=o(te,"TD",{align:!0});var kn=i(Ke);ml=l(kn,"Splits a word into characters and applies the merges learned during training"),kn.forEach(s),ul=m(te),Ve=o(te,"TD",{align:!0});var bn=i(Ve);cl=l(bn,"Finds the longest subword starting from the beginning that is in the vocabulary, then does the same for the rest of the word"),bn.forEach(s),hl=m(te),Xe=o(te,"TD",{align:!0});var $n=i(Xe);_l=l($n,"Finds the most likely split into tokens, using the scores learned during training"),$n.forEach(s),te.forEach(s),X.forEach(s),_s.forEach(s),os=m(e),Qe=o(e,"P",{});var vn=i(Qe);gl=l(vn,"\u73B0\u5728\u8BA9\u6211\u4EEC\u6DF1\u5165\u4E86\u89E3 BPE\uFF01"),vn.forEach(s),this.h()},h(){u(c,"name","hf:doc:metadata"),u(c,"content",JSON.stringify(Nn)),u(v,"id",""),u(re,"class","block dark:hidden"),En(re.src,dl="https://huggingface.co/datasets/huggingface-course/documentation-images/resolve/main/en/chapter6/tokenization_pipeline.svg")||u(re,"src",dl),u(re,"alt","The tokenization pipeline."),u(le,"class","hidden dark:block"),En(le.src,kl="https://huggingface.co/datasets/huggingface-course/documentation-images/resolve/main/en/chapter6/tokenization_pipeline-dark.svg")||u(le,"src",kl),u(le,"alt","The tokenization pipeline."),u(D,"class","flex justify-center"),u(ne,"id",""),u(oe,"href","http://www.unicode.org/reports/tr15/"),u(oe,"rel","nofollow"),u(ue,"id",""),u(je,"href","/course/chapter2"),u($e,"id",""),u(ve,"href","https://github.com/google/sentencepiece"),u(ve,"rel","nofollow"),u(Se,"href","/course/chapter7/7"),u(Ee,"id",""),u(Ge,"align","center"),u(Ne,"align","center"),u(He,"align","center"),u(qe,"align","center"),u(Be,"align","center"),u(Oe,"align","center"),u(Ae,"align","center"),u(De,"align","center"),u(Ce,"align","center"),u(Me,"align","center"),u(Ue,"align","center"),u(Fe,"align","center"),u(Ie,"align","center"),u(We,"align","center"),u(Le,"align","center"),u(Je,"align","center"),u(Ye,"align","center"),u(Ke,"align","center"),u(Ve,"align","center"),u(Xe,"align","center")},m(e,n){t(document.head,c),p(e,C,n),p(e,v,n),t(v,se),p(e,M,n),d(E,e,n),p(e,z,n),p(e,A,n),t(A,gs),p(e,jt,n),p(e,D,n),t(D,re),t(D,ds),t(D,le),p(e,wt,n),p(e,N,n),t(N,ks),t(N,st),t(st,bs),t(N,$s),t(N,rt),t(rt,vs),t(N,Es),p(e,yt,n),p(e,ne,n),t(ne,Ts),p(e,Pt,n),d(ae,e,n),p(e,xt,n),p(e,U,n),t(U,zs),t(U,oe),t(oe,js),t(U,ws),p(e,St,n),p(e,H,n),t(H,ys),t(H,lt),t(lt,Ps),t(H,xs),t(H,nt),t(nt,Ss),t(H,Rs),p(e,Rt,n),d(ie,e,n),p(e,Gt,n),d(pe,e,n),p(e,Nt,n),p(e,j,n),t(j,at),t(at,Gs),t(j,Ns),t(j,ot),t(ot,Hs),t(j,qs),t(j,it),t(it,Bs),t(j,Os),p(e,Ht,n),d(fe,e,n),p(e,qt,n),d(me,e,n),p(e,Bt,n),p(e,F,n),t(F,As),t(F,pt),t(pt,Ds),t(F,Cs),p(e,Ot,n),d(I,e,n),p(e,At,n),p(e,ue,n),t(ue,Ms),p(e,Dt,n),d(ce,e,n),p(e,Ct,n),p(e,W,n),t(W,Us),t(W,je),t(je,Fs),t(W,Is),p(e,Mt,n),p(e,T,n),t(T,Ws),t(T,ft),t(ft,Ls),t(T,Js),t(T,mt),t(mt,Ys),t(T,Ks),t(T,ut),t(ut,Vs),t(T,Xs),p(e,Ut,n),d(he,e,n),p(e,Ft,n),d(_e,e,n),p(e,It,n),p(e,q,n),t(q,Qs),t(q,ct),t(ct,Zs),t(q,er),t(q,ht),t(ht,tr),t(q,sr),p(e,Wt,n),p(e,we,n),t(we,rr),p(e,Lt,n),d(ge,e,n),p(e,Jt,n),p(e,L,n),t(L,lr),t(L,_t),t(_t,nr),t(L,ar),p(e,Yt,n),d(de,e,n),p(e,Kt,n),p(e,ye,n),t(ye,or),p(e,Vt,n),p(e,Pe,n),t(Pe,ir),p(e,Xt,n),d(ke,e,n),p(e,Qt,n),d(be,e,n),p(e,Zt,n),p(e,h,n),t(h,pr),t(h,gt),t(gt,fr),t(h,mr),t(h,dt),t(dt,ur),t(h,cr),t(h,kt),t(kt,hr),t(h,_r),t(h,bt),t(bt,gr),t(h,dr),p(e,es,n),p(e,xe,n),t(xe,kr),p(e,ts,n),p(e,$e,n),t($e,br),p(e,ss,n),p(e,w,n),t(w,ve),t(ve,$r),t(w,vr),t(w,$t),t($t,Er),t(w,Tr),t(w,Se),t(Se,zr),t(w,jr),p(e,rs,n),p(e,J,n),t(J,wr),t(J,vt),t(vt,yr),t(J,Pr),p(e,ls,n),p(e,Ee,n),t(Ee,xr),p(e,ns,n),p(e,Re,n),t(Re,Sr),p(e,as,n),p(e,Y,n),t(Y,Et),t(Et,y),t(y,Ge),t(Ge,Rr),t(y,Gr),t(y,Ne),t(Ne,Nr),t(y,Hr),t(y,He),t(He,qr),t(y,Br),t(y,qe),t(qe,Or),t(Y,Ar),t(Y,P),t(P,x),t(x,Be),t(Be,Dr),t(x,Cr),t(x,Oe),t(Oe,Mr),t(x,Ur),t(x,Ae),t(Ae,Fr),t(x,Ir),t(x,De),t(De,Wr),t(P,Lr),t(P,S),t(S,Ce),t(Ce,Jr),t(S,Yr),t(S,Me),t(Me,Kr),t(S,Vr),t(S,Ue),t(Ue,Xr),t(S,Qr),t(S,Fe),t(Fe,Zr),t(P,el),t(P,R),t(R,Ie),t(Ie,tl),t(R,sl),t(R,We),t(We,rl),t(R,ll),t(R,Le),t(Le,nl),t(R,al),t(R,Je),t(Je,ol),t(P,il),t(P,G),t(G,Ye),t(Ye,pl),t(G,fl),t(G,Ke),t(Ke,ml),t(G,ul),t(G,Ve),t(Ve,cl),t(G,hl),t(G,Xe),t(Xe,_l),p(e,os,n),p(e,Qe,n),t(Qe,gl),is=!0},p(e,[n]){const Tt={};n&2&&(Tt.$$scope={dirty:n,ctx:e}),I.$set(Tt)},i(e){is||(k(E.$$.fragment,e),k(ae.$$.fragment,e),k(ie.$$.fragment,e),k(pe.$$.fragment,e),k(fe.$$.fragment,e),k(me.$$.fragment,e),k(I.$$.fragment,e),k(ce.$$.fragment,e),k(he.$$.fragment,e),k(_e.$$.fragment,e),k(ge.$$.fragment,e),k(de.$$.fragment,e),k(ke.$$.fragment,e),k(be.$$.fragment,e),is=!0)},o(e){b(E.$$.fragment,e),b(ae.$$.fragment,e),b(ie.$$.fragment,e),b(pe.$$.fragment,e),b(fe.$$.fragment,e),b(me.$$.fragment,e),b(I.$$.fragment,e),b(ce.$$.fragment,e),b(he.$$.fragment,e),b(_e.$$.fragment,e),b(ge.$$.fragment,e),b(de.$$.fragment,e),b(ke.$$.fragment,e),b(be.$$.fragment,e),is=!1},d(e){s(c),e&&s(C),e&&s(v),e&&s(M),$(E,e),e&&s(z),e&&s(A),e&&s(jt),e&&s(D),e&&s(wt),e&&s(N),e&&s(yt),e&&s(ne),e&&s(Pt),$(ae,e),e&&s(xt),e&&s(U),e&&s(St),e&&s(H),e&&s(Rt),$(ie,e),e&&s(Gt),$(pe,e),e&&s(Nt),e&&s(j),e&&s(Ht),$(fe,e),e&&s(qt),$(me,e),e&&s(Bt),e&&s(F),e&&s(Ot),$(I,e),e&&s(At),e&&s(ue),e&&s(Dt),$(ce,e),e&&s(Ct),e&&s(W),e&&s(Mt),e&&s(T),e&&s(Ut),$(he,e),e&&s(Ft),$(_e,e),e&&s(It),e&&s(q),e&&s(Wt),e&&s(we),e&&s(Lt),$(ge,e),e&&s(Jt),e&&s(L),e&&s(Yt),$(de,e),e&&s(Kt),e&&s(ye),e&&s(Vt),e&&s(Pe),e&&s(Xt),$(ke,e),e&&s(Qt),$(be,e),e&&s(Zt),e&&s(h),e&&s(es),e&&s(xe),e&&s(ts),e&&s($e),e&&s(ss),e&&s(w),e&&s(rs),e&&s(J),e&&s(ls),e&&s(Ee),e&&s(ns),e&&s(Re),e&&s(as),e&&s(Y),e&&s(os),e&&s(Qe)}}}const Nn={local:"",sections:[{local:"",title:"\u6B63\u5E38\u5316"},{local:"",title:"\u9884\u6807\u8BB0\u5316"},{local:"",title:"\u53E5\u5B50"},{local:"",title:"\u7B97\u6CD5\u6982\u8FF0"}],title:"\u6807\u51C6\u5316\u548C\u9884\u6807\u8BB0\u5316"};function Hn(zt){return Pn(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class Mn extends zn{constructor(c){super();jn(this,c,Hn,Gn,wn,{})}}export{Mn as default,Nn as metadata};
