import{S as lr,i as pr,s as tr,e as r,k as u,t as p,w as j,M as er,c as i,d as a,m as c,a as o,h as t,x,b as U,G as n,g as e,y as g,q as m,o as $,B as _,v as rr}from"../../chunks/vendor-hf-doc-builder.js";import{T as ca}from"../../chunks/Tip-hf-doc-builder.js";import{Y as ir}from"../../chunks/Youtube-hf-doc-builder.js";import{I as or}from"../../chunks/IconCopyLink-hf-doc-builder.js";import{C as q}from"../../chunks/CodeBlock-hf-doc-builder.js";import{C as ur}from"../../chunks/CourseFloatingBanner-hf-doc-builder.js";import"../../chunks/DocNotebookDropdown-hf-doc-builder.js";function cr(O){let h,E;return{c(){h=r("p"),E=p("\u{1F4A1} \u672C\u8282\u6DF1\u5165\u4ECB\u7ECD\u4E86BPE,\u751A\u81F3\u5C55\u793A\u4E86\u4E00\u4E2A\u5B8C\u6574\u7684\u5B9E\u73B0\u3002\u5982\u679C\u4F60\u53EA\u60F3\u5927\u81F4\u4E86\u89E3\u6807\u8BB0\u5316\u7B97\u6CD5,\u53EF\u4EE5\u8DF3\u5230\u6700\u540E\u3002")},l(f){h=i(f,"P",{});var w=o(h);E=t(w,"\u{1F4A1} \u672C\u8282\u6DF1\u5165\u4ECB\u7ECD\u4E86BPE,\u751A\u81F3\u5C55\u793A\u4E86\u4E00\u4E2A\u5B8C\u6574\u7684\u5B9E\u73B0\u3002\u5982\u679C\u4F60\u53EA\u60F3\u5927\u81F4\u4E86\u89E3\u6807\u8BB0\u5316\u7B97\u6CD5,\u53EF\u4EE5\u8DF3\u5230\u6700\u540E\u3002"),w.forEach(a)},m(f,w){e(f,h,w),n(h,E)},d(f){f&&a(h)}}}function hr(O){let h,E,f,w,y;return{c(){h=r("p"),E=p("TGPT-2 \u548C RoBERTa \u6807\u8BB0\u5668(\u975E\u5E38\u76F8\u4F3C)\u6709\u4E00\u4E2A\u806A\u660E\u7684\u65B9\u6CD5\u6765\u5904\u7406\u8FD9\u4E2A\u95EE\u9898: \u4ED6\u4EEC\u4E0D\u628A\u5355\u8BCD\u770B\u6210\u662F\u7528 Unicode \u5B57\u7B26\u5199\u7684\uFF0C\u800C\u662F\u7528\u5B57\u8282\u5199\u7684\u3002\u8FD9\u6837,\u57FA\u672C\u8BCD\u6C47\u8868\u7684\u5927\u5C0F\u5F88\u5C0F(256),\u4F46\u4F60\u80FD\u60F3\u5230\u7684\u6BCF\u4E2A\u5B57\u7B26\u4ECD\u5C06\u88AB\u5305\u542B\u5728\u5185,\u800C\u4E0D\u4F1A\u6700\u7EC8\u8F6C\u6362\u4E3A\u672A\u77E5\u6807\u8BB0\u3002\u8FD9\u4E2A\u6280\u5DE7\u88AB\u79F0\u4E3A "),f=r("em"),w=p("\u5B57\u8282\u7EA7 BPE"),y=p("\u3002")},l(b){h=i(b,"P",{});var k=o(h);E=t(k,"TGPT-2 \u548C RoBERTa \u6807\u8BB0\u5668(\u975E\u5E38\u76F8\u4F3C)\u6709\u4E00\u4E2A\u806A\u660E\u7684\u65B9\u6CD5\u6765\u5904\u7406\u8FD9\u4E2A\u95EE\u9898: \u4ED6\u4EEC\u4E0D\u628A\u5355\u8BCD\u770B\u6210\u662F\u7528 Unicode \u5B57\u7B26\u5199\u7684\uFF0C\u800C\u662F\u7528\u5B57\u8282\u5199\u7684\u3002\u8FD9\u6837,\u57FA\u672C\u8BCD\u6C47\u8868\u7684\u5927\u5C0F\u5F88\u5C0F(256),\u4F46\u4F60\u80FD\u60F3\u5230\u7684\u6BCF\u4E2A\u5B57\u7B26\u4ECD\u5C06\u88AB\u5305\u542B\u5728\u5185,\u800C\u4E0D\u4F1A\u6700\u7EC8\u8F6C\u6362\u4E3A\u672A\u77E5\u6807\u8BB0\u3002\u8FD9\u4E2A\u6280\u5DE7\u88AB\u79F0\u4E3A "),f=i(k,"EM",{});var T=o(f);w=t(T,"\u5B57\u8282\u7EA7 BPE"),T.forEach(a),y=t(k,"\u3002"),k.forEach(a)},m(b,k){e(b,h,k),n(h,E),n(h,f),n(f,w),n(h,y)},d(b){b&&a(h)}}}function fr(O){let h,E,f,w,y;return{c(){h=r("p"),E=p("\u270F\uFE0F "),f=r("strong"),w=p("\u73B0\u5728\u8F6E\u5230\u4F60\u4E86!"),y=p("\u4F60\u8BA4\u4E3A\u4E0B\u4E00\u4E2A\u5408\u5E76\u89C4\u5219\u662F\u4EC0\u4E48\uFF1F")},l(b){h=i(b,"P",{});var k=o(h);E=t(k,"\u270F\uFE0F "),f=i(k,"STRONG",{});var T=o(f);w=t(T,"\u73B0\u5728\u8F6E\u5230\u4F60\u4E86!"),T.forEach(a),y=t(k,"\u4F60\u8BA4\u4E3A\u4E0B\u4E00\u4E2A\u5408\u5E76\u89C4\u5219\u662F\u4EC0\u4E48\uFF1F"),k.forEach(a)},m(b,k){e(b,h,k),n(h,E),n(h,f),n(f,w),n(h,y)},d(b){b&&a(h)}}}function jr(O){let h,E,f,w,y,b,k,T;return{c(){h=r("p"),E=p("\u270F\uFE0F "),f=r("strong"),w=p("\u73B0\u5728\u8F6E\u5230\u4F60\u4E86!"),y=p(" \u4F60\u8BA4\u4E3A\u8FD9\u4E2A\u8BCD "),b=r("code"),k=p('"unhug"'),T=p(" \u5C06\u5982\u4F55\u88AB\u6807\u8BB0\uFF1F")},l(S){h=i(S,"P",{});var D=o(h);E=t(D,"\u270F\uFE0F "),f=i(D,"STRONG",{});var H=o(f);w=t(H,"\u73B0\u5728\u8F6E\u5230\u4F60\u4E86!"),H.forEach(a),y=t(D," \u4F60\u8BA4\u4E3A\u8FD9\u4E2A\u8BCD "),b=i(D,"CODE",{});var ps=o(b);k=t(ps,'"unhug"'),ps.forEach(a),T=t(D," \u5C06\u5982\u4F55\u88AB\u6807\u8BB0\uFF1F"),D.forEach(a)},m(S,D){e(S,h,D),n(h,E),n(h,f),n(f,w),n(h,y),n(h,b),n(b,k),n(h,T)},d(S){S&&a(h)}}}function xr(O){let h,E,f,w,y;return{c(){h=r("p"),E=p("\u{1F4A1} \u5728\u540C\u4E00\u8BED\u6599\u5E93\u4E0A\u4F7F\u7528 "),f=r("code"),w=p("train_new_from_iterator()"),y=p(" \u4E0D\u4F1A\u4EA7\u751F\u5B8C\u5168\u76F8\u540C\u7684\u8BCD\u6C47\u8868\u3002\u8FD9\u662F\u56E0\u4E3A\u5F53\u6709\u6700\u9891\u7E41\u5BF9\u7684\u9009\u62E9\u65F6,\u6211\u4EEC\u9009\u62E9\u9047\u5230\u7684\u7B2C\u4E00\u4E2A, \u800C \u{1F917} Tokenizers \u5E93\u6839\u636E\u5185\u90E8ID\u9009\u62E9\u7B2C\u4E00\u4E2A\u3002")},l(b){h=i(b,"P",{});var k=o(h);E=t(k,"\u{1F4A1} \u5728\u540C\u4E00\u8BED\u6599\u5E93\u4E0A\u4F7F\u7528 "),f=i(k,"CODE",{});var T=o(f);w=t(T,"train_new_from_iterator()"),T.forEach(a),y=t(k," \u4E0D\u4F1A\u4EA7\u751F\u5B8C\u5168\u76F8\u540C\u7684\u8BCD\u6C47\u8868\u3002\u8FD9\u662F\u56E0\u4E3A\u5F53\u6709\u6700\u9891\u7E41\u5BF9\u7684\u9009\u62E9\u65F6,\u6211\u4EEC\u9009\u62E9\u9047\u5230\u7684\u7B2C\u4E00\u4E2A, \u800C \u{1F917} Tokenizers \u5E93\u6839\u636E\u5185\u90E8ID\u9009\u62E9\u7B2C\u4E00\u4E2A\u3002"),k.forEach(a)},m(b,k){e(b,h,k),n(h,E),n(h,f),n(f,w),n(h,y)},d(b){b&&a(h)}}}function gr(O){let h,E;return{c(){h=r("p"),E=p("\u26A0\uFE0F \u5982\u679C\u5B58\u5728\u672A\u77E5\u5B57\u7B26,\u6211\u4EEC\u7684\u5B9E\u73B0\u5C06\u629B\u51FA\u9519\u8BEF,\u56E0\u4E3A\u6211\u4EEC\u6CA1\u6709\u505A\u4EFB\u4F55\u5904\u7406\u5B83\u4EEC\u3002GPT-2 \u5B9E\u9645\u4E0A\u6CA1\u6709\u672A\u77E5\u6807\u8BB0(\u4F7F\u7528\u5B57\u8282\u7EA7 BPE \u65F6\u4E0D\u53EF\u80FD\u5F97\u5230\u672A\u77E5\u5B57\u7B26),\u4F46\u8FD9\u53EF\u80FD\u53D1\u751F\u5728\u8FD9\u91CC,\u56E0\u4E3A\u6211\u4EEC\u6CA1\u6709\u5728\u521D\u59CB\u8BCD\u6C47\u8868\u4E2D\u5305\u542B\u6240\u6709\u53EF\u80FD\u7684\u5B57\u8282\u3002 BPE \u7684\u8FD9\u65B9\u9762\u8D85\u51FA\u4E86\u672C\u8282\u7684\u8303\u56F4,\u56E0\u6B64\u6211\u4EEC\u5FFD\u7565\u4E86\u7EC6\u8282\u3002")},l(f){h=i(f,"P",{});var w=o(h);E=t(w,"\u26A0\uFE0F \u5982\u679C\u5B58\u5728\u672A\u77E5\u5B57\u7B26,\u6211\u4EEC\u7684\u5B9E\u73B0\u5C06\u629B\u51FA\u9519\u8BEF,\u56E0\u4E3A\u6211\u4EEC\u6CA1\u6709\u505A\u4EFB\u4F55\u5904\u7406\u5B83\u4EEC\u3002GPT-2 \u5B9E\u9645\u4E0A\u6CA1\u6709\u672A\u77E5\u6807\u8BB0(\u4F7F\u7528\u5B57\u8282\u7EA7 BPE \u65F6\u4E0D\u53EF\u80FD\u5F97\u5230\u672A\u77E5\u5B57\u7B26),\u4F46\u8FD9\u53EF\u80FD\u53D1\u751F\u5728\u8FD9\u91CC,\u56E0\u4E3A\u6211\u4EEC\u6CA1\u6709\u5728\u521D\u59CB\u8BCD\u6C47\u8868\u4E2D\u5305\u542B\u6240\u6709\u53EF\u80FD\u7684\u5B57\u8282\u3002 BPE \u7684\u8FD9\u65B9\u9762\u8D85\u51FA\u4E86\u672C\u8282\u7684\u8303\u56F4,\u56E0\u6B64\u6211\u4EEC\u5FFD\u7565\u4E86\u7EC6\u8282\u3002"),w.forEach(a)},m(f,w){e(f,h,w),n(h,E)},d(f){f&&a(h)}}}function mr(O){let h,E,f,w,y,b,k,T,S,D,H,ps,F,rn,ts,Bl,on,Ss,Nl,un,es,cn,V,Gl,ha,Hl,Al,hn,M,fn,K,Rl,fa,Il,Sl,jn,Ls,Ll,xn,Us,Ul,gn,rs,mn,C,Fl,ja,Vl,Ml,xa,Kl,Wl,ga,Yl,Jl,ma,Zl,Ql,$a,Xl,sp,$n,is,_n,P,ap,_a,np,lp,qa,pp,tp,ba,ep,rp,da,ip,op,wa,up,cp,ka,hp,fp,Ea,jp,xp,qn,A,gp,va,mp,$p,ya,_p,qp,bn,os,dn,B,bp,Pa,dp,wp,Ta,kp,Ep,Ca,vp,yp,wn,us,kn,R,Pp,za,Tp,Cp,Oa,zp,Op,En,cs,vn,Fs,Dp,yn,W,Pn,hs,Bp,Tn,Vs,Np,Cn,N,Da,Gp,Hp,Ba,Ap,Rp,Na,Ip,Sp,Ga,Lp,zn,Ms,Up,On,fs,Dn,d,Fp,Ha,Vp,Mp,Aa,Kp,Wp,Ra,Yp,Jp,Ia,Zp,Qp,Sa,Xp,st,La,at,nt,Ua,lt,pt,Fa,tt,et,Va,rt,it,Ma,ot,ut,Ka,ct,ht,Wa,ft,jt,Bn,Y,Nn,L,J,Ya,js,xt,Ja,gt,Gn,Ks,mt,Hn,Ws,$t,An,xs,Rn,Z,_t,Za,qt,bt,In,gs,Sn,Ys,dt,Ln,ms,Un,$s,Fn,Js,wt,Vn,_s,Mn,qs,Kn,Q,kt,Qa,Et,vt,Wn,bs,Yn,Zs,yt,Jn,ds,Zn,Qs,Pt,Qn,ws,Xn,Xs,Tt,sl,ks,al,Es,nl,sa,Ct,ll,vs,pl,ys,tl,I,zt,Xa,Ot,Dt,sn,Bt,Nt,el,Ps,rl,X,Gt,an,Ht,At,il,Ts,ol,aa,Rt,ul,Cs,cl,zs,hl,na,It,fl,Os,jl,la,St,xl,Ds,gl,Bs,ml,pa,Lt,$l,Ns,_l,Gs,ql,ss,bl,ta,Ut,dl,Hs,wl,ea,Ft,kl,As,El,Rs,vl,as,yl,ra,Vt,Pl;return b=new ur({props:{chapter:6,classNames:"absolute z-10 right-0 top-0",notebooks:[{label:"Google Colab",value:"https://colab.research.google.com/github/huggingface/notebooks/blob/master/course/zh-CN/chapter6/section5.ipynb"},{label:"Aws Studio",value:"https://studiolab.sagemaker.aws/import/github/huggingface/notebooks/blob/master/course/zh-CN/chapter6/section5.ipynb"}]}}),H=new ir({props:{id:"HEikzVL-lZU"}}),F=new ca({props:{$$slots:{default:[cr]},$$scope:{ctx:O}}}),es=new q({props:{code:'"hug", "pug", "pun", "bun", "hugs"',highlighted:'<span class="hljs-string">&quot;hug&quot;</span>, <span class="hljs-string">&quot;pug&quot;</span>, <span class="hljs-string">&quot;pun&quot;</span>, <span class="hljs-string">&quot;bun&quot;</span>, <span class="hljs-string">&quot;hugs&quot;</span>'}}),M=new ca({props:{$$slots:{default:[hr]},$$scope:{ctx:O}}}),rs=new q({props:{code:'("hug", 10), ("pug", 5), ("pun", 12), ("bun", 4), ("hugs", 5)',highlighted:'(<span class="hljs-string">&quot;hug&quot;</span><span class="hljs-punctuation">,</span> <span class="hljs-number">10</span>)<span class="hljs-punctuation">,</span> (<span class="hljs-string">&quot;pug&quot;</span><span class="hljs-punctuation">,</span> <span class="hljs-number">5</span>)<span class="hljs-punctuation">,</span> (<span class="hljs-string">&quot;pun&quot;</span><span class="hljs-punctuation">,</span> <span class="hljs-number">12</span>)<span class="hljs-punctuation">,</span> (<span class="hljs-string">&quot;bun&quot;</span><span class="hljs-punctuation">,</span> <span class="hljs-number">4</span>)<span class="hljs-punctuation">,</span> (<span class="hljs-string">&quot;hugs&quot;</span><span class="hljs-punctuation">,</span> <span class="hljs-number">5</span>)'}}),is=new q({props:{code:'("h" "u" "g", 10), ("p" "u" "g", 5), ("p" "u" "n", 12), ("b" "u" "n", 4), ("h" "u" "g" "s", 5)',highlighted:'(<span class="hljs-string">&quot;h&quot;</span> <span class="hljs-string">&quot;u&quot;</span> <span class="hljs-string">&quot;g&quot;</span><span class="hljs-punctuation">,</span> <span class="hljs-number">10</span>)<span class="hljs-punctuation">,</span> (<span class="hljs-string">&quot;p&quot;</span> <span class="hljs-string">&quot;u&quot;</span> <span class="hljs-string">&quot;g&quot;</span><span class="hljs-punctuation">,</span> <span class="hljs-number">5</span>)<span class="hljs-punctuation">,</span> (<span class="hljs-string">&quot;p&quot;</span> <span class="hljs-string">&quot;u&quot;</span> <span class="hljs-string">&quot;n&quot;</span><span class="hljs-punctuation">,</span> <span class="hljs-number">12</span>)<span class="hljs-punctuation">,</span> (<span class="hljs-string">&quot;b&quot;</span> <span class="hljs-string">&quot;u&quot;</span> <span class="hljs-string">&quot;n&quot;</span><span class="hljs-punctuation">,</span> <span class="hljs-number">4</span>)<span class="hljs-punctuation">,</span> (<span class="hljs-string">&quot;h&quot;</span> <span class="hljs-string">&quot;u&quot;</span> <span class="hljs-string">&quot;g&quot;</span> <span class="hljs-string">&quot;s&quot;</span><span class="hljs-punctuation">,</span> <span class="hljs-number">5</span>)'}}),os=new q({props:{code:`Vocabulary: ["b", "g", "h", "n", "p", "s", "u", "ug"]
Corpus: ("h" "ug", 10), ("p" "ug", 5), ("p" "u" "n", 12), ("b" "u" "n", 4), ("h" "ug" "s", 5)`,highlighted:`<span class="hljs-symbol">Vocabulary:</span> [<span class="hljs-string">&quot;b&quot;</span>, <span class="hljs-string">&quot;g&quot;</span>, <span class="hljs-string">&quot;h&quot;</span>, <span class="hljs-string">&quot;n&quot;</span>, <span class="hljs-string">&quot;p&quot;</span>, <span class="hljs-string">&quot;s&quot;</span>, <span class="hljs-string">&quot;u&quot;</span>, <span class="hljs-string">&quot;ug&quot;</span>]
<span class="hljs-symbol">Corpus:</span> (<span class="hljs-string">&quot;h&quot;</span> <span class="hljs-string">&quot;ug&quot;</span>, <span class="hljs-number">10</span>), (<span class="hljs-string">&quot;p&quot;</span> <span class="hljs-string">&quot;ug&quot;</span>, <span class="hljs-number">5</span>), (<span class="hljs-string">&quot;p&quot;</span> <span class="hljs-string">&quot;u&quot;</span> <span class="hljs-string">&quot;n&quot;</span>, <span class="hljs-number">12</span>), (<span class="hljs-string">&quot;b&quot;</span> <span class="hljs-string">&quot;u&quot;</span> <span class="hljs-string">&quot;n&quot;</span>, <span class="hljs-number">4</span>), (<span class="hljs-string">&quot;h&quot;</span> <span class="hljs-string">&quot;ug&quot;</span> <span class="hljs-string">&quot;s&quot;</span>, <span class="hljs-number">5</span>)`}}),us=new q({props:{code:`Vocabulary: ["b", "g", "h", "n", "p", "s", "u", "ug", "un"]
Corpus: ("h" "ug", 10), ("p" "ug", 5), ("p" "un", 12), ("b" "un", 4), ("h" "ug" "s", 5)`,highlighted:`<span class="hljs-symbol">Vocabulary:</span> [<span class="hljs-string">&quot;b&quot;</span>, <span class="hljs-string">&quot;g&quot;</span>, <span class="hljs-string">&quot;h&quot;</span>, <span class="hljs-string">&quot;n&quot;</span>, <span class="hljs-string">&quot;p&quot;</span>, <span class="hljs-string">&quot;s&quot;</span>, <span class="hljs-string">&quot;u&quot;</span>, <span class="hljs-string">&quot;ug&quot;</span>, <span class="hljs-string">&quot;un&quot;</span>]
<span class="hljs-symbol">Corpus:</span> (<span class="hljs-string">&quot;h&quot;</span> <span class="hljs-string">&quot;ug&quot;</span>, <span class="hljs-number">10</span>), (<span class="hljs-string">&quot;p&quot;</span> <span class="hljs-string">&quot;ug&quot;</span>, <span class="hljs-number">5</span>), (<span class="hljs-string">&quot;p&quot;</span> <span class="hljs-string">&quot;un&quot;</span>, <span class="hljs-number">12</span>), (<span class="hljs-string">&quot;b&quot;</span> <span class="hljs-string">&quot;un&quot;</span>, <span class="hljs-number">4</span>), (<span class="hljs-string">&quot;h&quot;</span> <span class="hljs-string">&quot;ug&quot;</span> <span class="hljs-string">&quot;s&quot;</span>, <span class="hljs-number">5</span>)`}}),cs=new q({props:{code:`Vocabulary: ["b", "g", "h", "n", "p", "s", "u", "ug", "un", "hug"]
Corpus: ("hug", 10), ("p" "ug", 5), ("p" "un", 12), ("b" "un", 4), ("hug" "s", 5)`,highlighted:`<span class="hljs-symbol">Vocabulary:</span> [<span class="hljs-string">&quot;b&quot;</span>, <span class="hljs-string">&quot;g&quot;</span>, <span class="hljs-string">&quot;h&quot;</span>, <span class="hljs-string">&quot;n&quot;</span>, <span class="hljs-string">&quot;p&quot;</span>, <span class="hljs-string">&quot;s&quot;</span>, <span class="hljs-string">&quot;u&quot;</span>, <span class="hljs-string">&quot;ug&quot;</span>, <span class="hljs-string">&quot;un&quot;</span>, <span class="hljs-string">&quot;hug&quot;</span>]
<span class="hljs-symbol">Corpus:</span> (<span class="hljs-string">&quot;hug&quot;</span>, <span class="hljs-number">10</span>), (<span class="hljs-string">&quot;p&quot;</span> <span class="hljs-string">&quot;ug&quot;</span>, <span class="hljs-number">5</span>), (<span class="hljs-string">&quot;p&quot;</span> <span class="hljs-string">&quot;un&quot;</span>, <span class="hljs-number">12</span>), (<span class="hljs-string">&quot;b&quot;</span> <span class="hljs-string">&quot;un&quot;</span>, <span class="hljs-number">4</span>), (<span class="hljs-string">&quot;hug&quot;</span> <span class="hljs-string">&quot;s&quot;</span>, <span class="hljs-number">5</span>)`}}),W=new ca({props:{$$slots:{default:[fr]},$$scope:{ctx:O}}}),fs=new q({props:{code:`("u", "g") -> "ug"
("u", "n") -> "un"
("h", "ug") -> "hug"`,highlighted:`<span class="hljs-function"><span class="hljs-params">(<span class="hljs-string">&quot;u&quot;</span>, <span class="hljs-string">&quot;g&quot;</span>)</span> -&gt;</span> <span class="hljs-string">&quot;ug&quot;</span>
<span class="hljs-function"><span class="hljs-params">(<span class="hljs-string">&quot;u&quot;</span>, <span class="hljs-string">&quot;n&quot;</span>)</span> -&gt;</span> <span class="hljs-string">&quot;un&quot;</span>
<span class="hljs-function"><span class="hljs-params">(<span class="hljs-string">&quot;h&quot;</span>, <span class="hljs-string">&quot;ug&quot;</span>)</span> -&gt;</span> <span class="hljs-string">&quot;hug&quot;</span>`}}),Y=new ca({props:{$$slots:{default:[jr]},$$scope:{ctx:O}}}),js=new or({}),xs=new q({props:{code:`corpus = [
    "This is the Hugging Face course.",
    "This chapter is about tokenization.",
    "This section shows several tokenizer algorithms.",
    "Hopefully, you will be able to understand how they are trained and generate tokens.",
]`,highlighted:`corpus = [
    <span class="hljs-string">&quot;This is the Hugging Face course.&quot;</span>,
    <span class="hljs-string">&quot;This chapter is about tokenization.&quot;</span>,
    <span class="hljs-string">&quot;This section shows several tokenizer algorithms.&quot;</span>,
    <span class="hljs-string">&quot;Hopefully, you will be able to understand how they are trained and generate tokens.&quot;</span>,
]`}}),gs=new q({props:{code:`from transformers import AutoTokenizer

tokenizer = AutoTokenizer.from_pretrained("gpt2")`,highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoTokenizer

tokenizer = AutoTokenizer.from_pretrained(<span class="hljs-string">&quot;gpt2&quot;</span>)`}}),ms=new q({props:{code:`from collections import defaultdict

word_freqs = defaultdict(int)

for text in corpus:
    words_with_offsets = tokenizer.backend_tokenizer.pre_tokenizer.pre_tokenize_str(text)
    new_words = [word for word, offset in words_with_offsets]
    for word in new_words:
        word_freqs[word] += 1

print(word_freqs)`,highlighted:`<span class="hljs-keyword">from</span> collections <span class="hljs-keyword">import</span> defaultdict

word_freqs = defaultdict(<span class="hljs-built_in">int</span>)

<span class="hljs-keyword">for</span> text <span class="hljs-keyword">in</span> corpus:
    words_with_offsets = tokenizer.backend_tokenizer.pre_tokenizer.pre_tokenize_str(text)
    new_words = [word <span class="hljs-keyword">for</span> word, offset <span class="hljs-keyword">in</span> words_with_offsets]
    <span class="hljs-keyword">for</span> word <span class="hljs-keyword">in</span> new_words:
        word_freqs[word] += <span class="hljs-number">1</span>

<span class="hljs-built_in">print</span>(word_freqs)`}}),$s=new q({props:{code:`defaultdict(int, {'This': 3, '\u0120is': 2, '\u0120the': 1, '\u0120Hugging': 1, '\u0120Face': 1, '\u0120Course': 1, '.': 4, '\u0120chapter': 1,
    '\u0120about': 1, '\u0120tokenization': 1, '\u0120section': 1, '\u0120shows': 1, '\u0120several': 1, '\u0120tokenizer': 1, '\u0120algorithms': 1,
    'Hopefully': 1, ',': 1, '\u0120you': 1, '\u0120will': 1, '\u0120be': 1, '\u0120able': 1, '\u0120to': 1, '\u0120understand': 1, '\u0120how': 1,
    '\u0120they': 1, '\u0120are': 1, '\u0120trained': 1, '\u0120and': 1, '\u0120generate': 1, '\u0120tokens': 1})`,highlighted:`defaultdict(<span class="hljs-built_in">int</span>, {<span class="hljs-string">&#x27;This&#x27;</span>: <span class="hljs-number">3</span>, <span class="hljs-string">&#x27;\u0120is&#x27;</span>: <span class="hljs-number">2</span>, <span class="hljs-string">&#x27;\u0120the&#x27;</span>: <span class="hljs-number">1</span>, <span class="hljs-string">&#x27;\u0120Hugging&#x27;</span>: <span class="hljs-number">1</span>, <span class="hljs-string">&#x27;\u0120Face&#x27;</span>: <span class="hljs-number">1</span>, <span class="hljs-string">&#x27;\u0120Course&#x27;</span>: <span class="hljs-number">1</span>, <span class="hljs-string">&#x27;.&#x27;</span>: <span class="hljs-number">4</span>, <span class="hljs-string">&#x27;\u0120chapter&#x27;</span>: <span class="hljs-number">1</span>,
    <span class="hljs-string">&#x27;\u0120about&#x27;</span>: <span class="hljs-number">1</span>, <span class="hljs-string">&#x27;\u0120tokenization&#x27;</span>: <span class="hljs-number">1</span>, <span class="hljs-string">&#x27;\u0120section&#x27;</span>: <span class="hljs-number">1</span>, <span class="hljs-string">&#x27;\u0120shows&#x27;</span>: <span class="hljs-number">1</span>, <span class="hljs-string">&#x27;\u0120several&#x27;</span>: <span class="hljs-number">1</span>, <span class="hljs-string">&#x27;\u0120tokenizer&#x27;</span>: <span class="hljs-number">1</span>, <span class="hljs-string">&#x27;\u0120algorithms&#x27;</span>: <span class="hljs-number">1</span>,
    <span class="hljs-string">&#x27;Hopefully&#x27;</span>: <span class="hljs-number">1</span>, <span class="hljs-string">&#x27;,&#x27;</span>: <span class="hljs-number">1</span>, <span class="hljs-string">&#x27;\u0120you&#x27;</span>: <span class="hljs-number">1</span>, <span class="hljs-string">&#x27;\u0120will&#x27;</span>: <span class="hljs-number">1</span>, <span class="hljs-string">&#x27;\u0120be&#x27;</span>: <span class="hljs-number">1</span>, <span class="hljs-string">&#x27;\u0120able&#x27;</span>: <span class="hljs-number">1</span>, <span class="hljs-string">&#x27;\u0120to&#x27;</span>: <span class="hljs-number">1</span>, <span class="hljs-string">&#x27;\u0120understand&#x27;</span>: <span class="hljs-number">1</span>, <span class="hljs-string">&#x27;\u0120how&#x27;</span>: <span class="hljs-number">1</span>,
    <span class="hljs-string">&#x27;\u0120they&#x27;</span>: <span class="hljs-number">1</span>, <span class="hljs-string">&#x27;\u0120are&#x27;</span>: <span class="hljs-number">1</span>, <span class="hljs-string">&#x27;\u0120trained&#x27;</span>: <span class="hljs-number">1</span>, <span class="hljs-string">&#x27;\u0120and&#x27;</span>: <span class="hljs-number">1</span>, <span class="hljs-string">&#x27;\u0120generate&#x27;</span>: <span class="hljs-number">1</span>, <span class="hljs-string">&#x27;\u0120tokens&#x27;</span>: <span class="hljs-number">1</span>})`}}),_s=new q({props:{code:`alphabet = []

for word in word_freqs.keys():
    for letter in word:
        if letter not in alphabet:
            alphabet.append(letter)
alphabet.sort()

print(alphabet)`,highlighted:`alphabet = []

<span class="hljs-keyword">for</span> word <span class="hljs-keyword">in</span> word_freqs.keys():
    <span class="hljs-keyword">for</span> letter <span class="hljs-keyword">in</span> word:
        <span class="hljs-keyword">if</span> letter <span class="hljs-keyword">not</span> <span class="hljs-keyword">in</span> alphabet:
            alphabet.append(letter)
alphabet.sort()

<span class="hljs-built_in">print</span>(alphabet)`}}),qs=new q({props:{code:`[ ',', '.', 'C', 'F', 'H', 'T', 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'k', 'l', 'm', 'n', 'o', 'p', 'r', 's',
  't', 'u', 'v', 'w', 'y', 'z', '\u0120']`,highlighted:`[ <span class="hljs-string">&#x27;,&#x27;</span>, <span class="hljs-string">&#x27;.&#x27;</span>, <span class="hljs-string">&#x27;C&#x27;</span>, <span class="hljs-string">&#x27;F&#x27;</span>, <span class="hljs-string">&#x27;H&#x27;</span>, <span class="hljs-string">&#x27;T&#x27;</span>, <span class="hljs-string">&#x27;a&#x27;</span>, <span class="hljs-string">&#x27;b&#x27;</span>, <span class="hljs-string">&#x27;c&#x27;</span>, <span class="hljs-string">&#x27;d&#x27;</span>, <span class="hljs-string">&#x27;e&#x27;</span>, <span class="hljs-string">&#x27;f&#x27;</span>, <span class="hljs-string">&#x27;g&#x27;</span>, <span class="hljs-string">&#x27;h&#x27;</span>, <span class="hljs-string">&#x27;i&#x27;</span>, <span class="hljs-string">&#x27;k&#x27;</span>, <span class="hljs-string">&#x27;l&#x27;</span>, <span class="hljs-string">&#x27;m&#x27;</span>, <span class="hljs-string">&#x27;n&#x27;</span>, <span class="hljs-string">&#x27;o&#x27;</span>, <span class="hljs-string">&#x27;p&#x27;</span>, <span class="hljs-string">&#x27;r&#x27;</span>, <span class="hljs-string">&#x27;s&#x27;</span>,
  <span class="hljs-string">&#x27;t&#x27;</span>, <span class="hljs-string">&#x27;u&#x27;</span>, <span class="hljs-string">&#x27;v&#x27;</span>, <span class="hljs-string">&#x27;w&#x27;</span>, <span class="hljs-string">&#x27;y&#x27;</span>, <span class="hljs-string">&#x27;z&#x27;</span>, <span class="hljs-string">&#x27;\u0120&#x27;</span>]`}}),bs=new q({props:{code:'vocab = ["<|endoftext|>"] + alphabet.copy()',highlighted:'vocab = [<span class="hljs-string">&quot;&lt;|endoftext|&gt;&quot;</span>] + alphabet.copy()'}}),ds=new q({props:{code:"splits = {word: [c for c in word] for word in word_freqs.keys()}",highlighted:'splits = {word: [c <span class="hljs-keyword">for</span> c <span class="hljs-keyword">in</span> word] <span class="hljs-keyword">for</span> word <span class="hljs-keyword">in</span> word_freqs.keys()}'}}),ws=new q({props:{code:`def compute_pair_freqs(splits):
    pair_freqs = defaultdict(int)
    for word, freq in word_freqs.items():
        split = splits[word]
        if len(split) == 1:
            continue
        for i in range(len(split) - 1):
            pair = (split[i], split[i + 1])
            pair_freqs[pair] += freq
    return pair_freqs`,highlighted:`<span class="hljs-keyword">def</span> <span class="hljs-title function_">compute_pair_freqs</span>(<span class="hljs-params">splits</span>):
    pair_freqs = defaultdict(<span class="hljs-built_in">int</span>)
    <span class="hljs-keyword">for</span> word, freq <span class="hljs-keyword">in</span> word_freqs.items():
        split = splits[word]
        <span class="hljs-keyword">if</span> <span class="hljs-built_in">len</span>(split) == <span class="hljs-number">1</span>:
            <span class="hljs-keyword">continue</span>
        <span class="hljs-keyword">for</span> i <span class="hljs-keyword">in</span> <span class="hljs-built_in">range</span>(<span class="hljs-built_in">len</span>(split) - <span class="hljs-number">1</span>):
            pair = (split[i], split[i + <span class="hljs-number">1</span>])
            pair_freqs[pair] += freq
    <span class="hljs-keyword">return</span> pair_freqs`}}),ks=new q({props:{code:`pair_freqs = compute_pair_freqs(splits)

for i, key in enumerate(pair_freqs.keys()):
    print(f"{key}: {pair_freqs[key]}")
    if i >= 5:
        break`,highlighted:`pair_freqs = compute_pair_freqs(splits)

<span class="hljs-keyword">for</span> i, key <span class="hljs-keyword">in</span> <span class="hljs-built_in">enumerate</span>(pair_freqs.keys()):
    <span class="hljs-built_in">print</span>(<span class="hljs-string">f&quot;<span class="hljs-subst">{key}</span>: <span class="hljs-subst">{pair_freqs[key]}</span>&quot;</span>)
    <span class="hljs-keyword">if</span> i &gt;= <span class="hljs-number">5</span>:
        <span class="hljs-keyword">break</span>`}}),Es=new q({props:{code:`('T', 'h'): 3
('h', 'i'): 3
('i', 's'): 5
('\u0120', 'i'): 2
('\u0120', 't'): 7
('t', 'h'): 3`,highlighted:`(<span class="hljs-string">&#x27;T&#x27;</span>, <span class="hljs-string">&#x27;h&#x27;</span>): <span class="hljs-number">3</span>
(<span class="hljs-string">&#x27;h&#x27;</span>, <span class="hljs-string">&#x27;i&#x27;</span>): <span class="hljs-number">3</span>
(<span class="hljs-string">&#x27;i&#x27;</span>, <span class="hljs-string">&#x27;s&#x27;</span>): <span class="hljs-number">5</span>
(<span class="hljs-string">&#x27;\u0120&#x27;</span>, <span class="hljs-string">&#x27;i&#x27;</span>): <span class="hljs-number">2</span>
(<span class="hljs-string">&#x27;\u0120&#x27;</span>, <span class="hljs-string">&#x27;t&#x27;</span>): <span class="hljs-number">7</span>
(<span class="hljs-string">&#x27;t&#x27;</span>, <span class="hljs-string">&#x27;h&#x27;</span>): <span class="hljs-number">3</span>`}}),vs=new q({props:{code:`best_pair = ""
max_freq = None

for pair, freq in pair_freqs.items():
    if max_freq is None or max_freq < freq:
        best_pair = pair
        max_freq = freq

print(best_pair, max_freq)`,highlighted:`best_pair = <span class="hljs-string">&quot;&quot;</span>
max_freq = <span class="hljs-literal">None</span>

<span class="hljs-keyword">for</span> pair, freq <span class="hljs-keyword">in</span> pair_freqs.items():
    <span class="hljs-keyword">if</span> max_freq <span class="hljs-keyword">is</span> <span class="hljs-literal">None</span> <span class="hljs-keyword">or</span> max_freq &lt; freq:
        best_pair = pair
        max_freq = freq

<span class="hljs-built_in">print</span>(best_pair, max_freq)`}}),ys=new q({props:{code:"('\u0120', 't') 7",highlighted:'(<span class="hljs-string">&#x27;\u0120&#x27;</span>, <span class="hljs-string">&#x27;t&#x27;</span>) <span class="hljs-number">7</span>'}}),Ps=new q({props:{code:`merges = {("\u0120", "t"): "\u0120t"}
vocab.append("\u0120t")`,highlighted:`merges = {(<span class="hljs-string">&quot;\u0120&quot;</span>, <span class="hljs-string">&quot;t&quot;</span>): <span class="hljs-string">&quot;\u0120t&quot;</span>}
vocab.append(<span class="hljs-string">&quot;\u0120t&quot;</span>)`}}),Ts=new q({props:{code:`def merge_pair(a, b, splits):
    for word in word_freqs:
        split = splits[word]
        if len(split) == 1:
            continue

        i = 0
        while i < len(split) - 1:
            if split[i] == a and split[i + 1] == b:
                split = split[:i] + [a + b] + split[i + 2 :]
            else:
                i += 1
        splits[word] = split
    return splits`,highlighted:`<span class="hljs-keyword">def</span> <span class="hljs-title function_">merge_pair</span>(<span class="hljs-params">a, b, splits</span>):
    <span class="hljs-keyword">for</span> word <span class="hljs-keyword">in</span> word_freqs:
        split = splits[word]
        <span class="hljs-keyword">if</span> <span class="hljs-built_in">len</span>(split) == <span class="hljs-number">1</span>:
            <span class="hljs-keyword">continue</span>

        i = <span class="hljs-number">0</span>
        <span class="hljs-keyword">while</span> i &lt; <span class="hljs-built_in">len</span>(split) - <span class="hljs-number">1</span>:
            <span class="hljs-keyword">if</span> split[i] == a <span class="hljs-keyword">and</span> split[i + <span class="hljs-number">1</span>] == b:
                split = split[:i] + [a + b] + split[i + <span class="hljs-number">2</span> :]
            <span class="hljs-keyword">else</span>:
                i += <span class="hljs-number">1</span>
        splits[word] = split
    <span class="hljs-keyword">return</span> splits`}}),Cs=new q({props:{code:`splits = merge_pair("\u0120", "t", splits)
print(splits["\u0120trained"])`,highlighted:`splits = merge_pair(<span class="hljs-string">&quot;\u0120&quot;</span>, <span class="hljs-string">&quot;t&quot;</span>, splits)
<span class="hljs-built_in">print</span>(splits[<span class="hljs-string">&quot;\u0120trained&quot;</span>])`}}),zs=new q({props:{code:"['\u0120t', 'r', 'a', 'i', 'n', 'e', 'd']",highlighted:'[<span class="hljs-string">&#x27;\u0120t&#x27;</span>, <span class="hljs-string">&#x27;r&#x27;</span>, <span class="hljs-string">&#x27;a&#x27;</span>, <span class="hljs-string">&#x27;i&#x27;</span>, <span class="hljs-string">&#x27;n&#x27;</span>, <span class="hljs-string">&#x27;e&#x27;</span>, <span class="hljs-string">&#x27;d&#x27;</span>]'}}),Os=new q({props:{code:`vocab_size = 50

while len(vocab) < vocab_size:
    pair_freqs = compute_pair_freqs(splits)
    best_pair = ""
    max_freq = None
    for pair, freq in pair_freqs.items():
        if max_freq is None or max_freq < freq:
            best_pair = pair
            max_freq = freq
    splits = merge_pair(*best_pair, splits)
    merges[best_pair] = best_pair[0] + best_pair[1]
    vocab.append(best_pair[0] + best_pair[1])`,highlighted:`vocab_size = <span class="hljs-number">50</span>

<span class="hljs-keyword">while</span> <span class="hljs-built_in">len</span>(vocab) &lt; vocab_size:
    pair_freqs = compute_pair_freqs(splits)
    best_pair = <span class="hljs-string">&quot;&quot;</span>
    max_freq = <span class="hljs-literal">None</span>
    <span class="hljs-keyword">for</span> pair, freq <span class="hljs-keyword">in</span> pair_freqs.items():
        <span class="hljs-keyword">if</span> max_freq <span class="hljs-keyword">is</span> <span class="hljs-literal">None</span> <span class="hljs-keyword">or</span> max_freq &lt; freq:
            best_pair = pair
            max_freq = freq
    splits = merge_pair(*best_pair, splits)
    merges[best_pair] = best_pair[<span class="hljs-number">0</span>] + best_pair[<span class="hljs-number">1</span>]
    vocab.append(best_pair[<span class="hljs-number">0</span>] + best_pair[<span class="hljs-number">1</span>])`}}),Ds=new q({props:{code:"print(merges)",highlighted:'<span class="hljs-built_in">print</span>(merges)'}}),Bs=new q({props:{code:`{('\u0120', 't'): '\u0120t', ('i', 's'): 'is', ('e', 'r'): 'er', ('\u0120', 'a'): '\u0120a', ('\u0120t', 'o'): '\u0120to', ('e', 'n'): 'en',
 ('T', 'h'): 'Th', ('Th', 'is'): 'This', ('o', 'u'): 'ou', ('s', 'e'): 'se', ('\u0120to', 'k'): '\u0120tok',
 ('\u0120tok', 'en'): '\u0120token', ('n', 'd'): 'nd', ('\u0120', 'is'): '\u0120is', ('\u0120t', 'h'): '\u0120th', ('\u0120th', 'e'): '\u0120the',
 ('i', 'n'): 'in', ('\u0120a', 'b'): '\u0120ab', ('\u0120token', 'i'): '\u0120tokeni'}`,highlighted:`{(<span class="hljs-string">&#x27;\u0120&#x27;</span>, <span class="hljs-string">&#x27;t&#x27;</span>): <span class="hljs-string">&#x27;\u0120t&#x27;</span>, (<span class="hljs-string">&#x27;i&#x27;</span>, <span class="hljs-string">&#x27;s&#x27;</span>): <span class="hljs-string">&#x27;is&#x27;</span>, (<span class="hljs-string">&#x27;e&#x27;</span>, <span class="hljs-string">&#x27;r&#x27;</span>): <span class="hljs-string">&#x27;er&#x27;</span>, (<span class="hljs-string">&#x27;\u0120&#x27;</span>, <span class="hljs-string">&#x27;a&#x27;</span>): <span class="hljs-string">&#x27;\u0120a&#x27;</span>, (<span class="hljs-string">&#x27;\u0120t&#x27;</span>, <span class="hljs-string">&#x27;o&#x27;</span>): <span class="hljs-string">&#x27;\u0120to&#x27;</span>, (<span class="hljs-string">&#x27;e&#x27;</span>, <span class="hljs-string">&#x27;n&#x27;</span>): <span class="hljs-string">&#x27;en&#x27;</span>,
 (<span class="hljs-string">&#x27;T&#x27;</span>, <span class="hljs-string">&#x27;h&#x27;</span>): <span class="hljs-string">&#x27;Th&#x27;</span>, (<span class="hljs-string">&#x27;Th&#x27;</span>, <span class="hljs-string">&#x27;is&#x27;</span>): <span class="hljs-string">&#x27;This&#x27;</span>, (<span class="hljs-string">&#x27;o&#x27;</span>, <span class="hljs-string">&#x27;u&#x27;</span>): <span class="hljs-string">&#x27;ou&#x27;</span>, (<span class="hljs-string">&#x27;s&#x27;</span>, <span class="hljs-string">&#x27;e&#x27;</span>): <span class="hljs-string">&#x27;se&#x27;</span>, (<span class="hljs-string">&#x27;\u0120to&#x27;</span>, <span class="hljs-string">&#x27;k&#x27;</span>): <span class="hljs-string">&#x27;\u0120tok&#x27;</span>,
 (<span class="hljs-string">&#x27;\u0120tok&#x27;</span>, <span class="hljs-string">&#x27;en&#x27;</span>): <span class="hljs-string">&#x27;\u0120token&#x27;</span>, (<span class="hljs-string">&#x27;n&#x27;</span>, <span class="hljs-string">&#x27;d&#x27;</span>): <span class="hljs-string">&#x27;nd&#x27;</span>, (<span class="hljs-string">&#x27;\u0120&#x27;</span>, <span class="hljs-string">&#x27;is&#x27;</span>): <span class="hljs-string">&#x27;\u0120is&#x27;</span>, (<span class="hljs-string">&#x27;\u0120t&#x27;</span>, <span class="hljs-string">&#x27;h&#x27;</span>): <span class="hljs-string">&#x27;\u0120th&#x27;</span>, (<span class="hljs-string">&#x27;\u0120th&#x27;</span>, <span class="hljs-string">&#x27;e&#x27;</span>): <span class="hljs-string">&#x27;\u0120the&#x27;</span>,
 (<span class="hljs-string">&#x27;i&#x27;</span>, <span class="hljs-string">&#x27;n&#x27;</span>): <span class="hljs-string">&#x27;in&#x27;</span>, (<span class="hljs-string">&#x27;\u0120a&#x27;</span>, <span class="hljs-string">&#x27;b&#x27;</span>): <span class="hljs-string">&#x27;\u0120ab&#x27;</span>, (<span class="hljs-string">&#x27;\u0120token&#x27;</span>, <span class="hljs-string">&#x27;i&#x27;</span>): <span class="hljs-string">&#x27;\u0120tokeni&#x27;</span>}`}}),Ns=new q({props:{code:"print(vocab)",highlighted:'<span class="hljs-built_in">print</span>(vocab)'}}),Gs=new q({props:{code:`['<|endoftext|>', ',', '.', 'C', 'F', 'H', 'T', 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'k', 'l', 'm', 'n', 'o',
 'p', 'r', 's', 't', 'u', 'v', 'w', 'y', 'z', '\u0120', '\u0120t', 'is', 'er', '\u0120a', '\u0120to', 'en', 'Th', 'This', 'ou', 'se',
 '\u0120tok', '\u0120token', 'nd', '\u0120is', '\u0120th', '\u0120the', 'in', '\u0120ab', '\u0120tokeni']`,highlighted:`[<span class="hljs-string">&#x27;&lt;|endoftext|&gt;&#x27;</span>, <span class="hljs-string">&#x27;,&#x27;</span>, <span class="hljs-string">&#x27;.&#x27;</span>, <span class="hljs-string">&#x27;C&#x27;</span>, <span class="hljs-string">&#x27;F&#x27;</span>, <span class="hljs-string">&#x27;H&#x27;</span>, <span class="hljs-string">&#x27;T&#x27;</span>, <span class="hljs-string">&#x27;a&#x27;</span>, <span class="hljs-string">&#x27;b&#x27;</span>, <span class="hljs-string">&#x27;c&#x27;</span>, <span class="hljs-string">&#x27;d&#x27;</span>, <span class="hljs-string">&#x27;e&#x27;</span>, <span class="hljs-string">&#x27;f&#x27;</span>, <span class="hljs-string">&#x27;g&#x27;</span>, <span class="hljs-string">&#x27;h&#x27;</span>, <span class="hljs-string">&#x27;i&#x27;</span>, <span class="hljs-string">&#x27;k&#x27;</span>, <span class="hljs-string">&#x27;l&#x27;</span>, <span class="hljs-string">&#x27;m&#x27;</span>, <span class="hljs-string">&#x27;n&#x27;</span>, <span class="hljs-string">&#x27;o&#x27;</span>,
 <span class="hljs-string">&#x27;p&#x27;</span>, <span class="hljs-string">&#x27;r&#x27;</span>, <span class="hljs-string">&#x27;s&#x27;</span>, <span class="hljs-string">&#x27;t&#x27;</span>, <span class="hljs-string">&#x27;u&#x27;</span>, <span class="hljs-string">&#x27;v&#x27;</span>, <span class="hljs-string">&#x27;w&#x27;</span>, <span class="hljs-string">&#x27;y&#x27;</span>, <span class="hljs-string">&#x27;z&#x27;</span>, <span class="hljs-string">&#x27;\u0120&#x27;</span>, <span class="hljs-string">&#x27;\u0120t&#x27;</span>, <span class="hljs-string">&#x27;is&#x27;</span>, <span class="hljs-string">&#x27;er&#x27;</span>, <span class="hljs-string">&#x27;\u0120a&#x27;</span>, <span class="hljs-string">&#x27;\u0120to&#x27;</span>, <span class="hljs-string">&#x27;en&#x27;</span>, <span class="hljs-string">&#x27;Th&#x27;</span>, <span class="hljs-string">&#x27;This&#x27;</span>, <span class="hljs-string">&#x27;ou&#x27;</span>, <span class="hljs-string">&#x27;se&#x27;</span>,
 <span class="hljs-string">&#x27;\u0120tok&#x27;</span>, <span class="hljs-string">&#x27;\u0120token&#x27;</span>, <span class="hljs-string">&#x27;nd&#x27;</span>, <span class="hljs-string">&#x27;\u0120is&#x27;</span>, <span class="hljs-string">&#x27;\u0120th&#x27;</span>, <span class="hljs-string">&#x27;\u0120the&#x27;</span>, <span class="hljs-string">&#x27;in&#x27;</span>, <span class="hljs-string">&#x27;\u0120ab&#x27;</span>, <span class="hljs-string">&#x27;\u0120tokeni&#x27;</span>]`}}),ss=new ca({props:{$$slots:{default:[xr]},$$scope:{ctx:O}}}),Hs=new q({props:{code:`def tokenize(text):
    pre_tokenize_result = tokenizer._tokenizer.pre_tokenizer.pre_tokenize_str(text)
    pre_tokenized_text = [word for word, offset in pre_tokenize_result]
    splits = [[l for l in word] for word in pre_tokenized_text]
    for pair, merge in merges.items():
        for idx, split in enumerate(splits):
            i = 0
            while i < len(split) - 1:
                if split[i] == pair[0] and split[i + 1] == pair[1]:
                    split = split[:i] + [merge] + split[i + 2 :]
                else:
                    i += 1
            splits[idx] = split

    return sum(splits, [])`,highlighted:`<span class="hljs-keyword">def</span> <span class="hljs-title function_">tokenize</span>(<span class="hljs-params">text</span>):
    pre_tokenize_result = tokenizer._tokenizer.pre_tokenizer.pre_tokenize_str(text)
    pre_tokenized_text = [word <span class="hljs-keyword">for</span> word, offset <span class="hljs-keyword">in</span> pre_tokenize_result]
    splits = [[l <span class="hljs-keyword">for</span> l <span class="hljs-keyword">in</span> word] <span class="hljs-keyword">for</span> word <span class="hljs-keyword">in</span> pre_tokenized_text]
    <span class="hljs-keyword">for</span> pair, merge <span class="hljs-keyword">in</span> merges.items():
        <span class="hljs-keyword">for</span> idx, split <span class="hljs-keyword">in</span> <span class="hljs-built_in">enumerate</span>(splits):
            i = <span class="hljs-number">0</span>
            <span class="hljs-keyword">while</span> i &lt; <span class="hljs-built_in">len</span>(split) - <span class="hljs-number">1</span>:
                <span class="hljs-keyword">if</span> split[i] == pair[<span class="hljs-number">0</span>] <span class="hljs-keyword">and</span> split[i + <span class="hljs-number">1</span>] == pair[<span class="hljs-number">1</span>]:
                    split = split[:i] + [merge] + split[i + <span class="hljs-number">2</span> :]
                <span class="hljs-keyword">else</span>:
                    i += <span class="hljs-number">1</span>
            splits[idx] = split

    <span class="hljs-keyword">return</span> <span class="hljs-built_in">sum</span>(splits, [])`}}),As=new q({props:{code:'tokenize("This is not a token.")',highlighted:'tokenize(<span class="hljs-string">&quot;This is not a token.&quot;</span>)'}}),Rs=new q({props:{code:"['This', '\u0120is', '\u0120', 'n', 'o', 't', '\u0120a', '\u0120token', '.']",highlighted:'[<span class="hljs-string">&#x27;This&#x27;</span>, <span class="hljs-string">&#x27;\u0120is&#x27;</span>, <span class="hljs-string">&#x27;\u0120&#x27;</span>, <span class="hljs-string">&#x27;n&#x27;</span>, <span class="hljs-string">&#x27;o&#x27;</span>, <span class="hljs-string">&#x27;t&#x27;</span>, <span class="hljs-string">&#x27;\u0120a&#x27;</span>, <span class="hljs-string">&#x27;\u0120token&#x27;</span>, <span class="hljs-string">&#x27;.&#x27;</span>]'}}),as=new ca({props:{warning:!0,$$slots:{default:[gr]},$$scope:{ctx:O}}}),{c(){h=r("meta"),E=u(),f=r("h1"),w=p("\u5B57\u8282\u5BF9\u7F16\u7801\u6807\u8BB0\u5316"),y=u(),j(b.$$.fragment),k=u(),T=r("p"),S=p("\u5B57\u8282\u5BF9\u7F16\u7801(BPE)\u6700\u521D\u88AB\u5F00\u53D1\u4E3A\u4E00\u79CD\u538B\u7F29\u6587\u672C\u7684\u7B97\u6CD5,\u7136\u540E\u5728\u9884\u8BAD\u7EC3 GPT \u6A21\u578B\u65F6\u88AB OpenAI \u7528\u4E8E\u6807\u8BB0\u5316\u3002\u8BB8\u591A Transformer \u6A21\u578B\u90FD\u4F7F\u7528\u5B83,\u5305\u62EC GPT\u3001GPT-2\u3001RoBERTa\u3001BART \u548C DeBERTa\u3002"),D=u(),j(H.$$.fragment),ps=u(),j(F.$$.fragment),rn=u(),ts=r("h2"),Bl=p("\u8BAD\u7EC3\u7B97\u6CD5"),on=u(),Ss=r("p"),Nl=p("BPE \u8BAD\u7EC3\u9996\u5148\u8BA1\u7B97\u8BED\u6599\u5E93\u4E2D\u4F7F\u7528\u7684\u552F\u4E00\u5355\u8BCD\u96C6(\u5728\u5B8C\u6210\u6807\u51C6\u5316\u548C\u9884\u6807\u8BB0\u5316\u6B65\u9AA4\u4E4B\u540E),\u7136\u540E\u901A\u8FC7\u83B7\u53D6\u7528\u4E8E\u7F16\u5199\u8FD9\u4E9B\u5355\u8BCD\u7684\u6240\u6709\u7B26\u53F7\u6765\u6784\u5EFA\u8BCD\u6C47\u8868\u3002\u4E00\u4E2A\u975E\u5E38\u7B80\u5355\u7684\u4F8B\u5B50,\u5047\u8BBE\u6211\u4EEC\u7684\u8BED\u6599\u5E93\u4F7F\u7528\u4E86\u8FD9\u4E94\u4E2A\u8BCD:"),un=u(),j(es.$$.fragment),cn=u(),V=r("p"),Gl=p("\u57FA\u7840\u8BCD\u6C47\u5C06\u662F "),ha=r("code"),Hl=p('["b", "g", "h", "n", "p", "s", "u"]'),Al=p("\u3002\u5BF9\u4E8E\u5B9E\u9645\u60C5\u51B5,\u57FA\u672C\u8BCD\u6C47\u8868\u5C06\u5305\u542B\u6240\u6709 ASCII \u5B57\u7B26,\u81F3\u5C11,\u53EF\u80FD\u8FD8\u5305\u542B\u4E00\u4E9B Unicode \u5B57\u7B26\u3002\u5982\u679C\u60A8\u6B63\u5728\u6807\u8BB0\u7684\u793A\u4F8B\u4F7F\u7528\u4E0D\u5728\u8BAD\u7EC3\u8BED\u6599\u5E93\u4E2D\u7684\u5B57\u7B26,\u5219\u8BE5\u5B57\u7B26\u5C06\u8F6C\u6362\u4E3A\u672A\u77E5\u6807\u8BB0\u3002\u8FD9\u5C31\u662F\u4E3A\u4EC0\u4E48\u8BB8\u591A NLP \u6A21\u578B\u5728\u5206\u6790\u5E26\u6709\u8868\u60C5\u7B26\u53F7\u7684\u5185\u5BB9\u65B9\u9762\u975E\u5E38\u7CDF\u7CD5\u7684\u539F\u56E0\u4E4B\u4E00\u3002"),hn=u(),j(M.$$.fragment),fn=u(),K=r("p"),Rl=p("\u83B7\u5F97\u8FD9\u4E2A\u57FA\u672C\u8BCD\u6C47\u540E,\u6211\u4EEC\u6DFB\u52A0\u65B0\u7684\u6807\u8BB0,\u76F4\u5230\u901A\u8FC7\u5B66\u4E60"),fa=r("em"),Il=p("\u5408\u5E76"),Sl=p("\u8FBE\u5230\u6240\u9700\u7684\u8BCD\u6C47\u91CF,\u8FD9\u662F\u5C06\u73B0\u6709\u8BCD\u6C47\u8868\u7684\u4E24\u4E2A\u5143\u7D20\u5408\u5E76\u4E3A\u4E00\u4E2A\u65B0\u5143\u7D20\u7684\u89C4\u5219\u3002\u56E0\u6B64,\u5728\u5F00\u59CB\u65F6,\u8FD9\u4E9B\u5408\u5E76\u5C06\u521B\u5EFA\u5177\u6709\u4E24\u4E2A\u5B57\u7B26\u7684\u6807\u8BB0,\u7136\u540E,\u968F\u7740\u8BAD\u7EC3\u7684\u8FDB\u884C,\u4F1A\u521B\u5EFA\u66F4\u957F\u7684\u5B50\u8BCD\u3002"),jn=u(),Ls=r("p"),Ll=p("\u5728\u5206\u8BCD\u5668\u8BAD\u7EC3\u671F\u95F4\u7684\u4EFB\u4F55\u4E00\u6B65,BPE \u7B97\u6CD5\u90FD\u4F1A\u641C\u7D22\u6700\u5E38\u89C1\u7684\u73B0\u6709\u6807\u8BB0\u5BF9 (\u201C\u5BF9\u201D,\u8FD9\u91CC\u6211\u4EEC\u6307\u7684\u662F\u5355\u8BCD\u4E2D\u7684\u4E24\u4E2A\u8FDE\u7EED\u6807\u8BB0)\u3002\u6700\u9891\u7E41\u7684\u4E00\u5BF9\u5C06\u88AB\u5408\u5E76,\u6211\u4EEC\u51B2\u6D17\u5E76\u91CD\u590D\u4E0B\u4E00\u6B65\u3002"),xn=u(),Us=r("p"),Ul=p("\u56DE\u5230\u6211\u4EEC\u4E4B\u524D\u7684\u4F8B\u5B50,\u8BA9\u6211\u4EEC\u5047\u8BBE\u5355\u8BCD\u5177\u6709\u4EE5\u4E0B\u9891\u7387:"),gn=u(),j(rs.$$.fragment),mn=u(),C=r("p"),Fl=p("\u610F\u5473\u7740 "),ja=r("code"),Vl=p('"hug"'),Ml=p(" \u5728\u8BED\u6599\u5E93\u4E2D\u51FA\u73B0\u4E8610\u6B21, "),xa=r("code"),Kl=p('"pug"'),Wl=p(" 5\u6B21, "),ga=r("code"),Yl=p('"pun"'),Jl=p(" 12\u6B21, "),ma=r("code"),Zl=p('"bun"'),Ql=p(" 4\u6B21, \u4EE5\u53CA "),$a=r("code"),Xl=p('"hugs"'),sp=p(" 5\u6B21\u3002\u6211\u4EEC\u901A\u8FC7\u5C06\u6BCF\u4E2A\u5355\u8BCD\u62C6\u5206\u4E3A\u5B57\u7B26(\u5F62\u6210\u6211\u4EEC\u521D\u59CB\u8BCD\u6C47\u8868\u7684\u5B57\u7B26)\u6765\u5F00\u59CB\u8BAD\u7EC3,\u8FD9\u6837\u6211\u4EEC\u5C31\u53EF\u4EE5\u5C06\u6BCF\u4E2A\u5355\u8BCD\u89C6\u4E3A\u4E00\u4E2A\u6807\u8BB0\u5217\u8868:"),$n=u(),j(is.$$.fragment),_n=u(),P=r("p"),ap=p("\u7136\u540E\u6211\u4EEC\u770B\u6210\u5BF9\u3002\u8FD9\u5BF9 "),_a=r("code"),np=p('("h", "u")'),lp=p(" \u51FA\u73B0\u5728\u5355\u8BCD "),qa=r("code"),pp=p('"hug"'),tp=p(" \u548C "),ba=r("code"),ep=p('"hugs"'),rp=p("\u4E2D,\u6240\u4EE5\u8BED\u6599\u5E93\u4E2D\u603B\u5171\u670915\u6B21\u3002\u4E0D\u8FC7,\u8FD9\u5E76\u4E0D\u662F\u6700\u9891\u7E41\u7684\u4E00\u5BF9:\u8FD9\u4E2A\u8363\u8A89\u5C5E\u4E8E "),da=r("code"),ip=p('("u", "g")'),op=p(",\u5B83\u51FA\u73B0\u5728 "),wa=r("code"),up=p('"hug"'),cp=p(", "),ka=r("code"),hp=p('"pug"'),fp=p(", \u4EE5\u53CA "),Ea=r("code"),jp=p('"hugs"'),xp=p("\u4E2D,\u5728\u8BCD\u6C47\u8868\u4E2D\u603B\u5171 20 \u6B21\u3002"),qn=u(),A=r("p"),gp=p("\u56E0\u6B64,\u6807\u8BB0\u5668\u5B66\u4E60\u7684\u7B2C\u4E00\u4E2A\u5408\u5E76\u89C4\u5219\u662F "),va=r("code"),mp=p('("u", "g") -> "ug"'),$p=p(",\u610F\u601D\u5C31\u662F "),ya=r("code"),_p=p('"ug"'),qp=p(" \u5C06\u88AB\u6DFB\u52A0\u5230\u8BCD\u6C47\u8868\u4E2D,\u5E76\u4E14\u8FD9\u5BF9\u5E94\u8BE5\u5408\u5E76\u5230\u8BED\u6599\u5E93\u7684\u6240\u6709\u5355\u8BCD\u4E2D\u3002\u5728\u8FD9\u4E2A\u9636\u6BB5\u7ED3\u675F\u65F6,\u8BCD\u6C47\u8868\u548C\u8BED\u6599\u5E93\u770B\u8D77\u6765\u50CF\u8FD9\u6837:"),bn=u(),j(os.$$.fragment),dn=u(),B=r("p"),bp=p("\u73B0\u5728\u6211\u4EEC\u6709\u4E00\u4E9B\u5BFC\u81F4\u6807\u8BB0\u957F\u4E8E\u4E24\u4E2A\u5B57\u7B26\u7684\u5BF9: \u4F8B\u5982 "),Pa=r("code"),dp=p('("h", "ug")'),wp=p(", \u5728\u8BED\u6599\u5E93\u4E2D\u51FA\u73B015\u6B21\u3002\u7136\u800C,\u8FD9\u4E2A\u9636\u6BB5\u6700\u9891\u7E41\u7684\u5BF9\u662F "),Ta=r("code"),kp=p('("u", "n")'),Ep=p(",\u5728\u8BED\u6599\u5E93\u4E2D\u51FA\u73B016\u6B21,\u6240\u4EE5\u5B66\u5230\u7684\u7B2C\u4E8C\u4E2A\u5408\u5E76\u89C4\u5219\u662F "),Ca=r("code"),vp=p('("u", "n") -> "un"'),yp=p("\u3002\u5C06\u5176\u6DFB\u52A0\u5230\u8BCD\u6C47\u8868\u5E76\u5408\u5E76\u6240\u6709\u73B0\u6709\u7684\u8FD9\u4E2A\u5BF9,\u5C06\u51FA\u73B0:"),wn=u(),j(us.$$.fragment),kn=u(),R=r("p"),Pp=p("\u73B0\u5728\u6700\u9891\u7E41\u7684\u4E00\u5BF9\u662F "),za=r("code"),Tp=p('("h", "ug")'),Cp=p(",\u6240\u4EE5\u6211\u4EEC\u5B66\u4E60\u4E86\u5408\u5E76\u89C4\u5219 "),Oa=r("code"),zp=p('("h", "ug") -> "hug"'),Op=p(",\u8FD9\u7ED9\u4E86\u6211\u4EEC\u7B2C\u4E00\u4E2A\u4E09\u4E2A\u5B57\u6BCD\u7684\u6807\u8BB0\u3002\u5408\u5E76\u540E,\u8BED\u6599\u5E93\u5982\u4E0B\u6240\u793A:"),En=u(),j(cs.$$.fragment),vn=u(),Fs=r("p"),Dp=p("\u6211\u4EEC\u7EE7\u7EED\u8FD9\u6837\u5408\u5E76,\u76F4\u5230\u8FBE\u5230\u6211\u4EEC\u6240\u9700\u7684\u8BCD\u6C47\u91CF\u3002"),yn=u(),j(W.$$.fragment),Pn=u(),hs=r("h2"),Bp=p("\u6807\u8BB0\u5316\u7B97\u6CD5"),Tn=u(),Vs=r("p"),Np=p("\u6807\u8BB0\u5316\u7D27\u8DDF\u8BAD\u7EC3\u8FC7\u7A0B,\u4ECE\u67D0\u79CD\u610F\u4E49\u4E0A\u8BF4,\u901A\u8FC7\u5E94\u7528\u4EE5\u4E0B\u6B65\u9AA4\u5BF9\u65B0\u8F93\u5165\u8FDB\u884C\u6807\u8BB0:"),Cn=u(),N=r("ol"),Da=r("li"),Gp=p("\u89C4\u8303\u5316"),Hp=u(),Ba=r("li"),Ap=p("\u9884\u6807\u8BB0\u5316"),Rp=u(),Na=r("li"),Ip=p("\u5C06\u5355\u8BCD\u62C6\u5206\u4E3A\u5355\u4E2A\u5B57\u7B26"),Sp=u(),Ga=r("li"),Lp=p("\u5C06\u5B66\u4E60\u7684\u5408\u5E76\u89C4\u5219\u6309\u987A\u5E8F\u5E94\u7528\u4E8E\u8FD9\u4E9B\u62C6\u5206"),zn=u(),Ms=r("p"),Up=p("\u8BA9\u6211\u4EEC\u4EE5\u6211\u4EEC\u5728\u8BAD\u7EC3\u671F\u95F4\u4F7F\u7528\u7684\u793A\u4F8B\u4E3A\u4F8B,\u5B66\u4E60\u4E09\u4E2A\u5408\u5E76\u89C4\u5219:"),On=u(),j(fs.$$.fragment),Dn=u(),d=r("p"),Fp=p("\u8FD9\u4E2A\u5355\u8BCD "),Ha=r("code"),Vp=p('"bug"'),Mp=p(" \u5C06\u88AB\u6807\u8BB0\u4E3A "),Aa=r("code"),Kp=p('["b", "ug"]'),Wp=p("\u3002\u7136\u800C "),Ra=r("code"),Yp=p('"mug"'),Jp=p(",\u5C06\u88AB\u6807\u8BB0\u4E3A "),Ia=r("code"),Zp=p('["[UNK]", "ug"]'),Qp=p(",\u56E0\u4E3A\u5B57\u6BCD "),Sa=r("code"),Xp=p('"m"'),st=p(" \u4E0D\u518D\u57FA\u672C\u8BCD\u6C47\u8868\u4E2D\u3002\u540C\u6837,\u5355\u8BCD"),La=r("code"),at=p('"thug"'),nt=p(" \u4F1A\u88AB\u6807\u8BB0\u4E3A "),Ua=r("code"),lt=p('["[UNK]", "hug"]'),pt=p(": \u5B57\u6BCD "),Fa=r("code"),tt=p('"t"'),et=p(" \u4E0D\u5728\u57FA\u672C\u8BCD\u6C47\u8868\u4E2D,\u5E94\u7528\u5408\u5E76\u89C4\u5219\u9996\u5148\u5BFC\u81F4 "),Va=r("code"),rt=p('"u"'),it=p(" \u548C "),Ma=r("code"),ot=p('"g"'),ut=p(" \u88AB\u5408\u5E76,\u7136\u540E\u662F "),Ka=r("code"),ct=p('"hu"'),ht=p(" \u548C "),Wa=r("code"),ft=p('"g"'),jt=p(" \u88AB\u5408\u5E76\u3002"),Bn=u(),j(Y.$$.fragment),Nn=u(),L=r("h2"),J=r("a"),Ya=r("span"),j(js.$$.fragment),xt=u(),Ja=r("span"),gt=p("\u5B9E\u73B0 BPE"),Gn=u(),Ks=r("p"),mt=p("\u73B0\u5728\u8BA9\u6211\u4EEC\u770B\u4E00\u4E0B BPE \u7B97\u6CD5\u7684\u5B9E\u73B0\u3002\u8FD9\u4E0D\u4F1A\u662F\u4F60\u53EF\u4EE5\u5728\u5927\u578B\u8BED\u6599\u5E93\u4E0A\u5B9E\u9645\u4F7F\u7528\u7684\u4F18\u5316\u7248\u672C;\u6211\u4EEC\u53EA\u662F\u60F3\u5411\u4F60\u5C55\u793A\u4EE3\u7801,\u4EE5\u4FBF\u4F60\u53EF\u4EE5\u66F4\u597D\u5730\u7406\u89E3\u7B97\u6CD5"),Hn=u(),Ws=r("p"),$t=p("\u9996\u5148\u6211\u4EEC\u9700\u8981\u4E00\u4E2A\u8BED\u6599\u5E93,\u6240\u4EE5\u8BA9\u6211\u4EEC\u7528\u51E0\u53E5\u8BDD\u521B\u5EFA\u4E00\u4E2A\u7B80\u5355\u7684\u8BED\u6599\u5E93:"),An=u(),j(xs.$$.fragment),Rn=u(),Z=r("p"),_t=p("\u63A5\u4E0B\u6765,\u6211\u4EEC\u9700\u8981\u5C06\u8BE5\u8BED\u6599\u5E93\u9884\u5148\u6807\u8BB0\u4E3A\u5355\u8BCD\u3002\u7531\u4E8E\u6211\u4EEC\u6B63\u5728\u590D\u5236 BPE \u6807\u8BB0\u5668(\u5982 GPT-2),\u6211\u4EEC\u5C06\u4F7F\u7528 "),Za=r("code"),qt=p("gpt2"),bt=p(" \u6807\u8BB0\u5668\u4F5C\u4E3A\u9884\u6807\u8BB0\u5316\u7684\u6807\u8BB0\u5668:"),In=u(),j(gs.$$.fragment),Sn=u(),Ys=r("p"),dt=p("\u7136\u540E\u6211\u4EEC\u5728\u8FDB\u884C\u9884\u6807\u8BB0\u5316\u65F6\u8BA1\u7B97\u8BED\u6599\u5E93\u4E2D\u6BCF\u4E2A\u5355\u8BCD\u7684\u9891\u7387:"),Ln=u(),j(ms.$$.fragment),Un=u(),j($s.$$.fragment),Fn=u(),Js=r("p"),wt=p("\u4E0B\u4E00\u6B65\u662F\u8BA1\u7B97\u57FA\u672C\u8BCD\u6C47,\u7531\u8BED\u6599\u5E93\u4E2D\u4F7F\u7528\u7684\u6240\u6709\u5B57\u7B26\u7EC4\u6210:"),Vn=u(),j(_s.$$.fragment),Mn=u(),j(qs.$$.fragment),Kn=u(),Q=r("p"),kt=p("\u6211\u4EEC\u8FD8\u5728\u8BE5\u8BCD\u6C47\u8868\u7684\u5F00\u5934\u6DFB\u52A0\u4E86\u6A21\u578B\u4F7F\u7528\u7684\u7279\u6B8A\u6807\u8BB0\u3002\u5BF9\u4E8EGPT-2,\u552F\u4E00\u7684\u7279\u6B8A\u6807\u8BB0\u662F "),Qa=r("code"),Et=p('"<|endoftext|>"'),vt=p(":"),Wn=u(),j(bs.$$.fragment),Yn=u(),Zs=r("p"),yt=p("\u6211\u4EEC\u73B0\u5728\u9700\u8981\u5C06\u6BCF\u4E2A\u5355\u8BCD\u62C6\u5206\u4E3A\u5355\u72EC\u7684\u5B57\u7B26,\u4EE5\u4FBF\u80FD\u591F\u5F00\u59CB\u8BAD\u7EC3:"),Jn=u(),j(ds.$$.fragment),Zn=u(),Qs=r("p"),Pt=p("\u73B0\u5728\u6211\u4EEC\u5DF2\u51C6\u5907\u597D\u8FDB\u884C\u8BAD\u7EC3,\u8BA9\u6211\u4EEC\u7F16\u5199\u4E00\u4E2A\u51FD\u6570\u6765\u8BA1\u7B97\u6BCF\u5BF9\u7684\u9891\u7387\u3002\u6211\u4EEC\u9700\u8981\u5728\u8BAD\u7EC3\u7684\u6BCF\u4E2A\u6B65\u9AA4\u4E2D\u4F7F\u7528\u5B83:"),Qn=u(),j(ws.$$.fragment),Xn=u(),Xs=r("p"),Tt=p("\u8BA9\u6211\u4EEC\u6765\u770B\u770B\u8FD9\u4E2A\u5B57\u5178\u5728\u521D\u59CB\u62C6\u5206\u540E\u7684\u4E00\u90E8\u5206:"),sl=u(),j(ks.$$.fragment),al=u(),j(Es.$$.fragment),nl=u(),sa=r("p"),Ct=p("\u73B0\u5728, \u627E\u5230\u6700\u9891\u7E41\u7684\u5BF9\u53EA\u9700\u8981\u4E00\u4E2A\u5FEB\u901F\u7684\u5FAA\u73AF:"),ll=u(),j(vs.$$.fragment),pl=u(),j(ys.$$.fragment),tl=u(),I=r("p"),zt=p("\u6240\u4EE5\u7B2C\u4E00\u4E2A\u8981\u5B66\u4E60\u7684\u5408\u5E76\u662F "),Xa=r("code"),Ot=p("('\u0120', 't') -> '\u0120t'"),Dt=p(", \u6211\u4EEC\u6DFB\u52A0 "),sn=r("code"),Bt=p("'\u0120t'"),Nt=p(" \u5230\u8BCD\u6C47\u8868:"),el=u(),j(Ps.$$.fragment),rl=u(),X=r("p"),Gt=p("\u8981\u7EE7\u7EED\u63A5\u4E0B\u6765\u7684\u6B65\u9AA4,\u6211\u4EEC\u9700\u8981\u5728\u6211\u4EEC\u7684"),an=r("code"),Ht=p("\u5206\u8BCD"),At=p("\u5B57\u5178\u4E2D\u5E94\u7528\u8BE5\u5408\u5E76\u3002\u8BA9\u6211\u4EEC\u4E3A\u6B64\u7F16\u5199\u53E6\u4E00\u4E2A\u51FD\u6570:"),il=u(),j(Ts.$$.fragment),ol=u(),aa=r("p"),Rt=p("\u6211\u4EEC\u53EF\u4EE5\u770B\u770B\u7B2C\u4E00\u6B21\u5408\u5E76\u7684\u7ED3\u679C:"),ul=u(),j(Cs.$$.fragment),cl=u(),j(zs.$$.fragment),hl=u(),na=r("p"),It=p("\u73B0\u5728\u6211\u4EEC\u6709\u4E86\u5FAA\u73AF\u6240\u9700\u7684\u4E00\u5207,\u76F4\u5230\u6211\u4EEC\u5B66\u4F1A\u4E86\u6211\u4EEC\u60F3\u8981\u7684\u6240\u6709\u5408\u5E76\u3002\u6211\u4EEC\u7684\u76EE\u6807\u662F\u8BCD\u6C47\u91CF\u8FBE\u523050:"),fl=u(),j(Os.$$.fragment),jl=u(),la=r("p"),St=p("\u7ED3\u679C,\u6211\u4EEC\u5B66\u4E60\u4E86 19 \u6761\u5408\u5E76\u89C4\u5219(\u521D\u59CB\u8BCD\u6C47\u8868\u7684\u5927\u5C0F 31 \u2014 30 \u5B57\u6BCD\u5B57\u7B26,\u52A0\u4E0A\u7279\u6B8A\u6807\u8BB0):"),xl=u(),j(Ds.$$.fragment),gl=u(),j(Bs.$$.fragment),ml=u(),pa=r("p"),Lt=p("\u8BCD\u6C47\u8868\u7531\u7279\u6B8A\u6807\u8BB0\u3001\u521D\u59CB\u5B57\u6BCD\u548C\u6240\u6709\u5408\u5E76\u7ED3\u679C\u7EC4\u6210:"),$l=u(),j(Ns.$$.fragment),_l=u(),j(Gs.$$.fragment),ql=u(),j(ss.$$.fragment),bl=u(),ta=r("p"),Ut=p("\u4E3A\u4E86\u5BF9\u65B0\u6587\u672C\u8FDB\u884C\u5206\u8BCD,\u6211\u4EEC\u5BF9\u5176\u8FDB\u884C\u9884\u5206\u8BCD\u3001\u62C6\u5206\uFF0C\u7136\u540E\u5E94\u7528\u5B66\u5230\u7684\u6240\u6709\u5408\u5E76\u89C4\u5219:"),dl=u(),j(Hs.$$.fragment),wl=u(),ea=r("p"),Ft=p("\u6211\u4EEC\u53EF\u4EE5\u5728\u4EFB\u4F55\u7531\u5B57\u6BCD\u8868\u4E2D\u7684\u5B57\u7B26\u7EC4\u6210\u7684\u6587\u672C\u4E0A\u5C1D\u8BD5\u8FD9\u4E2A:"),kl=u(),j(As.$$.fragment),El=u(),j(Rs.$$.fragment),vl=u(),j(as.$$.fragment),yl=u(),ra=r("p"),Vt=p("\u8FD9\u5C31\u662F BPE \u7B97\u6CD5\uFF01\u63A5\u4E0B\u6765,\u6211\u4EEC\u5C06\u770B\u770B WordPiece\u3002"),this.h()},l(s){const l=er('[data-svelte="svelte-1phssyn"]',document.head);h=i(l,"META",{name:!0,content:!0}),l.forEach(a),E=c(s),f=i(s,"H1",{id:!0});var nn=o(f);w=t(nn,"\u5B57\u8282\u5BF9\u7F16\u7801\u6807\u8BB0\u5316"),nn.forEach(a),y=c(s),x(b.$$.fragment,s),k=c(s),T=i(s,"P",{});var ln=o(T);S=t(ln,"\u5B57\u8282\u5BF9\u7F16\u7801(BPE)\u6700\u521D\u88AB\u5F00\u53D1\u4E3A\u4E00\u79CD\u538B\u7F29\u6587\u672C\u7684\u7B97\u6CD5,\u7136\u540E\u5728\u9884\u8BAD\u7EC3 GPT \u6A21\u578B\u65F6\u88AB OpenAI \u7528\u4E8E\u6807\u8BB0\u5316\u3002\u8BB8\u591A Transformer \u6A21\u578B\u90FD\u4F7F\u7528\u5B83,\u5305\u62EC GPT\u3001GPT-2\u3001RoBERTa\u3001BART \u548C DeBERTa\u3002"),ln.forEach(a),D=c(s),x(H.$$.fragment,s),ps=c(s),x(F.$$.fragment,s),rn=c(s),ts=i(s,"H2",{id:!0});var pn=o(ts);Bl=t(pn,"\u8BAD\u7EC3\u7B97\u6CD5"),pn.forEach(a),on=c(s),Ss=i(s,"P",{});var tn=o(Ss);Nl=t(tn,"BPE \u8BAD\u7EC3\u9996\u5148\u8BA1\u7B97\u8BED\u6599\u5E93\u4E2D\u4F7F\u7528\u7684\u552F\u4E00\u5355\u8BCD\u96C6(\u5728\u5B8C\u6210\u6807\u51C6\u5316\u548C\u9884\u6807\u8BB0\u5316\u6B65\u9AA4\u4E4B\u540E),\u7136\u540E\u901A\u8FC7\u83B7\u53D6\u7528\u4E8E\u7F16\u5199\u8FD9\u4E9B\u5355\u8BCD\u7684\u6240\u6709\u7B26\u53F7\u6765\u6784\u5EFA\u8BCD\u6C47\u8868\u3002\u4E00\u4E2A\u975E\u5E38\u7B80\u5355\u7684\u4F8B\u5B50,\u5047\u8BBE\u6211\u4EEC\u7684\u8BED\u6599\u5E93\u4F7F\u7528\u4E86\u8FD9\u4E94\u4E2A\u8BCD:"),tn.forEach(a),un=c(s),x(es.$$.fragment,s),cn=c(s),V=i(s,"P",{});var Is=o(V);Gl=t(Is,"\u57FA\u7840\u8BCD\u6C47\u5C06\u662F "),ha=i(Is,"CODE",{});var en=o(ha);Hl=t(en,'["b", "g", "h", "n", "p", "s", "u"]'),en.forEach(a),Al=t(Is,"\u3002\u5BF9\u4E8E\u5B9E\u9645\u60C5\u51B5,\u57FA\u672C\u8BCD\u6C47\u8868\u5C06\u5305\u542B\u6240\u6709 ASCII \u5B57\u7B26,\u81F3\u5C11,\u53EF\u80FD\u8FD8\u5305\u542B\u4E00\u4E9B Unicode \u5B57\u7B26\u3002\u5982\u679C\u60A8\u6B63\u5728\u6807\u8BB0\u7684\u793A\u4F8B\u4F7F\u7528\u4E0D\u5728\u8BAD\u7EC3\u8BED\u6599\u5E93\u4E2D\u7684\u5B57\u7B26,\u5219\u8BE5\u5B57\u7B26\u5C06\u8F6C\u6362\u4E3A\u672A\u77E5\u6807\u8BB0\u3002\u8FD9\u5C31\u662F\u4E3A\u4EC0\u4E48\u8BB8\u591A NLP \u6A21\u578B\u5728\u5206\u6790\u5E26\u6709\u8868\u60C5\u7B26\u53F7\u7684\u5185\u5BB9\u65B9\u9762\u975E\u5E38\u7CDF\u7CD5\u7684\u539F\u56E0\u4E4B\u4E00\u3002"),Is.forEach(a),hn=c(s),x(M.$$.fragment,s),fn=c(s),K=i(s,"P",{});var Tl=o(K);Rl=t(Tl,"\u83B7\u5F97\u8FD9\u4E2A\u57FA\u672C\u8BCD\u6C47\u540E,\u6211\u4EEC\u6DFB\u52A0\u65B0\u7684\u6807\u8BB0,\u76F4\u5230\u901A\u8FC7\u5B66\u4E60"),fa=i(Tl,"EM",{});var Mt=o(fa);Il=t(Mt,"\u5408\u5E76"),Mt.forEach(a),Sl=t(Tl,"\u8FBE\u5230\u6240\u9700\u7684\u8BCD\u6C47\u91CF,\u8FD9\u662F\u5C06\u73B0\u6709\u8BCD\u6C47\u8868\u7684\u4E24\u4E2A\u5143\u7D20\u5408\u5E76\u4E3A\u4E00\u4E2A\u65B0\u5143\u7D20\u7684\u89C4\u5219\u3002\u56E0\u6B64,\u5728\u5F00\u59CB\u65F6,\u8FD9\u4E9B\u5408\u5E76\u5C06\u521B\u5EFA\u5177\u6709\u4E24\u4E2A\u5B57\u7B26\u7684\u6807\u8BB0,\u7136\u540E,\u968F\u7740\u8BAD\u7EC3\u7684\u8FDB\u884C,\u4F1A\u521B\u5EFA\u66F4\u957F\u7684\u5B50\u8BCD\u3002"),Tl.forEach(a),jn=c(s),Ls=i(s,"P",{});var Kt=o(Ls);Ll=t(Kt,"\u5728\u5206\u8BCD\u5668\u8BAD\u7EC3\u671F\u95F4\u7684\u4EFB\u4F55\u4E00\u6B65,BPE \u7B97\u6CD5\u90FD\u4F1A\u641C\u7D22\u6700\u5E38\u89C1\u7684\u73B0\u6709\u6807\u8BB0\u5BF9 (\u201C\u5BF9\u201D,\u8FD9\u91CC\u6211\u4EEC\u6307\u7684\u662F\u5355\u8BCD\u4E2D\u7684\u4E24\u4E2A\u8FDE\u7EED\u6807\u8BB0)\u3002\u6700\u9891\u7E41\u7684\u4E00\u5BF9\u5C06\u88AB\u5408\u5E76,\u6211\u4EEC\u51B2\u6D17\u5E76\u91CD\u590D\u4E0B\u4E00\u6B65\u3002"),Kt.forEach(a),xn=c(s),Us=i(s,"P",{});var Wt=o(Us);Ul=t(Wt,"\u56DE\u5230\u6211\u4EEC\u4E4B\u524D\u7684\u4F8B\u5B50,\u8BA9\u6211\u4EEC\u5047\u8BBE\u5355\u8BCD\u5177\u6709\u4EE5\u4E0B\u9891\u7387:"),Wt.forEach(a),gn=c(s),x(rs.$$.fragment,s),mn=c(s),C=i(s,"P",{});var G=o(C);Fl=t(G,"\u610F\u5473\u7740 "),ja=i(G,"CODE",{});var Yt=o(ja);Vl=t(Yt,'"hug"'),Yt.forEach(a),Ml=t(G," \u5728\u8BED\u6599\u5E93\u4E2D\u51FA\u73B0\u4E8610\u6B21, "),xa=i(G,"CODE",{});var Jt=o(xa);Kl=t(Jt,'"pug"'),Jt.forEach(a),Wl=t(G," 5\u6B21, "),ga=i(G,"CODE",{});var Zt=o(ga);Yl=t(Zt,'"pun"'),Zt.forEach(a),Jl=t(G," 12\u6B21, "),ma=i(G,"CODE",{});var Qt=o(ma);Zl=t(Qt,'"bun"'),Qt.forEach(a),Ql=t(G," 4\u6B21, \u4EE5\u53CA "),$a=i(G,"CODE",{});var Xt=o($a);Xl=t(Xt,'"hugs"'),Xt.forEach(a),sp=t(G," 5\u6B21\u3002\u6211\u4EEC\u901A\u8FC7\u5C06\u6BCF\u4E2A\u5355\u8BCD\u62C6\u5206\u4E3A\u5B57\u7B26(\u5F62\u6210\u6211\u4EEC\u521D\u59CB\u8BCD\u6C47\u8868\u7684\u5B57\u7B26)\u6765\u5F00\u59CB\u8BAD\u7EC3,\u8FD9\u6837\u6211\u4EEC\u5C31\u53EF\u4EE5\u5C06\u6BCF\u4E2A\u5355\u8BCD\u89C6\u4E3A\u4E00\u4E2A\u6807\u8BB0\u5217\u8868:"),G.forEach(a),$n=c(s),x(is.$$.fragment,s),_n=c(s),P=i(s,"P",{});var z=o(P);ap=t(z,"\u7136\u540E\u6211\u4EEC\u770B\u6210\u5BF9\u3002\u8FD9\u5BF9 "),_a=i(z,"CODE",{});var se=o(_a);np=t(se,'("h", "u")'),se.forEach(a),lp=t(z," \u51FA\u73B0\u5728\u5355\u8BCD "),qa=i(z,"CODE",{});var ae=o(qa);pp=t(ae,'"hug"'),ae.forEach(a),tp=t(z," \u548C "),ba=i(z,"CODE",{});var ne=o(ba);ep=t(ne,'"hugs"'),ne.forEach(a),rp=t(z,"\u4E2D,\u6240\u4EE5\u8BED\u6599\u5E93\u4E2D\u603B\u5171\u670915\u6B21\u3002\u4E0D\u8FC7,\u8FD9\u5E76\u4E0D\u662F\u6700\u9891\u7E41\u7684\u4E00\u5BF9:\u8FD9\u4E2A\u8363\u8A89\u5C5E\u4E8E "),da=i(z,"CODE",{});var le=o(da);ip=t(le,'("u", "g")'),le.forEach(a),op=t(z,",\u5B83\u51FA\u73B0\u5728 "),wa=i(z,"CODE",{});var pe=o(wa);up=t(pe,'"hug"'),pe.forEach(a),cp=t(z,", "),ka=i(z,"CODE",{});var te=o(ka);hp=t(te,'"pug"'),te.forEach(a),fp=t(z,", \u4EE5\u53CA "),Ea=i(z,"CODE",{});var ee=o(Ea);jp=t(ee,'"hugs"'),ee.forEach(a),xp=t(z,"\u4E2D,\u5728\u8BCD\u6C47\u8868\u4E2D\u603B\u5171 20 \u6B21\u3002"),z.forEach(a),qn=c(s),A=i(s,"P",{});var ia=o(A);gp=t(ia,"\u56E0\u6B64,\u6807\u8BB0\u5668\u5B66\u4E60\u7684\u7B2C\u4E00\u4E2A\u5408\u5E76\u89C4\u5219\u662F "),va=i(ia,"CODE",{});var re=o(va);mp=t(re,'("u", "g") -> "ug"'),re.forEach(a),$p=t(ia,",\u610F\u601D\u5C31\u662F "),ya=i(ia,"CODE",{});var ie=o(ya);_p=t(ie,'"ug"'),ie.forEach(a),qp=t(ia," \u5C06\u88AB\u6DFB\u52A0\u5230\u8BCD\u6C47\u8868\u4E2D,\u5E76\u4E14\u8FD9\u5BF9\u5E94\u8BE5\u5408\u5E76\u5230\u8BED\u6599\u5E93\u7684\u6240\u6709\u5355\u8BCD\u4E2D\u3002\u5728\u8FD9\u4E2A\u9636\u6BB5\u7ED3\u675F\u65F6,\u8BCD\u6C47\u8868\u548C\u8BED\u6599\u5E93\u770B\u8D77\u6765\u50CF\u8FD9\u6837:"),ia.forEach(a),bn=c(s),x(os.$$.fragment,s),dn=c(s),B=i(s,"P",{});var ns=o(B);bp=t(ns,"\u73B0\u5728\u6211\u4EEC\u6709\u4E00\u4E9B\u5BFC\u81F4\u6807\u8BB0\u957F\u4E8E\u4E24\u4E2A\u5B57\u7B26\u7684\u5BF9: \u4F8B\u5982 "),Pa=i(ns,"CODE",{});var oe=o(Pa);dp=t(oe,'("h", "ug")'),oe.forEach(a),wp=t(ns,", \u5728\u8BED\u6599\u5E93\u4E2D\u51FA\u73B015\u6B21\u3002\u7136\u800C,\u8FD9\u4E2A\u9636\u6BB5\u6700\u9891\u7E41\u7684\u5BF9\u662F "),Ta=i(ns,"CODE",{});var ue=o(Ta);kp=t(ue,'("u", "n")'),ue.forEach(a),Ep=t(ns,",\u5728\u8BED\u6599\u5E93\u4E2D\u51FA\u73B016\u6B21,\u6240\u4EE5\u5B66\u5230\u7684\u7B2C\u4E8C\u4E2A\u5408\u5E76\u89C4\u5219\u662F "),Ca=i(ns,"CODE",{});var ce=o(Ca);vp=t(ce,'("u", "n") -> "un"'),ce.forEach(a),yp=t(ns,"\u3002\u5C06\u5176\u6DFB\u52A0\u5230\u8BCD\u6C47\u8868\u5E76\u5408\u5E76\u6240\u6709\u73B0\u6709\u7684\u8FD9\u4E2A\u5BF9,\u5C06\u51FA\u73B0:"),ns.forEach(a),wn=c(s),x(us.$$.fragment,s),kn=c(s),R=i(s,"P",{});var oa=o(R);Pp=t(oa,"\u73B0\u5728\u6700\u9891\u7E41\u7684\u4E00\u5BF9\u662F "),za=i(oa,"CODE",{});var he=o(za);Tp=t(he,'("h", "ug")'),he.forEach(a),Cp=t(oa,",\u6240\u4EE5\u6211\u4EEC\u5B66\u4E60\u4E86\u5408\u5E76\u89C4\u5219 "),Oa=i(oa,"CODE",{});var fe=o(Oa);zp=t(fe,'("h", "ug") -> "hug"'),fe.forEach(a),Op=t(oa,",\u8FD9\u7ED9\u4E86\u6211\u4EEC\u7B2C\u4E00\u4E2A\u4E09\u4E2A\u5B57\u6BCD\u7684\u6807\u8BB0\u3002\u5408\u5E76\u540E,\u8BED\u6599\u5E93\u5982\u4E0B\u6240\u793A:"),oa.forEach(a),En=c(s),x(cs.$$.fragment,s),vn=c(s),Fs=i(s,"P",{});var je=o(Fs);Dp=t(je,"\u6211\u4EEC\u7EE7\u7EED\u8FD9\u6837\u5408\u5E76,\u76F4\u5230\u8FBE\u5230\u6211\u4EEC\u6240\u9700\u7684\u8BCD\u6C47\u91CF\u3002"),je.forEach(a),yn=c(s),x(W.$$.fragment,s),Pn=c(s),hs=i(s,"H2",{id:!0});var xe=o(hs);Bp=t(xe,"\u6807\u8BB0\u5316\u7B97\u6CD5"),xe.forEach(a),Tn=c(s),Vs=i(s,"P",{});var ge=o(Vs);Np=t(ge,"\u6807\u8BB0\u5316\u7D27\u8DDF\u8BAD\u7EC3\u8FC7\u7A0B,\u4ECE\u67D0\u79CD\u610F\u4E49\u4E0A\u8BF4,\u901A\u8FC7\u5E94\u7528\u4EE5\u4E0B\u6B65\u9AA4\u5BF9\u65B0\u8F93\u5165\u8FDB\u884C\u6807\u8BB0:"),ge.forEach(a),Cn=c(s),N=i(s,"OL",{});var ls=o(N);Da=i(ls,"LI",{});var me=o(Da);Gp=t(me,"\u89C4\u8303\u5316"),me.forEach(a),Hp=c(ls),Ba=i(ls,"LI",{});var $e=o(Ba);Ap=t($e,"\u9884\u6807\u8BB0\u5316"),$e.forEach(a),Rp=c(ls),Na=i(ls,"LI",{});var _e=o(Na);Ip=t(_e,"\u5C06\u5355\u8BCD\u62C6\u5206\u4E3A\u5355\u4E2A\u5B57\u7B26"),_e.forEach(a),Sp=c(ls),Ga=i(ls,"LI",{});var qe=o(Ga);Lp=t(qe,"\u5C06\u5B66\u4E60\u7684\u5408\u5E76\u89C4\u5219\u6309\u987A\u5E8F\u5E94\u7528\u4E8E\u8FD9\u4E9B\u62C6\u5206"),qe.forEach(a),ls.forEach(a),zn=c(s),Ms=i(s,"P",{});var be=o(Ms);Up=t(be,"\u8BA9\u6211\u4EEC\u4EE5\u6211\u4EEC\u5728\u8BAD\u7EC3\u671F\u95F4\u4F7F\u7528\u7684\u793A\u4F8B\u4E3A\u4F8B,\u5B66\u4E60\u4E09\u4E2A\u5408\u5E76\u89C4\u5219:"),be.forEach(a),On=c(s),x(fs.$$.fragment,s),Dn=c(s),d=i(s,"P",{});var v=o(d);Fp=t(v,"\u8FD9\u4E2A\u5355\u8BCD "),Ha=i(v,"CODE",{});var de=o(Ha);Vp=t(de,'"bug"'),de.forEach(a),Mp=t(v," \u5C06\u88AB\u6807\u8BB0\u4E3A "),Aa=i(v,"CODE",{});var we=o(Aa);Kp=t(we,'["b", "ug"]'),we.forEach(a),Wp=t(v,"\u3002\u7136\u800C "),Ra=i(v,"CODE",{});var ke=o(Ra);Yp=t(ke,'"mug"'),ke.forEach(a),Jp=t(v,",\u5C06\u88AB\u6807\u8BB0\u4E3A "),Ia=i(v,"CODE",{});var Ee=o(Ia);Zp=t(Ee,'["[UNK]", "ug"]'),Ee.forEach(a),Qp=t(v,",\u56E0\u4E3A\u5B57\u6BCD "),Sa=i(v,"CODE",{});var ve=o(Sa);Xp=t(ve,'"m"'),ve.forEach(a),st=t(v," \u4E0D\u518D\u57FA\u672C\u8BCD\u6C47\u8868\u4E2D\u3002\u540C\u6837,\u5355\u8BCD"),La=i(v,"CODE",{});var ye=o(La);at=t(ye,'"thug"'),ye.forEach(a),nt=t(v," \u4F1A\u88AB\u6807\u8BB0\u4E3A "),Ua=i(v,"CODE",{});var Pe=o(Ua);lt=t(Pe,'["[UNK]", "hug"]'),Pe.forEach(a),pt=t(v,": \u5B57\u6BCD "),Fa=i(v,"CODE",{});var Te=o(Fa);tt=t(Te,'"t"'),Te.forEach(a),et=t(v," \u4E0D\u5728\u57FA\u672C\u8BCD\u6C47\u8868\u4E2D,\u5E94\u7528\u5408\u5E76\u89C4\u5219\u9996\u5148\u5BFC\u81F4 "),Va=i(v,"CODE",{});var Ce=o(Va);rt=t(Ce,'"u"'),Ce.forEach(a),it=t(v," \u548C "),Ma=i(v,"CODE",{});var ze=o(Ma);ot=t(ze,'"g"'),ze.forEach(a),ut=t(v," \u88AB\u5408\u5E76,\u7136\u540E\u662F "),Ka=i(v,"CODE",{});var Oe=o(Ka);ct=t(Oe,'"hu"'),Oe.forEach(a),ht=t(v," \u548C "),Wa=i(v,"CODE",{});var De=o(Wa);ft=t(De,'"g"'),De.forEach(a),jt=t(v," \u88AB\u5408\u5E76\u3002"),v.forEach(a),Bn=c(s),x(Y.$$.fragment,s),Nn=c(s),L=i(s,"H2",{class:!0});var Cl=o(L);J=i(Cl,"A",{id:!0,class:!0,href:!0});var Be=o(J);Ya=i(Be,"SPAN",{});var Ne=o(Ya);x(js.$$.fragment,Ne),Ne.forEach(a),Be.forEach(a),xt=c(Cl),Ja=i(Cl,"SPAN",{});var Ge=o(Ja);gt=t(Ge,"\u5B9E\u73B0 BPE"),Ge.forEach(a),Cl.forEach(a),Gn=c(s),Ks=i(s,"P",{});var He=o(Ks);mt=t(He,"\u73B0\u5728\u8BA9\u6211\u4EEC\u770B\u4E00\u4E0B BPE \u7B97\u6CD5\u7684\u5B9E\u73B0\u3002\u8FD9\u4E0D\u4F1A\u662F\u4F60\u53EF\u4EE5\u5728\u5927\u578B\u8BED\u6599\u5E93\u4E0A\u5B9E\u9645\u4F7F\u7528\u7684\u4F18\u5316\u7248\u672C;\u6211\u4EEC\u53EA\u662F\u60F3\u5411\u4F60\u5C55\u793A\u4EE3\u7801,\u4EE5\u4FBF\u4F60\u53EF\u4EE5\u66F4\u597D\u5730\u7406\u89E3\u7B97\u6CD5"),He.forEach(a),Hn=c(s),Ws=i(s,"P",{});var Ae=o(Ws);$t=t(Ae,"\u9996\u5148\u6211\u4EEC\u9700\u8981\u4E00\u4E2A\u8BED\u6599\u5E93,\u6240\u4EE5\u8BA9\u6211\u4EEC\u7528\u51E0\u53E5\u8BDD\u521B\u5EFA\u4E00\u4E2A\u7B80\u5355\u7684\u8BED\u6599\u5E93:"),Ae.forEach(a),An=c(s),x(xs.$$.fragment,s),Rn=c(s),Z=i(s,"P",{});var zl=o(Z);_t=t(zl,"\u63A5\u4E0B\u6765,\u6211\u4EEC\u9700\u8981\u5C06\u8BE5\u8BED\u6599\u5E93\u9884\u5148\u6807\u8BB0\u4E3A\u5355\u8BCD\u3002\u7531\u4E8E\u6211\u4EEC\u6B63\u5728\u590D\u5236 BPE \u6807\u8BB0\u5668(\u5982 GPT-2),\u6211\u4EEC\u5C06\u4F7F\u7528 "),Za=i(zl,"CODE",{});var Re=o(Za);qt=t(Re,"gpt2"),Re.forEach(a),bt=t(zl," \u6807\u8BB0\u5668\u4F5C\u4E3A\u9884\u6807\u8BB0\u5316\u7684\u6807\u8BB0\u5668:"),zl.forEach(a),In=c(s),x(gs.$$.fragment,s),Sn=c(s),Ys=i(s,"P",{});var Ie=o(Ys);dt=t(Ie,"\u7136\u540E\u6211\u4EEC\u5728\u8FDB\u884C\u9884\u6807\u8BB0\u5316\u65F6\u8BA1\u7B97\u8BED\u6599\u5E93\u4E2D\u6BCF\u4E2A\u5355\u8BCD\u7684\u9891\u7387:"),Ie.forEach(a),Ln=c(s),x(ms.$$.fragment,s),Un=c(s),x($s.$$.fragment,s),Fn=c(s),Js=i(s,"P",{});var Se=o(Js);wt=t(Se,"\u4E0B\u4E00\u6B65\u662F\u8BA1\u7B97\u57FA\u672C\u8BCD\u6C47,\u7531\u8BED\u6599\u5E93\u4E2D\u4F7F\u7528\u7684\u6240\u6709\u5B57\u7B26\u7EC4\u6210:"),Se.forEach(a),Vn=c(s),x(_s.$$.fragment,s),Mn=c(s),x(qs.$$.fragment,s),Kn=c(s),Q=i(s,"P",{});var Ol=o(Q);kt=t(Ol,"\u6211\u4EEC\u8FD8\u5728\u8BE5\u8BCD\u6C47\u8868\u7684\u5F00\u5934\u6DFB\u52A0\u4E86\u6A21\u578B\u4F7F\u7528\u7684\u7279\u6B8A\u6807\u8BB0\u3002\u5BF9\u4E8EGPT-2,\u552F\u4E00\u7684\u7279\u6B8A\u6807\u8BB0\u662F "),Qa=i(Ol,"CODE",{});var Le=o(Qa);Et=t(Le,'"<|endoftext|>"'),Le.forEach(a),vt=t(Ol,":"),Ol.forEach(a),Wn=c(s),x(bs.$$.fragment,s),Yn=c(s),Zs=i(s,"P",{});var Ue=o(Zs);yt=t(Ue,"\u6211\u4EEC\u73B0\u5728\u9700\u8981\u5C06\u6BCF\u4E2A\u5355\u8BCD\u62C6\u5206\u4E3A\u5355\u72EC\u7684\u5B57\u7B26,\u4EE5\u4FBF\u80FD\u591F\u5F00\u59CB\u8BAD\u7EC3:"),Ue.forEach(a),Jn=c(s),x(ds.$$.fragment,s),Zn=c(s),Qs=i(s,"P",{});var Fe=o(Qs);Pt=t(Fe,"\u73B0\u5728\u6211\u4EEC\u5DF2\u51C6\u5907\u597D\u8FDB\u884C\u8BAD\u7EC3,\u8BA9\u6211\u4EEC\u7F16\u5199\u4E00\u4E2A\u51FD\u6570\u6765\u8BA1\u7B97\u6BCF\u5BF9\u7684\u9891\u7387\u3002\u6211\u4EEC\u9700\u8981\u5728\u8BAD\u7EC3\u7684\u6BCF\u4E2A\u6B65\u9AA4\u4E2D\u4F7F\u7528\u5B83:"),Fe.forEach(a),Qn=c(s),x(ws.$$.fragment,s),Xn=c(s),Xs=i(s,"P",{});var Ve=o(Xs);Tt=t(Ve,"\u8BA9\u6211\u4EEC\u6765\u770B\u770B\u8FD9\u4E2A\u5B57\u5178\u5728\u521D\u59CB\u62C6\u5206\u540E\u7684\u4E00\u90E8\u5206:"),Ve.forEach(a),sl=c(s),x(ks.$$.fragment,s),al=c(s),x(Es.$$.fragment,s),nl=c(s),sa=i(s,"P",{});var Me=o(sa);Ct=t(Me,"\u73B0\u5728, \u627E\u5230\u6700\u9891\u7E41\u7684\u5BF9\u53EA\u9700\u8981\u4E00\u4E2A\u5FEB\u901F\u7684\u5FAA\u73AF:"),Me.forEach(a),ll=c(s),x(vs.$$.fragment,s),pl=c(s),x(ys.$$.fragment,s),tl=c(s),I=i(s,"P",{});var ua=o(I);zt=t(ua,"\u6240\u4EE5\u7B2C\u4E00\u4E2A\u8981\u5B66\u4E60\u7684\u5408\u5E76\u662F "),Xa=i(ua,"CODE",{});var Ke=o(Xa);Ot=t(Ke,"('\u0120', 't') -> '\u0120t'"),Ke.forEach(a),Dt=t(ua,", \u6211\u4EEC\u6DFB\u52A0 "),sn=i(ua,"CODE",{});var We=o(sn);Bt=t(We,"'\u0120t'"),We.forEach(a),Nt=t(ua," \u5230\u8BCD\u6C47\u8868:"),ua.forEach(a),el=c(s),x(Ps.$$.fragment,s),rl=c(s),X=i(s,"P",{});var Dl=o(X);Gt=t(Dl,"\u8981\u7EE7\u7EED\u63A5\u4E0B\u6765\u7684\u6B65\u9AA4,\u6211\u4EEC\u9700\u8981\u5728\u6211\u4EEC\u7684"),an=i(Dl,"CODE",{});var Ye=o(an);Ht=t(Ye,"\u5206\u8BCD"),Ye.forEach(a),At=t(Dl,"\u5B57\u5178\u4E2D\u5E94\u7528\u8BE5\u5408\u5E76\u3002\u8BA9\u6211\u4EEC\u4E3A\u6B64\u7F16\u5199\u53E6\u4E00\u4E2A\u51FD\u6570:"),Dl.forEach(a),il=c(s),x(Ts.$$.fragment,s),ol=c(s),aa=i(s,"P",{});var Je=o(aa);Rt=t(Je,"\u6211\u4EEC\u53EF\u4EE5\u770B\u770B\u7B2C\u4E00\u6B21\u5408\u5E76\u7684\u7ED3\u679C:"),Je.forEach(a),ul=c(s),x(Cs.$$.fragment,s),cl=c(s),x(zs.$$.fragment,s),hl=c(s),na=i(s,"P",{});var Ze=o(na);It=t(Ze,"\u73B0\u5728\u6211\u4EEC\u6709\u4E86\u5FAA\u73AF\u6240\u9700\u7684\u4E00\u5207,\u76F4\u5230\u6211\u4EEC\u5B66\u4F1A\u4E86\u6211\u4EEC\u60F3\u8981\u7684\u6240\u6709\u5408\u5E76\u3002\u6211\u4EEC\u7684\u76EE\u6807\u662F\u8BCD\u6C47\u91CF\u8FBE\u523050:"),Ze.forEach(a),fl=c(s),x(Os.$$.fragment,s),jl=c(s),la=i(s,"P",{});var Qe=o(la);St=t(Qe,"\u7ED3\u679C,\u6211\u4EEC\u5B66\u4E60\u4E86 19 \u6761\u5408\u5E76\u89C4\u5219(\u521D\u59CB\u8BCD\u6C47\u8868\u7684\u5927\u5C0F 31 \u2014 30 \u5B57\u6BCD\u5B57\u7B26,\u52A0\u4E0A\u7279\u6B8A\u6807\u8BB0):"),Qe.forEach(a),xl=c(s),x(Ds.$$.fragment,s),gl=c(s),x(Bs.$$.fragment,s),ml=c(s),pa=i(s,"P",{});var Xe=o(pa);Lt=t(Xe,"\u8BCD\u6C47\u8868\u7531\u7279\u6B8A\u6807\u8BB0\u3001\u521D\u59CB\u5B57\u6BCD\u548C\u6240\u6709\u5408\u5E76\u7ED3\u679C\u7EC4\u6210:"),Xe.forEach(a),$l=c(s),x(Ns.$$.fragment,s),_l=c(s),x(Gs.$$.fragment,s),ql=c(s),x(ss.$$.fragment,s),bl=c(s),ta=i(s,"P",{});var sr=o(ta);Ut=t(sr,"\u4E3A\u4E86\u5BF9\u65B0\u6587\u672C\u8FDB\u884C\u5206\u8BCD,\u6211\u4EEC\u5BF9\u5176\u8FDB\u884C\u9884\u5206\u8BCD\u3001\u62C6\u5206\uFF0C\u7136\u540E\u5E94\u7528\u5B66\u5230\u7684\u6240\u6709\u5408\u5E76\u89C4\u5219:"),sr.forEach(a),dl=c(s),x(Hs.$$.fragment,s),wl=c(s),ea=i(s,"P",{});var ar=o(ea);Ft=t(ar,"\u6211\u4EEC\u53EF\u4EE5\u5728\u4EFB\u4F55\u7531\u5B57\u6BCD\u8868\u4E2D\u7684\u5B57\u7B26\u7EC4\u6210\u7684\u6587\u672C\u4E0A\u5C1D\u8BD5\u8FD9\u4E2A:"),ar.forEach(a),kl=c(s),x(As.$$.fragment,s),El=c(s),x(Rs.$$.fragment,s),vl=c(s),x(as.$$.fragment,s),yl=c(s),ra=i(s,"P",{});var nr=o(ra);Vt=t(nr,"\u8FD9\u5C31\u662F BPE \u7B97\u6CD5\uFF01\u63A5\u4E0B\u6765,\u6211\u4EEC\u5C06\u770B\u770B WordPiece\u3002"),nr.forEach(a),this.h()},h(){U(h,"name","hf:doc:metadata"),U(h,"content",JSON.stringify($r)),U(f,"id",""),U(ts,"id",""),U(hs,"id",""),U(J,"id","bpe"),U(J,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),U(J,"href","#bpe"),U(L,"class","relative group")},m(s,l){n(document.head,h),e(s,E,l),e(s,f,l),n(f,w),e(s,y,l),g(b,s,l),e(s,k,l),e(s,T,l),n(T,S),e(s,D,l),g(H,s,l),e(s,ps,l),g(F,s,l),e(s,rn,l),e(s,ts,l),n(ts,Bl),e(s,on,l),e(s,Ss,l),n(Ss,Nl),e(s,un,l),g(es,s,l),e(s,cn,l),e(s,V,l),n(V,Gl),n(V,ha),n(ha,Hl),n(V,Al),e(s,hn,l),g(M,s,l),e(s,fn,l),e(s,K,l),n(K,Rl),n(K,fa),n(fa,Il),n(K,Sl),e(s,jn,l),e(s,Ls,l),n(Ls,Ll),e(s,xn,l),e(s,Us,l),n(Us,Ul),e(s,gn,l),g(rs,s,l),e(s,mn,l),e(s,C,l),n(C,Fl),n(C,ja),n(ja,Vl),n(C,Ml),n(C,xa),n(xa,Kl),n(C,Wl),n(C,ga),n(ga,Yl),n(C,Jl),n(C,ma),n(ma,Zl),n(C,Ql),n(C,$a),n($a,Xl),n(C,sp),e(s,$n,l),g(is,s,l),e(s,_n,l),e(s,P,l),n(P,ap),n(P,_a),n(_a,np),n(P,lp),n(P,qa),n(qa,pp),n(P,tp),n(P,ba),n(ba,ep),n(P,rp),n(P,da),n(da,ip),n(P,op),n(P,wa),n(wa,up),n(P,cp),n(P,ka),n(ka,hp),n(P,fp),n(P,Ea),n(Ea,jp),n(P,xp),e(s,qn,l),e(s,A,l),n(A,gp),n(A,va),n(va,mp),n(A,$p),n(A,ya),n(ya,_p),n(A,qp),e(s,bn,l),g(os,s,l),e(s,dn,l),e(s,B,l),n(B,bp),n(B,Pa),n(Pa,dp),n(B,wp),n(B,Ta),n(Ta,kp),n(B,Ep),n(B,Ca),n(Ca,vp),n(B,yp),e(s,wn,l),g(us,s,l),e(s,kn,l),e(s,R,l),n(R,Pp),n(R,za),n(za,Tp),n(R,Cp),n(R,Oa),n(Oa,zp),n(R,Op),e(s,En,l),g(cs,s,l),e(s,vn,l),e(s,Fs,l),n(Fs,Dp),e(s,yn,l),g(W,s,l),e(s,Pn,l),e(s,hs,l),n(hs,Bp),e(s,Tn,l),e(s,Vs,l),n(Vs,Np),e(s,Cn,l),e(s,N,l),n(N,Da),n(Da,Gp),n(N,Hp),n(N,Ba),n(Ba,Ap),n(N,Rp),n(N,Na),n(Na,Ip),n(N,Sp),n(N,Ga),n(Ga,Lp),e(s,zn,l),e(s,Ms,l),n(Ms,Up),e(s,On,l),g(fs,s,l),e(s,Dn,l),e(s,d,l),n(d,Fp),n(d,Ha),n(Ha,Vp),n(d,Mp),n(d,Aa),n(Aa,Kp),n(d,Wp),n(d,Ra),n(Ra,Yp),n(d,Jp),n(d,Ia),n(Ia,Zp),n(d,Qp),n(d,Sa),n(Sa,Xp),n(d,st),n(d,La),n(La,at),n(d,nt),n(d,Ua),n(Ua,lt),n(d,pt),n(d,Fa),n(Fa,tt),n(d,et),n(d,Va),n(Va,rt),n(d,it),n(d,Ma),n(Ma,ot),n(d,ut),n(d,Ka),n(Ka,ct),n(d,ht),n(d,Wa),n(Wa,ft),n(d,jt),e(s,Bn,l),g(Y,s,l),e(s,Nn,l),e(s,L,l),n(L,J),n(J,Ya),g(js,Ya,null),n(L,xt),n(L,Ja),n(Ja,gt),e(s,Gn,l),e(s,Ks,l),n(Ks,mt),e(s,Hn,l),e(s,Ws,l),n(Ws,$t),e(s,An,l),g(xs,s,l),e(s,Rn,l),e(s,Z,l),n(Z,_t),n(Z,Za),n(Za,qt),n(Z,bt),e(s,In,l),g(gs,s,l),e(s,Sn,l),e(s,Ys,l),n(Ys,dt),e(s,Ln,l),g(ms,s,l),e(s,Un,l),g($s,s,l),e(s,Fn,l),e(s,Js,l),n(Js,wt),e(s,Vn,l),g(_s,s,l),e(s,Mn,l),g(qs,s,l),e(s,Kn,l),e(s,Q,l),n(Q,kt),n(Q,Qa),n(Qa,Et),n(Q,vt),e(s,Wn,l),g(bs,s,l),e(s,Yn,l),e(s,Zs,l),n(Zs,yt),e(s,Jn,l),g(ds,s,l),e(s,Zn,l),e(s,Qs,l),n(Qs,Pt),e(s,Qn,l),g(ws,s,l),e(s,Xn,l),e(s,Xs,l),n(Xs,Tt),e(s,sl,l),g(ks,s,l),e(s,al,l),g(Es,s,l),e(s,nl,l),e(s,sa,l),n(sa,Ct),e(s,ll,l),g(vs,s,l),e(s,pl,l),g(ys,s,l),e(s,tl,l),e(s,I,l),n(I,zt),n(I,Xa),n(Xa,Ot),n(I,Dt),n(I,sn),n(sn,Bt),n(I,Nt),e(s,el,l),g(Ps,s,l),e(s,rl,l),e(s,X,l),n(X,Gt),n(X,an),n(an,Ht),n(X,At),e(s,il,l),g(Ts,s,l),e(s,ol,l),e(s,aa,l),n(aa,Rt),e(s,ul,l),g(Cs,s,l),e(s,cl,l),g(zs,s,l),e(s,hl,l),e(s,na,l),n(na,It),e(s,fl,l),g(Os,s,l),e(s,jl,l),e(s,la,l),n(la,St),e(s,xl,l),g(Ds,s,l),e(s,gl,l),g(Bs,s,l),e(s,ml,l),e(s,pa,l),n(pa,Lt),e(s,$l,l),g(Ns,s,l),e(s,_l,l),g(Gs,s,l),e(s,ql,l),g(ss,s,l),e(s,bl,l),e(s,ta,l),n(ta,Ut),e(s,dl,l),g(Hs,s,l),e(s,wl,l),e(s,ea,l),n(ea,Ft),e(s,kl,l),g(As,s,l),e(s,El,l),g(Rs,s,l),e(s,vl,l),g(as,s,l),e(s,yl,l),e(s,ra,l),n(ra,Vt),Pl=!0},p(s,[l]){const nn={};l&2&&(nn.$$scope={dirty:l,ctx:s}),F.$set(nn);const ln={};l&2&&(ln.$$scope={dirty:l,ctx:s}),M.$set(ln);const pn={};l&2&&(pn.$$scope={dirty:l,ctx:s}),W.$set(pn);const tn={};l&2&&(tn.$$scope={dirty:l,ctx:s}),Y.$set(tn);const Is={};l&2&&(Is.$$scope={dirty:l,ctx:s}),ss.$set(Is);const en={};l&2&&(en.$$scope={dirty:l,ctx:s}),as.$set(en)},i(s){Pl||(m(b.$$.fragment,s),m(H.$$.fragment,s),m(F.$$.fragment,s),m(es.$$.fragment,s),m(M.$$.fragment,s),m(rs.$$.fragment,s),m(is.$$.fragment,s),m(os.$$.fragment,s),m(us.$$.fragment,s),m(cs.$$.fragment,s),m(W.$$.fragment,s),m(fs.$$.fragment,s),m(Y.$$.fragment,s),m(js.$$.fragment,s),m(xs.$$.fragment,s),m(gs.$$.fragment,s),m(ms.$$.fragment,s),m($s.$$.fragment,s),m(_s.$$.fragment,s),m(qs.$$.fragment,s),m(bs.$$.fragment,s),m(ds.$$.fragment,s),m(ws.$$.fragment,s),m(ks.$$.fragment,s),m(Es.$$.fragment,s),m(vs.$$.fragment,s),m(ys.$$.fragment,s),m(Ps.$$.fragment,s),m(Ts.$$.fragment,s),m(Cs.$$.fragment,s),m(zs.$$.fragment,s),m(Os.$$.fragment,s),m(Ds.$$.fragment,s),m(Bs.$$.fragment,s),m(Ns.$$.fragment,s),m(Gs.$$.fragment,s),m(ss.$$.fragment,s),m(Hs.$$.fragment,s),m(As.$$.fragment,s),m(Rs.$$.fragment,s),m(as.$$.fragment,s),Pl=!0)},o(s){$(b.$$.fragment,s),$(H.$$.fragment,s),$(F.$$.fragment,s),$(es.$$.fragment,s),$(M.$$.fragment,s),$(rs.$$.fragment,s),$(is.$$.fragment,s),$(os.$$.fragment,s),$(us.$$.fragment,s),$(cs.$$.fragment,s),$(W.$$.fragment,s),$(fs.$$.fragment,s),$(Y.$$.fragment,s),$(js.$$.fragment,s),$(xs.$$.fragment,s),$(gs.$$.fragment,s),$(ms.$$.fragment,s),$($s.$$.fragment,s),$(_s.$$.fragment,s),$(qs.$$.fragment,s),$(bs.$$.fragment,s),$(ds.$$.fragment,s),$(ws.$$.fragment,s),$(ks.$$.fragment,s),$(Es.$$.fragment,s),$(vs.$$.fragment,s),$(ys.$$.fragment,s),$(Ps.$$.fragment,s),$(Ts.$$.fragment,s),$(Cs.$$.fragment,s),$(zs.$$.fragment,s),$(Os.$$.fragment,s),$(Ds.$$.fragment,s),$(Bs.$$.fragment,s),$(Ns.$$.fragment,s),$(Gs.$$.fragment,s),$(ss.$$.fragment,s),$(Hs.$$.fragment,s),$(As.$$.fragment,s),$(Rs.$$.fragment,s),$(as.$$.fragment,s),Pl=!1},d(s){a(h),s&&a(E),s&&a(f),s&&a(y),_(b,s),s&&a(k),s&&a(T),s&&a(D),_(H,s),s&&a(ps),_(F,s),s&&a(rn),s&&a(ts),s&&a(on),s&&a(Ss),s&&a(un),_(es,s),s&&a(cn),s&&a(V),s&&a(hn),_(M,s),s&&a(fn),s&&a(K),s&&a(jn),s&&a(Ls),s&&a(xn),s&&a(Us),s&&a(gn),_(rs,s),s&&a(mn),s&&a(C),s&&a($n),_(is,s),s&&a(_n),s&&a(P),s&&a(qn),s&&a(A),s&&a(bn),_(os,s),s&&a(dn),s&&a(B),s&&a(wn),_(us,s),s&&a(kn),s&&a(R),s&&a(En),_(cs,s),s&&a(vn),s&&a(Fs),s&&a(yn),_(W,s),s&&a(Pn),s&&a(hs),s&&a(Tn),s&&a(Vs),s&&a(Cn),s&&a(N),s&&a(zn),s&&a(Ms),s&&a(On),_(fs,s),s&&a(Dn),s&&a(d),s&&a(Bn),_(Y,s),s&&a(Nn),s&&a(L),_(js),s&&a(Gn),s&&a(Ks),s&&a(Hn),s&&a(Ws),s&&a(An),_(xs,s),s&&a(Rn),s&&a(Z),s&&a(In),_(gs,s),s&&a(Sn),s&&a(Ys),s&&a(Ln),_(ms,s),s&&a(Un),_($s,s),s&&a(Fn),s&&a(Js),s&&a(Vn),_(_s,s),s&&a(Mn),_(qs,s),s&&a(Kn),s&&a(Q),s&&a(Wn),_(bs,s),s&&a(Yn),s&&a(Zs),s&&a(Jn),_(ds,s),s&&a(Zn),s&&a(Qs),s&&a(Qn),_(ws,s),s&&a(Xn),s&&a(Xs),s&&a(sl),_(ks,s),s&&a(al),_(Es,s),s&&a(nl),s&&a(sa),s&&a(ll),_(vs,s),s&&a(pl),_(ys,s),s&&a(tl),s&&a(I),s&&a(el),_(Ps,s),s&&a(rl),s&&a(X),s&&a(il),_(Ts,s),s&&a(ol),s&&a(aa),s&&a(ul),_(Cs,s),s&&a(cl),_(zs,s),s&&a(hl),s&&a(na),s&&a(fl),_(Os,s),s&&a(jl),s&&a(la),s&&a(xl),_(Ds,s),s&&a(gl),_(Bs,s),s&&a(ml),s&&a(pa),s&&a($l),_(Ns,s),s&&a(_l),_(Gs,s),s&&a(ql),_(ss,s),s&&a(bl),s&&a(ta),s&&a(dl),_(Hs,s),s&&a(wl),s&&a(ea),s&&a(kl),_(As,s),s&&a(El),_(Rs,s),s&&a(vl),_(as,s),s&&a(yl),s&&a(ra)}}}const $r={local:"",sections:[{local:"",title:"\u8BAD\u7EC3\u7B97\u6CD5"},{local:"",title:"\u6807\u8BB0\u5316\u7B97\u6CD5"},{local:"bpe",title:"\u5B9E\u73B0 BPE"}],title:"\u5B57\u8282\u5BF9\u7F16\u7801\u6807\u8BB0\u5316"};function _r(O){return rr(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class yr extends lr{constructor(h){super();pr(this,h,_r,mr,tr,{})}}export{yr as default,$r as metadata};
