import{S as ti,i as pi,s as ei,e as r,k as c,w as m,t,U as ri,M as ii,c as i,d as a,m as h,a as o,x as f,h as p,V as oi,b as A,G as n,g as e,y as j,q as g,o as d,B as x,v as ci}from"../../chunks/vendor-hf-doc-builder.js";import{T as _n}from"../../chunks/Tip-hf-doc-builder.js";import{Y as hi}from"../../chunks/Youtube-hf-doc-builder.js";import{I as ni}from"../../chunks/IconCopyLink-hf-doc-builder.js";import{C as _}from"../../chunks/CodeBlock-hf-doc-builder.js";import{C as ui}from"../../chunks/CourseFloatingBanner-hf-doc-builder.js";import"../../chunks/DocNotebookDropdown-hf-doc-builder.js";function mi(S){let u,q;return{c(){u=r("p"),q=t("\u{1F4A1} \u672C\u8282\u6DF1\u5165\u4ECB\u7ECD WordPiece,\u751A\u81F3\u5C55\u793A\u5B8C\u6574\u7684\u5B9E\u73B0\u3002\u5982\u679C\u60A8\u53EA\u60F3\u5927\u81F4\u4E86\u89E3\u6807\u8BB0\u5316\u7B97\u6CD5,\u53EF\u4EE5\u8DF3\u5230\u6700\u540E\u3002")},l(b){u=i(b,"P",{});var $=o(u);q=p($,"\u{1F4A1} \u672C\u8282\u6DF1\u5165\u4ECB\u7ECD WordPiece,\u751A\u81F3\u5C55\u793A\u5B8C\u6574\u7684\u5B9E\u73B0\u3002\u5982\u679C\u60A8\u53EA\u60F3\u5927\u81F4\u4E86\u89E3\u6807\u8BB0\u5316\u7B97\u6CD5,\u53EF\u4EE5\u8DF3\u5230\u6700\u540E\u3002"),$.forEach(a)},m(b,$){e(b,u,$),n(u,q)},d(b){b&&a(u)}}}function fi(S){let u,q;return{c(){u=r("p"),q=t("\u26A0\uFE0F Google \u4ECE\u672A\u5F00\u6E90 WordPiece \u8BAD\u7EC3\u7B97\u6CD5\u7684\u5B9E\u73B0,\u56E0\u6B64\u4EE5\u4E0B\u662F\u6211\u4EEC\u57FA\u4E8E\u5DF2\u53D1\u8868\u6587\u732E\u7684\u6700\u4F73\u731C\u6D4B\u3002\u5B83\u53EF\u80FD\u4E0D\u662F 100% \u51C6\u786E\u7684\u3002")},l(b){u=i(b,"P",{});var $=o(u);q=p($,"\u26A0\uFE0F Google \u4ECE\u672A\u5F00\u6E90 WordPiece \u8BAD\u7EC3\u7B97\u6CD5\u7684\u5B9E\u73B0,\u56E0\u6B64\u4EE5\u4E0B\u662F\u6211\u4EEC\u57FA\u4E8E\u5DF2\u53D1\u8868\u6587\u732E\u7684\u6700\u4F73\u731C\u6D4B\u3002\u5B83\u53EF\u80FD\u4E0D\u662F 100% \u51C6\u786E\u7684\u3002"),$.forEach(a)},m(b,$){e(b,u,$),n(u,q)},d(b){b&&a(u)}}}function ji(S){let u,q,b,$,O;return{c(){u=r("p"),q=t("\u270F\uFE0F "),b=r("strong"),$=t("\u73B0\u5728\u8F6E\u5230\u4F60\u4E86!"),O=t(" \u4E0B\u4E00\u4E2A\u5408\u5E76\u89C4\u5219\u662F\u4EC0\u4E48\uFF1F")},l(w){u=i(w,"P",{});var E=o(u);q=p(E,"\u270F\uFE0F "),b=i(E,"STRONG",{});var B=o(b);$=p(B,"\u73B0\u5728\u8F6E\u5230\u4F60\u4E86!"),B.forEach(a),O=p(E," \u4E0B\u4E00\u4E2A\u5408\u5E76\u89C4\u5219\u662F\u4EC0\u4E48\uFF1F"),E.forEach(a)},m(w,E){e(w,u,E),n(u,q),n(u,b),n(b,$),n(u,O)},d(w){w&&a(u)}}}function gi(S){let u,q,b,$,O,w,E,B;return{c(){u=r("p"),q=t("\u270F\uFE0F "),b=r("strong"),$=t("\u73B0\u5728\u8F6E\u5230\u4F60\u4E86!"),O=c(),w=r("code"),E=t('"pugs"'),B=t(" \u5C06\u88AB\u5982\u4F55\u6807\u8BB0?")},l(G){u=i(G,"P",{});var W=o(u);q=p(W,"\u270F\uFE0F "),b=i(W,"STRONG",{});var K=o(b);$=p(K,"\u73B0\u5728\u8F6E\u5230\u4F60\u4E86!"),K.forEach(a),O=h(W),w=i(W,"CODE",{});var es=o(w);E=p(es,'"pugs"'),es.forEach(a),B=p(W," \u5C06\u88AB\u5982\u4F55\u6807\u8BB0?"),W.forEach(a)},m(G,W){e(G,u,W),n(u,q),n(u,b),n(b,$),n(u,O),n(u,w),n(w,E),n(u,B)},d(G){G&&a(u)}}}function di(S){let u,q,b,$,O;return{c(){u=r("p"),q=t("\u{1F4A1} \u5728\u540C\u4E00\u8BED\u6599\u5E93\u4E0A\u4F7F\u7528 "),b=r("code"),$=t("train_new_from_iterator()"),O=t(" \u4E0D\u4F1A\u4EA7\u751F\u5B8C\u5168\u76F8\u540C\u7684\u8BCD\u6C47\u8868\u3002\u8FD9\u662F\u56E0\u4E3A \u{1F917} Tokenizers \u5E93\u6CA1\u6709\u4E3A\u8BAD\u7EC3\u5B9E\u73B0 WordPiece(\u56E0\u4E3A\u6211\u4EEC\u4E0D\u5B8C\u5168\u786E\u5B9A\u5B83\u7684\u5185\u90E8\u7ED3\u6784),\u800C\u662F\u4F7F\u7528 BPE\u3002")},l(w){u=i(w,"P",{});var E=o(u);q=p(E,"\u{1F4A1} \u5728\u540C\u4E00\u8BED\u6599\u5E93\u4E0A\u4F7F\u7528 "),b=i(E,"CODE",{});var B=o(b);$=p(B,"train_new_from_iterator()"),B.forEach(a),O=p(E," \u4E0D\u4F1A\u4EA7\u751F\u5B8C\u5168\u76F8\u540C\u7684\u8BCD\u6C47\u8868\u3002\u8FD9\u662F\u56E0\u4E3A \u{1F917} Tokenizers \u5E93\u6CA1\u6709\u4E3A\u8BAD\u7EC3\u5B9E\u73B0 WordPiece(\u56E0\u4E3A\u6211\u4EEC\u4E0D\u5B8C\u5168\u786E\u5B9A\u5B83\u7684\u5185\u90E8\u7ED3\u6784),\u800C\u662F\u4F7F\u7528 BPE\u3002"),E.forEach(a)},m(w,E){e(w,u,E),n(u,q),n(u,b),n(b,$),n(u,O)},d(w){w&&a(u)}}}function xi(S){let u,q,b,$,O,w,E,B,G,W,K,es,Ls,Zl,$n,rs,wn,I,qn,is,st,vn,Y,kn,U,at,ga,nt,lt,da,tt,pt,En,os,yn,Gs,et,Pn,cs,rt,Cn,li='<span class="katex-display"><span class="katex"><span class="katex-mathml"><math xmlns="http://www.w3.org/1998/Math/MathML" display="block"><semantics><mrow><mrow><mi mathvariant="normal">s</mi><mi mathvariant="normal">c</mi><mi mathvariant="normal">o</mi><mi mathvariant="normal">r</mi><mi mathvariant="normal">e</mi></mrow><mo>=</mo><mo stretchy="false">(</mo><mrow><mi mathvariant="normal">f</mi><mi mathvariant="normal">r</mi><mi mathvariant="normal">e</mi><mi mathvariant="normal">q</mi><mi mathvariant="normal">_</mi><mi mathvariant="normal">o</mi><mi mathvariant="normal">f</mi><mi mathvariant="normal">_</mi><mi mathvariant="normal">p</mi><mi mathvariant="normal">a</mi><mi mathvariant="normal">i</mi><mi mathvariant="normal">r</mi></mrow><mo stretchy="false">)</mo><mi mathvariant="normal">/</mi><mo stretchy="false">(</mo><mrow><mi mathvariant="normal">f</mi><mi mathvariant="normal">r</mi><mi mathvariant="normal">e</mi><mi mathvariant="normal">q</mi><mi mathvariant="normal">_</mi><mi mathvariant="normal">o</mi><mi mathvariant="normal">f</mi><mi mathvariant="normal">_</mi><mi mathvariant="normal">f</mi><mi mathvariant="normal">i</mi><mi mathvariant="normal">r</mi><mi mathvariant="normal">s</mi><mi mathvariant="normal">t</mi><mi mathvariant="normal">_</mi><mi mathvariant="normal">e</mi><mi mathvariant="normal">l</mi><mi mathvariant="normal">e</mi><mi mathvariant="normal">m</mi><mi mathvariant="normal">e</mi><mi mathvariant="normal">n</mi><mi mathvariant="normal">t</mi></mrow><mo>\xD7</mo><mrow><mi mathvariant="normal">f</mi><mi mathvariant="normal">r</mi><mi mathvariant="normal">e</mi><mi mathvariant="normal">q</mi><mi mathvariant="normal">_</mi><mi mathvariant="normal">o</mi><mi mathvariant="normal">f</mi><mi mathvariant="normal">_</mi><mi mathvariant="normal">s</mi><mi mathvariant="normal">e</mi><mi mathvariant="normal">c</mi><mi mathvariant="normal">o</mi><mi mathvariant="normal">n</mi><mi mathvariant="normal">d</mi><mi mathvariant="normal">_</mi><mi mathvariant="normal">e</mi><mi mathvariant="normal">l</mi><mi mathvariant="normal">e</mi><mi mathvariant="normal">m</mi><mi mathvariant="normal">e</mi><mi mathvariant="normal">n</mi><mi mathvariant="normal">t</mi></mrow><mo stretchy="false">)</mo></mrow><annotation encoding="application/x-tex">\\mathrm{score} = (\\mathrm{freq\\_of\\_pair}) / (\\mathrm{freq\\_of\\_first\\_element} \\times \\mathrm{freq\\_of\\_second\\_element})</annotation></semantics></math></span><span class="katex-html" aria-hidden="true"><span class="base"><span class="strut" style="height:0.4306em;"></span><span class="mord"><span class="mord mathrm">score</span></span><span class="mspace" style="margin-right:0.2778em;"></span><span class="mrel">=</span><span class="mspace" style="margin-right:0.2778em;"></span></span><span class="base"><span class="strut" style="height:1.06em;vertical-align:-0.31em;"></span><span class="mopen">(</span><span class="mord"><span class="mord mathrm">freq_of_pair</span></span><span class="mclose">)</span><span class="mord">/</span><span class="mopen">(</span><span class="mord"><span class="mord mathrm">freq_of_first_element</span></span><span class="mspace" style="margin-right:0.2222em;"></span><span class="mbin">\xD7</span><span class="mspace" style="margin-right:0.2222em;"></span></span><span class="base"><span class="strut" style="height:1.06em;vertical-align:-0.31em;"></span><span class="mord"><span class="mord mathrm">freq_of_second_element</span></span><span class="mclose">)</span></span></span></span></span>',Dn,P,it,xa,ot,ct,ba,ht,ut,_a,mt,ft,$a,jt,gt,wa,dt,xt,qa,bt,_t,zn,Vs,$t,On,hs,Tn,Is,wt,Bn,us,Nn,y,qt,va,vt,kt,ka,Et,yt,Ea,Pt,Ct,ya,Dt,zt,Pa,Ot,Tt,Ca,Bt,Nt,Da,Ht,Wt,Hn,F,St,za,At,Kt,Oa,Ut,Ft,Wn,ms,Sn,R,Rt,Ta,Mt,Lt,Ba,Gt,Vt,An,fs,Kn,M,It,Na,Yt,Jt,Ha,Qt,Xt,Un,js,Fn,Ys,Zt,Rn,J,Mn,gs,sp,Ln,C,ap,Wa,np,lp,Sa,tp,pp,Aa,ep,rp,Ka,ip,op,Ua,cp,hp,Fa,up,mp,Gn,Q,fp,Ra,jp,gp,Vn,v,dp,Ma,xp,bp,La,_p,$p,Ga,wp,qp,Va,vp,kp,Ia,Ep,yp,Ya,Pp,Cp,Ja,Dp,zp,Qa,Op,Tp,In,k,Bp,Xa,Np,Hp,Za,Wp,Sp,sn,Ap,Kp,an,Up,Fp,nn,Rp,Mp,ln,Lp,Gp,tn,Vp,Ip,pn,Yp,Jp,Yn,X,Jn,V,Z,en,ds,Qp,rn,Xp,Qn,Js,Zp,Xn,Qs,se,Zn,xs,sl,ss,ae,on,ne,le,al,bs,nl,Xs,te,ll,_s,tl,$s,pl,as,pe,cn,ee,re,el,ws,rl,qs,il,ns,ie,hn,oe,ce,ol,vs,cl,ls,he,un,ue,me,hl,ks,ul,Zs,fe,ml,Es,fl,sa,je,jl,ys,gl,Ps,dl,aa,ge,xl,Cs,bl,Ds,_l,L,de,mn,xe,be,fn,_e,$e,$l,zs,wl,ts,we,jn,qe,ve,ql,Os,vl,na,ke,kl,Ts,El,Bs,yl,la,Ee,Pl,Ns,Cl,ta,ye,Dl,Hs,zl,Ws,Ol,pa,Pe,Tl,ps,Bl,ea,Ce,Nl,Ss,Hl,ra,De,Wl,As,Sl,Ks,Al,ia,ze,Kl,Us,Ul,oa,Oe,Fl,Fs,Rl,Rs,Ml,ca,Te,Ll;return w=new ni({}),K=new ui({props:{chapter:6,classNames:"absolute z-10 right-0 top-0",notebooks:[{label:"Google Colab",value:"https://colab.research.google.com/github/huggingface/notebooks/blob/master/course/zh-CN/chapter6/section6.ipynb"},{label:"Aws Studio",value:"https://studiolab.sagemaker.aws/import/github/huggingface/notebooks/blob/master/course/zh-CN/chapter6/section6.ipynb"}]}}),rs=new hi({props:{id:"qpv6ms_t_1A"}}),I=new _n({props:{$$slots:{default:[mi]},$$scope:{ctx:S}}}),Y=new _n({props:{warning:!0,$$slots:{default:[fi]},$$scope:{ctx:S}}}),os=new _({props:{code:"w ##o ##r ##d",highlighted:"w ##o ##r ##d"}}),hs=new _({props:{code:'("hug", 10), ("pug", 5), ("pun", 12), ("bun", 4), ("hugs", 5)',highlighted:'(<span class="hljs-string">&quot;hug&quot;</span><span class="hljs-punctuation">,</span> <span class="hljs-number">10</span>)<span class="hljs-punctuation">,</span> (<span class="hljs-string">&quot;pug&quot;</span><span class="hljs-punctuation">,</span> <span class="hljs-number">5</span>)<span class="hljs-punctuation">,</span> (<span class="hljs-string">&quot;pun&quot;</span><span class="hljs-punctuation">,</span> <span class="hljs-number">12</span>)<span class="hljs-punctuation">,</span> (<span class="hljs-string">&quot;bun&quot;</span><span class="hljs-punctuation">,</span> <span class="hljs-number">4</span>)<span class="hljs-punctuation">,</span> (<span class="hljs-string">&quot;hugs&quot;</span><span class="hljs-punctuation">,</span> <span class="hljs-number">5</span>)'}}),us=new _({props:{code:'("h" "##u" "##g", 10), ("p" "##u" "##g", 5), ("p" "##u" "##n", 12), ("b" "##u" "##n", 4), ("h" "##u" "##g" "##s", 5)',highlighted:'(<span class="hljs-string">&quot;h&quot;</span> <span class="hljs-string">&quot;#<span class="hljs-subst">#u</span>&quot;</span> <span class="hljs-string">&quot;#<span class="hljs-subst">#g</span>&quot;</span>, <span class="hljs-number">10</span>), (<span class="hljs-string">&quot;p&quot;</span> <span class="hljs-string">&quot;#<span class="hljs-subst">#u</span>&quot;</span> <span class="hljs-string">&quot;#<span class="hljs-subst">#g</span>&quot;</span>, <span class="hljs-number">5</span>), (<span class="hljs-string">&quot;p&quot;</span> <span class="hljs-string">&quot;#<span class="hljs-subst">#u</span>&quot;</span> <span class="hljs-string">&quot;#<span class="hljs-subst">#n</span>&quot;</span>, <span class="hljs-number">12</span>), (<span class="hljs-string">&quot;b&quot;</span> <span class="hljs-string">&quot;#<span class="hljs-subst">#u</span>&quot;</span> <span class="hljs-string">&quot;#<span class="hljs-subst">#n</span>&quot;</span>, <span class="hljs-number">4</span>), (<span class="hljs-string">&quot;h&quot;</span> <span class="hljs-string">&quot;#<span class="hljs-subst">#u</span>&quot;</span> <span class="hljs-string">&quot;#<span class="hljs-subst">#g</span>&quot;</span> <span class="hljs-string">&quot;#<span class="hljs-subst">#s</span>&quot;</span>, <span class="hljs-number">5</span>)'}}),ms=new _({props:{code:`Vocabulary: ["b", "h", "p", "##g", "##n", "##s", "##u", "##gs"]
Corpus: ("h" "##u" "##g", 10), ("p" "##u" "##g", 5), ("p" "##u" "##n", 12), ("b" "##u" "##n", 4), ("h" "##u" "##gs", 5)`,highlighted:`Vocabulary: [<span class="hljs-string">&quot;b&quot;</span>, <span class="hljs-string">&quot;h&quot;</span>, <span class="hljs-string">&quot;p&quot;</span>, <span class="hljs-string">&quot;#<span class="hljs-subst">#g</span>&quot;</span>, <span class="hljs-string">&quot;#<span class="hljs-subst">#n</span>&quot;</span>, <span class="hljs-string">&quot;#<span class="hljs-subst">#s</span>&quot;</span>, <span class="hljs-string">&quot;#<span class="hljs-subst">#u</span>&quot;</span>, <span class="hljs-string">&quot;#<span class="hljs-subst">#gs</span>&quot;</span>]
Corpus: (<span class="hljs-string">&quot;h&quot;</span> <span class="hljs-string">&quot;#<span class="hljs-subst">#u</span>&quot;</span> <span class="hljs-string">&quot;#<span class="hljs-subst">#g</span>&quot;</span>, <span class="hljs-number">10</span>), (<span class="hljs-string">&quot;p&quot;</span> <span class="hljs-string">&quot;#<span class="hljs-subst">#u</span>&quot;</span> <span class="hljs-string">&quot;#<span class="hljs-subst">#g</span>&quot;</span>, <span class="hljs-number">5</span>), (<span class="hljs-string">&quot;p&quot;</span> <span class="hljs-string">&quot;#<span class="hljs-subst">#u</span>&quot;</span> <span class="hljs-string">&quot;#<span class="hljs-subst">#n</span>&quot;</span>, <span class="hljs-number">12</span>), (<span class="hljs-string">&quot;b&quot;</span> <span class="hljs-string">&quot;#<span class="hljs-subst">#u</span>&quot;</span> <span class="hljs-string">&quot;#<span class="hljs-subst">#n</span>&quot;</span>, <span class="hljs-number">4</span>), (<span class="hljs-string">&quot;h&quot;</span> <span class="hljs-string">&quot;#<span class="hljs-subst">#u</span>&quot;</span> <span class="hljs-string">&quot;#<span class="hljs-subst">#gs</span>&quot;</span>, <span class="hljs-number">5</span>)`}}),fs=new _({props:{code:`Vocabulary: ["b", "h", "p", "##g", "##n", "##s", "##u", "##gs", "hu"]
Corpus: ("hu" "##g", 10), ("p" "##u" "##g", 5), ("p" "##u" "##n", 12), ("b" "##u" "##n", 4), ("hu" "##gs", 5)`,highlighted:`Vocabulary: [<span class="hljs-string">&quot;b&quot;</span>, <span class="hljs-string">&quot;h&quot;</span>, <span class="hljs-string">&quot;p&quot;</span>, <span class="hljs-string">&quot;#<span class="hljs-subst">#g</span>&quot;</span>, <span class="hljs-string">&quot;#<span class="hljs-subst">#n</span>&quot;</span>, <span class="hljs-string">&quot;#<span class="hljs-subst">#s</span>&quot;</span>, <span class="hljs-string">&quot;#<span class="hljs-subst">#u</span>&quot;</span>, <span class="hljs-string">&quot;#<span class="hljs-subst">#gs</span>&quot;</span>, <span class="hljs-string">&quot;hu&quot;</span>]
Corpus: (<span class="hljs-string">&quot;hu&quot;</span> <span class="hljs-string">&quot;#<span class="hljs-subst">#g</span>&quot;</span>, <span class="hljs-number">10</span>), (<span class="hljs-string">&quot;p&quot;</span> <span class="hljs-string">&quot;#<span class="hljs-subst">#u</span>&quot;</span> <span class="hljs-string">&quot;#<span class="hljs-subst">#g</span>&quot;</span>, <span class="hljs-number">5</span>), (<span class="hljs-string">&quot;p&quot;</span> <span class="hljs-string">&quot;#<span class="hljs-subst">#u</span>&quot;</span> <span class="hljs-string">&quot;#<span class="hljs-subst">#n</span>&quot;</span>, <span class="hljs-number">12</span>), (<span class="hljs-string">&quot;b&quot;</span> <span class="hljs-string">&quot;#<span class="hljs-subst">#u</span>&quot;</span> <span class="hljs-string">&quot;#<span class="hljs-subst">#n</span>&quot;</span>, <span class="hljs-number">4</span>), (<span class="hljs-string">&quot;hu&quot;</span> <span class="hljs-string">&quot;#<span class="hljs-subst">#gs</span>&quot;</span>, <span class="hljs-number">5</span>)`}}),js=new _({props:{code:`Vocabulary: ["b", "h", "p", "##g", "##n", "##s", "##u", "##gs", "hu", "hug"]
Corpus: ("hug", 10), ("p" "##u" "##g", 5), ("p" "##u" "##n", 12), ("b" "##u" "##n", 4), ("hu" "##gs", 5)`,highlighted:`Vocabulary: [<span class="hljs-string">&quot;b&quot;</span>, <span class="hljs-string">&quot;h&quot;</span>, <span class="hljs-string">&quot;p&quot;</span>, <span class="hljs-string">&quot;#<span class="hljs-subst">#g</span>&quot;</span>, <span class="hljs-string">&quot;#<span class="hljs-subst">#n</span>&quot;</span>, <span class="hljs-string">&quot;#<span class="hljs-subst">#s</span>&quot;</span>, <span class="hljs-string">&quot;#<span class="hljs-subst">#u</span>&quot;</span>, <span class="hljs-string">&quot;#<span class="hljs-subst">#gs</span>&quot;</span>, <span class="hljs-string">&quot;hu&quot;</span>, <span class="hljs-string">&quot;hug&quot;</span>]
Corpus: (<span class="hljs-string">&quot;hug&quot;</span>, <span class="hljs-number">10</span>), (<span class="hljs-string">&quot;p&quot;</span> <span class="hljs-string">&quot;#<span class="hljs-subst">#u</span>&quot;</span> <span class="hljs-string">&quot;#<span class="hljs-subst">#g</span>&quot;</span>, <span class="hljs-number">5</span>), (<span class="hljs-string">&quot;p&quot;</span> <span class="hljs-string">&quot;#<span class="hljs-subst">#u</span>&quot;</span> <span class="hljs-string">&quot;#<span class="hljs-subst">#n</span>&quot;</span>, <span class="hljs-number">12</span>), (<span class="hljs-string">&quot;b&quot;</span> <span class="hljs-string">&quot;#<span class="hljs-subst">#u</span>&quot;</span> <span class="hljs-string">&quot;#<span class="hljs-subst">#n</span>&quot;</span>, <span class="hljs-number">4</span>), (<span class="hljs-string">&quot;hu&quot;</span> <span class="hljs-string">&quot;#<span class="hljs-subst">#gs</span>&quot;</span>, <span class="hljs-number">5</span>)`}}),J=new _n({props:{$$slots:{default:[ji]},$$scope:{ctx:S}}}),X=new _n({props:{$$slots:{default:[gi]},$$scope:{ctx:S}}}),ds=new ni({}),xs=new _({props:{code:`corpus = [
    "This is the Hugging Face course.",
    "This chapter is about tokenization.",
    "This section shows several tokenizer algorithms.",
    "Hopefully, you will be able to understand how they are trained and generate tokens.",
]`,highlighted:`corpus = [
    <span class="hljs-string">&quot;This is the Hugging Face course.&quot;</span>,
    <span class="hljs-string">&quot;This chapter is about tokenization.&quot;</span>,
    <span class="hljs-string">&quot;This section shows several tokenizer algorithms.&quot;</span>,
    <span class="hljs-string">&quot;Hopefully, you will be able to understand how they are trained and generate tokens.&quot;</span>,
]`}}),bs=new _({props:{code:`from transformers import AutoTokenizer

tokenizer = AutoTokenizer.from_pretrained("bert-base-cased")`,highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoTokenizer

tokenizer = AutoTokenizer.from_pretrained(<span class="hljs-string">&quot;bert-base-cased&quot;</span>)`}}),_s=new _({props:{code:`from collections import defaultdict

word_freqs = defaultdict(int)
for text in corpus:
    words_with_offsets = tokenizer.backend_tokenizer.pre_tokenizer.pre_tokenize_str(text)
    new_words = [word for word, offset in words_with_offsets]
    for word in new_words:
        word_freqs[word] += 1

word_freqs`,highlighted:`<span class="hljs-keyword">from</span> collections <span class="hljs-keyword">import</span> defaultdict

word_freqs = defaultdict(<span class="hljs-built_in">int</span>)
<span class="hljs-keyword">for</span> text <span class="hljs-keyword">in</span> corpus:
    words_with_offsets = tokenizer.backend_tokenizer.pre_tokenizer.pre_tokenize_str(text)
    new_words = [word <span class="hljs-keyword">for</span> word, offset <span class="hljs-keyword">in</span> words_with_offsets]
    <span class="hljs-keyword">for</span> word <span class="hljs-keyword">in</span> new_words:
        word_freqs[word] += <span class="hljs-number">1</span>

word_freqs`}}),$s=new _({props:{code:`defaultdict(
    int, {'This': 3, 'is': 2, 'the': 1, 'Hugging': 1, 'Face': 1, 'Course': 1, '.': 4, 'chapter': 1, 'about': 1,
    'tokenization': 1, 'section': 1, 'shows': 1, 'several': 1, 'tokenizer': 1, 'algorithms': 1, 'Hopefully': 1,
    ',': 1, 'you': 1, 'will': 1, 'be': 1, 'able': 1, 'to': 1, 'understand': 1, 'how': 1, 'they': 1, 'are': 1,
    'trained': 1, 'and': 1, 'generate': 1, 'tokens': 1})`,highlighted:`defaultdict(
    <span class="hljs-built_in">int</span>, {<span class="hljs-string">&#x27;This&#x27;</span>: <span class="hljs-number">3</span>, <span class="hljs-string">&#x27;is&#x27;</span>: <span class="hljs-number">2</span>, <span class="hljs-string">&#x27;the&#x27;</span>: <span class="hljs-number">1</span>, <span class="hljs-string">&#x27;Hugging&#x27;</span>: <span class="hljs-number">1</span>, <span class="hljs-string">&#x27;Face&#x27;</span>: <span class="hljs-number">1</span>, <span class="hljs-string">&#x27;Course&#x27;</span>: <span class="hljs-number">1</span>, <span class="hljs-string">&#x27;.&#x27;</span>: <span class="hljs-number">4</span>, <span class="hljs-string">&#x27;chapter&#x27;</span>: <span class="hljs-number">1</span>, <span class="hljs-string">&#x27;about&#x27;</span>: <span class="hljs-number">1</span>,
    <span class="hljs-string">&#x27;tokenization&#x27;</span>: <span class="hljs-number">1</span>, <span class="hljs-string">&#x27;section&#x27;</span>: <span class="hljs-number">1</span>, <span class="hljs-string">&#x27;shows&#x27;</span>: <span class="hljs-number">1</span>, <span class="hljs-string">&#x27;several&#x27;</span>: <span class="hljs-number">1</span>, <span class="hljs-string">&#x27;tokenizer&#x27;</span>: <span class="hljs-number">1</span>, <span class="hljs-string">&#x27;algorithms&#x27;</span>: <span class="hljs-number">1</span>, <span class="hljs-string">&#x27;Hopefully&#x27;</span>: <span class="hljs-number">1</span>,
    <span class="hljs-string">&#x27;,&#x27;</span>: <span class="hljs-number">1</span>, <span class="hljs-string">&#x27;you&#x27;</span>: <span class="hljs-number">1</span>, <span class="hljs-string">&#x27;will&#x27;</span>: <span class="hljs-number">1</span>, <span class="hljs-string">&#x27;be&#x27;</span>: <span class="hljs-number">1</span>, <span class="hljs-string">&#x27;able&#x27;</span>: <span class="hljs-number">1</span>, <span class="hljs-string">&#x27;to&#x27;</span>: <span class="hljs-number">1</span>, <span class="hljs-string">&#x27;understand&#x27;</span>: <span class="hljs-number">1</span>, <span class="hljs-string">&#x27;how&#x27;</span>: <span class="hljs-number">1</span>, <span class="hljs-string">&#x27;they&#x27;</span>: <span class="hljs-number">1</span>, <span class="hljs-string">&#x27;are&#x27;</span>: <span class="hljs-number">1</span>,
    <span class="hljs-string">&#x27;trained&#x27;</span>: <span class="hljs-number">1</span>, <span class="hljs-string">&#x27;and&#x27;</span>: <span class="hljs-number">1</span>, <span class="hljs-string">&#x27;generate&#x27;</span>: <span class="hljs-number">1</span>, <span class="hljs-string">&#x27;tokens&#x27;</span>: <span class="hljs-number">1</span>})`}}),ws=new _({props:{code:`alphabet = []
for word in word_freqs.keys():
    if word[0] not in alphabet:
        alphabet.append(word[0])
    for letter in word[1:]:
        if f"##{letter}" not in alphabet:
            alphabet.append(f"##{letter}")

alphabet.sort()
alphabet

print(alphabet)`,highlighted:`alphabet = []
<span class="hljs-keyword">for</span> word <span class="hljs-keyword">in</span> word_freqs.keys():
    <span class="hljs-keyword">if</span> word[<span class="hljs-number">0</span>] <span class="hljs-keyword">not</span> <span class="hljs-keyword">in</span> alphabet:
        alphabet.append(word[<span class="hljs-number">0</span>])
    <span class="hljs-keyword">for</span> letter <span class="hljs-keyword">in</span> word[<span class="hljs-number">1</span>:]:
        <span class="hljs-keyword">if</span> <span class="hljs-string">f&quot;##<span class="hljs-subst">{letter}</span>&quot;</span> <span class="hljs-keyword">not</span> <span class="hljs-keyword">in</span> alphabet:
            alphabet.append(<span class="hljs-string">f&quot;##<span class="hljs-subst">{letter}</span>&quot;</span>)

alphabet.sort()
alphabet

<span class="hljs-built_in">print</span>(alphabet)`}}),qs=new _({props:{code:`['##a', '##b', '##c', '##d', '##e', '##f', '##g', '##h', '##i', '##k', '##l', '##m', '##n', '##o', '##p', '##r', '##s',
 '##t', '##u', '##v', '##w', '##y', '##z', ',', '.', 'C', 'F', 'H', 'T', 'a', 'b', 'c', 'g', 'h', 'i', 's', 't', 'u',
 'w', 'y']`,highlighted:`[<span class="hljs-string">&#x27;##a&#x27;</span>, <span class="hljs-string">&#x27;##b&#x27;</span>, <span class="hljs-string">&#x27;##c&#x27;</span>, <span class="hljs-string">&#x27;##d&#x27;</span>, <span class="hljs-string">&#x27;##e&#x27;</span>, <span class="hljs-string">&#x27;##f&#x27;</span>, <span class="hljs-string">&#x27;##g&#x27;</span>, <span class="hljs-string">&#x27;##h&#x27;</span>, <span class="hljs-string">&#x27;##i&#x27;</span>, <span class="hljs-string">&#x27;##k&#x27;</span>, <span class="hljs-string">&#x27;##l&#x27;</span>, <span class="hljs-string">&#x27;##m&#x27;</span>, <span class="hljs-string">&#x27;##n&#x27;</span>, <span class="hljs-string">&#x27;##o&#x27;</span>, <span class="hljs-string">&#x27;##p&#x27;</span>, <span class="hljs-string">&#x27;##r&#x27;</span>, <span class="hljs-string">&#x27;##s&#x27;</span>,
 <span class="hljs-string">&#x27;##t&#x27;</span>, <span class="hljs-string">&#x27;##u&#x27;</span>, <span class="hljs-string">&#x27;##v&#x27;</span>, <span class="hljs-string">&#x27;##w&#x27;</span>, <span class="hljs-string">&#x27;##y&#x27;</span>, <span class="hljs-string">&#x27;##z&#x27;</span>, <span class="hljs-string">&#x27;,&#x27;</span>, <span class="hljs-string">&#x27;.&#x27;</span>, <span class="hljs-string">&#x27;C&#x27;</span>, <span class="hljs-string">&#x27;F&#x27;</span>, <span class="hljs-string">&#x27;H&#x27;</span>, <span class="hljs-string">&#x27;T&#x27;</span>, <span class="hljs-string">&#x27;a&#x27;</span>, <span class="hljs-string">&#x27;b&#x27;</span>, <span class="hljs-string">&#x27;c&#x27;</span>, <span class="hljs-string">&#x27;g&#x27;</span>, <span class="hljs-string">&#x27;h&#x27;</span>, <span class="hljs-string">&#x27;i&#x27;</span>, <span class="hljs-string">&#x27;s&#x27;</span>, <span class="hljs-string">&#x27;t&#x27;</span>, <span class="hljs-string">&#x27;u&#x27;</span>,
 <span class="hljs-string">&#x27;w&#x27;</span>, <span class="hljs-string">&#x27;y&#x27;</span>]`}}),vs=new _({props:{code:'vocab = ["[PAD]", "[UNK]", "[CLS]", "[SEP]", "[MASK]"] + alphabet.copy()',highlighted:'vocab = [<span class="hljs-string">&quot;[PAD]&quot;</span>, <span class="hljs-string">&quot;[UNK]&quot;</span>, <span class="hljs-string">&quot;[CLS]&quot;</span>, <span class="hljs-string">&quot;[SEP]&quot;</span>, <span class="hljs-string">&quot;[MASK]&quot;</span>] + alphabet.copy()'}}),ks=new _({props:{code:`splits = {
    word: [c if i == 0 else f"##{c}" for i, c in enumerate(word)]
    for word in word_freqs.keys()
}`,highlighted:`splits = {
    word: [c <span class="hljs-keyword">if</span> i == <span class="hljs-number">0</span> <span class="hljs-keyword">else</span> <span class="hljs-string">f&quot;##<span class="hljs-subst">{c}</span>&quot;</span> <span class="hljs-keyword">for</span> i, c <span class="hljs-keyword">in</span> <span class="hljs-built_in">enumerate</span>(word)]
    <span class="hljs-keyword">for</span> word <span class="hljs-keyword">in</span> word_freqs.keys()
}`}}),Es=new _({props:{code:`def compute_pair_scores(splits):
    letter_freqs = defaultdict(int)
    pair_freqs = defaultdict(int)
    for word, freq in word_freqs.items():
        split = splits[word]
        if len(split) == 1:
            letter_freqs[split[0]] += freq
            continue
        for i in range(len(split) - 1):
            pair = (split[i], split[i + 1])
            letter_freqs[split[i]] += freq
            pair_freqs[pair] += freq
        letter_freqs[split[-1]] += freq

    scores = {
        pair: freq / (letter_freqs[pair[0]] * letter_freqs[pair[1]])
        for pair, freq in pair_freqs.items()
    }
    return scores`,highlighted:`<span class="hljs-keyword">def</span> <span class="hljs-title function_">compute_pair_scores</span>(<span class="hljs-params">splits</span>):
    letter_freqs = defaultdict(<span class="hljs-built_in">int</span>)
    pair_freqs = defaultdict(<span class="hljs-built_in">int</span>)
    <span class="hljs-keyword">for</span> word, freq <span class="hljs-keyword">in</span> word_freqs.items():
        split = splits[word]
        <span class="hljs-keyword">if</span> <span class="hljs-built_in">len</span>(split) == <span class="hljs-number">1</span>:
            letter_freqs[split[<span class="hljs-number">0</span>]] += freq
            <span class="hljs-keyword">continue</span>
        <span class="hljs-keyword">for</span> i <span class="hljs-keyword">in</span> <span class="hljs-built_in">range</span>(<span class="hljs-built_in">len</span>(split) - <span class="hljs-number">1</span>):
            pair = (split[i], split[i + <span class="hljs-number">1</span>])
            letter_freqs[split[i]] += freq
            pair_freqs[pair] += freq
        letter_freqs[split[-<span class="hljs-number">1</span>]] += freq

    scores = {
        pair: freq / (letter_freqs[pair[<span class="hljs-number">0</span>]] * letter_freqs[pair[<span class="hljs-number">1</span>]])
        <span class="hljs-keyword">for</span> pair, freq <span class="hljs-keyword">in</span> pair_freqs.items()
    }
    <span class="hljs-keyword">return</span> scores`}}),ys=new _({props:{code:`pair_scores = compute_pair_scores(splits)
for i, key in enumerate(pair_scores.keys()):
    print(f"{key}: {pair_scores[key]}")
    if i >= 5:
        break`,highlighted:`pair_scores = compute_pair_scores(splits)
<span class="hljs-keyword">for</span> i, key <span class="hljs-keyword">in</span> <span class="hljs-built_in">enumerate</span>(pair_scores.keys()):
    <span class="hljs-built_in">print</span>(<span class="hljs-string">f&quot;<span class="hljs-subst">{key}</span>: <span class="hljs-subst">{pair_scores[key]}</span>&quot;</span>)
    <span class="hljs-keyword">if</span> i &gt;= <span class="hljs-number">5</span>:
        <span class="hljs-keyword">break</span>`}}),Ps=new _({props:{code:`('T', '##h'): 0.125
('##h', '##i'): 0.03409090909090909
('##i', '##s'): 0.02727272727272727
('i', '##s'): 0.1
('t', '##h'): 0.03571428571428571
('##h', '##e'): 0.011904761904761904`,highlighted:`(<span class="hljs-string">&#x27;T&#x27;</span>, <span class="hljs-string">&#x27;##h&#x27;</span>): <span class="hljs-number">0.125</span>
(<span class="hljs-string">&#x27;##h&#x27;</span>, <span class="hljs-string">&#x27;##i&#x27;</span>): <span class="hljs-number">0.03409090909090909</span>
(<span class="hljs-string">&#x27;##i&#x27;</span>, <span class="hljs-string">&#x27;##s&#x27;</span>): <span class="hljs-number">0.02727272727272727</span>
(<span class="hljs-string">&#x27;i&#x27;</span>, <span class="hljs-string">&#x27;##s&#x27;</span>): <span class="hljs-number">0.1</span>
(<span class="hljs-string">&#x27;t&#x27;</span>, <span class="hljs-string">&#x27;##h&#x27;</span>): <span class="hljs-number">0.03571428571428571</span>
(<span class="hljs-string">&#x27;##h&#x27;</span>, <span class="hljs-string">&#x27;##e&#x27;</span>): <span class="hljs-number">0.011904761904761904</span>`}}),Cs=new _({props:{code:`best_pair = ""
max_score = None
for pair, score in pair_scores.items():
    if max_score is None or max_score < score:
        best_pair = pair
        max_score = score

print(best_pair, max_score)`,highlighted:`best_pair = <span class="hljs-string">&quot;&quot;</span>
max_score = <span class="hljs-literal">None</span>
<span class="hljs-keyword">for</span> pair, score <span class="hljs-keyword">in</span> pair_scores.items():
    <span class="hljs-keyword">if</span> max_score <span class="hljs-keyword">is</span> <span class="hljs-literal">None</span> <span class="hljs-keyword">or</span> max_score &lt; score:
        best_pair = pair
        max_score = score

<span class="hljs-built_in">print</span>(best_pair, max_score)`}}),Ds=new _({props:{code:"('a', '##b') 0.2",highlighted:'(<span class="hljs-string">&#x27;a&#x27;</span>, <span class="hljs-string">&#x27;##b&#x27;</span>) <span class="hljs-number">0.2</span>'}}),zs=new _({props:{code:'vocab.append("ab")',highlighted:'vocab.append(<span class="hljs-string">&quot;ab&quot;</span>)'}}),Os=new _({props:{code:`def merge_pair(a, b, splits):
    for word in word_freqs:
        split = splits[word]
        if len(split) == 1:
            continue
        i = 0
        while i < len(split) - 1:
            if split[i] == a and split[i + 1] == b:
                merge = a + b[2:] if b.startswith("##") else a + b
                split = split[:i] + [merge] + split[i + 2 :]
            else:
                i += 1
        splits[word] = split
    return splits`,highlighted:`<span class="hljs-keyword">def</span> <span class="hljs-title function_">merge_pair</span>(<span class="hljs-params">a, b, splits</span>):
    <span class="hljs-keyword">for</span> word <span class="hljs-keyword">in</span> word_freqs:
        split = splits[word]
        <span class="hljs-keyword">if</span> <span class="hljs-built_in">len</span>(split) == <span class="hljs-number">1</span>:
            <span class="hljs-keyword">continue</span>
        i = <span class="hljs-number">0</span>
        <span class="hljs-keyword">while</span> i &lt; <span class="hljs-built_in">len</span>(split) - <span class="hljs-number">1</span>:
            <span class="hljs-keyword">if</span> split[i] == a <span class="hljs-keyword">and</span> split[i + <span class="hljs-number">1</span>] == b:
                merge = a + b[<span class="hljs-number">2</span>:] <span class="hljs-keyword">if</span> b.startswith(<span class="hljs-string">&quot;##&quot;</span>) <span class="hljs-keyword">else</span> a + b
                split = split[:i] + [merge] + split[i + <span class="hljs-number">2</span> :]
            <span class="hljs-keyword">else</span>:
                i += <span class="hljs-number">1</span>
        splits[word] = split
    <span class="hljs-keyword">return</span> splits`}}),Ts=new _({props:{code:`splits = merge_pair("a", "##b", splits)
splits["about"]`,highlighted:`splits = merge_pair(<span class="hljs-string">&quot;a&quot;</span>, <span class="hljs-string">&quot;##b&quot;</span>, splits)
splits[<span class="hljs-string">&quot;about&quot;</span>]`}}),Bs=new _({props:{code:"['ab', '##o', '##u', '##t']",highlighted:'[<span class="hljs-string">&#x27;ab&#x27;</span>, <span class="hljs-string">&#x27;##o&#x27;</span>, <span class="hljs-string">&#x27;##u&#x27;</span>, <span class="hljs-string">&#x27;##t&#x27;</span>]'}}),Ns=new _({props:{code:`vocab_size = 70
while len(vocab) < vocab_size:
    scores = compute_pair_scores(splits)
    best_pair, max_score = "", None
    for pair, score in scores.items():
        if max_score is None or max_score < score:
            best_pair = pair
            max_score = score
    splits = merge_pair(*best_pair, splits)
    new_token = (
        best_pair[0] + best_pair[1][2:]
        if best_pair[1].startswith("##")
        else best_pair[0] + best_pair[1]
    )
    vocab.append(new_token)`,highlighted:`vocab_size = <span class="hljs-number">70</span>
<span class="hljs-keyword">while</span> <span class="hljs-built_in">len</span>(vocab) &lt; vocab_size:
    scores = compute_pair_scores(splits)
    best_pair, max_score = <span class="hljs-string">&quot;&quot;</span>, <span class="hljs-literal">None</span>
    <span class="hljs-keyword">for</span> pair, score <span class="hljs-keyword">in</span> scores.items():
        <span class="hljs-keyword">if</span> max_score <span class="hljs-keyword">is</span> <span class="hljs-literal">None</span> <span class="hljs-keyword">or</span> max_score &lt; score:
            best_pair = pair
            max_score = score
    splits = merge_pair(*best_pair, splits)
    new_token = (
        best_pair[<span class="hljs-number">0</span>] + best_pair[<span class="hljs-number">1</span>][<span class="hljs-number">2</span>:]
        <span class="hljs-keyword">if</span> best_pair[<span class="hljs-number">1</span>].startswith(<span class="hljs-string">&quot;##&quot;</span>)
        <span class="hljs-keyword">else</span> best_pair[<span class="hljs-number">0</span>] + best_pair[<span class="hljs-number">1</span>]
    )
    vocab.append(new_token)`}}),Hs=new _({props:{code:"print(vocab)",highlighted:'<span class="hljs-built_in">print</span>(vocab)'}}),Ws=new _({props:{code:`['[PAD]', '[UNK]', '[CLS]', '[SEP]', '[MASK]', '##a', '##b', '##c', '##d', '##e', '##f', '##g', '##h', '##i', '##k',
 '##l', '##m', '##n', '##o', '##p', '##r', '##s', '##t', '##u', '##v', '##w', '##y', '##z', ',', '.', 'C', 'F', 'H',
 'T', 'a', 'b', 'c', 'g', 'h', 'i', 's', 't', 'u', 'w', 'y', '##fu', 'Fa', 'Fac', '##ct', '##ful', '##full', '##fully',
 'Th', 'ch', '##hm', 'cha', 'chap', 'chapt', '##thm', 'Hu', 'Hug', 'Hugg', 'sh', 'th', 'is', '##thms', '##za', '##zat',
 '##ut']`,highlighted:`[<span class="hljs-string">&#x27;[PAD]&#x27;</span>, <span class="hljs-string">&#x27;[UNK]&#x27;</span>, <span class="hljs-string">&#x27;[CLS]&#x27;</span>, <span class="hljs-string">&#x27;[SEP]&#x27;</span>, <span class="hljs-string">&#x27;[MASK]&#x27;</span>, <span class="hljs-string">&#x27;##a&#x27;</span>, <span class="hljs-string">&#x27;##b&#x27;</span>, <span class="hljs-string">&#x27;##c&#x27;</span>, <span class="hljs-string">&#x27;##d&#x27;</span>, <span class="hljs-string">&#x27;##e&#x27;</span>, <span class="hljs-string">&#x27;##f&#x27;</span>, <span class="hljs-string">&#x27;##g&#x27;</span>, <span class="hljs-string">&#x27;##h&#x27;</span>, <span class="hljs-string">&#x27;##i&#x27;</span>, <span class="hljs-string">&#x27;##k&#x27;</span>,
 <span class="hljs-string">&#x27;##l&#x27;</span>, <span class="hljs-string">&#x27;##m&#x27;</span>, <span class="hljs-string">&#x27;##n&#x27;</span>, <span class="hljs-string">&#x27;##o&#x27;</span>, <span class="hljs-string">&#x27;##p&#x27;</span>, <span class="hljs-string">&#x27;##r&#x27;</span>, <span class="hljs-string">&#x27;##s&#x27;</span>, <span class="hljs-string">&#x27;##t&#x27;</span>, <span class="hljs-string">&#x27;##u&#x27;</span>, <span class="hljs-string">&#x27;##v&#x27;</span>, <span class="hljs-string">&#x27;##w&#x27;</span>, <span class="hljs-string">&#x27;##y&#x27;</span>, <span class="hljs-string">&#x27;##z&#x27;</span>, <span class="hljs-string">&#x27;,&#x27;</span>, <span class="hljs-string">&#x27;.&#x27;</span>, <span class="hljs-string">&#x27;C&#x27;</span>, <span class="hljs-string">&#x27;F&#x27;</span>, <span class="hljs-string">&#x27;H&#x27;</span>,
 <span class="hljs-string">&#x27;T&#x27;</span>, <span class="hljs-string">&#x27;a&#x27;</span>, <span class="hljs-string">&#x27;b&#x27;</span>, <span class="hljs-string">&#x27;c&#x27;</span>, <span class="hljs-string">&#x27;g&#x27;</span>, <span class="hljs-string">&#x27;h&#x27;</span>, <span class="hljs-string">&#x27;i&#x27;</span>, <span class="hljs-string">&#x27;s&#x27;</span>, <span class="hljs-string">&#x27;t&#x27;</span>, <span class="hljs-string">&#x27;u&#x27;</span>, <span class="hljs-string">&#x27;w&#x27;</span>, <span class="hljs-string">&#x27;y&#x27;</span>, <span class="hljs-string">&#x27;##fu&#x27;</span>, <span class="hljs-string">&#x27;Fa&#x27;</span>, <span class="hljs-string">&#x27;Fac&#x27;</span>, <span class="hljs-string">&#x27;##ct&#x27;</span>, <span class="hljs-string">&#x27;##ful&#x27;</span>, <span class="hljs-string">&#x27;##full&#x27;</span>, <span class="hljs-string">&#x27;##fully&#x27;</span>,
 <span class="hljs-string">&#x27;Th&#x27;</span>, <span class="hljs-string">&#x27;ch&#x27;</span>, <span class="hljs-string">&#x27;##hm&#x27;</span>, <span class="hljs-string">&#x27;cha&#x27;</span>, <span class="hljs-string">&#x27;chap&#x27;</span>, <span class="hljs-string">&#x27;chapt&#x27;</span>, <span class="hljs-string">&#x27;##thm&#x27;</span>, <span class="hljs-string">&#x27;Hu&#x27;</span>, <span class="hljs-string">&#x27;Hug&#x27;</span>, <span class="hljs-string">&#x27;Hugg&#x27;</span>, <span class="hljs-string">&#x27;sh&#x27;</span>, <span class="hljs-string">&#x27;th&#x27;</span>, <span class="hljs-string">&#x27;is&#x27;</span>, <span class="hljs-string">&#x27;##thms&#x27;</span>, <span class="hljs-string">&#x27;##za&#x27;</span>, <span class="hljs-string">&#x27;##zat&#x27;</span>,
 <span class="hljs-string">&#x27;##ut&#x27;</span>]`}}),ps=new _n({props:{$$slots:{default:[di]},$$scope:{ctx:S}}}),Ss=new _({props:{code:`def encode_word(word):
    tokens = []
    while len(word) > 0:
        i = len(word)
        while i > 0 and word[:i] not in vocab:
            i -= 1
        if i == 0:
            return ["[UNK]"]
        tokens.append(word[:i])
        word = word[i:]
        if len(word) > 0:
            word = f"##{word}"
    return tokens`,highlighted:`<span class="hljs-keyword">def</span> <span class="hljs-title function_">encode_word</span>(<span class="hljs-params">word</span>):
    tokens = []
    <span class="hljs-keyword">while</span> <span class="hljs-built_in">len</span>(word) &gt; <span class="hljs-number">0</span>:
        i = <span class="hljs-built_in">len</span>(word)
        <span class="hljs-keyword">while</span> i &gt; <span class="hljs-number">0</span> <span class="hljs-keyword">and</span> word[:i] <span class="hljs-keyword">not</span> <span class="hljs-keyword">in</span> vocab:
            i -= <span class="hljs-number">1</span>
        <span class="hljs-keyword">if</span> i == <span class="hljs-number">0</span>:
            <span class="hljs-keyword">return</span> [<span class="hljs-string">&quot;[UNK]&quot;</span>]
        tokens.append(word[:i])
        word = word[i:]
        <span class="hljs-keyword">if</span> <span class="hljs-built_in">len</span>(word) &gt; <span class="hljs-number">0</span>:
            word = <span class="hljs-string">f&quot;##<span class="hljs-subst">{word}</span>&quot;</span>
    <span class="hljs-keyword">return</span> tokens`}}),As=new _({props:{code:`print(encode_word("Hugging"))
print(encode_word("HOgging"))`,highlighted:`<span class="hljs-built_in">print</span>(encode_word(<span class="hljs-string">&quot;Hugging&quot;</span>))
<span class="hljs-built_in">print</span>(encode_word(<span class="hljs-string">&quot;HOgging&quot;</span>))`}}),Ks=new _({props:{code:`['Hugg', '##i', '##n', '##g']
['[UNK]']`,highlighted:`[<span class="hljs-string">&#x27;Hugg&#x27;</span>, <span class="hljs-string">&#x27;##i&#x27;</span>, <span class="hljs-string">&#x27;##n&#x27;</span>, <span class="hljs-string">&#x27;##g&#x27;</span>]
[<span class="hljs-string">&#x27;[UNK]&#x27;</span>]`}}),Us=new _({props:{code:`def tokenize(text):
    pre_tokenize_result = tokenizer._tokenizer.pre_tokenizer.pre_tokenize_str(text)
    pre_tokenized_text = [word for word, offset in pre_tokenize_result]
    encoded_words = [encode_word(word) for word in pre_tokenized_text]
    return sum(encoded_words, [])`,highlighted:`<span class="hljs-keyword">def</span> <span class="hljs-title function_">tokenize</span>(<span class="hljs-params">text</span>):
    pre_tokenize_result = tokenizer._tokenizer.pre_tokenizer.pre_tokenize_str(text)
    pre_tokenized_text = [word <span class="hljs-keyword">for</span> word, offset <span class="hljs-keyword">in</span> pre_tokenize_result]
    encoded_words = [encode_word(word) <span class="hljs-keyword">for</span> word <span class="hljs-keyword">in</span> pre_tokenized_text]
    <span class="hljs-keyword">return</span> <span class="hljs-built_in">sum</span>(encoded_words, [])`}}),Fs=new _({props:{code:'tokenize("This is the Hugging Face course!")',highlighted:'tokenize(<span class="hljs-string">&quot;This is the Hugging Face course!&quot;</span>)'}}),Rs=new _({props:{code:`['Th', '##i', '##s', 'is', 'th', '##e', 'Hugg', '##i', '##n', '##g', 'Fac', '##e', 'c', '##o', '##u', '##r', '##s',
 '##e', '[UNK]']`,highlighted:`[<span class="hljs-string">&#x27;Th&#x27;</span>, <span class="hljs-string">&#x27;##i&#x27;</span>, <span class="hljs-string">&#x27;##s&#x27;</span>, <span class="hljs-string">&#x27;is&#x27;</span>, <span class="hljs-string">&#x27;th&#x27;</span>, <span class="hljs-string">&#x27;##e&#x27;</span>, <span class="hljs-string">&#x27;Hugg&#x27;</span>, <span class="hljs-string">&#x27;##i&#x27;</span>, <span class="hljs-string">&#x27;##n&#x27;</span>, <span class="hljs-string">&#x27;##g&#x27;</span>, <span class="hljs-string">&#x27;Fac&#x27;</span>, <span class="hljs-string">&#x27;##e&#x27;</span>, <span class="hljs-string">&#x27;c&#x27;</span>, <span class="hljs-string">&#x27;##o&#x27;</span>, <span class="hljs-string">&#x27;##u&#x27;</span>, <span class="hljs-string">&#x27;##r&#x27;</span>, <span class="hljs-string">&#x27;##s&#x27;</span>,
 <span class="hljs-string">&#x27;##e&#x27;</span>, <span class="hljs-string">&#x27;[UNK]&#x27;</span>]`}}),{c(){u=r("meta"),q=c(),b=r("h1"),$=r("a"),O=r("span"),m(w.$$.fragment),E=c(),B=r("span"),G=t("WordPiece \u6807\u8BB0\u5316"),W=c(),m(K.$$.fragment),es=c(),Ls=r("p"),Zl=t("WordPiece \u662F Google \u4E3A\u9884\u8BAD\u7EC3 BERT \u800C\u5F00\u53D1\u7684\u6807\u8BB0\u5316\u7B97\u6CD5\u3002\u6B64\u540E,\u5B83\u5728\u4E0D\u5C11\u57FA\u4E8E BERT \u7684 Transformer \u6A21\u578B\u4E2D\u5F97\u5230\u91CD\u7528,\u4F8B\u5982 DistilBERT\u3001MobileBERT\u3001Funnel Transformers \u548C MPNET\u3002\u5B83\u5728\u8BAD\u7EC3\u65B9\u9762\u4E0E BPE \u975E\u5E38\u76F8\u4F3C,\u4F46\u5B9E\u9645\u6807\u8BB0\u5316\u7684\u65B9\u5F0F\u4E0D\u540C\u3002"),$n=c(),m(rs.$$.fragment),wn=c(),m(I.$$.fragment),qn=c(),is=r("h2"),st=t("\u8BAD\u7EC3\u7B97\u6CD5"),vn=c(),m(Y.$$.fragment),kn=c(),U=r("p"),at=t("\u4E0E BPE \u4E00\u6837,WordPiece \u4ECE\u4E00\u4E2A\u5C0F\u8BCD\u6C47\u8868\u5F00\u59CB,\u5305\u62EC\u6A21\u578B\u4F7F\u7528\u7684\u7279\u6B8A\u6807\u8BB0\u548C\u521D\u59CB\u5B57\u6BCD\u8868\u3002\u56E0\u4E3A\u5B83\u901A\u8FC7\u6DFB\u52A0\u524D\u7F00\u6765\u8BC6\u522B\u5B50\u8BCD (\u5982\u540C "),ga=r("code"),nt=t("##"),lt=t(" \u5BF9\u4E8E BERT),\u6BCF\u4E2A\u5355\u8BCD\u6700\u521D\u662F\u901A\u8FC7\u5C06\u8BE5\u524D\u7F00\u6DFB\u52A0\u5230\u5355\u8BCD\u5185\u7684\u6240\u6709\u5B57\u7B26\u6765\u62C6\u5206\u7684\u3002\u6240\u4EE5,\u4F8B\u5982 "),da=r("code"),tt=t('"word"'),pt=t(" ,\u50CF\u8FD9\u6837\u62C6\u5206:"),En=c(),m(os.$$.fragment),yn=c(),Gs=r("p"),et=t("\u56E0\u6B64,\u521D\u59CB\u5B57\u6BCD\u8868\u5305\u542B\u51FA\u73B0\u5728\u5355\u8BCD\u5F00\u5934\u7684\u6240\u6709\u5B57\u7B26\u4EE5\u53CA\u51FA\u73B0\u5728\u5355\u8BCD\u5185\u90E8\u7684\u4EE5 WordPiece \u524D\u7F00\u5F00\u5934\u7684\u5B57\u7B26\u3002"),Pn=c(),cs=r("p"),rt=t(`\u7136\u540E,\u518D\u6B21\u50CF BPE \u4E00\u6837,WordPiece \u5B66\u4E60\u5408\u5E76\u89C4\u5219\u3002\u4E3B\u8981\u533A\u522B\u5728\u4E8E\u9009\u62E9\u8981\u5408\u5E76\u7684\u5BF9\u7684\u65B9\u5F0F\u3002WordPiece \u4E0D\u662F\u9009\u62E9\u6700\u9891\u7E41\u7684\u5BF9\uFF0C\u800C\u662F\u4F7F\u7528\u4EE5\u4E0B\u516C\u5F0F\u8BA1\u7B97\u6BCF\u5BF9\u7684\u5206\u6570:
`),Cn=new ri,Dn=c(),P=r("p"),it=t("\u901A\u8FC7\u5C06\u914D\u5BF9\u7684\u9891\u7387\u9664\u4EE5\u5176\u6BCF\u4E2A\u90E8\u5206\u7684\u9891\u7387\u7684\u4E58\u79EF, \u8BE5\u7B97\u6CD5\u4F18\u5148\u5408\u5E76\u5355\u4E2A\u90E8\u5206\u5728\u8BCD\u6C47\u8868\u4E2D\u9891\u7387\u8F83\u4F4E\u7684\u5BF9\u3002\u4F8B\u5982,\u5B83\u4E0D\u4E00\u5B9A\u4F1A\u5408\u5E76 "),xa=r("code"),ot=t('("un", "##able")'),ct=t(" \u5373\u4F7F\u8FD9\u5BF9\u5728\u8BCD\u6C47\u8868\u4E2D\u51FA\u73B0\u7684\u9891\u7387\u5F88\u9AD8,\u56E0\u4E3A "),ba=r("code"),ht=t('"un"'),ut=t(" \u548C "),_a=r("code"),mt=t('"##able"'),ft=t(" \u5F88\u53EF\u80FD\u6BCF\u4E2A\u8BCD\u90FD\u51FA\u73B0\u5728\u5F88\u591A\u5176\u4ED6\u8BCD\u4E2D\u5E76\u4E14\u51FA\u73B0\u9891\u7387\u5F88\u9AD8\u3002\u76F8\u6BD4\u4E4B\u4E0B,\u50CF "),$a=r("code"),jt=t('("hu", "##gging")'),gt=t(" \u53EF\u80FD\u4F1A\u66F4\u5FEB\u5730\u5408\u5E76 (\u5047\u8BBE \u201Chugging\u201D \u7ECF\u5E38\u51FA\u73B0\u5728\u8BCD\u6C47\u8868\u4E2D),\u56E0\u4E3A "),wa=r("code"),dt=t('"hu"'),xt=t(" \u548C "),qa=r("code"),bt=t('"##gging"'),_t=t(" \u8FD9\u4E24\u4E2A\u8BCD\u5355\u72EC\u51FA\u73B0\u5730\u9891\u7387\u53EF\u80FD\u8F83\u4F4E\u3002"),zn=c(),Vs=r("p"),$t=t("\u8BA9\u6211\u4EEC\u770B\u770B\u6211\u4EEC\u5728 BPE \u8BAD\u7EC3\u793A\u4F8B\u4E2D\u4F7F\u7528\u7684\u76F8\u540C\u8BCD\u6C47:"),On=c(),m(hs.$$.fragment),Tn=c(),Is=r("p"),wt=t("\u8FD9\u91CC\u7684\u62C6\u5206\u5C06\u662F:"),Bn=c(),m(us.$$.fragment),Nn=c(),y=r("p"),qt=t("\u6240\u4EE5\u6700\u521D\u7684\u8BCD\u6C47\u5C06\u662F "),va=r("code"),vt=t('["b", "h", "p", "##g", "##n", "##s", "##u"]'),kt=t(" (\u5982\u679C\u6211\u4EEC\u6682\u65F6\u5FD8\u8BB0\u7279\u6B8A\u6807\u8BB0)\u3002\u6700\u9891\u7E41\u7684\u4E00\u5BF9\u662F "),ka=r("code"),Et=t('("##u", "##g")'),yt=t(" (\u76EE\u524D20\u6B21),\u4F46 "),Ea=r("code"),Pt=t('"##u"'),Ct=t(" \u5355\u72EC\u51FA\u73B0\u7684\u9891\u7387\u975E\u5E38\u9AD8,\u6240\u4EE5\u5B83\u7684\u5206\u6570\u4E0D\u662F\u6700\u9AD8\u7684(\u5B83\u662F 1 / 36)\u3002\u6240\u6709\u5E26\u6709 "),ya=r("code"),Dt=t('"##u"'),zt=t(" \u7684\u5BF9\u5B9E\u9645\u4E0A\u90FD\u6709\u76F8\u540C\u7684\u5206\u6570(1 / 36),\u6240\u4EE5\u5206\u6570\u6700\u9AD8\u7684\u5BF9\u662F "),Pa=r("code"),Ot=t('("##g", "##s")'),Tt=t(" \u2014 \u552F\u4E00\u6CA1\u6709 "),Ca=r("code"),Bt=t('"##u"'),Nt=t(" \u7684\u5BF9\u2014 1 / 20,\u6240\u4EE5\u5B66\u4E60\u7684\u7B2C\u4E00\u4E2A\u5408\u5E76\u662F "),Da=r("code"),Ht=t('("##g", "##s") -> ("##gs")'),Wt=t("\u3002"),Hn=c(),F=r("p"),St=t("\u8BF7\u6CE8\u610F,\u5F53\u6211\u4EEC\u5408\u5E76\u65F6,\u6211\u4EEC\u5220\u9664\u4E86\u4E24\u4E2A\u6807\u8BB0\u4E4B\u95F4\u7684 "),za=r("code"),At=t("##"),Kt=t(",\u6240\u4EE5\u6211\u4EEC\u6DFB\u52A0 "),Oa=r("code"),Ut=t('"##gs"'),Ft=t(" \u5230\u8BCD\u6C47\u8868\u4E2D,\u5E76\u5728\u8BED\u6599\u5E93\u7684\u5355\u8BCD\u4E2D\u5E94\u7528\u8BE5\u5408\u5E76:"),Wn=c(),m(ms.$$.fragment),Sn=c(),R=r("p"),Rt=t("\u5728\u8FD9\u4E00\u70B9\u4E2D, "),Ta=r("code"),Mt=t('"##u"'),Lt=t(" \u662F\u5728\u6240\u6709\u53EF\u80FD\u7684\u5BF9\u4E2D,\u56E0\u6B64\u5B83\u4EEC\u6700\u7EC8\u90FD\u5177\u6709\u76F8\u540C\u7684\u5206\u6570\u3002\u5047\u8BBE\u5728\u8FD9\u79CD\u60C5\u51B5\u4E0B,\u7B2C\u4E00\u5BF9\u88AB\u5408\u5E76, "),Ba=r("code"),Gt=t('("h", "##u") -> "hu"'),Vt=t("\u3002\u8FD9\u4F7F\u5F97\u6211\u4EEC:"),An=c(),m(fs.$$.fragment),Kn=c(),M=r("p"),It=t("\u7136\u540E\u4E0B\u4E00\u4E2A\u6700\u9AD8\u7684\u5206\u6570\u7531 "),Na=r("code"),Yt=t('("hu", "##g")'),Jt=t(" \u548C "),Ha=r("code"),Qt=t('("hu", "##gs")'),Xt=t(" \u5171\u4EAB(1/15,\u4E0E\u5176\u4ED6\u6240\u6709\u5BF9\u7684 1/21 \u76F8\u6BD4),\u56E0\u6B64\u5408\u5E76\u5F97\u5206\u6700\u9AD8\u7684\u7B2C\u4E00\u5BF9:"),Un=c(),m(js.$$.fragment),Fn=c(),Ys=r("p"),Zt=t("\u6211\u4EEC\u7EE7\u7EED\u8FD9\u6837\u5904\u7406,\u76F4\u5230\u8FBE\u5230\u6211\u4EEC\u6240\u9700\u7684\u8BCD\u6C47\u91CF\u3002"),Rn=c(),m(J.$$.fragment),Mn=c(),gs=r("h2"),sp=t("\u6807\u8BB0\u5316\u7B97\u6CD5"),Ln=c(),C=r("p"),ap=t("WordPiece \u548C BPE \u4E2D\u7684\u6807\u8BB0\u5316\u7684\u4E0D\u540C\u5728\u4E8E WordPiece \u53EA\u4FDD\u5B58\u6700\u7EC8\u8BCD\u6C47,\u800C\u4E0D\u662F\u5B66\u4E60\u7684\u5408\u5E76\u89C4\u5219\u3002\u4ECE\u8981\u6807\u8BB0\u7684\u5355\u8BCD\u5F00\u59CB,WordPiece \u627E\u5230\u8BCD\u6C47\u8868\u4E2D\u6700\u957F\u7684\u5B50\u8BCD,\u7136\u540E\u5BF9\u5176\u8FDB\u884C\u62C6\u5206\u3002\u4F8B\u5982,\u5982\u679C\u6211\u4EEC\u4F7F\u7528\u4E0A\u9762\u4F8B\u5B50\u4E2D\u5B66\u5230\u7684\u8BCD\u6C47,\u5BF9\u4E8E\u5355\u8BCD "),Wa=r("code"),np=t('"hugs"'),lp=t(",\u8BCD\u6C47\u8868\u4E2D\u4ECE\u5934\u5F00\u59CB\u7684\u6700\u957F\u5B50\u8BCD\u662F "),Sa=r("code"),tp=t('"hug"'),pp=t(",\u6240\u4EE5\u6211\u4EEC\u5728\u90A3\u91CC\u62C6\u5206\u5E76\u5F97\u5230 "),Aa=r("code"),ep=t('["hug", "##s"]'),rp=t("\u3002 \u7136\u540E\u6211\u4EEC\u7EE7\u7EED\u4F7F\u7528\u8BCD\u6C47\u8868\u4E2D\u7684 "),Ka=r("code"),ip=t('"##s"'),op=t(",\u56E0\u6B64 "),Ua=r("code"),cp=t('"hugs"'),hp=t(" \u7684\u6807\u8BB0\u5316\u662F "),Fa=r("code"),up=t('["hug", "##s"]'),mp=t("."),Gn=c(),Q=r("p"),fp=t("\u4F7F\u7528 BPE, \u6211\u4EEC\u5C06\u6309\u987A\u5E8F\u5E94\u7528\u5B66\u4E60\u5230\u7684\u5408\u5E76\u5E76\u5C06\u5176\u6807\u8BB0\u4E3A "),Ra=r("code"),jp=t('["hu", "##gs"]'),gp=t(",\u6240\u4EE5\u7F16\u7801\u4E0D\u540C\u3002"),Vn=c(),v=r("p"),dp=t("\u518D\u4E3E\u4E00\u4E2A\u4F8B\u5B50,\u8BA9\u6211\u4EEC\u770B\u770B "),Ma=r("code"),xp=t('"bugs"'),bp=t(" \u5C06\u5982\u4F55\u88AB\u6807\u8BB0\u5316\u3002 "),La=r("code"),_p=t('"b"'),$p=t(" \u662F\u4ECE\u8BCD\u6C47\u8868\u4E2D\u5355\u8BCD\u5F00\u5934\u5F00\u59CB\u7684\u6700\u957F\u5B50\u8BCD,\u6240\u4EE5\u6211\u4EEC\u5728\u90A3\u91CC\u62C6\u5206\u5E76\u5F97\u5230 "),Ga=r("code"),wp=t('["b", "##ugs"]'),qp=t("\u3002\u7136\u540E "),Va=r("code"),vp=t('"##u"'),kp=t(" \u662F\u8BCD\u6C47\u8868\u4E2D\u4ECE "),Ia=r("code"),Ep=t('"##ugs"'),yp=t(" \u5F00\u59CB\u7684\u6700\u957F\u7684\u5B50\u8BCD,\u6240\u4EE5\u6211\u4EEC\u5728\u90A3\u91CC\u62C6\u5206\u5E76\u5F97\u5230 "),Ya=r("code"),Pp=t('["b", "##u, "##gs"]'),Cp=t("\u3002\u6700\u540E, "),Ja=r("code"),Dp=t('"##gs"'),zp=t(" \u5728\u8BCD\u6C47\u8868\u4E2D,\u6240\u4EE5\u6700\u540E\u4E00\u4E2A\u5217\u8868\u662F "),Qa=r("code"),Op=t('"bugs"'),Tp=t(" \u7684\u6807\u8BB0\u5316\u3002"),In=c(),k=r("p"),Bp=t("\u5F53\u5206\u8BCD\u8FBE\u5230\u65E0\u6CD5\u5728\u8BCD\u6C47\u8868\u4E2D\u627E\u5230\u5B50\u8BCD\u7684\u9636\u6BB5\u65F6, \u6574\u4E2A\u8BCD\u88AB\u6807\u8BB0\u4E3A\u672A\u77E5 \u2014 \u4F8B\u5982, "),Xa=r("code"),Np=t('"mug"'),Hp=t(" \u5C06\u88AB\u6807\u8BB0\u4E3A "),Za=r("code"),Wp=t('["[UNK]"]'),Sp=t(",\u5C31\u50CF "),sn=r("code"),Ap=t('"bum"'),Kp=t(" (\u5373\u4F7F\u6211\u4EEC\u53EF\u4EE5\u4EE5 "),an=r("code"),Up=t('"b"'),Fp=t(" \u548C "),nn=r("code"),Rp=t('"##u"'),Mp=t(" \u5F00\u59CB, "),ln=r("code"),Lp=t('"##m"'),Gp=t(" \u4E0D\u5728\u8BCD\u6C47\u8868\u4E2D,\u7531\u6B64\u4EA7\u751F\u7684\u6807\u8BB0\u5C06\u53EA\u662F "),tn=r("code"),Vp=t('["[UNK]"]'),Ip=t(", \u4E0D\u662F "),pn=r("code"),Yp=t('["b", "##u", "[UNK]"]'),Jp=t(")\u3002\u8FD9\u662F\u4E0E BPE \u7684\u53E6\u4E00\u4E2A\u533A\u522B,BPE \u53EA\u4F1A\u5C06\u4E0D\u5728\u8BCD\u6C47\u8868\u4E2D\u7684\u5355\u4E2A\u5B57\u7B26\u5206\u7C7B\u4E3A\u672A\u77E5\u3002"),Yn=c(),m(X.$$.fragment),Jn=c(),V=r("h2"),Z=r("a"),en=r("span"),m(ds.$$.fragment),Qp=c(),rn=r("span"),Xp=t("\u5B9E\u73B0 WordPiece"),Qn=c(),Js=r("p"),Zp=t("\u73B0\u5728\u8BA9\u6211\u4EEC\u770B\u4E00\u4E0B WordPiece \u7B97\u6CD5\u7684\u5B9E\u73B0\u3002\u4E0E BPE \u4E00\u6837,\u8FD9\u53EA\u662F\u6559\u5B66,\u4F60\u5C06\u65E0\u6CD5\u5728\u5927\u578B\u8BED\u6599\u5E93\u4E2D\u4F7F\u7528\u5B83\u3002"),Xn=c(),Qs=r("p"),se=t("\u6211\u4EEC\u5C06\u4F7F\u7528\u4E0E BPE \u793A\u4F8B\u4E2D\u76F8\u540C\u7684\u8BED\u6599\u5E93:"),Zn=c(),m(xs.$$.fragment),sl=c(),ss=r("p"),ae=t("\u9996\u5148,\u6211\u4EEC\u9700\u8981\u5C06\u8BED\u6599\u5E93\u9884\u5148\u6807\u8BB0\u4E3A\u5355\u8BCD\u3002\u7531\u4E8E\u6211\u4EEC\u6B63\u5728\u590D\u5236 WordPiece \u6807\u8BB0\u5668 (\u5982 BERT),\u56E0\u6B64\u6211\u4EEC\u5C06\u4F7F\u7528 "),on=r("code"),ne=t("bert-base-cased"),le=t(" \u6807\u8BB0\u5668\u7528\u4E8E\u9884\u6807\u8BB0\u5316:"),al=c(),m(bs.$$.fragment),nl=c(),Xs=r("p"),te=t("\u7136\u540E\u6211\u4EEC\u5728\u8FDB\u884C\u9884\u6807\u8BB0\u5316\u65F6\u8BA1\u7B97\u8BED\u6599\u5E93\u4E2D\u6BCF\u4E2A\u5355\u8BCD\u7684\u9891\u7387:"),ll=c(),m(_s.$$.fragment),tl=c(),m($s.$$.fragment),pl=c(),as=r("p"),pe=t("\u6B63\u5982\u6211\u4EEC\u4E4B\u524D\u770B\u5230\u7684,\u5B57\u6BCD\u8868\u662F\u7531\u5355\u8BCD\u7684\u6240\u6709\u7B2C\u4E00\u4E2A\u5B57\u6BCD\u7EC4\u6210\u7684\u552F\u4E00\u96C6\u5408,\u4EE5\u53CA\u51FA\u73B0\u5728\u524D\u7F00\u4E3A "),cn=r("code"),ee=t("##"),re=t(" \u7684\u5176\u4ED6\u5B57\u6BCD:"),el=c(),m(ws.$$.fragment),rl=c(),m(qs.$$.fragment),il=c(),ns=r("p"),ie=t("\u6211\u4EEC\u8FD8\u5728\u8BE5\u8BCD\u6C47\u8868\u7684\u5F00\u5934\u6DFB\u52A0\u4E86\u6A21\u578B\u4F7F\u7528\u7684\u7279\u6B8A\u6807\u8BB0\u3002\u5728\u4F7F\u7528 BERT \u7684\u60C5\u51B5\u4E0B,\u5B83\u662F\u5217\u8868 "),hn=r("code"),oe=t('["[PAD]", "[UNK]", "[CLS]", "[SEP]", "[MASK]"]'),ce=t(":"),ol=c(),m(vs.$$.fragment),cl=c(),ls=r("p"),he=t("\u63A5\u4E0B\u6765\u6211\u4EEC\u9700\u8981\u62C6\u5206\u6BCF\u4E2A\u5355\u8BCD, \u6240\u6709\u4E0D\u662F\u7B2C\u4E00\u4E2A\u5B57\u6BCD\u7684\u5B57\u6BCD\u90FD\u4EE5 "),un=r("code"),ue=t("##"),me=t(" \u4E3A\u524D\u7F00:"),hl=c(),m(ks.$$.fragment),ul=c(),Zs=r("p"),fe=t("\u73B0\u5728\u6211\u4EEC\u5DF2\u7ECF\u51C6\u5907\u597D\u8BAD\u7EC3\u4E86,\u8BA9\u6211\u4EEC\u7F16\u5199\u4E00\u4E2A\u51FD\u6570\u6765\u8BA1\u7B97\u6BCF\u5BF9\u7684\u5206\u6570\u3002\u6211\u4EEC\u9700\u8981\u5728\u8BAD\u7EC3\u7684\u6BCF\u4E2A\u6B65\u9AA4\u4E2D\u4F7F\u7528\u5B83:"),ml=c(),m(Es.$$.fragment),fl=c(),sa=r("p"),je=t("\u8BA9\u6211\u4EEC\u6765\u770B\u770B\u8FD9\u4E2A\u5B57\u5178\u5728\u521D\u59CB\u62C6\u5206\u540E\u7684\u4E00\u90E8\u5206:"),jl=c(),m(ys.$$.fragment),gl=c(),m(Ps.$$.fragment),dl=c(),aa=r("p"),ge=t("\u73B0\u5728,\u627E\u5230\u5F97\u5206\u6700\u9AD8\u7684\u5BF9\u53EA\u9700\u8981\u4E00\u4E2A\u5FEB\u901F\u5FAA\u73AF:"),xl=c(),m(Cs.$$.fragment),bl=c(),m(Ds.$$.fragment),_l=c(),L=r("p"),de=t("\u6240\u4EE5\u7B2C\u4E00\u4E2A\u8981\u5B66\u4E60\u7684\u5408\u5E76\u662F "),mn=r("code"),xe=t("('a', '##b') -> 'ab'"),be=t(", \u5E76\u4E14\u6211\u4EEC\u6DFB\u52A0 "),fn=r("code"),_e=t("'ab'"),$e=t(" \u5230\u8BCD\u6C47\u8868\u4E2D:"),$l=c(),m(zs.$$.fragment),wl=c(),ts=r("p"),we=t("\u8981\u7EE7\u7EED\u63A5\u4E0B\u6765\u7684\u6B65\u9AA4,\u6211\u4EEC\u9700\u8981\u5728\u6211\u4EEC\u7684 "),jn=r("code"),qe=t("\u62C6\u5206"),ve=t(" \u5B57\u5178\u4E2D\u5E94\u7528\u8BE5\u5408\u5E76\u3002\u8BA9\u6211\u4EEC\u4E3A\u6B64\u7F16\u5199\u53E6\u4E00\u4E2A\u51FD\u6570:"),ql=c(),m(Os.$$.fragment),vl=c(),na=r("p"),ke=t("\u6211\u4EEC\u53EF\u4EE5\u770B\u770B\u7B2C\u4E00\u6B21\u5408\u5E76\u7684\u7ED3\u679C:"),kl=c(),m(Ts.$$.fragment),El=c(),m(Bs.$$.fragment),yl=c(),la=r("p"),Ee=t("\u73B0\u5728\u6211\u4EEC\u6709\u4E86\u5FAA\u73AF\u6240\u9700\u7684\u4E00\u5207,\u76F4\u5230\u6211\u4EEC\u5B66\u4F1A\u4E86\u6211\u4EEC\u60F3\u8981\u7684\u6240\u6709\u5408\u5E76\u3002\u6211\u4EEC\u7684\u76EE\u6807\u8BCD\u6C47\u91CF\u4E3A70:"),Pl=c(),m(Ns.$$.fragment),Cl=c(),ta=r("p"),ye=t("\u7136\u540E\u6211\u4EEC\u53EF\u4EE5\u67E5\u770B\u751F\u6210\u7684\u8BCD\u6C47\u8868:"),Dl=c(),m(Hs.$$.fragment),zl=c(),m(Ws.$$.fragment),Ol=c(),pa=r("p"),Pe=t("\u6B63\u5982\u6211\u4EEC\u6240\u770B\u5230\u7684,\u4E0E BPE \u76F8\u6BD4,\u8FD9\u4E2A\u6807\u8BB0\u5668\u5C06\u5355\u8BCD\u7684\u4E00\u90E8\u5206\u4F5C\u4E3A\u6807\u8BB0\u5B66\u4E60\u5F97\u66F4\u5FEB\u4E00\u4E9B\u3002"),Tl=c(),m(ps.$$.fragment),Bl=c(),ea=r("p"),Ce=t("\u4E3A\u4E86\u5BF9\u65B0\u6587\u672C\u8FDB\u884C\u5206\u8BCD,\u6211\u4EEC\u5BF9\u5176\u8FDB\u884C\u9884\u5206\u8BCD\u3001\u62C6\u5206,\u7136\u540E\u5BF9\u6BCF\u4E2A\u5355\u8BCD\u5E94\u7528\u5206\u8BCD\u7B97\u6CD5\u3002\u4E5F\u5C31\u662F\u8BF4,\u6211\u4EEC\u4ECE\u7B2C\u4E00\u4E2A\u8BCD\u7684\u5F00\u5934\u5BFB\u627E\u6700\u5927\u7684\u5B50\u8BCD\u5E76\u5C06\u5176\u62C6\u5206,\u7136\u540E\u6211\u4EEC\u5728\u7B2C\u4E8C\u90E8\u5206\u91CD\u590D\u8FD9\u4E2A\u8FC7\u7A0B,\u5BF9\u4E8E\u8BE5\u8BCD\u7684\u5176\u4F59\u90E8\u5206\u548C\u6587\u672C\u4E2D\u7684\u4EE5\u4E0B\u8BCD,\u4F9D\u6B64\u7C7B\u63A8:"),Nl=c(),m(Ss.$$.fragment),Hl=c(),ra=r("p"),De=t("\u8BA9\u6211\u4EEC\u7528\u8BCD\u6C47\u8868\u4E2D\u7684\u4E00\u4E2A\u5355\u8BCD\u548C\u53E6\u4E00\u4E2A\u4E0D\u5728\u8BCD\u6C47\u8868\u4E2D\u7684\u5355\u8BCD\u8FDB\u884C\u6D4B\u8BD5:"),Wl=c(),m(As.$$.fragment),Sl=c(),m(Ks.$$.fragment),Al=c(),ia=r("p"),ze=t("\u73B0\u5728,\u8BA9\u6211\u4EEC\u7F16\u5199\u4E00\u4E2A\u6807\u8BB0\u6587\u672C\u7684\u51FD\u6570:"),Kl=c(),m(Us.$$.fragment),Ul=c(),oa=r("p"),Oe=t("\u6211\u4EEC\u53EF\u4EE5\u5728\u4EFB\u4F55\u6587\u672C\u4E0A\u5C1D\u8BD5:"),Fl=c(),m(Fs.$$.fragment),Rl=c(),m(Rs.$$.fragment),Ml=c(),ca=r("p"),Te=t("\u8FD9\u5C31\u662F WordPiece \u7B97\u6CD5\u7684\u5168\u90E8\u5185\u5BB9!\u73B0\u5728\u8BA9\u6211\u4EEC\u6765\u770B\u770B Unigram\u3002"),this.h()},l(s){const l=ii('[data-svelte="svelte-1phssyn"]',document.head);u=i(l,"META",{name:!0,content:!0}),l.forEach(a),q=h(s),b=i(s,"H1",{class:!0});var Ms=o(b);$=i(Ms,"A",{id:!0,class:!0,href:!0});var gn=o($);O=i(gn,"SPAN",{});var dn=o(O);f(w.$$.fragment,dn),dn.forEach(a),gn.forEach(a),E=h(Ms),B=i(Ms,"SPAN",{});var xn=o(B);G=p(xn,"WordPiece \u6807\u8BB0\u5316"),xn.forEach(a),Ms.forEach(a),W=h(s),f(K.$$.fragment,s),es=h(s),Ls=i(s,"P",{});var bn=o(Ls);Zl=p(bn,"WordPiece \u662F Google \u4E3A\u9884\u8BAD\u7EC3 BERT \u800C\u5F00\u53D1\u7684\u6807\u8BB0\u5316\u7B97\u6CD5\u3002\u6B64\u540E,\u5B83\u5728\u4E0D\u5C11\u57FA\u4E8E BERT \u7684 Transformer \u6A21\u578B\u4E2D\u5F97\u5230\u91CD\u7528,\u4F8B\u5982 DistilBERT\u3001MobileBERT\u3001Funnel Transformers \u548C MPNET\u3002\u5B83\u5728\u8BAD\u7EC3\u65B9\u9762\u4E0E BPE \u975E\u5E38\u76F8\u4F3C,\u4F46\u5B9E\u9645\u6807\u8BB0\u5316\u7684\u65B9\u5F0F\u4E0D\u540C\u3002"),bn.forEach(a),$n=h(s),f(rs.$$.fragment,s),wn=h(s),f(I.$$.fragment,s),qn=h(s),is=i(s,"H2",{id:!0});var Ne=o(is);st=p(Ne,"\u8BAD\u7EC3\u7B97\u6CD5"),Ne.forEach(a),vn=h(s),f(Y.$$.fragment,s),kn=h(s),U=i(s,"P",{});var ha=o(U);at=p(ha,"\u4E0E BPE \u4E00\u6837,WordPiece \u4ECE\u4E00\u4E2A\u5C0F\u8BCD\u6C47\u8868\u5F00\u59CB,\u5305\u62EC\u6A21\u578B\u4F7F\u7528\u7684\u7279\u6B8A\u6807\u8BB0\u548C\u521D\u59CB\u5B57\u6BCD\u8868\u3002\u56E0\u4E3A\u5B83\u901A\u8FC7\u6DFB\u52A0\u524D\u7F00\u6765\u8BC6\u522B\u5B50\u8BCD (\u5982\u540C "),ga=i(ha,"CODE",{});var He=o(ga);nt=p(He,"##"),He.forEach(a),lt=p(ha," \u5BF9\u4E8E BERT),\u6BCF\u4E2A\u5355\u8BCD\u6700\u521D\u662F\u901A\u8FC7\u5C06\u8BE5\u524D\u7F00\u6DFB\u52A0\u5230\u5355\u8BCD\u5185\u7684\u6240\u6709\u5B57\u7B26\u6765\u62C6\u5206\u7684\u3002\u6240\u4EE5,\u4F8B\u5982 "),da=i(ha,"CODE",{});var We=o(da);tt=p(We,'"word"'),We.forEach(a),pt=p(ha," ,\u50CF\u8FD9\u6837\u62C6\u5206:"),ha.forEach(a),En=h(s),f(os.$$.fragment,s),yn=h(s),Gs=i(s,"P",{});var Se=o(Gs);et=p(Se,"\u56E0\u6B64,\u521D\u59CB\u5B57\u6BCD\u8868\u5305\u542B\u51FA\u73B0\u5728\u5355\u8BCD\u5F00\u5934\u7684\u6240\u6709\u5B57\u7B26\u4EE5\u53CA\u51FA\u73B0\u5728\u5355\u8BCD\u5185\u90E8\u7684\u4EE5 WordPiece \u524D\u7F00\u5F00\u5934\u7684\u5B57\u7B26\u3002"),Se.forEach(a),Pn=h(s),cs=i(s,"P",{});var Be=o(cs);rt=p(Be,`\u7136\u540E,\u518D\u6B21\u50CF BPE \u4E00\u6837,WordPiece \u5B66\u4E60\u5408\u5E76\u89C4\u5219\u3002\u4E3B\u8981\u533A\u522B\u5728\u4E8E\u9009\u62E9\u8981\u5408\u5E76\u7684\u5BF9\u7684\u65B9\u5F0F\u3002WordPiece \u4E0D\u662F\u9009\u62E9\u6700\u9891\u7E41\u7684\u5BF9\uFF0C\u800C\u662F\u4F7F\u7528\u4EE5\u4E0B\u516C\u5F0F\u8BA1\u7B97\u6BCF\u5BF9\u7684\u5206\u6570:
`),Cn=oi(Be),Be.forEach(a),Dn=h(s),P=i(s,"P",{});var N=o(P);it=p(N,"\u901A\u8FC7\u5C06\u914D\u5BF9\u7684\u9891\u7387\u9664\u4EE5\u5176\u6BCF\u4E2A\u90E8\u5206\u7684\u9891\u7387\u7684\u4E58\u79EF, \u8BE5\u7B97\u6CD5\u4F18\u5148\u5408\u5E76\u5355\u4E2A\u90E8\u5206\u5728\u8BCD\u6C47\u8868\u4E2D\u9891\u7387\u8F83\u4F4E\u7684\u5BF9\u3002\u4F8B\u5982,\u5B83\u4E0D\u4E00\u5B9A\u4F1A\u5408\u5E76 "),xa=i(N,"CODE",{});var Ae=o(xa);ot=p(Ae,'("un", "##able")'),Ae.forEach(a),ct=p(N," \u5373\u4F7F\u8FD9\u5BF9\u5728\u8BCD\u6C47\u8868\u4E2D\u51FA\u73B0\u7684\u9891\u7387\u5F88\u9AD8,\u56E0\u4E3A "),ba=i(N,"CODE",{});var Ke=o(ba);ht=p(Ke,'"un"'),Ke.forEach(a),ut=p(N," \u548C "),_a=i(N,"CODE",{});var Ue=o(_a);mt=p(Ue,'"##able"'),Ue.forEach(a),ft=p(N," \u5F88\u53EF\u80FD\u6BCF\u4E2A\u8BCD\u90FD\u51FA\u73B0\u5728\u5F88\u591A\u5176\u4ED6\u8BCD\u4E2D\u5E76\u4E14\u51FA\u73B0\u9891\u7387\u5F88\u9AD8\u3002\u76F8\u6BD4\u4E4B\u4E0B,\u50CF "),$a=i(N,"CODE",{});var Fe=o($a);jt=p(Fe,'("hu", "##gging")'),Fe.forEach(a),gt=p(N," \u53EF\u80FD\u4F1A\u66F4\u5FEB\u5730\u5408\u5E76 (\u5047\u8BBE \u201Chugging\u201D \u7ECF\u5E38\u51FA\u73B0\u5728\u8BCD\u6C47\u8868\u4E2D),\u56E0\u4E3A "),wa=i(N,"CODE",{});var Re=o(wa);dt=p(Re,'"hu"'),Re.forEach(a),xt=p(N," \u548C "),qa=i(N,"CODE",{});var Me=o(qa);bt=p(Me,'"##gging"'),Me.forEach(a),_t=p(N," \u8FD9\u4E24\u4E2A\u8BCD\u5355\u72EC\u51FA\u73B0\u5730\u9891\u7387\u53EF\u80FD\u8F83\u4F4E\u3002"),N.forEach(a),zn=h(s),Vs=i(s,"P",{});var Le=o(Vs);$t=p(Le,"\u8BA9\u6211\u4EEC\u770B\u770B\u6211\u4EEC\u5728 BPE \u8BAD\u7EC3\u793A\u4F8B\u4E2D\u4F7F\u7528\u7684\u76F8\u540C\u8BCD\u6C47:"),Le.forEach(a),On=h(s),f(hs.$$.fragment,s),Tn=h(s),Is=i(s,"P",{});var Ge=o(Is);wt=p(Ge,"\u8FD9\u91CC\u7684\u62C6\u5206\u5C06\u662F:"),Ge.forEach(a),Bn=h(s),f(us.$$.fragment,s),Nn=h(s),y=i(s,"P",{});var T=o(y);qt=p(T,"\u6240\u4EE5\u6700\u521D\u7684\u8BCD\u6C47\u5C06\u662F "),va=i(T,"CODE",{});var Ve=o(va);vt=p(Ve,'["b", "h", "p", "##g", "##n", "##s", "##u"]'),Ve.forEach(a),kt=p(T," (\u5982\u679C\u6211\u4EEC\u6682\u65F6\u5FD8\u8BB0\u7279\u6B8A\u6807\u8BB0)\u3002\u6700\u9891\u7E41\u7684\u4E00\u5BF9\u662F "),ka=i(T,"CODE",{});var Ie=o(ka);Et=p(Ie,'("##u", "##g")'),Ie.forEach(a),yt=p(T," (\u76EE\u524D20\u6B21),\u4F46 "),Ea=i(T,"CODE",{});var Ye=o(Ea);Pt=p(Ye,'"##u"'),Ye.forEach(a),Ct=p(T," \u5355\u72EC\u51FA\u73B0\u7684\u9891\u7387\u975E\u5E38\u9AD8,\u6240\u4EE5\u5B83\u7684\u5206\u6570\u4E0D\u662F\u6700\u9AD8\u7684(\u5B83\u662F 1 / 36)\u3002\u6240\u6709\u5E26\u6709 "),ya=i(T,"CODE",{});var Je=o(ya);Dt=p(Je,'"##u"'),Je.forEach(a),zt=p(T," \u7684\u5BF9\u5B9E\u9645\u4E0A\u90FD\u6709\u76F8\u540C\u7684\u5206\u6570(1 / 36),\u6240\u4EE5\u5206\u6570\u6700\u9AD8\u7684\u5BF9\u662F "),Pa=i(T,"CODE",{});var Qe=o(Pa);Ot=p(Qe,'("##g", "##s")'),Qe.forEach(a),Tt=p(T," \u2014 \u552F\u4E00\u6CA1\u6709 "),Ca=i(T,"CODE",{});var Xe=o(Ca);Bt=p(Xe,'"##u"'),Xe.forEach(a),Nt=p(T," \u7684\u5BF9\u2014 1 / 20,\u6240\u4EE5\u5B66\u4E60\u7684\u7B2C\u4E00\u4E2A\u5408\u5E76\u662F "),Da=i(T,"CODE",{});var Ze=o(Da);Ht=p(Ze,'("##g", "##s") -> ("##gs")'),Ze.forEach(a),Wt=p(T,"\u3002"),T.forEach(a),Hn=h(s),F=i(s,"P",{});var ua=o(F);St=p(ua,"\u8BF7\u6CE8\u610F,\u5F53\u6211\u4EEC\u5408\u5E76\u65F6,\u6211\u4EEC\u5220\u9664\u4E86\u4E24\u4E2A\u6807\u8BB0\u4E4B\u95F4\u7684 "),za=i(ua,"CODE",{});var sr=o(za);At=p(sr,"##"),sr.forEach(a),Kt=p(ua,",\u6240\u4EE5\u6211\u4EEC\u6DFB\u52A0 "),Oa=i(ua,"CODE",{});var ar=o(Oa);Ut=p(ar,'"##gs"'),ar.forEach(a),Ft=p(ua," \u5230\u8BCD\u6C47\u8868\u4E2D,\u5E76\u5728\u8BED\u6599\u5E93\u7684\u5355\u8BCD\u4E2D\u5E94\u7528\u8BE5\u5408\u5E76:"),ua.forEach(a),Wn=h(s),f(ms.$$.fragment,s),Sn=h(s),R=i(s,"P",{});var ma=o(R);Rt=p(ma,"\u5728\u8FD9\u4E00\u70B9\u4E2D, "),Ta=i(ma,"CODE",{});var nr=o(Ta);Mt=p(nr,'"##u"'),nr.forEach(a),Lt=p(ma," \u662F\u5728\u6240\u6709\u53EF\u80FD\u7684\u5BF9\u4E2D,\u56E0\u6B64\u5B83\u4EEC\u6700\u7EC8\u90FD\u5177\u6709\u76F8\u540C\u7684\u5206\u6570\u3002\u5047\u8BBE\u5728\u8FD9\u79CD\u60C5\u51B5\u4E0B,\u7B2C\u4E00\u5BF9\u88AB\u5408\u5E76, "),Ba=i(ma,"CODE",{});var lr=o(Ba);Gt=p(lr,'("h", "##u") -> "hu"'),lr.forEach(a),Vt=p(ma,"\u3002\u8FD9\u4F7F\u5F97\u6211\u4EEC:"),ma.forEach(a),An=h(s),f(fs.$$.fragment,s),Kn=h(s),M=i(s,"P",{});var fa=o(M);It=p(fa,"\u7136\u540E\u4E0B\u4E00\u4E2A\u6700\u9AD8\u7684\u5206\u6570\u7531 "),Na=i(fa,"CODE",{});var tr=o(Na);Yt=p(tr,'("hu", "##g")'),tr.forEach(a),Jt=p(fa," \u548C "),Ha=i(fa,"CODE",{});var pr=o(Ha);Qt=p(pr,'("hu", "##gs")'),pr.forEach(a),Xt=p(fa," \u5171\u4EAB(1/15,\u4E0E\u5176\u4ED6\u6240\u6709\u5BF9\u7684 1/21 \u76F8\u6BD4),\u56E0\u6B64\u5408\u5E76\u5F97\u5206\u6700\u9AD8\u7684\u7B2C\u4E00\u5BF9:"),fa.forEach(a),Un=h(s),f(js.$$.fragment,s),Fn=h(s),Ys=i(s,"P",{});var er=o(Ys);Zt=p(er,"\u6211\u4EEC\u7EE7\u7EED\u8FD9\u6837\u5904\u7406,\u76F4\u5230\u8FBE\u5230\u6211\u4EEC\u6240\u9700\u7684\u8BCD\u6C47\u91CF\u3002"),er.forEach(a),Rn=h(s),f(J.$$.fragment,s),Mn=h(s),gs=i(s,"H2",{id:!0});var rr=o(gs);sp=p(rr,"\u6807\u8BB0\u5316\u7B97\u6CD5"),rr.forEach(a),Ln=h(s),C=i(s,"P",{});var H=o(C);ap=p(H,"WordPiece \u548C BPE \u4E2D\u7684\u6807\u8BB0\u5316\u7684\u4E0D\u540C\u5728\u4E8E WordPiece \u53EA\u4FDD\u5B58\u6700\u7EC8\u8BCD\u6C47,\u800C\u4E0D\u662F\u5B66\u4E60\u7684\u5408\u5E76\u89C4\u5219\u3002\u4ECE\u8981\u6807\u8BB0\u7684\u5355\u8BCD\u5F00\u59CB,WordPiece \u627E\u5230\u8BCD\u6C47\u8868\u4E2D\u6700\u957F\u7684\u5B50\u8BCD,\u7136\u540E\u5BF9\u5176\u8FDB\u884C\u62C6\u5206\u3002\u4F8B\u5982,\u5982\u679C\u6211\u4EEC\u4F7F\u7528\u4E0A\u9762\u4F8B\u5B50\u4E2D\u5B66\u5230\u7684\u8BCD\u6C47,\u5BF9\u4E8E\u5355\u8BCD "),Wa=i(H,"CODE",{});var ir=o(Wa);np=p(ir,'"hugs"'),ir.forEach(a),lp=p(H,",\u8BCD\u6C47\u8868\u4E2D\u4ECE\u5934\u5F00\u59CB\u7684\u6700\u957F\u5B50\u8BCD\u662F "),Sa=i(H,"CODE",{});var or=o(Sa);tp=p(or,'"hug"'),or.forEach(a),pp=p(H,",\u6240\u4EE5\u6211\u4EEC\u5728\u90A3\u91CC\u62C6\u5206\u5E76\u5F97\u5230 "),Aa=i(H,"CODE",{});var cr=o(Aa);ep=p(cr,'["hug", "##s"]'),cr.forEach(a),rp=p(H,"\u3002 \u7136\u540E\u6211\u4EEC\u7EE7\u7EED\u4F7F\u7528\u8BCD\u6C47\u8868\u4E2D\u7684 "),Ka=i(H,"CODE",{});var hr=o(Ka);ip=p(hr,'"##s"'),hr.forEach(a),op=p(H,",\u56E0\u6B64 "),Ua=i(H,"CODE",{});var ur=o(Ua);cp=p(ur,'"hugs"'),ur.forEach(a),hp=p(H," \u7684\u6807\u8BB0\u5316\u662F "),Fa=i(H,"CODE",{});var mr=o(Fa);up=p(mr,'["hug", "##s"]'),mr.forEach(a),mp=p(H,"."),H.forEach(a),Gn=h(s),Q=i(s,"P",{});var Gl=o(Q);fp=p(Gl,"\u4F7F\u7528 BPE, \u6211\u4EEC\u5C06\u6309\u987A\u5E8F\u5E94\u7528\u5B66\u4E60\u5230\u7684\u5408\u5E76\u5E76\u5C06\u5176\u6807\u8BB0\u4E3A "),Ra=i(Gl,"CODE",{});var fr=o(Ra);jp=p(fr,'["hu", "##gs"]'),fr.forEach(a),gp=p(Gl,",\u6240\u4EE5\u7F16\u7801\u4E0D\u540C\u3002"),Gl.forEach(a),Vn=h(s),v=i(s,"P",{});var D=o(v);dp=p(D,"\u518D\u4E3E\u4E00\u4E2A\u4F8B\u5B50,\u8BA9\u6211\u4EEC\u770B\u770B "),Ma=i(D,"CODE",{});var jr=o(Ma);xp=p(jr,'"bugs"'),jr.forEach(a),bp=p(D," \u5C06\u5982\u4F55\u88AB\u6807\u8BB0\u5316\u3002 "),La=i(D,"CODE",{});var gr=o(La);_p=p(gr,'"b"'),gr.forEach(a),$p=p(D," \u662F\u4ECE\u8BCD\u6C47\u8868\u4E2D\u5355\u8BCD\u5F00\u5934\u5F00\u59CB\u7684\u6700\u957F\u5B50\u8BCD,\u6240\u4EE5\u6211\u4EEC\u5728\u90A3\u91CC\u62C6\u5206\u5E76\u5F97\u5230 "),Ga=i(D,"CODE",{});var dr=o(Ga);wp=p(dr,'["b", "##ugs"]'),dr.forEach(a),qp=p(D,"\u3002\u7136\u540E "),Va=i(D,"CODE",{});var xr=o(Va);vp=p(xr,'"##u"'),xr.forEach(a),kp=p(D," \u662F\u8BCD\u6C47\u8868\u4E2D\u4ECE "),Ia=i(D,"CODE",{});var br=o(Ia);Ep=p(br,'"##ugs"'),br.forEach(a),yp=p(D," \u5F00\u59CB\u7684\u6700\u957F\u7684\u5B50\u8BCD,\u6240\u4EE5\u6211\u4EEC\u5728\u90A3\u91CC\u62C6\u5206\u5E76\u5F97\u5230 "),Ya=i(D,"CODE",{});var _r=o(Ya);Pp=p(_r,'["b", "##u, "##gs"]'),_r.forEach(a),Cp=p(D,"\u3002\u6700\u540E, "),Ja=i(D,"CODE",{});var $r=o(Ja);Dp=p($r,'"##gs"'),$r.forEach(a),zp=p(D," \u5728\u8BCD\u6C47\u8868\u4E2D,\u6240\u4EE5\u6700\u540E\u4E00\u4E2A\u5217\u8868\u662F "),Qa=i(D,"CODE",{});var wr=o(Qa);Op=p(wr,'"bugs"'),wr.forEach(a),Tp=p(D," \u7684\u6807\u8BB0\u5316\u3002"),D.forEach(a),In=h(s),k=i(s,"P",{});var z=o(k);Bp=p(z,"\u5F53\u5206\u8BCD\u8FBE\u5230\u65E0\u6CD5\u5728\u8BCD\u6C47\u8868\u4E2D\u627E\u5230\u5B50\u8BCD\u7684\u9636\u6BB5\u65F6, \u6574\u4E2A\u8BCD\u88AB\u6807\u8BB0\u4E3A\u672A\u77E5 \u2014 \u4F8B\u5982, "),Xa=i(z,"CODE",{});var qr=o(Xa);Np=p(qr,'"mug"'),qr.forEach(a),Hp=p(z," \u5C06\u88AB\u6807\u8BB0\u4E3A "),Za=i(z,"CODE",{});var vr=o(Za);Wp=p(vr,'["[UNK]"]'),vr.forEach(a),Sp=p(z,",\u5C31\u50CF "),sn=i(z,"CODE",{});var kr=o(sn);Ap=p(kr,'"bum"'),kr.forEach(a),Kp=p(z," (\u5373\u4F7F\u6211\u4EEC\u53EF\u4EE5\u4EE5 "),an=i(z,"CODE",{});var Er=o(an);Up=p(Er,'"b"'),Er.forEach(a),Fp=p(z," \u548C "),nn=i(z,"CODE",{});var yr=o(nn);Rp=p(yr,'"##u"'),yr.forEach(a),Mp=p(z," \u5F00\u59CB, "),ln=i(z,"CODE",{});var Pr=o(ln);Lp=p(Pr,'"##m"'),Pr.forEach(a),Gp=p(z," \u4E0D\u5728\u8BCD\u6C47\u8868\u4E2D,\u7531\u6B64\u4EA7\u751F\u7684\u6807\u8BB0\u5C06\u53EA\u662F "),tn=i(z,"CODE",{});var Cr=o(tn);Vp=p(Cr,'["[UNK]"]'),Cr.forEach(a),Ip=p(z,", \u4E0D\u662F "),pn=i(z,"CODE",{});var Dr=o(pn);Yp=p(Dr,'["b", "##u", "[UNK]"]'),Dr.forEach(a),Jp=p(z,")\u3002\u8FD9\u662F\u4E0E BPE \u7684\u53E6\u4E00\u4E2A\u533A\u522B,BPE \u53EA\u4F1A\u5C06\u4E0D\u5728\u8BCD\u6C47\u8868\u4E2D\u7684\u5355\u4E2A\u5B57\u7B26\u5206\u7C7B\u4E3A\u672A\u77E5\u3002"),z.forEach(a),Yn=h(s),f(X.$$.fragment,s),Jn=h(s),V=i(s,"H2",{class:!0});var Vl=o(V);Z=i(Vl,"A",{id:!0,class:!0,href:!0});var zr=o(Z);en=i(zr,"SPAN",{});var Or=o(en);f(ds.$$.fragment,Or),Or.forEach(a),zr.forEach(a),Qp=h(Vl),rn=i(Vl,"SPAN",{});var Tr=o(rn);Xp=p(Tr,"\u5B9E\u73B0 WordPiece"),Tr.forEach(a),Vl.forEach(a),Qn=h(s),Js=i(s,"P",{});var Br=o(Js);Zp=p(Br,"\u73B0\u5728\u8BA9\u6211\u4EEC\u770B\u4E00\u4E0B WordPiece \u7B97\u6CD5\u7684\u5B9E\u73B0\u3002\u4E0E BPE \u4E00\u6837,\u8FD9\u53EA\u662F\u6559\u5B66,\u4F60\u5C06\u65E0\u6CD5\u5728\u5927\u578B\u8BED\u6599\u5E93\u4E2D\u4F7F\u7528\u5B83\u3002"),Br.forEach(a),Xn=h(s),Qs=i(s,"P",{});var Nr=o(Qs);se=p(Nr,"\u6211\u4EEC\u5C06\u4F7F\u7528\u4E0E BPE \u793A\u4F8B\u4E2D\u76F8\u540C\u7684\u8BED\u6599\u5E93:"),Nr.forEach(a),Zn=h(s),f(xs.$$.fragment,s),sl=h(s),ss=i(s,"P",{});var Il=o(ss);ae=p(Il,"\u9996\u5148,\u6211\u4EEC\u9700\u8981\u5C06\u8BED\u6599\u5E93\u9884\u5148\u6807\u8BB0\u4E3A\u5355\u8BCD\u3002\u7531\u4E8E\u6211\u4EEC\u6B63\u5728\u590D\u5236 WordPiece \u6807\u8BB0\u5668 (\u5982 BERT),\u56E0\u6B64\u6211\u4EEC\u5C06\u4F7F\u7528 "),on=i(Il,"CODE",{});var Hr=o(on);ne=p(Hr,"bert-base-cased"),Hr.forEach(a),le=p(Il," \u6807\u8BB0\u5668\u7528\u4E8E\u9884\u6807\u8BB0\u5316:"),Il.forEach(a),al=h(s),f(bs.$$.fragment,s),nl=h(s),Xs=i(s,"P",{});var Wr=o(Xs);te=p(Wr,"\u7136\u540E\u6211\u4EEC\u5728\u8FDB\u884C\u9884\u6807\u8BB0\u5316\u65F6\u8BA1\u7B97\u8BED\u6599\u5E93\u4E2D\u6BCF\u4E2A\u5355\u8BCD\u7684\u9891\u7387:"),Wr.forEach(a),ll=h(s),f(_s.$$.fragment,s),tl=h(s),f($s.$$.fragment,s),pl=h(s),as=i(s,"P",{});var Yl=o(as);pe=p(Yl,"\u6B63\u5982\u6211\u4EEC\u4E4B\u524D\u770B\u5230\u7684,\u5B57\u6BCD\u8868\u662F\u7531\u5355\u8BCD\u7684\u6240\u6709\u7B2C\u4E00\u4E2A\u5B57\u6BCD\u7EC4\u6210\u7684\u552F\u4E00\u96C6\u5408,\u4EE5\u53CA\u51FA\u73B0\u5728\u524D\u7F00\u4E3A "),cn=i(Yl,"CODE",{});var Sr=o(cn);ee=p(Sr,"##"),Sr.forEach(a),re=p(Yl," \u7684\u5176\u4ED6\u5B57\u6BCD:"),Yl.forEach(a),el=h(s),f(ws.$$.fragment,s),rl=h(s),f(qs.$$.fragment,s),il=h(s),ns=i(s,"P",{});var Jl=o(ns);ie=p(Jl,"\u6211\u4EEC\u8FD8\u5728\u8BE5\u8BCD\u6C47\u8868\u7684\u5F00\u5934\u6DFB\u52A0\u4E86\u6A21\u578B\u4F7F\u7528\u7684\u7279\u6B8A\u6807\u8BB0\u3002\u5728\u4F7F\u7528 BERT \u7684\u60C5\u51B5\u4E0B,\u5B83\u662F\u5217\u8868 "),hn=i(Jl,"CODE",{});var Ar=o(hn);oe=p(Ar,'["[PAD]", "[UNK]", "[CLS]", "[SEP]", "[MASK]"]'),Ar.forEach(a),ce=p(Jl,":"),Jl.forEach(a),ol=h(s),f(vs.$$.fragment,s),cl=h(s),ls=i(s,"P",{});var Ql=o(ls);he=p(Ql,"\u63A5\u4E0B\u6765\u6211\u4EEC\u9700\u8981\u62C6\u5206\u6BCF\u4E2A\u5355\u8BCD, \u6240\u6709\u4E0D\u662F\u7B2C\u4E00\u4E2A\u5B57\u6BCD\u7684\u5B57\u6BCD\u90FD\u4EE5 "),un=i(Ql,"CODE",{});var Kr=o(un);ue=p(Kr,"##"),Kr.forEach(a),me=p(Ql," \u4E3A\u524D\u7F00:"),Ql.forEach(a),hl=h(s),f(ks.$$.fragment,s),ul=h(s),Zs=i(s,"P",{});var Ur=o(Zs);fe=p(Ur,"\u73B0\u5728\u6211\u4EEC\u5DF2\u7ECF\u51C6\u5907\u597D\u8BAD\u7EC3\u4E86,\u8BA9\u6211\u4EEC\u7F16\u5199\u4E00\u4E2A\u51FD\u6570\u6765\u8BA1\u7B97\u6BCF\u5BF9\u7684\u5206\u6570\u3002\u6211\u4EEC\u9700\u8981\u5728\u8BAD\u7EC3\u7684\u6BCF\u4E2A\u6B65\u9AA4\u4E2D\u4F7F\u7528\u5B83:"),Ur.forEach(a),ml=h(s),f(Es.$$.fragment,s),fl=h(s),sa=i(s,"P",{});var Fr=o(sa);je=p(Fr,"\u8BA9\u6211\u4EEC\u6765\u770B\u770B\u8FD9\u4E2A\u5B57\u5178\u5728\u521D\u59CB\u62C6\u5206\u540E\u7684\u4E00\u90E8\u5206:"),Fr.forEach(a),jl=h(s),f(ys.$$.fragment,s),gl=h(s),f(Ps.$$.fragment,s),dl=h(s),aa=i(s,"P",{});var Rr=o(aa);ge=p(Rr,"\u73B0\u5728,\u627E\u5230\u5F97\u5206\u6700\u9AD8\u7684\u5BF9\u53EA\u9700\u8981\u4E00\u4E2A\u5FEB\u901F\u5FAA\u73AF:"),Rr.forEach(a),xl=h(s),f(Cs.$$.fragment,s),bl=h(s),f(Ds.$$.fragment,s),_l=h(s),L=i(s,"P",{});var ja=o(L);de=p(ja,"\u6240\u4EE5\u7B2C\u4E00\u4E2A\u8981\u5B66\u4E60\u7684\u5408\u5E76\u662F "),mn=i(ja,"CODE",{});var Mr=o(mn);xe=p(Mr,"('a', '##b') -> 'ab'"),Mr.forEach(a),be=p(ja,", \u5E76\u4E14\u6211\u4EEC\u6DFB\u52A0 "),fn=i(ja,"CODE",{});var Lr=o(fn);_e=p(Lr,"'ab'"),Lr.forEach(a),$e=p(ja," \u5230\u8BCD\u6C47\u8868\u4E2D:"),ja.forEach(a),$l=h(s),f(zs.$$.fragment,s),wl=h(s),ts=i(s,"P",{});var Xl=o(ts);we=p(Xl,"\u8981\u7EE7\u7EED\u63A5\u4E0B\u6765\u7684\u6B65\u9AA4,\u6211\u4EEC\u9700\u8981\u5728\u6211\u4EEC\u7684 "),jn=i(Xl,"CODE",{});var Gr=o(jn);qe=p(Gr,"\u62C6\u5206"),Gr.forEach(a),ve=p(Xl," \u5B57\u5178\u4E2D\u5E94\u7528\u8BE5\u5408\u5E76\u3002\u8BA9\u6211\u4EEC\u4E3A\u6B64\u7F16\u5199\u53E6\u4E00\u4E2A\u51FD\u6570:"),Xl.forEach(a),ql=h(s),f(Os.$$.fragment,s),vl=h(s),na=i(s,"P",{});var Vr=o(na);ke=p(Vr,"\u6211\u4EEC\u53EF\u4EE5\u770B\u770B\u7B2C\u4E00\u6B21\u5408\u5E76\u7684\u7ED3\u679C:"),Vr.forEach(a),kl=h(s),f(Ts.$$.fragment,s),El=h(s),f(Bs.$$.fragment,s),yl=h(s),la=i(s,"P",{});var Ir=o(la);Ee=p(Ir,"\u73B0\u5728\u6211\u4EEC\u6709\u4E86\u5FAA\u73AF\u6240\u9700\u7684\u4E00\u5207,\u76F4\u5230\u6211\u4EEC\u5B66\u4F1A\u4E86\u6211\u4EEC\u60F3\u8981\u7684\u6240\u6709\u5408\u5E76\u3002\u6211\u4EEC\u7684\u76EE\u6807\u8BCD\u6C47\u91CF\u4E3A70:"),Ir.forEach(a),Pl=h(s),f(Ns.$$.fragment,s),Cl=h(s),ta=i(s,"P",{});var Yr=o(ta);ye=p(Yr,"\u7136\u540E\u6211\u4EEC\u53EF\u4EE5\u67E5\u770B\u751F\u6210\u7684\u8BCD\u6C47\u8868:"),Yr.forEach(a),Dl=h(s),f(Hs.$$.fragment,s),zl=h(s),f(Ws.$$.fragment,s),Ol=h(s),pa=i(s,"P",{});var Jr=o(pa);Pe=p(Jr,"\u6B63\u5982\u6211\u4EEC\u6240\u770B\u5230\u7684,\u4E0E BPE \u76F8\u6BD4,\u8FD9\u4E2A\u6807\u8BB0\u5668\u5C06\u5355\u8BCD\u7684\u4E00\u90E8\u5206\u4F5C\u4E3A\u6807\u8BB0\u5B66\u4E60\u5F97\u66F4\u5FEB\u4E00\u4E9B\u3002"),Jr.forEach(a),Tl=h(s),f(ps.$$.fragment,s),Bl=h(s),ea=i(s,"P",{});var Qr=o(ea);Ce=p(Qr,"\u4E3A\u4E86\u5BF9\u65B0\u6587\u672C\u8FDB\u884C\u5206\u8BCD,\u6211\u4EEC\u5BF9\u5176\u8FDB\u884C\u9884\u5206\u8BCD\u3001\u62C6\u5206,\u7136\u540E\u5BF9\u6BCF\u4E2A\u5355\u8BCD\u5E94\u7528\u5206\u8BCD\u7B97\u6CD5\u3002\u4E5F\u5C31\u662F\u8BF4,\u6211\u4EEC\u4ECE\u7B2C\u4E00\u4E2A\u8BCD\u7684\u5F00\u5934\u5BFB\u627E\u6700\u5927\u7684\u5B50\u8BCD\u5E76\u5C06\u5176\u62C6\u5206,\u7136\u540E\u6211\u4EEC\u5728\u7B2C\u4E8C\u90E8\u5206\u91CD\u590D\u8FD9\u4E2A\u8FC7\u7A0B,\u5BF9\u4E8E\u8BE5\u8BCD\u7684\u5176\u4F59\u90E8\u5206\u548C\u6587\u672C\u4E2D\u7684\u4EE5\u4E0B\u8BCD,\u4F9D\u6B64\u7C7B\u63A8:"),Qr.forEach(a),Nl=h(s),f(Ss.$$.fragment,s),Hl=h(s),ra=i(s,"P",{});var Xr=o(ra);De=p(Xr,"\u8BA9\u6211\u4EEC\u7528\u8BCD\u6C47\u8868\u4E2D\u7684\u4E00\u4E2A\u5355\u8BCD\u548C\u53E6\u4E00\u4E2A\u4E0D\u5728\u8BCD\u6C47\u8868\u4E2D\u7684\u5355\u8BCD\u8FDB\u884C\u6D4B\u8BD5:"),Xr.forEach(a),Wl=h(s),f(As.$$.fragment,s),Sl=h(s),f(Ks.$$.fragment,s),Al=h(s),ia=i(s,"P",{});var Zr=o(ia);ze=p(Zr,"\u73B0\u5728,\u8BA9\u6211\u4EEC\u7F16\u5199\u4E00\u4E2A\u6807\u8BB0\u6587\u672C\u7684\u51FD\u6570:"),Zr.forEach(a),Kl=h(s),f(Us.$$.fragment,s),Ul=h(s),oa=i(s,"P",{});var si=o(oa);Oe=p(si,"\u6211\u4EEC\u53EF\u4EE5\u5728\u4EFB\u4F55\u6587\u672C\u4E0A\u5C1D\u8BD5:"),si.forEach(a),Fl=h(s),f(Fs.$$.fragment,s),Rl=h(s),f(Rs.$$.fragment,s),Ml=h(s),ca=i(s,"P",{});var ai=o(ca);Te=p(ai,"\u8FD9\u5C31\u662F WordPiece \u7B97\u6CD5\u7684\u5168\u90E8\u5185\u5BB9!\u73B0\u5728\u8BA9\u6211\u4EEC\u6765\u770B\u770B Unigram\u3002"),ai.forEach(a),this.h()},h(){A(u,"name","hf:doc:metadata"),A(u,"content",JSON.stringify(bi)),A($,"id","wordpiece"),A($,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),A($,"href","#wordpiece"),A(b,"class","relative group"),A(is,"id",""),Cn.a=null,A(gs,"id",""),A(Z,"id","wordpiece"),A(Z,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),A(Z,"href","#wordpiece"),A(V,"class","relative group")},m(s,l){n(document.head,u),e(s,q,l),e(s,b,l),n(b,$),n($,O),j(w,O,null),n(b,E),n(b,B),n(B,G),e(s,W,l),j(K,s,l),e(s,es,l),e(s,Ls,l),n(Ls,Zl),e(s,$n,l),j(rs,s,l),e(s,wn,l),j(I,s,l),e(s,qn,l),e(s,is,l),n(is,st),e(s,vn,l),j(Y,s,l),e(s,kn,l),e(s,U,l),n(U,at),n(U,ga),n(ga,nt),n(U,lt),n(U,da),n(da,tt),n(U,pt),e(s,En,l),j(os,s,l),e(s,yn,l),e(s,Gs,l),n(Gs,et),e(s,Pn,l),e(s,cs,l),n(cs,rt),Cn.m(li,cs),e(s,Dn,l),e(s,P,l),n(P,it),n(P,xa),n(xa,ot),n(P,ct),n(P,ba),n(ba,ht),n(P,ut),n(P,_a),n(_a,mt),n(P,ft),n(P,$a),n($a,jt),n(P,gt),n(P,wa),n(wa,dt),n(P,xt),n(P,qa),n(qa,bt),n(P,_t),e(s,zn,l),e(s,Vs,l),n(Vs,$t),e(s,On,l),j(hs,s,l),e(s,Tn,l),e(s,Is,l),n(Is,wt),e(s,Bn,l),j(us,s,l),e(s,Nn,l),e(s,y,l),n(y,qt),n(y,va),n(va,vt),n(y,kt),n(y,ka),n(ka,Et),n(y,yt),n(y,Ea),n(Ea,Pt),n(y,Ct),n(y,ya),n(ya,Dt),n(y,zt),n(y,Pa),n(Pa,Ot),n(y,Tt),n(y,Ca),n(Ca,Bt),n(y,Nt),n(y,Da),n(Da,Ht),n(y,Wt),e(s,Hn,l),e(s,F,l),n(F,St),n(F,za),n(za,At),n(F,Kt),n(F,Oa),n(Oa,Ut),n(F,Ft),e(s,Wn,l),j(ms,s,l),e(s,Sn,l),e(s,R,l),n(R,Rt),n(R,Ta),n(Ta,Mt),n(R,Lt),n(R,Ba),n(Ba,Gt),n(R,Vt),e(s,An,l),j(fs,s,l),e(s,Kn,l),e(s,M,l),n(M,It),n(M,Na),n(Na,Yt),n(M,Jt),n(M,Ha),n(Ha,Qt),n(M,Xt),e(s,Un,l),j(js,s,l),e(s,Fn,l),e(s,Ys,l),n(Ys,Zt),e(s,Rn,l),j(J,s,l),e(s,Mn,l),e(s,gs,l),n(gs,sp),e(s,Ln,l),e(s,C,l),n(C,ap),n(C,Wa),n(Wa,np),n(C,lp),n(C,Sa),n(Sa,tp),n(C,pp),n(C,Aa),n(Aa,ep),n(C,rp),n(C,Ka),n(Ka,ip),n(C,op),n(C,Ua),n(Ua,cp),n(C,hp),n(C,Fa),n(Fa,up),n(C,mp),e(s,Gn,l),e(s,Q,l),n(Q,fp),n(Q,Ra),n(Ra,jp),n(Q,gp),e(s,Vn,l),e(s,v,l),n(v,dp),n(v,Ma),n(Ma,xp),n(v,bp),n(v,La),n(La,_p),n(v,$p),n(v,Ga),n(Ga,wp),n(v,qp),n(v,Va),n(Va,vp),n(v,kp),n(v,Ia),n(Ia,Ep),n(v,yp),n(v,Ya),n(Ya,Pp),n(v,Cp),n(v,Ja),n(Ja,Dp),n(v,zp),n(v,Qa),n(Qa,Op),n(v,Tp),e(s,In,l),e(s,k,l),n(k,Bp),n(k,Xa),n(Xa,Np),n(k,Hp),n(k,Za),n(Za,Wp),n(k,Sp),n(k,sn),n(sn,Ap),n(k,Kp),n(k,an),n(an,Up),n(k,Fp),n(k,nn),n(nn,Rp),n(k,Mp),n(k,ln),n(ln,Lp),n(k,Gp),n(k,tn),n(tn,Vp),n(k,Ip),n(k,pn),n(pn,Yp),n(k,Jp),e(s,Yn,l),j(X,s,l),e(s,Jn,l),e(s,V,l),n(V,Z),n(Z,en),j(ds,en,null),n(V,Qp),n(V,rn),n(rn,Xp),e(s,Qn,l),e(s,Js,l),n(Js,Zp),e(s,Xn,l),e(s,Qs,l),n(Qs,se),e(s,Zn,l),j(xs,s,l),e(s,sl,l),e(s,ss,l),n(ss,ae),n(ss,on),n(on,ne),n(ss,le),e(s,al,l),j(bs,s,l),e(s,nl,l),e(s,Xs,l),n(Xs,te),e(s,ll,l),j(_s,s,l),e(s,tl,l),j($s,s,l),e(s,pl,l),e(s,as,l),n(as,pe),n(as,cn),n(cn,ee),n(as,re),e(s,el,l),j(ws,s,l),e(s,rl,l),j(qs,s,l),e(s,il,l),e(s,ns,l),n(ns,ie),n(ns,hn),n(hn,oe),n(ns,ce),e(s,ol,l),j(vs,s,l),e(s,cl,l),e(s,ls,l),n(ls,he),n(ls,un),n(un,ue),n(ls,me),e(s,hl,l),j(ks,s,l),e(s,ul,l),e(s,Zs,l),n(Zs,fe),e(s,ml,l),j(Es,s,l),e(s,fl,l),e(s,sa,l),n(sa,je),e(s,jl,l),j(ys,s,l),e(s,gl,l),j(Ps,s,l),e(s,dl,l),e(s,aa,l),n(aa,ge),e(s,xl,l),j(Cs,s,l),e(s,bl,l),j(Ds,s,l),e(s,_l,l),e(s,L,l),n(L,de),n(L,mn),n(mn,xe),n(L,be),n(L,fn),n(fn,_e),n(L,$e),e(s,$l,l),j(zs,s,l),e(s,wl,l),e(s,ts,l),n(ts,we),n(ts,jn),n(jn,qe),n(ts,ve),e(s,ql,l),j(Os,s,l),e(s,vl,l),e(s,na,l),n(na,ke),e(s,kl,l),j(Ts,s,l),e(s,El,l),j(Bs,s,l),e(s,yl,l),e(s,la,l),n(la,Ee),e(s,Pl,l),j(Ns,s,l),e(s,Cl,l),e(s,ta,l),n(ta,ye),e(s,Dl,l),j(Hs,s,l),e(s,zl,l),j(Ws,s,l),e(s,Ol,l),e(s,pa,l),n(pa,Pe),e(s,Tl,l),j(ps,s,l),e(s,Bl,l),e(s,ea,l),n(ea,Ce),e(s,Nl,l),j(Ss,s,l),e(s,Hl,l),e(s,ra,l),n(ra,De),e(s,Wl,l),j(As,s,l),e(s,Sl,l),j(Ks,s,l),e(s,Al,l),e(s,ia,l),n(ia,ze),e(s,Kl,l),j(Us,s,l),e(s,Ul,l),e(s,oa,l),n(oa,Oe),e(s,Fl,l),j(Fs,s,l),e(s,Rl,l),j(Rs,s,l),e(s,Ml,l),e(s,ca,l),n(ca,Te),Ll=!0},p(s,[l]){const Ms={};l&2&&(Ms.$$scope={dirty:l,ctx:s}),I.$set(Ms);const gn={};l&2&&(gn.$$scope={dirty:l,ctx:s}),Y.$set(gn);const dn={};l&2&&(dn.$$scope={dirty:l,ctx:s}),J.$set(dn);const xn={};l&2&&(xn.$$scope={dirty:l,ctx:s}),X.$set(xn);const bn={};l&2&&(bn.$$scope={dirty:l,ctx:s}),ps.$set(bn)},i(s){Ll||(g(w.$$.fragment,s),g(K.$$.fragment,s),g(rs.$$.fragment,s),g(I.$$.fragment,s),g(Y.$$.fragment,s),g(os.$$.fragment,s),g(hs.$$.fragment,s),g(us.$$.fragment,s),g(ms.$$.fragment,s),g(fs.$$.fragment,s),g(js.$$.fragment,s),g(J.$$.fragment,s),g(X.$$.fragment,s),g(ds.$$.fragment,s),g(xs.$$.fragment,s),g(bs.$$.fragment,s),g(_s.$$.fragment,s),g($s.$$.fragment,s),g(ws.$$.fragment,s),g(qs.$$.fragment,s),g(vs.$$.fragment,s),g(ks.$$.fragment,s),g(Es.$$.fragment,s),g(ys.$$.fragment,s),g(Ps.$$.fragment,s),g(Cs.$$.fragment,s),g(Ds.$$.fragment,s),g(zs.$$.fragment,s),g(Os.$$.fragment,s),g(Ts.$$.fragment,s),g(Bs.$$.fragment,s),g(Ns.$$.fragment,s),g(Hs.$$.fragment,s),g(Ws.$$.fragment,s),g(ps.$$.fragment,s),g(Ss.$$.fragment,s),g(As.$$.fragment,s),g(Ks.$$.fragment,s),g(Us.$$.fragment,s),g(Fs.$$.fragment,s),g(Rs.$$.fragment,s),Ll=!0)},o(s){d(w.$$.fragment,s),d(K.$$.fragment,s),d(rs.$$.fragment,s),d(I.$$.fragment,s),d(Y.$$.fragment,s),d(os.$$.fragment,s),d(hs.$$.fragment,s),d(us.$$.fragment,s),d(ms.$$.fragment,s),d(fs.$$.fragment,s),d(js.$$.fragment,s),d(J.$$.fragment,s),d(X.$$.fragment,s),d(ds.$$.fragment,s),d(xs.$$.fragment,s),d(bs.$$.fragment,s),d(_s.$$.fragment,s),d($s.$$.fragment,s),d(ws.$$.fragment,s),d(qs.$$.fragment,s),d(vs.$$.fragment,s),d(ks.$$.fragment,s),d(Es.$$.fragment,s),d(ys.$$.fragment,s),d(Ps.$$.fragment,s),d(Cs.$$.fragment,s),d(Ds.$$.fragment,s),d(zs.$$.fragment,s),d(Os.$$.fragment,s),d(Ts.$$.fragment,s),d(Bs.$$.fragment,s),d(Ns.$$.fragment,s),d(Hs.$$.fragment,s),d(Ws.$$.fragment,s),d(ps.$$.fragment,s),d(Ss.$$.fragment,s),d(As.$$.fragment,s),d(Ks.$$.fragment,s),d(Us.$$.fragment,s),d(Fs.$$.fragment,s),d(Rs.$$.fragment,s),Ll=!1},d(s){a(u),s&&a(q),s&&a(b),x(w),s&&a(W),x(K,s),s&&a(es),s&&a(Ls),s&&a($n),x(rs,s),s&&a(wn),x(I,s),s&&a(qn),s&&a(is),s&&a(vn),x(Y,s),s&&a(kn),s&&a(U),s&&a(En),x(os,s),s&&a(yn),s&&a(Gs),s&&a(Pn),s&&a(cs),s&&a(Dn),s&&a(P),s&&a(zn),s&&a(Vs),s&&a(On),x(hs,s),s&&a(Tn),s&&a(Is),s&&a(Bn),x(us,s),s&&a(Nn),s&&a(y),s&&a(Hn),s&&a(F),s&&a(Wn),x(ms,s),s&&a(Sn),s&&a(R),s&&a(An),x(fs,s),s&&a(Kn),s&&a(M),s&&a(Un),x(js,s),s&&a(Fn),s&&a(Ys),s&&a(Rn),x(J,s),s&&a(Mn),s&&a(gs),s&&a(Ln),s&&a(C),s&&a(Gn),s&&a(Q),s&&a(Vn),s&&a(v),s&&a(In),s&&a(k),s&&a(Yn),x(X,s),s&&a(Jn),s&&a(V),x(ds),s&&a(Qn),s&&a(Js),s&&a(Xn),s&&a(Qs),s&&a(Zn),x(xs,s),s&&a(sl),s&&a(ss),s&&a(al),x(bs,s),s&&a(nl),s&&a(Xs),s&&a(ll),x(_s,s),s&&a(tl),x($s,s),s&&a(pl),s&&a(as),s&&a(el),x(ws,s),s&&a(rl),x(qs,s),s&&a(il),s&&a(ns),s&&a(ol),x(vs,s),s&&a(cl),s&&a(ls),s&&a(hl),x(ks,s),s&&a(ul),s&&a(Zs),s&&a(ml),x(Es,s),s&&a(fl),s&&a(sa),s&&a(jl),x(ys,s),s&&a(gl),x(Ps,s),s&&a(dl),s&&a(aa),s&&a(xl),x(Cs,s),s&&a(bl),x(Ds,s),s&&a(_l),s&&a(L),s&&a($l),x(zs,s),s&&a(wl),s&&a(ts),s&&a(ql),x(Os,s),s&&a(vl),s&&a(na),s&&a(kl),x(Ts,s),s&&a(El),x(Bs,s),s&&a(yl),s&&a(la),s&&a(Pl),x(Ns,s),s&&a(Cl),s&&a(ta),s&&a(Dl),x(Hs,s),s&&a(zl),x(Ws,s),s&&a(Ol),s&&a(pa),s&&a(Tl),x(ps,s),s&&a(Bl),s&&a(ea),s&&a(Nl),x(Ss,s),s&&a(Hl),s&&a(ra),s&&a(Wl),x(As,s),s&&a(Sl),x(Ks,s),s&&a(Al),s&&a(ia),s&&a(Kl),x(Us,s),s&&a(Ul),s&&a(oa),s&&a(Fl),x(Fs,s),s&&a(Rl),x(Rs,s),s&&a(Ml),s&&a(ca)}}}const bi={local:"wordpiece",sections:[{local:"",title:"\u8BAD\u7EC3\u7B97\u6CD5"},{local:"",title:"\u6807\u8BB0\u5316\u7B97\u6CD5"},{local:"wordpiece",title:"\u5B9E\u73B0 WordPiece"}],title:"WordPiece \u6807\u8BB0\u5316  "};function _i(S){return ci(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class Pi extends ti{constructor(u){super();pi(this,u,_i,xi,ei,{})}}export{Pi as default,bi as metadata};
