import{S as mr,i as ir,s as cr,e as o,k as i,w as f,t as l,U as fe,M as ur,c as r,d as a,m as c,a as m,x as d,h as p,V as de,b as T,G as t,g as e,y as _,q as g,o as j,B as $,v as hr}from"../../chunks/vendor-hf-doc-builder.js";import{T as jn}from"../../chunks/Tip-hf-doc-builder.js";import{Y as fr}from"../../chunks/Youtube-hf-doc-builder.js";import{I as er}from"../../chunks/IconCopyLink-hf-doc-builder.js";import{C as w}from"../../chunks/CodeBlock-hf-doc-builder.js";import{C as dr}from"../../chunks/CourseFloatingBanner-hf-doc-builder.js";import"../../chunks/DocNotebookDropdown-hf-doc-builder.js";function _r(z){let u,q;return{c(){u=o("p"),q=l("\u{1F4A1} \u672C\u8282\u6DF1\u5165\u4ECB\u7ECD\u4E86 Unigram,\u751A\u81F3\u5C55\u793A\u4E86\u4E00\u4E2A\u5B8C\u6574\u7684\u5B9E\u73B0\u3002\u5982\u679C\u4F60\u53EA\u60F3\u5927\u81F4\u4E86\u89E3\u6807\u8BB0\u5316\u7B97\u6CD5,\u53EF\u4EE5\u8DF3\u5230\u6700\u540E\u3002")},l(h){u=r(h,"P",{});var b=m(u);q=p(b,"\u{1F4A1} \u672C\u8282\u6DF1\u5165\u4ECB\u7ECD\u4E86 Unigram,\u751A\u81F3\u5C55\u793A\u4E86\u4E00\u4E2A\u5B8C\u6574\u7684\u5B9E\u73B0\u3002\u5982\u679C\u4F60\u53EA\u60F3\u5927\u81F4\u4E86\u89E3\u6807\u8BB0\u5316\u7B97\u6CD5,\u53EF\u4EE5\u8DF3\u5230\u6700\u540E\u3002"),b.forEach(a)},m(h,b){e(h,u,b),t(u,q)},d(h){h&&a(u)}}}function gr(z){let u,q,h,b,P;return{c(){u=o("p"),q=l("\u270F\uFE0F "),h=o("strong"),b=l("\u73B0\u5728\u8F6E\u5230\u4F60\u4E86!"),P=l(" \u7F16\u5199\u4EE3\u7801\u6765\u8BA1\u7B97\u4E0A\u9762\u7684\u9891\u7387,\u5E76\u4ED4\u7EC6\u68C0\u67E5\u663E\u793A\u7684\u7ED3\u679C\u4EE5\u53CA\u603B\u548C\u662F\u5426\u6B63\u786E\u3002")},l(k){u=r(k,"P",{});var E=m(u);q=p(E,"\u270F\uFE0F "),h=r(E,"STRONG",{});var O=m(h);b=p(O,"\u73B0\u5728\u8F6E\u5230\u4F60\u4E86!"),O.forEach(a),P=p(E," \u7F16\u5199\u4EE3\u7801\u6765\u8BA1\u7B97\u4E0A\u9762\u7684\u9891\u7387,\u5E76\u4ED4\u7EC6\u68C0\u67E5\u663E\u793A\u7684\u7ED3\u679C\u4EE5\u53CA\u603B\u548C\u662F\u5426\u6B63\u786E\u3002"),E.forEach(a)},m(k,E){e(k,u,E),t(u,q),t(u,h),t(h,b),t(u,P)},d(k){k&&a(u)}}}function jr(z){let u,q,h,b,P,k,E,O;return{c(){u=o("p"),q=l("\u270F\uFE0F "),h=o("strong"),b=l("\u73B0\u5728\u8F6E\u5230\u4F60\u4E86!"),P=l(" \u786E\u5B9A\u5355\u8BCD "),k=o("code"),E=l('"huggun"'),O=l(" \u7684\u6807\u8BB0\u5316\u53CA\u5176\u5206\u6570\u3002")},l(R){u=r(R,"P",{});var C=m(u);q=p(C,"\u270F\uFE0F "),h=r(C,"STRONG",{});var B=m(h);b=p(B,"\u73B0\u5728\u8F6E\u5230\u4F60\u4E86!"),B.forEach(a),P=p(C," \u786E\u5B9A\u5355\u8BCD "),k=r(C,"CODE",{});var is=m(k);E=p(is,'"huggun"'),is.forEach(a),O=p(C," \u7684\u6807\u8BB0\u5316\u53CA\u5176\u5206\u6570\u3002"),C.forEach(a)},m(R,C){e(R,u,C),t(u,q),t(u,h),t(h,b),t(u,P),t(u,k),t(k,E),t(u,O)},d(R){R&&a(u)}}}function $r(z){let u,q;return{c(){u=o("p"),q=l("\u{1F4A1} SentencePiece \u4F7F\u7528\u4E00\u79CD\u79F0\u4E3A\u589E\u5F3A\u540E\u7F00\u6570\u7EC4(ESA)\u7684\u66F4\u9AD8\u6548\u7B97\u6CD5\u6765\u521B\u5EFA\u521D\u59CB\u8BCD\u6C47\u8868\u3002")},l(h){u=r(h,"P",{});var b=m(u);q=p(b,"\u{1F4A1} SentencePiece \u4F7F\u7528\u4E00\u79CD\u79F0\u4E3A\u589E\u5F3A\u540E\u7F00\u6570\u7EC4(ESA)\u7684\u66F4\u9AD8\u6548\u7B97\u6CD5\u6765\u521B\u5EFA\u521D\u59CB\u8BCD\u6C47\u8868\u3002"),b.forEach(a)},m(h,b){e(h,u,b),t(u,q)},d(h){h&&a(u)}}}function br(z){let u,q;return{c(){u=o("p"),q=l("\u{1F4A1} \u8FD9\u79CD\u65B9\u6CD5\u975E\u5E38\u4F4E\u6548,\u56E0\u6B64 SentencePiece \u4F7F\u7528\u4E86\u6CA1\u6709\u6807\u8BB0 X \u7684\u6A21\u578B\u635F\u5931\u7684\u8FD1\u4F3C\u503C:\u5B83\u4E0D\u662F\u4ECE\u5934\u5F00\u59CB,\u800C\u662F\u901A\u8FC7\u5176\u5728\u5269\u4F59\u8BCD\u6C47\u8868\u4E2D\u7684\u5206\u6BB5\u66FF\u6362\u6807\u8BB0 X\u3002\u8FD9\u6837,\u6240\u6709\u5206\u6570\u53EF\u4EE5\u4E0E\u6A21\u578B\u635F\u5931\u540C\u65F6\u8BA1\u7B97\u3002")},l(h){u=r(h,"P",{});var b=m(u);q=p(b,"\u{1F4A1} \u8FD9\u79CD\u65B9\u6CD5\u975E\u5E38\u4F4E\u6548,\u56E0\u6B64 SentencePiece \u4F7F\u7528\u4E86\u6CA1\u6709\u6807\u8BB0 X \u7684\u6A21\u578B\u635F\u5931\u7684\u8FD1\u4F3C\u503C:\u5B83\u4E0D\u662F\u4ECE\u5934\u5F00\u59CB,\u800C\u662F\u901A\u8FC7\u5176\u5728\u5269\u4F59\u8BCD\u6C47\u8868\u4E2D\u7684\u5206\u6BB5\u66FF\u6362\u6807\u8BB0 X\u3002\u8FD9\u6837,\u6240\u6709\u5206\u6570\u53EF\u4EE5\u4E0E\u6A21\u578B\u635F\u5931\u540C\u65F6\u8BA1\u7B97\u3002"),b.forEach(a)},m(h,b){e(h,u,b),t(u,q)},d(h){h&&a(u)}}}function wr(z){let u,q,h,b,P,k,E,O,R,C,B,is,Gs,_e,$n,cs,bn,G,wn,us,ge,qn,Is,je,kn,Ys,$e,yn,A,be,vn,lr='<span class="katex"><span class="katex-mathml"><math xmlns="http://www.w3.org/1998/Math/MathML"><semantics><mrow><mi>p</mi></mrow><annotation encoding="application/x-tex">p</annotation></semantics></math></span><span class="katex-html" aria-hidden="true"><span class="base"><span class="strut" style="height:0.625em;vertical-align:-0.1944em;"></span><span class="mord mathnormal">p</span></span></span></span>',En,xn,pr='<span class="katex"><span class="katex-mathml"><math xmlns="http://www.w3.org/1998/Math/MathML"><semantics><mrow><mi>p</mi></mrow><annotation encoding="application/x-tex">p</annotation></semantics></math></span><span class="katex-html" aria-hidden="true"><span class="base"><span class="strut" style="height:0.625em;vertical-align:-0.1944em;"></span><span class="mord mathnormal">p</span></span></span></span>',Pn,Cn,Zs,we,Tn,Js,qe,zn,Ks,ke,On,hs,Dn,Qs,ye,Un,fs,Nn,ds,ve,Hn,sa,Ee,Sn,x,xe,Ea,Pe,Ce,xa,Te,ze,Pa,Oe,De,Ca,Ue,Ne,Mn,aa,He,Bn,_s,An,I,Se,Ta,Me,Be,Ln,Y,Wn,M,Ae,za,Le,We,Oa,Xe,Re,Xn,or='<span class="katex-display"><span class="katex"><span class="katex-mathml"><math xmlns="http://www.w3.org/1998/Math/MathML" display="block"><semantics><mrow><mi>P</mi><mo stretchy="false">(</mo><mo stretchy="false">[</mo><mi mathvariant="normal">\u2018</mi><mi mathvariant="normal">\u2018</mi><mi>p</mi><mi mathvariant="normal">&quot;</mi><mo separator="true">,</mo><mi mathvariant="normal">\u2018</mi><mi mathvariant="normal">\u2018</mi><mi>u</mi><mi mathvariant="normal">&quot;</mi><mo separator="true">,</mo><mi mathvariant="normal">\u2018</mi><mi mathvariant="normal">\u2018</mi><mi>g</mi><mi mathvariant="normal">&quot;</mi><mo stretchy="false">]</mo><mo stretchy="false">)</mo><mo>=</mo><mi>P</mi><mo stretchy="false">(</mo><mi mathvariant="normal">\u2018</mi><mi mathvariant="normal">\u2018</mi><mi>p</mi><mi mathvariant="normal">&quot;</mi><mo stretchy="false">)</mo><mo>\xD7</mo><mi>P</mi><mo stretchy="false">(</mo><mi mathvariant="normal">\u2018</mi><mi mathvariant="normal">\u2018</mi><mi>u</mi><mi mathvariant="normal">&quot;</mi><mo stretchy="false">)</mo><mo>\xD7</mo><mi>P</mi><mo stretchy="false">(</mo><mi mathvariant="normal">\u2018</mi><mi mathvariant="normal">\u2018</mi><mi>g</mi><mi mathvariant="normal">&quot;</mi><mo stretchy="false">)</mo><mo>=</mo><mfrac><mn>5</mn><mn>210</mn></mfrac><mo>\xD7</mo><mfrac><mn>36</mn><mn>210</mn></mfrac><mo>\xD7</mo><mfrac><mn>20</mn><mn>210</mn></mfrac><mo>=</mo><mn>0.000389</mn></mrow><annotation encoding="application/x-tex">P([``p&quot;, ``u&quot;, ``g&quot;]) = P(``p&quot;) \\times P(``u&quot;) \\times P(``g&quot;) = \\frac{5}{210} \\times \\frac{36}{210} \\times \\frac{20}{210} = 0.000389</annotation></semantics></math></span><span class="katex-html" aria-hidden="true"><span class="base"><span class="strut" style="height:1em;vertical-align:-0.25em;"></span><span class="mord mathnormal" style="margin-right:0.13889em;">P</span><span class="mopen">([</span><span class="mord">\u2018\u2018</span><span class="mord mathnormal">p</span><span class="mord">&quot;</span><span class="mpunct">,</span><span class="mspace" style="margin-right:0.1667em;"></span><span class="mord">\u2018\u2018</span><span class="mord mathnormal">u</span><span class="mord">&quot;</span><span class="mpunct">,</span><span class="mspace" style="margin-right:0.1667em;"></span><span class="mord">\u2018\u2018</span><span class="mord mathnormal" style="margin-right:0.03588em;">g</span><span class="mord">&quot;</span><span class="mclose">])</span><span class="mspace" style="margin-right:0.2778em;"></span><span class="mrel">=</span><span class="mspace" style="margin-right:0.2778em;"></span></span><span class="base"><span class="strut" style="height:1em;vertical-align:-0.25em;"></span><span class="mord mathnormal" style="margin-right:0.13889em;">P</span><span class="mopen">(</span><span class="mord">\u2018\u2018</span><span class="mord mathnormal">p</span><span class="mord">&quot;</span><span class="mclose">)</span><span class="mspace" style="margin-right:0.2222em;"></span><span class="mbin">\xD7</span><span class="mspace" style="margin-right:0.2222em;"></span></span><span class="base"><span class="strut" style="height:1em;vertical-align:-0.25em;"></span><span class="mord mathnormal" style="margin-right:0.13889em;">P</span><span class="mopen">(</span><span class="mord">\u2018\u2018</span><span class="mord mathnormal">u</span><span class="mord">&quot;</span><span class="mclose">)</span><span class="mspace" style="margin-right:0.2222em;"></span><span class="mbin">\xD7</span><span class="mspace" style="margin-right:0.2222em;"></span></span><span class="base"><span class="strut" style="height:1em;vertical-align:-0.25em;"></span><span class="mord mathnormal" style="margin-right:0.13889em;">P</span><span class="mopen">(</span><span class="mord">\u2018\u2018</span><span class="mord mathnormal" style="margin-right:0.03588em;">g</span><span class="mord">&quot;</span><span class="mclose">)</span><span class="mspace" style="margin-right:0.2778em;"></span><span class="mrel">=</span><span class="mspace" style="margin-right:0.2778em;"></span></span><span class="base"><span class="strut" style="height:2.0074em;vertical-align:-0.686em;"></span><span class="mord"><span class="mopen nulldelimiter"></span><span class="mfrac"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:1.3214em;"><span style="top:-2.314em;"><span class="pstrut" style="height:3em;"></span><span class="mord"><span class="mord">210</span></span></span><span style="top:-3.23em;"><span class="pstrut" style="height:3em;"></span><span class="frac-line" style="border-bottom-width:0.04em;"></span></span><span style="top:-3.677em;"><span class="pstrut" style="height:3em;"></span><span class="mord"><span class="mord">5</span></span></span></span><span class="vlist-s">\u200B</span></span><span class="vlist-r"><span class="vlist" style="height:0.686em;"><span></span></span></span></span></span><span class="mclose nulldelimiter"></span></span><span class="mspace" style="margin-right:0.2222em;"></span><span class="mbin">\xD7</span><span class="mspace" style="margin-right:0.2222em;"></span></span><span class="base"><span class="strut" style="height:2.0074em;vertical-align:-0.686em;"></span><span class="mord"><span class="mopen nulldelimiter"></span><span class="mfrac"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:1.3214em;"><span style="top:-2.314em;"><span class="pstrut" style="height:3em;"></span><span class="mord"><span class="mord">210</span></span></span><span style="top:-3.23em;"><span class="pstrut" style="height:3em;"></span><span class="frac-line" style="border-bottom-width:0.04em;"></span></span><span style="top:-3.677em;"><span class="pstrut" style="height:3em;"></span><span class="mord"><span class="mord">36</span></span></span></span><span class="vlist-s">\u200B</span></span><span class="vlist-r"><span class="vlist" style="height:0.686em;"><span></span></span></span></span></span><span class="mclose nulldelimiter"></span></span><span class="mspace" style="margin-right:0.2222em;"></span><span class="mbin">\xD7</span><span class="mspace" style="margin-right:0.2222em;"></span></span><span class="base"><span class="strut" style="height:2.0074em;vertical-align:-0.686em;"></span><span class="mord"><span class="mopen nulldelimiter"></span><span class="mfrac"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:1.3214em;"><span style="top:-2.314em;"><span class="pstrut" style="height:3em;"></span><span class="mord"><span class="mord">210</span></span></span><span style="top:-3.23em;"><span class="pstrut" style="height:3em;"></span><span class="frac-line" style="border-bottom-width:0.04em;"></span></span><span style="top:-3.677em;"><span class="pstrut" style="height:3em;"></span><span class="mord"><span class="mord">20</span></span></span></span><span class="vlist-s">\u200B</span></span><span class="vlist-r"><span class="vlist" style="height:0.686em;"><span></span></span></span></span></span><span class="mclose nulldelimiter"></span></span><span class="mspace" style="margin-right:0.2778em;"></span><span class="mrel">=</span><span class="mspace" style="margin-right:0.2778em;"></span></span><span class="base"><span class="strut" style="height:0.6444em;"></span><span class="mord">0.000389</span></span></span></span></span>',Rn,F,Fe,Da,Ve,Ge,Fn,rr='<span class="katex-display"><span class="katex"><span class="katex-mathml"><math xmlns="http://www.w3.org/1998/Math/MathML" display="block"><semantics><mrow><mi>P</mi><mo stretchy="false">(</mo><mo stretchy="false">[</mo><mi mathvariant="normal">\u2018</mi><mi mathvariant="normal">\u2018</mi><mi>p</mi><mi>u</mi><mi mathvariant="normal">&quot;</mi><mo separator="true">,</mo><mi mathvariant="normal">\u2018</mi><mi mathvariant="normal">\u2018</mi><mi>g</mi><mi mathvariant="normal">&quot;</mi><mo stretchy="false">]</mo><mo stretchy="false">)</mo><mo>=</mo><mi>P</mi><mo stretchy="false">(</mo><mi mathvariant="normal">\u2018</mi><mi mathvariant="normal">\u2018</mi><mi>p</mi><mi>u</mi><mi mathvariant="normal">&quot;</mi><mo stretchy="false">)</mo><mo>\xD7</mo><mi>P</mi><mo stretchy="false">(</mo><mi mathvariant="normal">\u2018</mi><mi mathvariant="normal">\u2018</mi><mi>g</mi><mi mathvariant="normal">&quot;</mi><mo stretchy="false">)</mo><mo>=</mo><mfrac><mn>5</mn><mn>210</mn></mfrac><mo>\xD7</mo><mfrac><mn>20</mn><mn>210</mn></mfrac><mo>=</mo><mn>0.0022676</mn></mrow><annotation encoding="application/x-tex">P([``pu&quot;, ``g&quot;]) = P(``pu&quot;) \\times P(``g&quot;) = \\frac{5}{210} \\times \\frac{20}{210} = 0.0022676</annotation></semantics></math></span><span class="katex-html" aria-hidden="true"><span class="base"><span class="strut" style="height:1em;vertical-align:-0.25em;"></span><span class="mord mathnormal" style="margin-right:0.13889em;">P</span><span class="mopen">([</span><span class="mord">\u2018\u2018</span><span class="mord mathnormal">p</span><span class="mord mathnormal">u</span><span class="mord">&quot;</span><span class="mpunct">,</span><span class="mspace" style="margin-right:0.1667em;"></span><span class="mord">\u2018\u2018</span><span class="mord mathnormal" style="margin-right:0.03588em;">g</span><span class="mord">&quot;</span><span class="mclose">])</span><span class="mspace" style="margin-right:0.2778em;"></span><span class="mrel">=</span><span class="mspace" style="margin-right:0.2778em;"></span></span><span class="base"><span class="strut" style="height:1em;vertical-align:-0.25em;"></span><span class="mord mathnormal" style="margin-right:0.13889em;">P</span><span class="mopen">(</span><span class="mord">\u2018\u2018</span><span class="mord mathnormal">p</span><span class="mord mathnormal">u</span><span class="mord">&quot;</span><span class="mclose">)</span><span class="mspace" style="margin-right:0.2222em;"></span><span class="mbin">\xD7</span><span class="mspace" style="margin-right:0.2222em;"></span></span><span class="base"><span class="strut" style="height:1em;vertical-align:-0.25em;"></span><span class="mord mathnormal" style="margin-right:0.13889em;">P</span><span class="mopen">(</span><span class="mord">\u2018\u2018</span><span class="mord mathnormal" style="margin-right:0.03588em;">g</span><span class="mord">&quot;</span><span class="mclose">)</span><span class="mspace" style="margin-right:0.2778em;"></span><span class="mrel">=</span><span class="mspace" style="margin-right:0.2778em;"></span></span><span class="base"><span class="strut" style="height:2.0074em;vertical-align:-0.686em;"></span><span class="mord"><span class="mopen nulldelimiter"></span><span class="mfrac"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:1.3214em;"><span style="top:-2.314em;"><span class="pstrut" style="height:3em;"></span><span class="mord"><span class="mord">210</span></span></span><span style="top:-3.23em;"><span class="pstrut" style="height:3em;"></span><span class="frac-line" style="border-bottom-width:0.04em;"></span></span><span style="top:-3.677em;"><span class="pstrut" style="height:3em;"></span><span class="mord"><span class="mord">5</span></span></span></span><span class="vlist-s">\u200B</span></span><span class="vlist-r"><span class="vlist" style="height:0.686em;"><span></span></span></span></span></span><span class="mclose nulldelimiter"></span></span><span class="mspace" style="margin-right:0.2222em;"></span><span class="mbin">\xD7</span><span class="mspace" style="margin-right:0.2222em;"></span></span><span class="base"><span class="strut" style="height:2.0074em;vertical-align:-0.686em;"></span><span class="mord"><span class="mopen nulldelimiter"></span><span class="mfrac"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:1.3214em;"><span style="top:-2.314em;"><span class="pstrut" style="height:3em;"></span><span class="mord"><span class="mord">210</span></span></span><span style="top:-3.23em;"><span class="pstrut" style="height:3em;"></span><span class="frac-line" style="border-bottom-width:0.04em;"></span></span><span style="top:-3.677em;"><span class="pstrut" style="height:3em;"></span><span class="mord"><span class="mord">20</span></span></span></span><span class="vlist-s">\u200B</span></span><span class="vlist-r"><span class="vlist" style="height:0.686em;"><span></span></span></span></span></span><span class="mclose nulldelimiter"></span></span><span class="mspace" style="margin-right:0.2778em;"></span><span class="mrel">=</span><span class="mspace" style="margin-right:0.2778em;"></span></span><span class="base"><span class="strut" style="height:0.6444em;"></span><span class="mord">0.0022676</span></span></span></span></span>',Vn,na,Ie,Gn,Z,Ye,Ua,Ze,Je,In,gs,Yn,D,Ke,Na,Qe,sl,Ha,al,nl,Sa,tl,el,Zn,y,ll,Ma,pl,ol,Ba,rl,ml,Aa,il,cl,La,ul,hl,Wa,fl,dl,Jn,ta,_l,Kn,J,gl,Xa,jl,$l,Qn,js,st,L,bl,Ra,wl,ql,Fa,kl,yl,at,K,nt,$s,vl,tt,ea,El,et,Q,xl,Va,Pl,Cl,lt,la,Tl,pt,bs,ot,pa,zl,rt,ws,mt,oa,Ol,it,qs,ct,U,Dl,Ga,Ul,Nl,Ia,Hl,Sl,Ya,Ml,Bl,ut,N,Al,Za,Ll,Wl,Ja,Xl,Rl,Ka,Fl,Vl,ht,ks,ft,ra,Gl,dt,ys,_t,W,Il,Qa,Yl,Zl,sn,Jl,Kl,gt,V,ss,an,vs,Ql,nn,sp,jt,ma,ap,$t,ia,np,bt,Es,wt,as,tp,tn,ep,lp,qt,xs,kt,ca,pp,yt,Ps,vt,ua,op,Et,Cs,xt,Ts,Pt,ha,rp,Ct,zs,Tt,ns,zt,fa,mp,Ot,Os,Dt,ts,ip,en,cp,up,Ut,es,hp,ln,fp,dp,Nt,da,_p,Ht,Ds,St,_a,gp,Mt,Us,Bt,Ns,At,ga,jp,Lt,Hs,Wt,ja,$p,Xt,Ss,Rt,Ms,Ft,$a,bp,Vt,Bs,Gt,ba,wp,It,As,Yt,v,qp,pn,kp,yp,on,vp,Ep,rn,xp,Pp,mn,Cp,Tp,cn,zp,Op,Zt,Ls,Jt,ls,Kt,wa,Dp,Qt,Ws,se,ps,Up,un,Np,Hp,ae,Xs,ne,Rs,te,qa,Sp,ee;return k=new er({}),B=new dr({props:{chapter:6,classNames:"absolute z-10 right-0 top-0",notebooks:[{label:"Google Colab",value:"https://colab.research.google.com/github/huggingface/notebooks/blob/master/course/zh-CN/chapter6/section7.ipynb"},{label:"Aws Studio",value:"https://studiolab.sagemaker.aws/import/github/huggingface/notebooks/blob/master/course/zh-CN/chapter6/section7.ipynb"}]}}),cs=new fr({props:{id:"TGZfZVuF9Yc"}}),G=new jn({props:{$$slots:{default:[_r]},$$scope:{ctx:z}}}),hs=new w({props:{code:'("hug", 10), ("pug", 5), ("pun", 12), ("bun", 4), ("hugs", 5)',highlighted:'(<span class="hljs-string">&quot;hug&quot;</span><span class="hljs-punctuation">,</span> <span class="hljs-number">10</span>)<span class="hljs-punctuation">,</span> (<span class="hljs-string">&quot;pug&quot;</span><span class="hljs-punctuation">,</span> <span class="hljs-number">5</span>)<span class="hljs-punctuation">,</span> (<span class="hljs-string">&quot;pun&quot;</span><span class="hljs-punctuation">,</span> <span class="hljs-number">12</span>)<span class="hljs-punctuation">,</span> (<span class="hljs-string">&quot;bun&quot;</span><span class="hljs-punctuation">,</span> <span class="hljs-number">4</span>)<span class="hljs-punctuation">,</span> (<span class="hljs-string">&quot;hugs&quot;</span><span class="hljs-punctuation">,</span> <span class="hljs-number">5</span>)'}}),fs=new w({props:{code:'["h", "u", "g", "hu", "ug", "p", "pu", "n", "un", "b", "bu", "s", "hug", "gs", "ugs"]',highlighted:'<span class="hljs-selector-attr">[<span class="hljs-string">&quot;h&quot;</span>, <span class="hljs-string">&quot;u&quot;</span>, <span class="hljs-string">&quot;g&quot;</span>, <span class="hljs-string">&quot;hu&quot;</span>, <span class="hljs-string">&quot;ug&quot;</span>, <span class="hljs-string">&quot;p&quot;</span>, <span class="hljs-string">&quot;pu&quot;</span>, <span class="hljs-string">&quot;n&quot;</span>, <span class="hljs-string">&quot;un&quot;</span>, <span class="hljs-string">&quot;b&quot;</span>, <span class="hljs-string">&quot;bu&quot;</span>, <span class="hljs-string">&quot;s&quot;</span>, <span class="hljs-string">&quot;hug&quot;</span>, <span class="hljs-string">&quot;gs&quot;</span>, <span class="hljs-string">&quot;ugs&quot;</span>]</span>'}}),_s=new w({props:{code:`("h", 15) ("u", 36) ("g", 20) ("hu", 15) ("ug", 20) ("p", 17) ("pu", 17) ("n", 16)
("un", 16) ("b", 4) ("bu", 4) ("s", 5) ("hug", 15) ("gs", 5) ("ugs", 5)`,highlighted:`(<span class="hljs-string">&quot;h&quot;</span><span class="hljs-punctuation">,</span> <span class="hljs-number">15</span>) (<span class="hljs-string">&quot;u&quot;</span><span class="hljs-punctuation">,</span> <span class="hljs-number">36</span>) (<span class="hljs-string">&quot;g&quot;</span><span class="hljs-punctuation">,</span> <span class="hljs-number">20</span>) (<span class="hljs-string">&quot;hu&quot;</span><span class="hljs-punctuation">,</span> <span class="hljs-number">15</span>) (<span class="hljs-string">&quot;ug&quot;</span><span class="hljs-punctuation">,</span> <span class="hljs-number">20</span>) (<span class="hljs-string">&quot;p&quot;</span><span class="hljs-punctuation">,</span> <span class="hljs-number">17</span>) (<span class="hljs-string">&quot;pu&quot;</span><span class="hljs-punctuation">,</span> <span class="hljs-number">17</span>) (<span class="hljs-string">&quot;n&quot;</span><span class="hljs-punctuation">,</span> <span class="hljs-number">16</span>)
(<span class="hljs-string">&quot;un&quot;</span><span class="hljs-punctuation">,</span> <span class="hljs-number">16</span>) (<span class="hljs-string">&quot;b&quot;</span><span class="hljs-punctuation">,</span> <span class="hljs-number">4</span>) (<span class="hljs-string">&quot;bu&quot;</span><span class="hljs-punctuation">,</span> <span class="hljs-number">4</span>) (<span class="hljs-string">&quot;s&quot;</span><span class="hljs-punctuation">,</span> <span class="hljs-number">5</span>) (<span class="hljs-string">&quot;hug&quot;</span><span class="hljs-punctuation">,</span> <span class="hljs-number">15</span>) (<span class="hljs-string">&quot;gs&quot;</span><span class="hljs-punctuation">,</span> <span class="hljs-number">5</span>) (<span class="hljs-string">&quot;ugs&quot;</span><span class="hljs-punctuation">,</span> <span class="hljs-number">5</span>)`}}),Y=new jn({props:{$$slots:{default:[gr]},$$scope:{ctx:z}}}),gs=new w({props:{code:`["p", "u", "g"] : 0.000389
["p", "ug"] : 0.0022676
["pu", "g"] : 0.0022676`,highlighted:`[<span class="hljs-string">&quot;p&quot;</span>, <span class="hljs-string">&quot;u&quot;</span>, <span class="hljs-string">&quot;g&quot;</span>] : 0.000389
[<span class="hljs-string">&quot;p&quot;</span>, <span class="hljs-string">&quot;ug&quot;</span>] : 0.0022676
[<span class="hljs-string">&quot;pu&quot;</span>, <span class="hljs-string">&quot;g&quot;</span>] : 0.0022676`}}),js=new w({props:{code:`Character 0 (u): "u" (score 0.171429)
Character 1 (n): "un" (score 0.076191)
Character 2 (h): "un" "h" (score 0.005442)
Character 3 (u): "un" "hu" (score 0.005442)
Character 4 (g): "un" "hug" (score 0.005442)`,highlighted:`<span class="hljs-attribute">Character</span> <span class="hljs-number">0</span> (u): <span class="hljs-string">&quot;u&quot;</span> (score <span class="hljs-number">0</span>.<span class="hljs-number">171429</span>)
<span class="hljs-attribute">Character</span> <span class="hljs-number">1</span> (n): <span class="hljs-string">&quot;un&quot;</span> (score <span class="hljs-number">0</span>.<span class="hljs-number">076191</span>)
<span class="hljs-attribute">Character</span> <span class="hljs-number">2</span> (h): <span class="hljs-string">&quot;un&quot;</span> <span class="hljs-string">&quot;h&quot;</span> (score <span class="hljs-number">0</span>.<span class="hljs-number">005442</span>)
<span class="hljs-attribute">Character</span> <span class="hljs-number">3</span> (u): <span class="hljs-string">&quot;un&quot;</span> <span class="hljs-string">&quot;hu&quot;</span> (score <span class="hljs-number">0</span>.<span class="hljs-number">005442</span>)
<span class="hljs-attribute">Character</span> <span class="hljs-number">4</span> (g): <span class="hljs-string">&quot;un&quot;</span> <span class="hljs-string">&quot;hug&quot;</span> (score <span class="hljs-number">0</span>.<span class="hljs-number">005442</span>)`}}),K=new jn({props:{$$slots:{default:[jr]},$$scope:{ctx:z}}}),bs=new w({props:{code:'("hug", 10), ("pug", 5), ("pun", 12), ("bun", 4), ("hugs", 5)',highlighted:'(<span class="hljs-string">&quot;hug&quot;</span><span class="hljs-punctuation">,</span> <span class="hljs-number">10</span>)<span class="hljs-punctuation">,</span> (<span class="hljs-string">&quot;pug&quot;</span><span class="hljs-punctuation">,</span> <span class="hljs-number">5</span>)<span class="hljs-punctuation">,</span> (<span class="hljs-string">&quot;pun&quot;</span><span class="hljs-punctuation">,</span> <span class="hljs-number">12</span>)<span class="hljs-punctuation">,</span> (<span class="hljs-string">&quot;bun&quot;</span><span class="hljs-punctuation">,</span> <span class="hljs-number">4</span>)<span class="hljs-punctuation">,</span> (<span class="hljs-string">&quot;hugs&quot;</span><span class="hljs-punctuation">,</span> <span class="hljs-number">5</span>)'}}),ws=new w({props:{code:`"hug": ["hug"] (score 0.071428)
"pug": ["pu", "g"] (score 0.007710)
"pun": ["pu", "n"] (score 0.006168)
"bun": ["bu", "n"] (score 0.001451)
"hugs": ["hug", "s"] (score 0.001701)`,highlighted:`<span class="hljs-string">&quot;hug&quot;</span>: [<span class="hljs-string">&quot;hug&quot;</span>] <span class="hljs-comment">(score 0.071428)</span>
<span class="hljs-string">&quot;pug&quot;</span>: [<span class="hljs-string">&quot;pu&quot;</span>, <span class="hljs-string">&quot;g&quot;</span>] <span class="hljs-comment">(score 0.007710)</span>
<span class="hljs-string">&quot;pun&quot;</span>: [<span class="hljs-string">&quot;pu&quot;</span>, <span class="hljs-string">&quot;n&quot;</span>] <span class="hljs-comment">(score 0.006168)</span>
<span class="hljs-string">&quot;bun&quot;</span>: [<span class="hljs-string">&quot;bu&quot;</span>, <span class="hljs-string">&quot;n&quot;</span>] <span class="hljs-comment">(score 0.001451)</span>
<span class="hljs-string">&quot;hugs&quot;</span>: [<span class="hljs-string">&quot;hug&quot;</span>, <span class="hljs-string">&quot;s&quot;</span>] <span class="hljs-comment">(score 0.001701)</span>`}}),qs=new w({props:{code:"10 * (-log(0.071428)) + 5 * (-log(0.007710)) + 12 * (-log(0.006168)) + 4 * (-log(0.001451)) + 5 * (-log(0.001701)) = 169.8",highlighted:'<span class="hljs-attribute">10</span> * (-log(<span class="hljs-number">0</span>.<span class="hljs-number">071428</span>)) + <span class="hljs-number">5</span> * (-log(<span class="hljs-number">0</span>.<span class="hljs-number">007710</span>)) + <span class="hljs-number">12</span> * (-log(<span class="hljs-number">0</span>.<span class="hljs-number">006168</span>)) + <span class="hljs-number">4</span> * (-log(<span class="hljs-number">0</span>.<span class="hljs-number">001451</span>)) + <span class="hljs-number">5</span> * (-log(<span class="hljs-number">0</span>.<span class="hljs-number">001701</span>)) = <span class="hljs-number">169</span>.<span class="hljs-number">8</span>'}}),ks=new w({props:{code:`"hug": ["hu", "g"] (score 0.006802)
"hugs": ["hu", "gs"] (score 0.001701)`,highlighted:`<span class="hljs-string">&quot;hug&quot;</span>: [<span class="hljs-string">&quot;hu&quot;</span>, <span class="hljs-string">&quot;g&quot;</span>] <span class="hljs-comment">(score 0.006802)</span>
<span class="hljs-string">&quot;hugs&quot;</span>: [<span class="hljs-string">&quot;hu&quot;</span>, <span class="hljs-string">&quot;gs&quot;</span>] <span class="hljs-comment">(score 0.001701)</span>`}}),ys=new w({props:{code:"- 10 * (-log(0.071428)) + 10 * (-log(0.006802)) = 23.5",highlighted:'- <span class="hljs-number">10</span> * (<span class="hljs-name">-log</span>(<span class="hljs-number">0.071428</span>)) + <span class="hljs-number">10</span> * (<span class="hljs-name">-log</span>(<span class="hljs-number">0.006802</span>)) = <span class="hljs-number">23.5</span>'}}),vs=new er({}),Es=new w({props:{code:`corpus = [
    "This is the Hugging Face course.",
    "This chapter is about tokenization.",
    "This section shows several tokenizer algorithms.",
    "Hopefully, you will be able to understand how they are trained and generate tokens.",
]`,highlighted:`corpus = [
    <span class="hljs-string">&quot;This is the Hugging Face course.&quot;</span>,
    <span class="hljs-string">&quot;This chapter is about tokenization.&quot;</span>,
    <span class="hljs-string">&quot;This section shows several tokenizer algorithms.&quot;</span>,
    <span class="hljs-string">&quot;Hopefully, you will be able to understand how they are trained and generate tokens.&quot;</span>,
]`}}),xs=new w({props:{code:`from transformers import AutoTokenizer

tokenizer = AutoTokenizer.from_pretrained("xlnet-base-cased")`,highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoTokenizer

tokenizer = AutoTokenizer.from_pretrained(<span class="hljs-string">&quot;xlnet-base-cased&quot;</span>)`}}),Ps=new w({props:{code:`from collections import defaultdict

word_freqs = defaultdict(int)
for text in corpus:
    words_with_offsets = tokenizer.backend_tokenizer.pre_tokenizer.pre_tokenize_str(text)
    new_words = [word for word, offset in words_with_offsets]
    for word in new_words:
        word_freqs[word] += 1

word_freqs`,highlighted:`<span class="hljs-keyword">from</span> collections <span class="hljs-keyword">import</span> defaultdict

word_freqs = defaultdict(<span class="hljs-built_in">int</span>)
<span class="hljs-keyword">for</span> text <span class="hljs-keyword">in</span> corpus:
    words_with_offsets = tokenizer.backend_tokenizer.pre_tokenizer.pre_tokenize_str(text)
    new_words = [word <span class="hljs-keyword">for</span> word, offset <span class="hljs-keyword">in</span> words_with_offsets]
    <span class="hljs-keyword">for</span> word <span class="hljs-keyword">in</span> new_words:
        word_freqs[word] += <span class="hljs-number">1</span>

word_freqs`}}),Cs=new w({props:{code:`char_freqs = defaultdict(int)
subwords_freqs = defaultdict(int)
for word, freq in word_freqs.items():
    for i in range(len(word)):
        char_freqs[word[i]] += freq
        # Loop through the subwords of length at least 2
        for j in range(i + 2, len(word) + 1):
            subwords_freqs[word[i:j]] += freq

# Sort subwords by frequency
sorted_subwords = sorted(subwords_freqs.items(), key=lambda x: x[1], reverse=True)
sorted_subwords[:10]`,highlighted:`char_freqs = defaultdict(<span class="hljs-built_in">int</span>)
subwords_freqs = defaultdict(<span class="hljs-built_in">int</span>)
<span class="hljs-keyword">for</span> word, freq <span class="hljs-keyword">in</span> word_freqs.items():
    <span class="hljs-keyword">for</span> i <span class="hljs-keyword">in</span> <span class="hljs-built_in">range</span>(<span class="hljs-built_in">len</span>(word)):
        char_freqs[word[i]] += freq
        <span class="hljs-comment"># Loop through the subwords of length at least 2</span>
        <span class="hljs-keyword">for</span> j <span class="hljs-keyword">in</span> <span class="hljs-built_in">range</span>(i + <span class="hljs-number">2</span>, <span class="hljs-built_in">len</span>(word) + <span class="hljs-number">1</span>):
            subwords_freqs[word[i:j]] += freq

<span class="hljs-comment"># Sort subwords by frequency</span>
sorted_subwords = <span class="hljs-built_in">sorted</span>(subwords_freqs.items(), key=<span class="hljs-keyword">lambda</span> x: x[<span class="hljs-number">1</span>], reverse=<span class="hljs-literal">True</span>)
sorted_subwords[:<span class="hljs-number">10</span>]`}}),Ts=new w({props:{code:"[('\u2581t', 7), ('is', 5), ('er', 5), ('\u2581a', 5), ('\u2581to', 4), ('to', 4), ('en', 4), ('\u2581T', 3), ('\u2581Th', 3), ('\u2581Thi', 3)]",highlighted:'[(<span class="hljs-string">&#x27;\u2581t&#x27;</span>, <span class="hljs-number">7</span>), (<span class="hljs-string">&#x27;is&#x27;</span>, <span class="hljs-number">5</span>), (<span class="hljs-string">&#x27;er&#x27;</span>, <span class="hljs-number">5</span>), (<span class="hljs-string">&#x27;\u2581a&#x27;</span>, <span class="hljs-number">5</span>), (<span class="hljs-string">&#x27;\u2581to&#x27;</span>, <span class="hljs-number">4</span>), (<span class="hljs-string">&#x27;to&#x27;</span>, <span class="hljs-number">4</span>), (<span class="hljs-string">&#x27;en&#x27;</span>, <span class="hljs-number">4</span>), (<span class="hljs-string">&#x27;\u2581T&#x27;</span>, <span class="hljs-number">3</span>), (<span class="hljs-string">&#x27;\u2581Th&#x27;</span>, <span class="hljs-number">3</span>), (<span class="hljs-string">&#x27;\u2581Thi&#x27;</span>, <span class="hljs-number">3</span>)]'}}),zs=new w({props:{code:`token_freqs = list(char_freqs.items()) + sorted_subwords[: 300 - len(char_freqs)]
token_freqs = {token: freq for token, freq in token_freqs}`,highlighted:`token_freqs = <span class="hljs-built_in">list</span>(char_freqs.items()) + sorted_subwords[: <span class="hljs-number">300</span> - <span class="hljs-built_in">len</span>(char_freqs)]
token_freqs = {token: freq <span class="hljs-keyword">for</span> token, freq <span class="hljs-keyword">in</span> token_freqs}`}}),ns=new jn({props:{$$slots:{default:[$r]},$$scope:{ctx:z}}}),Os=new w({props:{code:`from math import log

total_sum = sum([freq for token, freq in token_freqs.items()])
model = {token: -log(freq / total_sum) for token, freq in token_freqs.items()}`,highlighted:`<span class="hljs-keyword">from</span> math <span class="hljs-keyword">import</span> log

total_sum = <span class="hljs-built_in">sum</span>([freq <span class="hljs-keyword">for</span> token, freq <span class="hljs-keyword">in</span> token_freqs.items()])
model = {token: -log(freq / total_sum) <span class="hljs-keyword">for</span> token, freq <span class="hljs-keyword">in</span> token_freqs.items()}`}}),Ds=new w({props:{code:`def encode_word(word, model):
    best_segmentations = [{"start": 0, "score": 1}] + [
        {"start": None, "score": None} for _ in range(len(word))
    ]
    for start_idx in range(len(word)):
        # This should be properly filled by the previous steps of the loop
        best_score_at_start = best_segmentations[start_idx]["score"]
        for end_idx in range(start_idx + 1, len(word) + 1):
            token = word[start_idx:end_idx]
            if token in model and best_score_at_start is not None:
                score = model[token] + best_score_at_start
                # If we have found a better segmentation ending at end_idx, we update
                if (
                    best_segmentations[end_idx]["score"] is None
                    or best_segmentations[end_idx]["score"] > score
                ):
                    best_segmentations[end_idx] = {"start": start_idx, "score": score}

    segmentation = best_segmentations[-1]
    if segmentation["score"] is None:
        # We did not find a tokenization of the word -> unknown
        return ["<unk>"], None

    score = segmentation["score"]
    start = segmentation["start"]
    end = len(word)
    tokens = []
    while start != 0:
        tokens.insert(0, word[start:end])
        next_start = best_segmentations[start]["start"]
        end = start
        start = next_start
    tokens.insert(0, word[start:end])
    return tokens, score`,highlighted:`<span class="hljs-keyword">def</span> <span class="hljs-title function_">encode_word</span>(<span class="hljs-params">word, model</span>):
    best_segmentations = [{<span class="hljs-string">&quot;start&quot;</span>: <span class="hljs-number">0</span>, <span class="hljs-string">&quot;score&quot;</span>: <span class="hljs-number">1</span>}] + [
        {<span class="hljs-string">&quot;start&quot;</span>: <span class="hljs-literal">None</span>, <span class="hljs-string">&quot;score&quot;</span>: <span class="hljs-literal">None</span>} <span class="hljs-keyword">for</span> _ <span class="hljs-keyword">in</span> <span class="hljs-built_in">range</span>(<span class="hljs-built_in">len</span>(word))
    ]
    <span class="hljs-keyword">for</span> start_idx <span class="hljs-keyword">in</span> <span class="hljs-built_in">range</span>(<span class="hljs-built_in">len</span>(word)):
        <span class="hljs-comment"># This should be properly filled by the previous steps of the loop</span>
        best_score_at_start = best_segmentations[start_idx][<span class="hljs-string">&quot;score&quot;</span>]
        <span class="hljs-keyword">for</span> end_idx <span class="hljs-keyword">in</span> <span class="hljs-built_in">range</span>(start_idx + <span class="hljs-number">1</span>, <span class="hljs-built_in">len</span>(word) + <span class="hljs-number">1</span>):
            token = word[start_idx:end_idx]
            <span class="hljs-keyword">if</span> token <span class="hljs-keyword">in</span> model <span class="hljs-keyword">and</span> best_score_at_start <span class="hljs-keyword">is</span> <span class="hljs-keyword">not</span> <span class="hljs-literal">None</span>:
                score = model[token] + best_score_at_start
                <span class="hljs-comment"># If we have found a better segmentation ending at end_idx, we update</span>
                <span class="hljs-keyword">if</span> (
                    best_segmentations[end_idx][<span class="hljs-string">&quot;score&quot;</span>] <span class="hljs-keyword">is</span> <span class="hljs-literal">None</span>
                    <span class="hljs-keyword">or</span> best_segmentations[end_idx][<span class="hljs-string">&quot;score&quot;</span>] &gt; score
                ):
                    best_segmentations[end_idx] = {<span class="hljs-string">&quot;start&quot;</span>: start_idx, <span class="hljs-string">&quot;score&quot;</span>: score}

    segmentation = best_segmentations[-<span class="hljs-number">1</span>]
    <span class="hljs-keyword">if</span> segmentation[<span class="hljs-string">&quot;score&quot;</span>] <span class="hljs-keyword">is</span> <span class="hljs-literal">None</span>:
        <span class="hljs-comment"># We did not find a tokenization of the word -&gt; unknown</span>
        <span class="hljs-keyword">return</span> [<span class="hljs-string">&quot;&lt;unk&gt;&quot;</span>], <span class="hljs-literal">None</span>

    score = segmentation[<span class="hljs-string">&quot;score&quot;</span>]
    start = segmentation[<span class="hljs-string">&quot;start&quot;</span>]
    end = <span class="hljs-built_in">len</span>(word)
    tokens = []
    <span class="hljs-keyword">while</span> start != <span class="hljs-number">0</span>:
        tokens.insert(<span class="hljs-number">0</span>, word[start:end])
        next_start = best_segmentations[start][<span class="hljs-string">&quot;start&quot;</span>]
        end = start
        start = next_start
    tokens.insert(<span class="hljs-number">0</span>, word[start:end])
    <span class="hljs-keyword">return</span> tokens, score`}}),Us=new w({props:{code:`print(encode_word("Hopefully", model))
print(encode_word("This", model))`,highlighted:`<span class="hljs-built_in">print</span>(encode_word(<span class="hljs-string">&quot;Hopefully&quot;</span>, model))
<span class="hljs-built_in">print</span>(encode_word(<span class="hljs-string">&quot;This&quot;</span>, model))`}}),Ns=new w({props:{code:`(['H', 'o', 'p', 'e', 'f', 'u', 'll', 'y'], 41.5157494601402)
(['This'], 6.288267030694535)`,highlighted:`([<span class="hljs-string">&#x27;H&#x27;</span>, <span class="hljs-string">&#x27;o&#x27;</span>, <span class="hljs-string">&#x27;p&#x27;</span>, <span class="hljs-string">&#x27;e&#x27;</span>, <span class="hljs-string">&#x27;f&#x27;</span>, <span class="hljs-string">&#x27;u&#x27;</span>, <span class="hljs-string">&#x27;ll&#x27;</span>, <span class="hljs-string">&#x27;y&#x27;</span>], <span class="hljs-number">41.5157494601402</span>)
([<span class="hljs-string">&#x27;This&#x27;</span>], <span class="hljs-number">6.288267030694535</span>)`}}),Hs=new w({props:{code:`def compute_loss(model):
    loss = 0
    for word, freq in word_freqs.items():
        _, word_loss = encode_word(word, model)
        loss += freq * word_loss
    return loss`,highlighted:`<span class="hljs-keyword">def</span> <span class="hljs-title function_">compute_loss</span>(<span class="hljs-params">model</span>):
    loss = <span class="hljs-number">0</span>
    <span class="hljs-keyword">for</span> word, freq <span class="hljs-keyword">in</span> word_freqs.items():
        _, word_loss = encode_word(word, model)
        loss += freq * word_loss
    <span class="hljs-keyword">return</span> loss`}}),Ss=new w({props:{code:"compute_loss(model)",highlighted:"compute_loss(model)"}}),Ms=new w({props:{code:"413.10377642940875",highlighted:'<span class="hljs-number">413.10377642940875</span>'}}),Bs=new w({props:{code:`import copy


def compute_scores(model):
    scores = {}
    model_loss = compute_loss(model)
    for token, score in model.items():
        # We always keep tokens of length 1
        if len(token) == 1:
            continue
        model_without_token = copy.deepcopy(model)
        _ = model_without_token.pop(token)
        scores[token] = compute_loss(model_without_token) - model_loss
    return scores`,highlighted:`<span class="hljs-keyword">import</span> copy


<span class="hljs-keyword">def</span> <span class="hljs-title function_">compute_scores</span>(<span class="hljs-params">model</span>):
    scores = {}
    model_loss = compute_loss(model)
    <span class="hljs-keyword">for</span> token, score <span class="hljs-keyword">in</span> model.items():
        <span class="hljs-comment"># We always keep tokens of length 1</span>
        <span class="hljs-keyword">if</span> <span class="hljs-built_in">len</span>(token) == <span class="hljs-number">1</span>:
            <span class="hljs-keyword">continue</span>
        model_without_token = copy.deepcopy(model)
        _ = model_without_token.pop(token)
        scores[token] = compute_loss(model_without_token) - model_loss
    <span class="hljs-keyword">return</span> scores`}}),As=new w({props:{code:`scores = compute_scores(model)
print(scores["ll"])
print(scores["his"])`,highlighted:`scores = compute_scores(model)
<span class="hljs-built_in">print</span>(scores[<span class="hljs-string">&quot;ll&quot;</span>])
<span class="hljs-built_in">print</span>(scores[<span class="hljs-string">&quot;his&quot;</span>])`}}),Ls=new w({props:{code:`6.376412403623874
0.0`,highlighted:`<span class="hljs-number">6.376412403623874</span>
<span class="hljs-number">0.0</span>`}}),ls=new jn({props:{$$slots:{default:[br]},$$scope:{ctx:z}}}),Ws=new w({props:{code:`percent_to_remove = 0.1
while len(model) > 100:
    scores = compute_scores(model)
    sorted_scores = sorted(scores.items(), key=lambda x: x[1])
    # Remove percent_to_remove tokens with the lowest scores.
    for i in range(int(len(model) * percent_to_remove)):
        _ = token_freqs.pop(sorted_scores[i][0])

    total_sum = sum([freq for token, freq in token_freqs.items()])
    model = {token: -log(freq / total_sum) for token, freq in token_freqs.items()}`,highlighted:`percent_to_remove = <span class="hljs-number">0.1</span>
<span class="hljs-keyword">while</span> <span class="hljs-built_in">len</span>(model) &gt; <span class="hljs-number">100</span>:
    scores = compute_scores(model)
    sorted_scores = <span class="hljs-built_in">sorted</span>(scores.items(), key=<span class="hljs-keyword">lambda</span> x: x[<span class="hljs-number">1</span>])
    <span class="hljs-comment"># Remove percent_to_remove tokens with the lowest scores.</span>
    <span class="hljs-keyword">for</span> i <span class="hljs-keyword">in</span> <span class="hljs-built_in">range</span>(<span class="hljs-built_in">int</span>(<span class="hljs-built_in">len</span>(model) * percent_to_remove)):
        _ = token_freqs.pop(sorted_scores[i][<span class="hljs-number">0</span>])

    total_sum = <span class="hljs-built_in">sum</span>([freq <span class="hljs-keyword">for</span> token, freq <span class="hljs-keyword">in</span> token_freqs.items()])
    model = {token: -log(freq / total_sum) <span class="hljs-keyword">for</span> token, freq <span class="hljs-keyword">in</span> token_freqs.items()}`}}),Xs=new w({props:{code:`def tokenize(text, model):
    words_with_offsets = tokenizer.backend_tokenizer.pre_tokenizer.pre_tokenize_str(text)
    pre_tokenized_text = [word for word, offset in words_with_offsets]
    encoded_words = [encode_word(word, model)[0] for word in pre_tokenized_text]
    return sum(encoded_words, [])


tokenize("This is the Hugging Face course.", model)`,highlighted:`<span class="hljs-keyword">def</span> <span class="hljs-title function_">tokenize</span>(<span class="hljs-params">text, model</span>):
    words_with_offsets = tokenizer.backend_tokenizer.pre_tokenizer.pre_tokenize_str(text)
    pre_tokenized_text = [word <span class="hljs-keyword">for</span> word, offset <span class="hljs-keyword">in</span> words_with_offsets]
    encoded_words = [encode_word(word, model)[<span class="hljs-number">0</span>] <span class="hljs-keyword">for</span> word <span class="hljs-keyword">in</span> pre_tokenized_text]
    <span class="hljs-keyword">return</span> <span class="hljs-built_in">sum</span>(encoded_words, [])


tokenize(<span class="hljs-string">&quot;This is the Hugging Face course.&quot;</span>, model)`}}),Rs=new w({props:{code:"['\u2581This', '\u2581is', '\u2581the', '\u2581Hugging', '\u2581Face', '\u2581', 'c', 'ou', 'r', 's', 'e', '.']",highlighted:'[<span class="hljs-string">&#x27;\u2581This&#x27;</span>, <span class="hljs-string">&#x27;\u2581is&#x27;</span>, <span class="hljs-string">&#x27;\u2581the&#x27;</span>, <span class="hljs-string">&#x27;\u2581Hugging&#x27;</span>, <span class="hljs-string">&#x27;\u2581Face&#x27;</span>, <span class="hljs-string">&#x27;\u2581&#x27;</span>, <span class="hljs-string">&#x27;c&#x27;</span>, <span class="hljs-string">&#x27;ou&#x27;</span>, <span class="hljs-string">&#x27;r&#x27;</span>, <span class="hljs-string">&#x27;s&#x27;</span>, <span class="hljs-string">&#x27;e&#x27;</span>, <span class="hljs-string">&#x27;.&#x27;</span>]'}}),{c(){u=o("meta"),q=i(),h=o("h1"),b=o("a"),P=o("span"),f(k.$$.fragment),E=i(),O=o("span"),R=l("Unigram\u6807\u8BB0\u5316"),C=i(),f(B.$$.fragment),is=i(),Gs=o("p"),_e=l("\u5728 SentencePiece \u4E2D\u7ECF\u5E38\u4F7F\u7528 Unigram \u7B97\u6CD5,\u8BE5\u7B97\u6CD5\u662F AlBERT\u3001T5\u3001mBART\u3001Big Bird \u548C XLNet \u7B49\u6A21\u578B\u4F7F\u7528\u7684\u6807\u8BB0\u5316\u7B97\u6CD5\u3002"),$n=i(),f(cs.$$.fragment),bn=i(),f(G.$$.fragment),wn=i(),us=o("h2"),ge=l("\u8BAD\u7EC3\u7B97\u6CD5"),qn=i(),Is=o("p"),je=l("\u4E0E BPE \u548C WordPiece \u76F8\u6BD4,Unigram \u5728\u53E6\u4E00\u4E2A\u65B9\u5411\u4E0A\u5DE5\u4F5C:\u5B83\u4ECE\u4E00\u4E2A\u8F83\u5927\u7684\u8BCD\u6C47\u8868\u5F00\u59CB,\u7136\u540E\u4ECE\u4E2D\u5220\u9664\u6807\u8BB0,\u76F4\u5230\u8FBE\u5230\u6240\u9700\u7684\u8BCD\u6C47\u8868\u5927\u5C0F\u3002\u6709\u591A\u79CD\u9009\u9879\u53EF\u7528\u4E8E\u6784\u5EFA\u57FA\u672C\u8BCD\u6C47\u8868:\u4F8B\u5982,\u6211\u4EEC\u53EF\u4EE5\u91C7\u7528\u9884\u6807\u8BB0\u5316\u5355\u8BCD\u4E2D\u6700\u5E38\u89C1\u7684\u5B50\u4E32,\u6216\u8005\u5728\u5177\u6709\u5927\u8BCD\u6C47\u91CF\u7684\u521D\u59CB\u8BED\u6599\u5E93\u4E0A\u5E94\u7528 BPE\u3002"),kn=i(),Ys=o("p"),$e=l("\u5728\u8BAD\u7EC3\u7684\u6BCF\u4E00\u6B65,Unigram \u7B97\u6CD5\u90FD\u4F1A\u5728\u7ED9\u5B9A\u5F53\u524D\u8BCD\u6C47\u7684\u60C5\u51B5\u4E0B\u8BA1\u7B97\u8BED\u6599\u5E93\u7684\u635F\u5931\u3002\u7136\u540E,\u5BF9\u4E8E\u8BCD\u6C47\u8868\u4E2D\u7684\u6BCF\u4E2A\u7B26\u53F7,\u7B97\u6CD5\u8BA1\u7B97\u5982\u679C\u5220\u9664\u8BE5\u7B26\u53F7,\u6574\u4F53\u635F\u5931\u4F1A\u589E\u52A0\u591A\u5C11,\u5E76\u5BFB\u627E\u589E\u52A0\u6700\u5C11\u7684\u7B26\u53F7\u3002\u8FD9\u4E9B\u7B26\u53F7\u5BF9\u8BED\u6599\u5E93\u7684\u6574\u4F53\u635F\u5931\u5F71\u54CD\u8F83\u5C0F,\u56E0\u6B64\u4ECE\u67D0\u79CD\u610F\u4E49\u4E0A\u8BF4,\u5B83\u4EEC\u201C\u4E0D\u592A\u9700\u8981\u201D\u5E76\u4E14\u662F\u79FB\u9664\u7684\u6700\u4F73\u5019\u9009\u8005\u3002"),yn=i(),A=o("p"),be=l("\u8FD9\u662F\u4E00\u4E2A\u975E\u5E38\u6602\u8D35\u7684\u64CD\u4F5C,\u6240\u4EE5\u6211\u4EEC\u4E0D\u53EA\u662F\u5220\u9664\u4E0E\u6700\u4F4E\u635F\u5931\u589E\u52A0\u76F8\u5173\u7684\u5355\u4E2A\u7B26\u53F7,\u800C\u4E14"),vn=new fe,En=l(" ("),xn=new fe,Pn=l("\u662F\u4E00\u4E2A\u53EF\u4EE5\u63A7\u5236\u7684\u8D85\u53C2\u6570,\u901A\u5E38\u662F 10 \u6216 20)\u4E0E\u6700\u4F4E\u635F\u5931\u589E\u52A0\u76F8\u5173\u7684\u7B26\u53F7\u7684\u767E\u5206\u6BD4\u3002\u7136\u540E\u91CD\u590D\u8FD9\u4E2A\u8FC7\u7A0B,\u76F4\u5230\u8BCD\u6C47\u91CF\u8FBE\u5230\u6240\u9700\u7684\u5927\u5C0F\u3002"),Cn=i(),Zs=o("p"),we=l("\u8BF7\u6CE8\u610F,\u6211\u4EEC\u4ECE\u4E0D\u5220\u9664\u57FA\u672C\u5B57\u7B26,\u4EE5\u786E\u4FDD\u53EF\u4EE5\u6807\u8BB0\u4EFB\u4F55\u5355\u8BCD\u3002"),Tn=i(),Js=o("p"),qe=l("\u73B0\u5728,\u8FD9\u4ECD\u7136\u6709\u70B9\u6A21\u7CCA:\u7B97\u6CD5\u7684\u4E3B\u8981\u90E8\u5206\u662F\u8BA1\u7B97\u8BED\u6599\u5E93\u7684\u635F\u5931,\u5E76\u67E5\u770B\u5F53\u6211\u4EEC\u4ECE\u8BCD\u6C47\u8868\u4E2D\u5220\u9664\u4E00\u4E9B\u6807\u8BB0\u65F6\u5B83\u4F1A\u5982\u4F55\u53D8\u5316,\u4F46\u6211\u4EEC\u8FD8\u6CA1\u6709\u89E3\u91CA\u5982\u4F55\u505A\u5230\u8FD9\u4E00\u70B9\u3002\u8FD9\u4E00\u6B65\u4F9D\u8D56\u4E8E Unigram \u6A21\u578B\u7684\u6807\u8BB0\u5316\u7B97\u6CD5,\u56E0\u6B64\u6211\u4EEC\u63A5\u4E0B\u6765\u5C06\u6DF1\u5165\u7814\u7A76\u3002"),zn=i(),Ks=o("p"),ke=l("\u6211\u4EEC\u5C06\u91CD\u7528\u524D\u9762\u793A\u4F8B\u4E2D\u7684\u8BED\u6599\u5E93:"),On=i(),f(hs.$$.fragment),Dn=i(),Qs=o("p"),ye=l("\u5BF9\u4E8E\u6B64\u793A\u4F8B,\u6211\u4EEC\u5C06\u91C7\u7528\u521D\u59CB\u8BCD\u6C47\u8868\u7684\u6240\u6709\u4E25\u683C\u5B50\u5B57\u7B26\u4E32:"),Un=i(),f(fs.$$.fragment),Nn=i(),ds=o("h2"),ve=l("\u6807\u8BB0\u5316\u7B97\u6CD5"),Hn=i(),sa=o("p"),Ee=l("Unigram \u6A21\u578B\u662F\u4E00\u79CD\u8BED\u8A00\u6A21\u578B,\u5B83\u8BA4\u4E3A\u6BCF\u4E2A\u6807\u8BB0\u90FD\u72EC\u7ACB\u4E8E\u5B83\u4E4B\u524D\u7684\u6807\u8BB0\u3002\u5B83\u662F\u6700\u7B80\u5355\u7684\u8BED\u8A00\u6A21\u578B,\u4ECE\u67D0\u79CD\u610F\u4E49\u4E0A\u8BF4, \u7ED9\u5B9A\u5148\u524D\u4E0A\u4E0B\u6587\u7684\u6807\u8BB0 X \u7684\u6982\u7387\u5C31\u662F\u6807\u8BB0 X \u7684\u6982\u7387\u3002\u56E0\u6B64,\u5982\u679C\u6211\u4EEC\u4F7F\u7528 Unigram \u8BED\u8A00\u6A21\u578B\u751F\u6210\u6587\u672C,\u6211\u4EEC\u5C06\u59CB\u7EC8\u9884\u6D4B\u6700\u5E38\u89C1\u7684\u6807\u8BB0\u3002"),Sn=i(),x=o("p"),xe=l("\u7ED9\u5B9A\u6807\u8BB0\u7684\u6982\u7387\u662F\u5B83\u5728\u539F\u59CB\u8BED\u6599\u5E93\u4E2D\u7684\u9891\u7387(\u6211\u4EEC\u627E\u5230\u5B83\u7684\u6B21\u6570),\u9664\u4EE5\u8BCD\u6C47\u8868\u4E2D\u6240\u6709\u6807\u8BB0\u7684\u6240\u6709\u9891\u7387\u7684\u603B\u548C(\u4EE5\u786E\u4FDD\u6982\u7387\u603B\u548C\u4E3A 1)\u3002\u4F8B\u5982, "),Ea=o("code"),Pe=l('"ug"'),Ce=l(" \u5728 "),xa=o("code"),Te=l('"hug"'),ze=l(" \u3001 "),Pa=o("code"),Oe=l('"pug"'),De=l(" \u4EE5\u53CA "),Ca=o("code"),Ue=l('"hugs"'),Ne=l(" \u4E2D,\u6240\u4EE5\u5B83\u5728\u6211\u4EEC\u7684\u8BED\u6599\u5E93\u4E2D\u7684\u9891\u7387\u4E3A 20\u3002"),Mn=i(),aa=o("p"),He=l("\u4EE5\u4E0B\u662F\u8BCD\u6C47\u8868\u4E2D\u6240\u6709\u53EF\u80FD\u7684\u5B50\u8BCD\u7684\u51FA\u73B0\u9891\u7387:"),Bn=i(),f(_s.$$.fragment),An=i(),I=o("p"),Se=l("\u6240\u4EE5,\u6240\u6709\u9891\u7387\u4E4B\u548C\u4E3A210, \u5E76\u4E14\u5B50\u8BCD "),Ta=o("code"),Me=l('"ug"'),Be=l(" \u51FA\u73B0\u7684\u6982\u7387\u662F 20/210\u3002"),Ln=i(),f(Y.$$.fragment),Wn=i(),M=o("p"),Ae=l("\u73B0\u5728,\u4E3A\u4E86\u5BF9\u7ED9\u5B9A\u7684\u5355\u8BCD\u8FDB\u884C\u6807\u8BB0,\u6211\u4EEC\u5C06\u6240\u6709\u53EF\u80FD\u7684\u5206\u5272\u89C6\u4E3A\u6807\u8BB0,\u5E76\u6839\u636E Unigram \u6A21\u578B\u8BA1\u7B97\u6BCF\u4E2A\u5206\u5272\u7684\u6982\u7387\u3002\u7531\u4E8E\u6240\u6709\u6807\u8BB0\u90FD\u88AB\u8BA4\u4E3A\u662F\u72EC\u7ACB\u7684,\u6240\u4EE5\u8FD9\u4E2A\u6982\u7387\u53EA\u662F\u6BCF\u4E2A\u6807\u8BB0\u6982\u7387\u7684\u4E58\u79EF\u3002\u4F8B\u5982, "),za=o("code"),Le=l('"pug"'),We=l(" \u7684\u6807\u8BB0\u5316 "),Oa=o("code"),Xe=l('["p", "u", "g"]'),Re=l(` \u7684\u6982\u7387\u4E3A:
`),Xn=new fe,Rn=i(),F=o("p"),Fe=l("\u76F8\u6BD4\u4E4B\u4E0B,\u6807\u8BB0\u5316 "),Da=o("code"),Ve=l('["pu", "g"]'),Ge=l(` \u7684\u6982\u7387\u4E3A:
`),Fn=new fe,Vn=i(),na=o("p"),Ie=l("\u6240\u4EE5\u4E00\u4E2A\u66F4\u6709\u53EF\u80FD\u3002\u4E00\u822C\u6765\u8BF4,\u5177\u6709\u5C3D\u53EF\u80FD\u5C11\u7684\u6807\u8BB0\u7684\u6807\u8BB0\u5316\u5C06\u5177\u6709\u6700\u9AD8\u7684\u6982\u7387(\u56E0\u4E3A\u6BCF\u4E2A\u6807\u8BB0\u91CD\u590D\u9664\u4EE5 210),\u8FD9\u5BF9\u5E94\u4E8E\u6211\u4EEC\u76F4\u89C2\u60F3\u8981\u7684:\u5C06\u4E00\u4E2A\u5355\u8BCD\u5206\u6210\u5C3D\u53EF\u80FD\u5C11\u7684\u6807\u8BB0\u3002"),Gn=i(),Z=o("p"),Ye=l("\u4F7F\u7528 Unigram \u6A21\u578B\u5BF9\u5355\u8BCD\u8FDB\u884C\u5206\u8BCD\u662F\u6982\u7387\u6700\u9AD8\u7684\u5206\u8BCD\u3002\u5728\u793A\u4F8B "),Ua=o("code"),Ze=l('"pug"'),Je=l(" \u4E2D,\u8FD9\u91CC\u662F\u6211\u4EEC\u4E3A\u6BCF\u4E2A\u53EF\u80FD\u7684\u5206\u5272\u83B7\u5F97\u7684\u6982\u7387:"),In=i(),f(gs.$$.fragment),Yn=i(),D=o("p"),Ke=l("\u6240\u4EE5, "),Na=o("code"),Qe=l('"pug"'),sl=l(" \u5C06\u88AB\u6807\u8BB0\u4E3A "),Ha=o("code"),al=l('["p", "ug"]'),nl=l(" \u6216\u8005 "),Sa=o("code"),tl=l('["pu", "g"]'),el=l(",  \u53D6\u51B3\u4E8E\u9996\u5148\u9047\u5230\u8FD9\u4E9B\u5206\u5272\u4E2D\u7684\u54EA\u4E00\u4E2A(\u8BF7\u6CE8\u610F,\u5728\u66F4\u5927\u7684\u8BED\u6599\u5E93\u4E2D,\u8FD9\u6837\u7684\u76F8\u7B49\u7684\u60C5\u51B5\u5F88\u5C11\u89C1)\u3002"),Zn=i(),y=o("p"),ll=l("\u5728\u8FD9\u79CD\u60C5\u51B5\u4E0B,\u5F88\u5BB9\u6613\u627E\u5230\u6240\u6709\u53EF\u80FD\u7684\u5206\u5272\u5E76\u8BA1\u7B97\u5B83\u4EEC\u7684\u6982\u7387,\u4F46\u4E00\u822C\u6765\u8BF4\u4F1A\u6709\u70B9\u56F0\u96BE\u3002\u6709\u4E00\u79CD\u7528\u4E8E\u6B64\u7684\u7ECF\u5178\u7B97\u6CD5,\u79F0\u4E3A "),Ma=o("em"),pl=l("\u7EF4\u7279\u6BD4(Viterbi)\u7B97\u6CD5"),ol=l("\u3002\u672C\u8D28\u4E0A,\u6211\u4EEC\u53EF\u4EE5\u6784\u5EFA\u4E00\u4E2A\u56FE\u6765\u68C0\u6D4B\u7ED9\u5B9A\u5355\u8BCD\u7684\u53EF\u80FD\u5206\u5272,\u5982\u679C\u4ECE"),Ba=o("em"),rl=l("a"),ml=l("\u5230"),Aa=o("em"),il=l("b"),cl=l("\u7684\u5B50\u8BCD\u5728\u8BCD\u6C47\u8868\u4E2D,\u5219\u4ECE\u5B57\u7B26"),La=o("em"),ul=l("a"),hl=l("\u5230\u5B57\u7B26"),Wa=o("em"),fl=l("b"),dl=l("\u4E4B\u95F4\u5B58\u5728\u4E00\u4E2A\u5206\u652F,\u5E76\u5C06\u5B50\u8BCD\u7684\u6982\u7387\u5F52\u56E0\u4E8E\u8BE5\u5206\u652F\u3002"),Jn=i(),ta=o("p"),_l=l("\u4E3A\u4E86\u5728\u8BE5\u56FE\u4E2D\u627E\u5230\u5C06\u5177\u6709\u6700\u4F73\u5206\u6570\u7684\u8DEF\u5F84,\u7EF4\u7279\u6BD4\u7B97\u6CD5\u4E3A\u5355\u8BCD\u4E2D\u7684\u6BCF\u4E2A\u4F4D\u7F6E\u786E\u5B9A\u5728\u8BE5\u4F4D\u7F6E\u7ED3\u675F\u7684\u5177\u6709\u6700\u4F73\u5206\u6570\u7684\u5206\u6BB5\u3002\u7531\u4E8E\u6211\u4EEC\u4ECE\u5F00\u59CB\u5230\u7ED3\u675F,\u53EF\u4EE5\u901A\u8FC7\u5FAA\u73AF\u904D\u5386\u4EE5\u5F53\u524D\u4F4D\u7F6E\u7ED3\u5C3E\u7684\u6240\u6709\u5B50\u8BCD,\u7136\u540E\u4F7F\u7528\u8BE5\u5B50\u8BCD\u5F00\u59CB\u4F4D\u7F6E\u7684\u6700\u4F73\u6807\u8BB0\u5316\u5206\u6570\u6765\u627E\u5230\u6700\u4F73\u5206\u6570\u3002\u7136\u540E,\u6211\u4EEC\u53EA\u9700\u8981\u5C55\u5F00\u5230\u8FBE\u7EC8\u70B9\u6240\u91C7\u53D6\u7684\u8DEF\u5F84\u3002"),Kn=i(),J=o("p"),gl=l("\u8BA9\u6211\u4EEC\u770B\u4E00\u4E2A\u4F7F\u7528\u6211\u4EEC\u7684\u8BCD\u6C47\u8868\u548C\u5355\u8BCD "),Xa=o("code"),jl=l('"unhug"'),$l=l(" \u7684\u4F8B\u5B50\u3002\u5BF9\u4E8E\u6BCF\u4E2A\u4F4D\u7F6E,\u4EE5\u6700\u597D\u7684\u5206\u6570\u7ED3\u5C3E\u7684\u5B50\u8BCD\u5982\u4E0B:"),Qn=i(),f(js.$$.fragment),st=i(),L=o("p"),bl=l("\u56E0\u6B64 "),Ra=o("code"),wl=l('"unhug"'),ql=l(" \u5C06\u88AB\u6807\u8BB0\u4E3A "),Fa=o("code"),kl=l('["un", "hug"]'),yl=l("\u3002"),at=i(),f(K.$$.fragment),nt=i(),$s=o("h2"),vl=l("\u56DE\u5230\u8BAD\u7EC3"),tt=i(),ea=o("p"),El=l("\u73B0\u5728\u6211\u4EEC\u5DF2\u7ECF\u4E86\u89E3\u4E86\u6807\u8BB0\u5316\u7684\u5DE5\u4F5C\u539F\u7406,\u6211\u4EEC\u53EF\u4EE5\u66F4\u6DF1\u5165\u5730\u7814\u7A76\u8BAD\u7EC3\u671F\u95F4\u4F7F\u7528\u7684\u635F\u5931\u3002\u5728\u4EFB\u4F55\u7ED9\u5B9A\u7684\u9636\u6BB5,\u8FD9\u4E2A\u635F\u5931\u662F\u901A\u8FC7\u5BF9\u8BED\u6599\u5E93\u4E2D\u7684\u6BCF\u4E2A\u5355\u8BCD\u8FDB\u884C\u6807\u8BB0\u6765\u8BA1\u7B97\u7684,\u4F7F\u7528\u5F53\u524D\u8BCD\u6C47\u8868\u548C\u7531\u8BED\u6599\u5E93\u4E2D\u6BCF\u4E2A\u6807\u8BB0\u7684\u9891\u7387\u786E\u5B9A\u7684 Unigram \u6A21\u578B(\u5982\u524D\u6240\u8FF0)\u3002"),et=i(),Q=o("p"),xl=l("\u8BED\u6599\u5E93\u4E2D\u7684\u6BCF\u4E2A\u8BCD\u90FD\u6709\u4E00\u4E2A\u5206\u6570,\u635F\u5931\u662F\u8FD9\u4E9B\u5206\u6570\u7684\u8D1F\u5BF9\u6570\u4F3C\u7136 \u2014 \u5373\u6240\u6709\u8BCD\u7684\u8BED\u6599\u5E93\u4E2D\u6240\u6709\u8BCD\u7684\u603B\u548C "),Va=o("code"),Pl=l("-log(P(word))"),Cl=l("\u3002"),lt=i(),la=o("p"),Tl=l("\u8BA9\u6211\u4EEC\u7528\u4EE5\u4E0B\u8BED\u6599\u5E93\u56DE\u5230\u6211\u4EEC\u7684\u4F8B\u5B50:"),pt=i(),f(bs.$$.fragment),ot=i(),pa=o("p"),zl=l("\u6BCF\u4E2A\u5355\u8BCD\u7684\u6807\u8BB0\u5316\u53CA\u5176\u5404\u81EA\u7684\u5206\u6570\u662F:"),rt=i(),f(ws.$$.fragment),mt=i(),oa=o("p"),Ol=l("\u6240\u4EE5\u635F\u5931\u662F:"),it=i(),f(qs.$$.fragment),ct=i(),U=o("p"),Dl=l("\u73B0\u5728\u6211\u4EEC\u9700\u8981\u8BA1\u7B97\u5220\u9664\u6BCF\u4E2A\u6807\u8BB0\u5982\u4F55\u5F71\u54CD\u635F\u5931\u3002\u8FD9\u76F8\u5F53\u4E4F\u5473,\u6240\u4EE5\u6211\u4EEC\u5728\u8FD9\u91CC\u53EA\u5BF9\u4E24\u4E2A\u6807\u8BB0\u8FDB\u884C\u64CD\u4F5C,\u5E76\u4FDD\u5B58\u6574\u4E2A\u8FC7\u7A0B\u4EE5\u5907\u6709\u4EE3\u7801\u6765\u5E2E\u52A9\u6211\u4EEC\u3002\u5728\u8FD9\u4E2A(\u975E\u5E38)\u7279\u6B8A\u7684\u60C5\u51B5\u4E0B,\u6211\u4EEC\u5BF9\u6240\u6709\u5355\u8BCD\u6709\u4E24\u4E2A\u7B49\u6548\u7684\u6807\u8BB0:\u6B63\u5982\u6211\u4EEC\u4E4B\u524D\u770B\u5230\u7684,\u4F8B\u5982, "),Ga=o("code"),Ul=l('"pug"'),Nl=l(" \u53EF\u4EE5\u4EE5\u76F8\u540C\u7684\u5206\u6570\u88AB\u6807\u8BB0\u4E3A "),Ia=o("code"),Hl=l('["p", "ug"]'),Sl=l("\u3002\u56E0\u6B64,\u53BB\u9664\u8BCD\u6C47\u8868\u4E2D\u7684 "),Ya=o("code"),Ml=l('"pu"'),Bl=l(" \u6807\u8BB0\u5C06\u7ED9\u51FA\u5B8C\u5168\u76F8\u540C\u7684\u635F\u5931\u3002"),ut=i(),N=o("p"),Al=l("\u53E6\u4E00\u65B9\u9762,\u53BB\u9664 "),Za=o("code"),Ll=l('"hug"'),Wl=l(" \u635F\u5931\u53D8\u5F97\u66F4\u7CDF, \u56E0\u4E3A "),Ja=o("code"),Xl=l('"hug"'),Rl=l(" \u548C "),Ka=o("code"),Fl=l('"hugs"'),Vl=l(" \u7684\u6807\u8BB0\u5316\u4F1A\u53D8\u6210:"),ht=i(),f(ks.$$.fragment),ft=i(),ra=o("p"),Gl=l("\u8FD9\u4E9B\u53D8\u5316\u5C06\u5BFC\u81F4\u635F\u5931\u589E\u52A0:"),dt=i(),f(ys.$$.fragment),_t=i(),W=o("p"),Il=l("\u56E0\u6B64, \u6807\u8BB0 "),Qa=o("code"),Yl=l('"pu"'),Zl=l("\u53EF\u80FD\u4F1A\u4ECE\u8BCD\u6C47\u8868\u4E2D\u5220\u9664,\u4F46\u4E0D\u4F1A\u5220\u9664 "),sn=o("code"),Jl=l('"hug"'),Kl=l("."),gt=i(),V=o("h2"),ss=o("a"),an=o("span"),f(vs.$$.fragment),Ql=i(),nn=o("span"),sp=l("\u5B9E\u73B0 Unigram"),jt=i(),ma=o("p"),ap=l("\u73B0\u5728\u8BA9\u6211\u4EEC\u5728\u4EE3\u7801\u4E2D\u5B9E\u73B0\u6211\u4EEC\u8FC4\u4ECA\u4E3A\u6B62\u770B\u5230\u7684\u6240\u6709\u5185\u5BB9\u3002\u4E0E BPE \u548C WordPiece \u4E00\u6837,\u8FD9\u4E0D\u662F Unigram \u7B97\u6CD5\u7684\u6709\u6548\u5B9E\u73B0(\u6070\u6070\u76F8\u53CD),\u4F46\u5B83\u5E94\u8BE5\u53EF\u4EE5\u5E2E\u52A9\u4F60\u66F4\u597D\u5730\u7406\u89E3\u5B83\u3002"),$t=i(),ia=o("p"),np=l("\u6211\u4EEC\u5C06\u4F7F\u7528\u4E0E\u4E4B\u524D\u76F8\u540C\u7684\u8BED\u6599\u5E93\u4F5C\u4E3A\u793A\u4F8B:"),bt=i(),f(Es.$$.fragment),wt=i(),as=o("p"),tp=l("\u8FD9\u4E00\u6B21,\u6211\u4EEC\u5C06\u4F7F\u7528 "),tn=o("code"),ep=l("xlnet-base-cased"),lp=l(" \u4F5C\u4E3A\u6211\u4EEC\u7684\u6A21\u578B:"),qt=i(),f(xs.$$.fragment),kt=i(),ca=o("p"),pp=l("\u4E0E BPE \u548C WordPiece \u4E00\u6837,\u6211\u4EEC\u9996\u5148\u8BA1\u7B97\u8BED\u6599\u5E93\u4E2D\u6BCF\u4E2A\u5355\u8BCD\u7684\u51FA\u73B0\u6B21\u6570:"),yt=i(),f(Ps.$$.fragment),vt=i(),ua=o("p"),op=l("\u7136\u540E,\u6211\u4EEC\u9700\u8981\u5C06\u6211\u4EEC\u7684\u8BCD\u6C47\u8868\u521D\u59CB\u5316\u4E3A\u5927\u4E8E\u6211\u4EEC\u6700\u7EC8\u60F3\u8981\u7684\u8BCD\u6C47\u91CF\u3002\u6211\u4EEC\u5FC5\u987B\u5305\u542B\u6240\u6709\u57FA\u672C\u5B57\u7B26(\u5426\u5219\u6211\u4EEC\u5C06\u65E0\u6CD5\u6807\u8BB0\u6BCF\u4E2A\u5355\u8BCD),\u4F46\u5BF9\u4E8E\u8F83\u5927\u7684\u5B50\u5B57\u7B26\u4E32,\u6211\u4EEC\u5C06\u53EA\u4FDD\u7559\u6700\u5E38\u89C1\u7684\u5B57\u7B26,\u56E0\u6B64\u6211\u4EEC\u6309\u9891\u7387\u5BF9\u5B83\u4EEC\u8FDB\u884C\u6392\u5E8F:"),Et=i(),f(Cs.$$.fragment),xt=i(),f(Ts.$$.fragment),Pt=i(),ha=o("p"),rp=l("\u6211\u4EEC\u7528\u6700\u4F18\u7684\u5B50\u8BCD\u5BF9\u5B57\u7B26\u8FDB\u884C\u5206\u7EC4,\u4EE5\u83B7\u5F97\u5927\u5C0F\u4E3A 300 \u7684\u521D\u59CB\u8BCD\u6C47\u8868:"),Ct=i(),f(zs.$$.fragment),Tt=i(),f(ns.$$.fragment),zt=i(),fa=o("p"),mp=l("\u63A5\u4E0B\u6765,\u6211\u4EEC\u8BA1\u7B97\u6240\u6709\u9891\u7387\u7684\u603B\u548C,\u5C06\u9891\u7387\u8F6C\u6362\u4E3A\u6982\u7387\u3002\u5BF9\u4E8E\u6211\u4EEC\u7684\u6A21\u578B,\u6211\u4EEC\u5C06\u5B58\u50A8\u6982\u7387\u7684\u5BF9\u6570,\u56E0\u4E3A\u6DFB\u52A0\u5BF9\u6570\u6BD4\u4E58\u4EE5\u5C0F\u6570\u5728\u6570\u503C\u4E0A\u66F4\u7A33\u5B9A,\u8FD9\u5C06\u7B80\u5316\u6A21\u578B\u635F\u5931\u7684\u8BA1\u7B97:"),Ot=i(),f(Os.$$.fragment),Dt=i(),ts=o("p"),ip=l("N\u73B0\u5728\u4E3B\u8981\u529F\u80FD\u662F\u4F7F\u7528 Viterbi \u7B97\u6CD5\u6807\u8BB0\u5355\u8BCD\u7684\u529F\u80FD\u3002\u6B63\u5982\u6211\u4EEC\u4E4B\u524D\u770B\u5230\u7684,\u8BE5\u7B97\u6CD5\u8BA1\u7B97\u5355\u8BCD\u7684\u6BCF\u4E2A\u5B50\u4E32\u7684\u6700\u4F73\u5206\u6BB5,\u6211\u4EEC\u5C06\u5176\u5B58\u50A8\u5728\u540D\u4E3A "),en=o("code"),cp=l("best_segmentations"),up=l(" \u7684\u53D8\u91CF\u4E2D\u3002\u6211\u4EEC\u5C06\u5728\u5355\u8BCD\u7684\u6BCF\u4E2A\u4F4D\u7F6E(\u4ECE 0 \u5230\u5176\u603B\u957F\u5EA6)\u5B58\u50A8\u4E00\u4E2A\u5B57\u5178,\u6709\u4E24\u4E2A\u952E:\u6700\u4F73\u5206\u5272\u4E2D\u6700\u540E\u4E00\u4E2A\u6807\u8BB0\u7684\u5F00\u59CB\u7D22\u5F15,\u4EE5\u53CA\u6700\u4F73\u5206\u5272\u7684\u5206\u6570\u3002\u4F7F\u7528\u6700\u540E\u4E00\u4E2A\u6807\u8BB0\u7684\u5F00\u59CB\u7D22\u5F15,\u4E00\u65E6\u5217\u8868\u5B8C\u5168\u586B\u5145,\u6211\u4EEC\u5C06\u80FD\u591F\u68C0\u7D22\u5B8C\u6574\u7684\u5206\u6BB5\u3002"),Ut=i(),es=o("p"),hp=l("\u586B\u5145\u5217\u8868\u53EA\u9700\u4E24\u4E2A\u5FAA\u73AF:\u4E3B\u5FAA\u73AF\u904D\u5386\u6BCF\u4E2A\u8D77\u59CB\u4F4D\u7F6E,\u7B2C\u4E8C\u4E2A\u5FAA\u73AF\u5C1D\u8BD5\u4ECE\u8BE5\u8D77\u59CB\u4F4D\u7F6E\u5F00\u59CB\u7684\u6240\u6709\u5B50\u5B57\u7B26\u4E32\u3002\u5982\u679C\u5B50\u4E32\u5728\u8BCD\u6C47\u8868\u4E2D,\u6211\u4EEC\u6709\u4E00\u4E2A\u65B0\u7684\u8BCD\u5206\u6BB5,\u76F4\u5230\u8BE5\u7ED3\u675F\u4F4D\u7F6E,\u6211\u4EEC\u5C06\u5176\u4E0E "),ln=o("code"),fp=l("best_segmentations"),dp=l(" \u76F8\u6BD4\u8F83\u3002"),Nt=i(),da=o("p"),_p=l("\u4E00\u65E6\u4E3B\u5FAA\u73AF\u5B8C\u6210,\u6211\u4EEC\u5C31\u4ECE\u7ED3\u5C3E\u5F00\u59CB,\u4ECE\u4E00\u4E2A\u5F00\u59CB\u4F4D\u7F6E\u8DF3\u5230\u4E0B\u4E00\u4E2A,\u8BB0\u5F55\u6211\u4EEC\u524D\u8FDB\u7684\u6807\u8BB0,\u76F4\u5230\u6211\u4EEC\u5230\u8FBE\u5355\u8BCD\u7684\u5F00\u5934:"),Ht=i(),f(Ds.$$.fragment),St=i(),_a=o("p"),gp=l("\u6211\u4EEC\u5DF2\u7ECF\u53EF\u4EE5\u5728\u4E00\u4E9B\u8BCD\u4E0A\u5C1D\u8BD5\u6211\u4EEC\u7684\u521D\u59CB\u6A21\u578B:"),Mt=i(),f(Us.$$.fragment),Bt=i(),f(Ns.$$.fragment),At=i(),ga=o("p"),jp=l("\u73B0\u5728\u5F88\u5BB9\u6613\u8BA1\u7B97\u6A21\u578B\u5728\u8BED\u6599\u5E93\u4E0A\u7684\u635F\u5931!"),Lt=i(),f(Hs.$$.fragment),Wt=i(),ja=o("p"),$p=l("\u6211\u4EEC\u53EF\u4EE5\u68C0\u67E5\u5B83\u662F\u5426\u9002\u7528\u4E8E\u6211\u4EEC\u62E5\u6709\u7684\u6A21\u578B:"),Xt=i(),f(Ss.$$.fragment),Rt=i(),f(Ms.$$.fragment),Ft=i(),$a=o("p"),bp=l("\u8BA1\u7B97\u6BCF\u4E2A\u6807\u8BB0\u7684\u5206\u6570\u4E5F\u4E0D\u662F\u5F88\u96BE;\u6211\u4EEC\u53EA\u9700\u8981\u8BA1\u7B97\u901A\u8FC7\u5220\u9664\u6BCF\u4E2A\u6807\u8BB0\u83B7\u5F97\u7684\u6A21\u578B\u7684\u635F\u5931:"),Vt=i(),f(Bs.$$.fragment),Gt=i(),ba=o("p"),wp=l("\u6211\u4EEC\u53EF\u4EE5\u5728\u7ED9\u5B9A\u7684\u6807\u8BB0\u4E0A\u5C1D\u8BD5:"),It=i(),f(As.$$.fragment),Yt=i(),v=o("p"),qp=l("\u81EA\u4ECE "),pn=o("code"),kp=l('"ll"'),yp=l(" \u7528\u4E8E\u6807\u8BB0\u5316 "),on=o("code"),vp=l('"Hopefully"'),Ep=l(", \u5220\u9664\u5B83\u53EF\u80FD\u4F1A\u8BA9\u6211\u4EEC\u4F7F\u7528\u6807\u8BB0 "),rn=o("code"),xp=l('"l"'),Pp=l(" \u4E24\u6B21\u76F8\u53CD,\u6211\u4EEC\u9884\u8BA1\u5B83\u5C06\u4EA7\u751F\u6B63\u635F\u5931\u3002 "),mn=o("code"),Cp=l('"his"'),Tp=l(" \u4EC5\u5728\u5355\u8BCD"),cn=o("code"),zp=l('"This"'),Op=l(" \u5185\u4F7F\u7528,\u5B83\u88AB\u6807\u8BB0\u4E3A\u81EA\u8EAB,\u6240\u4EE5\u6211\u4EEC\u671F\u671B\u5B83\u7684\u635F\u5931\u4E3A\u96F6\u3002\u7ED3\u679C\u5982\u4E0B:"),Zt=i(),f(Ls.$$.fragment),Jt=i(),f(ls.$$.fragment),Kt=i(),wa=o("p"),Dp=l("\u5B8C\u6210\u6240\u6709\u8FD9\u4E9B\u540E,\u6211\u4EEC\u9700\u8981\u505A\u7684\u6700\u540E\u4E00\u4EF6\u4E8B\u662F\u5C06\u6A21\u578B\u4F7F\u7528\u7684\u7279\u6B8A\u6807\u8BB0\u6DFB\u52A0\u5230\u8BCD\u6C47\u8868\u4E2D,\u7136\u540E\u5FAA\u73AF\u76F4\u5230\u6211\u4EEC\u4ECE\u8BCD\u6C47\u8868\u4E2D\u4FEE\u526A\u4E86\u8DB3\u591F\u7684\u6807\u8BB0\u4EE5\u8FBE\u5230\u6211\u4EEC\u60F3\u8981\u7684\u5927\u5C0F:"),Qt=i(),f(Ws.$$.fragment),se=i(),ps=o("p"),Up=l("\u7136\u540E,\u4E3A\u4E86\u6807\u8BB0\u4E00\u4E9B\u6587\u672C,\u6211\u4EEC\u53EA\u9700\u8981\u5E94\u7528\u9884\u6807\u8BB0\u5316,\u7136\u540E\u4F7F\u7528\u6211\u4EEC\u7684 "),un=o("code"),Np=l("encode_word()"),Hp=l(" \u51FD\u6570:"),ae=i(),f(Xs.$$.fragment),ne=i(),f(Rs.$$.fragment),te=i(),qa=o("p"),Sp=l("Unigram \u5C31\u662F\u8FD9\u6837!\u5E0C\u671B\u73B0\u5728\u4F60\u611F\u89C9\u81EA\u5DF1\u662F\u6807\u8BB0\u5668\u6240\u6709\u65B9\u9762\u7684\u4E13\u5BB6\u3002\u5728\u4E0B\u4E00\u8282\u4E2D,\u6211\u4EEC\u5C06\u6DF1\u5165\u7814\u7A76 \u{1F917} Tokenizers \u5E93\u7684\u6784\u5EFA\u5757,\u5E76\u5411\u60A8\u5C55\u793A\u5982\u4F55\u4F7F\u7528\u5B83\u4EEC\u6765\u6784\u5EFA\u60A8\u81EA\u5DF1\u7684\u6807\u8BB0\u5668\u3002"),this.h()},l(s){const n=ur('[data-svelte="svelte-1phssyn"]',document.head);u=r(n,"META",{name:!0,content:!0}),n.forEach(a),q=c(s),h=r(s,"H1",{class:!0});var Fs=m(h);b=r(Fs,"A",{id:!0,class:!0,href:!0});var hn=m(b);P=r(hn,"SPAN",{});var fn=m(P);d(k.$$.fragment,fn),fn.forEach(a),hn.forEach(a),E=c(Fs),O=r(Fs,"SPAN",{});var dn=m(O);R=p(dn,"Unigram\u6807\u8BB0\u5316"),dn.forEach(a),Fs.forEach(a),C=c(s),d(B.$$.fragment,s),is=c(s),Gs=r(s,"P",{});var _n=m(Gs);_e=p(_n,"\u5728 SentencePiece \u4E2D\u7ECF\u5E38\u4F7F\u7528 Unigram \u7B97\u6CD5,\u8BE5\u7B97\u6CD5\u662F AlBERT\u3001T5\u3001mBART\u3001Big Bird \u548C XLNet \u7B49\u6A21\u578B\u4F7F\u7528\u7684\u6807\u8BB0\u5316\u7B97\u6CD5\u3002"),_n.forEach(a),$n=c(s),d(cs.$$.fragment,s),bn=c(s),d(G.$$.fragment,s),wn=c(s),us=r(s,"H2",{id:!0});var Mp=m(us);ge=p(Mp,"\u8BAD\u7EC3\u7B97\u6CD5"),Mp.forEach(a),qn=c(s),Is=r(s,"P",{});var Bp=m(Is);je=p(Bp,"\u4E0E BPE \u548C WordPiece \u76F8\u6BD4,Unigram \u5728\u53E6\u4E00\u4E2A\u65B9\u5411\u4E0A\u5DE5\u4F5C:\u5B83\u4ECE\u4E00\u4E2A\u8F83\u5927\u7684\u8BCD\u6C47\u8868\u5F00\u59CB,\u7136\u540E\u4ECE\u4E2D\u5220\u9664\u6807\u8BB0,\u76F4\u5230\u8FBE\u5230\u6240\u9700\u7684\u8BCD\u6C47\u8868\u5927\u5C0F\u3002\u6709\u591A\u79CD\u9009\u9879\u53EF\u7528\u4E8E\u6784\u5EFA\u57FA\u672C\u8BCD\u6C47\u8868:\u4F8B\u5982,\u6211\u4EEC\u53EF\u4EE5\u91C7\u7528\u9884\u6807\u8BB0\u5316\u5355\u8BCD\u4E2D\u6700\u5E38\u89C1\u7684\u5B50\u4E32,\u6216\u8005\u5728\u5177\u6709\u5927\u8BCD\u6C47\u91CF\u7684\u521D\u59CB\u8BED\u6599\u5E93\u4E0A\u5E94\u7528 BPE\u3002"),Bp.forEach(a),kn=c(s),Ys=r(s,"P",{});var Ap=m(Ys);$e=p(Ap,"\u5728\u8BAD\u7EC3\u7684\u6BCF\u4E00\u6B65,Unigram \u7B97\u6CD5\u90FD\u4F1A\u5728\u7ED9\u5B9A\u5F53\u524D\u8BCD\u6C47\u7684\u60C5\u51B5\u4E0B\u8BA1\u7B97\u8BED\u6599\u5E93\u7684\u635F\u5931\u3002\u7136\u540E,\u5BF9\u4E8E\u8BCD\u6C47\u8868\u4E2D\u7684\u6BCF\u4E2A\u7B26\u53F7,\u7B97\u6CD5\u8BA1\u7B97\u5982\u679C\u5220\u9664\u8BE5\u7B26\u53F7,\u6574\u4F53\u635F\u5931\u4F1A\u589E\u52A0\u591A\u5C11,\u5E76\u5BFB\u627E\u589E\u52A0\u6700\u5C11\u7684\u7B26\u53F7\u3002\u8FD9\u4E9B\u7B26\u53F7\u5BF9\u8BED\u6599\u5E93\u7684\u6574\u4F53\u635F\u5931\u5F71\u54CD\u8F83\u5C0F,\u56E0\u6B64\u4ECE\u67D0\u79CD\u610F\u4E49\u4E0A\u8BF4,\u5B83\u4EEC\u201C\u4E0D\u592A\u9700\u8981\u201D\u5E76\u4E14\u662F\u79FB\u9664\u7684\u6700\u4F73\u5019\u9009\u8005\u3002"),Ap.forEach(a),yn=c(s),A=r(s,"P",{});var ka=m(A);be=p(ka,"\u8FD9\u662F\u4E00\u4E2A\u975E\u5E38\u6602\u8D35\u7684\u64CD\u4F5C,\u6240\u4EE5\u6211\u4EEC\u4E0D\u53EA\u662F\u5220\u9664\u4E0E\u6700\u4F4E\u635F\u5931\u589E\u52A0\u76F8\u5173\u7684\u5355\u4E2A\u7B26\u53F7,\u800C\u4E14"),vn=de(ka),En=p(ka," ("),xn=de(ka),Pn=p(ka,"\u662F\u4E00\u4E2A\u53EF\u4EE5\u63A7\u5236\u7684\u8D85\u53C2\u6570,\u901A\u5E38\u662F 10 \u6216 20)\u4E0E\u6700\u4F4E\u635F\u5931\u589E\u52A0\u76F8\u5173\u7684\u7B26\u53F7\u7684\u767E\u5206\u6BD4\u3002\u7136\u540E\u91CD\u590D\u8FD9\u4E2A\u8FC7\u7A0B,\u76F4\u5230\u8BCD\u6C47\u91CF\u8FBE\u5230\u6240\u9700\u7684\u5927\u5C0F\u3002"),ka.forEach(a),Cn=c(s),Zs=r(s,"P",{});var Lp=m(Zs);we=p(Lp,"\u8BF7\u6CE8\u610F,\u6211\u4EEC\u4ECE\u4E0D\u5220\u9664\u57FA\u672C\u5B57\u7B26,\u4EE5\u786E\u4FDD\u53EF\u4EE5\u6807\u8BB0\u4EFB\u4F55\u5355\u8BCD\u3002"),Lp.forEach(a),Tn=c(s),Js=r(s,"P",{});var Wp=m(Js);qe=p(Wp,"\u73B0\u5728,\u8FD9\u4ECD\u7136\u6709\u70B9\u6A21\u7CCA:\u7B97\u6CD5\u7684\u4E3B\u8981\u90E8\u5206\u662F\u8BA1\u7B97\u8BED\u6599\u5E93\u7684\u635F\u5931,\u5E76\u67E5\u770B\u5F53\u6211\u4EEC\u4ECE\u8BCD\u6C47\u8868\u4E2D\u5220\u9664\u4E00\u4E9B\u6807\u8BB0\u65F6\u5B83\u4F1A\u5982\u4F55\u53D8\u5316,\u4F46\u6211\u4EEC\u8FD8\u6CA1\u6709\u89E3\u91CA\u5982\u4F55\u505A\u5230\u8FD9\u4E00\u70B9\u3002\u8FD9\u4E00\u6B65\u4F9D\u8D56\u4E8E Unigram \u6A21\u578B\u7684\u6807\u8BB0\u5316\u7B97\u6CD5,\u56E0\u6B64\u6211\u4EEC\u63A5\u4E0B\u6765\u5C06\u6DF1\u5165\u7814\u7A76\u3002"),Wp.forEach(a),zn=c(s),Ks=r(s,"P",{});var Xp=m(Ks);ke=p(Xp,"\u6211\u4EEC\u5C06\u91CD\u7528\u524D\u9762\u793A\u4F8B\u4E2D\u7684\u8BED\u6599\u5E93:"),Xp.forEach(a),On=c(s),d(hs.$$.fragment,s),Dn=c(s),Qs=r(s,"P",{});var Rp=m(Qs);ye=p(Rp,"\u5BF9\u4E8E\u6B64\u793A\u4F8B,\u6211\u4EEC\u5C06\u91C7\u7528\u521D\u59CB\u8BCD\u6C47\u8868\u7684\u6240\u6709\u4E25\u683C\u5B50\u5B57\u7B26\u4E32:"),Rp.forEach(a),Un=c(s),d(fs.$$.fragment,s),Nn=c(s),ds=r(s,"H2",{id:!0});var Fp=m(ds);ve=p(Fp,"\u6807\u8BB0\u5316\u7B97\u6CD5"),Fp.forEach(a),Hn=c(s),sa=r(s,"P",{});var Vp=m(sa);Ee=p(Vp,"Unigram \u6A21\u578B\u662F\u4E00\u79CD\u8BED\u8A00\u6A21\u578B,\u5B83\u8BA4\u4E3A\u6BCF\u4E2A\u6807\u8BB0\u90FD\u72EC\u7ACB\u4E8E\u5B83\u4E4B\u524D\u7684\u6807\u8BB0\u3002\u5B83\u662F\u6700\u7B80\u5355\u7684\u8BED\u8A00\u6A21\u578B,\u4ECE\u67D0\u79CD\u610F\u4E49\u4E0A\u8BF4, \u7ED9\u5B9A\u5148\u524D\u4E0A\u4E0B\u6587\u7684\u6807\u8BB0 X \u7684\u6982\u7387\u5C31\u662F\u6807\u8BB0 X \u7684\u6982\u7387\u3002\u56E0\u6B64,\u5982\u679C\u6211\u4EEC\u4F7F\u7528 Unigram \u8BED\u8A00\u6A21\u578B\u751F\u6210\u6587\u672C,\u6211\u4EEC\u5C06\u59CB\u7EC8\u9884\u6D4B\u6700\u5E38\u89C1\u7684\u6807\u8BB0\u3002"),Vp.forEach(a),Sn=c(s),x=r(s,"P",{});var X=m(x);xe=p(X,"\u7ED9\u5B9A\u6807\u8BB0\u7684\u6982\u7387\u662F\u5B83\u5728\u539F\u59CB\u8BED\u6599\u5E93\u4E2D\u7684\u9891\u7387(\u6211\u4EEC\u627E\u5230\u5B83\u7684\u6B21\u6570),\u9664\u4EE5\u8BCD\u6C47\u8868\u4E2D\u6240\u6709\u6807\u8BB0\u7684\u6240\u6709\u9891\u7387\u7684\u603B\u548C(\u4EE5\u786E\u4FDD\u6982\u7387\u603B\u548C\u4E3A 1)\u3002\u4F8B\u5982, "),Ea=r(X,"CODE",{});var Gp=m(Ea);Pe=p(Gp,'"ug"'),Gp.forEach(a),Ce=p(X," \u5728 "),xa=r(X,"CODE",{});var Ip=m(xa);Te=p(Ip,'"hug"'),Ip.forEach(a),ze=p(X," \u3001 "),Pa=r(X,"CODE",{});var Yp=m(Pa);Oe=p(Yp,'"pug"'),Yp.forEach(a),De=p(X," \u4EE5\u53CA "),Ca=r(X,"CODE",{});var Zp=m(Ca);Ue=p(Zp,'"hugs"'),Zp.forEach(a),Ne=p(X," \u4E2D,\u6240\u4EE5\u5B83\u5728\u6211\u4EEC\u7684\u8BED\u6599\u5E93\u4E2D\u7684\u9891\u7387\u4E3A 20\u3002"),X.forEach(a),Mn=c(s),aa=r(s,"P",{});var Jp=m(aa);He=p(Jp,"\u4EE5\u4E0B\u662F\u8BCD\u6C47\u8868\u4E2D\u6240\u6709\u53EF\u80FD\u7684\u5B50\u8BCD\u7684\u51FA\u73B0\u9891\u7387:"),Jp.forEach(a),Bn=c(s),d(_s.$$.fragment,s),An=c(s),I=r(s,"P",{});var le=m(I);Se=p(le,"\u6240\u4EE5,\u6240\u6709\u9891\u7387\u4E4B\u548C\u4E3A210, \u5E76\u4E14\u5B50\u8BCD "),Ta=r(le,"CODE",{});var Kp=m(Ta);Me=p(Kp,'"ug"'),Kp.forEach(a),Be=p(le," \u51FA\u73B0\u7684\u6982\u7387\u662F 20/210\u3002"),le.forEach(a),Ln=c(s),d(Y.$$.fragment,s),Wn=c(s),M=r(s,"P",{});var Vs=m(M);Ae=p(Vs,"\u73B0\u5728,\u4E3A\u4E86\u5BF9\u7ED9\u5B9A\u7684\u5355\u8BCD\u8FDB\u884C\u6807\u8BB0,\u6211\u4EEC\u5C06\u6240\u6709\u53EF\u80FD\u7684\u5206\u5272\u89C6\u4E3A\u6807\u8BB0,\u5E76\u6839\u636E Unigram \u6A21\u578B\u8BA1\u7B97\u6BCF\u4E2A\u5206\u5272\u7684\u6982\u7387\u3002\u7531\u4E8E\u6240\u6709\u6807\u8BB0\u90FD\u88AB\u8BA4\u4E3A\u662F\u72EC\u7ACB\u7684,\u6240\u4EE5\u8FD9\u4E2A\u6982\u7387\u53EA\u662F\u6BCF\u4E2A\u6807\u8BB0\u6982\u7387\u7684\u4E58\u79EF\u3002\u4F8B\u5982, "),za=r(Vs,"CODE",{});var Qp=m(za);Le=p(Qp,'"pug"'),Qp.forEach(a),We=p(Vs," \u7684\u6807\u8BB0\u5316 "),Oa=r(Vs,"CODE",{});var so=m(Oa);Xe=p(so,'["p", "u", "g"]'),so.forEach(a),Re=p(Vs,` \u7684\u6982\u7387\u4E3A:
`),Xn=de(Vs),Vs.forEach(a),Rn=c(s),F=r(s,"P",{});var gn=m(F);Fe=p(gn,"\u76F8\u6BD4\u4E4B\u4E0B,\u6807\u8BB0\u5316 "),Da=r(gn,"CODE",{});var ao=m(Da);Ve=p(ao,'["pu", "g"]'),ao.forEach(a),Ge=p(gn,` \u7684\u6982\u7387\u4E3A:
`),Fn=de(gn),gn.forEach(a),Vn=c(s),na=r(s,"P",{});var no=m(na);Ie=p(no,"\u6240\u4EE5\u4E00\u4E2A\u66F4\u6709\u53EF\u80FD\u3002\u4E00\u822C\u6765\u8BF4,\u5177\u6709\u5C3D\u53EF\u80FD\u5C11\u7684\u6807\u8BB0\u7684\u6807\u8BB0\u5316\u5C06\u5177\u6709\u6700\u9AD8\u7684\u6982\u7387(\u56E0\u4E3A\u6BCF\u4E2A\u6807\u8BB0\u91CD\u590D\u9664\u4EE5 210),\u8FD9\u5BF9\u5E94\u4E8E\u6211\u4EEC\u76F4\u89C2\u60F3\u8981\u7684:\u5C06\u4E00\u4E2A\u5355\u8BCD\u5206\u6210\u5C3D\u53EF\u80FD\u5C11\u7684\u6807\u8BB0\u3002"),no.forEach(a),Gn=c(s),Z=r(s,"P",{});var pe=m(Z);Ye=p(pe,"\u4F7F\u7528 Unigram \u6A21\u578B\u5BF9\u5355\u8BCD\u8FDB\u884C\u5206\u8BCD\u662F\u6982\u7387\u6700\u9AD8\u7684\u5206\u8BCD\u3002\u5728\u793A\u4F8B "),Ua=r(pe,"CODE",{});var to=m(Ua);Ze=p(to,'"pug"'),to.forEach(a),Je=p(pe," \u4E2D,\u8FD9\u91CC\u662F\u6211\u4EEC\u4E3A\u6BCF\u4E2A\u53EF\u80FD\u7684\u5206\u5272\u83B7\u5F97\u7684\u6982\u7387:"),pe.forEach(a),In=c(s),d(gs.$$.fragment,s),Yn=c(s),D=r(s,"P",{});var os=m(D);Ke=p(os,"\u6240\u4EE5, "),Na=r(os,"CODE",{});var eo=m(Na);Qe=p(eo,'"pug"'),eo.forEach(a),sl=p(os," \u5C06\u88AB\u6807\u8BB0\u4E3A "),Ha=r(os,"CODE",{});var lo=m(Ha);al=p(lo,'["p", "ug"]'),lo.forEach(a),nl=p(os," \u6216\u8005 "),Sa=r(os,"CODE",{});var po=m(Sa);tl=p(po,'["pu", "g"]'),po.forEach(a),el=p(os,",  \u53D6\u51B3\u4E8E\u9996\u5148\u9047\u5230\u8FD9\u4E9B\u5206\u5272\u4E2D\u7684\u54EA\u4E00\u4E2A(\u8BF7\u6CE8\u610F,\u5728\u66F4\u5927\u7684\u8BED\u6599\u5E93\u4E2D,\u8FD9\u6837\u7684\u76F8\u7B49\u7684\u60C5\u51B5\u5F88\u5C11\u89C1)\u3002"),os.forEach(a),Zn=c(s),y=r(s,"P",{});var H=m(y);ll=p(H,"\u5728\u8FD9\u79CD\u60C5\u51B5\u4E0B,\u5F88\u5BB9\u6613\u627E\u5230\u6240\u6709\u53EF\u80FD\u7684\u5206\u5272\u5E76\u8BA1\u7B97\u5B83\u4EEC\u7684\u6982\u7387,\u4F46\u4E00\u822C\u6765\u8BF4\u4F1A\u6709\u70B9\u56F0\u96BE\u3002\u6709\u4E00\u79CD\u7528\u4E8E\u6B64\u7684\u7ECF\u5178\u7B97\u6CD5,\u79F0\u4E3A "),Ma=r(H,"EM",{});var oo=m(Ma);pl=p(oo,"\u7EF4\u7279\u6BD4(Viterbi)\u7B97\u6CD5"),oo.forEach(a),ol=p(H,"\u3002\u672C\u8D28\u4E0A,\u6211\u4EEC\u53EF\u4EE5\u6784\u5EFA\u4E00\u4E2A\u56FE\u6765\u68C0\u6D4B\u7ED9\u5B9A\u5355\u8BCD\u7684\u53EF\u80FD\u5206\u5272,\u5982\u679C\u4ECE"),Ba=r(H,"EM",{});var ro=m(Ba);rl=p(ro,"a"),ro.forEach(a),ml=p(H,"\u5230"),Aa=r(H,"EM",{});var mo=m(Aa);il=p(mo,"b"),mo.forEach(a),cl=p(H,"\u7684\u5B50\u8BCD\u5728\u8BCD\u6C47\u8868\u4E2D,\u5219\u4ECE\u5B57\u7B26"),La=r(H,"EM",{});var io=m(La);ul=p(io,"a"),io.forEach(a),hl=p(H,"\u5230\u5B57\u7B26"),Wa=r(H,"EM",{});var co=m(Wa);fl=p(co,"b"),co.forEach(a),dl=p(H,"\u4E4B\u95F4\u5B58\u5728\u4E00\u4E2A\u5206\u652F,\u5E76\u5C06\u5B50\u8BCD\u7684\u6982\u7387\u5F52\u56E0\u4E8E\u8BE5\u5206\u652F\u3002"),H.forEach(a),Jn=c(s),ta=r(s,"P",{});var uo=m(ta);_l=p(uo,"\u4E3A\u4E86\u5728\u8BE5\u56FE\u4E2D\u627E\u5230\u5C06\u5177\u6709\u6700\u4F73\u5206\u6570\u7684\u8DEF\u5F84,\u7EF4\u7279\u6BD4\u7B97\u6CD5\u4E3A\u5355\u8BCD\u4E2D\u7684\u6BCF\u4E2A\u4F4D\u7F6E\u786E\u5B9A\u5728\u8BE5\u4F4D\u7F6E\u7ED3\u675F\u7684\u5177\u6709\u6700\u4F73\u5206\u6570\u7684\u5206\u6BB5\u3002\u7531\u4E8E\u6211\u4EEC\u4ECE\u5F00\u59CB\u5230\u7ED3\u675F,\u53EF\u4EE5\u901A\u8FC7\u5FAA\u73AF\u904D\u5386\u4EE5\u5F53\u524D\u4F4D\u7F6E\u7ED3\u5C3E\u7684\u6240\u6709\u5B50\u8BCD,\u7136\u540E\u4F7F\u7528\u8BE5\u5B50\u8BCD\u5F00\u59CB\u4F4D\u7F6E\u7684\u6700\u4F73\u6807\u8BB0\u5316\u5206\u6570\u6765\u627E\u5230\u6700\u4F73\u5206\u6570\u3002\u7136\u540E,\u6211\u4EEC\u53EA\u9700\u8981\u5C55\u5F00\u5230\u8FBE\u7EC8\u70B9\u6240\u91C7\u53D6\u7684\u8DEF\u5F84\u3002"),uo.forEach(a),Kn=c(s),J=r(s,"P",{});var oe=m(J);gl=p(oe,"\u8BA9\u6211\u4EEC\u770B\u4E00\u4E2A\u4F7F\u7528\u6211\u4EEC\u7684\u8BCD\u6C47\u8868\u548C\u5355\u8BCD "),Xa=r(oe,"CODE",{});var ho=m(Xa);jl=p(ho,'"unhug"'),ho.forEach(a),$l=p(oe," \u7684\u4F8B\u5B50\u3002\u5BF9\u4E8E\u6BCF\u4E2A\u4F4D\u7F6E,\u4EE5\u6700\u597D\u7684\u5206\u6570\u7ED3\u5C3E\u7684\u5B50\u8BCD\u5982\u4E0B:"),oe.forEach(a),Qn=c(s),d(js.$$.fragment,s),st=c(s),L=r(s,"P",{});var ya=m(L);bl=p(ya,"\u56E0\u6B64 "),Ra=r(ya,"CODE",{});var fo=m(Ra);wl=p(fo,'"unhug"'),fo.forEach(a),ql=p(ya," \u5C06\u88AB\u6807\u8BB0\u4E3A "),Fa=r(ya,"CODE",{});var _o=m(Fa);kl=p(_o,'["un", "hug"]'),_o.forEach(a),yl=p(ya,"\u3002"),ya.forEach(a),at=c(s),d(K.$$.fragment,s),nt=c(s),$s=r(s,"H2",{id:!0});var go=m($s);vl=p(go,"\u56DE\u5230\u8BAD\u7EC3"),go.forEach(a),tt=c(s),ea=r(s,"P",{});var jo=m(ea);El=p(jo,"\u73B0\u5728\u6211\u4EEC\u5DF2\u7ECF\u4E86\u89E3\u4E86\u6807\u8BB0\u5316\u7684\u5DE5\u4F5C\u539F\u7406,\u6211\u4EEC\u53EF\u4EE5\u66F4\u6DF1\u5165\u5730\u7814\u7A76\u8BAD\u7EC3\u671F\u95F4\u4F7F\u7528\u7684\u635F\u5931\u3002\u5728\u4EFB\u4F55\u7ED9\u5B9A\u7684\u9636\u6BB5,\u8FD9\u4E2A\u635F\u5931\u662F\u901A\u8FC7\u5BF9\u8BED\u6599\u5E93\u4E2D\u7684\u6BCF\u4E2A\u5355\u8BCD\u8FDB\u884C\u6807\u8BB0\u6765\u8BA1\u7B97\u7684,\u4F7F\u7528\u5F53\u524D\u8BCD\u6C47\u8868\u548C\u7531\u8BED\u6599\u5E93\u4E2D\u6BCF\u4E2A\u6807\u8BB0\u7684\u9891\u7387\u786E\u5B9A\u7684 Unigram \u6A21\u578B(\u5982\u524D\u6240\u8FF0)\u3002"),jo.forEach(a),et=c(s),Q=r(s,"P",{});var re=m(Q);xl=p(re,"\u8BED\u6599\u5E93\u4E2D\u7684\u6BCF\u4E2A\u8BCD\u90FD\u6709\u4E00\u4E2A\u5206\u6570,\u635F\u5931\u662F\u8FD9\u4E9B\u5206\u6570\u7684\u8D1F\u5BF9\u6570\u4F3C\u7136 \u2014 \u5373\u6240\u6709\u8BCD\u7684\u8BED\u6599\u5E93\u4E2D\u6240\u6709\u8BCD\u7684\u603B\u548C "),Va=r(re,"CODE",{});var $o=m(Va);Pl=p($o,"-log(P(word))"),$o.forEach(a),Cl=p(re,"\u3002"),re.forEach(a),lt=c(s),la=r(s,"P",{});var bo=m(la);Tl=p(bo,"\u8BA9\u6211\u4EEC\u7528\u4EE5\u4E0B\u8BED\u6599\u5E93\u56DE\u5230\u6211\u4EEC\u7684\u4F8B\u5B50:"),bo.forEach(a),pt=c(s),d(bs.$$.fragment,s),ot=c(s),pa=r(s,"P",{});var wo=m(pa);zl=p(wo,"\u6BCF\u4E2A\u5355\u8BCD\u7684\u6807\u8BB0\u5316\u53CA\u5176\u5404\u81EA\u7684\u5206\u6570\u662F:"),wo.forEach(a),rt=c(s),d(ws.$$.fragment,s),mt=c(s),oa=r(s,"P",{});var qo=m(oa);Ol=p(qo,"\u6240\u4EE5\u635F\u5931\u662F:"),qo.forEach(a),it=c(s),d(qs.$$.fragment,s),ct=c(s),U=r(s,"P",{});var rs=m(U);Dl=p(rs,"\u73B0\u5728\u6211\u4EEC\u9700\u8981\u8BA1\u7B97\u5220\u9664\u6BCF\u4E2A\u6807\u8BB0\u5982\u4F55\u5F71\u54CD\u635F\u5931\u3002\u8FD9\u76F8\u5F53\u4E4F\u5473,\u6240\u4EE5\u6211\u4EEC\u5728\u8FD9\u91CC\u53EA\u5BF9\u4E24\u4E2A\u6807\u8BB0\u8FDB\u884C\u64CD\u4F5C,\u5E76\u4FDD\u5B58\u6574\u4E2A\u8FC7\u7A0B\u4EE5\u5907\u6709\u4EE3\u7801\u6765\u5E2E\u52A9\u6211\u4EEC\u3002\u5728\u8FD9\u4E2A(\u975E\u5E38)\u7279\u6B8A\u7684\u60C5\u51B5\u4E0B,\u6211\u4EEC\u5BF9\u6240\u6709\u5355\u8BCD\u6709\u4E24\u4E2A\u7B49\u6548\u7684\u6807\u8BB0:\u6B63\u5982\u6211\u4EEC\u4E4B\u524D\u770B\u5230\u7684,\u4F8B\u5982, "),Ga=r(rs,"CODE",{});var ko=m(Ga);Ul=p(ko,'"pug"'),ko.forEach(a),Nl=p(rs," \u53EF\u4EE5\u4EE5\u76F8\u540C\u7684\u5206\u6570\u88AB\u6807\u8BB0\u4E3A "),Ia=r(rs,"CODE",{});var yo=m(Ia);Hl=p(yo,'["p", "ug"]'),yo.forEach(a),Sl=p(rs,"\u3002\u56E0\u6B64,\u53BB\u9664\u8BCD\u6C47\u8868\u4E2D\u7684 "),Ya=r(rs,"CODE",{});var vo=m(Ya);Ml=p(vo,'"pu"'),vo.forEach(a),Bl=p(rs," \u6807\u8BB0\u5C06\u7ED9\u51FA\u5B8C\u5168\u76F8\u540C\u7684\u635F\u5931\u3002"),rs.forEach(a),ut=c(s),N=r(s,"P",{});var ms=m(N);Al=p(ms,"\u53E6\u4E00\u65B9\u9762,\u53BB\u9664 "),Za=r(ms,"CODE",{});var Eo=m(Za);Ll=p(Eo,'"hug"'),Eo.forEach(a),Wl=p(ms," \u635F\u5931\u53D8\u5F97\u66F4\u7CDF, \u56E0\u4E3A "),Ja=r(ms,"CODE",{});var xo=m(Ja);Xl=p(xo,'"hug"'),xo.forEach(a),Rl=p(ms," \u548C "),Ka=r(ms,"CODE",{});var Po=m(Ka);Fl=p(Po,'"hugs"'),Po.forEach(a),Vl=p(ms," \u7684\u6807\u8BB0\u5316\u4F1A\u53D8\u6210:"),ms.forEach(a),ht=c(s),d(ks.$$.fragment,s),ft=c(s),ra=r(s,"P",{});var Co=m(ra);Gl=p(Co,"\u8FD9\u4E9B\u53D8\u5316\u5C06\u5BFC\u81F4\u635F\u5931\u589E\u52A0:"),Co.forEach(a),dt=c(s),d(ys.$$.fragment,s),_t=c(s),W=r(s,"P",{});var va=m(W);Il=p(va,"\u56E0\u6B64, \u6807\u8BB0 "),Qa=r(va,"CODE",{});var To=m(Qa);Yl=p(To,'"pu"'),To.forEach(a),Zl=p(va,"\u53EF\u80FD\u4F1A\u4ECE\u8BCD\u6C47\u8868\u4E2D\u5220\u9664,\u4F46\u4E0D\u4F1A\u5220\u9664 "),sn=r(va,"CODE",{});var zo=m(sn);Jl=p(zo,'"hug"'),zo.forEach(a),Kl=p(va,"."),va.forEach(a),gt=c(s),V=r(s,"H2",{class:!0});var me=m(V);ss=r(me,"A",{id:!0,class:!0,href:!0});var Oo=m(ss);an=r(Oo,"SPAN",{});var Do=m(an);d(vs.$$.fragment,Do),Do.forEach(a),Oo.forEach(a),Ql=c(me),nn=r(me,"SPAN",{});var Uo=m(nn);sp=p(Uo,"\u5B9E\u73B0 Unigram"),Uo.forEach(a),me.forEach(a),jt=c(s),ma=r(s,"P",{});var No=m(ma);ap=p(No,"\u73B0\u5728\u8BA9\u6211\u4EEC\u5728\u4EE3\u7801\u4E2D\u5B9E\u73B0\u6211\u4EEC\u8FC4\u4ECA\u4E3A\u6B62\u770B\u5230\u7684\u6240\u6709\u5185\u5BB9\u3002\u4E0E BPE \u548C WordPiece \u4E00\u6837,\u8FD9\u4E0D\u662F Unigram \u7B97\u6CD5\u7684\u6709\u6548\u5B9E\u73B0(\u6070\u6070\u76F8\u53CD),\u4F46\u5B83\u5E94\u8BE5\u53EF\u4EE5\u5E2E\u52A9\u4F60\u66F4\u597D\u5730\u7406\u89E3\u5B83\u3002"),No.forEach(a),$t=c(s),ia=r(s,"P",{});var Ho=m(ia);np=p(Ho,"\u6211\u4EEC\u5C06\u4F7F\u7528\u4E0E\u4E4B\u524D\u76F8\u540C\u7684\u8BED\u6599\u5E93\u4F5C\u4E3A\u793A\u4F8B:"),Ho.forEach(a),bt=c(s),d(Es.$$.fragment,s),wt=c(s),as=r(s,"P",{});var ie=m(as);tp=p(ie,"\u8FD9\u4E00\u6B21,\u6211\u4EEC\u5C06\u4F7F\u7528 "),tn=r(ie,"CODE",{});var So=m(tn);ep=p(So,"xlnet-base-cased"),So.forEach(a),lp=p(ie," \u4F5C\u4E3A\u6211\u4EEC\u7684\u6A21\u578B:"),ie.forEach(a),qt=c(s),d(xs.$$.fragment,s),kt=c(s),ca=r(s,"P",{});var Mo=m(ca);pp=p(Mo,"\u4E0E BPE \u548C WordPiece \u4E00\u6837,\u6211\u4EEC\u9996\u5148\u8BA1\u7B97\u8BED\u6599\u5E93\u4E2D\u6BCF\u4E2A\u5355\u8BCD\u7684\u51FA\u73B0\u6B21\u6570:"),Mo.forEach(a),yt=c(s),d(Ps.$$.fragment,s),vt=c(s),ua=r(s,"P",{});var Bo=m(ua);op=p(Bo,"\u7136\u540E,\u6211\u4EEC\u9700\u8981\u5C06\u6211\u4EEC\u7684\u8BCD\u6C47\u8868\u521D\u59CB\u5316\u4E3A\u5927\u4E8E\u6211\u4EEC\u6700\u7EC8\u60F3\u8981\u7684\u8BCD\u6C47\u91CF\u3002\u6211\u4EEC\u5FC5\u987B\u5305\u542B\u6240\u6709\u57FA\u672C\u5B57\u7B26(\u5426\u5219\u6211\u4EEC\u5C06\u65E0\u6CD5\u6807\u8BB0\u6BCF\u4E2A\u5355\u8BCD),\u4F46\u5BF9\u4E8E\u8F83\u5927\u7684\u5B50\u5B57\u7B26\u4E32,\u6211\u4EEC\u5C06\u53EA\u4FDD\u7559\u6700\u5E38\u89C1\u7684\u5B57\u7B26,\u56E0\u6B64\u6211\u4EEC\u6309\u9891\u7387\u5BF9\u5B83\u4EEC\u8FDB\u884C\u6392\u5E8F:"),Bo.forEach(a),Et=c(s),d(Cs.$$.fragment,s),xt=c(s),d(Ts.$$.fragment,s),Pt=c(s),ha=r(s,"P",{});var Ao=m(ha);rp=p(Ao,"\u6211\u4EEC\u7528\u6700\u4F18\u7684\u5B50\u8BCD\u5BF9\u5B57\u7B26\u8FDB\u884C\u5206\u7EC4,\u4EE5\u83B7\u5F97\u5927\u5C0F\u4E3A 300 \u7684\u521D\u59CB\u8BCD\u6C47\u8868:"),Ao.forEach(a),Ct=c(s),d(zs.$$.fragment,s),Tt=c(s),d(ns.$$.fragment,s),zt=c(s),fa=r(s,"P",{});var Lo=m(fa);mp=p(Lo,"\u63A5\u4E0B\u6765,\u6211\u4EEC\u8BA1\u7B97\u6240\u6709\u9891\u7387\u7684\u603B\u548C,\u5C06\u9891\u7387\u8F6C\u6362\u4E3A\u6982\u7387\u3002\u5BF9\u4E8E\u6211\u4EEC\u7684\u6A21\u578B,\u6211\u4EEC\u5C06\u5B58\u50A8\u6982\u7387\u7684\u5BF9\u6570,\u56E0\u4E3A\u6DFB\u52A0\u5BF9\u6570\u6BD4\u4E58\u4EE5\u5C0F\u6570\u5728\u6570\u503C\u4E0A\u66F4\u7A33\u5B9A,\u8FD9\u5C06\u7B80\u5316\u6A21\u578B\u635F\u5931\u7684\u8BA1\u7B97:"),Lo.forEach(a),Ot=c(s),d(Os.$$.fragment,s),Dt=c(s),ts=r(s,"P",{});var ce=m(ts);ip=p(ce,"N\u73B0\u5728\u4E3B\u8981\u529F\u80FD\u662F\u4F7F\u7528 Viterbi \u7B97\u6CD5\u6807\u8BB0\u5355\u8BCD\u7684\u529F\u80FD\u3002\u6B63\u5982\u6211\u4EEC\u4E4B\u524D\u770B\u5230\u7684,\u8BE5\u7B97\u6CD5\u8BA1\u7B97\u5355\u8BCD\u7684\u6BCF\u4E2A\u5B50\u4E32\u7684\u6700\u4F73\u5206\u6BB5,\u6211\u4EEC\u5C06\u5176\u5B58\u50A8\u5728\u540D\u4E3A "),en=r(ce,"CODE",{});var Wo=m(en);cp=p(Wo,"best_segmentations"),Wo.forEach(a),up=p(ce," \u7684\u53D8\u91CF\u4E2D\u3002\u6211\u4EEC\u5C06\u5728\u5355\u8BCD\u7684\u6BCF\u4E2A\u4F4D\u7F6E(\u4ECE 0 \u5230\u5176\u603B\u957F\u5EA6)\u5B58\u50A8\u4E00\u4E2A\u5B57\u5178,\u6709\u4E24\u4E2A\u952E:\u6700\u4F73\u5206\u5272\u4E2D\u6700\u540E\u4E00\u4E2A\u6807\u8BB0\u7684\u5F00\u59CB\u7D22\u5F15,\u4EE5\u53CA\u6700\u4F73\u5206\u5272\u7684\u5206\u6570\u3002\u4F7F\u7528\u6700\u540E\u4E00\u4E2A\u6807\u8BB0\u7684\u5F00\u59CB\u7D22\u5F15,\u4E00\u65E6\u5217\u8868\u5B8C\u5168\u586B\u5145,\u6211\u4EEC\u5C06\u80FD\u591F\u68C0\u7D22\u5B8C\u6574\u7684\u5206\u6BB5\u3002"),ce.forEach(a),Ut=c(s),es=r(s,"P",{});var ue=m(es);hp=p(ue,"\u586B\u5145\u5217\u8868\u53EA\u9700\u4E24\u4E2A\u5FAA\u73AF:\u4E3B\u5FAA\u73AF\u904D\u5386\u6BCF\u4E2A\u8D77\u59CB\u4F4D\u7F6E,\u7B2C\u4E8C\u4E2A\u5FAA\u73AF\u5C1D\u8BD5\u4ECE\u8BE5\u8D77\u59CB\u4F4D\u7F6E\u5F00\u59CB\u7684\u6240\u6709\u5B50\u5B57\u7B26\u4E32\u3002\u5982\u679C\u5B50\u4E32\u5728\u8BCD\u6C47\u8868\u4E2D,\u6211\u4EEC\u6709\u4E00\u4E2A\u65B0\u7684\u8BCD\u5206\u6BB5,\u76F4\u5230\u8BE5\u7ED3\u675F\u4F4D\u7F6E,\u6211\u4EEC\u5C06\u5176\u4E0E "),ln=r(ue,"CODE",{});var Xo=m(ln);fp=p(Xo,"best_segmentations"),Xo.forEach(a),dp=p(ue," \u76F8\u6BD4\u8F83\u3002"),ue.forEach(a),Nt=c(s),da=r(s,"P",{});var Ro=m(da);_p=p(Ro,"\u4E00\u65E6\u4E3B\u5FAA\u73AF\u5B8C\u6210,\u6211\u4EEC\u5C31\u4ECE\u7ED3\u5C3E\u5F00\u59CB,\u4ECE\u4E00\u4E2A\u5F00\u59CB\u4F4D\u7F6E\u8DF3\u5230\u4E0B\u4E00\u4E2A,\u8BB0\u5F55\u6211\u4EEC\u524D\u8FDB\u7684\u6807\u8BB0,\u76F4\u5230\u6211\u4EEC\u5230\u8FBE\u5355\u8BCD\u7684\u5F00\u5934:"),Ro.forEach(a),Ht=c(s),d(Ds.$$.fragment,s),St=c(s),_a=r(s,"P",{});var Fo=m(_a);gp=p(Fo,"\u6211\u4EEC\u5DF2\u7ECF\u53EF\u4EE5\u5728\u4E00\u4E9B\u8BCD\u4E0A\u5C1D\u8BD5\u6211\u4EEC\u7684\u521D\u59CB\u6A21\u578B:"),Fo.forEach(a),Mt=c(s),d(Us.$$.fragment,s),Bt=c(s),d(Ns.$$.fragment,s),At=c(s),ga=r(s,"P",{});var Vo=m(ga);jp=p(Vo,"\u73B0\u5728\u5F88\u5BB9\u6613\u8BA1\u7B97\u6A21\u578B\u5728\u8BED\u6599\u5E93\u4E0A\u7684\u635F\u5931!"),Vo.forEach(a),Lt=c(s),d(Hs.$$.fragment,s),Wt=c(s),ja=r(s,"P",{});var Go=m(ja);$p=p(Go,"\u6211\u4EEC\u53EF\u4EE5\u68C0\u67E5\u5B83\u662F\u5426\u9002\u7528\u4E8E\u6211\u4EEC\u62E5\u6709\u7684\u6A21\u578B:"),Go.forEach(a),Xt=c(s),d(Ss.$$.fragment,s),Rt=c(s),d(Ms.$$.fragment,s),Ft=c(s),$a=r(s,"P",{});var Io=m($a);bp=p(Io,"\u8BA1\u7B97\u6BCF\u4E2A\u6807\u8BB0\u7684\u5206\u6570\u4E5F\u4E0D\u662F\u5F88\u96BE;\u6211\u4EEC\u53EA\u9700\u8981\u8BA1\u7B97\u901A\u8FC7\u5220\u9664\u6BCF\u4E2A\u6807\u8BB0\u83B7\u5F97\u7684\u6A21\u578B\u7684\u635F\u5931:"),Io.forEach(a),Vt=c(s),d(Bs.$$.fragment,s),Gt=c(s),ba=r(s,"P",{});var Yo=m(ba);wp=p(Yo,"\u6211\u4EEC\u53EF\u4EE5\u5728\u7ED9\u5B9A\u7684\u6807\u8BB0\u4E0A\u5C1D\u8BD5:"),Yo.forEach(a),It=c(s),d(As.$$.fragment,s),Yt=c(s),v=r(s,"P",{});var S=m(v);qp=p(S,"\u81EA\u4ECE "),pn=r(S,"CODE",{});var Zo=m(pn);kp=p(Zo,'"ll"'),Zo.forEach(a),yp=p(S," \u7528\u4E8E\u6807\u8BB0\u5316 "),on=r(S,"CODE",{});var Jo=m(on);vp=p(Jo,'"Hopefully"'),Jo.forEach(a),Ep=p(S,", \u5220\u9664\u5B83\u53EF\u80FD\u4F1A\u8BA9\u6211\u4EEC\u4F7F\u7528\u6807\u8BB0 "),rn=r(S,"CODE",{});var Ko=m(rn);xp=p(Ko,'"l"'),Ko.forEach(a),Pp=p(S," \u4E24\u6B21\u76F8\u53CD,\u6211\u4EEC\u9884\u8BA1\u5B83\u5C06\u4EA7\u751F\u6B63\u635F\u5931\u3002 "),mn=r(S,"CODE",{});var Qo=m(mn);Cp=p(Qo,'"his"'),Qo.forEach(a),Tp=p(S," \u4EC5\u5728\u5355\u8BCD"),cn=r(S,"CODE",{});var sr=m(cn);zp=p(sr,'"This"'),sr.forEach(a),Op=p(S," \u5185\u4F7F\u7528,\u5B83\u88AB\u6807\u8BB0\u4E3A\u81EA\u8EAB,\u6240\u4EE5\u6211\u4EEC\u671F\u671B\u5B83\u7684\u635F\u5931\u4E3A\u96F6\u3002\u7ED3\u679C\u5982\u4E0B:"),S.forEach(a),Zt=c(s),d(Ls.$$.fragment,s),Jt=c(s),d(ls.$$.fragment,s),Kt=c(s),wa=r(s,"P",{});var ar=m(wa);Dp=p(ar,"\u5B8C\u6210\u6240\u6709\u8FD9\u4E9B\u540E,\u6211\u4EEC\u9700\u8981\u505A\u7684\u6700\u540E\u4E00\u4EF6\u4E8B\u662F\u5C06\u6A21\u578B\u4F7F\u7528\u7684\u7279\u6B8A\u6807\u8BB0\u6DFB\u52A0\u5230\u8BCD\u6C47\u8868\u4E2D,\u7136\u540E\u5FAA\u73AF\u76F4\u5230\u6211\u4EEC\u4ECE\u8BCD\u6C47\u8868\u4E2D\u4FEE\u526A\u4E86\u8DB3\u591F\u7684\u6807\u8BB0\u4EE5\u8FBE\u5230\u6211\u4EEC\u60F3\u8981\u7684\u5927\u5C0F:"),ar.forEach(a),Qt=c(s),d(Ws.$$.fragment,s),se=c(s),ps=r(s,"P",{});var he=m(ps);Up=p(he,"\u7136\u540E,\u4E3A\u4E86\u6807\u8BB0\u4E00\u4E9B\u6587\u672C,\u6211\u4EEC\u53EA\u9700\u8981\u5E94\u7528\u9884\u6807\u8BB0\u5316,\u7136\u540E\u4F7F\u7528\u6211\u4EEC\u7684 "),un=r(he,"CODE",{});var nr=m(un);Np=p(nr,"encode_word()"),nr.forEach(a),Hp=p(he," \u51FD\u6570:"),he.forEach(a),ae=c(s),d(Xs.$$.fragment,s),ne=c(s),d(Rs.$$.fragment,s),te=c(s),qa=r(s,"P",{});var tr=m(qa);Sp=p(tr,"Unigram \u5C31\u662F\u8FD9\u6837!\u5E0C\u671B\u73B0\u5728\u4F60\u611F\u89C9\u81EA\u5DF1\u662F\u6807\u8BB0\u5668\u6240\u6709\u65B9\u9762\u7684\u4E13\u5BB6\u3002\u5728\u4E0B\u4E00\u8282\u4E2D,\u6211\u4EEC\u5C06\u6DF1\u5165\u7814\u7A76 \u{1F917} Tokenizers \u5E93\u7684\u6784\u5EFA\u5757,\u5E76\u5411\u60A8\u5C55\u793A\u5982\u4F55\u4F7F\u7528\u5B83\u4EEC\u6765\u6784\u5EFA\u60A8\u81EA\u5DF1\u7684\u6807\u8BB0\u5668\u3002"),tr.forEach(a),this.h()},h(){T(u,"name","hf:doc:metadata"),T(u,"content",JSON.stringify(qr)),T(b,"id","unigram"),T(b,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),T(b,"href","#unigram"),T(h,"class","relative group"),T(us,"id",""),vn.a=En,xn.a=Pn,T(ds,"id",""),Xn.a=null,Fn.a=null,T($s,"id",""),T(ss,"id","unigram"),T(ss,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),T(ss,"href","#unigram"),T(V,"class","relative group")},m(s,n){t(document.head,u),e(s,q,n),e(s,h,n),t(h,b),t(b,P),_(k,P,null),t(h,E),t(h,O),t(O,R),e(s,C,n),_(B,s,n),e(s,is,n),e(s,Gs,n),t(Gs,_e),e(s,$n,n),_(cs,s,n),e(s,bn,n),_(G,s,n),e(s,wn,n),e(s,us,n),t(us,ge),e(s,qn,n),e(s,Is,n),t(Is,je),e(s,kn,n),e(s,Ys,n),t(Ys,$e),e(s,yn,n),e(s,A,n),t(A,be),vn.m(lr,A),t(A,En),xn.m(pr,A),t(A,Pn),e(s,Cn,n),e(s,Zs,n),t(Zs,we),e(s,Tn,n),e(s,Js,n),t(Js,qe),e(s,zn,n),e(s,Ks,n),t(Ks,ke),e(s,On,n),_(hs,s,n),e(s,Dn,n),e(s,Qs,n),t(Qs,ye),e(s,Un,n),_(fs,s,n),e(s,Nn,n),e(s,ds,n),t(ds,ve),e(s,Hn,n),e(s,sa,n),t(sa,Ee),e(s,Sn,n),e(s,x,n),t(x,xe),t(x,Ea),t(Ea,Pe),t(x,Ce),t(x,xa),t(xa,Te),t(x,ze),t(x,Pa),t(Pa,Oe),t(x,De),t(x,Ca),t(Ca,Ue),t(x,Ne),e(s,Mn,n),e(s,aa,n),t(aa,He),e(s,Bn,n),_(_s,s,n),e(s,An,n),e(s,I,n),t(I,Se),t(I,Ta),t(Ta,Me),t(I,Be),e(s,Ln,n),_(Y,s,n),e(s,Wn,n),e(s,M,n),t(M,Ae),t(M,za),t(za,Le),t(M,We),t(M,Oa),t(Oa,Xe),t(M,Re),Xn.m(or,M),e(s,Rn,n),e(s,F,n),t(F,Fe),t(F,Da),t(Da,Ve),t(F,Ge),Fn.m(rr,F),e(s,Vn,n),e(s,na,n),t(na,Ie),e(s,Gn,n),e(s,Z,n),t(Z,Ye),t(Z,Ua),t(Ua,Ze),t(Z,Je),e(s,In,n),_(gs,s,n),e(s,Yn,n),e(s,D,n),t(D,Ke),t(D,Na),t(Na,Qe),t(D,sl),t(D,Ha),t(Ha,al),t(D,nl),t(D,Sa),t(Sa,tl),t(D,el),e(s,Zn,n),e(s,y,n),t(y,ll),t(y,Ma),t(Ma,pl),t(y,ol),t(y,Ba),t(Ba,rl),t(y,ml),t(y,Aa),t(Aa,il),t(y,cl),t(y,La),t(La,ul),t(y,hl),t(y,Wa),t(Wa,fl),t(y,dl),e(s,Jn,n),e(s,ta,n),t(ta,_l),e(s,Kn,n),e(s,J,n),t(J,gl),t(J,Xa),t(Xa,jl),t(J,$l),e(s,Qn,n),_(js,s,n),e(s,st,n),e(s,L,n),t(L,bl),t(L,Ra),t(Ra,wl),t(L,ql),t(L,Fa),t(Fa,kl),t(L,yl),e(s,at,n),_(K,s,n),e(s,nt,n),e(s,$s,n),t($s,vl),e(s,tt,n),e(s,ea,n),t(ea,El),e(s,et,n),e(s,Q,n),t(Q,xl),t(Q,Va),t(Va,Pl),t(Q,Cl),e(s,lt,n),e(s,la,n),t(la,Tl),e(s,pt,n),_(bs,s,n),e(s,ot,n),e(s,pa,n),t(pa,zl),e(s,rt,n),_(ws,s,n),e(s,mt,n),e(s,oa,n),t(oa,Ol),e(s,it,n),_(qs,s,n),e(s,ct,n),e(s,U,n),t(U,Dl),t(U,Ga),t(Ga,Ul),t(U,Nl),t(U,Ia),t(Ia,Hl),t(U,Sl),t(U,Ya),t(Ya,Ml),t(U,Bl),e(s,ut,n),e(s,N,n),t(N,Al),t(N,Za),t(Za,Ll),t(N,Wl),t(N,Ja),t(Ja,Xl),t(N,Rl),t(N,Ka),t(Ka,Fl),t(N,Vl),e(s,ht,n),_(ks,s,n),e(s,ft,n),e(s,ra,n),t(ra,Gl),e(s,dt,n),_(ys,s,n),e(s,_t,n),e(s,W,n),t(W,Il),t(W,Qa),t(Qa,Yl),t(W,Zl),t(W,sn),t(sn,Jl),t(W,Kl),e(s,gt,n),e(s,V,n),t(V,ss),t(ss,an),_(vs,an,null),t(V,Ql),t(V,nn),t(nn,sp),e(s,jt,n),e(s,ma,n),t(ma,ap),e(s,$t,n),e(s,ia,n),t(ia,np),e(s,bt,n),_(Es,s,n),e(s,wt,n),e(s,as,n),t(as,tp),t(as,tn),t(tn,ep),t(as,lp),e(s,qt,n),_(xs,s,n),e(s,kt,n),e(s,ca,n),t(ca,pp),e(s,yt,n),_(Ps,s,n),e(s,vt,n),e(s,ua,n),t(ua,op),e(s,Et,n),_(Cs,s,n),e(s,xt,n),_(Ts,s,n),e(s,Pt,n),e(s,ha,n),t(ha,rp),e(s,Ct,n),_(zs,s,n),e(s,Tt,n),_(ns,s,n),e(s,zt,n),e(s,fa,n),t(fa,mp),e(s,Ot,n),_(Os,s,n),e(s,Dt,n),e(s,ts,n),t(ts,ip),t(ts,en),t(en,cp),t(ts,up),e(s,Ut,n),e(s,es,n),t(es,hp),t(es,ln),t(ln,fp),t(es,dp),e(s,Nt,n),e(s,da,n),t(da,_p),e(s,Ht,n),_(Ds,s,n),e(s,St,n),e(s,_a,n),t(_a,gp),e(s,Mt,n),_(Us,s,n),e(s,Bt,n),_(Ns,s,n),e(s,At,n),e(s,ga,n),t(ga,jp),e(s,Lt,n),_(Hs,s,n),e(s,Wt,n),e(s,ja,n),t(ja,$p),e(s,Xt,n),_(Ss,s,n),e(s,Rt,n),_(Ms,s,n),e(s,Ft,n),e(s,$a,n),t($a,bp),e(s,Vt,n),_(Bs,s,n),e(s,Gt,n),e(s,ba,n),t(ba,wp),e(s,It,n),_(As,s,n),e(s,Yt,n),e(s,v,n),t(v,qp),t(v,pn),t(pn,kp),t(v,yp),t(v,on),t(on,vp),t(v,Ep),t(v,rn),t(rn,xp),t(v,Pp),t(v,mn),t(mn,Cp),t(v,Tp),t(v,cn),t(cn,zp),t(v,Op),e(s,Zt,n),_(Ls,s,n),e(s,Jt,n),_(ls,s,n),e(s,Kt,n),e(s,wa,n),t(wa,Dp),e(s,Qt,n),_(Ws,s,n),e(s,se,n),e(s,ps,n),t(ps,Up),t(ps,un),t(un,Np),t(ps,Hp),e(s,ae,n),_(Xs,s,n),e(s,ne,n),_(Rs,s,n),e(s,te,n),e(s,qa,n),t(qa,Sp),ee=!0},p(s,[n]){const Fs={};n&2&&(Fs.$$scope={dirty:n,ctx:s}),G.$set(Fs);const hn={};n&2&&(hn.$$scope={dirty:n,ctx:s}),Y.$set(hn);const fn={};n&2&&(fn.$$scope={dirty:n,ctx:s}),K.$set(fn);const dn={};n&2&&(dn.$$scope={dirty:n,ctx:s}),ns.$set(dn);const _n={};n&2&&(_n.$$scope={dirty:n,ctx:s}),ls.$set(_n)},i(s){ee||(g(k.$$.fragment,s),g(B.$$.fragment,s),g(cs.$$.fragment,s),g(G.$$.fragment,s),g(hs.$$.fragment,s),g(fs.$$.fragment,s),g(_s.$$.fragment,s),g(Y.$$.fragment,s),g(gs.$$.fragment,s),g(js.$$.fragment,s),g(K.$$.fragment,s),g(bs.$$.fragment,s),g(ws.$$.fragment,s),g(qs.$$.fragment,s),g(ks.$$.fragment,s),g(ys.$$.fragment,s),g(vs.$$.fragment,s),g(Es.$$.fragment,s),g(xs.$$.fragment,s),g(Ps.$$.fragment,s),g(Cs.$$.fragment,s),g(Ts.$$.fragment,s),g(zs.$$.fragment,s),g(ns.$$.fragment,s),g(Os.$$.fragment,s),g(Ds.$$.fragment,s),g(Us.$$.fragment,s),g(Ns.$$.fragment,s),g(Hs.$$.fragment,s),g(Ss.$$.fragment,s),g(Ms.$$.fragment,s),g(Bs.$$.fragment,s),g(As.$$.fragment,s),g(Ls.$$.fragment,s),g(ls.$$.fragment,s),g(Ws.$$.fragment,s),g(Xs.$$.fragment,s),g(Rs.$$.fragment,s),ee=!0)},o(s){j(k.$$.fragment,s),j(B.$$.fragment,s),j(cs.$$.fragment,s),j(G.$$.fragment,s),j(hs.$$.fragment,s),j(fs.$$.fragment,s),j(_s.$$.fragment,s),j(Y.$$.fragment,s),j(gs.$$.fragment,s),j(js.$$.fragment,s),j(K.$$.fragment,s),j(bs.$$.fragment,s),j(ws.$$.fragment,s),j(qs.$$.fragment,s),j(ks.$$.fragment,s),j(ys.$$.fragment,s),j(vs.$$.fragment,s),j(Es.$$.fragment,s),j(xs.$$.fragment,s),j(Ps.$$.fragment,s),j(Cs.$$.fragment,s),j(Ts.$$.fragment,s),j(zs.$$.fragment,s),j(ns.$$.fragment,s),j(Os.$$.fragment,s),j(Ds.$$.fragment,s),j(Us.$$.fragment,s),j(Ns.$$.fragment,s),j(Hs.$$.fragment,s),j(Ss.$$.fragment,s),j(Ms.$$.fragment,s),j(Bs.$$.fragment,s),j(As.$$.fragment,s),j(Ls.$$.fragment,s),j(ls.$$.fragment,s),j(Ws.$$.fragment,s),j(Xs.$$.fragment,s),j(Rs.$$.fragment,s),ee=!1},d(s){a(u),s&&a(q),s&&a(h),$(k),s&&a(C),$(B,s),s&&a(is),s&&a(Gs),s&&a($n),$(cs,s),s&&a(bn),$(G,s),s&&a(wn),s&&a(us),s&&a(qn),s&&a(Is),s&&a(kn),s&&a(Ys),s&&a(yn),s&&a(A),s&&a(Cn),s&&a(Zs),s&&a(Tn),s&&a(Js),s&&a(zn),s&&a(Ks),s&&a(On),$(hs,s),s&&a(Dn),s&&a(Qs),s&&a(Un),$(fs,s),s&&a(Nn),s&&a(ds),s&&a(Hn),s&&a(sa),s&&a(Sn),s&&a(x),s&&a(Mn),s&&a(aa),s&&a(Bn),$(_s,s),s&&a(An),s&&a(I),s&&a(Ln),$(Y,s),s&&a(Wn),s&&a(M),s&&a(Rn),s&&a(F),s&&a(Vn),s&&a(na),s&&a(Gn),s&&a(Z),s&&a(In),$(gs,s),s&&a(Yn),s&&a(D),s&&a(Zn),s&&a(y),s&&a(Jn),s&&a(ta),s&&a(Kn),s&&a(J),s&&a(Qn),$(js,s),s&&a(st),s&&a(L),s&&a(at),$(K,s),s&&a(nt),s&&a($s),s&&a(tt),s&&a(ea),s&&a(et),s&&a(Q),s&&a(lt),s&&a(la),s&&a(pt),$(bs,s),s&&a(ot),s&&a(pa),s&&a(rt),$(ws,s),s&&a(mt),s&&a(oa),s&&a(it),$(qs,s),s&&a(ct),s&&a(U),s&&a(ut),s&&a(N),s&&a(ht),$(ks,s),s&&a(ft),s&&a(ra),s&&a(dt),$(ys,s),s&&a(_t),s&&a(W),s&&a(gt),s&&a(V),$(vs),s&&a(jt),s&&a(ma),s&&a($t),s&&a(ia),s&&a(bt),$(Es,s),s&&a(wt),s&&a(as),s&&a(qt),$(xs,s),s&&a(kt),s&&a(ca),s&&a(yt),$(Ps,s),s&&a(vt),s&&a(ua),s&&a(Et),$(Cs,s),s&&a(xt),$(Ts,s),s&&a(Pt),s&&a(ha),s&&a(Ct),$(zs,s),s&&a(Tt),$(ns,s),s&&a(zt),s&&a(fa),s&&a(Ot),$(Os,s),s&&a(Dt),s&&a(ts),s&&a(Ut),s&&a(es),s&&a(Nt),s&&a(da),s&&a(Ht),$(Ds,s),s&&a(St),s&&a(_a),s&&a(Mt),$(Us,s),s&&a(Bt),$(Ns,s),s&&a(At),s&&a(ga),s&&a(Lt),$(Hs,s),s&&a(Wt),s&&a(ja),s&&a(Xt),$(Ss,s),s&&a(Rt),$(Ms,s),s&&a(Ft),s&&a($a),s&&a(Vt),$(Bs,s),s&&a(Gt),s&&a(ba),s&&a(It),$(As,s),s&&a(Yt),s&&a(v),s&&a(Zt),$(Ls,s),s&&a(Jt),$(ls,s),s&&a(Kt),s&&a(wa),s&&a(Qt),$(Ws,s),s&&a(se),s&&a(ps),s&&a(ae),$(Xs,s),s&&a(ne),$(Rs,s),s&&a(te),s&&a(qa)}}}const qr={local:"unigram",sections:[{local:"",title:"\u8BAD\u7EC3\u7B97\u6CD5"},{local:"",title:"\u6807\u8BB0\u5316\u7B97\u6CD5"},{local:"",title:"\u56DE\u5230\u8BAD\u7EC3"},{local:"unigram",title:"\u5B9E\u73B0 Unigram"}],title:"Unigram\u6807\u8BB0\u5316"};function kr(z){return hr(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class zr extends mr{constructor(u){super();ir(this,u,kr,wr,cr,{})}}export{zr as default,qr as metadata};
