import{S as Nc,i as Dc,s as Sc,e as o,k as i,w as u,t as l,U as Te,M as Oc,c,d as n,m as r,a as h,x as g,h as p,V as Ce,b as $,G as t,g as e,y as f,q as d,o as k,B as v,v as Uc}from"../../chunks/vendor-hf-doc-builder.js";import{T as Tt}from"../../chunks/Tip-hf-doc-builder.js";import{Y as Ac}from"../../chunks/Youtube-hf-doc-builder.js";import{I as Ct}from"../../chunks/IconCopyLink-hf-doc-builder.js";import{C as j}from"../../chunks/CodeBlock-hf-doc-builder.js";import{D as Bc}from"../../chunks/DocNotebookDropdown-hf-doc-builder.js";function Mc(C){let m,y;return{c(){m=o("p"),y=l("\u{1F4A1} Ph\u1EA7n n\xE0y s\u1EBD \u0111i s\xE2u v\xE0o Unigram c\u0169ng nh\u01B0 to\xE0n b\u1ED9 c\xE1ch tri\u1EC3n khai. B\u1EA1n c\xF3 th\u1EC3 b\u1ECF qua ph\u1EA7n cu\u1ED1i n\u1EBFu b\u1EA1n ch\u1EC9 quan t\xE2m t\u1ED5ng quan thu\u1EADt to\xE1n tokenize.")},l(b){m=c(b,"P",{});var _=h(m);y=p(_,"\u{1F4A1} Ph\u1EA7n n\xE0y s\u1EBD \u0111i s\xE2u v\xE0o Unigram c\u0169ng nh\u01B0 to\xE0n b\u1ED9 c\xE1ch tri\u1EC3n khai. B\u1EA1n c\xF3 th\u1EC3 b\u1ECF qua ph\u1EA7n cu\u1ED1i n\u1EBFu b\u1EA1n ch\u1EC9 quan t\xE2m t\u1ED5ng quan thu\u1EADt to\xE1n tokenize."),_.forEach(n)},m(b,_){e(b,m,_),t(m,y)},d(b){b&&n(m)}}}function Hc(C){let m,y,b,_,z;return{c(){m=o("p"),y=l("\u270F\uFE0F "),b=o("strong"),_=l("Gi\u1EDD \u0111\u1EBFn l\u01B0\u1EE3t b\u1EA1n!"),z=l(" Vi\u1EBFt \u0111o\u1EA1n m\xE3 \u0111\u1EC3 t\xEDnh t\u1EA7n su\u1EA5t tr\xEAn v\xE0 ki\u1EC3m tra l\u1EA1i k\u1EBFt qu\u1EA3 hi\u1EC3n th\u1ECB c\u0169ng nh\u01B0 t\u1ED5ng \u0111\xE3 \u0111\xFAng ch\u01B0a.")},l(q){m=c(q,"P",{});var E=h(m);y=p(E,"\u270F\uFE0F "),b=c(E,"STRONG",{});var V=h(b);_=p(V,"Gi\u1EDD \u0111\u1EBFn l\u01B0\u1EE3t b\u1EA1n!"),V.forEach(n),z=p(E," Vi\u1EBFt \u0111o\u1EA1n m\xE3 \u0111\u1EC3 t\xEDnh t\u1EA7n su\u1EA5t tr\xEAn v\xE0 ki\u1EC3m tra l\u1EA1i k\u1EBFt qu\u1EA3 hi\u1EC3n th\u1ECB c\u0169ng nh\u01B0 t\u1ED5ng \u0111\xE3 \u0111\xFAng ch\u01B0a."),E.forEach(n)},m(q,E){e(q,m,E),t(m,y),t(m,b),t(b,_),t(m,z)},d(q){q&&n(m)}}}function Gc(C){let m,y,b,_,z,q,E,V;return{c(){m=o("p"),y=l("\u270F\uFE0F "),b=o("strong"),_=l("Gi\u1EDD \u0111\u1EBFn l\u01B0\u1EE3t b\u1EA1n!"),z=l(" X\xE1c \u0111\u1ECBnh token c\u1EE7a t\u1EEB "),q=o("code"),E=l('"huggun"'),V=l(", v\xE0 \u0111i\u1EC3m c\u1EA3u ch\xFAng.")},l(X){m=c(X,"P",{});var T=h(m);y=p(T,"\u270F\uFE0F "),b=c(T,"STRONG",{});var B=h(b);_=p(B,"Gi\u1EDD \u0111\u1EBFn l\u01B0\u1EE3t b\u1EA1n!"),B.forEach(n),z=p(T," X\xE1c \u0111\u1ECBnh token c\u1EE7a t\u1EEB "),q=c(T,"CODE",{});var ds=h(q);E=p(ds,'"huggun"'),ds.forEach(n),V=p(T,", v\xE0 \u0111i\u1EC3m c\u1EA3u ch\xFAng."),T.forEach(n)},m(X,T){e(X,m,T),t(m,y),t(m,b),t(b,_),t(m,z),t(m,q),t(q,E),t(m,V)},d(X){X&&n(m)}}}function Lc(C){let m,y;return{c(){m=o("p"),y=l("\u{1F4A1} SentencePiece s\u1EED d\u1EE5ng m\u1ED9t thu\u1EADt to\xE1n hi\u1EC7u qu\u1EA3 h\u01A1n g\u1ECDi l\xE0 Enhanced Suffix Array (ESA) \u0111\u1EC3 t\u1EA1o ra b\u1ED9 t\u1EEB v\u1EF1ng ban \u0111\u1EA7u.")},l(b){m=c(b,"P",{});var _=h(m);y=p(_,"\u{1F4A1} SentencePiece s\u1EED d\u1EE5ng m\u1ED9t thu\u1EADt to\xE1n hi\u1EC7u qu\u1EA3 h\u01A1n g\u1ECDi l\xE0 Enhanced Suffix Array (ESA) \u0111\u1EC3 t\u1EA1o ra b\u1ED9 t\u1EEB v\u1EF1ng ban \u0111\u1EA7u."),_.forEach(n)},m(b,_){e(b,m,_),t(m,y)},d(b){b&&n(m)}}}function Xc(C){let m,y;return{c(){m=o("p"),y=l("\u{1F4A1} Ph\u01B0\u01A1ng ph\xE1p n\xE0y r\u1EA5t kh\xF4ng hi\u1EC7u qu\u1EA3, n\xEAn SentencePiece  s\u1EED d\u1EE5ng m\u1ED9t x\u1EA5p x\u1EC9 c\u1EE7a h\xE0m m\u1EA5t m\xE1t c\u1EE7a m\xF4 h\xECnh m\xE0 kh\xF4ng d\xF9ng token X: thay v\xEC b\u1EAFt \u0111\u1EA7u t\u1EEB \u0111\u1EA7u, n\xF3 ch\u1EC9 thay th\u1EBF token X b\u1EDFi ph\xE2n \u0111o\u1EA1n b\xEAn tr\xE1i c\u1EE7a n\xF3 trong b\u1ED9 t\u1EEB v\u1EF1ng. B\u1EB1ng c\xE1ch n\xE0y, t\u1EA5t c\u1EA3 \u0111i\u1EC3m c\xF3 th\u1EC3 \u0111\u01B0\u1EE3c t\xEDnh trong c\xF9ng m\u1ED9t l\u1EA7n \u0111\u1ED3ng th\u1EDDi v\u1EDBi s\u1EF1 m\u1EA5t m\xE1t c\u1EE7a m\xF4 h\xECnh.")},l(b){m=c(b,"P",{});var _=h(m);y=p(_,"\u{1F4A1} Ph\u01B0\u01A1ng ph\xE1p n\xE0y r\u1EA5t kh\xF4ng hi\u1EC7u qu\u1EA3, n\xEAn SentencePiece  s\u1EED d\u1EE5ng m\u1ED9t x\u1EA5p x\u1EC9 c\u1EE7a h\xE0m m\u1EA5t m\xE1t c\u1EE7a m\xF4 h\xECnh m\xE0 kh\xF4ng d\xF9ng token X: thay v\xEC b\u1EAFt \u0111\u1EA7u t\u1EEB \u0111\u1EA7u, n\xF3 ch\u1EC9 thay th\u1EBF token X b\u1EDFi ph\xE2n \u0111o\u1EA1n b\xEAn tr\xE1i c\u1EE7a n\xF3 trong b\u1ED9 t\u1EEB v\u1EF1ng. B\u1EB1ng c\xE1ch n\xE0y, t\u1EA5t c\u1EA3 \u0111i\u1EC3m c\xF3 th\u1EC3 \u0111\u01B0\u1EE3c t\xEDnh trong c\xF9ng m\u1ED9t l\u1EA7n \u0111\u1ED3ng th\u1EDDi v\u1EDBi s\u1EF1 m\u1EA5t m\xE1t c\u1EE7a m\xF4 h\xECnh."),_.forEach(n)},m(b,_){e(b,m,_),t(m,y)},d(b){b&&n(m)}}}function Fc(C){let m,y,b,_,z,q,E,V,X,T,B,ds,Js,Ve,Vt,ks,Nt,Y,Dt,F,I,Nn,vs,Ne,Dn,De,St,sn,Se,Ot,nn,Oe,Ut,M,Ue,At,zc='<span class="katex"><span class="katex-mathml"><math xmlns="http://www.w3.org/1998/Math/MathML"><semantics><mrow><mi>p</mi></mrow><annotation encoding="application/x-tex">p</annotation></semantics></math></span><span class="katex-html" aria-hidden="true"><span class="base"><span class="strut" style="height:0.625em;vertical-align:-0.1944em;"></span><span class="mord mathnormal">p</span></span></span></span>',Bt,Mt,Tc='<span class="katex"><span class="katex-mathml"><math xmlns="http://www.w3.org/1998/Math/MathML"><semantics><mrow><mi>p</mi></mrow><annotation encoding="application/x-tex">p</annotation></semantics></math></span><span class="katex-html" aria-hidden="true"><span class="base"><span class="strut" style="height:0.625em;vertical-align:-0.1944em;"></span><span class="mord mathnormal">p</span></span></span></span>',Ht,Gt,tn,Ae,Lt,an,Be,Xt,en,Me,Ft,bs,Rt,ln,He,Wt,_s,Qt,R,Z,Sn,js,Ge,On,Le,Kt,pn,Xe,Yt,P,Fe,Un,Re,We,An,Qe,Ke,Bn,Ye,Ie,Mn,Ze,Je,It,on,sl,Zt,ys,Jt,J,nl,Hn,tl,al,sa,ss,na,A,el,Gn,ll,pl,Ln,ol,cl,ta,Cc='<span class="katex-display"><span class="katex"><span class="katex-mathml"><math xmlns="http://www.w3.org/1998/Math/MathML" display="block"><semantics><mrow><mi>P</mi><mo stretchy="false">(</mo><mo stretchy="false">[</mo><mi mathvariant="normal">\u2018</mi><mi mathvariant="normal">\u2018</mi><mi>p</mi><mi mathvariant="normal">&quot;</mi><mo separator="true">,</mo><mi mathvariant="normal">\u2018</mi><mi mathvariant="normal">\u2018</mi><mi>u</mi><mi mathvariant="normal">&quot;</mi><mo separator="true">,</mo><mi mathvariant="normal">\u2018</mi><mi mathvariant="normal">\u2018</mi><mi>g</mi><mi mathvariant="normal">&quot;</mi><mo stretchy="false">]</mo><mo stretchy="false">)</mo><mo>=</mo><mi>P</mi><mo stretchy="false">(</mo><mi mathvariant="normal">\u2018</mi><mi mathvariant="normal">\u2018</mi><mi>p</mi><mi mathvariant="normal">&quot;</mi><mo stretchy="false">)</mo><mo>\xD7</mo><mi>P</mi><mo stretchy="false">(</mo><mi mathvariant="normal">\u2018</mi><mi mathvariant="normal">\u2018</mi><mi>u</mi><mi mathvariant="normal">&quot;</mi><mo stretchy="false">)</mo><mo>\xD7</mo><mi>P</mi><mo stretchy="false">(</mo><mi mathvariant="normal">\u2018</mi><mi mathvariant="normal">\u2018</mi><mi>g</mi><mi mathvariant="normal">&quot;</mi><mo stretchy="false">)</mo><mo>=</mo><mfrac><mn>5</mn><mn>210</mn></mfrac><mo>\xD7</mo><mfrac><mn>36</mn><mn>210</mn></mfrac><mo>\xD7</mo><mfrac><mn>20</mn><mn>210</mn></mfrac><mo>=</mo><mn>0.000389</mn></mrow><annotation encoding="application/x-tex">P([``p&quot;, ``u&quot;, ``g&quot;]) = P(``p&quot;) \\times P(``u&quot;) \\times P(``g&quot;) = \\frac{5}{210} \\times \\frac{36}{210} \\times \\frac{20}{210} = 0.000389</annotation></semantics></math></span><span class="katex-html" aria-hidden="true"><span class="base"><span class="strut" style="height:1em;vertical-align:-0.25em;"></span><span class="mord mathnormal" style="margin-right:0.13889em;">P</span><span class="mopen">([</span><span class="mord">\u2018\u2018</span><span class="mord mathnormal">p</span><span class="mord">&quot;</span><span class="mpunct">,</span><span class="mspace" style="margin-right:0.1667em;"></span><span class="mord">\u2018\u2018</span><span class="mord mathnormal">u</span><span class="mord">&quot;</span><span class="mpunct">,</span><span class="mspace" style="margin-right:0.1667em;"></span><span class="mord">\u2018\u2018</span><span class="mord mathnormal" style="margin-right:0.03588em;">g</span><span class="mord">&quot;</span><span class="mclose">])</span><span class="mspace" style="margin-right:0.2778em;"></span><span class="mrel">=</span><span class="mspace" style="margin-right:0.2778em;"></span></span><span class="base"><span class="strut" style="height:1em;vertical-align:-0.25em;"></span><span class="mord mathnormal" style="margin-right:0.13889em;">P</span><span class="mopen">(</span><span class="mord">\u2018\u2018</span><span class="mord mathnormal">p</span><span class="mord">&quot;</span><span class="mclose">)</span><span class="mspace" style="margin-right:0.2222em;"></span><span class="mbin">\xD7</span><span class="mspace" style="margin-right:0.2222em;"></span></span><span class="base"><span class="strut" style="height:1em;vertical-align:-0.25em;"></span><span class="mord mathnormal" style="margin-right:0.13889em;">P</span><span class="mopen">(</span><span class="mord">\u2018\u2018</span><span class="mord mathnormal">u</span><span class="mord">&quot;</span><span class="mclose">)</span><span class="mspace" style="margin-right:0.2222em;"></span><span class="mbin">\xD7</span><span class="mspace" style="margin-right:0.2222em;"></span></span><span class="base"><span class="strut" style="height:1em;vertical-align:-0.25em;"></span><span class="mord mathnormal" style="margin-right:0.13889em;">P</span><span class="mopen">(</span><span class="mord">\u2018\u2018</span><span class="mord mathnormal" style="margin-right:0.03588em;">g</span><span class="mord">&quot;</span><span class="mclose">)</span><span class="mspace" style="margin-right:0.2778em;"></span><span class="mrel">=</span><span class="mspace" style="margin-right:0.2778em;"></span></span><span class="base"><span class="strut" style="height:2.0074em;vertical-align:-0.686em;"></span><span class="mord"><span class="mopen nulldelimiter"></span><span class="mfrac"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:1.3214em;"><span style="top:-2.314em;"><span class="pstrut" style="height:3em;"></span><span class="mord"><span class="mord">210</span></span></span><span style="top:-3.23em;"><span class="pstrut" style="height:3em;"></span><span class="frac-line" style="border-bottom-width:0.04em;"></span></span><span style="top:-3.677em;"><span class="pstrut" style="height:3em;"></span><span class="mord"><span class="mord">5</span></span></span></span><span class="vlist-s">\u200B</span></span><span class="vlist-r"><span class="vlist" style="height:0.686em;"><span></span></span></span></span></span><span class="mclose nulldelimiter"></span></span><span class="mspace" style="margin-right:0.2222em;"></span><span class="mbin">\xD7</span><span class="mspace" style="margin-right:0.2222em;"></span></span><span class="base"><span class="strut" style="height:2.0074em;vertical-align:-0.686em;"></span><span class="mord"><span class="mopen nulldelimiter"></span><span class="mfrac"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:1.3214em;"><span style="top:-2.314em;"><span class="pstrut" style="height:3em;"></span><span class="mord"><span class="mord">210</span></span></span><span style="top:-3.23em;"><span class="pstrut" style="height:3em;"></span><span class="frac-line" style="border-bottom-width:0.04em;"></span></span><span style="top:-3.677em;"><span class="pstrut" style="height:3em;"></span><span class="mord"><span class="mord">36</span></span></span></span><span class="vlist-s">\u200B</span></span><span class="vlist-r"><span class="vlist" style="height:0.686em;"><span></span></span></span></span></span><span class="mclose nulldelimiter"></span></span><span class="mspace" style="margin-right:0.2222em;"></span><span class="mbin">\xD7</span><span class="mspace" style="margin-right:0.2222em;"></span></span><span class="base"><span class="strut" style="height:2.0074em;vertical-align:-0.686em;"></span><span class="mord"><span class="mopen nulldelimiter"></span><span class="mfrac"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:1.3214em;"><span style="top:-2.314em;"><span class="pstrut" style="height:3em;"></span><span class="mord"><span class="mord">210</span></span></span><span style="top:-3.23em;"><span class="pstrut" style="height:3em;"></span><span class="frac-line" style="border-bottom-width:0.04em;"></span></span><span style="top:-3.677em;"><span class="pstrut" style="height:3em;"></span><span class="mord"><span class="mord">20</span></span></span></span><span class="vlist-s">\u200B</span></span><span class="vlist-r"><span class="vlist" style="height:0.686em;"><span></span></span></span></span></span><span class="mclose nulldelimiter"></span></span><span class="mspace" style="margin-right:0.2778em;"></span><span class="mrel">=</span><span class="mspace" style="margin-right:0.2778em;"></span></span><span class="base"><span class="strut" style="height:0.6444em;"></span><span class="mord">0.000389</span></span></span></span></span>',aa,W,hl,Xn,il,rl,ea,Vc='<span class="katex-display"><span class="katex"><span class="katex-mathml"><math xmlns="http://www.w3.org/1998/Math/MathML" display="block"><semantics><mrow><mi>P</mi><mo stretchy="false">(</mo><mo stretchy="false">[</mo><mi mathvariant="normal">\u2018</mi><mi mathvariant="normal">\u2018</mi><mi>p</mi><mi>u</mi><mi mathvariant="normal">&quot;</mi><mo separator="true">,</mo><mi mathvariant="normal">\u2018</mi><mi mathvariant="normal">\u2018</mi><mi>g</mi><mi mathvariant="normal">&quot;</mi><mo stretchy="false">]</mo><mo stretchy="false">)</mo><mo>=</mo><mi>P</mi><mo stretchy="false">(</mo><mi mathvariant="normal">\u2018</mi><mi mathvariant="normal">\u2018</mi><mi>p</mi><mi>u</mi><mi mathvariant="normal">&quot;</mi><mo stretchy="false">)</mo><mo>\xD7</mo><mi>P</mi><mo stretchy="false">(</mo><mi mathvariant="normal">\u2018</mi><mi mathvariant="normal">\u2018</mi><mi>g</mi><mi mathvariant="normal">&quot;</mi><mo stretchy="false">)</mo><mo>=</mo><mfrac><mn>5</mn><mn>210</mn></mfrac><mo>\xD7</mo><mfrac><mn>20</mn><mn>210</mn></mfrac><mo>=</mo><mn>0.0022676</mn></mrow><annotation encoding="application/x-tex">P([``pu&quot;, ``g&quot;]) = P(``pu&quot;) \\times P(``g&quot;) = \\frac{5}{210} \\times \\frac{20}{210} = 0.0022676</annotation></semantics></math></span><span class="katex-html" aria-hidden="true"><span class="base"><span class="strut" style="height:1em;vertical-align:-0.25em;"></span><span class="mord mathnormal" style="margin-right:0.13889em;">P</span><span class="mopen">([</span><span class="mord">\u2018\u2018</span><span class="mord mathnormal">p</span><span class="mord mathnormal">u</span><span class="mord">&quot;</span><span class="mpunct">,</span><span class="mspace" style="margin-right:0.1667em;"></span><span class="mord">\u2018\u2018</span><span class="mord mathnormal" style="margin-right:0.03588em;">g</span><span class="mord">&quot;</span><span class="mclose">])</span><span class="mspace" style="margin-right:0.2778em;"></span><span class="mrel">=</span><span class="mspace" style="margin-right:0.2778em;"></span></span><span class="base"><span class="strut" style="height:1em;vertical-align:-0.25em;"></span><span class="mord mathnormal" style="margin-right:0.13889em;">P</span><span class="mopen">(</span><span class="mord">\u2018\u2018</span><span class="mord mathnormal">p</span><span class="mord mathnormal">u</span><span class="mord">&quot;</span><span class="mclose">)</span><span class="mspace" style="margin-right:0.2222em;"></span><span class="mbin">\xD7</span><span class="mspace" style="margin-right:0.2222em;"></span></span><span class="base"><span class="strut" style="height:1em;vertical-align:-0.25em;"></span><span class="mord mathnormal" style="margin-right:0.13889em;">P</span><span class="mopen">(</span><span class="mord">\u2018\u2018</span><span class="mord mathnormal" style="margin-right:0.03588em;">g</span><span class="mord">&quot;</span><span class="mclose">)</span><span class="mspace" style="margin-right:0.2778em;"></span><span class="mrel">=</span><span class="mspace" style="margin-right:0.2778em;"></span></span><span class="base"><span class="strut" style="height:2.0074em;vertical-align:-0.686em;"></span><span class="mord"><span class="mopen nulldelimiter"></span><span class="mfrac"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:1.3214em;"><span style="top:-2.314em;"><span class="pstrut" style="height:3em;"></span><span class="mord"><span class="mord">210</span></span></span><span style="top:-3.23em;"><span class="pstrut" style="height:3em;"></span><span class="frac-line" style="border-bottom-width:0.04em;"></span></span><span style="top:-3.677em;"><span class="pstrut" style="height:3em;"></span><span class="mord"><span class="mord">5</span></span></span></span><span class="vlist-s">\u200B</span></span><span class="vlist-r"><span class="vlist" style="height:0.686em;"><span></span></span></span></span></span><span class="mclose nulldelimiter"></span></span><span class="mspace" style="margin-right:0.2222em;"></span><span class="mbin">\xD7</span><span class="mspace" style="margin-right:0.2222em;"></span></span><span class="base"><span class="strut" style="height:2.0074em;vertical-align:-0.686em;"></span><span class="mord"><span class="mopen nulldelimiter"></span><span class="mfrac"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:1.3214em;"><span style="top:-2.314em;"><span class="pstrut" style="height:3em;"></span><span class="mord"><span class="mord">210</span></span></span><span style="top:-3.23em;"><span class="pstrut" style="height:3em;"></span><span class="frac-line" style="border-bottom-width:0.04em;"></span></span><span style="top:-3.677em;"><span class="pstrut" style="height:3em;"></span><span class="mord"><span class="mord">20</span></span></span></span><span class="vlist-s">\u200B</span></span><span class="vlist-r"><span class="vlist" style="height:0.686em;"><span></span></span></span></span></span><span class="mclose nulldelimiter"></span></span><span class="mspace" style="margin-right:0.2778em;"></span><span class="mrel">=</span><span class="mspace" style="margin-right:0.2778em;"></span></span><span class="base"><span class="strut" style="height:0.6444em;"></span><span class="mord">0.0022676</span></span></span></span></span>',la,cn,ml,pa,ns,ul,Fn,gl,fl,oa,$s,ca,N,dl,Rn,kl,vl,Wn,bl,_l,Qn,jl,yl,ha,w,$l,Kn,ql,wl,Yn,xl,El,In,Pl,zl,Zn,Tl,Cl,Jn,Vl,Nl,ia,hn,Dl,ra,ts,Sl,st,Ol,Ul,ma,qs,ua,H,Al,nt,Bl,Ml,tt,Hl,Gl,ga,as,fa,Q,es,at,ws,Ll,et,Xl,da,rn,Fl,ka,ls,Rl,lt,Wl,Ql,va,mn,Kl,ba,xs,_a,un,Yl,ja,Es,ya,gn,Il,$a,Ps,qa,D,Zl,pt,Jl,sp,ot,np,tp,ct,ap,ep,wa,S,lp,ht,pp,op,it,cp,hp,rt,ip,rp,xa,zs,Ea,fn,mp,Pa,Ts,za,G,up,mt,gp,fp,ut,dp,kp,Ta,K,ps,gt,Cs,vp,ft,bp,Ca,dn,_p,Va,kn,jp,Na,Vs,Da,os,yp,dt,$p,qp,Sa,Ns,Oa,vn,wp,Ua,Ds,Aa,bn,xp,Ba,Ss,Ma,Os,Ha,_n,Ep,Ga,Us,La,cs,Xa,jn,Pp,Fa,As,Ra,hs,zp,kt,Tp,Cp,Wa,is,Vp,vt,Np,Dp,Qa,yn,Sp,Ka,Bs,Ya,$n,Op,Ia,Ms,Za,Hs,Ja,qn,Up,se,Gs,ne,wn,Ap,te,Ls,ae,Xs,ee,xn,Bp,le,Fs,pe,En,Mp,oe,Rs,ce,x,Hp,bt,Gp,Lp,_t,Xp,Fp,jt,Rp,Wp,yt,Qp,Kp,$t,Yp,Ip,he,Ws,ie,rs,re,Pn,Zp,me,Qs,ue,ms,Jp,qt,so,no,ge,Ks,fe,Ys,de,zn,to,ke;return q=new Ct({}),B=new Bc({props:{classNames:"absolute z-10 right-0 top-0",options:[{label:"Google Colab",value:"https://colab.research.google.com/github/huggingface/notebooks/blob/master/course/vi/chapter6/section7.ipynb"},{label:"Aws Studio",value:"https://studiolab.sagemaker.aws/import/github/huggingface/notebooks/blob/master/course/vi/chapter6/section7.ipynb"}]}}),ks=new Ac({props:{id:"TGZfZVuF9Yc"}}),Y=new Tt({props:{$$slots:{default:[Mc]},$$scope:{ctx:C}}}),vs=new Ct({}),bs=new j({props:{code:'("hug", 10), ("pug", 5), ("pun", 12), ("bun", 4), ("hugs", 5)',highlighted:'(<span class="hljs-string">&quot;hug&quot;</span><span class="hljs-punctuation">,</span> <span class="hljs-number">10</span>)<span class="hljs-punctuation">,</span> (<span class="hljs-string">&quot;pug&quot;</span><span class="hljs-punctuation">,</span> <span class="hljs-number">5</span>)<span class="hljs-punctuation">,</span> (<span class="hljs-string">&quot;pun&quot;</span><span class="hljs-punctuation">,</span> <span class="hljs-number">12</span>)<span class="hljs-punctuation">,</span> (<span class="hljs-string">&quot;bun&quot;</span><span class="hljs-punctuation">,</span> <span class="hljs-number">4</span>)<span class="hljs-punctuation">,</span> (<span class="hljs-string">&quot;hugs&quot;</span><span class="hljs-punctuation">,</span> <span class="hljs-number">5</span>)'}}),_s=new j({props:{code:'["h", "u", "g", "hu", "ug", "p", "pu", "n", "un", "b", "bu", "s", "hug", "gs", "ugs"]',highlighted:'<span class="hljs-selector-attr">[<span class="hljs-string">&quot;h&quot;</span>, <span class="hljs-string">&quot;u&quot;</span>, <span class="hljs-string">&quot;g&quot;</span>, <span class="hljs-string">&quot;hu&quot;</span>, <span class="hljs-string">&quot;ug&quot;</span>, <span class="hljs-string">&quot;p&quot;</span>, <span class="hljs-string">&quot;pu&quot;</span>, <span class="hljs-string">&quot;n&quot;</span>, <span class="hljs-string">&quot;un&quot;</span>, <span class="hljs-string">&quot;b&quot;</span>, <span class="hljs-string">&quot;bu&quot;</span>, <span class="hljs-string">&quot;s&quot;</span>, <span class="hljs-string">&quot;hug&quot;</span>, <span class="hljs-string">&quot;gs&quot;</span>, <span class="hljs-string">&quot;ugs&quot;</span>]</span>'}}),js=new Ct({}),ys=new j({props:{code:`("h", 15) ("u", 36) ("g", 20) ("hu", 15) ("ug", 20) ("p", 17) ("pu", 17) ("n", 16)
("un", 16) ("b", 4) ("bu", 4) ("s", 5) ("hug", 15) ("gs", 5) ("ugs", 5)`,highlighted:`(<span class="hljs-string">&quot;h&quot;</span><span class="hljs-punctuation">,</span> <span class="hljs-number">15</span>) (<span class="hljs-string">&quot;u&quot;</span><span class="hljs-punctuation">,</span> <span class="hljs-number">36</span>) (<span class="hljs-string">&quot;g&quot;</span><span class="hljs-punctuation">,</span> <span class="hljs-number">20</span>) (<span class="hljs-string">&quot;hu&quot;</span><span class="hljs-punctuation">,</span> <span class="hljs-number">15</span>) (<span class="hljs-string">&quot;ug&quot;</span><span class="hljs-punctuation">,</span> <span class="hljs-number">20</span>) (<span class="hljs-string">&quot;p&quot;</span><span class="hljs-punctuation">,</span> <span class="hljs-number">17</span>) (<span class="hljs-string">&quot;pu&quot;</span><span class="hljs-punctuation">,</span> <span class="hljs-number">17</span>) (<span class="hljs-string">&quot;n&quot;</span><span class="hljs-punctuation">,</span> <span class="hljs-number">16</span>)
(<span class="hljs-string">&quot;un&quot;</span><span class="hljs-punctuation">,</span> <span class="hljs-number">16</span>) (<span class="hljs-string">&quot;b&quot;</span><span class="hljs-punctuation">,</span> <span class="hljs-number">4</span>) (<span class="hljs-string">&quot;bu&quot;</span><span class="hljs-punctuation">,</span> <span class="hljs-number">4</span>) (<span class="hljs-string">&quot;s&quot;</span><span class="hljs-punctuation">,</span> <span class="hljs-number">5</span>) (<span class="hljs-string">&quot;hug&quot;</span><span class="hljs-punctuation">,</span> <span class="hljs-number">15</span>) (<span class="hljs-string">&quot;gs&quot;</span><span class="hljs-punctuation">,</span> <span class="hljs-number">5</span>) (<span class="hljs-string">&quot;ugs&quot;</span><span class="hljs-punctuation">,</span> <span class="hljs-number">5</span>)`}}),ss=new Tt({props:{$$slots:{default:[Hc]},$$scope:{ctx:C}}}),$s=new j({props:{code:`["p", "u", "g"] : 0.000389
["p", "ug"] : 0.0022676
["pu", "g"] : 0.0022676`,highlighted:`[<span class="hljs-string">&quot;p&quot;</span>, <span class="hljs-string">&quot;u&quot;</span>, <span class="hljs-string">&quot;g&quot;</span>] : 0.000389
[<span class="hljs-string">&quot;p&quot;</span>, <span class="hljs-string">&quot;ug&quot;</span>] : 0.0022676
[<span class="hljs-string">&quot;pu&quot;</span>, <span class="hljs-string">&quot;g&quot;</span>] : 0.0022676`}}),qs=new j({props:{code:`Character 0 (u): "u" (score 0.171429)
Character 1 (n): "un" (score 0.076191)
Character 2 (h): "un" "h" (score 0.005442)
Character 3 (u): "un" "hu" (score 0.005442)
Character 4 (g): "un" "hug" (score 0.005442)`,highlighted:`<span class="hljs-attribute">Character</span> <span class="hljs-number">0</span> (u): <span class="hljs-string">&quot;u&quot;</span> (score <span class="hljs-number">0</span>.<span class="hljs-number">171429</span>)
<span class="hljs-attribute">Character</span> <span class="hljs-number">1</span> (n): <span class="hljs-string">&quot;un&quot;</span> (score <span class="hljs-number">0</span>.<span class="hljs-number">076191</span>)
<span class="hljs-attribute">Character</span> <span class="hljs-number">2</span> (h): <span class="hljs-string">&quot;un&quot;</span> <span class="hljs-string">&quot;h&quot;</span> (score <span class="hljs-number">0</span>.<span class="hljs-number">005442</span>)
<span class="hljs-attribute">Character</span> <span class="hljs-number">3</span> (u): <span class="hljs-string">&quot;un&quot;</span> <span class="hljs-string">&quot;hu&quot;</span> (score <span class="hljs-number">0</span>.<span class="hljs-number">005442</span>)
<span class="hljs-attribute">Character</span> <span class="hljs-number">4</span> (g): <span class="hljs-string">&quot;un&quot;</span> <span class="hljs-string">&quot;hug&quot;</span> (score <span class="hljs-number">0</span>.<span class="hljs-number">005442</span>)`}}),as=new Tt({props:{$$slots:{default:[Gc]},$$scope:{ctx:C}}}),ws=new Ct({}),xs=new j({props:{code:'("hug", 10), ("pug", 5), ("pun", 12), ("bun", 4), ("hugs", 5)',highlighted:'(<span class="hljs-string">&quot;hug&quot;</span><span class="hljs-punctuation">,</span> <span class="hljs-number">10</span>)<span class="hljs-punctuation">,</span> (<span class="hljs-string">&quot;pug&quot;</span><span class="hljs-punctuation">,</span> <span class="hljs-number">5</span>)<span class="hljs-punctuation">,</span> (<span class="hljs-string">&quot;pun&quot;</span><span class="hljs-punctuation">,</span> <span class="hljs-number">12</span>)<span class="hljs-punctuation">,</span> (<span class="hljs-string">&quot;bun&quot;</span><span class="hljs-punctuation">,</span> <span class="hljs-number">4</span>)<span class="hljs-punctuation">,</span> (<span class="hljs-string">&quot;hugs&quot;</span><span class="hljs-punctuation">,</span> <span class="hljs-number">5</span>)'}}),Es=new j({props:{code:`"hug": ["hug"] (score 0.071428)
"pug": ["pu", "g"] (score 0.007710)
"pun": ["pu", "n"] (score 0.006168)
"bun": ["bu", "n"] (score 0.001451)
"hugs": ["hug", "s"] (score 0.001701)`,highlighted:`<span class="hljs-string">&quot;hug&quot;</span>: [<span class="hljs-string">&quot;hug&quot;</span>] <span class="hljs-comment">(score 0.071428)</span>
<span class="hljs-string">&quot;pug&quot;</span>: [<span class="hljs-string">&quot;pu&quot;</span>, <span class="hljs-string">&quot;g&quot;</span>] <span class="hljs-comment">(score 0.007710)</span>
<span class="hljs-string">&quot;pun&quot;</span>: [<span class="hljs-string">&quot;pu&quot;</span>, <span class="hljs-string">&quot;n&quot;</span>] <span class="hljs-comment">(score 0.006168)</span>
<span class="hljs-string">&quot;bun&quot;</span>: [<span class="hljs-string">&quot;bu&quot;</span>, <span class="hljs-string">&quot;n&quot;</span>] <span class="hljs-comment">(score 0.001451)</span>
<span class="hljs-string">&quot;hugs&quot;</span>: [<span class="hljs-string">&quot;hug&quot;</span>, <span class="hljs-string">&quot;s&quot;</span>] <span class="hljs-comment">(score 0.001701)</span>`}}),Ps=new j({props:{code:"10 * (-log(0.071428)) + 5 * (-log(0.007710)) + 12 * (-log(0.006168)) + 4 * (-log(0.001451)) + 5 * (-log(0.001701)) = 169.8",highlighted:'<span class="hljs-attribute">10</span> * (-log(<span class="hljs-number">0</span>.<span class="hljs-number">071428</span>)) + <span class="hljs-number">5</span> * (-log(<span class="hljs-number">0</span>.<span class="hljs-number">007710</span>)) + <span class="hljs-number">12</span> * (-log(<span class="hljs-number">0</span>.<span class="hljs-number">006168</span>)) + <span class="hljs-number">4</span> * (-log(<span class="hljs-number">0</span>.<span class="hljs-number">001451</span>)) + <span class="hljs-number">5</span> * (-log(<span class="hljs-number">0</span>.<span class="hljs-number">001701</span>)) = <span class="hljs-number">169</span>.<span class="hljs-number">8</span>'}}),zs=new j({props:{code:`"hug": ["hu", "g"] (score 0.006802)
"hugs": ["hu", "gs"] (score 0.001701)`,highlighted:`<span class="hljs-string">&quot;hug&quot;</span>: [<span class="hljs-string">&quot;hu&quot;</span>, <span class="hljs-string">&quot;g&quot;</span>] <span class="hljs-comment">(score 0.006802)</span>
<span class="hljs-string">&quot;hugs&quot;</span>: [<span class="hljs-string">&quot;hu&quot;</span>, <span class="hljs-string">&quot;gs&quot;</span>] <span class="hljs-comment">(score 0.001701)</span>`}}),Ts=new j({props:{code:"- 10 * (-log(0.071428)) + 10 * (-log(0.006802)) = 23.5",highlighted:'- <span class="hljs-number">10</span> * (<span class="hljs-name">-log</span>(<span class="hljs-number">0.071428</span>)) + <span class="hljs-number">10</span> * (<span class="hljs-name">-log</span>(<span class="hljs-number">0.006802</span>)) = <span class="hljs-number">23.5</span>'}}),Cs=new Ct({}),Vs=new j({props:{code:`corpus = [
    "This is the Hugging Face Course.",
    "This chapter is about tokenization.",
    "This section shows several tokenizer algorithms.",
    "Hopefully, you will be able to understand how they are trained and generate tokens.",
]`,highlighted:`corpus = [
    <span class="hljs-string">&quot;This is the Hugging Face Course.&quot;</span>,
    <span class="hljs-string">&quot;This chapter is about tokenization.&quot;</span>,
    <span class="hljs-string">&quot;This section shows several tokenizer algorithms.&quot;</span>,
    <span class="hljs-string">&quot;Hopefully, you will be able to understand how they are trained and generate tokens.&quot;</span>,
]`}}),Ns=new j({props:{code:`from transformers import AutoTokenizer

tokenizer = AutoTokenizer.from_pretrained("xlnet-base-cased")`,highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoTokenizer

tokenizer = AutoTokenizer.from_pretrained(<span class="hljs-string">&quot;xlnet-base-cased&quot;</span>)`}}),Ds=new j({props:{code:`from collections import defaultdict

word_freqs = defaultdict(int)
for text in corpus:
    words_with_offsets = tokenizer.backend_tokenizer.pre_tokenizer.pre_tokenize_str(text)
    new_words = [word for word, offset in words_with_offsets]
    for word in new_words:
        word_freqs[word] += 1

word_freqs`,highlighted:`<span class="hljs-keyword">from</span> collections <span class="hljs-keyword">import</span> defaultdict

word_freqs = defaultdict(<span class="hljs-built_in">int</span>)
<span class="hljs-keyword">for</span> text <span class="hljs-keyword">in</span> corpus:
    words_with_offsets = tokenizer.backend_tokenizer.pre_tokenizer.pre_tokenize_str(text)
    new_words = [word <span class="hljs-keyword">for</span> word, offset <span class="hljs-keyword">in</span> words_with_offsets]
    <span class="hljs-keyword">for</span> word <span class="hljs-keyword">in</span> new_words:
        word_freqs[word] += <span class="hljs-number">1</span>

word_freqs`}}),Ss=new j({props:{code:`char_freqs = defaultdict(int)
subwords_freqs = defaultdict(int)
for word, freq in word_freqs.items():
    for i in range(len(word)):
        char_freqs[word[i]] += freq
        # L\u1EB7p qua c\xE1c t\u1EEB con c\xF3 \u0111\u1ED9 d\xE0i >= 2
        for j in range(i + 2, len(word) + 1):
            subwords_freqs[word[i:j]] += freq

# S\u1EAFp x\u1EBFp c\xE1c t\u1EEB con theo t\u1EA7n su\u1EA5t
sorted_subwords = sorted(subwords_freqs.items(), key=lambda x: x[1], reverse=True)
sorted_subwords[:10]`,highlighted:`char_freqs = defaultdict(<span class="hljs-built_in">int</span>)
subwords_freqs = defaultdict(<span class="hljs-built_in">int</span>)
<span class="hljs-keyword">for</span> word, freq <span class="hljs-keyword">in</span> word_freqs.items():
    <span class="hljs-keyword">for</span> i <span class="hljs-keyword">in</span> <span class="hljs-built_in">range</span>(<span class="hljs-built_in">len</span>(word)):
        char_freqs[word[i]] += freq
        <span class="hljs-comment"># L\u1EB7p qua c\xE1c t\u1EEB con c\xF3 \u0111\u1ED9 d\xE0i &gt;= 2</span>
        <span class="hljs-keyword">for</span> j <span class="hljs-keyword">in</span> <span class="hljs-built_in">range</span>(i + <span class="hljs-number">2</span>, <span class="hljs-built_in">len</span>(word) + <span class="hljs-number">1</span>):
            subwords_freqs[word[i:j]] += freq

<span class="hljs-comment"># S\u1EAFp x\u1EBFp c\xE1c t\u1EEB con theo t\u1EA7n su\u1EA5t</span>
sorted_subwords = <span class="hljs-built_in">sorted</span>(subwords_freqs.items(), key=<span class="hljs-keyword">lambda</span> x: x[<span class="hljs-number">1</span>], reverse=<span class="hljs-literal">True</span>)
sorted_subwords[:<span class="hljs-number">10</span>]`}}),Os=new j({props:{code:"[('\u2581t', 7), ('is', 5), ('er', 5), ('\u2581a', 5), ('\u2581to', 4), ('to', 4), ('en', 4), ('\u2581T', 3), ('\u2581Th', 3), ('\u2581Thi', 3)]",highlighted:'[(<span class="hljs-string">&#x27;\u2581t&#x27;</span>, <span class="hljs-number">7</span>), (<span class="hljs-string">&#x27;is&#x27;</span>, <span class="hljs-number">5</span>), (<span class="hljs-string">&#x27;er&#x27;</span>, <span class="hljs-number">5</span>), (<span class="hljs-string">&#x27;\u2581a&#x27;</span>, <span class="hljs-number">5</span>), (<span class="hljs-string">&#x27;\u2581to&#x27;</span>, <span class="hljs-number">4</span>), (<span class="hljs-string">&#x27;to&#x27;</span>, <span class="hljs-number">4</span>), (<span class="hljs-string">&#x27;en&#x27;</span>, <span class="hljs-number">4</span>), (<span class="hljs-string">&#x27;\u2581T&#x27;</span>, <span class="hljs-number">3</span>), (<span class="hljs-string">&#x27;\u2581Th&#x27;</span>, <span class="hljs-number">3</span>), (<span class="hljs-string">&#x27;\u2581Thi&#x27;</span>, <span class="hljs-number">3</span>)]'}}),Us=new j({props:{code:`token_freqs = list(char_freqs.items()) + sorted_subwords[: 300 - len(char_freqs)]
token_freqs = {token: freq for token, freq in token_freqs}`,highlighted:`token_freqs = <span class="hljs-built_in">list</span>(char_freqs.items()) + sorted_subwords[: <span class="hljs-number">300</span> - <span class="hljs-built_in">len</span>(char_freqs)]
token_freqs = {token: freq <span class="hljs-keyword">for</span> token, freq <span class="hljs-keyword">in</span> token_freqs}`}}),cs=new Tt({props:{$$slots:{default:[Lc]},$$scope:{ctx:C}}}),As=new j({props:{code:`from math import log

total_sum = sum([freq for token, freq in token_freqs.items()])
model = {token: -log(freq / total_sum) for token, freq in token_freqs.items()}`,highlighted:`<span class="hljs-keyword">from</span> math <span class="hljs-keyword">import</span> log

total_sum = <span class="hljs-built_in">sum</span>([freq <span class="hljs-keyword">for</span> token, freq <span class="hljs-keyword">in</span> token_freqs.items()])
model = {token: -log(freq / total_sum) <span class="hljs-keyword">for</span> token, freq <span class="hljs-keyword">in</span> token_freqs.items()}`}}),Bs=new j({props:{code:`def encode_word(word, model):
    best_segmentations = [{"start": 0, "score": 1}] + [
        {"start": None, "score": None} for _ in range(len(word))
    ]
    for start_idx in range(len(word)):
        # N\xF3 n\xEAn \u0111\u01B0\u1EE3c l\u1EA5p \u0111\u1EA7y b\u1EDFi c\xE1c b\u01B0\u1EDBc ph\xEDa tr\u01B0\u1EDBc c\u1EE7a v\xF2ng l\u1EB7p
        best_score_at_start = best_segmentations[start_idx]["score"]
        for end_idx in range(start_idx + 1, len(word) + 1):
            token = word[start_idx:end_idx]
            if token in model and best_score_at_start is not None:
                score = model[token] + best_score_at_start
                # N\u1EBFu ch\xFAng ta t\xECm th\u1EA5y m\u1ED9t ph\xE2n \u0111o\u1EA1n k\u1EBFt th\xFAc t\u1ED1t h\u01A1n t\u1EA1i end_idx, ch\xFAng ta c\u1EADp nh\u1EADt
                if (
                    best_segmentations[end_idx]["score"] is None
                    or best_segmentations[end_idx]["score"] > score
                ):
                    best_segmentations[end_idx] = {"start": start_idx, "score": score}

    segmentation = best_segmentations[-1]
    if segmentation["score"] is None:
        # Ta \u0111\xE3 kh\xF4ng t\xECm th\u1EA5y tokenize c\u1EE7a t\u1EEB -> kh\xF4ng x\xE1c \u0111\u1ECBnh
        return ["<unk>"], None

    score = segmentation["score"]
    start = segmentation["start"]
    end = len(word)
    tokens = []
    while start != 0:
        tokens.insert(0, word[start:end])
        next_start = best_segmentations[start]["start"]
        end = start
        start = next_start
    tokens.insert(0, word[start:end])
    return tokens, score`,highlighted:`<span class="hljs-keyword">def</span> <span class="hljs-title function_">encode_word</span>(<span class="hljs-params">word, model</span>):
    best_segmentations = [{<span class="hljs-string">&quot;start&quot;</span>: <span class="hljs-number">0</span>, <span class="hljs-string">&quot;score&quot;</span>: <span class="hljs-number">1</span>}] + [
        {<span class="hljs-string">&quot;start&quot;</span>: <span class="hljs-literal">None</span>, <span class="hljs-string">&quot;score&quot;</span>: <span class="hljs-literal">None</span>} <span class="hljs-keyword">for</span> _ <span class="hljs-keyword">in</span> <span class="hljs-built_in">range</span>(<span class="hljs-built_in">len</span>(word))
    ]
    <span class="hljs-keyword">for</span> start_idx <span class="hljs-keyword">in</span> <span class="hljs-built_in">range</span>(<span class="hljs-built_in">len</span>(word)):
        <span class="hljs-comment"># N\xF3 n\xEAn \u0111\u01B0\u1EE3c l\u1EA5p \u0111\u1EA7y b\u1EDFi c\xE1c b\u01B0\u1EDBc ph\xEDa tr\u01B0\u1EDBc c\u1EE7a v\xF2ng l\u1EB7p</span>
        best_score_at_start = best_segmentations[start_idx][<span class="hljs-string">&quot;score&quot;</span>]
        <span class="hljs-keyword">for</span> end_idx <span class="hljs-keyword">in</span> <span class="hljs-built_in">range</span>(start_idx + <span class="hljs-number">1</span>, <span class="hljs-built_in">len</span>(word) + <span class="hljs-number">1</span>):
            token = word[start_idx:end_idx]
            <span class="hljs-keyword">if</span> token <span class="hljs-keyword">in</span> model <span class="hljs-keyword">and</span> best_score_at_start <span class="hljs-keyword">is</span> <span class="hljs-keyword">not</span> <span class="hljs-literal">None</span>:
                score = model[token] + best_score_at_start
                <span class="hljs-comment"># N\u1EBFu ch\xFAng ta t\xECm th\u1EA5y m\u1ED9t ph\xE2n \u0111o\u1EA1n k\u1EBFt th\xFAc t\u1ED1t h\u01A1n t\u1EA1i end_idx, ch\xFAng ta c\u1EADp nh\u1EADt</span>
                <span class="hljs-keyword">if</span> (
                    best_segmentations[end_idx][<span class="hljs-string">&quot;score&quot;</span>] <span class="hljs-keyword">is</span> <span class="hljs-literal">None</span>
                    <span class="hljs-keyword">or</span> best_segmentations[end_idx][<span class="hljs-string">&quot;score&quot;</span>] &gt; score
                ):
                    best_segmentations[end_idx] = {<span class="hljs-string">&quot;start&quot;</span>: start_idx, <span class="hljs-string">&quot;score&quot;</span>: score}

    segmentation = best_segmentations[-<span class="hljs-number">1</span>]
    <span class="hljs-keyword">if</span> segmentation[<span class="hljs-string">&quot;score&quot;</span>] <span class="hljs-keyword">is</span> <span class="hljs-literal">None</span>:
        <span class="hljs-comment"># Ta \u0111\xE3 kh\xF4ng t\xECm th\u1EA5y tokenize c\u1EE7a t\u1EEB -&gt; kh\xF4ng x\xE1c \u0111\u1ECBnh</span>
        <span class="hljs-keyword">return</span> [<span class="hljs-string">&quot;&lt;unk&gt;&quot;</span>], <span class="hljs-literal">None</span>

    score = segmentation[<span class="hljs-string">&quot;score&quot;</span>]
    start = segmentation[<span class="hljs-string">&quot;start&quot;</span>]
    end = <span class="hljs-built_in">len</span>(word)
    tokens = []
    <span class="hljs-keyword">while</span> start != <span class="hljs-number">0</span>:
        tokens.insert(<span class="hljs-number">0</span>, word[start:end])
        next_start = best_segmentations[start][<span class="hljs-string">&quot;start&quot;</span>]
        end = start
        start = next_start
    tokens.insert(<span class="hljs-number">0</span>, word[start:end])
    <span class="hljs-keyword">return</span> tokens, score`}}),Ms=new j({props:{code:`print(encode_word("Hopefully", model))
print(encode_word("This", model))`,highlighted:`<span class="hljs-built_in">print</span>(encode_word(<span class="hljs-string">&quot;Hopefully&quot;</span>, model))
<span class="hljs-built_in">print</span>(encode_word(<span class="hljs-string">&quot;This&quot;</span>, model))`}}),Hs=new j({props:{code:`(['H', 'o', 'p', 'e', 'f', 'u', 'll', 'y'], 41.5157494601402)
(['This'], 6.288267030694535)`,highlighted:`([<span class="hljs-string">&#x27;H&#x27;</span>, <span class="hljs-string">&#x27;o&#x27;</span>, <span class="hljs-string">&#x27;p&#x27;</span>, <span class="hljs-string">&#x27;e&#x27;</span>, <span class="hljs-string">&#x27;f&#x27;</span>, <span class="hljs-string">&#x27;u&#x27;</span>, <span class="hljs-string">&#x27;ll&#x27;</span>, <span class="hljs-string">&#x27;y&#x27;</span>], <span class="hljs-number">41.5157494601402</span>)
([<span class="hljs-string">&#x27;This&#x27;</span>], <span class="hljs-number">6.288267030694535</span>)`}}),Gs=new j({props:{code:`def compute_loss(model):
    loss = 0
    for word, freq in word_freqs.items():
        _, word_loss = encode_word(word, model)
        loss += freq * word_loss
    return loss`,highlighted:`<span class="hljs-keyword">def</span> <span class="hljs-title function_">compute_loss</span>(<span class="hljs-params">model</span>):
    loss = <span class="hljs-number">0</span>
    <span class="hljs-keyword">for</span> word, freq <span class="hljs-keyword">in</span> word_freqs.items():
        _, word_loss = encode_word(word, model)
        loss += freq * word_loss
    <span class="hljs-keyword">return</span> loss`}}),Ls=new j({props:{code:"compute_loss(model)",highlighted:"compute_loss(model)"}}),Xs=new j({props:{code:"413.10377642940875",highlighted:'<span class="hljs-number">413.10377642940875</span>'}}),Fs=new j({props:{code:`import copy


def compute_scores(model):
    scores = {}
    model_loss = compute_loss(model)
    for token, score in model.items():
        # Ta lu\xF4n gi\u1EEF \u0111\u1ED9 d\xE0i c\xE1c token b\u1EB1ng 1
        if len(token) == 1:
            continue
        model_without_token = copy.deepcopy(model)
        _ = model_without_token.pop(token)
        scores[token] = compute_loss(model_without_token) - model_loss
    return scores`,highlighted:`<span class="hljs-keyword">import</span> copy


<span class="hljs-keyword">def</span> <span class="hljs-title function_">compute_scores</span>(<span class="hljs-params">model</span>):
    scores = {}
    model_loss = compute_loss(model)
    <span class="hljs-keyword">for</span> token, score <span class="hljs-keyword">in</span> model.items():
        <span class="hljs-comment"># Ta lu\xF4n gi\u1EEF \u0111\u1ED9 d\xE0i c\xE1c token b\u1EB1ng 1</span>
        <span class="hljs-keyword">if</span> <span class="hljs-built_in">len</span>(token) == <span class="hljs-number">1</span>:
            <span class="hljs-keyword">continue</span>
        model_without_token = copy.deepcopy(model)
        _ = model_without_token.pop(token)
        scores[token] = compute_loss(model_without_token) - model_loss
    <span class="hljs-keyword">return</span> scores`}}),Rs=new j({props:{code:`scores = compute_scores(model)
print(scores["ll"])
print(scores["his"])`,highlighted:`scores = compute_scores(model)
<span class="hljs-built_in">print</span>(scores[<span class="hljs-string">&quot;ll&quot;</span>])
<span class="hljs-built_in">print</span>(scores[<span class="hljs-string">&quot;his&quot;</span>])`}}),Ws=new j({props:{code:`6.376412403623874
0.0`,highlighted:`<span class="hljs-number">6.376412403623874</span>
<span class="hljs-number">0.0</span>`}}),rs=new Tt({props:{$$slots:{default:[Xc]},$$scope:{ctx:C}}}),Qs=new j({props:{code:`percent_to_remove = 0.1
while len(model) > 100:
    scores = compute_scores(model)
    sorted_scores = sorted(scores.items(), key=lambda x: x[1])
    # Lo\u1EA1i token percent_to_remove v\u1EDBi \u0111i\u1EC3m th\u1EA5p nh\u1EA5t.
    for i in range(int(len(model) * percent_to_remove)):
        _ = token_freqs.pop(sorted_scores[i][0])
    total_sum = sum([freq for token, freq in token_freqs.items()])
    model = {token: -log(freq / total_sum) for token, freq in token_freqs.items()}`,highlighted:`percent_to_remove = <span class="hljs-number">0.1</span>
<span class="hljs-keyword">while</span> <span class="hljs-built_in">len</span>(model) &gt; <span class="hljs-number">100</span>:
    scores = compute_scores(model)
    sorted_scores = <span class="hljs-built_in">sorted</span>(scores.items(), key=<span class="hljs-keyword">lambda</span> x: x[<span class="hljs-number">1</span>])
    <span class="hljs-comment"># Lo\u1EA1i token percent_to_remove v\u1EDBi \u0111i\u1EC3m th\u1EA5p nh\u1EA5t.</span>
    <span class="hljs-keyword">for</span> i <span class="hljs-keyword">in</span> <span class="hljs-built_in">range</span>(<span class="hljs-built_in">int</span>(<span class="hljs-built_in">len</span>(model) * percent_to_remove)):
        _ = token_freqs.pop(sorted_scores[i][<span class="hljs-number">0</span>])
    total_sum = <span class="hljs-built_in">sum</span>([freq <span class="hljs-keyword">for</span> token, freq <span class="hljs-keyword">in</span> token_freqs.items()])
    model = {token: -log(freq / total_sum) <span class="hljs-keyword">for</span> token, freq <span class="hljs-keyword">in</span> token_freqs.items()}`}}),Ks=new j({props:{code:`def tokenize(text, model):
    words_with_offsets = tokenizer.backend_tokenizer.pre_tokenizer.pre_tokenize_str(text)
    pre_tokenized_text = [word for word, offset in words_with_offsets]
    encoded_words = [encode_word(word, model)[0] for word in pre_tokenized_text]
    return sum(encoded_words, [])


tokenize("This is the Hugging Face course.", model)`,highlighted:`<span class="hljs-keyword">def</span> <span class="hljs-title function_">tokenize</span>(<span class="hljs-params">text, model</span>):
    words_with_offsets = tokenizer.backend_tokenizer.pre_tokenizer.pre_tokenize_str(text)
    pre_tokenized_text = [word <span class="hljs-keyword">for</span> word, offset <span class="hljs-keyword">in</span> words_with_offsets]
    encoded_words = [encode_word(word, model)[<span class="hljs-number">0</span>] <span class="hljs-keyword">for</span> word <span class="hljs-keyword">in</span> pre_tokenized_text]
    <span class="hljs-keyword">return</span> <span class="hljs-built_in">sum</span>(encoded_words, [])


tokenize(<span class="hljs-string">&quot;This is the Hugging Face course.&quot;</span>, model)`}}),Ys=new j({props:{code:"['\u2581This', '\u2581is', '\u2581the', '\u2581Hugging', '\u2581Face', '\u2581', 'c', 'ou', 'r', 's', 'e', '.']",highlighted:'[<span class="hljs-string">&#x27;\u2581This&#x27;</span>, <span class="hljs-string">&#x27;\u2581is&#x27;</span>, <span class="hljs-string">&#x27;\u2581the&#x27;</span>, <span class="hljs-string">&#x27;\u2581Hugging&#x27;</span>, <span class="hljs-string">&#x27;\u2581Face&#x27;</span>, <span class="hljs-string">&#x27;\u2581&#x27;</span>, <span class="hljs-string">&#x27;c&#x27;</span>, <span class="hljs-string">&#x27;ou&#x27;</span>, <span class="hljs-string">&#x27;r&#x27;</span>, <span class="hljs-string">&#x27;s&#x27;</span>, <span class="hljs-string">&#x27;e&#x27;</span>, <span class="hljs-string">&#x27;.&#x27;</span>]'}}),{c(){m=o("meta"),y=i(),b=o("h1"),_=o("a"),z=o("span"),u(q.$$.fragment),E=i(),V=o("span"),X=l("Unigram tokenization"),T=i(),u(B.$$.fragment),ds=i(),Js=o("p"),Ve=l("Thu\u1EADt to\xE1n Unigram th\u01B0\u1EDDng \u0111\u01B0\u1EE3c s\u1EED dung trong SentencePiece,  \u0111\xE2y l\xE0 m\u1ED9t thu\u1EADt to\xE1n tokenize cho c\xE1c m\xF4 h\xECnh nh\u01B0 AlBERT, T5, mBART, Big Bird, v\xE0 XLNet."),Vt=i(),u(ks.$$.fragment),Nt=i(),u(Y.$$.fragment),Dt=i(),F=o("h2"),I=o("a"),Nn=o("span"),u(vs.$$.fragment),Ne=i(),Dn=o("span"),De=l("Thu\u1EADt to\xE1n hu\u1EA5n luy\u1EC7n"),St=i(),sn=o("p"),Se=l("So v\u1EDBi BPE v\xE0 WordPiece, Unigram ho\u1EA1t \u0111\u1ED9ng theo h\u01B0\u1EDBng kh\xE1c: n\xF3 b\u1EAFt \u0111\u1EA7u t\u1EEB m\u1ED9t t\u1EEB v\u1EF1ng l\u1EDBn v\xE0 lo\u1EA1i b\u1ECF c\xE1c token cho \u0111\u1EBFn khi n\xF3 \u0111\u1EA1t \u0111\u1EBFn k\xEDch th\u01B0\u1EDBc t\u1EEB v\u1EF1ng mong mu\u1ED1n. C\xF3 m\u1ED9t s\u1ED1 t\xF9y ch\u1ECDn \u0111\u1EC3 s\u1EED d\u1EE5ng \u0111\u1EC3 x\xE2y d\u1EF1ng v\u1ED1n t\u1EEB v\u1EF1ng c\u01A1 b\u1EA3n: v\xED d\u1EE5: ch\xFAng ta c\xF3 th\u1EC3 l\u1EA5y c\xE1c chu\u1ED7i con ph\u1ED5 bi\u1EBFn nh\u1EA5t trong c\xE1c t\u1EEB \u0111\u01B0\u1EE3c ti\u1EC1n tokenize ho\u1EB7c \xE1p d\u1EE5ng BPE tr\xEAn kho ng\u1EEF li\u1EC7u ban \u0111\u1EA7u c\xF3 k\xEDch th\u01B0\u1EDBc t\u1EEB v\u1EF1ng l\u1EDBn."),Ot=i(),nn=o("p"),Oe=l("T\u1EA1i m\u1ED7i b\u01B0\u1EDBc c\u1EE7a qu\xE1 tr\xECnh hu\u1EA5n luy\u1EC7n, thu\u1EADt to\xE1n Unigram t\xEDnh to\xE1n s\u1EF1 m\u1EA5t m\xE1t tr\xEAn kho ng\u1EEF li\u1EC7u \u0111\u01B0\u1EE3c cung c\u1EA5p t\u1EEB v\u1EF1ng hi\u1EC7n t\u1EA1i. Sau \u0111\xF3, \u0111\u1ED1i v\u1EDBi m\u1ED7i k\xFD hi\u1EC7u trong t\u1EEB v\u1EF1ng, thu\u1EADt to\xE1n s\u1EBD t\xEDnh to\xE1n m\u1EE9c \u0111\u1ED9 t\u1ED5n th\u1EA5t t\u1ED5ng th\u1EC3 s\u1EBD t\u0103ng l\xEAn bao nhi\xEAu n\u1EBFu k\xFD hi\u1EC7u b\u1ECB x\xF3a v\xE0 t\xECm ki\u1EBFm c\xE1c k\xFD hi\u1EC7u l\xE0m t\u0103ng n\xF3 \xEDt nh\u1EA5t. Nh\u1EEFng bi\u1EC3u t\u01B0\u1EE3ng \u0111\xF3 c\xF3 \u1EA3nh h\u01B0\u1EDFng th\u1EA5p h\u01A1n \u0111\u1EBFn s\u1EF1 m\u1EA5t m\xE1t t\u1ED5ng th\u1EC3 \u0111\u1ED1i v\u1EDBi kho d\u1EEF li\u1EC7u, v\xEC v\u1EADy theo m\u1ED9t ngh\u0129a n\xE0o \u0111\xF3, ch\xFAng \u201C\xEDt c\u1EA7n thi\u1EBFt h\u01A1n\u201D v\xE0 l\xE0 nh\u1EEFng \u1EE9ng c\u1EED vi\xEAn t\u1ED1t nh\u1EA5t \u0111\u1EC3 lo\u1EA1i b\u1ECF."),Ut=i(),M=o("p"),Ue=l("\u0110\xE2y l\xE0 m\u1ED9t ho\u1EA1t \u0111\u1ED9ng r\u1EA5t t\u1ED1n k\xE9m, v\xEC v\u1EADy ch\xFAng t\xF4i kh\xF4ng ch\u1EC9 lo\u1EA1i b\u1ECF m\u1ED9t bi\u1EC3u t\u01B0\u1EE3ng li\xEAn quan \u0111\u1EBFn m\u1EE9c t\u0103ng t\u1ED5n th\u1EA5t th\u1EA5p nh\u1EA5t, m\xE0 "),At=new Te,Bt=l(" ("),Mt=new Te,Ht=l(" l\xE0 m\u1ED9t si\xEAu tham s\u1ED1 b\u1EA1n c\xF3 th\u1EC3 ki\u1EC3m so\xE1t, th\u01B0\u1EDDng l\xE0 10 ho\u1EB7c 20) ph\u1EA7n tr\u0103m c\xE1c k\xFD hi\u1EC7u li\xEAn quan \u0111\u1EBFn m\u1EE9c t\u0103ng t\u1ED5n th\u1EA5t th\u1EA5p nh\u1EA5t. Qu\xE1 tr\xECnh n\xE0y sau \u0111\xF3 \u0111\u01B0\u1EE3c l\u1EB7p l\u1EA1i cho \u0111\u1EBFn khi t\u1EEB v\u1EF1ng \u0111\u1EA1t \u0111\u01B0\u1EE3c k\xEDch th\u01B0\u1EDBc mong mu\u1ED1n."),Gt=i(),tn=o("p"),Ae=l("L\u01B0u \xFD r\u1EB1ng ch\xFAng ta kh\xF4ng bao gi\u1EDD x\xF3a c\xE1c k\xFD t\u1EF1 c\u01A1 s\u1EDF, \u0111\u1EC3 \u0111\u1EA3m b\u1EA3o r\u1EB1ng b\u1EA5t k\u1EF3 t\u1EEB n\xE0o c\u0169ng c\xF3 th\u1EC3 \u0111\u01B0\u1EE3c tokenize."),Lt=i(),an=o("p"),Be=l("B\xE2y gi\u1EDD, \u0111i\u1EC1u n\xE0y v\u1EABn c\xF2n h\u01A1i m\u01A1 h\u1ED3: ph\u1EA7n ch\xEDnh c\u1EE7a thu\u1EADt to\xE1n l\xE0 t\xEDnh to\xE1n s\u1EF1 m\u1EA5t m\xE1t trong kho ng\u1EEF li\u1EC7u v\xE0 xem n\xF3 thay \u0111\u1ED5i nh\u01B0 th\u1EBF n\xE0o khi ch\xFAng t\xF4i x\xF3a m\u1ED9t s\u1ED1 token kh\u1ECFi t\u1EEB v\u1EF1ng, nh\u01B0ng ch\xFAng ta ch\u01B0a gi\u1EA3i th\xEDch c\xE1ch th\u1EF1c hi\u1EC7n \u0111i\u1EC1u n\xE0y. B\u01B0\u1EDBc n\xE0y d\u1EF1a tr\xEAn thu\u1EADt to\xE1n tokenize c\u1EE7a m\xF4 h\xECnh Unigram, v\xEC v\u1EADy ch\xFAng ta s\u1EBD \u0111i s\xE2u v\xE0o ph\u1EA7n ti\u1EBFp theo."),Xt=i(),en=o("p"),Me=l("Ch\xFAng ta s\u1EBD t\xE1i s\u1EED d\u1EE5ng kho ng\u1EEF li\u1EC7u t\u1EEB c\xE1c v\xED d\u1EE5 tr\u01B0\u1EDBc:"),Ft=i(),u(bs.$$.fragment),Rt=i(),ln=o("p"),He=l("and for this example, we will take all strict substrings for the initial vocabulary :"),Wt=i(),u(_s.$$.fragment),Qt=i(),R=o("h2"),Z=o("a"),Sn=o("span"),u(js.$$.fragment),Ge=i(),On=o("span"),Le=l("Thu\u1EADt to\xE1n tokenize"),Kt=i(),pn=o("p"),Xe=l("M\xF4 h\xECnh Unigram l\xE0 m\u1ED9t lo\u1EA1i m\xF4 h\xECnh ng\xF4n ng\u1EEF coi m\u1ED7i token l\xE0 \u0111\u1ED9c l\u1EADp v\u1EDBi c\xE1c token tr\u01B0\u1EDBc n\xF3. \u0110\xF3 l\xE0 m\xF4 h\xECnh ng\xF4n ng\u1EEF \u0111\u01A1n gi\u1EA3n nh\u1EA5t, theo ngh\u0129a x\xE1c su\u1EA5t c\u1EE7a token X trong b\u1ED1i c\u1EA3nh tr\u01B0\u1EDBc \u0111\xF3 ch\u1EC9 l\xE0 x\xE1c su\u1EA5t c\u1EE7a token X. V\xEC v\u1EADy, n\u1EBFu ch\xFAng ta s\u1EED d\u1EE5ng m\xF4 h\xECnh ng\xF4n ng\u1EEF Unigram \u0111\u1EC3 t\u1EA1o v\u0103n b\u1EA3n, ch\xFAng ta s\u1EBD lu\xF4n d\u1EF1 \u0111o\xE1n token ph\u1ED5 bi\u1EBFn nh\u1EA5t."),Yt=i(),P=o("p"),Fe=l("X\xE1c su\u1EA5t c\u1EE7a m\u1ED9t token nh\u1EA5t \u0111\u1ECBnh l\xE0 t\u1EA7n su\u1EA5t c\u1EE7a n\xF3 (s\u1ED1 l\u1EA7n ch\xFAng ta t\xECm th\u1EA5y n\xF3) trong kho t\xE0i li\u1EC7u g\u1ED1c, chia cho t\u1ED5ng t\u1EA5t c\u1EA3 c\xE1c t\u1EA7n s\u1ED1 c\u1EE7a t\u1EA5t c\u1EA3 c\xE1c token trong t\u1EEB v\u1EF1ng (\u0111\u1EC3 \u0111\u1EA3m b\u1EA3o x\xE1c su\u1EA5t t\u1ED5ng b\u1EB1ng 1). V\xED d\u1EE5: "),Un=o("code"),Re=l('"ug"'),We=l(" c\xF3 trong "),An=o("code"),Qe=l('"hug"'),Ke=l(", "),Bn=o("code"),Ye=l('"pug"'),Ie=l(" v\xE0 "),Mn=o("code"),Ze=l('"hugs"'),Je=l(", v\xEC v\u1EADy n\xF3 c\xF3 t\u1EA7n su\u1EA5t l\xE0 20 trong kho ng\u1EEF li\u1EC7u c\u1EE7a ch\xFAng t\xF4i."),It=i(),on=o("p"),sl=l("D\u01B0\u1EDBi \u0111\xE2y l\xE0 t\u1EA7n su\u1EA5t c\u1EE7a t\u1EA5t c\u1EA3 c\xE1c t\u1EEB ph\u1EE5 c\xF3 th\u1EC3 c\xF3 trong t\u1EEB v\u1EF1ng:"),Zt=i(),u(ys.$$.fragment),Jt=i(),J=o("p"),nl=l("V\u1EADy n\xEAn, t\u1ED5ng c\u1EE7a t\u1EA5t c\u1EA3 c\xE1c t\u1EA7n su\u1EA5t l\xE0 210, v\xE0 x\xE1c su\u1EA5t c\u1EE7a t\u1EEB ph\u1EE5 "),Hn=o("code"),tl=l('"ug"'),al=l(" l\xE0 20/210."),sa=i(),u(ss.$$.fragment),na=i(),A=o("p"),el=l("Gi\u1EDD, \u0111\u1EC3 tokenize m\u1ED9t t\u1EEB cho tr\u01B0\u1EDBc, ch\xFAng ta s\u1EBD nh\xECn v\xE0o t\u1EA5t c\u1EA3 c\xE1c ph\u1EA7n \u0111o\u1EA1n th\xE0nh token v\xE0 t\xEDnh x\xE1c su\u1EA5t c\u1EE7a t\u1EEBng c\xE1i theo m\xF4 h\xECnh Unigram. V\xEC t\u1EA5t c\u1EA3 token \u0111\u01B0\u1EE3c cho l\xE0 \u0111\u1ED9c l\u1EADp, x\xE1c su\u1EA5t n\xE0y ch\u1EC9 l\xE0 t\xEDch c\u1EE7a x\xE1c su\u1EA5t m\u1ED7i token. V\xED d\u1EE5, "),Gn=o("code"),ll=l('["p", "u", "g"]'),pl=l(" c\u1EE7a "),Ln=o("code"),ol=l('"pug"'),cl=l(` c\xF3 x\xE1c su\u1EA5t:
`),ta=new Te,aa=i(),W=o("p"),hl=l("T\u01B0\u01A1ng t\u1EF1, "),Xn=o("code"),il=l('["pu", "g"]'),rl=l(` c\xF3 x\xE1c su\u1EA5t:
`),ea=new Te,la=i(),cn=o("p"),ml=l("N\xF3i chung, vi\u1EC7c tokenize c\xF3 \xEDt token nh\u1EA5t c\xF3 th\u1EC3 s\u1EBD c\xF3 x\xE1c su\u1EA5t cao nh\u1EA5t (v\xEC ph\xE9p chia cho 210 l\u1EB7p l\u1EA1i cho m\u1ED7i token), t\u01B0\u01A1ng \u1EE9ng v\u1EDBi nh\u1EEFng g\xEC ch\xFAng ta mu\u1ED1n tr\u1EF1c quan: chia m\u1ED9t t\u1EEB th\xE0nh s\u1ED1 l\u01B0\u1EE3ng token nh\u1EA5t c\xF3 th\u1EC3."),pa=i(),ns=o("p"),ul=l("Tokenize c\u1EE7a m\u1ED9t t\u1EEB v\u1EDBi m\xF4 h\xECnh Unigram sau \u0111\xF3 l\xE0 token c\xF3 x\xE1c su\u1EA5t cao nh\u1EA5t. Trong v\xED d\u1EE5 v\u1EC1 "),Fn=o("code"),gl=l('"pug"'),fl=l(", \u0111\xE2y l\xE0 c\xE1c x\xE1c su\u1EA5t m\xE0 ch\xFAng ta s\u1EBD nh\u1EADn \u0111\u01B0\u1EE3c cho m\u1ED7i ph\xE2n \u0111o\u1EA1n c\xF3 th\u1EC3 c\xF3:"),oa=i(),u($s.$$.fragment),ca=i(),N=o("p"),dl=l("V\xEC v\u1EADy, "),Rn=o("code"),kl=l('"pug"'),vl=l(" s\u1EBD \u0111\u01B0\u1EE3c tokenize l\xE0 "),Wn=o("code"),bl=l('["p", "ug"]'),_l=l("  ho\u1EB7c "),Qn=o("code"),jl=l('["pu", "g"]'),yl=l(", t\xF9y thu\u1ED9c v\xE0o ph\xE2n \u0111o\u1EA1n n\xE0o trong s\u1ED1 \u0111\xF3 \u0111\u01B0\u1EE3c g\u1EB7p \u0111\u1EA7u ti\xEAn (l\u01B0u \xFD r\u1EB1ng trong m\u1ED9t ph\xE2n \u0111o\u1EA1n l\u1EDBn h\u01A1n ng\u1EEF li\u1EC7u, nh\u1EEFng tr\u01B0\u1EDDng h\u1EE3p b\xECnh \u0111\u1EB3ng nh\u01B0 th\u1EBF n\xE0y s\u1EBD r\u1EA5t hi\u1EBFm)."),ha=i(),w=o("p"),$l=l("Trong tr\u01B0\u1EDDng h\u1EE3p n\xE0y, th\u1EADt d\u1EC5 d\xE0ng \u0111\u1EC3 t\xECm t\u1EA5t c\u1EA3 c\xE1c ph\xE2n \u0111o\u1EA1n c\xF3 th\u1EC3 c\xF3 v\xE0 t\xEDnh to\xE1n x\xE1c su\u1EA5t c\u1EE7a ch\xFAng, nh\u01B0ng n\xF3i chung s\u1EBD kh\xF3 h\u01A1n m\u1ED9t ch\xFAt. C\xF3 m\u1ED9t thu\u1EADt to\xE1n c\u1ED5 \u0111i\u1EC3n \u0111\u01B0\u1EE3c s\u1EED d\u1EE5ng cho vi\u1EC7c n\xE0y, \u0111\u01B0\u1EE3c g\u1ECDi l\xE0 thu\u1EADt to\xE1n "),Kn=o("em"),ql=l("Viterbi"),wl=l(". V\u1EC1 c\u01A1 b\u1EA3n, ch\xFAng ta c\xF3 th\u1EC3 x\xE2y d\u1EF1ng m\u1ED9t bi\u1EC3u \u0111\u1ED3 \u0111\u1EC3 ph\xE1t hi\u1EC7n c\xE1c ph\xE2n \u0111o\u1EA1n c\xF3 th\u1EC3 c\xF3 c\u1EE7a m\u1ED9t t\u1EEB nh\u1EA5t \u0111\u1ECBnh b\u1EB1ng c\xE1ch n\xF3i r\u1EB1ng c\xF3 m\u1ED9t nh\xE1nh t\u1EEB k\xFD t\u1EF1 "),Yn=o("em"),xl=l("a"),El=l(" \u0111\u1EBFn k\xFD t\u1EF1 "),In=o("em"),Pl=l("b"),zl=l(" n\u1EBFu t\u1EEB ph\u1EE5 t\u1EEB "),Zn=o("em"),Tl=l("a"),Cl=l(" \u0111\u1EBFn "),Jn=o("em"),Vl=l("b"),Nl=l(" n\u1EB1m trong t\u1EEB v\u1EF1ng v\xE0 quy cho nh\xE1nh \u0111\xF3 x\xE1c su\u1EA5t c\u1EE7a t\u1EEB ph\u1EE5 ."),ia=i(),hn=o("p"),Dl=l("\u0110\u1EC3 t\xECm \u0111\u01B0\u1EDDng d\u1EABn trong bi\u1EC3u \u0111\u1ED3 \u0111\xF3 s\u1EBD c\xF3 \u0111i\u1EC3m t\u1ED1t nh\u1EA5t, thu\u1EADt to\xE1n Viterbi x\xE1c \u0111\u1ECBnh, \u0111\u1ED1i v\u1EDBi m\u1ED7i v\u1ECB tr\xED trong t\u1EEB, ph\xE2n \u0111o\u1EA1n c\xF3 \u0111i\u1EC3m t\u1ED1t nh\u1EA5t k\u1EBFt th\xFAc t\u1EA1i v\u1ECB tr\xED \u0111\xF3. V\xEC ch\xFAng ta \u0111i t\u1EEB \u0111\u1EA7u \u0111\u1EBFn cu\u1ED1i, \u0111i\u1EC3m t\u1ED1t nh\u1EA5t c\xF3 th\u1EC3 \u0111\u01B0\u1EE3c t\xECm th\u1EA5y b\u1EB1ng c\xE1ch l\u1EB7p qua t\u1EA5t c\u1EA3 c\xE1c t\u1EEB ph\u1EE5 k\u1EBFt th\xFAc \u1EDF v\u1ECB tr\xED hi\u1EC7n t\u1EA1i v\xE0 sau \u0111\xF3 s\u1EED d\u1EE5ng \u0111i\u1EC3m token t\u1ED1t nh\u1EA5t t\u1EEB \u200B\u200Bv\u1ECB tr\xED m\xE0 t\u1EEB ph\u1EE5 n\xE0y b\u1EAFt \u0111\u1EA7u. Sau \u0111\xF3, ch\xFAng ta ch\u1EC9 c\u1EA7n b\u1ECF qua con \u0111\u01B0\u1EDDng \u0111\xE3 th\u1EF1c hi\u1EC7n \u0111\u1EC3 \u0111\u1EBFn cu\u1ED1i."),ra=i(),ts=o("p"),Sl=l("H\xE3y xem m\u1ED9t v\xED d\u1EE5 s\u1EED d\u1EE5ng t\u1EEB v\u1EF1ng c\u1EE7a ch\xFAng ta v\xE0 t\u1EEB "),st=o("code"),Ol=l('"unhug"'),Ul=l(". \u0110\u1ED1i v\u1EDBi m\u1ED7i v\u1ECB tr\xED, c\xE1c t\u1EEB ph\u1EE5 c\xF3 \u0111i\u1EC3m s\u1ED1 t\u1ED1t nh\u1EA5t k\u1EBFt th\xFAc \u1EDF \u0111\xF3 nh\u01B0 sau:"),ma=i(),u(qs.$$.fragment),ua=i(),H=o("p"),Al=l("V\u1EADy "),nt=o("code"),Bl=l('"unhug"'),Ml=l(" c\xF3 th\u1EC3 tokenize th\xE0nh "),tt=o("code"),Hl=l('["un", "hug"]'),Gl=l("."),ga=i(),u(as.$$.fragment),fa=i(),Q=o("h2"),es=o("a"),at=o("span"),u(ws.$$.fragment),Ll=i(),et=o("span"),Xl=l("Quay l\u1EA1i hu\u1EA5n luy\u1EC7n"),da=i(),rn=o("p"),Fl=l("B\xE2y gi\u1EDD ch\xFAng ta \u0111\xE3 th\u1EA5y c\xE1ch th\u1EE9c ho\u1EA1t \u0111\u1ED9ng c\u1EE7a tokenize, ch\xFAng ta c\xF3 th\u1EC3 t\xECm hi\u1EC3u s\xE2u h\u01A1n m\u1ED9t ch\xFAt v\u1EC1 s\u1EF1 m\u1EA5t m\xE1t \u0111\u01B0\u1EE3c s\u1EED d\u1EE5ng trong qu\xE1 tr\xECnh hu\u1EA5n luy\u1EC7n. \u1EDE b\u1EA5t k\u1EF3 giai \u0111o\u1EA1n nh\u1EA5t \u0111\u1ECBnh n\xE0o, s\u1EF1 m\u1EA5t m\xE1t n\xE0y \u0111\u01B0\u1EE3c t\xEDnh to\xE1n b\u1EB1ng c\xE1ch tokenize m\u1ECDi t\u1EEB trong kho ng\u1EEF li\u1EC7u, s\u1EED d\u1EE5ng t\u1EEB v\u1EF1ng hi\u1EC7n t\u1EA1i v\xE0 m\xF4 h\xECnh Unigram \u0111\u01B0\u1EE3c x\xE1c \u0111\u1ECBnh b\u1EDFi t\u1EA7n s\u1ED1 c\u1EE7a m\u1ED7i token trong kho ng\u1EEF li\u1EC7u (nh\u01B0 \u0111\xE3 th\u1EA5y tr\u01B0\u1EDBc \u0111\xE2y)."),ka=i(),ls=o("p"),Rl=l("M\u1ED7i t\u1EEB trong kho ng\u1EEF li\u1EC7u \u0111\u1EC1u c\xF3 m\u1ED9t \u0111i\u1EC3m v\xE0 s\u1EF1 m\u1EA5t m\xE1t l\xE0 kh\u1EA3 n\u0103ng b\u1ECB \xE2m c\u1EE7a nh\u1EEFng \u0111i\u1EC3m s\u1ED1 \u0111\xF3 - ngh\u0129a l\xE0 t\u1ED5ng cho t\u1EA5t c\u1EA3 c\xE1c t\u1EEB trong kho ng\u1EEF li\u1EC7u c\u1EE7a t\u1EA5t c\u1EA3 c\xE1c "),lt=o("code"),Wl=l("-log(P(word))"),Ql=l("."),va=i(),mn=o("p"),Kl=l("C\xF9ng xem v\xED d\u1EE5 c\u1EE7a ch\xFAng ta v\u1EDBi kho ng\u1EEF li\u1EC7u d\u01B0\u1EDBi \u0111\xE2y:"),ba=i(),u(xs.$$.fragment),_a=i(),un=o("p"),Yl=l("K\u1EBFt qu\u1EA3 tokenize m\u1ED7i t\u1EEB v\xE0 \u0111i\u1EC3m t\u01B0\u01A1ng \u1EE9ng c\u1EE7a ch\xFAng nh\u01B0 sau:"),ja=i(),u(Es.$$.fragment),ya=i(),gn=o("p"),Il=l("V\xE0 s\u1EF1 m\u1EA5t m\xE1t b\u1EB1ng:"),$a=i(),u(Ps.$$.fragment),qa=i(),D=o("p"),Zl=l("Gi\u1EDD th\xEC ta c\u1EA7n t\xEDnh xem vi\u1EC7c lo\u1EA1i b\u1ECF m\u1ED7i token s\u1EBD \u1EA3nh h\u01B0\u1EDFng th\u1EBF n\xE0o t\u1EDBi s\u1EF1 m\u1EA5t m\xE1t. \u0110i\u1EC1u n\xE0y kh\xE1 t\u1EBB nh\u1EA1t, v\xEC v\u1EADy ch\xFAng ta s\u1EBD ch\u1EC9 l\xE0m \u0111i\u1EC1u \u0111\xF3 cho hai token \u1EDF \u0111\xE2y v\xE0 l\u01B0u to\xE0n b\u1ED9 qu\xE1 tr\xECnh khi ch\xFAng ta c\xF3 \u0111o\u1EA1n m\xE3 tr\u1EE3 gi\xFAp. Trong tr\u01B0\u1EDDng h\u1EE3p (r\u1EA5t) c\u1EE5 th\u1EC3 n\xE0y, ch\xFAng t\xF4i c\xF3 hai token t\u01B0\u01A1ng \u0111\u01B0\u01A1ng c\u1EE7a t\u1EA5t c\u1EA3 c\xE1c t\u1EEB: nh\u01B0 ch\xFAng ta \u0111\xE3 th\u1EA5y tr\u01B0\u1EDBc \u0111\xF3, v\xED d\u1EE5: "),pt=o("code"),Jl=l('"pug"'),sp=l(" c\xF3 th\u1EC3 \u0111\u01B0\u1EE3c tokenize "),ot=o("code"),np=l('["p","ug"]'),tp=l(" v\u1EDBi c\xF9ng s\u1ED1 \u0111i\u1EC3m. Do \u0111\xF3, lo\u1EA1i b\u1ECF m\xE3 th\xF4ng b\xE1o "),ct=o("code"),ap=l('"pu"'),ep=l(" kh\u1ECFi t\u1EEB v\u1EF1ng s\u1EBD g\xE2y ra s\u1EF1 m\u1EA5t m\xE1t t\u01B0\u01A1ng t\u1EF1."),wa=i(),S=o("p"),lp=l("M\u1EB7t kh\xE1c, vi\u1EC7c lo\u1EA1i b\u1ECF "),ht=o("code"),pp=l('"hug"'),op=l(" s\u1EBD l\xE0m cho s\u1EF1 m\u1EA5t m\xE1t tr\u1EDF n\xEAn t\u1ED3i t\u1EC7 h\u01A1n, b\u1EDFi v\xEC token c\u1EE7a "),it=o("code"),cp=l('"hug"'),hp=l(" v\xE0 "),rt=o("code"),ip=l('"hugs"'),rp=l(" s\u1EBD tr\u1EDF th\xE0nh:"),xa=i(),u(zs.$$.fragment),Ea=i(),fn=o("p"),mp=l("Nh\u1EEFng thay \u0111\u1ED5i n\xE0y s\u1EBD \u1EA3nh h\u01B0\u1EDFng \u0111\u1EBFn v\xE0 l\xE0m s\u1EF1 m\u1EA5t m\xE1t t\u0103ng l\xEAn"),Pa=i(),u(Ts.$$.fragment),za=i(),G=o("p"),up=l("V\xEC v\u1EADy, token "),mt=o("code"),gp=l('"pu"'),fp=l(" ch\u1EAFc ch\u1EAFn s\u1EBD b\u1ECB lo\u1EA1i kh\u1ECFi b\u1ED9 t\u1EEB v\u1EF1ng, nh\u01B0ng "),ut=o("code"),dp=l('"hug"'),kp=l(" th\xEC kh\xF4ng."),Ta=i(),K=o("h2"),ps=o("a"),gt=o("span"),u(Cs.$$.fragment),vp=i(),ft=o("span"),bp=l("Tri\u1EC3n khai Unigram"),Ca=i(),dn=o("p"),_p=l("Gi\u1EDD h\xE3y c\xF9ng tri\u1EC3n khai m\u1ECDi th\u1EE9 ta \u0111\xE3 c\xF9ng xem th\xF4ng qua \u0111aonj m\xE3. Gi\u1ED1ng nh\u01B0 BPE v\xE0 WordPiece, \u0111\xE2y kh\xF4ng ph\u1EA3i l\xE0 m\u1ED9t c\xE1ch tri\u1EC3n khai hi\u1EC3u qu\u1EA3 c\u1EE7a thu\u1EADt to\xE1n (kh\xE1 ng\u01B0\u1EE3c l\u1EA1i), nh\u01B0ng n\xF3 s\u1EBD gi\xFAp b\u1EA1n hi\u1EC3u h\u01A1n v\u1EC1 Unigram."),Va=i(),kn=o("p"),jp=l("Ta s\u1EBD s\u1EED d\xF9ng c\xF9ng b\u1ED9 ng\u1EEF li\u1EC7u \u0111\xE3 s\u1EED d\u1EE5ng nh\u01B0 m\u1ED9t v\xED d\u1EE5:"),Na=i(),u(Vs.$$.fragment),Da=i(),os=o("p"),yp=l("L\u1EA7n n\xE0y, ta s\u1EBD s\u1EED d\u1EE5ng "),dt=o("code"),$p=l("xlnet-base-cased"),qp=l(" nh\u01B0 m\xF4 h\xECnh:"),Sa=i(),u(Ns.$$.fragment),Oa=i(),vn=o("p"),wp=l("T\u01B0\u01A1ng t\u1EF1 BPE v\xE0 WordPiece, ta s\u1EBD b\u1EAFt \u0111\u1EA7u \u0111\u1EBFm t\u1EA7n su\u1EA5t xu\u1EA5t hi\u1EC7n c\u1EE7a m\u1ED7i t\u1EEB trong kho ng\u1EEF li\u1EC7u:"),Ua=i(),u(Ds.$$.fragment),Aa=i(),bn=o("p"),xp=l("Sau \u0111\xF3, ta c\u1EA7n kh\u1EDFi t\u1EA1o b\u1ED9 t\u1EEB v\u1EF1ng v\u1EDBi s\u1ED1 l\u01B0\u1EE3ng l\u1EDBn h\u01A1n k\xEDch th\u01B0\u1EDBc ta mu\u1ED1n cu\u1ED1i c\xF9ng. Ta ph\u1EA3i t\u1ED5ng k\u1EBFt t\u1EA5t c\u1EA3 c\xE1c k\xED t\u1EF1 c\u01A1 b\u1EA3n (n\u1EBFu kh\xF4ng ta s\u1EBD kh\xF4ng th\u1EC3 tokenize t\u1EA5t c\u1EA3 c\xE1c t\u1EEB), nh\u01B0ng v\u1EDBi c\xE1c chu\u1ED7i con l\u1EDBn h\u01A1n ta s\u1EBD gi\u1EEF ph\u1EA7n th\xF4ng d\u1EE5ng nh\u1EA5t v\xE0 s\u1EAFp x\u1EBFp ch\xFAng theo t\u1EA7n su\u1EA5t:"),Ba=i(),u(Ss.$$.fragment),Ma=i(),u(Os.$$.fragment),Ha=i(),_n=o("p"),Ep=l("Ta nh\xF3m c\xE1c k\xED t\u1EF1 c\xF3 c\xE1c t\u1EEB con t\u1ED1t nh\u1EA5t v\xE0o b\u1ED9 t\u1EEB v\u1EF1ng ban \u0111\u1EA7u c\xF3 k\xEDch th\u01B0\u1EDBc l\xE0 300:"),Ga=i(),u(Us.$$.fragment),La=i(),u(cs.$$.fragment),Xa=i(),jn=o("p"),Pp=l("Ti\u1EBFp theo, ch\xFAng ta t\xEDnh t\u1ED5ng t\u1EA7n su\u1EA5t \u0111\u1EC3 bi\u1EBFn \u0111\u1ED5i c\xE1c t\u1EA7n su\u1EA5t n\xE0y th\xE0nh x\xE1c su\u1EA5t. V\u1EDBi m\xF4 h\xECnh, ch\xFAng ta s\u1EBD l\u01B0u c\xE1c log c\u1EE7a x\xE1c xu\u1EA5t, v\xEC n\xF3 \u1ED5n \u0111\u1ECBnh h\u01A1n v\u1EC1 m\u1EB7t s\u1ED1 h\u1ECDc khi c\u1ED9ng logarit h\u01A1n l\xE0 nh\xE2n c\xE1c s\u1ED1 nh\u1ECF v\xE0 \u0111i\u1EC1u n\xE0y s\u1EBD \u0111\u01A1n gi\u1EA3n h\xF3a vi\u1EC7c t\xEDnh to\xE1n m\u1EA5t m\xE1t c\u1EE7a m\xF4 h\xECnh:"),Fa=i(),u(As.$$.fragment),Ra=i(),hs=o("p"),zp=l("Gi\u1EDD th\xEC c\xE1c h\xE0m ch\xEDnh l\xE0 h\xE0m tokenize t\u1EEB s\u1EED d\u1EE5ng thu\u1EADt to\xE1n Viterbi. Nh\u01B0 \u0111\xE3 th\u1EA5y tr\u01B0\u1EDBc \u0111\xF3, thu\u1EADt to\xE1n t\xEDnh ph\xE2n \u0111o\u1EA1n t\u1ED1t nh\u1EA5t c\u1EE7a m\u1ED7i chu\u1ED7i con c\u1EE7a t\u1EEB, \u0111\u01B0\u1EE3c l\u01B0u d\u01B0\u1EDBi bi\u1EBFn  "),kt=o("code"),Tp=l("best_segmentations"),Cp=l(". Ch\xFAng ta s\u1EBD l\u01B0u m\u1ED7i v\u1ECB tr\xED m\u1ED9t t\u1EEB \u0111i\u1EC3n trong t\u1EEB (t\u1EEB 0 cho t\u1EDBi \u0111\u1ED9 d\xE0i c\u1EE7a n\xF3), v\u1EDBi hai kho\xE1: ch\u1EC9 m\u1EE5c \u0111i\u1EC3m b\u1EAFt \u0111\u1EA7u c\u1EE7a token cu\u1ED1i trong ph\u1EA7n \u0111o\u1EA1n t\u1ED1t nh\u1EA5t, v\xE0 \u0111i\u1EC3m c\u1EE7a ph\xE2n \u0111o\u1EA1n t\u1ED1t nh\u1EA5t. V\u1EDBi ch\u1EC9 m\u1EE5c c\u1EE7a \u0111i\u1EC3m b\u1EAFt \u0111\u1EA7u c\u1EE7a token cu\u1ED1i trong ph\u1EA7n \u0111o\u1EA1n t\u1ED1t nh\u1EA5t, ta s\u1EBD c\xF3 th\u1EC3 truy v\u1EA5n to\xE0n b\u1ED9 ph\xE2n \u0111o\u1EA1n m\u1ED9t khi danh s\xE1ch \u0111\u01B0\u1EE3c \u0111i\u1EC1n \u0111\u1EE7."),Wa=i(),is=o("p"),Vp=l("Vi\u1EC7c \u0111i\u1EC1n danh s\xE1ch \u0111\u01B0\u1EE3c th\u1EF1c hi\u1EC7n ch\u1EC9 v\u1EDBi hai v\xF2ng l\u1EB7p: v\xF2ng l\u1EB7p ch\xEDnh \u0111i qua t\u1EEBng v\u1ECB tr\xED b\u1EAFt \u0111\u1EA7u v\xE0 v\xF2ng l\u1EB7p th\u1EE9 hai th\u1EED t\u1EA5t c\u1EA3 c\xE1c chu\u1ED7i con b\u1EAFt \u0111\u1EA7u t\u1EEB v\u1ECB tr\xED b\u1EAFt \u0111\u1EA7u \u0111\xF3. N\u1EBFu chu\u1ED7i con c\xF3 trong t\u1EEB v\u1EF1ng, ch\xFAng ta c\xF3 m\u1ED9t ph\xE2n \u0111o\u1EA1n m\u1EDBi c\u1EE7a t\u1EEB cho \u0111\u1EBFn v\u1ECB tr\xED k\u1EBFt th\xFAc \u0111\xF3, v\xE0 so s\xE1nh v\u1EDBi nh\u1EEFng g\xEC c\xF3 trong "),vt=o("code"),Np=l("best_segmentations"),Dp=l("."),Qa=i(),yn=o("p"),Sp=l("Khi v\xF2ng l\u1EB7p ch\xEDnh k\u1EBFt th\xFAc, ch\xFAng ta ch\u1EC9 b\u1EAFt \u0111\u1EA7u t\u1EEB cu\u1ED1i v\xE0 nh\u1EA3y t\u1EEB v\u1ECB tr\xED b\u1EAFt \u0111\u1EA7u n\xE0y sang v\u1ECB tr\xED ti\u1EBFp theo, ghi l\u1EA1i c\xE1c token khi ch\xFAng ta \u0111i, cho \u0111\u1EBFn khi ch\xFAng ta \u0111\u1EBFn v\u1ECB tr\xED \u0111\u1EA7u c\u1EE7a t\u1EEB:"),Ka=i(),u(Bs.$$.fragment),Ya=i(),$n=o("p"),Op=l("Ta c\xF3 th\u1EC3 s\u1EB5n s\xE0ng th\u1EED m\xF4 h\xECnh ban \u0111\u1EA7u l\xEAn m\u1ED9t s\u1ED1 t\u1EEB:"),Ia=i(),u(Ms.$$.fragment),Za=i(),u(Hs.$$.fragment),Ja=i(),qn=o("p"),Up=l("Gi\u1EDD th\xEC th\u1EADt d\u1EC5 d\xE0ng \u0111\u1EC3 t\xEDnh s\u1EF1 m\u1EA5t m\xE1t c\u1EE7a m\xF4 h\xECnh tr\xEAn kho ng\u1EEF li\u1EC7u!"),se=i(),u(Gs.$$.fragment),ne=i(),wn=o("p"),Ap=l("Ta c\xF3 th\u1EC3 ki\u1EC3m tra c\xE1ch n\xF3 ho\u1EA1t \u0111\u1ED9ng tr\xEAn m\xF4 h\xECnh ta c\xF3:"),te=i(),u(Ls.$$.fragment),ae=i(),u(Xs.$$.fragment),ee=i(),xn=o("p"),Bp=l("Vi\u1EC7c t\xEDnh \u0111i\u1EC3m cho m\u1ED7i token kh\xF4ng qu\u1EA3 kh\xF3; ta ch\u1EC9 ph\u1EA3i t\xEDnh s\u1EF1 m\u1EA5t m\xE1t c\u1EE7a m\xF4 h\xECnh khi xo\xE1 m\u1ED7i token:"),le=i(),u(Fs.$$.fragment),pe=i(),En=o("p"),Mp=l("Ta c\xF3 th\u1EC3 th\u1EED v\u1EDBi token cho tr\u01B0\u1EDBc:"),oe=i(),u(Rs.$$.fragment),ce=i(),x=o("p"),Hp=l("V\xEC "),bt=o("code"),Gp=l('"ll"'),Lp=l(" \u0111\u01B0\u1EE3c s\u1EED d\u1EE5ng trong qu\xE1 tr\xECnh tokenize  "),_t=o("code"),Xp=l('"Hopefully"'),Fp=l(", v\xE0 lo\u1EA1i b\u1ECF n\xF3 ch\u1EAFc ch\u1EAFn s\u1EBD l\xE0m ta thay v\xE0o \u0111\xF3 s\u1EED d\u1EE5ng "),jt=o("code"),Rp=l('"l"'),Wp=l(" hai l\u1EA7n, ta k\xEC v\u1ECDng n\xF3 s\u1EBD \u0111em l\u1EA1i s\u1EF1 m\u1EA5t m\xE1t d\u01B0\u01A1ng. "),yt=o("code"),Qp=l('"his"'),Kp=l(" ch\u1EC9 \u0111\u01B0\u1EE3c s\u1EED d\u1EE5ng trong t\u1EEB "),$t=o("code"),Yp=l('"This"'),Ip=l(", n\xF3 \u0111\u01B0\u1EE3c tokenize th\xE0nh ch\xEDnh n\xF3, n\xEAn ta k\xEC v\u1ECDng n\xF3 s\u1EBD kh\xF4ng c\xF3 m\u1EA5t m\xE1t. V\xE0 \u0111\xE2y l\xE0 k\u1EBFt qu\u1EA3:"),he=i(),u(Ws.$$.fragment),ie=i(),u(rs.$$.fragment),re=i(),Pn=o("p"),Zp=l("V\u1EDBi t\u1EA5t c\u1EA3 nh\u1EEFng \u0111i\u1EC1u tr\xEAn, \u0111i\u1EC1u cu\u1ED1i c\xF9ng ta c\u1EA7n ph\u1EA3i l\xE0m l\xE0 th\xEAm c\xE1c token \u0111\u1EB7c bi\u1EC7t c\u1EE7a m\xF4 h\xECnh v\xE0o b\u1ED9 t\u1EEB v\u1EF1ng, sau \u0111\xF3 l\u1EB7p cho \u0111\u1EBFn khi ch\xFAng ta c\u1EAFt \u0111\u1EE7 s\u1ED1 token ta mong mu\u1ED1n cho k\xEDch c\u1EE1 b\u1ED9 t\u1EEB v\u1EF1ng:"),me=i(),u(Qs.$$.fragment),ue=i(),ms=o("p"),Jp=l("Sau \u0111\xF3, \u0111\u1EC3 tokenize c\xE1c \u0111o\u1EA1n v\u0103n b\u1EA3n, ta ch\u1EC9 c\u1EA7n \xE1p d\u1EE5ng pre-tokenization v\xE0 sau \u0111\u1ECF s\u1EED d\u1EE5ng h\xE0m "),qt=o("code"),so=l("encode_word()"),no=l(":"),ge=i(),u(Ks.$$.fragment),fe=i(),u(Ys.$$.fragment),de=i(),zn=o("p"),to=l("V\xE0 \u0111\xF3 l\xE0 Unigram! Hy v\u1ECDng r\u1EB1ng b\xE2y gi\u1EDD b\u1EA1n c\u1EA3m th\u1EA5y nh\u01B0 m\u1ED9t chuy\xEAn gia trong t\u1EA5t c\u1EA3 m\u1ECDi tokenizer. Trong ph\u1EA7n ti\u1EBFp theo, ch\xFAng ta s\u1EBD \u0111i s\xE2u v\xE0o c\xE1c kh\u1ED1i c\u1EE7a th\u01B0 vi\u1EC7n \u{1F917} Tokenizers v\xE0 ch\u1EC9 cho b\u1EA1n c\xE1ch b\u1EA1n c\xF3 th\u1EC3 s\u1EED d\u1EE5ng ch\xFAng \u0111\u1EC3 t\u1EA1o tokenizer c\u1EE7a ri\xEAng m\xECnh."),this.h()},l(s){const a=Oc('[data-svelte="svelte-1phssyn"]',document.head);m=c(a,"META",{name:!0,content:!0}),a.forEach(n),y=r(s),b=c(s,"H1",{class:!0});var Is=h(b);_=c(Is,"A",{id:!0,class:!0,href:!0});var wt=h(_);z=c(wt,"SPAN",{});var xt=h(z);g(q.$$.fragment,xt),xt.forEach(n),wt.forEach(n),E=r(Is),V=c(Is,"SPAN",{});var Et=h(V);X=p(Et,"Unigram tokenization"),Et.forEach(n),Is.forEach(n),T=r(s),g(B.$$.fragment,s),ds=r(s),Js=c(s,"P",{});var Pt=h(Js);Ve=p(Pt,"Thu\u1EADt to\xE1n Unigram th\u01B0\u1EDDng \u0111\u01B0\u1EE3c s\u1EED dung trong SentencePiece,  \u0111\xE2y l\xE0 m\u1ED9t thu\u1EADt to\xE1n tokenize cho c\xE1c m\xF4 h\xECnh nh\u01B0 AlBERT, T5, mBART, Big Bird, v\xE0 XLNet."),Pt.forEach(n),Vt=r(s),g(ks.$$.fragment,s),Nt=r(s),g(Y.$$.fragment,s),Dt=r(s),F=c(s,"H2",{class:!0});var ve=h(F);I=c(ve,"A",{id:!0,class:!0,href:!0});var ao=h(I);Nn=c(ao,"SPAN",{});var eo=h(Nn);g(vs.$$.fragment,eo),eo.forEach(n),ao.forEach(n),Ne=r(ve),Dn=c(ve,"SPAN",{});var lo=h(Dn);De=p(lo,"Thu\u1EADt to\xE1n hu\u1EA5n luy\u1EC7n"),lo.forEach(n),ve.forEach(n),St=r(s),sn=c(s,"P",{});var po=h(sn);Se=p(po,"So v\u1EDBi BPE v\xE0 WordPiece, Unigram ho\u1EA1t \u0111\u1ED9ng theo h\u01B0\u1EDBng kh\xE1c: n\xF3 b\u1EAFt \u0111\u1EA7u t\u1EEB m\u1ED9t t\u1EEB v\u1EF1ng l\u1EDBn v\xE0 lo\u1EA1i b\u1ECF c\xE1c token cho \u0111\u1EBFn khi n\xF3 \u0111\u1EA1t \u0111\u1EBFn k\xEDch th\u01B0\u1EDBc t\u1EEB v\u1EF1ng mong mu\u1ED1n. C\xF3 m\u1ED9t s\u1ED1 t\xF9y ch\u1ECDn \u0111\u1EC3 s\u1EED d\u1EE5ng \u0111\u1EC3 x\xE2y d\u1EF1ng v\u1ED1n t\u1EEB v\u1EF1ng c\u01A1 b\u1EA3n: v\xED d\u1EE5: ch\xFAng ta c\xF3 th\u1EC3 l\u1EA5y c\xE1c chu\u1ED7i con ph\u1ED5 bi\u1EBFn nh\u1EA5t trong c\xE1c t\u1EEB \u0111\u01B0\u1EE3c ti\u1EC1n tokenize ho\u1EB7c \xE1p d\u1EE5ng BPE tr\xEAn kho ng\u1EEF li\u1EC7u ban \u0111\u1EA7u c\xF3 k\xEDch th\u01B0\u1EDBc t\u1EEB v\u1EF1ng l\u1EDBn."),po.forEach(n),Ot=r(s),nn=c(s,"P",{});var oo=h(nn);Oe=p(oo,"T\u1EA1i m\u1ED7i b\u01B0\u1EDBc c\u1EE7a qu\xE1 tr\xECnh hu\u1EA5n luy\u1EC7n, thu\u1EADt to\xE1n Unigram t\xEDnh to\xE1n s\u1EF1 m\u1EA5t m\xE1t tr\xEAn kho ng\u1EEF li\u1EC7u \u0111\u01B0\u1EE3c cung c\u1EA5p t\u1EEB v\u1EF1ng hi\u1EC7n t\u1EA1i. Sau \u0111\xF3, \u0111\u1ED1i v\u1EDBi m\u1ED7i k\xFD hi\u1EC7u trong t\u1EEB v\u1EF1ng, thu\u1EADt to\xE1n s\u1EBD t\xEDnh to\xE1n m\u1EE9c \u0111\u1ED9 t\u1ED5n th\u1EA5t t\u1ED5ng th\u1EC3 s\u1EBD t\u0103ng l\xEAn bao nhi\xEAu n\u1EBFu k\xFD hi\u1EC7u b\u1ECB x\xF3a v\xE0 t\xECm ki\u1EBFm c\xE1c k\xFD hi\u1EC7u l\xE0m t\u0103ng n\xF3 \xEDt nh\u1EA5t. Nh\u1EEFng bi\u1EC3u t\u01B0\u1EE3ng \u0111\xF3 c\xF3 \u1EA3nh h\u01B0\u1EDFng th\u1EA5p h\u01A1n \u0111\u1EBFn s\u1EF1 m\u1EA5t m\xE1t t\u1ED5ng th\u1EC3 \u0111\u1ED1i v\u1EDBi kho d\u1EEF li\u1EC7u, v\xEC v\u1EADy theo m\u1ED9t ngh\u0129a n\xE0o \u0111\xF3, ch\xFAng \u201C\xEDt c\u1EA7n thi\u1EBFt h\u01A1n\u201D v\xE0 l\xE0 nh\u1EEFng \u1EE9ng c\u1EED vi\xEAn t\u1ED1t nh\u1EA5t \u0111\u1EC3 lo\u1EA1i b\u1ECF."),oo.forEach(n),Ut=r(s),M=c(s,"P",{});var Tn=h(M);Ue=p(Tn,"\u0110\xE2y l\xE0 m\u1ED9t ho\u1EA1t \u0111\u1ED9ng r\u1EA5t t\u1ED1n k\xE9m, v\xEC v\u1EADy ch\xFAng t\xF4i kh\xF4ng ch\u1EC9 lo\u1EA1i b\u1ECF m\u1ED9t bi\u1EC3u t\u01B0\u1EE3ng li\xEAn quan \u0111\u1EBFn m\u1EE9c t\u0103ng t\u1ED5n th\u1EA5t th\u1EA5p nh\u1EA5t, m\xE0 "),At=Ce(Tn),Bt=p(Tn," ("),Mt=Ce(Tn),Ht=p(Tn," l\xE0 m\u1ED9t si\xEAu tham s\u1ED1 b\u1EA1n c\xF3 th\u1EC3 ki\u1EC3m so\xE1t, th\u01B0\u1EDDng l\xE0 10 ho\u1EB7c 20) ph\u1EA7n tr\u0103m c\xE1c k\xFD hi\u1EC7u li\xEAn quan \u0111\u1EBFn m\u1EE9c t\u0103ng t\u1ED5n th\u1EA5t th\u1EA5p nh\u1EA5t. Qu\xE1 tr\xECnh n\xE0y sau \u0111\xF3 \u0111\u01B0\u1EE3c l\u1EB7p l\u1EA1i cho \u0111\u1EBFn khi t\u1EEB v\u1EF1ng \u0111\u1EA1t \u0111\u01B0\u1EE3c k\xEDch th\u01B0\u1EDBc mong mu\u1ED1n."),Tn.forEach(n),Gt=r(s),tn=c(s,"P",{});var co=h(tn);Ae=p(co,"L\u01B0u \xFD r\u1EB1ng ch\xFAng ta kh\xF4ng bao gi\u1EDD x\xF3a c\xE1c k\xFD t\u1EF1 c\u01A1 s\u1EDF, \u0111\u1EC3 \u0111\u1EA3m b\u1EA3o r\u1EB1ng b\u1EA5t k\u1EF3 t\u1EEB n\xE0o c\u0169ng c\xF3 th\u1EC3 \u0111\u01B0\u1EE3c tokenize."),co.forEach(n),Lt=r(s),an=c(s,"P",{});var ho=h(an);Be=p(ho,"B\xE2y gi\u1EDD, \u0111i\u1EC1u n\xE0y v\u1EABn c\xF2n h\u01A1i m\u01A1 h\u1ED3: ph\u1EA7n ch\xEDnh c\u1EE7a thu\u1EADt to\xE1n l\xE0 t\xEDnh to\xE1n s\u1EF1 m\u1EA5t m\xE1t trong kho ng\u1EEF li\u1EC7u v\xE0 xem n\xF3 thay \u0111\u1ED5i nh\u01B0 th\u1EBF n\xE0o khi ch\xFAng t\xF4i x\xF3a m\u1ED9t s\u1ED1 token kh\u1ECFi t\u1EEB v\u1EF1ng, nh\u01B0ng ch\xFAng ta ch\u01B0a gi\u1EA3i th\xEDch c\xE1ch th\u1EF1c hi\u1EC7n \u0111i\u1EC1u n\xE0y. B\u01B0\u1EDBc n\xE0y d\u1EF1a tr\xEAn thu\u1EADt to\xE1n tokenize c\u1EE7a m\xF4 h\xECnh Unigram, v\xEC v\u1EADy ch\xFAng ta s\u1EBD \u0111i s\xE2u v\xE0o ph\u1EA7n ti\u1EBFp theo."),ho.forEach(n),Xt=r(s),en=c(s,"P",{});var io=h(en);Me=p(io,"Ch\xFAng ta s\u1EBD t\xE1i s\u1EED d\u1EE5ng kho ng\u1EEF li\u1EC7u t\u1EEB c\xE1c v\xED d\u1EE5 tr\u01B0\u1EDBc:"),io.forEach(n),Ft=r(s),g(bs.$$.fragment,s),Rt=r(s),ln=c(s,"P",{});var ro=h(ln);He=p(ro,"and for this example, we will take all strict substrings for the initial vocabulary :"),ro.forEach(n),Wt=r(s),g(_s.$$.fragment,s),Qt=r(s),R=c(s,"H2",{class:!0});var be=h(R);Z=c(be,"A",{id:!0,class:!0,href:!0});var mo=h(Z);Sn=c(mo,"SPAN",{});var uo=h(Sn);g(js.$$.fragment,uo),uo.forEach(n),mo.forEach(n),Ge=r(be),On=c(be,"SPAN",{});var go=h(On);Le=p(go,"Thu\u1EADt to\xE1n tokenize"),go.forEach(n),be.forEach(n),Kt=r(s),pn=c(s,"P",{});var fo=h(pn);Xe=p(fo,"M\xF4 h\xECnh Unigram l\xE0 m\u1ED9t lo\u1EA1i m\xF4 h\xECnh ng\xF4n ng\u1EEF coi m\u1ED7i token l\xE0 \u0111\u1ED9c l\u1EADp v\u1EDBi c\xE1c token tr\u01B0\u1EDBc n\xF3. \u0110\xF3 l\xE0 m\xF4 h\xECnh ng\xF4n ng\u1EEF \u0111\u01A1n gi\u1EA3n nh\u1EA5t, theo ngh\u0129a x\xE1c su\u1EA5t c\u1EE7a token X trong b\u1ED1i c\u1EA3nh tr\u01B0\u1EDBc \u0111\xF3 ch\u1EC9 l\xE0 x\xE1c su\u1EA5t c\u1EE7a token X. V\xEC v\u1EADy, n\u1EBFu ch\xFAng ta s\u1EED d\u1EE5ng m\xF4 h\xECnh ng\xF4n ng\u1EEF Unigram \u0111\u1EC3 t\u1EA1o v\u0103n b\u1EA3n, ch\xFAng ta s\u1EBD lu\xF4n d\u1EF1 \u0111o\xE1n token ph\u1ED5 bi\u1EBFn nh\u1EA5t."),fo.forEach(n),Yt=r(s),P=c(s,"P",{});var L=h(P);Fe=p(L,"X\xE1c su\u1EA5t c\u1EE7a m\u1ED9t token nh\u1EA5t \u0111\u1ECBnh l\xE0 t\u1EA7n su\u1EA5t c\u1EE7a n\xF3 (s\u1ED1 l\u1EA7n ch\xFAng ta t\xECm th\u1EA5y n\xF3) trong kho t\xE0i li\u1EC7u g\u1ED1c, chia cho t\u1ED5ng t\u1EA5t c\u1EA3 c\xE1c t\u1EA7n s\u1ED1 c\u1EE7a t\u1EA5t c\u1EA3 c\xE1c token trong t\u1EEB v\u1EF1ng (\u0111\u1EC3 \u0111\u1EA3m b\u1EA3o x\xE1c su\u1EA5t t\u1ED5ng b\u1EB1ng 1). V\xED d\u1EE5: "),Un=c(L,"CODE",{});var ko=h(Un);Re=p(ko,'"ug"'),ko.forEach(n),We=p(L," c\xF3 trong "),An=c(L,"CODE",{});var vo=h(An);Qe=p(vo,'"hug"'),vo.forEach(n),Ke=p(L,", "),Bn=c(L,"CODE",{});var bo=h(Bn);Ye=p(bo,'"pug"'),bo.forEach(n),Ie=p(L," v\xE0 "),Mn=c(L,"CODE",{});var _o=h(Mn);Ze=p(_o,'"hugs"'),_o.forEach(n),Je=p(L,", v\xEC v\u1EADy n\xF3 c\xF3 t\u1EA7n su\u1EA5t l\xE0 20 trong kho ng\u1EEF li\u1EC7u c\u1EE7a ch\xFAng t\xF4i."),L.forEach(n),It=r(s),on=c(s,"P",{});var jo=h(on);sl=p(jo,"D\u01B0\u1EDBi \u0111\xE2y l\xE0 t\u1EA7n su\u1EA5t c\u1EE7a t\u1EA5t c\u1EA3 c\xE1c t\u1EEB ph\u1EE5 c\xF3 th\u1EC3 c\xF3 trong t\u1EEB v\u1EF1ng:"),jo.forEach(n),Zt=r(s),g(ys.$$.fragment,s),Jt=r(s),J=c(s,"P",{});var _e=h(J);nl=p(_e,"V\u1EADy n\xEAn, t\u1ED5ng c\u1EE7a t\u1EA5t c\u1EA3 c\xE1c t\u1EA7n su\u1EA5t l\xE0 210, v\xE0 x\xE1c su\u1EA5t c\u1EE7a t\u1EEB ph\u1EE5 "),Hn=c(_e,"CODE",{});var yo=h(Hn);tl=p(yo,'"ug"'),yo.forEach(n),al=p(_e," l\xE0 20/210."),_e.forEach(n),sa=r(s),g(ss.$$.fragment,s),na=r(s),A=c(s,"P",{});var Zs=h(A);el=p(Zs,"Gi\u1EDD, \u0111\u1EC3 tokenize m\u1ED9t t\u1EEB cho tr\u01B0\u1EDBc, ch\xFAng ta s\u1EBD nh\xECn v\xE0o t\u1EA5t c\u1EA3 c\xE1c ph\u1EA7n \u0111o\u1EA1n th\xE0nh token v\xE0 t\xEDnh x\xE1c su\u1EA5t c\u1EE7a t\u1EEBng c\xE1i theo m\xF4 h\xECnh Unigram. V\xEC t\u1EA5t c\u1EA3 token \u0111\u01B0\u1EE3c cho l\xE0 \u0111\u1ED9c l\u1EADp, x\xE1c su\u1EA5t n\xE0y ch\u1EC9 l\xE0 t\xEDch c\u1EE7a x\xE1c su\u1EA5t m\u1ED7i token. V\xED d\u1EE5, "),Gn=c(Zs,"CODE",{});var $o=h(Gn);ll=p($o,'["p", "u", "g"]'),$o.forEach(n),pl=p(Zs," c\u1EE7a "),Ln=c(Zs,"CODE",{});var qo=h(Ln);ol=p(qo,'"pug"'),qo.forEach(n),cl=p(Zs,` c\xF3 x\xE1c su\u1EA5t:
`),ta=Ce(Zs),Zs.forEach(n),aa=r(s),W=c(s,"P",{});var zt=h(W);hl=p(zt,"T\u01B0\u01A1ng t\u1EF1, "),Xn=c(zt,"CODE",{});var wo=h(Xn);il=p(wo,'["pu", "g"]'),wo.forEach(n),rl=p(zt,` c\xF3 x\xE1c su\u1EA5t:
`),ea=Ce(zt),zt.forEach(n),la=r(s),cn=c(s,"P",{});var xo=h(cn);ml=p(xo,"N\xF3i chung, vi\u1EC7c tokenize c\xF3 \xEDt token nh\u1EA5t c\xF3 th\u1EC3 s\u1EBD c\xF3 x\xE1c su\u1EA5t cao nh\u1EA5t (v\xEC ph\xE9p chia cho 210 l\u1EB7p l\u1EA1i cho m\u1ED7i token), t\u01B0\u01A1ng \u1EE9ng v\u1EDBi nh\u1EEFng g\xEC ch\xFAng ta mu\u1ED1n tr\u1EF1c quan: chia m\u1ED9t t\u1EEB th\xE0nh s\u1ED1 l\u01B0\u1EE3ng token nh\u1EA5t c\xF3 th\u1EC3."),xo.forEach(n),pa=r(s),ns=c(s,"P",{});var je=h(ns);ul=p(je,"Tokenize c\u1EE7a m\u1ED9t t\u1EEB v\u1EDBi m\xF4 h\xECnh Unigram sau \u0111\xF3 l\xE0 token c\xF3 x\xE1c su\u1EA5t cao nh\u1EA5t. Trong v\xED d\u1EE5 v\u1EC1 "),Fn=c(je,"CODE",{});var Eo=h(Fn);gl=p(Eo,'"pug"'),Eo.forEach(n),fl=p(je,", \u0111\xE2y l\xE0 c\xE1c x\xE1c su\u1EA5t m\xE0 ch\xFAng ta s\u1EBD nh\u1EADn \u0111\u01B0\u1EE3c cho m\u1ED7i ph\xE2n \u0111o\u1EA1n c\xF3 th\u1EC3 c\xF3:"),je.forEach(n),oa=r(s),g($s.$$.fragment,s),ca=r(s),N=c(s,"P",{});var us=h(N);dl=p(us,"V\xEC v\u1EADy, "),Rn=c(us,"CODE",{});var Po=h(Rn);kl=p(Po,'"pug"'),Po.forEach(n),vl=p(us," s\u1EBD \u0111\u01B0\u1EE3c tokenize l\xE0 "),Wn=c(us,"CODE",{});var zo=h(Wn);bl=p(zo,'["p", "ug"]'),zo.forEach(n),_l=p(us,"  ho\u1EB7c "),Qn=c(us,"CODE",{});var To=h(Qn);jl=p(To,'["pu", "g"]'),To.forEach(n),yl=p(us,", t\xF9y thu\u1ED9c v\xE0o ph\xE2n \u0111o\u1EA1n n\xE0o trong s\u1ED1 \u0111\xF3 \u0111\u01B0\u1EE3c g\u1EB7p \u0111\u1EA7u ti\xEAn (l\u01B0u \xFD r\u1EB1ng trong m\u1ED9t ph\xE2n \u0111o\u1EA1n l\u1EDBn h\u01A1n ng\u1EEF li\u1EC7u, nh\u1EEFng tr\u01B0\u1EDDng h\u1EE3p b\xECnh \u0111\u1EB3ng nh\u01B0 th\u1EBF n\xE0y s\u1EBD r\u1EA5t hi\u1EBFm)."),us.forEach(n),ha=r(s),w=c(s,"P",{});var O=h(w);$l=p(O,"Trong tr\u01B0\u1EDDng h\u1EE3p n\xE0y, th\u1EADt d\u1EC5 d\xE0ng \u0111\u1EC3 t\xECm t\u1EA5t c\u1EA3 c\xE1c ph\xE2n \u0111o\u1EA1n c\xF3 th\u1EC3 c\xF3 v\xE0 t\xEDnh to\xE1n x\xE1c su\u1EA5t c\u1EE7a ch\xFAng, nh\u01B0ng n\xF3i chung s\u1EBD kh\xF3 h\u01A1n m\u1ED9t ch\xFAt. C\xF3 m\u1ED9t thu\u1EADt to\xE1n c\u1ED5 \u0111i\u1EC3n \u0111\u01B0\u1EE3c s\u1EED d\u1EE5ng cho vi\u1EC7c n\xE0y, \u0111\u01B0\u1EE3c g\u1ECDi l\xE0 thu\u1EADt to\xE1n "),Kn=c(O,"EM",{});var Co=h(Kn);ql=p(Co,"Viterbi"),Co.forEach(n),wl=p(O,". V\u1EC1 c\u01A1 b\u1EA3n, ch\xFAng ta c\xF3 th\u1EC3 x\xE2y d\u1EF1ng m\u1ED9t bi\u1EC3u \u0111\u1ED3 \u0111\u1EC3 ph\xE1t hi\u1EC7n c\xE1c ph\xE2n \u0111o\u1EA1n c\xF3 th\u1EC3 c\xF3 c\u1EE7a m\u1ED9t t\u1EEB nh\u1EA5t \u0111\u1ECBnh b\u1EB1ng c\xE1ch n\xF3i r\u1EB1ng c\xF3 m\u1ED9t nh\xE1nh t\u1EEB k\xFD t\u1EF1 "),Yn=c(O,"EM",{});var Vo=h(Yn);xl=p(Vo,"a"),Vo.forEach(n),El=p(O," \u0111\u1EBFn k\xFD t\u1EF1 "),In=c(O,"EM",{});var No=h(In);Pl=p(No,"b"),No.forEach(n),zl=p(O," n\u1EBFu t\u1EEB ph\u1EE5 t\u1EEB "),Zn=c(O,"EM",{});var Do=h(Zn);Tl=p(Do,"a"),Do.forEach(n),Cl=p(O," \u0111\u1EBFn "),Jn=c(O,"EM",{});var So=h(Jn);Vl=p(So,"b"),So.forEach(n),Nl=p(O," n\u1EB1m trong t\u1EEB v\u1EF1ng v\xE0 quy cho nh\xE1nh \u0111\xF3 x\xE1c su\u1EA5t c\u1EE7a t\u1EEB ph\u1EE5 ."),O.forEach(n),ia=r(s),hn=c(s,"P",{});var Oo=h(hn);Dl=p(Oo,"\u0110\u1EC3 t\xECm \u0111\u01B0\u1EDDng d\u1EABn trong bi\u1EC3u \u0111\u1ED3 \u0111\xF3 s\u1EBD c\xF3 \u0111i\u1EC3m t\u1ED1t nh\u1EA5t, thu\u1EADt to\xE1n Viterbi x\xE1c \u0111\u1ECBnh, \u0111\u1ED1i v\u1EDBi m\u1ED7i v\u1ECB tr\xED trong t\u1EEB, ph\xE2n \u0111o\u1EA1n c\xF3 \u0111i\u1EC3m t\u1ED1t nh\u1EA5t k\u1EBFt th\xFAc t\u1EA1i v\u1ECB tr\xED \u0111\xF3. V\xEC ch\xFAng ta \u0111i t\u1EEB \u0111\u1EA7u \u0111\u1EBFn cu\u1ED1i, \u0111i\u1EC3m t\u1ED1t nh\u1EA5t c\xF3 th\u1EC3 \u0111\u01B0\u1EE3c t\xECm th\u1EA5y b\u1EB1ng c\xE1ch l\u1EB7p qua t\u1EA5t c\u1EA3 c\xE1c t\u1EEB ph\u1EE5 k\u1EBFt th\xFAc \u1EDF v\u1ECB tr\xED hi\u1EC7n t\u1EA1i v\xE0 sau \u0111\xF3 s\u1EED d\u1EE5ng \u0111i\u1EC3m token t\u1ED1t nh\u1EA5t t\u1EEB \u200B\u200Bv\u1ECB tr\xED m\xE0 t\u1EEB ph\u1EE5 n\xE0y b\u1EAFt \u0111\u1EA7u. Sau \u0111\xF3, ch\xFAng ta ch\u1EC9 c\u1EA7n b\u1ECF qua con \u0111\u01B0\u1EDDng \u0111\xE3 th\u1EF1c hi\u1EC7n \u0111\u1EC3 \u0111\u1EBFn cu\u1ED1i."),Oo.forEach(n),ra=r(s),ts=c(s,"P",{});var ye=h(ts);Sl=p(ye,"H\xE3y xem m\u1ED9t v\xED d\u1EE5 s\u1EED d\u1EE5ng t\u1EEB v\u1EF1ng c\u1EE7a ch\xFAng ta v\xE0 t\u1EEB "),st=c(ye,"CODE",{});var Uo=h(st);Ol=p(Uo,'"unhug"'),Uo.forEach(n),Ul=p(ye,". \u0110\u1ED1i v\u1EDBi m\u1ED7i v\u1ECB tr\xED, c\xE1c t\u1EEB ph\u1EE5 c\xF3 \u0111i\u1EC3m s\u1ED1 t\u1ED1t nh\u1EA5t k\u1EBFt th\xFAc \u1EDF \u0111\xF3 nh\u01B0 sau:"),ye.forEach(n),ma=r(s),g(qs.$$.fragment,s),ua=r(s),H=c(s,"P",{});var Cn=h(H);Al=p(Cn,"V\u1EADy "),nt=c(Cn,"CODE",{});var Ao=h(nt);Bl=p(Ao,'"unhug"'),Ao.forEach(n),Ml=p(Cn," c\xF3 th\u1EC3 tokenize th\xE0nh "),tt=c(Cn,"CODE",{});var Bo=h(tt);Hl=p(Bo,'["un", "hug"]'),Bo.forEach(n),Gl=p(Cn,"."),Cn.forEach(n),ga=r(s),g(as.$$.fragment,s),fa=r(s),Q=c(s,"H2",{class:!0});var $e=h(Q);es=c($e,"A",{id:!0,class:!0,href:!0});var Mo=h(es);at=c(Mo,"SPAN",{});var Ho=h(at);g(ws.$$.fragment,Ho),Ho.forEach(n),Mo.forEach(n),Ll=r($e),et=c($e,"SPAN",{});var Go=h(et);Xl=p(Go,"Quay l\u1EA1i hu\u1EA5n luy\u1EC7n"),Go.forEach(n),$e.forEach(n),da=r(s),rn=c(s,"P",{});var Lo=h(rn);Fl=p(Lo,"B\xE2y gi\u1EDD ch\xFAng ta \u0111\xE3 th\u1EA5y c\xE1ch th\u1EE9c ho\u1EA1t \u0111\u1ED9ng c\u1EE7a tokenize, ch\xFAng ta c\xF3 th\u1EC3 t\xECm hi\u1EC3u s\xE2u h\u01A1n m\u1ED9t ch\xFAt v\u1EC1 s\u1EF1 m\u1EA5t m\xE1t \u0111\u01B0\u1EE3c s\u1EED d\u1EE5ng trong qu\xE1 tr\xECnh hu\u1EA5n luy\u1EC7n. \u1EDE b\u1EA5t k\u1EF3 giai \u0111o\u1EA1n nh\u1EA5t \u0111\u1ECBnh n\xE0o, s\u1EF1 m\u1EA5t m\xE1t n\xE0y \u0111\u01B0\u1EE3c t\xEDnh to\xE1n b\u1EB1ng c\xE1ch tokenize m\u1ECDi t\u1EEB trong kho ng\u1EEF li\u1EC7u, s\u1EED d\u1EE5ng t\u1EEB v\u1EF1ng hi\u1EC7n t\u1EA1i v\xE0 m\xF4 h\xECnh Unigram \u0111\u01B0\u1EE3c x\xE1c \u0111\u1ECBnh b\u1EDFi t\u1EA7n s\u1ED1 c\u1EE7a m\u1ED7i token trong kho ng\u1EEF li\u1EC7u (nh\u01B0 \u0111\xE3 th\u1EA5y tr\u01B0\u1EDBc \u0111\xE2y)."),Lo.forEach(n),ka=r(s),ls=c(s,"P",{});var qe=h(ls);Rl=p(qe,"M\u1ED7i t\u1EEB trong kho ng\u1EEF li\u1EC7u \u0111\u1EC1u c\xF3 m\u1ED9t \u0111i\u1EC3m v\xE0 s\u1EF1 m\u1EA5t m\xE1t l\xE0 kh\u1EA3 n\u0103ng b\u1ECB \xE2m c\u1EE7a nh\u1EEFng \u0111i\u1EC3m s\u1ED1 \u0111\xF3 - ngh\u0129a l\xE0 t\u1ED5ng cho t\u1EA5t c\u1EA3 c\xE1c t\u1EEB trong kho ng\u1EEF li\u1EC7u c\u1EE7a t\u1EA5t c\u1EA3 c\xE1c "),lt=c(qe,"CODE",{});var Xo=h(lt);Wl=p(Xo,"-log(P(word))"),Xo.forEach(n),Ql=p(qe,"."),qe.forEach(n),va=r(s),mn=c(s,"P",{});var Fo=h(mn);Kl=p(Fo,"C\xF9ng xem v\xED d\u1EE5 c\u1EE7a ch\xFAng ta v\u1EDBi kho ng\u1EEF li\u1EC7u d\u01B0\u1EDBi \u0111\xE2y:"),Fo.forEach(n),ba=r(s),g(xs.$$.fragment,s),_a=r(s),un=c(s,"P",{});var Ro=h(un);Yl=p(Ro,"K\u1EBFt qu\u1EA3 tokenize m\u1ED7i t\u1EEB v\xE0 \u0111i\u1EC3m t\u01B0\u01A1ng \u1EE9ng c\u1EE7a ch\xFAng nh\u01B0 sau:"),Ro.forEach(n),ja=r(s),g(Es.$$.fragment,s),ya=r(s),gn=c(s,"P",{});var Wo=h(gn);Il=p(Wo,"V\xE0 s\u1EF1 m\u1EA5t m\xE1t b\u1EB1ng:"),Wo.forEach(n),$a=r(s),g(Ps.$$.fragment,s),qa=r(s),D=c(s,"P",{});var gs=h(D);Zl=p(gs,"Gi\u1EDD th\xEC ta c\u1EA7n t\xEDnh xem vi\u1EC7c lo\u1EA1i b\u1ECF m\u1ED7i token s\u1EBD \u1EA3nh h\u01B0\u1EDFng th\u1EBF n\xE0o t\u1EDBi s\u1EF1 m\u1EA5t m\xE1t. \u0110i\u1EC1u n\xE0y kh\xE1 t\u1EBB nh\u1EA1t, v\xEC v\u1EADy ch\xFAng ta s\u1EBD ch\u1EC9 l\xE0m \u0111i\u1EC1u \u0111\xF3 cho hai token \u1EDF \u0111\xE2y v\xE0 l\u01B0u to\xE0n b\u1ED9 qu\xE1 tr\xECnh khi ch\xFAng ta c\xF3 \u0111o\u1EA1n m\xE3 tr\u1EE3 gi\xFAp. Trong tr\u01B0\u1EDDng h\u1EE3p (r\u1EA5t) c\u1EE5 th\u1EC3 n\xE0y, ch\xFAng t\xF4i c\xF3 hai token t\u01B0\u01A1ng \u0111\u01B0\u01A1ng c\u1EE7a t\u1EA5t c\u1EA3 c\xE1c t\u1EEB: nh\u01B0 ch\xFAng ta \u0111\xE3 th\u1EA5y tr\u01B0\u1EDBc \u0111\xF3, v\xED d\u1EE5: "),pt=c(gs,"CODE",{});var Qo=h(pt);Jl=p(Qo,'"pug"'),Qo.forEach(n),sp=p(gs," c\xF3 th\u1EC3 \u0111\u01B0\u1EE3c tokenize "),ot=c(gs,"CODE",{});var Ko=h(ot);np=p(Ko,'["p","ug"]'),Ko.forEach(n),tp=p(gs," v\u1EDBi c\xF9ng s\u1ED1 \u0111i\u1EC3m. Do \u0111\xF3, lo\u1EA1i b\u1ECF m\xE3 th\xF4ng b\xE1o "),ct=c(gs,"CODE",{});var Yo=h(ct);ap=p(Yo,'"pu"'),Yo.forEach(n),ep=p(gs," kh\u1ECFi t\u1EEB v\u1EF1ng s\u1EBD g\xE2y ra s\u1EF1 m\u1EA5t m\xE1t t\u01B0\u01A1ng t\u1EF1."),gs.forEach(n),wa=r(s),S=c(s,"P",{});var fs=h(S);lp=p(fs,"M\u1EB7t kh\xE1c, vi\u1EC7c lo\u1EA1i b\u1ECF "),ht=c(fs,"CODE",{});var Io=h(ht);pp=p(Io,'"hug"'),Io.forEach(n),op=p(fs," s\u1EBD l\xE0m cho s\u1EF1 m\u1EA5t m\xE1t tr\u1EDF n\xEAn t\u1ED3i t\u1EC7 h\u01A1n, b\u1EDFi v\xEC token c\u1EE7a "),it=c(fs,"CODE",{});var Zo=h(it);cp=p(Zo,'"hug"'),Zo.forEach(n),hp=p(fs," v\xE0 "),rt=c(fs,"CODE",{});var Jo=h(rt);ip=p(Jo,'"hugs"'),Jo.forEach(n),rp=p(fs," s\u1EBD tr\u1EDF th\xE0nh:"),fs.forEach(n),xa=r(s),g(zs.$$.fragment,s),Ea=r(s),fn=c(s,"P",{});var sc=h(fn);mp=p(sc,"Nh\u1EEFng thay \u0111\u1ED5i n\xE0y s\u1EBD \u1EA3nh h\u01B0\u1EDFng \u0111\u1EBFn v\xE0 l\xE0m s\u1EF1 m\u1EA5t m\xE1t t\u0103ng l\xEAn"),sc.forEach(n),Pa=r(s),g(Ts.$$.fragment,s),za=r(s),G=c(s,"P",{});var Vn=h(G);up=p(Vn,"V\xEC v\u1EADy, token "),mt=c(Vn,"CODE",{});var nc=h(mt);gp=p(nc,'"pu"'),nc.forEach(n),fp=p(Vn," ch\u1EAFc ch\u1EAFn s\u1EBD b\u1ECB lo\u1EA1i kh\u1ECFi b\u1ED9 t\u1EEB v\u1EF1ng, nh\u01B0ng "),ut=c(Vn,"CODE",{});var tc=h(ut);dp=p(tc,'"hug"'),tc.forEach(n),kp=p(Vn," th\xEC kh\xF4ng."),Vn.forEach(n),Ta=r(s),K=c(s,"H2",{class:!0});var we=h(K);ps=c(we,"A",{id:!0,class:!0,href:!0});var ac=h(ps);gt=c(ac,"SPAN",{});var ec=h(gt);g(Cs.$$.fragment,ec),ec.forEach(n),ac.forEach(n),vp=r(we),ft=c(we,"SPAN",{});var lc=h(ft);bp=p(lc,"Tri\u1EC3n khai Unigram"),lc.forEach(n),we.forEach(n),Ca=r(s),dn=c(s,"P",{});var pc=h(dn);_p=p(pc,"Gi\u1EDD h\xE3y c\xF9ng tri\u1EC3n khai m\u1ECDi th\u1EE9 ta \u0111\xE3 c\xF9ng xem th\xF4ng qua \u0111aonj m\xE3. Gi\u1ED1ng nh\u01B0 BPE v\xE0 WordPiece, \u0111\xE2y kh\xF4ng ph\u1EA3i l\xE0 m\u1ED9t c\xE1ch tri\u1EC3n khai hi\u1EC3u qu\u1EA3 c\u1EE7a thu\u1EADt to\xE1n (kh\xE1 ng\u01B0\u1EE3c l\u1EA1i), nh\u01B0ng n\xF3 s\u1EBD gi\xFAp b\u1EA1n hi\u1EC3u h\u01A1n v\u1EC1 Unigram."),pc.forEach(n),Va=r(s),kn=c(s,"P",{});var oc=h(kn);jp=p(oc,"Ta s\u1EBD s\u1EED d\xF9ng c\xF9ng b\u1ED9 ng\u1EEF li\u1EC7u \u0111\xE3 s\u1EED d\u1EE5ng nh\u01B0 m\u1ED9t v\xED d\u1EE5:"),oc.forEach(n),Na=r(s),g(Vs.$$.fragment,s),Da=r(s),os=c(s,"P",{});var xe=h(os);yp=p(xe,"L\u1EA7n n\xE0y, ta s\u1EBD s\u1EED d\u1EE5ng "),dt=c(xe,"CODE",{});var cc=h(dt);$p=p(cc,"xlnet-base-cased"),cc.forEach(n),qp=p(xe," nh\u01B0 m\xF4 h\xECnh:"),xe.forEach(n),Sa=r(s),g(Ns.$$.fragment,s),Oa=r(s),vn=c(s,"P",{});var hc=h(vn);wp=p(hc,"T\u01B0\u01A1ng t\u1EF1 BPE v\xE0 WordPiece, ta s\u1EBD b\u1EAFt \u0111\u1EA7u \u0111\u1EBFm t\u1EA7n su\u1EA5t xu\u1EA5t hi\u1EC7n c\u1EE7a m\u1ED7i t\u1EEB trong kho ng\u1EEF li\u1EC7u:"),hc.forEach(n),Ua=r(s),g(Ds.$$.fragment,s),Aa=r(s),bn=c(s,"P",{});var ic=h(bn);xp=p(ic,"Sau \u0111\xF3, ta c\u1EA7n kh\u1EDFi t\u1EA1o b\u1ED9 t\u1EEB v\u1EF1ng v\u1EDBi s\u1ED1 l\u01B0\u1EE3ng l\u1EDBn h\u01A1n k\xEDch th\u01B0\u1EDBc ta mu\u1ED1n cu\u1ED1i c\xF9ng. Ta ph\u1EA3i t\u1ED5ng k\u1EBFt t\u1EA5t c\u1EA3 c\xE1c k\xED t\u1EF1 c\u01A1 b\u1EA3n (n\u1EBFu kh\xF4ng ta s\u1EBD kh\xF4ng th\u1EC3 tokenize t\u1EA5t c\u1EA3 c\xE1c t\u1EEB), nh\u01B0ng v\u1EDBi c\xE1c chu\u1ED7i con l\u1EDBn h\u01A1n ta s\u1EBD gi\u1EEF ph\u1EA7n th\xF4ng d\u1EE5ng nh\u1EA5t v\xE0 s\u1EAFp x\u1EBFp ch\xFAng theo t\u1EA7n su\u1EA5t:"),ic.forEach(n),Ba=r(s),g(Ss.$$.fragment,s),Ma=r(s),g(Os.$$.fragment,s),Ha=r(s),_n=c(s,"P",{});var rc=h(_n);Ep=p(rc,"Ta nh\xF3m c\xE1c k\xED t\u1EF1 c\xF3 c\xE1c t\u1EEB con t\u1ED1t nh\u1EA5t v\xE0o b\u1ED9 t\u1EEB v\u1EF1ng ban \u0111\u1EA7u c\xF3 k\xEDch th\u01B0\u1EDBc l\xE0 300:"),rc.forEach(n),Ga=r(s),g(Us.$$.fragment,s),La=r(s),g(cs.$$.fragment,s),Xa=r(s),jn=c(s,"P",{});var mc=h(jn);Pp=p(mc,"Ti\u1EBFp theo, ch\xFAng ta t\xEDnh t\u1ED5ng t\u1EA7n su\u1EA5t \u0111\u1EC3 bi\u1EBFn \u0111\u1ED5i c\xE1c t\u1EA7n su\u1EA5t n\xE0y th\xE0nh x\xE1c su\u1EA5t. V\u1EDBi m\xF4 h\xECnh, ch\xFAng ta s\u1EBD l\u01B0u c\xE1c log c\u1EE7a x\xE1c xu\u1EA5t, v\xEC n\xF3 \u1ED5n \u0111\u1ECBnh h\u01A1n v\u1EC1 m\u1EB7t s\u1ED1 h\u1ECDc khi c\u1ED9ng logarit h\u01A1n l\xE0 nh\xE2n c\xE1c s\u1ED1 nh\u1ECF v\xE0 \u0111i\u1EC1u n\xE0y s\u1EBD \u0111\u01A1n gi\u1EA3n h\xF3a vi\u1EC7c t\xEDnh to\xE1n m\u1EA5t m\xE1t c\u1EE7a m\xF4 h\xECnh:"),mc.forEach(n),Fa=r(s),g(As.$$.fragment,s),Ra=r(s),hs=c(s,"P",{});var Ee=h(hs);zp=p(Ee,"Gi\u1EDD th\xEC c\xE1c h\xE0m ch\xEDnh l\xE0 h\xE0m tokenize t\u1EEB s\u1EED d\u1EE5ng thu\u1EADt to\xE1n Viterbi. Nh\u01B0 \u0111\xE3 th\u1EA5y tr\u01B0\u1EDBc \u0111\xF3, thu\u1EADt to\xE1n t\xEDnh ph\xE2n \u0111o\u1EA1n t\u1ED1t nh\u1EA5t c\u1EE7a m\u1ED7i chu\u1ED7i con c\u1EE7a t\u1EEB, \u0111\u01B0\u1EE3c l\u01B0u d\u01B0\u1EDBi bi\u1EBFn  "),kt=c(Ee,"CODE",{});var uc=h(kt);Tp=p(uc,"best_segmentations"),uc.forEach(n),Cp=p(Ee,". Ch\xFAng ta s\u1EBD l\u01B0u m\u1ED7i v\u1ECB tr\xED m\u1ED9t t\u1EEB \u0111i\u1EC3n trong t\u1EEB (t\u1EEB 0 cho t\u1EDBi \u0111\u1ED9 d\xE0i c\u1EE7a n\xF3), v\u1EDBi hai kho\xE1: ch\u1EC9 m\u1EE5c \u0111i\u1EC3m b\u1EAFt \u0111\u1EA7u c\u1EE7a token cu\u1ED1i trong ph\u1EA7n \u0111o\u1EA1n t\u1ED1t nh\u1EA5t, v\xE0 \u0111i\u1EC3m c\u1EE7a ph\xE2n \u0111o\u1EA1n t\u1ED1t nh\u1EA5t. V\u1EDBi ch\u1EC9 m\u1EE5c c\u1EE7a \u0111i\u1EC3m b\u1EAFt \u0111\u1EA7u c\u1EE7a token cu\u1ED1i trong ph\u1EA7n \u0111o\u1EA1n t\u1ED1t nh\u1EA5t, ta s\u1EBD c\xF3 th\u1EC3 truy v\u1EA5n to\xE0n b\u1ED9 ph\xE2n \u0111o\u1EA1n m\u1ED9t khi danh s\xE1ch \u0111\u01B0\u1EE3c \u0111i\u1EC1n \u0111\u1EE7."),Ee.forEach(n),Wa=r(s),is=c(s,"P",{});var Pe=h(is);Vp=p(Pe,"Vi\u1EC7c \u0111i\u1EC1n danh s\xE1ch \u0111\u01B0\u1EE3c th\u1EF1c hi\u1EC7n ch\u1EC9 v\u1EDBi hai v\xF2ng l\u1EB7p: v\xF2ng l\u1EB7p ch\xEDnh \u0111i qua t\u1EEBng v\u1ECB tr\xED b\u1EAFt \u0111\u1EA7u v\xE0 v\xF2ng l\u1EB7p th\u1EE9 hai th\u1EED t\u1EA5t c\u1EA3 c\xE1c chu\u1ED7i con b\u1EAFt \u0111\u1EA7u t\u1EEB v\u1ECB tr\xED b\u1EAFt \u0111\u1EA7u \u0111\xF3. N\u1EBFu chu\u1ED7i con c\xF3 trong t\u1EEB v\u1EF1ng, ch\xFAng ta c\xF3 m\u1ED9t ph\xE2n \u0111o\u1EA1n m\u1EDBi c\u1EE7a t\u1EEB cho \u0111\u1EBFn v\u1ECB tr\xED k\u1EBFt th\xFAc \u0111\xF3, v\xE0 so s\xE1nh v\u1EDBi nh\u1EEFng g\xEC c\xF3 trong "),vt=c(Pe,"CODE",{});var gc=h(vt);Np=p(gc,"best_segmentations"),gc.forEach(n),Dp=p(Pe,"."),Pe.forEach(n),Qa=r(s),yn=c(s,"P",{});var fc=h(yn);Sp=p(fc,"Khi v\xF2ng l\u1EB7p ch\xEDnh k\u1EBFt th\xFAc, ch\xFAng ta ch\u1EC9 b\u1EAFt \u0111\u1EA7u t\u1EEB cu\u1ED1i v\xE0 nh\u1EA3y t\u1EEB v\u1ECB tr\xED b\u1EAFt \u0111\u1EA7u n\xE0y sang v\u1ECB tr\xED ti\u1EBFp theo, ghi l\u1EA1i c\xE1c token khi ch\xFAng ta \u0111i, cho \u0111\u1EBFn khi ch\xFAng ta \u0111\u1EBFn v\u1ECB tr\xED \u0111\u1EA7u c\u1EE7a t\u1EEB:"),fc.forEach(n),Ka=r(s),g(Bs.$$.fragment,s),Ya=r(s),$n=c(s,"P",{});var dc=h($n);Op=p(dc,"Ta c\xF3 th\u1EC3 s\u1EB5n s\xE0ng th\u1EED m\xF4 h\xECnh ban \u0111\u1EA7u l\xEAn m\u1ED9t s\u1ED1 t\u1EEB:"),dc.forEach(n),Ia=r(s),g(Ms.$$.fragment,s),Za=r(s),g(Hs.$$.fragment,s),Ja=r(s),qn=c(s,"P",{});var kc=h(qn);Up=p(kc,"Gi\u1EDD th\xEC th\u1EADt d\u1EC5 d\xE0ng \u0111\u1EC3 t\xEDnh s\u1EF1 m\u1EA5t m\xE1t c\u1EE7a m\xF4 h\xECnh tr\xEAn kho ng\u1EEF li\u1EC7u!"),kc.forEach(n),se=r(s),g(Gs.$$.fragment,s),ne=r(s),wn=c(s,"P",{});var vc=h(wn);Ap=p(vc,"Ta c\xF3 th\u1EC3 ki\u1EC3m tra c\xE1ch n\xF3 ho\u1EA1t \u0111\u1ED9ng tr\xEAn m\xF4 h\xECnh ta c\xF3:"),vc.forEach(n),te=r(s),g(Ls.$$.fragment,s),ae=r(s),g(Xs.$$.fragment,s),ee=r(s),xn=c(s,"P",{});var bc=h(xn);Bp=p(bc,"Vi\u1EC7c t\xEDnh \u0111i\u1EC3m cho m\u1ED7i token kh\xF4ng qu\u1EA3 kh\xF3; ta ch\u1EC9 ph\u1EA3i t\xEDnh s\u1EF1 m\u1EA5t m\xE1t c\u1EE7a m\xF4 h\xECnh khi xo\xE1 m\u1ED7i token:"),bc.forEach(n),le=r(s),g(Fs.$$.fragment,s),pe=r(s),En=c(s,"P",{});var _c=h(En);Mp=p(_c,"Ta c\xF3 th\u1EC3 th\u1EED v\u1EDBi token cho tr\u01B0\u1EDBc:"),_c.forEach(n),oe=r(s),g(Rs.$$.fragment,s),ce=r(s),x=c(s,"P",{});var U=h(x);Hp=p(U,"V\xEC "),bt=c(U,"CODE",{});var jc=h(bt);Gp=p(jc,'"ll"'),jc.forEach(n),Lp=p(U," \u0111\u01B0\u1EE3c s\u1EED d\u1EE5ng trong qu\xE1 tr\xECnh tokenize  "),_t=c(U,"CODE",{});var yc=h(_t);Xp=p(yc,'"Hopefully"'),yc.forEach(n),Fp=p(U,", v\xE0 lo\u1EA1i b\u1ECF n\xF3 ch\u1EAFc ch\u1EAFn s\u1EBD l\xE0m ta thay v\xE0o \u0111\xF3 s\u1EED d\u1EE5ng "),jt=c(U,"CODE",{});var $c=h(jt);Rp=p($c,'"l"'),$c.forEach(n),Wp=p(U," hai l\u1EA7n, ta k\xEC v\u1ECDng n\xF3 s\u1EBD \u0111em l\u1EA1i s\u1EF1 m\u1EA5t m\xE1t d\u01B0\u01A1ng. "),yt=c(U,"CODE",{});var qc=h(yt);Qp=p(qc,'"his"'),qc.forEach(n),Kp=p(U," ch\u1EC9 \u0111\u01B0\u1EE3c s\u1EED d\u1EE5ng trong t\u1EEB "),$t=c(U,"CODE",{});var wc=h($t);Yp=p(wc,'"This"'),wc.forEach(n),Ip=p(U,", n\xF3 \u0111\u01B0\u1EE3c tokenize th\xE0nh ch\xEDnh n\xF3, n\xEAn ta k\xEC v\u1ECDng n\xF3 s\u1EBD kh\xF4ng c\xF3 m\u1EA5t m\xE1t. V\xE0 \u0111\xE2y l\xE0 k\u1EBFt qu\u1EA3:"),U.forEach(n),he=r(s),g(Ws.$$.fragment,s),ie=r(s),g(rs.$$.fragment,s),re=r(s),Pn=c(s,"P",{});var xc=h(Pn);Zp=p(xc,"V\u1EDBi t\u1EA5t c\u1EA3 nh\u1EEFng \u0111i\u1EC1u tr\xEAn, \u0111i\u1EC1u cu\u1ED1i c\xF9ng ta c\u1EA7n ph\u1EA3i l\xE0m l\xE0 th\xEAm c\xE1c token \u0111\u1EB7c bi\u1EC7t c\u1EE7a m\xF4 h\xECnh v\xE0o b\u1ED9 t\u1EEB v\u1EF1ng, sau \u0111\xF3 l\u1EB7p cho \u0111\u1EBFn khi ch\xFAng ta c\u1EAFt \u0111\u1EE7 s\u1ED1 token ta mong mu\u1ED1n cho k\xEDch c\u1EE1 b\u1ED9 t\u1EEB v\u1EF1ng:"),xc.forEach(n),me=r(s),g(Qs.$$.fragment,s),ue=r(s),ms=c(s,"P",{});var ze=h(ms);Jp=p(ze,"Sau \u0111\xF3, \u0111\u1EC3 tokenize c\xE1c \u0111o\u1EA1n v\u0103n b\u1EA3n, ta ch\u1EC9 c\u1EA7n \xE1p d\u1EE5ng pre-tokenization v\xE0 sau \u0111\u1ECF s\u1EED d\u1EE5ng h\xE0m "),qt=c(ze,"CODE",{});var Ec=h(qt);so=p(Ec,"encode_word()"),Ec.forEach(n),no=p(ze,":"),ze.forEach(n),ge=r(s),g(Ks.$$.fragment,s),fe=r(s),g(Ys.$$.fragment,s),de=r(s),zn=c(s,"P",{});var Pc=h(zn);to=p(Pc,"V\xE0 \u0111\xF3 l\xE0 Unigram! Hy v\u1ECDng r\u1EB1ng b\xE2y gi\u1EDD b\u1EA1n c\u1EA3m th\u1EA5y nh\u01B0 m\u1ED9t chuy\xEAn gia trong t\u1EA5t c\u1EA3 m\u1ECDi tokenizer. Trong ph\u1EA7n ti\u1EBFp theo, ch\xFAng ta s\u1EBD \u0111i s\xE2u v\xE0o c\xE1c kh\u1ED1i c\u1EE7a th\u01B0 vi\u1EC7n \u{1F917} Tokenizers v\xE0 ch\u1EC9 cho b\u1EA1n c\xE1ch b\u1EA1n c\xF3 th\u1EC3 s\u1EED d\u1EE5ng ch\xFAng \u0111\u1EC3 t\u1EA1o tokenizer c\u1EE7a ri\xEAng m\xECnh."),Pc.forEach(n),this.h()},h(){$(m,"name","hf:doc:metadata"),$(m,"content",JSON.stringify(Rc)),$(_,"id","unigram-tokenization"),$(_,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),$(_,"href","#unigram-tokenization"),$(b,"class","relative group"),$(I,"id","thut-ton-hun-luyn"),$(I,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),$(I,"href","#thut-ton-hun-luyn"),$(F,"class","relative group"),At.a=Bt,Mt.a=Ht,$(Z,"id","thut-ton-tokenize"),$(Z,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),$(Z,"href","#thut-ton-tokenize"),$(R,"class","relative group"),ta.a=null,ea.a=null,$(es,"id","quay-li-hun-luyn"),$(es,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),$(es,"href","#quay-li-hun-luyn"),$(Q,"class","relative group"),$(ps,"id","trin-khai-unigram"),$(ps,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),$(ps,"href","#trin-khai-unigram"),$(K,"class","relative group")},m(s,a){t(document.head,m),e(s,y,a),e(s,b,a),t(b,_),t(_,z),f(q,z,null),t(b,E),t(b,V),t(V,X),e(s,T,a),f(B,s,a),e(s,ds,a),e(s,Js,a),t(Js,Ve),e(s,Vt,a),f(ks,s,a),e(s,Nt,a),f(Y,s,a),e(s,Dt,a),e(s,F,a),t(F,I),t(I,Nn),f(vs,Nn,null),t(F,Ne),t(F,Dn),t(Dn,De),e(s,St,a),e(s,sn,a),t(sn,Se),e(s,Ot,a),e(s,nn,a),t(nn,Oe),e(s,Ut,a),e(s,M,a),t(M,Ue),At.m(zc,M),t(M,Bt),Mt.m(Tc,M),t(M,Ht),e(s,Gt,a),e(s,tn,a),t(tn,Ae),e(s,Lt,a),e(s,an,a),t(an,Be),e(s,Xt,a),e(s,en,a),t(en,Me),e(s,Ft,a),f(bs,s,a),e(s,Rt,a),e(s,ln,a),t(ln,He),e(s,Wt,a),f(_s,s,a),e(s,Qt,a),e(s,R,a),t(R,Z),t(Z,Sn),f(js,Sn,null),t(R,Ge),t(R,On),t(On,Le),e(s,Kt,a),e(s,pn,a),t(pn,Xe),e(s,Yt,a),e(s,P,a),t(P,Fe),t(P,Un),t(Un,Re),t(P,We),t(P,An),t(An,Qe),t(P,Ke),t(P,Bn),t(Bn,Ye),t(P,Ie),t(P,Mn),t(Mn,Ze),t(P,Je),e(s,It,a),e(s,on,a),t(on,sl),e(s,Zt,a),f(ys,s,a),e(s,Jt,a),e(s,J,a),t(J,nl),t(J,Hn),t(Hn,tl),t(J,al),e(s,sa,a),f(ss,s,a),e(s,na,a),e(s,A,a),t(A,el),t(A,Gn),t(Gn,ll),t(A,pl),t(A,Ln),t(Ln,ol),t(A,cl),ta.m(Cc,A),e(s,aa,a),e(s,W,a),t(W,hl),t(W,Xn),t(Xn,il),t(W,rl),ea.m(Vc,W),e(s,la,a),e(s,cn,a),t(cn,ml),e(s,pa,a),e(s,ns,a),t(ns,ul),t(ns,Fn),t(Fn,gl),t(ns,fl),e(s,oa,a),f($s,s,a),e(s,ca,a),e(s,N,a),t(N,dl),t(N,Rn),t(Rn,kl),t(N,vl),t(N,Wn),t(Wn,bl),t(N,_l),t(N,Qn),t(Qn,jl),t(N,yl),e(s,ha,a),e(s,w,a),t(w,$l),t(w,Kn),t(Kn,ql),t(w,wl),t(w,Yn),t(Yn,xl),t(w,El),t(w,In),t(In,Pl),t(w,zl),t(w,Zn),t(Zn,Tl),t(w,Cl),t(w,Jn),t(Jn,Vl),t(w,Nl),e(s,ia,a),e(s,hn,a),t(hn,Dl),e(s,ra,a),e(s,ts,a),t(ts,Sl),t(ts,st),t(st,Ol),t(ts,Ul),e(s,ma,a),f(qs,s,a),e(s,ua,a),e(s,H,a),t(H,Al),t(H,nt),t(nt,Bl),t(H,Ml),t(H,tt),t(tt,Hl),t(H,Gl),e(s,ga,a),f(as,s,a),e(s,fa,a),e(s,Q,a),t(Q,es),t(es,at),f(ws,at,null),t(Q,Ll),t(Q,et),t(et,Xl),e(s,da,a),e(s,rn,a),t(rn,Fl),e(s,ka,a),e(s,ls,a),t(ls,Rl),t(ls,lt),t(lt,Wl),t(ls,Ql),e(s,va,a),e(s,mn,a),t(mn,Kl),e(s,ba,a),f(xs,s,a),e(s,_a,a),e(s,un,a),t(un,Yl),e(s,ja,a),f(Es,s,a),e(s,ya,a),e(s,gn,a),t(gn,Il),e(s,$a,a),f(Ps,s,a),e(s,qa,a),e(s,D,a),t(D,Zl),t(D,pt),t(pt,Jl),t(D,sp),t(D,ot),t(ot,np),t(D,tp),t(D,ct),t(ct,ap),t(D,ep),e(s,wa,a),e(s,S,a),t(S,lp),t(S,ht),t(ht,pp),t(S,op),t(S,it),t(it,cp),t(S,hp),t(S,rt),t(rt,ip),t(S,rp),e(s,xa,a),f(zs,s,a),e(s,Ea,a),e(s,fn,a),t(fn,mp),e(s,Pa,a),f(Ts,s,a),e(s,za,a),e(s,G,a),t(G,up),t(G,mt),t(mt,gp),t(G,fp),t(G,ut),t(ut,dp),t(G,kp),e(s,Ta,a),e(s,K,a),t(K,ps),t(ps,gt),f(Cs,gt,null),t(K,vp),t(K,ft),t(ft,bp),e(s,Ca,a),e(s,dn,a),t(dn,_p),e(s,Va,a),e(s,kn,a),t(kn,jp),e(s,Na,a),f(Vs,s,a),e(s,Da,a),e(s,os,a),t(os,yp),t(os,dt),t(dt,$p),t(os,qp),e(s,Sa,a),f(Ns,s,a),e(s,Oa,a),e(s,vn,a),t(vn,wp),e(s,Ua,a),f(Ds,s,a),e(s,Aa,a),e(s,bn,a),t(bn,xp),e(s,Ba,a),f(Ss,s,a),e(s,Ma,a),f(Os,s,a),e(s,Ha,a),e(s,_n,a),t(_n,Ep),e(s,Ga,a),f(Us,s,a),e(s,La,a),f(cs,s,a),e(s,Xa,a),e(s,jn,a),t(jn,Pp),e(s,Fa,a),f(As,s,a),e(s,Ra,a),e(s,hs,a),t(hs,zp),t(hs,kt),t(kt,Tp),t(hs,Cp),e(s,Wa,a),e(s,is,a),t(is,Vp),t(is,vt),t(vt,Np),t(is,Dp),e(s,Qa,a),e(s,yn,a),t(yn,Sp),e(s,Ka,a),f(Bs,s,a),e(s,Ya,a),e(s,$n,a),t($n,Op),e(s,Ia,a),f(Ms,s,a),e(s,Za,a),f(Hs,s,a),e(s,Ja,a),e(s,qn,a),t(qn,Up),e(s,se,a),f(Gs,s,a),e(s,ne,a),e(s,wn,a),t(wn,Ap),e(s,te,a),f(Ls,s,a),e(s,ae,a),f(Xs,s,a),e(s,ee,a),e(s,xn,a),t(xn,Bp),e(s,le,a),f(Fs,s,a),e(s,pe,a),e(s,En,a),t(En,Mp),e(s,oe,a),f(Rs,s,a),e(s,ce,a),e(s,x,a),t(x,Hp),t(x,bt),t(bt,Gp),t(x,Lp),t(x,_t),t(_t,Xp),t(x,Fp),t(x,jt),t(jt,Rp),t(x,Wp),t(x,yt),t(yt,Qp),t(x,Kp),t(x,$t),t($t,Yp),t(x,Ip),e(s,he,a),f(Ws,s,a),e(s,ie,a),f(rs,s,a),e(s,re,a),e(s,Pn,a),t(Pn,Zp),e(s,me,a),f(Qs,s,a),e(s,ue,a),e(s,ms,a),t(ms,Jp),t(ms,qt),t(qt,so),t(ms,no),e(s,ge,a),f(Ks,s,a),e(s,fe,a),f(Ys,s,a),e(s,de,a),e(s,zn,a),t(zn,to),ke=!0},p(s,[a]){const Is={};a&2&&(Is.$$scope={dirty:a,ctx:s}),Y.$set(Is);const wt={};a&2&&(wt.$$scope={dirty:a,ctx:s}),ss.$set(wt);const xt={};a&2&&(xt.$$scope={dirty:a,ctx:s}),as.$set(xt);const Et={};a&2&&(Et.$$scope={dirty:a,ctx:s}),cs.$set(Et);const Pt={};a&2&&(Pt.$$scope={dirty:a,ctx:s}),rs.$set(Pt)},i(s){ke||(d(q.$$.fragment,s),d(B.$$.fragment,s),d(ks.$$.fragment,s),d(Y.$$.fragment,s),d(vs.$$.fragment,s),d(bs.$$.fragment,s),d(_s.$$.fragment,s),d(js.$$.fragment,s),d(ys.$$.fragment,s),d(ss.$$.fragment,s),d($s.$$.fragment,s),d(qs.$$.fragment,s),d(as.$$.fragment,s),d(ws.$$.fragment,s),d(xs.$$.fragment,s),d(Es.$$.fragment,s),d(Ps.$$.fragment,s),d(zs.$$.fragment,s),d(Ts.$$.fragment,s),d(Cs.$$.fragment,s),d(Vs.$$.fragment,s),d(Ns.$$.fragment,s),d(Ds.$$.fragment,s),d(Ss.$$.fragment,s),d(Os.$$.fragment,s),d(Us.$$.fragment,s),d(cs.$$.fragment,s),d(As.$$.fragment,s),d(Bs.$$.fragment,s),d(Ms.$$.fragment,s),d(Hs.$$.fragment,s),d(Gs.$$.fragment,s),d(Ls.$$.fragment,s),d(Xs.$$.fragment,s),d(Fs.$$.fragment,s),d(Rs.$$.fragment,s),d(Ws.$$.fragment,s),d(rs.$$.fragment,s),d(Qs.$$.fragment,s),d(Ks.$$.fragment,s),d(Ys.$$.fragment,s),ke=!0)},o(s){k(q.$$.fragment,s),k(B.$$.fragment,s),k(ks.$$.fragment,s),k(Y.$$.fragment,s),k(vs.$$.fragment,s),k(bs.$$.fragment,s),k(_s.$$.fragment,s),k(js.$$.fragment,s),k(ys.$$.fragment,s),k(ss.$$.fragment,s),k($s.$$.fragment,s),k(qs.$$.fragment,s),k(as.$$.fragment,s),k(ws.$$.fragment,s),k(xs.$$.fragment,s),k(Es.$$.fragment,s),k(Ps.$$.fragment,s),k(zs.$$.fragment,s),k(Ts.$$.fragment,s),k(Cs.$$.fragment,s),k(Vs.$$.fragment,s),k(Ns.$$.fragment,s),k(Ds.$$.fragment,s),k(Ss.$$.fragment,s),k(Os.$$.fragment,s),k(Us.$$.fragment,s),k(cs.$$.fragment,s),k(As.$$.fragment,s),k(Bs.$$.fragment,s),k(Ms.$$.fragment,s),k(Hs.$$.fragment,s),k(Gs.$$.fragment,s),k(Ls.$$.fragment,s),k(Xs.$$.fragment,s),k(Fs.$$.fragment,s),k(Rs.$$.fragment,s),k(Ws.$$.fragment,s),k(rs.$$.fragment,s),k(Qs.$$.fragment,s),k(Ks.$$.fragment,s),k(Ys.$$.fragment,s),ke=!1},d(s){n(m),s&&n(y),s&&n(b),v(q),s&&n(T),v(B,s),s&&n(ds),s&&n(Js),s&&n(Vt),v(ks,s),s&&n(Nt),v(Y,s),s&&n(Dt),s&&n(F),v(vs),s&&n(St),s&&n(sn),s&&n(Ot),s&&n(nn),s&&n(Ut),s&&n(M),s&&n(Gt),s&&n(tn),s&&n(Lt),s&&n(an),s&&n(Xt),s&&n(en),s&&n(Ft),v(bs,s),s&&n(Rt),s&&n(ln),s&&n(Wt),v(_s,s),s&&n(Qt),s&&n(R),v(js),s&&n(Kt),s&&n(pn),s&&n(Yt),s&&n(P),s&&n(It),s&&n(on),s&&n(Zt),v(ys,s),s&&n(Jt),s&&n(J),s&&n(sa),v(ss,s),s&&n(na),s&&n(A),s&&n(aa),s&&n(W),s&&n(la),s&&n(cn),s&&n(pa),s&&n(ns),s&&n(oa),v($s,s),s&&n(ca),s&&n(N),s&&n(ha),s&&n(w),s&&n(ia),s&&n(hn),s&&n(ra),s&&n(ts),s&&n(ma),v(qs,s),s&&n(ua),s&&n(H),s&&n(ga),v(as,s),s&&n(fa),s&&n(Q),v(ws),s&&n(da),s&&n(rn),s&&n(ka),s&&n(ls),s&&n(va),s&&n(mn),s&&n(ba),v(xs,s),s&&n(_a),s&&n(un),s&&n(ja),v(Es,s),s&&n(ya),s&&n(gn),s&&n($a),v(Ps,s),s&&n(qa),s&&n(D),s&&n(wa),s&&n(S),s&&n(xa),v(zs,s),s&&n(Ea),s&&n(fn),s&&n(Pa),v(Ts,s),s&&n(za),s&&n(G),s&&n(Ta),s&&n(K),v(Cs),s&&n(Ca),s&&n(dn),s&&n(Va),s&&n(kn),s&&n(Na),v(Vs,s),s&&n(Da),s&&n(os),s&&n(Sa),v(Ns,s),s&&n(Oa),s&&n(vn),s&&n(Ua),v(Ds,s),s&&n(Aa),s&&n(bn),s&&n(Ba),v(Ss,s),s&&n(Ma),v(Os,s),s&&n(Ha),s&&n(_n),s&&n(Ga),v(Us,s),s&&n(La),v(cs,s),s&&n(Xa),s&&n(jn),s&&n(Fa),v(As,s),s&&n(Ra),s&&n(hs),s&&n(Wa),s&&n(is),s&&n(Qa),s&&n(yn),s&&n(Ka),v(Bs,s),s&&n(Ya),s&&n($n),s&&n(Ia),v(Ms,s),s&&n(Za),v(Hs,s),s&&n(Ja),s&&n(qn),s&&n(se),v(Gs,s),s&&n(ne),s&&n(wn),s&&n(te),v(Ls,s),s&&n(ae),v(Xs,s),s&&n(ee),s&&n(xn),s&&n(le),v(Fs,s),s&&n(pe),s&&n(En),s&&n(oe),v(Rs,s),s&&n(ce),s&&n(x),s&&n(he),v(Ws,s),s&&n(ie),v(rs,s),s&&n(re),s&&n(Pn),s&&n(me),v(Qs,s),s&&n(ue),s&&n(ms),s&&n(ge),v(Ks,s),s&&n(fe),v(Ys,s),s&&n(de),s&&n(zn)}}}const Rc={local:"unigram-tokenization",sections:[{local:"thut-ton-hun-luyn",title:"Thu\u1EADt to\xE1n hu\u1EA5n luy\u1EC7n"},{local:"thut-ton-tokenize",title:"Thu\u1EADt to\xE1n tokenize"},{local:"quay-li-hun-luyn",title:"Quay l\u1EA1i hu\u1EA5n luy\u1EC7n"},{local:"trin-khai-unigram",title:"Tri\u1EC3n khai Unigram"}],title:"Unigram tokenization"};function Wc(C){return Uc(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class sh extends Nc{constructor(m){super();Dc(this,m,Wc,Fc,Sc,{})}}export{sh as default,Rc as metadata};
