import{S as pl,i as gl,s as ul,e as l,k as o,w as g,t as e,M as ml,c,d as t,m as p,a as r,x as u,h as a,b as _,G as s,g as i,y as m,q as f,o as k,B as d,v as fl}from"../../chunks/vendor-hf-doc-builder.js";import{T as kl}from"../../chunks/Tip-hf-doc-builder.js";import{Y as dl}from"../../chunks/Youtube-hf-doc-builder.js";import{I as $h}from"../../chunks/IconCopyLink-hf-doc-builder.js";import{C as b}from"../../chunks/CodeBlock-hf-doc-builder.js";import{C as _l}from"../../chunks/CourseFloatingBanner-hf-doc-builder.js";import"../../chunks/DocNotebookDropdown-hf-doc-builder.js";function xl(ts){let y,B;return{c(){y=l("p"),B=e("\u26A0\uFE0F Hu\u1EA5n luy\u1EC7n m\u1ED9t tokenizer kh\xF4ng gi\u1ED1ng nh\u01B0 hu\u1EA5n luy\u1EC7n m\u1ED9t m\xF4 h\xECnh! Hu\u1EA5n luy\u1EC7n m\xF4 h\xECnh s\u1EED d\u1EE5ng gi\u1EA3m \u0111\u1ED9 d\u1ED1c ng\u1EABu nhi\xEAn \u0111\u1EC3 l\xE0m cho t\u1ED5n th\u1EA5t nh\u1ECF h\u01A1n m\u1ED9t ch\xFAt cho m\u1ED7i \u0111\u1EE3t. N\xF3 \u0111\u01B0\u1EE3c ng\u1EABu nhi\xEAn h\xF3a b\u1EDFi t\u1EF1 nhi\xEAn (c\xF3 ngh\u0129a l\xE0 b\u1EA1n ph\u1EA3i \u0111\u1EB7t m\u1ED9t gi\xE1 tr\u1ECB seed \u0111\u1EC3 c\xF3 \u0111\u01B0\u1EE3c k\u1EBFt qu\u1EA3 t\u01B0\u01A1ng t\u1EF1 khi th\u1EF1c hi\u1EC7n c\xF9ng th\u1EF1c hi\u1EC7n hu\u1EA5n luy\u1EC7n hai l\u1EA7n). Hu\u1EA5n luy\u1EC7n m\u1ED9t tr\xECnh tokenize l\xE0 m\u1ED9t quy tr\xECnh th\u1ED1ng k\xEA c\u1ED1 g\u1EAFng x\xE1c \u0111\u1ECBnh nh\u1EEFng t\u1EEB ph\u1EE5 n\xE0o t\u1ED1t nh\u1EA5t \u0111\u1EC3 ch\u1ECDn cho m\u1ED9t kho d\u1EEF li\u1EC7u nh\u1EA5t \u0111\u1ECBnh, v\xE0 c\xE1c quy t\u1EAFc \u0111\u01B0\u1EE3c s\u1EED d\u1EE5ng \u0111\u1EC3 ch\u1ECDn ch\xFAng d\u1EF1a tr\xEAn thu\u1EADt to\xE1n tokenize. N\xF3 mang t\xEDnh c\u1ED1 \u0111\u1ECBnh, ngh\u0129a l\xE0 b\u1EA1n lu\xF4n nh\u1EADn \u0111\u01B0\u1EE3c c\xF9ng m\u1ED9t k\u1EBFt qu\u1EA3 khi hu\u1EA5n luy\u1EC7n v\u1EDBi c\xF9ng m\u1ED9t thu\u1EADt to\xE1n tr\xEAn c\xF9ng m\u1ED9t kho t\xE0i li\u1EC7u.")},l(j){y=c(j,"P",{});var w=r(y);B=a(w,"\u26A0\uFE0F Hu\u1EA5n luy\u1EC7n m\u1ED9t tokenizer kh\xF4ng gi\u1ED1ng nh\u01B0 hu\u1EA5n luy\u1EC7n m\u1ED9t m\xF4 h\xECnh! Hu\u1EA5n luy\u1EC7n m\xF4 h\xECnh s\u1EED d\u1EE5ng gi\u1EA3m \u0111\u1ED9 d\u1ED1c ng\u1EABu nhi\xEAn \u0111\u1EC3 l\xE0m cho t\u1ED5n th\u1EA5t nh\u1ECF h\u01A1n m\u1ED9t ch\xFAt cho m\u1ED7i \u0111\u1EE3t. N\xF3 \u0111\u01B0\u1EE3c ng\u1EABu nhi\xEAn h\xF3a b\u1EDFi t\u1EF1 nhi\xEAn (c\xF3 ngh\u0129a l\xE0 b\u1EA1n ph\u1EA3i \u0111\u1EB7t m\u1ED9t gi\xE1 tr\u1ECB seed \u0111\u1EC3 c\xF3 \u0111\u01B0\u1EE3c k\u1EBFt qu\u1EA3 t\u01B0\u01A1ng t\u1EF1 khi th\u1EF1c hi\u1EC7n c\xF9ng th\u1EF1c hi\u1EC7n hu\u1EA5n luy\u1EC7n hai l\u1EA7n). Hu\u1EA5n luy\u1EC7n m\u1ED9t tr\xECnh tokenize l\xE0 m\u1ED9t quy tr\xECnh th\u1ED1ng k\xEA c\u1ED1 g\u1EAFng x\xE1c \u0111\u1ECBnh nh\u1EEFng t\u1EEB ph\u1EE5 n\xE0o t\u1ED1t nh\u1EA5t \u0111\u1EC3 ch\u1ECDn cho m\u1ED9t kho d\u1EEF li\u1EC7u nh\u1EA5t \u0111\u1ECBnh, v\xE0 c\xE1c quy t\u1EAFc \u0111\u01B0\u1EE3c s\u1EED d\u1EE5ng \u0111\u1EC3 ch\u1ECDn ch\xFAng d\u1EF1a tr\xEAn thu\u1EADt to\xE1n tokenize. N\xF3 mang t\xEDnh c\u1ED1 \u0111\u1ECBnh, ngh\u0129a l\xE0 b\u1EA1n lu\xF4n nh\u1EADn \u0111\u01B0\u1EE3c c\xF9ng m\u1ED9t k\u1EBFt qu\u1EA3 khi hu\u1EA5n luy\u1EC7n v\u1EDBi c\xF9ng m\u1ED9t thu\u1EADt to\xE1n tr\xEAn c\xF9ng m\u1ED9t kho t\xE0i li\u1EC7u."),w.forEach(t)},m(j,w){i(j,y,w),s(y,B)},d(j){j&&t(y)}}}function bl(ts){let y,B,j,w,ot,nn,jh,pt,zh,ss,tn,hs,E,wh,Gn,Eh,qh,gt,Ph,Ch,ut,Ah,Th,es,sn,as,R,is,S,U,mt,hn,Dh,ft,Nh,ls,K,Oh,kt,Lh,Hh,cs,z,Sh,en,Gh,Mh,dt,Bh,Rh,an,Uh,Kh,ln,Vh,Ih,rs,cn,os,Mn,Yh,ps,rn,gs,on,us,C,Fh,_t,Qh,Xh,xt,Jh,Zh,ms,pn,fs,Bn,Wh,ks,gn,ds,V,ne,bt,te,se,_s,Rn,he,xs,un,bs,Un,ee,vs,mn,ys,A,ae,vt,ie,le,yt,ce,re,$s,Kn,oe,js,fn,zs,Vn,pe,ws,kn,Es,In,ge,qs,dn,Ps,T,ue,$t,me,fe,jt,ke,de,Cs,_n,As,Yn,_e,Ts,G,I,zt,xn,xe,wt,be,Ds,Fn,ve,Ns,bn,Os,Qn,ye,Ls,Xn,$e,Hs,vn,Ss,yn,Gs,q,je,Et,ze,we,qt,Ee,qe,Pt,Pe,Ce,Ms,Y,Ae,Ct,Te,De,Bs,$n,Rs,Jn,Ne,Us,D,Oe,At,Le,He,jn,Se,Ge,Ks,F,Me,Zn,Be,Re,Vs,N,Ue,zn,Ke,Ve,Tt,Ie,Ye,Is,wn,Ys,En,Fs,$,Fe,Dt,Qe,Xe,Nt,Je,Ze,Ot,We,na,Lt,ta,sa,Ht,ha,ea,Qs,qn,Xs,Pn,Js,Wn,aa,Zs,Cn,Ws,An,nh,x,ia,St,la,ca,Gt,ra,oa,Mt,pa,ga,Bt,ua,ma,Rt,fa,ka,Ut,da,_a,Kt,xa,ba,Vt,va,ya,It,$a,ja,Yt,za,wa,th,M,Q,Ft,Tn,Ea,Qt,qa,sh,X,Pa,Xt,Ca,Aa,hh,Dn,eh,J,Ta,Jt,Da,Na,ah,Nn,ih,nt,Oa,lh,On,ch,tt,La,rh,Ln,oh,O,Ha,Zt,Sa,Ga,Wt,Ma,Ba,ph,Hn,gh,L,Ra,st,Ua,Ka,ns,Va,Ia,uh;return nn=new $h({}),tn=new _l({props:{chapter:6,classNames:"absolute z-10 right-0 top-0",notebooks:[{label:"Google Colab",value:"https://colab.research.google.com/github/huggingface/notebooks/blob/master/course/vi/chapter6/section2.ipynb"},{label:"Aws Studio",value:"https://studiolab.sagemaker.aws/import/github/huggingface/notebooks/blob/master/course/vi/chapter6/section2.ipynb"}]}}),sn=new dl({props:{id:"DJimQynXZsQ"}}),R=new kl({props:{warning:!0,$$slots:{default:[xl]},$$scope:{ctx:ts}}}),hn=new $h({}),cn=new b({props:{code:`from datasets import load_dataset

# Qu\xE1 tr\xECnh n\xE0y c\xF3 th\u1EC3 m\u1EA5t m\u1ED9t v\xE0i ph\xFAt \u0111\u1EC3 t\u1EA3i, v\xEC v\u1EADy h\xE3y l\u1EA5y c\xE0 ph\xEA ho\u1EB7c tr\xE0 trong khi ch\u1EDD \u0111\u1EE3i!
raw_datasets = load_dataset("code_search_net", "python")`,highlighted:`<span class="hljs-keyword">from</span> datasets <span class="hljs-keyword">import</span> load_dataset

<span class="hljs-comment"># Qu\xE1 tr\xECnh n\xE0y c\xF3 th\u1EC3 m\u1EA5t m\u1ED9t v\xE0i ph\xFAt \u0111\u1EC3 t\u1EA3i, v\xEC v\u1EADy h\xE3y l\u1EA5y c\xE0 ph\xEA ho\u1EB7c tr\xE0 trong khi ch\u1EDD \u0111\u1EE3i!</span>
raw_datasets = load_dataset(<span class="hljs-string">&quot;code_search_net&quot;</span>, <span class="hljs-string">&quot;python&quot;</span>)`}}),rn=new b({props:{code:'raw_datasets["train"]',highlighted:'raw_datasets[<span class="hljs-string">&quot;train&quot;</span>]'}}),on=new b({props:{code:`Dataset({
    features: ['repository_name', 'func_path_in_repository', 'func_name', 'whole_func_string', 'language', 
      'func_code_string', 'func_code_tokens', 'func_documentation_string', 'func_documentation_tokens', 'split_name', 
      'func_code_url'
    ],
    num_rows: 412178
})`,highlighted:`Dataset({
    features: [<span class="hljs-string">&#x27;repository_name&#x27;</span>, <span class="hljs-string">&#x27;func_path_in_repository&#x27;</span>, <span class="hljs-string">&#x27;func_name&#x27;</span>, <span class="hljs-string">&#x27;whole_func_string&#x27;</span>, <span class="hljs-string">&#x27;language&#x27;</span>, 
      <span class="hljs-string">&#x27;func_code_string&#x27;</span>, <span class="hljs-string">&#x27;func_code_tokens&#x27;</span>, <span class="hljs-string">&#x27;func_documentation_string&#x27;</span>, <span class="hljs-string">&#x27;func_documentation_tokens&#x27;</span>, <span class="hljs-string">&#x27;split_name&#x27;</span>, 
      <span class="hljs-string">&#x27;func_code_url&#x27;</span>
    ],
    num_rows: <span class="hljs-number">412178</span>
})`}}),pn=new b({props:{code:'print(raw_datasets["train"][123456]["whole_func_string"])',highlighted:'<span class="hljs-built_in">print</span>(raw_datasets[<span class="hljs-string">&quot;train&quot;</span>][<span class="hljs-number">123456</span>][<span class="hljs-string">&quot;whole_func_string&quot;</span>])'}}),gn=new b({props:{code:`def handle_simple_responses(
      self, timeout_ms=None, info_cb=DEFAULT_MESSAGE_CALLBACK):
    """Accepts normal responses from the device.

    Args:
      timeout_ms: Timeout in milliseconds to wait for each response.
      info_cb: Optional callback for text sent from the bootloader.

    Returns:
      OKAY packet's message.
    """
    return self._accept_responses('OKAY', info_cb, timeout_ms=timeout_ms)`,highlighted:`<span class="hljs-keyword">def</span> <span class="hljs-title function_">handle_simple_responses</span>(<span class="hljs-params">
      self, timeout_ms=<span class="hljs-literal">None</span>, info_cb=DEFAULT_MESSAGE_CALLBACK</span>):
    <span class="hljs-string">&quot;&quot;&quot;Accepts normal responses from the device.

    Args:
      timeout_ms: Timeout in milliseconds to wait for each response.
      info_cb: Optional callback for text sent from the bootloader.

    Returns:
      OKAY packet&#x27;s message.
    &quot;&quot;&quot;</span>
    <span class="hljs-keyword">return</span> self._accept_responses(<span class="hljs-string">&#x27;OKAY&#x27;</span>, info_cb, timeout_ms=timeout_ms)`}}),un=new b({props:{code:`# \u0110\u1EEBng b\u1ECF ghi ch\xFA d\xF2ng b\xEAn d\u01B0\u1EDBi tr\u1EEB khi t\u1EADp d\u1EEF li\u1EC7u c\u1EE7a b\u1EA1n nh\u1ECF!
# training_corpus = [raw_datasets["train"][i: i + 1000]["whole_func_string"] for i in range(0, len(raw_datasets["train"]), 1000)]`,highlighted:`<span class="hljs-comment"># \u0110\u1EEBng b\u1ECF ghi ch\xFA d\xF2ng b\xEAn d\u01B0\u1EDBi tr\u1EEB khi t\u1EADp d\u1EEF li\u1EC7u c\u1EE7a b\u1EA1n nh\u1ECF!</span>
<span class="hljs-comment"># training_corpus = [raw_datasets[&quot;train&quot;][i: i + 1000][&quot;whole_func_string&quot;] for i in range(0, len(raw_datasets[&quot;train&quot;]), 1000)]</span>`}}),mn=new b({props:{code:`training_corpus = (
    raw_datasets["train"][i : i + 1000]["whole_func_string"]
    for i in range(0, len(raw_datasets["train"]), 1000)
)`,highlighted:`training_corpus = (
    raw_datasets[<span class="hljs-string">&quot;train&quot;</span>][i : i + <span class="hljs-number">1000</span>][<span class="hljs-string">&quot;whole_func_string&quot;</span>]
    <span class="hljs-keyword">for</span> i <span class="hljs-keyword">in</span> <span class="hljs-built_in">range</span>(<span class="hljs-number">0</span>, <span class="hljs-built_in">len</span>(raw_datasets[<span class="hljs-string">&quot;train&quot;</span>]), <span class="hljs-number">1000</span>)
)`}}),fn=new b({props:{code:`gen = (i for i in range(10))
print(list(gen))
print(list(gen))`,highlighted:`gen = (i <span class="hljs-keyword">for</span> i <span class="hljs-keyword">in</span> <span class="hljs-built_in">range</span>(<span class="hljs-number">10</span>))
<span class="hljs-built_in">print</span>(<span class="hljs-built_in">list</span>(gen))
<span class="hljs-built_in">print</span>(<span class="hljs-built_in">list</span>(gen))`}}),kn=new b({props:{code:`[0, 1, 2, 3, 4, 5, 6, 7, 8, 9]
[]`,highlighted:`[<span class="hljs-number">0</span>, <span class="hljs-number">1</span>, <span class="hljs-number">2</span>, <span class="hljs-number">3</span>, <span class="hljs-number">4</span>, <span class="hljs-number">5</span>, <span class="hljs-number">6</span>, <span class="hljs-number">7</span>, <span class="hljs-number">8</span>, <span class="hljs-number">9</span>]
[]`}}),dn=new b({props:{code:`def get_training_corpus():
    return (
        raw_datasets["train"][i : i + 1000]["whole_func_string"]
        for i in range(0, len(raw_datasets["train"]), 1000)
    )


training_corpus = get_training_corpus()`,highlighted:`<span class="hljs-keyword">def</span> <span class="hljs-title function_">get_training_corpus</span>():
    <span class="hljs-keyword">return</span> (
        raw_datasets[<span class="hljs-string">&quot;train&quot;</span>][i : i + <span class="hljs-number">1000</span>][<span class="hljs-string">&quot;whole_func_string&quot;</span>]
        <span class="hljs-keyword">for</span> i <span class="hljs-keyword">in</span> <span class="hljs-built_in">range</span>(<span class="hljs-number">0</span>, <span class="hljs-built_in">len</span>(raw_datasets[<span class="hljs-string">&quot;train&quot;</span>]), <span class="hljs-number">1000</span>)
    )


training_corpus = get_training_corpus()`}}),_n=new b({props:{code:`def get_training_corpus():
    dataset = raw_datasets["train"]
    for start_idx in range(0, len(dataset), 1000):
        samples = dataset[start_idx : start_idx + 1000]
        yield samples["whole_func_string"]`,highlighted:`<span class="hljs-keyword">def</span> <span class="hljs-title function_">get_training_corpus</span>():
    dataset = raw_datasets[<span class="hljs-string">&quot;train&quot;</span>]
    <span class="hljs-keyword">for</span> start_idx <span class="hljs-keyword">in</span> <span class="hljs-built_in">range</span>(<span class="hljs-number">0</span>, <span class="hljs-built_in">len</span>(dataset), <span class="hljs-number">1000</span>):
        samples = dataset[start_idx : start_idx + <span class="hljs-number">1000</span>]
        <span class="hljs-keyword">yield</span> samples[<span class="hljs-string">&quot;whole_func_string&quot;</span>]`}}),xn=new $h({}),bn=new b({props:{code:`from transformers import AutoTokenizer

old_tokenizer = AutoTokenizer.from_pretrained("gpt2")`,highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoTokenizer

old_tokenizer = AutoTokenizer.from_pretrained(<span class="hljs-string">&quot;gpt2&quot;</span>)`}}),vn=new b({props:{code:`example = '''def add_numbers(a, b):
    """Add the two numbers \`a\` and \`b\`."""
    return a + b'''

tokens = old_tokenizer.tokenize(example)
tokens`,highlighted:`example = <span class="hljs-string">&#x27;&#x27;&#x27;def add_numbers(a, b):
    &quot;&quot;&quot;Add the two numbers \`a\` and \`b\`.&quot;&quot;&quot;
    return a + b&#x27;&#x27;&#x27;</span>

tokens = old_tokenizer.tokenize(example)
tokens`}}),yn=new b({props:{code:`['def', '\u0120add', '_', 'n', 'umbers', '(', 'a', ',', '\u0120b', '):', '\u010A', '\u0120', '\u0120', '\u0120', '\u0120"""', 'Add', '\u0120the', '\u0120two',
 '\u0120numbers', '\u0120\`', 'a', '\`', '\u0120and', '\u0120\`', 'b', '\`', '."', '""', '\u010A', '\u0120', '\u0120', '\u0120', '\u0120return', '\u0120a', '\u0120+', '\u0120b']`,highlighted:'[<span class="hljs-string">&#x27;def&#x27;</span>, <span class="hljs-string">&#x27;\u0120add&#x27;</span>, <span class="hljs-string">&#x27;_&#x27;</span>, <span class="hljs-string">&#x27;n&#x27;</span>, <span class="hljs-string">&#x27;umbers&#x27;</span>, <span class="hljs-string">&#x27;(&#x27;</span>, <span class="hljs-string">&#x27;a&#x27;</span>, <span class="hljs-string">&#x27;,&#x27;</span>, <span class="hljs-string">&#x27;\u0120b&#x27;</span>, <span class="hljs-string">&#x27;):&#x27;</span>, <span class="hljs-string">&#x27;\u010A&#x27;</span>, <span class="hljs-string">&#x27;\u0120&#x27;</span>, <span class="hljs-string">&#x27;\u0120&#x27;</span>, <span class="hljs-string">&#x27;\u0120&#x27;</span>, <span class="hljs-string">&#x27;\u0120&quot;&quot;&quot;&#x27;</span>, <span class="hljs-string">&#x27;Add&#x27;</span>, <span class="hljs-string">&#x27;\u0120the&#x27;</span>, <span class="hljs-string">&#x27;\u0120two&#x27;</span>,\n <span class="hljs-string">&#x27;\u0120numbers&#x27;</span>, <span class="hljs-string">&#x27;\u0120`&#x27;</span>, <span class="hljs-string">&#x27;a&#x27;</span>, <span class="hljs-string">&#x27;`&#x27;</span>, <span class="hljs-string">&#x27;\u0120and&#x27;</span>, <span class="hljs-string">&#x27;\u0120`&#x27;</span>, <span class="hljs-string">&#x27;b&#x27;</span>, <span class="hljs-string">&#x27;`&#x27;</span>, <span class="hljs-string">&#x27;.&quot;&#x27;</span>, <span class="hljs-string">&#x27;&quot;&quot;&#x27;</span>, <span class="hljs-string">&#x27;\u010A&#x27;</span>, <span class="hljs-string">&#x27;\u0120&#x27;</span>, <span class="hljs-string">&#x27;\u0120&#x27;</span>, <span class="hljs-string">&#x27;\u0120&#x27;</span>, <span class="hljs-string">&#x27;\u0120return&#x27;</span>, <span class="hljs-string">&#x27;\u0120a&#x27;</span>, <span class="hljs-string">&#x27;\u0120+&#x27;</span>, <span class="hljs-string">&#x27;\u0120b&#x27;</span>]'}}),$n=new b({props:{code:"tokenizer = old_tokenizer.train_new_from_iterator(training_corpus, 52000)",highlighted:'tokenizer = old_tokenizer.train_new_from_iterator(training_corpus, <span class="hljs-number">52000</span>)'}}),wn=new b({props:{code:`tokens = tokenizer.tokenize(example)
tokens`,highlighted:`tokens = tokenizer.tokenize(example)
tokens`}}),En=new b({props:{code:`['def', '\u0120add', '_', 'numbers', '(', 'a', ',', '\u0120b', '):', '\u010A\u0120\u0120\u0120', '\u0120"""', 'Add', '\u0120the', '\u0120two', '\u0120numbers', '\u0120\`',
 'a', '\`', '\u0120and', '\u0120\`', 'b', '\`."""', '\u010A\u0120\u0120\u0120', '\u0120return', '\u0120a', '\u0120+', '\u0120b']`,highlighted:'[<span class="hljs-string">&#x27;def&#x27;</span>, <span class="hljs-string">&#x27;\u0120add&#x27;</span>, <span class="hljs-string">&#x27;_&#x27;</span>, <span class="hljs-string">&#x27;numbers&#x27;</span>, <span class="hljs-string">&#x27;(&#x27;</span>, <span class="hljs-string">&#x27;a&#x27;</span>, <span class="hljs-string">&#x27;,&#x27;</span>, <span class="hljs-string">&#x27;\u0120b&#x27;</span>, <span class="hljs-string">&#x27;):&#x27;</span>, <span class="hljs-string">&#x27;\u010A\u0120\u0120\u0120&#x27;</span>, <span class="hljs-string">&#x27;\u0120&quot;&quot;&quot;&#x27;</span>, <span class="hljs-string">&#x27;Add&#x27;</span>, <span class="hljs-string">&#x27;\u0120the&#x27;</span>, <span class="hljs-string">&#x27;\u0120two&#x27;</span>, <span class="hljs-string">&#x27;\u0120numbers&#x27;</span>, <span class="hljs-string">&#x27;\u0120`&#x27;</span>,\n <span class="hljs-string">&#x27;a&#x27;</span>, <span class="hljs-string">&#x27;`&#x27;</span>, <span class="hljs-string">&#x27;\u0120and&#x27;</span>, <span class="hljs-string">&#x27;\u0120`&#x27;</span>, <span class="hljs-string">&#x27;b&#x27;</span>, <span class="hljs-string">&#x27;`.&quot;&quot;&quot;&#x27;</span>, <span class="hljs-string">&#x27;\u010A\u0120\u0120\u0120&#x27;</span>, <span class="hljs-string">&#x27;\u0120return&#x27;</span>, <span class="hljs-string">&#x27;\u0120a&#x27;</span>, <span class="hljs-string">&#x27;\u0120+&#x27;</span>, <span class="hljs-string">&#x27;\u0120b&#x27;</span>]'}}),qn=new b({props:{code:`print(len(tokens))
print(len(old_tokenizer.tokenize(example)))`,highlighted:`<span class="hljs-built_in">print</span>(<span class="hljs-built_in">len</span>(tokens))
<span class="hljs-built_in">print</span>(<span class="hljs-built_in">len</span>(old_tokenizer.tokenize(example)))`}}),Pn=new b({props:{code:`27
36`,highlighted:`<span class="hljs-number">27</span>
<span class="hljs-number">36</span>`}}),Cn=new b({props:{code:`example = """class LinearLayer():
    def __init__(self, input_size, output_size):
        self.weight = torch.randn(input_size, output_size)
        self.bias = torch.zeros(output_size)

    def __call__(self, x):
        return x @ self.weights + self.bias
    """
tokenizer.tokenize(example)`,highlighted:`example = <span class="hljs-string">&quot;&quot;&quot;class LinearLayer():
    def __init__(self, input_size, output_size):
        self.weight = torch.randn(input_size, output_size)
        self.bias = torch.zeros(output_size)

    def __call__(self, x):
        return x @ self.weights + self.bias
    &quot;&quot;&quot;</span>
tokenizer.tokenize(example)`}}),An=new b({props:{code:`['class', '\u0120Linear', 'Layer', '():', '\u010A\u0120\u0120\u0120', '\u0120def', '\u0120__', 'init', '__(', 'self', ',', '\u0120input', '_', 'size', ',',
 '\u0120output', '_', 'size', '):', '\u010A\u0120\u0120\u0120\u0120\u0120\u0120\u0120', '\u0120self', '.', 'weight', '\u0120=', '\u0120torch', '.', 'randn', '(', 'input', '_',
 'size', ',', '\u0120output', '_', 'size', ')', '\u010A\u0120\u0120\u0120\u0120\u0120\u0120\u0120', '\u0120self', '.', 'bias', '\u0120=', '\u0120torch', '.', 'zeros', '(',
 'output', '_', 'size', ')', '\u010A\u010A\u0120\u0120\u0120', '\u0120def', '\u0120__', 'call', '__(', 'self', ',', '\u0120x', '):', '\u010A\u0120\u0120\u0120\u0120\u0120\u0120\u0120',
 '\u0120return', '\u0120x', '\u0120@', '\u0120self', '.', 'weights', '\u0120+', '\u0120self', '.', 'bias', '\u010A\u0120\u0120\u0120\u0120']`,highlighted:`[<span class="hljs-string">&#x27;class&#x27;</span>, <span class="hljs-string">&#x27;\u0120Linear&#x27;</span>, <span class="hljs-string">&#x27;Layer&#x27;</span>, <span class="hljs-string">&#x27;():&#x27;</span>, <span class="hljs-string">&#x27;\u010A\u0120\u0120\u0120&#x27;</span>, <span class="hljs-string">&#x27;\u0120def&#x27;</span>, <span class="hljs-string">&#x27;\u0120__&#x27;</span>, <span class="hljs-string">&#x27;init&#x27;</span>, <span class="hljs-string">&#x27;__(&#x27;</span>, <span class="hljs-string">&#x27;self&#x27;</span>, <span class="hljs-string">&#x27;,&#x27;</span>, <span class="hljs-string">&#x27;\u0120input&#x27;</span>, <span class="hljs-string">&#x27;_&#x27;</span>, <span class="hljs-string">&#x27;size&#x27;</span>, <span class="hljs-string">&#x27;,&#x27;</span>,
 <span class="hljs-string">&#x27;\u0120output&#x27;</span>, <span class="hljs-string">&#x27;_&#x27;</span>, <span class="hljs-string">&#x27;size&#x27;</span>, <span class="hljs-string">&#x27;):&#x27;</span>, <span class="hljs-string">&#x27;\u010A\u0120\u0120\u0120\u0120\u0120\u0120\u0120&#x27;</span>, <span class="hljs-string">&#x27;\u0120self&#x27;</span>, <span class="hljs-string">&#x27;.&#x27;</span>, <span class="hljs-string">&#x27;weight&#x27;</span>, <span class="hljs-string">&#x27;\u0120=&#x27;</span>, <span class="hljs-string">&#x27;\u0120torch&#x27;</span>, <span class="hljs-string">&#x27;.&#x27;</span>, <span class="hljs-string">&#x27;randn&#x27;</span>, <span class="hljs-string">&#x27;(&#x27;</span>, <span class="hljs-string">&#x27;input&#x27;</span>, <span class="hljs-string">&#x27;_&#x27;</span>,
 <span class="hljs-string">&#x27;size&#x27;</span>, <span class="hljs-string">&#x27;,&#x27;</span>, <span class="hljs-string">&#x27;\u0120output&#x27;</span>, <span class="hljs-string">&#x27;_&#x27;</span>, <span class="hljs-string">&#x27;size&#x27;</span>, <span class="hljs-string">&#x27;)&#x27;</span>, <span class="hljs-string">&#x27;\u010A\u0120\u0120\u0120\u0120\u0120\u0120\u0120&#x27;</span>, <span class="hljs-string">&#x27;\u0120self&#x27;</span>, <span class="hljs-string">&#x27;.&#x27;</span>, <span class="hljs-string">&#x27;bias&#x27;</span>, <span class="hljs-string">&#x27;\u0120=&#x27;</span>, <span class="hljs-string">&#x27;\u0120torch&#x27;</span>, <span class="hljs-string">&#x27;.&#x27;</span>, <span class="hljs-string">&#x27;zeros&#x27;</span>, <span class="hljs-string">&#x27;(&#x27;</span>,
 <span class="hljs-string">&#x27;output&#x27;</span>, <span class="hljs-string">&#x27;_&#x27;</span>, <span class="hljs-string">&#x27;size&#x27;</span>, <span class="hljs-string">&#x27;)&#x27;</span>, <span class="hljs-string">&#x27;\u010A\u010A\u0120\u0120\u0120&#x27;</span>, <span class="hljs-string">&#x27;\u0120def&#x27;</span>, <span class="hljs-string">&#x27;\u0120__&#x27;</span>, <span class="hljs-string">&#x27;call&#x27;</span>, <span class="hljs-string">&#x27;__(&#x27;</span>, <span class="hljs-string">&#x27;self&#x27;</span>, <span class="hljs-string">&#x27;,&#x27;</span>, <span class="hljs-string">&#x27;\u0120x&#x27;</span>, <span class="hljs-string">&#x27;):&#x27;</span>, <span class="hljs-string">&#x27;\u010A\u0120\u0120\u0120\u0120\u0120\u0120\u0120&#x27;</span>,
 <span class="hljs-string">&#x27;\u0120return&#x27;</span>, <span class="hljs-string">&#x27;\u0120x&#x27;</span>, <span class="hljs-string">&#x27;\u0120@&#x27;</span>, <span class="hljs-string">&#x27;\u0120self&#x27;</span>, <span class="hljs-string">&#x27;.&#x27;</span>, <span class="hljs-string">&#x27;weights&#x27;</span>, <span class="hljs-string">&#x27;\u0120+&#x27;</span>, <span class="hljs-string">&#x27;\u0120self&#x27;</span>, <span class="hljs-string">&#x27;.&#x27;</span>, <span class="hljs-string">&#x27;bias&#x27;</span>, <span class="hljs-string">&#x27;\u010A\u0120\u0120\u0120\u0120&#x27;</span>]`}}),Tn=new $h({}),Dn=new b({props:{code:'tokenizer.save_pretrained("code-search-net-tokenizer")',highlighted:'tokenizer.save_pretrained(<span class="hljs-string">&quot;code-search-net-tokenizer&quot;</span>)'}}),Nn=new b({props:{code:`from huggingface_hub import notebook_login

notebook_login()`,highlighted:`<span class="hljs-keyword">from</span> huggingface_hub <span class="hljs-keyword">import</span> notebook_login

notebook_login()`}}),On=new b({props:{code:"huggingface-cli login",highlighted:"huggingface-cli login"}}),Ln=new b({props:{code:'tokenizer.push_to_hub("code-search-net-tokenizer")',highlighted:'tokenizer.push_to_hub(<span class="hljs-string">&quot;code-search-net-tokenizer&quot;</span>)'}}),Hn=new b({props:{code:`# Thay "huggingface-course" d\u01B0\u1EDBi \u0111\u1EA5y v\u1EDBi t\xEAn kh\xF4ng gian th\u1EF1c s\u1EF1 s\u1EED d\u1EE5ng tokenizer ri\xEAng c\u1EE7a b\u1EA1n
tokenizer = AutoTokenizer.from_pretrained("huggingface-course/code-search-net-tokenizer")`,highlighted:`<span class="hljs-comment"># Thay &quot;huggingface-course&quot; d\u01B0\u1EDBi \u0111\u1EA5y v\u1EDBi t\xEAn kh\xF4ng gian th\u1EF1c s\u1EF1 s\u1EED d\u1EE5ng tokenizer ri\xEAng c\u1EE7a b\u1EA1n</span>
tokenizer = AutoTokenizer.from_pretrained(<span class="hljs-string">&quot;huggingface-course/code-search-net-tokenizer&quot;</span>)`}}),{c(){y=l("meta"),B=o(),j=l("h1"),w=l("a"),ot=l("span"),g(nn.$$.fragment),jh=o(),pt=l("span"),zh=e("Hu\u1EA5n luy\u1EC7n m\u1ED9t tokenizer m\u1EDBi t\u1EEB c\xE1i c\u0169"),ss=o(),g(tn.$$.fragment),hs=o(),E=l("p"),wh=e("N\u1EBFu m\xF4 h\xECnh ng\xF4n ng\u1EEF kh\xF4ng c\xF3 s\u1EB5n ng\xF4n ng\u1EEF b\u1EA1n quan t\xE2m ho\u1EB7c n\u1EBFu kho t\xE0i li\u1EC7u c\u1EE7a b\u1EA1n r\u1EA5t kh\xE1c v\u1EDBi kho m\xE0 m\xF4 h\xECnh ng\xF4n ng\u1EEF c\u1EE7a b\u1EA1n \u0111\xE3 hu\u1EA5n luy\u1EC7n, b\u1EA1n r\u1EA5t c\xF3 th\u1EC3 s\u1EBD mu\u1ED1n hu\u1EA5n luy\u1EC7n l\u1EA1i m\xF4 h\xECnh t\u1EEB \u0111\u1EA7u b\u1EB1ng c\xE1ch s\u1EED d\u1EE5ng tr\xECnh tokenize ph\xF9 h\u1EE3p v\u1EDBi d\u1EEF li\u1EC7u c\u1EE7a b\u1EA1n. \u0110i\u1EC1u \u0111\xF3 s\u1EBD y\xEAu c\u1EA7u hu\u1EA5n luy\u1EC7n m\u1ED9t tr\xECnh tokenize m\u1EDBi tr\xEAn t\u1EADp d\u1EEF li\u1EC7u c\u1EE7a b\u1EA1n. Nh\u01B0ng ch\xEDnh x\xE1c th\xEC \u0111i\u1EC1u \u0111\xF3 c\xF3 ngh\u0129a l\xE0 g\xEC? Khi ch\xFAng ta l\u1EA7n \u0111\u1EA7u xem x\xE9t c\xE1c tokenizer trong "),Gn=l("a"),Eh=e("Ch\u01B0\u01A1ng 2"),qh=e(", ch\xFAng ta th\u1EA5y r\u1EB1ng h\u1EA7u h\u1EBFt c\xE1c m\xF4 h\xECnh Transformer s\u1EED d\u1EE5ng thu\u1EADt to\xE1n tokenize "),gt=l("em"),Ph=e("t\u1EEB ph\u1EE5"),Ch=e(". \u0110\u1EC3 x\xE1c \u0111\u1ECBnh nh\u1EEFng t\u1EEB ph\u1EE5 n\xE0o \u0111\u01B0\u1EE3c quan t\xE2m v\xE0 xu\u1EA5t hi\u1EC7n th\u01B0\u1EDDng xuy\xEAn nh\u1EA5t trong kho ng\u1EEF li\u1EC7u hi\u1EC7n c\xF3, tr\xECnh tokenize c\u1EA7n ph\u1EA3i xem x\xE9t k\u1EF9 t\u1EA5t c\u1EA3 c\xE1c v\u0103n b\u1EA3n trong kho ng\u1EEF li\u1EC7u - m\u1ED9t qu\xE1 tr\xECnh m\xE0 ch\xFAng ta g\u1ECDi l\xE0 "),ut=l("em"),Ah=e("hu\u1EA5n luy\u1EC7n"),Th=e(". C\xE1c quy t\u1EAFc chi ph\u1ED1i vi\u1EC7c hu\u1EA5n luy\u1EC7n n\xE0y ph\u1EE5 thu\u1ED9c v\xE0o lo\u1EA1i tokenizer \u0111\u01B0\u1EE3c s\u1EED d\u1EE5ng v\xE0 ch\xFAng ta s\u1EBD xem x\xE9t ba thu\u1EADt to\xE1n ch\xEDnh \u1EDF ph\u1EA7n sau c\u1EE7a ch\u01B0\u01A1ng n\xE0y."),es=o(),g(sn.$$.fragment),as=o(),g(R.$$.fragment),is=o(),S=l("h2"),U=l("a"),mt=l("span"),g(hn.$$.fragment),Dh=o(),ft=l("span"),Nh=e("T\u1EADp h\u1EE3p m\u1ED9t kho ng\u1EEF li\u1EC7u"),ls=o(),K=l("p"),Oh=e("C\xF3 m\u1ED9t API r\u1EA5t \u0111\u01A1n gi\u1EA3n trong \u{1F917} Transformers m\xE0 b\u1EA1n c\xF3 th\u1EC3 s\u1EED d\u1EE5ng \u0111\u1EC3 hu\u1EA5n luy\u1EC7n m\u1ED9t tokenizer m\u1EDBi c\xF3 c\xF9ng \u0111\u1EB7c \u0111i\u1EC3m v\u1EDBi c\xE1i hi\u1EC7n c\xF3: "),kt=l("code"),Lh=e("AutoTokenizer.train_new_from_iterator()"),Hh=e(". \u0110\u1EC3 th\u1EA5y \u0111i\u1EC1u n\xE0y trong th\u1EF1c t\u1EBF, gi\u1EA3 s\u1EED ch\xFAng ta mu\u1ED1n hu\u1EA5n luy\u1EC7n GPT-2 t\u1EEB \u0111\u1EA7u, nh\u01B0ng b\u1EB1ng m\u1ED9t ng\xF4n ng\u1EEF kh\xE1c ngo\xE0i ti\u1EBFng Anh. Nhi\u1EC7m v\u1EE5 \u0111\u1EA7u ti\xEAn c\u1EE7a ch\xFAng ta s\u1EBD l\xE0 thu th\u1EADp nhi\u1EC1u d\u1EEF li\u1EC7u b\u1EB1ng ng\xF4n ng\u1EEF \u0111\xF3 trong m\u1ED9t kho d\u1EEF li\u1EC7u hu\u1EA5n luy\u1EC7n. \u0110\u1EC3 cung c\u1EA5p c\xE1c m\u1EABu m\xE0 m\u1ECDi ng\u01B0\u1EDDi c\xF3 hi\u1EC3u \u0111\u01B0\u1EE3c, ch\xFAng ta s\u1EBD kh\xF4ng s\u1EED d\u1EE5ng ng\xF4n ng\u1EEF nh\u01B0 ti\u1EBFng Nga ho\u1EB7c ti\u1EBFng Trung \u1EDF \u0111\xE2y, m\xE0 l\xE0 ng\xF4n ng\u1EEF ti\u1EBFng Anh chuy\xEAn d\u1EE5ng: \u0111o\u1EA1n m\xE3 Python."),cs=o(),z=l("p"),Sh=e("Th\u01B0 vi\u1EC7n "),en=l("a"),Gh=e("\u{1F917} Datasets"),Mh=e(" c\xF3 th\u1EC3 gi\xFAp ch\xFAng ta t\u1EADp h\u1EE3p m\u1ED9t kho d\u1EEF li\u1EC7u m\xE3 ngu\u1ED3n Python. Ch\xFAng ta s\u1EBD s\u1EED d\u1EE5ng h\xE0m "),dt=l("code"),Bh=e("load_dataset()"),Rh=e(" th\xF4ng th\u01B0\u1EDDng \u0111\u1EC3 t\u1EA3i xu\u1ED1ng v\xE0 l\u01B0u v\xE0o b\u1ED9 nh\u1EDB cache c\u1EE7a t\u1EADp d\u1EEF li\u1EC7u "),an=l("a"),Uh=e("CodeSearchNet"),Kh=e(". T\u1EADp d\u1EEF li\u1EC7u n\xE0y \u0111\u01B0\u1EE3c t\u1EA1o cho "),ln=l("a"),Vh=e("th\u1EED th\xE1ch CodeSearchNet"),Ih=e(" v\xE0 ch\u1EE9a h\xE0ng tri\u1EC7u h\xE0m t\u1EEB c\xE1c th\u01B0 vi\u1EC7n m\xE3 ngu\u1ED3n m\u1EDF tr\xEAn GitHub b\u1EB1ng m\u1ED9t s\u1ED1 ng\xF4n ng\u1EEF l\u1EADp tr\xECnh. \u1EDE \u0111\xE2y, ch\xFAng ta s\u1EBD t\u1EA3i ph\u1EA7n Python c\u1EE7a t\u1EADp d\u1EEF li\u1EC7u n\xE0y:"),rs=o(),g(cn.$$.fragment),os=o(),Mn=l("p"),Yh=e("Ch\xFAng ta c\xF3 th\u1EC3 xem x\xE9t ph\u1EA7n t\xE1ch hu\u1EA5n luy\u1EC7n \u0111\u1EC3 xem ta c\xF3 quy\u1EC1n truy c\u1EADp v\xE0o nh\u1EEFng c\u1ED9t n\xE0o:"),ps=o(),g(rn.$$.fragment),gs=o(),g(on.$$.fragment),us=o(),C=l("p"),Fh=e("Ch\xFAng ta c\xF3 th\u1EC3 th\u1EA5y t\u1EADp d\u1EEF li\u1EC7u t\xE1ch chu\u1ED7i t\xE0i li\u1EC7u m\xF4 t\u1EA3 kh\u1ECFi \u0111o\u1EA1n m\xE3 v\xE0 \u0111\u1EC1 xu\u1EA5t tokenize c\u1EA3 hai. \u1EDE \u0111\xE2y, ch\xFAng ta s\u1EBD ch\u1EC9 s\u1EED d\u1EE5ng c\u1ED9t "),_t=l("code"),Qh=e("whole_func_string"),Xh=e(" \u0111\u1EC3 hu\u1EA5n luy\u1EC7n tr\xECnh tokenize. Ch\xFAng ta c\xF3 th\u1EC3 xem x\xE9t m\u1EABu m\u1ED9t trong nh\u1EEFng h\xE0m n\xE0y b\u1EB1ng c\xE1ch l\u1EADp ch\u1EC9 m\u1EE5c v\xE0o ph\u1EA7n "),xt=l("code"),Jh=e("train"),Zh=e(":"),ms=o(),g(pn.$$.fragment),fs=o(),Bn=l("p"),Wh=e("n\xF3 n\xEAn tr\u1EA3 v\u1EC1 k\u1EBFt qu\u1EA3 nh\u01B0 d\u01B0\u1EDBi \u0111\xE2y:"),ks=o(),g(gn.$$.fragment),ds=o(),V=l("p"),ne=e("\u0110i\u1EC1u \u0111\u1EA7u ti\xEAn ch\xFAng ta c\u1EA7n l\xE0m l\xE0 chuy\u1EC3n \u0111\u1ED5i t\u1EADp d\u1EEF li\u1EC7u th\xE0nh m\u1ED9t "),bt=l("em"),te=e("iterator"),se=e(" danh s\xE1ch c\xE1c v\u0103n b\u1EA3n - v\xED d\u1EE5, m\u1ED9t danh s\xE1ch c\xE1c v\u0103n b\u1EA3n. Vi\u1EC7c s\u1EED d\u1EE5ng danh s\xE1ch v\u0103n b\u1EA3n s\u1EBD cho ph\xE9p tokenizer ho\u1EA1t \u0111\u1ED9ng nhanh h\u01A1n (hu\u1EA5n luy\u1EC7n h\xE0ng lo\u1EA1t v\u0103n b\u1EA3n thay v\xEC x\u1EED l\xFD t\u1EEBng v\u0103n b\u1EA3n ri\xEAng l\u1EBB) v\xE0 n\xF3 ph\u1EA3i l\xE0 m\u1ED9t tr\xECnh l\u1EB7p n\u1EBFu ch\xFAng ta mu\u1ED1n tr\xE1nh c\xF3 m\u1ECDi th\u1EE9 trong b\u1ED9 nh\u1EDB c\xF9ng m\u1ED9t l\xFAc. N\u1EBFu kho d\u1EEF li\u1EC7u c\u1EE7a b\u1EA1n l\u1EDBn, b\u1EA1n s\u1EBD mu\u1ED1n t\u1EADn d\u1EE5ng l\u1EE3i th\u1EBF th\u1EF1c ti\u1EC5n l\xE0 \u{1F917} Datasets kh\xF4ng t\u1EA3i m\u1ECDi th\u1EE9 v\xE0o RAM m\xE0 l\u01B0u tr\u1EEF c\xE1c ph\u1EA7n t\u1EED c\u1EE7a t\u1EADp d\u1EEF li\u1EC7u tr\xEAn \u0111\u0129a."),_s=o(),Rn=l("p"),he=e("L\xE0m nh\u01B0 sau s\u1EBD t\u1EA1o m\u1ED9t danh s\xE1ch c\xE1c danh s\xE1ch v\u1EDBi m\u1ED7i danh s\xE1ch g\u1ED3m 1,000 v\u0103n b\u1EA3n, nh\u01B0ng s\u1EBD t\u1EA3i m\u1ECDi th\u1EE9 v\xE0o b\u1ED9 nh\u1EDB:"),xs=o(),g(un.$$.fragment),bs=o(),Un=l("p"),ee=e("S\u1EED d\u1EE5ng tr\xECnh t\u1EA1o Python, ch\xFAng ta c\xF3 th\u1EC3 tr\xE1nh vi\u1EC7c Python t\u1EA3i b\u1EA5t k\u1EF3 th\u1EE9 g\xEC v\xE0o b\u1ED9 nh\u1EDB cho \u0111\u1EBFn khi n\xF3 th\u1EF1c s\u1EF1 c\u1EA7n thi\u1EBFt. \u0110\u1EC3 t\u1EA1o m\u1ED9t tr\xECnh t\u1EA1o nh\u01B0 v\u1EADy, b\u1EA1n ch\u1EC9 c\u1EA7n thay d\u1EA5u ngo\u1EB7c vu\xF4ng b\u1EB1ng d\u1EA5u ngo\u1EB7c \u0111\u01A1n:"),vs=o(),g(mn.$$.fragment),ys=o(),A=l("p"),ae=e("D\xF2ng m\xE3 n\xE0y kh\xF4ng t\xECm n\u1EA1p b\u1EA5t k\u1EF3 ph\u1EA7n t\u1EED n\xE0o c\u1EE7a t\u1EADp d\u1EEF li\u1EC7u; n\xF3 ch\u1EC9 t\u1EA1o m\u1ED9t \u0111\u1ED1i t\u01B0\u1EE3ng m\xE0 b\u1EA1n c\xF3 th\u1EC3 s\u1EED d\u1EE5ng trong v\xF2ng l\u1EB7p Python "),vt=l("code"),ie=e("for"),le=e(". C\xE1c v\u0103n b\u1EA3n s\u1EBD ch\u1EC9 \u0111\u01B0\u1EE3c t\u1EA3i khi b\u1EA1n c\u1EA7n (ngh\u0129a l\xE0 khi b\u1EA1n \u0111ang \u1EDF b\u01B0\u1EDBc c\u1EE7a v\xF2ng l\u1EB7p "),yt=l("code"),ce=e("for"),re=e(" m\xE0 y\xEAu c\u1EA7u ch\xFAng) v\xE0 ch\u1EC9 1,000 v\u0103n b\u1EA3n s\u1EBD \u0111\u01B0\u1EE3c t\u1EA3i m\u1ED7i l\u1EA7n. B\u1EB1ng c\xE1ch n\xE0y, b\u1EA1n s\u1EBD kh\xF4ng s\u1EED d\u1EE5ng h\u1EBFt b\u1ED9 nh\u1EDB c\u1EE7a m\xECnh ngay c\u1EA3 khi b\u1EA1n \u0111ang x\u1EED l\xFD m\u1ED9t t\u1EADp d\u1EEF li\u1EC7u l\u1EDBn."),$s=o(),Kn=l("p"),oe=e("V\u1EA5n \u0111\u1EC1 v\u1EDBi m\u1ED9t \u0111\u1ED1i t\u01B0\u1EE3ng t\u1EA1o l\xE0 n\xF3 ch\u1EC9 c\xF3 th\u1EC3 \u0111\u01B0\u1EE3c s\u1EED d\u1EE5ng m\u1ED9t l\u1EA7n. V\xEC v\u1EADy, thay v\xEC \u0111i\u1EC1u n\xE0y cho ta danh s\xE1ch 10 ch\u1EEF s\u1ED1 \u0111\u1EA7u ti\xEAn hai l\u1EA7n:"),js=o(),g(fn.$$.fragment),zs=o(),Vn=l("p"),pe=e("ch\xFAng ta c\xF3 th\u1EC3 l\u1EA5y ch\xFAng trong m\u1ED9t l\u1EA7n v\xE0 sau \u0111\xF3 danh s\xE1ng s\u1EBD tr\u1ED1ng:"),ws=o(),g(kn.$$.fragment),Es=o(),In=l("p"),ge=e("\u0110\xF3 l\xE0 l\xED do ch\xFAng ta \u0111\u1ECBnh ngh\u0129a m\u1ED9t h\xE0m thay v\xE0o \u0111\xF3 tr\u1EA3 v\u1EC1 m\u1ED9t tr\xECnh t\u1EA1o:"),qs=o(),g(dn.$$.fragment),Ps=o(),T=l("p"),ue=e("Ta c\xF3 th\u1EC3 \u0111\u1ECBnh ngh\u0129a tr\xECnh t\u1EA1o b\xEAn trong v\xF2ng l\u1EB7p "),$t=l("code"),me=e("for"),fe=e(" s\u1EED d\u1EE5ng "),jt=l("code"),ke=e("yield"),de=e(":"),Cs=o(),g(_n.$$.fragment),As=o(),Yn=l("p"),_e=e("s\u1EBD t\u1EA1o ra tr\xECnh t\u1EA1o ho\xE0n to\xE0n gi\u1ED1ng nh\u01B0 tr\u01B0\u1EDBc \u0111\xE2y, nh\u01B0ng cho ph\xE9p b\u1EA1n s\u1EED d\u1EE5ng logic ph\u1EE9c t\u1EA1p h\u01A1n b\u1EA1n c\xF3 th\u1EC3 trong m\u1ED9t bao h\xE0m."),Ts=o(),G=l("h2"),I=l("a"),zt=l("span"),g(xn.$$.fragment),xe=o(),wt=l("span"),be=e("Hu\u1EA5n luy\u1EC7n m\u1ED9t tokenizer m\u1EDBi"),Ds=o(),Fn=l("p"),ve=e("B\xE2y gi\u1EDD ch\xFAng ta \u0111\xE3 c\xF3 kho v\u0103n b\u1EA3n c\u1EE7a m\xECnh d\u01B0\u1EDBi d\u1EA1ng m\u1ED9t tr\xECnh l\u1EB7p c\xE1c lo\u1EA1t v\u0103n b\u1EA3n, ch\xFAng ta \u0111\xE3 s\u1EB5n s\xE0ng \u0111\u1EC3 hu\u1EA5n luy\u1EC7n m\u1ED9t tr\xECnh tokenize m\u1EDBi. \u0110\u1EC3 th\u1EF1c hi\u1EC7n vi\u1EC7c n\xE0y, tr\u01B0\u1EDBc ti\xEAn ch\xFAng ta c\u1EA7n t\u1EA3i tokenizer m\xE0 ch\xFAng ta mu\u1ED1n gh\xE9p n\u1ED1i v\u1EDBi m\xF4 h\xECnh c\u1EE7a m\xECnh (\u1EDF \u0111\xE2y, GPT-2):"),Ns=o(),g(bn.$$.fragment),Os=o(),Qn=l("p"),ye=e("M\u1EB7c d\xF9 ch\xFAng ta s\u1EBD hu\u1EA5n luy\u1EC7n m\u1ED9t tokenizer, nh\u01B0ng b\u1EA1n n\xEAn l\xE0m \u0111i\u1EC1u n\xE0y \u0111\u1EC3 tr\xE1nh b\u1EAFt \u0111\u1EA7u ho\xE0n to\xE0n t\u1EEB \u0111\u1EA7u. B\u1EB1ng c\xE1ch n\xE0y, ch\xFAng ta s\u1EBD kh\xF4ng ph\u1EA3i ch\u1EC9 \u0111\u1ECBnh b\u1EA5t k\u1EF3 \u0111i\u1EC1u g\xEC v\u1EC1 thu\u1EADt to\xE1n tokenize ho\u1EB7c c\xE1c token \u0111\u1EB7c bi\u1EC7t m\xE0 ta mu\u1ED1n s\u1EED d\u1EE5ng; tokenizer m\u1EDBi s\u1EBD gi\u1ED1ng h\u1EC7t nh\u01B0 GPT-2 v\xE0 \u0111i\u1EC1u duy nh\u1EA5t s\u1EBD thay \u0111\u1ED5i l\xE0 t\u1EEB v\u1EF1ng, s\u1EBD \u0111\u01B0\u1EE3c x\xE1c \u0111\u1ECBnh b\u1EDFi qu\xE1 tr\xECnh hu\u1EA5n luy\u1EC7n tr\xEAn kho ng\u1EEF li\u1EC7u c\u1EE7a ch\xFAng t\xF4i."),Ls=o(),Xn=l("p"),$e=e("\u0110\u1EA7u ti\xEAn, ch\xFAng ta h\xE3y xem c\xE1ch m\xE0 tokenizer n\xE0y s\u1EBD x\u1EED l\xFD m\u1ED9t h\xE0m m\u1EABu th\u1EBF n\xE0o:"),Hs=o(),g(vn.$$.fragment),Ss=o(),g(yn.$$.fragment),Gs=o(),q=l("p"),je=e("Tokenizer n\xE0y c\xF3 m\u1ED9t s\u1ED1 k\xFD hi\u1EC7u \u0111\u1EB7c bi\u1EC7t, nh\u01B0 "),Et=l("code"),ze=e("\u0120"),we=e(" v\xE0 "),qt=l("code"),Ee=e("\u010A"),qe=e(", t\u01B0\u01A1ng \u1EE9ng bi\u1EC3u th\u1ECB d\u1EA5u c\xE1ch v\xE0 d\xF2ng m\u1EDBi. Nh\u01B0 ch\xFAng ta c\xF3 th\u1EC3 th\u1EA5y, \u0111i\u1EC1u n\xE0y kh\xF4ng qu\xE1 hi\u1EC7u qu\u1EA3: tokenizer tr\u1EA3 v\u1EC1 c\xE1c m\xE3 th\xF4ng b\xE1o ri\xEAng l\u1EBB cho t\u1EEBng kho\u1EA3ng tr\u1EAFng, khi n\xF3 c\xF3 th\u1EC3 nh\xF3m c\xE1c m\u1EE9c th\u1EE5t l\u1EC1 l\u1EA1i v\u1EDBi nhau (v\xEC c\xF3 b\u1ED9 b\u1ED1n ho\u1EB7c t\xE1m d\u1EA5u c\xE1ch s\u1EBD r\u1EA5t ph\u1ED5 bi\u1EBFn trong m\xE3). N\xF3 c\u0169ng t\xE1ch t\xEAn h\xE0m h\u01A1i k\u1EF3 l\u1EA1, nh\xECn kh\xF4ng quen c\xE1c t\u1EEB c\xF3 k\xFD t\u1EF1 "),Pt=l("code"),Pe=e("_"),Ce=e("."),Ms=o(),Y=l("p"),Ae=e("H\xE3y hu\u1EA5n luy\u1EC7n m\u1ED9t tokenizer m\u1EDBi v\xE0 xem li\u1EC7u n\xF3 c\xF3 gi\u1EA3i quy\u1EBFt \u0111\u01B0\u1EE3c nh\u1EEFng v\u1EA5n \u0111\u1EC1 \u0111\xF3 kh\xF4ng. \u0110\u1ED1i v\u1EDBi \u0111i\u1EC1u n\xE0y, ch\xFAng ta s\u1EBD s\u1EED d\u1EE5ng ph\u01B0\u01A1ng th\u1EE9c "),Ct=l("code"),Te=e("train_new_from_iterator()"),De=e(":"),Bs=o(),g($n.$$.fragment),Rs=o(),Jn=l("p"),Ne=e("L\u1EC7nh n\xE0y c\xF3 th\u1EC3 m\u1EA5t m\u1ED9t ch\xFAt th\u1EDDi gian n\u1EBFu kho d\u1EEF li\u1EC7u c\u1EE7a b\u1EA1n r\u1EA5t l\u1EDBn, nh\u01B0ng \u0111\u1ED1i v\u1EDBi t\u1EADp d\u1EEF li\u1EC7u 1.6GB v\u0103n b\u1EA3n n\xE0y, n\xF3 r\u1EA5t nhanh (1 ph\xFAt 16 gi\xE2y tr\xEAn CPU AMD Ryzen 9 3900X v\u1EDBi 12 l\xF5i)."),Us=o(),D=l("p"),Oe=e("L\u01B0u \xFD r\u1EB1ng "),At=l("code"),Le=e("AutoTokenizer.train_new_from_iterator()"),He=e(" ch\u1EC9 ho\u1EA1t \u0111\u1ED9ng n\u1EBFu tokenizer b\u1EA1n \u0111ang s\u1EED d\u1EE5ng l\xE0 tokenizer \u201Cnhanh\u201D. Nh\u01B0 b\u1EA1n s\u1EBD th\u1EA5y trong ph\u1EA7n ti\u1EBFp theo, th\u01B0 vi\u1EC7n \u{1F917} Transformers ch\u1EE9a hai lo\u1EA1i tokenizers: m\u1ED9t s\u1ED1 \u0111\u01B0\u1EE3c vi\u1EBFt ho\xE0n to\xE0n b\u1EB1ng Python v\xE0 nh\u1EEFng lo\u1EA1i kh\xE1c (lo\u1EA1i nhanh) \u0111\u01B0\u1EE3c h\u1ED7 tr\u1EE3 b\u1EDFi th\u01B0 vi\u1EC7n \u{1F917} Tokenizers, \u0111\u01B0\u1EE3c vi\u1EBFt b\u1EB1ng ng\xF4n ng\u1EEF l\u1EADp tr\xECnh "),jn=l("a"),Se=e("Rust"),Ge=e(". Python l\xE0 ng\xF4n ng\u1EEF th\u01B0\u1EDDng \u0111\u01B0\u1EE3c s\u1EED d\u1EE5ng nh\u1EA5t cho c\xE1c \u1EE9ng d\u1EE5ng khoa h\u1ECDc d\u1EEF li\u1EC7u v\xE0 h\u1ECDc s\xE2u, nh\u01B0ng khi b\u1EA5t k\u1EF3 th\u1EE9 g\xEC c\u1EA7n \u0111\u01B0\u1EE3c song song h\xF3a cho nhanh, n\xF3 ph\u1EA3i \u0111\u01B0\u1EE3c vi\u1EBFt b\u1EB1ng m\u1ED9t ng\xF4n ng\u1EEF kh\xE1c. V\xED d\u1EE5, c\xE1c ph\xE9p nh\xE2n ma tr\u1EADn l\xE0 c\u1ED1t l\xF5i c\u1EE7a t\xEDnh to\xE1n m\xF4 h\xECnh \u0111\u01B0\u1EE3c vi\u1EBFt b\u1EB1ng CUDA, m\u1ED9t th\u01B0 vi\u1EC7n C \u0111\u01B0\u1EE3c t\u1ED1i \u01B0u h\xF3a cho GPU."),Ks=o(),F=l("p"),Me=e("Vi\u1EC7c hu\u1EA5n luy\u1EC7n m\u1ED9t tokenizer ho\xE0n to\xE0n m\u1EDBi b\u1EB1ng Python thu\u1EA7n t\xFAy s\u1EBD r\u1EA5t ch\u1EADm, \u0111\xF3 l\xE0 l\xFD do t\u1EA1i sao ch\xFAng t\xF4i \u0111\xE3 ph\xE1t tri\u1EC3n th\u01B0 vi\u1EC7n \u{1F917} Tokenizer. L\u01B0u \xFD r\u1EB1ng c\u0169ng gi\u1ED1ng nh\u01B0 b\u1EA1n kh\xF4ng ph\u1EA3i h\u1ECDc ng\xF4n ng\u1EEF CUDA \u0111\u1EC3 c\xF3 th\u1EC3 th\u1EF1c thi m\xF4 h\xECnh c\u1EE7a m\xECnh tr\xEAn m\u1ED9t lo\u1EA1t \u0111\u1EA7u v\xE0o tr\xEAn GPU, b\u1EA1n s\u1EBD kh\xF4ng c\u1EA7n ph\u1EA3i h\u1ECDc Rust \u0111\u1EC3 s\u1EED d\u1EE5ng tr\xECnh tokenizer nhanh. Th\u01B0 vi\u1EC7n \u{1F917} Tokenizers cung c\u1EA5p c\xE1c li\xEAn k\u1EBFt Python cho nhi\u1EC1u ph\u01B0\u01A1ng th\u1EE9c g\u1ECDi n\u1ED9i b\u1ED9 m\u1ED9t s\u1ED1 \u0111o\u1EA1n m\xE3 trong Rust; v\xED d\u1EE5: \u0111\u1EC3 song song hu\u1EA5n luy\u1EC7n tr\xECnh tokenize m\u1EDBi c\u1EE7a b\u1EA1n ho\u1EB7c, nh\u01B0 ch\xFAng ta \u0111\xE3 th\u1EA5y trong "),Zn=l("a"),Be=e("Ch\u01B0\u01A1ng 3"),Re=e(", tokenize m\u1ED9t lo\u1EA1t \u0111\u1EA7u v\xE0o."),Vs=o(),N=l("p"),Ue=e("H\u1EA7u h\u1EBFt c\xE1c m\xF4 h\xECnh Transformer \u0111\u1EC1u c\xF3 s\u1EB5n c\xF4ng c\u1EE5 tokenize nhanh (c\xF3 m\u1ED9t s\u1ED1 ngo\u1EA1i l\u1EC7 m\xE0 b\u1EA1n c\xF3 th\u1EC3 ki\u1EC3m tra "),zn=l("a"),Ke=e("t\u1EA1i \u0111\xE2y"),Ve=e(") v\xE0 API "),Tt=l("code"),Ie=e("AutoTokenizer"),Ye=e(" lu\xF4n ch\u1ECDn t\u1ED1c tokenizer nhanh cho b\u1EA1n n\u1EBFu n\xF3 c\xF3 s\u1EB5n. Trong ph\u1EA7n ti\u1EBFp theo, ch\xFAng ta s\u1EBD xem x\xE9t m\u1ED9t s\u1ED1 t\xEDnh n\u0103ng \u0111\u1EB7c bi\u1EC7t kh\xE1c m\xE0 c\xE1c tokenize nhanh c\xF3 m\xE0 th\u1EF1c s\u1EF1 h\u1EEFu \xEDch cho c\xE1c t\xE1c v\u1EE5 nh\u01B0 ph\xE2n lo\u1EA1i token v\xE0 h\u1ECFi \u0111\xE1p. Tuy nhi\xEAn, tr\u01B0\u1EDBc khi \u0111i s\xE2u v\xE0o v\u1EA5n \u0111\u1EC1 \u0111\xF3, ch\xFAng ta h\xE3y th\u1EED tokenizer ho\xE0n to\xE0n m\u1EDBi c\u1EE7a ch\xFAng ta tr\xEAn m\u1EABu tr\u01B0\u1EDBc:"),Is=o(),g(wn.$$.fragment),Ys=o(),g(En.$$.fragment),Fs=o(),$=l("p"),Fe=e("\u1EDE \u0111\xE2y ch\xFAng ta l\u1EA1i th\u1EA5y c\xE1c k\xFD hi\u1EC7u \u0111\u1EB7c bi\u1EC7t "),Dt=l("code"),Qe=e("\u0120"),Xe=e(" v\xE0 "),Nt=l("code"),Je=e("\u010A"),Ze=e(" bi\u1EC3u th\u1ECB d\u1EA5u c\xE1ch v\xE0 d\xF2ng m\u1EDBi, nh\u01B0ng ch\xFAng ta c\u0169ng c\xF3 th\u1EC3 th\u1EA5y r\u1EB1ng tr\xECnh tokenize \u0111\xE3 h\u1ECDc \u0111\u01B0\u1EE3c m\u1ED9t s\u1ED1 token r\u1EA5t c\u1EE5 th\u1EC3 cho m\u1ED9t kho c\xE1c h\xE0m Python: v\xED d\u1EE5: c\xF3 m\u1ED9t token "),Ot=l("code"),We=e("\u010A\u0120\u0120\u0120"),na=e(" \u0111\u1EA1i di\u1EC7n cho m\u1ED9t th\u1EE5t l\u1EC1 v\xE0 token "),Lt=l("code"),ta=e('\u0120"""'),sa=e(" \u0111\u1EA1i di\u1EC7n cho ba d\u1EA5u ngo\u1EB7c k\xE9p b\u1EAFt \u0111\u1EA7u m\u1ED9t chu\u1ED7i t\xE0i li\u1EC7u. Tokenizer c\u0169ng ph\xE2n chia ch\xEDnh x\xE1c t\xEAn h\xE0m tr\xEAn "),Ht=l("code"),ha=e("_"),ea=e(". \u0110\xE2y l\xE0 m\u1ED9t bi\u1EC5u di\u1EC5n kh\xE1 nh\u1ECF g\u1ECDn; t\u01B0\u01A1ng \u0111\u1ED1i, s\u1EED d\u1EE5ng tokenizer \u0111\u01A1n gi\u1EA3n b\u1EB1ng ti\u1EBFng Anh tr\xEAn c\xF9ng m\u1ED9t m\u1EABu s\u1EBD cho ta m\u1ED9t c\xE2u d\xE0i h\u01A1n:"),Qs=o(),g(qn.$$.fragment),Xs=o(),g(Pn.$$.fragment),Js=o(),Wn=l("p"),aa=e("H\xE3y c\xF9ng nh\xECn v\xE0o v\xED d\u1EE5 sau:"),Zs=o(),g(Cn.$$.fragment),Ws=o(),g(An.$$.fragment),nh=o(),x=l("p"),ia=e("Ngo\xE0i token t\u01B0\u01A1ng \u1EE9ng v\u1EDBi th\u1EE5t l\u1EC1, \u1EDF \u0111\xE2y ch\xFAng ta c\u0169ng c\xF3 th\u1EC3 th\u1EA5y token cho th\u1EE5t l\u1EC1 k\xE9p:"),St=l("code"),la=e("\u010A\u0120\u0120\u0120\u0120\u0120\u0120\u0120"),ca=e(". C\xE1c t\u1EEB \u0111\u1EB7c bi\u1EC7t trong Python nh\u01B0 "),Gt=l("code"),ra=e("class"),oa=e(", "),Mt=l("code"),pa=e("init"),ga=e(", "),Bt=l("code"),ua=e("call"),ma=e(", "),Rt=l("code"),fa=e("self"),ka=e(", v\xE0 "),Ut=l("code"),da=e("return"),_a=e(", m\u1ED7i t\u1EEB \u0111\u01B0\u1EE3c tokenize th\xE0nh m\u1ED9t token v\xE0 ch\xFAng ta c\xF3 th\u1EC3 th\u1EA5y c\u0169ng nh\u01B0 t\xE1ch "),Kt=l("code"),xa=e("_"),ba=e("  v\xE0 "),Vt=l("code"),va=e("."),ya=e(",  tokenizer ph\xE2n chia ch\xEDnh x\xE1c c\xE1c t\xEAn: "),It=l("code"),$a=e("LinearLayer"),ja=e(" \u0111\u01B0\u1EE3c tokenize l\xE0 "),Yt=l("code"),za=e('["\u0120Linear", "Layer"]'),wa=e("."),th=o(),M=l("h2"),Q=l("a"),Ft=l("span"),g(Tn.$$.fragment),Ea=o(),Qt=l("span"),qa=e("L\u01B0u tokenizer"),sh=o(),X=l("p"),Pa=e("\u0110\u1EC3 \u0111\u1EA3m b\u1EA3o r\u1EB1ng ch\xFAng ta c\xF3 th\u1EC3 s\u1EED d\u1EE5ng n\xF3 sau n\xE0y, ch\xFAng ta c\u1EA7n ph\u1EA3i l\u01B0u tokenizer m\u1EDBi c\u1EE7a m\xECnh. Gi\u1ED1ng nh\u01B0 \u0111\u1ED1i v\u1EDBi c\xE1c m\xF4 h\xECnh, \u0111i\u1EC1u n\xE0y \u0111\u01B0\u1EE3c th\u1EF1c hi\u1EC7n v\u1EDBi ph\u01B0\u01A1ng th\u1EE9c "),Xt=l("code"),Ca=e("save_pretrained()"),Aa=e(":"),hh=o(),g(Dn.$$.fragment),eh=o(),J=l("p"),Ta=e("Thao t\xE1c n\xE0y s\u1EBD t\u1EA1o m\u1ED9t th\u01B0 m\u1EE5c m\u1EDBi c\xF3 t\xEAn "),Jt=l("em"),Da=e("code-search-net-tokenizer"),Na=e(", s\u1EBD ch\u1EE9a t\u1EA5t c\u1EA3 c\xE1c t\u1EC7p m\xE0 tokenizer c\u1EA7n \u0111\u01B0\u1EE3c t\u1EA3i l\u1EA1i. N\u1EBFu b\u1EA1n mu\u1ED1n chia s\u1EBB tokenizer n\xE0y v\u1EDBi \u0111\u1ED3ng nghi\u1EC7p v\xE0 b\u1EA1n b\xE8 c\u1EE7a m\xECnh, b\u1EA1n c\xF3 th\u1EC3 t\u1EA3i n\xF3 l\xEAn Hub b\u1EB1ng c\xE1ch \u0111\u0103ng nh\u1EADp v\xE0o t\xE0i kho\u1EA3n c\u1EE7a m\xECnh. N\u1EBFu b\u1EA1n \u0111ang l\xE0m vi\u1EC7c tr\xEAn notebook, c\xF3 m\u1ED9t h\xE0m ti\u1EC7n \xEDch gi\xFAp b\u1EA1n l\xE0m \u0111i\u1EC1u n\xE0y:"),ah=o(),g(Nn.$$.fragment),ih=o(),nt=l("p"),Oa=e("Thao t\xE1c n\xE0y s\u1EBD hi\u1EC3n th\u1ECB m\u1ED9t ti\u1EC7n \xEDch m\xE0 b\u1EA1n c\xF3 th\u1EC3 nh\u1EADp th\xF4ng tin \u0111\u0103ng nh\u1EADp Hugging Face c\u1EE7a m\xECnh. N\u1EBFu b\u1EA1n kh\xF4ng l\xE0m vi\u1EC7c trong notebook, ch\u1EC9 c\u1EA7n nh\u1EADp d\xF2ng sau v\xE0o thi\u1EBFt b\u1ECB \u0111\u1EA7u cu\u1ED1i c\u1EE7a b\u1EA1n:"),lh=o(),g(On.$$.fragment),ch=o(),tt=l("p"),La=e("Khi b\u1EA1n \u0111\xE3 \u0111\u0103ng nh\u1EADp, b\u1EA1n c\xF3 th\u1EC3 \u0111\u1EA9y tokenizer c\u1EE7a m\xECnh b\u1EB1ng c\xE1ch th\u1EF1c hi\u1EC7n l\u1EC7nh sau:"),rh=o(),g(Ln.$$.fragment),oh=o(),O=l("p"),Ha=e("Thao t\xE1c n\xE0y s\u1EBD t\u1EA1o m\u1ED9t kho l\u01B0u tr\u1EEF m\u1EDBi trong kh\xF4ng gian t\xEAn c\u1EE7a b\u1EA1n v\u1EDBi t\xEAn "),Zt=l("code"),Sa=e("code-search-net-tokenizer"),Ga=e(", ch\u1EE9a t\u1EC7p tokenizer. Sau \u0111\xF3 b\u1EA1n c\xF3 th\u1EC3 t\u1EA3i tokenizer t\u1EEB b\u1EA5t k\xEC \u0111\xE2u v\u1EDBi ph\u01B0\u01A1ng th\u1EE9c "),Wt=l("code"),Ma=e("from_pretrained()"),Ba=e(":"),ph=o(),g(Hn.$$.fragment),gh=o(),L=l("p"),Ra=e("Gi\u1EDD b\u1EA1n \u0111\xE3 s\u1EB5n s\xE0ng \u0111\u1EC3 hu\u1EA5n luy\u1EC7n m\u1ED9t m\xF4 h\xECnh ng\xF4n ng\u1EEF t\u1EEB \u0111\u1EA7u v\xE0 vi\u1EC7c tinh ch\u1EC9nh n\xF3 trong t\u1EA7m tay c\u1EE7a b\u1EA1n! Ch\xFAng ta s\u1EBD t\xECm hi\u1EC3u \u0111i\u1EC1u \u0111\xF3 trong "),st=l("a"),Ua=e("Ch\u01B0\u01A1ng 7"),Ka=e(", nh\u01B0ng tr\u01B0\u1EDBc ti\xEAn, trong ph\u1EA7n c\xF2n l\u1EA1i c\u1EE7a ch\u01B0\u01A1ng n\xE0y, ch\xFAng ta s\u1EBD xem x\xE9t k\u1EF9 h\u01A1n v\u1EC1 c\xE1c tr\xECnh tokenize nhanh v\xE0 kh\xE1m ph\xE1 chi ti\u1EBFt nh\u1EEFng g\xEC th\u1EF1c s\u1EF1 x\u1EA3y ra khi ch\xFAng ta g\u1ECDi ph\u01B0\u01A1ng th\u1EE9c "),ns=l("code"),Va=e("train_new_from_iterator()"),Ia=e("."),this.h()},l(n){const h=ml('[data-svelte="svelte-1phssyn"]',document.head);y=c(h,"META",{name:!0,content:!0}),h.forEach(t),B=p(n),j=c(n,"H1",{class:!0});var Sn=r(j);w=c(Sn,"A",{id:!0,class:!0,href:!0});var Ya=r(w);ot=c(Ya,"SPAN",{});var Fa=r(ot);u(nn.$$.fragment,Fa),Fa.forEach(t),Ya.forEach(t),jh=p(Sn),pt=c(Sn,"SPAN",{});var Qa=r(pt);zh=a(Qa,"Hu\u1EA5n luy\u1EC7n m\u1ED9t tokenizer m\u1EDBi t\u1EEB c\xE1i c\u0169"),Qa.forEach(t),Sn.forEach(t),ss=p(n),u(tn.$$.fragment,n),hs=p(n),E=c(n,"P",{});var Z=r(E);wh=a(Z,"N\u1EBFu m\xF4 h\xECnh ng\xF4n ng\u1EEF kh\xF4ng c\xF3 s\u1EB5n ng\xF4n ng\u1EEF b\u1EA1n quan t\xE2m ho\u1EB7c n\u1EBFu kho t\xE0i li\u1EC7u c\u1EE7a b\u1EA1n r\u1EA5t kh\xE1c v\u1EDBi kho m\xE0 m\xF4 h\xECnh ng\xF4n ng\u1EEF c\u1EE7a b\u1EA1n \u0111\xE3 hu\u1EA5n luy\u1EC7n, b\u1EA1n r\u1EA5t c\xF3 th\u1EC3 s\u1EBD mu\u1ED1n hu\u1EA5n luy\u1EC7n l\u1EA1i m\xF4 h\xECnh t\u1EEB \u0111\u1EA7u b\u1EB1ng c\xE1ch s\u1EED d\u1EE5ng tr\xECnh tokenize ph\xF9 h\u1EE3p v\u1EDBi d\u1EEF li\u1EC7u c\u1EE7a b\u1EA1n. \u0110i\u1EC1u \u0111\xF3 s\u1EBD y\xEAu c\u1EA7u hu\u1EA5n luy\u1EC7n m\u1ED9t tr\xECnh tokenize m\u1EDBi tr\xEAn t\u1EADp d\u1EEF li\u1EC7u c\u1EE7a b\u1EA1n. Nh\u01B0ng ch\xEDnh x\xE1c th\xEC \u0111i\u1EC1u \u0111\xF3 c\xF3 ngh\u0129a l\xE0 g\xEC? Khi ch\xFAng ta l\u1EA7n \u0111\u1EA7u xem x\xE9t c\xE1c tokenizer trong "),Gn=c(Z,"A",{href:!0});var Xa=r(Gn);Eh=a(Xa,"Ch\u01B0\u01A1ng 2"),Xa.forEach(t),qh=a(Z,", ch\xFAng ta th\u1EA5y r\u1EB1ng h\u1EA7u h\u1EBFt c\xE1c m\xF4 h\xECnh Transformer s\u1EED d\u1EE5ng thu\u1EADt to\xE1n tokenize "),gt=c(Z,"EM",{});var Ja=r(gt);Ph=a(Ja,"t\u1EEB ph\u1EE5"),Ja.forEach(t),Ch=a(Z,". \u0110\u1EC3 x\xE1c \u0111\u1ECBnh nh\u1EEFng t\u1EEB ph\u1EE5 n\xE0o \u0111\u01B0\u1EE3c quan t\xE2m v\xE0 xu\u1EA5t hi\u1EC7n th\u01B0\u1EDDng xuy\xEAn nh\u1EA5t trong kho ng\u1EEF li\u1EC7u hi\u1EC7n c\xF3, tr\xECnh tokenize c\u1EA7n ph\u1EA3i xem x\xE9t k\u1EF9 t\u1EA5t c\u1EA3 c\xE1c v\u0103n b\u1EA3n trong kho ng\u1EEF li\u1EC7u - m\u1ED9t qu\xE1 tr\xECnh m\xE0 ch\xFAng ta g\u1ECDi l\xE0 "),ut=c(Z,"EM",{});var Za=r(ut);Ah=a(Za,"hu\u1EA5n luy\u1EC7n"),Za.forEach(t),Th=a(Z,". C\xE1c quy t\u1EAFc chi ph\u1ED1i vi\u1EC7c hu\u1EA5n luy\u1EC7n n\xE0y ph\u1EE5 thu\u1ED9c v\xE0o lo\u1EA1i tokenizer \u0111\u01B0\u1EE3c s\u1EED d\u1EE5ng v\xE0 ch\xFAng ta s\u1EBD xem x\xE9t ba thu\u1EADt to\xE1n ch\xEDnh \u1EDF ph\u1EA7n sau c\u1EE7a ch\u01B0\u01A1ng n\xE0y."),Z.forEach(t),es=p(n),u(sn.$$.fragment,n),as=p(n),u(R.$$.fragment,n),is=p(n),S=c(n,"H2",{class:!0});var mh=r(S);U=c(mh,"A",{id:!0,class:!0,href:!0});var Wa=r(U);mt=c(Wa,"SPAN",{});var ni=r(mt);u(hn.$$.fragment,ni),ni.forEach(t),Wa.forEach(t),Dh=p(mh),ft=c(mh,"SPAN",{});var ti=r(ft);Nh=a(ti,"T\u1EADp h\u1EE3p m\u1ED9t kho ng\u1EEF li\u1EC7u"),ti.forEach(t),mh.forEach(t),ls=p(n),K=c(n,"P",{});var fh=r(K);Oh=a(fh,"C\xF3 m\u1ED9t API r\u1EA5t \u0111\u01A1n gi\u1EA3n trong \u{1F917} Transformers m\xE0 b\u1EA1n c\xF3 th\u1EC3 s\u1EED d\u1EE5ng \u0111\u1EC3 hu\u1EA5n luy\u1EC7n m\u1ED9t tokenizer m\u1EDBi c\xF3 c\xF9ng \u0111\u1EB7c \u0111i\u1EC3m v\u1EDBi c\xE1i hi\u1EC7n c\xF3: "),kt=c(fh,"CODE",{});var si=r(kt);Lh=a(si,"AutoTokenizer.train_new_from_iterator()"),si.forEach(t),Hh=a(fh,". \u0110\u1EC3 th\u1EA5y \u0111i\u1EC1u n\xE0y trong th\u1EF1c t\u1EBF, gi\u1EA3 s\u1EED ch\xFAng ta mu\u1ED1n hu\u1EA5n luy\u1EC7n GPT-2 t\u1EEB \u0111\u1EA7u, nh\u01B0ng b\u1EB1ng m\u1ED9t ng\xF4n ng\u1EEF kh\xE1c ngo\xE0i ti\u1EBFng Anh. Nhi\u1EC7m v\u1EE5 \u0111\u1EA7u ti\xEAn c\u1EE7a ch\xFAng ta s\u1EBD l\xE0 thu th\u1EADp nhi\u1EC1u d\u1EEF li\u1EC7u b\u1EB1ng ng\xF4n ng\u1EEF \u0111\xF3 trong m\u1ED9t kho d\u1EEF li\u1EC7u hu\u1EA5n luy\u1EC7n. \u0110\u1EC3 cung c\u1EA5p c\xE1c m\u1EABu m\xE0 m\u1ECDi ng\u01B0\u1EDDi c\xF3 hi\u1EC3u \u0111\u01B0\u1EE3c, ch\xFAng ta s\u1EBD kh\xF4ng s\u1EED d\u1EE5ng ng\xF4n ng\u1EEF nh\u01B0 ti\u1EBFng Nga ho\u1EB7c ti\u1EBFng Trung \u1EDF \u0111\xE2y, m\xE0 l\xE0 ng\xF4n ng\u1EEF ti\u1EBFng Anh chuy\xEAn d\u1EE5ng: \u0111o\u1EA1n m\xE3 Python."),fh.forEach(t),cs=p(n),z=c(n,"P",{});var H=r(z);Sh=a(H,"Th\u01B0 vi\u1EC7n "),en=c(H,"A",{href:!0,rel:!0});var hi=r(en);Gh=a(hi,"\u{1F917} Datasets"),hi.forEach(t),Mh=a(H," c\xF3 th\u1EC3 gi\xFAp ch\xFAng ta t\u1EADp h\u1EE3p m\u1ED9t kho d\u1EEF li\u1EC7u m\xE3 ngu\u1ED3n Python. Ch\xFAng ta s\u1EBD s\u1EED d\u1EE5ng h\xE0m "),dt=c(H,"CODE",{});var ei=r(dt);Bh=a(ei,"load_dataset()"),ei.forEach(t),Rh=a(H," th\xF4ng th\u01B0\u1EDDng \u0111\u1EC3 t\u1EA3i xu\u1ED1ng v\xE0 l\u01B0u v\xE0o b\u1ED9 nh\u1EDB cache c\u1EE7a t\u1EADp d\u1EEF li\u1EC7u "),an=c(H,"A",{href:!0,rel:!0});var ai=r(an);Uh=a(ai,"CodeSearchNet"),ai.forEach(t),Kh=a(H,". T\u1EADp d\u1EEF li\u1EC7u n\xE0y \u0111\u01B0\u1EE3c t\u1EA1o cho "),ln=c(H,"A",{href:!0,rel:!0});var ii=r(ln);Vh=a(ii,"th\u1EED th\xE1ch CodeSearchNet"),ii.forEach(t),Ih=a(H," v\xE0 ch\u1EE9a h\xE0ng tri\u1EC7u h\xE0m t\u1EEB c\xE1c th\u01B0 vi\u1EC7n m\xE3 ngu\u1ED3n m\u1EDF tr\xEAn GitHub b\u1EB1ng m\u1ED9t s\u1ED1 ng\xF4n ng\u1EEF l\u1EADp tr\xECnh. \u1EDE \u0111\xE2y, ch\xFAng ta s\u1EBD t\u1EA3i ph\u1EA7n Python c\u1EE7a t\u1EADp d\u1EEF li\u1EC7u n\xE0y:"),H.forEach(t),rs=p(n),u(cn.$$.fragment,n),os=p(n),Mn=c(n,"P",{});var li=r(Mn);Yh=a(li,"Ch\xFAng ta c\xF3 th\u1EC3 xem x\xE9t ph\u1EA7n t\xE1ch hu\u1EA5n luy\u1EC7n \u0111\u1EC3 xem ta c\xF3 quy\u1EC1n truy c\u1EADp v\xE0o nh\u1EEFng c\u1ED9t n\xE0o:"),li.forEach(t),ps=p(n),u(rn.$$.fragment,n),gs=p(n),u(on.$$.fragment,n),us=p(n),C=c(n,"P",{});var ht=r(C);Fh=a(ht,"Ch\xFAng ta c\xF3 th\u1EC3 th\u1EA5y t\u1EADp d\u1EEF li\u1EC7u t\xE1ch chu\u1ED7i t\xE0i li\u1EC7u m\xF4 t\u1EA3 kh\u1ECFi \u0111o\u1EA1n m\xE3 v\xE0 \u0111\u1EC1 xu\u1EA5t tokenize c\u1EA3 hai. \u1EDE \u0111\xE2y, ch\xFAng ta s\u1EBD ch\u1EC9 s\u1EED d\u1EE5ng c\u1ED9t "),_t=c(ht,"CODE",{});var ci=r(_t);Qh=a(ci,"whole_func_string"),ci.forEach(t),Xh=a(ht," \u0111\u1EC3 hu\u1EA5n luy\u1EC7n tr\xECnh tokenize. Ch\xFAng ta c\xF3 th\u1EC3 xem x\xE9t m\u1EABu m\u1ED9t trong nh\u1EEFng h\xE0m n\xE0y b\u1EB1ng c\xE1ch l\u1EADp ch\u1EC9 m\u1EE5c v\xE0o ph\u1EA7n "),xt=c(ht,"CODE",{});var ri=r(xt);Jh=a(ri,"train"),ri.forEach(t),Zh=a(ht,":"),ht.forEach(t),ms=p(n),u(pn.$$.fragment,n),fs=p(n),Bn=c(n,"P",{});var oi=r(Bn);Wh=a(oi,"n\xF3 n\xEAn tr\u1EA3 v\u1EC1 k\u1EBFt qu\u1EA3 nh\u01B0 d\u01B0\u1EDBi \u0111\xE2y:"),oi.forEach(t),ks=p(n),u(gn.$$.fragment,n),ds=p(n),V=c(n,"P",{});var kh=r(V);ne=a(kh,"\u0110i\u1EC1u \u0111\u1EA7u ti\xEAn ch\xFAng ta c\u1EA7n l\xE0m l\xE0 chuy\u1EC3n \u0111\u1ED5i t\u1EADp d\u1EEF li\u1EC7u th\xE0nh m\u1ED9t "),bt=c(kh,"EM",{});var pi=r(bt);te=a(pi,"iterator"),pi.forEach(t),se=a(kh," danh s\xE1ch c\xE1c v\u0103n b\u1EA3n - v\xED d\u1EE5, m\u1ED9t danh s\xE1ch c\xE1c v\u0103n b\u1EA3n. Vi\u1EC7c s\u1EED d\u1EE5ng danh s\xE1ch v\u0103n b\u1EA3n s\u1EBD cho ph\xE9p tokenizer ho\u1EA1t \u0111\u1ED9ng nhanh h\u01A1n (hu\u1EA5n luy\u1EC7n h\xE0ng lo\u1EA1t v\u0103n b\u1EA3n thay v\xEC x\u1EED l\xFD t\u1EEBng v\u0103n b\u1EA3n ri\xEAng l\u1EBB) v\xE0 n\xF3 ph\u1EA3i l\xE0 m\u1ED9t tr\xECnh l\u1EB7p n\u1EBFu ch\xFAng ta mu\u1ED1n tr\xE1nh c\xF3 m\u1ECDi th\u1EE9 trong b\u1ED9 nh\u1EDB c\xF9ng m\u1ED9t l\xFAc. N\u1EBFu kho d\u1EEF li\u1EC7u c\u1EE7a b\u1EA1n l\u1EDBn, b\u1EA1n s\u1EBD mu\u1ED1n t\u1EADn d\u1EE5ng l\u1EE3i th\u1EBF th\u1EF1c ti\u1EC5n l\xE0 \u{1F917} Datasets kh\xF4ng t\u1EA3i m\u1ECDi th\u1EE9 v\xE0o RAM m\xE0 l\u01B0u tr\u1EEF c\xE1c ph\u1EA7n t\u1EED c\u1EE7a t\u1EADp d\u1EEF li\u1EC7u tr\xEAn \u0111\u0129a."),kh.forEach(t),_s=p(n),Rn=c(n,"P",{});var gi=r(Rn);he=a(gi,"L\xE0m nh\u01B0 sau s\u1EBD t\u1EA1o m\u1ED9t danh s\xE1ch c\xE1c danh s\xE1ch v\u1EDBi m\u1ED7i danh s\xE1ch g\u1ED3m 1,000 v\u0103n b\u1EA3n, nh\u01B0ng s\u1EBD t\u1EA3i m\u1ECDi th\u1EE9 v\xE0o b\u1ED9 nh\u1EDB:"),gi.forEach(t),xs=p(n),u(un.$$.fragment,n),bs=p(n),Un=c(n,"P",{});var ui=r(Un);ee=a(ui,"S\u1EED d\u1EE5ng tr\xECnh t\u1EA1o Python, ch\xFAng ta c\xF3 th\u1EC3 tr\xE1nh vi\u1EC7c Python t\u1EA3i b\u1EA5t k\u1EF3 th\u1EE9 g\xEC v\xE0o b\u1ED9 nh\u1EDB cho \u0111\u1EBFn khi n\xF3 th\u1EF1c s\u1EF1 c\u1EA7n thi\u1EBFt. \u0110\u1EC3 t\u1EA1o m\u1ED9t tr\xECnh t\u1EA1o nh\u01B0 v\u1EADy, b\u1EA1n ch\u1EC9 c\u1EA7n thay d\u1EA5u ngo\u1EB7c vu\xF4ng b\u1EB1ng d\u1EA5u ngo\u1EB7c \u0111\u01A1n:"),ui.forEach(t),vs=p(n),u(mn.$$.fragment,n),ys=p(n),A=c(n,"P",{});var et=r(A);ae=a(et,"D\xF2ng m\xE3 n\xE0y kh\xF4ng t\xECm n\u1EA1p b\u1EA5t k\u1EF3 ph\u1EA7n t\u1EED n\xE0o c\u1EE7a t\u1EADp d\u1EEF li\u1EC7u; n\xF3 ch\u1EC9 t\u1EA1o m\u1ED9t \u0111\u1ED1i t\u01B0\u1EE3ng m\xE0 b\u1EA1n c\xF3 th\u1EC3 s\u1EED d\u1EE5ng trong v\xF2ng l\u1EB7p Python "),vt=c(et,"CODE",{});var mi=r(vt);ie=a(mi,"for"),mi.forEach(t),le=a(et,". C\xE1c v\u0103n b\u1EA3n s\u1EBD ch\u1EC9 \u0111\u01B0\u1EE3c t\u1EA3i khi b\u1EA1n c\u1EA7n (ngh\u0129a l\xE0 khi b\u1EA1n \u0111ang \u1EDF b\u01B0\u1EDBc c\u1EE7a v\xF2ng l\u1EB7p "),yt=c(et,"CODE",{});var fi=r(yt);ce=a(fi,"for"),fi.forEach(t),re=a(et," m\xE0 y\xEAu c\u1EA7u ch\xFAng) v\xE0 ch\u1EC9 1,000 v\u0103n b\u1EA3n s\u1EBD \u0111\u01B0\u1EE3c t\u1EA3i m\u1ED7i l\u1EA7n. B\u1EB1ng c\xE1ch n\xE0y, b\u1EA1n s\u1EBD kh\xF4ng s\u1EED d\u1EE5ng h\u1EBFt b\u1ED9 nh\u1EDB c\u1EE7a m\xECnh ngay c\u1EA3 khi b\u1EA1n \u0111ang x\u1EED l\xFD m\u1ED9t t\u1EADp d\u1EEF li\u1EC7u l\u1EDBn."),et.forEach(t),$s=p(n),Kn=c(n,"P",{});var ki=r(Kn);oe=a(ki,"V\u1EA5n \u0111\u1EC1 v\u1EDBi m\u1ED9t \u0111\u1ED1i t\u01B0\u1EE3ng t\u1EA1o l\xE0 n\xF3 ch\u1EC9 c\xF3 th\u1EC3 \u0111\u01B0\u1EE3c s\u1EED d\u1EE5ng m\u1ED9t l\u1EA7n. V\xEC v\u1EADy, thay v\xEC \u0111i\u1EC1u n\xE0y cho ta danh s\xE1ch 10 ch\u1EEF s\u1ED1 \u0111\u1EA7u ti\xEAn hai l\u1EA7n:"),ki.forEach(t),js=p(n),u(fn.$$.fragment,n),zs=p(n),Vn=c(n,"P",{});var di=r(Vn);pe=a(di,"ch\xFAng ta c\xF3 th\u1EC3 l\u1EA5y ch\xFAng trong m\u1ED9t l\u1EA7n v\xE0 sau \u0111\xF3 danh s\xE1ng s\u1EBD tr\u1ED1ng:"),di.forEach(t),ws=p(n),u(kn.$$.fragment,n),Es=p(n),In=c(n,"P",{});var _i=r(In);ge=a(_i,"\u0110\xF3 l\xE0 l\xED do ch\xFAng ta \u0111\u1ECBnh ngh\u0129a m\u1ED9t h\xE0m thay v\xE0o \u0111\xF3 tr\u1EA3 v\u1EC1 m\u1ED9t tr\xECnh t\u1EA1o:"),_i.forEach(t),qs=p(n),u(dn.$$.fragment,n),Ps=p(n),T=c(n,"P",{});var at=r(T);ue=a(at,"Ta c\xF3 th\u1EC3 \u0111\u1ECBnh ngh\u0129a tr\xECnh t\u1EA1o b\xEAn trong v\xF2ng l\u1EB7p "),$t=c(at,"CODE",{});var xi=r($t);me=a(xi,"for"),xi.forEach(t),fe=a(at," s\u1EED d\u1EE5ng "),jt=c(at,"CODE",{});var bi=r(jt);ke=a(bi,"yield"),bi.forEach(t),de=a(at,":"),at.forEach(t),Cs=p(n),u(_n.$$.fragment,n),As=p(n),Yn=c(n,"P",{});var vi=r(Yn);_e=a(vi,"s\u1EBD t\u1EA1o ra tr\xECnh t\u1EA1o ho\xE0n to\xE0n gi\u1ED1ng nh\u01B0 tr\u01B0\u1EDBc \u0111\xE2y, nh\u01B0ng cho ph\xE9p b\u1EA1n s\u1EED d\u1EE5ng logic ph\u1EE9c t\u1EA1p h\u01A1n b\u1EA1n c\xF3 th\u1EC3 trong m\u1ED9t bao h\xE0m."),vi.forEach(t),Ts=p(n),G=c(n,"H2",{class:!0});var dh=r(G);I=c(dh,"A",{id:!0,class:!0,href:!0});var yi=r(I);zt=c(yi,"SPAN",{});var $i=r(zt);u(xn.$$.fragment,$i),$i.forEach(t),yi.forEach(t),xe=p(dh),wt=c(dh,"SPAN",{});var ji=r(wt);be=a(ji,"Hu\u1EA5n luy\u1EC7n m\u1ED9t tokenizer m\u1EDBi"),ji.forEach(t),dh.forEach(t),Ds=p(n),Fn=c(n,"P",{});var zi=r(Fn);ve=a(zi,"B\xE2y gi\u1EDD ch\xFAng ta \u0111\xE3 c\xF3 kho v\u0103n b\u1EA3n c\u1EE7a m\xECnh d\u01B0\u1EDBi d\u1EA1ng m\u1ED9t tr\xECnh l\u1EB7p c\xE1c lo\u1EA1t v\u0103n b\u1EA3n, ch\xFAng ta \u0111\xE3 s\u1EB5n s\xE0ng \u0111\u1EC3 hu\u1EA5n luy\u1EC7n m\u1ED9t tr\xECnh tokenize m\u1EDBi. \u0110\u1EC3 th\u1EF1c hi\u1EC7n vi\u1EC7c n\xE0y, tr\u01B0\u1EDBc ti\xEAn ch\xFAng ta c\u1EA7n t\u1EA3i tokenizer m\xE0 ch\xFAng ta mu\u1ED1n gh\xE9p n\u1ED1i v\u1EDBi m\xF4 h\xECnh c\u1EE7a m\xECnh (\u1EDF \u0111\xE2y, GPT-2):"),zi.forEach(t),Ns=p(n),u(bn.$$.fragment,n),Os=p(n),Qn=c(n,"P",{});var wi=r(Qn);ye=a(wi,"M\u1EB7c d\xF9 ch\xFAng ta s\u1EBD hu\u1EA5n luy\u1EC7n m\u1ED9t tokenizer, nh\u01B0ng b\u1EA1n n\xEAn l\xE0m \u0111i\u1EC1u n\xE0y \u0111\u1EC3 tr\xE1nh b\u1EAFt \u0111\u1EA7u ho\xE0n to\xE0n t\u1EEB \u0111\u1EA7u. B\u1EB1ng c\xE1ch n\xE0y, ch\xFAng ta s\u1EBD kh\xF4ng ph\u1EA3i ch\u1EC9 \u0111\u1ECBnh b\u1EA5t k\u1EF3 \u0111i\u1EC1u g\xEC v\u1EC1 thu\u1EADt to\xE1n tokenize ho\u1EB7c c\xE1c token \u0111\u1EB7c bi\u1EC7t m\xE0 ta mu\u1ED1n s\u1EED d\u1EE5ng; tokenizer m\u1EDBi s\u1EBD gi\u1ED1ng h\u1EC7t nh\u01B0 GPT-2 v\xE0 \u0111i\u1EC1u duy nh\u1EA5t s\u1EBD thay \u0111\u1ED5i l\xE0 t\u1EEB v\u1EF1ng, s\u1EBD \u0111\u01B0\u1EE3c x\xE1c \u0111\u1ECBnh b\u1EDFi qu\xE1 tr\xECnh hu\u1EA5n luy\u1EC7n tr\xEAn kho ng\u1EEF li\u1EC7u c\u1EE7a ch\xFAng t\xF4i."),wi.forEach(t),Ls=p(n),Xn=c(n,"P",{});var Ei=r(Xn);$e=a(Ei,"\u0110\u1EA7u ti\xEAn, ch\xFAng ta h\xE3y xem c\xE1ch m\xE0 tokenizer n\xE0y s\u1EBD x\u1EED l\xFD m\u1ED9t h\xE0m m\u1EABu th\u1EBF n\xE0o:"),Ei.forEach(t),Hs=p(n),u(vn.$$.fragment,n),Ss=p(n),u(yn.$$.fragment,n),Gs=p(n),q=c(n,"P",{});var W=r(q);je=a(W,"Tokenizer n\xE0y c\xF3 m\u1ED9t s\u1ED1 k\xFD hi\u1EC7u \u0111\u1EB7c bi\u1EC7t, nh\u01B0 "),Et=c(W,"CODE",{});var qi=r(Et);ze=a(qi,"\u0120"),qi.forEach(t),we=a(W," v\xE0 "),qt=c(W,"CODE",{});var Pi=r(qt);Ee=a(Pi,"\u010A"),Pi.forEach(t),qe=a(W,", t\u01B0\u01A1ng \u1EE9ng bi\u1EC3u th\u1ECB d\u1EA5u c\xE1ch v\xE0 d\xF2ng m\u1EDBi. Nh\u01B0 ch\xFAng ta c\xF3 th\u1EC3 th\u1EA5y, \u0111i\u1EC1u n\xE0y kh\xF4ng qu\xE1 hi\u1EC7u qu\u1EA3: tokenizer tr\u1EA3 v\u1EC1 c\xE1c m\xE3 th\xF4ng b\xE1o ri\xEAng l\u1EBB cho t\u1EEBng kho\u1EA3ng tr\u1EAFng, khi n\xF3 c\xF3 th\u1EC3 nh\xF3m c\xE1c m\u1EE9c th\u1EE5t l\u1EC1 l\u1EA1i v\u1EDBi nhau (v\xEC c\xF3 b\u1ED9 b\u1ED1n ho\u1EB7c t\xE1m d\u1EA5u c\xE1ch s\u1EBD r\u1EA5t ph\u1ED5 bi\u1EBFn trong m\xE3). N\xF3 c\u0169ng t\xE1ch t\xEAn h\xE0m h\u01A1i k\u1EF3 l\u1EA1, nh\xECn kh\xF4ng quen c\xE1c t\u1EEB c\xF3 k\xFD t\u1EF1 "),Pt=c(W,"CODE",{});var Ci=r(Pt);Pe=a(Ci,"_"),Ci.forEach(t),Ce=a(W,"."),W.forEach(t),Ms=p(n),Y=c(n,"P",{});var _h=r(Y);Ae=a(_h,"H\xE3y hu\u1EA5n luy\u1EC7n m\u1ED9t tokenizer m\u1EDBi v\xE0 xem li\u1EC7u n\xF3 c\xF3 gi\u1EA3i quy\u1EBFt \u0111\u01B0\u1EE3c nh\u1EEFng v\u1EA5n \u0111\u1EC1 \u0111\xF3 kh\xF4ng. \u0110\u1ED1i v\u1EDBi \u0111i\u1EC1u n\xE0y, ch\xFAng ta s\u1EBD s\u1EED d\u1EE5ng ph\u01B0\u01A1ng th\u1EE9c "),Ct=c(_h,"CODE",{});var Ai=r(Ct);Te=a(Ai,"train_new_from_iterator()"),Ai.forEach(t),De=a(_h,":"),_h.forEach(t),Bs=p(n),u($n.$$.fragment,n),Rs=p(n),Jn=c(n,"P",{});var Ti=r(Jn);Ne=a(Ti,"L\u1EC7nh n\xE0y c\xF3 th\u1EC3 m\u1EA5t m\u1ED9t ch\xFAt th\u1EDDi gian n\u1EBFu kho d\u1EEF li\u1EC7u c\u1EE7a b\u1EA1n r\u1EA5t l\u1EDBn, nh\u01B0ng \u0111\u1ED1i v\u1EDBi t\u1EADp d\u1EEF li\u1EC7u 1.6GB v\u0103n b\u1EA3n n\xE0y, n\xF3 r\u1EA5t nhanh (1 ph\xFAt 16 gi\xE2y tr\xEAn CPU AMD Ryzen 9 3900X v\u1EDBi 12 l\xF5i)."),Ti.forEach(t),Us=p(n),D=c(n,"P",{});var it=r(D);Oe=a(it,"L\u01B0u \xFD r\u1EB1ng "),At=c(it,"CODE",{});var Di=r(At);Le=a(Di,"AutoTokenizer.train_new_from_iterator()"),Di.forEach(t),He=a(it," ch\u1EC9 ho\u1EA1t \u0111\u1ED9ng n\u1EBFu tokenizer b\u1EA1n \u0111ang s\u1EED d\u1EE5ng l\xE0 tokenizer \u201Cnhanh\u201D. Nh\u01B0 b\u1EA1n s\u1EBD th\u1EA5y trong ph\u1EA7n ti\u1EBFp theo, th\u01B0 vi\u1EC7n \u{1F917} Transformers ch\u1EE9a hai lo\u1EA1i tokenizers: m\u1ED9t s\u1ED1 \u0111\u01B0\u1EE3c vi\u1EBFt ho\xE0n to\xE0n b\u1EB1ng Python v\xE0 nh\u1EEFng lo\u1EA1i kh\xE1c (lo\u1EA1i nhanh) \u0111\u01B0\u1EE3c h\u1ED7 tr\u1EE3 b\u1EDFi th\u01B0 vi\u1EC7n \u{1F917} Tokenizers, \u0111\u01B0\u1EE3c vi\u1EBFt b\u1EB1ng ng\xF4n ng\u1EEF l\u1EADp tr\xECnh "),jn=c(it,"A",{href:!0,rel:!0});var Ni=r(jn);Se=a(Ni,"Rust"),Ni.forEach(t),Ge=a(it,". Python l\xE0 ng\xF4n ng\u1EEF th\u01B0\u1EDDng \u0111\u01B0\u1EE3c s\u1EED d\u1EE5ng nh\u1EA5t cho c\xE1c \u1EE9ng d\u1EE5ng khoa h\u1ECDc d\u1EEF li\u1EC7u v\xE0 h\u1ECDc s\xE2u, nh\u01B0ng khi b\u1EA5t k\u1EF3 th\u1EE9 g\xEC c\u1EA7n \u0111\u01B0\u1EE3c song song h\xF3a cho nhanh, n\xF3 ph\u1EA3i \u0111\u01B0\u1EE3c vi\u1EBFt b\u1EB1ng m\u1ED9t ng\xF4n ng\u1EEF kh\xE1c. V\xED d\u1EE5, c\xE1c ph\xE9p nh\xE2n ma tr\u1EADn l\xE0 c\u1ED1t l\xF5i c\u1EE7a t\xEDnh to\xE1n m\xF4 h\xECnh \u0111\u01B0\u1EE3c vi\u1EBFt b\u1EB1ng CUDA, m\u1ED9t th\u01B0 vi\u1EC7n C \u0111\u01B0\u1EE3c t\u1ED1i \u01B0u h\xF3a cho GPU."),it.forEach(t),Ks=p(n),F=c(n,"P",{});var xh=r(F);Me=a(xh,"Vi\u1EC7c hu\u1EA5n luy\u1EC7n m\u1ED9t tokenizer ho\xE0n to\xE0n m\u1EDBi b\u1EB1ng Python thu\u1EA7n t\xFAy s\u1EBD r\u1EA5t ch\u1EADm, \u0111\xF3 l\xE0 l\xFD do t\u1EA1i sao ch\xFAng t\xF4i \u0111\xE3 ph\xE1t tri\u1EC3n th\u01B0 vi\u1EC7n \u{1F917} Tokenizer. L\u01B0u \xFD r\u1EB1ng c\u0169ng gi\u1ED1ng nh\u01B0 b\u1EA1n kh\xF4ng ph\u1EA3i h\u1ECDc ng\xF4n ng\u1EEF CUDA \u0111\u1EC3 c\xF3 th\u1EC3 th\u1EF1c thi m\xF4 h\xECnh c\u1EE7a m\xECnh tr\xEAn m\u1ED9t lo\u1EA1t \u0111\u1EA7u v\xE0o tr\xEAn GPU, b\u1EA1n s\u1EBD kh\xF4ng c\u1EA7n ph\u1EA3i h\u1ECDc Rust \u0111\u1EC3 s\u1EED d\u1EE5ng tr\xECnh tokenizer nhanh. Th\u01B0 vi\u1EC7n \u{1F917} Tokenizers cung c\u1EA5p c\xE1c li\xEAn k\u1EBFt Python cho nhi\u1EC1u ph\u01B0\u01A1ng th\u1EE9c g\u1ECDi n\u1ED9i b\u1ED9 m\u1ED9t s\u1ED1 \u0111o\u1EA1n m\xE3 trong Rust; v\xED d\u1EE5: \u0111\u1EC3 song song hu\u1EA5n luy\u1EC7n tr\xECnh tokenize m\u1EDBi c\u1EE7a b\u1EA1n ho\u1EB7c, nh\u01B0 ch\xFAng ta \u0111\xE3 th\u1EA5y trong "),Zn=c(xh,"A",{href:!0});var Oi=r(Zn);Be=a(Oi,"Ch\u01B0\u01A1ng 3"),Oi.forEach(t),Re=a(xh,", tokenize m\u1ED9t lo\u1EA1t \u0111\u1EA7u v\xE0o."),xh.forEach(t),Vs=p(n),N=c(n,"P",{});var lt=r(N);Ue=a(lt,"H\u1EA7u h\u1EBFt c\xE1c m\xF4 h\xECnh Transformer \u0111\u1EC1u c\xF3 s\u1EB5n c\xF4ng c\u1EE5 tokenize nhanh (c\xF3 m\u1ED9t s\u1ED1 ngo\u1EA1i l\u1EC7 m\xE0 b\u1EA1n c\xF3 th\u1EC3 ki\u1EC3m tra "),zn=c(lt,"A",{href:!0,rel:!0});var Li=r(zn);Ke=a(Li,"t\u1EA1i \u0111\xE2y"),Li.forEach(t),Ve=a(lt,") v\xE0 API "),Tt=c(lt,"CODE",{});var Hi=r(Tt);Ie=a(Hi,"AutoTokenizer"),Hi.forEach(t),Ye=a(lt," lu\xF4n ch\u1ECDn t\u1ED1c tokenizer nhanh cho b\u1EA1n n\u1EBFu n\xF3 c\xF3 s\u1EB5n. Trong ph\u1EA7n ti\u1EBFp theo, ch\xFAng ta s\u1EBD xem x\xE9t m\u1ED9t s\u1ED1 t\xEDnh n\u0103ng \u0111\u1EB7c bi\u1EC7t kh\xE1c m\xE0 c\xE1c tokenize nhanh c\xF3 m\xE0 th\u1EF1c s\u1EF1 h\u1EEFu \xEDch cho c\xE1c t\xE1c v\u1EE5 nh\u01B0 ph\xE2n lo\u1EA1i token v\xE0 h\u1ECFi \u0111\xE1p. Tuy nhi\xEAn, tr\u01B0\u1EDBc khi \u0111i s\xE2u v\xE0o v\u1EA5n \u0111\u1EC1 \u0111\xF3, ch\xFAng ta h\xE3y th\u1EED tokenizer ho\xE0n to\xE0n m\u1EDBi c\u1EE7a ch\xFAng ta tr\xEAn m\u1EABu tr\u01B0\u1EDBc:"),lt.forEach(t),Is=p(n),u(wn.$$.fragment,n),Ys=p(n),u(En.$$.fragment,n),Fs=p(n),$=c(n,"P",{});var P=r($);Fe=a(P,"\u1EDE \u0111\xE2y ch\xFAng ta l\u1EA1i th\u1EA5y c\xE1c k\xFD hi\u1EC7u \u0111\u1EB7c bi\u1EC7t "),Dt=c(P,"CODE",{});var Si=r(Dt);Qe=a(Si,"\u0120"),Si.forEach(t),Xe=a(P," v\xE0 "),Nt=c(P,"CODE",{});var Gi=r(Nt);Je=a(Gi,"\u010A"),Gi.forEach(t),Ze=a(P," bi\u1EC3u th\u1ECB d\u1EA5u c\xE1ch v\xE0 d\xF2ng m\u1EDBi, nh\u01B0ng ch\xFAng ta c\u0169ng c\xF3 th\u1EC3 th\u1EA5y r\u1EB1ng tr\xECnh tokenize \u0111\xE3 h\u1ECDc \u0111\u01B0\u1EE3c m\u1ED9t s\u1ED1 token r\u1EA5t c\u1EE5 th\u1EC3 cho m\u1ED9t kho c\xE1c h\xE0m Python: v\xED d\u1EE5: c\xF3 m\u1ED9t token "),Ot=c(P,"CODE",{});var Mi=r(Ot);We=a(Mi,"\u010A\u0120\u0120\u0120"),Mi.forEach(t),na=a(P," \u0111\u1EA1i di\u1EC7n cho m\u1ED9t th\u1EE5t l\u1EC1 v\xE0 token "),Lt=c(P,"CODE",{});var Bi=r(Lt);ta=a(Bi,'\u0120"""'),Bi.forEach(t),sa=a(P," \u0111\u1EA1i di\u1EC7n cho ba d\u1EA5u ngo\u1EB7c k\xE9p b\u1EAFt \u0111\u1EA7u m\u1ED9t chu\u1ED7i t\xE0i li\u1EC7u. Tokenizer c\u0169ng ph\xE2n chia ch\xEDnh x\xE1c t\xEAn h\xE0m tr\xEAn "),Ht=c(P,"CODE",{});var Ri=r(Ht);ha=a(Ri,"_"),Ri.forEach(t),ea=a(P,". \u0110\xE2y l\xE0 m\u1ED9t bi\u1EC5u di\u1EC5n kh\xE1 nh\u1ECF g\u1ECDn; t\u01B0\u01A1ng \u0111\u1ED1i, s\u1EED d\u1EE5ng tokenizer \u0111\u01A1n gi\u1EA3n b\u1EB1ng ti\u1EBFng Anh tr\xEAn c\xF9ng m\u1ED9t m\u1EABu s\u1EBD cho ta m\u1ED9t c\xE2u d\xE0i h\u01A1n:"),P.forEach(t),Qs=p(n),u(qn.$$.fragment,n),Xs=p(n),u(Pn.$$.fragment,n),Js=p(n),Wn=c(n,"P",{});var Ui=r(Wn);aa=a(Ui,"H\xE3y c\xF9ng nh\xECn v\xE0o v\xED d\u1EE5 sau:"),Ui.forEach(t),Zs=p(n),u(Cn.$$.fragment,n),Ws=p(n),u(An.$$.fragment,n),nh=p(n),x=c(n,"P",{});var v=r(x);ia=a(v,"Ngo\xE0i token t\u01B0\u01A1ng \u1EE9ng v\u1EDBi th\u1EE5t l\u1EC1, \u1EDF \u0111\xE2y ch\xFAng ta c\u0169ng c\xF3 th\u1EC3 th\u1EA5y token cho th\u1EE5t l\u1EC1 k\xE9p:"),St=c(v,"CODE",{});var Ki=r(St);la=a(Ki,"\u010A\u0120\u0120\u0120\u0120\u0120\u0120\u0120"),Ki.forEach(t),ca=a(v,". C\xE1c t\u1EEB \u0111\u1EB7c bi\u1EC7t trong Python nh\u01B0 "),Gt=c(v,"CODE",{});var Vi=r(Gt);ra=a(Vi,"class"),Vi.forEach(t),oa=a(v,", "),Mt=c(v,"CODE",{});var Ii=r(Mt);pa=a(Ii,"init"),Ii.forEach(t),ga=a(v,", "),Bt=c(v,"CODE",{});var Yi=r(Bt);ua=a(Yi,"call"),Yi.forEach(t),ma=a(v,", "),Rt=c(v,"CODE",{});var Fi=r(Rt);fa=a(Fi,"self"),Fi.forEach(t),ka=a(v,", v\xE0 "),Ut=c(v,"CODE",{});var Qi=r(Ut);da=a(Qi,"return"),Qi.forEach(t),_a=a(v,", m\u1ED7i t\u1EEB \u0111\u01B0\u1EE3c tokenize th\xE0nh m\u1ED9t token v\xE0 ch\xFAng ta c\xF3 th\u1EC3 th\u1EA5y c\u0169ng nh\u01B0 t\xE1ch "),Kt=c(v,"CODE",{});var Xi=r(Kt);xa=a(Xi,"_"),Xi.forEach(t),ba=a(v,"  v\xE0 "),Vt=c(v,"CODE",{});var Ji=r(Vt);va=a(Ji,"."),Ji.forEach(t),ya=a(v,",  tokenizer ph\xE2n chia ch\xEDnh x\xE1c c\xE1c t\xEAn: "),It=c(v,"CODE",{});var Zi=r(It);$a=a(Zi,"LinearLayer"),Zi.forEach(t),ja=a(v," \u0111\u01B0\u1EE3c tokenize l\xE0 "),Yt=c(v,"CODE",{});var Wi=r(Yt);za=a(Wi,'["\u0120Linear", "Layer"]'),Wi.forEach(t),wa=a(v,"."),v.forEach(t),th=p(n),M=c(n,"H2",{class:!0});var bh=r(M);Q=c(bh,"A",{id:!0,class:!0,href:!0});var nl=r(Q);Ft=c(nl,"SPAN",{});var tl=r(Ft);u(Tn.$$.fragment,tl),tl.forEach(t),nl.forEach(t),Ea=p(bh),Qt=c(bh,"SPAN",{});var sl=r(Qt);qa=a(sl,"L\u01B0u tokenizer"),sl.forEach(t),bh.forEach(t),sh=p(n),X=c(n,"P",{});var vh=r(X);Pa=a(vh,"\u0110\u1EC3 \u0111\u1EA3m b\u1EA3o r\u1EB1ng ch\xFAng ta c\xF3 th\u1EC3 s\u1EED d\u1EE5ng n\xF3 sau n\xE0y, ch\xFAng ta c\u1EA7n ph\u1EA3i l\u01B0u tokenizer m\u1EDBi c\u1EE7a m\xECnh. Gi\u1ED1ng nh\u01B0 \u0111\u1ED1i v\u1EDBi c\xE1c m\xF4 h\xECnh, \u0111i\u1EC1u n\xE0y \u0111\u01B0\u1EE3c th\u1EF1c hi\u1EC7n v\u1EDBi ph\u01B0\u01A1ng th\u1EE9c "),Xt=c(vh,"CODE",{});var hl=r(Xt);Ca=a(hl,"save_pretrained()"),hl.forEach(t),Aa=a(vh,":"),vh.forEach(t),hh=p(n),u(Dn.$$.fragment,n),eh=p(n),J=c(n,"P",{});var yh=r(J);Ta=a(yh,"Thao t\xE1c n\xE0y s\u1EBD t\u1EA1o m\u1ED9t th\u01B0 m\u1EE5c m\u1EDBi c\xF3 t\xEAn "),Jt=c(yh,"EM",{});var el=r(Jt);Da=a(el,"code-search-net-tokenizer"),el.forEach(t),Na=a(yh,", s\u1EBD ch\u1EE9a t\u1EA5t c\u1EA3 c\xE1c t\u1EC7p m\xE0 tokenizer c\u1EA7n \u0111\u01B0\u1EE3c t\u1EA3i l\u1EA1i. N\u1EBFu b\u1EA1n mu\u1ED1n chia s\u1EBB tokenizer n\xE0y v\u1EDBi \u0111\u1ED3ng nghi\u1EC7p v\xE0 b\u1EA1n b\xE8 c\u1EE7a m\xECnh, b\u1EA1n c\xF3 th\u1EC3 t\u1EA3i n\xF3 l\xEAn Hub b\u1EB1ng c\xE1ch \u0111\u0103ng nh\u1EADp v\xE0o t\xE0i kho\u1EA3n c\u1EE7a m\xECnh. N\u1EBFu b\u1EA1n \u0111ang l\xE0m vi\u1EC7c tr\xEAn notebook, c\xF3 m\u1ED9t h\xE0m ti\u1EC7n \xEDch gi\xFAp b\u1EA1n l\xE0m \u0111i\u1EC1u n\xE0y:"),yh.forEach(t),ah=p(n),u(Nn.$$.fragment,n),ih=p(n),nt=c(n,"P",{});var al=r(nt);Oa=a(al,"Thao t\xE1c n\xE0y s\u1EBD hi\u1EC3n th\u1ECB m\u1ED9t ti\u1EC7n \xEDch m\xE0 b\u1EA1n c\xF3 th\u1EC3 nh\u1EADp th\xF4ng tin \u0111\u0103ng nh\u1EADp Hugging Face c\u1EE7a m\xECnh. N\u1EBFu b\u1EA1n kh\xF4ng l\xE0m vi\u1EC7c trong notebook, ch\u1EC9 c\u1EA7n nh\u1EADp d\xF2ng sau v\xE0o thi\u1EBFt b\u1ECB \u0111\u1EA7u cu\u1ED1i c\u1EE7a b\u1EA1n:"),al.forEach(t),lh=p(n),u(On.$$.fragment,n),ch=p(n),tt=c(n,"P",{});var il=r(tt);La=a(il,"Khi b\u1EA1n \u0111\xE3 \u0111\u0103ng nh\u1EADp, b\u1EA1n c\xF3 th\u1EC3 \u0111\u1EA9y tokenizer c\u1EE7a m\xECnh b\u1EB1ng c\xE1ch th\u1EF1c hi\u1EC7n l\u1EC7nh sau:"),il.forEach(t),rh=p(n),u(Ln.$$.fragment,n),oh=p(n),O=c(n,"P",{});var ct=r(O);Ha=a(ct,"Thao t\xE1c n\xE0y s\u1EBD t\u1EA1o m\u1ED9t kho l\u01B0u tr\u1EEF m\u1EDBi trong kh\xF4ng gian t\xEAn c\u1EE7a b\u1EA1n v\u1EDBi t\xEAn "),Zt=c(ct,"CODE",{});var ll=r(Zt);Sa=a(ll,"code-search-net-tokenizer"),ll.forEach(t),Ga=a(ct,", ch\u1EE9a t\u1EC7p tokenizer. Sau \u0111\xF3 b\u1EA1n c\xF3 th\u1EC3 t\u1EA3i tokenizer t\u1EEB b\u1EA5t k\xEC \u0111\xE2u v\u1EDBi ph\u01B0\u01A1ng th\u1EE9c "),Wt=c(ct,"CODE",{});var cl=r(Wt);Ma=a(cl,"from_pretrained()"),cl.forEach(t),Ba=a(ct,":"),ct.forEach(t),ph=p(n),u(Hn.$$.fragment,n),gh=p(n),L=c(n,"P",{});var rt=r(L);Ra=a(rt,"Gi\u1EDD b\u1EA1n \u0111\xE3 s\u1EB5n s\xE0ng \u0111\u1EC3 hu\u1EA5n luy\u1EC7n m\u1ED9t m\xF4 h\xECnh ng\xF4n ng\u1EEF t\u1EEB \u0111\u1EA7u v\xE0 vi\u1EC7c tinh ch\u1EC9nh n\xF3 trong t\u1EA7m tay c\u1EE7a b\u1EA1n! Ch\xFAng ta s\u1EBD t\xECm hi\u1EC3u \u0111i\u1EC1u \u0111\xF3 trong "),st=c(rt,"A",{href:!0});var rl=r(st);Ua=a(rl,"Ch\u01B0\u01A1ng 7"),rl.forEach(t),Ka=a(rt,", nh\u01B0ng tr\u01B0\u1EDBc ti\xEAn, trong ph\u1EA7n c\xF2n l\u1EA1i c\u1EE7a ch\u01B0\u01A1ng n\xE0y, ch\xFAng ta s\u1EBD xem x\xE9t k\u1EF9 h\u01A1n v\u1EC1 c\xE1c tr\xECnh tokenize nhanh v\xE0 kh\xE1m ph\xE1 chi ti\u1EBFt nh\u1EEFng g\xEC th\u1EF1c s\u1EF1 x\u1EA3y ra khi ch\xFAng ta g\u1ECDi ph\u01B0\u01A1ng th\u1EE9c "),ns=c(rt,"CODE",{});var ol=r(ns);Va=a(ol,"train_new_from_iterator()"),ol.forEach(t),Ia=a(rt,"."),rt.forEach(t),this.h()},h(){_(y,"name","hf:doc:metadata"),_(y,"content",JSON.stringify(vl)),_(w,"id","hun-luyn-mt-tokenizer-mi-t-ci-c"),_(w,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),_(w,"href","#hun-luyn-mt-tokenizer-mi-t-ci-c"),_(j,"class","relative group"),_(Gn,"href","/course/chapter2"),_(U,"id","tp-hp-mt-kho-ng-liu"),_(U,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),_(U,"href","#tp-hp-mt-kho-ng-liu"),_(S,"class","relative group"),_(en,"href","https://github.com/huggingface/datasets"),_(en,"rel","nofollow"),_(an,"href","https://huggingface.co/datasets/code_search_net"),_(an,"rel","nofollow"),_(ln,"href","https://wandb.ai/github/CodeSearchNet/benchmark"),_(ln,"rel","nofollow"),_(I,"id","hun-luyn-mt-tokenizer-mi"),_(I,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),_(I,"href","#hun-luyn-mt-tokenizer-mi"),_(G,"class","relative group"),_(jn,"href","https://www.rust-lang.org"),_(jn,"rel","nofollow"),_(Zn,"href","/course/chapter3"),_(zn,"href","https://huggingface.co/transformers/#supported-frameworks"),_(zn,"rel","nofollow"),_(Q,"id","lu-tokenizer"),_(Q,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),_(Q,"href","#lu-tokenizer"),_(M,"class","relative group"),_(st,"href","/course/chap7")},m(n,h){s(document.head,y),i(n,B,h),i(n,j,h),s(j,w),s(w,ot),m(nn,ot,null),s(j,jh),s(j,pt),s(pt,zh),i(n,ss,h),m(tn,n,h),i(n,hs,h),i(n,E,h),s(E,wh),s(E,Gn),s(Gn,Eh),s(E,qh),s(E,gt),s(gt,Ph),s(E,Ch),s(E,ut),s(ut,Ah),s(E,Th),i(n,es,h),m(sn,n,h),i(n,as,h),m(R,n,h),i(n,is,h),i(n,S,h),s(S,U),s(U,mt),m(hn,mt,null),s(S,Dh),s(S,ft),s(ft,Nh),i(n,ls,h),i(n,K,h),s(K,Oh),s(K,kt),s(kt,Lh),s(K,Hh),i(n,cs,h),i(n,z,h),s(z,Sh),s(z,en),s(en,Gh),s(z,Mh),s(z,dt),s(dt,Bh),s(z,Rh),s(z,an),s(an,Uh),s(z,Kh),s(z,ln),s(ln,Vh),s(z,Ih),i(n,rs,h),m(cn,n,h),i(n,os,h),i(n,Mn,h),s(Mn,Yh),i(n,ps,h),m(rn,n,h),i(n,gs,h),m(on,n,h),i(n,us,h),i(n,C,h),s(C,Fh),s(C,_t),s(_t,Qh),s(C,Xh),s(C,xt),s(xt,Jh),s(C,Zh),i(n,ms,h),m(pn,n,h),i(n,fs,h),i(n,Bn,h),s(Bn,Wh),i(n,ks,h),m(gn,n,h),i(n,ds,h),i(n,V,h),s(V,ne),s(V,bt),s(bt,te),s(V,se),i(n,_s,h),i(n,Rn,h),s(Rn,he),i(n,xs,h),m(un,n,h),i(n,bs,h),i(n,Un,h),s(Un,ee),i(n,vs,h),m(mn,n,h),i(n,ys,h),i(n,A,h),s(A,ae),s(A,vt),s(vt,ie),s(A,le),s(A,yt),s(yt,ce),s(A,re),i(n,$s,h),i(n,Kn,h),s(Kn,oe),i(n,js,h),m(fn,n,h),i(n,zs,h),i(n,Vn,h),s(Vn,pe),i(n,ws,h),m(kn,n,h),i(n,Es,h),i(n,In,h),s(In,ge),i(n,qs,h),m(dn,n,h),i(n,Ps,h),i(n,T,h),s(T,ue),s(T,$t),s($t,me),s(T,fe),s(T,jt),s(jt,ke),s(T,de),i(n,Cs,h),m(_n,n,h),i(n,As,h),i(n,Yn,h),s(Yn,_e),i(n,Ts,h),i(n,G,h),s(G,I),s(I,zt),m(xn,zt,null),s(G,xe),s(G,wt),s(wt,be),i(n,Ds,h),i(n,Fn,h),s(Fn,ve),i(n,Ns,h),m(bn,n,h),i(n,Os,h),i(n,Qn,h),s(Qn,ye),i(n,Ls,h),i(n,Xn,h),s(Xn,$e),i(n,Hs,h),m(vn,n,h),i(n,Ss,h),m(yn,n,h),i(n,Gs,h),i(n,q,h),s(q,je),s(q,Et),s(Et,ze),s(q,we),s(q,qt),s(qt,Ee),s(q,qe),s(q,Pt),s(Pt,Pe),s(q,Ce),i(n,Ms,h),i(n,Y,h),s(Y,Ae),s(Y,Ct),s(Ct,Te),s(Y,De),i(n,Bs,h),m($n,n,h),i(n,Rs,h),i(n,Jn,h),s(Jn,Ne),i(n,Us,h),i(n,D,h),s(D,Oe),s(D,At),s(At,Le),s(D,He),s(D,jn),s(jn,Se),s(D,Ge),i(n,Ks,h),i(n,F,h),s(F,Me),s(F,Zn),s(Zn,Be),s(F,Re),i(n,Vs,h),i(n,N,h),s(N,Ue),s(N,zn),s(zn,Ke),s(N,Ve),s(N,Tt),s(Tt,Ie),s(N,Ye),i(n,Is,h),m(wn,n,h),i(n,Ys,h),m(En,n,h),i(n,Fs,h),i(n,$,h),s($,Fe),s($,Dt),s(Dt,Qe),s($,Xe),s($,Nt),s(Nt,Je),s($,Ze),s($,Ot),s(Ot,We),s($,na),s($,Lt),s(Lt,ta),s($,sa),s($,Ht),s(Ht,ha),s($,ea),i(n,Qs,h),m(qn,n,h),i(n,Xs,h),m(Pn,n,h),i(n,Js,h),i(n,Wn,h),s(Wn,aa),i(n,Zs,h),m(Cn,n,h),i(n,Ws,h),m(An,n,h),i(n,nh,h),i(n,x,h),s(x,ia),s(x,St),s(St,la),s(x,ca),s(x,Gt),s(Gt,ra),s(x,oa),s(x,Mt),s(Mt,pa),s(x,ga),s(x,Bt),s(Bt,ua),s(x,ma),s(x,Rt),s(Rt,fa),s(x,ka),s(x,Ut),s(Ut,da),s(x,_a),s(x,Kt),s(Kt,xa),s(x,ba),s(x,Vt),s(Vt,va),s(x,ya),s(x,It),s(It,$a),s(x,ja),s(x,Yt),s(Yt,za),s(x,wa),i(n,th,h),i(n,M,h),s(M,Q),s(Q,Ft),m(Tn,Ft,null),s(M,Ea),s(M,Qt),s(Qt,qa),i(n,sh,h),i(n,X,h),s(X,Pa),s(X,Xt),s(Xt,Ca),s(X,Aa),i(n,hh,h),m(Dn,n,h),i(n,eh,h),i(n,J,h),s(J,Ta),s(J,Jt),s(Jt,Da),s(J,Na),i(n,ah,h),m(Nn,n,h),i(n,ih,h),i(n,nt,h),s(nt,Oa),i(n,lh,h),m(On,n,h),i(n,ch,h),i(n,tt,h),s(tt,La),i(n,rh,h),m(Ln,n,h),i(n,oh,h),i(n,O,h),s(O,Ha),s(O,Zt),s(Zt,Sa),s(O,Ga),s(O,Wt),s(Wt,Ma),s(O,Ba),i(n,ph,h),m(Hn,n,h),i(n,gh,h),i(n,L,h),s(L,Ra),s(L,st),s(st,Ua),s(L,Ka),s(L,ns),s(ns,Va),s(L,Ia),uh=!0},p(n,[h]){const Sn={};h&2&&(Sn.$$scope={dirty:h,ctx:n}),R.$set(Sn)},i(n){uh||(f(nn.$$.fragment,n),f(tn.$$.fragment,n),f(sn.$$.fragment,n),f(R.$$.fragment,n),f(hn.$$.fragment,n),f(cn.$$.fragment,n),f(rn.$$.fragment,n),f(on.$$.fragment,n),f(pn.$$.fragment,n),f(gn.$$.fragment,n),f(un.$$.fragment,n),f(mn.$$.fragment,n),f(fn.$$.fragment,n),f(kn.$$.fragment,n),f(dn.$$.fragment,n),f(_n.$$.fragment,n),f(xn.$$.fragment,n),f(bn.$$.fragment,n),f(vn.$$.fragment,n),f(yn.$$.fragment,n),f($n.$$.fragment,n),f(wn.$$.fragment,n),f(En.$$.fragment,n),f(qn.$$.fragment,n),f(Pn.$$.fragment,n),f(Cn.$$.fragment,n),f(An.$$.fragment,n),f(Tn.$$.fragment,n),f(Dn.$$.fragment,n),f(Nn.$$.fragment,n),f(On.$$.fragment,n),f(Ln.$$.fragment,n),f(Hn.$$.fragment,n),uh=!0)},o(n){k(nn.$$.fragment,n),k(tn.$$.fragment,n),k(sn.$$.fragment,n),k(R.$$.fragment,n),k(hn.$$.fragment,n),k(cn.$$.fragment,n),k(rn.$$.fragment,n),k(on.$$.fragment,n),k(pn.$$.fragment,n),k(gn.$$.fragment,n),k(un.$$.fragment,n),k(mn.$$.fragment,n),k(fn.$$.fragment,n),k(kn.$$.fragment,n),k(dn.$$.fragment,n),k(_n.$$.fragment,n),k(xn.$$.fragment,n),k(bn.$$.fragment,n),k(vn.$$.fragment,n),k(yn.$$.fragment,n),k($n.$$.fragment,n),k(wn.$$.fragment,n),k(En.$$.fragment,n),k(qn.$$.fragment,n),k(Pn.$$.fragment,n),k(Cn.$$.fragment,n),k(An.$$.fragment,n),k(Tn.$$.fragment,n),k(Dn.$$.fragment,n),k(Nn.$$.fragment,n),k(On.$$.fragment,n),k(Ln.$$.fragment,n),k(Hn.$$.fragment,n),uh=!1},d(n){t(y),n&&t(B),n&&t(j),d(nn),n&&t(ss),d(tn,n),n&&t(hs),n&&t(E),n&&t(es),d(sn,n),n&&t(as),d(R,n),n&&t(is),n&&t(S),d(hn),n&&t(ls),n&&t(K),n&&t(cs),n&&t(z),n&&t(rs),d(cn,n),n&&t(os),n&&t(Mn),n&&t(ps),d(rn,n),n&&t(gs),d(on,n),n&&t(us),n&&t(C),n&&t(ms),d(pn,n),n&&t(fs),n&&t(Bn),n&&t(ks),d(gn,n),n&&t(ds),n&&t(V),n&&t(_s),n&&t(Rn),n&&t(xs),d(un,n),n&&t(bs),n&&t(Un),n&&t(vs),d(mn,n),n&&t(ys),n&&t(A),n&&t($s),n&&t(Kn),n&&t(js),d(fn,n),n&&t(zs),n&&t(Vn),n&&t(ws),d(kn,n),n&&t(Es),n&&t(In),n&&t(qs),d(dn,n),n&&t(Ps),n&&t(T),n&&t(Cs),d(_n,n),n&&t(As),n&&t(Yn),n&&t(Ts),n&&t(G),d(xn),n&&t(Ds),n&&t(Fn),n&&t(Ns),d(bn,n),n&&t(Os),n&&t(Qn),n&&t(Ls),n&&t(Xn),n&&t(Hs),d(vn,n),n&&t(Ss),d(yn,n),n&&t(Gs),n&&t(q),n&&t(Ms),n&&t(Y),n&&t(Bs),d($n,n),n&&t(Rs),n&&t(Jn),n&&t(Us),n&&t(D),n&&t(Ks),n&&t(F),n&&t(Vs),n&&t(N),n&&t(Is),d(wn,n),n&&t(Ys),d(En,n),n&&t(Fs),n&&t($),n&&t(Qs),d(qn,n),n&&t(Xs),d(Pn,n),n&&t(Js),n&&t(Wn),n&&t(Zs),d(Cn,n),n&&t(Ws),d(An,n),n&&t(nh),n&&t(x),n&&t(th),n&&t(M),d(Tn),n&&t(sh),n&&t(X),n&&t(hh),d(Dn,n),n&&t(eh),n&&t(J),n&&t(ah),d(Nn,n),n&&t(ih),n&&t(nt),n&&t(lh),d(On,n),n&&t(ch),n&&t(tt),n&&t(rh),d(Ln,n),n&&t(oh),n&&t(O),n&&t(ph),d(Hn,n),n&&t(gh),n&&t(L)}}}const vl={local:"hun-luyn-mt-tokenizer-mi-t-ci-c",sections:[{local:"tp-hp-mt-kho-ng-liu",title:"T\u1EADp h\u1EE3p m\u1ED9t kho ng\u1EEF li\u1EC7u"},{local:"hun-luyn-mt-tokenizer-mi",title:"Hu\u1EA5n luy\u1EC7n m\u1ED9t tokenizer m\u1EDBi"},{local:"lu-tokenizer",title:"L\u01B0u tokenizer"}],title:"Hu\u1EA5n luy\u1EC7n m\u1ED9t tokenizer m\u1EDBi t\u1EEB c\xE1i c\u0169"};function yl(ts){return fl(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class Cl extends pl{constructor(y){super();gl(this,y,yl,bl,ul,{})}}export{Cl as default,vl as metadata};
