import{S as Nr,i as Or,s as Cr,e as p,k as f,w as u,t as a,M as Lr,c as o,d as t,m as _,a as i,x as h,h as l,b as k,G as n,g as r,y as c,q as x,o as m,B as g,v as Dr}from"../../chunks/vendor-hf-doc-builder.js";import{T as Ur}from"../../chunks/Tip-hf-doc-builder.js";import{Y as Mr}from"../../chunks/Youtube-hf-doc-builder.js";import{I as Hr}from"../../chunks/IconCopyLink-hf-doc-builder.js";import{C as j}from"../../chunks/CodeBlock-hf-doc-builder.js";import{C as Br}from"../../chunks/CourseFloatingBanner-hf-doc-builder.js";import"../../chunks/DocNotebookDropdown-hf-doc-builder.js";function Ir(Mt){let w,L;return{c(){w=p("p"),L=a("\u26A0\uFE0F \u8BAD\u7EC3\u6807\u8BB0\u5668\u4E0E\u8BAD\u7EC3\u6A21\u578B\u4E0D\u540C\uFF01\u6A21\u578B\u8BAD\u7EC3\u4F7F\u7528\u968F\u673A\u68AF\u5EA6\u4E0B\u964D\u4F7F\u6BCF\u4E2Abatch\u7684loss\u5C0F\u4E00\u70B9\u3002\u5B83\u672C\u8D28\u4E0A\u662F\u968F\u673A\u7684\uFF08\u8FD9\u610F\u5473\u7740\u5728\u8FDB\u884C\u4E24\u6B21\u76F8\u540C\u7684\u8BAD\u7EC3\u65F6\uFF0C\u60A8\u5FC5\u987B\u8BBE\u7F6E\u4E00\u4E9B\u968F\u673A\u6570\u79CD\u5B50\u624D\u80FD\u83B7\u5F97\u76F8\u540C\u7684\u7ED3\u679C\uFF09\u3002\u8BAD\u7EC3\u6807\u8BB0\u5668\u662F\u4E00\u4E2A\u7EDF\u8BA1\u8FC7\u7A0B\uFF0C\u5B83\u8BD5\u56FE\u786E\u5B9A\u54EA\u4E9B\u5B50\u8BCD\u6700\u9002\u5408\u4E3A\u7ED9\u5B9A\u7684\u8BED\u6599\u5E93\u9009\u62E9\uFF0C\u7528\u4E8E\u9009\u62E9\u5B83\u4EEC\u7684\u786E\u5207\u89C4\u5219\u53D6\u51B3\u4E8E\u5206\u8BCD\u7B97\u6CD5\u3002\u5B83\u662F\u786E\u5B9A\u6027\u7684\uFF0C\u8FD9\u610F\u5473\u7740\u5728\u76F8\u540C\u7684\u8BED\u6599\u5E93\u4E0A\u4F7F\u7528\u76F8\u540C\u7684\u7B97\u6CD5\u8FDB\u884C\u8BAD\u7EC3\u65F6\uFF0C\u60A8\u603B\u662F\u4F1A\u5F97\u5230\u76F8\u540C\u7684\u7ED3\u679C\u3002")},l(v){w=o(v,"P",{});var z=i(w);L=l(z,"\u26A0\uFE0F \u8BAD\u7EC3\u6807\u8BB0\u5668\u4E0E\u8BAD\u7EC3\u6A21\u578B\u4E0D\u540C\uFF01\u6A21\u578B\u8BAD\u7EC3\u4F7F\u7528\u968F\u673A\u68AF\u5EA6\u4E0B\u964D\u4F7F\u6BCF\u4E2Abatch\u7684loss\u5C0F\u4E00\u70B9\u3002\u5B83\u672C\u8D28\u4E0A\u662F\u968F\u673A\u7684\uFF08\u8FD9\u610F\u5473\u7740\u5728\u8FDB\u884C\u4E24\u6B21\u76F8\u540C\u7684\u8BAD\u7EC3\u65F6\uFF0C\u60A8\u5FC5\u987B\u8BBE\u7F6E\u4E00\u4E9B\u968F\u673A\u6570\u79CD\u5B50\u624D\u80FD\u83B7\u5F97\u76F8\u540C\u7684\u7ED3\u679C\uFF09\u3002\u8BAD\u7EC3\u6807\u8BB0\u5668\u662F\u4E00\u4E2A\u7EDF\u8BA1\u8FC7\u7A0B\uFF0C\u5B83\u8BD5\u56FE\u786E\u5B9A\u54EA\u4E9B\u5B50\u8BCD\u6700\u9002\u5408\u4E3A\u7ED9\u5B9A\u7684\u8BED\u6599\u5E93\u9009\u62E9\uFF0C\u7528\u4E8E\u9009\u62E9\u5B83\u4EEC\u7684\u786E\u5207\u89C4\u5219\u53D6\u51B3\u4E8E\u5206\u8BCD\u7B97\u6CD5\u3002\u5B83\u662F\u786E\u5B9A\u6027\u7684\uFF0C\u8FD9\u610F\u5473\u7740\u5728\u76F8\u540C\u7684\u8BED\u6599\u5E93\u4E0A\u4F7F\u7528\u76F8\u540C\u7684\u7B97\u6CD5\u8FDB\u884C\u8BAD\u7EC3\u65F6\uFF0C\u60A8\u603B\u662F\u4F1A\u5F97\u5230\u76F8\u540C\u7684\u7ED3\u679C\u3002"),z.forEach(t)},m(v,z){r(v,w,z),n(w,L)},d(v){v&&t(w)}}}function Kr(Mt){let w,L,v,z,et,F,re,at,pe,Ht,X,Bt,y,oe,As,ie,fe,lt,_e,ue,rt,he,ce,It,J,Kt,D,Yt,Q,xe,Ft,U,me,pt,ge,$e,Xt,E,Z,je,be,ot,ke,we,V,de,ve,W,ze,Ee,Jt,ss,Qt,Gs,ye,Zt,ts,Vt,ns,Wt,T,Pe,it,qe,Te,ft,Ae,Ge,sn,es,tn,Ss,Se,nn,as,en,Rs,Re,an,Ns,Ne,ln,ls,rn,Os,Oe,pn,rs,on,A,Ce,_t,Le,De,ut,Ue,Me,fn,Cs,He,_n,ps,un,Ls,Be,hn,os,cn,Ds,Ie,xn,is,mn,G,Ke,ht,Ye,Fe,ct,Xe,Je,gn,fs,$n,Us,Qe,jn,_s,Ze,bn,Ms,Ve,kn,us,wn,Hs,We,dn,Bs,sa,vn,hs,zn,cs,En,P,ta,xt,na,ea,mt,aa,la,gt,ra,pa,yn,M,oa,$t,ia,fa,Pn,xs,qn,Is,_a,Tn,S,ua,jt,ha,ca,ms,xa,ma,An,H,ga,Ks,$a,ja,Gn,R,ba,gs,ka,wa,bt,da,va,Sn,$s,Rn,js,Nn,d,za,kt,Ea,ya,wt,Pa,qa,dt,Ta,Aa,vt,Ga,Sa,zt,Ra,Na,On,bs,Cn,ks,Ln,Ys,Oa,Dn,ws,Un,ds,Mn,$,Ca,Et,La,Da,yt,Ua,Ma,Pt,Ha,Ba,qt,Ia,Ka,Tt,Ya,Fa,At,Xa,Ja,Gt,Qa,Za,St,Va,Wa,Rt,sl,tl,Nt,nl,el,Hn,vs,al,Bn,B,ll,Ot,rl,pl,In,zs,Kn,I,ol,Ct,il,fl,Yn,Es,Fn,Fs,_l,Xn,ys,Jn,Xs,ul,Qn,Ps,Zn,N,hl,Lt,cl,xl,Dt,ml,gl,Vn,qs,Wn,O,$l,Js,jl,bl,Ut,kl,wl,se;return F=new Hr({}),X=new Br({props:{chapter:6,classNames:"absolute z-10 right-0 top-0",notebooks:[{label:"Google Colab",value:"https://colab.research.google.com/github/huggingface/notebooks/blob/master/course/zh-CN/chapter6/section2.ipynb"},{label:"Aws Studio",value:"https://studiolab.sagemaker.aws/import/github/huggingface/notebooks/blob/master/course/zh-CN/chapter6/section2.ipynb"}]}}),J=new Mr({props:{id:"DJimQynXZsQ"}}),D=new Ur({props:{warning:!0,$$slots:{default:[Ir]},$$scope:{ctx:Mt}}}),ss=new j({props:{code:`from datasets import load_dataset

# This can take a few minutes to load, so grab a coffee or tea while you wait!
raw_datasets = load_dataset("code_search_net", "python")`,highlighted:`<span class="hljs-keyword">from</span> datasets <span class="hljs-keyword">import</span> load_dataset

<span class="hljs-comment"># This can take a few minutes to load, so grab a coffee or tea while you wait!</span>
raw_datasets = load_dataset(<span class="hljs-string">&quot;code_search_net&quot;</span>, <span class="hljs-string">&quot;python&quot;</span>)`}}),ts=new j({props:{code:'raw_datasets["train"]',highlighted:'raw_datasets[<span class="hljs-string">&quot;train&quot;</span>]'}}),ns=new j({props:{code:`Dataset({
    features: ['repository_name', 'func_path_in_repository', 'func_name', 'whole_func_string', 'language', 
      'func_code_string', 'func_code_tokens', 'func_documentation_string', 'func_documentation_tokens', 'split_name', 
      'func_code_url'
    ],
    num_rows: 412178
})`,highlighted:`Dataset({
    features: [<span class="hljs-string">&#x27;repository_name&#x27;</span>, <span class="hljs-string">&#x27;func_path_in_repository&#x27;</span>, <span class="hljs-string">&#x27;func_name&#x27;</span>, <span class="hljs-string">&#x27;whole_func_string&#x27;</span>, <span class="hljs-string">&#x27;language&#x27;</span>, 
      <span class="hljs-string">&#x27;func_code_string&#x27;</span>, <span class="hljs-string">&#x27;func_code_tokens&#x27;</span>, <span class="hljs-string">&#x27;func_documentation_string&#x27;</span>, <span class="hljs-string">&#x27;func_documentation_tokens&#x27;</span>, <span class="hljs-string">&#x27;split_name&#x27;</span>, 
      <span class="hljs-string">&#x27;func_code_url&#x27;</span>
    ],
    num_rows: <span class="hljs-number">412178</span>
})`}}),es=new j({props:{code:'print(raw_datasets["train"][123456]["whole_func_string"])',highlighted:'<span class="hljs-built_in">print</span>(raw_datasets[<span class="hljs-string">&quot;train&quot;</span>][<span class="hljs-number">123456</span>][<span class="hljs-string">&quot;whole_func_string&quot;</span>])'}}),as=new j({props:{code:`def handle_simple_responses(
      self, timeout_ms=None, info_cb=DEFAULT_MESSAGE_CALLBACK):
    """Accepts normal responses from the device.

    Args:
      timeout_ms: Timeout in milliseconds to wait for each response.
      info_cb: Optional callback for text sent from the bootloader.

    Returns:
      OKAY packet's message.
    """
    return self._accept_responses('OKAY', info_cb, timeout_ms=timeout_ms)`,highlighted:`<span class="hljs-keyword">def</span> <span class="hljs-title function_">handle_simple_responses</span>(<span class="hljs-params">
      self, timeout_ms=<span class="hljs-literal">None</span>, info_cb=DEFAULT_MESSAGE_CALLBACK</span>):
    <span class="hljs-string">&quot;&quot;&quot;Accepts normal responses from the device.

    Args:
      timeout_ms: Timeout in milliseconds to wait for each response.
      info_cb: Optional callback for text sent from the bootloader.

    Returns:
      OKAY packet&#x27;s message.
    &quot;&quot;&quot;</span>
    <span class="hljs-keyword">return</span> self._accept_responses(<span class="hljs-string">&#x27;OKAY&#x27;</span>, info_cb, timeout_ms=timeout_ms)`}}),ls=new j({props:{code:`# Don't uncomment the following line unless your dataset is small!
# training_corpus = [raw_datasets["train"][i: i + 1000]["whole_func_string"] for i in range(0, len(raw_datasets["train"]), 1000)]`,highlighted:`<span class="hljs-comment"># Don&#x27;t uncomment the following line unless your dataset is small!</span>
<span class="hljs-comment"># training_corpus = [raw_datasets[&quot;train&quot;][i: i + 1000][&quot;whole_func_string&quot;] for i in range(0, len(raw_datasets[&quot;train&quot;]), 1000)]</span>`}}),rs=new j({props:{code:`training_corpus = (
    raw_datasets["train"][i : i + 1000]["whole_func_string"]
    for i in range(0, len(raw_datasets["train"]), 1000)
)`,highlighted:`training_corpus = (
    raw_datasets[<span class="hljs-string">&quot;train&quot;</span>][i : i + <span class="hljs-number">1000</span>][<span class="hljs-string">&quot;whole_func_string&quot;</span>]
    <span class="hljs-keyword">for</span> i <span class="hljs-keyword">in</span> <span class="hljs-built_in">range</span>(<span class="hljs-number">0</span>, <span class="hljs-built_in">len</span>(raw_datasets[<span class="hljs-string">&quot;train&quot;</span>]), <span class="hljs-number">1000</span>)
)`}}),ps=new j({props:{code:`gen = (i for i in range(10))
print(list(gen))
print(list(gen))`,highlighted:`gen = (i <span class="hljs-keyword">for</span> i <span class="hljs-keyword">in</span> <span class="hljs-built_in">range</span>(<span class="hljs-number">10</span>))
<span class="hljs-built_in">print</span>(<span class="hljs-built_in">list</span>(gen))
<span class="hljs-built_in">print</span>(<span class="hljs-built_in">list</span>(gen))`}}),os=new j({props:{code:`[0, 1, 2, 3, 4, 5, 6, 7, 8, 9]
[]`,highlighted:`[<span class="hljs-number">0</span>, <span class="hljs-number">1</span>, <span class="hljs-number">2</span>, <span class="hljs-number">3</span>, <span class="hljs-number">4</span>, <span class="hljs-number">5</span>, <span class="hljs-number">6</span>, <span class="hljs-number">7</span>, <span class="hljs-number">8</span>, <span class="hljs-number">9</span>]
[]`}}),is=new j({props:{code:`def get_training_corpus():
    return (
        raw_datasets["train"][i : i + 1000]["whole_func_string"]
        for i in range(0, len(raw_datasets["train"]), 1000)
    )


training_corpus = get_training_corpus()`,highlighted:`<span class="hljs-keyword">def</span> <span class="hljs-title function_">get_training_corpus</span>():
    <span class="hljs-keyword">return</span> (
        raw_datasets[<span class="hljs-string">&quot;train&quot;</span>][i : i + <span class="hljs-number">1000</span>][<span class="hljs-string">&quot;whole_func_string&quot;</span>]
        <span class="hljs-keyword">for</span> i <span class="hljs-keyword">in</span> <span class="hljs-built_in">range</span>(<span class="hljs-number">0</span>, <span class="hljs-built_in">len</span>(raw_datasets[<span class="hljs-string">&quot;train&quot;</span>]), <span class="hljs-number">1000</span>)
    )


training_corpus = get_training_corpus()`}}),fs=new j({props:{code:`def get_training_corpus():
    dataset = raw_datasets["train"]
    for start_idx in range(0, len(dataset), 1000):
        samples = dataset[start_idx : start_idx + 1000]
        yield samples["whole_func_string"]`,highlighted:`<span class="hljs-keyword">def</span> <span class="hljs-title function_">get_training_corpus</span>():
    dataset = raw_datasets[<span class="hljs-string">&quot;train&quot;</span>]
    <span class="hljs-keyword">for</span> start_idx <span class="hljs-keyword">in</span> <span class="hljs-built_in">range</span>(<span class="hljs-number">0</span>, <span class="hljs-built_in">len</span>(dataset), <span class="hljs-number">1000</span>):
        samples = dataset[start_idx : start_idx + <span class="hljs-number">1000</span>]
        <span class="hljs-keyword">yield</span> samples[<span class="hljs-string">&quot;whole_func_string&quot;</span>]`}}),us=new j({props:{code:`from transformers import AutoTokenizer

old_tokenizer = AutoTokenizer.from_pretrained("gpt2")`,highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoTokenizer

old_tokenizer = AutoTokenizer.from_pretrained(<span class="hljs-string">&quot;gpt2&quot;</span>)`}}),hs=new j({props:{code:`example = '''def add_numbers(a, b):
    """Add the two numbers \`a\` and \`b\`."""
    return a + b'''

tokens = old_tokenizer.tokenize(example)
tokens`,highlighted:`example = <span class="hljs-string">&#x27;&#x27;&#x27;def add_numbers(a, b):
    &quot;&quot;&quot;Add the two numbers \`a\` and \`b\`.&quot;&quot;&quot;
    return a + b&#x27;&#x27;&#x27;</span>

tokens = old_tokenizer.tokenize(example)
tokens`}}),cs=new j({props:{code:`['def', '\u0120add', '_', 'n', 'umbers', '(', 'a', ',', '\u0120b', '):', '\u010A', '\u0120', '\u0120', '\u0120', '\u0120"""', 'Add', '\u0120the', '\u0120two',
 '\u0120numbers', '\u0120\`', 'a', '\`', '\u0120and', '\u0120\`', 'b', '\`', '."', '""', '\u010A', '\u0120', '\u0120', '\u0120', '\u0120return', '\u0120a', '\u0120+', '\u0120b']`,highlighted:'[<span class="hljs-string">&#x27;def&#x27;</span>, <span class="hljs-string">&#x27;\u0120add&#x27;</span>, <span class="hljs-string">&#x27;_&#x27;</span>, <span class="hljs-string">&#x27;n&#x27;</span>, <span class="hljs-string">&#x27;umbers&#x27;</span>, <span class="hljs-string">&#x27;(&#x27;</span>, <span class="hljs-string">&#x27;a&#x27;</span>, <span class="hljs-string">&#x27;,&#x27;</span>, <span class="hljs-string">&#x27;\u0120b&#x27;</span>, <span class="hljs-string">&#x27;):&#x27;</span>, <span class="hljs-string">&#x27;\u010A&#x27;</span>, <span class="hljs-string">&#x27;\u0120&#x27;</span>, <span class="hljs-string">&#x27;\u0120&#x27;</span>, <span class="hljs-string">&#x27;\u0120&#x27;</span>, <span class="hljs-string">&#x27;\u0120&quot;&quot;&quot;&#x27;</span>, <span class="hljs-string">&#x27;Add&#x27;</span>, <span class="hljs-string">&#x27;\u0120the&#x27;</span>, <span class="hljs-string">&#x27;\u0120two&#x27;</span>,\n <span class="hljs-string">&#x27;\u0120numbers&#x27;</span>, <span class="hljs-string">&#x27;\u0120`&#x27;</span>, <span class="hljs-string">&#x27;a&#x27;</span>, <span class="hljs-string">&#x27;`&#x27;</span>, <span class="hljs-string">&#x27;\u0120and&#x27;</span>, <span class="hljs-string">&#x27;\u0120`&#x27;</span>, <span class="hljs-string">&#x27;b&#x27;</span>, <span class="hljs-string">&#x27;`&#x27;</span>, <span class="hljs-string">&#x27;.&quot;&#x27;</span>, <span class="hljs-string">&#x27;&quot;&quot;&#x27;</span>, <span class="hljs-string">&#x27;\u010A&#x27;</span>, <span class="hljs-string">&#x27;\u0120&#x27;</span>, <span class="hljs-string">&#x27;\u0120&#x27;</span>, <span class="hljs-string">&#x27;\u0120&#x27;</span>, <span class="hljs-string">&#x27;\u0120return&#x27;</span>, <span class="hljs-string">&#x27;\u0120a&#x27;</span>, <span class="hljs-string">&#x27;\u0120+&#x27;</span>, <span class="hljs-string">&#x27;\u0120b&#x27;</span>]'}}),xs=new j({props:{code:"tokenizer = old_tokenizer.train_new_from_iterator(training_corpus, 52000)",highlighted:'tokenizer = old_tokenizer.train_new_from_iterator(training_corpus, <span class="hljs-number">52000</span>)'}}),$s=new j({props:{code:`tokens = tokenizer.tokenize(example)
tokens`,highlighted:`tokens = tokenizer.tokenize(example)
tokens`}}),js=new j({props:{code:`['def', '\u0120add', '_', 'numbers', '(', 'a', ',', '\u0120b', '):', '\u010A\u0120\u0120\u0120', '\u0120"""', 'Add', '\u0120the', '\u0120two', '\u0120numbers', '\u0120\`',
 'a', '\`', '\u0120and', '\u0120\`', 'b', '\`."""', '\u010A\u0120\u0120\u0120', '\u0120return', '\u0120a', '\u0120+', '\u0120b']`,highlighted:'[<span class="hljs-string">&#x27;def&#x27;</span>, <span class="hljs-string">&#x27;\u0120add&#x27;</span>, <span class="hljs-string">&#x27;_&#x27;</span>, <span class="hljs-string">&#x27;numbers&#x27;</span>, <span class="hljs-string">&#x27;(&#x27;</span>, <span class="hljs-string">&#x27;a&#x27;</span>, <span class="hljs-string">&#x27;,&#x27;</span>, <span class="hljs-string">&#x27;\u0120b&#x27;</span>, <span class="hljs-string">&#x27;):&#x27;</span>, <span class="hljs-string">&#x27;\u010A\u0120\u0120\u0120&#x27;</span>, <span class="hljs-string">&#x27;\u0120&quot;&quot;&quot;&#x27;</span>, <span class="hljs-string">&#x27;Add&#x27;</span>, <span class="hljs-string">&#x27;\u0120the&#x27;</span>, <span class="hljs-string">&#x27;\u0120two&#x27;</span>, <span class="hljs-string">&#x27;\u0120numbers&#x27;</span>, <span class="hljs-string">&#x27;\u0120`&#x27;</span>,\n <span class="hljs-string">&#x27;a&#x27;</span>, <span class="hljs-string">&#x27;`&#x27;</span>, <span class="hljs-string">&#x27;\u0120and&#x27;</span>, <span class="hljs-string">&#x27;\u0120`&#x27;</span>, <span class="hljs-string">&#x27;b&#x27;</span>, <span class="hljs-string">&#x27;`.&quot;&quot;&quot;&#x27;</span>, <span class="hljs-string">&#x27;\u010A\u0120\u0120\u0120&#x27;</span>, <span class="hljs-string">&#x27;\u0120return&#x27;</span>, <span class="hljs-string">&#x27;\u0120a&#x27;</span>, <span class="hljs-string">&#x27;\u0120+&#x27;</span>, <span class="hljs-string">&#x27;\u0120b&#x27;</span>]'}}),bs=new j({props:{code:`print(len(tokens))
print(len(old_tokenizer.tokenize(example)))`,highlighted:`<span class="hljs-built_in">print</span>(<span class="hljs-built_in">len</span>(tokens))
<span class="hljs-built_in">print</span>(<span class="hljs-built_in">len</span>(old_tokenizer.tokenize(example)))`}}),ks=new j({props:{code:`27
36`,highlighted:`<span class="hljs-number">27</span>
<span class="hljs-number">36</span>`}}),ws=new j({props:{code:`example = """class LinearLayer():
    def __init__(self, input_size, output_size):
        self.weight = torch.randn(input_size, output_size)
        self.bias = torch.zeros(output_size)

    def __call__(self, x):
        return x @ self.weights + self.bias
    """
tokenizer.tokenize(example)`,highlighted:`example = <span class="hljs-string">&quot;&quot;&quot;class LinearLayer():
    def __init__(self, input_size, output_size):
        self.weight = torch.randn(input_size, output_size)
        self.bias = torch.zeros(output_size)

    def __call__(self, x):
        return x @ self.weights + self.bias
    &quot;&quot;&quot;</span>
tokenizer.tokenize(example)`}}),ds=new j({props:{code:`['class', '\u0120Linear', 'Layer', '():', '\u010A\u0120\u0120\u0120', '\u0120def', '\u0120__', 'init', '__(', 'self', ',', '\u0120input', '_', 'size', ',',
 '\u0120output', '_', 'size', '):', '\u010A\u0120\u0120\u0120\u0120\u0120\u0120\u0120', '\u0120self', '.', 'weight', '\u0120=', '\u0120torch', '.', 'randn', '(', 'input', '_',
 'size', ',', '\u0120output', '_', 'size', ')', '\u010A\u0120\u0120\u0120\u0120\u0120\u0120\u0120', '\u0120self', '.', 'bias', '\u0120=', '\u0120torch', '.', 'zeros', '(',
 'output', '_', 'size', ')', '\u010A\u010A\u0120\u0120\u0120', '\u0120def', '\u0120__', 'call', '__(', 'self', ',', '\u0120x', '):', '\u010A\u0120\u0120\u0120\u0120\u0120\u0120\u0120',
 '\u0120return', '\u0120x', '\u0120@', '\u0120self', '.', 'weights', '\u0120+', '\u0120self', '.', 'bias', '\u010A\u0120\u0120\u0120\u0120']`,highlighted:`[<span class="hljs-string">&#x27;class&#x27;</span>, <span class="hljs-string">&#x27;\u0120Linear&#x27;</span>, <span class="hljs-string">&#x27;Layer&#x27;</span>, <span class="hljs-string">&#x27;():&#x27;</span>, <span class="hljs-string">&#x27;\u010A\u0120\u0120\u0120&#x27;</span>, <span class="hljs-string">&#x27;\u0120def&#x27;</span>, <span class="hljs-string">&#x27;\u0120__&#x27;</span>, <span class="hljs-string">&#x27;init&#x27;</span>, <span class="hljs-string">&#x27;__(&#x27;</span>, <span class="hljs-string">&#x27;self&#x27;</span>, <span class="hljs-string">&#x27;,&#x27;</span>, <span class="hljs-string">&#x27;\u0120input&#x27;</span>, <span class="hljs-string">&#x27;_&#x27;</span>, <span class="hljs-string">&#x27;size&#x27;</span>, <span class="hljs-string">&#x27;,&#x27;</span>,
 <span class="hljs-string">&#x27;\u0120output&#x27;</span>, <span class="hljs-string">&#x27;_&#x27;</span>, <span class="hljs-string">&#x27;size&#x27;</span>, <span class="hljs-string">&#x27;):&#x27;</span>, <span class="hljs-string">&#x27;\u010A\u0120\u0120\u0120\u0120\u0120\u0120\u0120&#x27;</span>, <span class="hljs-string">&#x27;\u0120self&#x27;</span>, <span class="hljs-string">&#x27;.&#x27;</span>, <span class="hljs-string">&#x27;weight&#x27;</span>, <span class="hljs-string">&#x27;\u0120=&#x27;</span>, <span class="hljs-string">&#x27;\u0120torch&#x27;</span>, <span class="hljs-string">&#x27;.&#x27;</span>, <span class="hljs-string">&#x27;randn&#x27;</span>, <span class="hljs-string">&#x27;(&#x27;</span>, <span class="hljs-string">&#x27;input&#x27;</span>, <span class="hljs-string">&#x27;_&#x27;</span>,
 <span class="hljs-string">&#x27;size&#x27;</span>, <span class="hljs-string">&#x27;,&#x27;</span>, <span class="hljs-string">&#x27;\u0120output&#x27;</span>, <span class="hljs-string">&#x27;_&#x27;</span>, <span class="hljs-string">&#x27;size&#x27;</span>, <span class="hljs-string">&#x27;)&#x27;</span>, <span class="hljs-string">&#x27;\u010A\u0120\u0120\u0120\u0120\u0120\u0120\u0120&#x27;</span>, <span class="hljs-string">&#x27;\u0120self&#x27;</span>, <span class="hljs-string">&#x27;.&#x27;</span>, <span class="hljs-string">&#x27;bias&#x27;</span>, <span class="hljs-string">&#x27;\u0120=&#x27;</span>, <span class="hljs-string">&#x27;\u0120torch&#x27;</span>, <span class="hljs-string">&#x27;.&#x27;</span>, <span class="hljs-string">&#x27;zeros&#x27;</span>, <span class="hljs-string">&#x27;(&#x27;</span>,
 <span class="hljs-string">&#x27;output&#x27;</span>, <span class="hljs-string">&#x27;_&#x27;</span>, <span class="hljs-string">&#x27;size&#x27;</span>, <span class="hljs-string">&#x27;)&#x27;</span>, <span class="hljs-string">&#x27;\u010A\u010A\u0120\u0120\u0120&#x27;</span>, <span class="hljs-string">&#x27;\u0120def&#x27;</span>, <span class="hljs-string">&#x27;\u0120__&#x27;</span>, <span class="hljs-string">&#x27;call&#x27;</span>, <span class="hljs-string">&#x27;__(&#x27;</span>, <span class="hljs-string">&#x27;self&#x27;</span>, <span class="hljs-string">&#x27;,&#x27;</span>, <span class="hljs-string">&#x27;\u0120x&#x27;</span>, <span class="hljs-string">&#x27;):&#x27;</span>, <span class="hljs-string">&#x27;\u010A\u0120\u0120\u0120\u0120\u0120\u0120\u0120&#x27;</span>,
 <span class="hljs-string">&#x27;\u0120return&#x27;</span>, <span class="hljs-string">&#x27;\u0120x&#x27;</span>, <span class="hljs-string">&#x27;\u0120@&#x27;</span>, <span class="hljs-string">&#x27;\u0120self&#x27;</span>, <span class="hljs-string">&#x27;.&#x27;</span>, <span class="hljs-string">&#x27;weights&#x27;</span>, <span class="hljs-string">&#x27;\u0120+&#x27;</span>, <span class="hljs-string">&#x27;\u0120self&#x27;</span>, <span class="hljs-string">&#x27;.&#x27;</span>, <span class="hljs-string">&#x27;bias&#x27;</span>, <span class="hljs-string">&#x27;\u010A\u0120\u0120\u0120\u0120&#x27;</span>]`}}),zs=new j({props:{code:'tokenizer.save_pretrained("code-search-net-tokenizer")',highlighted:'tokenizer.save_pretrained(<span class="hljs-string">&quot;code-search-net-tokenizer&quot;</span>)'}}),Es=new j({props:{code:`from huggingface_hub import notebook_login

notebook_login()`,highlighted:`<span class="hljs-keyword">from</span> huggingface_hub <span class="hljs-keyword">import</span> notebook_login

notebook_login()`}}),ys=new j({props:{code:"huggingface-cli login",highlighted:"huggingface-cli login"}}),Ps=new j({props:{code:'tokenizer.push_to_hub("code-search-net-tokenizer")',highlighted:'tokenizer.push_to_hub(<span class="hljs-string">&quot;code-search-net-tokenizer&quot;</span>)'}}),qs=new j({props:{code:`# Replace "huggingface-course" below with your actual namespace to use your own tokenizer
tokenizer = AutoTokenizer.from_pretrained("huggingface-course/code-search-net-tokenizer")`,highlighted:`<span class="hljs-comment"># Replace &quot;huggingface-course&quot; below with your actual namespace to use your own tokenizer</span>
tokenizer = AutoTokenizer.from_pretrained(<span class="hljs-string">&quot;huggingface-course/code-search-net-tokenizer&quot;</span>)`}}),{c(){w=p("meta"),L=f(),v=p("h1"),z=p("a"),et=p("span"),u(F.$$.fragment),re=f(),at=p("span"),pe=a("\u6839\u636E\u5DF2\u6709\u7684tokenizer\u8BAD\u7EC3\u65B0\u7684tokenizer"),Ht=f(),u(X.$$.fragment),Bt=f(),y=p("p"),oe=a("\u5982\u679C\u60A8\u611F\u5174\u8DA3\u7684\u8BED\u8A00\u4E2D\u6CA1\u6709\u53EF\u7528\u7684\u8BED\u8A00\u6A21\u578B\uFF0C\u6216\u8005\u5982\u679C\u60A8\u7684\u8BED\u6599\u5E93\u4E0E\u60A8\u7684\u8BED\u8A00\u6A21\u578B\u6240\u8BAD\u7EC3\u7684\u8BED\u6599\u5E93\u6709\u5F88\u5927\u4E0D\u540C\uFF0C\u60A8\u5F88\u53EF\u80FD\u5E0C\u671B\u4ECE\u9002\u5408\u60A8\u7684\u6570\u636E\u7684\u6807\u8BB0\u5668\u4ECE\u5934\u5F00\u59CB\u91CD\u65B0\u8BAD\u7EC3\u6A21\u578B . \u8FD9\u5C06\u9700\u8981\u5728\u60A8\u7684\u6570\u636E\u96C6\u4E0A\u8BAD\u7EC3\u4E00\u4E2A\u65B0\u7684\u6807\u8BB0\u5668\u3002 \u4F46\u8FD9\u7A76\u7ADF\u662F\u4EC0\u4E48\u610F\u601D\uFF1F \u5F53\u6211\u4EEC\u5728 "),As=p("a"),ie=a("\u7B2C\u4E8C\u7AE0"),fe=a(" \u4E2D\u7B2C\u4E00\u6B21\u67E5\u770B\u6807\u8BB0\u5668\u65F6\uFF0C\u6211\u4EEC\u770B\u5230\u5927\u591A\u6570 Transformer \u6A21\u578B\u4F7F\u7528"),lt=p("em"),_e=a("\u5B50\u8BCD\u5206\u8BCD\u7B97\u6CD5"),ue=a("\u3002 \u4E3A\u4E86\u8BC6\u522B\u54EA\u4E9B\u5B50\u8BCD\u662F\u611F\u5174\u8DA3\u7684\u5E76\u4E14\u5728\u624B\u5934\u7684\u8BED\u6599\u5E93\u4E2D\u6700\u5E38\u51FA\u73B0\uFF0C\u6807\u8BB0\u5668\u9700\u8981\u4ED4\u7EC6\u67E5\u770B\u8BED\u6599\u5E93\u4E2D\u7684\u6240\u6709\u6587\u672C\u2014\u2014\u6211\u4EEC\u79F0\u4E4B\u4E3A"),rt=p("em"),he=a("training"),ce=a("\u7684\u8FC7\u7A0B\u3002 \u8FD9\u79CD\u8BAD\u7EC3\u7684\u786E\u5207\u89C4\u5219\u53D6\u51B3\u4E8E\u6240\u4F7F\u7528\u7684\u6807\u8BB0\u5668\u7684\u7C7B\u578B\uFF0C\u6211\u4EEC\u5C06\u5728\u672C\u7AE0\u540E\u9762\u8BA8\u8BBA\u4E09\u79CD\u4E3B\u8981\u7B97\u6CD5\u3002"),It=f(),u(J.$$.fragment),Kt=f(),u(D.$$.fragment),Yt=f(),Q=p("h2"),xe=a("\u51C6\u5907\u8BED\u6599\u5E93"),Ft=f(),U=p("p"),me=a("\u{1F917} Transformers \u4E2D\u6709\u4E00\u4E2A\u975E\u5E38\u7B80\u5355\u7684 API\uFF0C\u4F60\u53EF\u4EE5\u7528\u5B83\u6765\u8BAD\u7EC3\u4E00\u4E2A\u65B0\u7684\u6807\u8BB0\u5668\uFF0C\u4F7F\u5B83\u4E0E\u73B0\u6709\u6807\u8BB0\u5668\u76F8\u540C\u7684\u7279\u5F81\uFF1A "),pt=p("strong"),ge=a("AutoTokenizer.train_new_from_iterator()"),$e=a(" .\u4E3A\u4E86\u590D\u73B0\u8FD9\u4E00\u70B9\uFF0C\u5047\u8BBE\u6211\u4EEC\u60F3\u4ECE\u5934\u5F00\u59CB\u8BAD\u7EC3 GPT-2\uFF0C\u4F46\u4F7F\u7528\u82F1\u8BED\u4EE5\u5916\u7684\u8BED\u8A00\u3002\u6211\u4EEC\u7684\u9996\u8981\u4EFB\u52A1\u662F\u5728\u8BAD\u7EC3\u8BED\u6599\u5E93\u4E2D\u6536\u96C6\u8BE5\u8BED\u8A00\u7684\u5927\u91CF\u6570\u636E\u3002\u4E3A\u4E86\u63D0\u4F9B\u6BCF\u4E2A\u4EBA\u90FD\u80FD\u7406\u89E3\u7684\u793A\u4F8B\uFF0C\u6211\u4EEC\u5728\u8FD9\u91CC\u4E0D\u4F1A\u4F7F\u7528\u4FC4\u8BED\u6216\u4E2D\u6587\u4E4B\u7C7B\u7684\u8BED\u8A00\uFF0C\u800C\u662F\u4F7F\u7528\u5728\u7279\u5B9A\u9886\u57DF\u7684\u82F1\u8BED\u8BED\u8A00\uFF1APython \u4EE3\u7801\u3002"),Xt=f(),E=p("p"),Z=p("a"),je=a("\u{1F917} Datasets"),be=a("\u5E93\u53EF\u4EE5\u5E2E\u52A9\u6211\u4EEC\u7EC4\u88C5\u4E00\u4E2A Python \u6E90\u4EE3\u7801\u8BED\u6599\u5E93\u3002\u6211\u4EEC\u5C06\u4F7F\u7528"),ot=p("strong"),ke=a("load_dataset()"),we=a("\u529F\u80FD\u4E0B\u8F7D\u548C\u7F13\u5B58"),V=p("a"),de=a("CodeSearchNet"),ve=a("\u6570\u636E\u96C6\u3002\u8BE5\u6570\u636E\u96C6\u662F\u4E3A"),W=p("a"),ze=a("CodeSearchNet \u6311\u6218"),Ee=a("\u800C\u521B\u5EFA\u7684\u5E76\u5305\u542B\u6765\u81EA GitHub \u4E0A\u5F00\u6E90\u5E93\u7684\u6570\u767E\u4E07\u79CD\u7F16\u7A0B\u8BED\u8A00\u7684\u51FD\u6570\u3002\u5728\u8FD9\u91CC\uFF0C\u6211\u4EEC\u5C06\u52A0\u8F7D\u6B64\u6570\u636E\u96C6\u7684 Python \u90E8\u5206\uFF1A"),Jt=f(),u(ss.$$.fragment),Qt=f(),Gs=p("p"),ye=a("\u6211\u4EEC\u53EF\u4EE5\u67E5\u770B\u8BAD\u7EC3\u96C6\u7684\u90E8\u5206\uFF0C\u4EE5\u67E5\u770B\u6211\u4EEC\u6570\u636E\u96C6\u4E2D\u6709\u54EA\u4E9B\u5217\uFF1A"),Zt=f(),u(ts.$$.fragment),Vt=f(),u(ns.$$.fragment),Wt=f(),T=p("p"),Pe=a("\u6211\u4EEC\u53EF\u4EE5\u770B\u5230\u6570\u636E\u96C6\u5C06\u6587\u6863\u5B57\u7B26\u4E32\u4E0E\u4EE3\u7801\u5206\u5F00\uFF0C\u5E76\u4E14\u6709\u4ED6\u4EEC\u5404\u81EA\u7684\u6807\u8BB0\u5316\u540E\u7684\u7ED3\u679C\u3002 \u8FD9\u91CC\u3002 \u6211\u4EEC\u5C06\u53EA\u4F7F\u7528 "),it=p("code"),qe=a("whole_func_string"),Te=a(" \u5217\u6765\u8BAD\u7EC3\u6211\u4EEC\u7684\u6807\u8BB0\u5668\u3002 \u6211\u4EEC\u53EF\u4EE5\u901A\u8FC7\u6307\u5B9A\u5230 "),ft=p("code"),Ae=a("train"),Ge=a(" \u4E2D\u7684\u4E00\u90E8\u5206\u6765\u67E5\u770B\u8FD9\u4E9B\u51FD\u6570\u7684\u4E00\u4E2A\u793A\u4F8B\uFF1A"),sn=f(),u(es.$$.fragment),tn=f(),Ss=p("p"),Se=a("\u5E94\u8BE5\u6253\u5370\u4EE5\u4E0B\u5185\u5BB9\uFF1A"),nn=f(),u(as.$$.fragment),en=f(),Rs=p("p"),Re=a("\u6211\u4EEC\u9700\u8981\u505A\u7684\u7B2C\u4E00\u4EF6\u4E8B\u662F\u5C06\u6570\u636E\u96C6\u8F6C\u6362\u4E3A\u8FED\u4EE3\u5668\u6587\u672C\u5217\u8868 - \u4F8B\u5982\uFF0C\u6587\u672C\u5217\u8868\u3002\u4F7F\u7528\u6587\u672C\u5217\u8868\u5C06\u4F7F\u6211\u4EEC\u7684\u6807\u8BB0\u5668\u8FD0\u884C\u5F97\u66F4\u5FEB\uFF08\u8BAD\u7EC3\u6210\u6279\u6587\u672C\u800C\u4E0D\u662F\u4E00\u4E2A\u63A5\u4E00\u4E2A\u5730\u5904\u7406\u5355\u4E2A\u6587\u672C\uFF09\uFF0C\u5982\u679C\u6211\u4EEC\u60F3\u907F\u514D\u4E00\u6B21\u5C06\u6240\u6709\u5185\u5BB9\u90FD\u653E\u5728\u5185\u5B58\u4E2D\uFF0C\u5B83\u5E94\u8BE5\u662F\u4E00\u4E2A\u8FED\u4EE3\u5668\u3002\u5982\u679C\u4F60\u7684\u8BED\u6599\u5E93\u5F88\u5927\uFF0C\u4F60\u4F1A\u60F3\u8981\u5229\u7528\u8FD9\u6837\u4E00\u4E2A\u7279\u6027\uFF1A\u{1F917} Datasets \u4E0D\u4F1A\u5C06\u6240\u6709\u5185\u5BB9\u90FD\u52A0\u8F7D\u5230 RAM \u4E2D\uFF0C\u800C\u662F\u5C06\u6570\u636E\u96C6\u7684\u5143\u7D20\u5B58\u50A8\u5728\u78C1\u76D8\u4E0A\u3002"),an=f(),Ns=p("p"),Ne=a("\u6267\u884C\u4EE5\u4E0B\u64CD\u4F5C\u5C06\u521B\u5EFA\u4E00\u4E2A\u5305\u542B 1,000 \u4E2A\u6587\u672C\u7684\u5217\u8868\u7684\u5217\u8868\uFF0C\u4F46\u4F1A\u5C06\u6240\u6709\u5185\u5BB9\u52A0\u8F7D\u5230\u5185\u5B58\u4E2D\uFF1A"),ln=f(),u(ls.$$.fragment),rn=f(),Os=p("p"),Oe=a("\u4F7F\u7528 Python \u751F\u6210\u5668\uFF0C\u6211\u4EEC\u53EF\u4EE5\u907F\u514D Python \u5C06\u4EFB\u4F55\u5185\u5BB9\u52A0\u8F7D\u5230\u5185\u5B58\u4E2D\uFF0C\u76F4\u5230\u771F\u6B63\u9700\u8981\u4E3A\u6B62\u3002\u8981\u521B\u5EFA\u8FD9\u6837\u7684\u751F\u6210\u5668\uFF0C\u60A8\u53EA\u9700\u8981\u5C06\u62EC\u53F7\u66FF\u6362\u4E3A\u5706\u62EC\u53F7\uFF1A"),pn=f(),u(rs.$$.fragment),on=f(),A=p("p"),Ce=a("\u8FD9\u884C\u4EE3\u7801\u4E0D\u4F1A\u83B7\u53D6\u6570\u636E\u96C6\u7684\u4EFB\u4F55\u5143\u7D20\uFF1B\u5B83\u53EA\u662F\u521B\u5EFA\u4E86\u4E00\u4E2A\u53EF\u4EE5\u5728 Python \u4E2D\u4F7F\u7528\u7684\u5BF9\u8C61 "),_t=p("strong"),Le=a("for"),De=a(" \u73AF\u5F62\u3002\u6587\u672C\u53EA\u4F1A\u5728\u60A8\u9700\u8981\u65F6\u52A0\u8F7D\uFF08\u5373\uFF0C\u5F53\u60A8\u5904\u4E8E "),ut=p("strong"),Ue=a("for"),Me=a(" \u9700\u8981\u5B83\u4EEC\u7684\u5FAA\u73AF\uFF09\uFF0C\u5E76\u4E14\u4E00\u6B21\u53EA\u4F1A\u52A0\u8F7D 1,000 \u4E2A\u6587\u672C\u3002\u8FD9\u6837\uFF0C\u5373\u4F7F\u60A8\u6B63\u5728\u5904\u7406\u5E9E\u5927\u7684\u6570\u636E\u96C6\uFF0C\u4E5F\u4E0D\u4F1A\u8017\u5C3D\u6240\u6709\u5185\u5B58\u3002"),fn=f(),Cs=p("p"),He=a("\u751F\u6210\u5668\u5BF9\u8C61\u7684\u95EE\u9898\u5728\u4E8E\u5B83\u53EA\u80FD\u4F7F\u7528\u4E00\u6B21\uFF0C\u6BCF\u6B21\u8BBF\u95EE\u5B83\u5C06\u7ED9\u51FA\u4E0B\u4E00\u4E2A\u503C\u3002 \u4E0B\u9762\u662F\u4E00\u4E2A\u4F8B\u5B50\uFF1A"),_n=f(),u(ps.$$.fragment),un=f(),Ls=p("p"),Be=a("\u6211\u4EEC\u7B2C\u4E00\u6B21\u5F97\u5230\u4E86\u8FD9\u4E2A\u5217\u8868\uFF0C\u7136\u540E\u662F\u4E00\u4E2A\u7A7A\u5217\u8868\uFF1A"),hn=f(),u(os.$$.fragment),cn=f(),Ds=p("p"),Ie=a("\u8FD9\u5C31\u662F\u6211\u4EEC\u5B9A\u4E49\u4E00\u4E2A\u8FD4\u56DE\u751F\u6210\u5668\u7684\u51FD\u6570\u7684\u539F\u56E0\uFF1A"),xn=f(),u(is.$$.fragment),mn=f(),G=p("p"),Ke=a("\u60A8\u8FD8\u53EF\u4EE5\u5728\u4E00\u4E2A "),ht=p("strong"),Ye=a("for"),Fe=a(" \u5FAA\u73AF\u5185\u90E8\u4F7F\u7528 "),ct=p("strong"),Xe=a("yield"),Je=a(" \u5173\u952E\u5B57\u5B9A\u4E49\u60A8\u7684\u751F\u6210\u5668\uFF1A"),gn=f(),u(fs.$$.fragment),$n=f(),Us=p("p"),Qe=a("\u8FD9\u5C06\u4EA7\u751F\u4E0E\u4EE5\u524D\u5B8C\u5168\u76F8\u540C\u7684\u751F\u6210\u5668\uFF0C\u4F46\u5141\u8BB8\u60A8\u4F7F\u7528\u6BD4\u5217\u8868\u751F\u6210\u5F0F\u4E2D\u66F4\u590D\u6742\u7684\u903B\u8F91\u3002"),jn=f(),_s=p("h2"),Ze=a("\u8BAD\u7EC3\u4E00\u4E2A\u65B0\u7684\u6807\u8BB0\u5668"),bn=f(),Ms=p("p"),Ve=a("\u73B0\u5728\u6211\u4EEC\u7684\u8BED\u6599\u5E93\u662F\u6587\u672C\u6279\u91CF\u8FED\u4EE3\u5668\u7684\u5F62\u5F0F\uFF0C\u6211\u4EEC\u51C6\u5907\u8BAD\u7EC3\u4E00\u4E2A\u65B0\u7684\u6807\u8BB0\u5668\u3002\u4E3A\u6B64\uFF0C\u6211\u4EEC\u9996\u5148\u9700\u8981\u52A0\u8F7D\u8981\u4E0E\u6A21\u578B\u914D\u5BF9\u7684\u6807\u8BB0\u5668\uFF08\u6B64\u5904\u4E3A GPT-2\uFF09\uFF1A"),kn=f(),u(us.$$.fragment),wn=f(),Hs=p("p"),We=a("\u5373\u4F7F\u6211\u4EEC\u8981\u8BAD\u7EC3\u4E00\u4E2A\u65B0\u7684\u6807\u8BB0\u5668\uFF0C\u6700\u597D\u8FD8\u662F\u8FD9\u6837\u505A\u4EE5\u907F\u514D\u5B8C\u5168\u4ECE\u5934\u5F00\u59CB\u3002\u8FD9\u6837\uFF0C\u6211\u4EEC\u5C31\u4E0D\u5FC5\u6307\u5B9A\u4EFB\u4F55\u5173\u4E8E\u6807\u8BB0\u5316\u7B97\u6CD5\u6216\u6211\u4EEC\u60F3\u8981\u4F7F\u7528\u7684\u7279\u6B8A\u6807\u8BB0\uFF1B\u6211\u4EEC\u7684\u65B0\u6807\u8BB0\u5668\u5C06\u4E0E GPT-2 \u5B8C\u5168\u76F8\u540C\uFF0C\u552F\u4E00\u4F1A\u6539\u53D8\u7684\u662F\u8F93\u5165\u7684\u6570\u636E\uFF0C\u8FD9\u5C06\u53D6\u51B3\u4E8E\u6211\u4EEC\u8BAD\u7EC3\u7684\u8BED\u6599\u3002"),dn=f(),Bs=p("p"),sa=a("\u9996\u5148\u8BA9\u6211\u4EEC\u770B\u770B\u8FD9\u4E2A\u6807\u8BB0\u5668\u5C06\u5982\u4F55\u5904\u7406\u793A\u4F8B\u7684\u6570\u636E\uFF1A"),vn=f(),u(hs.$$.fragment),zn=f(),u(cs.$$.fragment),En=f(),P=p("p"),ta=a("\u8FD9\u4E2A\u6807\u8BB0\u5668\u6709\u4E00\u4E9B\u7279\u6B8A\u7684\u7B26\u53F7\uFF0C\u6BD4\u5982 "),xt=p("strong"),na=a("\u010A"),ea=a(" \u548C "),mt=p("strong"),aa=a("\u0120"),la=a(" \uFF0C\u5206\u522B\u8868\u793A\u7A7A\u683C\u548C\u6362\u884C\u7B26\u3002\u6B63\u5982\u6211\u4EEC\u6240\u770B\u5230\u7684\uFF0C\u8FD9\u4E0D\u662F\u592A\u6709\u6548\uFF1A\u6807\u8BB0\u5668\u4E3A\u6BCF\u4E2A\u7A7A\u683C\u8FD4\u56DE\u5355\u72EC\u7684\u6807\u8BB0\uFF0C\u5F53\u5B83\u53EF\u4EE5\u5C06\u7F29\u8FDB\u7EA7\u522B\u7EC4\u5408\u5728\u4E00\u8D77\u65F6\uFF08\u56E0\u4E3A\u5728\u4EE3\u7801\u4E2D\u5177\u6709\u56DB\u4E2A\u6216\u516B\u4E2A\u7A7A\u683C\u7684\u96C6\u5408\u5C06\u975E\u5E38\u666E\u904D\uFF09\u3002\u5B83\u4E5F\u6709\u70B9\u5947\u602A\u5730\u62C6\u5206\u4E86\u51FD\u6570\u540D\u79F0\uFF0C\u800C\u4E60\u60EF\u4F7F\u7528"),gt=p("strong"),ra=a("_"),pa=a("\u7684\u51FD\u6570\u547D\u540D\u7684\u65B9\u6CD5\u3002"),yn=f(),M=p("p"),oa=a("\u8BA9\u6211\u4EEC\u8BAD\u7EC3\u4E00\u4E2A\u65B0\u7684\u6807\u8BB0\u5668\uFF0C\u770B\u770B\u5B83\u662F\u5426\u80FD\u89E3\u51B3\u8FD9\u4E9B\u95EE\u9898\u3002\u4E3A\u6B64\uFF0C\u6211\u4EEC\u5C06\u4F7F\u7528 "),$t=p("strong"),ia=a("train_new_from_iterator()"),fa=a(" \u65B9\u6CD5\uFF1A"),Pn=f(),u(xs.$$.fragment),qn=f(),Is=p("p"),_a=a("\u5982\u679C\u60A8\u7684\u8BED\u6599\u5E93\u975E\u5E38\u5927\uFF0C\u6B64\u547D\u4EE4\u53EF\u80FD\u9700\u8981\u4E00\u4E9B\u65F6\u95F4\uFF0C\u4F46\u5BF9\u4E8E\u8FD9\u4E2A 1.6 GB \u6587\u672C\u6570\u636E\u96C6\uFF0C\u5B83\u7684\u901F\u5EA6\u975E\u5E38\u5FEB\uFF08\u5728\u5177\u6709 12 \u4E2A\u5185\u6838\u7684 AMD Ryzen 9 3900X CPU \u4E0A\u4E3A 1 \u5206 16 \u79D2\uFF09\u3002"),Tn=f(),S=p("p"),ua=a("\u6CE8\u610F "),jt=p("strong"),ha=a("AutoTokenizer.train_new_from_iterator()"),ca=a(" \u4EC5\u5F53\u60A8\u4F7F\u7528\u7684\u6807\u8BB0\u5668\u662F\u201C\u5FEB\u901F\uFF08fast\uFF09\u201D\u6807\u8BB0\u5668\u65F6\u624D\u6709\u6548\u3002\u6B63\u5982\u60A8\u5C06\u5728\u4E0B\u4E00\u8282\u4E2D\u770B\u5230\u7684\uFF0C\u{1F917} Transformers \u5E93\u5305\u542B\u4E24\u79CD\u7C7B\u578B\u7684\u6807\u8BB0\u5668\uFF1A\u4E00\u4E9B\u5B8C\u5168\u7528 Python \u7F16\u5199\uFF0C\u800C\u53E6\u4E00\u4E9B\uFF08\u5FEB\u901F\u7684\uFF09\u7531 \u{1F917} Tokenizers \u5E93\u652F\u6301\uFF0C\u8BE5\u5E93\u7528"),ms=p("a"),xa=a("Rust"),ma=a("\u7F16\u7A0B\u8BED\u8A00\u7F16\u5199\u3002 Python \u662F\u6700\u5E38\u7528\u4E8E\u6570\u636E\u79D1\u5B66\u548C\u6DF1\u5EA6\u5B66\u4E60\u5E94\u7528\u7A0B\u5E8F\u7684\u8BED\u8A00\uFF0C\u4F46\u662F\u5F53\u9700\u8981\u5E76\u884C\u5316\u4EE5\u63D0\u9AD8\u901F\u5EA6\u65F6\uFF0C\u5FC5\u987B\u7528\u53E6\u4E00\u79CD\u8BED\u8A00\u7F16\u5199\u3002\u4F8B\u5982\uFF0C\u6A21\u578B\u8BA1\u7B97\u6838\u5FC3\u7684\u77E9\u9635\u4E58\u6CD5\u662F\u7528 CUDA \u7F16\u5199\u7684\uFF0CCUDA \u662F\u4E00\u4E2A\u9488\u5BF9 GPU \u7684\u4F18\u5316 C \u5E93\u3002"),An=f(),H=p("p"),ga=a("\u7528\u7EAF Python \u8BAD\u7EC3\u4E00\u4E2A\u5168\u65B0\u7684\u6807\u8BB0\u5668\u4F1A\u975E\u5E38\u7F13\u6162\uFF0C\u8FD9\u5C31\u662F\u6211\u4EEC\u5F00\u53D1 \u{1F917} Tokenizers\u5E93\u7684\u539F\u56E0\u3002\u8BF7\u6CE8\u610F\uFF0C\u6B63\u5982\u60A8\u65E0\u9700\u5B66\u4E60 CUDA \u8BED\u8A00\u5373\u53EF\u5728 GPU \u4E0A\u6267\u884C\u60A8\u7684\u6A21\u578B\u4E00\u6837\uFF0C\u60A8\u4E5F\u65E0\u9700\u5B66\u4E60 Rust \u5373\u53EF\u4F7F\u7528\u5FEB\u901F\u6807\u8BB0\u5668\u3002 \u{1F917} Tokenizers \u5E93\u4E3A\u8BB8\u591A\u5185\u90E8\u8C03\u7528 Rust \u4EE3\u7801\u7684\u65B9\u6CD5\u63D0\u4F9B Python \u7ED1\u5B9A\uFF1B\u4F8B\u5982\uFF0C\u5E76\u884C\u5316\u65B0\u6807\u8BB0\u5668\u7684\u8BAD\u7EC3\uFF0C\u6216\u8005\uFF0C\u6B63\u5982\u6211\u4EEC\u5728"),Ks=p("a"),$a=a("\u7B2C\u4E09\u7AE0"),ja=a("\u4E2D\u770B\u5230\u7684\uFF0C\u5BF9\u4E00\u6279\u8F93\u5165\u8FDB\u884C\u6807\u8BB0\u5316\u3002"),Gn=f(),R=p("p"),ba=a("\u5927\u591A\u6570 Transformer \u6A21\u578B\u90FD\u6709\u53EF\u7528\u7684\u5FEB\u901F\u6807\u8BB0\u5668\uFF08\u60A8\u53EF\u4EE5"),gs=p("a"),ka=a("\u5728\u8FD9\u91CC"),wa=a("\u68C0\u67E5\u4E00\u4E9B\u4F8B\u5916\u60C5\u51B5)\uFF0C\u5982\u679C "),bt=p("strong"),da=a("AutoTokenizer"),va=a(" \u53EF\u7528\uFF0CAPI \u603B\u662F\u4E3A\u60A8\u9009\u62E9\u5FEB\u901F\u6807\u8BB0\u5668\u3002\u5728\u4E0B\u4E00\u8282\u4E2D\uFF0C\u6211\u4EEC\u5C06\u770B\u770B\u5FEB\u901F\u6807\u8BB0\u5668\u5177\u6709\u7684\u5176\u4ED6\u4E00\u4E9B\u7279\u6B8A\u529F\u80FD\uFF0C\u8FD9\u4E9B\u529F\u80FD\u5BF9\u4E8E\u6807\u8BB0\u5206\u7C7B\u548C\u95EE\u7B54\u7B49\u4EFB\u52A1\u975E\u5E38\u6709\u7528\u3002\u7136\u800C\uFF0C\u5728\u6DF1\u5165\u7814\u7A76\u4E4B\u524D\uFF0C\u8BA9\u6211\u4EEC\u5728\u4E0A\u4E00\u4E2A\u793A\u4F8B\u4E2D\u5C1D\u8BD5\u6211\u4EEC\u5168\u65B0\u7684\u6807\u8BB0\u5668\uFF1A"),Sn=f(),u($s.$$.fragment),Rn=f(),u(js.$$.fragment),Nn=f(),d=p("p"),za=a("\u5728\u8FD9\u91CC\u6211\u4EEC\u518D\u6B21\u770B\u5230\u7279\u6B8A\u7B26\u53F7 "),kt=p("strong"),Ea=a("\u010A"),ya=a(" \u548C "),wt=p("strong"),Pa=a("\u0120"),qa=a(" \u8868\u793A\u7A7A\u683C\u548C\u6362\u884C\u7B26\uFF0C\u4F46\u6211\u4EEC\u4E5F\u53EF\u4EE5\u770B\u5230\u6211\u4EEC\u7684\u6807\u8BB0\u5668\u5B66\u4E60\u4E86\u4E00\u4E9B\u9AD8\u5EA6\u7279\u5B9A\u4E8E Python \u51FD\u6570\u8BED\u6599\u5E93\u7684\u6807\u8BB0\uFF1A\u4F8B\u5982\uFF0C\u6709\u4E00\u4E2A "),dt=p("strong"),Ta=a("\u010A\u0120\u0120\u0120"),Aa=a(" \u8868\u793A\u7F29\u8FDB\u7684\u6807\u8BB0\uFF0C\u4EE5\u53CA "),vt=p("strong"),Ga=a("\u0120"),Sa=a(" \u8868\u793A\u5F00\u59CB\u6587\u6863\u5B57\u7B26\u4E32\u7684\u4E09\u4E2A\u5F15\u53F7\u7684\u6807\u8BB0\u3002\u6807\u8BB0\u5668\u8FD8\u6B63\u786E\u4F7F\u7528"),zt=p("strong"),Ra=a("_"),Na=a("\u547D\u540D\u7684\u89C4\u8303\u5C06\u51FD\u6570\u540D\u79F0\u62C6\u5206\u4E3A  .\u8FD9\u662F\u4E00\u4E2A\u975E\u5E38\u7D27\u51D1\u7684\u8868\u793A\uFF1B\u76F8\u6BD4\u4E4B\u4E0B\uFF0C\u5728\u540C\u4E00\u4E2A\u4F8B\u5B50\u4E2D\u4F7F\u7528\u7B80\u5355\u7684\u82F1\u8BED\u6807\u8BB0\u5668\u4F1A\u7ED9\u6211\u4EEC\u4E00\u4E2A\u66F4\u957F\u7684\u53E5\u5B50\uFF1A"),On=f(),u(bs.$$.fragment),Cn=f(),u(ks.$$.fragment),Ln=f(),Ys=p("p"),Oa=a("\u8BA9\u6211\u4EEC\u518D\u770B\u4E00\u4E2A\u4F8B\u5B50\uFF1A"),Dn=f(),u(ws.$$.fragment),Un=f(),u(ds.$$.fragment),Mn=f(),$=p("p"),Ca=a("\u9664\u4E86\u4E00\u4E2A\u7F29\u8FDB\u5BF9\u5E94\u7684token\uFF0C\u8FD9\u91CC\u6211\u4EEC\u8FD8\u53EF\u4EE5\u770B\u5230\u4E00\u4E2A\u53CC\u7F29\u8FDB\u7684token\uFF1A "),Et=p("strong"),La=a("\u010A\u0120\u0120\u0120\u0120\u0120\u0120\u0120"),Da=a(" .\u7279\u6B8A\u7684 Python \u8BCD\u5982 "),yt=p("strong"),Ua=a("class"),Ma=a(" , "),Pt=p("strong"),Ha=a("init"),Ba=a(" , "),qt=p("strong"),Ia=a("call"),Ka=a(" , "),Tt=p("strong"),Ya=a("self"),Fa=a(" \uFF0C \u548C "),At=p("strong"),Xa=a("return"),Ja=a(" \u6BCF\u4E2A\u90FD\u88AB\u6807\u8BB0\u4E3A\u4E00\u4E2A\u6807\u8BB0\uFF0C\u6211\u4EEC\u53EF\u4EE5\u770B\u5230\uFF0C\u4EE5\u53CA\u5206\u88C2 "),Gt=p("strong"),Qa=a("_"),Za=a(" \u548C "),St=p("strong"),Va=a("."),Wa=a(" \u6807\u8BB0\u5668\u751A\u81F3\u53EF\u4EE5\u6B63\u786E\u62C6\u5206\u9A7C\u5CF0\u5F0F\u540D\u79F0\uFF1A "),Rt=p("strong"),sl=a("LinearLayer"),tl=a(" \u88AB\u6807\u8BB0\u4E3A "),Nt=p("strong"),nl=a("[\u0120Linear, Layer]"),el=a(" ."),Hn=f(),vs=p("h2"),al=a("\u4FDD\u5B58\u6807\u8BB0\u5668"),Bn=f(),B=p("p"),ll=a("\u4E3A\u4E86\u786E\u4FDD\u6211\u4EEC\u4EE5\u540E\u53EF\u4EE5\u4F7F\u7528\u5B83\uFF0C\u6211\u4EEC\u9700\u8981\u4FDD\u5B58\u6211\u4EEC\u7684\u65B0\u6807\u8BB0\u5668\u3002\u5C31\u50CF\u6A21\u578B\u4E00\u6837\uFF0C\u662F\u901A\u8FC7 "),Ot=p("strong"),rl=a("save_pretrained()"),pl=a(" \u65B9\u6CD5\uFF1A"),In=f(),u(zs.$$.fragment),Kn=f(),I=p("p"),ol=a("\u8FD9\u5C06\u521B\u5EFA\u4E00\u4E2A\u540D\u4E3A\u7684"),Ct=p("em"),il=a("code-search-net-tokenizer"),fl=a("\u7684\u65B0\u6587\u4EF6\u5939\uFF0C\u5B83\u5C06\u5305\u542B\u91CD\u65B0\u52A0\u8F7D\u6807\u8BB0\u5668\u6240\u9700\u8981\u7684\u6240\u6709\u6587\u4EF6\u3002\u5982\u679C\u60A8\u60F3\u4E0E\u60A8\u7684\u540C\u4E8B\u548C\u670B\u53CB\u5206\u4EAB\u8FD9\u4E2A\u6807\u8BB0\u5668\uFF0C\u60A8\u53EF\u4EE5\u901A\u8FC7\u767B\u5F55\u60A8\u7684\u5E10\u6237\u5C06\u5176\u4E0A\u4F20\u5230 Hub\u3002\u5982\u679C\u60A8\u5728notebook\u4E0A\u5DE5\u4F5C\uFF0C\u6709\u4E00\u4E2A\u65B9\u4FBF\u7684\u529F\u80FD\u53EF\u4EE5\u5E2E\u52A9\u60A8\uFF1A"),Yn=f(),u(Es.$$.fragment),Fn=f(),Fs=p("p"),_l=a("\u8FD9\u5C06\u663E\u793A\u4E00\u4E2A\u5C0F\u90E8\u4EF6\uFF0C\u60A8\u53EF\u4EE5\u5728\u5176\u4E2D\u8F93\u5165\u60A8\u7684 Hugging Face \u767B\u5F55\u51ED\u636E\u3002\u5982\u679C\u60A8\u4E0D\u662F\u5728notebook\u4E0A\u5DE5\u4F5C\uFF0C\u53EA\u9700\u5728\u7EC8\u7AEF\u4E2D\u8F93\u5165\u4EE5\u4E0B\u884C\uFF1A"),Xn=f(),u(ys.$$.fragment),Jn=f(),Xs=p("p"),ul=a("\u767B\u5F55\u540E\uFF0C\u60A8\u53EF\u4EE5\u901A\u8FC7\u6267\u884C\u4EE5\u4E0B\u547D\u4EE4\u6765\u63A8\u9001\u60A8\u7684\u6807\u8BB0\u5668\uFF1A"),Qn=f(),u(Ps.$$.fragment),Zn=f(),N=p("p"),hl=a("\u8FD9\u5C06\u5728\u60A8\u7684\u547D\u540D\u7A7A\u95F4\u4E2D\u521B\u5EFA\u4E00\u4E2A\u540D\u4E3A"),Lt=p("strong"),cl=a("code-search-net-tokenizer"),xl=a("\u7684\u65B0\u5B58\u50A8\u5E93  \uFF0C\u5305\u542B\u6807\u8BB0\u5668\u6587\u4EF6\u3002\u7136\u540E\uFF0C\u60A8\u53EF\u4EE5\u4F7F\u7528\u4EE5\u4E0B\u547D\u4EE4\u4ECE\u4EFB\u4F55\u5730\u65B9\u52A0\u8F7D\u6807\u8BB0\u5668\u7684 "),Dt=p("strong"),ml=a("from_pretrained()"),gl=a(" \u65B9\u6CD5\uFF1A"),Vn=f(),u(qs.$$.fragment),Wn=f(),O=p("p"),$l=a("\u60A8\u73B0\u5728\u5DF2\u51C6\u5907\u597D\u4ECE\u5934\u5F00\u59CB\u8BAD\u7EC3\u8BED\u8A00\u6A21\u578B\u5E76\u6839\u636E\u60A8\u624B\u5934\u7684\u4EFB\u52A1\u5BF9\u5176\u8FDB\u884C\u5FAE\u8C03\uFF01\u6211\u4EEC\u5C06\u5728"),Js=p("a"),jl=a("\u7B2C\u4E03\u7AE0"),bl=a("\u8FDB\u884C\u8FD9\u90E8\u5206\u3002\u4F46\u9996\u5148\uFF0C\u5728\u672C\u7AE0\u7684\u5176\u4F59\u90E8\u5206\uFF0C\u6211\u4EEC\u5C06\u4ED4\u7EC6\u7814\u7A76\u5FEB\u901F\u6807\u8BB0\u5668\uFF0C\u5E76\u8BE6\u7EC6\u63A2\u8BA8\u8C03\u7528 "),Ut=p("strong"),kl=a("train_new_from_iterator()"),wl=a(" \u65B9\u6CD5\u65F6\u5B9E\u9645\u53D1\u751F\u7684\u60C5\u51B5 ."),this.h()},l(s){const e=Lr('[data-svelte="svelte-1phssyn"]',document.head);w=o(e,"META",{name:!0,content:!0}),e.forEach(t),L=_(s),v=o(s,"H1",{class:!0});var Ts=i(v);z=o(Ts,"A",{id:!0,class:!0,href:!0});var dl=i(z);et=o(dl,"SPAN",{});var vl=i(et);h(F.$$.fragment,vl),vl.forEach(t),dl.forEach(t),re=_(Ts),at=o(Ts,"SPAN",{});var zl=i(at);pe=l(zl,"\u6839\u636E\u5DF2\u6709\u7684tokenizer\u8BAD\u7EC3\u65B0\u7684tokenizer"),zl.forEach(t),Ts.forEach(t),Ht=_(s),h(X.$$.fragment,s),Bt=_(s),y=o(s,"P",{});var K=i(y);oe=l(K,"\u5982\u679C\u60A8\u611F\u5174\u8DA3\u7684\u8BED\u8A00\u4E2D\u6CA1\u6709\u53EF\u7528\u7684\u8BED\u8A00\u6A21\u578B\uFF0C\u6216\u8005\u5982\u679C\u60A8\u7684\u8BED\u6599\u5E93\u4E0E\u60A8\u7684\u8BED\u8A00\u6A21\u578B\u6240\u8BAD\u7EC3\u7684\u8BED\u6599\u5E93\u6709\u5F88\u5927\u4E0D\u540C\uFF0C\u60A8\u5F88\u53EF\u80FD\u5E0C\u671B\u4ECE\u9002\u5408\u60A8\u7684\u6570\u636E\u7684\u6807\u8BB0\u5668\u4ECE\u5934\u5F00\u59CB\u91CD\u65B0\u8BAD\u7EC3\u6A21\u578B . \u8FD9\u5C06\u9700\u8981\u5728\u60A8\u7684\u6570\u636E\u96C6\u4E0A\u8BAD\u7EC3\u4E00\u4E2A\u65B0\u7684\u6807\u8BB0\u5668\u3002 \u4F46\u8FD9\u7A76\u7ADF\u662F\u4EC0\u4E48\u610F\u601D\uFF1F \u5F53\u6211\u4EEC\u5728 "),As=o(K,"A",{href:!0});var El=i(As);ie=l(El,"\u7B2C\u4E8C\u7AE0"),El.forEach(t),fe=l(K," \u4E2D\u7B2C\u4E00\u6B21\u67E5\u770B\u6807\u8BB0\u5668\u65F6\uFF0C\u6211\u4EEC\u770B\u5230\u5927\u591A\u6570 Transformer \u6A21\u578B\u4F7F\u7528"),lt=o(K,"EM",{});var yl=i(lt);_e=l(yl,"\u5B50\u8BCD\u5206\u8BCD\u7B97\u6CD5"),yl.forEach(t),ue=l(K,"\u3002 \u4E3A\u4E86\u8BC6\u522B\u54EA\u4E9B\u5B50\u8BCD\u662F\u611F\u5174\u8DA3\u7684\u5E76\u4E14\u5728\u624B\u5934\u7684\u8BED\u6599\u5E93\u4E2D\u6700\u5E38\u51FA\u73B0\uFF0C\u6807\u8BB0\u5668\u9700\u8981\u4ED4\u7EC6\u67E5\u770B\u8BED\u6599\u5E93\u4E2D\u7684\u6240\u6709\u6587\u672C\u2014\u2014\u6211\u4EEC\u79F0\u4E4B\u4E3A"),rt=o(K,"EM",{});var Pl=i(rt);he=l(Pl,"training"),Pl.forEach(t),ce=l(K,"\u7684\u8FC7\u7A0B\u3002 \u8FD9\u79CD\u8BAD\u7EC3\u7684\u786E\u5207\u89C4\u5219\u53D6\u51B3\u4E8E\u6240\u4F7F\u7528\u7684\u6807\u8BB0\u5668\u7684\u7C7B\u578B\uFF0C\u6211\u4EEC\u5C06\u5728\u672C\u7AE0\u540E\u9762\u8BA8\u8BBA\u4E09\u79CD\u4E3B\u8981\u7B97\u6CD5\u3002"),K.forEach(t),It=_(s),h(J.$$.fragment,s),Kt=_(s),h(D.$$.fragment,s),Yt=_(s),Q=o(s,"H2",{id:!0});var ql=i(Q);xe=l(ql,"\u51C6\u5907\u8BED\u6599\u5E93"),ql.forEach(t),Ft=_(s),U=o(s,"P",{});var te=i(U);me=l(te,"\u{1F917} Transformers \u4E2D\u6709\u4E00\u4E2A\u975E\u5E38\u7B80\u5355\u7684 API\uFF0C\u4F60\u53EF\u4EE5\u7528\u5B83\u6765\u8BAD\u7EC3\u4E00\u4E2A\u65B0\u7684\u6807\u8BB0\u5668\uFF0C\u4F7F\u5B83\u4E0E\u73B0\u6709\u6807\u8BB0\u5668\u76F8\u540C\u7684\u7279\u5F81\uFF1A "),pt=o(te,"STRONG",{});var Tl=i(pt);ge=l(Tl,"AutoTokenizer.train_new_from_iterator()"),Tl.forEach(t),$e=l(te," .\u4E3A\u4E86\u590D\u73B0\u8FD9\u4E00\u70B9\uFF0C\u5047\u8BBE\u6211\u4EEC\u60F3\u4ECE\u5934\u5F00\u59CB\u8BAD\u7EC3 GPT-2\uFF0C\u4F46\u4F7F\u7528\u82F1\u8BED\u4EE5\u5916\u7684\u8BED\u8A00\u3002\u6211\u4EEC\u7684\u9996\u8981\u4EFB\u52A1\u662F\u5728\u8BAD\u7EC3\u8BED\u6599\u5E93\u4E2D\u6536\u96C6\u8BE5\u8BED\u8A00\u7684\u5927\u91CF\u6570\u636E\u3002\u4E3A\u4E86\u63D0\u4F9B\u6BCF\u4E2A\u4EBA\u90FD\u80FD\u7406\u89E3\u7684\u793A\u4F8B\uFF0C\u6211\u4EEC\u5728\u8FD9\u91CC\u4E0D\u4F1A\u4F7F\u7528\u4FC4\u8BED\u6216\u4E2D\u6587\u4E4B\u7C7B\u7684\u8BED\u8A00\uFF0C\u800C\u662F\u4F7F\u7528\u5728\u7279\u5B9A\u9886\u57DF\u7684\u82F1\u8BED\u8BED\u8A00\uFF1APython \u4EE3\u7801\u3002"),te.forEach(t),Xt=_(s),E=o(s,"P",{});var C=i(E);Z=o(C,"A",{href:!0,rel:!0});var Al=i(Z);je=l(Al,"\u{1F917} Datasets"),Al.forEach(t),be=l(C,"\u5E93\u53EF\u4EE5\u5E2E\u52A9\u6211\u4EEC\u7EC4\u88C5\u4E00\u4E2A Python \u6E90\u4EE3\u7801\u8BED\u6599\u5E93\u3002\u6211\u4EEC\u5C06\u4F7F\u7528"),ot=o(C,"STRONG",{});var Gl=i(ot);ke=l(Gl,"load_dataset()"),Gl.forEach(t),we=l(C,"\u529F\u80FD\u4E0B\u8F7D\u548C\u7F13\u5B58"),V=o(C,"A",{href:!0,rel:!0});var Sl=i(V);de=l(Sl,"CodeSearchNet"),Sl.forEach(t),ve=l(C,"\u6570\u636E\u96C6\u3002\u8BE5\u6570\u636E\u96C6\u662F\u4E3A"),W=o(C,"A",{href:!0,rel:!0});var Rl=i(W);ze=l(Rl,"CodeSearchNet \u6311\u6218"),Rl.forEach(t),Ee=l(C,"\u800C\u521B\u5EFA\u7684\u5E76\u5305\u542B\u6765\u81EA GitHub \u4E0A\u5F00\u6E90\u5E93\u7684\u6570\u767E\u4E07\u79CD\u7F16\u7A0B\u8BED\u8A00\u7684\u51FD\u6570\u3002\u5728\u8FD9\u91CC\uFF0C\u6211\u4EEC\u5C06\u52A0\u8F7D\u6B64\u6570\u636E\u96C6\u7684 Python \u90E8\u5206\uFF1A"),C.forEach(t),Jt=_(s),h(ss.$$.fragment,s),Qt=_(s),Gs=o(s,"P",{});var Nl=i(Gs);ye=l(Nl,"\u6211\u4EEC\u53EF\u4EE5\u67E5\u770B\u8BAD\u7EC3\u96C6\u7684\u90E8\u5206\uFF0C\u4EE5\u67E5\u770B\u6211\u4EEC\u6570\u636E\u96C6\u4E2D\u6709\u54EA\u4E9B\u5217\uFF1A"),Nl.forEach(t),Zt=_(s),h(ts.$$.fragment,s),Vt=_(s),h(ns.$$.fragment,s),Wt=_(s),T=o(s,"P",{});var Qs=i(T);Pe=l(Qs,"\u6211\u4EEC\u53EF\u4EE5\u770B\u5230\u6570\u636E\u96C6\u5C06\u6587\u6863\u5B57\u7B26\u4E32\u4E0E\u4EE3\u7801\u5206\u5F00\uFF0C\u5E76\u4E14\u6709\u4ED6\u4EEC\u5404\u81EA\u7684\u6807\u8BB0\u5316\u540E\u7684\u7ED3\u679C\u3002 \u8FD9\u91CC\u3002 \u6211\u4EEC\u5C06\u53EA\u4F7F\u7528 "),it=o(Qs,"CODE",{});var Ol=i(it);qe=l(Ol,"whole_func_string"),Ol.forEach(t),Te=l(Qs," \u5217\u6765\u8BAD\u7EC3\u6211\u4EEC\u7684\u6807\u8BB0\u5668\u3002 \u6211\u4EEC\u53EF\u4EE5\u901A\u8FC7\u6307\u5B9A\u5230 "),ft=o(Qs,"CODE",{});var Cl=i(ft);Ae=l(Cl,"train"),Cl.forEach(t),Ge=l(Qs," \u4E2D\u7684\u4E00\u90E8\u5206\u6765\u67E5\u770B\u8FD9\u4E9B\u51FD\u6570\u7684\u4E00\u4E2A\u793A\u4F8B\uFF1A"),Qs.forEach(t),sn=_(s),h(es.$$.fragment,s),tn=_(s),Ss=o(s,"P",{});var Ll=i(Ss);Se=l(Ll,"\u5E94\u8BE5\u6253\u5370\u4EE5\u4E0B\u5185\u5BB9\uFF1A"),Ll.forEach(t),nn=_(s),h(as.$$.fragment,s),en=_(s),Rs=o(s,"P",{});var Dl=i(Rs);Re=l(Dl,"\u6211\u4EEC\u9700\u8981\u505A\u7684\u7B2C\u4E00\u4EF6\u4E8B\u662F\u5C06\u6570\u636E\u96C6\u8F6C\u6362\u4E3A\u8FED\u4EE3\u5668\u6587\u672C\u5217\u8868 - \u4F8B\u5982\uFF0C\u6587\u672C\u5217\u8868\u3002\u4F7F\u7528\u6587\u672C\u5217\u8868\u5C06\u4F7F\u6211\u4EEC\u7684\u6807\u8BB0\u5668\u8FD0\u884C\u5F97\u66F4\u5FEB\uFF08\u8BAD\u7EC3\u6210\u6279\u6587\u672C\u800C\u4E0D\u662F\u4E00\u4E2A\u63A5\u4E00\u4E2A\u5730\u5904\u7406\u5355\u4E2A\u6587\u672C\uFF09\uFF0C\u5982\u679C\u6211\u4EEC\u60F3\u907F\u514D\u4E00\u6B21\u5C06\u6240\u6709\u5185\u5BB9\u90FD\u653E\u5728\u5185\u5B58\u4E2D\uFF0C\u5B83\u5E94\u8BE5\u662F\u4E00\u4E2A\u8FED\u4EE3\u5668\u3002\u5982\u679C\u4F60\u7684\u8BED\u6599\u5E93\u5F88\u5927\uFF0C\u4F60\u4F1A\u60F3\u8981\u5229\u7528\u8FD9\u6837\u4E00\u4E2A\u7279\u6027\uFF1A\u{1F917} Datasets \u4E0D\u4F1A\u5C06\u6240\u6709\u5185\u5BB9\u90FD\u52A0\u8F7D\u5230 RAM \u4E2D\uFF0C\u800C\u662F\u5C06\u6570\u636E\u96C6\u7684\u5143\u7D20\u5B58\u50A8\u5728\u78C1\u76D8\u4E0A\u3002"),Dl.forEach(t),an=_(s),Ns=o(s,"P",{});var Ul=i(Ns);Ne=l(Ul,"\u6267\u884C\u4EE5\u4E0B\u64CD\u4F5C\u5C06\u521B\u5EFA\u4E00\u4E2A\u5305\u542B 1,000 \u4E2A\u6587\u672C\u7684\u5217\u8868\u7684\u5217\u8868\uFF0C\u4F46\u4F1A\u5C06\u6240\u6709\u5185\u5BB9\u52A0\u8F7D\u5230\u5185\u5B58\u4E2D\uFF1A"),Ul.forEach(t),ln=_(s),h(ls.$$.fragment,s),rn=_(s),Os=o(s,"P",{});var Ml=i(Os);Oe=l(Ml,"\u4F7F\u7528 Python \u751F\u6210\u5668\uFF0C\u6211\u4EEC\u53EF\u4EE5\u907F\u514D Python \u5C06\u4EFB\u4F55\u5185\u5BB9\u52A0\u8F7D\u5230\u5185\u5B58\u4E2D\uFF0C\u76F4\u5230\u771F\u6B63\u9700\u8981\u4E3A\u6B62\u3002\u8981\u521B\u5EFA\u8FD9\u6837\u7684\u751F\u6210\u5668\uFF0C\u60A8\u53EA\u9700\u8981\u5C06\u62EC\u53F7\u66FF\u6362\u4E3A\u5706\u62EC\u53F7\uFF1A"),Ml.forEach(t),pn=_(s),h(rs.$$.fragment,s),on=_(s),A=o(s,"P",{});var Zs=i(A);Ce=l(Zs,"\u8FD9\u884C\u4EE3\u7801\u4E0D\u4F1A\u83B7\u53D6\u6570\u636E\u96C6\u7684\u4EFB\u4F55\u5143\u7D20\uFF1B\u5B83\u53EA\u662F\u521B\u5EFA\u4E86\u4E00\u4E2A\u53EF\u4EE5\u5728 Python \u4E2D\u4F7F\u7528\u7684\u5BF9\u8C61 "),_t=o(Zs,"STRONG",{});var Hl=i(_t);Le=l(Hl,"for"),Hl.forEach(t),De=l(Zs," \u73AF\u5F62\u3002\u6587\u672C\u53EA\u4F1A\u5728\u60A8\u9700\u8981\u65F6\u52A0\u8F7D\uFF08\u5373\uFF0C\u5F53\u60A8\u5904\u4E8E "),ut=o(Zs,"STRONG",{});var Bl=i(ut);Ue=l(Bl,"for"),Bl.forEach(t),Me=l(Zs," \u9700\u8981\u5B83\u4EEC\u7684\u5FAA\u73AF\uFF09\uFF0C\u5E76\u4E14\u4E00\u6B21\u53EA\u4F1A\u52A0\u8F7D 1,000 \u4E2A\u6587\u672C\u3002\u8FD9\u6837\uFF0C\u5373\u4F7F\u60A8\u6B63\u5728\u5904\u7406\u5E9E\u5927\u7684\u6570\u636E\u96C6\uFF0C\u4E5F\u4E0D\u4F1A\u8017\u5C3D\u6240\u6709\u5185\u5B58\u3002"),Zs.forEach(t),fn=_(s),Cs=o(s,"P",{});var Il=i(Cs);He=l(Il,"\u751F\u6210\u5668\u5BF9\u8C61\u7684\u95EE\u9898\u5728\u4E8E\u5B83\u53EA\u80FD\u4F7F\u7528\u4E00\u6B21\uFF0C\u6BCF\u6B21\u8BBF\u95EE\u5B83\u5C06\u7ED9\u51FA\u4E0B\u4E00\u4E2A\u503C\u3002 \u4E0B\u9762\u662F\u4E00\u4E2A\u4F8B\u5B50\uFF1A"),Il.forEach(t),_n=_(s),h(ps.$$.fragment,s),un=_(s),Ls=o(s,"P",{});var Kl=i(Ls);Be=l(Kl,"\u6211\u4EEC\u7B2C\u4E00\u6B21\u5F97\u5230\u4E86\u8FD9\u4E2A\u5217\u8868\uFF0C\u7136\u540E\u662F\u4E00\u4E2A\u7A7A\u5217\u8868\uFF1A"),Kl.forEach(t),hn=_(s),h(os.$$.fragment,s),cn=_(s),Ds=o(s,"P",{});var Yl=i(Ds);Ie=l(Yl,"\u8FD9\u5C31\u662F\u6211\u4EEC\u5B9A\u4E49\u4E00\u4E2A\u8FD4\u56DE\u751F\u6210\u5668\u7684\u51FD\u6570\u7684\u539F\u56E0\uFF1A"),Yl.forEach(t),xn=_(s),h(is.$$.fragment,s),mn=_(s),G=o(s,"P",{});var Vs=i(G);Ke=l(Vs,"\u60A8\u8FD8\u53EF\u4EE5\u5728\u4E00\u4E2A "),ht=o(Vs,"STRONG",{});var Fl=i(ht);Ye=l(Fl,"for"),Fl.forEach(t),Fe=l(Vs," \u5FAA\u73AF\u5185\u90E8\u4F7F\u7528 "),ct=o(Vs,"STRONG",{});var Xl=i(ct);Xe=l(Xl,"yield"),Xl.forEach(t),Je=l(Vs," \u5173\u952E\u5B57\u5B9A\u4E49\u60A8\u7684\u751F\u6210\u5668\uFF1A"),Vs.forEach(t),gn=_(s),h(fs.$$.fragment,s),$n=_(s),Us=o(s,"P",{});var Jl=i(Us);Qe=l(Jl,"\u8FD9\u5C06\u4EA7\u751F\u4E0E\u4EE5\u524D\u5B8C\u5168\u76F8\u540C\u7684\u751F\u6210\u5668\uFF0C\u4F46\u5141\u8BB8\u60A8\u4F7F\u7528\u6BD4\u5217\u8868\u751F\u6210\u5F0F\u4E2D\u66F4\u590D\u6742\u7684\u903B\u8F91\u3002"),Jl.forEach(t),jn=_(s),_s=o(s,"H2",{id:!0});var Ql=i(_s);Ze=l(Ql,"\u8BAD\u7EC3\u4E00\u4E2A\u65B0\u7684\u6807\u8BB0\u5668"),Ql.forEach(t),bn=_(s),Ms=o(s,"P",{});var Zl=i(Ms);Ve=l(Zl,"\u73B0\u5728\u6211\u4EEC\u7684\u8BED\u6599\u5E93\u662F\u6587\u672C\u6279\u91CF\u8FED\u4EE3\u5668\u7684\u5F62\u5F0F\uFF0C\u6211\u4EEC\u51C6\u5907\u8BAD\u7EC3\u4E00\u4E2A\u65B0\u7684\u6807\u8BB0\u5668\u3002\u4E3A\u6B64\uFF0C\u6211\u4EEC\u9996\u5148\u9700\u8981\u52A0\u8F7D\u8981\u4E0E\u6A21\u578B\u914D\u5BF9\u7684\u6807\u8BB0\u5668\uFF08\u6B64\u5904\u4E3A GPT-2\uFF09\uFF1A"),Zl.forEach(t),kn=_(s),h(us.$$.fragment,s),wn=_(s),Hs=o(s,"P",{});var Vl=i(Hs);We=l(Vl,"\u5373\u4F7F\u6211\u4EEC\u8981\u8BAD\u7EC3\u4E00\u4E2A\u65B0\u7684\u6807\u8BB0\u5668\uFF0C\u6700\u597D\u8FD8\u662F\u8FD9\u6837\u505A\u4EE5\u907F\u514D\u5B8C\u5168\u4ECE\u5934\u5F00\u59CB\u3002\u8FD9\u6837\uFF0C\u6211\u4EEC\u5C31\u4E0D\u5FC5\u6307\u5B9A\u4EFB\u4F55\u5173\u4E8E\u6807\u8BB0\u5316\u7B97\u6CD5\u6216\u6211\u4EEC\u60F3\u8981\u4F7F\u7528\u7684\u7279\u6B8A\u6807\u8BB0\uFF1B\u6211\u4EEC\u7684\u65B0\u6807\u8BB0\u5668\u5C06\u4E0E GPT-2 \u5B8C\u5168\u76F8\u540C\uFF0C\u552F\u4E00\u4F1A\u6539\u53D8\u7684\u662F\u8F93\u5165\u7684\u6570\u636E\uFF0C\u8FD9\u5C06\u53D6\u51B3\u4E8E\u6211\u4EEC\u8BAD\u7EC3\u7684\u8BED\u6599\u3002"),Vl.forEach(t),dn=_(s),Bs=o(s,"P",{});var Wl=i(Bs);sa=l(Wl,"\u9996\u5148\u8BA9\u6211\u4EEC\u770B\u770B\u8FD9\u4E2A\u6807\u8BB0\u5668\u5C06\u5982\u4F55\u5904\u7406\u793A\u4F8B\u7684\u6570\u636E\uFF1A"),Wl.forEach(t),vn=_(s),h(hs.$$.fragment,s),zn=_(s),h(cs.$$.fragment,s),En=_(s),P=o(s,"P",{});var Y=i(P);ta=l(Y,"\u8FD9\u4E2A\u6807\u8BB0\u5668\u6709\u4E00\u4E9B\u7279\u6B8A\u7684\u7B26\u53F7\uFF0C\u6BD4\u5982 "),xt=o(Y,"STRONG",{});var sr=i(xt);na=l(sr,"\u010A"),sr.forEach(t),ea=l(Y," \u548C "),mt=o(Y,"STRONG",{});var tr=i(mt);aa=l(tr,"\u0120"),tr.forEach(t),la=l(Y," \uFF0C\u5206\u522B\u8868\u793A\u7A7A\u683C\u548C\u6362\u884C\u7B26\u3002\u6B63\u5982\u6211\u4EEC\u6240\u770B\u5230\u7684\uFF0C\u8FD9\u4E0D\u662F\u592A\u6709\u6548\uFF1A\u6807\u8BB0\u5668\u4E3A\u6BCF\u4E2A\u7A7A\u683C\u8FD4\u56DE\u5355\u72EC\u7684\u6807\u8BB0\uFF0C\u5F53\u5B83\u53EF\u4EE5\u5C06\u7F29\u8FDB\u7EA7\u522B\u7EC4\u5408\u5728\u4E00\u8D77\u65F6\uFF08\u56E0\u4E3A\u5728\u4EE3\u7801\u4E2D\u5177\u6709\u56DB\u4E2A\u6216\u516B\u4E2A\u7A7A\u683C\u7684\u96C6\u5408\u5C06\u975E\u5E38\u666E\u904D\uFF09\u3002\u5B83\u4E5F\u6709\u70B9\u5947\u602A\u5730\u62C6\u5206\u4E86\u51FD\u6570\u540D\u79F0\uFF0C\u800C\u4E60\u60EF\u4F7F\u7528"),gt=o(Y,"STRONG",{});var nr=i(gt);ra=l(nr,"_"),nr.forEach(t),pa=l(Y,"\u7684\u51FD\u6570\u547D\u540D\u7684\u65B9\u6CD5\u3002"),Y.forEach(t),yn=_(s),M=o(s,"P",{});var ne=i(M);oa=l(ne,"\u8BA9\u6211\u4EEC\u8BAD\u7EC3\u4E00\u4E2A\u65B0\u7684\u6807\u8BB0\u5668\uFF0C\u770B\u770B\u5B83\u662F\u5426\u80FD\u89E3\u51B3\u8FD9\u4E9B\u95EE\u9898\u3002\u4E3A\u6B64\uFF0C\u6211\u4EEC\u5C06\u4F7F\u7528 "),$t=o(ne,"STRONG",{});var er=i($t);ia=l(er,"train_new_from_iterator()"),er.forEach(t),fa=l(ne," \u65B9\u6CD5\uFF1A"),ne.forEach(t),Pn=_(s),h(xs.$$.fragment,s),qn=_(s),Is=o(s,"P",{});var ar=i(Is);_a=l(ar,"\u5982\u679C\u60A8\u7684\u8BED\u6599\u5E93\u975E\u5E38\u5927\uFF0C\u6B64\u547D\u4EE4\u53EF\u80FD\u9700\u8981\u4E00\u4E9B\u65F6\u95F4\uFF0C\u4F46\u5BF9\u4E8E\u8FD9\u4E2A 1.6 GB \u6587\u672C\u6570\u636E\u96C6\uFF0C\u5B83\u7684\u901F\u5EA6\u975E\u5E38\u5FEB\uFF08\u5728\u5177\u6709 12 \u4E2A\u5185\u6838\u7684 AMD Ryzen 9 3900X CPU \u4E0A\u4E3A 1 \u5206 16 \u79D2\uFF09\u3002"),ar.forEach(t),Tn=_(s),S=o(s,"P",{});var Ws=i(S);ua=l(Ws,"\u6CE8\u610F "),jt=o(Ws,"STRONG",{});var lr=i(jt);ha=l(lr,"AutoTokenizer.train_new_from_iterator()"),lr.forEach(t),ca=l(Ws," \u4EC5\u5F53\u60A8\u4F7F\u7528\u7684\u6807\u8BB0\u5668\u662F\u201C\u5FEB\u901F\uFF08fast\uFF09\u201D\u6807\u8BB0\u5668\u65F6\u624D\u6709\u6548\u3002\u6B63\u5982\u60A8\u5C06\u5728\u4E0B\u4E00\u8282\u4E2D\u770B\u5230\u7684\uFF0C\u{1F917} Transformers \u5E93\u5305\u542B\u4E24\u79CD\u7C7B\u578B\u7684\u6807\u8BB0\u5668\uFF1A\u4E00\u4E9B\u5B8C\u5168\u7528 Python \u7F16\u5199\uFF0C\u800C\u53E6\u4E00\u4E9B\uFF08\u5FEB\u901F\u7684\uFF09\u7531 \u{1F917} Tokenizers \u5E93\u652F\u6301\uFF0C\u8BE5\u5E93\u7528"),ms=o(Ws,"A",{href:!0,rel:!0});var rr=i(ms);xa=l(rr,"Rust"),rr.forEach(t),ma=l(Ws,"\u7F16\u7A0B\u8BED\u8A00\u7F16\u5199\u3002 Python \u662F\u6700\u5E38\u7528\u4E8E\u6570\u636E\u79D1\u5B66\u548C\u6DF1\u5EA6\u5B66\u4E60\u5E94\u7528\u7A0B\u5E8F\u7684\u8BED\u8A00\uFF0C\u4F46\u662F\u5F53\u9700\u8981\u5E76\u884C\u5316\u4EE5\u63D0\u9AD8\u901F\u5EA6\u65F6\uFF0C\u5FC5\u987B\u7528\u53E6\u4E00\u79CD\u8BED\u8A00\u7F16\u5199\u3002\u4F8B\u5982\uFF0C\u6A21\u578B\u8BA1\u7B97\u6838\u5FC3\u7684\u77E9\u9635\u4E58\u6CD5\u662F\u7528 CUDA \u7F16\u5199\u7684\uFF0CCUDA \u662F\u4E00\u4E2A\u9488\u5BF9 GPU \u7684\u4F18\u5316 C \u5E93\u3002"),Ws.forEach(t),An=_(s),H=o(s,"P",{});var ee=i(H);ga=l(ee,"\u7528\u7EAF Python \u8BAD\u7EC3\u4E00\u4E2A\u5168\u65B0\u7684\u6807\u8BB0\u5668\u4F1A\u975E\u5E38\u7F13\u6162\uFF0C\u8FD9\u5C31\u662F\u6211\u4EEC\u5F00\u53D1 \u{1F917} Tokenizers\u5E93\u7684\u539F\u56E0\u3002\u8BF7\u6CE8\u610F\uFF0C\u6B63\u5982\u60A8\u65E0\u9700\u5B66\u4E60 CUDA \u8BED\u8A00\u5373\u53EF\u5728 GPU \u4E0A\u6267\u884C\u60A8\u7684\u6A21\u578B\u4E00\u6837\uFF0C\u60A8\u4E5F\u65E0\u9700\u5B66\u4E60 Rust \u5373\u53EF\u4F7F\u7528\u5FEB\u901F\u6807\u8BB0\u5668\u3002 \u{1F917} Tokenizers \u5E93\u4E3A\u8BB8\u591A\u5185\u90E8\u8C03\u7528 Rust \u4EE3\u7801\u7684\u65B9\u6CD5\u63D0\u4F9B Python \u7ED1\u5B9A\uFF1B\u4F8B\u5982\uFF0C\u5E76\u884C\u5316\u65B0\u6807\u8BB0\u5668\u7684\u8BAD\u7EC3\uFF0C\u6216\u8005\uFF0C\u6B63\u5982\u6211\u4EEC\u5728"),Ks=o(ee,"A",{href:!0});var pr=i(Ks);$a=l(pr,"\u7B2C\u4E09\u7AE0"),pr.forEach(t),ja=l(ee,"\u4E2D\u770B\u5230\u7684\uFF0C\u5BF9\u4E00\u6279\u8F93\u5165\u8FDB\u884C\u6807\u8BB0\u5316\u3002"),ee.forEach(t),Gn=_(s),R=o(s,"P",{});var st=i(R);ba=l(st,"\u5927\u591A\u6570 Transformer \u6A21\u578B\u90FD\u6709\u53EF\u7528\u7684\u5FEB\u901F\u6807\u8BB0\u5668\uFF08\u60A8\u53EF\u4EE5"),gs=o(st,"A",{href:!0,rel:!0});var or=i(gs);ka=l(or,"\u5728\u8FD9\u91CC"),or.forEach(t),wa=l(st,"\u68C0\u67E5\u4E00\u4E9B\u4F8B\u5916\u60C5\u51B5)\uFF0C\u5982\u679C "),bt=o(st,"STRONG",{});var ir=i(bt);da=l(ir,"AutoTokenizer"),ir.forEach(t),va=l(st," \u53EF\u7528\uFF0CAPI \u603B\u662F\u4E3A\u60A8\u9009\u62E9\u5FEB\u901F\u6807\u8BB0\u5668\u3002\u5728\u4E0B\u4E00\u8282\u4E2D\uFF0C\u6211\u4EEC\u5C06\u770B\u770B\u5FEB\u901F\u6807\u8BB0\u5668\u5177\u6709\u7684\u5176\u4ED6\u4E00\u4E9B\u7279\u6B8A\u529F\u80FD\uFF0C\u8FD9\u4E9B\u529F\u80FD\u5BF9\u4E8E\u6807\u8BB0\u5206\u7C7B\u548C\u95EE\u7B54\u7B49\u4EFB\u52A1\u975E\u5E38\u6709\u7528\u3002\u7136\u800C\uFF0C\u5728\u6DF1\u5165\u7814\u7A76\u4E4B\u524D\uFF0C\u8BA9\u6211\u4EEC\u5728\u4E0A\u4E00\u4E2A\u793A\u4F8B\u4E2D\u5C1D\u8BD5\u6211\u4EEC\u5168\u65B0\u7684\u6807\u8BB0\u5668\uFF1A"),st.forEach(t),Sn=_(s),h($s.$$.fragment,s),Rn=_(s),h(js.$$.fragment,s),Nn=_(s),d=o(s,"P",{});var q=i(d);za=l(q,"\u5728\u8FD9\u91CC\u6211\u4EEC\u518D\u6B21\u770B\u5230\u7279\u6B8A\u7B26\u53F7 "),kt=o(q,"STRONG",{});var fr=i(kt);Ea=l(fr,"\u010A"),fr.forEach(t),ya=l(q," \u548C "),wt=o(q,"STRONG",{});var _r=i(wt);Pa=l(_r,"\u0120"),_r.forEach(t),qa=l(q," \u8868\u793A\u7A7A\u683C\u548C\u6362\u884C\u7B26\uFF0C\u4F46\u6211\u4EEC\u4E5F\u53EF\u4EE5\u770B\u5230\u6211\u4EEC\u7684\u6807\u8BB0\u5668\u5B66\u4E60\u4E86\u4E00\u4E9B\u9AD8\u5EA6\u7279\u5B9A\u4E8E Python \u51FD\u6570\u8BED\u6599\u5E93\u7684\u6807\u8BB0\uFF1A\u4F8B\u5982\uFF0C\u6709\u4E00\u4E2A "),dt=o(q,"STRONG",{});var ur=i(dt);Ta=l(ur,"\u010A\u0120\u0120\u0120"),ur.forEach(t),Aa=l(q," \u8868\u793A\u7F29\u8FDB\u7684\u6807\u8BB0\uFF0C\u4EE5\u53CA "),vt=o(q,"STRONG",{});var hr=i(vt);Ga=l(hr,"\u0120"),hr.forEach(t),Sa=l(q," \u8868\u793A\u5F00\u59CB\u6587\u6863\u5B57\u7B26\u4E32\u7684\u4E09\u4E2A\u5F15\u53F7\u7684\u6807\u8BB0\u3002\u6807\u8BB0\u5668\u8FD8\u6B63\u786E\u4F7F\u7528"),zt=o(q,"STRONG",{});var cr=i(zt);Ra=l(cr,"_"),cr.forEach(t),Na=l(q,"\u547D\u540D\u7684\u89C4\u8303\u5C06\u51FD\u6570\u540D\u79F0\u62C6\u5206\u4E3A  .\u8FD9\u662F\u4E00\u4E2A\u975E\u5E38\u7D27\u51D1\u7684\u8868\u793A\uFF1B\u76F8\u6BD4\u4E4B\u4E0B\uFF0C\u5728\u540C\u4E00\u4E2A\u4F8B\u5B50\u4E2D\u4F7F\u7528\u7B80\u5355\u7684\u82F1\u8BED\u6807\u8BB0\u5668\u4F1A\u7ED9\u6211\u4EEC\u4E00\u4E2A\u66F4\u957F\u7684\u53E5\u5B50\uFF1A"),q.forEach(t),On=_(s),h(bs.$$.fragment,s),Cn=_(s),h(ks.$$.fragment,s),Ln=_(s),Ys=o(s,"P",{});var xr=i(Ys);Oa=l(xr,"\u8BA9\u6211\u4EEC\u518D\u770B\u4E00\u4E2A\u4F8B\u5B50\uFF1A"),xr.forEach(t),Dn=_(s),h(ws.$$.fragment,s),Un=_(s),h(ds.$$.fragment,s),Mn=_(s),$=o(s,"P",{});var b=i($);Ca=l(b,"\u9664\u4E86\u4E00\u4E2A\u7F29\u8FDB\u5BF9\u5E94\u7684token\uFF0C\u8FD9\u91CC\u6211\u4EEC\u8FD8\u53EF\u4EE5\u770B\u5230\u4E00\u4E2A\u53CC\u7F29\u8FDB\u7684token\uFF1A "),Et=o(b,"STRONG",{});var mr=i(Et);La=l(mr,"\u010A\u0120\u0120\u0120\u0120\u0120\u0120\u0120"),mr.forEach(t),Da=l(b," .\u7279\u6B8A\u7684 Python \u8BCD\u5982 "),yt=o(b,"STRONG",{});var gr=i(yt);Ua=l(gr,"class"),gr.forEach(t),Ma=l(b," , "),Pt=o(b,"STRONG",{});var $r=i(Pt);Ha=l($r,"init"),$r.forEach(t),Ba=l(b," , "),qt=o(b,"STRONG",{});var jr=i(qt);Ia=l(jr,"call"),jr.forEach(t),Ka=l(b," , "),Tt=o(b,"STRONG",{});var br=i(Tt);Ya=l(br,"self"),br.forEach(t),Fa=l(b," \uFF0C \u548C "),At=o(b,"STRONG",{});var kr=i(At);Xa=l(kr,"return"),kr.forEach(t),Ja=l(b," \u6BCF\u4E2A\u90FD\u88AB\u6807\u8BB0\u4E3A\u4E00\u4E2A\u6807\u8BB0\uFF0C\u6211\u4EEC\u53EF\u4EE5\u770B\u5230\uFF0C\u4EE5\u53CA\u5206\u88C2 "),Gt=o(b,"STRONG",{});var wr=i(Gt);Qa=l(wr,"_"),wr.forEach(t),Za=l(b," \u548C "),St=o(b,"STRONG",{});var dr=i(St);Va=l(dr,"."),dr.forEach(t),Wa=l(b," \u6807\u8BB0\u5668\u751A\u81F3\u53EF\u4EE5\u6B63\u786E\u62C6\u5206\u9A7C\u5CF0\u5F0F\u540D\u79F0\uFF1A "),Rt=o(b,"STRONG",{});var vr=i(Rt);sl=l(vr,"LinearLayer"),vr.forEach(t),tl=l(b," \u88AB\u6807\u8BB0\u4E3A "),Nt=o(b,"STRONG",{});var zr=i(Nt);nl=l(zr,"[\u0120Linear, Layer]"),zr.forEach(t),el=l(b," ."),b.forEach(t),Hn=_(s),vs=o(s,"H2",{id:!0});var Er=i(vs);al=l(Er,"\u4FDD\u5B58\u6807\u8BB0\u5668"),Er.forEach(t),Bn=_(s),B=o(s,"P",{});var ae=i(B);ll=l(ae,"\u4E3A\u4E86\u786E\u4FDD\u6211\u4EEC\u4EE5\u540E\u53EF\u4EE5\u4F7F\u7528\u5B83\uFF0C\u6211\u4EEC\u9700\u8981\u4FDD\u5B58\u6211\u4EEC\u7684\u65B0\u6807\u8BB0\u5668\u3002\u5C31\u50CF\u6A21\u578B\u4E00\u6837\uFF0C\u662F\u901A\u8FC7 "),Ot=o(ae,"STRONG",{});var yr=i(Ot);rl=l(yr,"save_pretrained()"),yr.forEach(t),pl=l(ae," \u65B9\u6CD5\uFF1A"),ae.forEach(t),In=_(s),h(zs.$$.fragment,s),Kn=_(s),I=o(s,"P",{});var le=i(I);ol=l(le,"\u8FD9\u5C06\u521B\u5EFA\u4E00\u4E2A\u540D\u4E3A\u7684"),Ct=o(le,"EM",{});var Pr=i(Ct);il=l(Pr,"code-search-net-tokenizer"),Pr.forEach(t),fl=l(le,"\u7684\u65B0\u6587\u4EF6\u5939\uFF0C\u5B83\u5C06\u5305\u542B\u91CD\u65B0\u52A0\u8F7D\u6807\u8BB0\u5668\u6240\u9700\u8981\u7684\u6240\u6709\u6587\u4EF6\u3002\u5982\u679C\u60A8\u60F3\u4E0E\u60A8\u7684\u540C\u4E8B\u548C\u670B\u53CB\u5206\u4EAB\u8FD9\u4E2A\u6807\u8BB0\u5668\uFF0C\u60A8\u53EF\u4EE5\u901A\u8FC7\u767B\u5F55\u60A8\u7684\u5E10\u6237\u5C06\u5176\u4E0A\u4F20\u5230 Hub\u3002\u5982\u679C\u60A8\u5728notebook\u4E0A\u5DE5\u4F5C\uFF0C\u6709\u4E00\u4E2A\u65B9\u4FBF\u7684\u529F\u80FD\u53EF\u4EE5\u5E2E\u52A9\u60A8\uFF1A"),le.forEach(t),Yn=_(s),h(Es.$$.fragment,s),Fn=_(s),Fs=o(s,"P",{});var qr=i(Fs);_l=l(qr,"\u8FD9\u5C06\u663E\u793A\u4E00\u4E2A\u5C0F\u90E8\u4EF6\uFF0C\u60A8\u53EF\u4EE5\u5728\u5176\u4E2D\u8F93\u5165\u60A8\u7684 Hugging Face \u767B\u5F55\u51ED\u636E\u3002\u5982\u679C\u60A8\u4E0D\u662F\u5728notebook\u4E0A\u5DE5\u4F5C\uFF0C\u53EA\u9700\u5728\u7EC8\u7AEF\u4E2D\u8F93\u5165\u4EE5\u4E0B\u884C\uFF1A"),qr.forEach(t),Xn=_(s),h(ys.$$.fragment,s),Jn=_(s),Xs=o(s,"P",{});var Tr=i(Xs);ul=l(Tr,"\u767B\u5F55\u540E\uFF0C\u60A8\u53EF\u4EE5\u901A\u8FC7\u6267\u884C\u4EE5\u4E0B\u547D\u4EE4\u6765\u63A8\u9001\u60A8\u7684\u6807\u8BB0\u5668\uFF1A"),Tr.forEach(t),Qn=_(s),h(Ps.$$.fragment,s),Zn=_(s),N=o(s,"P",{});var tt=i(N);hl=l(tt,"\u8FD9\u5C06\u5728\u60A8\u7684\u547D\u540D\u7A7A\u95F4\u4E2D\u521B\u5EFA\u4E00\u4E2A\u540D\u4E3A"),Lt=o(tt,"STRONG",{});var Ar=i(Lt);cl=l(Ar,"code-search-net-tokenizer"),Ar.forEach(t),xl=l(tt,"\u7684\u65B0\u5B58\u50A8\u5E93  \uFF0C\u5305\u542B\u6807\u8BB0\u5668\u6587\u4EF6\u3002\u7136\u540E\uFF0C\u60A8\u53EF\u4EE5\u4F7F\u7528\u4EE5\u4E0B\u547D\u4EE4\u4ECE\u4EFB\u4F55\u5730\u65B9\u52A0\u8F7D\u6807\u8BB0\u5668\u7684 "),Dt=o(tt,"STRONG",{});var Gr=i(Dt);ml=l(Gr,"from_pretrained()"),Gr.forEach(t),gl=l(tt," \u65B9\u6CD5\uFF1A"),tt.forEach(t),Vn=_(s),h(qs.$$.fragment,s),Wn=_(s),O=o(s,"P",{});var nt=i(O);$l=l(nt,"\u60A8\u73B0\u5728\u5DF2\u51C6\u5907\u597D\u4ECE\u5934\u5F00\u59CB\u8BAD\u7EC3\u8BED\u8A00\u6A21\u578B\u5E76\u6839\u636E\u60A8\u624B\u5934\u7684\u4EFB\u52A1\u5BF9\u5176\u8FDB\u884C\u5FAE\u8C03\uFF01\u6211\u4EEC\u5C06\u5728"),Js=o(nt,"A",{href:!0});var Sr=i(Js);jl=l(Sr,"\u7B2C\u4E03\u7AE0"),Sr.forEach(t),bl=l(nt,"\u8FDB\u884C\u8FD9\u90E8\u5206\u3002\u4F46\u9996\u5148\uFF0C\u5728\u672C\u7AE0\u7684\u5176\u4F59\u90E8\u5206\uFF0C\u6211\u4EEC\u5C06\u4ED4\u7EC6\u7814\u7A76\u5FEB\u901F\u6807\u8BB0\u5668\uFF0C\u5E76\u8BE6\u7EC6\u63A2\u8BA8\u8C03\u7528 "),Ut=o(nt,"STRONG",{});var Rr=i(Ut);kl=l(Rr,"train_new_from_iterator()"),Rr.forEach(t),wl=l(nt," \u65B9\u6CD5\u65F6\u5B9E\u9645\u53D1\u751F\u7684\u60C5\u51B5 ."),nt.forEach(t),this.h()},h(){k(w,"name","hf:doc:metadata"),k(w,"content",JSON.stringify(Yr)),k(z,"id","tokenizertokenizer"),k(z,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),k(z,"href","#tokenizertokenizer"),k(v,"class","relative group"),k(As,"href","/course/chapter2"),k(Q,"id",""),k(Z,"href","https://github.com/huggingface/datasets"),k(Z,"rel","nofollow"),k(V,"href","https://huggingface.co/datasets/code_search_net"),k(V,"rel","nofollow"),k(W,"href","https://wandb.ai/github/CodeSearchNet/benchmark"),k(W,"rel","nofollow"),k(_s,"id",""),k(ms,"href","https://www.rust-lang.org"),k(ms,"rel","nofollow"),k(Ks,"href","/course/chapter3"),k(gs,"href","https://huggingface.co/transformers/#supported-frameworks"),k(gs,"rel","nofollow"),k(vs,"id",""),k(Js,"href","/course/chapter7")},m(s,e){n(document.head,w),r(s,L,e),r(s,v,e),n(v,z),n(z,et),c(F,et,null),n(v,re),n(v,at),n(at,pe),r(s,Ht,e),c(X,s,e),r(s,Bt,e),r(s,y,e),n(y,oe),n(y,As),n(As,ie),n(y,fe),n(y,lt),n(lt,_e),n(y,ue),n(y,rt),n(rt,he),n(y,ce),r(s,It,e),c(J,s,e),r(s,Kt,e),c(D,s,e),r(s,Yt,e),r(s,Q,e),n(Q,xe),r(s,Ft,e),r(s,U,e),n(U,me),n(U,pt),n(pt,ge),n(U,$e),r(s,Xt,e),r(s,E,e),n(E,Z),n(Z,je),n(E,be),n(E,ot),n(ot,ke),n(E,we),n(E,V),n(V,de),n(E,ve),n(E,W),n(W,ze),n(E,Ee),r(s,Jt,e),c(ss,s,e),r(s,Qt,e),r(s,Gs,e),n(Gs,ye),r(s,Zt,e),c(ts,s,e),r(s,Vt,e),c(ns,s,e),r(s,Wt,e),r(s,T,e),n(T,Pe),n(T,it),n(it,qe),n(T,Te),n(T,ft),n(ft,Ae),n(T,Ge),r(s,sn,e),c(es,s,e),r(s,tn,e),r(s,Ss,e),n(Ss,Se),r(s,nn,e),c(as,s,e),r(s,en,e),r(s,Rs,e),n(Rs,Re),r(s,an,e),r(s,Ns,e),n(Ns,Ne),r(s,ln,e),c(ls,s,e),r(s,rn,e),r(s,Os,e),n(Os,Oe),r(s,pn,e),c(rs,s,e),r(s,on,e),r(s,A,e),n(A,Ce),n(A,_t),n(_t,Le),n(A,De),n(A,ut),n(ut,Ue),n(A,Me),r(s,fn,e),r(s,Cs,e),n(Cs,He),r(s,_n,e),c(ps,s,e),r(s,un,e),r(s,Ls,e),n(Ls,Be),r(s,hn,e),c(os,s,e),r(s,cn,e),r(s,Ds,e),n(Ds,Ie),r(s,xn,e),c(is,s,e),r(s,mn,e),r(s,G,e),n(G,Ke),n(G,ht),n(ht,Ye),n(G,Fe),n(G,ct),n(ct,Xe),n(G,Je),r(s,gn,e),c(fs,s,e),r(s,$n,e),r(s,Us,e),n(Us,Qe),r(s,jn,e),r(s,_s,e),n(_s,Ze),r(s,bn,e),r(s,Ms,e),n(Ms,Ve),r(s,kn,e),c(us,s,e),r(s,wn,e),r(s,Hs,e),n(Hs,We),r(s,dn,e),r(s,Bs,e),n(Bs,sa),r(s,vn,e),c(hs,s,e),r(s,zn,e),c(cs,s,e),r(s,En,e),r(s,P,e),n(P,ta),n(P,xt),n(xt,na),n(P,ea),n(P,mt),n(mt,aa),n(P,la),n(P,gt),n(gt,ra),n(P,pa),r(s,yn,e),r(s,M,e),n(M,oa),n(M,$t),n($t,ia),n(M,fa),r(s,Pn,e),c(xs,s,e),r(s,qn,e),r(s,Is,e),n(Is,_a),r(s,Tn,e),r(s,S,e),n(S,ua),n(S,jt),n(jt,ha),n(S,ca),n(S,ms),n(ms,xa),n(S,ma),r(s,An,e),r(s,H,e),n(H,ga),n(H,Ks),n(Ks,$a),n(H,ja),r(s,Gn,e),r(s,R,e),n(R,ba),n(R,gs),n(gs,ka),n(R,wa),n(R,bt),n(bt,da),n(R,va),r(s,Sn,e),c($s,s,e),r(s,Rn,e),c(js,s,e),r(s,Nn,e),r(s,d,e),n(d,za),n(d,kt),n(kt,Ea),n(d,ya),n(d,wt),n(wt,Pa),n(d,qa),n(d,dt),n(dt,Ta),n(d,Aa),n(d,vt),n(vt,Ga),n(d,Sa),n(d,zt),n(zt,Ra),n(d,Na),r(s,On,e),c(bs,s,e),r(s,Cn,e),c(ks,s,e),r(s,Ln,e),r(s,Ys,e),n(Ys,Oa),r(s,Dn,e),c(ws,s,e),r(s,Un,e),c(ds,s,e),r(s,Mn,e),r(s,$,e),n($,Ca),n($,Et),n(Et,La),n($,Da),n($,yt),n(yt,Ua),n($,Ma),n($,Pt),n(Pt,Ha),n($,Ba),n($,qt),n(qt,Ia),n($,Ka),n($,Tt),n(Tt,Ya),n($,Fa),n($,At),n(At,Xa),n($,Ja),n($,Gt),n(Gt,Qa),n($,Za),n($,St),n(St,Va),n($,Wa),n($,Rt),n(Rt,sl),n($,tl),n($,Nt),n(Nt,nl),n($,el),r(s,Hn,e),r(s,vs,e),n(vs,al),r(s,Bn,e),r(s,B,e),n(B,ll),n(B,Ot),n(Ot,rl),n(B,pl),r(s,In,e),c(zs,s,e),r(s,Kn,e),r(s,I,e),n(I,ol),n(I,Ct),n(Ct,il),n(I,fl),r(s,Yn,e),c(Es,s,e),r(s,Fn,e),r(s,Fs,e),n(Fs,_l),r(s,Xn,e),c(ys,s,e),r(s,Jn,e),r(s,Xs,e),n(Xs,ul),r(s,Qn,e),c(Ps,s,e),r(s,Zn,e),r(s,N,e),n(N,hl),n(N,Lt),n(Lt,cl),n(N,xl),n(N,Dt),n(Dt,ml),n(N,gl),r(s,Vn,e),c(qs,s,e),r(s,Wn,e),r(s,O,e),n(O,$l),n(O,Js),n(Js,jl),n(O,bl),n(O,Ut),n(Ut,kl),n(O,wl),se=!0},p(s,[e]){const Ts={};e&2&&(Ts.$$scope={dirty:e,ctx:s}),D.$set(Ts)},i(s){se||(x(F.$$.fragment,s),x(X.$$.fragment,s),x(J.$$.fragment,s),x(D.$$.fragment,s),x(ss.$$.fragment,s),x(ts.$$.fragment,s),x(ns.$$.fragment,s),x(es.$$.fragment,s),x(as.$$.fragment,s),x(ls.$$.fragment,s),x(rs.$$.fragment,s),x(ps.$$.fragment,s),x(os.$$.fragment,s),x(is.$$.fragment,s),x(fs.$$.fragment,s),x(us.$$.fragment,s),x(hs.$$.fragment,s),x(cs.$$.fragment,s),x(xs.$$.fragment,s),x($s.$$.fragment,s),x(js.$$.fragment,s),x(bs.$$.fragment,s),x(ks.$$.fragment,s),x(ws.$$.fragment,s),x(ds.$$.fragment,s),x(zs.$$.fragment,s),x(Es.$$.fragment,s),x(ys.$$.fragment,s),x(Ps.$$.fragment,s),x(qs.$$.fragment,s),se=!0)},o(s){m(F.$$.fragment,s),m(X.$$.fragment,s),m(J.$$.fragment,s),m(D.$$.fragment,s),m(ss.$$.fragment,s),m(ts.$$.fragment,s),m(ns.$$.fragment,s),m(es.$$.fragment,s),m(as.$$.fragment,s),m(ls.$$.fragment,s),m(rs.$$.fragment,s),m(ps.$$.fragment,s),m(os.$$.fragment,s),m(is.$$.fragment,s),m(fs.$$.fragment,s),m(us.$$.fragment,s),m(hs.$$.fragment,s),m(cs.$$.fragment,s),m(xs.$$.fragment,s),m($s.$$.fragment,s),m(js.$$.fragment,s),m(bs.$$.fragment,s),m(ks.$$.fragment,s),m(ws.$$.fragment,s),m(ds.$$.fragment,s),m(zs.$$.fragment,s),m(Es.$$.fragment,s),m(ys.$$.fragment,s),m(Ps.$$.fragment,s),m(qs.$$.fragment,s),se=!1},d(s){t(w),s&&t(L),s&&t(v),g(F),s&&t(Ht),g(X,s),s&&t(Bt),s&&t(y),s&&t(It),g(J,s),s&&t(Kt),g(D,s),s&&t(Yt),s&&t(Q),s&&t(Ft),s&&t(U),s&&t(Xt),s&&t(E),s&&t(Jt),g(ss,s),s&&t(Qt),s&&t(Gs),s&&t(Zt),g(ts,s),s&&t(Vt),g(ns,s),s&&t(Wt),s&&t(T),s&&t(sn),g(es,s),s&&t(tn),s&&t(Ss),s&&t(nn),g(as,s),s&&t(en),s&&t(Rs),s&&t(an),s&&t(Ns),s&&t(ln),g(ls,s),s&&t(rn),s&&t(Os),s&&t(pn),g(rs,s),s&&t(on),s&&t(A),s&&t(fn),s&&t(Cs),s&&t(_n),g(ps,s),s&&t(un),s&&t(Ls),s&&t(hn),g(os,s),s&&t(cn),s&&t(Ds),s&&t(xn),g(is,s),s&&t(mn),s&&t(G),s&&t(gn),g(fs,s),s&&t($n),s&&t(Us),s&&t(jn),s&&t(_s),s&&t(bn),s&&t(Ms),s&&t(kn),g(us,s),s&&t(wn),s&&t(Hs),s&&t(dn),s&&t(Bs),s&&t(vn),g(hs,s),s&&t(zn),g(cs,s),s&&t(En),s&&t(P),s&&t(yn),s&&t(M),s&&t(Pn),g(xs,s),s&&t(qn),s&&t(Is),s&&t(Tn),s&&t(S),s&&t(An),s&&t(H),s&&t(Gn),s&&t(R),s&&t(Sn),g($s,s),s&&t(Rn),g(js,s),s&&t(Nn),s&&t(d),s&&t(On),g(bs,s),s&&t(Cn),g(ks,s),s&&t(Ln),s&&t(Ys),s&&t(Dn),g(ws,s),s&&t(Un),g(ds,s),s&&t(Mn),s&&t($),s&&t(Hn),s&&t(vs),s&&t(Bn),s&&t(B),s&&t(In),g(zs,s),s&&t(Kn),s&&t(I),s&&t(Yn),g(Es,s),s&&t(Fn),s&&t(Fs),s&&t(Xn),g(ys,s),s&&t(Jn),s&&t(Xs),s&&t(Qn),g(Ps,s),s&&t(Zn),s&&t(N),s&&t(Vn),g(qs,s),s&&t(Wn),s&&t(O)}}}const Yr={local:"tokenizertokenizer",sections:[{local:"",title:"\u51C6\u5907\u8BED\u6599\u5E93"},{local:"",title:"\u8BAD\u7EC3\u4E00\u4E2A\u65B0\u7684\u6807\u8BB0\u5668"},{local:"",title:"\u4FDD\u5B58\u6807\u8BB0\u5668"}],title:"\u6839\u636E\u5DF2\u6709\u7684tokenizer\u8BAD\u7EC3\u65B0\u7684tokenizer"};function Fr(Mt){return Dr(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class tp extends Nr{constructor(w){super();Or(this,w,Fr,Kr,Cr,{})}}export{tp as default,Yr as metadata};
