import{S as Eo,i as yo,s as To,e as o,k as c,w as g,t as a,M as zo,c as r,d as e,m as h,a as i,x as j,h as n,b as f,F as t,g as p,y as w,q as k,o as b,B as $}from"../../chunks/vendor-4833417e.js";import{T as pa}from"../../chunks/Tip-fffd6df1.js";import{Y as xo}from"../../chunks/Youtube-27813aed.js";import{I as Xt}from"../../chunks/IconCopyLink-4b81c553.js";import{C}from"../../chunks/CodeBlock-6a3d1b46.js";import{C as qo}from"../../chunks/CodeBlockFw-27a176a0.js";import"../../chunks/CopyButton-dacfbfaf.js";function Co(P){let m,x,d,_,y;return{c(){m=o("p"),x=a("See the token classification "),d=o("a"),_=a("task page"),y=a(" for more information about other forms of token classification and their associated models, datasets, and metrics."),this.h()},l(u){m=r(u,"P",{});var v=i(m);x=n(v,"See the token classification "),d=r(v,"A",{href:!0,rel:!0});var T=i(d);_=n(T,"task page"),T.forEach(e),y=n(v," for more information about other forms of token classification and their associated models, datasets, and metrics."),v.forEach(e),this.h()},h(){f(d,"href","https://huggingface.co/tasks/token-classification"),f(d,"rel","nofollow")},m(u,v){p(u,m,v),t(m,x),t(m,d),t(d,_),t(m,y)},d(u){u&&e(m)}}}function Ao(P){let m,x,d,_,y,u,v,T;return{c(){m=o("p"),x=a("If you aren\u2019t familiar with fine-tuning a model with the "),d=o("a"),_=a("Trainer"),y=a(", take a look at the basic tutorial "),u=o("a"),v=a("here"),T=a("!"),this.h()},l(z){m=r(z,"P",{});var E=i(m);x=n(E,"If you aren\u2019t familiar with fine-tuning a model with the "),d=r(E,"A",{href:!0});var A=i(d);_=n(A,"Trainer"),A.forEach(e),y=n(E,", take a look at the basic tutorial "),u=r(E,"A",{href:!0});var O=i(u);v=n(O,"here"),O.forEach(e),T=n(E,"!"),E.forEach(e),this.h()},h(){f(d,"href","/docs/transformers/pr_15839/en/main_classes/trainer#transformers.Trainer"),f(u,"href","training#finetune-with-trainer")},m(z,E){p(z,m,E),t(m,x),t(m,d),t(d,_),t(m,y),t(m,u),t(u,v),t(m,T)},d(z){z&&e(m)}}}function Do(P){let m,x,d,_,y;return{c(){m=o("p"),x=a("If you aren\u2019t familiar with fine-tuning a model with Keras, take a look at the basic tutorial "),d=o("a"),_=a("here"),y=a("!"),this.h()},l(u){m=r(u,"P",{});var v=i(m);x=n(v,"If you aren\u2019t familiar with fine-tuning a model with Keras, take a look at the basic tutorial "),d=r(v,"A",{href:!0});var T=i(d);_=n(T,"here"),T.forEach(e),y=n(v,"!"),v.forEach(e),this.h()},h(){f(d,"href","training#finetune-with-keras")},m(u,v){p(u,m,v),t(m,x),t(m,d),t(d,_),t(m,y)},d(u){u&&e(m)}}}function Fo(P){let m,x,d,_,y,u,v,T;return{c(){m=o("p"),x=a(`For a more in-depth example of how to fine-tune a model for token classification, take a look at the corresponding
`),d=o("a"),_=a("PyTorch notebook"),y=a(`
or `),u=o("a"),v=a("TensorFlow notebook"),T=a("."),this.h()},l(z){m=r(z,"P",{});var E=i(m);x=n(E,`For a more in-depth example of how to fine-tune a model for token classification, take a look at the corresponding
`),d=r(E,"A",{href:!0,rel:!0});var A=i(d);_=n(A,"PyTorch notebook"),A.forEach(e),y=n(E,`
or `),u=r(E,"A",{href:!0,rel:!0});var O=i(u);v=n(O,"TensorFlow notebook"),O.forEach(e),T=n(E,"."),E.forEach(e),this.h()},h(){f(d,"href","https://colab.research.google.com/github/huggingface/notebooks/blob/master/examples/token_classification.ipynb"),f(d,"rel","nofollow"),f(u,"href","https://colab.research.google.com/github/huggingface/notebooks/blob/master/examples/token_classification-tf.ipynb"),f(u,"rel","nofollow")},m(z,E){p(z,m,E),t(m,x),t(m,d),t(d,_),t(m,y),t(m,u),t(u,v),t(m,T)},d(z){z&&e(m)}}}function Po(P){let m,x,d,_,y,u,v,T,z,E,A,O,Ks,ca,se,L,ha,ms,fa,ma,ds,da,ua,te,Z,ee,H,J,dt,us,_a,ut,ga,ae,Vs,ja,ne,_s,le,Zs,wa,oe,gs,re,G,ka,_t,ba,$a,ie,js,pe,N,va,gt,xa,Ea,jt,ya,Ta,ce,I,Js,wt,za,qa,Ca,B,kt,Aa,Da,bt,Fa,Pa,$t,Sa,Oa,La,Gs,vt,Na,Ia,he,Y,Q,xt,ws,Ba,Et,Ma,fe,ks,me,X,Ra,yt,Ua,Wa,de,bs,ue,ss,Ha,Tt,Ya,Ka,_e,$s,ge,M,Va,zt,Za,Ja,qt,Ga,Qa,je,R,vs,Xa,xs,Ct,sn,tn,en,S,an,At,nn,ln,Dt,on,rn,Ft,pn,cn,hn,Es,fn,Pt,mn,dn,we,Qs,un,ke,ys,be,D,_n,Ts,St,gn,jn,Ot,wn,kn,Lt,bn,$n,$e,zs,ve,q,vn,Xs,xn,En,Nt,yn,Tn,It,zn,qn,Bt,Cn,An,xe,qs,Ee,K,ts,Mt,Cs,Dn,Rt,Fn,ye,es,Pn,st,Sn,On,Te,As,ze,as,qe,tt,Ln,Ce,U,Ds,Nn,et,In,Bn,Mn,Fs,Rn,at,Un,Wn,Hn,Ps,Yn,nt,Kn,Vn,Ae,Ss,De,V,ns,Ut,Os,Zn,Wt,Jn,Fe,lt,Gn,Pe,ls,Se,F,Qn,Ht,Xn,sl,Ls,Yt,tl,el,Kt,al,nl,Oe,Ns,Le,ot,ll,Ne,Is,Ie,os,ol,rt,rl,il,Be,Bs,Me,rs,pl,Ms,Vt,cl,hl,Re,Rs,Ue,is,fl,Us,Zt,ml,dl,We,Ws,He,ps,Ye;return u=new Xt({}),A=new xo({props:{id:"wVHdVlPScxA"}}),Z=new pa({props:{$$slots:{default:[Co]},$$scope:{ctx:P}}}),us=new Xt({}),_s=new C({props:{code:`from datasets import load_dataset

wnut = load_dataset("wnut_17")`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> datasets <span class="hljs-keyword">import</span> load_dataset

<span class="hljs-meta">&gt;&gt;&gt; </span>wnut = load_dataset(<span class="hljs-string">&quot;wnut_17&quot;</span>)`}}),gs=new C({props:{code:'wnut["train"][0]',highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>wnut[<span class="hljs-string">&quot;train&quot;</span>][<span class="hljs-number">0</span>]
{<span class="hljs-string">&#x27;id&#x27;</span>: <span class="hljs-string">&#x27;0&#x27;</span>,
 <span class="hljs-string">&#x27;ner_tags&#x27;</span>: [<span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">7</span>, <span class="hljs-number">8</span>, <span class="hljs-number">8</span>, <span class="hljs-number">0</span>, <span class="hljs-number">7</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>],
 <span class="hljs-string">&#x27;tokens&#x27;</span>: [<span class="hljs-string">&#x27;@paulwalk&#x27;</span>, <span class="hljs-string">&#x27;It&#x27;</span>, <span class="hljs-string">&quot;&#x27;s&quot;</span>, <span class="hljs-string">&#x27;the&#x27;</span>, <span class="hljs-string">&#x27;view&#x27;</span>, <span class="hljs-string">&#x27;from&#x27;</span>, <span class="hljs-string">&#x27;where&#x27;</span>, <span class="hljs-string">&#x27;I&#x27;</span>, <span class="hljs-string">&quot;&#x27;m&quot;</span>, <span class="hljs-string">&#x27;living&#x27;</span>, <span class="hljs-string">&#x27;for&#x27;</span>, <span class="hljs-string">&#x27;two&#x27;</span>, <span class="hljs-string">&#x27;weeks&#x27;</span>, <span class="hljs-string">&#x27;.&#x27;</span>, <span class="hljs-string">&#x27;Empire&#x27;</span>, <span class="hljs-string">&#x27;State&#x27;</span>, <span class="hljs-string">&#x27;Building&#x27;</span>, <span class="hljs-string">&#x27;=&#x27;</span>, <span class="hljs-string">&#x27;ESB&#x27;</span>, <span class="hljs-string">&#x27;.&#x27;</span>, <span class="hljs-string">&#x27;Pretty&#x27;</span>, <span class="hljs-string">&#x27;bad&#x27;</span>, <span class="hljs-string">&#x27;storm&#x27;</span>, <span class="hljs-string">&#x27;here&#x27;</span>, <span class="hljs-string">&#x27;last&#x27;</span>, <span class="hljs-string">&#x27;evening&#x27;</span>, <span class="hljs-string">&#x27;.&#x27;</span>]
}`}}),js=new C({props:{code:`label_list = wnut["train"].features[f"ner_tags"].feature.names
label_list`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>label_list = wnut[<span class="hljs-string">&quot;train&quot;</span>].features[<span class="hljs-string">f&quot;ner_tags&quot;</span>].feature.names
<span class="hljs-meta">&gt;&gt;&gt; </span>label_list
[
    <span class="hljs-string">&quot;O&quot;</span>,
    <span class="hljs-string">&quot;B-corporation&quot;</span>,
    <span class="hljs-string">&quot;I-corporation&quot;</span>,
    <span class="hljs-string">&quot;B-creative-work&quot;</span>,
    <span class="hljs-string">&quot;I-creative-work&quot;</span>,
    <span class="hljs-string">&quot;B-group&quot;</span>,
    <span class="hljs-string">&quot;I-group&quot;</span>,
    <span class="hljs-string">&quot;B-location&quot;</span>,
    <span class="hljs-string">&quot;I-location&quot;</span>,
    <span class="hljs-string">&quot;B-person&quot;</span>,
    <span class="hljs-string">&quot;I-person&quot;</span>,
    <span class="hljs-string">&quot;B-product&quot;</span>,
    <span class="hljs-string">&quot;I-product&quot;</span>,
]`}}),ws=new Xt({}),ks=new xo({props:{id:"iY2AZYdZAr0"}}),bs=new C({props:{code:`from transformers import AutoTokenizer

tokenizer = AutoTokenizer.from_pretrained("distilbert-base-uncased")`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoTokenizer

<span class="hljs-meta">&gt;&gt;&gt; </span>tokenizer = AutoTokenizer.from_pretrained(<span class="hljs-string">&quot;distilbert-base-uncased&quot;</span>)`}}),$s=new C({props:{code:`tokenized_input = tokenizer(example["tokens"], is_split_into_words=True)
tokens = tokenizer.convert_ids_to_tokens(tokenized_input["input_ids"])
tokens`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>tokenized_input = tokenizer(example[<span class="hljs-string">&quot;tokens&quot;</span>], is_split_into_words=<span class="hljs-literal">True</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>tokens = tokenizer.convert_ids_to_tokens(tokenized_input[<span class="hljs-string">&quot;input_ids&quot;</span>])
<span class="hljs-meta">&gt;&gt;&gt; </span>tokens
[<span class="hljs-string">&#x27;[CLS]&#x27;</span>, <span class="hljs-string">&#x27;@&#x27;</span>, <span class="hljs-string">&#x27;paul&#x27;</span>, <span class="hljs-string">&#x27;##walk&#x27;</span>, <span class="hljs-string">&#x27;it&#x27;</span>, <span class="hljs-string">&quot;&#x27;&quot;</span>, <span class="hljs-string">&#x27;s&#x27;</span>, <span class="hljs-string">&#x27;the&#x27;</span>, <span class="hljs-string">&#x27;view&#x27;</span>, <span class="hljs-string">&#x27;from&#x27;</span>, <span class="hljs-string">&#x27;where&#x27;</span>, <span class="hljs-string">&#x27;i&#x27;</span>, <span class="hljs-string">&quot;&#x27;&quot;</span>, <span class="hljs-string">&#x27;m&#x27;</span>, <span class="hljs-string">&#x27;living&#x27;</span>, <span class="hljs-string">&#x27;for&#x27;</span>, <span class="hljs-string">&#x27;two&#x27;</span>, <span class="hljs-string">&#x27;weeks&#x27;</span>, <span class="hljs-string">&#x27;.&#x27;</span>, <span class="hljs-string">&#x27;empire&#x27;</span>, <span class="hljs-string">&#x27;state&#x27;</span>, <span class="hljs-string">&#x27;building&#x27;</span>, <span class="hljs-string">&#x27;=&#x27;</span>, <span class="hljs-string">&#x27;es&#x27;</span>, <span class="hljs-string">&#x27;##b&#x27;</span>, <span class="hljs-string">&#x27;.&#x27;</span>, <span class="hljs-string">&#x27;pretty&#x27;</span>, <span class="hljs-string">&#x27;bad&#x27;</span>, <span class="hljs-string">&#x27;storm&#x27;</span>, <span class="hljs-string">&#x27;here&#x27;</span>, <span class="hljs-string">&#x27;last&#x27;</span>, <span class="hljs-string">&#x27;evening&#x27;</span>, <span class="hljs-string">&#x27;.&#x27;</span>, <span class="hljs-string">&#x27;[SEP]&#x27;</span>]`}}),ys=new C({props:{code:`def tokenize_and_align_labels(examples):
    tokenized_inputs = tokenizer(examples["tokens"], truncation=True, is_split_into_words=True)

    labels = []
    for i, label in enumerate(examples[f"ner_tags"]):
        word_ids = tokenized_inputs.word_ids(batch_index=i)  # Map tokens to their respective word.
        previous_word_idx = None
        label_ids = []
        for word_idx in word_ids:  # Set the special tokens to -100.
            if word_idx is None:
                label_ids.append(-100)
            elif word_idx != previous_word_idx:  # Only label the first token of a given word.
                label_ids.append(label[word_idx])
            else:
                label_ids.append(-100)
            previous_word_idx = word_idx
        labels.append(label_ids)

    tokenized_inputs["labels"] = labels
    return tokenized_inputs`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">def</span> <span class="hljs-title function_">tokenize_and_align_labels</span>(<span class="hljs-params">examples</span>):
<span class="hljs-meta">... </span>    tokenized_inputs = tokenizer(examples[<span class="hljs-string">&quot;tokens&quot;</span>], truncation=<span class="hljs-literal">True</span>, is_split_into_words=<span class="hljs-literal">True</span>)

<span class="hljs-meta">... </span>    labels = []
<span class="hljs-meta">... </span>    <span class="hljs-keyword">for</span> i, label <span class="hljs-keyword">in</span> <span class="hljs-built_in">enumerate</span>(examples[<span class="hljs-string">f&quot;ner_tags&quot;</span>]):
<span class="hljs-meta">... </span>        word_ids = tokenized_inputs.word_ids(batch_index=i)  <span class="hljs-comment"># Map tokens to their respective word.</span>
<span class="hljs-meta">... </span>        previous_word_idx = <span class="hljs-literal">None</span>
<span class="hljs-meta">... </span>        label_ids = []
<span class="hljs-meta">... </span>        <span class="hljs-keyword">for</span> word_idx <span class="hljs-keyword">in</span> word_ids:  <span class="hljs-comment"># Set the special tokens to -100.</span>
<span class="hljs-meta">... </span>            <span class="hljs-keyword">if</span> word_idx <span class="hljs-keyword">is</span> <span class="hljs-literal">None</span>:
<span class="hljs-meta">... </span>                label_ids.append(-<span class="hljs-number">100</span>)
<span class="hljs-meta">... </span>            <span class="hljs-keyword">elif</span> word_idx != previous_word_idx:  <span class="hljs-comment"># Only label the first token of a given word.</span>
<span class="hljs-meta">... </span>                label_ids.append(label[word_idx])
<span class="hljs-meta">... </span>            <span class="hljs-keyword">else</span>:
<span class="hljs-meta">... </span>                label_ids.append(-<span class="hljs-number">100</span>)
<span class="hljs-meta">... </span>            previous_word_idx = word_idx
<span class="hljs-meta">... </span>        labels.append(label_ids)

<span class="hljs-meta">... </span>    tokenized_inputs[<span class="hljs-string">&quot;labels&quot;</span>] = labels
<span class="hljs-meta">... </span>    <span class="hljs-keyword">return</span> tokenized_inputs`}}),zs=new C({props:{code:"tokenized_wnut = wnut.map(tokenize_and_align_labels, batched=True)",highlighted:'<span class="hljs-meta">&gt;&gt;&gt; </span>tokenized_wnut = wnut.<span class="hljs-built_in">map</span>(tokenize_and_align_labels, batched=<span class="hljs-literal">True</span>)'}}),qs=new qo({props:{group1:{id:"pt",code:`from transformers import DataCollatorForTokenClassification

data_collator = DataCollatorForTokenClassification(tokenizer=tokenizer)`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> DataCollatorForTokenClassification

<span class="hljs-meta">&gt;&gt;&gt; </span>data_collator = DataCollatorForTokenClassification(tokenizer=tokenizer)`},group2:{id:"tf",code:`from transformers import DataCollatorForTokenClassification

data_collator = DataCollatorForTokenClassification(tokenizer=tokenizer, return_tensors="tf")`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> DataCollatorForTokenClassification

<span class="hljs-meta">&gt;&gt;&gt; </span>data_collator = DataCollatorForTokenClassification(tokenizer=tokenizer, return_tensors=<span class="hljs-string">&quot;tf&quot;</span>)`}}}),Cs=new Xt({}),As=new C({props:{code:`from transformers import AutoModelForTokenClassification, TrainingArguments, Trainer

model = AutoModelForTokenClassification.from_pretrained("distilbert-base-uncased", num_labels=14)`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoModelForTokenClassification, TrainingArguments, Trainer

<span class="hljs-meta">&gt;&gt;&gt; </span>model = AutoModelForTokenClassification.from_pretrained(<span class="hljs-string">&quot;distilbert-base-uncased&quot;</span>, num_labels=<span class="hljs-number">14</span>)`}}),as=new pa({props:{$$slots:{default:[Ao]},$$scope:{ctx:P}}}),Ss=new C({props:{code:`training_args = TrainingArguments(
    output_dir="./results",
    evaluation_strategy="epoch",
    learning_rate=2e-5,
    per_device_train_batch_size=16,
    per_device_eval_batch_size=16,
    num_train_epochs=3,
    weight_decay=0.01,
)

trainer = Trainer(
    model=model,
    args=training_args,
    train_dataset=tokenized_wnut["train"],
    eval_dataset=tokenized_wnut["test"],
    tokenizer=tokenizer,
    data_collator=data_collator,
)

trainer.train()`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>training_args = TrainingArguments(
<span class="hljs-meta">... </span>    output_dir=<span class="hljs-string">&quot;./results&quot;</span>,
<span class="hljs-meta">... </span>    evaluation_strategy=<span class="hljs-string">&quot;epoch&quot;</span>,
<span class="hljs-meta">... </span>    learning_rate=<span class="hljs-number">2e-5</span>,
<span class="hljs-meta">... </span>    per_device_train_batch_size=<span class="hljs-number">16</span>,
<span class="hljs-meta">... </span>    per_device_eval_batch_size=<span class="hljs-number">16</span>,
<span class="hljs-meta">... </span>    num_train_epochs=<span class="hljs-number">3</span>,
<span class="hljs-meta">... </span>    weight_decay=<span class="hljs-number">0.01</span>,
<span class="hljs-meta">... </span>)

<span class="hljs-meta">&gt;&gt;&gt; </span>trainer = Trainer(
<span class="hljs-meta">... </span>    model=model,
<span class="hljs-meta">... </span>    args=training_args,
<span class="hljs-meta">... </span>    train_dataset=tokenized_wnut[<span class="hljs-string">&quot;train&quot;</span>],
<span class="hljs-meta">... </span>    eval_dataset=tokenized_wnut[<span class="hljs-string">&quot;test&quot;</span>],
<span class="hljs-meta">... </span>    tokenizer=tokenizer,
<span class="hljs-meta">... </span>    data_collator=data_collator,
<span class="hljs-meta">... </span>)

<span class="hljs-meta">&gt;&gt;&gt; </span>trainer.train()`}}),Os=new Xt({}),ls=new pa({props:{$$slots:{default:[Do]},$$scope:{ctx:P}}}),Ns=new C({props:{code:`tf_train_set = tokenized_wnut["train"].to_tf_dataset(
    columns=["attention_mask", "input_ids", "labels"],
    shuffle=True,
    batch_size=16,
    collate_fn=data_collator,
)

tf_validation_set = tokenized_wnut["validation"].to_tf_dataset(
    columns=["attention_mask", "input_ids", "labels"],
    shuffle=False,
    batch_size=16,
    collate_fn=data_collator,
)`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>tf_train_set = tokenized_wnut[<span class="hljs-string">&quot;train&quot;</span>].to_tf_dataset(
<span class="hljs-meta">... </span>    columns=[<span class="hljs-string">&quot;attention_mask&quot;</span>, <span class="hljs-string">&quot;input_ids&quot;</span>, <span class="hljs-string">&quot;labels&quot;</span>],
<span class="hljs-meta">... </span>    shuffle=<span class="hljs-literal">True</span>,
<span class="hljs-meta">... </span>    batch_size=<span class="hljs-number">16</span>,
<span class="hljs-meta">... </span>    collate_fn=data_collator,
<span class="hljs-meta">... </span>)

<span class="hljs-meta">&gt;&gt;&gt; </span>tf_validation_set = tokenized_wnut[<span class="hljs-string">&quot;validation&quot;</span>].to_tf_dataset(
<span class="hljs-meta">... </span>    columns=[<span class="hljs-string">&quot;attention_mask&quot;</span>, <span class="hljs-string">&quot;input_ids&quot;</span>, <span class="hljs-string">&quot;labels&quot;</span>],
<span class="hljs-meta">... </span>    shuffle=<span class="hljs-literal">False</span>,
<span class="hljs-meta">... </span>    batch_size=<span class="hljs-number">16</span>,
<span class="hljs-meta">... </span>    collate_fn=data_collator,
<span class="hljs-meta">... </span>)`}}),Is=new C({props:{code:`from transformers import create_optimizer

batch_size = 16
num_train_epochs = 3
num_train_steps = (len(tokenized_wnut["train"]) // batch_size) * num_train_epochs
optimizer, lr_schedule = create_optimizer(
    init_lr=2e-5,
    num_train_steps=num_train_steps,
    weight_decay_rate=0.01,
    num_warmup_steps=0,
)`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> create_optimizer

<span class="hljs-meta">&gt;&gt;&gt; </span>batch_size = <span class="hljs-number">16</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>num_train_epochs = <span class="hljs-number">3</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>num_train_steps = (<span class="hljs-built_in">len</span>(tokenized_wnut[<span class="hljs-string">&quot;train&quot;</span>]) // batch_size) * num_train_epochs
<span class="hljs-meta">&gt;&gt;&gt; </span>optimizer, lr_schedule = create_optimizer(
<span class="hljs-meta">... </span>    init_lr=<span class="hljs-number">2e-5</span>,
<span class="hljs-meta">... </span>    num_train_steps=num_train_steps,
<span class="hljs-meta">... </span>    weight_decay_rate=<span class="hljs-number">0.01</span>,
<span class="hljs-meta">... </span>    num_warmup_steps=<span class="hljs-number">0</span>,
<span class="hljs-meta">... </span>)`}}),Bs=new C({props:{code:`from transformers import TFAutoModelForTokenClassification

model = TFAutoModelForTokenClassification.from_pretrained("distilbert-base-uncased", num_labels=2)`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> TFAutoModelForTokenClassification

<span class="hljs-meta">&gt;&gt;&gt; </span>model = TFAutoModelForTokenClassification.from_pretrained(<span class="hljs-string">&quot;distilbert-base-uncased&quot;</span>, num_labels=<span class="hljs-number">2</span>)`}}),Rs=new C({props:{code:`import tensorflow as tf

model.compile(optimizer=optimizer)`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">import</span> tensorflow <span class="hljs-keyword">as</span> tf

<span class="hljs-meta">&gt;&gt;&gt; </span>model.<span class="hljs-built_in">compile</span>(optimizer=optimizer)`}}),Ws=new C({props:{code:"model.fit(x=tf_train_set, validation_data=tf_validation_set, epochs=3)",highlighted:'<span class="hljs-meta">&gt;&gt;&gt; </span>model.fit(x=tf_train_set, validation_data=tf_validation_set, epochs=<span class="hljs-number">3</span>)'}}),ps=new pa({props:{$$slots:{default:[Fo]},$$scope:{ctx:P}}}),{c(){m=o("meta"),x=c(),d=o("h1"),_=o("a"),y=o("span"),g(u.$$.fragment),v=c(),T=o("span"),z=a("Token classification"),E=c(),g(A.$$.fragment),O=c(),Ks=o("p"),ca=a("Token classification assigns a label to individual tokens in a sentence. One of the most common token classification tasks is Named Entity Recognition (NER). NER attempts to find a label for each entity in a sentence, such as a person, location, or organization."),se=c(),L=o("p"),ha=a("This guide will show you how to fine-tune "),ms=o("a"),fa=a("DistilBERT"),ma=a(" on the "),ds=o("a"),da=a("WNUT 17"),ua=a(" dataset to detect new entities."),te=c(),g(Z.$$.fragment),ee=c(),H=o("h2"),J=o("a"),dt=o("span"),g(us.$$.fragment),_a=c(),ut=o("span"),ga=a("Load WNUT 17 dataset"),ae=c(),Vs=o("p"),ja=a("Load the WNUT 17 dataset from the \u{1F917} Datasets library:"),ne=c(),g(_s.$$.fragment),le=c(),Zs=o("p"),wa=a("Then take a look at an example:"),oe=c(),g(gs.$$.fragment),re=c(),G=o("p"),ka=a("Each number in "),_t=o("code"),ba=a("ner_tags"),$a=a(" represents an entity. Convert the number to a label name for more information:"),ie=c(),g(js.$$.fragment),pe=c(),N=o("p"),va=a("The "),gt=o("code"),xa=a("ner_tag"),Ea=a(" describes an entity, such as a corporation, location, or person. The letter that prefixes each "),jt=o("code"),ya=a("ner_tag"),Ta=a(" indicates the token position of the entity:"),ce=c(),I=o("ul"),Js=o("li"),wt=o("code"),za=a("B-"),qa=a(" indicates the beginning of an entity."),Ca=c(),B=o("li"),kt=o("code"),Aa=a("I-"),Da=a(" indicates a token is contained inside the same entity (e.g., the "),bt=o("code"),Fa=a("State"),Pa=a(` token is a part of an entity like
`),$t=o("code"),Sa=a("Empire State Building"),Oa=a(")."),La=c(),Gs=o("li"),vt=o("code"),Na=a("0"),Ia=a(" indicates the token doesn\u2019t correspond to any entity."),he=c(),Y=o("h2"),Q=o("a"),xt=o("span"),g(ws.$$.fragment),Ba=c(),Et=o("span"),Ma=a("Preprocess"),fe=c(),g(ks.$$.fragment),me=c(),X=o("p"),Ra=a("Load the DistilBERT tokenizer to process the "),yt=o("code"),Ua=a("tokens"),Wa=a(":"),de=c(),g(bs.$$.fragment),ue=c(),ss=o("p"),Ha=a("Since the input has already been split into words, set "),Tt=o("code"),Ya=a("is_split_into_words=True"),Ka=a(" to tokenize the words into subwords:"),_e=c(),g($s.$$.fragment),ge=c(),M=o("p"),Va=a("Adding the special tokens "),zt=o("code"),Za=a("[CLS]"),Ja=a(" and "),qt=o("code"),Ga=a("[SEP]"),Qa=a(" and subword tokenization creates a mismatch between the input and labels. A single word corresponding to a single label may be split into two subwords. You will need to realign the tokens and labels by:"),je=c(),R=o("ol"),vs=o("li"),Xa=a("Mapping all tokens to their corresponding word with the "),xs=o("a"),Ct=o("code"),sn=a("word_ids"),tn=a(" method."),en=c(),S=o("li"),an=a("Assigning the label "),At=o("code"),nn=a("-100"),ln=a(" to the special tokens "),Dt=o("code"),on=a("[CLS]"),rn=a(" and "),Ft=o("code"),pn=a("[SEP]"),cn=a(` so the PyTorch loss function ignores
them.`),hn=c(),Es=o("li"),fn=a("Only labeling the first token of a given word. Assign "),Pt=o("code"),mn=a("-100"),dn=a(" to other subtokens from the same word."),we=c(),Qs=o("p"),un=a("Here is how you can create a function to realign the tokens and labels, and truncate sequences to be no longer than DistilBERT\u2019s maximum input length::"),ke=c(),g(ys.$$.fragment),be=c(),D=o("p"),_n=a("Use \u{1F917} Datasets "),Ts=o("a"),St=o("code"),gn=a("map"),jn=a(" function to tokenize and align the labels over the entire dataset. You can speed up the "),Ot=o("code"),wn=a("map"),kn=a(" function by setting "),Lt=o("code"),bn=a("batched=True"),$n=a(" to process multiple elements of the dataset at once:"),$e=c(),g(zs.$$.fragment),ve=c(),q=o("p"),vn=a("Use "),Xs=o("a"),xn=a("DataCollatorForTokenClassification"),En=a(" to create a batch of examples. It will also "),Nt=o("em"),yn=a("dynamically pad"),Tn=a(" your text and labels to the length of the longest element in its batch, so they are a uniform length. While it is possible to pad your text in the "),It=o("code"),zn=a("tokenizer"),qn=a(" function by setting "),Bt=o("code"),Cn=a("padding=True"),An=a(", dynamic padding is more efficient."),xe=c(),g(qs.$$.fragment),Ee=c(),K=o("h2"),ts=o("a"),Mt=o("span"),g(Cs.$$.fragment),Dn=c(),Rt=o("span"),Fn=a("Fine-tune with Trainer"),ye=c(),es=o("p"),Pn=a("Load DistilBERT with "),st=o("a"),Sn=a("AutoModelForTokenClassification"),On=a(" along with the number of expected labels:"),Te=c(),g(As.$$.fragment),ze=c(),g(as.$$.fragment),qe=c(),tt=o("p"),Ln=a("At this point, only three steps remain:"),Ce=c(),U=o("ol"),Ds=o("li"),Nn=a("Define your training hyperparameters in "),et=o("a"),In=a("TrainingArguments"),Bn=a("."),Mn=c(),Fs=o("li"),Rn=a("Pass the training arguments to "),at=o("a"),Un=a("Trainer"),Wn=a(" along with the model, dataset, tokenizer, and data collator."),Hn=c(),Ps=o("li"),Yn=a("Call "),nt=o("a"),Kn=a("train()"),Vn=a(" to fine-tune your model."),Ae=c(),g(Ss.$$.fragment),De=c(),V=o("h2"),ns=o("a"),Ut=o("span"),g(Os.$$.fragment),Zn=c(),Wt=o("span"),Jn=a("Fine-tune with TensorFlow"),Fe=c(),lt=o("p"),Gn=a("To fine-tune a model in TensorFlow is just as easy, with only a few differences."),Pe=c(),g(ls.$$.fragment),Se=c(),F=o("p"),Qn=a("Convert your datasets to the "),Ht=o("code"),Xn=a("tf.data.Dataset"),sl=a(" format with "),Ls=o("a"),Yt=o("code"),tl=a("to_tf_dataset"),el=a(". Specify inputs and labels in "),Kt=o("code"),al=a("columns"),nl=a(", whether to shuffle the dataset order, batch size, and the data collator:"),Oe=c(),g(Ns.$$.fragment),Le=c(),ot=o("p"),ll=a("Set up an optimizer function, learning rate schedule, and some training hyperparameters:"),Ne=c(),g(Is.$$.fragment),Ie=c(),os=o("p"),ol=a("Load DistilBERT with "),rt=o("a"),rl=a("TFAutoModelForTokenClassification"),il=a(" along with the number of expected labels:"),Be=c(),g(Bs.$$.fragment),Me=c(),rs=o("p"),pl=a("Configure the model for training with "),Ms=o("a"),Vt=o("code"),cl=a("compile"),hl=a(":"),Re=c(),g(Rs.$$.fragment),Ue=c(),is=o("p"),fl=a("Call "),Us=o("a"),Zt=o("code"),ml=a("fit"),dl=a(" to fine-tune the model:"),We=c(),g(Ws.$$.fragment),He=c(),g(ps.$$.fragment),this.h()},l(s){const l=zo('[data-svelte="svelte-1phssyn"]',document.head);m=r(l,"META",{name:!0,content:!0}),l.forEach(e),x=h(s),d=r(s,"H1",{class:!0});var Hs=i(d);_=r(Hs,"A",{id:!0,class:!0,href:!0});var Jt=i(_);y=r(Jt,"SPAN",{});var Gt=i(y);j(u.$$.fragment,Gt),Gt.forEach(e),Jt.forEach(e),v=h(Hs),T=r(Hs,"SPAN",{});var Qt=i(T);z=n(Qt,"Token classification"),Qt.forEach(e),Hs.forEach(e),E=h(s),j(A.$$.fragment,s),O=h(s),Ks=r(s,"P",{});var gl=i(Ks);ca=n(gl,"Token classification assigns a label to individual tokens in a sentence. One of the most common token classification tasks is Named Entity Recognition (NER). NER attempts to find a label for each entity in a sentence, such as a person, location, or organization."),gl.forEach(e),se=h(s),L=r(s,"P",{});var it=i(L);ha=n(it,"This guide will show you how to fine-tune "),ms=r(it,"A",{href:!0,rel:!0});var jl=i(ms);fa=n(jl,"DistilBERT"),jl.forEach(e),ma=n(it," on the "),ds=r(it,"A",{href:!0,rel:!0});var wl=i(ds);da=n(wl,"WNUT 17"),wl.forEach(e),ua=n(it," dataset to detect new entities."),it.forEach(e),te=h(s),j(Z.$$.fragment,s),ee=h(s),H=r(s,"H2",{class:!0});var Ke=i(H);J=r(Ke,"A",{id:!0,class:!0,href:!0});var kl=i(J);dt=r(kl,"SPAN",{});var bl=i(dt);j(us.$$.fragment,bl),bl.forEach(e),kl.forEach(e),_a=h(Ke),ut=r(Ke,"SPAN",{});var $l=i(ut);ga=n($l,"Load WNUT 17 dataset"),$l.forEach(e),Ke.forEach(e),ae=h(s),Vs=r(s,"P",{});var vl=i(Vs);ja=n(vl,"Load the WNUT 17 dataset from the \u{1F917} Datasets library:"),vl.forEach(e),ne=h(s),j(_s.$$.fragment,s),le=h(s),Zs=r(s,"P",{});var xl=i(Zs);wa=n(xl,"Then take a look at an example:"),xl.forEach(e),oe=h(s),j(gs.$$.fragment,s),re=h(s),G=r(s,"P",{});var Ve=i(G);ka=n(Ve,"Each number in "),_t=r(Ve,"CODE",{});var El=i(_t);ba=n(El,"ner_tags"),El.forEach(e),$a=n(Ve," represents an entity. Convert the number to a label name for more information:"),Ve.forEach(e),ie=h(s),j(js.$$.fragment,s),pe=h(s),N=r(s,"P",{});var pt=i(N);va=n(pt,"The "),gt=r(pt,"CODE",{});var yl=i(gt);xa=n(yl,"ner_tag"),yl.forEach(e),Ea=n(pt," describes an entity, such as a corporation, location, or person. The letter that prefixes each "),jt=r(pt,"CODE",{});var Tl=i(jt);ya=n(Tl,"ner_tag"),Tl.forEach(e),Ta=n(pt," indicates the token position of the entity:"),pt.forEach(e),ce=h(s),I=r(s,"UL",{});var ct=i(I);Js=r(ct,"LI",{});var ul=i(Js);wt=r(ul,"CODE",{});var zl=i(wt);za=n(zl,"B-"),zl.forEach(e),qa=n(ul," indicates the beginning of an entity."),ul.forEach(e),Ca=h(ct),B=r(ct,"LI",{});var Ys=i(B);kt=r(Ys,"CODE",{});var ql=i(kt);Aa=n(ql,"I-"),ql.forEach(e),Da=n(Ys," indicates a token is contained inside the same entity (e.g., the "),bt=r(Ys,"CODE",{});var Cl=i(bt);Fa=n(Cl,"State"),Cl.forEach(e),Pa=n(Ys,` token is a part of an entity like
`),$t=r(Ys,"CODE",{});var Al=i($t);Sa=n(Al,"Empire State Building"),Al.forEach(e),Oa=n(Ys,")."),Ys.forEach(e),La=h(ct),Gs=r(ct,"LI",{});var _l=i(Gs);vt=r(_l,"CODE",{});var Dl=i(vt);Na=n(Dl,"0"),Dl.forEach(e),Ia=n(_l," indicates the token doesn\u2019t correspond to any entity."),_l.forEach(e),ct.forEach(e),he=h(s),Y=r(s,"H2",{class:!0});var Ze=i(Y);Q=r(Ze,"A",{id:!0,class:!0,href:!0});var Fl=i(Q);xt=r(Fl,"SPAN",{});var Pl=i(xt);j(ws.$$.fragment,Pl),Pl.forEach(e),Fl.forEach(e),Ba=h(Ze),Et=r(Ze,"SPAN",{});var Sl=i(Et);Ma=n(Sl,"Preprocess"),Sl.forEach(e),Ze.forEach(e),fe=h(s),j(ks.$$.fragment,s),me=h(s),X=r(s,"P",{});var Je=i(X);Ra=n(Je,"Load the DistilBERT tokenizer to process the "),yt=r(Je,"CODE",{});var Ol=i(yt);Ua=n(Ol,"tokens"),Ol.forEach(e),Wa=n(Je,":"),Je.forEach(e),de=h(s),j(bs.$$.fragment,s),ue=h(s),ss=r(s,"P",{});var Ge=i(ss);Ha=n(Ge,"Since the input has already been split into words, set "),Tt=r(Ge,"CODE",{});var Ll=i(Tt);Ya=n(Ll,"is_split_into_words=True"),Ll.forEach(e),Ka=n(Ge," to tokenize the words into subwords:"),Ge.forEach(e),_e=h(s),j($s.$$.fragment,s),ge=h(s),M=r(s,"P",{});var ht=i(M);Va=n(ht,"Adding the special tokens "),zt=r(ht,"CODE",{});var Nl=i(zt);Za=n(Nl,"[CLS]"),Nl.forEach(e),Ja=n(ht," and "),qt=r(ht,"CODE",{});var Il=i(qt);Ga=n(Il,"[SEP]"),Il.forEach(e),Qa=n(ht," and subword tokenization creates a mismatch between the input and labels. A single word corresponding to a single label may be split into two subwords. You will need to realign the tokens and labels by:"),ht.forEach(e),je=h(s),R=r(s,"OL",{});var ft=i(R);vs=r(ft,"LI",{});var Qe=i(vs);Xa=n(Qe,"Mapping all tokens to their corresponding word with the "),xs=r(Qe,"A",{href:!0,rel:!0});var Bl=i(xs);Ct=r(Bl,"CODE",{});var Ml=i(Ct);sn=n(Ml,"word_ids"),Ml.forEach(e),Bl.forEach(e),tn=n(Qe," method."),Qe.forEach(e),en=h(ft),S=r(ft,"LI",{});var cs=i(S);an=n(cs,"Assigning the label "),At=r(cs,"CODE",{});var Rl=i(At);nn=n(Rl,"-100"),Rl.forEach(e),ln=n(cs," to the special tokens "),Dt=r(cs,"CODE",{});var Ul=i(Dt);on=n(Ul,"[CLS]"),Ul.forEach(e),rn=n(cs," and "),Ft=r(cs,"CODE",{});var Wl=i(Ft);pn=n(Wl,"[SEP]"),Wl.forEach(e),cn=n(cs,` so the PyTorch loss function ignores
them.`),cs.forEach(e),hn=h(ft),Es=r(ft,"LI",{});var Xe=i(Es);fn=n(Xe,"Only labeling the first token of a given word. Assign "),Pt=r(Xe,"CODE",{});var Hl=i(Pt);mn=n(Hl,"-100"),Hl.forEach(e),dn=n(Xe," to other subtokens from the same word."),Xe.forEach(e),ft.forEach(e),we=h(s),Qs=r(s,"P",{});var Yl=i(Qs);un=n(Yl,"Here is how you can create a function to realign the tokens and labels, and truncate sequences to be no longer than DistilBERT\u2019s maximum input length::"),Yl.forEach(e),ke=h(s),j(ys.$$.fragment,s),be=h(s),D=r(s,"P",{});var hs=i(D);_n=n(hs,"Use \u{1F917} Datasets "),Ts=r(hs,"A",{href:!0,rel:!0});var Kl=i(Ts);St=r(Kl,"CODE",{});var Vl=i(St);gn=n(Vl,"map"),Vl.forEach(e),Kl.forEach(e),jn=n(hs," function to tokenize and align the labels over the entire dataset. You can speed up the "),Ot=r(hs,"CODE",{});var Zl=i(Ot);wn=n(Zl,"map"),Zl.forEach(e),kn=n(hs," function by setting "),Lt=r(hs,"CODE",{});var Jl=i(Lt);bn=n(Jl,"batched=True"),Jl.forEach(e),$n=n(hs," to process multiple elements of the dataset at once:"),hs.forEach(e),$e=h(s),j(zs.$$.fragment,s),ve=h(s),q=r(s,"P",{});var W=i(q);vn=n(W,"Use "),Xs=r(W,"A",{href:!0});var Gl=i(Xs);xn=n(Gl,"DataCollatorForTokenClassification"),Gl.forEach(e),En=n(W," to create a batch of examples. It will also "),Nt=r(W,"EM",{});var Ql=i(Nt);yn=n(Ql,"dynamically pad"),Ql.forEach(e),Tn=n(W," your text and labels to the length of the longest element in its batch, so they are a uniform length. While it is possible to pad your text in the "),It=r(W,"CODE",{});var Xl=i(It);zn=n(Xl,"tokenizer"),Xl.forEach(e),qn=n(W," function by setting "),Bt=r(W,"CODE",{});var so=i(Bt);Cn=n(so,"padding=True"),so.forEach(e),An=n(W,", dynamic padding is more efficient."),W.forEach(e),xe=h(s),j(qs.$$.fragment,s),Ee=h(s),K=r(s,"H2",{class:!0});var sa=i(K);ts=r(sa,"A",{id:!0,class:!0,href:!0});var to=i(ts);Mt=r(to,"SPAN",{});var eo=i(Mt);j(Cs.$$.fragment,eo),eo.forEach(e),to.forEach(e),Dn=h(sa),Rt=r(sa,"SPAN",{});var ao=i(Rt);Fn=n(ao,"Fine-tune with Trainer"),ao.forEach(e),sa.forEach(e),ye=h(s),es=r(s,"P",{});var ta=i(es);Pn=n(ta,"Load DistilBERT with "),st=r(ta,"A",{href:!0});var no=i(st);Sn=n(no,"AutoModelForTokenClassification"),no.forEach(e),On=n(ta," along with the number of expected labels:"),ta.forEach(e),Te=h(s),j(As.$$.fragment,s),ze=h(s),j(as.$$.fragment,s),qe=h(s),tt=r(s,"P",{});var lo=i(tt);Ln=n(lo,"At this point, only three steps remain:"),lo.forEach(e),Ce=h(s),U=r(s,"OL",{});var mt=i(U);Ds=r(mt,"LI",{});var ea=i(Ds);Nn=n(ea,"Define your training hyperparameters in "),et=r(ea,"A",{href:!0});var oo=i(et);In=n(oo,"TrainingArguments"),oo.forEach(e),Bn=n(ea,"."),ea.forEach(e),Mn=h(mt),Fs=r(mt,"LI",{});var aa=i(Fs);Rn=n(aa,"Pass the training arguments to "),at=r(aa,"A",{href:!0});var ro=i(at);Un=n(ro,"Trainer"),ro.forEach(e),Wn=n(aa," along with the model, dataset, tokenizer, and data collator."),aa.forEach(e),Hn=h(mt),Ps=r(mt,"LI",{});var na=i(Ps);Yn=n(na,"Call "),nt=r(na,"A",{href:!0});var io=i(nt);Kn=n(io,"train()"),io.forEach(e),Vn=n(na," to fine-tune your model."),na.forEach(e),mt.forEach(e),Ae=h(s),j(Ss.$$.fragment,s),De=h(s),V=r(s,"H2",{class:!0});var la=i(V);ns=r(la,"A",{id:!0,class:!0,href:!0});var po=i(ns);Ut=r(po,"SPAN",{});var co=i(Ut);j(Os.$$.fragment,co),co.forEach(e),po.forEach(e),Zn=h(la),Wt=r(la,"SPAN",{});var ho=i(Wt);Jn=n(ho,"Fine-tune with TensorFlow"),ho.forEach(e),la.forEach(e),Fe=h(s),lt=r(s,"P",{});var fo=i(lt);Gn=n(fo,"To fine-tune a model in TensorFlow is just as easy, with only a few differences."),fo.forEach(e),Pe=h(s),j(ls.$$.fragment,s),Se=h(s),F=r(s,"P",{});var fs=i(F);Qn=n(fs,"Convert your datasets to the "),Ht=r(fs,"CODE",{});var mo=i(Ht);Xn=n(mo,"tf.data.Dataset"),mo.forEach(e),sl=n(fs," format with "),Ls=r(fs,"A",{href:!0,rel:!0});var uo=i(Ls);Yt=r(uo,"CODE",{});var _o=i(Yt);tl=n(_o,"to_tf_dataset"),_o.forEach(e),uo.forEach(e),el=n(fs,". Specify inputs and labels in "),Kt=r(fs,"CODE",{});var go=i(Kt);al=n(go,"columns"),go.forEach(e),nl=n(fs,", whether to shuffle the dataset order, batch size, and the data collator:"),fs.forEach(e),Oe=h(s),j(Ns.$$.fragment,s),Le=h(s),ot=r(s,"P",{});var jo=i(ot);ll=n(jo,"Set up an optimizer function, learning rate schedule, and some training hyperparameters:"),jo.forEach(e),Ne=h(s),j(Is.$$.fragment,s),Ie=h(s),os=r(s,"P",{});var oa=i(os);ol=n(oa,"Load DistilBERT with "),rt=r(oa,"A",{href:!0});var wo=i(rt);rl=n(wo,"TFAutoModelForTokenClassification"),wo.forEach(e),il=n(oa," along with the number of expected labels:"),oa.forEach(e),Be=h(s),j(Bs.$$.fragment,s),Me=h(s),rs=r(s,"P",{});var ra=i(rs);pl=n(ra,"Configure the model for training with "),Ms=r(ra,"A",{href:!0,rel:!0});var ko=i(Ms);Vt=r(ko,"CODE",{});var bo=i(Vt);cl=n(bo,"compile"),bo.forEach(e),ko.forEach(e),hl=n(ra,":"),ra.forEach(e),Re=h(s),j(Rs.$$.fragment,s),Ue=h(s),is=r(s,"P",{});var ia=i(is);fl=n(ia,"Call "),Us=r(ia,"A",{href:!0,rel:!0});var $o=i(Us);Zt=r($o,"CODE",{});var vo=i(Zt);ml=n(vo,"fit"),vo.forEach(e),$o.forEach(e),dl=n(ia," to fine-tune the model:"),ia.forEach(e),We=h(s),j(Ws.$$.fragment,s),He=h(s),j(ps.$$.fragment,s),this.h()},h(){f(m,"name","hf:doc:metadata"),f(m,"content",JSON.stringify(So)),f(_,"id","token-classification"),f(_,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),f(_,"href","#token-classification"),f(d,"class","relative group"),f(ms,"href","https://huggingface.co/distilbert-base-uncased"),f(ms,"rel","nofollow"),f(ds,"href","https://huggingface.co/datasets/wnut_17"),f(ds,"rel","nofollow"),f(J,"id","load-wnut-17-dataset"),f(J,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),f(J,"href","#load-wnut-17-dataset"),f(H,"class","relative group"),f(Q,"id","preprocess"),f(Q,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),f(Q,"href","#preprocess"),f(Y,"class","relative group"),f(xs,"href","https://huggingface.co/docs/tokenizers/python/latest/api/reference.html#tokenizers.Encoding.word_ids"),f(xs,"rel","nofollow"),f(Ts,"href","https://huggingface.co/docs/datasets/package_reference/main_classes.html#datasets.Dataset.map"),f(Ts,"rel","nofollow"),f(Xs,"href","/docs/transformers/pr_15839/en/main_classes/data_collator#transformers.DataCollatorForTokenClassification"),f(ts,"id","finetune-with-trainer"),f(ts,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),f(ts,"href","#finetune-with-trainer"),f(K,"class","relative group"),f(st,"href","/docs/transformers/pr_15839/en/model_doc/auto#transformers.AutoModelForTokenClassification"),f(et,"href","/docs/transformers/pr_15839/en/main_classes/trainer#transformers.TrainingArguments"),f(at,"href","/docs/transformers/pr_15839/en/main_classes/trainer#transformers.Trainer"),f(nt,"href","/docs/transformers/pr_15839/en/main_classes/trainer#transformers.Trainer.train"),f(ns,"id","finetune-with-tensorflow"),f(ns,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),f(ns,"href","#finetune-with-tensorflow"),f(V,"class","relative group"),f(Ls,"href","https://huggingface.co/docs/datasets/package_reference/main_classes.html#datasets.Dataset.to_tf_dataset"),f(Ls,"rel","nofollow"),f(rt,"href","/docs/transformers/pr_15839/en/model_doc/auto#transformers.TFAutoModelForTokenClassification"),f(Ms,"href","https://keras.io/api/models/model_training_apis/#compile-method"),f(Ms,"rel","nofollow"),f(Us,"href","https://keras.io/api/models/model_training_apis/#fit-method"),f(Us,"rel","nofollow")},m(s,l){t(document.head,m),p(s,x,l),p(s,d,l),t(d,_),t(_,y),w(u,y,null),t(d,v),t(d,T),t(T,z),p(s,E,l),w(A,s,l),p(s,O,l),p(s,Ks,l),t(Ks,ca),p(s,se,l),p(s,L,l),t(L,ha),t(L,ms),t(ms,fa),t(L,ma),t(L,ds),t(ds,da),t(L,ua),p(s,te,l),w(Z,s,l),p(s,ee,l),p(s,H,l),t(H,J),t(J,dt),w(us,dt,null),t(H,_a),t(H,ut),t(ut,ga),p(s,ae,l),p(s,Vs,l),t(Vs,ja),p(s,ne,l),w(_s,s,l),p(s,le,l),p(s,Zs,l),t(Zs,wa),p(s,oe,l),w(gs,s,l),p(s,re,l),p(s,G,l),t(G,ka),t(G,_t),t(_t,ba),t(G,$a),p(s,ie,l),w(js,s,l),p(s,pe,l),p(s,N,l),t(N,va),t(N,gt),t(gt,xa),t(N,Ea),t(N,jt),t(jt,ya),t(N,Ta),p(s,ce,l),p(s,I,l),t(I,Js),t(Js,wt),t(wt,za),t(Js,qa),t(I,Ca),t(I,B),t(B,kt),t(kt,Aa),t(B,Da),t(B,bt),t(bt,Fa),t(B,Pa),t(B,$t),t($t,Sa),t(B,Oa),t(I,La),t(I,Gs),t(Gs,vt),t(vt,Na),t(Gs,Ia),p(s,he,l),p(s,Y,l),t(Y,Q),t(Q,xt),w(ws,xt,null),t(Y,Ba),t(Y,Et),t(Et,Ma),p(s,fe,l),w(ks,s,l),p(s,me,l),p(s,X,l),t(X,Ra),t(X,yt),t(yt,Ua),t(X,Wa),p(s,de,l),w(bs,s,l),p(s,ue,l),p(s,ss,l),t(ss,Ha),t(ss,Tt),t(Tt,Ya),t(ss,Ka),p(s,_e,l),w($s,s,l),p(s,ge,l),p(s,M,l),t(M,Va),t(M,zt),t(zt,Za),t(M,Ja),t(M,qt),t(qt,Ga),t(M,Qa),p(s,je,l),p(s,R,l),t(R,vs),t(vs,Xa),t(vs,xs),t(xs,Ct),t(Ct,sn),t(vs,tn),t(R,en),t(R,S),t(S,an),t(S,At),t(At,nn),t(S,ln),t(S,Dt),t(Dt,on),t(S,rn),t(S,Ft),t(Ft,pn),t(S,cn),t(R,hn),t(R,Es),t(Es,fn),t(Es,Pt),t(Pt,mn),t(Es,dn),p(s,we,l),p(s,Qs,l),t(Qs,un),p(s,ke,l),w(ys,s,l),p(s,be,l),p(s,D,l),t(D,_n),t(D,Ts),t(Ts,St),t(St,gn),t(D,jn),t(D,Ot),t(Ot,wn),t(D,kn),t(D,Lt),t(Lt,bn),t(D,$n),p(s,$e,l),w(zs,s,l),p(s,ve,l),p(s,q,l),t(q,vn),t(q,Xs),t(Xs,xn),t(q,En),t(q,Nt),t(Nt,yn),t(q,Tn),t(q,It),t(It,zn),t(q,qn),t(q,Bt),t(Bt,Cn),t(q,An),p(s,xe,l),w(qs,s,l),p(s,Ee,l),p(s,K,l),t(K,ts),t(ts,Mt),w(Cs,Mt,null),t(K,Dn),t(K,Rt),t(Rt,Fn),p(s,ye,l),p(s,es,l),t(es,Pn),t(es,st),t(st,Sn),t(es,On),p(s,Te,l),w(As,s,l),p(s,ze,l),w(as,s,l),p(s,qe,l),p(s,tt,l),t(tt,Ln),p(s,Ce,l),p(s,U,l),t(U,Ds),t(Ds,Nn),t(Ds,et),t(et,In),t(Ds,Bn),t(U,Mn),t(U,Fs),t(Fs,Rn),t(Fs,at),t(at,Un),t(Fs,Wn),t(U,Hn),t(U,Ps),t(Ps,Yn),t(Ps,nt),t(nt,Kn),t(Ps,Vn),p(s,Ae,l),w(Ss,s,l),p(s,De,l),p(s,V,l),t(V,ns),t(ns,Ut),w(Os,Ut,null),t(V,Zn),t(V,Wt),t(Wt,Jn),p(s,Fe,l),p(s,lt,l),t(lt,Gn),p(s,Pe,l),w(ls,s,l),p(s,Se,l),p(s,F,l),t(F,Qn),t(F,Ht),t(Ht,Xn),t(F,sl),t(F,Ls),t(Ls,Yt),t(Yt,tl),t(F,el),t(F,Kt),t(Kt,al),t(F,nl),p(s,Oe,l),w(Ns,s,l),p(s,Le,l),p(s,ot,l),t(ot,ll),p(s,Ne,l),w(Is,s,l),p(s,Ie,l),p(s,os,l),t(os,ol),t(os,rt),t(rt,rl),t(os,il),p(s,Be,l),w(Bs,s,l),p(s,Me,l),p(s,rs,l),t(rs,pl),t(rs,Ms),t(Ms,Vt),t(Vt,cl),t(rs,hl),p(s,Re,l),w(Rs,s,l),p(s,Ue,l),p(s,is,l),t(is,fl),t(is,Us),t(Us,Zt),t(Zt,ml),t(is,dl),p(s,We,l),w(Ws,s,l),p(s,He,l),w(ps,s,l),Ye=!0},p(s,[l]){const Hs={};l&2&&(Hs.$$scope={dirty:l,ctx:s}),Z.$set(Hs);const Jt={};l&2&&(Jt.$$scope={dirty:l,ctx:s}),as.$set(Jt);const Gt={};l&2&&(Gt.$$scope={dirty:l,ctx:s}),ls.$set(Gt);const Qt={};l&2&&(Qt.$$scope={dirty:l,ctx:s}),ps.$set(Qt)},i(s){Ye||(k(u.$$.fragment,s),k(A.$$.fragment,s),k(Z.$$.fragment,s),k(us.$$.fragment,s),k(_s.$$.fragment,s),k(gs.$$.fragment,s),k(js.$$.fragment,s),k(ws.$$.fragment,s),k(ks.$$.fragment,s),k(bs.$$.fragment,s),k($s.$$.fragment,s),k(ys.$$.fragment,s),k(zs.$$.fragment,s),k(qs.$$.fragment,s),k(Cs.$$.fragment,s),k(As.$$.fragment,s),k(as.$$.fragment,s),k(Ss.$$.fragment,s),k(Os.$$.fragment,s),k(ls.$$.fragment,s),k(Ns.$$.fragment,s),k(Is.$$.fragment,s),k(Bs.$$.fragment,s),k(Rs.$$.fragment,s),k(Ws.$$.fragment,s),k(ps.$$.fragment,s),Ye=!0)},o(s){b(u.$$.fragment,s),b(A.$$.fragment,s),b(Z.$$.fragment,s),b(us.$$.fragment,s),b(_s.$$.fragment,s),b(gs.$$.fragment,s),b(js.$$.fragment,s),b(ws.$$.fragment,s),b(ks.$$.fragment,s),b(bs.$$.fragment,s),b($s.$$.fragment,s),b(ys.$$.fragment,s),b(zs.$$.fragment,s),b(qs.$$.fragment,s),b(Cs.$$.fragment,s),b(As.$$.fragment,s),b(as.$$.fragment,s),b(Ss.$$.fragment,s),b(Os.$$.fragment,s),b(ls.$$.fragment,s),b(Ns.$$.fragment,s),b(Is.$$.fragment,s),b(Bs.$$.fragment,s),b(Rs.$$.fragment,s),b(Ws.$$.fragment,s),b(ps.$$.fragment,s),Ye=!1},d(s){e(m),s&&e(x),s&&e(d),$(u),s&&e(E),$(A,s),s&&e(O),s&&e(Ks),s&&e(se),s&&e(L),s&&e(te),$(Z,s),s&&e(ee),s&&e(H),$(us),s&&e(ae),s&&e(Vs),s&&e(ne),$(_s,s),s&&e(le),s&&e(Zs),s&&e(oe),$(gs,s),s&&e(re),s&&e(G),s&&e(ie),$(js,s),s&&e(pe),s&&e(N),s&&e(ce),s&&e(I),s&&e(he),s&&e(Y),$(ws),s&&e(fe),$(ks,s),s&&e(me),s&&e(X),s&&e(de),$(bs,s),s&&e(ue),s&&e(ss),s&&e(_e),$($s,s),s&&e(ge),s&&e(M),s&&e(je),s&&e(R),s&&e(we),s&&e(Qs),s&&e(ke),$(ys,s),s&&e(be),s&&e(D),s&&e($e),$(zs,s),s&&e(ve),s&&e(q),s&&e(xe),$(qs,s),s&&e(Ee),s&&e(K),$(Cs),s&&e(ye),s&&e(es),s&&e(Te),$(As,s),s&&e(ze),$(as,s),s&&e(qe),s&&e(tt),s&&e(Ce),s&&e(U),s&&e(Ae),$(Ss,s),s&&e(De),s&&e(V),$(Os),s&&e(Fe),s&&e(lt),s&&e(Pe),$(ls,s),s&&e(Se),s&&e(F),s&&e(Oe),$(Ns,s),s&&e(Le),s&&e(ot),s&&e(Ne),$(Is,s),s&&e(Ie),s&&e(os),s&&e(Be),$(Bs,s),s&&e(Me),s&&e(rs),s&&e(Re),$(Rs,s),s&&e(Ue),s&&e(is),s&&e(We),$(Ws,s),s&&e(He),$(ps,s)}}}const So={local:"token-classification",sections:[{local:"load-wnut-17-dataset",title:"Load WNUT 17 dataset"},{local:"preprocess",title:"Preprocess"},{local:"finetune-with-trainer",title:"Fine-tune with Trainer"},{local:"finetune-with-tensorflow",title:"Fine-tune with TensorFlow"}],title:"Token classification"};function Oo(P,m,x){let{fw:d}=m;return P.$$set=_=>{"fw"in _&&x(0,d=_.fw)},[d]}class Wo extends Eo{constructor(m){super();yo(this,m,Oo,Po,To,{fw:0})}}export{Wo as default,So as metadata};
