import{S as dn,i as _n,s as gn,e as r,k as _,w as v,t as l,M as $n,c as i,d as e,m as g,a as p,x as E,h as o,b as j,G as t,g as f,y,q as T,o as z,B as q,v as jn,L as un}from"../../chunks/vendor-hf-doc-builder.js";import{T as de}from"../../chunks/Tip-hf-doc-builder.js";import{Y as hn}from"../../chunks/Youtube-hf-doc-builder.js";import{I as ue}from"../../chunks/IconCopyLink-hf-doc-builder.js";import{C as ts}from"../../chunks/CodeBlock-hf-doc-builder.js";import{F as mn,M as _e}from"../../chunks/Markdown-hf-doc-builder.js";function kn(P){let a,m,n,u,k;return{c(){a=r("p"),m=l("See the token classification "),n=r("a"),u=l("task page"),k=l(" for more information about other forms of token classification and their associated models, datasets, and metrics."),this.h()},l($){a=i($,"P",{});var w=p(a);m=o(w,"See the token classification "),n=i(w,"A",{href:!0,rel:!0});var C=p(n);u=o(C,"task page"),C.forEach(e),k=o(w," for more information about other forms of token classification and their associated models, datasets, and metrics."),w.forEach(e),this.h()},h(){j(n,"href","https://huggingface.co/tasks/token-classification"),j(n,"rel","nofollow")},m($,w){f($,a,w),t(a,m),t(a,n),t(n,u),t(a,k)},d($){$&&e(a)}}}function wn(P){let a,m;return a=new ts({props:{code:`from transformers import DataCollatorForTokenClassification

data_collator = DataCollatorForTokenClassification(tokenizer=tokenizer)`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> DataCollatorForTokenClassification

<span class="hljs-meta">&gt;&gt;&gt; </span>data_collator = DataCollatorForTokenClassification(tokenizer=tokenizer)`}}),{c(){v(a.$$.fragment)},l(n){E(a.$$.fragment,n)},m(n,u){y(a,n,u),m=!0},p:un,i(n){m||(T(a.$$.fragment,n),m=!0)},o(n){z(a.$$.fragment,n),m=!1},d(n){q(a,n)}}}function bn(P){let a,m;return a=new _e({props:{$$slots:{default:[wn]},$$scope:{ctx:P}}}),{c(){v(a.$$.fragment)},l(n){E(a.$$.fragment,n)},m(n,u){y(a,n,u),m=!0},p(n,u){const k={};u&2&&(k.$$scope={dirty:u,ctx:n}),a.$set(k)},i(n){m||(T(a.$$.fragment,n),m=!0)},o(n){z(a.$$.fragment,n),m=!1},d(n){q(a,n)}}}function xn(P){let a,m;return a=new ts({props:{code:`from transformers import DataCollatorForTokenClassification

data_collator = DataCollatorForTokenClassification(tokenizer=tokenizer, return_tensors="tf")`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> DataCollatorForTokenClassification

<span class="hljs-meta">&gt;&gt;&gt; </span>data_collator = DataCollatorForTokenClassification(tokenizer=tokenizer, return_tensors=<span class="hljs-string">&quot;tf&quot;</span>)`}}),{c(){v(a.$$.fragment)},l(n){E(a.$$.fragment,n)},m(n,u){y(a,n,u),m=!0},p:un,i(n){m||(T(a.$$.fragment,n),m=!0)},o(n){z(a.$$.fragment,n),m=!1},d(n){q(a,n)}}}function vn(P){let a,m;return a=new _e({props:{$$slots:{default:[xn]},$$scope:{ctx:P}}}),{c(){v(a.$$.fragment)},l(n){E(a.$$.fragment,n)},m(n,u){y(a,n,u),m=!0},p(n,u){const k={};u&2&&(k.$$scope={dirty:u,ctx:n}),a.$set(k)},i(n){m||(T(a.$$.fragment,n),m=!0)},o(n){z(a.$$.fragment,n),m=!1},d(n){q(a,n)}}}function En(P){let a,m,n,u,k,$,w,C;return{c(){a=r("p"),m=l("If you aren\u2019t familiar with fine-tuning a model with the "),n=r("a"),u=l("Trainer"),k=l(", take a look at the basic tutorial "),$=r("a"),w=l("here"),C=l("!"),this.h()},l(A){a=i(A,"P",{});var x=p(a);m=o(x,"If you aren\u2019t familiar with fine-tuning a model with the "),n=i(x,"A",{href:!0});var F=p(n);u=o(F,"Trainer"),F.forEach(e),k=o(x,", take a look at the basic tutorial "),$=i(x,"A",{href:!0});var W=p($);w=o(W,"here"),W.forEach(e),C=o(x,"!"),x.forEach(e),this.h()},h(){j(n,"href","/docs/transformers/pr_17548/en/main_classes/trainer#transformers.Trainer"),j($,"href","../training#finetune-with-trainer")},m(A,x){f(A,a,x),t(a,m),t(a,n),t(n,u),t(a,k),t(a,$),t($,w),t(a,C)},d(A){A&&e(a)}}}function yn(P){let a,m,n,u,k,$,w,C,A,x,F,W,G,L,K,S,es,Y,$s,ms,O,us,M,ds,R,_s,I,V,N,J,Q,os,U,as;return w=new ts({props:{code:`from transformers import AutoModelForTokenClassification, TrainingArguments, Trainer

model = AutoModelForTokenClassification.from_pretrained("distilbert-base-uncased", num_labels=14)`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoModelForTokenClassification, TrainingArguments, Trainer

<span class="hljs-meta">&gt;&gt;&gt; </span>model = AutoModelForTokenClassification.from_pretrained(<span class="hljs-string">&quot;distilbert-base-uncased&quot;</span>, num_labels=<span class="hljs-number">14</span>)`}}),A=new de({props:{$$slots:{default:[En]},$$scope:{ctx:P}}}),U=new ts({props:{code:`training_args = TrainingArguments(
    output_dir="./results",
    evaluation_strategy="epoch",
    learning_rate=2e-5,
    per_device_train_batch_size=16,
    per_device_eval_batch_size=16,
    num_train_epochs=3,
    weight_decay=0.01,
)

trainer = Trainer(
    model=model,
    args=training_args,
    train_dataset=tokenized_wnut["train"],
    eval_dataset=tokenized_wnut["test"],
    tokenizer=tokenizer,
    data_collator=data_collator,
)

trainer.train()`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>training_args = TrainingArguments(
<span class="hljs-meta">... </span>    output_dir=<span class="hljs-string">&quot;./results&quot;</span>,
<span class="hljs-meta">... </span>    evaluation_strategy=<span class="hljs-string">&quot;epoch&quot;</span>,
<span class="hljs-meta">... </span>    learning_rate=<span class="hljs-number">2e-5</span>,
<span class="hljs-meta">... </span>    per_device_train_batch_size=<span class="hljs-number">16</span>,
<span class="hljs-meta">... </span>    per_device_eval_batch_size=<span class="hljs-number">16</span>,
<span class="hljs-meta">... </span>    num_train_epochs=<span class="hljs-number">3</span>,
<span class="hljs-meta">... </span>    weight_decay=<span class="hljs-number">0.01</span>,
<span class="hljs-meta">... </span>)

<span class="hljs-meta">&gt;&gt;&gt; </span>trainer = Trainer(
<span class="hljs-meta">... </span>    model=model,
<span class="hljs-meta">... </span>    args=training_args,
<span class="hljs-meta">... </span>    train_dataset=tokenized_wnut[<span class="hljs-string">&quot;train&quot;</span>],
<span class="hljs-meta">... </span>    eval_dataset=tokenized_wnut[<span class="hljs-string">&quot;test&quot;</span>],
<span class="hljs-meta">... </span>    tokenizer=tokenizer,
<span class="hljs-meta">... </span>    data_collator=data_collator,
<span class="hljs-meta">... </span>)

<span class="hljs-meta">&gt;&gt;&gt; </span>trainer.train()`}}),{c(){a=r("p"),m=l("Load DistilBERT with "),n=r("a"),u=l("AutoModelForTokenClassification"),k=l(" along with the number of expected labels:"),$=_(),v(w.$$.fragment),C=_(),v(A.$$.fragment),x=_(),F=r("p"),W=l("At this point, only three steps remain:"),G=_(),L=r("ol"),K=r("li"),S=l("Define your training hyperparameters in "),es=r("a"),Y=l("TrainingArguments"),$s=l("."),ms=_(),O=r("li"),us=l("Pass the training arguments to "),M=r("a"),ds=l("Trainer"),R=l(" along with the model, dataset, tokenizer, and data collator."),_s=_(),I=r("li"),V=l("Call "),N=r("a"),J=l("train()"),Q=l(" to fine-tune your model."),os=_(),v(U.$$.fragment),this.h()},l(d){a=i(d,"P",{});var D=p(a);m=o(D,"Load DistilBERT with "),n=i(D,"A",{href:!0});var ns=p(n);u=o(ns,"AutoModelForTokenClassification"),ns.forEach(e),k=o(D," along with the number of expected labels:"),D.forEach(e),$=g(d),E(w.$$.fragment,d),C=g(d),E(A.$$.fragment,d),x=g(d),F=i(d,"P",{});var H=p(F);W=o(H,"At this point, only three steps remain:"),H.forEach(e),G=g(d),L=i(d,"OL",{});var X=p(L);K=i(X,"LI",{});var B=p(K);S=o(B,"Define your training hyperparameters in "),es=i(B,"A",{href:!0});var ks=p(es);Y=o(ks,"TrainingArguments"),ks.forEach(e),$s=o(B,"."),B.forEach(e),ms=g(X),O=i(X,"LI",{});var Z=p(O);us=o(Z,"Pass the training arguments to "),M=i(Z,"A",{href:!0});var ss=p(M);ds=o(ss,"Trainer"),ss.forEach(e),R=o(Z," along with the model, dataset, tokenizer, and data collator."),Z.forEach(e),_s=g(X),I=i(X,"LI",{});var is=p(I);V=o(is,"Call "),N=i(is,"A",{href:!0});var ls=p(N);J=o(ls,"train()"),ls.forEach(e),Q=o(is," to fine-tune your model."),is.forEach(e),X.forEach(e),os=g(d),E(U.$$.fragment,d),this.h()},h(){j(n,"href","/docs/transformers/pr_17548/en/model_doc/auto#transformers.AutoModelForTokenClassification"),j(es,"href","/docs/transformers/pr_17548/en/main_classes/trainer#transformers.TrainingArguments"),j(M,"href","/docs/transformers/pr_17548/en/main_classes/trainer#transformers.Trainer"),j(N,"href","/docs/transformers/pr_17548/en/main_classes/trainer#transformers.Trainer.train")},m(d,D){f(d,a,D),t(a,m),t(a,n),t(n,u),t(a,k),f(d,$,D),y(w,d,D),f(d,C,D),y(A,d,D),f(d,x,D),f(d,F,D),t(F,W),f(d,G,D),f(d,L,D),t(L,K),t(K,S),t(K,es),t(es,Y),t(K,$s),t(L,ms),t(L,O),t(O,us),t(O,M),t(M,ds),t(O,R),t(L,_s),t(L,I),t(I,V),t(I,N),t(N,J),t(I,Q),f(d,os,D),y(U,d,D),as=!0},p(d,D){const ns={};D&2&&(ns.$$scope={dirty:D,ctx:d}),A.$set(ns)},i(d){as||(T(w.$$.fragment,d),T(A.$$.fragment,d),T(U.$$.fragment,d),as=!0)},o(d){z(w.$$.fragment,d),z(A.$$.fragment,d),z(U.$$.fragment,d),as=!1},d(d){d&&e(a),d&&e($),q(w,d),d&&e(C),q(A,d),d&&e(x),d&&e(F),d&&e(G),d&&e(L),d&&e(os),q(U,d)}}}function Tn(P){let a,m;return a=new _e({props:{$$slots:{default:[yn]},$$scope:{ctx:P}}}),{c(){v(a.$$.fragment)},l(n){E(a.$$.fragment,n)},m(n,u){y(a,n,u),m=!0},p(n,u){const k={};u&2&&(k.$$scope={dirty:u,ctx:n}),a.$set(k)},i(n){m||(T(a.$$.fragment,n),m=!0)},o(n){z(a.$$.fragment,n),m=!1},d(n){q(a,n)}}}function zn(P){let a,m,n,u,k;return{c(){a=r("p"),m=l("If you aren\u2019t familiar with fine-tuning a model with Keras, take a look at the basic tutorial "),n=r("a"),u=l("here"),k=l("!"),this.h()},l($){a=i($,"P",{});var w=p(a);m=o(w,"If you aren\u2019t familiar with fine-tuning a model with Keras, take a look at the basic tutorial "),n=i(w,"A",{href:!0});var C=p(n);u=o(C,"here"),C.forEach(e),k=o(w,"!"),w.forEach(e),this.h()},h(){j(n,"href","training#finetune-with-keras")},m($,w){f($,a,w),t(a,m),t(a,n),t(n,u),t(a,k)},d($){$&&e(a)}}}function qn(P){let a,m,n,u,k,$,w,C,A,x,F,W,G,L,K,S,es,Y,$s,ms,O,us,M,ds,R,_s,I,V,N,J,Q,os,U,as,d,D,ns,H,X,B,ks,Z,ss,is,ls,Cs,ps,As;return L=new ts({props:{code:`tf_train_set = tokenized_wnut["train"].to_tf_dataset(
    columns=["attention_mask", "input_ids", "labels"],
    shuffle=True,
    batch_size=16,
    collate_fn=data_collator,
)

tf_validation_set = tokenized_wnut["validation"].to_tf_dataset(
    columns=["attention_mask", "input_ids", "labels"],
    shuffle=False,
    batch_size=16,
    collate_fn=data_collator,
)`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>tf_train_set = tokenized_wnut[<span class="hljs-string">&quot;train&quot;</span>].to_tf_dataset(
<span class="hljs-meta">... </span>    columns=[<span class="hljs-string">&quot;attention_mask&quot;</span>, <span class="hljs-string">&quot;input_ids&quot;</span>, <span class="hljs-string">&quot;labels&quot;</span>],
<span class="hljs-meta">... </span>    shuffle=<span class="hljs-literal">True</span>,
<span class="hljs-meta">... </span>    batch_size=<span class="hljs-number">16</span>,
<span class="hljs-meta">... </span>    collate_fn=data_collator,
<span class="hljs-meta">... </span>)

<span class="hljs-meta">&gt;&gt;&gt; </span>tf_validation_set = tokenized_wnut[<span class="hljs-string">&quot;validation&quot;</span>].to_tf_dataset(
<span class="hljs-meta">... </span>    columns=[<span class="hljs-string">&quot;attention_mask&quot;</span>, <span class="hljs-string">&quot;input_ids&quot;</span>, <span class="hljs-string">&quot;labels&quot;</span>],
<span class="hljs-meta">... </span>    shuffle=<span class="hljs-literal">False</span>,
<span class="hljs-meta">... </span>    batch_size=<span class="hljs-number">16</span>,
<span class="hljs-meta">... </span>    collate_fn=data_collator,
<span class="hljs-meta">... </span>)`}}),S=new de({props:{$$slots:{default:[zn]},$$scope:{ctx:P}}}),O=new ts({props:{code:`from transformers import create_optimizer

batch_size = 16
num_train_epochs = 3
num_train_steps = (len(tokenized_wnut["train"]) // batch_size) * num_train_epochs
optimizer, lr_schedule = create_optimizer(
    init_lr=2e-5,
    num_train_steps=num_train_steps,
    weight_decay_rate=0.01,
    num_warmup_steps=0,
)`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> create_optimizer

<span class="hljs-meta">&gt;&gt;&gt; </span>batch_size = <span class="hljs-number">16</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>num_train_epochs = <span class="hljs-number">3</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>num_train_steps = (<span class="hljs-built_in">len</span>(tokenized_wnut[<span class="hljs-string">&quot;train&quot;</span>]) // batch_size) * num_train_epochs
<span class="hljs-meta">&gt;&gt;&gt; </span>optimizer, lr_schedule = create_optimizer(
<span class="hljs-meta">... </span>    init_lr=<span class="hljs-number">2e-5</span>,
<span class="hljs-meta">... </span>    num_train_steps=num_train_steps,
<span class="hljs-meta">... </span>    weight_decay_rate=<span class="hljs-number">0.01</span>,
<span class="hljs-meta">... </span>    num_warmup_steps=<span class="hljs-number">0</span>,
<span class="hljs-meta">... </span>)`}}),N=new ts({props:{code:`from transformers import TFAutoModelForTokenClassification

model = TFAutoModelForTokenClassification.from_pretrained("distilbert-base-uncased", num_labels=2)`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> TFAutoModelForTokenClassification

<span class="hljs-meta">&gt;&gt;&gt; </span>model = TFAutoModelForTokenClassification.from_pretrained(<span class="hljs-string">&quot;distilbert-base-uncased&quot;</span>, num_labels=<span class="hljs-number">2</span>)`}}),H=new ts({props:{code:`import tensorflow as tf

model.compile(optimizer=optimizer)`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">import</span> tensorflow <span class="hljs-keyword">as</span> tf

<span class="hljs-meta">&gt;&gt;&gt; </span>model.<span class="hljs-built_in">compile</span>(optimizer=optimizer)`}}),ps=new ts({props:{code:"model.fit(x=tf_train_set, validation_data=tf_validation_set, epochs=3)",highlighted:'<span class="hljs-meta">&gt;&gt;&gt; </span>model.fit(x=tf_train_set, validation_data=tf_validation_set, epochs=<span class="hljs-number">3</span>)'}}),{c(){a=r("p"),m=l("To fine-tune a model in TensorFlow, start by converting your datasets to the "),n=r("code"),u=l("tf.data.Dataset"),k=l(" format with "),$=r("a"),w=r("code"),C=l("to_tf_dataset"),A=l(". Specify inputs and labels in "),x=r("code"),F=l("columns"),W=l(", whether to shuffle the dataset order, batch size, and the data collator:"),G=_(),v(L.$$.fragment),K=_(),v(S.$$.fragment),es=_(),Y=r("p"),$s=l("Set up an optimizer function, learning rate schedule, and some training hyperparameters:"),ms=_(),v(O.$$.fragment),us=_(),M=r("p"),ds=l("Load DistilBERT with "),R=r("a"),_s=l("TFAutoModelForTokenClassification"),I=l(" along with the number of expected labels:"),V=_(),v(N.$$.fragment),J=_(),Q=r("p"),os=l("Configure the model for training with "),U=r("a"),as=r("code"),d=l("compile"),D=l(":"),ns=_(),v(H.$$.fragment),X=_(),B=r("p"),ks=l("Call "),Z=r("a"),ss=r("code"),is=l("fit"),ls=l(" to fine-tune the model:"),Cs=_(),v(ps.$$.fragment),this.h()},l(c){a=i(c,"P",{});var b=p(a);m=o(b,"To fine-tune a model in TensorFlow, start by converting your datasets to the "),n=i(b,"CODE",{});var cs=p(n);u=o(cs,"tf.data.Dataset"),cs.forEach(e),k=o(b," format with "),$=i(b,"A",{href:!0,rel:!0});var Us=p($);w=i(Us,"CODE",{});var fs=p(w);C=o(fs,"to_tf_dataset"),fs.forEach(e),Us.forEach(e),A=o(b,". Specify inputs and labels in "),x=i(b,"CODE",{});var at=p(x);F=o(at,"columns"),at.forEach(e),W=o(b,", whether to shuffle the dataset order, batch size, and the data collator:"),b.forEach(e),G=g(c),E(L.$$.fragment,c),K=g(c),E(S.$$.fragment,c),es=g(c),Y=i(c,"P",{});var Ds=p(Y);$s=o(Ds,"Set up an optimizer function, learning rate schedule, and some training hyperparameters:"),Ds.forEach(e),ms=g(c),E(O.$$.fragment,c),us=g(c),M=i(c,"P",{});var Ps=p(M);ds=o(Ps,"Load DistilBERT with "),R=i(Ps,"A",{href:!0});var nt=p(R);_s=o(nt,"TFAutoModelForTokenClassification"),nt.forEach(e),I=o(Ps," along with the number of expected labels:"),Ps.forEach(e),V=g(c),E(N.$$.fragment,c),J=g(c),Q=i(c,"P",{});var ws=p(Q);os=o(ws,"Configure the model for training with "),U=i(ws,"A",{href:!0,rel:!0});var lt=p(U);as=i(lt,"CODE",{});var ot=p(as);d=o(ot,"compile"),ot.forEach(e),lt.forEach(e),D=o(ws,":"),ws.forEach(e),ns=g(c),E(H.$$.fragment,c),X=g(c),B=i(c,"P",{});var ys=p(B);ks=o(ys,"Call "),Z=i(ys,"A",{href:!0,rel:!0});var hs=p(Z);ss=i(hs,"CODE",{});var Ts=p(ss);is=o(Ts,"fit"),Ts.forEach(e),hs.forEach(e),ls=o(ys," to fine-tune the model:"),ys.forEach(e),Cs=g(c),E(ps.$$.fragment,c),this.h()},h(){j($,"href","https://huggingface.co/docs/datasets/package_reference/main_classes.html#datasets.Dataset.to_tf_dataset"),j($,"rel","nofollow"),j(R,"href","/docs/transformers/pr_17548/en/model_doc/auto#transformers.TFAutoModelForTokenClassification"),j(U,"href","https://keras.io/api/models/model_training_apis/#compile-method"),j(U,"rel","nofollow"),j(Z,"href","https://keras.io/api/models/model_training_apis/#fit-method"),j(Z,"rel","nofollow")},m(c,b){f(c,a,b),t(a,m),t(a,n),t(n,u),t(a,k),t(a,$),t($,w),t(w,C),t(a,A),t(a,x),t(x,F),t(a,W),f(c,G,b),y(L,c,b),f(c,K,b),y(S,c,b),f(c,es,b),f(c,Y,b),t(Y,$s),f(c,ms,b),y(O,c,b),f(c,us,b),f(c,M,b),t(M,ds),t(M,R),t(R,_s),t(M,I),f(c,V,b),y(N,c,b),f(c,J,b),f(c,Q,b),t(Q,os),t(Q,U),t(U,as),t(as,d),t(Q,D),f(c,ns,b),y(H,c,b),f(c,X,b),f(c,B,b),t(B,ks),t(B,Z),t(Z,ss),t(ss,is),t(B,ls),f(c,Cs,b),y(ps,c,b),As=!0},p(c,b){const cs={};b&2&&(cs.$$scope={dirty:b,ctx:c}),S.$set(cs)},i(c){As||(T(L.$$.fragment,c),T(S.$$.fragment,c),T(O.$$.fragment,c),T(N.$$.fragment,c),T(H.$$.fragment,c),T(ps.$$.fragment,c),As=!0)},o(c){z(L.$$.fragment,c),z(S.$$.fragment,c),z(O.$$.fragment,c),z(N.$$.fragment,c),z(H.$$.fragment,c),z(ps.$$.fragment,c),As=!1},d(c){c&&e(a),c&&e(G),q(L,c),c&&e(K),q(S,c),c&&e(es),c&&e(Y),c&&e(ms),q(O,c),c&&e(us),c&&e(M),c&&e(V),q(N,c),c&&e(J),c&&e(Q),c&&e(ns),q(H,c),c&&e(X),c&&e(B),c&&e(Cs),q(ps,c)}}}function Cn(P){let a,m;return a=new _e({props:{$$slots:{default:[qn]},$$scope:{ctx:P}}}),{c(){v(a.$$.fragment)},l(n){E(a.$$.fragment,n)},m(n,u){y(a,n,u),m=!0},p(n,u){const k={};u&2&&(k.$$scope={dirty:u,ctx:n}),a.$set(k)},i(n){m||(T(a.$$.fragment,n),m=!0)},o(n){z(a.$$.fragment,n),m=!1},d(n){q(a,n)}}}function An(P){let a,m,n,u,k,$,w,C;return{c(){a=r("p"),m=l(`For a more in-depth example of how to fine-tune a model for token classification, take a look at the corresponding
`),n=r("a"),u=l("PyTorch notebook"),k=l(`
or `),$=r("a"),w=l("TensorFlow notebook"),C=l("."),this.h()},l(A){a=i(A,"P",{});var x=p(a);m=o(x,`For a more in-depth example of how to fine-tune a model for token classification, take a look at the corresponding
`),n=i(x,"A",{href:!0,rel:!0});var F=p(n);u=o(F,"PyTorch notebook"),F.forEach(e),k=o(x,`
or `),$=i(x,"A",{href:!0,rel:!0});var W=p($);w=o(W,"TensorFlow notebook"),W.forEach(e),C=o(x,"."),x.forEach(e),this.h()},h(){j(n,"href","https://colab.research.google.com/github/huggingface/notebooks/blob/main/examples/token_classification.ipynb"),j(n,"rel","nofollow"),j($,"href","https://colab.research.google.com/github/huggingface/notebooks/blob/main/examples/token_classification-tf.ipynb"),j($,"rel","nofollow")},m(A,x){f(A,a,x),t(a,m),t(a,n),t(n,u),t(a,k),t(a,$),t($,w),t(a,C)},d(A){A&&e(a)}}}function Dn(P){let a,m,n,u,k,$,w,C,A,x,F,W,G,L,K,S,es,Y,$s,ms,O,us,M,ds,R,_s,I,V,N,J,Q,os,U,as,d,D,ns,H,X,B,ks,Z,ss,is,ls,Cs,ps,As,c,b,cs,Us,fs,at,Ds,Ps,nt,ws,lt,ot,ys,hs,Ts,dt,ge,$e,je,bs,_t,ke,we,gt,be,xe,$t,ve,Ee,ye,rt,jt,Te,ze,Rt,zs,Fs,kt,Ws,qe,wt,Ce,Ut,Ys,Wt,Ss,Ae,bt,De,Pe,Yt,Hs,Ht,Os,Fe,xt,Se,Oe,Kt,Ks,Vt,xs,Le,vt,Ie,Ne,Et,Be,Me,Zt,vs,Vs,Re,Zs,yt,Ue,We,Ye,js,He,Tt,Ke,Ve,zt,Ze,Ge,qt,Je,Qe,Xe,Gs,sa,Ct,ta,ea,Gt,it,aa,Jt,Js,Qt,gs,na,Qs,At,la,oa,Dt,ra,ia,Pt,pa,ca,Xt,Xs,se,rs,fa,pt,ha,ma,Ft,ua,da,St,_a,ga,Ot,$a,ja,te,Ls,ee,qs,Is,Lt,st,ka,It,wa,ae,Ns,ne,Bs,le;return $=new ue({}),F=new hn({props:{id:"wVHdVlPScxA"}}),R=new de({props:{$$slots:{default:[kn]},$$scope:{ctx:P}}}),J=new ue({}),H=new ts({props:{code:`from datasets import load_dataset

wnut = load_dataset("wnut_17")`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> datasets <span class="hljs-keyword">import</span> load_dataset

<span class="hljs-meta">&gt;&gt;&gt; </span>wnut = load_dataset(<span class="hljs-string">&quot;wnut_17&quot;</span>)`}}),ss=new ts({props:{code:'wnut["train"][0]',highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>wnut[<span class="hljs-string">&quot;train&quot;</span>][<span class="hljs-number">0</span>]
{<span class="hljs-string">&#x27;id&#x27;</span>: <span class="hljs-string">&#x27;0&#x27;</span>,
 <span class="hljs-string">&#x27;ner_tags&#x27;</span>: [<span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">7</span>, <span class="hljs-number">8</span>, <span class="hljs-number">8</span>, <span class="hljs-number">0</span>, <span class="hljs-number">7</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>],
 <span class="hljs-string">&#x27;tokens&#x27;</span>: [<span class="hljs-string">&#x27;@paulwalk&#x27;</span>, <span class="hljs-string">&#x27;It&#x27;</span>, <span class="hljs-string">&quot;&#x27;s&quot;</span>, <span class="hljs-string">&#x27;the&#x27;</span>, <span class="hljs-string">&#x27;view&#x27;</span>, <span class="hljs-string">&#x27;from&#x27;</span>, <span class="hljs-string">&#x27;where&#x27;</span>, <span class="hljs-string">&#x27;I&#x27;</span>, <span class="hljs-string">&quot;&#x27;m&quot;</span>, <span class="hljs-string">&#x27;living&#x27;</span>, <span class="hljs-string">&#x27;for&#x27;</span>, <span class="hljs-string">&#x27;two&#x27;</span>, <span class="hljs-string">&#x27;weeks&#x27;</span>, <span class="hljs-string">&#x27;.&#x27;</span>, <span class="hljs-string">&#x27;Empire&#x27;</span>, <span class="hljs-string">&#x27;State&#x27;</span>, <span class="hljs-string">&#x27;Building&#x27;</span>, <span class="hljs-string">&#x27;=&#x27;</span>, <span class="hljs-string">&#x27;ESB&#x27;</span>, <span class="hljs-string">&#x27;.&#x27;</span>, <span class="hljs-string">&#x27;Pretty&#x27;</span>, <span class="hljs-string">&#x27;bad&#x27;</span>, <span class="hljs-string">&#x27;storm&#x27;</span>, <span class="hljs-string">&#x27;here&#x27;</span>, <span class="hljs-string">&#x27;last&#x27;</span>, <span class="hljs-string">&#x27;evening&#x27;</span>, <span class="hljs-string">&#x27;.&#x27;</span>]
}`}}),cs=new ts({props:{code:`label_list = wnut["train"].features[f"ner_tags"].feature.names
label_list`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>label_list = wnut[<span class="hljs-string">&quot;train&quot;</span>].features[<span class="hljs-string">f&quot;ner_tags&quot;</span>].feature.names
<span class="hljs-meta">&gt;&gt;&gt; </span>label_list
[
    <span class="hljs-string">&quot;O&quot;</span>,
    <span class="hljs-string">&quot;B-corporation&quot;</span>,
    <span class="hljs-string">&quot;I-corporation&quot;</span>,
    <span class="hljs-string">&quot;B-creative-work&quot;</span>,
    <span class="hljs-string">&quot;I-creative-work&quot;</span>,
    <span class="hljs-string">&quot;B-group&quot;</span>,
    <span class="hljs-string">&quot;I-group&quot;</span>,
    <span class="hljs-string">&quot;B-location&quot;</span>,
    <span class="hljs-string">&quot;I-location&quot;</span>,
    <span class="hljs-string">&quot;B-person&quot;</span>,
    <span class="hljs-string">&quot;I-person&quot;</span>,
    <span class="hljs-string">&quot;B-product&quot;</span>,
    <span class="hljs-string">&quot;I-product&quot;</span>,
]`}}),Ws=new ue({}),Ys=new hn({props:{id:"iY2AZYdZAr0"}}),Hs=new ts({props:{code:`from transformers import AutoTokenizer

tokenizer = AutoTokenizer.from_pretrained("distilbert-base-uncased")`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoTokenizer

<span class="hljs-meta">&gt;&gt;&gt; </span>tokenizer = AutoTokenizer.from_pretrained(<span class="hljs-string">&quot;distilbert-base-uncased&quot;</span>)`}}),Ks=new ts({props:{code:`tokenized_input = tokenizer(example["tokens"], is_split_into_words=True)
tokens = tokenizer.convert_ids_to_tokens(tokenized_input["input_ids"])
tokens`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>tokenized_input = tokenizer(example[<span class="hljs-string">&quot;tokens&quot;</span>], is_split_into_words=<span class="hljs-literal">True</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>tokens = tokenizer.convert_ids_to_tokens(tokenized_input[<span class="hljs-string">&quot;input_ids&quot;</span>])
<span class="hljs-meta">&gt;&gt;&gt; </span>tokens
[<span class="hljs-string">&#x27;[CLS]&#x27;</span>, <span class="hljs-string">&#x27;@&#x27;</span>, <span class="hljs-string">&#x27;paul&#x27;</span>, <span class="hljs-string">&#x27;##walk&#x27;</span>, <span class="hljs-string">&#x27;it&#x27;</span>, <span class="hljs-string">&quot;&#x27;&quot;</span>, <span class="hljs-string">&#x27;s&#x27;</span>, <span class="hljs-string">&#x27;the&#x27;</span>, <span class="hljs-string">&#x27;view&#x27;</span>, <span class="hljs-string">&#x27;from&#x27;</span>, <span class="hljs-string">&#x27;where&#x27;</span>, <span class="hljs-string">&#x27;i&#x27;</span>, <span class="hljs-string">&quot;&#x27;&quot;</span>, <span class="hljs-string">&#x27;m&#x27;</span>, <span class="hljs-string">&#x27;living&#x27;</span>, <span class="hljs-string">&#x27;for&#x27;</span>, <span class="hljs-string">&#x27;two&#x27;</span>, <span class="hljs-string">&#x27;weeks&#x27;</span>, <span class="hljs-string">&#x27;.&#x27;</span>, <span class="hljs-string">&#x27;empire&#x27;</span>, <span class="hljs-string">&#x27;state&#x27;</span>, <span class="hljs-string">&#x27;building&#x27;</span>, <span class="hljs-string">&#x27;=&#x27;</span>, <span class="hljs-string">&#x27;es&#x27;</span>, <span class="hljs-string">&#x27;##b&#x27;</span>, <span class="hljs-string">&#x27;.&#x27;</span>, <span class="hljs-string">&#x27;pretty&#x27;</span>, <span class="hljs-string">&#x27;bad&#x27;</span>, <span class="hljs-string">&#x27;storm&#x27;</span>, <span class="hljs-string">&#x27;here&#x27;</span>, <span class="hljs-string">&#x27;last&#x27;</span>, <span class="hljs-string">&#x27;evening&#x27;</span>, <span class="hljs-string">&#x27;.&#x27;</span>, <span class="hljs-string">&#x27;[SEP]&#x27;</span>]`}}),Js=new ts({props:{code:`def tokenize_and_align_labels(examples):
    tokenized_inputs = tokenizer(examples["tokens"], truncation=True, is_split_into_words=True)

    labels = []
    for i, label in enumerate(examples[f"ner_tags"]):
        word_ids = tokenized_inputs.word_ids(batch_index=i)  # Map tokens to their respective word.
        previous_word_idx = None
        label_ids = []
        for word_idx in word_ids:  # Set the special tokens to -100.
            if word_idx is None:
                label_ids.append(-100)
            elif word_idx != previous_word_idx:  # Only label the first token of a given word.
                label_ids.append(label[word_idx])
            else:
                label_ids.append(-100)
            previous_word_idx = word_idx
        labels.append(label_ids)

    tokenized_inputs["labels"] = labels
    return tokenized_inputs`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">def</span> <span class="hljs-title function_">tokenize_and_align_labels</span>(<span class="hljs-params">examples</span>):
<span class="hljs-meta">... </span>    tokenized_inputs = tokenizer(examples[<span class="hljs-string">&quot;tokens&quot;</span>], truncation=<span class="hljs-literal">True</span>, is_split_into_words=<span class="hljs-literal">True</span>)

<span class="hljs-meta">... </span>    labels = []
<span class="hljs-meta">... </span>    <span class="hljs-keyword">for</span> i, label <span class="hljs-keyword">in</span> <span class="hljs-built_in">enumerate</span>(examples[<span class="hljs-string">f&quot;ner_tags&quot;</span>]):
<span class="hljs-meta">... </span>        word_ids = tokenized_inputs.word_ids(batch_index=i)  <span class="hljs-comment"># Map tokens to their respective word.</span>
<span class="hljs-meta">... </span>        previous_word_idx = <span class="hljs-literal">None</span>
<span class="hljs-meta">... </span>        label_ids = []
<span class="hljs-meta">... </span>        <span class="hljs-keyword">for</span> word_idx <span class="hljs-keyword">in</span> word_ids:  <span class="hljs-comment"># Set the special tokens to -100.</span>
<span class="hljs-meta">... </span>            <span class="hljs-keyword">if</span> word_idx <span class="hljs-keyword">is</span> <span class="hljs-literal">None</span>:
<span class="hljs-meta">... </span>                label_ids.append(-<span class="hljs-number">100</span>)
<span class="hljs-meta">... </span>            <span class="hljs-keyword">elif</span> word_idx != previous_word_idx:  <span class="hljs-comment"># Only label the first token of a given word.</span>
<span class="hljs-meta">... </span>                label_ids.append(label[word_idx])
<span class="hljs-meta">... </span>            <span class="hljs-keyword">else</span>:
<span class="hljs-meta">... </span>                label_ids.append(-<span class="hljs-number">100</span>)
<span class="hljs-meta">... </span>            previous_word_idx = word_idx
<span class="hljs-meta">... </span>        labels.append(label_ids)

<span class="hljs-meta">... </span>    tokenized_inputs[<span class="hljs-string">&quot;labels&quot;</span>] = labels
<span class="hljs-meta">... </span>    <span class="hljs-keyword">return</span> tokenized_inputs`}}),Xs=new ts({props:{code:"tokenized_wnut = wnut.map(tokenize_and_align_labels, batched=True)",highlighted:'<span class="hljs-meta">&gt;&gt;&gt; </span>tokenized_wnut = wnut.<span class="hljs-built_in">map</span>(tokenize_and_align_labels, batched=<span class="hljs-literal">True</span>)'}}),Ls=new mn({props:{pytorch:!0,tensorflow:!0,jax:!1,$$slots:{tensorflow:[vn],pytorch:[bn]},$$scope:{ctx:P}}}),st=new ue({}),Ns=new mn({props:{pytorch:!0,tensorflow:!0,jax:!1,$$slots:{tensorflow:[Cn],pytorch:[Tn]},$$scope:{ctx:P}}}),Bs=new de({props:{$$slots:{default:[An]},$$scope:{ctx:P}}}),{c(){a=r("meta"),m=_(),n=r("h1"),u=r("a"),k=r("span"),v($.$$.fragment),w=_(),C=r("span"),A=l("Token classification"),x=_(),v(F.$$.fragment),W=_(),G=r("p"),L=l("Token classification assigns a label to individual tokens in a sentence. One of the most common token classification tasks is Named Entity Recognition (NER). NER attempts to find a label for each entity in a sentence, such as a person, location, or organization."),K=_(),S=r("p"),es=l("This guide will show you how to fine-tune "),Y=r("a"),$s=l("DistilBERT"),ms=l(" on the "),O=r("a"),us=l("WNUT 17"),M=l(" dataset to detect new entities."),ds=_(),v(R.$$.fragment),_s=_(),I=r("h2"),V=r("a"),N=r("span"),v(J.$$.fragment),Q=_(),os=r("span"),U=l("Load WNUT 17 dataset"),as=_(),d=r("p"),D=l("Load the WNUT 17 dataset from the \u{1F917} Datasets library:"),ns=_(),v(H.$$.fragment),X=_(),B=r("p"),ks=l("Then take a look at an example:"),Z=_(),v(ss.$$.fragment),is=_(),ls=r("p"),Cs=l("Each number in "),ps=r("code"),As=l("ner_tags"),c=l(" represents an entity. Convert the number to a label name for more information:"),b=_(),v(cs.$$.fragment),Us=_(),fs=r("p"),at=l("The "),Ds=r("code"),Ps=l("ner_tag"),nt=l(" describes an entity, such as a corporation, location, or person. The letter that prefixes each "),ws=r("code"),lt=l("ner_tag"),ot=l(" indicates the token position of the entity:"),ys=_(),hs=r("ul"),Ts=r("li"),dt=r("code"),ge=l("B-"),$e=l(" indicates the beginning of an entity."),je=_(),bs=r("li"),_t=r("code"),ke=l("I-"),we=l(" indicates a token is contained inside the same entity (e.g., the "),gt=r("code"),be=l("State"),xe=l(` token is a part of an entity like
`),$t=r("code"),ve=l("Empire State Building"),Ee=l(")."),ye=_(),rt=r("li"),jt=r("code"),Te=l("0"),ze=l(" indicates the token doesn\u2019t correspond to any entity."),Rt=_(),zs=r("h2"),Fs=r("a"),kt=r("span"),v(Ws.$$.fragment),qe=_(),wt=r("span"),Ce=l("Preprocess"),Ut=_(),v(Ys.$$.fragment),Wt=_(),Ss=r("p"),Ae=l("Load the DistilBERT tokenizer to process the "),bt=r("code"),De=l("tokens"),Pe=l(":"),Yt=_(),v(Hs.$$.fragment),Ht=_(),Os=r("p"),Fe=l("Since the input has already been split into words, set "),xt=r("code"),Se=l("is_split_into_words=True"),Oe=l(" to tokenize the words into subwords:"),Kt=_(),v(Ks.$$.fragment),Vt=_(),xs=r("p"),Le=l("Adding the special tokens "),vt=r("code"),Ie=l("[CLS]"),Ne=l(" and "),Et=r("code"),Be=l("[SEP]"),Me=l(" and subword tokenization creates a mismatch between the input and labels. A single word corresponding to a single label may be split into two subwords. You will need to realign the tokens and labels by:"),Zt=_(),vs=r("ol"),Vs=r("li"),Re=l("Mapping all tokens to their corresponding word with the "),Zs=r("a"),yt=r("code"),Ue=l("word_ids"),We=l(" method."),Ye=_(),js=r("li"),He=l("Assigning the label "),Tt=r("code"),Ke=l("-100"),Ve=l(" to the special tokens "),zt=r("code"),Ze=l("[CLS]"),Ge=l(" and "),qt=r("code"),Je=l("[SEP]"),Qe=l(` so the PyTorch loss function ignores
them.`),Xe=_(),Gs=r("li"),sa=l("Only labeling the first token of a given word. Assign "),Ct=r("code"),ta=l("-100"),ea=l(" to other subtokens from the same word."),Gt=_(),it=r("p"),aa=l("Here is how you can create a function to realign the tokens and labels, and truncate sequences to be no longer than DistilBERT\u2019s maximum input length::"),Jt=_(),v(Js.$$.fragment),Qt=_(),gs=r("p"),na=l("Use \u{1F917} Datasets "),Qs=r("a"),At=r("code"),la=l("map"),oa=l(" function to tokenize and align the labels over the entire dataset. You can speed up the "),Dt=r("code"),ra=l("map"),ia=l(" function by setting "),Pt=r("code"),pa=l("batched=True"),ca=l(" to process multiple elements of the dataset at once:"),Xt=_(),v(Xs.$$.fragment),se=_(),rs=r("p"),fa=l("Use "),pt=r("a"),ha=l("DataCollatorForTokenClassification"),ma=l(" to create a batch of examples. It will also "),Ft=r("em"),ua=l("dynamically pad"),da=l(" your text and labels to the length of the longest element in its batch, so they are a uniform length. While it is possible to pad your text in the "),St=r("code"),_a=l("tokenizer"),ga=l(" function by setting "),Ot=r("code"),$a=l("padding=True"),ja=l(", dynamic padding is more efficient."),te=_(),v(Ls.$$.fragment),ee=_(),qs=r("h2"),Is=r("a"),Lt=r("span"),v(st.$$.fragment),ka=_(),It=r("span"),wa=l("Train"),ae=_(),v(Ns.$$.fragment),ne=_(),v(Bs.$$.fragment),this.h()},l(s){const h=$n('[data-svelte="svelte-1phssyn"]',document.head);a=i(h,"META",{name:!0,content:!0}),h.forEach(e),m=g(s),n=i(s,"H1",{class:!0});var tt=p(n);u=i(tt,"A",{id:!0,class:!0,href:!0});var Nt=p(u);k=i(Nt,"SPAN",{});var Bt=p(k);E($.$$.fragment,Bt),Bt.forEach(e),Nt.forEach(e),w=g(tt),C=i(tt,"SPAN",{});var Mt=p(C);A=o(Mt,"Token classification"),Mt.forEach(e),tt.forEach(e),x=g(s),E(F.$$.fragment,s),W=g(s),G=i(s,"P",{});var va=p(G);L=o(va,"Token classification assigns a label to individual tokens in a sentence. One of the most common token classification tasks is Named Entity Recognition (NER). NER attempts to find a label for each entity in a sentence, such as a person, location, or organization."),va.forEach(e),K=g(s),S=i(s,"P",{});var ct=p(S);es=o(ct,"This guide will show you how to fine-tune "),Y=i(ct,"A",{href:!0,rel:!0});var Ea=p(Y);$s=o(Ea,"DistilBERT"),Ea.forEach(e),ms=o(ct," on the "),O=i(ct,"A",{href:!0,rel:!0});var ya=p(O);us=o(ya,"WNUT 17"),ya.forEach(e),M=o(ct," dataset to detect new entities."),ct.forEach(e),ds=g(s),E(R.$$.fragment,s),_s=g(s),I=i(s,"H2",{class:!0});var oe=p(I);V=i(oe,"A",{id:!0,class:!0,href:!0});var Ta=p(V);N=i(Ta,"SPAN",{});var za=p(N);E(J.$$.fragment,za),za.forEach(e),Ta.forEach(e),Q=g(oe),os=i(oe,"SPAN",{});var qa=p(os);U=o(qa,"Load WNUT 17 dataset"),qa.forEach(e),oe.forEach(e),as=g(s),d=i(s,"P",{});var Ca=p(d);D=o(Ca,"Load the WNUT 17 dataset from the \u{1F917} Datasets library:"),Ca.forEach(e),ns=g(s),E(H.$$.fragment,s),X=g(s),B=i(s,"P",{});var Aa=p(B);ks=o(Aa,"Then take a look at an example:"),Aa.forEach(e),Z=g(s),E(ss.$$.fragment,s),is=g(s),ls=i(s,"P",{});var re=p(ls);Cs=o(re,"Each number in "),ps=i(re,"CODE",{});var Da=p(ps);As=o(Da,"ner_tags"),Da.forEach(e),c=o(re," represents an entity. Convert the number to a label name for more information:"),re.forEach(e),b=g(s),E(cs.$$.fragment,s),Us=g(s),fs=i(s,"P",{});var ft=p(fs);at=o(ft,"The "),Ds=i(ft,"CODE",{});var Pa=p(Ds);Ps=o(Pa,"ner_tag"),Pa.forEach(e),nt=o(ft," describes an entity, such as a corporation, location, or person. The letter that prefixes each "),ws=i(ft,"CODE",{});var Fa=p(ws);lt=o(Fa,"ner_tag"),Fa.forEach(e),ot=o(ft," indicates the token position of the entity:"),ft.forEach(e),ys=g(s),hs=i(s,"UL",{});var ht=p(hs);Ts=i(ht,"LI",{});var ba=p(Ts);dt=i(ba,"CODE",{});var Sa=p(dt);ge=o(Sa,"B-"),Sa.forEach(e),$e=o(ba," indicates the beginning of an entity."),ba.forEach(e),je=g(ht),bs=i(ht,"LI",{});var et=p(bs);_t=i(et,"CODE",{});var Oa=p(_t);ke=o(Oa,"I-"),Oa.forEach(e),we=o(et," indicates a token is contained inside the same entity (e.g., the "),gt=i(et,"CODE",{});var La=p(gt);be=o(La,"State"),La.forEach(e),xe=o(et,` token is a part of an entity like
`),$t=i(et,"CODE",{});var Ia=p($t);ve=o(Ia,"Empire State Building"),Ia.forEach(e),Ee=o(et,")."),et.forEach(e),ye=g(ht),rt=i(ht,"LI",{});var xa=p(rt);jt=i(xa,"CODE",{});var Na=p(jt);Te=o(Na,"0"),Na.forEach(e),ze=o(xa," indicates the token doesn\u2019t correspond to any entity."),xa.forEach(e),ht.forEach(e),Rt=g(s),zs=i(s,"H2",{class:!0});var ie=p(zs);Fs=i(ie,"A",{id:!0,class:!0,href:!0});var Ba=p(Fs);kt=i(Ba,"SPAN",{});var Ma=p(kt);E(Ws.$$.fragment,Ma),Ma.forEach(e),Ba.forEach(e),qe=g(ie),wt=i(ie,"SPAN",{});var Ra=p(wt);Ce=o(Ra,"Preprocess"),Ra.forEach(e),ie.forEach(e),Ut=g(s),E(Ys.$$.fragment,s),Wt=g(s),Ss=i(s,"P",{});var pe=p(Ss);Ae=o(pe,"Load the DistilBERT tokenizer to process the "),bt=i(pe,"CODE",{});var Ua=p(bt);De=o(Ua,"tokens"),Ua.forEach(e),Pe=o(pe,":"),pe.forEach(e),Yt=g(s),E(Hs.$$.fragment,s),Ht=g(s),Os=i(s,"P",{});var ce=p(Os);Fe=o(ce,"Since the input has already been split into words, set "),xt=i(ce,"CODE",{});var Wa=p(xt);Se=o(Wa,"is_split_into_words=True"),Wa.forEach(e),Oe=o(ce," to tokenize the words into subwords:"),ce.forEach(e),Kt=g(s),E(Ks.$$.fragment,s),Vt=g(s),xs=i(s,"P",{});var mt=p(xs);Le=o(mt,"Adding the special tokens "),vt=i(mt,"CODE",{});var Ya=p(vt);Ie=o(Ya,"[CLS]"),Ya.forEach(e),Ne=o(mt," and "),Et=i(mt,"CODE",{});var Ha=p(Et);Be=o(Ha,"[SEP]"),Ha.forEach(e),Me=o(mt," and subword tokenization creates a mismatch between the input and labels. A single word corresponding to a single label may be split into two subwords. You will need to realign the tokens and labels by:"),mt.forEach(e),Zt=g(s),vs=i(s,"OL",{});var ut=p(vs);Vs=i(ut,"LI",{});var fe=p(Vs);Re=o(fe,"Mapping all tokens to their corresponding word with the "),Zs=i(fe,"A",{href:!0,rel:!0});var Ka=p(Zs);yt=i(Ka,"CODE",{});var Va=p(yt);Ue=o(Va,"word_ids"),Va.forEach(e),Ka.forEach(e),We=o(fe," method."),fe.forEach(e),Ye=g(ut),js=i(ut,"LI",{});var Ms=p(js);He=o(Ms,"Assigning the label "),Tt=i(Ms,"CODE",{});var Za=p(Tt);Ke=o(Za,"-100"),Za.forEach(e),Ve=o(Ms," to the special tokens "),zt=i(Ms,"CODE",{});var Ga=p(zt);Ze=o(Ga,"[CLS]"),Ga.forEach(e),Ge=o(Ms," and "),qt=i(Ms,"CODE",{});var Ja=p(qt);Je=o(Ja,"[SEP]"),Ja.forEach(e),Qe=o(Ms,` so the PyTorch loss function ignores
them.`),Ms.forEach(e),Xe=g(ut),Gs=i(ut,"LI",{});var he=p(Gs);sa=o(he,"Only labeling the first token of a given word. Assign "),Ct=i(he,"CODE",{});var Qa=p(Ct);ta=o(Qa,"-100"),Qa.forEach(e),ea=o(he," to other subtokens from the same word."),he.forEach(e),ut.forEach(e),Gt=g(s),it=i(s,"P",{});var Xa=p(it);aa=o(Xa,"Here is how you can create a function to realign the tokens and labels, and truncate sequences to be no longer than DistilBERT\u2019s maximum input length::"),Xa.forEach(e),Jt=g(s),E(Js.$$.fragment,s),Qt=g(s),gs=i(s,"P",{});var Rs=p(gs);na=o(Rs,"Use \u{1F917} Datasets "),Qs=i(Rs,"A",{href:!0,rel:!0});var sn=p(Qs);At=i(sn,"CODE",{});var tn=p(At);la=o(tn,"map"),tn.forEach(e),sn.forEach(e),oa=o(Rs," function to tokenize and align the labels over the entire dataset. You can speed up the "),Dt=i(Rs,"CODE",{});var en=p(Dt);ra=o(en,"map"),en.forEach(e),ia=o(Rs," function by setting "),Pt=i(Rs,"CODE",{});var an=p(Pt);pa=o(an,"batched=True"),an.forEach(e),ca=o(Rs," to process multiple elements of the dataset at once:"),Rs.forEach(e),Xt=g(s),E(Xs.$$.fragment,s),se=g(s),rs=i(s,"P",{});var Es=p(rs);fa=o(Es,"Use "),pt=i(Es,"A",{href:!0});var nn=p(pt);ha=o(nn,"DataCollatorForTokenClassification"),nn.forEach(e),ma=o(Es," to create a batch of examples. It will also "),Ft=i(Es,"EM",{});var ln=p(Ft);ua=o(ln,"dynamically pad"),ln.forEach(e),da=o(Es," your text and labels to the length of the longest element in its batch, so they are a uniform length. While it is possible to pad your text in the "),St=i(Es,"CODE",{});var on=p(St);_a=o(on,"tokenizer"),on.forEach(e),ga=o(Es," function by setting "),Ot=i(Es,"CODE",{});var rn=p(Ot);$a=o(rn,"padding=True"),rn.forEach(e),ja=o(Es,", dynamic padding is more efficient."),Es.forEach(e),te=g(s),E(Ls.$$.fragment,s),ee=g(s),qs=i(s,"H2",{class:!0});var me=p(qs);Is=i(me,"A",{id:!0,class:!0,href:!0});var pn=p(Is);Lt=i(pn,"SPAN",{});var cn=p(Lt);E(st.$$.fragment,cn),cn.forEach(e),pn.forEach(e),ka=g(me),It=i(me,"SPAN",{});var fn=p(It);wa=o(fn,"Train"),fn.forEach(e),me.forEach(e),ae=g(s),E(Ns.$$.fragment,s),ne=g(s),E(Bs.$$.fragment,s),this.h()},h(){j(a,"name","hf:doc:metadata"),j(a,"content",JSON.stringify(Pn)),j(u,"id","token-classification"),j(u,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),j(u,"href","#token-classification"),j(n,"class","relative group"),j(Y,"href","https://huggingface.co/distilbert-base-uncased"),j(Y,"rel","nofollow"),j(O,"href","https://huggingface.co/datasets/wnut_17"),j(O,"rel","nofollow"),j(V,"id","load-wnut-17-dataset"),j(V,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),j(V,"href","#load-wnut-17-dataset"),j(I,"class","relative group"),j(Fs,"id","preprocess"),j(Fs,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),j(Fs,"href","#preprocess"),j(zs,"class","relative group"),j(Zs,"href","https://huggingface.co/docs/tokenizers/python/latest/api/reference.html#tokenizers.Encoding.word_ids"),j(Zs,"rel","nofollow"),j(Qs,"href","https://huggingface.co/docs/datasets/package_reference/main_classes.html#datasets.Dataset.map"),j(Qs,"rel","nofollow"),j(pt,"href","/docs/transformers/pr_17548/en/main_classes/data_collator#transformers.DataCollatorForTokenClassification"),j(Is,"id","train"),j(Is,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),j(Is,"href","#train"),j(qs,"class","relative group")},m(s,h){t(document.head,a),f(s,m,h),f(s,n,h),t(n,u),t(u,k),y($,k,null),t(n,w),t(n,C),t(C,A),f(s,x,h),y(F,s,h),f(s,W,h),f(s,G,h),t(G,L),f(s,K,h),f(s,S,h),t(S,es),t(S,Y),t(Y,$s),t(S,ms),t(S,O),t(O,us),t(S,M),f(s,ds,h),y(R,s,h),f(s,_s,h),f(s,I,h),t(I,V),t(V,N),y(J,N,null),t(I,Q),t(I,os),t(os,U),f(s,as,h),f(s,d,h),t(d,D),f(s,ns,h),y(H,s,h),f(s,X,h),f(s,B,h),t(B,ks),f(s,Z,h),y(ss,s,h),f(s,is,h),f(s,ls,h),t(ls,Cs),t(ls,ps),t(ps,As),t(ls,c),f(s,b,h),y(cs,s,h),f(s,Us,h),f(s,fs,h),t(fs,at),t(fs,Ds),t(Ds,Ps),t(fs,nt),t(fs,ws),t(ws,lt),t(fs,ot),f(s,ys,h),f(s,hs,h),t(hs,Ts),t(Ts,dt),t(dt,ge),t(Ts,$e),t(hs,je),t(hs,bs),t(bs,_t),t(_t,ke),t(bs,we),t(bs,gt),t(gt,be),t(bs,xe),t(bs,$t),t($t,ve),t(bs,Ee),t(hs,ye),t(hs,rt),t(rt,jt),t(jt,Te),t(rt,ze),f(s,Rt,h),f(s,zs,h),t(zs,Fs),t(Fs,kt),y(Ws,kt,null),t(zs,qe),t(zs,wt),t(wt,Ce),f(s,Ut,h),y(Ys,s,h),f(s,Wt,h),f(s,Ss,h),t(Ss,Ae),t(Ss,bt),t(bt,De),t(Ss,Pe),f(s,Yt,h),y(Hs,s,h),f(s,Ht,h),f(s,Os,h),t(Os,Fe),t(Os,xt),t(xt,Se),t(Os,Oe),f(s,Kt,h),y(Ks,s,h),f(s,Vt,h),f(s,xs,h),t(xs,Le),t(xs,vt),t(vt,Ie),t(xs,Ne),t(xs,Et),t(Et,Be),t(xs,Me),f(s,Zt,h),f(s,vs,h),t(vs,Vs),t(Vs,Re),t(Vs,Zs),t(Zs,yt),t(yt,Ue),t(Vs,We),t(vs,Ye),t(vs,js),t(js,He),t(js,Tt),t(Tt,Ke),t(js,Ve),t(js,zt),t(zt,Ze),t(js,Ge),t(js,qt),t(qt,Je),t(js,Qe),t(vs,Xe),t(vs,Gs),t(Gs,sa),t(Gs,Ct),t(Ct,ta),t(Gs,ea),f(s,Gt,h),f(s,it,h),t(it,aa),f(s,Jt,h),y(Js,s,h),f(s,Qt,h),f(s,gs,h),t(gs,na),t(gs,Qs),t(Qs,At),t(At,la),t(gs,oa),t(gs,Dt),t(Dt,ra),t(gs,ia),t(gs,Pt),t(Pt,pa),t(gs,ca),f(s,Xt,h),y(Xs,s,h),f(s,se,h),f(s,rs,h),t(rs,fa),t(rs,pt),t(pt,ha),t(rs,ma),t(rs,Ft),t(Ft,ua),t(rs,da),t(rs,St),t(St,_a),t(rs,ga),t(rs,Ot),t(Ot,$a),t(rs,ja),f(s,te,h),y(Ls,s,h),f(s,ee,h),f(s,qs,h),t(qs,Is),t(Is,Lt),y(st,Lt,null),t(qs,ka),t(qs,It),t(It,wa),f(s,ae,h),y(Ns,s,h),f(s,ne,h),y(Bs,s,h),le=!0},p(s,[h]){const tt={};h&2&&(tt.$$scope={dirty:h,ctx:s}),R.$set(tt);const Nt={};h&2&&(Nt.$$scope={dirty:h,ctx:s}),Ls.$set(Nt);const Bt={};h&2&&(Bt.$$scope={dirty:h,ctx:s}),Ns.$set(Bt);const Mt={};h&2&&(Mt.$$scope={dirty:h,ctx:s}),Bs.$set(Mt)},i(s){le||(T($.$$.fragment,s),T(F.$$.fragment,s),T(R.$$.fragment,s),T(J.$$.fragment,s),T(H.$$.fragment,s),T(ss.$$.fragment,s),T(cs.$$.fragment,s),T(Ws.$$.fragment,s),T(Ys.$$.fragment,s),T(Hs.$$.fragment,s),T(Ks.$$.fragment,s),T(Js.$$.fragment,s),T(Xs.$$.fragment,s),T(Ls.$$.fragment,s),T(st.$$.fragment,s),T(Ns.$$.fragment,s),T(Bs.$$.fragment,s),le=!0)},o(s){z($.$$.fragment,s),z(F.$$.fragment,s),z(R.$$.fragment,s),z(J.$$.fragment,s),z(H.$$.fragment,s),z(ss.$$.fragment,s),z(cs.$$.fragment,s),z(Ws.$$.fragment,s),z(Ys.$$.fragment,s),z(Hs.$$.fragment,s),z(Ks.$$.fragment,s),z(Js.$$.fragment,s),z(Xs.$$.fragment,s),z(Ls.$$.fragment,s),z(st.$$.fragment,s),z(Ns.$$.fragment,s),z(Bs.$$.fragment,s),le=!1},d(s){e(a),s&&e(m),s&&e(n),q($),s&&e(x),q(F,s),s&&e(W),s&&e(G),s&&e(K),s&&e(S),s&&e(ds),q(R,s),s&&e(_s),s&&e(I),q(J),s&&e(as),s&&e(d),s&&e(ns),q(H,s),s&&e(X),s&&e(B),s&&e(Z),q(ss,s),s&&e(is),s&&e(ls),s&&e(b),q(cs,s),s&&e(Us),s&&e(fs),s&&e(ys),s&&e(hs),s&&e(Rt),s&&e(zs),q(Ws),s&&e(Ut),q(Ys,s),s&&e(Wt),s&&e(Ss),s&&e(Yt),q(Hs,s),s&&e(Ht),s&&e(Os),s&&e(Kt),q(Ks,s),s&&e(Vt),s&&e(xs),s&&e(Zt),s&&e(vs),s&&e(Gt),s&&e(it),s&&e(Jt),q(Js,s),s&&e(Qt),s&&e(gs),s&&e(Xt),q(Xs,s),s&&e(se),s&&e(rs),s&&e(te),q(Ls,s),s&&e(ee),s&&e(qs),q(st),s&&e(ae),q(Ns,s),s&&e(ne),q(Bs,s)}}}const Pn={local:"token-classification",sections:[{local:"load-wnut-17-dataset",title:"Load WNUT 17 dataset"},{local:"preprocess",title:"Preprocess"},{local:"train",title:"Train"}],title:"Token classification"};function Fn(P){return jn(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class Mn extends dn{constructor(a){super();_n(this,a,Fn,Dn,gn,{})}}export{Mn as default,Pn as metadata};
