import{S as mn,i as dn,s as un,e as r,k as _,w as E,t as l,M as _n,c as i,d as t,m as g,a as p,x as y,h as o,b as j,G as e,g as f,y as T,q as z,o as q,B as C,v as gn,L as hn}from"../../chunks/vendor-hf-doc-builder.js";import{T as ht}from"../../chunks/Tip-hf-doc-builder.js";import{Y as cn}from"../../chunks/Youtube-hf-doc-builder.js";import{I as ft}from"../../chunks/IconCopyLink-hf-doc-builder.js";import{C as ss}from"../../chunks/CodeBlock-hf-doc-builder.js";import{F as fn,M as mt}from"../../chunks/Markdown-hf-doc-builder.js";function $n(S){let a,m,n,u,k;return{c(){a=r("p"),m=l("See the token classification "),n=r("a"),u=l("task page"),k=l(" for more information about other forms of token classification and their associated models, datasets, and metrics."),this.h()},l($){a=i($,"P",{});var w=p(a);m=o(w,"See the token classification "),n=i(w,"A",{href:!0,rel:!0});var D=p(n);u=o(D,"task page"),D.forEach(t),k=o(w," for more information about other forms of token classification and their associated models, datasets, and metrics."),w.forEach(t),this.h()},h(){j(n,"href","https://huggingface.co/tasks/token-classification"),j(n,"rel","nofollow")},m($,w){f($,a,w),e(a,m),e(a,n),e(n,u),e(a,k)},d($){$&&t(a)}}}function jn(S){let a,m;return a=new ss({props:{code:`from transformers import DataCollatorForTokenClassification

data_collator = DataCollatorForTokenClassification(tokenizer=tokenizer)`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> DataCollatorForTokenClassification

<span class="hljs-meta">&gt;&gt;&gt; </span>data_collator = DataCollatorForTokenClassification(tokenizer=tokenizer)`}}),{c(){E(a.$$.fragment)},l(n){y(a.$$.fragment,n)},m(n,u){T(a,n,u),m=!0},p:hn,i(n){m||(z(a.$$.fragment,n),m=!0)},o(n){q(a.$$.fragment,n),m=!1},d(n){C(a,n)}}}function kn(S){let a,m;return a=new mt({props:{$$slots:{default:[jn]},$$scope:{ctx:S}}}),{c(){E(a.$$.fragment)},l(n){y(a.$$.fragment,n)},m(n,u){T(a,n,u),m=!0},p(n,u){const k={};u&2&&(k.$$scope={dirty:u,ctx:n}),a.$set(k)},i(n){m||(z(a.$$.fragment,n),m=!0)},o(n){q(a.$$.fragment,n),m=!1},d(n){C(a,n)}}}function wn(S){let a,m;return a=new ss({props:{code:`from transformers import DataCollatorForTokenClassification

data_collator = DataCollatorForTokenClassification(tokenizer=tokenizer, return_tensors="tf")`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> DataCollatorForTokenClassification

<span class="hljs-meta">&gt;&gt;&gt; </span>data_collator = DataCollatorForTokenClassification(tokenizer=tokenizer, return_tensors=<span class="hljs-string">&quot;tf&quot;</span>)`}}),{c(){E(a.$$.fragment)},l(n){y(a.$$.fragment,n)},m(n,u){T(a,n,u),m=!0},p:hn,i(n){m||(z(a.$$.fragment,n),m=!0)},o(n){q(a.$$.fragment,n),m=!1},d(n){C(a,n)}}}function bn(S){let a,m;return a=new mt({props:{$$slots:{default:[wn]},$$scope:{ctx:S}}}),{c(){E(a.$$.fragment)},l(n){y(a.$$.fragment,n)},m(n,u){T(a,n,u),m=!0},p(n,u){const k={};u&2&&(k.$$scope={dirty:u,ctx:n}),a.$set(k)},i(n){m||(z(a.$$.fragment,n),m=!0)},o(n){q(a.$$.fragment,n),m=!1},d(n){C(a,n)}}}function xn(S){let a,m,n,u,k,$,w,D;return{c(){a=r("p"),m=l("If you aren\u2019t familiar with fine-tuning a model with the "),n=r("a"),u=l("Trainer"),k=l(", take a look at the basic tutorial "),$=r("a"),w=l("here"),D=l("!"),this.h()},l(A){a=i(A,"P",{});var b=p(a);m=o(b,"If you aren\u2019t familiar with fine-tuning a model with the "),n=i(b,"A",{href:!0});var F=p(n);u=o(F,"Trainer"),F.forEach(t),k=o(b,", take a look at the basic tutorial "),$=i(b,"A",{href:!0});var O=p($);w=o(O,"here"),O.forEach(t),D=o(b,"!"),b.forEach(t),this.h()},h(){j(n,"href","/docs/transformers/pr_19907/en/main_classes/trainer#transformers.Trainer"),j($,"href","../training#finetune-with-trainer")},m(A,b){f(A,a,b),e(a,m),e(a,n),e(n,u),e(a,k),e(a,$),e($,w),e(a,D)},d(A){A&&t(a)}}}function vn(S){let a,m,n,u,k,$,w,D,A,b,F,O,V,I,Z,B,R,G,J,hs,L,ms,ts,is,N,ps,P,Q,M,Y,ds,as,X,H;return w=new ss({props:{code:`from transformers import AutoModelForTokenClassification, TrainingArguments, Trainer

model = AutoModelForTokenClassification.from_pretrained("distilbert-base-uncased", num_labels=14)`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoModelForTokenClassification, TrainingArguments, Trainer

<span class="hljs-meta">&gt;&gt;&gt; </span>model = AutoModelForTokenClassification.from_pretrained(<span class="hljs-string">&quot;distilbert-base-uncased&quot;</span>, num_labels=<span class="hljs-number">14</span>)`}}),A=new ht({props:{$$slots:{default:[xn]},$$scope:{ctx:S}}}),X=new ss({props:{code:`training_args = TrainingArguments(
    output_dir="./results",
    evaluation_strategy="epoch",
    learning_rate=2e-5,
    per_device_train_batch_size=16,
    per_device_eval_batch_size=16,
    num_train_epochs=3,
    weight_decay=0.01,
)

trainer = Trainer(
    model=model,
    args=training_args,
    train_dataset=tokenized_wnut["train"],
    eval_dataset=tokenized_wnut["test"],
    tokenizer=tokenizer,
    data_collator=data_collator,
)

trainer.train()`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>training_args = TrainingArguments(
<span class="hljs-meta">... </span>    output_dir=<span class="hljs-string">&quot;./results&quot;</span>,
<span class="hljs-meta">... </span>    evaluation_strategy=<span class="hljs-string">&quot;epoch&quot;</span>,
<span class="hljs-meta">... </span>    learning_rate=<span class="hljs-number">2e-5</span>,
<span class="hljs-meta">... </span>    per_device_train_batch_size=<span class="hljs-number">16</span>,
<span class="hljs-meta">... </span>    per_device_eval_batch_size=<span class="hljs-number">16</span>,
<span class="hljs-meta">... </span>    num_train_epochs=<span class="hljs-number">3</span>,
<span class="hljs-meta">... </span>    weight_decay=<span class="hljs-number">0.01</span>,
<span class="hljs-meta">... </span>)

<span class="hljs-meta">&gt;&gt;&gt; </span>trainer = Trainer(
<span class="hljs-meta">... </span>    model=model,
<span class="hljs-meta">... </span>    args=training_args,
<span class="hljs-meta">... </span>    train_dataset=tokenized_wnut[<span class="hljs-string">&quot;train&quot;</span>],
<span class="hljs-meta">... </span>    eval_dataset=tokenized_wnut[<span class="hljs-string">&quot;test&quot;</span>],
<span class="hljs-meta">... </span>    tokenizer=tokenizer,
<span class="hljs-meta">... </span>    data_collator=data_collator,
<span class="hljs-meta">... </span>)

<span class="hljs-meta">&gt;&gt;&gt; </span>trainer.train()`}}),{c(){a=r("p"),m=l("Load DistilBERT with "),n=r("a"),u=l("AutoModelForTokenClassification"),k=l(" along with the number of expected labels:"),$=_(),E(w.$$.fragment),D=_(),E(A.$$.fragment),b=_(),F=r("p"),O=l("At this point, only three steps remain:"),V=_(),I=r("ol"),Z=r("li"),B=l("Define your training hyperparameters in "),R=r("a"),G=l("TrainingArguments"),J=l("."),hs=_(),L=r("li"),ms=l("Pass the training arguments to "),ts=r("a"),is=l("Trainer"),N=l(" along with the model, dataset, tokenizer, and data collator."),ps=_(),P=r("li"),Q=l("Call "),M=r("a"),Y=l("train()"),ds=l(" to fine-tune your model."),as=_(),E(X.$$.fragment),this.h()},l(d){a=i(d,"P",{});var v=p(a);m=o(v,"Load DistilBERT with "),n=i(v,"A",{href:!0});var ns=p(n);u=o(ns,"AutoModelForTokenClassification"),ns.forEach(t),k=o(v," along with the number of expected labels:"),v.forEach(t),$=g(d),y(w.$$.fragment,d),D=g(d),y(A.$$.fragment,d),b=g(d),F=i(d,"P",{});var U=p(F);O=o(U,"At this point, only three steps remain:"),U.forEach(t),V=g(d),I=i(d,"OL",{});var K=p(I);Z=i(K,"LI",{});var es=p(Z);B=o(es,"Define your training hyperparameters in "),R=i(es,"A",{href:!0});var gs=p(R);G=o(gs,"TrainingArguments"),gs.forEach(t),J=o(es,"."),es.forEach(t),hs=g(K),L=i(K,"LI",{});var ls=p(L);ms=o(ls,"Pass the training arguments to "),ts=i(ls,"A",{href:!0});var W=p(ts);is=o(W,"Trainer"),W.forEach(t),N=o(ls," along with the model, dataset, tokenizer, and data collator."),ls.forEach(t),ps=g(K),P=i(K,"LI",{});var os=p(P);Q=o(os,"Call "),M=i(os,"A",{href:!0});var c=p(M);Y=o(c,"train()"),c.forEach(t),ds=o(os," to fine-tune your model."),os.forEach(t),K.forEach(t),as=g(d),y(X.$$.fragment,d),this.h()},h(){j(n,"href","/docs/transformers/pr_19907/en/model_doc/auto#transformers.AutoModelForTokenClassification"),j(R,"href","/docs/transformers/pr_19907/en/main_classes/trainer#transformers.TrainingArguments"),j(ts,"href","/docs/transformers/pr_19907/en/main_classes/trainer#transformers.Trainer"),j(M,"href","/docs/transformers/pr_19907/en/main_classes/trainer#transformers.Trainer.train")},m(d,v){f(d,a,v),e(a,m),e(a,n),e(n,u),e(a,k),f(d,$,v),T(w,d,v),f(d,D,v),T(A,d,v),f(d,b,v),f(d,F,v),e(F,O),f(d,V,v),f(d,I,v),e(I,Z),e(Z,B),e(Z,R),e(R,G),e(Z,J),e(I,hs),e(I,L),e(L,ms),e(L,ts),e(ts,is),e(L,N),e(I,ps),e(I,P),e(P,Q),e(P,M),e(M,Y),e(P,ds),f(d,as,v),T(X,d,v),H=!0},p(d,v){const ns={};v&2&&(ns.$$scope={dirty:v,ctx:d}),A.$set(ns)},i(d){H||(z(w.$$.fragment,d),z(A.$$.fragment,d),z(X.$$.fragment,d),H=!0)},o(d){q(w.$$.fragment,d),q(A.$$.fragment,d),q(X.$$.fragment,d),H=!1},d(d){d&&t(a),d&&t($),C(w,d),d&&t(D),C(A,d),d&&t(b),d&&t(F),d&&t(V),d&&t(I),d&&t(as),C(X,d)}}}function En(S){let a,m;return a=new mt({props:{$$slots:{default:[vn]},$$scope:{ctx:S}}}),{c(){E(a.$$.fragment)},l(n){y(a.$$.fragment,n)},m(n,u){T(a,n,u),m=!0},p(n,u){const k={};u&2&&(k.$$scope={dirty:u,ctx:n}),a.$set(k)},i(n){m||(z(a.$$.fragment,n),m=!0)},o(n){q(a.$$.fragment,n),m=!1},d(n){C(a,n)}}}function yn(S){let a,m,n,u,k;return{c(){a=r("p"),m=l("If you aren\u2019t familiar with fine-tuning a model with Keras, take a look at the basic tutorial "),n=r("a"),u=l("here"),k=l("!"),this.h()},l($){a=i($,"P",{});var w=p(a);m=o(w,"If you aren\u2019t familiar with fine-tuning a model with Keras, take a look at the basic tutorial "),n=i(w,"A",{href:!0});var D=p(n);u=o(D,"here"),D.forEach(t),k=o(w,"!"),w.forEach(t),this.h()},h(){j(n,"href","training#finetune-with-keras")},m($,w){f($,a,w),e(a,m),e(a,n),e(n,u),e(a,k)},d($){$&&t(a)}}}function Tn(S){let a,m,n,u,k,$,w,D,A,b,F,O,V,I,Z,B,R,G,J,hs,L,ms,ts,is,N,ps,P,Q,M,Y,ds,as,X,H,d,v,ns,U,K,es,gs,ls,W,os;return b=new ss({props:{code:`tf_train_set = model.prepare_tf_dataset(
    tokenized_wnut["train"],
    shuffle=True,
    batch_size=16,
    collate_fn=data_collator,
)

tf_validation_set = model.prepare_tf_dataset(
    tokenized_wnut["validation"],
    shuffle=False,
    batch_size=16,
    collate_fn=data_collator,
)`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>tf_train_set = model.prepare_tf_dataset(
<span class="hljs-meta">... </span>    tokenized_wnut[<span class="hljs-string">&quot;train&quot;</span>],
<span class="hljs-meta">... </span>    shuffle=<span class="hljs-literal">True</span>,
<span class="hljs-meta">... </span>    batch_size=<span class="hljs-number">16</span>,
<span class="hljs-meta">... </span>    collate_fn=data_collator,
<span class="hljs-meta">... </span>)

<span class="hljs-meta">&gt;&gt;&gt; </span>tf_validation_set = model.prepare_tf_dataset(
<span class="hljs-meta">... </span>    tokenized_wnut[<span class="hljs-string">&quot;validation&quot;</span>],
<span class="hljs-meta">... </span>    shuffle=<span class="hljs-literal">False</span>,
<span class="hljs-meta">... </span>    batch_size=<span class="hljs-number">16</span>,
<span class="hljs-meta">... </span>    collate_fn=data_collator,
<span class="hljs-meta">... </span>)`}}),O=new ht({props:{$$slots:{default:[yn]},$$scope:{ctx:S}}}),R=new ss({props:{code:`from transformers import create_optimizer

batch_size = 16
num_train_epochs = 3
num_train_steps = (len(tokenized_wnut["train"]) // batch_size) * num_train_epochs
optimizer, lr_schedule = create_optimizer(
    init_lr=2e-5,
    num_train_steps=num_train_steps,
    weight_decay_rate=0.01,
    num_warmup_steps=0,
)`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> create_optimizer

<span class="hljs-meta">&gt;&gt;&gt; </span>batch_size = <span class="hljs-number">16</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>num_train_epochs = <span class="hljs-number">3</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>num_train_steps = (<span class="hljs-built_in">len</span>(tokenized_wnut[<span class="hljs-string">&quot;train&quot;</span>]) // batch_size) * num_train_epochs
<span class="hljs-meta">&gt;&gt;&gt; </span>optimizer, lr_schedule = create_optimizer(
<span class="hljs-meta">... </span>    init_lr=<span class="hljs-number">2e-5</span>,
<span class="hljs-meta">... </span>    num_train_steps=num_train_steps,
<span class="hljs-meta">... </span>    weight_decay_rate=<span class="hljs-number">0.01</span>,
<span class="hljs-meta">... </span>    num_warmup_steps=<span class="hljs-number">0</span>,
<span class="hljs-meta">... </span>)`}}),N=new ss({props:{code:`from transformers import TFAutoModelForTokenClassification

model = TFAutoModelForTokenClassification.from_pretrained("distilbert-base-uncased", num_labels=2)`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> TFAutoModelForTokenClassification

<span class="hljs-meta">&gt;&gt;&gt; </span>model = TFAutoModelForTokenClassification.from_pretrained(<span class="hljs-string">&quot;distilbert-base-uncased&quot;</span>, num_labels=<span class="hljs-number">2</span>)`}}),H=new ss({props:{code:`import tensorflow as tf

model.compile(optimizer=optimizer)`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">import</span> tensorflow <span class="hljs-keyword">as</span> tf

<span class="hljs-meta">&gt;&gt;&gt; </span>model.<span class="hljs-built_in">compile</span>(optimizer=optimizer)`}}),W=new ss({props:{code:"model.fit(x=tf_train_set, validation_data=tf_validation_set, epochs=3)",highlighted:'<span class="hljs-meta">&gt;&gt;&gt; </span>model.fit(x=tf_train_set, validation_data=tf_validation_set, epochs=<span class="hljs-number">3</span>)'}}),{c(){a=r("p"),m=l("To fine-tune a model in TensorFlow, start by converting your datasets to the "),n=r("code"),u=l("tf.data.Dataset"),k=l(" format with "),$=r("a"),w=l("prepare_tf_dataset()"),D=l("."),A=_(),E(b.$$.fragment),F=_(),E(O.$$.fragment),V=_(),I=r("p"),Z=l("Set up an optimizer function, learning rate schedule, and some training hyperparameters:"),B=_(),E(R.$$.fragment),G=_(),J=r("p"),hs=l("Load DistilBERT with "),L=r("a"),ms=l("TFAutoModelForTokenClassification"),ts=l(" along with the number of expected labels:"),is=_(),E(N.$$.fragment),ps=_(),P=r("p"),Q=l("Configure the model for training with "),M=r("a"),Y=r("code"),ds=l("compile"),as=l(":"),X=_(),E(H.$$.fragment),d=_(),v=r("p"),ns=l("Call "),U=r("a"),K=r("code"),es=l("fit"),gs=l(" to fine-tune the model:"),ls=_(),E(W.$$.fragment),this.h()},l(c){a=i(c,"P",{});var x=p(a);m=o(x,"To fine-tune a model in TensorFlow, start by converting your datasets to the "),n=i(x,"CODE",{});var us=p(n);u=o(us,"tf.data.Dataset"),us.forEach(t),k=o(x," format with "),$=i(x,"A",{href:!0});var Js=p($);w=o(Js,"prepare_tf_dataset()"),Js.forEach(t),D=o(x,"."),x.forEach(t),A=g(c),y(b.$$.fragment,c),F=g(c),y(O.$$.fragment,c),V=g(c),I=i(c,"P",{});var Qs=p(I);Z=o(Qs,"Set up an optimizer function, learning rate schedule, and some training hyperparameters:"),Qs.forEach(t),B=g(c),y(R.$$.fragment,c),G=g(c),J=i(c,"P",{});var Es=p(J);hs=o(Es,"Load DistilBERT with "),L=i(Es,"A",{href:!0});var $s=p(L);ms=o($s,"TFAutoModelForTokenClassification"),$s.forEach(t),ts=o(Es," along with the number of expected labels:"),Es.forEach(t),is=g(c),y(N.$$.fragment,c),ps=g(c),P=i(c,"P",{});var ys=p(P);Q=o(ys,"Configure the model for training with "),M=i(ys,"A",{href:!0,rel:!0});var cs=p(M);Y=i(cs,"CODE",{});var Xs=p(Y);ds=o(Xs,"compile"),Xs.forEach(t),cs.forEach(t),as=o(ys,":"),ys.forEach(t),X=g(c),y(H.$$.fragment,c),d=g(c),v=i(c,"P",{});var js=p(v);ns=o(js,"Call "),U=i(js,"A",{href:!0,rel:!0});var se=p(U);K=i(se,"CODE",{});var ee=p(K);es=o(ee,"fit"),ee.forEach(t),se.forEach(t),gs=o(js," to fine-tune the model:"),js.forEach(t),ls=g(c),y(W.$$.fragment,c),this.h()},h(){j($,"href","/docs/transformers/pr_19907/en/main_classes/model#transformers.TFPreTrainedModel.prepare_tf_dataset"),j(L,"href","/docs/transformers/pr_19907/en/model_doc/auto#transformers.TFAutoModelForTokenClassification"),j(M,"href","https://keras.io/api/models/model_training_apis/#compile-method"),j(M,"rel","nofollow"),j(U,"href","https://keras.io/api/models/model_training_apis/#fit-method"),j(U,"rel","nofollow")},m(c,x){f(c,a,x),e(a,m),e(a,n),e(n,u),e(a,k),e(a,$),e($,w),e(a,D),f(c,A,x),T(b,c,x),f(c,F,x),T(O,c,x),f(c,V,x),f(c,I,x),e(I,Z),f(c,B,x),T(R,c,x),f(c,G,x),f(c,J,x),e(J,hs),e(J,L),e(L,ms),e(J,ts),f(c,is,x),T(N,c,x),f(c,ps,x),f(c,P,x),e(P,Q),e(P,M),e(M,Y),e(Y,ds),e(P,as),f(c,X,x),T(H,c,x),f(c,d,x),f(c,v,x),e(v,ns),e(v,U),e(U,K),e(K,es),e(v,gs),f(c,ls,x),T(W,c,x),os=!0},p(c,x){const us={};x&2&&(us.$$scope={dirty:x,ctx:c}),O.$set(us)},i(c){os||(z(b.$$.fragment,c),z(O.$$.fragment,c),z(R.$$.fragment,c),z(N.$$.fragment,c),z(H.$$.fragment,c),z(W.$$.fragment,c),os=!0)},o(c){q(b.$$.fragment,c),q(O.$$.fragment,c),q(R.$$.fragment,c),q(N.$$.fragment,c),q(H.$$.fragment,c),q(W.$$.fragment,c),os=!1},d(c){c&&t(a),c&&t(A),C(b,c),c&&t(F),C(O,c),c&&t(V),c&&t(I),c&&t(B),C(R,c),c&&t(G),c&&t(J),c&&t(is),C(N,c),c&&t(ps),c&&t(P),c&&t(X),C(H,c),c&&t(d),c&&t(v),c&&t(ls),C(W,c)}}}function zn(S){let a,m;return a=new mt({props:{$$slots:{default:[Tn]},$$scope:{ctx:S}}}),{c(){E(a.$$.fragment)},l(n){y(a.$$.fragment,n)},m(n,u){T(a,n,u),m=!0},p(n,u){const k={};u&2&&(k.$$scope={dirty:u,ctx:n}),a.$set(k)},i(n){m||(z(a.$$.fragment,n),m=!0)},o(n){q(a.$$.fragment,n),m=!1},d(n){C(a,n)}}}function qn(S){let a,m,n,u,k,$,w,D;return{c(){a=r("p"),m=l(`For a more in-depth example of how to fine-tune a model for token classification, take a look at the corresponding
`),n=r("a"),u=l("PyTorch notebook"),k=l(`
or `),$=r("a"),w=l("TensorFlow notebook"),D=l("."),this.h()},l(A){a=i(A,"P",{});var b=p(a);m=o(b,`For a more in-depth example of how to fine-tune a model for token classification, take a look at the corresponding
`),n=i(b,"A",{href:!0,rel:!0});var F=p(n);u=o(F,"PyTorch notebook"),F.forEach(t),k=o(b,`
or `),$=i(b,"A",{href:!0,rel:!0});var O=p($);w=o(O,"TensorFlow notebook"),O.forEach(t),D=o(b,"."),b.forEach(t),this.h()},h(){j(n,"href","https://colab.research.google.com/github/huggingface/notebooks/blob/main/examples/token_classification.ipynb"),j(n,"rel","nofollow"),j($,"href","https://colab.research.google.com/github/huggingface/notebooks/blob/main/examples/token_classification-tf.ipynb"),j($,"rel","nofollow")},m(A,b){f(A,a,b),e(a,m),e(a,n),e(n,u),e(a,k),e(a,$),e($,w),e(a,D)},d(A){A&&t(a)}}}function Cn(S){let a,m,n,u,k,$,w,D,A,b,F,O,V,I,Z,B,R,G,J,hs,L,ms,ts,is,N,ps,P,Q,M,Y,ds,as,X,H,d,v,ns,U,K,es,gs,ls,W,os,c,x,us,Js,Qs,Es,$s,ys,cs,Xs,js,se,ee,fe,dt,ut,Ie,ks,te,he,_t,gt,$t,ws,me,jt,kt,de,wt,bt,ue,xt,vt,Et,ae,_e,yt,Tt,Ne,Ts,qs,ge,Is,zt,$e,qt,Me,Ns,Be,Cs,Ct,je,At,Dt,Re,Ms,Ue,As,Pt,ke,Ft,St,We,Bs,Ye,bs,Ot,we,Lt,It,be,Nt,Mt,He,xs,Rs,Bt,Us,xe,Rt,Ut,Wt,_s,Yt,ve,Ht,Kt,Ee,Vt,Zt,ye,Gt,Jt,Qt,Ws,Xt,Te,sa,ea,Ke,ne,ta,Ve,Ys,Ze,fs,aa,Hs,na,la,ze,oa,ra,qe,ia,pa,Ge,Ks,Je,rs,ca,le,fa,ha,Ce,ma,da,Ae,ua,_a,De,ga,$a,Qe,Ds,Xe,zs,Ps,Pe,Vs,ja,Fe,ka,st,Fs,et,Ss,tt;return $=new ft({}),F=new cn({props:{id:"wVHdVlPScxA"}}),N=new ht({props:{$$slots:{default:[$n]},$$scope:{ctx:S}}}),Y=new ft({}),U=new ss({props:{code:`from datasets import load_dataset

wnut = load_dataset("wnut_17")`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> datasets <span class="hljs-keyword">import</span> load_dataset

<span class="hljs-meta">&gt;&gt;&gt; </span>wnut = load_dataset(<span class="hljs-string">&quot;wnut_17&quot;</span>)`}}),W=new ss({props:{code:'wnut["train"][0]',highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>wnut[<span class="hljs-string">&quot;train&quot;</span>][<span class="hljs-number">0</span>]
{<span class="hljs-string">&#x27;id&#x27;</span>: <span class="hljs-string">&#x27;0&#x27;</span>,
 <span class="hljs-string">&#x27;ner_tags&#x27;</span>: [<span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">7</span>, <span class="hljs-number">8</span>, <span class="hljs-number">8</span>, <span class="hljs-number">0</span>, <span class="hljs-number">7</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>],
 <span class="hljs-string">&#x27;tokens&#x27;</span>: [<span class="hljs-string">&#x27;@paulwalk&#x27;</span>, <span class="hljs-string">&#x27;It&#x27;</span>, <span class="hljs-string">&quot;&#x27;s&quot;</span>, <span class="hljs-string">&#x27;the&#x27;</span>, <span class="hljs-string">&#x27;view&#x27;</span>, <span class="hljs-string">&#x27;from&#x27;</span>, <span class="hljs-string">&#x27;where&#x27;</span>, <span class="hljs-string">&#x27;I&#x27;</span>, <span class="hljs-string">&quot;&#x27;m&quot;</span>, <span class="hljs-string">&#x27;living&#x27;</span>, <span class="hljs-string">&#x27;for&#x27;</span>, <span class="hljs-string">&#x27;two&#x27;</span>, <span class="hljs-string">&#x27;weeks&#x27;</span>, <span class="hljs-string">&#x27;.&#x27;</span>, <span class="hljs-string">&#x27;Empire&#x27;</span>, <span class="hljs-string">&#x27;State&#x27;</span>, <span class="hljs-string">&#x27;Building&#x27;</span>, <span class="hljs-string">&#x27;=&#x27;</span>, <span class="hljs-string">&#x27;ESB&#x27;</span>, <span class="hljs-string">&#x27;.&#x27;</span>, <span class="hljs-string">&#x27;Pretty&#x27;</span>, <span class="hljs-string">&#x27;bad&#x27;</span>, <span class="hljs-string">&#x27;storm&#x27;</span>, <span class="hljs-string">&#x27;here&#x27;</span>, <span class="hljs-string">&#x27;last&#x27;</span>, <span class="hljs-string">&#x27;evening&#x27;</span>, <span class="hljs-string">&#x27;.&#x27;</span>]
}`}}),$s=new ss({props:{code:`label_list = wnut["train"].features[f"ner_tags"].feature.names
label_list`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>label_list = wnut[<span class="hljs-string">&quot;train&quot;</span>].features[<span class="hljs-string">f&quot;ner_tags&quot;</span>].feature.names
<span class="hljs-meta">&gt;&gt;&gt; </span>label_list
[
    <span class="hljs-string">&quot;O&quot;</span>,
    <span class="hljs-string">&quot;B-corporation&quot;</span>,
    <span class="hljs-string">&quot;I-corporation&quot;</span>,
    <span class="hljs-string">&quot;B-creative-work&quot;</span>,
    <span class="hljs-string">&quot;I-creative-work&quot;</span>,
    <span class="hljs-string">&quot;B-group&quot;</span>,
    <span class="hljs-string">&quot;I-group&quot;</span>,
    <span class="hljs-string">&quot;B-location&quot;</span>,
    <span class="hljs-string">&quot;I-location&quot;</span>,
    <span class="hljs-string">&quot;B-person&quot;</span>,
    <span class="hljs-string">&quot;I-person&quot;</span>,
    <span class="hljs-string">&quot;B-product&quot;</span>,
    <span class="hljs-string">&quot;I-product&quot;</span>,
]`}}),Is=new ft({}),Ns=new cn({props:{id:"iY2AZYdZAr0"}}),Ms=new ss({props:{code:`from transformers import AutoTokenizer

tokenizer = AutoTokenizer.from_pretrained("distilbert-base-uncased")`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoTokenizer

<span class="hljs-meta">&gt;&gt;&gt; </span>tokenizer = AutoTokenizer.from_pretrained(<span class="hljs-string">&quot;distilbert-base-uncased&quot;</span>)`}}),Bs=new ss({props:{code:`tokenized_input = tokenizer(example["tokens"], is_split_into_words=True)
tokens = tokenizer.convert_ids_to_tokens(tokenized_input["input_ids"])
tokens`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>tokenized_input = tokenizer(example[<span class="hljs-string">&quot;tokens&quot;</span>], is_split_into_words=<span class="hljs-literal">True</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>tokens = tokenizer.convert_ids_to_tokens(tokenized_input[<span class="hljs-string">&quot;input_ids&quot;</span>])
<span class="hljs-meta">&gt;&gt;&gt; </span>tokens
[<span class="hljs-string">&#x27;[CLS]&#x27;</span>, <span class="hljs-string">&#x27;@&#x27;</span>, <span class="hljs-string">&#x27;paul&#x27;</span>, <span class="hljs-string">&#x27;##walk&#x27;</span>, <span class="hljs-string">&#x27;it&#x27;</span>, <span class="hljs-string">&quot;&#x27;&quot;</span>, <span class="hljs-string">&#x27;s&#x27;</span>, <span class="hljs-string">&#x27;the&#x27;</span>, <span class="hljs-string">&#x27;view&#x27;</span>, <span class="hljs-string">&#x27;from&#x27;</span>, <span class="hljs-string">&#x27;where&#x27;</span>, <span class="hljs-string">&#x27;i&#x27;</span>, <span class="hljs-string">&quot;&#x27;&quot;</span>, <span class="hljs-string">&#x27;m&#x27;</span>, <span class="hljs-string">&#x27;living&#x27;</span>, <span class="hljs-string">&#x27;for&#x27;</span>, <span class="hljs-string">&#x27;two&#x27;</span>, <span class="hljs-string">&#x27;weeks&#x27;</span>, <span class="hljs-string">&#x27;.&#x27;</span>, <span class="hljs-string">&#x27;empire&#x27;</span>, <span class="hljs-string">&#x27;state&#x27;</span>, <span class="hljs-string">&#x27;building&#x27;</span>, <span class="hljs-string">&#x27;=&#x27;</span>, <span class="hljs-string">&#x27;es&#x27;</span>, <span class="hljs-string">&#x27;##b&#x27;</span>, <span class="hljs-string">&#x27;.&#x27;</span>, <span class="hljs-string">&#x27;pretty&#x27;</span>, <span class="hljs-string">&#x27;bad&#x27;</span>, <span class="hljs-string">&#x27;storm&#x27;</span>, <span class="hljs-string">&#x27;here&#x27;</span>, <span class="hljs-string">&#x27;last&#x27;</span>, <span class="hljs-string">&#x27;evening&#x27;</span>, <span class="hljs-string">&#x27;.&#x27;</span>, <span class="hljs-string">&#x27;[SEP]&#x27;</span>]`}}),Ys=new ss({props:{code:`def tokenize_and_align_labels(examples):
    tokenized_inputs = tokenizer(examples["tokens"], truncation=True, is_split_into_words=True)

    labels = []
    for i, label in enumerate(examples[f"ner_tags"]):
        word_ids = tokenized_inputs.word_ids(batch_index=i)  # Map tokens to their respective word.
        previous_word_idx = None
        label_ids = []
        for word_idx in word_ids:  # Set the special tokens to -100.
            if word_idx is None:
                label_ids.append(-100)
            elif word_idx != previous_word_idx:  # Only label the first token of a given word.
                label_ids.append(label[word_idx])
            else:
                label_ids.append(-100)
            previous_word_idx = word_idx
        labels.append(label_ids)

    tokenized_inputs["labels"] = labels
    return tokenized_inputs`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">def</span> <span class="hljs-title function_">tokenize_and_align_labels</span>(<span class="hljs-params">examples</span>):
<span class="hljs-meta">... </span>    tokenized_inputs = tokenizer(examples[<span class="hljs-string">&quot;tokens&quot;</span>], truncation=<span class="hljs-literal">True</span>, is_split_into_words=<span class="hljs-literal">True</span>)

<span class="hljs-meta">... </span>    labels = []
<span class="hljs-meta">... </span>    <span class="hljs-keyword">for</span> i, label <span class="hljs-keyword">in</span> <span class="hljs-built_in">enumerate</span>(examples[<span class="hljs-string">f&quot;ner_tags&quot;</span>]):
<span class="hljs-meta">... </span>        word_ids = tokenized_inputs.word_ids(batch_index=i)  <span class="hljs-comment"># Map tokens to their respective word.</span>
<span class="hljs-meta">... </span>        previous_word_idx = <span class="hljs-literal">None</span>
<span class="hljs-meta">... </span>        label_ids = []
<span class="hljs-meta">... </span>        <span class="hljs-keyword">for</span> word_idx <span class="hljs-keyword">in</span> word_ids:  <span class="hljs-comment"># Set the special tokens to -100.</span>
<span class="hljs-meta">... </span>            <span class="hljs-keyword">if</span> word_idx <span class="hljs-keyword">is</span> <span class="hljs-literal">None</span>:
<span class="hljs-meta">... </span>                label_ids.append(-<span class="hljs-number">100</span>)
<span class="hljs-meta">... </span>            <span class="hljs-keyword">elif</span> word_idx != previous_word_idx:  <span class="hljs-comment"># Only label the first token of a given word.</span>
<span class="hljs-meta">... </span>                label_ids.append(label[word_idx])
<span class="hljs-meta">... </span>            <span class="hljs-keyword">else</span>:
<span class="hljs-meta">... </span>                label_ids.append(-<span class="hljs-number">100</span>)
<span class="hljs-meta">... </span>            previous_word_idx = word_idx
<span class="hljs-meta">... </span>        labels.append(label_ids)

<span class="hljs-meta">... </span>    tokenized_inputs[<span class="hljs-string">&quot;labels&quot;</span>] = labels
<span class="hljs-meta">... </span>    <span class="hljs-keyword">return</span> tokenized_inputs`}}),Ks=new ss({props:{code:"tokenized_wnut = wnut.map(tokenize_and_align_labels, batched=True)",highlighted:'<span class="hljs-meta">&gt;&gt;&gt; </span>tokenized_wnut = wnut.<span class="hljs-built_in">map</span>(tokenize_and_align_labels, batched=<span class="hljs-literal">True</span>)'}}),Ds=new fn({props:{pytorch:!0,tensorflow:!0,jax:!1,$$slots:{tensorflow:[bn],pytorch:[kn]},$$scope:{ctx:S}}}),Vs=new ft({}),Fs=new fn({props:{pytorch:!0,tensorflow:!0,jax:!1,$$slots:{tensorflow:[zn],pytorch:[En]},$$scope:{ctx:S}}}),Ss=new ht({props:{$$slots:{default:[qn]},$$scope:{ctx:S}}}),{c(){a=r("meta"),m=_(),n=r("h1"),u=r("a"),k=r("span"),E($.$$.fragment),w=_(),D=r("span"),A=l("Token classification"),b=_(),E(F.$$.fragment),O=_(),V=r("p"),I=l("Token classification assigns a label to individual tokens in a sentence. One of the most common token classification tasks is Named Entity Recognition (NER). NER attempts to find a label for each entity in a sentence, such as a person, location, or organization."),Z=_(),B=r("p"),R=l("This guide will show you how to fine-tune "),G=r("a"),J=l("DistilBERT"),hs=l(" on the "),L=r("a"),ms=l("WNUT 17"),ts=l(" dataset to detect new entities."),is=_(),E(N.$$.fragment),ps=_(),P=r("h2"),Q=r("a"),M=r("span"),E(Y.$$.fragment),ds=_(),as=r("span"),X=l("Load WNUT 17 dataset"),H=_(),d=r("p"),v=l("Load the WNUT 17 dataset from the \u{1F917} Datasets library:"),ns=_(),E(U.$$.fragment),K=_(),es=r("p"),gs=l("Then take a look at an example:"),ls=_(),E(W.$$.fragment),os=_(),c=r("p"),x=l("Each number in "),us=r("code"),Js=l("ner_tags"),Qs=l(" represents an entity. Convert the number to a label name for more information:"),Es=_(),E($s.$$.fragment),ys=_(),cs=r("p"),Xs=l("The "),js=r("code"),se=l("ner_tag"),ee=l(" describes an entity, such as a corporation, location, or person. The letter that prefixes each "),fe=r("code"),dt=l("ner_tag"),ut=l(" indicates the token position of the entity:"),Ie=_(),ks=r("ul"),te=r("li"),he=r("code"),_t=l("B-"),gt=l(" indicates the beginning of an entity."),$t=_(),ws=r("li"),me=r("code"),jt=l("I-"),kt=l(" indicates a token is contained inside the same entity (e.g., the "),de=r("code"),wt=l("State"),bt=l(` token is a part of an entity like
`),ue=r("code"),xt=l("Empire State Building"),vt=l(")."),Et=_(),ae=r("li"),_e=r("code"),yt=l("0"),Tt=l(" indicates the token doesn\u2019t correspond to any entity."),Ne=_(),Ts=r("h2"),qs=r("a"),ge=r("span"),E(Is.$$.fragment),zt=_(),$e=r("span"),qt=l("Preprocess"),Me=_(),E(Ns.$$.fragment),Be=_(),Cs=r("p"),Ct=l("Load the DistilBERT tokenizer to process the "),je=r("code"),At=l("tokens"),Dt=l(":"),Re=_(),E(Ms.$$.fragment),Ue=_(),As=r("p"),Pt=l("Since the input has already been split into words, set "),ke=r("code"),Ft=l("is_split_into_words=True"),St=l(" to tokenize the words into subwords:"),We=_(),E(Bs.$$.fragment),Ye=_(),bs=r("p"),Ot=l("Adding the special tokens "),we=r("code"),Lt=l("[CLS]"),It=l(" and "),be=r("code"),Nt=l("[SEP]"),Mt=l(" and subword tokenization creates a mismatch between the input and labels. A single word corresponding to a single label may be split into two subwords. You will need to realign the tokens and labels by:"),He=_(),xs=r("ol"),Rs=r("li"),Bt=l("Mapping all tokens to their corresponding word with the "),Us=r("a"),xe=r("code"),Rt=l("word_ids"),Ut=l(" method."),Wt=_(),_s=r("li"),Yt=l("Assigning the label "),ve=r("code"),Ht=l("-100"),Kt=l(" to the special tokens "),Ee=r("code"),Vt=l("[CLS]"),Zt=l(" and "),ye=r("code"),Gt=l("[SEP]"),Jt=l(` so the PyTorch loss function ignores
them.`),Qt=_(),Ws=r("li"),Xt=l("Only labeling the first token of a given word. Assign "),Te=r("code"),sa=l("-100"),ea=l(" to other subtokens from the same word."),Ke=_(),ne=r("p"),ta=l("Here is how you can create a function to realign the tokens and labels, and truncate sequences to be no longer than DistilBERT\u2019s maximum input length::"),Ve=_(),E(Ys.$$.fragment),Ze=_(),fs=r("p"),aa=l("Use \u{1F917} Datasets "),Hs=r("a"),na=l("map"),la=l(" function to tokenize and align the labels over the entire dataset. You can speed up the "),ze=r("code"),oa=l("map"),ra=l(" function by setting "),qe=r("code"),ia=l("batched=True"),pa=l(" to process multiple elements of the dataset at once:"),Ge=_(),E(Ks.$$.fragment),Je=_(),rs=r("p"),ca=l("Use "),le=r("a"),fa=l("DataCollatorForTokenClassification"),ha=l(" to create a batch of examples. It will also "),Ce=r("em"),ma=l("dynamically pad"),da=l(" your text and labels to the length of the longest element in its batch, so they are a uniform length. While it is possible to pad your text in the "),Ae=r("code"),ua=l("tokenizer"),_a=l(" function by setting "),De=r("code"),ga=l("padding=True"),$a=l(", dynamic padding is more efficient."),Qe=_(),E(Ds.$$.fragment),Xe=_(),zs=r("h2"),Ps=r("a"),Pe=r("span"),E(Vs.$$.fragment),ja=_(),Fe=r("span"),ka=l("Train"),st=_(),E(Fs.$$.fragment),et=_(),E(Ss.$$.fragment),this.h()},l(s){const h=_n('[data-svelte="svelte-1phssyn"]',document.head);a=i(h,"META",{name:!0,content:!0}),h.forEach(t),m=g(s),n=i(s,"H1",{class:!0});var Zs=p(n);u=i(Zs,"A",{id:!0,class:!0,href:!0});var Se=p(u);k=i(Se,"SPAN",{});var Oe=p(k);y($.$$.fragment,Oe),Oe.forEach(t),Se.forEach(t),w=g(Zs),D=i(Zs,"SPAN",{});var Le=p(D);A=o(Le,"Token classification"),Le.forEach(t),Zs.forEach(t),b=g(s),y(F.$$.fragment,s),O=g(s),V=i(s,"P",{});var xa=p(V);I=o(xa,"Token classification assigns a label to individual tokens in a sentence. One of the most common token classification tasks is Named Entity Recognition (NER). NER attempts to find a label for each entity in a sentence, such as a person, location, or organization."),xa.forEach(t),Z=g(s),B=i(s,"P",{});var oe=p(B);R=o(oe,"This guide will show you how to fine-tune "),G=i(oe,"A",{href:!0,rel:!0});var va=p(G);J=o(va,"DistilBERT"),va.forEach(t),hs=o(oe," on the "),L=i(oe,"A",{href:!0,rel:!0});var Ea=p(L);ms=o(Ea,"WNUT 17"),Ea.forEach(t),ts=o(oe," dataset to detect new entities."),oe.forEach(t),is=g(s),y(N.$$.fragment,s),ps=g(s),P=i(s,"H2",{class:!0});var at=p(P);Q=i(at,"A",{id:!0,class:!0,href:!0});var ya=p(Q);M=i(ya,"SPAN",{});var Ta=p(M);y(Y.$$.fragment,Ta),Ta.forEach(t),ya.forEach(t),ds=g(at),as=i(at,"SPAN",{});var za=p(as);X=o(za,"Load WNUT 17 dataset"),za.forEach(t),at.forEach(t),H=g(s),d=i(s,"P",{});var qa=p(d);v=o(qa,"Load the WNUT 17 dataset from the \u{1F917} Datasets library:"),qa.forEach(t),ns=g(s),y(U.$$.fragment,s),K=g(s),es=i(s,"P",{});var Ca=p(es);gs=o(Ca,"Then take a look at an example:"),Ca.forEach(t),ls=g(s),y(W.$$.fragment,s),os=g(s),c=i(s,"P",{});var nt=p(c);x=o(nt,"Each number in "),us=i(nt,"CODE",{});var Aa=p(us);Js=o(Aa,"ner_tags"),Aa.forEach(t),Qs=o(nt," represents an entity. Convert the number to a label name for more information:"),nt.forEach(t),Es=g(s),y($s.$$.fragment,s),ys=g(s),cs=i(s,"P",{});var re=p(cs);Xs=o(re,"The "),js=i(re,"CODE",{});var Da=p(js);se=o(Da,"ner_tag"),Da.forEach(t),ee=o(re," describes an entity, such as a corporation, location, or person. The letter that prefixes each "),fe=i(re,"CODE",{});var Pa=p(fe);dt=o(Pa,"ner_tag"),Pa.forEach(t),ut=o(re," indicates the token position of the entity:"),re.forEach(t),Ie=g(s),ks=i(s,"UL",{});var ie=p(ks);te=i(ie,"LI",{});var wa=p(te);he=i(wa,"CODE",{});var Fa=p(he);_t=o(Fa,"B-"),Fa.forEach(t),gt=o(wa," indicates the beginning of an entity."),wa.forEach(t),$t=g(ie),ws=i(ie,"LI",{});var Gs=p(ws);me=i(Gs,"CODE",{});var Sa=p(me);jt=o(Sa,"I-"),Sa.forEach(t),kt=o(Gs," indicates a token is contained inside the same entity (e.g., the "),de=i(Gs,"CODE",{});var Oa=p(de);wt=o(Oa,"State"),Oa.forEach(t),bt=o(Gs,` token is a part of an entity like
`),ue=i(Gs,"CODE",{});var La=p(ue);xt=o(La,"Empire State Building"),La.forEach(t),vt=o(Gs,")."),Gs.forEach(t),Et=g(ie),ae=i(ie,"LI",{});var ba=p(ae);_e=i(ba,"CODE",{});var Ia=p(_e);yt=o(Ia,"0"),Ia.forEach(t),Tt=o(ba," indicates the token doesn\u2019t correspond to any entity."),ba.forEach(t),ie.forEach(t),Ne=g(s),Ts=i(s,"H2",{class:!0});var lt=p(Ts);qs=i(lt,"A",{id:!0,class:!0,href:!0});var Na=p(qs);ge=i(Na,"SPAN",{});var Ma=p(ge);y(Is.$$.fragment,Ma),Ma.forEach(t),Na.forEach(t),zt=g(lt),$e=i(lt,"SPAN",{});var Ba=p($e);qt=o(Ba,"Preprocess"),Ba.forEach(t),lt.forEach(t),Me=g(s),y(Ns.$$.fragment,s),Be=g(s),Cs=i(s,"P",{});var ot=p(Cs);Ct=o(ot,"Load the DistilBERT tokenizer to process the "),je=i(ot,"CODE",{});var Ra=p(je);At=o(Ra,"tokens"),Ra.forEach(t),Dt=o(ot,":"),ot.forEach(t),Re=g(s),y(Ms.$$.fragment,s),Ue=g(s),As=i(s,"P",{});var rt=p(As);Pt=o(rt,"Since the input has already been split into words, set "),ke=i(rt,"CODE",{});var Ua=p(ke);Ft=o(Ua,"is_split_into_words=True"),Ua.forEach(t),St=o(rt," to tokenize the words into subwords:"),rt.forEach(t),We=g(s),y(Bs.$$.fragment,s),Ye=g(s),bs=i(s,"P",{});var pe=p(bs);Ot=o(pe,"Adding the special tokens "),we=i(pe,"CODE",{});var Wa=p(we);Lt=o(Wa,"[CLS]"),Wa.forEach(t),It=o(pe," and "),be=i(pe,"CODE",{});var Ya=p(be);Nt=o(Ya,"[SEP]"),Ya.forEach(t),Mt=o(pe," and subword tokenization creates a mismatch between the input and labels. A single word corresponding to a single label may be split into two subwords. You will need to realign the tokens and labels by:"),pe.forEach(t),He=g(s),xs=i(s,"OL",{});var ce=p(xs);Rs=i(ce,"LI",{});var it=p(Rs);Bt=o(it,"Mapping all tokens to their corresponding word with the "),Us=i(it,"A",{href:!0,rel:!0});var Ha=p(Us);xe=i(Ha,"CODE",{});var Ka=p(xe);Rt=o(Ka,"word_ids"),Ka.forEach(t),Ha.forEach(t),Ut=o(it," method."),it.forEach(t),Wt=g(ce),_s=i(ce,"LI",{});var Os=p(_s);Yt=o(Os,"Assigning the label "),ve=i(Os,"CODE",{});var Va=p(ve);Ht=o(Va,"-100"),Va.forEach(t),Kt=o(Os," to the special tokens "),Ee=i(Os,"CODE",{});var Za=p(Ee);Vt=o(Za,"[CLS]"),Za.forEach(t),Zt=o(Os," and "),ye=i(Os,"CODE",{});var Ga=p(ye);Gt=o(Ga,"[SEP]"),Ga.forEach(t),Jt=o(Os,` so the PyTorch loss function ignores
them.`),Os.forEach(t),Qt=g(ce),Ws=i(ce,"LI",{});var pt=p(Ws);Xt=o(pt,"Only labeling the first token of a given word. Assign "),Te=i(pt,"CODE",{});var Ja=p(Te);sa=o(Ja,"-100"),Ja.forEach(t),ea=o(pt," to other subtokens from the same word."),pt.forEach(t),ce.forEach(t),Ke=g(s),ne=i(s,"P",{});var Qa=p(ne);ta=o(Qa,"Here is how you can create a function to realign the tokens and labels, and truncate sequences to be no longer than DistilBERT\u2019s maximum input length::"),Qa.forEach(t),Ve=g(s),y(Ys.$$.fragment,s),Ze=g(s),fs=i(s,"P",{});var Ls=p(fs);aa=o(Ls,"Use \u{1F917} Datasets "),Hs=i(Ls,"A",{href:!0,rel:!0});var Xa=p(Hs);na=o(Xa,"map"),Xa.forEach(t),la=o(Ls," function to tokenize and align the labels over the entire dataset. You can speed up the "),ze=i(Ls,"CODE",{});var sn=p(ze);oa=o(sn,"map"),sn.forEach(t),ra=o(Ls," function by setting "),qe=i(Ls,"CODE",{});var en=p(qe);ia=o(en,"batched=True"),en.forEach(t),pa=o(Ls," to process multiple elements of the dataset at once:"),Ls.forEach(t),Ge=g(s),y(Ks.$$.fragment,s),Je=g(s),rs=i(s,"P",{});var vs=p(rs);ca=o(vs,"Use "),le=i(vs,"A",{href:!0});var tn=p(le);fa=o(tn,"DataCollatorForTokenClassification"),tn.forEach(t),ha=o(vs," to create a batch of examples. It will also "),Ce=i(vs,"EM",{});var an=p(Ce);ma=o(an,"dynamically pad"),an.forEach(t),da=o(vs," your text and labels to the length of the longest element in its batch, so they are a uniform length. While it is possible to pad your text in the "),Ae=i(vs,"CODE",{});var nn=p(Ae);ua=o(nn,"tokenizer"),nn.forEach(t),_a=o(vs," function by setting "),De=i(vs,"CODE",{});var ln=p(De);ga=o(ln,"padding=True"),ln.forEach(t),$a=o(vs,", dynamic padding is more efficient."),vs.forEach(t),Qe=g(s),y(Ds.$$.fragment,s),Xe=g(s),zs=i(s,"H2",{class:!0});var ct=p(zs);Ps=i(ct,"A",{id:!0,class:!0,href:!0});var on=p(Ps);Pe=i(on,"SPAN",{});var rn=p(Pe);y(Vs.$$.fragment,rn),rn.forEach(t),on.forEach(t),ja=g(ct),Fe=i(ct,"SPAN",{});var pn=p(Fe);ka=o(pn,"Train"),pn.forEach(t),ct.forEach(t),st=g(s),y(Fs.$$.fragment,s),et=g(s),y(Ss.$$.fragment,s),this.h()},h(){j(a,"name","hf:doc:metadata"),j(a,"content",JSON.stringify(An)),j(u,"id","token-classification"),j(u,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),j(u,"href","#token-classification"),j(n,"class","relative group"),j(G,"href","https://huggingface.co/distilbert-base-uncased"),j(G,"rel","nofollow"),j(L,"href","https://huggingface.co/datasets/wnut_17"),j(L,"rel","nofollow"),j(Q,"id","load-wnut-17-dataset"),j(Q,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),j(Q,"href","#load-wnut-17-dataset"),j(P,"class","relative group"),j(qs,"id","preprocess"),j(qs,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),j(qs,"href","#preprocess"),j(Ts,"class","relative group"),j(Us,"href","https://huggingface.co/docs/tokenizers/python/latest/api/reference.html#tokenizers.Encoding.word_ids"),j(Us,"rel","nofollow"),j(Hs,"href","https://huggingface.co/docs/datasets/main/en/package_reference/main_classes#datasets.Dataset.map"),j(Hs,"rel","nofollow"),j(le,"href","/docs/transformers/pr_19907/en/main_classes/data_collator#transformers.DataCollatorForTokenClassification"),j(Ps,"id","train"),j(Ps,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),j(Ps,"href","#train"),j(zs,"class","relative group")},m(s,h){e(document.head,a),f(s,m,h),f(s,n,h),e(n,u),e(u,k),T($,k,null),e(n,w),e(n,D),e(D,A),f(s,b,h),T(F,s,h),f(s,O,h),f(s,V,h),e(V,I),f(s,Z,h),f(s,B,h),e(B,R),e(B,G),e(G,J),e(B,hs),e(B,L),e(L,ms),e(B,ts),f(s,is,h),T(N,s,h),f(s,ps,h),f(s,P,h),e(P,Q),e(Q,M),T(Y,M,null),e(P,ds),e(P,as),e(as,X),f(s,H,h),f(s,d,h),e(d,v),f(s,ns,h),T(U,s,h),f(s,K,h),f(s,es,h),e(es,gs),f(s,ls,h),T(W,s,h),f(s,os,h),f(s,c,h),e(c,x),e(c,us),e(us,Js),e(c,Qs),f(s,Es,h),T($s,s,h),f(s,ys,h),f(s,cs,h),e(cs,Xs),e(cs,js),e(js,se),e(cs,ee),e(cs,fe),e(fe,dt),e(cs,ut),f(s,Ie,h),f(s,ks,h),e(ks,te),e(te,he),e(he,_t),e(te,gt),e(ks,$t),e(ks,ws),e(ws,me),e(me,jt),e(ws,kt),e(ws,de),e(de,wt),e(ws,bt),e(ws,ue),e(ue,xt),e(ws,vt),e(ks,Et),e(ks,ae),e(ae,_e),e(_e,yt),e(ae,Tt),f(s,Ne,h),f(s,Ts,h),e(Ts,qs),e(qs,ge),T(Is,ge,null),e(Ts,zt),e(Ts,$e),e($e,qt),f(s,Me,h),T(Ns,s,h),f(s,Be,h),f(s,Cs,h),e(Cs,Ct),e(Cs,je),e(je,At),e(Cs,Dt),f(s,Re,h),T(Ms,s,h),f(s,Ue,h),f(s,As,h),e(As,Pt),e(As,ke),e(ke,Ft),e(As,St),f(s,We,h),T(Bs,s,h),f(s,Ye,h),f(s,bs,h),e(bs,Ot),e(bs,we),e(we,Lt),e(bs,It),e(bs,be),e(be,Nt),e(bs,Mt),f(s,He,h),f(s,xs,h),e(xs,Rs),e(Rs,Bt),e(Rs,Us),e(Us,xe),e(xe,Rt),e(Rs,Ut),e(xs,Wt),e(xs,_s),e(_s,Yt),e(_s,ve),e(ve,Ht),e(_s,Kt),e(_s,Ee),e(Ee,Vt),e(_s,Zt),e(_s,ye),e(ye,Gt),e(_s,Jt),e(xs,Qt),e(xs,Ws),e(Ws,Xt),e(Ws,Te),e(Te,sa),e(Ws,ea),f(s,Ke,h),f(s,ne,h),e(ne,ta),f(s,Ve,h),T(Ys,s,h),f(s,Ze,h),f(s,fs,h),e(fs,aa),e(fs,Hs),e(Hs,na),e(fs,la),e(fs,ze),e(ze,oa),e(fs,ra),e(fs,qe),e(qe,ia),e(fs,pa),f(s,Ge,h),T(Ks,s,h),f(s,Je,h),f(s,rs,h),e(rs,ca),e(rs,le),e(le,fa),e(rs,ha),e(rs,Ce),e(Ce,ma),e(rs,da),e(rs,Ae),e(Ae,ua),e(rs,_a),e(rs,De),e(De,ga),e(rs,$a),f(s,Qe,h),T(Ds,s,h),f(s,Xe,h),f(s,zs,h),e(zs,Ps),e(Ps,Pe),T(Vs,Pe,null),e(zs,ja),e(zs,Fe),e(Fe,ka),f(s,st,h),T(Fs,s,h),f(s,et,h),T(Ss,s,h),tt=!0},p(s,[h]){const Zs={};h&2&&(Zs.$$scope={dirty:h,ctx:s}),N.$set(Zs);const Se={};h&2&&(Se.$$scope={dirty:h,ctx:s}),Ds.$set(Se);const Oe={};h&2&&(Oe.$$scope={dirty:h,ctx:s}),Fs.$set(Oe);const Le={};h&2&&(Le.$$scope={dirty:h,ctx:s}),Ss.$set(Le)},i(s){tt||(z($.$$.fragment,s),z(F.$$.fragment,s),z(N.$$.fragment,s),z(Y.$$.fragment,s),z(U.$$.fragment,s),z(W.$$.fragment,s),z($s.$$.fragment,s),z(Is.$$.fragment,s),z(Ns.$$.fragment,s),z(Ms.$$.fragment,s),z(Bs.$$.fragment,s),z(Ys.$$.fragment,s),z(Ks.$$.fragment,s),z(Ds.$$.fragment,s),z(Vs.$$.fragment,s),z(Fs.$$.fragment,s),z(Ss.$$.fragment,s),tt=!0)},o(s){q($.$$.fragment,s),q(F.$$.fragment,s),q(N.$$.fragment,s),q(Y.$$.fragment,s),q(U.$$.fragment,s),q(W.$$.fragment,s),q($s.$$.fragment,s),q(Is.$$.fragment,s),q(Ns.$$.fragment,s),q(Ms.$$.fragment,s),q(Bs.$$.fragment,s),q(Ys.$$.fragment,s),q(Ks.$$.fragment,s),q(Ds.$$.fragment,s),q(Vs.$$.fragment,s),q(Fs.$$.fragment,s),q(Ss.$$.fragment,s),tt=!1},d(s){t(a),s&&t(m),s&&t(n),C($),s&&t(b),C(F,s),s&&t(O),s&&t(V),s&&t(Z),s&&t(B),s&&t(is),C(N,s),s&&t(ps),s&&t(P),C(Y),s&&t(H),s&&t(d),s&&t(ns),C(U,s),s&&t(K),s&&t(es),s&&t(ls),C(W,s),s&&t(os),s&&t(c),s&&t(Es),C($s,s),s&&t(ys),s&&t(cs),s&&t(Ie),s&&t(ks),s&&t(Ne),s&&t(Ts),C(Is),s&&t(Me),C(Ns,s),s&&t(Be),s&&t(Cs),s&&t(Re),C(Ms,s),s&&t(Ue),s&&t(As),s&&t(We),C(Bs,s),s&&t(Ye),s&&t(bs),s&&t(He),s&&t(xs),s&&t(Ke),s&&t(ne),s&&t(Ve),C(Ys,s),s&&t(Ze),s&&t(fs),s&&t(Ge),C(Ks,s),s&&t(Je),s&&t(rs),s&&t(Qe),C(Ds,s),s&&t(Xe),s&&t(zs),C(Vs),s&&t(st),C(Fs,s),s&&t(et),C(Ss,s)}}}const An={local:"token-classification",sections:[{local:"load-wnut-17-dataset",title:"Load WNUT 17 dataset"},{local:"preprocess",title:"Preprocess"},{local:"train",title:"Train"}],title:"Token classification"};function Dn(S){return gn(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class Nn extends mn{constructor(a){super();dn(this,a,Dn,Cn,un,{})}}export{Nn as default,An as metadata};
