import{S as ho,i as _o,s as go,e as r,k as _,w as x,t as n,M as $o,c as i,d as e,m as g,a as p,x as E,h as l,b as k,G as a,g as d,y as q,q as z,o as C,B as T,v as jo,L as fo}from"../../chunks/vendor-hf-doc-builder.js";import{T as he}from"../../chunks/Tip-hf-doc-builder.js";import{Y as mo}from"../../chunks/Youtube-hf-doc-builder.js";import{I as fe}from"../../chunks/IconCopyLink-hf-doc-builder.js";import{C as as}from"../../chunks/CodeBlock-hf-doc-builder.js";import{F as uo,M as _e}from"../../chunks/Markdown-hf-doc-builder.js";function ko(P){let t,u,o,f,j;return{c(){t=r("p"),u=n("Consulte a "),o=r("a"),f=n("p\xE1gina de tarefas de classifica\xE7\xE3o de tokens"),j=n(" para obter mais informa\xE7\xF5es sobre outras formas de classifica\xE7\xE3o de tokens e seus modelos, conjuntos de dados e m\xE9tricas associadas."),this.h()},l($){t=i($,"P",{});var v=p(t);u=l(v,"Consulte a "),o=i(v,"A",{href:!0,rel:!0});var D=p(o);f=l(D,"p\xE1gina de tarefas de classifica\xE7\xE3o de tokens"),D.forEach(e),j=l(v," para obter mais informa\xE7\xF5es sobre outras formas de classifica\xE7\xE3o de tokens e seus modelos, conjuntos de dados e m\xE9tricas associadas."),v.forEach(e),this.h()},h(){k(o,"href","https://huggingface.co/tasks/token-classification"),k(o,"rel","nofollow")},m($,v){d($,t,v),a(t,u),a(t,o),a(o,f),a(t,j)},d($){$&&e(t)}}}function vo(P){let t,u;return t=new as({props:{code:`from transformers import DataCollatorForTokenClassification

data_collator = DataCollatorForTokenClassification(tokenizer=tokenizer)`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> DataCollatorForTokenClassification

<span class="hljs-meta">&gt;&gt;&gt; </span>data_collator = DataCollatorForTokenClassification(tokenizer=tokenizer)`}}),{c(){x(t.$$.fragment)},l(o){E(t.$$.fragment,o)},m(o,f){q(t,o,f),u=!0},p:fo,i(o){u||(z(t.$$.fragment,o),u=!0)},o(o){C(t.$$.fragment,o),u=!1},d(o){T(t,o)}}}function bo(P){let t,u;return t=new _e({props:{$$slots:{default:[vo]},$$scope:{ctx:P}}}),{c(){x(t.$$.fragment)},l(o){E(t.$$.fragment,o)},m(o,f){q(t,o,f),u=!0},p(o,f){const j={};f&2&&(j.$$scope={dirty:f,ctx:o}),t.$set(j)},i(o){u||(z(t.$$.fragment,o),u=!0)},o(o){C(t.$$.fragment,o),u=!1},d(o){T(t,o)}}}function wo(P){let t,u;return t=new as({props:{code:`from transformers import DataCollatorForTokenClassification

data_collator = DataCollatorForTokenClassification(tokenizer=tokenizer, return_tensors="tf")`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> DataCollatorForTokenClassification

<span class="hljs-meta">&gt;&gt;&gt; </span>data_collator = DataCollatorForTokenClassification(tokenizer=tokenizer, return_tensors=<span class="hljs-string">&quot;tf&quot;</span>)`}}),{c(){x(t.$$.fragment)},l(o){E(t.$$.fragment,o)},m(o,f){q(t,o,f),u=!0},p:fo,i(o){u||(z(t.$$.fragment,o),u=!0)},o(o){C(t.$$.fragment,o),u=!1},d(o){T(t,o)}}}function xo(P){let t,u;return t=new _e({props:{$$slots:{default:[wo]},$$scope:{ctx:P}}}),{c(){x(t.$$.fragment)},l(o){E(t.$$.fragment,o)},m(o,f){q(t,o,f),u=!0},p(o,f){const j={};f&2&&(j.$$scope={dirty:f,ctx:o}),t.$set(j)},i(o){u||(z(t.$$.fragment,o),u=!0)},o(o){C(t.$$.fragment,o),u=!1},d(o){T(t,o)}}}function Eo(P){let t,u,o,f,j,$,v,D;return{c(){t=r("p"),u=n("Se voc\xEA n\xE3o estiver familiarizado com o fine-tuning de um modelo com o "),o=r("code"),f=n("Trainer"),j=n(", d\xEA uma olhada no tutorial b\xE1sico "),$=r("a"),v=n("aqui"),D=n("!"),this.h()},l(A){t=i(A,"P",{});var w=p(t);u=l(w,"Se voc\xEA n\xE3o estiver familiarizado com o fine-tuning de um modelo com o "),o=i(w,"CODE",{});var O=p(o);f=l(O,"Trainer"),O.forEach(e),j=l(w,", d\xEA uma olhada no tutorial b\xE1sico "),$=i(w,"A",{href:!0});var L=p($);v=l(L,"aqui"),L.forEach(e),D=l(w,"!"),w.forEach(e),this.h()},h(){k($,"href","../training#finetune-with-trainer")},m(A,w){d(A,t,w),a(t,u),a(t,o),a(o,f),a(t,j),a(t,$),a($,v),a(t,D)},d(A){A&&e(t)}}}function qo(P){let t,u,o,f,j,$,v,D,A,w,O,L,G,N,Y,F,ns,U,$s,us,S,fs,W,hs,V,_s,B,K,I,J,Q,ls,R,es;return v=new as({props:{code:`from transformers import AutoModelForTokenClassification, TrainingArguments, Trainer

model = AutoModelForTokenClassification.from_pretrained("distilbert-base-uncased", num_labels=14)`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoModelForTokenClassification, TrainingArguments, Trainer

<span class="hljs-meta">&gt;&gt;&gt; </span>model = AutoModelForTokenClassification.from_pretrained(<span class="hljs-string">&quot;distilbert-base-uncased&quot;</span>, num_labels=<span class="hljs-number">14</span>)`}}),A=new he({props:{$$slots:{default:[Eo]},$$scope:{ctx:P}}}),R=new as({props:{code:`training_args = TrainingArguments(
    output_dir="./results",
    evaluation_strategy="epoch",
    learning_rate=2e-5,
    per_device_train_batch_size=16,
    per_device_eval_batch_size=16,
    num_train_epochs=3,
    weight_decay=0.01,
)

trainer = Trainer(
    model=model,
    args=training_args,
    train_dataset=tokenized_wnut["train"],
    eval_dataset=tokenized_wnut["test"],
    tokenizer=tokenizer,
    data_collator=data_collator,
)

trainer.train()`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>training_args = TrainingArguments(
<span class="hljs-meta">... </span>    output_dir=<span class="hljs-string">&quot;./results&quot;</span>,
<span class="hljs-meta">... </span>    evaluation_strategy=<span class="hljs-string">&quot;epoch&quot;</span>,
<span class="hljs-meta">... </span>    learning_rate=<span class="hljs-number">2e-5</span>,
<span class="hljs-meta">... </span>    per_device_train_batch_size=<span class="hljs-number">16</span>,
<span class="hljs-meta">... </span>    per_device_eval_batch_size=<span class="hljs-number">16</span>,
<span class="hljs-meta">... </span>    num_train_epochs=<span class="hljs-number">3</span>,
<span class="hljs-meta">... </span>    weight_decay=<span class="hljs-number">0.01</span>,
<span class="hljs-meta">... </span>)

<span class="hljs-meta">&gt;&gt;&gt; </span>trainer = Trainer(
<span class="hljs-meta">... </span>    model=model,
<span class="hljs-meta">... </span>    args=training_args,
<span class="hljs-meta">... </span>    train_dataset=tokenized_wnut[<span class="hljs-string">&quot;train&quot;</span>],
<span class="hljs-meta">... </span>    eval_dataset=tokenized_wnut[<span class="hljs-string">&quot;test&quot;</span>],
<span class="hljs-meta">... </span>    tokenizer=tokenizer,
<span class="hljs-meta">... </span>    data_collator=data_collator,
<span class="hljs-meta">... </span>)

<span class="hljs-meta">&gt;&gt;&gt; </span>trainer.train()`}}),{c(){t=r("p"),u=n("Carregue o DistilBERT com o "),o=r("code"),f=n("AutoModelForTokenClassification"),j=n(" junto com o n\xFAmero de r\xF3tulos esperados:"),$=_(),x(v.$$.fragment),D=_(),x(A.$$.fragment),w=_(),O=r("p"),L=n("Nesse ponto, restam apenas tr\xEAs passos:"),G=_(),N=r("ol"),Y=r("li"),F=n("Definir seus hiperpar\xE2metros de treinamento em "),ns=r("code"),U=n("TrainingArguments"),$s=n("."),us=_(),S=r("li"),fs=n("Passar os argumentos de treinamento para o "),W=r("code"),hs=n("Trainer"),V=n(" junto com o modelo, conjunto de dados, tokenizador e o data collator."),_s=_(),B=r("li"),K=n("Chamar a fun\xE7\xE3o "),I=r("code"),J=n("train()"),Q=n(" para executar o fine-tuning do seu modelo."),ls=_(),x(R.$$.fragment)},l(h){t=i(h,"P",{});var y=p(t);u=l(y,"Carregue o DistilBERT com o "),o=i(y,"CODE",{});var ts=p(o);f=l(ts,"AutoModelForTokenClassification"),ts.forEach(e),j=l(y," junto com o n\xFAmero de r\xF3tulos esperados:"),y.forEach(e),$=g(h),E(v.$$.fragment,h),D=g(h),E(A.$$.fragment,h),w=g(h),O=i(h,"P",{});var H=p(O);L=l(H,"Nesse ponto, restam apenas tr\xEAs passos:"),H.forEach(e),G=g(h),N=i(h,"OL",{});var X=p(N);Y=i(X,"LI",{});var M=p(Y);F=l(M,"Definir seus hiperpar\xE2metros de treinamento em "),ns=i(M,"CODE",{});var ks=p(ns);U=l(ks,"TrainingArguments"),ks.forEach(e),$s=l(M,"."),M.forEach(e),us=g(X),S=i(X,"LI",{});var Z=p(S);fs=l(Z,"Passar os argumentos de treinamento para o "),W=i(Z,"CODE",{});var ss=p(W);hs=l(ss,"Trainer"),ss.forEach(e),V=l(Z," junto com o modelo, conjunto de dados, tokenizador e o data collator."),Z.forEach(e),_s=g(X),B=i(X,"LI",{});var is=p(B);K=l(is,"Chamar a fun\xE7\xE3o "),I=i(is,"CODE",{});var os=p(I);J=l(os,"train()"),os.forEach(e),Q=l(is," para executar o fine-tuning do seu modelo."),is.forEach(e),X.forEach(e),ls=g(h),E(R.$$.fragment,h)},m(h,y){d(h,t,y),a(t,u),a(t,o),a(o,f),a(t,j),d(h,$,y),q(v,h,y),d(h,D,y),q(A,h,y),d(h,w,y),d(h,O,y),a(O,L),d(h,G,y),d(h,N,y),a(N,Y),a(Y,F),a(Y,ns),a(ns,U),a(Y,$s),a(N,us),a(N,S),a(S,fs),a(S,W),a(W,hs),a(S,V),a(N,_s),a(N,B),a(B,K),a(B,I),a(I,J),a(B,Q),d(h,ls,y),q(R,h,y),es=!0},p(h,y){const ts={};y&2&&(ts.$$scope={dirty:y,ctx:h}),A.$set(ts)},i(h){es||(z(v.$$.fragment,h),z(A.$$.fragment,h),z(R.$$.fragment,h),es=!0)},o(h){C(v.$$.fragment,h),C(A.$$.fragment,h),C(R.$$.fragment,h),es=!1},d(h){h&&e(t),h&&e($),T(v,h),h&&e(D),T(A,h),h&&e(w),h&&e(O),h&&e(G),h&&e(N),h&&e(ls),T(R,h)}}}function zo(P){let t,u;return t=new _e({props:{$$slots:{default:[qo]},$$scope:{ctx:P}}}),{c(){x(t.$$.fragment)},l(o){E(t.$$.fragment,o)},m(o,f){q(t,o,f),u=!0},p(o,f){const j={};f&2&&(j.$$scope={dirty:f,ctx:o}),t.$set(j)},i(o){u||(z(t.$$.fragment,o),u=!0)},o(o){C(t.$$.fragment,o),u=!1},d(o){T(t,o)}}}function Co(P){let t,u,o,f,j;return{c(){t=r("p"),u=n("Se voc\xEA n\xE3o estiver familiarizado com o fine-tuning de um modelo com o Keras, d\xEA uma olhada no tutorial b\xE1sico "),o=r("a"),f=n("aqui"),j=n("!"),this.h()},l($){t=i($,"P",{});var v=p(t);u=l(v,"Se voc\xEA n\xE3o estiver familiarizado com o fine-tuning de um modelo com o Keras, d\xEA uma olhada no tutorial b\xE1sico "),o=i(v,"A",{href:!0});var D=p(o);f=l(D,"aqui"),D.forEach(e),j=l(v,"!"),v.forEach(e),this.h()},h(){k(o,"href","training#finetune-with-keras")},m($,v){d($,t,v),a(t,u),a(t,o),a(o,f),a(t,j)},d($){$&&e(t)}}}function To(P){let t,u,o,f,j,$,v,D,A,w,O,L,G,N,Y,F,ns,U,$s,us,S,fs,W,hs,V,_s,B,K,I,J,Q,ls,R,es,h,y,ts,H,X,M,ks,Z,ss,is,os,Ds,ps,As;return N=new as({props:{code:`tf_train_set = tokenized_wnut["train"].to_tf_dataset(
    columns=["attention_mask", "input_ids", "labels"],
    shuffle=True,
    batch_size=16,
    collate_fn=data_collator,
)

tf_validation_set = tokenized_wnut["validation"].to_tf_dataset(
    columns=["attention_mask", "input_ids", "labels"],
    shuffle=False,
    batch_size=16,
    collate_fn=data_collator,
)`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>tf_train_set = tokenized_wnut[<span class="hljs-string">&quot;train&quot;</span>].to_tf_dataset(
<span class="hljs-meta">... </span>    columns=[<span class="hljs-string">&quot;attention_mask&quot;</span>, <span class="hljs-string">&quot;input_ids&quot;</span>, <span class="hljs-string">&quot;labels&quot;</span>],
<span class="hljs-meta">... </span>    shuffle=<span class="hljs-literal">True</span>,
<span class="hljs-meta">... </span>    batch_size=<span class="hljs-number">16</span>,
<span class="hljs-meta">... </span>    collate_fn=data_collator,
<span class="hljs-meta">... </span>)

<span class="hljs-meta">&gt;&gt;&gt; </span>tf_validation_set = tokenized_wnut[<span class="hljs-string">&quot;validation&quot;</span>].to_tf_dataset(
<span class="hljs-meta">... </span>    columns=[<span class="hljs-string">&quot;attention_mask&quot;</span>, <span class="hljs-string">&quot;input_ids&quot;</span>, <span class="hljs-string">&quot;labels&quot;</span>],
<span class="hljs-meta">... </span>    shuffle=<span class="hljs-literal">False</span>,
<span class="hljs-meta">... </span>    batch_size=<span class="hljs-number">16</span>,
<span class="hljs-meta">... </span>    collate_fn=data_collator,
<span class="hljs-meta">... </span>)`}}),F=new he({props:{$$slots:{default:[Co]},$$scope:{ctx:P}}}),S=new as({props:{code:`from transformers import create_optimizer

batch_size = 16
num_train_epochs = 3
num_train_steps = (len(tokenized_wnut["train"]) // batch_size) * num_train_epochs
optimizer, lr_schedule = create_optimizer(
    init_lr=2e-5,
    num_train_steps=num_train_steps,
    weight_decay_rate=0.01,
    num_warmup_steps=0,
)`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> create_optimizer

<span class="hljs-meta">&gt;&gt;&gt; </span>batch_size = <span class="hljs-number">16</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>num_train_epochs = <span class="hljs-number">3</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>num_train_steps = (<span class="hljs-built_in">len</span>(tokenized_wnut[<span class="hljs-string">&quot;train&quot;</span>]) // batch_size) * num_train_epochs
<span class="hljs-meta">&gt;&gt;&gt; </span>optimizer, lr_schedule = create_optimizer(
<span class="hljs-meta">... </span>    init_lr=<span class="hljs-number">2e-5</span>,
<span class="hljs-meta">... </span>    num_train_steps=num_train_steps,
<span class="hljs-meta">... </span>    weight_decay_rate=<span class="hljs-number">0.01</span>,
<span class="hljs-meta">... </span>    num_warmup_steps=<span class="hljs-number">0</span>,
<span class="hljs-meta">... </span>)`}}),I=new as({props:{code:`from transformers import TFAutoModelForTokenClassification

model = TFAutoModelForTokenClassification.from_pretrained("distilbert-base-uncased", num_labels=2)`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> TFAutoModelForTokenClassification

<span class="hljs-meta">&gt;&gt;&gt; </span>model = TFAutoModelForTokenClassification.from_pretrained(<span class="hljs-string">&quot;distilbert-base-uncased&quot;</span>, num_labels=<span class="hljs-number">2</span>)`}}),H=new as({props:{code:`import tensorflow as tf

model.compile(optimizer=optimizer)`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">import</span> tensorflow <span class="hljs-keyword">as</span> tf

<span class="hljs-meta">&gt;&gt;&gt; </span>model.<span class="hljs-built_in">compile</span>(optimizer=optimizer)`}}),ps=new as({props:{code:"model.fit(x=tf_train_set, validation_data=tf_validation_set, epochs=3)",highlighted:'<span class="hljs-meta">&gt;&gt;&gt; </span>model.fit(x=tf_train_set, validation_data=tf_validation_set, epochs=<span class="hljs-number">3</span>)'}}),{c(){t=r("p"),u=n("Para executar o fine-tuning de um modelo no TensorFlow, comece convertendo seu conjunto de dados para o formato "),o=r("code"),f=n("tf.data.Dataset"),j=n(" com "),$=r("a"),v=r("code"),D=n("to_tf_dataset"),A=n(". Nessa execu\xE7\xE3o voc\xEA dever\xE1 especificar as entradas e r\xF3tulos (no par\xE2metro "),w=r("code"),O=n("columns"),L=n("), se deseja embaralhar o conjunto de dados, o tamanho do batch e o data collator:"),G=_(),x(N.$$.fragment),Y=_(),x(F.$$.fragment),ns=_(),U=r("p"),$s=n("Configure o otimizador e alguns hiperpar\xE2metros de treinamento:"),us=_(),x(S.$$.fragment),fs=_(),W=r("p"),hs=n("Carregue o DistilBERT com o "),V=r("code"),_s=n("TFAutoModelForTokenClassification"),B=n(" junto com o n\xFAmero de r\xF3tulos esperados:"),K=_(),x(I.$$.fragment),J=_(),Q=r("p"),ls=n("Configure o modelo para treinamento com o m\xE9todo "),R=r("a"),es=r("code"),h=n("compile"),y=n(":"),ts=_(),x(H.$$.fragment),X=_(),M=r("p"),ks=n("Chame o m\xE9todo "),Z=r("a"),ss=r("code"),is=n("fit"),os=n(" para executar o fine-tuning do modelo:"),Ds=_(),x(ps.$$.fragment),this.h()},l(c){t=i(c,"P",{});var b=p(t);u=l(b,"Para executar o fine-tuning de um modelo no TensorFlow, comece convertendo seu conjunto de dados para o formato "),o=i(b,"CODE",{});var cs=p(o);f=l(cs,"tf.data.Dataset"),cs.forEach(e),j=l(b," com "),$=i(b,"A",{href:!0,rel:!0});var Us=p($);v=i(Us,"CODE",{});var ds=p(v);D=l(ds,"to_tf_dataset"),ds.forEach(e),Us.forEach(e),A=l(b,". Nessa execu\xE7\xE3o voc\xEA dever\xE1 especificar as entradas e r\xF3tulos (no par\xE2metro "),w=i(b,"CODE",{});var ta=p(w);O=l(ta,"columns"),ta.forEach(e),L=l(b,"), se deseja embaralhar o conjunto de dados, o tamanho do batch e o data collator:"),b.forEach(e),G=g(c),E(N.$$.fragment,c),Y=g(c),E(F.$$.fragment,c),ns=g(c),U=i(c,"P",{});var ys=p(U);$s=l(ys,"Configure o otimizador e alguns hiperpar\xE2metros de treinamento:"),ys.forEach(e),us=g(c),E(S.$$.fragment,c),fs=g(c),W=i(c,"P",{});var Ps=p(W);hs=l(Ps,"Carregue o DistilBERT com o "),V=i(Ps,"CODE",{});var oa=p(V);_s=l(oa,"TFAutoModelForTokenClassification"),oa.forEach(e),B=l(Ps," junto com o n\xFAmero de r\xF3tulos esperados:"),Ps.forEach(e),K=g(c),E(I.$$.fragment,c),J=g(c),Q=i(c,"P",{});var vs=p(Q);ls=l(vs,"Configure o modelo para treinamento com o m\xE9todo "),R=i(vs,"A",{href:!0,rel:!0});var na=p(R);es=i(na,"CODE",{});var la=p(es);h=l(la,"compile"),la.forEach(e),na.forEach(e),y=l(vs,":"),vs.forEach(e),ts=g(c),E(H.$$.fragment,c),X=g(c),M=i(c,"P",{});var qs=p(M);ks=l(qs,"Chame o m\xE9todo "),Z=i(qs,"A",{href:!0,rel:!0});var ms=p(Z);ss=i(ms,"CODE",{});var zs=p(ss);is=l(zs,"fit"),zs.forEach(e),ms.forEach(e),os=l(qs," para executar o fine-tuning do modelo:"),qs.forEach(e),Ds=g(c),E(ps.$$.fragment,c),this.h()},h(){k($,"href","https://huggingface.co/docs/datasets/package_reference/main_classes.html#datasets.Dataset.to_tf_dataset"),k($,"rel","nofollow"),k(R,"href","https://keras.io/api/models/model_training_apis/#compile-method"),k(R,"rel","nofollow"),k(Z,"href","https://keras.io/api/models/model_training_apis/#fit-method"),k(Z,"rel","nofollow")},m(c,b){d(c,t,b),a(t,u),a(t,o),a(o,f),a(t,j),a(t,$),a($,v),a(v,D),a(t,A),a(t,w),a(w,O),a(t,L),d(c,G,b),q(N,c,b),d(c,Y,b),q(F,c,b),d(c,ns,b),d(c,U,b),a(U,$s),d(c,us,b),q(S,c,b),d(c,fs,b),d(c,W,b),a(W,hs),a(W,V),a(V,_s),a(W,B),d(c,K,b),q(I,c,b),d(c,J,b),d(c,Q,b),a(Q,ls),a(Q,R),a(R,es),a(es,h),a(Q,y),d(c,ts,b),q(H,c,b),d(c,X,b),d(c,M,b),a(M,ks),a(M,Z),a(Z,ss),a(ss,is),a(M,os),d(c,Ds,b),q(ps,c,b),As=!0},p(c,b){const cs={};b&2&&(cs.$$scope={dirty:b,ctx:c}),F.$set(cs)},i(c){As||(z(N.$$.fragment,c),z(F.$$.fragment,c),z(S.$$.fragment,c),z(I.$$.fragment,c),z(H.$$.fragment,c),z(ps.$$.fragment,c),As=!0)},o(c){C(N.$$.fragment,c),C(F.$$.fragment,c),C(S.$$.fragment,c),C(I.$$.fragment,c),C(H.$$.fragment,c),C(ps.$$.fragment,c),As=!1},d(c){c&&e(t),c&&e(G),T(N,c),c&&e(Y),T(F,c),c&&e(ns),c&&e(U),c&&e(us),T(S,c),c&&e(fs),c&&e(W),c&&e(K),T(I,c),c&&e(J),c&&e(Q),c&&e(ts),T(H,c),c&&e(X),c&&e(M),c&&e(Ds),T(ps,c)}}}function Do(P){let t,u;return t=new _e({props:{$$slots:{default:[To]},$$scope:{ctx:P}}}),{c(){x(t.$$.fragment)},l(o){E(t.$$.fragment,o)},m(o,f){q(t,o,f),u=!0},p(o,f){const j={};f&2&&(j.$$scope={dirty:f,ctx:o}),t.$set(j)},i(o){u||(z(t.$$.fragment,o),u=!0)},o(o){C(t.$$.fragment,o),u=!1},d(o){T(t,o)}}}function Ao(P){let t,u,o,f,j,$,v,D;return{c(){t=r("p"),u=n("Para obter um exemplo mais aprofundado de como executar o fine-tuning de um modelo para classifica\xE7\xE3o de tokens, d\xEA uma olhada nesse "),o=r("a"),f=n("notebook utilizando PyTorch"),j=n(" ou nesse "),$=r("a"),v=n("notebook utilizando TensorFlow"),D=n("."),this.h()},l(A){t=i(A,"P",{});var w=p(t);u=l(w,"Para obter um exemplo mais aprofundado de como executar o fine-tuning de um modelo para classifica\xE7\xE3o de tokens, d\xEA uma olhada nesse "),o=i(w,"A",{href:!0,rel:!0});var O=p(o);f=l(O,"notebook utilizando PyTorch"),O.forEach(e),j=l(w," ou nesse "),$=i(w,"A",{href:!0,rel:!0});var L=p($);v=l(L,"notebook utilizando TensorFlow"),L.forEach(e),D=l(w,"."),w.forEach(e),this.h()},h(){k(o,"href","https://colab.research.google.com/github/huggingface/notebooks/blob/main/examples/token_classification.ipynb"),k(o,"rel","nofollow"),k($,"href","https://colab.research.google.com/github/huggingface/notebooks/blob/main/examples/token_classification-tf.ipynb"),k($,"rel","nofollow")},m(A,w){d(A,t,w),a(t,u),a(t,o),a(o,f),a(t,j),a(t,$),a($,v),a(t,D)},d(A){A&&e(t)}}}function yo(P){let t,u,o,f,j,$,v,D,A,w,O,L,G,N,Y,F,ns,U,$s,us,S,fs,W,hs,V,_s,B,K,I,J,Q,ls,R,es,h,y,ts,H,X,M,ks,Z,ss,is,os,Ds,ps,As,c,b,cs,Us,ds,ta,ys,Ps,oa,vs,na,la,qs,ms,zs,fa,ge,$e,je,bs,ha,ke,ve,_a,be,we,ga,xe,Ee,qe,ra,$a,ze,Ce,La,Cs,Os,ja,Ws,Te,ka,De,Ua,Vs,Wa,Fs,Ae,va,ye,Pe,Va,Hs,Ha,Ss,Oe,ba,Fe,Se,Ya,Ys,Ka,ws,Ne,wa,Be,Me,xa,Ie,Re,Za,xs,Ks,Le,Zs,Ea,Ue,We,Ve,js,He,qa,Ye,Ke,za,Ze,Ge,Ca,Je,Qe,Xe,Gs,st,Ta,at,et,Ga,ia,tt,Ja,Js,Qa,gs,ot,Qs,Da,nt,lt,Aa,rt,it,ya,pt,ct,Xa,Xs,se,rs,dt,Pa,mt,ut,Oa,ft,ht,Fa,_t,gt,Sa,$t,jt,ae,Ns,ee,Ts,Bs,Na,sa,kt,Ba,vt,te,Ms,oe,Is,ne;return $=new fe({}),O=new mo({props:{id:"wVHdVlPScxA"}}),V=new he({props:{$$slots:{default:[ko]},$$scope:{ctx:P}}}),J=new fe({}),H=new as({props:{code:`from datasets import load_dataset

wnut = load_dataset("wnut_17")`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> datasets <span class="hljs-keyword">import</span> load_dataset

<span class="hljs-meta">&gt;&gt;&gt; </span>wnut = load_dataset(<span class="hljs-string">&quot;wnut_17&quot;</span>)`}}),ss=new as({props:{code:'wnut["train"][0]',highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>wnut[<span class="hljs-string">&quot;train&quot;</span>][<span class="hljs-number">0</span>]
{<span class="hljs-string">&#x27;id&#x27;</span>: <span class="hljs-string">&#x27;0&#x27;</span>,
 <span class="hljs-string">&#x27;ner_tags&#x27;</span>: [<span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">7</span>, <span class="hljs-number">8</span>, <span class="hljs-number">8</span>, <span class="hljs-number">0</span>, <span class="hljs-number">7</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>],
 <span class="hljs-string">&#x27;tokens&#x27;</span>: [<span class="hljs-string">&#x27;@paulwalk&#x27;</span>, <span class="hljs-string">&#x27;It&#x27;</span>, <span class="hljs-string">&quot;&#x27;s&quot;</span>, <span class="hljs-string">&#x27;the&#x27;</span>, <span class="hljs-string">&#x27;view&#x27;</span>, <span class="hljs-string">&#x27;from&#x27;</span>, <span class="hljs-string">&#x27;where&#x27;</span>, <span class="hljs-string">&#x27;I&#x27;</span>, <span class="hljs-string">&quot;&#x27;m&quot;</span>, <span class="hljs-string">&#x27;living&#x27;</span>, <span class="hljs-string">&#x27;for&#x27;</span>, <span class="hljs-string">&#x27;two&#x27;</span>, <span class="hljs-string">&#x27;weeks&#x27;</span>, <span class="hljs-string">&#x27;.&#x27;</span>, <span class="hljs-string">&#x27;Empire&#x27;</span>, <span class="hljs-string">&#x27;State&#x27;</span>, <span class="hljs-string">&#x27;Building&#x27;</span>, <span class="hljs-string">&#x27;=&#x27;</span>, <span class="hljs-string">&#x27;ESB&#x27;</span>, <span class="hljs-string">&#x27;.&#x27;</span>, <span class="hljs-string">&#x27;Pretty&#x27;</span>, <span class="hljs-string">&#x27;bad&#x27;</span>, <span class="hljs-string">&#x27;storm&#x27;</span>, <span class="hljs-string">&#x27;here&#x27;</span>, <span class="hljs-string">&#x27;last&#x27;</span>, <span class="hljs-string">&#x27;evening&#x27;</span>, <span class="hljs-string">&#x27;.&#x27;</span>]
}`}}),cs=new as({props:{code:`label_list = wnut["train"].features[f"ner_tags"].feature.names
label_list`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>label_list = wnut[<span class="hljs-string">&quot;train&quot;</span>].features[<span class="hljs-string">f&quot;ner_tags&quot;</span>].feature.names
<span class="hljs-meta">&gt;&gt;&gt; </span>label_list
[
    <span class="hljs-string">&quot;O&quot;</span>,
    <span class="hljs-string">&quot;B-corporation&quot;</span>,
    <span class="hljs-string">&quot;I-corporation&quot;</span>,
    <span class="hljs-string">&quot;B-creative-work&quot;</span>,
    <span class="hljs-string">&quot;I-creative-work&quot;</span>,
    <span class="hljs-string">&quot;B-group&quot;</span>,
    <span class="hljs-string">&quot;I-group&quot;</span>,
    <span class="hljs-string">&quot;B-location&quot;</span>,
    <span class="hljs-string">&quot;I-location&quot;</span>,
    <span class="hljs-string">&quot;B-person&quot;</span>,
    <span class="hljs-string">&quot;I-person&quot;</span>,
    <span class="hljs-string">&quot;B-product&quot;</span>,
    <span class="hljs-string">&quot;I-product&quot;</span>,
]`}}),Ws=new fe({}),Vs=new mo({props:{id:"iY2AZYdZAr0"}}),Hs=new as({props:{code:`from transformers import AutoTokenizer

tokenizer = AutoTokenizer.from_pretrained("distilbert-base-uncased")`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoTokenizer

<span class="hljs-meta">&gt;&gt;&gt; </span>tokenizer = AutoTokenizer.from_pretrained(<span class="hljs-string">&quot;distilbert-base-uncased&quot;</span>)`}}),Ys=new as({props:{code:`tokenized_input = tokenizer(example["tokens"], is_split_into_words=True)
tokens = tokenizer.convert_ids_to_tokens(tokenized_input["input_ids"])
tokens`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>tokenized_input = tokenizer(example[<span class="hljs-string">&quot;tokens&quot;</span>], is_split_into_words=<span class="hljs-literal">True</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>tokens = tokenizer.convert_ids_to_tokens(tokenized_input[<span class="hljs-string">&quot;input_ids&quot;</span>])
<span class="hljs-meta">&gt;&gt;&gt; </span>tokens
[<span class="hljs-string">&#x27;[CLS]&#x27;</span>, <span class="hljs-string">&#x27;@&#x27;</span>, <span class="hljs-string">&#x27;paul&#x27;</span>, <span class="hljs-string">&#x27;##walk&#x27;</span>, <span class="hljs-string">&#x27;it&#x27;</span>, <span class="hljs-string">&quot;&#x27;&quot;</span>, <span class="hljs-string">&#x27;s&#x27;</span>, <span class="hljs-string">&#x27;the&#x27;</span>, <span class="hljs-string">&#x27;view&#x27;</span>, <span class="hljs-string">&#x27;from&#x27;</span>, <span class="hljs-string">&#x27;where&#x27;</span>, <span class="hljs-string">&#x27;i&#x27;</span>, <span class="hljs-string">&quot;&#x27;&quot;</span>, <span class="hljs-string">&#x27;m&#x27;</span>, <span class="hljs-string">&#x27;living&#x27;</span>, <span class="hljs-string">&#x27;for&#x27;</span>, <span class="hljs-string">&#x27;two&#x27;</span>, <span class="hljs-string">&#x27;weeks&#x27;</span>, <span class="hljs-string">&#x27;.&#x27;</span>, <span class="hljs-string">&#x27;empire&#x27;</span>, <span class="hljs-string">&#x27;state&#x27;</span>, <span class="hljs-string">&#x27;building&#x27;</span>, <span class="hljs-string">&#x27;=&#x27;</span>, <span class="hljs-string">&#x27;es&#x27;</span>, <span class="hljs-string">&#x27;##b&#x27;</span>, <span class="hljs-string">&#x27;.&#x27;</span>, <span class="hljs-string">&#x27;pretty&#x27;</span>, <span class="hljs-string">&#x27;bad&#x27;</span>, <span class="hljs-string">&#x27;storm&#x27;</span>, <span class="hljs-string">&#x27;here&#x27;</span>, <span class="hljs-string">&#x27;last&#x27;</span>, <span class="hljs-string">&#x27;evening&#x27;</span>, <span class="hljs-string">&#x27;.&#x27;</span>, <span class="hljs-string">&#x27;[SEP]&#x27;</span>]`}}),Js=new as({props:{code:`def tokenize_and_align_labels(examples):
    tokenized_inputs = tokenizer(examples["tokens"], truncation=True, is_split_into_words=True)

    labels = []
    for i, label in enumerate(examples[f"ner_tags"]):
        word_ids = tokenized_inputs.word_ids(batch_index=i)  # Map tokens to their respective word.
        previous_word_idx = None
        label_ids = []
        for word_idx in word_ids:  # Set the special tokens to -100.
            if word_idx is None:
                label_ids.append(-100)
            elif word_idx != previous_word_idx:  # Only label the first token of a given word.
                label_ids.append(label[word_idx])
            else:
                label_ids.append(-100)
            previous_word_idx = word_idx
        labels.append(label_ids)

    tokenized_inputs["labels"] = labels
    return tokenized_inputs`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">def</span> <span class="hljs-title function_">tokenize_and_align_labels</span>(<span class="hljs-params">examples</span>):
<span class="hljs-meta">... </span>    tokenized_inputs = tokenizer(examples[<span class="hljs-string">&quot;tokens&quot;</span>], truncation=<span class="hljs-literal">True</span>, is_split_into_words=<span class="hljs-literal">True</span>)

<span class="hljs-meta">... </span>    labels = []
<span class="hljs-meta">... </span>    <span class="hljs-keyword">for</span> i, label <span class="hljs-keyword">in</span> <span class="hljs-built_in">enumerate</span>(examples[<span class="hljs-string">f&quot;ner_tags&quot;</span>]):
<span class="hljs-meta">... </span>        word_ids = tokenized_inputs.word_ids(batch_index=i)  <span class="hljs-comment"># Map tokens to their respective word.</span>
<span class="hljs-meta">... </span>        previous_word_idx = <span class="hljs-literal">None</span>
<span class="hljs-meta">... </span>        label_ids = []
<span class="hljs-meta">... </span>        <span class="hljs-keyword">for</span> word_idx <span class="hljs-keyword">in</span> word_ids:  <span class="hljs-comment"># Set the special tokens to -100.</span>
<span class="hljs-meta">... </span>            <span class="hljs-keyword">if</span> word_idx <span class="hljs-keyword">is</span> <span class="hljs-literal">None</span>:
<span class="hljs-meta">... </span>                label_ids.append(-<span class="hljs-number">100</span>)
<span class="hljs-meta">... </span>            <span class="hljs-keyword">elif</span> word_idx != previous_word_idx:  <span class="hljs-comment"># Only label the first token of a given word.</span>
<span class="hljs-meta">... </span>                label_ids.append(label[word_idx])
<span class="hljs-meta">... </span>            <span class="hljs-keyword">else</span>:
<span class="hljs-meta">... </span>                label_ids.append(-<span class="hljs-number">100</span>)
<span class="hljs-meta">... </span>            previous_word_idx = word_idx
<span class="hljs-meta">... </span>        labels.append(label_ids)

<span class="hljs-meta">... </span>    tokenized_inputs[<span class="hljs-string">&quot;labels&quot;</span>] = labels
<span class="hljs-meta">... </span>    <span class="hljs-keyword">return</span> tokenized_inputs`}}),Xs=new as({props:{code:"tokenized_wnut = wnut.map(tokenize_and_align_labels, batched=True)",highlighted:'<span class="hljs-meta">&gt;&gt;&gt; </span>tokenized_wnut = wnut.<span class="hljs-built_in">map</span>(tokenize_and_align_labels, batched=<span class="hljs-literal">True</span>)'}}),Ns=new uo({props:{pytorch:!0,tensorflow:!0,jax:!1,$$slots:{tensorflow:[xo],pytorch:[bo]},$$scope:{ctx:P}}}),sa=new fe({}),Ms=new uo({props:{pytorch:!0,tensorflow:!0,jax:!1,$$slots:{tensorflow:[Do],pytorch:[zo]},$$scope:{ctx:P}}}),Is=new he({props:{$$slots:{default:[Ao]},$$scope:{ctx:P}}}),{c(){t=r("meta"),u=_(),o=r("h1"),f=r("a"),j=r("span"),x($.$$.fragment),v=_(),D=r("span"),A=n("Classifica\xE7\xE3o de tokens"),w=_(),x(O.$$.fragment),L=_(),G=r("p"),N=n("A classifica\xE7\xE3o de tokens atribui um r\xF3tulo a tokens individuais em uma frase. Uma das tarefas de classifica\xE7\xE3o de tokens mais comuns \xE9 o Reconhecimento de Entidade Nomeada, tamb\xE9m chamada de NER (sigla em ingl\xEAs para Named Entity Recognition). O NER tenta encontrar um r\xF3tulo para cada entidade em uma frase, como uma pessoa, local ou organiza\xE7\xE3o."),Y=_(),F=r("p"),ns=n("Este guia mostrar\xE1 como realizar o fine-tuning do "),U=r("a"),$s=n("DistilBERT"),us=n(" no conjunto de dados "),S=r("a"),fs=n("WNUT 17"),W=n(" para detectar novas entidades."),hs=_(),x(V.$$.fragment),_s=_(),B=r("h2"),K=r("a"),I=r("span"),x(J.$$.fragment),Q=_(),ls=r("span"),R=n("Carregando o conjunto de dados WNUT 17"),es=_(),h=r("p"),y=n("Carregue o conjunto de dados WNUT 17 da biblioteca \u{1F917} Datasets:"),ts=_(),x(H.$$.fragment),X=_(),M=r("p"),ks=n("E d\xEA uma olhada em um exemplo:"),Z=_(),x(ss.$$.fragment),is=_(),os=r("p"),Ds=n("Cada n\xFAmero em "),ps=r("code"),As=n("ner_tags"),c=n(" representa uma entidade. Converta o n\xFAmero em um r\xF3tulo para obter mais informa\xE7\xF5es:"),b=_(),x(cs.$$.fragment),Us=_(),ds=r("p"),ta=n("O "),ys=r("code"),Ps=n("ner_tag"),oa=n(" descreve uma entidade, como uma organiza\xE7\xE3o, local ou pessoa. A letra que prefixa cada "),vs=r("code"),na=n("ner_tag"),la=n(" indica a posi\xE7\xE3o do token da entidade:"),qs=_(),ms=r("ul"),zs=r("li"),fa=r("code"),ge=n("B-"),$e=n(" indica o in\xEDcio de uma entidade."),je=_(),bs=r("li"),ha=r("code"),ke=n("I-"),ve=n(" indica que um token est\xE1 contido dentro da mesma entidade (por exemplo, o token "),_a=r("code"),be=n("State"),we=n(" pode fazer parte de uma entidade como "),ga=r("code"),xe=n("Empire State Building"),Ee=n(")."),qe=_(),ra=r("li"),$a=r("code"),ze=n("0"),Ce=n(" indica que o token n\xE3o corresponde a nenhuma entidade."),La=_(),Cs=r("h2"),Os=r("a"),ja=r("span"),x(Ws.$$.fragment),Te=_(),ka=r("span"),De=n("Pr\xE9-processamento"),Ua=_(),x(Vs.$$.fragment),Wa=_(),Fs=r("p"),Ae=n("Carregue o tokenizer do DistilBERT para processar os "),va=r("code"),ye=n("tokens"),Pe=n(":"),Va=_(),x(Hs.$$.fragment),Ha=_(),Ss=r("p"),Oe=n("Como a entrada j\xE1 foi dividida em palavras, defina "),ba=r("code"),Fe=n("is_split_into_words=True"),Se=n(" para tokenizar as palavras em subpalavras:"),Ya=_(),x(Ys.$$.fragment),Ka=_(),ws=r("p"),Ne=n("Ao adicionar os tokens especiais "),wa=r("code"),Be=n("[CLS]"),Me=n(" e "),xa=r("code"),Ie=n("[SEP]"),Re=n(" e a tokeniza\xE7\xE3o de subpalavras uma incompatibilidade \xE9 gerada entre a entrada e os r\xF3tulos. Uma \xFAnica palavra correspondente a um \xFAnico r\xF3tulo pode ser dividida em duas subpalavras. Voc\xEA precisar\xE1 realinhar os tokens e os r\xF3tulos da seguinte forma:"),Za=_(),xs=r("ol"),Ks=r("li"),Le=n("Mapeie todos os tokens para a palavra correspondente com o m\xE9todo "),Zs=r("a"),Ea=r("code"),Ue=n("word_ids"),We=n("."),Ve=_(),js=r("li"),He=n("Atribuindo o r\xF3tulo "),qa=r("code"),Ye=n("-100"),Ke=n(" aos tokens especiais "),za=r("code"),Ze=n("[CLS]"),Ge=n(" e "),Ca=r("code"),Je=n("[SEP]"),Qe=n(" para que a fun\xE7\xE3o de loss do PyTorch ignore eles."),Xe=_(),Gs=r("li"),st=n("Rotular apenas o primeiro token de uma determinada palavra. Atribuindo "),Ta=r("code"),at=n("-100"),et=n(" a outros subtokens da mesma palavra."),Ga=_(),ia=r("p"),tt=n("Aqui est\xE1 como voc\xEA pode criar uma fun\xE7\xE3o para realinhar os tokens e r\xF3tulos e truncar sequ\xEAncias para n\xE3o serem maiores que o comprimento m\xE1ximo de entrada do DistilBERT:"),Ja=_(),x(Js.$$.fragment),Qa=_(),gs=r("p"),ot=n("Use a fun\xE7\xE3o "),Qs=r("a"),Da=r("code"),nt=n("map"),lt=n(" do \u{1F917} Datasets para tokenizar e alinhar os r\xF3tulos em todo o conjunto de dados. Voc\xEA pode acelerar a fun\xE7\xE3o "),Aa=r("code"),rt=n("map"),it=n(" configurando "),ya=r("code"),pt=n("batched=True"),ct=n(" para processar v\xE1rios elementos do conjunto de dados de uma s\xF3 vez:"),Xa=_(),x(Xs.$$.fragment),se=_(),rs=r("p"),dt=n("Use o "),Pa=r("code"),mt=n("DataCollatorForTokenClassification"),ut=n(" para criar um batch de exemplos. Ele tamb\xE9m "),Oa=r("em"),ft=n("preencher\xE1 dinamicamente"),ht=n(" seu texto e r\xF3tulos para o comprimento do elemento mais longo em seu batch, para que tenham um comprimento uniforme. Embora seja poss\xEDvel preencher seu texto na fun\xE7\xE3o "),Fa=r("code"),_t=n("tokenizer"),gt=n(" configurando "),Sa=r("code"),$t=n("padding=True"),jt=n(", o preenchimento din\xE2mico \xE9 mais eficiente."),ae=_(),x(Ns.$$.fragment),ee=_(),Ts=r("h2"),Bs=r("a"),Na=r("span"),x(sa.$$.fragment),kt=_(),Ba=r("span"),vt=n("Treinamento"),te=_(),x(Ms.$$.fragment),oe=_(),x(Is.$$.fragment),this.h()},l(s){const m=$o('[data-svelte="svelte-1phssyn"]',document.head);t=i(m,"META",{name:!0,content:!0}),m.forEach(e),u=g(s),o=i(s,"H1",{class:!0});var aa=p(o);f=i(aa,"A",{id:!0,class:!0,href:!0});var Ma=p(f);j=i(Ma,"SPAN",{});var Ia=p(j);E($.$$.fragment,Ia),Ia.forEach(e),Ma.forEach(e),v=g(aa),D=i(aa,"SPAN",{});var Ra=p(D);A=l(Ra,"Classifica\xE7\xE3o de tokens"),Ra.forEach(e),aa.forEach(e),w=g(s),E(O.$$.fragment,s),L=g(s),G=i(s,"P",{});var xt=p(G);N=l(xt,"A classifica\xE7\xE3o de tokens atribui um r\xF3tulo a tokens individuais em uma frase. Uma das tarefas de classifica\xE7\xE3o de tokens mais comuns \xE9 o Reconhecimento de Entidade Nomeada, tamb\xE9m chamada de NER (sigla em ingl\xEAs para Named Entity Recognition). O NER tenta encontrar um r\xF3tulo para cada entidade em uma frase, como uma pessoa, local ou organiza\xE7\xE3o."),xt.forEach(e),Y=g(s),F=i(s,"P",{});var pa=p(F);ns=l(pa,"Este guia mostrar\xE1 como realizar o fine-tuning do "),U=i(pa,"A",{href:!0,rel:!0});var Et=p(U);$s=l(Et,"DistilBERT"),Et.forEach(e),us=l(pa," no conjunto de dados "),S=i(pa,"A",{href:!0,rel:!0});var qt=p(S);fs=l(qt,"WNUT 17"),qt.forEach(e),W=l(pa," para detectar novas entidades."),pa.forEach(e),hs=g(s),E(V.$$.fragment,s),_s=g(s),B=i(s,"H2",{class:!0});var le=p(B);K=i(le,"A",{id:!0,class:!0,href:!0});var zt=p(K);I=i(zt,"SPAN",{});var Ct=p(I);E(J.$$.fragment,Ct),Ct.forEach(e),zt.forEach(e),Q=g(le),ls=i(le,"SPAN",{});var Tt=p(ls);R=l(Tt,"Carregando o conjunto de dados WNUT 17"),Tt.forEach(e),le.forEach(e),es=g(s),h=i(s,"P",{});var Dt=p(h);y=l(Dt,"Carregue o conjunto de dados WNUT 17 da biblioteca \u{1F917} Datasets:"),Dt.forEach(e),ts=g(s),E(H.$$.fragment,s),X=g(s),M=i(s,"P",{});var At=p(M);ks=l(At,"E d\xEA uma olhada em um exemplo:"),At.forEach(e),Z=g(s),E(ss.$$.fragment,s),is=g(s),os=i(s,"P",{});var re=p(os);Ds=l(re,"Cada n\xFAmero em "),ps=i(re,"CODE",{});var yt=p(ps);As=l(yt,"ner_tags"),yt.forEach(e),c=l(re," representa uma entidade. Converta o n\xFAmero em um r\xF3tulo para obter mais informa\xE7\xF5es:"),re.forEach(e),b=g(s),E(cs.$$.fragment,s),Us=g(s),ds=i(s,"P",{});var ca=p(ds);ta=l(ca,"O "),ys=i(ca,"CODE",{});var Pt=p(ys);Ps=l(Pt,"ner_tag"),Pt.forEach(e),oa=l(ca," descreve uma entidade, como uma organiza\xE7\xE3o, local ou pessoa. A letra que prefixa cada "),vs=i(ca,"CODE",{});var Ot=p(vs);na=l(Ot,"ner_tag"),Ot.forEach(e),la=l(ca," indica a posi\xE7\xE3o do token da entidade:"),ca.forEach(e),qs=g(s),ms=i(s,"UL",{});var da=p(ms);zs=i(da,"LI",{});var bt=p(zs);fa=i(bt,"CODE",{});var Ft=p(fa);ge=l(Ft,"B-"),Ft.forEach(e),$e=l(bt," indica o in\xEDcio de uma entidade."),bt.forEach(e),je=g(da),bs=i(da,"LI",{});var ea=p(bs);ha=i(ea,"CODE",{});var St=p(ha);ke=l(St,"I-"),St.forEach(e),ve=l(ea," indica que um token est\xE1 contido dentro da mesma entidade (por exemplo, o token "),_a=i(ea,"CODE",{});var Nt=p(_a);be=l(Nt,"State"),Nt.forEach(e),we=l(ea," pode fazer parte de uma entidade como "),ga=i(ea,"CODE",{});var Bt=p(ga);xe=l(Bt,"Empire State Building"),Bt.forEach(e),Ee=l(ea,")."),ea.forEach(e),qe=g(da),ra=i(da,"LI",{});var wt=p(ra);$a=i(wt,"CODE",{});var Mt=p($a);ze=l(Mt,"0"),Mt.forEach(e),Ce=l(wt," indica que o token n\xE3o corresponde a nenhuma entidade."),wt.forEach(e),da.forEach(e),La=g(s),Cs=i(s,"H2",{class:!0});var ie=p(Cs);Os=i(ie,"A",{id:!0,class:!0,href:!0});var It=p(Os);ja=i(It,"SPAN",{});var Rt=p(ja);E(Ws.$$.fragment,Rt),Rt.forEach(e),It.forEach(e),Te=g(ie),ka=i(ie,"SPAN",{});var Lt=p(ka);De=l(Lt,"Pr\xE9-processamento"),Lt.forEach(e),ie.forEach(e),Ua=g(s),E(Vs.$$.fragment,s),Wa=g(s),Fs=i(s,"P",{});var pe=p(Fs);Ae=l(pe,"Carregue o tokenizer do DistilBERT para processar os "),va=i(pe,"CODE",{});var Ut=p(va);ye=l(Ut,"tokens"),Ut.forEach(e),Pe=l(pe,":"),pe.forEach(e),Va=g(s),E(Hs.$$.fragment,s),Ha=g(s),Ss=i(s,"P",{});var ce=p(Ss);Oe=l(ce,"Como a entrada j\xE1 foi dividida em palavras, defina "),ba=i(ce,"CODE",{});var Wt=p(ba);Fe=l(Wt,"is_split_into_words=True"),Wt.forEach(e),Se=l(ce," para tokenizar as palavras em subpalavras:"),ce.forEach(e),Ya=g(s),E(Ys.$$.fragment,s),Ka=g(s),ws=i(s,"P",{});var ma=p(ws);Ne=l(ma,"Ao adicionar os tokens especiais "),wa=i(ma,"CODE",{});var Vt=p(wa);Be=l(Vt,"[CLS]"),Vt.forEach(e),Me=l(ma," e "),xa=i(ma,"CODE",{});var Ht=p(xa);Ie=l(Ht,"[SEP]"),Ht.forEach(e),Re=l(ma," e a tokeniza\xE7\xE3o de subpalavras uma incompatibilidade \xE9 gerada entre a entrada e os r\xF3tulos. Uma \xFAnica palavra correspondente a um \xFAnico r\xF3tulo pode ser dividida em duas subpalavras. Voc\xEA precisar\xE1 realinhar os tokens e os r\xF3tulos da seguinte forma:"),ma.forEach(e),Za=g(s),xs=i(s,"OL",{});var ua=p(xs);Ks=i(ua,"LI",{});var de=p(Ks);Le=l(de,"Mapeie todos os tokens para a palavra correspondente com o m\xE9todo "),Zs=i(de,"A",{href:!0,rel:!0});var Yt=p(Zs);Ea=i(Yt,"CODE",{});var Kt=p(Ea);Ue=l(Kt,"word_ids"),Kt.forEach(e),Yt.forEach(e),We=l(de,"."),de.forEach(e),Ve=g(ua),js=i(ua,"LI",{});var Rs=p(js);He=l(Rs,"Atribuindo o r\xF3tulo "),qa=i(Rs,"CODE",{});var Zt=p(qa);Ye=l(Zt,"-100"),Zt.forEach(e),Ke=l(Rs," aos tokens especiais "),za=i(Rs,"CODE",{});var Gt=p(za);Ze=l(Gt,"[CLS]"),Gt.forEach(e),Ge=l(Rs," e "),Ca=i(Rs,"CODE",{});var Jt=p(Ca);Je=l(Jt,"[SEP]"),Jt.forEach(e),Qe=l(Rs," para que a fun\xE7\xE3o de loss do PyTorch ignore eles."),Rs.forEach(e),Xe=g(ua),Gs=i(ua,"LI",{});var me=p(Gs);st=l(me,"Rotular apenas o primeiro token de uma determinada palavra. Atribuindo "),Ta=i(me,"CODE",{});var Qt=p(Ta);at=l(Qt,"-100"),Qt.forEach(e),et=l(me," a outros subtokens da mesma palavra."),me.forEach(e),ua.forEach(e),Ga=g(s),ia=i(s,"P",{});var Xt=p(ia);tt=l(Xt,"Aqui est\xE1 como voc\xEA pode criar uma fun\xE7\xE3o para realinhar os tokens e r\xF3tulos e truncar sequ\xEAncias para n\xE3o serem maiores que o comprimento m\xE1ximo de entrada do DistilBERT:"),Xt.forEach(e),Ja=g(s),E(Js.$$.fragment,s),Qa=g(s),gs=i(s,"P",{});var Ls=p(gs);ot=l(Ls,"Use a fun\xE7\xE3o "),Qs=i(Ls,"A",{href:!0,rel:!0});var so=p(Qs);Da=i(so,"CODE",{});var ao=p(Da);nt=l(ao,"map"),ao.forEach(e),so.forEach(e),lt=l(Ls," do \u{1F917} Datasets para tokenizar e alinhar os r\xF3tulos em todo o conjunto de dados. Voc\xEA pode acelerar a fun\xE7\xE3o "),Aa=i(Ls,"CODE",{});var eo=p(Aa);rt=l(eo,"map"),eo.forEach(e),it=l(Ls," configurando "),ya=i(Ls,"CODE",{});var to=p(ya);pt=l(to,"batched=True"),to.forEach(e),ct=l(Ls," para processar v\xE1rios elementos do conjunto de dados de uma s\xF3 vez:"),Ls.forEach(e),Xa=g(s),E(Xs.$$.fragment,s),se=g(s),rs=i(s,"P",{});var Es=p(rs);dt=l(Es,"Use o "),Pa=i(Es,"CODE",{});var oo=p(Pa);mt=l(oo,"DataCollatorForTokenClassification"),oo.forEach(e),ut=l(Es," para criar um batch de exemplos. Ele tamb\xE9m "),Oa=i(Es,"EM",{});var no=p(Oa);ft=l(no,"preencher\xE1 dinamicamente"),no.forEach(e),ht=l(Es," seu texto e r\xF3tulos para o comprimento do elemento mais longo em seu batch, para que tenham um comprimento uniforme. Embora seja poss\xEDvel preencher seu texto na fun\xE7\xE3o "),Fa=i(Es,"CODE",{});var lo=p(Fa);_t=l(lo,"tokenizer"),lo.forEach(e),gt=l(Es," configurando "),Sa=i(Es,"CODE",{});var ro=p(Sa);$t=l(ro,"padding=True"),ro.forEach(e),jt=l(Es,", o preenchimento din\xE2mico \xE9 mais eficiente."),Es.forEach(e),ae=g(s),E(Ns.$$.fragment,s),ee=g(s),Ts=i(s,"H2",{class:!0});var ue=p(Ts);Bs=i(ue,"A",{id:!0,class:!0,href:!0});var io=p(Bs);Na=i(io,"SPAN",{});var po=p(Na);E(sa.$$.fragment,po),po.forEach(e),io.forEach(e),kt=g(ue),Ba=i(ue,"SPAN",{});var co=p(Ba);vt=l(co,"Treinamento"),co.forEach(e),ue.forEach(e),te=g(s),E(Ms.$$.fragment,s),oe=g(s),E(Is.$$.fragment,s),this.h()},h(){k(t,"name","hf:doc:metadata"),k(t,"content",JSON.stringify(Po)),k(f,"id","classificao-de-tokens"),k(f,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),k(f,"href","#classificao-de-tokens"),k(o,"class","relative group"),k(U,"href","https://huggingface.co/distilbert-base-uncased"),k(U,"rel","nofollow"),k(S,"href","https://huggingface.co/datasets/wnut_17"),k(S,"rel","nofollow"),k(K,"id","carregando-o-conjunto-de-dados-wnut-17"),k(K,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),k(K,"href","#carregando-o-conjunto-de-dados-wnut-17"),k(B,"class","relative group"),k(Os,"id","prprocessamento"),k(Os,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),k(Os,"href","#prprocessamento"),k(Cs,"class","relative group"),k(Zs,"href","https://huggingface.co/docs/tokenizers/python/latest/api/reference.html#tokenizers.Encoding.word_ids"),k(Zs,"rel","nofollow"),k(Qs,"href","https://huggingface.co/docs/datasets/package_reference/main_classes.html#datasets.Dataset.map"),k(Qs,"rel","nofollow"),k(Bs,"id","treinamento"),k(Bs,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),k(Bs,"href","#treinamento"),k(Ts,"class","relative group")},m(s,m){a(document.head,t),d(s,u,m),d(s,o,m),a(o,f),a(f,j),q($,j,null),a(o,v),a(o,D),a(D,A),d(s,w,m),q(O,s,m),d(s,L,m),d(s,G,m),a(G,N),d(s,Y,m),d(s,F,m),a(F,ns),a(F,U),a(U,$s),a(F,us),a(F,S),a(S,fs),a(F,W),d(s,hs,m),q(V,s,m),d(s,_s,m),d(s,B,m),a(B,K),a(K,I),q(J,I,null),a(B,Q),a(B,ls),a(ls,R),d(s,es,m),d(s,h,m),a(h,y),d(s,ts,m),q(H,s,m),d(s,X,m),d(s,M,m),a(M,ks),d(s,Z,m),q(ss,s,m),d(s,is,m),d(s,os,m),a(os,Ds),a(os,ps),a(ps,As),a(os,c),d(s,b,m),q(cs,s,m),d(s,Us,m),d(s,ds,m),a(ds,ta),a(ds,ys),a(ys,Ps),a(ds,oa),a(ds,vs),a(vs,na),a(ds,la),d(s,qs,m),d(s,ms,m),a(ms,zs),a(zs,fa),a(fa,ge),a(zs,$e),a(ms,je),a(ms,bs),a(bs,ha),a(ha,ke),a(bs,ve),a(bs,_a),a(_a,be),a(bs,we),a(bs,ga),a(ga,xe),a(bs,Ee),a(ms,qe),a(ms,ra),a(ra,$a),a($a,ze),a(ra,Ce),d(s,La,m),d(s,Cs,m),a(Cs,Os),a(Os,ja),q(Ws,ja,null),a(Cs,Te),a(Cs,ka),a(ka,De),d(s,Ua,m),q(Vs,s,m),d(s,Wa,m),d(s,Fs,m),a(Fs,Ae),a(Fs,va),a(va,ye),a(Fs,Pe),d(s,Va,m),q(Hs,s,m),d(s,Ha,m),d(s,Ss,m),a(Ss,Oe),a(Ss,ba),a(ba,Fe),a(Ss,Se),d(s,Ya,m),q(Ys,s,m),d(s,Ka,m),d(s,ws,m),a(ws,Ne),a(ws,wa),a(wa,Be),a(ws,Me),a(ws,xa),a(xa,Ie),a(ws,Re),d(s,Za,m),d(s,xs,m),a(xs,Ks),a(Ks,Le),a(Ks,Zs),a(Zs,Ea),a(Ea,Ue),a(Ks,We),a(xs,Ve),a(xs,js),a(js,He),a(js,qa),a(qa,Ye),a(js,Ke),a(js,za),a(za,Ze),a(js,Ge),a(js,Ca),a(Ca,Je),a(js,Qe),a(xs,Xe),a(xs,Gs),a(Gs,st),a(Gs,Ta),a(Ta,at),a(Gs,et),d(s,Ga,m),d(s,ia,m),a(ia,tt),d(s,Ja,m),q(Js,s,m),d(s,Qa,m),d(s,gs,m),a(gs,ot),a(gs,Qs),a(Qs,Da),a(Da,nt),a(gs,lt),a(gs,Aa),a(Aa,rt),a(gs,it),a(gs,ya),a(ya,pt),a(gs,ct),d(s,Xa,m),q(Xs,s,m),d(s,se,m),d(s,rs,m),a(rs,dt),a(rs,Pa),a(Pa,mt),a(rs,ut),a(rs,Oa),a(Oa,ft),a(rs,ht),a(rs,Fa),a(Fa,_t),a(rs,gt),a(rs,Sa),a(Sa,$t),a(rs,jt),d(s,ae,m),q(Ns,s,m),d(s,ee,m),d(s,Ts,m),a(Ts,Bs),a(Bs,Na),q(sa,Na,null),a(Ts,kt),a(Ts,Ba),a(Ba,vt),d(s,te,m),q(Ms,s,m),d(s,oe,m),q(Is,s,m),ne=!0},p(s,[m]){const aa={};m&2&&(aa.$$scope={dirty:m,ctx:s}),V.$set(aa);const Ma={};m&2&&(Ma.$$scope={dirty:m,ctx:s}),Ns.$set(Ma);const Ia={};m&2&&(Ia.$$scope={dirty:m,ctx:s}),Ms.$set(Ia);const Ra={};m&2&&(Ra.$$scope={dirty:m,ctx:s}),Is.$set(Ra)},i(s){ne||(z($.$$.fragment,s),z(O.$$.fragment,s),z(V.$$.fragment,s),z(J.$$.fragment,s),z(H.$$.fragment,s),z(ss.$$.fragment,s),z(cs.$$.fragment,s),z(Ws.$$.fragment,s),z(Vs.$$.fragment,s),z(Hs.$$.fragment,s),z(Ys.$$.fragment,s),z(Js.$$.fragment,s),z(Xs.$$.fragment,s),z(Ns.$$.fragment,s),z(sa.$$.fragment,s),z(Ms.$$.fragment,s),z(Is.$$.fragment,s),ne=!0)},o(s){C($.$$.fragment,s),C(O.$$.fragment,s),C(V.$$.fragment,s),C(J.$$.fragment,s),C(H.$$.fragment,s),C(ss.$$.fragment,s),C(cs.$$.fragment,s),C(Ws.$$.fragment,s),C(Vs.$$.fragment,s),C(Hs.$$.fragment,s),C(Ys.$$.fragment,s),C(Js.$$.fragment,s),C(Xs.$$.fragment,s),C(Ns.$$.fragment,s),C(sa.$$.fragment,s),C(Ms.$$.fragment,s),C(Is.$$.fragment,s),ne=!1},d(s){e(t),s&&e(u),s&&e(o),T($),s&&e(w),T(O,s),s&&e(L),s&&e(G),s&&e(Y),s&&e(F),s&&e(hs),T(V,s),s&&e(_s),s&&e(B),T(J),s&&e(es),s&&e(h),s&&e(ts),T(H,s),s&&e(X),s&&e(M),s&&e(Z),T(ss,s),s&&e(is),s&&e(os),s&&e(b),T(cs,s),s&&e(Us),s&&e(ds),s&&e(qs),s&&e(ms),s&&e(La),s&&e(Cs),T(Ws),s&&e(Ua),T(Vs,s),s&&e(Wa),s&&e(Fs),s&&e(Va),T(Hs,s),s&&e(Ha),s&&e(Ss),s&&e(Ya),T(Ys,s),s&&e(Ka),s&&e(ws),s&&e(Za),s&&e(xs),s&&e(Ga),s&&e(ia),s&&e(Ja),T(Js,s),s&&e(Qa),s&&e(gs),s&&e(Xa),T(Xs,s),s&&e(se),s&&e(rs),s&&e(ae),T(Ns,s),s&&e(ee),s&&e(Ts),T(sa),s&&e(te),T(Ms,s),s&&e(oe),T(Is,s)}}}const Po={local:"classificao-de-tokens",sections:[{local:"carregando-o-conjunto-de-dados-wnut-17",title:"Carregando o conjunto de dados WNUT 17"},{local:"prprocessamento",title:"Pr\xE9-processamento"},{local:"treinamento",title:"Treinamento"}],title:"Classifica\xE7\xE3o de tokens"};function Oo(P){return jo(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class Ro extends ho{constructor(t){super();_o(this,t,Oo,yo,go,{})}}export{Ro as default,Po as metadata};
