import{S as dm,i as cm,s as mm,e as l,k as f,w as d,t as n,M as _m,c as i,d as t,m as h,a as p,x as c,h as o,b as u,F as s,g as r,y as m,q as _,o as w,B as g}from"../chunks/vendor-4833417e.js";import{T as Su}from"../chunks/Tip-fffd6df1.js";import{I as z}from"../chunks/IconCopyLink-4b81c553.js";import{C as b}from"../chunks/CodeBlock-6a3d1b46.js";import{D as wm}from"../chunks/DocNotebookDropdown-ecff2a90.js";import"../chunks/CopyButton-dacfbfaf.js";function gm(V){let $,q,k,y,E,v,P,T;return{c(){$=l("p"),q=n(`For a more in-depth example of how to fine-tune a model for text classification, take a look at the corresponding
`),k=l("a"),y=n("PyTorch notebook"),E=n(`
or `),v=l("a"),P=n("TensorFlow notebook"),T=n("."),this.h()},l(x){$=i(x,"P",{});var j=p($);q=o(j,`For a more in-depth example of how to fine-tune a model for text classification, take a look at the corresponding
`),k=i(j,"A",{href:!0,rel:!0});var A=p(k);y=o(A,"PyTorch notebook"),A.forEach(t),E=o(j,`
or `),v=i(j,"A",{href:!0,rel:!0});var C=p(v);P=o(C,"TensorFlow notebook"),C.forEach(t),T=o(j,"."),j.forEach(t),this.h()},h(){u(k,"href","https://colab.research.google.com/github/huggingface/notebooks/blob/master/examples/text_classification.ipynb"),u(k,"rel","nofollow"),u(v,"href","https://colab.research.google.com/github/huggingface/notebooks/blob/master/examples/text_classification-tf.ipynb"),u(v,"rel","nofollow")},m(x,j){r(x,$,j),s($,q),s($,k),s(k,y),s($,E),s($,v),s(v,P),s($,T)},d(x){x&&t($)}}}function bm(V){let $,q,k,y,E,v,P,T;return{c(){$=l("p"),q=n(`For a more in-depth example of how to fine-tune a model for token classification, take a look at the corresponding
`),k=l("a"),y=n("PyTorch notebook"),E=n(`
or `),v=l("a"),P=n("TensorFlow notebook"),T=n("."),this.h()},l(x){$=i(x,"P",{});var j=p($);q=o(j,`For a more in-depth example of how to fine-tune a model for token classification, take a look at the corresponding
`),k=i(j,"A",{href:!0,rel:!0});var A=p(k);y=o(A,"PyTorch notebook"),A.forEach(t),E=o(j,`
or `),v=i(j,"A",{href:!0,rel:!0});var C=p(v);P=o(C,"TensorFlow notebook"),C.forEach(t),T=o(j,"."),j.forEach(t),this.h()},h(){u(k,"href","https://colab.research.google.com/github/huggingface/notebooks/blob/master/examples/token_classification.ipynb"),u(k,"rel","nofollow"),u(v,"href","https://colab.research.google.com/github/huggingface/notebooks/blob/master/examples/token_classification-tf.ipynb"),u(v,"rel","nofollow")},m(x,j){r(x,$,j),s($,q),s($,k),s(k,y),s($,E),s($,v),s(v,P),s($,T)},d(x){x&&t($)}}}function $m(V){let $,q,k,y,E,v,P,T;return{c(){$=l("p"),q=n(`For a more in-depth example of how to fine-tune a model for question answering, take a look at the corresponding
`),k=l("a"),y=n("PyTorch notebook"),E=n(`
or `),v=l("a"),P=n("TensorFlow notebook"),T=n("."),this.h()},l(x){$=i(x,"P",{});var j=p($);q=o(j,`For a more in-depth example of how to fine-tune a model for question answering, take a look at the corresponding
`),k=i(j,"A",{href:!0,rel:!0});var A=p(k);y=o(A,"PyTorch notebook"),A.forEach(t),E=o(j,`
or `),v=i(j,"A",{href:!0,rel:!0});var C=p(v);P=o(C,"TensorFlow notebook"),C.forEach(t),T=o(j,"."),j.forEach(t),this.h()},h(){u(k,"href","https://colab.research.google.com/github/huggingface/notebooks/blob/master/examples/question_answering.ipynb"),u(k,"rel","nofollow"),u(v,"href","https://colab.research.google.com/github/huggingface/notebooks/blob/master/examples/question_answering-tf.ipynb"),u(v,"rel","nofollow")},m(x,j){r(x,$,j),s($,q),s($,k),s(k,y),s($,E),s($,v),s(v,P),s($,T)},d(x){x&&t($)}}}function km(V){let $,q,k,y,E,v,P,T,x,j,A,C,qs,gi,to,ue,bi,st,$i,ki,so,at,ao,Es,vi,no,I,Da,Ts,ji,yi,Sa,As,xi,qi,Na,zs,Ei,oo,Ps,ro,J,de,Ia,nt,Ti,Ma,Ai,lo,ce,zi,ot,Pi,Fi,io,me,po,K,_e,La,rt,Ci,Oa,Di,fo,Fs,Si,ho,lt,uo,we,Ni,Ba,Ii,Mi,co,it,mo,X,ge,Wa,pt,Li,Ha,Oi,_o,M,Bi,Cs,Wi,Hi,ft,Qi,Ri,wo,ht,go,Ds,Ui,bo,ut,$o,L,Yi,Qa,Gi,Vi,Ra,Ji,Ki,ko,dt,vo,F,Xi,Ua,Zi,ep,Ya,tp,sp,Ga,ap,np,Va,op,rp,jo,ct,yo,Z,be,Ja,mt,lp,Ka,ip,xo,$e,pp,Ss,fp,hp,qo,_t,Eo,Ns,up,To,O,wt,dp,Is,cp,mp,_p,gt,wp,Ms,gp,bp,$p,bt,kp,Xa,vp,jp,Ao,$t,zo,ee,ke,Za,kt,yp,en,xp,Po,Ls,qp,Fo,D,Ep,Os,Tp,Ap,tn,zp,Pp,sn,Fp,Cp,Co,vt,Do,S,Dp,an,Sp,Np,nn,Ip,Mp,on,Lp,Op,So,jt,No,Bs,Bp,Io,yt,Mo,ve,Wp,Ws,Hp,Qp,Lo,xt,Oo,Hs,Rp,Bo,qt,Wo,je,Up,rn,Yp,Gp,Ho,Et,Qo,Qs,Ro,te,ye,ln,Tt,Vp,pn,Jp,Uo,xe,Kp,At,Xp,Zp,Yo,qe,Go,se,Ee,fn,zt,ef,hn,tf,Vo,Rs,sf,Jo,Pt,Ko,Us,af,Xo,Ft,Zo,Ys,nf,er,Ct,tr,Gs,of,sr,B,Vs,un,rf,lf,pf,W,dn,ff,hf,cn,uf,df,mn,cf,mf,_f,Js,_n,wf,gf,ar,ae,Te,wn,Dt,bf,gn,$f,nr,Ae,kf,Ks,vf,jf,or,St,rr,ze,yf,bn,xf,qf,lr,Nt,ir,H,Ef,$n,Tf,Af,kn,zf,Pf,pr,Q,It,Ff,vn,Cf,Df,Sf,ne,Nf,jn,If,Mf,yn,Lf,Of,Bf,Mt,Wf,xn,Hf,Qf,fr,Xs,Rf,hr,Lt,ur,Pe,Uf,qn,Yf,Gf,dr,Ot,cr,Zs,Vf,mr,Bt,_r,oe,Fe,En,Wt,Jf,Tn,Kf,wr,Ce,Xf,ea,Zf,eh,gr,Ht,br,De,th,ta,sh,ah,$r,Qt,kr,Se,nh,sa,oh,rh,vr,Rt,jr,aa,lh,yr,Ut,xr,re,Ne,An,Yt,ih,zn,ph,qr,na,fh,Er,Gt,Tr,R,hh,Pn,uh,dh,Fn,ch,mh,Ar,Vt,zr,Ie,_h,oa,wh,gh,Pr,Jt,Fr,ra,bh,Cr,Kt,Dr,la,$h,Sr,Xt,Nr,Me,kh,Cn,vh,jh,Ir,Zt,Mr,ia,Lr,le,Le,Dn,es,yh,Sn,xh,Or,Oe,qh,ts,Eh,Th,Br,Be,Wr,ie,We,Nn,ss,Ah,In,zh,Hr,pa,Ph,Qr,as,Rr,fa,Fh,Ur,ns,Yr,pe,He,Mn,os,Ch,Ln,Dh,Gr,Qe,Sh,ha,Nh,Ih,Vr,rs,Jr,ua,Mh,Kr,U,N,Lh,On,Oh,Bh,Bn,Wh,Hh,Wn,Qh,Rh,Uh,ls,Yh,Hn,Gh,Vh,Jh,is,Kh,Qn,Xh,Zh,Xr,da,eu,Zr,ps,el,Re,tu,Rn,su,au,tl,fs,sl,ca,nu,al,hs,nl,fe,Ue,Un,us,ou,Yn,ru,ol,Ye,lu,ma,iu,pu,rl,ds,ll,Ge,fu,_a,hu,uu,il,cs,pl,Ve,du,wa,cu,mu,fl,ms,hl,ga,_u,ul,_s,dl,he,Je,Gn,ws,wu,Vn,gu,cl,ba,bu,ml,gs,_l,Y,$u,Jn,ku,vu,Kn,ju,yu,wl,bs,gl,$a,xu,bl,$s,$l,Ke,qu,ka,Eu,Tu,kl,ks,vl,va,Au,jl,vs,yl,Xe,zu,Xn,Pu,Fu,xl,js,ql;return v=new z({}),A=new wm({props:{classNames:"absolute z-10 right-0 top-0",options:[{label:"Mixed",value:"https://colab.research.google.com/github/huggingface/notebooks/blob/master/transformers_doc/custom_datasets.ipynb"},{label:"PyTorch",value:"https://colab.research.google.com/github/huggingface/notebooks/blob/master/transformers_doc/pytorch/custom_datasets.ipynb"},{label:"TensorFlow",value:"https://colab.research.google.com/github/huggingface/notebooks/blob/master/transformers_doc/tensorflow/custom_datasets.ipynb"},{label:"Mixed",value:"https://studiolab.sagemaker.aws/import/github/huggingface/notebooks/blob/master/transformers_doc/custom_datasets.ipynb"},{label:"PyTorch",value:"https://studiolab.sagemaker.aws/import/github/huggingface/notebooks/blob/master/transformers_doc/pytorch/custom_datasets.ipynb"},{label:"TensorFlow",value:"https://studiolab.sagemaker.aws/import/github/huggingface/notebooks/blob/master/transformers_doc/tensorflow/custom_datasets.ipynb"}]}}),at=new b({props:{code:"pip install datasets",highlighted:"pip install datasets"}}),nt=new z({}),me=new Su({props:{$$slots:{default:[gm]},$$scope:{ctx:V}}}),rt=new z({}),lt=new b({props:{code:`from datasets import load_dataset

imdb = load_dataset("imdb")`,highlighted:`<span class="hljs-keyword">from</span> datasets <span class="hljs-keyword">import</span> load_dataset

imdb = load_dataset(<span class="hljs-string">&quot;imdb&quot;</span>)`}}),it=new b({props:{code:`imdb["train"][0]
{
    "label": 1,
    "text": "Bromwell High is a cartoon comedy. It ran at the same time as some other programs about school life, such as \\"Teachers\\". My 35 years in the teaching profession lead me to believe that Bromwell High's satire is much closer to reality than is \\"Teachers\\". The scramble to survive financially, the insightful students who can see right through their pathetic teachers' pomp, the pettiness of the whole situation, all remind me of the schools I knew and their students. When I saw the episode in which a student repeatedly tried to burn down the school, I immediately recalled ......... at .......... High. A classic line: INSPECTOR: I'm here to sack one of your teachers. STUDENT: Welcome to Bromwell High. I expect that many adults of my age think that Bromwell High is far fetched. What a pity that it isn't!",
}`,highlighted:`imdb[<span class="hljs-string">&quot;train&quot;</span>][<span class="hljs-number">0</span>]
{
    <span class="hljs-string">&quot;label&quot;</span>: <span class="hljs-number">1</span>,
    <span class="hljs-string">&quot;text&quot;</span>: <span class="hljs-string">&quot;Bromwell High is a cartoon comedy. It ran at the same time as some other programs about school life, such as \\&quot;Teachers\\&quot;. My 35 years in the teaching profession lead me to believe that Bromwell High&#x27;s satire is much closer to reality than is \\&quot;Teachers\\&quot;. The scramble to survive financially, the insightful students who can see right through their pathetic teachers&#x27; pomp, the pettiness of the whole situation, all remind me of the schools I knew and their students. When I saw the episode in which a student repeatedly tried to burn down the school, I immediately recalled ......... at .......... High. A classic line: INSPECTOR: I&#x27;m here to sack one of your teachers. STUDENT: Welcome to Bromwell High. I expect that many adults of my age think that Bromwell High is far fetched. What a pity that it isn&#x27;t!&quot;</span>,
}`}}),pt=new z({}),ht=new b({props:{code:`from transformers import AutoTokenizer

tokenizer = AutoTokenizer.from_pretrained("distilbert-base-uncased")`,highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoTokenizer

tokenizer = AutoTokenizer.from_pretrained(<span class="hljs-string">&quot;distilbert-base-uncased&quot;</span>)`}}),ut=new b({props:{code:`def preprocess_function(examples):
    return tokenizer(examples["text"], truncation=True)`,highlighted:`<span class="hljs-keyword">def</span> <span class="hljs-title function_">preprocess_function</span>(<span class="hljs-params">examples</span>):
    <span class="hljs-keyword">return</span> tokenizer(examples[<span class="hljs-string">&quot;text&quot;</span>], truncation=<span class="hljs-literal">True</span>)`}}),dt=new b({props:{code:"tokenized_imdb = imdb.map(preprocess_function, batched=True)",highlighted:'tokenized_imdb = imdb.<span class="hljs-built_in">map</span>(preprocess_function, batched=<span class="hljs-literal">True</span>)'}}),ct=new b({props:{code:`from transformers import DataCollatorWithPadding

data_collator = DataCollatorWithPadding(tokenizer=tokenizer)`,highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> DataCollatorWithPadding

data_collator = DataCollatorWithPadding(tokenizer=tokenizer)`}}),mt=new z({}),_t=new b({props:{code:`from transformers import AutoModelForSequenceClassification

model = AutoModelForSequenceClassification.from_pretrained("distilbert-base-uncased", num_labels=2)`,highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoModelForSequenceClassification

model = AutoModelForSequenceClassification.from_pretrained(<span class="hljs-string">&quot;distilbert-base-uncased&quot;</span>, num_labels=<span class="hljs-number">2</span>)`}}),$t=new b({props:{code:`from transformers import TrainingArguments, Trainer

training_args = TrainingArguments(
    output_dir="./results",
    learning_rate=2e-5,
    per_device_train_batch_size=16,
    per_device_eval_batch_size=16,
    num_train_epochs=5,
    weight_decay=0.01,
)

trainer = Trainer(
    model=model,
    args=training_args,
    train_dataset=tokenized_imdb["train"],
    eval_dataset=tokenized_imdb["test"],
    tokenizer=tokenizer,
    data_collator=data_collator,
)

trainer.train()`,highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> TrainingArguments, Trainer

training_args = TrainingArguments(
    output_dir=<span class="hljs-string">&quot;./results&quot;</span>,
    learning_rate=<span class="hljs-number">2e-5</span>,
    per_device_train_batch_size=<span class="hljs-number">16</span>,
    per_device_eval_batch_size=<span class="hljs-number">16</span>,
    num_train_epochs=<span class="hljs-number">5</span>,
    weight_decay=<span class="hljs-number">0.01</span>,
)

trainer = Trainer(
    model=model,
    args=training_args,
    train_dataset=tokenized_imdb[<span class="hljs-string">&quot;train&quot;</span>],
    eval_dataset=tokenized_imdb[<span class="hljs-string">&quot;test&quot;</span>],
    tokenizer=tokenizer,
    data_collator=data_collator,
)

trainer.train()`}}),kt=new z({}),vt=new b({props:{code:`from transformers import DataCollatorWithPadding

data_collator = DataCollatorWithPadding(tokenizer, return_tensors="tf")`,highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> DataCollatorWithPadding

data_collator = DataCollatorWithPadding(tokenizer, return_tensors=<span class="hljs-string">&quot;tf&quot;</span>)`}}),jt=new b({props:{code:`tf_train_dataset = tokenized_imdb["train"].to_tf_dataset(
    columns=["attention_mask", "input_ids", "label"],
    shuffle=True,
    batch_size=16,
    collate_fn=data_collator,
)

tf_validation_dataset = tokenized_imdb["train"].to_tf_dataset(
    columns=["attention_mask", "input_ids", "label"],
    shuffle=False,
    batch_size=16,
    collate_fn=data_collator,
)`,highlighted:`tf_train_dataset = tokenized_imdb[<span class="hljs-string">&quot;train&quot;</span>].to_tf_dataset(
    columns=[<span class="hljs-string">&quot;attention_mask&quot;</span>, <span class="hljs-string">&quot;input_ids&quot;</span>, <span class="hljs-string">&quot;label&quot;</span>],
    shuffle=<span class="hljs-literal">True</span>,
    batch_size=<span class="hljs-number">16</span>,
    collate_fn=data_collator,
)

tf_validation_dataset = tokenized_imdb[<span class="hljs-string">&quot;train&quot;</span>].to_tf_dataset(
    columns=[<span class="hljs-string">&quot;attention_mask&quot;</span>, <span class="hljs-string">&quot;input_ids&quot;</span>, <span class="hljs-string">&quot;label&quot;</span>],
    shuffle=<span class="hljs-literal">False</span>,
    batch_size=<span class="hljs-number">16</span>,
    collate_fn=data_collator,
)`}}),yt=new b({props:{code:`from transformers import create_optimizer
import tensorflow as tf

batch_size = 16
num_epochs = 5
batches_per_epoch = len(tokenized_imdb["train"]) // batch_size
total_train_steps = int(batches_per_epoch * num_epochs)
optimizer, schedule = create_optimizer(init_lr=2e-5, num_warmup_steps=0, num_train_steps=total_train_steps)`,highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> create_optimizer
<span class="hljs-keyword">import</span> tensorflow <span class="hljs-keyword">as</span> tf

batch_size = <span class="hljs-number">16</span>
num_epochs = <span class="hljs-number">5</span>
batches_per_epoch = <span class="hljs-built_in">len</span>(tokenized_imdb[<span class="hljs-string">&quot;train&quot;</span>]) // batch_size
total_train_steps = <span class="hljs-built_in">int</span>(batches_per_epoch * num_epochs)
optimizer, schedule = create_optimizer(init_lr=<span class="hljs-number">2e-5</span>, num_warmup_steps=<span class="hljs-number">0</span>, num_train_steps=total_train_steps)`}}),xt=new b({props:{code:`from transformers import TFAutoModelForSequenceClassification

model = TFAutoModelForSequenceClassification.from_pretrained("distilbert-base-uncased", num_labels=2)`,highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> TFAutoModelForSequenceClassification

model = TFAutoModelForSequenceClassification.from_pretrained(<span class="hljs-string">&quot;distilbert-base-uncased&quot;</span>, num_labels=<span class="hljs-number">2</span>)`}}),qt=new b({props:{code:`import tensorflow as tf

model.compile(optimizer=optimizer)`,highlighted:`<span class="hljs-keyword">import</span> tensorflow <span class="hljs-keyword">as</span> tf

model.<span class="hljs-built_in">compile</span>(optimizer=optimizer)`}}),Et=new b({props:{code:`model.fit(
    tf_train_set,
    validation_data=tf_validation_set,
    epochs=num_train_epochs,
)`,highlighted:`model.fit(
    tf_train_set,
    validation_data=tf_validation_set,
    epochs=num_train_epochs,
)`}}),Tt=new z({}),qe=new Su({props:{$$slots:{default:[bm]},$$scope:{ctx:V}}}),zt=new z({}),Pt=new b({props:{code:`from datasets import load_dataset

wnut = load_dataset("wnut_17")`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> datasets <span class="hljs-keyword">import</span> load_dataset

<span class="hljs-meta">&gt;&gt;&gt; </span>wnut = load_dataset(<span class="hljs-string">&quot;wnut_17&quot;</span>)`}}),Ft=new b({props:{code:'wnut["train"][0]',highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>wnut[<span class="hljs-string">&quot;train&quot;</span>][<span class="hljs-number">0</span>]
{<span class="hljs-string">&#x27;id&#x27;</span>: <span class="hljs-string">&#x27;0&#x27;</span>,
 <span class="hljs-string">&#x27;ner_tags&#x27;</span>: [<span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">7</span>, <span class="hljs-number">8</span>, <span class="hljs-number">8</span>, <span class="hljs-number">0</span>, <span class="hljs-number">7</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>],
 <span class="hljs-string">&#x27;tokens&#x27;</span>: [<span class="hljs-string">&#x27;@paulwalk&#x27;</span>, <span class="hljs-string">&#x27;It&#x27;</span>, <span class="hljs-string">&quot;&#x27;s&quot;</span>, <span class="hljs-string">&#x27;the&#x27;</span>, <span class="hljs-string">&#x27;view&#x27;</span>, <span class="hljs-string">&#x27;from&#x27;</span>, <span class="hljs-string">&#x27;where&#x27;</span>, <span class="hljs-string">&#x27;I&#x27;</span>, <span class="hljs-string">&quot;&#x27;m&quot;</span>, <span class="hljs-string">&#x27;living&#x27;</span>, <span class="hljs-string">&#x27;for&#x27;</span>, <span class="hljs-string">&#x27;two&#x27;</span>, <span class="hljs-string">&#x27;weeks&#x27;</span>, <span class="hljs-string">&#x27;.&#x27;</span>, <span class="hljs-string">&#x27;Empire&#x27;</span>, <span class="hljs-string">&#x27;State&#x27;</span>, <span class="hljs-string">&#x27;Building&#x27;</span>, <span class="hljs-string">&#x27;=&#x27;</span>, <span class="hljs-string">&#x27;ESB&#x27;</span>, <span class="hljs-string">&#x27;.&#x27;</span>, <span class="hljs-string">&#x27;Pretty&#x27;</span>, <span class="hljs-string">&#x27;bad&#x27;</span>, <span class="hljs-string">&#x27;storm&#x27;</span>, <span class="hljs-string">&#x27;here&#x27;</span>, <span class="hljs-string">&#x27;last&#x27;</span>, <span class="hljs-string">&#x27;evening&#x27;</span>, <span class="hljs-string">&#x27;.&#x27;</span>]
}`}}),Ct=new b({props:{code:`label_list = wnut["train"].features[f"ner_tags"].feature.names
label_list`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>label_list = wnut[<span class="hljs-string">&quot;train&quot;</span>].features[<span class="hljs-string">f&quot;ner_tags&quot;</span>].feature.names
<span class="hljs-meta">&gt;&gt;&gt; </span>label_list
[
    <span class="hljs-string">&quot;O&quot;</span>,
    <span class="hljs-string">&quot;B-corporation&quot;</span>,
    <span class="hljs-string">&quot;I-corporation&quot;</span>,
    <span class="hljs-string">&quot;B-creative-work&quot;</span>,
    <span class="hljs-string">&quot;I-creative-work&quot;</span>,
    <span class="hljs-string">&quot;B-group&quot;</span>,
    <span class="hljs-string">&quot;I-group&quot;</span>,
    <span class="hljs-string">&quot;B-location&quot;</span>,
    <span class="hljs-string">&quot;I-location&quot;</span>,
    <span class="hljs-string">&quot;B-person&quot;</span>,
    <span class="hljs-string">&quot;I-person&quot;</span>,
    <span class="hljs-string">&quot;B-product&quot;</span>,
    <span class="hljs-string">&quot;I-product&quot;</span>,
]`}}),Dt=new z({}),St=new b({props:{code:`from transformers import AutoTokenizer

tokenizer = AutoTokenizer.from_pretrained("distilbert-base-uncased")`,highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoTokenizer

tokenizer = AutoTokenizer.from_pretrained(<span class="hljs-string">&quot;distilbert-base-uncased&quot;</span>)`}}),Nt=new b({props:{code:`tokenized_input = tokenizer(example["tokens"], is_split_into_words=True)
tokens = tokenizer.convert_ids_to_tokens(tokenized_input["input_ids"])
tokens`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>tokenized_input = tokenizer(example[<span class="hljs-string">&quot;tokens&quot;</span>], is_split_into_words=<span class="hljs-literal">True</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>tokens = tokenizer.convert_ids_to_tokens(tokenized_input[<span class="hljs-string">&quot;input_ids&quot;</span>])
<span class="hljs-meta">&gt;&gt;&gt; </span>tokens
[<span class="hljs-string">&#x27;[CLS]&#x27;</span>, <span class="hljs-string">&#x27;@&#x27;</span>, <span class="hljs-string">&#x27;paul&#x27;</span>, <span class="hljs-string">&#x27;##walk&#x27;</span>, <span class="hljs-string">&#x27;it&#x27;</span>, <span class="hljs-string">&quot;&#x27;&quot;</span>, <span class="hljs-string">&#x27;s&#x27;</span>, <span class="hljs-string">&#x27;the&#x27;</span>, <span class="hljs-string">&#x27;view&#x27;</span>, <span class="hljs-string">&#x27;from&#x27;</span>, <span class="hljs-string">&#x27;where&#x27;</span>, <span class="hljs-string">&#x27;i&#x27;</span>, <span class="hljs-string">&quot;&#x27;&quot;</span>, <span class="hljs-string">&#x27;m&#x27;</span>, <span class="hljs-string">&#x27;living&#x27;</span>, <span class="hljs-string">&#x27;for&#x27;</span>, <span class="hljs-string">&#x27;two&#x27;</span>, <span class="hljs-string">&#x27;weeks&#x27;</span>, <span class="hljs-string">&#x27;.&#x27;</span>, <span class="hljs-string">&#x27;empire&#x27;</span>, <span class="hljs-string">&#x27;state&#x27;</span>, <span class="hljs-string">&#x27;building&#x27;</span>, <span class="hljs-string">&#x27;=&#x27;</span>, <span class="hljs-string">&#x27;es&#x27;</span>, <span class="hljs-string">&#x27;##b&#x27;</span>, <span class="hljs-string">&#x27;.&#x27;</span>, <span class="hljs-string">&#x27;pretty&#x27;</span>, <span class="hljs-string">&#x27;bad&#x27;</span>, <span class="hljs-string">&#x27;storm&#x27;</span>, <span class="hljs-string">&#x27;here&#x27;</span>, <span class="hljs-string">&#x27;last&#x27;</span>, <span class="hljs-string">&#x27;evening&#x27;</span>, <span class="hljs-string">&#x27;.&#x27;</span>, <span class="hljs-string">&#x27;[SEP]&#x27;</span>]`}}),Lt=new b({props:{code:`def tokenize_and_align_labels(examples):
    tokenized_inputs = tokenizer(examples["tokens"], truncation=True, is_split_into_words=True)

    labels = []
    for i, label in enumerate(examples[f"ner_tags"]):
        word_ids = tokenized_inputs.word_ids(batch_index=i)  # Map tokens to their respective word.
        previous_word_idx = None
        label_ids = []
        for word_idx in word_ids:  # Set the special tokens to -100.
            if word_idx is None:
                label_ids.append(-100)
            elif word_idx != previous_word_idx:  # Only label the first token of a given word.
                label_ids.append(label[word_idx])
            else:
                label_ids.append(-100)
            previous_word_idx = word_idx
        labels.append(label_ids)

    tokenized_inputs["labels"] = labels
    return tokenized_inputs`,highlighted:`<span class="hljs-keyword">def</span> <span class="hljs-title function_">tokenize_and_align_labels</span>(<span class="hljs-params">examples</span>):
    tokenized_inputs = tokenizer(examples[<span class="hljs-string">&quot;tokens&quot;</span>], truncation=<span class="hljs-literal">True</span>, is_split_into_words=<span class="hljs-literal">True</span>)

    labels = []
    <span class="hljs-keyword">for</span> i, label <span class="hljs-keyword">in</span> <span class="hljs-built_in">enumerate</span>(examples[<span class="hljs-string">f&quot;ner_tags&quot;</span>]):
        word_ids = tokenized_inputs.word_ids(batch_index=i)  <span class="hljs-comment"># Map tokens to their respective word.</span>
        previous_word_idx = <span class="hljs-literal">None</span>
        label_ids = []
        <span class="hljs-keyword">for</span> word_idx <span class="hljs-keyword">in</span> word_ids:  <span class="hljs-comment"># Set the special tokens to -100.</span>
            <span class="hljs-keyword">if</span> word_idx <span class="hljs-keyword">is</span> <span class="hljs-literal">None</span>:
                label_ids.append(-<span class="hljs-number">100</span>)
            <span class="hljs-keyword">elif</span> word_idx != previous_word_idx:  <span class="hljs-comment"># Only label the first token of a given word.</span>
                label_ids.append(label[word_idx])
            <span class="hljs-keyword">else</span>:
                label_ids.append(-<span class="hljs-number">100</span>)
            previous_word_idx = word_idx
        labels.append(label_ids)

    tokenized_inputs[<span class="hljs-string">&quot;labels&quot;</span>] = labels
    <span class="hljs-keyword">return</span> tokenized_inputs`}}),Ot=new b({props:{code:"tokenized_wnut = wnut.map(tokenize_and_align_labels, batched=True)",highlighted:'tokenized_wnut = wnut.<span class="hljs-built_in">map</span>(tokenize_and_align_labels, batched=<span class="hljs-literal">True</span>)'}}),Bt=new b({props:{code:`from transformers import DataCollatorForTokenClassification

data_collator = DataCollatorForTokenClassification(tokenizer)`,highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> DataCollatorForTokenClassification

data_collator = DataCollatorForTokenClassification(tokenizer)`}}),Wt=new z({}),Ht=new b({props:{code:`from transformers import AutoModelForTokenClassification, TrainingArguments, Trainer

model = AutoModelForTokenClassification.from_pretrained("distilbert-base-uncased", num_labels=len(label_list))`,highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoModelForTokenClassification, TrainingArguments, Trainer

model = AutoModelForTokenClassification.from_pretrained(<span class="hljs-string">&quot;distilbert-base-uncased&quot;</span>, num_labels=<span class="hljs-built_in">len</span>(label_list))`}}),Qt=new b({props:{code:`training_args = TrainingArguments(
    output_dir="./results",
    evaluation_strategy="epoch",
    learning_rate=2e-5,
    per_device_train_batch_size=16,
    per_device_eval_batch_size=16,
    num_train_epochs=3,
    weight_decay=0.01,
)`,highlighted:`training_args = TrainingArguments(
    output_dir=<span class="hljs-string">&quot;./results&quot;</span>,
    evaluation_strategy=<span class="hljs-string">&quot;epoch&quot;</span>,
    learning_rate=<span class="hljs-number">2e-5</span>,
    per_device_train_batch_size=<span class="hljs-number">16</span>,
    per_device_eval_batch_size=<span class="hljs-number">16</span>,
    num_train_epochs=<span class="hljs-number">3</span>,
    weight_decay=<span class="hljs-number">0.01</span>,
)`}}),Rt=new b({props:{code:`trainer = Trainer(
    model=model,
    args=training_args,
    train_dataset=tokenized_wnut["train"],
    eval_dataset=tokenized_wnut["test"],
    data_collator=data_collator,
    tokenizer=tokenizer,
)`,highlighted:`trainer = Trainer(
    model=model,
    args=training_args,
    train_dataset=tokenized_wnut[<span class="hljs-string">&quot;train&quot;</span>],
    eval_dataset=tokenized_wnut[<span class="hljs-string">&quot;test&quot;</span>],
    data_collator=data_collator,
    tokenizer=tokenizer,
)`}}),Ut=new b({props:{code:"trainer.train()",highlighted:"trainer.train()"}}),Yt=new z({}),Gt=new b({props:{code:`from transformers import DataCollatorForTokenClassification

data_collator = DataCollatorForTokenClassification(tokenizer, return_tensors="tf")`,highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> DataCollatorForTokenClassification

data_collator = DataCollatorForTokenClassification(tokenizer, return_tensors=<span class="hljs-string">&quot;tf&quot;</span>)`}}),Vt=new b({props:{code:`tf_train_set = tokenized_wnut["train"].to_tf_dataset(
    columns=["attention_mask", "input_ids", "labels"],
    shuffle=True,
    batch_size=16,
    collate_fn=data_collator,
)

tf_validation_set = tokenized_wnut["validation"].to_tf_dataset(
    columns=["attention_mask", "input_ids", "labels"],
    shuffle=False,
    batch_size=16,
    collate_fn=data_collator,
)`,highlighted:`tf_train_set = tokenized_wnut[<span class="hljs-string">&quot;train&quot;</span>].to_tf_dataset(
    columns=[<span class="hljs-string">&quot;attention_mask&quot;</span>, <span class="hljs-string">&quot;input_ids&quot;</span>, <span class="hljs-string">&quot;labels&quot;</span>],
    shuffle=<span class="hljs-literal">True</span>,
    batch_size=<span class="hljs-number">16</span>,
    collate_fn=data_collator,
)

tf_validation_set = tokenized_wnut[<span class="hljs-string">&quot;validation&quot;</span>].to_tf_dataset(
    columns=[<span class="hljs-string">&quot;attention_mask&quot;</span>, <span class="hljs-string">&quot;input_ids&quot;</span>, <span class="hljs-string">&quot;labels&quot;</span>],
    shuffle=<span class="hljs-literal">False</span>,
    batch_size=<span class="hljs-number">16</span>,
    collate_fn=data_collator,
)`}}),Jt=new b({props:{code:`from transformers import TFAutoModelForTokenClassification

model = TFAutoModelForTokenClassification.from_pretrained("distilbert-base-uncased", num_labels=len(label_list))`,highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> TFAutoModelForTokenClassification

model = TFAutoModelForTokenClassification.from_pretrained(<span class="hljs-string">&quot;distilbert-base-uncased&quot;</span>, num_labels=<span class="hljs-built_in">len</span>(label_list))`}}),Kt=new b({props:{code:`from transformers import create_optimizer

batch_size = 16
num_train_epochs = 3
num_train_steps = (len(tokenized_datasets["train"]) // batch_size) * num_train_epochs
optimizer, lr_schedule = create_optimizer(
    init_lr=2e-5,
    num_train_steps=num_train_steps,
    weight_decay_rate=0.01,
    num_warmup_steps=0,
)`,highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> create_optimizer

batch_size = <span class="hljs-number">16</span>
num_train_epochs = <span class="hljs-number">3</span>
num_train_steps = (<span class="hljs-built_in">len</span>(tokenized_datasets[<span class="hljs-string">&quot;train&quot;</span>]) // batch_size) * num_train_epochs
optimizer, lr_schedule = create_optimizer(
    init_lr=<span class="hljs-number">2e-5</span>,
    num_train_steps=num_train_steps,
    weight_decay_rate=<span class="hljs-number">0.01</span>,
    num_warmup_steps=<span class="hljs-number">0</span>,
)`}}),Xt=new b({props:{code:`import tensorflow as tf

model.compile(optimizer=optimizer)`,highlighted:`<span class="hljs-keyword">import</span> tensorflow <span class="hljs-keyword">as</span> tf

model.<span class="hljs-built_in">compile</span>(optimizer=optimizer)`}}),Zt=new b({props:{code:`model.fit(
    tf_train_set,
    validation_data=tf_validation_set,
    epochs=num_train_epochs,
)`,highlighted:`model.fit(
    tf_train_set,
    validation_data=tf_validation_set,
    epochs=num_train_epochs,
)`}}),es=new z({}),Be=new Su({props:{$$slots:{default:[$m]},$$scope:{ctx:V}}}),ss=new z({}),as=new b({props:{code:`from datasets import load_dataset

squad = load_dataset("squad")`,highlighted:`<span class="hljs-keyword">from</span> datasets <span class="hljs-keyword">import</span> load_dataset

squad = load_dataset(<span class="hljs-string">&quot;squad&quot;</span>)`}}),ns=new b({props:{code:'squad["train"][0]',highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>squad[<span class="hljs-string">&quot;train&quot;</span>][<span class="hljs-number">0</span>]
{<span class="hljs-string">&#x27;answers&#x27;</span>: {<span class="hljs-string">&#x27;answer_start&#x27;</span>: [<span class="hljs-number">515</span>], <span class="hljs-string">&#x27;text&#x27;</span>: [<span class="hljs-string">&#x27;Saint Bernadette Soubirous&#x27;</span>]},
 <span class="hljs-string">&#x27;context&#x27;</span>: <span class="hljs-string">&#x27;Architecturally, the school has a Catholic character. Atop the Main Building\\&#x27;s gold dome is a golden statue of the Virgin Mary. Immediately in front of the Main Building and facing it, is a copper statue of Christ with arms upraised with the legend &quot;Venite Ad Me Omnes&quot;. Next to the Main Building is the Basilica of the Sacred Heart. Immediately behind the basilica is the Grotto, a Marian place of prayer and reflection. It is a replica of the grotto at Lourdes, France where the Virgin Mary reputedly appeared to Saint Bernadette Soubirous in 1858. At the end of the main drive (and in a direct line that connects through 3 statues and the Gold Dome), is a simple, modern stone statue of Mary.&#x27;</span>,
 <span class="hljs-string">&#x27;id&#x27;</span>: <span class="hljs-string">&#x27;5733be284776f41900661182&#x27;</span>,
 <span class="hljs-string">&#x27;question&#x27;</span>: <span class="hljs-string">&#x27;To whom did the Virgin Mary allegedly appear in 1858 in Lourdes France?&#x27;</span>,
 <span class="hljs-string">&#x27;title&#x27;</span>: <span class="hljs-string">&#x27;University_of_Notre_Dame&#x27;</span>
}`}}),os=new z({}),rs=new b({props:{code:`from transformers import AutoTokenizer

tokenizer = AutoTokenizer.from_pretrained("distilbert-base-uncased")`,highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoTokenizer

tokenizer = AutoTokenizer.from_pretrained(<span class="hljs-string">&quot;distilbert-base-uncased&quot;</span>)`}}),ps=new b({props:{code:`def preprocess_function(examples):
    questions = [q.strip() for q in examples["question"]]
    inputs = tokenizer(
        questions,
        examples["context"],
        max_length=384,
        truncation="only_second",
        return_offsets_mapping=True,
        padding="max_length",
    )

    offset_mapping = inputs.pop("offset_mapping")
    answers = examples["answers"]
    start_positions = []
    end_positions = []

    for i, offset in enumerate(offset_mapping):
        answer = answers[i]
        start_char = answer["answer_start"][0]
        end_char = answer["answer_start"][0] + len(answer["text"][0])
        sequence_ids = inputs.sequence_ids(i)

        # Find the start and end of the context
        idx = 0
        while sequence_ids[idx] != 1:
            idx += 1
        context_start = idx
        while sequence_ids[idx] == 1:
            idx += 1
        context_end = idx - 1

        # If the answer is not fully inside the context, label it (0, 0)
        if offset[context_start][0] > end_char or offset[context_end][1] < start_char:
            start_positions.append(0)
            end_positions.append(0)
        else:
            # Otherwise it's the start and end token positions
            idx = context_start
            while idx <= context_end and offset[idx][0] <= start_char:
                idx += 1
            start_positions.append(idx - 1)

            idx = context_end
            while idx >= context_start and offset[idx][1] >= end_char:
                idx -= 1
            end_positions.append(idx + 1)

    inputs["start_positions"] = start_positions
    inputs["end_positions"] = end_positions
    return inputs`,highlighted:`<span class="hljs-keyword">def</span> <span class="hljs-title function_">preprocess_function</span>(<span class="hljs-params">examples</span>):
    questions = [q.strip() <span class="hljs-keyword">for</span> q <span class="hljs-keyword">in</span> examples[<span class="hljs-string">&quot;question&quot;</span>]]
    inputs = tokenizer(
        questions,
        examples[<span class="hljs-string">&quot;context&quot;</span>],
        max_length=<span class="hljs-number">384</span>,
        truncation=<span class="hljs-string">&quot;only_second&quot;</span>,
        return_offsets_mapping=<span class="hljs-literal">True</span>,
        padding=<span class="hljs-string">&quot;max_length&quot;</span>,
    )

    offset_mapping = inputs.pop(<span class="hljs-string">&quot;offset_mapping&quot;</span>)
    answers = examples[<span class="hljs-string">&quot;answers&quot;</span>]
    start_positions = []
    end_positions = []

    <span class="hljs-keyword">for</span> i, offset <span class="hljs-keyword">in</span> <span class="hljs-built_in">enumerate</span>(offset_mapping):
        answer = answers[i]
        start_char = answer[<span class="hljs-string">&quot;answer_start&quot;</span>][<span class="hljs-number">0</span>]
        end_char = answer[<span class="hljs-string">&quot;answer_start&quot;</span>][<span class="hljs-number">0</span>] + <span class="hljs-built_in">len</span>(answer[<span class="hljs-string">&quot;text&quot;</span>][<span class="hljs-number">0</span>])
        sequence_ids = inputs.sequence_ids(i)

        <span class="hljs-comment"># Find the start and end of the context</span>
        idx = <span class="hljs-number">0</span>
        <span class="hljs-keyword">while</span> sequence_ids[idx] != <span class="hljs-number">1</span>:
            idx += <span class="hljs-number">1</span>
        context_start = idx
        <span class="hljs-keyword">while</span> sequence_ids[idx] == <span class="hljs-number">1</span>:
            idx += <span class="hljs-number">1</span>
        context_end = idx - <span class="hljs-number">1</span>

        <span class="hljs-comment"># If the answer is not fully inside the context, label it (0, 0)</span>
        <span class="hljs-keyword">if</span> offset[context_start][<span class="hljs-number">0</span>] &gt; end_char <span class="hljs-keyword">or</span> offset[context_end][<span class="hljs-number">1</span>] &lt; start_char:
            start_positions.append(<span class="hljs-number">0</span>)
            end_positions.append(<span class="hljs-number">0</span>)
        <span class="hljs-keyword">else</span>:
            <span class="hljs-comment"># Otherwise it&#x27;s the start and end token positions</span>
            idx = context_start
            <span class="hljs-keyword">while</span> idx &lt;= context_end <span class="hljs-keyword">and</span> offset[idx][<span class="hljs-number">0</span>] &lt;= start_char:
                idx += <span class="hljs-number">1</span>
            start_positions.append(idx - <span class="hljs-number">1</span>)

            idx = context_end
            <span class="hljs-keyword">while</span> idx &gt;= context_start <span class="hljs-keyword">and</span> offset[idx][<span class="hljs-number">1</span>] &gt;= end_char:
                idx -= <span class="hljs-number">1</span>
            end_positions.append(idx + <span class="hljs-number">1</span>)

    inputs[<span class="hljs-string">&quot;start_positions&quot;</span>] = start_positions
    inputs[<span class="hljs-string">&quot;end_positions&quot;</span>] = end_positions
    <span class="hljs-keyword">return</span> inputs`}}),fs=new b({props:{code:'tokenized_squad = squad.map(preprocess_function, batched=True, remove_columns=squad["train"].column_names)',highlighted:'tokenized_squad = squad.<span class="hljs-built_in">map</span>(preprocess_function, batched=<span class="hljs-literal">True</span>, remove_columns=squad[<span class="hljs-string">&quot;train&quot;</span>].column_names)'}}),hs=new b({props:{code:`from transformers import default_data_collator

data_collator = default_data_collator`,highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> default_data_collator

data_collator = default_data_collator`}}),us=new z({}),ds=new b({props:{code:`from transformers import AutoModelForQuestionAnswering, TrainingArguments, Trainer

model = AutoModelForQuestionAnswering.from_pretrained("distilbert-base-uncased")`,highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoModelForQuestionAnswering, TrainingArguments, Trainer

model = AutoModelForQuestionAnswering.from_pretrained(<span class="hljs-string">&quot;distilbert-base-uncased&quot;</span>)`}}),cs=new b({props:{code:`training_args = TrainingArguments(
    output_dir="./results",
    evaluation_strategy="epoch",
    learning_rate=2e-5,
    per_device_train_batch_size=16,
    per_device_eval_batch_size=16,
    num_train_epochs=3,
    weight_decay=0.01,
)`,highlighted:`training_args = TrainingArguments(
    output_dir=<span class="hljs-string">&quot;./results&quot;</span>,
    evaluation_strategy=<span class="hljs-string">&quot;epoch&quot;</span>,
    learning_rate=<span class="hljs-number">2e-5</span>,
    per_device_train_batch_size=<span class="hljs-number">16</span>,
    per_device_eval_batch_size=<span class="hljs-number">16</span>,
    num_train_epochs=<span class="hljs-number">3</span>,
    weight_decay=<span class="hljs-number">0.01</span>,
)`}}),ms=new b({props:{code:`trainer = Trainer(
    model=model,
    args=training_args,
    train_dataset=tokenized_squad["train"],
    eval_dataset=tokenized_squad["validation"],
    data_collator=data_collator,
    tokenizer=tokenizer,
)`,highlighted:`trainer = Trainer(
    model=model,
    args=training_args,
    train_dataset=tokenized_squad[<span class="hljs-string">&quot;train&quot;</span>],
    eval_dataset=tokenized_squad[<span class="hljs-string">&quot;validation&quot;</span>],
    data_collator=data_collator,
    tokenizer=tokenizer,
)`}}),_s=new b({props:{code:"trainer.train()",highlighted:"trainer.train()"}}),ws=new z({}),gs=new b({props:{code:`from transformers.data.data_collator import tf_default_collator

data_collator = tf_default_collator`,highlighted:`<span class="hljs-keyword">from</span> transformers.data.data_collator <span class="hljs-keyword">import</span> tf_default_collator

data_collator = tf_default_collator`}}),bs=new b({props:{code:`tf_train_set = tokenized_squad["train"].to_tf_dataset(
    columns=["attention_mask", "input_ids", "start_positions", "end_positions"],
    dummy_labels=True,
    shuffle=True,
    batch_size=16,
    collate_fn=data_collator,
)

tf_validation_set = tokenized_squad["validation"].to_tf_dataset(
    columns=["attention_mask", "input_ids", "start_positions", "end_positions"],
    dummy_labels=True,
    shuffle=False,
    batch_size=16,
    collate_fn=data_collator,
)`,highlighted:`tf_train_set = tokenized_squad[<span class="hljs-string">&quot;train&quot;</span>].to_tf_dataset(
    columns=[<span class="hljs-string">&quot;attention_mask&quot;</span>, <span class="hljs-string">&quot;input_ids&quot;</span>, <span class="hljs-string">&quot;start_positions&quot;</span>, <span class="hljs-string">&quot;end_positions&quot;</span>],
    dummy_labels=<span class="hljs-literal">True</span>,
    shuffle=<span class="hljs-literal">True</span>,
    batch_size=<span class="hljs-number">16</span>,
    collate_fn=data_collator,
)

tf_validation_set = tokenized_squad[<span class="hljs-string">&quot;validation&quot;</span>].to_tf_dataset(
    columns=[<span class="hljs-string">&quot;attention_mask&quot;</span>, <span class="hljs-string">&quot;input_ids&quot;</span>, <span class="hljs-string">&quot;start_positions&quot;</span>, <span class="hljs-string">&quot;end_positions&quot;</span>],
    dummy_labels=<span class="hljs-literal">True</span>,
    shuffle=<span class="hljs-literal">False</span>,
    batch_size=<span class="hljs-number">16</span>,
    collate_fn=data_collator,
)`}}),$s=new b({props:{code:`from transformers import create_optimizer

batch_size = 16
num_epochs = 2
total_train_steps = (len(tokenized_squad["train"]) // batch_size) * num_epochs
optimizer, schedule = create_optimizer(
    init_lr=2e-5,
    num_warmup_steps=0,
    num_train_steps=total_train_steps,
)`,highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> create_optimizer

batch_size = <span class="hljs-number">16</span>
num_epochs = <span class="hljs-number">2</span>
total_train_steps = (<span class="hljs-built_in">len</span>(tokenized_squad[<span class="hljs-string">&quot;train&quot;</span>]) // batch_size) * num_epochs
optimizer, schedule = create_optimizer(
    init_lr=<span class="hljs-number">2e-5</span>,
    num_warmup_steps=<span class="hljs-number">0</span>,
    num_train_steps=total_train_steps,
)`}}),ks=new b({props:{code:`from transformers import TFAutoModelForQuestionAnswering

model = TFAutoModelForQuestionAnswering("distilbert-base-uncased")`,highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> TFAutoModelForQuestionAnswering

model = TFAutoModelForQuestionAnswering(<span class="hljs-string">&quot;distilbert-base-uncased&quot;</span>)`}}),vs=new b({props:{code:`import tensorflow as tf

model.compile(optimizer=optimizer)`,highlighted:`<span class="hljs-keyword">import</span> tensorflow <span class="hljs-keyword">as</span> tf

model.<span class="hljs-built_in">compile</span>(optimizer=optimizer)`}}),js=new b({props:{code:`model.fit(
    tf_train_set,
    validation_data=tf_validation_set,
    epochs=num_train_epochs,
)`,highlighted:`model.fit(
    tf_train_set,
    validation_data=tf_validation_set,
    epochs=num_train_epochs,
)`}}),{c(){$=l("meta"),q=f(),k=l("h1"),y=l("a"),E=l("span"),d(v.$$.fragment),P=f(),T=l("span"),x=n("How to fine-tune a model for common downstream tasks"),j=f(),d(A.$$.fragment),C=f(),qs=l("p"),gi=n(`This guide will show you how to fine-tune \u{1F917} Transformers models for common downstream tasks. You will use the \u{1F917}
Datasets library to quickly load and preprocess the datasets, getting them ready for training with PyTorch and
TensorFlow.`),to=f(),ue=l("p"),bi=n(`Before you begin, make sure you have the \u{1F917} Datasets library installed. For more detailed installation instructions,
refer to the \u{1F917} Datasets `),st=l("a"),$i=n("installation page"),ki=n(`. All of the
examples in this guide will use \u{1F917} Datasets to load and preprocess a dataset.`),so=f(),d(at.$$.fragment),ao=f(),Es=l("p"),vi=n("Learn how to fine-tune a model for:"),no=f(),I=l("ul"),Da=l("li"),Ts=l("a"),ji=n("seq_imdb"),yi=f(),Sa=l("li"),As=l("a"),xi=n("tok_ner"),qi=f(),Na=l("li"),zs=l("a"),Ei=n("qa_squad"),oo=f(),Ps=l("a"),ro=f(),J=l("h2"),de=l("a"),Ia=l("span"),d(nt.$$.fragment),Ti=f(),Ma=l("span"),Ai=n("Sequence classification with IMDb reviews"),lo=f(),ce=l("p"),zi=n(`Sequence classification refers to the task of classifying sequences of text according to a given number of classes. In
this example, learn how to fine-tune a model on the `),ot=l("a"),Pi=n("IMDb dataset"),Fi=n(` to determine
whether a review is positive or negative.`),io=f(),d(me.$$.fragment),po=f(),K=l("h3"),_e=l("a"),La=l("span"),d(rt.$$.fragment),Ci=f(),Oa=l("span"),Di=n("Load IMDb dataset"),fo=f(),Fs=l("p"),Si=n("The \u{1F917} Datasets library makes it simple to load a dataset:"),ho=f(),d(lt.$$.fragment),uo=f(),we=l("p"),Ni=n("This loads a "),Ba=l("code"),Ii=n("DatasetDict"),Mi=n(" object which you can index into to view an example:"),co=f(),d(it.$$.fragment),mo=f(),X=l("h3"),ge=l("a"),Wa=l("span"),d(pt.$$.fragment),Li=f(),Ha=l("span"),Oi=n("Preprocess"),_o=f(),M=l("p"),Bi=n(`The next step is to tokenize the text into a readable format by the model. It is important to load the same tokenizer a
model was trained with to ensure appropriately tokenized words. Load the DistilBERT tokenizer with the
`),Cs=l("a"),Wi=n("AutoTokenizer"),Hi=n(" because we will eventually train a classifier using a pretrained "),ft=l("a"),Qi=n("DistilBERT"),Ri=n(" model:"),wo=f(),d(ht.$$.fragment),go=f(),Ds=l("p"),Ui=n(`Now that you have instantiated a tokenizer, create a function that will tokenize the text. You should also truncate
longer sequences in the text to be no longer than the model\u2019s maximum input length:`),bo=f(),d(ut.$$.fragment),$o=f(),L=l("p"),Yi=n("Use \u{1F917} Datasets "),Qa=l("code"),Gi=n("map"),Vi=n(` function to apply the preprocessing function to the entire dataset. You can also set
`),Ra=l("code"),Ji=n("batched=True"),Ki=n(` to apply the preprocessing function to multiple elements of the dataset at once for faster
preprocessing:`),ko=f(),d(dt.$$.fragment),vo=f(),F=l("p"),Xi=n("Lastly, pad your text so they are a uniform length. While it is possible to pad your text in the "),Ua=l("code"),Zi=n("tokenizer"),ep=n(` function
by setting `),Ya=l("code"),tp=n("padding=True"),sp=n(`, it is more efficient to only pad the text to the length of the longest element in its
batch. This is known as `),Ga=l("strong"),ap=n("dynamic padding"),np=n(". You can do this with the "),Va=l("code"),op=n("DataCollatorWithPadding"),rp=n(" function:"),jo=f(),d(ct.$$.fragment),yo=f(),Z=l("h3"),be=l("a"),Ja=l("span"),d(mt.$$.fragment),lp=f(),Ka=l("span"),ip=n("Fine-tune with the Trainer API"),xo=f(),$e=l("p"),pp=n("Now load your model with the "),Ss=l("a"),fp=n("AutoModelForSequenceClassification"),hp=n(" class along with the number of expected labels:"),qo=f(),d(_t.$$.fragment),Eo=f(),Ns=l("p"),up=n("At this point, only three steps remain:"),To=f(),O=l("ol"),wt=l("li"),dp=n("Define your training hyperparameters in "),Is=l("a"),cp=n("TrainingArguments"),mp=n("."),_p=f(),gt=l("li"),wp=n("Pass the training arguments to a "),Ms=l("a"),gp=n("Trainer"),bp=n(" along with the model, dataset, tokenizer, and data collator."),$p=f(),bt=l("li"),kp=n("Call "),Xa=l("code"),vp=n("Trainer.train()"),jp=n(" to fine-tune your model."),Ao=f(),d($t.$$.fragment),zo=f(),ee=l("h3"),ke=l("a"),Za=l("span"),d(kt.$$.fragment),yp=f(),en=l("span"),xp=n("Fine-tune with TensorFlow"),Po=f(),Ls=l("p"),qp=n("Fine-tuning with TensorFlow is just as easy, with only a few differences."),Fo=f(),D=l("p"),Ep=n("Start by batching the processed examples together with dynamic padding using the "),Os=l("a"),Tp=n("DataCollatorWithPadding"),Ap=n(` function.
Make sure you set `),tn=l("code"),zp=n('return_tensors="tf"'),Pp=n(" to return "),sn=l("code"),Fp=n("tf.Tensor"),Cp=n(" outputs instead of PyTorch tensors!"),Co=f(),d(vt.$$.fragment),Do=f(),S=l("p"),Dp=n("Next, convert your datasets to the "),an=l("code"),Sp=n("tf.data.Dataset"),Np=n(" format with "),nn=l("code"),Ip=n("to_tf_dataset"),Mp=n(`. Specify inputs and labels in the
`),on=l("code"),Lp=n("columns"),Op=n(" argument:"),So=f(),d(jt.$$.fragment),No=f(),Bs=l("p"),Bp=n("Set up an optimizer function, learning rate schedule, and some training hyperparameters:"),Io=f(),d(yt.$$.fragment),Mo=f(),ve=l("p"),Wp=n("Load your model with the "),Ws=l("a"),Hp=n("TFAutoModelForSequenceClassification"),Qp=n(" class along with the number of expected labels:"),Lo=f(),d(xt.$$.fragment),Oo=f(),Hs=l("p"),Rp=n("Compile the model:"),Bo=f(),d(qt.$$.fragment),Wo=f(),je=l("p"),Up=n("Finally, fine-tune the model by calling "),rn=l("code"),Yp=n("model.fit"),Gp=n(":"),Ho=f(),d(Et.$$.fragment),Qo=f(),Qs=l("a"),Ro=f(),te=l("h2"),ye=l("a"),ln=l("span"),d(Tt.$$.fragment),Vp=f(),pn=l("span"),Jp=n("Token classification with WNUT emerging entities"),Uo=f(),xe=l("p"),Kp=n(`Token classification refers to the task of classifying individual tokens in a sentence. One of the most common token
classification tasks is Named Entity Recognition (NER). NER attempts to find a label for each entity in a sentence,
such as a person, location, or organization. In this example, learn how to fine-tune a model on the `),At=l("a"),Xp=n("WNUT 17"),Zp=n(" dataset to detect new entities."),Yo=f(),d(qe.$$.fragment),Go=f(),se=l("h3"),Ee=l("a"),fn=l("span"),d(zt.$$.fragment),ef=f(),hn=l("span"),tf=n("Load WNUT 17 dataset"),Vo=f(),Rs=l("p"),sf=n("Load the WNUT 17 dataset from the \u{1F917} Datasets library:"),Jo=f(),d(Pt.$$.fragment),Ko=f(),Us=l("p"),af=n("A quick look at the dataset shows the labels associated with each word in the sentence:"),Xo=f(),d(Ft.$$.fragment),Zo=f(),Ys=l("p"),nf=n("View the specific NER tags by:"),er=f(),d(Ct.$$.fragment),tr=f(),Gs=l("p"),of=n("A letter prefixes each NER tag which can mean:"),sr=f(),B=l("ul"),Vs=l("li"),un=l("code"),rf=n("B-"),lf=n(" indicates the beginning of an entity."),pf=f(),W=l("li"),dn=l("code"),ff=n("I-"),hf=n(" indicates a token is contained inside the same entity (e.g., the "),cn=l("code"),uf=n("State"),df=n(` token is a part of an entity like
`),mn=l("code"),cf=n("Empire State Building"),mf=n(")."),_f=f(),Js=l("li"),_n=l("code"),wf=n("0"),gf=n(" indicates the token doesn\u2019t correspond to any entity."),ar=f(),ae=l("h3"),Te=l("a"),wn=l("span"),d(Dt.$$.fragment),bf=f(),gn=l("span"),$f=n("Preprocess"),nr=f(),Ae=l("p"),kf=n("Now you need to tokenize the text. Load the DistilBERT tokenizer with an "),Ks=l("a"),vf=n("AutoTokenizer"),jf=n(":"),or=f(),d(St.$$.fragment),rr=f(),ze=l("p"),yf=n("Since the input has already been split into words, set "),bn=l("code"),xf=n("is_split_into_words=True"),qf=n(` to tokenize the words into
subwords:`),lr=f(),d(Nt.$$.fragment),ir=f(),H=l("p"),Ef=n("The addition of the special tokens "),$n=l("code"),Tf=n("[CLS]"),Af=n(" and "),kn=l("code"),zf=n("[SEP]"),Pf=n(` and subword tokenization creates a mismatch between the
input and labels. Realign the labels and tokens by:`),pr=f(),Q=l("ol"),It=l("li"),Ff=n("Mapping all tokens to their corresponding word with the "),vn=l("code"),Cf=n("word_ids"),Df=n(" method."),Sf=f(),ne=l("li"),Nf=n("Assigning the label "),jn=l("code"),If=n("-100"),Mf=n(" to the special tokens "),yn=l("code"),Lf=n("[CLS]"),Of=n(" and \u201C[SEP]``` so the PyTorch loss function ignores\nthem."),Bf=f(),Mt=l("li"),Wf=n("Only labeling the first token of a given word. Assign "),xn=l("code"),Hf=n("-100"),Qf=n(" to the other subtokens from the same word."),fr=f(),Xs=l("p"),Rf=n("Here is how you can create a function that will realign the labels and tokens:"),hr=f(),d(Lt.$$.fragment),ur=f(),Pe=l("p"),Uf=n("Now tokenize and align the labels over the entire dataset with \u{1F917} Datasets "),qn=l("code"),Yf=n("map"),Gf=n(" function:"),dr=f(),d(Ot.$$.fragment),cr=f(),Zs=l("p"),Vf=n("Finally, pad your text and labels, so they are a uniform length:"),mr=f(),d(Bt.$$.fragment),_r=f(),oe=l("h3"),Fe=l("a"),En=l("span"),d(Wt.$$.fragment),Jf=f(),Tn=l("span"),Kf=n("Fine-tune with the Trainer API"),wr=f(),Ce=l("p"),Xf=n("Load your model with the "),ea=l("a"),Zf=n("AutoModelForTokenClassification"),eh=n(" class along with the number of expected labels:"),gr=f(),d(Ht.$$.fragment),br=f(),De=l("p"),th=n("Gather your training arguments in "),ta=l("a"),sh=n("TrainingArguments"),ah=n(":"),$r=f(),d(Qt.$$.fragment),kr=f(),Se=l("p"),nh=n("Collect your model, training arguments, dataset, data collator, and tokenizer in "),sa=l("a"),oh=n("Trainer"),rh=n(":"),vr=f(),d(Rt.$$.fragment),jr=f(),aa=l("p"),lh=n("Fine-tune your model:"),yr=f(),d(Ut.$$.fragment),xr=f(),re=l("h3"),Ne=l("a"),An=l("span"),d(Yt.$$.fragment),ih=f(),zn=l("span"),ph=n("Fine-tune with TensorFlow"),qr=f(),na=l("p"),fh=n("Batch your examples together and pad your text and labels, so they are a uniform length:"),Er=f(),d(Gt.$$.fragment),Tr=f(),R=l("p"),hh=n("Convert your datasets to the "),Pn=l("code"),uh=n("tf.data.Dataset"),dh=n(" format with "),Fn=l("code"),ch=n("to_tf_dataset"),mh=n(":"),Ar=f(),d(Vt.$$.fragment),zr=f(),Ie=l("p"),_h=n("Load the model with the "),oa=l("a"),wh=n("TFAutoModelForTokenClassification"),gh=n(" class along with the number of expected labels:"),Pr=f(),d(Jt.$$.fragment),Fr=f(),ra=l("p"),bh=n("Set up an optimizer function, learning rate schedule, and some training hyperparameters:"),Cr=f(),d(Kt.$$.fragment),Dr=f(),la=l("p"),$h=n("Compile the model:"),Sr=f(),d(Xt.$$.fragment),Nr=f(),Me=l("p"),kh=n("Call "),Cn=l("code"),vh=n("model.fit"),jh=n(" to fine-tune your model:"),Ir=f(),d(Zt.$$.fragment),Mr=f(),ia=l("a"),Lr=f(),le=l("h2"),Le=l("a"),Dn=l("span"),d(es.$$.fragment),yh=f(),Sn=l("span"),xh=n("Question Answering with SQuAD"),Or=f(),Oe=l("p"),qh=n(`There are many types of question answering (QA) tasks. Extractive QA focuses on identifying the answer from the text
given a question. In this example, learn how to fine-tune a model on the `),ts=l("a"),Eh=n("SQuAD"),Th=n(" dataset."),Br=f(),d(Be.$$.fragment),Wr=f(),ie=l("h3"),We=l("a"),Nn=l("span"),d(ss.$$.fragment),Ah=f(),In=l("span"),zh=n("Load SQuAD dataset"),Hr=f(),pa=l("p"),Ph=n("Load the SQuAD dataset from the \u{1F917} Datasets library:"),Qr=f(),d(as.$$.fragment),Rr=f(),fa=l("p"),Fh=n("Take a look at an example from the dataset:"),Ur=f(),d(ns.$$.fragment),Yr=f(),pe=l("h3"),He=l("a"),Mn=l("span"),d(os.$$.fragment),Ch=f(),Ln=l("span"),Dh=n("Preprocess"),Gr=f(),Qe=l("p"),Sh=n("Load the DistilBERT tokenizer with an "),ha=l("a"),Nh=n("AutoTokenizer"),Ih=n(":"),Vr=f(),d(rs.$$.fragment),Jr=f(),ua=l("p"),Mh=n("There are a few things to be aware of when preprocessing text for question answering:"),Kr=f(),U=l("ol"),N=l("li"),Lh=n("Some examples in a dataset may have a very long "),On=l("code"),Oh=n("context"),Bh=n(` that exceeds the maximum input length of the model. You
can deal with this by truncating the `),Bn=l("code"),Wh=n("context"),Hh=n(" and set "),Wn=l("code"),Qh=n('truncation="only_second"'),Rh=n("."),Uh=f(),ls=l("li"),Yh=n(`Next, you need to map the start and end positions of the answer to the original context. Set
`),Hn=l("code"),Gh=n("return_offset_mapping=True"),Vh=n(" to handle this."),Jh=f(),is=l("li"),Kh=n("With the mapping in hand, you can find the start and end tokens of the answer. Use the "),Qn=l("code"),Xh=n("sequence_ids"),Zh=n(` method to
find which part of the offset corresponds to the question, and which part of the offset corresponds to the context.`),Xr=f(),da=l("p"),eu=n("Assemble everything in a preprocessing function as shown below:"),Zr=f(),d(ps.$$.fragment),el=f(),Re=l("p"),tu=n("Apply the preprocessing function over the entire dataset with \u{1F917} Datasets "),Rn=l("code"),su=n("map"),au=n(" function:"),tl=f(),d(fs.$$.fragment),sl=f(),ca=l("p"),nu=n("Batch the processed examples together:"),al=f(),d(hs.$$.fragment),nl=f(),fe=l("h3"),Ue=l("a"),Un=l("span"),d(us.$$.fragment),ou=f(),Yn=l("span"),ru=n("Fine-tune with the Trainer API"),ol=f(),Ye=l("p"),lu=n("Load your model with the "),ma=l("a"),iu=n("AutoModelForQuestionAnswering"),pu=n(" class:"),rl=f(),d(ds.$$.fragment),ll=f(),Ge=l("p"),fu=n("Gather your training arguments in "),_a=l("a"),hu=n("TrainingArguments"),uu=n(":"),il=f(),d(cs.$$.fragment),pl=f(),Ve=l("p"),du=n("Collect your model, training arguments, dataset, data collator, and tokenizer in "),wa=l("a"),cu=n("Trainer"),mu=n(":"),fl=f(),d(ms.$$.fragment),hl=f(),ga=l("p"),_u=n("Fine-tune your model:"),ul=f(),d(_s.$$.fragment),dl=f(),he=l("h3"),Je=l("a"),Gn=l("span"),d(ws.$$.fragment),wu=f(),Vn=l("span"),gu=n("Fine-tune with TensorFlow"),cl=f(),ba=l("p"),bu=n("Batch the processed examples together with a TensorFlow default data collator:"),ml=f(),d(gs.$$.fragment),_l=f(),Y=l("p"),$u=n("Convert your datasets to the "),Jn=l("code"),ku=n("tf.data.Dataset"),vu=n(" format with the "),Kn=l("code"),ju=n("to_tf_dataset"),yu=n(" function:"),wl=f(),d(bs.$$.fragment),gl=f(),$a=l("p"),xu=n("Set up an optimizer function, learning rate schedule, and some training hyperparameters:"),bl=f(),d($s.$$.fragment),$l=f(),Ke=l("p"),qu=n("Load your model with the "),ka=l("a"),Eu=n("TFAutoModelForQuestionAnswering"),Tu=n(" class:"),kl=f(),d(ks.$$.fragment),vl=f(),va=l("p"),Au=n("Compile the model:"),jl=f(),d(vs.$$.fragment),yl=f(),Xe=l("p"),zu=n("Call "),Xn=l("code"),Pu=n("model.fit"),Fu=n(" to fine-tune the model:"),xl=f(),d(js.$$.fragment),this.h()},l(e){const a=_m('[data-svelte="svelte-1phssyn"]',document.head);$=i(a,"META",{name:!0,content:!0}),a.forEach(t),q=h(e),k=i(e,"H1",{class:!0});var ys=p(k);y=i(ys,"A",{id:!0,class:!0,href:!0});var Zn=p(y);E=i(Zn,"SPAN",{});var eo=p(E);c(v.$$.fragment,eo),eo.forEach(t),Zn.forEach(t),P=h(ys),T=i(ys,"SPAN",{});var Nu=p(T);x=o(Nu,"How to fine-tune a model for common downstream tasks"),Nu.forEach(t),ys.forEach(t),j=h(e),c(A.$$.fragment,e),C=h(e),qs=i(e,"P",{});var Iu=p(qs);gi=o(Iu,`This guide will show you how to fine-tune \u{1F917} Transformers models for common downstream tasks. You will use the \u{1F917}
Datasets library to quickly load and preprocess the datasets, getting them ready for training with PyTorch and
TensorFlow.`),Iu.forEach(t),to=h(e),ue=i(e,"P",{});var El=p(ue);bi=o(El,`Before you begin, make sure you have the \u{1F917} Datasets library installed. For more detailed installation instructions,
refer to the \u{1F917} Datasets `),st=i(El,"A",{href:!0,rel:!0});var Mu=p(st);$i=o(Mu,"installation page"),Mu.forEach(t),ki=o(El,`. All of the
examples in this guide will use \u{1F917} Datasets to load and preprocess a dataset.`),El.forEach(t),so=h(e),c(at.$$.fragment,e),ao=h(e),Es=i(e,"P",{});var Lu=p(Es);vi=o(Lu,"Learn how to fine-tune a model for:"),Lu.forEach(t),no=h(e),I=i(e,"UL",{});var ja=p(I);Da=i(ja,"LI",{});var Ou=p(Da);Ts=i(Ou,"A",{href:!0});var Bu=p(Ts);ji=o(Bu,"seq_imdb"),Bu.forEach(t),Ou.forEach(t),yi=h(ja),Sa=i(ja,"LI",{});var Wu=p(Sa);As=i(Wu,"A",{href:!0});var Hu=p(As);xi=o(Hu,"tok_ner"),Hu.forEach(t),Wu.forEach(t),qi=h(ja),Na=i(ja,"LI",{});var Qu=p(Na);zs=i(Qu,"A",{href:!0});var Ru=p(zs);Ei=o(Ru,"qa_squad"),Ru.forEach(t),Qu.forEach(t),ja.forEach(t),oo=h(e),Ps=i(e,"A",{id:!0}),p(Ps).forEach(t),ro=h(e),J=i(e,"H2",{class:!0});var Tl=p(J);de=i(Tl,"A",{id:!0,class:!0,href:!0});var Uu=p(de);Ia=i(Uu,"SPAN",{});var Yu=p(Ia);c(nt.$$.fragment,Yu),Yu.forEach(t),Uu.forEach(t),Ti=h(Tl),Ma=i(Tl,"SPAN",{});var Gu=p(Ma);Ai=o(Gu,"Sequence classification with IMDb reviews"),Gu.forEach(t),Tl.forEach(t),lo=h(e),ce=i(e,"P",{});var Al=p(ce);zi=o(Al,`Sequence classification refers to the task of classifying sequences of text according to a given number of classes. In
this example, learn how to fine-tune a model on the `),ot=i(Al,"A",{href:!0,rel:!0});var Vu=p(ot);Pi=o(Vu,"IMDb dataset"),Vu.forEach(t),Fi=o(Al,` to determine
whether a review is positive or negative.`),Al.forEach(t),io=h(e),c(me.$$.fragment,e),po=h(e),K=i(e,"H3",{class:!0});var zl=p(K);_e=i(zl,"A",{id:!0,class:!0,href:!0});var Ju=p(_e);La=i(Ju,"SPAN",{});var Ku=p(La);c(rt.$$.fragment,Ku),Ku.forEach(t),Ju.forEach(t),Ci=h(zl),Oa=i(zl,"SPAN",{});var Xu=p(Oa);Di=o(Xu,"Load IMDb dataset"),Xu.forEach(t),zl.forEach(t),fo=h(e),Fs=i(e,"P",{});var Zu=p(Fs);Si=o(Zu,"The \u{1F917} Datasets library makes it simple to load a dataset:"),Zu.forEach(t),ho=h(e),c(lt.$$.fragment,e),uo=h(e),we=i(e,"P",{});var Pl=p(we);Ni=o(Pl,"This loads a "),Ba=i(Pl,"CODE",{});var ed=p(Ba);Ii=o(ed,"DatasetDict"),ed.forEach(t),Mi=o(Pl," object which you can index into to view an example:"),Pl.forEach(t),co=h(e),c(it.$$.fragment,e),mo=h(e),X=i(e,"H3",{class:!0});var Fl=p(X);ge=i(Fl,"A",{id:!0,class:!0,href:!0});var td=p(ge);Wa=i(td,"SPAN",{});var sd=p(Wa);c(pt.$$.fragment,sd),sd.forEach(t),td.forEach(t),Li=h(Fl),Ha=i(Fl,"SPAN",{});var ad=p(Ha);Oi=o(ad,"Preprocess"),ad.forEach(t),Fl.forEach(t),_o=h(e),M=i(e,"P",{});var ya=p(M);Bi=o(ya,`The next step is to tokenize the text into a readable format by the model. It is important to load the same tokenizer a
model was trained with to ensure appropriately tokenized words. Load the DistilBERT tokenizer with the
`),Cs=i(ya,"A",{href:!0});var nd=p(Cs);Wi=o(nd,"AutoTokenizer"),nd.forEach(t),Hi=o(ya," because we will eventually train a classifier using a pretrained "),ft=i(ya,"A",{href:!0,rel:!0});var od=p(ft);Qi=o(od,"DistilBERT"),od.forEach(t),Ri=o(ya," model:"),ya.forEach(t),wo=h(e),c(ht.$$.fragment,e),go=h(e),Ds=i(e,"P",{});var rd=p(Ds);Ui=o(rd,`Now that you have instantiated a tokenizer, create a function that will tokenize the text. You should also truncate
longer sequences in the text to be no longer than the model\u2019s maximum input length:`),rd.forEach(t),bo=h(e),c(ut.$$.fragment,e),$o=h(e),L=i(e,"P",{});var xa=p(L);Yi=o(xa,"Use \u{1F917} Datasets "),Qa=i(xa,"CODE",{});var ld=p(Qa);Gi=o(ld,"map"),ld.forEach(t),Vi=o(xa,` function to apply the preprocessing function to the entire dataset. You can also set
`),Ra=i(xa,"CODE",{});var id=p(Ra);Ji=o(id,"batched=True"),id.forEach(t),Ki=o(xa,` to apply the preprocessing function to multiple elements of the dataset at once for faster
preprocessing:`),xa.forEach(t),ko=h(e),c(dt.$$.fragment,e),vo=h(e),F=i(e,"P",{});var G=p(F);Xi=o(G,"Lastly, pad your text so they are a uniform length. While it is possible to pad your text in the "),Ua=i(G,"CODE",{});var pd=p(Ua);Zi=o(pd,"tokenizer"),pd.forEach(t),ep=o(G,` function
by setting `),Ya=i(G,"CODE",{});var fd=p(Ya);tp=o(fd,"padding=True"),fd.forEach(t),sp=o(G,`, it is more efficient to only pad the text to the length of the longest element in its
batch. This is known as `),Ga=i(G,"STRONG",{});var hd=p(Ga);ap=o(hd,"dynamic padding"),hd.forEach(t),np=o(G,". You can do this with the "),Va=i(G,"CODE",{});var ud=p(Va);op=o(ud,"DataCollatorWithPadding"),ud.forEach(t),rp=o(G," function:"),G.forEach(t),jo=h(e),c(ct.$$.fragment,e),yo=h(e),Z=i(e,"H3",{class:!0});var Cl=p(Z);be=i(Cl,"A",{id:!0,class:!0,href:!0});var dd=p(be);Ja=i(dd,"SPAN",{});var cd=p(Ja);c(mt.$$.fragment,cd),cd.forEach(t),dd.forEach(t),lp=h(Cl),Ka=i(Cl,"SPAN",{});var md=p(Ka);ip=o(md,"Fine-tune with the Trainer API"),md.forEach(t),Cl.forEach(t),xo=h(e),$e=i(e,"P",{});var Dl=p($e);pp=o(Dl,"Now load your model with the "),Ss=i(Dl,"A",{href:!0});var _d=p(Ss);fp=o(_d,"AutoModelForSequenceClassification"),_d.forEach(t),hp=o(Dl," class along with the number of expected labels:"),Dl.forEach(t),qo=h(e),c(_t.$$.fragment,e),Eo=h(e),Ns=i(e,"P",{});var wd=p(Ns);up=o(wd,"At this point, only three steps remain:"),wd.forEach(t),To=h(e),O=i(e,"OL",{});var qa=p(O);wt=i(qa,"LI",{});var Sl=p(wt);dp=o(Sl,"Define your training hyperparameters in "),Is=i(Sl,"A",{href:!0});var gd=p(Is);cp=o(gd,"TrainingArguments"),gd.forEach(t),mp=o(Sl,"."),Sl.forEach(t),_p=h(qa),gt=i(qa,"LI",{});var Nl=p(gt);wp=o(Nl,"Pass the training arguments to a "),Ms=i(Nl,"A",{href:!0});var bd=p(Ms);gp=o(bd,"Trainer"),bd.forEach(t),bp=o(Nl," along with the model, dataset, tokenizer, and data collator."),Nl.forEach(t),$p=h(qa),bt=i(qa,"LI",{});var Il=p(bt);kp=o(Il,"Call "),Xa=i(Il,"CODE",{});var $d=p(Xa);vp=o($d,"Trainer.train()"),$d.forEach(t),jp=o(Il," to fine-tune your model."),Il.forEach(t),qa.forEach(t),Ao=h(e),c($t.$$.fragment,e),zo=h(e),ee=i(e,"H3",{class:!0});var Ml=p(ee);ke=i(Ml,"A",{id:!0,class:!0,href:!0});var kd=p(ke);Za=i(kd,"SPAN",{});var vd=p(Za);c(kt.$$.fragment,vd),vd.forEach(t),kd.forEach(t),yp=h(Ml),en=i(Ml,"SPAN",{});var jd=p(en);xp=o(jd,"Fine-tune with TensorFlow"),jd.forEach(t),Ml.forEach(t),Po=h(e),Ls=i(e,"P",{});var yd=p(Ls);qp=o(yd,"Fine-tuning with TensorFlow is just as easy, with only a few differences."),yd.forEach(t),Fo=h(e),D=i(e,"P",{});var Ze=p(D);Ep=o(Ze,"Start by batching the processed examples together with dynamic padding using the "),Os=i(Ze,"A",{href:!0});var xd=p(Os);Tp=o(xd,"DataCollatorWithPadding"),xd.forEach(t),Ap=o(Ze,` function.
Make sure you set `),tn=i(Ze,"CODE",{});var qd=p(tn);zp=o(qd,'return_tensors="tf"'),qd.forEach(t),Pp=o(Ze," to return "),sn=i(Ze,"CODE",{});var Ed=p(sn);Fp=o(Ed,"tf.Tensor"),Ed.forEach(t),Cp=o(Ze," outputs instead of PyTorch tensors!"),Ze.forEach(t),Co=h(e),c(vt.$$.fragment,e),Do=h(e),S=i(e,"P",{});var et=p(S);Dp=o(et,"Next, convert your datasets to the "),an=i(et,"CODE",{});var Td=p(an);Sp=o(Td,"tf.data.Dataset"),Td.forEach(t),Np=o(et," format with "),nn=i(et,"CODE",{});var Ad=p(nn);Ip=o(Ad,"to_tf_dataset"),Ad.forEach(t),Mp=o(et,`. Specify inputs and labels in the
`),on=i(et,"CODE",{});var zd=p(on);Lp=o(zd,"columns"),zd.forEach(t),Op=o(et," argument:"),et.forEach(t),So=h(e),c(jt.$$.fragment,e),No=h(e),Bs=i(e,"P",{});var Pd=p(Bs);Bp=o(Pd,"Set up an optimizer function, learning rate schedule, and some training hyperparameters:"),Pd.forEach(t),Io=h(e),c(yt.$$.fragment,e),Mo=h(e),ve=i(e,"P",{});var Ll=p(ve);Wp=o(Ll,"Load your model with the "),Ws=i(Ll,"A",{href:!0});var Fd=p(Ws);Hp=o(Fd,"TFAutoModelForSequenceClassification"),Fd.forEach(t),Qp=o(Ll," class along with the number of expected labels:"),Ll.forEach(t),Lo=h(e),c(xt.$$.fragment,e),Oo=h(e),Hs=i(e,"P",{});var Cd=p(Hs);Rp=o(Cd,"Compile the model:"),Cd.forEach(t),Bo=h(e),c(qt.$$.fragment,e),Wo=h(e),je=i(e,"P",{});var Ol=p(je);Up=o(Ol,"Finally, fine-tune the model by calling "),rn=i(Ol,"CODE",{});var Dd=p(rn);Yp=o(Dd,"model.fit"),Dd.forEach(t),Gp=o(Ol,":"),Ol.forEach(t),Ho=h(e),c(Et.$$.fragment,e),Qo=h(e),Qs=i(e,"A",{id:!0}),p(Qs).forEach(t),Ro=h(e),te=i(e,"H2",{class:!0});var Bl=p(te);ye=i(Bl,"A",{id:!0,class:!0,href:!0});var Sd=p(ye);ln=i(Sd,"SPAN",{});var Nd=p(ln);c(Tt.$$.fragment,Nd),Nd.forEach(t),Sd.forEach(t),Vp=h(Bl),pn=i(Bl,"SPAN",{});var Id=p(pn);Jp=o(Id,"Token classification with WNUT emerging entities"),Id.forEach(t),Bl.forEach(t),Uo=h(e),xe=i(e,"P",{});var Wl=p(xe);Kp=o(Wl,`Token classification refers to the task of classifying individual tokens in a sentence. One of the most common token
classification tasks is Named Entity Recognition (NER). NER attempts to find a label for each entity in a sentence,
such as a person, location, or organization. In this example, learn how to fine-tune a model on the `),At=i(Wl,"A",{href:!0,rel:!0});var Md=p(At);Xp=o(Md,"WNUT 17"),Md.forEach(t),Zp=o(Wl," dataset to detect new entities."),Wl.forEach(t),Yo=h(e),c(qe.$$.fragment,e),Go=h(e),se=i(e,"H3",{class:!0});var Hl=p(se);Ee=i(Hl,"A",{id:!0,class:!0,href:!0});var Ld=p(Ee);fn=i(Ld,"SPAN",{});var Od=p(fn);c(zt.$$.fragment,Od),Od.forEach(t),Ld.forEach(t),ef=h(Hl),hn=i(Hl,"SPAN",{});var Bd=p(hn);tf=o(Bd,"Load WNUT 17 dataset"),Bd.forEach(t),Hl.forEach(t),Vo=h(e),Rs=i(e,"P",{});var Wd=p(Rs);sf=o(Wd,"Load the WNUT 17 dataset from the \u{1F917} Datasets library:"),Wd.forEach(t),Jo=h(e),c(Pt.$$.fragment,e),Ko=h(e),Us=i(e,"P",{});var Hd=p(Us);af=o(Hd,"A quick look at the dataset shows the labels associated with each word in the sentence:"),Hd.forEach(t),Xo=h(e),c(Ft.$$.fragment,e),Zo=h(e),Ys=i(e,"P",{});var Qd=p(Ys);nf=o(Qd,"View the specific NER tags by:"),Qd.forEach(t),er=h(e),c(Ct.$$.fragment,e),tr=h(e),Gs=i(e,"P",{});var Rd=p(Gs);of=o(Rd,"A letter prefixes each NER tag which can mean:"),Rd.forEach(t),sr=h(e),B=i(e,"UL",{});var Ea=p(B);Vs=i(Ea,"LI",{});var Cu=p(Vs);un=i(Cu,"CODE",{});var Ud=p(un);rf=o(Ud,"B-"),Ud.forEach(t),lf=o(Cu," indicates the beginning of an entity."),Cu.forEach(t),pf=h(Ea),W=i(Ea,"LI",{});var xs=p(W);dn=i(xs,"CODE",{});var Yd=p(dn);ff=o(Yd,"I-"),Yd.forEach(t),hf=o(xs," indicates a token is contained inside the same entity (e.g., the "),cn=i(xs,"CODE",{});var Gd=p(cn);uf=o(Gd,"State"),Gd.forEach(t),df=o(xs,` token is a part of an entity like
`),mn=i(xs,"CODE",{});var Vd=p(mn);cf=o(Vd,"Empire State Building"),Vd.forEach(t),mf=o(xs,")."),xs.forEach(t),_f=h(Ea),Js=i(Ea,"LI",{});var Du=p(Js);_n=i(Du,"CODE",{});var Jd=p(_n);wf=o(Jd,"0"),Jd.forEach(t),gf=o(Du," indicates the token doesn\u2019t correspond to any entity."),Du.forEach(t),Ea.forEach(t),ar=h(e),ae=i(e,"H3",{class:!0});var Ql=p(ae);Te=i(Ql,"A",{id:!0,class:!0,href:!0});var Kd=p(Te);wn=i(Kd,"SPAN",{});var Xd=p(wn);c(Dt.$$.fragment,Xd),Xd.forEach(t),Kd.forEach(t),bf=h(Ql),gn=i(Ql,"SPAN",{});var Zd=p(gn);$f=o(Zd,"Preprocess"),Zd.forEach(t),Ql.forEach(t),nr=h(e),Ae=i(e,"P",{});var Rl=p(Ae);kf=o(Rl,"Now you need to tokenize the text. Load the DistilBERT tokenizer with an "),Ks=i(Rl,"A",{href:!0});var ec=p(Ks);vf=o(ec,"AutoTokenizer"),ec.forEach(t),jf=o(Rl,":"),Rl.forEach(t),or=h(e),c(St.$$.fragment,e),rr=h(e),ze=i(e,"P",{});var Ul=p(ze);yf=o(Ul,"Since the input has already been split into words, set "),bn=i(Ul,"CODE",{});var tc=p(bn);xf=o(tc,"is_split_into_words=True"),tc.forEach(t),qf=o(Ul,` to tokenize the words into
subwords:`),Ul.forEach(t),lr=h(e),c(Nt.$$.fragment,e),ir=h(e),H=i(e,"P",{});var Ta=p(H);Ef=o(Ta,"The addition of the special tokens "),$n=i(Ta,"CODE",{});var sc=p($n);Tf=o(sc,"[CLS]"),sc.forEach(t),Af=o(Ta," and "),kn=i(Ta,"CODE",{});var ac=p(kn);zf=o(ac,"[SEP]"),ac.forEach(t),Pf=o(Ta,` and subword tokenization creates a mismatch between the
input and labels. Realign the labels and tokens by:`),Ta.forEach(t),pr=h(e),Q=i(e,"OL",{});var Aa=p(Q);It=i(Aa,"LI",{});var Yl=p(It);Ff=o(Yl,"Mapping all tokens to their corresponding word with the "),vn=i(Yl,"CODE",{});var nc=p(vn);Cf=o(nc,"word_ids"),nc.forEach(t),Df=o(Yl," method."),Yl.forEach(t),Sf=h(Aa),ne=i(Aa,"LI",{});var za=p(ne);Nf=o(za,"Assigning the label "),jn=i(za,"CODE",{});var oc=p(jn);If=o(oc,"-100"),oc.forEach(t),Mf=o(za," to the special tokens "),yn=i(za,"CODE",{});var rc=p(yn);Lf=o(rc,"[CLS]"),rc.forEach(t),Of=o(za," and \u201C[SEP]``` so the PyTorch loss function ignores\nthem."),za.forEach(t),Bf=h(Aa),Mt=i(Aa,"LI",{});var Gl=p(Mt);Wf=o(Gl,"Only labeling the first token of a given word. Assign "),xn=i(Gl,"CODE",{});var lc=p(xn);Hf=o(lc,"-100"),lc.forEach(t),Qf=o(Gl," to the other subtokens from the same word."),Gl.forEach(t),Aa.forEach(t),fr=h(e),Xs=i(e,"P",{});var ic=p(Xs);Rf=o(ic,"Here is how you can create a function that will realign the labels and tokens:"),ic.forEach(t),hr=h(e),c(Lt.$$.fragment,e),ur=h(e),Pe=i(e,"P",{});var Vl=p(Pe);Uf=o(Vl,"Now tokenize and align the labels over the entire dataset with \u{1F917} Datasets "),qn=i(Vl,"CODE",{});var pc=p(qn);Yf=o(pc,"map"),pc.forEach(t),Gf=o(Vl," function:"),Vl.forEach(t),dr=h(e),c(Ot.$$.fragment,e),cr=h(e),Zs=i(e,"P",{});var fc=p(Zs);Vf=o(fc,"Finally, pad your text and labels, so they are a uniform length:"),fc.forEach(t),mr=h(e),c(Bt.$$.fragment,e),_r=h(e),oe=i(e,"H3",{class:!0});var Jl=p(oe);Fe=i(Jl,"A",{id:!0,class:!0,href:!0});var hc=p(Fe);En=i(hc,"SPAN",{});var uc=p(En);c(Wt.$$.fragment,uc),uc.forEach(t),hc.forEach(t),Jf=h(Jl),Tn=i(Jl,"SPAN",{});var dc=p(Tn);Kf=o(dc,"Fine-tune with the Trainer API"),dc.forEach(t),Jl.forEach(t),wr=h(e),Ce=i(e,"P",{});var Kl=p(Ce);Xf=o(Kl,"Load your model with the "),ea=i(Kl,"A",{href:!0});var cc=p(ea);Zf=o(cc,"AutoModelForTokenClassification"),cc.forEach(t),eh=o(Kl," class along with the number of expected labels:"),Kl.forEach(t),gr=h(e),c(Ht.$$.fragment,e),br=h(e),De=i(e,"P",{});var Xl=p(De);th=o(Xl,"Gather your training arguments in "),ta=i(Xl,"A",{href:!0});var mc=p(ta);sh=o(mc,"TrainingArguments"),mc.forEach(t),ah=o(Xl,":"),Xl.forEach(t),$r=h(e),c(Qt.$$.fragment,e),kr=h(e),Se=i(e,"P",{});var Zl=p(Se);nh=o(Zl,"Collect your model, training arguments, dataset, data collator, and tokenizer in "),sa=i(Zl,"A",{href:!0});var _c=p(sa);oh=o(_c,"Trainer"),_c.forEach(t),rh=o(Zl,":"),Zl.forEach(t),vr=h(e),c(Rt.$$.fragment,e),jr=h(e),aa=i(e,"P",{});var wc=p(aa);lh=o(wc,"Fine-tune your model:"),wc.forEach(t),yr=h(e),c(Ut.$$.fragment,e),xr=h(e),re=i(e,"H3",{class:!0});var ei=p(re);Ne=i(ei,"A",{id:!0,class:!0,href:!0});var gc=p(Ne);An=i(gc,"SPAN",{});var bc=p(An);c(Yt.$$.fragment,bc),bc.forEach(t),gc.forEach(t),ih=h(ei),zn=i(ei,"SPAN",{});var $c=p(zn);ph=o($c,"Fine-tune with TensorFlow"),$c.forEach(t),ei.forEach(t),qr=h(e),na=i(e,"P",{});var kc=p(na);fh=o(kc,"Batch your examples together and pad your text and labels, so they are a uniform length:"),kc.forEach(t),Er=h(e),c(Gt.$$.fragment,e),Tr=h(e),R=i(e,"P",{});var Pa=p(R);hh=o(Pa,"Convert your datasets to the "),Pn=i(Pa,"CODE",{});var vc=p(Pn);uh=o(vc,"tf.data.Dataset"),vc.forEach(t),dh=o(Pa," format with "),Fn=i(Pa,"CODE",{});var jc=p(Fn);ch=o(jc,"to_tf_dataset"),jc.forEach(t),mh=o(Pa,":"),Pa.forEach(t),Ar=h(e),c(Vt.$$.fragment,e),zr=h(e),Ie=i(e,"P",{});var ti=p(Ie);_h=o(ti,"Load the model with the "),oa=i(ti,"A",{href:!0});var yc=p(oa);wh=o(yc,"TFAutoModelForTokenClassification"),yc.forEach(t),gh=o(ti," class along with the number of expected labels:"),ti.forEach(t),Pr=h(e),c(Jt.$$.fragment,e),Fr=h(e),ra=i(e,"P",{});var xc=p(ra);bh=o(xc,"Set up an optimizer function, learning rate schedule, and some training hyperparameters:"),xc.forEach(t),Cr=h(e),c(Kt.$$.fragment,e),Dr=h(e),la=i(e,"P",{});var qc=p(la);$h=o(qc,"Compile the model:"),qc.forEach(t),Sr=h(e),c(Xt.$$.fragment,e),Nr=h(e),Me=i(e,"P",{});var si=p(Me);kh=o(si,"Call "),Cn=i(si,"CODE",{});var Ec=p(Cn);vh=o(Ec,"model.fit"),Ec.forEach(t),jh=o(si," to fine-tune your model:"),si.forEach(t),Ir=h(e),c(Zt.$$.fragment,e),Mr=h(e),ia=i(e,"A",{id:!0}),p(ia).forEach(t),Lr=h(e),le=i(e,"H2",{class:!0});var ai=p(le);Le=i(ai,"A",{id:!0,class:!0,href:!0});var Tc=p(Le);Dn=i(Tc,"SPAN",{});var Ac=p(Dn);c(es.$$.fragment,Ac),Ac.forEach(t),Tc.forEach(t),yh=h(ai),Sn=i(ai,"SPAN",{});var zc=p(Sn);xh=o(zc,"Question Answering with SQuAD"),zc.forEach(t),ai.forEach(t),Or=h(e),Oe=i(e,"P",{});var ni=p(Oe);qh=o(ni,`There are many types of question answering (QA) tasks. Extractive QA focuses on identifying the answer from the text
given a question. In this example, learn how to fine-tune a model on the `),ts=i(ni,"A",{href:!0,rel:!0});var Pc=p(ts);Eh=o(Pc,"SQuAD"),Pc.forEach(t),Th=o(ni," dataset."),ni.forEach(t),Br=h(e),c(Be.$$.fragment,e),Wr=h(e),ie=i(e,"H3",{class:!0});var oi=p(ie);We=i(oi,"A",{id:!0,class:!0,href:!0});var Fc=p(We);Nn=i(Fc,"SPAN",{});var Cc=p(Nn);c(ss.$$.fragment,Cc),Cc.forEach(t),Fc.forEach(t),Ah=h(oi),In=i(oi,"SPAN",{});var Dc=p(In);zh=o(Dc,"Load SQuAD dataset"),Dc.forEach(t),oi.forEach(t),Hr=h(e),pa=i(e,"P",{});var Sc=p(pa);Ph=o(Sc,"Load the SQuAD dataset from the \u{1F917} Datasets library:"),Sc.forEach(t),Qr=h(e),c(as.$$.fragment,e),Rr=h(e),fa=i(e,"P",{});var Nc=p(fa);Fh=o(Nc,"Take a look at an example from the dataset:"),Nc.forEach(t),Ur=h(e),c(ns.$$.fragment,e),Yr=h(e),pe=i(e,"H3",{class:!0});var ri=p(pe);He=i(ri,"A",{id:!0,class:!0,href:!0});var Ic=p(He);Mn=i(Ic,"SPAN",{});var Mc=p(Mn);c(os.$$.fragment,Mc),Mc.forEach(t),Ic.forEach(t),Ch=h(ri),Ln=i(ri,"SPAN",{});var Lc=p(Ln);Dh=o(Lc,"Preprocess"),Lc.forEach(t),ri.forEach(t),Gr=h(e),Qe=i(e,"P",{});var li=p(Qe);Sh=o(li,"Load the DistilBERT tokenizer with an "),ha=i(li,"A",{href:!0});var Oc=p(ha);Nh=o(Oc,"AutoTokenizer"),Oc.forEach(t),Ih=o(li,":"),li.forEach(t),Vr=h(e),c(rs.$$.fragment,e),Jr=h(e),ua=i(e,"P",{});var Bc=p(ua);Mh=o(Bc,"There are a few things to be aware of when preprocessing text for question answering:"),Bc.forEach(t),Kr=h(e),U=i(e,"OL",{});var Fa=p(U);N=i(Fa,"LI",{});var tt=p(N);Lh=o(tt,"Some examples in a dataset may have a very long "),On=i(tt,"CODE",{});var Wc=p(On);Oh=o(Wc,"context"),Wc.forEach(t),Bh=o(tt,` that exceeds the maximum input length of the model. You
can deal with this by truncating the `),Bn=i(tt,"CODE",{});var Hc=p(Bn);Wh=o(Hc,"context"),Hc.forEach(t),Hh=o(tt," and set "),Wn=i(tt,"CODE",{});var Qc=p(Wn);Qh=o(Qc,'truncation="only_second"'),Qc.forEach(t),Rh=o(tt,"."),tt.forEach(t),Uh=h(Fa),ls=i(Fa,"LI",{});var ii=p(ls);Yh=o(ii,`Next, you need to map the start and end positions of the answer to the original context. Set
`),Hn=i(ii,"CODE",{});var Rc=p(Hn);Gh=o(Rc,"return_offset_mapping=True"),Rc.forEach(t),Vh=o(ii," to handle this."),ii.forEach(t),Jh=h(Fa),is=i(Fa,"LI",{});var pi=p(is);Kh=o(pi,"With the mapping in hand, you can find the start and end tokens of the answer. Use the "),Qn=i(pi,"CODE",{});var Uc=p(Qn);Xh=o(Uc,"sequence_ids"),Uc.forEach(t),Zh=o(pi,` method to
find which part of the offset corresponds to the question, and which part of the offset corresponds to the context.`),pi.forEach(t),Fa.forEach(t),Xr=h(e),da=i(e,"P",{});var Yc=p(da);eu=o(Yc,"Assemble everything in a preprocessing function as shown below:"),Yc.forEach(t),Zr=h(e),c(ps.$$.fragment,e),el=h(e),Re=i(e,"P",{});var fi=p(Re);tu=o(fi,"Apply the preprocessing function over the entire dataset with \u{1F917} Datasets "),Rn=i(fi,"CODE",{});var Gc=p(Rn);su=o(Gc,"map"),Gc.forEach(t),au=o(fi," function:"),fi.forEach(t),tl=h(e),c(fs.$$.fragment,e),sl=h(e),ca=i(e,"P",{});var Vc=p(ca);nu=o(Vc,"Batch the processed examples together:"),Vc.forEach(t),al=h(e),c(hs.$$.fragment,e),nl=h(e),fe=i(e,"H3",{class:!0});var hi=p(fe);Ue=i(hi,"A",{id:!0,class:!0,href:!0});var Jc=p(Ue);Un=i(Jc,"SPAN",{});var Kc=p(Un);c(us.$$.fragment,Kc),Kc.forEach(t),Jc.forEach(t),ou=h(hi),Yn=i(hi,"SPAN",{});var Xc=p(Yn);ru=o(Xc,"Fine-tune with the Trainer API"),Xc.forEach(t),hi.forEach(t),ol=h(e),Ye=i(e,"P",{});var ui=p(Ye);lu=o(ui,"Load your model with the "),ma=i(ui,"A",{href:!0});var Zc=p(ma);iu=o(Zc,"AutoModelForQuestionAnswering"),Zc.forEach(t),pu=o(ui," class:"),ui.forEach(t),rl=h(e),c(ds.$$.fragment,e),ll=h(e),Ge=i(e,"P",{});var di=p(Ge);fu=o(di,"Gather your training arguments in "),_a=i(di,"A",{href:!0});var em=p(_a);hu=o(em,"TrainingArguments"),em.forEach(t),uu=o(di,":"),di.forEach(t),il=h(e),c(cs.$$.fragment,e),pl=h(e),Ve=i(e,"P",{});var ci=p(Ve);du=o(ci,"Collect your model, training arguments, dataset, data collator, and tokenizer in "),wa=i(ci,"A",{href:!0});var tm=p(wa);cu=o(tm,"Trainer"),tm.forEach(t),mu=o(ci,":"),ci.forEach(t),fl=h(e),c(ms.$$.fragment,e),hl=h(e),ga=i(e,"P",{});var sm=p(ga);_u=o(sm,"Fine-tune your model:"),sm.forEach(t),ul=h(e),c(_s.$$.fragment,e),dl=h(e),he=i(e,"H3",{class:!0});var mi=p(he);Je=i(mi,"A",{id:!0,class:!0,href:!0});var am=p(Je);Gn=i(am,"SPAN",{});var nm=p(Gn);c(ws.$$.fragment,nm),nm.forEach(t),am.forEach(t),wu=h(mi),Vn=i(mi,"SPAN",{});var om=p(Vn);gu=o(om,"Fine-tune with TensorFlow"),om.forEach(t),mi.forEach(t),cl=h(e),ba=i(e,"P",{});var rm=p(ba);bu=o(rm,"Batch the processed examples together with a TensorFlow default data collator:"),rm.forEach(t),ml=h(e),c(gs.$$.fragment,e),_l=h(e),Y=i(e,"P",{});var Ca=p(Y);$u=o(Ca,"Convert your datasets to the "),Jn=i(Ca,"CODE",{});var lm=p(Jn);ku=o(lm,"tf.data.Dataset"),lm.forEach(t),vu=o(Ca," format with the "),Kn=i(Ca,"CODE",{});var im=p(Kn);ju=o(im,"to_tf_dataset"),im.forEach(t),yu=o(Ca," function:"),Ca.forEach(t),wl=h(e),c(bs.$$.fragment,e),gl=h(e),$a=i(e,"P",{});var pm=p($a);xu=o(pm,"Set up an optimizer function, learning rate schedule, and some training hyperparameters:"),pm.forEach(t),bl=h(e),c($s.$$.fragment,e),$l=h(e),Ke=i(e,"P",{});var _i=p(Ke);qu=o(_i,"Load your model with the "),ka=i(_i,"A",{href:!0});var fm=p(ka);Eu=o(fm,"TFAutoModelForQuestionAnswering"),fm.forEach(t),Tu=o(_i," class:"),_i.forEach(t),kl=h(e),c(ks.$$.fragment,e),vl=h(e),va=i(e,"P",{});var hm=p(va);Au=o(hm,"Compile the model:"),hm.forEach(t),jl=h(e),c(vs.$$.fragment,e),yl=h(e),Xe=i(e,"P",{});var wi=p(Xe);zu=o(wi,"Call "),Xn=i(wi,"CODE",{});var um=p(Xn);Pu=o(um,"model.fit"),um.forEach(t),Fu=o(wi," to fine-tune the model:"),wi.forEach(t),xl=h(e),c(js.$$.fragment,e),this.h()},h(){u($,"name","hf:doc:metadata"),u($,"content",JSON.stringify(vm)),u(y,"id","how-to-finetune-a-model-for-common-downstream-tasks"),u(y,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),u(y,"href","#how-to-finetune-a-model-for-common-downstream-tasks"),u(k,"class","relative group"),u(st,"href","https://huggingface.co/docs/datasets/installation.html"),u(st,"rel","nofollow"),u(Ts,"href","#seq_imdb"),u(As,"href","#tok_ner"),u(zs,"href","#qa_squad"),u(Ps,"id","seq_imdb"),u(de,"id","sequence-classification-with-imdb-reviews"),u(de,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),u(de,"href","#sequence-classification-with-imdb-reviews"),u(J,"class","relative group"),u(ot,"href","https://huggingface.co/datasets/imdb"),u(ot,"rel","nofollow"),u(_e,"id","load-imdb-dataset"),u(_e,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),u(_e,"href","#load-imdb-dataset"),u(K,"class","relative group"),u(ge,"id","preprocess"),u(ge,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),u(ge,"href","#preprocess"),u(X,"class","relative group"),u(Cs,"href","/docs/transformers/pr_15839/en/model_doc/auto#transformers.AutoTokenizer"),u(ft,"href","https://huggingface.co/distilbert-base-uncased"),u(ft,"rel","nofollow"),u(be,"id","finetune-with-the-trainer-api"),u(be,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),u(be,"href","#finetune-with-the-trainer-api"),u(Z,"class","relative group"),u(Ss,"href","/docs/transformers/pr_15839/en/model_doc/auto#transformers.AutoModelForSequenceClassification"),u(Is,"href","/docs/transformers/pr_15839/en/main_classes/trainer#transformers.TrainingArguments"),u(Ms,"href","/docs/transformers/pr_15839/en/main_classes/trainer#transformers.Trainer"),u(ke,"id","finetune-with-tensorflow"),u(ke,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),u(ke,"href","#finetune-with-tensorflow"),u(ee,"class","relative group"),u(Os,"href","/docs/transformers/pr_15839/en/main_classes/data_collator#transformers.DataCollatorWithPadding"),u(Ws,"href","/docs/transformers/pr_15839/en/model_doc/auto#transformers.TFAutoModelForSequenceClassification"),u(Qs,"id","tok_ner"),u(ye,"id","token-classification-with-wnut-emerging-entities"),u(ye,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),u(ye,"href","#token-classification-with-wnut-emerging-entities"),u(te,"class","relative group"),u(At,"href","https://huggingface.co/datasets/wnut_17"),u(At,"rel","nofollow"),u(Ee,"id","load-wnut-17-dataset"),u(Ee,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),u(Ee,"href","#load-wnut-17-dataset"),u(se,"class","relative group"),u(Te,"id","preprocess"),u(Te,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),u(Te,"href","#preprocess"),u(ae,"class","relative group"),u(Ks,"href","/docs/transformers/pr_15839/en/model_doc/auto#transformers.AutoTokenizer"),u(Fe,"id","finetune-with-the-trainer-api"),u(Fe,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),u(Fe,"href","#finetune-with-the-trainer-api"),u(oe,"class","relative group"),u(ea,"href","/docs/transformers/pr_15839/en/model_doc/auto#transformers.AutoModelForTokenClassification"),u(ta,"href","/docs/transformers/pr_15839/en/main_classes/trainer#transformers.TrainingArguments"),u(sa,"href","/docs/transformers/pr_15839/en/main_classes/trainer#transformers.Trainer"),u(Ne,"id","finetune-with-tensorflow"),u(Ne,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),u(Ne,"href","#finetune-with-tensorflow"),u(re,"class","relative group"),u(oa,"href","/docs/transformers/pr_15839/en/model_doc/auto#transformers.TFAutoModelForTokenClassification"),u(ia,"id","qa_squad"),u(Le,"id","question-answering-with-squad"),u(Le,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),u(Le,"href","#question-answering-with-squad"),u(le,"class","relative group"),u(ts,"href","https://huggingface.co/datasets/squad"),u(ts,"rel","nofollow"),u(We,"id","load-squad-dataset"),u(We,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),u(We,"href","#load-squad-dataset"),u(ie,"class","relative group"),u(He,"id","preprocess"),u(He,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),u(He,"href","#preprocess"),u(pe,"class","relative group"),u(ha,"href","/docs/transformers/pr_15839/en/model_doc/auto#transformers.AutoTokenizer"),u(Ue,"id","finetune-with-the-trainer-api"),u(Ue,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),u(Ue,"href","#finetune-with-the-trainer-api"),u(fe,"class","relative group"),u(ma,"href","/docs/transformers/pr_15839/en/model_doc/auto#transformers.AutoModelForQuestionAnswering"),u(_a,"href","/docs/transformers/pr_15839/en/main_classes/trainer#transformers.TrainingArguments"),u(wa,"href","/docs/transformers/pr_15839/en/main_classes/trainer#transformers.Trainer"),u(Je,"id","finetune-with-tensorflow"),u(Je,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),u(Je,"href","#finetune-with-tensorflow"),u(he,"class","relative group"),u(ka,"href","/docs/transformers/pr_15839/en/model_doc/auto#transformers.TFAutoModelForQuestionAnswering")},m(e,a){s(document.head,$),r(e,q,a),r(e,k,a),s(k,y),s(y,E),m(v,E,null),s(k,P),s(k,T),s(T,x),r(e,j,a),m(A,e,a),r(e,C,a),r(e,qs,a),s(qs,gi),r(e,to,a),r(e,ue,a),s(ue,bi),s(ue,st),s(st,$i),s(ue,ki),r(e,so,a),m(at,e,a),r(e,ao,a),r(e,Es,a),s(Es,vi),r(e,no,a),r(e,I,a),s(I,Da),s(Da,Ts),s(Ts,ji),s(I,yi),s(I,Sa),s(Sa,As),s(As,xi),s(I,qi),s(I,Na),s(Na,zs),s(zs,Ei),r(e,oo,a),r(e,Ps,a),r(e,ro,a),r(e,J,a),s(J,de),s(de,Ia),m(nt,Ia,null),s(J,Ti),s(J,Ma),s(Ma,Ai),r(e,lo,a),r(e,ce,a),s(ce,zi),s(ce,ot),s(ot,Pi),s(ce,Fi),r(e,io,a),m(me,e,a),r(e,po,a),r(e,K,a),s(K,_e),s(_e,La),m(rt,La,null),s(K,Ci),s(K,Oa),s(Oa,Di),r(e,fo,a),r(e,Fs,a),s(Fs,Si),r(e,ho,a),m(lt,e,a),r(e,uo,a),r(e,we,a),s(we,Ni),s(we,Ba),s(Ba,Ii),s(we,Mi),r(e,co,a),m(it,e,a),r(e,mo,a),r(e,X,a),s(X,ge),s(ge,Wa),m(pt,Wa,null),s(X,Li),s(X,Ha),s(Ha,Oi),r(e,_o,a),r(e,M,a),s(M,Bi),s(M,Cs),s(Cs,Wi),s(M,Hi),s(M,ft),s(ft,Qi),s(M,Ri),r(e,wo,a),m(ht,e,a),r(e,go,a),r(e,Ds,a),s(Ds,Ui),r(e,bo,a),m(ut,e,a),r(e,$o,a),r(e,L,a),s(L,Yi),s(L,Qa),s(Qa,Gi),s(L,Vi),s(L,Ra),s(Ra,Ji),s(L,Ki),r(e,ko,a),m(dt,e,a),r(e,vo,a),r(e,F,a),s(F,Xi),s(F,Ua),s(Ua,Zi),s(F,ep),s(F,Ya),s(Ya,tp),s(F,sp),s(F,Ga),s(Ga,ap),s(F,np),s(F,Va),s(Va,op),s(F,rp),r(e,jo,a),m(ct,e,a),r(e,yo,a),r(e,Z,a),s(Z,be),s(be,Ja),m(mt,Ja,null),s(Z,lp),s(Z,Ka),s(Ka,ip),r(e,xo,a),r(e,$e,a),s($e,pp),s($e,Ss),s(Ss,fp),s($e,hp),r(e,qo,a),m(_t,e,a),r(e,Eo,a),r(e,Ns,a),s(Ns,up),r(e,To,a),r(e,O,a),s(O,wt),s(wt,dp),s(wt,Is),s(Is,cp),s(wt,mp),s(O,_p),s(O,gt),s(gt,wp),s(gt,Ms),s(Ms,gp),s(gt,bp),s(O,$p),s(O,bt),s(bt,kp),s(bt,Xa),s(Xa,vp),s(bt,jp),r(e,Ao,a),m($t,e,a),r(e,zo,a),r(e,ee,a),s(ee,ke),s(ke,Za),m(kt,Za,null),s(ee,yp),s(ee,en),s(en,xp),r(e,Po,a),r(e,Ls,a),s(Ls,qp),r(e,Fo,a),r(e,D,a),s(D,Ep),s(D,Os),s(Os,Tp),s(D,Ap),s(D,tn),s(tn,zp),s(D,Pp),s(D,sn),s(sn,Fp),s(D,Cp),r(e,Co,a),m(vt,e,a),r(e,Do,a),r(e,S,a),s(S,Dp),s(S,an),s(an,Sp),s(S,Np),s(S,nn),s(nn,Ip),s(S,Mp),s(S,on),s(on,Lp),s(S,Op),r(e,So,a),m(jt,e,a),r(e,No,a),r(e,Bs,a),s(Bs,Bp),r(e,Io,a),m(yt,e,a),r(e,Mo,a),r(e,ve,a),s(ve,Wp),s(ve,Ws),s(Ws,Hp),s(ve,Qp),r(e,Lo,a),m(xt,e,a),r(e,Oo,a),r(e,Hs,a),s(Hs,Rp),r(e,Bo,a),m(qt,e,a),r(e,Wo,a),r(e,je,a),s(je,Up),s(je,rn),s(rn,Yp),s(je,Gp),r(e,Ho,a),m(Et,e,a),r(e,Qo,a),r(e,Qs,a),r(e,Ro,a),r(e,te,a),s(te,ye),s(ye,ln),m(Tt,ln,null),s(te,Vp),s(te,pn),s(pn,Jp),r(e,Uo,a),r(e,xe,a),s(xe,Kp),s(xe,At),s(At,Xp),s(xe,Zp),r(e,Yo,a),m(qe,e,a),r(e,Go,a),r(e,se,a),s(se,Ee),s(Ee,fn),m(zt,fn,null),s(se,ef),s(se,hn),s(hn,tf),r(e,Vo,a),r(e,Rs,a),s(Rs,sf),r(e,Jo,a),m(Pt,e,a),r(e,Ko,a),r(e,Us,a),s(Us,af),r(e,Xo,a),m(Ft,e,a),r(e,Zo,a),r(e,Ys,a),s(Ys,nf),r(e,er,a),m(Ct,e,a),r(e,tr,a),r(e,Gs,a),s(Gs,of),r(e,sr,a),r(e,B,a),s(B,Vs),s(Vs,un),s(un,rf),s(Vs,lf),s(B,pf),s(B,W),s(W,dn),s(dn,ff),s(W,hf),s(W,cn),s(cn,uf),s(W,df),s(W,mn),s(mn,cf),s(W,mf),s(B,_f),s(B,Js),s(Js,_n),s(_n,wf),s(Js,gf),r(e,ar,a),r(e,ae,a),s(ae,Te),s(Te,wn),m(Dt,wn,null),s(ae,bf),s(ae,gn),s(gn,$f),r(e,nr,a),r(e,Ae,a),s(Ae,kf),s(Ae,Ks),s(Ks,vf),s(Ae,jf),r(e,or,a),m(St,e,a),r(e,rr,a),r(e,ze,a),s(ze,yf),s(ze,bn),s(bn,xf),s(ze,qf),r(e,lr,a),m(Nt,e,a),r(e,ir,a),r(e,H,a),s(H,Ef),s(H,$n),s($n,Tf),s(H,Af),s(H,kn),s(kn,zf),s(H,Pf),r(e,pr,a),r(e,Q,a),s(Q,It),s(It,Ff),s(It,vn),s(vn,Cf),s(It,Df),s(Q,Sf),s(Q,ne),s(ne,Nf),s(ne,jn),s(jn,If),s(ne,Mf),s(ne,yn),s(yn,Lf),s(ne,Of),s(Q,Bf),s(Q,Mt),s(Mt,Wf),s(Mt,xn),s(xn,Hf),s(Mt,Qf),r(e,fr,a),r(e,Xs,a),s(Xs,Rf),r(e,hr,a),m(Lt,e,a),r(e,ur,a),r(e,Pe,a),s(Pe,Uf),s(Pe,qn),s(qn,Yf),s(Pe,Gf),r(e,dr,a),m(Ot,e,a),r(e,cr,a),r(e,Zs,a),s(Zs,Vf),r(e,mr,a),m(Bt,e,a),r(e,_r,a),r(e,oe,a),s(oe,Fe),s(Fe,En),m(Wt,En,null),s(oe,Jf),s(oe,Tn),s(Tn,Kf),r(e,wr,a),r(e,Ce,a),s(Ce,Xf),s(Ce,ea),s(ea,Zf),s(Ce,eh),r(e,gr,a),m(Ht,e,a),r(e,br,a),r(e,De,a),s(De,th),s(De,ta),s(ta,sh),s(De,ah),r(e,$r,a),m(Qt,e,a),r(e,kr,a),r(e,Se,a),s(Se,nh),s(Se,sa),s(sa,oh),s(Se,rh),r(e,vr,a),m(Rt,e,a),r(e,jr,a),r(e,aa,a),s(aa,lh),r(e,yr,a),m(Ut,e,a),r(e,xr,a),r(e,re,a),s(re,Ne),s(Ne,An),m(Yt,An,null),s(re,ih),s(re,zn),s(zn,ph),r(e,qr,a),r(e,na,a),s(na,fh),r(e,Er,a),m(Gt,e,a),r(e,Tr,a),r(e,R,a),s(R,hh),s(R,Pn),s(Pn,uh),s(R,dh),s(R,Fn),s(Fn,ch),s(R,mh),r(e,Ar,a),m(Vt,e,a),r(e,zr,a),r(e,Ie,a),s(Ie,_h),s(Ie,oa),s(oa,wh),s(Ie,gh),r(e,Pr,a),m(Jt,e,a),r(e,Fr,a),r(e,ra,a),s(ra,bh),r(e,Cr,a),m(Kt,e,a),r(e,Dr,a),r(e,la,a),s(la,$h),r(e,Sr,a),m(Xt,e,a),r(e,Nr,a),r(e,Me,a),s(Me,kh),s(Me,Cn),s(Cn,vh),s(Me,jh),r(e,Ir,a),m(Zt,e,a),r(e,Mr,a),r(e,ia,a),r(e,Lr,a),r(e,le,a),s(le,Le),s(Le,Dn),m(es,Dn,null),s(le,yh),s(le,Sn),s(Sn,xh),r(e,Or,a),r(e,Oe,a),s(Oe,qh),s(Oe,ts),s(ts,Eh),s(Oe,Th),r(e,Br,a),m(Be,e,a),r(e,Wr,a),r(e,ie,a),s(ie,We),s(We,Nn),m(ss,Nn,null),s(ie,Ah),s(ie,In),s(In,zh),r(e,Hr,a),r(e,pa,a),s(pa,Ph),r(e,Qr,a),m(as,e,a),r(e,Rr,a),r(e,fa,a),s(fa,Fh),r(e,Ur,a),m(ns,e,a),r(e,Yr,a),r(e,pe,a),s(pe,He),s(He,Mn),m(os,Mn,null),s(pe,Ch),s(pe,Ln),s(Ln,Dh),r(e,Gr,a),r(e,Qe,a),s(Qe,Sh),s(Qe,ha),s(ha,Nh),s(Qe,Ih),r(e,Vr,a),m(rs,e,a),r(e,Jr,a),r(e,ua,a),s(ua,Mh),r(e,Kr,a),r(e,U,a),s(U,N),s(N,Lh),s(N,On),s(On,Oh),s(N,Bh),s(N,Bn),s(Bn,Wh),s(N,Hh),s(N,Wn),s(Wn,Qh),s(N,Rh),s(U,Uh),s(U,ls),s(ls,Yh),s(ls,Hn),s(Hn,Gh),s(ls,Vh),s(U,Jh),s(U,is),s(is,Kh),s(is,Qn),s(Qn,Xh),s(is,Zh),r(e,Xr,a),r(e,da,a),s(da,eu),r(e,Zr,a),m(ps,e,a),r(e,el,a),r(e,Re,a),s(Re,tu),s(Re,Rn),s(Rn,su),s(Re,au),r(e,tl,a),m(fs,e,a),r(e,sl,a),r(e,ca,a),s(ca,nu),r(e,al,a),m(hs,e,a),r(e,nl,a),r(e,fe,a),s(fe,Ue),s(Ue,Un),m(us,Un,null),s(fe,ou),s(fe,Yn),s(Yn,ru),r(e,ol,a),r(e,Ye,a),s(Ye,lu),s(Ye,ma),s(ma,iu),s(Ye,pu),r(e,rl,a),m(ds,e,a),r(e,ll,a),r(e,Ge,a),s(Ge,fu),s(Ge,_a),s(_a,hu),s(Ge,uu),r(e,il,a),m(cs,e,a),r(e,pl,a),r(e,Ve,a),s(Ve,du),s(Ve,wa),s(wa,cu),s(Ve,mu),r(e,fl,a),m(ms,e,a),r(e,hl,a),r(e,ga,a),s(ga,_u),r(e,ul,a),m(_s,e,a),r(e,dl,a),r(e,he,a),s(he,Je),s(Je,Gn),m(ws,Gn,null),s(he,wu),s(he,Vn),s(Vn,gu),r(e,cl,a),r(e,ba,a),s(ba,bu),r(e,ml,a),m(gs,e,a),r(e,_l,a),r(e,Y,a),s(Y,$u),s(Y,Jn),s(Jn,ku),s(Y,vu),s(Y,Kn),s(Kn,ju),s(Y,yu),r(e,wl,a),m(bs,e,a),r(e,gl,a),r(e,$a,a),s($a,xu),r(e,bl,a),m($s,e,a),r(e,$l,a),r(e,Ke,a),s(Ke,qu),s(Ke,ka),s(ka,Eu),s(Ke,Tu),r(e,kl,a),m(ks,e,a),r(e,vl,a),r(e,va,a),s(va,Au),r(e,jl,a),m(vs,e,a),r(e,yl,a),r(e,Xe,a),s(Xe,zu),s(Xe,Xn),s(Xn,Pu),s(Xe,Fu),r(e,xl,a),m(js,e,a),ql=!0},p(e,[a]){const ys={};a&2&&(ys.$$scope={dirty:a,ctx:e}),me.$set(ys);const Zn={};a&2&&(Zn.$$scope={dirty:a,ctx:e}),qe.$set(Zn);const eo={};a&2&&(eo.$$scope={dirty:a,ctx:e}),Be.$set(eo)},i(e){ql||(_(v.$$.fragment,e),_(A.$$.fragment,e),_(at.$$.fragment,e),_(nt.$$.fragment,e),_(me.$$.fragment,e),_(rt.$$.fragment,e),_(lt.$$.fragment,e),_(it.$$.fragment,e),_(pt.$$.fragment,e),_(ht.$$.fragment,e),_(ut.$$.fragment,e),_(dt.$$.fragment,e),_(ct.$$.fragment,e),_(mt.$$.fragment,e),_(_t.$$.fragment,e),_($t.$$.fragment,e),_(kt.$$.fragment,e),_(vt.$$.fragment,e),_(jt.$$.fragment,e),_(yt.$$.fragment,e),_(xt.$$.fragment,e),_(qt.$$.fragment,e),_(Et.$$.fragment,e),_(Tt.$$.fragment,e),_(qe.$$.fragment,e),_(zt.$$.fragment,e),_(Pt.$$.fragment,e),_(Ft.$$.fragment,e),_(Ct.$$.fragment,e),_(Dt.$$.fragment,e),_(St.$$.fragment,e),_(Nt.$$.fragment,e),_(Lt.$$.fragment,e),_(Ot.$$.fragment,e),_(Bt.$$.fragment,e),_(Wt.$$.fragment,e),_(Ht.$$.fragment,e),_(Qt.$$.fragment,e),_(Rt.$$.fragment,e),_(Ut.$$.fragment,e),_(Yt.$$.fragment,e),_(Gt.$$.fragment,e),_(Vt.$$.fragment,e),_(Jt.$$.fragment,e),_(Kt.$$.fragment,e),_(Xt.$$.fragment,e),_(Zt.$$.fragment,e),_(es.$$.fragment,e),_(Be.$$.fragment,e),_(ss.$$.fragment,e),_(as.$$.fragment,e),_(ns.$$.fragment,e),_(os.$$.fragment,e),_(rs.$$.fragment,e),_(ps.$$.fragment,e),_(fs.$$.fragment,e),_(hs.$$.fragment,e),_(us.$$.fragment,e),_(ds.$$.fragment,e),_(cs.$$.fragment,e),_(ms.$$.fragment,e),_(_s.$$.fragment,e),_(ws.$$.fragment,e),_(gs.$$.fragment,e),_(bs.$$.fragment,e),_($s.$$.fragment,e),_(ks.$$.fragment,e),_(vs.$$.fragment,e),_(js.$$.fragment,e),ql=!0)},o(e){w(v.$$.fragment,e),w(A.$$.fragment,e),w(at.$$.fragment,e),w(nt.$$.fragment,e),w(me.$$.fragment,e),w(rt.$$.fragment,e),w(lt.$$.fragment,e),w(it.$$.fragment,e),w(pt.$$.fragment,e),w(ht.$$.fragment,e),w(ut.$$.fragment,e),w(dt.$$.fragment,e),w(ct.$$.fragment,e),w(mt.$$.fragment,e),w(_t.$$.fragment,e),w($t.$$.fragment,e),w(kt.$$.fragment,e),w(vt.$$.fragment,e),w(jt.$$.fragment,e),w(yt.$$.fragment,e),w(xt.$$.fragment,e),w(qt.$$.fragment,e),w(Et.$$.fragment,e),w(Tt.$$.fragment,e),w(qe.$$.fragment,e),w(zt.$$.fragment,e),w(Pt.$$.fragment,e),w(Ft.$$.fragment,e),w(Ct.$$.fragment,e),w(Dt.$$.fragment,e),w(St.$$.fragment,e),w(Nt.$$.fragment,e),w(Lt.$$.fragment,e),w(Ot.$$.fragment,e),w(Bt.$$.fragment,e),w(Wt.$$.fragment,e),w(Ht.$$.fragment,e),w(Qt.$$.fragment,e),w(Rt.$$.fragment,e),w(Ut.$$.fragment,e),w(Yt.$$.fragment,e),w(Gt.$$.fragment,e),w(Vt.$$.fragment,e),w(Jt.$$.fragment,e),w(Kt.$$.fragment,e),w(Xt.$$.fragment,e),w(Zt.$$.fragment,e),w(es.$$.fragment,e),w(Be.$$.fragment,e),w(ss.$$.fragment,e),w(as.$$.fragment,e),w(ns.$$.fragment,e),w(os.$$.fragment,e),w(rs.$$.fragment,e),w(ps.$$.fragment,e),w(fs.$$.fragment,e),w(hs.$$.fragment,e),w(us.$$.fragment,e),w(ds.$$.fragment,e),w(cs.$$.fragment,e),w(ms.$$.fragment,e),w(_s.$$.fragment,e),w(ws.$$.fragment,e),w(gs.$$.fragment,e),w(bs.$$.fragment,e),w($s.$$.fragment,e),w(ks.$$.fragment,e),w(vs.$$.fragment,e),w(js.$$.fragment,e),ql=!1},d(e){t($),e&&t(q),e&&t(k),g(v),e&&t(j),g(A,e),e&&t(C),e&&t(qs),e&&t(to),e&&t(ue),e&&t(so),g(at,e),e&&t(ao),e&&t(Es),e&&t(no),e&&t(I),e&&t(oo),e&&t(Ps),e&&t(ro),e&&t(J),g(nt),e&&t(lo),e&&t(ce),e&&t(io),g(me,e),e&&t(po),e&&t(K),g(rt),e&&t(fo),e&&t(Fs),e&&t(ho),g(lt,e),e&&t(uo),e&&t(we),e&&t(co),g(it,e),e&&t(mo),e&&t(X),g(pt),e&&t(_o),e&&t(M),e&&t(wo),g(ht,e),e&&t(go),e&&t(Ds),e&&t(bo),g(ut,e),e&&t($o),e&&t(L),e&&t(ko),g(dt,e),e&&t(vo),e&&t(F),e&&t(jo),g(ct,e),e&&t(yo),e&&t(Z),g(mt),e&&t(xo),e&&t($e),e&&t(qo),g(_t,e),e&&t(Eo),e&&t(Ns),e&&t(To),e&&t(O),e&&t(Ao),g($t,e),e&&t(zo),e&&t(ee),g(kt),e&&t(Po),e&&t(Ls),e&&t(Fo),e&&t(D),e&&t(Co),g(vt,e),e&&t(Do),e&&t(S),e&&t(So),g(jt,e),e&&t(No),e&&t(Bs),e&&t(Io),g(yt,e),e&&t(Mo),e&&t(ve),e&&t(Lo),g(xt,e),e&&t(Oo),e&&t(Hs),e&&t(Bo),g(qt,e),e&&t(Wo),e&&t(je),e&&t(Ho),g(Et,e),e&&t(Qo),e&&t(Qs),e&&t(Ro),e&&t(te),g(Tt),e&&t(Uo),e&&t(xe),e&&t(Yo),g(qe,e),e&&t(Go),e&&t(se),g(zt),e&&t(Vo),e&&t(Rs),e&&t(Jo),g(Pt,e),e&&t(Ko),e&&t(Us),e&&t(Xo),g(Ft,e),e&&t(Zo),e&&t(Ys),e&&t(er),g(Ct,e),e&&t(tr),e&&t(Gs),e&&t(sr),e&&t(B),e&&t(ar),e&&t(ae),g(Dt),e&&t(nr),e&&t(Ae),e&&t(or),g(St,e),e&&t(rr),e&&t(ze),e&&t(lr),g(Nt,e),e&&t(ir),e&&t(H),e&&t(pr),e&&t(Q),e&&t(fr),e&&t(Xs),e&&t(hr),g(Lt,e),e&&t(ur),e&&t(Pe),e&&t(dr),g(Ot,e),e&&t(cr),e&&t(Zs),e&&t(mr),g(Bt,e),e&&t(_r),e&&t(oe),g(Wt),e&&t(wr),e&&t(Ce),e&&t(gr),g(Ht,e),e&&t(br),e&&t(De),e&&t($r),g(Qt,e),e&&t(kr),e&&t(Se),e&&t(vr),g(Rt,e),e&&t(jr),e&&t(aa),e&&t(yr),g(Ut,e),e&&t(xr),e&&t(re),g(Yt),e&&t(qr),e&&t(na),e&&t(Er),g(Gt,e),e&&t(Tr),e&&t(R),e&&t(Ar),g(Vt,e),e&&t(zr),e&&t(Ie),e&&t(Pr),g(Jt,e),e&&t(Fr),e&&t(ra),e&&t(Cr),g(Kt,e),e&&t(Dr),e&&t(la),e&&t(Sr),g(Xt,e),e&&t(Nr),e&&t(Me),e&&t(Ir),g(Zt,e),e&&t(Mr),e&&t(ia),e&&t(Lr),e&&t(le),g(es),e&&t(Or),e&&t(Oe),e&&t(Br),g(Be,e),e&&t(Wr),e&&t(ie),g(ss),e&&t(Hr),e&&t(pa),e&&t(Qr),g(as,e),e&&t(Rr),e&&t(fa),e&&t(Ur),g(ns,e),e&&t(Yr),e&&t(pe),g(os),e&&t(Gr),e&&t(Qe),e&&t(Vr),g(rs,e),e&&t(Jr),e&&t(ua),e&&t(Kr),e&&t(U),e&&t(Xr),e&&t(da),e&&t(Zr),g(ps,e),e&&t(el),e&&t(Re),e&&t(tl),g(fs,e),e&&t(sl),e&&t(ca),e&&t(al),g(hs,e),e&&t(nl),e&&t(fe),g(us),e&&t(ol),e&&t(Ye),e&&t(rl),g(ds,e),e&&t(ll),e&&t(Ge),e&&t(il),g(cs,e),e&&t(pl),e&&t(Ve),e&&t(fl),g(ms,e),e&&t(hl),e&&t(ga),e&&t(ul),g(_s,e),e&&t(dl),e&&t(he),g(ws),e&&t(cl),e&&t(ba),e&&t(ml),g(gs,e),e&&t(_l),e&&t(Y),e&&t(wl),g(bs,e),e&&t(gl),e&&t($a),e&&t(bl),g($s,e),e&&t($l),e&&t(Ke),e&&t(kl),g(ks,e),e&&t(vl),e&&t(va),e&&t(jl),g(vs,e),e&&t(yl),e&&t(Xe),e&&t(xl),g(js,e)}}}const vm={local:"how-to-finetune-a-model-for-common-downstream-tasks",sections:[{local:"sequence-classification-with-imdb-reviews",sections:[{local:"load-imdb-dataset",title:"Load IMDb dataset"},{local:"preprocess",title:"Preprocess"},{local:"finetune-with-the-trainer-api",title:"Fine-tune with the Trainer API"},{local:"finetune-with-tensorflow",title:"Fine-tune with TensorFlow"}],title:"Sequence classification with IMDb reviews"},{local:"token-classification-with-wnut-emerging-entities",sections:[{local:"load-wnut-17-dataset",title:"Load WNUT 17 dataset"},{local:"preprocess",title:"Preprocess"},{local:"finetune-with-the-trainer-api",title:"Fine-tune with the Trainer API"},{local:"finetune-with-tensorflow",title:"Fine-tune with TensorFlow"}],title:"Token classification with WNUT emerging entities"},{local:"question-answering-with-squad",sections:[{local:"load-squad-dataset",title:"Load SQuAD dataset"},{local:"preprocess",title:"Preprocess"},{local:"finetune-with-the-trainer-api",title:"Fine-tune with the Trainer API"},{local:"finetune-with-tensorflow",title:"Fine-tune with TensorFlow"}],title:"Question Answering with SQuAD"}],title:"How to fine-tune a model for common downstream tasks"};function jm(V,$,q){let{fw:k}=$;return V.$$set=y=>{"fw"in y&&q(0,k=y.fw)},[k]}class zm extends dm{constructor($){super();cm(this,$,jm,km,mm,{fw:0})}}export{zm as default,vm as metadata};
