import{S as is,i as ps,s as fs,e as o,k as m,w,t as p,M as cs,c as n,d as e,m as h,a as l,x as y,h as f,b as u,G as t,g as i,y as b,q as k,o as j,B as E,v as ms}from"../chunks/vendor-hf-doc-builder.js";import{T as xa}from"../chunks/Tip-hf-doc-builder.js";import{Y as qa}from"../chunks/Youtube-hf-doc-builder.js";import{I as Lt}from"../chunks/IconCopyLink-hf-doc-builder.js";import{C as J}from"../chunks/CodeBlock-hf-doc-builder.js";import{D as hs}from"../chunks/DocNotebookDropdown-hf-doc-builder.js";import{F as ls,M as za}from"../chunks/Markdown-hf-doc-builder.js";function us(Me){let g,A;return{c(){g=o("p"),A=p(`You will see a warning about some of the pretrained weights not being used and some weights being randomly
initialized. Don\u2019t worry, this is completely normal! The pretrained head of the BERT model is discarded, and replaced with a randomly initialized classification head. You will fine-tune this new model head on your sequence classification task, transferring the knowledge of the pretrained model to it.`)},l(d){g=n(d,"P",{});var T=l(g);A=f(T,`You will see a warning about some of the pretrained weights not being used and some weights being randomly
initialized. Don\u2019t worry, this is completely normal! The pretrained head of the BERT model is discarded, and replaced with a randomly initialized classification head. You will fine-tune this new model head on your sequence classification task, transferring the knowledge of the pretrained model to it.`),T.forEach(e)},m(d,T){i(d,g,T),t(g,A)},d(d){d&&e(g)}}}function ds(Me){let g,A,d,T,q,N,X,x,D,I,S,Z,Q,Oe,Be,M,Se,le,G,te,ge,Ae,ae,Ze,ie,Qe,U,lt,F,L,V,O,it,He,pe,pt,xt,Pe,_e,yt,Fe,B,Ve,z,H,fe,ce,Ye,qe,$e,et,C,ve,xe,Y,se,we,Ne,R,ye,tt,bt,re,me,Re,c,P,K,oe,ze,be,W,ft,Ie,ke,ee,je,We,Ge,ne,Ce,kt,Le,he,Ue,ue,ct,at,Wt,zt,mt,Ee,Te,Ke,de,Mt,ht,Gt,st,Ct,ut,dt,rt,jt,Ft,Et,Tt,At,gt,Je,_t,$t,Ut,vt,ot,s,_;return g=new qa({props:{id:"nvBXf7s7vTI"}}),le=new J({props:{code:`from transformers import AutoModelForSequenceClassification

model = AutoModelForSequenceClassification.from_pretrained("bert-base-cased", num_labels=5)`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoModelForSequenceClassification

<span class="hljs-meta">&gt;&gt;&gt; </span>model = AutoModelForSequenceClassification.from_pretrained(<span class="hljs-string">&quot;bert-base-cased&quot;</span>, num_labels=<span class="hljs-number">5</span>)`}}),te=new xa({props:{$$slots:{default:[us]},$$scope:{ctx:Me}}}),ie=new Lt({}),B=new J({props:{code:`from transformers import TrainingArguments

training_args = TrainingArguments(output_dir="test_trainer")`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> TrainingArguments

<span class="hljs-meta">&gt;&gt;&gt; </span>training_args = TrainingArguments(output_dir=<span class="hljs-string">&quot;test_trainer&quot;</span>)`}}),ce=new Lt({}),ze=new J({props:{code:`import numpy as np
from datasets import load_metric

metric = load_metric("accuracy")`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">import</span> numpy <span class="hljs-keyword">as</span> np
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> datasets <span class="hljs-keyword">import</span> load_metric

<span class="hljs-meta">&gt;&gt;&gt; </span>metric = load_metric(<span class="hljs-string">&quot;accuracy&quot;</span>)`}}),he=new J({props:{code:`def compute_metrics(eval_pred):
    logits, labels = eval_pred
    predictions = np.argmax(logits, axis=-1)
    return metric.compute(predictions=predictions, references=labels)`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">def</span> <span class="hljs-title function_">compute_metrics</span>(<span class="hljs-params">eval_pred</span>):
<span class="hljs-meta">... </span>    logits, labels = eval_pred
<span class="hljs-meta">... </span>    predictions = np.argmax(logits, axis=-<span class="hljs-number">1</span>)
<span class="hljs-meta">... </span>    <span class="hljs-keyword">return</span> metric.compute(predictions=predictions, references=labels)`}}),Ee=new J({props:{code:`from transformers import TrainingArguments, Trainer

training_args = TrainingArguments(output_dir="test_trainer", evaluation_strategy="epoch")`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> TrainingArguments, Trainer

<span class="hljs-meta">&gt;&gt;&gt; </span>training_args = TrainingArguments(output_dir=<span class="hljs-string">&quot;test_trainer&quot;</span>, evaluation_strategy=<span class="hljs-string">&quot;epoch&quot;</span>)`}}),ht=new Lt({}),At=new J({props:{code:`trainer = Trainer(
    model=model,
    args=training_args,
    train_dataset=small_train_dataset,
    eval_dataset=small_eval_dataset,
    compute_metrics=compute_metrics,
)`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>trainer = Trainer(
<span class="hljs-meta">... </span>    model=model,
<span class="hljs-meta">... </span>    args=training_args,
<span class="hljs-meta">... </span>    train_dataset=small_train_dataset,
<span class="hljs-meta">... </span>    eval_dataset=small_eval_dataset,
<span class="hljs-meta">... </span>    compute_metrics=compute_metrics,
<span class="hljs-meta">... </span>)`}}),s=new J({props:{code:"trainer.train()",highlighted:'<span class="hljs-meta">&gt;&gt;&gt; </span>trainer.train()'}}),{c(){w(g.$$.fragment),A=m(),d=o("p"),T=p("\u{1F917} Transformers provides a "),q=o("a"),N=p("Trainer"),X=p(" class optimized for training \u{1F917} Transformers models, making it easier to start training without manually writing your own training loop. The "),x=o("a"),D=p("Trainer"),I=p(" API supports a wide range of training options and features such as logging, gradient accumulation, and mixed precision."),S=m(),Z=o("p"),Q=p("Start by loading your model and specify the number of expected labels. From the Yelp Review "),Oe=o("a"),Be=p("dataset card"),M=p(", you know there are five labels:"),Se=m(),w(le.$$.fragment),G=m(),w(te.$$.fragment),ge=m(),Ae=o("h3"),ae=o("a"),Ze=o("span"),w(ie.$$.fragment),Qe=m(),U=o("span"),lt=p("Training hyperparameters"),F=m(),L=o("p"),V=p("Next, create a "),O=o("a"),it=p("TrainingArguments"),He=p(" class which contains all the hyperparameters you can tune as well as flags for activating different training options. For this tutorial you can start with the default training "),pe=o("a"),pt=p("hyperparameters"),xt=p(", but feel free to experiment with these to find your optimal settings."),Pe=m(),_e=o("p"),yt=p("Specify where to save the checkpoints from your training:"),Fe=m(),w(B.$$.fragment),Ve=m(),z=o("h3"),H=o("a"),fe=o("span"),w(ce.$$.fragment),Ye=m(),qe=o("span"),$e=p("Metrics"),et=m(),C=o("p"),ve=o("a"),xe=p("Trainer"),Y=p(" does not automatically evaluate model performance during training. You will need to pass "),se=o("a"),we=p("Trainer"),Ne=p(" a function to compute and report metrics. The \u{1F917} Datasets library provides a simple "),R=o("a"),ye=o("code"),tt=p("accuracy"),bt=p(" function you can load with the "),re=o("code"),me=p("load_metric"),Re=p(" (see this "),c=o("a"),P=p("tutorial"),K=p(" for more information) function:"),oe=m(),w(ze.$$.fragment),be=m(),W=o("p"),ft=p("Call "),Ie=o("code"),ke=p("compute"),ee=p(" on "),je=o("code"),We=p("metric"),Ge=p(" to calculate the accuracy of your predictions. Before passing your predictions to "),ne=o("code"),Ce=p("compute"),kt=p(", you need to convert the predictions to logits (remember all \u{1F917} Transformers models return logits):"),Le=m(),w(he.$$.fragment),Ue=m(),ue=o("p"),ct=p("If you\u2019d like to monitor your evaluation metrics during fine-tuning, specify the "),at=o("code"),Wt=p("evaluation_strategy"),zt=p(" parameter in your training arguments to report the evaluation metric at the end of each epoch:"),mt=m(),w(Ee.$$.fragment),Te=m(),Ke=o("h3"),de=o("a"),Mt=o("span"),w(ht.$$.fragment),Gt=m(),st=o("span"),Ct=p("Trainer"),ut=m(),dt=o("p"),rt=p("Create a "),jt=o("a"),Ft=p("Trainer"),Et=p(" object with your model, training arguments, training and test datasets, and evaluation function:"),Tt=m(),w(At.$$.fragment),gt=m(),Je=o("p"),_t=p("Then fine-tune your model by calling "),$t=o("a"),Ut=p("train()"),vt=p(":"),ot=m(),w(s.$$.fragment),this.h()},l(r){y(g.$$.fragment,r),A=h(r),d=n(r,"P",{});var v=l(d);T=f(v,"\u{1F917} Transformers provides a "),q=n(v,"A",{href:!0});var Kt=l(q);N=f(Kt,"Trainer"),Kt.forEach(e),X=f(v," class optimized for training \u{1F917} Transformers models, making it easier to start training without manually writing your own training loop. The "),x=n(v,"A",{href:!0});var ta=l(x);D=f(ta,"Trainer"),ta.forEach(e),I=f(v," API supports a wide range of training options and features such as logging, gradient accumulation, and mixed precision."),v.forEach(e),S=h(r),Z=n(r,"P",{});var Pt=l(Z);Q=f(Pt,"Start by loading your model and specify the number of expected labels. From the Yelp Review "),Oe=n(Pt,"A",{href:!0,rel:!0});var Ot=l(Oe);Be=f(Ot,"dataset card"),Ot.forEach(e),M=f(Pt,", you know there are five labels:"),Pt.forEach(e),Se=h(r),y(le.$$.fragment,r),G=h(r),y(te.$$.fragment,r),ge=h(r),Ae=n(r,"H3",{class:!0});var wt=l(Ae);ae=n(wt,"A",{id:!0,class:!0,href:!0});var la=l(ae);Ze=n(la,"SPAN",{});var ia=l(Ze);y(ie.$$.fragment,ia),ia.forEach(e),la.forEach(e),Qe=h(wt),U=n(wt,"SPAN",{});var aa=l(U);lt=f(aa,"Training hyperparameters"),aa.forEach(e),wt.forEach(e),F=h(r),L=n(r,"P",{});var Xe=l(L);V=f(Xe,"Next, create a "),O=n(Xe,"A",{href:!0});var sa=l(O);it=f(sa,"TrainingArguments"),sa.forEach(e),He=f(Xe," class which contains all the hyperparameters you can tune as well as flags for activating different training options. For this tutorial you can start with the default training "),pe=n(Xe,"A",{href:!0,rel:!0});var Dt=l(pe);pt=f(Dt,"hyperparameters"),Dt.forEach(e),xt=f(Xe,", but feel free to experiment with these to find your optimal settings."),Xe.forEach(e),Pe=h(r),_e=n(r,"P",{});var Nt=l(_e);yt=f(Nt,"Specify where to save the checkpoints from your training:"),Nt.forEach(e),Fe=h(r),y(B.$$.fragment,r),Ve=h(r),z=n(r,"H3",{class:!0});var Bt=l(z);H=n(Bt,"A",{id:!0,class:!0,href:!0});var St=l(H);fe=n(St,"SPAN",{});var pa=l(fe);y(ce.$$.fragment,pa),pa.forEach(e),St.forEach(e),Ye=h(Bt),qe=n(Bt,"SPAN",{});var Ht=l(qe);$e=f(Ht,"Metrics"),Ht.forEach(e),Bt.forEach(e),et=h(r),C=n(r,"P",{});var nt=l(C);ve=n(nt,"A",{href:!0});var ra=l(ve);xe=f(ra,"Trainer"),ra.forEach(e),Y=f(nt," does not automatically evaluate model performance during training. You will need to pass "),se=n(nt,"A",{href:!0});var qt=l(se);we=f(qt,"Trainer"),qt.forEach(e),Ne=f(nt," a function to compute and report metrics. The \u{1F917} Datasets library provides a simple "),R=n(nt,"A",{href:!0,rel:!0});var Qt=l(R);ye=n(Qt,"CODE",{});var Jt=l(ye);tt=f(Jt,"accuracy"),Jt.forEach(e),Qt.forEach(e),bt=f(nt," function you can load with the "),re=n(nt,"CODE",{});var fa=l(re);me=f(fa,"load_metric"),fa.forEach(e),Re=f(nt," (see this "),c=n(nt,"A",{href:!0,rel:!0});var ca=l(c);P=f(ca,"tutorial"),ca.forEach(e),K=f(nt," for more information) function:"),nt.forEach(e),oe=h(r),y(ze.$$.fragment,r),be=h(r),W=n(r,"P",{});var De=l(W);ft=f(De,"Call "),Ie=n(De,"CODE",{});var Vt=l(Ie);ke=f(Vt,"compute"),Vt.forEach(e),ee=f(De," on "),je=n(De,"CODE",{});var ma=l(je);We=f(ma,"metric"),ma.forEach(e),Ge=f(De," to calculate the accuracy of your predictions. Before passing your predictions to "),ne=n(De,"CODE",{});var ha=l(ne);Ce=f(ha,"compute"),ha.forEach(e),kt=f(De,", you need to convert the predictions to logits (remember all \u{1F917} Transformers models return logits):"),De.forEach(e),Le=h(r),y(he.$$.fragment,r),Ue=h(r),ue=n(r,"P",{});var It=l(ue);ct=f(It,"If you\u2019d like to monitor your evaluation metrics during fine-tuning, specify the "),at=n(It,"CODE",{});var Yt=l(at);Wt=f(Yt,"evaluation_strategy"),Yt.forEach(e),zt=f(It," parameter in your training arguments to report the evaluation metric at the end of each epoch:"),It.forEach(e),mt=h(r),y(Ee.$$.fragment,r),Te=h(r),Ke=n(r,"H3",{class:!0});var Xt=l(Ke);de=n(Xt,"A",{id:!0,class:!0,href:!0});var a=l(de);Mt=n(a,"SPAN",{});var $=l(Mt);y(ht.$$.fragment,$),$.forEach(e),a.forEach(e),Gt=h(Xt),st=n(Xt,"SPAN",{});var Rt=l(st);Ct=f(Rt,"Trainer"),Rt.forEach(e),Xt.forEach(e),ut=h(r),dt=n(r,"P",{});var oa=l(dt);rt=f(oa,"Create a "),jt=n(oa,"A",{href:!0});var ea=l(jt);Ft=f(ea,"Trainer"),ea.forEach(e),Et=f(oa," object with your model, training arguments, training and test datasets, and evaluation function:"),oa.forEach(e),Tt=h(r),y(At.$$.fragment,r),gt=h(r),Je=n(r,"P",{});var na=l(Je);_t=f(na,"Then fine-tune your model by calling "),$t=n(na,"A",{href:!0});var Zt=l($t);Ut=f(Zt,"train()"),Zt.forEach(e),vt=f(na,":"),na.forEach(e),ot=h(r),y(s.$$.fragment,r),this.h()},h(){u(q,"href","/docs/transformers/pr_17548/en/main_classes/trainer#transformers.Trainer"),u(x,"href","/docs/transformers/pr_17548/en/main_classes/trainer#transformers.Trainer"),u(Oe,"href","https://huggingface.co/datasets/yelp_review_full#data-fields"),u(Oe,"rel","nofollow"),u(ae,"id","training-hyperparameters"),u(ae,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),u(ae,"href","#training-hyperparameters"),u(Ae,"class","relative group"),u(O,"href","/docs/transformers/pr_17548/en/main_classes/trainer#transformers.TrainingArguments"),u(pe,"href","https://huggingface.co/docs/transformers/main_classes/trainer#transformers.TrainingArguments"),u(pe,"rel","nofollow"),u(H,"id","metrics"),u(H,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),u(H,"href","#metrics"),u(z,"class","relative group"),u(ve,"href","/docs/transformers/pr_17548/en/main_classes/trainer#transformers.Trainer"),u(se,"href","/docs/transformers/pr_17548/en/main_classes/trainer#transformers.Trainer"),u(R,"href","https://huggingface.co/metrics/accuracy"),u(R,"rel","nofollow"),u(c,"href","https://huggingface.co/docs/datasets/metrics.html"),u(c,"rel","nofollow"),u(de,"id","trainer"),u(de,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),u(de,"href","#trainer"),u(Ke,"class","relative group"),u(jt,"href","/docs/transformers/pr_17548/en/main_classes/trainer#transformers.Trainer"),u($t,"href","/docs/transformers/pr_17548/en/main_classes/trainer#transformers.Trainer.train")},m(r,v){b(g,r,v),i(r,A,v),i(r,d,v),t(d,T),t(d,q),t(q,N),t(d,X),t(d,x),t(x,D),t(d,I),i(r,S,v),i(r,Z,v),t(Z,Q),t(Z,Oe),t(Oe,Be),t(Z,M),i(r,Se,v),b(le,r,v),i(r,G,v),b(te,r,v),i(r,ge,v),i(r,Ae,v),t(Ae,ae),t(ae,Ze),b(ie,Ze,null),t(Ae,Qe),t(Ae,U),t(U,lt),i(r,F,v),i(r,L,v),t(L,V),t(L,O),t(O,it),t(L,He),t(L,pe),t(pe,pt),t(L,xt),i(r,Pe,v),i(r,_e,v),t(_e,yt),i(r,Fe,v),b(B,r,v),i(r,Ve,v),i(r,z,v),t(z,H),t(H,fe),b(ce,fe,null),t(z,Ye),t(z,qe),t(qe,$e),i(r,et,v),i(r,C,v),t(C,ve),t(ve,xe),t(C,Y),t(C,se),t(se,we),t(C,Ne),t(C,R),t(R,ye),t(ye,tt),t(C,bt),t(C,re),t(re,me),t(C,Re),t(C,c),t(c,P),t(C,K),i(r,oe,v),b(ze,r,v),i(r,be,v),i(r,W,v),t(W,ft),t(W,Ie),t(Ie,ke),t(W,ee),t(W,je),t(je,We),t(W,Ge),t(W,ne),t(ne,Ce),t(W,kt),i(r,Le,v),b(he,r,v),i(r,Ue,v),i(r,ue,v),t(ue,ct),t(ue,at),t(at,Wt),t(ue,zt),i(r,mt,v),b(Ee,r,v),i(r,Te,v),i(r,Ke,v),t(Ke,de),t(de,Mt),b(ht,Mt,null),t(Ke,Gt),t(Ke,st),t(st,Ct),i(r,ut,v),i(r,dt,v),t(dt,rt),t(dt,jt),t(jt,Ft),t(dt,Et),i(r,Tt,v),b(At,r,v),i(r,gt,v),i(r,Je,v),t(Je,_t),t(Je,$t),t($t,Ut),t(Je,vt),i(r,ot,v),b(s,r,v),_=!0},p(r,v){const Kt={};v&2&&(Kt.$$scope={dirty:v,ctx:r}),te.$set(Kt)},i(r){_||(k(g.$$.fragment,r),k(le.$$.fragment,r),k(te.$$.fragment,r),k(ie.$$.fragment,r),k(B.$$.fragment,r),k(ce.$$.fragment,r),k(ze.$$.fragment,r),k(he.$$.fragment,r),k(Ee.$$.fragment,r),k(ht.$$.fragment,r),k(At.$$.fragment,r),k(s.$$.fragment,r),_=!0)},o(r){j(g.$$.fragment,r),j(le.$$.fragment,r),j(te.$$.fragment,r),j(ie.$$.fragment,r),j(B.$$.fragment,r),j(ce.$$.fragment,r),j(ze.$$.fragment,r),j(he.$$.fragment,r),j(Ee.$$.fragment,r),j(ht.$$.fragment,r),j(At.$$.fragment,r),j(s.$$.fragment,r),_=!1},d(r){E(g,r),r&&e(A),r&&e(d),r&&e(S),r&&e(Z),r&&e(Se),E(le,r),r&&e(G),E(te,r),r&&e(ge),r&&e(Ae),E(ie),r&&e(F),r&&e(L),r&&e(Pe),r&&e(_e),r&&e(Fe),E(B,r),r&&e(Ve),r&&e(z),E(ce),r&&e(et),r&&e(C),r&&e(oe),E(ze,r),r&&e(be),r&&e(W),r&&e(Le),E(he,r),r&&e(Ue),r&&e(ue),r&&e(mt),E(Ee,r),r&&e(Te),r&&e(Ke),E(ht),r&&e(ut),r&&e(dt),r&&e(Tt),E(At,r),r&&e(gt),r&&e(Je),r&&e(ot),E(s,r)}}}function gs(Me){let g,A;return g=new za({props:{$$slots:{default:[ds]},$$scope:{ctx:Me}}}),{c(){w(g.$$.fragment)},l(d){y(g.$$.fragment,d)},m(d,T){b(g,d,T),A=!0},p(d,T){const q={};T&2&&(q.$$scope={dirty:T,ctx:d}),g.$set(q)},i(d){A||(k(g.$$.fragment,d),A=!0)},o(d){j(g.$$.fragment,d),A=!1},d(d){E(g,d)}}}function _s(Me){let g,A,d,T,q,N,X;return{c(){g=o("p"),A=o("a"),d=p("Trainer"),T=p(" uses "),q=o("a"),N=p("DataCollatorWithPadding"),X=p(" by default so you don\u2019t need to explicitly specify a data collator."),this.h()},l(x){g=n(x,"P",{});var D=l(g);A=n(D,"A",{href:!0});var I=l(A);d=f(I,"Trainer"),I.forEach(e),T=f(D," uses "),q=n(D,"A",{href:!0});var S=l(q);N=f(S,"DataCollatorWithPadding"),S.forEach(e),X=f(D," by default so you don\u2019t need to explicitly specify a data collator."),D.forEach(e),this.h()},h(){u(A,"href","/docs/transformers/pr_17548/en/main_classes/trainer#transformers.Trainer"),u(q,"href","/docs/transformers/pr_17548/en/main_classes/data_collator#transformers.DataCollatorWithPadding")},m(x,D){i(x,g,D),t(g,A),t(A,d),t(g,T),t(g,q),t(q,N),t(g,X)},d(x){x&&e(g)}}}function $s(Me){let g,A,d,T,q,N,X,x,D,I,S,Z,Q,Oe,Be,M,Se,le,G,te,ge,Ae,ae,Ze,ie,Qe,U,lt,F,L,V,O,it,He,pe,pt,xt,Pe,_e,yt,Fe,B,Ve,z,H,fe,ce,Ye,qe,$e,et,C,ve,xe,Y,se,we,Ne,R,ye,tt,bt,re,me,Re;return d=new qa({props:{id:"rnTGBy2ax1c"}}),S=new Lt({}),ie=new J({props:{code:`from transformers import DefaultDataCollator

data_collator = DefaultDataCollator(return_tensors="tf")`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> DefaultDataCollator

<span class="hljs-meta">&gt;&gt;&gt; </span>data_collator = DefaultDataCollator(return_tensors=<span class="hljs-string">&quot;tf&quot;</span>)`}}),U=new xa({props:{$$slots:{default:[_s]},$$scope:{ctx:Me}}}),B=new J({props:{code:`tf_train_dataset = small_train_dataset.to_tf_dataset(
    columns=["attention_mask", "input_ids", "token_type_ids"],
    label_cols=["labels"],
    shuffle=True,
    collate_fn=data_collator,
    batch_size=8,
)

tf_validation_dataset = small_eval_dataset.to_tf_dataset(
    columns=["attention_mask", "input_ids", "token_type_ids"],
    label_cols=["labels"],
    shuffle=False,
    collate_fn=data_collator,
    batch_size=8,
)`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>tf_train_dataset = small_train_dataset.to_tf_dataset(
<span class="hljs-meta">... </span>    columns=[<span class="hljs-string">&quot;attention_mask&quot;</span>, <span class="hljs-string">&quot;input_ids&quot;</span>, <span class="hljs-string">&quot;token_type_ids&quot;</span>],
<span class="hljs-meta">... </span>    label_cols=[<span class="hljs-string">&quot;labels&quot;</span>],
<span class="hljs-meta">... </span>    shuffle=<span class="hljs-literal">True</span>,
<span class="hljs-meta">... </span>    collate_fn=data_collator,
<span class="hljs-meta">... </span>    batch_size=<span class="hljs-number">8</span>,
<span class="hljs-meta">... </span>)

<span class="hljs-meta">&gt;&gt;&gt; </span>tf_validation_dataset = small_eval_dataset.to_tf_dataset(
<span class="hljs-meta">... </span>    columns=[<span class="hljs-string">&quot;attention_mask&quot;</span>, <span class="hljs-string">&quot;input_ids&quot;</span>, <span class="hljs-string">&quot;token_type_ids&quot;</span>],
<span class="hljs-meta">... </span>    label_cols=[<span class="hljs-string">&quot;labels&quot;</span>],
<span class="hljs-meta">... </span>    shuffle=<span class="hljs-literal">False</span>,
<span class="hljs-meta">... </span>    collate_fn=data_collator,
<span class="hljs-meta">... </span>    batch_size=<span class="hljs-number">8</span>,
<span class="hljs-meta">... </span>)`}}),ce=new Lt({}),Y=new J({props:{code:`import tensorflow as tf
from transformers import TFAutoModelForSequenceClassification

model = TFAutoModelForSequenceClassification.from_pretrained("bert-base-cased", num_labels=5)`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">import</span> tensorflow <span class="hljs-keyword">as</span> tf
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> TFAutoModelForSequenceClassification

<span class="hljs-meta">&gt;&gt;&gt; </span>model = TFAutoModelForSequenceClassification.from_pretrained(<span class="hljs-string">&quot;bert-base-cased&quot;</span>, num_labels=<span class="hljs-number">5</span>)`}}),me=new J({props:{code:`model.compile(
    optimizer=tf.keras.optimizers.Adam(learning_rate=5e-5),
    loss=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True),
    metrics=tf.metrics.SparseCategoricalAccuracy(),
)

model.fit(tf_train_dataset, validation_data=tf_validation_dataset, epochs=3)`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>model.<span class="hljs-built_in">compile</span>(
<span class="hljs-meta">... </span>    optimizer=tf.keras.optimizers.Adam(learning_rate=<span class="hljs-number">5e-5</span>),
<span class="hljs-meta">... </span>    loss=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=<span class="hljs-literal">True</span>),
<span class="hljs-meta">... </span>    metrics=tf.metrics.SparseCategoricalAccuracy(),
<span class="hljs-meta">... </span>)

<span class="hljs-meta">&gt;&gt;&gt; </span>model.fit(tf_train_dataset, validation_data=tf_validation_dataset, epochs=<span class="hljs-number">3</span>)`}}),{c(){g=o("a"),A=m(),w(d.$$.fragment),T=m(),q=o("p"),N=p("\u{1F917} Transformers models also supports training in TensorFlow with the Keras API."),X=m(),x=o("h3"),D=o("a"),I=o("span"),w(S.$$.fragment),Z=m(),Q=o("span"),Oe=p("Convert dataset to TensorFlow format"),Be=m(),M=o("p"),Se=p("The "),le=o("a"),G=p("DefaultDataCollator"),te=p(" assembles tensors into a batch for the model to train on. Make sure you specify "),ge=o("code"),Ae=p("return_tensors"),ae=p(" to return TensorFlow tensors:"),Ze=m(),w(ie.$$.fragment),Qe=m(),w(U.$$.fragment),lt=m(),F=o("p"),L=p("Next, convert the tokenized datasets to TensorFlow datasets with the "),V=o("a"),O=o("code"),it=p("to_tf_dataset"),He=p(" method. Specify your inputs in "),pe=o("code"),pt=p("columns"),xt=p(", and your label in "),Pe=o("code"),_e=p("label_cols"),yt=p(":"),Fe=m(),w(B.$$.fragment),Ve=m(),z=o("h3"),H=o("a"),fe=o("span"),w(ce.$$.fragment),Ye=m(),qe=o("span"),$e=p("Compile and fit"),et=m(),C=o("p"),ve=p("Load a TensorFlow model with the expected number of labels:"),xe=m(),w(Y.$$.fragment),se=m(),we=o("p"),Ne=p("Then compile and fine-tune your model with "),R=o("a"),ye=o("code"),tt=p("fit"),bt=p(" as you would with any other Keras model:"),re=m(),w(me.$$.fragment),this.h()},l(c){g=n(c,"A",{id:!0}),l(g).forEach(e),A=h(c),y(d.$$.fragment,c),T=h(c),q=n(c,"P",{});var P=l(q);N=f(P,"\u{1F917} Transformers models also supports training in TensorFlow with the Keras API."),P.forEach(e),X=h(c),x=n(c,"H3",{class:!0});var K=l(x);D=n(K,"A",{id:!0,class:!0,href:!0});var oe=l(D);I=n(oe,"SPAN",{});var ze=l(I);y(S.$$.fragment,ze),ze.forEach(e),oe.forEach(e),Z=h(K),Q=n(K,"SPAN",{});var be=l(Q);Oe=f(be,"Convert dataset to TensorFlow format"),be.forEach(e),K.forEach(e),Be=h(c),M=n(c,"P",{});var W=l(M);Se=f(W,"The "),le=n(W,"A",{href:!0});var ft=l(le);G=f(ft,"DefaultDataCollator"),ft.forEach(e),te=f(W," assembles tensors into a batch for the model to train on. Make sure you specify "),ge=n(W,"CODE",{});var Ie=l(ge);Ae=f(Ie,"return_tensors"),Ie.forEach(e),ae=f(W," to return TensorFlow tensors:"),W.forEach(e),Ze=h(c),y(ie.$$.fragment,c),Qe=h(c),y(U.$$.fragment,c),lt=h(c),F=n(c,"P",{});var ke=l(F);L=f(ke,"Next, convert the tokenized datasets to TensorFlow datasets with the "),V=n(ke,"A",{href:!0,rel:!0});var ee=l(V);O=n(ee,"CODE",{});var je=l(O);it=f(je,"to_tf_dataset"),je.forEach(e),ee.forEach(e),He=f(ke," method. Specify your inputs in "),pe=n(ke,"CODE",{});var We=l(pe);pt=f(We,"columns"),We.forEach(e),xt=f(ke,", and your label in "),Pe=n(ke,"CODE",{});var Ge=l(Pe);_e=f(Ge,"label_cols"),Ge.forEach(e),yt=f(ke,":"),ke.forEach(e),Fe=h(c),y(B.$$.fragment,c),Ve=h(c),z=n(c,"H3",{class:!0});var ne=l(z);H=n(ne,"A",{id:!0,class:!0,href:!0});var Ce=l(H);fe=n(Ce,"SPAN",{});var kt=l(fe);y(ce.$$.fragment,kt),kt.forEach(e),Ce.forEach(e),Ye=h(ne),qe=n(ne,"SPAN",{});var Le=l(qe);$e=f(Le,"Compile and fit"),Le.forEach(e),ne.forEach(e),et=h(c),C=n(c,"P",{});var he=l(C);ve=f(he,"Load a TensorFlow model with the expected number of labels:"),he.forEach(e),xe=h(c),y(Y.$$.fragment,c),se=h(c),we=n(c,"P",{});var Ue=l(we);Ne=f(Ue,"Then compile and fine-tune your model with "),R=n(Ue,"A",{href:!0,rel:!0});var ue=l(R);ye=n(ue,"CODE",{});var ct=l(ye);tt=f(ct,"fit"),ct.forEach(e),ue.forEach(e),bt=f(Ue," as you would with any other Keras model:"),Ue.forEach(e),re=h(c),y(me.$$.fragment,c),this.h()},h(){u(g,"id","keras"),u(D,"id","convert-dataset-to-tensorflow-format"),u(D,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),u(D,"href","#convert-dataset-to-tensorflow-format"),u(x,"class","relative group"),u(le,"href","/docs/transformers/pr_17548/en/main_classes/data_collator#transformers.DefaultDataCollator"),u(V,"href","https://huggingface.co/docs/datasets/package_reference/main_classes.html#datasets.Dataset.to_tf_dataset"),u(V,"rel","nofollow"),u(H,"id","compile-and-fit"),u(H,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),u(H,"href","#compile-and-fit"),u(z,"class","relative group"),u(R,"href","https://keras.io/api/models/model_training_apis/"),u(R,"rel","nofollow")},m(c,P){i(c,g,P),i(c,A,P),b(d,c,P),i(c,T,P),i(c,q,P),t(q,N),i(c,X,P),i(c,x,P),t(x,D),t(D,I),b(S,I,null),t(x,Z),t(x,Q),t(Q,Oe),i(c,Be,P),i(c,M,P),t(M,Se),t(M,le),t(le,G),t(M,te),t(M,ge),t(ge,Ae),t(M,ae),i(c,Ze,P),b(ie,c,P),i(c,Qe,P),b(U,c,P),i(c,lt,P),i(c,F,P),t(F,L),t(F,V),t(V,O),t(O,it),t(F,He),t(F,pe),t(pe,pt),t(F,xt),t(F,Pe),t(Pe,_e),t(F,yt),i(c,Fe,P),b(B,c,P),i(c,Ve,P),i(c,z,P),t(z,H),t(H,fe),b(ce,fe,null),t(z,Ye),t(z,qe),t(qe,$e),i(c,et,P),i(c,C,P),t(C,ve),i(c,xe,P),b(Y,c,P),i(c,se,P),i(c,we,P),t(we,Ne),t(we,R),t(R,ye),t(ye,tt),t(we,bt),i(c,re,P),b(me,c,P),Re=!0},p(c,P){const K={};P&2&&(K.$$scope={dirty:P,ctx:c}),U.$set(K)},i(c){Re||(k(d.$$.fragment,c),k(S.$$.fragment,c),k(ie.$$.fragment,c),k(U.$$.fragment,c),k(B.$$.fragment,c),k(ce.$$.fragment,c),k(Y.$$.fragment,c),k(me.$$.fragment,c),Re=!0)},o(c){j(d.$$.fragment,c),j(S.$$.fragment,c),j(ie.$$.fragment,c),j(U.$$.fragment,c),j(B.$$.fragment,c),j(ce.$$.fragment,c),j(Y.$$.fragment,c),j(me.$$.fragment,c),Re=!1},d(c){c&&e(g),c&&e(A),E(d,c),c&&e(T),c&&e(q),c&&e(X),c&&e(x),E(S),c&&e(Be),c&&e(M),c&&e(Ze),E(ie,c),c&&e(Qe),E(U,c),c&&e(lt),c&&e(F),c&&e(Fe),E(B,c),c&&e(Ve),c&&e(z),E(ce),c&&e(et),c&&e(C),c&&e(xe),E(Y,c),c&&e(se),c&&e(we),c&&e(re),E(me,c)}}}function vs(Me){let g,A;return g=new za({props:{$$slots:{default:[$s]},$$scope:{ctx:Me}}}),{c(){w(g.$$.fragment)},l(d){y(g.$$.fragment,d)},m(d,T){b(g,d,T),A=!0},p(d,T){const q={};T&2&&(q.$$scope={dirty:T,ctx:d}),g.$set(q)},i(d){A||(k(g.$$.fragment,d),A=!0)},o(d){j(g.$$.fragment,d),A=!1},d(d){E(g,d)}}}function ws(Me){let g,A,d,T,q,N,X,x;return{c(){g=o("p"),A=p("Get free access to a cloud GPU if you don\u2019t have one with a hosted notebook like "),d=o("a"),T=p("Colaboratory"),q=p(" or "),N=o("a"),X=p("SageMaker StudioLab"),x=p("."),this.h()},l(D){g=n(D,"P",{});var I=l(g);A=f(I,"Get free access to a cloud GPU if you don\u2019t have one with a hosted notebook like "),d=n(I,"A",{href:!0,rel:!0});var S=l(d);T=f(S,"Colaboratory"),S.forEach(e),q=f(I," or "),N=n(I,"A",{href:!0,rel:!0});var Z=l(N);X=f(Z,"SageMaker StudioLab"),Z.forEach(e),x=f(I,"."),I.forEach(e),this.h()},h(){u(d,"href","https://colab.research.google.com/"),u(d,"rel","nofollow"),u(N,"href","https://studiolab.sagemaker.aws/"),u(N,"rel","nofollow")},m(D,I){i(D,g,I),t(g,A),t(g,d),t(d,T),t(g,q),t(g,N),t(N,X),t(g,x)},d(D){D&&e(g)}}}function ys(Me){let g,A,d,T,q,N,X,x,D,I,S,Z,Q,Oe,Be,M,Se,le,G,te,ge,Ae,ae,Ze,ie,Qe,U,lt,F,L,V,O,it,He,pe,pt,xt,Pe,_e,yt,Fe,B,Ve,z,H,fe,ce,Ye,qe,$e,et,C,ve,xe,Y,se,we,Ne,R,ye,tt,bt,re,me,Re,c,P,K,oe,ze,be,W,ft,Ie,ke,ee,je,We,Ge,ne,Ce,kt,Le,he,Ue,ue,ct,at,Wt,zt,mt,Ee,Te,Ke,de,Mt,ht,Gt,st,Ct,ut,dt,rt,jt,Ft,Et,Tt,At,gt,Je,_t,$t,Ut,vt,ot,s,_,r,v,Kt,ta,Pt,Ot,wt,la,ia,aa,Xe,sa,Dt,Nt,Bt,St,pa,Ht,nt,ra,qt,Qt,Jt,fa,ca,De,Vt,ma,ha,It,Yt,Xt;return g=new qa({props:{id:"Dh9CL8fyG80"}}),S=new J({props:{code:`del model
del pytorch_model
del trainer
torch.cuda.empty_cache()`,highlighted:`<span class="hljs-keyword">del</span> model
<span class="hljs-keyword">del</span> pytorch_model
<span class="hljs-keyword">del</span> trainer
torch.cuda.empty_cache()`}}),U=new J({props:{code:'tokenized_datasets = tokenized_datasets.remove_columns(["text"])',highlighted:'<span class="hljs-meta">&gt;&gt;&gt; </span>tokenized_datasets = tokenized_datasets.remove_columns([<span class="hljs-string">&quot;text&quot;</span>])'}}),B=new J({props:{code:'tokenized_datasets = tokenized_datasets.rename_column("label", "labels")',highlighted:'<span class="hljs-meta">&gt;&gt;&gt; </span>tokenized_datasets = tokenized_datasets.rename_column(<span class="hljs-string">&quot;label&quot;</span>, <span class="hljs-string">&quot;labels&quot;</span>)'}}),Ye=new J({props:{code:'tokenized_datasets.set_format("torch")',highlighted:'<span class="hljs-meta">&gt;&gt;&gt; </span>tokenized_datasets.set_format(<span class="hljs-string">&quot;torch&quot;</span>)'}}),ve=new J({props:{code:`small_train_dataset = tokenized_datasets["train"].shuffle(seed=42).select(range(1000))
small_eval_dataset = tokenized_datasets["test"].shuffle(seed=42).select(range(1000))`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>small_train_dataset = tokenized_datasets[<span class="hljs-string">&quot;train&quot;</span>].shuffle(seed=<span class="hljs-number">42</span>).select(<span class="hljs-built_in">range</span>(<span class="hljs-number">1000</span>))
<span class="hljs-meta">&gt;&gt;&gt; </span>small_eval_dataset = tokenized_datasets[<span class="hljs-string">&quot;test&quot;</span>].shuffle(seed=<span class="hljs-number">42</span>).select(<span class="hljs-built_in">range</span>(<span class="hljs-number">1000</span>))`}}),Ne=new Lt({}),oe=new J({props:{code:`from torch.utils.data import DataLoader

train_dataloader = DataLoader(small_train_dataset, shuffle=True, batch_size=8)
eval_dataloader = DataLoader(small_eval_dataset, batch_size=8)`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> torch.utils.data <span class="hljs-keyword">import</span> DataLoader

<span class="hljs-meta">&gt;&gt;&gt; </span>train_dataloader = DataLoader(small_train_dataset, shuffle=<span class="hljs-literal">True</span>, batch_size=<span class="hljs-number">8</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>eval_dataloader = DataLoader(small_eval_dataset, batch_size=<span class="hljs-number">8</span>)`}}),Ie=new J({props:{code:`from transformers import AutoModelForSequenceClassification

model = AutoModelForSequenceClassification.from_pretrained("bert-base-cased", num_labels=5)`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoModelForSequenceClassification

<span class="hljs-meta">&gt;&gt;&gt; </span>model = AutoModelForSequenceClassification.from_pretrained(<span class="hljs-string">&quot;bert-base-cased&quot;</span>, num_labels=<span class="hljs-number">5</span>)`}}),Ge=new Lt({}),mt=new J({props:{code:`from torch.optim import AdamW

optimizer = AdamW(model.parameters(), lr=5e-5)`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> torch.optim <span class="hljs-keyword">import</span> AdamW

<span class="hljs-meta">&gt;&gt;&gt; </span>optimizer = AdamW(model.parameters(), lr=<span class="hljs-number">5e-5</span>)`}}),st=new J({props:{code:`from transformers import get_scheduler

num_epochs = 3
num_training_steps = num_epochs * len(train_dataloader)
lr_scheduler = get_scheduler(
    name="linear", optimizer=optimizer, num_warmup_steps=0, num_training_steps=num_training_steps
)`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> get_scheduler

<span class="hljs-meta">&gt;&gt;&gt; </span>num_epochs = <span class="hljs-number">3</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>num_training_steps = num_epochs * <span class="hljs-built_in">len</span>(train_dataloader)
<span class="hljs-meta">&gt;&gt;&gt; </span>lr_scheduler = get_scheduler(
<span class="hljs-meta">... </span>    name=<span class="hljs-string">&quot;linear&quot;</span>, optimizer=optimizer, num_warmup_steps=<span class="hljs-number">0</span>, num_training_steps=num_training_steps
<span class="hljs-meta">... </span>)`}}),Tt=new J({props:{code:`import torch

device = torch.device("cuda") if torch.cuda.is_available() else torch.device("cpu")
model.to(device)`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">import</span> torch

<span class="hljs-meta">&gt;&gt;&gt; </span>device = torch.device(<span class="hljs-string">&quot;cuda&quot;</span>) <span class="hljs-keyword">if</span> torch.cuda.is_available() <span class="hljs-keyword">else</span> torch.device(<span class="hljs-string">&quot;cpu&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>model.to(device)`}}),gt=new xa({props:{$$slots:{default:[ws]},$$scope:{ctx:Me}}}),_=new Lt({}),Xe=new J({props:{code:`from tqdm.auto import tqdm

progress_bar = tqdm(range(num_training_steps))

model.train()
for epoch in range(num_epochs):
    for batch in train_dataloader:
        batch = {k: v.to(device) for k, v in batch.items()}
        outputs = model(**batch)
        loss = outputs.loss
        loss.backward()

        optimizer.step()
        lr_scheduler.step()
        optimizer.zero_grad()
        progress_bar.update(1)`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> tqdm.auto <span class="hljs-keyword">import</span> tqdm

<span class="hljs-meta">&gt;&gt;&gt; </span>progress_bar = tqdm(<span class="hljs-built_in">range</span>(num_training_steps))

<span class="hljs-meta">&gt;&gt;&gt; </span>model.train()
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">for</span> epoch <span class="hljs-keyword">in</span> <span class="hljs-built_in">range</span>(num_epochs):
<span class="hljs-meta">... </span>    <span class="hljs-keyword">for</span> batch <span class="hljs-keyword">in</span> train_dataloader:
<span class="hljs-meta">... </span>        batch = {k: v.to(device) <span class="hljs-keyword">for</span> k, v <span class="hljs-keyword">in</span> batch.items()}
<span class="hljs-meta">... </span>        outputs = model(**batch)
<span class="hljs-meta">... </span>        loss = outputs.loss
<span class="hljs-meta">... </span>        loss.backward()

<span class="hljs-meta">... </span>        optimizer.step()
<span class="hljs-meta">... </span>        lr_scheduler.step()
<span class="hljs-meta">... </span>        optimizer.zero_grad()
<span class="hljs-meta">... </span>        progress_bar.update(<span class="hljs-number">1</span>)`}}),St=new Lt({}),Yt=new J({props:{code:`metric = load_metric("accuracy")
model.eval()
for batch in eval_dataloader:
    batch = {k: v.to(device) for k, v in batch.items()}
    with torch.no_grad():
        outputs = model(**batch)

    logits = outputs.logits
    predictions = torch.argmax(logits, dim=-1)
    metric.add_batch(predictions=predictions, references=batch["labels"])

metric.compute()`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>metric = load_metric(<span class="hljs-string">&quot;accuracy&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>model.<span class="hljs-built_in">eval</span>()
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">for</span> batch <span class="hljs-keyword">in</span> eval_dataloader:
<span class="hljs-meta">... </span>    batch = {k: v.to(device) <span class="hljs-keyword">for</span> k, v <span class="hljs-keyword">in</span> batch.items()}
<span class="hljs-meta">... </span>    <span class="hljs-keyword">with</span> torch.no_grad():
<span class="hljs-meta">... </span>        outputs = model(**batch)

<span class="hljs-meta">... </span>    logits = outputs.logits
<span class="hljs-meta">... </span>    predictions = torch.argmax(logits, dim=-<span class="hljs-number">1</span>)
<span class="hljs-meta">... </span>    metric.add_batch(predictions=predictions, references=batch[<span class="hljs-string">&quot;labels&quot;</span>])

<span class="hljs-meta">&gt;&gt;&gt; </span>metric.compute()`}}),{c(){w(g.$$.fragment),A=m(),d=o("p"),T=o("a"),q=p("Trainer"),N=p(" takes care of the training loop and allows you to fine-tune a model in a single line of code. For users who prefer to write their own training loop, you can also fine-tune a \u{1F917} Transformers model in native PyTorch."),X=m(),x=o("p"),D=p("At this point, you may need to restart your notebook or execute the following code to free some memory:"),I=m(),w(S.$$.fragment),Z=m(),Q=o("p"),Oe=p("Next, manually postprocess "),Be=o("code"),M=p("tokenized_dataset"),Se=p(" to prepare it for training."),le=m(),G=o("ol"),te=o("li"),ge=o("p"),Ae=p("Remove the "),ae=o("code"),Ze=p("text"),ie=p(" column because the model does not accept raw text as an input:"),Qe=m(),w(U.$$.fragment),lt=m(),F=o("li"),L=o("p"),V=p("Rename the "),O=o("code"),it=p("label"),He=p(" column to "),pe=o("code"),pt=p("labels"),xt=p(" because the model expects the argument to be named "),Pe=o("code"),_e=p("labels"),yt=p(":"),Fe=m(),w(B.$$.fragment),Ve=m(),z=o("li"),H=o("p"),fe=p("Set the format of the dataset to return PyTorch tensors instead of lists:"),ce=m(),w(Ye.$$.fragment),qe=m(),$e=o("p"),et=p("Then create a smaller subset of the dataset as previously shown to speed up the fine-tuning:"),C=m(),w(ve.$$.fragment),xe=m(),Y=o("h3"),se=o("a"),we=o("span"),w(Ne.$$.fragment),R=m(),ye=o("span"),tt=p("DataLoader"),bt=m(),re=o("p"),me=p("Create a "),Re=o("code"),c=p("DataLoader"),P=p(" for your training and test datasets so you can iterate over batches of data:"),K=m(),w(oe.$$.fragment),ze=m(),be=o("p"),W=p("Load your model with the number of expected labels:"),ft=m(),w(Ie.$$.fragment),ke=m(),ee=o("h3"),je=o("a"),We=o("span"),w(Ge.$$.fragment),ne=m(),Ce=o("span"),kt=p("Optimizer and learning rate scheduler"),Le=m(),he=o("p"),Ue=p("Create an optimizer and learning rate scheduler to fine-tune the model. Let\u2019s use the "),ue=o("a"),ct=o("code"),at=p("AdamW"),Wt=p(" optimizer from PyTorch:"),zt=m(),w(mt.$$.fragment),Ee=m(),Te=o("p"),Ke=p("Create the default learning rate scheduler from "),de=o("a"),Mt=p("Trainer"),ht=p(":"),Gt=m(),w(st.$$.fragment),Ct=m(),ut=o("p"),dt=p("Lastly, specify "),rt=o("code"),jt=p("device"),Ft=p(" to use a GPU if you have access to one. Otherwise, training on a CPU may take several hours instead of a couple of minutes."),Et=m(),w(Tt.$$.fragment),At=m(),w(gt.$$.fragment),Je=m(),_t=o("p"),$t=p("Great, now you are ready to train! \u{1F973}"),Ut=m(),vt=o("h3"),ot=o("a"),s=o("span"),w(_.$$.fragment),r=m(),v=o("span"),Kt=p("Training loop"),ta=m(),Pt=o("p"),Ot=p("To keep track of your training progress, use the "),wt=o("a"),la=p("tqdm"),ia=p(" library to add a progress bar over the number of training steps:"),aa=m(),w(Xe.$$.fragment),sa=m(),Dt=o("h3"),Nt=o("a"),Bt=o("span"),w(St.$$.fragment),pa=m(),Ht=o("span"),nt=p("Metrics"),ra=m(),qt=o("p"),Qt=p("Just like how you need to add an evaluation function to "),Jt=o("a"),fa=p("Trainer"),ca=p(", you need to do the same when you write your own training loop. But instead of calculating and reporting the metric at the end of each epoch, this time you will accumulate all the batches with "),De=o("a"),Vt=o("code"),ma=p("add_batch"),ha=p(" and calculate the metric at the very end."),It=m(),w(Yt.$$.fragment),this.h()},l(a){y(g.$$.fragment,a),A=h(a),d=n(a,"P",{});var $=l(d);T=n($,"A",{href:!0});var Rt=l(T);q=f(Rt,"Trainer"),Rt.forEach(e),N=f($," takes care of the training loop and allows you to fine-tune a model in a single line of code. For users who prefer to write their own training loop, you can also fine-tune a \u{1F917} Transformers model in native PyTorch."),$.forEach(e),X=h(a),x=n(a,"P",{});var oa=l(x);D=f(oa,"At this point, you may need to restart your notebook or execute the following code to free some memory:"),oa.forEach(e),I=h(a),y(S.$$.fragment,a),Z=h(a),Q=n(a,"P",{});var ea=l(Q);Oe=f(ea,"Next, manually postprocess "),Be=n(ea,"CODE",{});var na=l(Be);M=f(na,"tokenized_dataset"),na.forEach(e),Se=f(ea," to prepare it for training."),ea.forEach(e),le=h(a),G=n(a,"OL",{});var Zt=l(G);te=n(Zt,"LI",{});var ua=l(te);ge=n(ua,"P",{});var ga=l(ge);Ae=f(ga,"Remove the "),ae=n(ga,"CODE",{});var Ca=l(ae);Ze=f(Ca,"text"),Ca.forEach(e),ie=f(ga," column because the model does not accept raw text as an input:"),ga.forEach(e),Qe=h(ua),y(U.$$.fragment,ua),ua.forEach(e),lt=h(Zt),F=n(Zt,"LI",{});var $a=l(F);L=n($a,"P",{});var da=l(L);V=f(da,"Rename the "),O=n(da,"CODE",{});var Da=l(O);it=f(Da,"label"),Da.forEach(e),He=f(da," column to "),pe=n(da,"CODE",{});var Sa=l(pe);pt=f(Sa,"labels"),Sa.forEach(e),xt=f(da," because the model expects the argument to be named "),Pe=n(da,"CODE",{});var Fa=l(Pe);_e=f(Fa,"labels"),Fa.forEach(e),yt=f(da,":"),da.forEach(e),Fe=h($a),y(B.$$.fragment,$a),$a.forEach(e),Ve=h(Zt),z=n(Zt,"LI",{});var va=l(z);H=n(va,"P",{});var Na=l(H);fe=f(Na,"Set the format of the dataset to return PyTorch tensors instead of lists:"),Na.forEach(e),ce=h(va),y(Ye.$$.fragment,va),va.forEach(e),Zt.forEach(e),qe=h(a),$e=n(a,"P",{});var Ia=l($e);et=f(Ia,"Then create a smaller subset of the dataset as previously shown to speed up the fine-tuning:"),Ia.forEach(e),C=h(a),y(ve.$$.fragment,a),xe=h(a),Y=n(a,"H3",{class:!0});var wa=l(Y);se=n(wa,"A",{id:!0,class:!0,href:!0});var La=l(se);we=n(La,"SPAN",{});var Ma=l(we);y(Ne.$$.fragment,Ma),Ma.forEach(e),La.forEach(e),R=h(wa),ye=n(wa,"SPAN",{});var Oa=l(ye);tt=f(Oa,"DataLoader"),Oa.forEach(e),wa.forEach(e),bt=h(a),re=n(a,"P",{});var ya=l(re);me=f(ya,"Create a "),Re=n(ya,"CODE",{});var Ba=l(Re);c=f(Ba,"DataLoader"),Ba.forEach(e),P=f(ya," for your training and test datasets so you can iterate over batches of data:"),ya.forEach(e),K=h(a),y(oe.$$.fragment,a),ze=h(a),be=n(a,"P",{});var Ha=l(be);W=f(Ha,"Load your model with the number of expected labels:"),Ha.forEach(e),ft=h(a),y(Ie.$$.fragment,a),ke=h(a),ee=n(a,"H3",{class:!0});var ba=l(ee);je=n(ba,"A",{id:!0,class:!0,href:!0});var Ya=l(je);We=n(Ya,"SPAN",{});var Ra=l(We);y(Ge.$$.fragment,Ra),Ra.forEach(e),Ya.forEach(e),ne=h(ba),Ce=n(ba,"SPAN",{});var Wa=l(Ce);kt=f(Wa,"Optimizer and learning rate scheduler"),Wa.forEach(e),ba.forEach(e),Le=h(a),he=n(a,"P",{});var ka=l(he);Ue=f(ka,"Create an optimizer and learning rate scheduler to fine-tune the model. Let\u2019s use the "),ue=n(ka,"A",{href:!0,rel:!0});var Ga=l(ue);ct=n(Ga,"CODE",{});var Ua=l(ct);at=f(Ua,"AdamW"),Ua.forEach(e),Ga.forEach(e),Wt=f(ka," optimizer from PyTorch:"),ka.forEach(e),zt=h(a),y(mt.$$.fragment,a),Ee=h(a),Te=n(a,"P",{});var ja=l(Te);Ke=f(ja,"Create the default learning rate scheduler from "),de=n(ja,"A",{href:!0});var Ka=l(de);Mt=f(Ka,"Trainer"),Ka.forEach(e),ht=f(ja,":"),ja.forEach(e),Gt=h(a),y(st.$$.fragment,a),Ct=h(a),ut=n(a,"P",{});var Ea=l(ut);dt=f(Ea,"Lastly, specify "),rt=n(Ea,"CODE",{});var Ja=l(rt);jt=f(Ja,"device"),Ja.forEach(e),Ft=f(Ea," to use a GPU if you have access to one. Otherwise, training on a CPU may take several hours instead of a couple of minutes."),Ea.forEach(e),Et=h(a),y(Tt.$$.fragment,a),At=h(a),y(gt.$$.fragment,a),Je=h(a),_t=n(a,"P",{});var Xa=l(_t);$t=f(Xa,"Great, now you are ready to train! \u{1F973}"),Xa.forEach(e),Ut=h(a),vt=n(a,"H3",{class:!0});var Ta=l(vt);ot=n(Ta,"A",{id:!0,class:!0,href:!0});var Za=l(ot);s=n(Za,"SPAN",{});var Qa=l(s);y(_.$$.fragment,Qa),Qa.forEach(e),Za.forEach(e),r=h(Ta),v=n(Ta,"SPAN",{});var Va=l(v);Kt=f(Va,"Training loop"),Va.forEach(e),Ta.forEach(e),ta=h(a),Pt=n(a,"P",{});var Aa=l(Pt);Ot=f(Aa,"To keep track of your training progress, use the "),wt=n(Aa,"A",{href:!0,rel:!0});var es=l(wt);la=f(es,"tqdm"),es.forEach(e),ia=f(Aa," library to add a progress bar over the number of training steps:"),Aa.forEach(e),aa=h(a),y(Xe.$$.fragment,a),sa=h(a),Dt=n(a,"H3",{class:!0});var Pa=l(Dt);Nt=n(Pa,"A",{id:!0,class:!0,href:!0});var ts=l(Nt);Bt=n(ts,"SPAN",{});var as=l(Bt);y(St.$$.fragment,as),as.forEach(e),ts.forEach(e),pa=h(Pa),Ht=n(Pa,"SPAN",{});var ss=l(Ht);nt=f(ss,"Metrics"),ss.forEach(e),Pa.forEach(e),ra=h(a),qt=n(a,"P",{});var _a=l(qt);Qt=f(_a,"Just like how you need to add an evaluation function to "),Jt=n(_a,"A",{href:!0});var rs=l(Jt);fa=f(rs,"Trainer"),rs.forEach(e),ca=f(_a,", you need to do the same when you write your own training loop. But instead of calculating and reporting the metric at the end of each epoch, this time you will accumulate all the batches with "),De=n(_a,"A",{href:!0,rel:!0});var os=l(De);Vt=n(os,"CODE",{});var ns=l(Vt);ma=f(ns,"add_batch"),ns.forEach(e),os.forEach(e),ha=f(_a," and calculate the metric at the very end."),_a.forEach(e),It=h(a),y(Yt.$$.fragment,a),this.h()},h(){u(T,"href","/docs/transformers/pr_17548/en/main_classes/trainer#transformers.Trainer"),u(se,"id","dataloader"),u(se,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),u(se,"href","#dataloader"),u(Y,"class","relative group"),u(je,"id","optimizer-and-learning-rate-scheduler"),u(je,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),u(je,"href","#optimizer-and-learning-rate-scheduler"),u(ee,"class","relative group"),u(ue,"href","https://pytorch.org/docs/stable/generated/torch.optim.AdamW.html"),u(ue,"rel","nofollow"),u(de,"href","/docs/transformers/pr_17548/en/main_classes/trainer#transformers.Trainer"),u(ot,"id","training-loop"),u(ot,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),u(ot,"href","#training-loop"),u(vt,"class","relative group"),u(wt,"href","https://tqdm.github.io/"),u(wt,"rel","nofollow"),u(Nt,"id","metrics"),u(Nt,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),u(Nt,"href","#metrics"),u(Dt,"class","relative group"),u(Jt,"href","/docs/transformers/pr_17548/en/main_classes/trainer#transformers.Trainer"),u(De,"href","https://huggingface.co/docs/datasets/package_reference/main_classes.html?highlight=add_batch#datasets.Metric.add_batch"),u(De,"rel","nofollow")},m(a,$){b(g,a,$),i(a,A,$),i(a,d,$),t(d,T),t(T,q),t(d,N),i(a,X,$),i(a,x,$),t(x,D),i(a,I,$),b(S,a,$),i(a,Z,$),i(a,Q,$),t(Q,Oe),t(Q,Be),t(Be,M),t(Q,Se),i(a,le,$),i(a,G,$),t(G,te),t(te,ge),t(ge,Ae),t(ge,ae),t(ae,Ze),t(ge,ie),t(te,Qe),b(U,te,null),t(G,lt),t(G,F),t(F,L),t(L,V),t(L,O),t(O,it),t(L,He),t(L,pe),t(pe,pt),t(L,xt),t(L,Pe),t(Pe,_e),t(L,yt),t(F,Fe),b(B,F,null),t(G,Ve),t(G,z),t(z,H),t(H,fe),t(z,ce),b(Ye,z,null),i(a,qe,$),i(a,$e,$),t($e,et),i(a,C,$),b(ve,a,$),i(a,xe,$),i(a,Y,$),t(Y,se),t(se,we),b(Ne,we,null),t(Y,R),t(Y,ye),t(ye,tt),i(a,bt,$),i(a,re,$),t(re,me),t(re,Re),t(Re,c),t(re,P),i(a,K,$),b(oe,a,$),i(a,ze,$),i(a,be,$),t(be,W),i(a,ft,$),b(Ie,a,$),i(a,ke,$),i(a,ee,$),t(ee,je),t(je,We),b(Ge,We,null),t(ee,ne),t(ee,Ce),t(Ce,kt),i(a,Le,$),i(a,he,$),t(he,Ue),t(he,ue),t(ue,ct),t(ct,at),t(he,Wt),i(a,zt,$),b(mt,a,$),i(a,Ee,$),i(a,Te,$),t(Te,Ke),t(Te,de),t(de,Mt),t(Te,ht),i(a,Gt,$),b(st,a,$),i(a,Ct,$),i(a,ut,$),t(ut,dt),t(ut,rt),t(rt,jt),t(ut,Ft),i(a,Et,$),b(Tt,a,$),i(a,At,$),b(gt,a,$),i(a,Je,$),i(a,_t,$),t(_t,$t),i(a,Ut,$),i(a,vt,$),t(vt,ot),t(ot,s),b(_,s,null),t(vt,r),t(vt,v),t(v,Kt),i(a,ta,$),i(a,Pt,$),t(Pt,Ot),t(Pt,wt),t(wt,la),t(Pt,ia),i(a,aa,$),b(Xe,a,$),i(a,sa,$),i(a,Dt,$),t(Dt,Nt),t(Nt,Bt),b(St,Bt,null),t(Dt,pa),t(Dt,Ht),t(Ht,nt),i(a,ra,$),i(a,qt,$),t(qt,Qt),t(qt,Jt),t(Jt,fa),t(qt,ca),t(qt,De),t(De,Vt),t(Vt,ma),t(qt,ha),i(a,It,$),b(Yt,a,$),Xt=!0},p(a,$){const Rt={};$&2&&(Rt.$$scope={dirty:$,ctx:a}),gt.$set(Rt)},i(a){Xt||(k(g.$$.fragment,a),k(S.$$.fragment,a),k(U.$$.fragment,a),k(B.$$.fragment,a),k(Ye.$$.fragment,a),k(ve.$$.fragment,a),k(Ne.$$.fragment,a),k(oe.$$.fragment,a),k(Ie.$$.fragment,a),k(Ge.$$.fragment,a),k(mt.$$.fragment,a),k(st.$$.fragment,a),k(Tt.$$.fragment,a),k(gt.$$.fragment,a),k(_.$$.fragment,a),k(Xe.$$.fragment,a),k(St.$$.fragment,a),k(Yt.$$.fragment,a),Xt=!0)},o(a){j(g.$$.fragment,a),j(S.$$.fragment,a),j(U.$$.fragment,a),j(B.$$.fragment,a),j(Ye.$$.fragment,a),j(ve.$$.fragment,a),j(Ne.$$.fragment,a),j(oe.$$.fragment,a),j(Ie.$$.fragment,a),j(Ge.$$.fragment,a),j(mt.$$.fragment,a),j(st.$$.fragment,a),j(Tt.$$.fragment,a),j(gt.$$.fragment,a),j(_.$$.fragment,a),j(Xe.$$.fragment,a),j(St.$$.fragment,a),j(Yt.$$.fragment,a),Xt=!1},d(a){E(g,a),a&&e(A),a&&e(d),a&&e(X),a&&e(x),a&&e(I),E(S,a),a&&e(Z),a&&e(Q),a&&e(le),a&&e(G),E(U),E(B),E(Ye),a&&e(qe),a&&e($e),a&&e(C),E(ve,a),a&&e(xe),a&&e(Y),E(Ne),a&&e(bt),a&&e(re),a&&e(K),E(oe,a),a&&e(ze),a&&e(be),a&&e(ft),E(Ie,a),a&&e(ke),a&&e(ee),E(Ge),a&&e(Le),a&&e(he),a&&e(zt),E(mt,a),a&&e(Ee),a&&e(Te),a&&e(Gt),E(st,a),a&&e(Ct),a&&e(ut),a&&e(Et),E(Tt,a),a&&e(At),E(gt,a),a&&e(Je),a&&e(_t),a&&e(Ut),a&&e(vt),E(_),a&&e(ta),a&&e(Pt),a&&e(aa),E(Xe,a),a&&e(sa),a&&e(Dt),E(St),a&&e(ra),a&&e(qt),a&&e(It),E(Yt,a)}}}function bs(Me){let g,A;return g=new za({props:{$$slots:{default:[ys]},$$scope:{ctx:Me}}}),{c(){w(g.$$.fragment)},l(d){y(g.$$.fragment,d)},m(d,T){b(g,d,T),A=!0},p(d,T){const q={};T&2&&(q.$$scope={dirty:T,ctx:d}),g.$set(q)},i(d){A||(k(g.$$.fragment,d),A=!0)},o(d){j(g.$$.fragment,d),A=!1},d(d){E(g,d)}}}function ks(Me){let g,A,d,T,q,N,X,x,D,I,S,Z,Q,Oe,Be,M,Se,le,G,te,ge,Ae,ae,Ze,ie,Qe,U,lt,F,L,V,O,it,He,pe,pt,xt,Pe,_e,yt,Fe,B,Ve,z,H,fe,ce,Ye,qe,$e,et,C,ve,xe,Y,se,we,Ne,R,ye,tt,bt,re,me,Re,c,P,K,oe,ze,be,W,ft,Ie,ke,ee,je,We,Ge,ne,Ce,kt,Le,he,Ue,ue,ct,at,Wt,zt,mt,Ee,Te,Ke,de,Mt,ht,Gt,st,Ct,ut,dt,rt,jt,Ft,Et,Tt,At,gt,Je,_t,$t,Ut,vt,ot;return N=new Lt({}),S=new hs({props:{classNames:"absolute z-10 right-0 top-0",options:[{label:"Mixed",value:"https://colab.research.google.com/github/huggingface/notebooks/blob/main/transformers_doc/en/training.ipynb"},{label:"PyTorch",value:"https://colab.research.google.com/github/huggingface/notebooks/blob/main/transformers_doc/en/pytorch/training.ipynb"},{label:"TensorFlow",value:"https://colab.research.google.com/github/huggingface/notebooks/blob/main/transformers_doc/en/tensorflow/training.ipynb"},{label:"Mixed",value:"https://studiolab.sagemaker.aws/import/github/huggingface/notebooks/blob/main/transformers_doc/en/training.ipynb"},{label:"PyTorch",value:"https://studiolab.sagemaker.aws/import/github/huggingface/notebooks/blob/main/transformers_doc/en/pytorch/training.ipynb"},{label:"TensorFlow",value:"https://studiolab.sagemaker.aws/import/github/huggingface/notebooks/blob/main/transformers_doc/en/tensorflow/training.ipynb"}]}}),He=new Lt({}),_e=new qa({props:{id:"_BZearw7f0w"}}),$e=new J({props:{code:`from datasets import load_dataset

dataset = load_dataset("yelp_review_full")
dataset["train"][100]`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> datasets <span class="hljs-keyword">import</span> load_dataset

<span class="hljs-meta">&gt;&gt;&gt; </span>dataset = load_dataset(<span class="hljs-string">&quot;yelp_review_full&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>dataset[<span class="hljs-string">&quot;train&quot;</span>][<span class="hljs-number">100</span>]
{<span class="hljs-string">&#x27;label&#x27;</span>: <span class="hljs-number">0</span>,
 <span class="hljs-string">&#x27;text&#x27;</span>: <span class="hljs-string">&#x27;My expectations for McDonalds are t rarely high. But for one to still fail so spectacularly...that takes something special!\\\\nThe cashier took my friends\\&#x27;s order, then promptly ignored me. I had to force myself in front of a cashier who opened his register to wait on the person BEHIND me. I waited over five minutes for a gigantic order that included precisely one kid\\&#x27;s meal. After watching two people who ordered after me be handed their food, I asked where mine was. The manager started yelling at the cashiers for \\\\&quot;serving off their orders\\\\&quot; when they didn\\&#x27;t have their food. But neither cashier was anywhere near those controls, and the manager was the one serving food to customers and clearing the boards.\\\\nThe manager was rude when giving me my order. She didn\\&#x27;t make sure that I had everything ON MY RECEIPT, and never even had the decency to apologize that I felt I was getting poor service.\\\\nI\\&#x27;ve eaten at various McDonalds restaurants for over 30 years. I\\&#x27;ve worked at more than one location. I expect bad days, bad moods, and the occasional mistake. But I have yet to have a decent experience at this store. It will remain a place I avoid unless someone in my party needs to avoid illness from low blood sugar. Perhaps I should go back to the racially biased service of Steak n Shake instead!&#x27;</span>}`}}),R=new J({props:{code:`from transformers import AutoTokenizer

tokenizer = AutoTokenizer.from_pretrained("bert-base-cased")


def tokenize_function(examples):
    return tokenizer(examples["text"], padding="max_length", truncation=True)


tokenized_datasets = dataset.map(tokenize_function, batched=True)`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoTokenizer

<span class="hljs-meta">&gt;&gt;&gt; </span>tokenizer = AutoTokenizer.from_pretrained(<span class="hljs-string">&quot;bert-base-cased&quot;</span>)


<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">def</span> <span class="hljs-title function_">tokenize_function</span>(<span class="hljs-params">examples</span>):
<span class="hljs-meta">... </span>    <span class="hljs-keyword">return</span> tokenizer(examples[<span class="hljs-string">&quot;text&quot;</span>], padding=<span class="hljs-string">&quot;max_length&quot;</span>, truncation=<span class="hljs-literal">True</span>)


<span class="hljs-meta">&gt;&gt;&gt; </span>tokenized_datasets = dataset.<span class="hljs-built_in">map</span>(tokenize_function, batched=<span class="hljs-literal">True</span>)`}}),me=new J({props:{code:`small_train_dataset = tokenized_datasets["train"].shuffle(seed=42).select(range(1000))
small_eval_dataset = tokenized_datasets["test"].shuffle(seed=42).select(range(1000))`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>small_train_dataset = tokenized_datasets[<span class="hljs-string">&quot;train&quot;</span>].shuffle(seed=<span class="hljs-number">42</span>).select(<span class="hljs-built_in">range</span>(<span class="hljs-number">1000</span>))
<span class="hljs-meta">&gt;&gt;&gt; </span>small_eval_dataset = tokenized_datasets[<span class="hljs-string">&quot;test&quot;</span>].shuffle(seed=<span class="hljs-number">42</span>).select(<span class="hljs-built_in">range</span>(<span class="hljs-number">1000</span>))`}}),be=new Lt({}),ee=new ls({props:{pytorch:!0,tensorflow:!0,jax:!1,$$slots:{tensorflow:[vs],pytorch:[gs]},$$scope:{ctx:Me}}}),Le=new Lt({}),at=new ls({props:{pytorch:!0,tensorflow:!1,jax:!1,$$slots:{pytorch:[bs]},$$scope:{ctx:Me}}}),de=new Lt({}),{c(){g=o("meta"),A=m(),d=o("h1"),T=o("a"),q=o("span"),w(N.$$.fragment),X=m(),x=o("span"),D=p("Fine-tune a pretrained model"),I=m(),w(S.$$.fragment),Z=m(),Q=o("p"),Oe=p("There are significant benefits to using a pretrained model. It reduces computation costs, your carbon footprint, and allows you to use state-of-the-art models without having to train one from scratch. \u{1F917} Transformers provides access to thousands of pretrained models for a wide range of tasks. When you use a pretrained model, you train it on a dataset specific to your task. This is known as fine-tuning, an incredibly powerful training technique. In this tutorial, you will fine-tune a pretrained model with a deep learning framework of your choice:"),Be=m(),M=o("ul"),Se=o("li"),le=p("Fine-tune a pretrained model with \u{1F917} Transformers "),G=o("a"),te=p("Trainer"),ge=p("."),Ae=m(),ae=o("li"),Ze=p("Fine-tune a pretrained model in TensorFlow with Keras."),ie=m(),Qe=o("li"),U=p("Fine-tune a pretrained model in native PyTorch."),lt=m(),F=o("a"),L=m(),V=o("h2"),O=o("a"),it=o("span"),w(He.$$.fragment),pe=m(),pt=o("span"),xt=p("Prepare a dataset"),Pe=m(),w(_e.$$.fragment),yt=m(),Fe=o("p"),B=p("Before you can fine-tune a pretrained model, download a dataset and prepare it for training. The previous tutorial showed you how to process data for training, and now you get an opportunity to put those skills to the test!"),Ve=m(),z=o("p"),H=p("Begin by loading the "),fe=o("a"),ce=p("Yelp Reviews"),Ye=p(" dataset:"),qe=m(),w($e.$$.fragment),et=m(),C=o("p"),ve=p("As you now know, you need a tokenizer to process the text and include a padding and truncation strategy to handle any variable sequence lengths. To process your dataset in one step, use \u{1F917} Datasets "),xe=o("a"),Y=o("code"),se=p("map"),we=p(" method to apply a preprocessing function over the entire dataset:"),Ne=m(),w(R.$$.fragment),ye=m(),tt=o("p"),bt=p("If you like, you can create a smaller subset of the full dataset to fine-tune on to reduce the time it takes:"),re=m(),w(me.$$.fragment),Re=m(),c=o("a"),P=m(),K=o("h2"),oe=o("a"),ze=o("span"),w(be.$$.fragment),W=m(),ft=o("span"),Ie=p("Train"),ke=m(),w(ee.$$.fragment),je=m(),We=o("a"),Ge=m(),ne=o("h2"),Ce=o("a"),kt=o("span"),w(Le.$$.fragment),he=m(),Ue=o("span"),ue=p("Train in native PyTorch"),ct=m(),w(at.$$.fragment),Wt=m(),zt=o("a"),mt=m(),Ee=o("h2"),Te=o("a"),Ke=o("span"),w(de.$$.fragment),Mt=m(),ht=o("span"),Gt=p("Additional resources"),st=m(),Ct=o("p"),ut=p("For more fine-tuning examples, refer to:"),dt=m(),rt=o("ul"),jt=o("li"),Ft=o("p"),Et=o("a"),Tt=p("\u{1F917} Transformers Examples"),At=p(` includes scripts
to train common NLP tasks in PyTorch and TensorFlow.`),gt=m(),Je=o("li"),_t=o("p"),$t=o("a"),Ut=p("\u{1F917} Transformers Notebooks"),vt=p(" contains various notebooks on how to fine-tune a model for specific tasks in PyTorch and TensorFlow."),this.h()},l(s){const _=cs('[data-svelte="svelte-1phssyn"]',document.head);g=n(_,"META",{name:!0,content:!0}),_.forEach(e),A=h(s),d=n(s,"H1",{class:!0});var r=l(d);T=n(r,"A",{id:!0,class:!0,href:!0});var v=l(T);q=n(v,"SPAN",{});var Kt=l(q);y(N.$$.fragment,Kt),Kt.forEach(e),v.forEach(e),X=h(r),x=n(r,"SPAN",{});var ta=l(x);D=f(ta,"Fine-tune a pretrained model"),ta.forEach(e),r.forEach(e),I=h(s),y(S.$$.fragment,s),Z=h(s),Q=n(s,"P",{});var Pt=l(Q);Oe=f(Pt,"There are significant benefits to using a pretrained model. It reduces computation costs, your carbon footprint, and allows you to use state-of-the-art models without having to train one from scratch. \u{1F917} Transformers provides access to thousands of pretrained models for a wide range of tasks. When you use a pretrained model, you train it on a dataset specific to your task. This is known as fine-tuning, an incredibly powerful training technique. In this tutorial, you will fine-tune a pretrained model with a deep learning framework of your choice:"),Pt.forEach(e),Be=h(s),M=n(s,"UL",{});var Ot=l(M);Se=n(Ot,"LI",{});var wt=l(Se);le=f(wt,"Fine-tune a pretrained model with \u{1F917} Transformers "),G=n(wt,"A",{href:!0});var la=l(G);te=f(la,"Trainer"),la.forEach(e),ge=f(wt,"."),wt.forEach(e),Ae=h(Ot),ae=n(Ot,"LI",{});var ia=l(ae);Ze=f(ia,"Fine-tune a pretrained model in TensorFlow with Keras."),ia.forEach(e),ie=h(Ot),Qe=n(Ot,"LI",{});var aa=l(Qe);U=f(aa,"Fine-tune a pretrained model in native PyTorch."),aa.forEach(e),Ot.forEach(e),lt=h(s),F=n(s,"A",{id:!0}),l(F).forEach(e),L=h(s),V=n(s,"H2",{class:!0});var Xe=l(V);O=n(Xe,"A",{id:!0,class:!0,href:!0});var sa=l(O);it=n(sa,"SPAN",{});var Dt=l(it);y(He.$$.fragment,Dt),Dt.forEach(e),sa.forEach(e),pe=h(Xe),pt=n(Xe,"SPAN",{});var Nt=l(pt);xt=f(Nt,"Prepare a dataset"),Nt.forEach(e),Xe.forEach(e),Pe=h(s),y(_e.$$.fragment,s),yt=h(s),Fe=n(s,"P",{});var Bt=l(Fe);B=f(Bt,"Before you can fine-tune a pretrained model, download a dataset and prepare it for training. The previous tutorial showed you how to process data for training, and now you get an opportunity to put those skills to the test!"),Bt.forEach(e),Ve=h(s),z=n(s,"P",{});var St=l(z);H=f(St,"Begin by loading the "),fe=n(St,"A",{href:!0,rel:!0});var pa=l(fe);ce=f(pa,"Yelp Reviews"),pa.forEach(e),Ye=f(St," dataset:"),St.forEach(e),qe=h(s),y($e.$$.fragment,s),et=h(s),C=n(s,"P",{});var Ht=l(C);ve=f(Ht,"As you now know, you need a tokenizer to process the text and include a padding and truncation strategy to handle any variable sequence lengths. To process your dataset in one step, use \u{1F917} Datasets "),xe=n(Ht,"A",{href:!0,rel:!0});var nt=l(xe);Y=n(nt,"CODE",{});var ra=l(Y);se=f(ra,"map"),ra.forEach(e),nt.forEach(e),we=f(Ht," method to apply a preprocessing function over the entire dataset:"),Ht.forEach(e),Ne=h(s),y(R.$$.fragment,s),ye=h(s),tt=n(s,"P",{});var qt=l(tt);bt=f(qt,"If you like, you can create a smaller subset of the full dataset to fine-tune on to reduce the time it takes:"),qt.forEach(e),re=h(s),y(me.$$.fragment,s),Re=h(s),c=n(s,"A",{id:!0}),l(c).forEach(e),P=h(s),K=n(s,"H2",{class:!0});var Qt=l(K);oe=n(Qt,"A",{id:!0,class:!0,href:!0});var Jt=l(oe);ze=n(Jt,"SPAN",{});var fa=l(ze);y(be.$$.fragment,fa),fa.forEach(e),Jt.forEach(e),W=h(Qt),ft=n(Qt,"SPAN",{});var ca=l(ft);Ie=f(ca,"Train"),ca.forEach(e),Qt.forEach(e),ke=h(s),y(ee.$$.fragment,s),je=h(s),We=n(s,"A",{id:!0}),l(We).forEach(e),Ge=h(s),ne=n(s,"H2",{class:!0});var De=l(ne);Ce=n(De,"A",{id:!0,class:!0,href:!0});var Vt=l(Ce);kt=n(Vt,"SPAN",{});var ma=l(kt);y(Le.$$.fragment,ma),ma.forEach(e),Vt.forEach(e),he=h(De),Ue=n(De,"SPAN",{});var ha=l(Ue);ue=f(ha,"Train in native PyTorch"),ha.forEach(e),De.forEach(e),ct=h(s),y(at.$$.fragment,s),Wt=h(s),zt=n(s,"A",{id:!0}),l(zt).forEach(e),mt=h(s),Ee=n(s,"H2",{class:!0});var It=l(Ee);Te=n(It,"A",{id:!0,class:!0,href:!0});var Yt=l(Te);Ke=n(Yt,"SPAN",{});var Xt=l(Ke);y(de.$$.fragment,Xt),Xt.forEach(e),Yt.forEach(e),Mt=h(It),ht=n(It,"SPAN",{});var a=l(ht);Gt=f(a,"Additional resources"),a.forEach(e),It.forEach(e),st=h(s),Ct=n(s,"P",{});var $=l(Ct);ut=f($,"For more fine-tuning examples, refer to:"),$.forEach(e),dt=h(s),rt=n(s,"UL",{});var Rt=l(rt);jt=n(Rt,"LI",{});var oa=l(jt);Ft=n(oa,"P",{});var ea=l(Ft);Et=n(ea,"A",{href:!0,rel:!0});var na=l(Et);Tt=f(na,"\u{1F917} Transformers Examples"),na.forEach(e),At=f(ea,` includes scripts
to train common NLP tasks in PyTorch and TensorFlow.`),ea.forEach(e),oa.forEach(e),gt=h(Rt),Je=n(Rt,"LI",{});var Zt=l(Je);_t=n(Zt,"P",{});var ua=l(_t);$t=n(ua,"A",{href:!0});var ga=l($t);Ut=f(ga,"\u{1F917} Transformers Notebooks"),ga.forEach(e),vt=f(ua," contains various notebooks on how to fine-tune a model for specific tasks in PyTorch and TensorFlow."),ua.forEach(e),Zt.forEach(e),Rt.forEach(e),this.h()},h(){u(g,"name","hf:doc:metadata"),u(g,"content",JSON.stringify(js)),u(T,"id","finetune-a-pretrained-model"),u(T,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),u(T,"href","#finetune-a-pretrained-model"),u(d,"class","relative group"),u(G,"href","/docs/transformers/pr_17548/en/main_classes/trainer#transformers.Trainer"),u(F,"id","data-processing"),u(O,"id","prepare-a-dataset"),u(O,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),u(O,"href","#prepare-a-dataset"),u(V,"class","relative group"),u(fe,"href","https://huggingface.co/datasets/yelp_review_full"),u(fe,"rel","nofollow"),u(xe,"href","https://huggingface.co/docs/datasets/process.html#map"),u(xe,"rel","nofollow"),u(c,"id","trainer"),u(oe,"id","train"),u(oe,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),u(oe,"href","#train"),u(K,"class","relative group"),u(We,"id","pytorch_native"),u(Ce,"id","train-in-native-pytorch"),u(Ce,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),u(Ce,"href","#train-in-native-pytorch"),u(ne,"class","relative group"),u(zt,"id","additional-resources"),u(Te,"id","additional-resources"),u(Te,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),u(Te,"href","#additional-resources"),u(Ee,"class","relative group"),u(Et,"href","https://github.com/huggingface/transformers/tree/main/examples"),u(Et,"rel","nofollow"),u($t,"href","notebooks")},m(s,_){t(document.head,g),i(s,A,_),i(s,d,_),t(d,T),t(T,q),b(N,q,null),t(d,X),t(d,x),t(x,D),i(s,I,_),b(S,s,_),i(s,Z,_),i(s,Q,_),t(Q,Oe),i(s,Be,_),i(s,M,_),t(M,Se),t(Se,le),t(Se,G),t(G,te),t(Se,ge),t(M,Ae),t(M,ae),t(ae,Ze),t(M,ie),t(M,Qe),t(Qe,U),i(s,lt,_),i(s,F,_),i(s,L,_),i(s,V,_),t(V,O),t(O,it),b(He,it,null),t(V,pe),t(V,pt),t(pt,xt),i(s,Pe,_),b(_e,s,_),i(s,yt,_),i(s,Fe,_),t(Fe,B),i(s,Ve,_),i(s,z,_),t(z,H),t(z,fe),t(fe,ce),t(z,Ye),i(s,qe,_),b($e,s,_),i(s,et,_),i(s,C,_),t(C,ve),t(C,xe),t(xe,Y),t(Y,se),t(C,we),i(s,Ne,_),b(R,s,_),i(s,ye,_),i(s,tt,_),t(tt,bt),i(s,re,_),b(me,s,_),i(s,Re,_),i(s,c,_),i(s,P,_),i(s,K,_),t(K,oe),t(oe,ze),b(be,ze,null),t(K,W),t(K,ft),t(ft,Ie),i(s,ke,_),b(ee,s,_),i(s,je,_),i(s,We,_),i(s,Ge,_),i(s,ne,_),t(ne,Ce),t(Ce,kt),b(Le,kt,null),t(ne,he),t(ne,Ue),t(Ue,ue),i(s,ct,_),b(at,s,_),i(s,Wt,_),i(s,zt,_),i(s,mt,_),i(s,Ee,_),t(Ee,Te),t(Te,Ke),b(de,Ke,null),t(Ee,Mt),t(Ee,ht),t(ht,Gt),i(s,st,_),i(s,Ct,_),t(Ct,ut),i(s,dt,_),i(s,rt,_),t(rt,jt),t(jt,Ft),t(Ft,Et),t(Et,Tt),t(Ft,At),t(rt,gt),t(rt,Je),t(Je,_t),t(_t,$t),t($t,Ut),t(_t,vt),ot=!0},p(s,[_]){const r={};_&2&&(r.$$scope={dirty:_,ctx:s}),ee.$set(r);const v={};_&2&&(v.$$scope={dirty:_,ctx:s}),at.$set(v)},i(s){ot||(k(N.$$.fragment,s),k(S.$$.fragment,s),k(He.$$.fragment,s),k(_e.$$.fragment,s),k($e.$$.fragment,s),k(R.$$.fragment,s),k(me.$$.fragment,s),k(be.$$.fragment,s),k(ee.$$.fragment,s),k(Le.$$.fragment,s),k(at.$$.fragment,s),k(de.$$.fragment,s),ot=!0)},o(s){j(N.$$.fragment,s),j(S.$$.fragment,s),j(He.$$.fragment,s),j(_e.$$.fragment,s),j($e.$$.fragment,s),j(R.$$.fragment,s),j(me.$$.fragment,s),j(be.$$.fragment,s),j(ee.$$.fragment,s),j(Le.$$.fragment,s),j(at.$$.fragment,s),j(de.$$.fragment,s),ot=!1},d(s){e(g),s&&e(A),s&&e(d),E(N),s&&e(I),E(S,s),s&&e(Z),s&&e(Q),s&&e(Be),s&&e(M),s&&e(lt),s&&e(F),s&&e(L),s&&e(V),E(He),s&&e(Pe),E(_e,s),s&&e(yt),s&&e(Fe),s&&e(Ve),s&&e(z),s&&e(qe),E($e,s),s&&e(et),s&&e(C),s&&e(Ne),E(R,s),s&&e(ye),s&&e(tt),s&&e(re),E(me,s),s&&e(Re),s&&e(c),s&&e(P),s&&e(K),E(be),s&&e(ke),E(ee,s),s&&e(je),s&&e(We),s&&e(Ge),s&&e(ne),E(Le),s&&e(ct),E(at,s),s&&e(Wt),s&&e(zt),s&&e(mt),s&&e(Ee),E(de),s&&e(st),s&&e(Ct),s&&e(dt),s&&e(rt)}}}const js={local:"finetune-a-pretrained-model",sections:[{local:"prepare-a-dataset",title:"Prepare a dataset"},{local:"train",sections:[{local:"training-hyperparameters",title:"Training hyperparameters"},{local:"metrics",title:"Metrics"},{local:"trainer",title:"Trainer"},{local:"convert-dataset-to-tensorflow-format",title:"Convert dataset to TensorFlow format"},{local:"compile-and-fit",title:"Compile and fit"}],title:"Train"},{local:"train-in-native-pytorch",sections:[{local:"dataloader",title:"DataLoader"},{local:"optimizer-and-learning-rate-scheduler",title:"Optimizer and learning rate scheduler"},{local:"training-loop",title:"Training loop"},{local:"metrics",title:"Metrics"}],title:"Train in native PyTorch"},{local:"additional-resources",title:"Additional resources"}],title:"Fine-tune a pretrained model"};function Es(Me){return ms(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class Ds extends is{constructor(g){super();ps(this,g,Es,ks,fs,{})}}export{Ds as default,js as metadata};
