import{S as Of,i as Bf,s as Hf,e as r,k as h,w as c,t as i,M as Yf,c as o,d as t,m,a as n,x as u,h as p,b as f,F as a,g as l,y as d,q as g,o as _,B as v}from"../chunks/vendor-4833417e.js";import{T as Bi}from"../chunks/Tip-fffd6df1.js";import{Y as Wo}from"../chunks/Youtube-27813aed.js";import{I as x}from"../chunks/IconCopyLink-4b81c553.js";import{C as y}from"../chunks/CodeBlock-6a3d1b46.js";import{D as Wf}from"../chunks/DocNotebookDropdown-ecff2a90.js";import"../chunks/CopyButton-dacfbfaf.js";function Rf(Y){let $,k;return{c(){$=r("p"),k=i(`You will see a warning about some of the pretrained weights not being used and some weights being randomly
initialized. Don\u2019t worry, this is completely normal! The pretrained head of the BERT model is discarded, and replaced with a randomly initialized classification head. You will fine-tune this new model head on your sequence classification task, transferring the knowledge of the pretrained model to it.`)},l(w){$=o(w,"P",{});var b=n($);k=p(b,`You will see a warning about some of the pretrained weights not being used and some weights being randomly
initialized. Don\u2019t worry, this is completely normal! The pretrained head of the BERT model is discarded, and replaced with a randomly initialized classification head. You will fine-tune this new model head on your sequence classification task, transferring the knowledge of the pretrained model to it.`),b.forEach(t)},m(w,b){l(w,$,b),a($,k)},d(w){w&&t($)}}}function Gf(Y){let $,k,w,b,T,j,F;return{c(){$=r("p"),k=r("a"),w=i("Trainer"),b=i(" uses "),T=r("a"),j=i("DataCollatorWithPadding"),F=i(" by default so you don\u2019t need to explicitly specify a data collator."),this.h()},l(P){$=o(P,"P",{});var E=n($);k=o(E,"A",{href:!0});var A=n(k);w=p(A,"Trainer"),A.forEach(t),b=p(E," uses "),T=o(E,"A",{href:!0});var z=n(T);j=p(z,"DataCollatorWithPadding"),z.forEach(t),F=p(E," by default so you don\u2019t need to explicitly specify a data collator."),E.forEach(t),this.h()},h(){f(k,"href","/docs/transformers/pr_15839/en/main_classes/trainer#transformers.Trainer"),f(T,"href","/docs/transformers/pr_15839/en/main_classes/data_collator#transformers.DataCollatorWithPadding")},m(P,E){l(P,$,E),a($,k),a(k,w),a($,b),a($,T),a(T,j),a($,F)},d(P){P&&t($)}}}function Kf(Y){let $,k,w,b,T,j,F,P;return{c(){$=r("p"),k=i("Get free access to a cloud GPU if you don\u2019t have one with a hosted notebook like "),w=r("a"),b=i("Colaboratory"),T=i(" or "),j=r("a"),F=i("SageMaker StudioLab"),P=i("."),this.h()},l(E){$=o(E,"P",{});var A=n($);k=p(A,"Get free access to a cloud GPU if you don\u2019t have one with a hosted notebook like "),w=o(A,"A",{href:!0,rel:!0});var z=n(w);b=p(z,"Colaboratory"),z.forEach(t),T=p(A," or "),j=o(A,"A",{href:!0,rel:!0});var Le=n(j);F=p(Le,"SageMaker StudioLab"),Le.forEach(t),P=p(A,"."),A.forEach(t),this.h()},h(){f(w,"href","https://colab.research.google.com/"),f(w,"rel","nofollow"),f(j,"href","https://studiolab.sagemaker.aws/"),f(j,"rel","nofollow")},m(E,A){l(E,$,A),a($,k),a($,w),a(w,b),a($,T),a($,j),a(j,F),a($,P)},d(E){E&&t($)}}}function Uf(Y){let $,k,w,b,T,j,F,P,E,A,z,Le,Ut,Ro,Fs,I,Me,Go,Jt,Ko,Uo,Jo,Ca,Xo,Zo,Da,Qo,Ss,Xt,Ns,W,re,Fa,Oe,Vo,Sa,en,Is,Be,Ls,Zt,tn,Ms,oe,an,He,sn,rn,Os,Ye,Bs,ne,on,We,Na,nn,ln,Hs,Re,Ys,Qt,pn,Ws,Ge,Rs,Vt,Gs,R,le,Ia,Ke,fn,ea,hn,La,mn,Ks,Ue,Us,L,cn,ta,un,dn,aa,gn,_n,Js,ie,vn,Je,$n,wn,Xs,Xe,Zs,pe,Qs,G,fe,Ma,Ze,yn,Oa,bn,Vs,M,kn,sa,jn,En,Qe,Tn,An,er,ra,Pn,tr,Ve,ar,K,he,Ba,et,qn,Ha,xn,sr,q,oa,zn,Cn,na,Dn,Fn,tt,Ya,Sn,Nn,Wa,In,Ln,at,Mn,On,rr,st,or,C,Bn,Ra,Hn,Yn,Ga,Wn,Rn,Ka,Gn,Kn,nr,rt,lr,me,Un,Ua,Jn,Xn,ir,ot,pr,U,ce,Ja,nt,Zn,Xa,Qn,fr,ue,Vn,la,el,tl,hr,lt,mr,de,al,ia,sl,rl,cr,it,ur,pa,dr,J,ge,Za,pt,ol,Qa,nl,gr,ft,_r,fa,ll,vr,X,_e,Va,ht,il,es,pl,$r,O,fl,ha,hl,ml,ts,cl,ul,wr,mt,yr,ve,br,D,dl,ct,as,gl,_l,ss,vl,$l,rs,wl,yl,kr,ut,jr,Z,$e,os,dt,bl,ns,kl,Er,ma,jl,Tr,gt,Ar,we,El,_t,ls,Tl,Al,Pr,vt,qr,ca,xr,Q,ye,is,$t,Pl,ps,ql,zr,wt,Cr,yt,ua,xl,zl,Dr,da,Cl,Fr,bt,Sr,be,Dl,fs,Fl,Sl,Nr,B,kt,jt,Nl,hs,Il,Ll,Ml,Et,Ol,Tt,S,Bl,ms,Hl,Yl,cs,Wl,Rl,us,Gl,Kl,Ul,At,Jl,Pt,ds,Xl,Zl,qt,Ir,ga,Ql,Lr,xt,Mr,V,ke,gs,zt,Vl,_s,ei,Or,je,ti,vs,ai,si,Br,Ct,Hr,_a,ri,Yr,Dt,Wr,ee,Ee,$s,Ft,oi,ws,ni,Rr,Te,li,St,ys,ii,pi,Gr,Nt,Kr,Ae,fi,va,hi,mi,Ur,It,Jr,Pe,ci,bs,ui,di,Xr,Lt,Zr,qe,Qr,$a,gi,Vr,te,xe,ks,Mt,_i,js,vi,eo,ze,$i,Ot,wi,yi,to,Bt,ao,ae,Ce,Es,Ht,bi,Ts,ki,so,H,ji,wa,Ei,Ti,Yt,As,Ai,Pi,ro,Wt,oo,ya,no,se,De,Ps,Rt,qi,qs,xi,lo,ba,zi,io,Fe,xs,ka,Gt,Ci,Di,Fi,zs,ja,Ea,Si,Ni,po;return j=new x({}),z=new Wf({props:{classNames:"absolute z-10 right-0 top-0",options:[{label:"Mixed",value:"https://colab.research.google.com/github/huggingface/notebooks/blob/master/transformers_doc/training.ipynb"},{label:"PyTorch",value:"https://colab.research.google.com/github/huggingface/notebooks/blob/master/transformers_doc/pytorch/training.ipynb"},{label:"TensorFlow",value:"https://colab.research.google.com/github/huggingface/notebooks/blob/master/transformers_doc/tensorflow/training.ipynb"},{label:"Mixed",value:"https://studiolab.sagemaker.aws/import/github/huggingface/notebooks/blob/master/transformers_doc/training.ipynb"},{label:"PyTorch",value:"https://studiolab.sagemaker.aws/import/github/huggingface/notebooks/blob/master/transformers_doc/pytorch/training.ipynb"},{label:"TensorFlow",value:"https://studiolab.sagemaker.aws/import/github/huggingface/notebooks/blob/master/transformers_doc/tensorflow/training.ipynb"}]}}),Oe=new x({}),Be=new Wo({props:{id:"_BZearw7f0w"}}),Ye=new y({props:{code:`from datasets import load_dataset

dataset = load_dataset("yelp_review_full")
dataset[100]`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> datasets <span class="hljs-keyword">import</span> load_dataset

<span class="hljs-meta">&gt;&gt;&gt; </span>dataset = load_dataset(<span class="hljs-string">&quot;yelp_review_full&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>dataset[<span class="hljs-number">100</span>]
{<span class="hljs-string">&#x27;label&#x27;</span>: <span class="hljs-number">0</span>,
 <span class="hljs-string">&#x27;text&#x27;</span>: <span class="hljs-string">&#x27;My expectations for McDonalds are t rarely high. But for one to still fail so spectacularly...that takes something special!\\\\nThe cashier took my friends\\&#x27;s order, then promptly ignored me. I had to force myself in front of a cashier who opened his register to wait on the person BEHIND me. I waited over five minutes for a gigantic order that included precisely one kid\\&#x27;s meal. After watching two people who ordered after me be handed their food, I asked where mine was. The manager started yelling at the cashiers for \\\\&quot;serving off their orders\\\\&quot; when they didn\\&#x27;t have their food. But neither cashier was anywhere near those controls, and the manager was the one serving food to customers and clearing the boards.\\\\nThe manager was rude when giving me my order. She didn\\&#x27;t make sure that I had everything ON MY RECEIPT, and never even had the decency to apologize that I felt I was getting poor service.\\\\nI\\&#x27;ve eaten at various McDonalds restaurants for over 30 years. I\\&#x27;ve worked at more than one location. I expect bad days, bad moods, and the occasional mistake. But I have yet to have a decent experience at this store. It will remain a place I avoid unless someone in my party needs to avoid illness from low blood sugar. Perhaps I should go back to the racially biased service of Steak n Shake instead!&#x27;</span>}`}}),Re=new y({props:{code:`from transformers import AutoTokenizer

tokenizer = AutoTokenizer.from_pretrained("bert-base-cased")


def tokenize_function(examples):
    return tokenizer(examples["text"], padding="max_length", truncation=True)


tokenized_datasets = dataset.map(tokenize_function, batched=True)`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoTokenizer

<span class="hljs-meta">&gt;&gt;&gt; </span>tokenizer = AutoTokenizer.from_pretrained(<span class="hljs-string">&quot;bert-base-cased&quot;</span>)


<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">def</span> <span class="hljs-title function_">tokenize_function</span>(<span class="hljs-params">examples</span>):
<span class="hljs-meta">... </span>    <span class="hljs-keyword">return</span> tokenizer(examples[<span class="hljs-string">&quot;text&quot;</span>], padding=<span class="hljs-string">&quot;max_length&quot;</span>, truncation=<span class="hljs-literal">True</span>)


<span class="hljs-meta">&gt;&gt;&gt; </span>tokenized_datasets = dataset.<span class="hljs-built_in">map</span>(tokenize_function, batched=<span class="hljs-literal">True</span>)`}}),Ge=new y({props:{code:`small_train_dataset = tokenized_datasets["train"].shuffle(seed=42).select(range(1000))
small_eval_dataset = tokenized_datasets["test"].shuffle(seed=42).select(range(1000))`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>small_train_dataset = tokenized_datasets[<span class="hljs-string">&quot;train&quot;</span>].shuffle(seed=<span class="hljs-number">42</span>).select(<span class="hljs-built_in">range</span>(<span class="hljs-number">1000</span>))
<span class="hljs-meta">&gt;&gt;&gt; </span>small_eval_dataset = tokenized_datasets[<span class="hljs-string">&quot;test&quot;</span>].shuffle(seed=<span class="hljs-number">42</span>).select(<span class="hljs-built_in">range</span>(<span class="hljs-number">1000</span>))`}}),Ke=new x({}),Ue=new Wo({props:{id:"nvBXf7s7vTI"}}),Xe=new y({props:{code:`from transformers import AutoModelForSequenceClassification

model = AutoModelForSequenceClassification.from_pretrained("bert-base-cased", num_labels=5)`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoModelForSequenceClassification

<span class="hljs-meta">&gt;&gt;&gt; </span>model = AutoModelForSequenceClassification.from_pretrained(<span class="hljs-string">&quot;bert-base-cased&quot;</span>, num_labels=<span class="hljs-number">5</span>)`}}),pe=new Bi({props:{$$slots:{default:[Rf]},$$scope:{ctx:Y}}}),Ze=new x({}),Ve=new y({props:{code:`from transformers import TrainingArguments

training_args = TrainingArguments(output_dir="test_trainer")`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> TrainingArguments

<span class="hljs-meta">&gt;&gt;&gt; </span>training_args = TrainingArguments(output_dir=<span class="hljs-string">&quot;test_trainer&quot;</span>)`}}),et=new x({}),st=new y({props:{code:`import numpy as np
from datasets import load_metric

metric = load_metric("accuracy")`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">import</span> numpy <span class="hljs-keyword">as</span> np
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> datasets <span class="hljs-keyword">import</span> load_metric

<span class="hljs-meta">&gt;&gt;&gt; </span>metric = load_metric(<span class="hljs-string">&quot;accuracy&quot;</span>)`}}),rt=new y({props:{code:`def compute_metrics(eval_pred):
    logits, labels = eval_pred
    predictions = np.argmax(logits, axis=-1)
    return metric.compute(predictions=predictions, references=labels)`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">def</span> <span class="hljs-title function_">compute_metrics</span>(<span class="hljs-params">eval_pred</span>):
<span class="hljs-meta">... </span>    logits, labels = eval_pred
<span class="hljs-meta">... </span>    predictions = np.argmax(logits, axis=-<span class="hljs-number">1</span>)
<span class="hljs-meta">... </span>    <span class="hljs-keyword">return</span> metric.compute(predictions=predictions, references=labels)`}}),ot=new y({props:{code:`from transformers import TrainingArguments

training_args = TrainingArguments(output_dir="test_trainer", evaluation_strategy="epoch")`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> TrainingArguments

<span class="hljs-meta">&gt;&gt;&gt; </span>training_args = TrainingArguments(output_dir=<span class="hljs-string">&quot;test_trainer&quot;</span>, evaluation_strategy=<span class="hljs-string">&quot;epoch&quot;</span>)`}}),nt=new x({}),lt=new y({props:{code:`trainer = Trainer(
    model=model,
    args=training_args,
    train_dataset=small_train_dataset,
    eval_dataset=small_eval_dataset,
    compute_metrics=compute_metrics,
)`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>trainer = Trainer(
<span class="hljs-meta">... </span>    model=model,
<span class="hljs-meta">... </span>    args=training_args,
<span class="hljs-meta">... </span>    train_dataset=small_train_dataset,
<span class="hljs-meta">... </span>    eval_dataset=small_eval_dataset,
<span class="hljs-meta">... </span>    compute_metrics=compute_metrics,
<span class="hljs-meta">... </span>)`}}),it=new y({props:{code:"trainer.train()",highlighted:'<span class="hljs-meta">&gt;&gt;&gt; </span>trainer.train()'}}),pt=new x({}),ft=new Wo({props:{id:"rnTGBy2ax1c"}}),ht=new x({}),mt=new y({props:{code:`from transformers import DefaultDataCollator

data_collator = DefaultDataCollator(return_tensors="tf")`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> DefaultDataCollator

<span class="hljs-meta">&gt;&gt;&gt; </span>data_collator = DefaultDataCollator(return_tensors=<span class="hljs-string">&quot;tf&quot;</span>)`}}),ve=new Bi({props:{$$slots:{default:[Gf]},$$scope:{ctx:Y}}}),ut=new y({props:{code:`tf_train_dataset = small_train_dataset.to_tf_dataset(
    columns=["attention_mask", "input_ids", "token_type_ids"],
    label_cols=["labels"],
    shuffle=True,
    collate_fn=data_collator,
    batch_size=8,
)

tf_validation_dataset = small_eval_dataset.to_tf_dataset(
    columns=["attention_mask", "input_ids", "token_type_ids"],
    label_cols=["labels"],
    shuffle=False,
    collate_fn=data_collator,
    batch_size=8,
)`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>tf_train_dataset = small_train_dataset.to_tf_dataset(
<span class="hljs-meta">... </span>    columns=[<span class="hljs-string">&quot;attention_mask&quot;</span>, <span class="hljs-string">&quot;input_ids&quot;</span>, <span class="hljs-string">&quot;token_type_ids&quot;</span>],
<span class="hljs-meta">... </span>    label_cols=[<span class="hljs-string">&quot;labels&quot;</span>],
<span class="hljs-meta">... </span>    shuffle=<span class="hljs-literal">True</span>,
<span class="hljs-meta">... </span>    collate_fn=data_collator,
<span class="hljs-meta">... </span>    batch_size=<span class="hljs-number">8</span>,
<span class="hljs-meta">... </span>)

<span class="hljs-meta">&gt;&gt;&gt; </span>tf_validation_dataset = small_eval_dataset.to_tf_dataset(
<span class="hljs-meta">... </span>    columns=[<span class="hljs-string">&quot;attention_mask&quot;</span>, <span class="hljs-string">&quot;input_ids&quot;</span>, <span class="hljs-string">&quot;token_type_ids&quot;</span>],
<span class="hljs-meta">... </span>    label_cols=[<span class="hljs-string">&quot;labels&quot;</span>],
<span class="hljs-meta">... </span>    shuffle=<span class="hljs-literal">False</span>,
<span class="hljs-meta">... </span>    collate_fn=data_collator,
<span class="hljs-meta">... </span>    batch_size=<span class="hljs-number">8</span>,
<span class="hljs-meta">... </span>)`}}),dt=new x({}),gt=new y({props:{code:`import tensorflow as tf
from transformers import TFAutoModelForSequenceClassification

model = TFAutoModelForSequenceClassification.from_pretrained("bert-base-cased", num_labels=5)`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">import</span> tensorflow <span class="hljs-keyword">as</span> tf
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> TFAutoModelForSequenceClassification

<span class="hljs-meta">&gt;&gt;&gt; </span>model = TFAutoModelForSequenceClassification.from_pretrained(<span class="hljs-string">&quot;bert-base-cased&quot;</span>, num_labels=<span class="hljs-number">5</span>)`}}),vt=new y({props:{code:`model.compile(
    optimizer=tf.keras.optimizers.Adam(learning_rate=5e-5),
    loss=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True),
    metrics=tf.metrics.SparseCategoricalAccuracy(),
)

model.fit(tf_train_dataset, validation_data=tf_validation_dataset, epochs=3)`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>model.<span class="hljs-built_in">compile</span>(
<span class="hljs-meta">... </span>    optimizer=tf.keras.optimizers.Adam(learning_rate=<span class="hljs-number">5e-5</span>),
<span class="hljs-meta">... </span>    loss=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=<span class="hljs-literal">True</span>),
<span class="hljs-meta">... </span>    metrics=tf.metrics.SparseCategoricalAccuracy(),
<span class="hljs-meta">... </span>)

<span class="hljs-meta">&gt;&gt;&gt; </span>model.fit(tf_train_dataset, validation_data=tf_validation_dataset, epochs=<span class="hljs-number">3</span>)`}}),$t=new x({}),wt=new Wo({props:{id:"Dh9CL8fyG80"}}),bt=new y({props:{code:`del model
del pytorch_model
del trainer
torch.cuda.empty_cache()`,highlighted:`<span class="hljs-keyword">del</span> model
<span class="hljs-keyword">del</span> pytorch_model
<span class="hljs-keyword">del</span> trainer
torch.cuda.empty_cache()`}}),Et=new y({props:{code:'tokenized_datasets = tokenized_datasets.remove_columns(["text"])',highlighted:'<span class="hljs-meta">&gt;&gt;&gt; </span>tokenized_datasets = tokenized_datasets.remove_columns([<span class="hljs-string">&quot;text&quot;</span>])'}}),At=new y({props:{code:'tokenized_datasets = tokenized_datasets.rename_column("label", "labels")',highlighted:'<span class="hljs-meta">&gt;&gt;&gt; </span>tokenized_datasets = tokenized_datasets.rename_column(<span class="hljs-string">&quot;label&quot;</span>, <span class="hljs-string">&quot;labels&quot;</span>)'}}),qt=new y({props:{code:'tokenized_datasets.set_format("torch")',highlighted:'<span class="hljs-meta">&gt;&gt;&gt; </span>tokenized_datasets.set_format(<span class="hljs-string">&quot;torch&quot;</span>)'}}),xt=new y({props:{code:`small_train_dataset = tokenized_datasets["train"].shuffle(seed=42).select(range(1000))
small_eval_dataset = tokenized_datasets["test"].shuffle(seed=42).select(range(1000))`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>small_train_dataset = tokenized_datasets[<span class="hljs-string">&quot;train&quot;</span>].shuffle(seed=<span class="hljs-number">42</span>).select(<span class="hljs-built_in">range</span>(<span class="hljs-number">1000</span>))
<span class="hljs-meta">&gt;&gt;&gt; </span>small_eval_dataset = tokenized_datasets[<span class="hljs-string">&quot;test&quot;</span>].shuffle(seed=<span class="hljs-number">42</span>).select(<span class="hljs-built_in">range</span>(<span class="hljs-number">1000</span>))`}}),zt=new x({}),Ct=new y({props:{code:`from torch.utils.data import DataLoader

train_dataloader = DataLoader(small_train_dataset, shuffle=True, batch_size=8)
eval_dataloader = DataLoader(small_eval_dataset, batch_size=8)`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> torch.utils.data <span class="hljs-keyword">import</span> DataLoader

<span class="hljs-meta">&gt;&gt;&gt; </span>train_dataloader = DataLoader(small_train_dataset, shuffle=<span class="hljs-literal">True</span>, batch_size=<span class="hljs-number">8</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>eval_dataloader = DataLoader(small_eval_dataset, batch_size=<span class="hljs-number">8</span>)`}}),Dt=new y({props:{code:`from transformers import AutoModelForSequenceClassification

model = AutoModelForSequenceClassification.from_pretrained("bert-base-cased", num_labels=5)`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoModelForSequenceClassification

<span class="hljs-meta">&gt;&gt;&gt; </span>model = AutoModelForSequenceClassification.from_pretrained(<span class="hljs-string">&quot;bert-base-cased&quot;</span>, num_labels=<span class="hljs-number">5</span>)`}}),Ft=new x({}),Nt=new y({props:{code:`from torch.optim import AdamW

optimizer = AdamW(model.parameters(), lr=5e-5)`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> torch.optim <span class="hljs-keyword">import</span> AdamW

<span class="hljs-meta">&gt;&gt;&gt; </span>optimizer = AdamW(model.parameters(), lr=<span class="hljs-number">5e-5</span>)`}}),It=new y({props:{code:`from transformers import get_scheduler

num_epochs = 3
num_training_steps = num_epochs * len(train_dataloader)
lr_scheduler = get_scheduler(
    name="linear", optimizer=optimizer, num_warmup_steps=0, num_training_steps=num_training_steps
)`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> get_scheduler

<span class="hljs-meta">&gt;&gt;&gt; </span>num_epochs = <span class="hljs-number">3</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>num_training_steps = num_epochs * <span class="hljs-built_in">len</span>(train_dataloader)
<span class="hljs-meta">&gt;&gt;&gt; </span>lr_scheduler = get_scheduler(
<span class="hljs-meta">... </span>    name=<span class="hljs-string">&quot;linear&quot;</span>, optimizer=optimizer, num_warmup_steps=<span class="hljs-number">0</span>, num_training_steps=num_training_steps
<span class="hljs-meta">... </span>)`}}),Lt=new y({props:{code:`import torch

device = torch.device("cuda") if torch.cuda.is_available() else torch.device("cpu")
model.to(device)`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">import</span> torch

<span class="hljs-meta">&gt;&gt;&gt; </span>device = torch.device(<span class="hljs-string">&quot;cuda&quot;</span>) <span class="hljs-keyword">if</span> torch.cuda.is_available() <span class="hljs-keyword">else</span> torch.device(<span class="hljs-string">&quot;cpu&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>model.to(device)`}}),qe=new Bi({props:{$$slots:{default:[Kf]},$$scope:{ctx:Y}}}),Mt=new x({}),Bt=new y({props:{code:`from tqdm.auto import tqdm

progress_bar = tqdm(range(num_training_steps))

model.train()
for epoch in range(num_epochs):
    for batch in train_dataloader:
        batch = {k: v.to(device) for k, v in batch.items()}
        outputs = model(**batch)
        loss = outputs.loss
        loss.backward()

        optimizer.step()
        lr_scheduler.step()
        optimizer.zero_grad()
        progress_bar.update(1)`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> tqdm.auto <span class="hljs-keyword">import</span> tqdm

<span class="hljs-meta">&gt;&gt;&gt; </span>progress_bar = tqdm(<span class="hljs-built_in">range</span>(num_training_steps))

<span class="hljs-meta">&gt;&gt;&gt; </span>model.train()
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">for</span> epoch <span class="hljs-keyword">in</span> <span class="hljs-built_in">range</span>(num_epochs):
<span class="hljs-meta">... </span>    <span class="hljs-keyword">for</span> batch <span class="hljs-keyword">in</span> train_dataloader:
<span class="hljs-meta">... </span>        batch = {k: v.to(device) <span class="hljs-keyword">for</span> k, v <span class="hljs-keyword">in</span> batch.items()}
<span class="hljs-meta">... </span>        outputs = model(**batch)
<span class="hljs-meta">... </span>        loss = outputs.loss
<span class="hljs-meta">... </span>        loss.backward()

<span class="hljs-meta">... </span>        optimizer.step()
<span class="hljs-meta">... </span>        lr_scheduler.step()
<span class="hljs-meta">... </span>        optimizer.zero_grad()
<span class="hljs-meta">... </span>        progress_bar.update(<span class="hljs-number">1</span>)`}}),Ht=new x({}),Wt=new y({props:{code:`metric = load_metric("accuracy")
model.eval()
for batch in eval_dataloader:
    batch = {k: v.to(device) for k, v in batch.items()}
    with torch.no_grad():
        outputs = model(**batch)

    logits = outputs.logits
    predictions = torch.argmax(logits, dim=-1)
    metric.add_batch(predictions=predictions, references=batch["labels"])

metric.compute()`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>metric = load_metric(<span class="hljs-string">&quot;accuracy&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>model.<span class="hljs-built_in">eval</span>()
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">for</span> batch <span class="hljs-keyword">in</span> eval_dataloader:
<span class="hljs-meta">... </span>    batch = {k: v.to(device) <span class="hljs-keyword">for</span> k, v <span class="hljs-keyword">in</span> batch.items()}
<span class="hljs-meta">... </span>    <span class="hljs-keyword">with</span> torch.no_grad():
<span class="hljs-meta">... </span>        outputs = model(**batch)

<span class="hljs-meta">... </span>    logits = outputs.logits
<span class="hljs-meta">... </span>    predictions = torch.argmax(logits, dim=-<span class="hljs-number">1</span>)
<span class="hljs-meta">... </span>    metric.add_batch(predictions=predictions, references=batch[<span class="hljs-string">&quot;labels&quot;</span>])

<span class="hljs-meta">&gt;&gt;&gt; </span>metric.compute()`}}),Rt=new x({}),{c(){$=r("meta"),k=h(),w=r("h1"),b=r("a"),T=r("span"),c(j.$$.fragment),F=h(),P=r("span"),E=i("Fine-tune a pretrained model"),A=h(),c(z.$$.fragment),Le=h(),Ut=r("p"),Ro=i("There are significant benefits to using a pretrained model. It reduces computation costs, your carbon footprint, and allows you to use state-of-the-art models without having to train one from scratch. \u{1F917} Transformers provides access to thousands of pretrained models for a wide range of tasks. When you use a pretrained model, you train it on a dataset specific to your task. This is known as fine-tuning, an incredibly powerful training technique. In this tutorial, you will fine-tune a pretrained model with a deep learning framework of your choice:"),Fs=h(),I=r("ul"),Me=r("li"),Go=i("Fine-tune a pretrained model with \u{1F917} Transformers "),Jt=r("a"),Ko=i("Trainer"),Uo=i("."),Jo=h(),Ca=r("li"),Xo=i("Fine-tune a pretrained model in TensorFlow with Keras."),Zo=h(),Da=r("li"),Qo=i("Fine-tune a pretrained model in native PyTorch."),Ss=h(),Xt=r("a"),Ns=h(),W=r("h2"),re=r("a"),Fa=r("span"),c(Oe.$$.fragment),Vo=h(),Sa=r("span"),en=i("Prepare a dataset"),Is=h(),c(Be.$$.fragment),Ls=h(),Zt=r("p"),tn=i("Before you can fine-tune a pretrained model, download a dataset and prepare it for training. The previous tutorial showed you how to process data for training, and now you get an opportunity to put those skills to the test!"),Ms=h(),oe=r("p"),an=i("Begin by loading the "),He=r("a"),sn=i("Yelp Reviews"),rn=i(" dataset:"),Os=h(),c(Ye.$$.fragment),Bs=h(),ne=r("p"),on=i("As you now know, you need a tokenizer to process the text and include a padding and truncation strategy to handle any variable sequence lengths. To process your dataset in one step, use \u{1F917} Datasets "),We=r("a"),Na=r("code"),nn=i("map"),ln=i(" method to apply a preprocessing function over the entire dataset:"),Hs=h(),c(Re.$$.fragment),Ys=h(),Qt=r("p"),pn=i("If you like, you can create a smaller subset of the full dataset to fine-tune on to reduce the time it takes:"),Ws=h(),c(Ge.$$.fragment),Rs=h(),Vt=r("a"),Gs=h(),R=r("h2"),le=r("a"),Ia=r("span"),c(Ke.$$.fragment),fn=h(),ea=r("span"),hn=i("Fine-tune with "),La=r("code"),mn=i("Trainer"),Ks=h(),c(Ue.$$.fragment),Us=h(),L=r("p"),cn=i("\u{1F917} Transformers provides a "),ta=r("a"),un=i("Trainer"),dn=i(" class optimized for training \u{1F917} Transformers models, making it easier to start training without manually writing your own training loop. The "),aa=r("a"),gn=i("Trainer"),_n=i(" API supports a wide range of training options and features such as logging, gradient accumulation, and mixed precision."),Js=h(),ie=r("p"),vn=i("Start by loading your model and specify the number of expected labels. From the Yelp Review "),Je=r("a"),$n=i("dataset card"),wn=i(", you know there are five labels:"),Xs=h(),c(Xe.$$.fragment),Zs=h(),c(pe.$$.fragment),Qs=h(),G=r("h3"),fe=r("a"),Ma=r("span"),c(Ze.$$.fragment),yn=h(),Oa=r("span"),bn=i("Training hyperparameters"),Vs=h(),M=r("p"),kn=i("Next, create a "),sa=r("a"),jn=i("TrainingArguments"),En=i(" class which contains all the hyperparameters you can tune as well as flags for activating different training options. For this tutorial you can start with the default training "),Qe=r("a"),Tn=i("hyperparameters"),An=i(", but feel free to experiment with these to find your optimal settings."),er=h(),ra=r("p"),Pn=i("Specify where to save the checkpoints from your training:"),tr=h(),c(Ve.$$.fragment),ar=h(),K=r("h3"),he=r("a"),Ba=r("span"),c(et.$$.fragment),qn=h(),Ha=r("span"),xn=i("Metrics"),sr=h(),q=r("p"),oa=r("a"),zn=i("Trainer"),Cn=i(" does not automatically evaluate model performance during training. You will need to pass "),na=r("a"),Dn=i("Trainer"),Fn=i(" a function to compute and report metrics. The \u{1F917} Datasets library provides a simple "),tt=r("a"),Ya=r("code"),Sn=i("accuracy"),Nn=i(" function you can load with the "),Wa=r("code"),In=i("load_metric"),Ln=i(" (see this "),at=r("a"),Mn=i("tutorial"),On=i(" for more information) function:"),rr=h(),c(st.$$.fragment),or=h(),C=r("p"),Bn=i("Call "),Ra=r("code"),Hn=i("compute"),Yn=i(" on "),Ga=r("code"),Wn=i("metric"),Rn=i(" to calculate the accuracy of your predictions. Before passing your predictions to "),Ka=r("code"),Gn=i("compute"),Kn=i(", you need to convert the predictions to logits (remember all \u{1F917} Transformers models return logits):"),nr=h(),c(rt.$$.fragment),lr=h(),me=r("p"),Un=i("If you\u2019d like to monitor your evaluation metrics during fine-tuning, specify the "),Ua=r("code"),Jn=i("evaluation_strategy"),Xn=i(" parameter in your training arguments to report the evaluation metric at the end of each epoch:"),ir=h(),c(ot.$$.fragment),pr=h(),U=r("h3"),ce=r("a"),Ja=r("span"),c(nt.$$.fragment),Zn=h(),Xa=r("span"),Qn=i("Trainer"),fr=h(),ue=r("p"),Vn=i("Create a "),la=r("a"),el=i("Trainer"),tl=i(" object with your model, training arguments, training and test datasets, and evaluation function:"),hr=h(),c(lt.$$.fragment),mr=h(),de=r("p"),al=i("Then fine-tune your model by calling "),ia=r("a"),sl=i("train()"),rl=i(":"),cr=h(),c(it.$$.fragment),ur=h(),pa=r("a"),dr=h(),J=r("h2"),ge=r("a"),Za=r("span"),c(pt.$$.fragment),ol=h(),Qa=r("span"),nl=i("Fine-tune with Keras"),gr=h(),c(ft.$$.fragment),_r=h(),fa=r("p"),ll=i("\u{1F917} Transformers models also supports training in TensorFlow with the Keras API. You only need to make a few changes before you can fine-tune."),vr=h(),X=r("h3"),_e=r("a"),Va=r("span"),c(ht.$$.fragment),il=h(),es=r("span"),pl=i("Convert dataset to TensorFlow format"),$r=h(),O=r("p"),fl=i("The "),ha=r("a"),hl=i("DefaultDataCollator"),ml=i(" assembles tensors into a batch for the model to train on. Make sure you specify "),ts=r("code"),cl=i("return_tensors"),ul=i(" to return TensorFlow tensors:"),wr=h(),c(mt.$$.fragment),yr=h(),c(ve.$$.fragment),br=h(),D=r("p"),dl=i("Next, convert the tokenized datasets to TensorFlow datasets with the "),ct=r("a"),as=r("code"),gl=i("to_tf_dataset"),_l=i(" method. Specify your inputs in "),ss=r("code"),vl=i("columns"),$l=i(", and your label in "),rs=r("code"),wl=i("label_cols"),yl=i(":"),kr=h(),c(ut.$$.fragment),jr=h(),Z=r("h3"),$e=r("a"),os=r("span"),c(dt.$$.fragment),bl=h(),ns=r("span"),kl=i("Compile and fit"),Er=h(),ma=r("p"),jl=i("Load a TensorFlow model with the expected number of labels:"),Tr=h(),c(gt.$$.fragment),Ar=h(),we=r("p"),El=i("Then compile and fine-tune your model with "),_t=r("a"),ls=r("code"),Tl=i("fit"),Al=i(" as you would with any other Keras model:"),Pr=h(),c(vt.$$.fragment),qr=h(),ca=r("a"),xr=h(),Q=r("h2"),ye=r("a"),is=r("span"),c($t.$$.fragment),Pl=h(),ps=r("span"),ql=i("Fine-tune in native PyTorch"),zr=h(),c(wt.$$.fragment),Cr=h(),yt=r("p"),ua=r("a"),xl=i("Trainer"),zl=i(" takes care of the training loop and allows you to fine-tune a model in a single line of code. For users who prefer to write their own training loop, you can also fine-tune a \u{1F917} Transformers model in native PyTorch."),Dr=h(),da=r("p"),Cl=i("At this point, you may need to restart your notebook or execute the following code to free some memory:"),Fr=h(),c(bt.$$.fragment),Sr=h(),be=r("p"),Dl=i("Next, manually postprocess "),fs=r("code"),Fl=i("tokenized_dataset"),Sl=i(" to prepare it for training."),Nr=h(),B=r("ol"),kt=r("li"),jt=r("p"),Nl=i("Remove the "),hs=r("code"),Il=i("text"),Ll=i(" column because the model does not accept raw text as an input:"),Ml=h(),c(Et.$$.fragment),Ol=h(),Tt=r("li"),S=r("p"),Bl=i("Rename the "),ms=r("code"),Hl=i("label"),Yl=i(" column to "),cs=r("code"),Wl=i("labels"),Rl=i(" because the model expects the argument to be named "),us=r("code"),Gl=i("labels"),Kl=i(":"),Ul=h(),c(At.$$.fragment),Jl=h(),Pt=r("li"),ds=r("p"),Xl=i("Set the format of the dataset to return PyTorch tensors instead of lists:"),Zl=h(),c(qt.$$.fragment),Ir=h(),ga=r("p"),Ql=i("Then create a smaller subset of the dataset as previously shown to speed up the fine-tuning:"),Lr=h(),c(xt.$$.fragment),Mr=h(),V=r("h3"),ke=r("a"),gs=r("span"),c(zt.$$.fragment),Vl=h(),_s=r("span"),ei=i("DataLoader"),Or=h(),je=r("p"),ti=i("Create a "),vs=r("code"),ai=i("DataLoader"),si=i(" for your training and test datasets so you can iterate over batches of data:"),Br=h(),c(Ct.$$.fragment),Hr=h(),_a=r("p"),ri=i("Load your model with the number of expected labels:"),Yr=h(),c(Dt.$$.fragment),Wr=h(),ee=r("h3"),Ee=r("a"),$s=r("span"),c(Ft.$$.fragment),oi=h(),ws=r("span"),ni=i("Optimizer and learning rate scheduler"),Rr=h(),Te=r("p"),li=i("Create an optimizer and learning rate scheduler to fine-tune the model. Let\u2019s use the "),St=r("a"),ys=r("code"),ii=i("AdamW"),pi=i(" optimizer from PyTorch:"),Gr=h(),c(Nt.$$.fragment),Kr=h(),Ae=r("p"),fi=i("Create the default learning rate scheduler from "),va=r("a"),hi=i("Trainer"),mi=i(":"),Ur=h(),c(It.$$.fragment),Jr=h(),Pe=r("p"),ci=i("Lastly, specify "),bs=r("code"),ui=i("device"),di=i(" to use a GPU if you have access to one. Otherwise, training on a CPU may take several hours instead of a couple of minutes."),Xr=h(),c(Lt.$$.fragment),Zr=h(),c(qe.$$.fragment),Qr=h(),$a=r("p"),gi=i("Great, now you are ready to train! \u{1F973}"),Vr=h(),te=r("h3"),xe=r("a"),ks=r("span"),c(Mt.$$.fragment),_i=h(),js=r("span"),vi=i("Training loop"),eo=h(),ze=r("p"),$i=i("To keep track of your training progress, use the "),Ot=r("a"),wi=i("tqdm"),yi=i(" library to add a progress bar over the number of training steps:"),to=h(),c(Bt.$$.fragment),ao=h(),ae=r("h3"),Ce=r("a"),Es=r("span"),c(Ht.$$.fragment),bi=h(),Ts=r("span"),ki=i("Metrics"),so=h(),H=r("p"),ji=i("Just like how you need to add an evaluation function to "),wa=r("a"),Ei=i("Trainer"),Ti=i(", you need to do the same when you write your own training loop. But instead of calculating and reporting the metric at the end of each epoch, this time you will accumulate all the batches with "),Yt=r("a"),As=r("code"),Ai=i("add_batch"),Pi=i(" and calculate the metric at the very end."),ro=h(),c(Wt.$$.fragment),oo=h(),ya=r("a"),no=h(),se=r("h2"),De=r("a"),Ps=r("span"),c(Rt.$$.fragment),qi=h(),qs=r("span"),xi=i("Additional resources"),lo=h(),ba=r("p"),zi=i("For more fine-tuning examples, refer to:"),io=h(),Fe=r("ul"),xs=r("li"),ka=r("p"),Gt=r("a"),Ci=i("\u{1F917} Transformers Examples"),Di=i(` includes scripts
to train common NLP tasks in PyTorch and TensorFlow.`),Fi=h(),zs=r("li"),ja=r("p"),Ea=r("a"),Si=i("\u{1F917} Transformers Notebooks"),Ni=i(" contains various notebooks on how to fine-tune a model for specific tasks in PyTorch and TensorFlow."),this.h()},l(e){const s=Yf('[data-svelte="svelte-1phssyn"]',document.head);$=o(s,"META",{name:!0,content:!0}),s.forEach(t),k=m(e),w=o(e,"H1",{class:!0});var Kt=n(w);b=o(Kt,"A",{id:!0,class:!0,href:!0});var Cs=n(b);T=o(Cs,"SPAN",{});var Ds=n(T);u(j.$$.fragment,Ds),Ds.forEach(t),Cs.forEach(t),F=m(Kt),P=o(Kt,"SPAN",{});var Hi=n(P);E=p(Hi,"Fine-tune a pretrained model"),Hi.forEach(t),Kt.forEach(t),A=m(e),u(z.$$.fragment,e),Le=m(e),Ut=o(e,"P",{});var Yi=n(Ut);Ro=p(Yi,"There are significant benefits to using a pretrained model. It reduces computation costs, your carbon footprint, and allows you to use state-of-the-art models without having to train one from scratch. \u{1F917} Transformers provides access to thousands of pretrained models for a wide range of tasks. When you use a pretrained model, you train it on a dataset specific to your task. This is known as fine-tuning, an incredibly powerful training technique. In this tutorial, you will fine-tune a pretrained model with a deep learning framework of your choice:"),Yi.forEach(t),Fs=m(e),I=o(e,"UL",{});var Ta=n(I);Me=o(Ta,"LI",{});var fo=n(Me);Go=p(fo,"Fine-tune a pretrained model with \u{1F917} Transformers "),Jt=o(fo,"A",{href:!0});var Wi=n(Jt);Ko=p(Wi,"Trainer"),Wi.forEach(t),Uo=p(fo,"."),fo.forEach(t),Jo=m(Ta),Ca=o(Ta,"LI",{});var Ri=n(Ca);Xo=p(Ri,"Fine-tune a pretrained model in TensorFlow with Keras."),Ri.forEach(t),Zo=m(Ta),Da=o(Ta,"LI",{});var Gi=n(Da);Qo=p(Gi,"Fine-tune a pretrained model in native PyTorch."),Gi.forEach(t),Ta.forEach(t),Ss=m(e),Xt=o(e,"A",{id:!0}),n(Xt).forEach(t),Ns=m(e),W=o(e,"H2",{class:!0});var ho=n(W);re=o(ho,"A",{id:!0,class:!0,href:!0});var Ki=n(re);Fa=o(Ki,"SPAN",{});var Ui=n(Fa);u(Oe.$$.fragment,Ui),Ui.forEach(t),Ki.forEach(t),Vo=m(ho),Sa=o(ho,"SPAN",{});var Ji=n(Sa);en=p(Ji,"Prepare a dataset"),Ji.forEach(t),ho.forEach(t),Is=m(e),u(Be.$$.fragment,e),Ls=m(e),Zt=o(e,"P",{});var Xi=n(Zt);tn=p(Xi,"Before you can fine-tune a pretrained model, download a dataset and prepare it for training. The previous tutorial showed you how to process data for training, and now you get an opportunity to put those skills to the test!"),Xi.forEach(t),Ms=m(e),oe=o(e,"P",{});var mo=n(oe);an=p(mo,"Begin by loading the "),He=o(mo,"A",{href:!0,rel:!0});var Zi=n(He);sn=p(Zi,"Yelp Reviews"),Zi.forEach(t),rn=p(mo," dataset:"),mo.forEach(t),Os=m(e),u(Ye.$$.fragment,e),Bs=m(e),ne=o(e,"P",{});var co=n(ne);on=p(co,"As you now know, you need a tokenizer to process the text and include a padding and truncation strategy to handle any variable sequence lengths. To process your dataset in one step, use \u{1F917} Datasets "),We=o(co,"A",{href:!0,rel:!0});var Qi=n(We);Na=o(Qi,"CODE",{});var Vi=n(Na);nn=p(Vi,"map"),Vi.forEach(t),Qi.forEach(t),ln=p(co," method to apply a preprocessing function over the entire dataset:"),co.forEach(t),Hs=m(e),u(Re.$$.fragment,e),Ys=m(e),Qt=o(e,"P",{});var ep=n(Qt);pn=p(ep,"If you like, you can create a smaller subset of the full dataset to fine-tune on to reduce the time it takes:"),ep.forEach(t),Ws=m(e),u(Ge.$$.fragment,e),Rs=m(e),Vt=o(e,"A",{id:!0}),n(Vt).forEach(t),Gs=m(e),R=o(e,"H2",{class:!0});var uo=n(R);le=o(uo,"A",{id:!0,class:!0,href:!0});var tp=n(le);Ia=o(tp,"SPAN",{});var ap=n(Ia);u(Ke.$$.fragment,ap),ap.forEach(t),tp.forEach(t),fn=m(uo),ea=o(uo,"SPAN",{});var Ii=n(ea);hn=p(Ii,"Fine-tune with "),La=o(Ii,"CODE",{});var sp=n(La);mn=p(sp,"Trainer"),sp.forEach(t),Ii.forEach(t),uo.forEach(t),Ks=m(e),u(Ue.$$.fragment,e),Us=m(e),L=o(e,"P",{});var Aa=n(L);cn=p(Aa,"\u{1F917} Transformers provides a "),ta=o(Aa,"A",{href:!0});var rp=n(ta);un=p(rp,"Trainer"),rp.forEach(t),dn=p(Aa," class optimized for training \u{1F917} Transformers models, making it easier to start training without manually writing your own training loop. The "),aa=o(Aa,"A",{href:!0});var op=n(aa);gn=p(op,"Trainer"),op.forEach(t),_n=p(Aa," API supports a wide range of training options and features such as logging, gradient accumulation, and mixed precision."),Aa.forEach(t),Js=m(e),ie=o(e,"P",{});var go=n(ie);vn=p(go,"Start by loading your model and specify the number of expected labels. From the Yelp Review "),Je=o(go,"A",{href:!0,rel:!0});var np=n(Je);$n=p(np,"dataset card"),np.forEach(t),wn=p(go,", you know there are five labels:"),go.forEach(t),Xs=m(e),u(Xe.$$.fragment,e),Zs=m(e),u(pe.$$.fragment,e),Qs=m(e),G=o(e,"H3",{class:!0});var _o=n(G);fe=o(_o,"A",{id:!0,class:!0,href:!0});var lp=n(fe);Ma=o(lp,"SPAN",{});var ip=n(Ma);u(Ze.$$.fragment,ip),ip.forEach(t),lp.forEach(t),yn=m(_o),Oa=o(_o,"SPAN",{});var pp=n(Oa);bn=p(pp,"Training hyperparameters"),pp.forEach(t),_o.forEach(t),Vs=m(e),M=o(e,"P",{});var Pa=n(M);kn=p(Pa,"Next, create a "),sa=o(Pa,"A",{href:!0});var fp=n(sa);jn=p(fp,"TrainingArguments"),fp.forEach(t),En=p(Pa," class which contains all the hyperparameters you can tune as well as flags for activating different training options. For this tutorial you can start with the default training "),Qe=o(Pa,"A",{href:!0,rel:!0});var hp=n(Qe);Tn=p(hp,"hyperparameters"),hp.forEach(t),An=p(Pa,", but feel free to experiment with these to find your optimal settings."),Pa.forEach(t),er=m(e),ra=o(e,"P",{});var mp=n(ra);Pn=p(mp,"Specify where to save the checkpoints from your training:"),mp.forEach(t),tr=m(e),u(Ve.$$.fragment,e),ar=m(e),K=o(e,"H3",{class:!0});var vo=n(K);he=o(vo,"A",{id:!0,class:!0,href:!0});var cp=n(he);Ba=o(cp,"SPAN",{});var up=n(Ba);u(et.$$.fragment,up),up.forEach(t),cp.forEach(t),qn=m(vo),Ha=o(vo,"SPAN",{});var dp=n(Ha);xn=p(dp,"Metrics"),dp.forEach(t),vo.forEach(t),sr=m(e),q=o(e,"P",{});var N=n(q);oa=o(N,"A",{href:!0});var gp=n(oa);zn=p(gp,"Trainer"),gp.forEach(t),Cn=p(N," does not automatically evaluate model performance during training. You will need to pass "),na=o(N,"A",{href:!0});var _p=n(na);Dn=p(_p,"Trainer"),_p.forEach(t),Fn=p(N," a function to compute and report metrics. The \u{1F917} Datasets library provides a simple "),tt=o(N,"A",{href:!0,rel:!0});var vp=n(tt);Ya=o(vp,"CODE",{});var $p=n(Ya);Sn=p($p,"accuracy"),$p.forEach(t),vp.forEach(t),Nn=p(N," function you can load with the "),Wa=o(N,"CODE",{});var wp=n(Wa);In=p(wp,"load_metric"),wp.forEach(t),Ln=p(N," (see this "),at=o(N,"A",{href:!0,rel:!0});var yp=n(at);Mn=p(yp,"tutorial"),yp.forEach(t),On=p(N," for more information) function:"),N.forEach(t),rr=m(e),u(st.$$.fragment,e),or=m(e),C=o(e,"P",{});var Se=n(C);Bn=p(Se,"Call "),Ra=o(Se,"CODE",{});var bp=n(Ra);Hn=p(bp,"compute"),bp.forEach(t),Yn=p(Se," on "),Ga=o(Se,"CODE",{});var kp=n(Ga);Wn=p(kp,"metric"),kp.forEach(t),Rn=p(Se," to calculate the accuracy of your predictions. Before passing your predictions to "),Ka=o(Se,"CODE",{});var jp=n(Ka);Gn=p(jp,"compute"),jp.forEach(t),Kn=p(Se,", you need to convert the predictions to logits (remember all \u{1F917} Transformers models return logits):"),Se.forEach(t),nr=m(e),u(rt.$$.fragment,e),lr=m(e),me=o(e,"P",{});var $o=n(me);Un=p($o,"If you\u2019d like to monitor your evaluation metrics during fine-tuning, specify the "),Ua=o($o,"CODE",{});var Ep=n(Ua);Jn=p(Ep,"evaluation_strategy"),Ep.forEach(t),Xn=p($o," parameter in your training arguments to report the evaluation metric at the end of each epoch:"),$o.forEach(t),ir=m(e),u(ot.$$.fragment,e),pr=m(e),U=o(e,"H3",{class:!0});var wo=n(U);ce=o(wo,"A",{id:!0,class:!0,href:!0});var Tp=n(ce);Ja=o(Tp,"SPAN",{});var Ap=n(Ja);u(nt.$$.fragment,Ap),Ap.forEach(t),Tp.forEach(t),Zn=m(wo),Xa=o(wo,"SPAN",{});var Pp=n(Xa);Qn=p(Pp,"Trainer"),Pp.forEach(t),wo.forEach(t),fr=m(e),ue=o(e,"P",{});var yo=n(ue);Vn=p(yo,"Create a "),la=o(yo,"A",{href:!0});var qp=n(la);el=p(qp,"Trainer"),qp.forEach(t),tl=p(yo," object with your model, training arguments, training and test datasets, and evaluation function:"),yo.forEach(t),hr=m(e),u(lt.$$.fragment,e),mr=m(e),de=o(e,"P",{});var bo=n(de);al=p(bo,"Then fine-tune your model by calling "),ia=o(bo,"A",{href:!0});var xp=n(ia);sl=p(xp,"train()"),xp.forEach(t),rl=p(bo,":"),bo.forEach(t),cr=m(e),u(it.$$.fragment,e),ur=m(e),pa=o(e,"A",{id:!0}),n(pa).forEach(t),dr=m(e),J=o(e,"H2",{class:!0});var ko=n(J);ge=o(ko,"A",{id:!0,class:!0,href:!0});var zp=n(ge);Za=o(zp,"SPAN",{});var Cp=n(Za);u(pt.$$.fragment,Cp),Cp.forEach(t),zp.forEach(t),ol=m(ko),Qa=o(ko,"SPAN",{});var Dp=n(Qa);nl=p(Dp,"Fine-tune with Keras"),Dp.forEach(t),ko.forEach(t),gr=m(e),u(ft.$$.fragment,e),_r=m(e),fa=o(e,"P",{});var Fp=n(fa);ll=p(Fp,"\u{1F917} Transformers models also supports training in TensorFlow with the Keras API. You only need to make a few changes before you can fine-tune."),Fp.forEach(t),vr=m(e),X=o(e,"H3",{class:!0});var jo=n(X);_e=o(jo,"A",{id:!0,class:!0,href:!0});var Sp=n(_e);Va=o(Sp,"SPAN",{});var Np=n(Va);u(ht.$$.fragment,Np),Np.forEach(t),Sp.forEach(t),il=m(jo),es=o(jo,"SPAN",{});var Ip=n(es);pl=p(Ip,"Convert dataset to TensorFlow format"),Ip.forEach(t),jo.forEach(t),$r=m(e),O=o(e,"P",{});var qa=n(O);fl=p(qa,"The "),ha=o(qa,"A",{href:!0});var Lp=n(ha);hl=p(Lp,"DefaultDataCollator"),Lp.forEach(t),ml=p(qa," assembles tensors into a batch for the model to train on. Make sure you specify "),ts=o(qa,"CODE",{});var Mp=n(ts);cl=p(Mp,"return_tensors"),Mp.forEach(t),ul=p(qa," to return TensorFlow tensors:"),qa.forEach(t),wr=m(e),u(mt.$$.fragment,e),yr=m(e),u(ve.$$.fragment,e),br=m(e),D=o(e,"P",{});var Ne=n(D);dl=p(Ne,"Next, convert the tokenized datasets to TensorFlow datasets with the "),ct=o(Ne,"A",{href:!0,rel:!0});var Op=n(ct);as=o(Op,"CODE",{});var Bp=n(as);gl=p(Bp,"to_tf_dataset"),Bp.forEach(t),Op.forEach(t),_l=p(Ne," method. Specify your inputs in "),ss=o(Ne,"CODE",{});var Hp=n(ss);vl=p(Hp,"columns"),Hp.forEach(t),$l=p(Ne,", and your label in "),rs=o(Ne,"CODE",{});var Yp=n(rs);wl=p(Yp,"label_cols"),Yp.forEach(t),yl=p(Ne,":"),Ne.forEach(t),kr=m(e),u(ut.$$.fragment,e),jr=m(e),Z=o(e,"H3",{class:!0});var Eo=n(Z);$e=o(Eo,"A",{id:!0,class:!0,href:!0});var Wp=n($e);os=o(Wp,"SPAN",{});var Rp=n(os);u(dt.$$.fragment,Rp),Rp.forEach(t),Wp.forEach(t),bl=m(Eo),ns=o(Eo,"SPAN",{});var Gp=n(ns);kl=p(Gp,"Compile and fit"),Gp.forEach(t),Eo.forEach(t),Er=m(e),ma=o(e,"P",{});var Kp=n(ma);jl=p(Kp,"Load a TensorFlow model with the expected number of labels:"),Kp.forEach(t),Tr=m(e),u(gt.$$.fragment,e),Ar=m(e),we=o(e,"P",{});var To=n(we);El=p(To,"Then compile and fine-tune your model with "),_t=o(To,"A",{href:!0,rel:!0});var Up=n(_t);ls=o(Up,"CODE",{});var Jp=n(ls);Tl=p(Jp,"fit"),Jp.forEach(t),Up.forEach(t),Al=p(To," as you would with any other Keras model:"),To.forEach(t),Pr=m(e),u(vt.$$.fragment,e),qr=m(e),ca=o(e,"A",{id:!0}),n(ca).forEach(t),xr=m(e),Q=o(e,"H2",{class:!0});var Ao=n(Q);ye=o(Ao,"A",{id:!0,class:!0,href:!0});var Xp=n(ye);is=o(Xp,"SPAN",{});var Zp=n(is);u($t.$$.fragment,Zp),Zp.forEach(t),Xp.forEach(t),Pl=m(Ao),ps=o(Ao,"SPAN",{});var Qp=n(ps);ql=p(Qp,"Fine-tune in native PyTorch"),Qp.forEach(t),Ao.forEach(t),zr=m(e),u(wt.$$.fragment,e),Cr=m(e),yt=o(e,"P",{});var Li=n(yt);ua=o(Li,"A",{href:!0});var Vp=n(ua);xl=p(Vp,"Trainer"),Vp.forEach(t),zl=p(Li," takes care of the training loop and allows you to fine-tune a model in a single line of code. For users who prefer to write their own training loop, you can also fine-tune a \u{1F917} Transformers model in native PyTorch."),Li.forEach(t),Dr=m(e),da=o(e,"P",{});var ef=n(da);Cl=p(ef,"At this point, you may need to restart your notebook or execute the following code to free some memory:"),ef.forEach(t),Fr=m(e),u(bt.$$.fragment,e),Sr=m(e),be=o(e,"P",{});var Po=n(be);Dl=p(Po,"Next, manually postprocess "),fs=o(Po,"CODE",{});var tf=n(fs);Fl=p(tf,"tokenized_dataset"),tf.forEach(t),Sl=p(Po," to prepare it for training."),Po.forEach(t),Nr=m(e),B=o(e,"OL",{});var xa=n(B);kt=o(xa,"LI",{});var qo=n(kt);jt=o(qo,"P",{});var xo=n(jt);Nl=p(xo,"Remove the "),hs=o(xo,"CODE",{});var af=n(hs);Il=p(af,"text"),af.forEach(t),Ll=p(xo," column because the model does not accept raw text as an input:"),xo.forEach(t),Ml=m(qo),u(Et.$$.fragment,qo),qo.forEach(t),Ol=m(xa),Tt=o(xa,"LI",{});var zo=n(Tt);S=o(zo,"P",{});var Ie=n(S);Bl=p(Ie,"Rename the "),ms=o(Ie,"CODE",{});var sf=n(ms);Hl=p(sf,"label"),sf.forEach(t),Yl=p(Ie," column to "),cs=o(Ie,"CODE",{});var rf=n(cs);Wl=p(rf,"labels"),rf.forEach(t),Rl=p(Ie," because the model expects the argument to be named "),us=o(Ie,"CODE",{});var of=n(us);Gl=p(of,"labels"),of.forEach(t),Kl=p(Ie,":"),Ie.forEach(t),Ul=m(zo),u(At.$$.fragment,zo),zo.forEach(t),Jl=m(xa),Pt=o(xa,"LI",{});var Co=n(Pt);ds=o(Co,"P",{});var nf=n(ds);Xl=p(nf,"Set the format of the dataset to return PyTorch tensors instead of lists:"),nf.forEach(t),Zl=m(Co),u(qt.$$.fragment,Co),Co.forEach(t),xa.forEach(t),Ir=m(e),ga=o(e,"P",{});var lf=n(ga);Ql=p(lf,"Then create a smaller subset of the dataset as previously shown to speed up the fine-tuning:"),lf.forEach(t),Lr=m(e),u(xt.$$.fragment,e),Mr=m(e),V=o(e,"H3",{class:!0});var Do=n(V);ke=o(Do,"A",{id:!0,class:!0,href:!0});var pf=n(ke);gs=o(pf,"SPAN",{});var ff=n(gs);u(zt.$$.fragment,ff),ff.forEach(t),pf.forEach(t),Vl=m(Do),_s=o(Do,"SPAN",{});var hf=n(_s);ei=p(hf,"DataLoader"),hf.forEach(t),Do.forEach(t),Or=m(e),je=o(e,"P",{});var Fo=n(je);ti=p(Fo,"Create a "),vs=o(Fo,"CODE",{});var mf=n(vs);ai=p(mf,"DataLoader"),mf.forEach(t),si=p(Fo," for your training and test datasets so you can iterate over batches of data:"),Fo.forEach(t),Br=m(e),u(Ct.$$.fragment,e),Hr=m(e),_a=o(e,"P",{});var cf=n(_a);ri=p(cf,"Load your model with the number of expected labels:"),cf.forEach(t),Yr=m(e),u(Dt.$$.fragment,e),Wr=m(e),ee=o(e,"H3",{class:!0});var So=n(ee);Ee=o(So,"A",{id:!0,class:!0,href:!0});var uf=n(Ee);$s=o(uf,"SPAN",{});var df=n($s);u(Ft.$$.fragment,df),df.forEach(t),uf.forEach(t),oi=m(So),ws=o(So,"SPAN",{});var gf=n(ws);ni=p(gf,"Optimizer and learning rate scheduler"),gf.forEach(t),So.forEach(t),Rr=m(e),Te=o(e,"P",{});var No=n(Te);li=p(No,"Create an optimizer and learning rate scheduler to fine-tune the model. Let\u2019s use the "),St=o(No,"A",{href:!0,rel:!0});var _f=n(St);ys=o(_f,"CODE",{});var vf=n(ys);ii=p(vf,"AdamW"),vf.forEach(t),_f.forEach(t),pi=p(No," optimizer from PyTorch:"),No.forEach(t),Gr=m(e),u(Nt.$$.fragment,e),Kr=m(e),Ae=o(e,"P",{});var Io=n(Ae);fi=p(Io,"Create the default learning rate scheduler from "),va=o(Io,"A",{href:!0});var $f=n(va);hi=p($f,"Trainer"),$f.forEach(t),mi=p(Io,":"),Io.forEach(t),Ur=m(e),u(It.$$.fragment,e),Jr=m(e),Pe=o(e,"P",{});var Lo=n(Pe);ci=p(Lo,"Lastly, specify "),bs=o(Lo,"CODE",{});var wf=n(bs);ui=p(wf,"device"),wf.forEach(t),di=p(Lo," to use a GPU if you have access to one. Otherwise, training on a CPU may take several hours instead of a couple of minutes."),Lo.forEach(t),Xr=m(e),u(Lt.$$.fragment,e),Zr=m(e),u(qe.$$.fragment,e),Qr=m(e),$a=o(e,"P",{});var yf=n($a);gi=p(yf,"Great, now you are ready to train! \u{1F973}"),yf.forEach(t),Vr=m(e),te=o(e,"H3",{class:!0});var Mo=n(te);xe=o(Mo,"A",{id:!0,class:!0,href:!0});var bf=n(xe);ks=o(bf,"SPAN",{});var kf=n(ks);u(Mt.$$.fragment,kf),kf.forEach(t),bf.forEach(t),_i=m(Mo),js=o(Mo,"SPAN",{});var jf=n(js);vi=p(jf,"Training loop"),jf.forEach(t),Mo.forEach(t),eo=m(e),ze=o(e,"P",{});var Oo=n(ze);$i=p(Oo,"To keep track of your training progress, use the "),Ot=o(Oo,"A",{href:!0,rel:!0});var Ef=n(Ot);wi=p(Ef,"tqdm"),Ef.forEach(t),yi=p(Oo," library to add a progress bar over the number of training steps:"),Oo.forEach(t),to=m(e),u(Bt.$$.fragment,e),ao=m(e),ae=o(e,"H3",{class:!0});var Bo=n(ae);Ce=o(Bo,"A",{id:!0,class:!0,href:!0});var Tf=n(Ce);Es=o(Tf,"SPAN",{});var Af=n(Es);u(Ht.$$.fragment,Af),Af.forEach(t),Tf.forEach(t),bi=m(Bo),Ts=o(Bo,"SPAN",{});var Pf=n(Ts);ki=p(Pf,"Metrics"),Pf.forEach(t),Bo.forEach(t),so=m(e),H=o(e,"P",{});var za=n(H);ji=p(za,"Just like how you need to add an evaluation function to "),wa=o(za,"A",{href:!0});var qf=n(wa);Ei=p(qf,"Trainer"),qf.forEach(t),Ti=p(za,", you need to do the same when you write your own training loop. But instead of calculating and reporting the metric at the end of each epoch, this time you will accumulate all the batches with "),Yt=o(za,"A",{href:!0,rel:!0});var xf=n(Yt);As=o(xf,"CODE",{});var zf=n(As);Ai=p(zf,"add_batch"),zf.forEach(t),xf.forEach(t),Pi=p(za," and calculate the metric at the very end."),za.forEach(t),ro=m(e),u(Wt.$$.fragment,e),oo=m(e),ya=o(e,"A",{id:!0}),n(ya).forEach(t),no=m(e),se=o(e,"H2",{class:!0});var Ho=n(se);De=o(Ho,"A",{id:!0,class:!0,href:!0});var Cf=n(De);Ps=o(Cf,"SPAN",{});var Df=n(Ps);u(Rt.$$.fragment,Df),Df.forEach(t),Cf.forEach(t),qi=m(Ho),qs=o(Ho,"SPAN",{});var Ff=n(qs);xi=p(Ff,"Additional resources"),Ff.forEach(t),Ho.forEach(t),lo=m(e),ba=o(e,"P",{});var Sf=n(ba);zi=p(Sf,"For more fine-tuning examples, refer to:"),Sf.forEach(t),io=m(e),Fe=o(e,"UL",{});var Yo=n(Fe);xs=o(Yo,"LI",{});var Nf=n(xs);ka=o(Nf,"P",{});var Mi=n(ka);Gt=o(Mi,"A",{href:!0,rel:!0});var If=n(Gt);Ci=p(If,"\u{1F917} Transformers Examples"),If.forEach(t),Di=p(Mi,` includes scripts
to train common NLP tasks in PyTorch and TensorFlow.`),Mi.forEach(t),Nf.forEach(t),Fi=m(Yo),zs=o(Yo,"LI",{});var Lf=n(zs);ja=o(Lf,"P",{});var Oi=n(ja);Ea=o(Oi,"A",{href:!0});var Mf=n(Ea);Si=p(Mf,"\u{1F917} Transformers Notebooks"),Mf.forEach(t),Ni=p(Oi," contains various notebooks on how to fine-tune a model for specific tasks in PyTorch and TensorFlow."),Oi.forEach(t),Lf.forEach(t),Yo.forEach(t),this.h()},h(){f($,"name","hf:doc:metadata"),f($,"content",JSON.stringify(Jf)),f(b,"id","finetune-a-pretrained-model"),f(b,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),f(b,"href","#finetune-a-pretrained-model"),f(w,"class","relative group"),f(Jt,"href","/docs/transformers/pr_15839/en/main_classes/trainer#transformers.Trainer"),f(Xt,"id","data-processing"),f(re,"id","prepare-a-dataset"),f(re,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),f(re,"href","#prepare-a-dataset"),f(W,"class","relative group"),f(He,"href","https://huggingface.co/datasets/yelp_review_full"),f(He,"rel","nofollow"),f(We,"href","https://huggingface.co/docs/datasets/process.html#map"),f(We,"rel","nofollow"),f(Vt,"id","trainer"),f(le,"id","finetune-with-trainer"),f(le,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),f(le,"href","#finetune-with-trainer"),f(R,"class","relative group"),f(ta,"href","/docs/transformers/pr_15839/en/main_classes/trainer#transformers.Trainer"),f(aa,"href","/docs/transformers/pr_15839/en/main_classes/trainer#transformers.Trainer"),f(Je,"href","https://huggingface.co/datasets/yelp_review_full#data-fields"),f(Je,"rel","nofollow"),f(fe,"id","training-hyperparameters"),f(fe,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),f(fe,"href","#training-hyperparameters"),f(G,"class","relative group"),f(sa,"href","/docs/transformers/pr_15839/en/main_classes/trainer#transformers.TrainingArguments"),f(Qe,"href","https://huggingface.co/docs/transformers/main_classes/trainer#transformers.TrainingArguments"),f(Qe,"rel","nofollow"),f(he,"id","metrics"),f(he,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),f(he,"href","#metrics"),f(K,"class","relative group"),f(oa,"href","/docs/transformers/pr_15839/en/main_classes/trainer#transformers.Trainer"),f(na,"href","/docs/transformers/pr_15839/en/main_classes/trainer#transformers.Trainer"),f(tt,"href","https://huggingface.co/metrics/accuracy"),f(tt,"rel","nofollow"),f(at,"href","https://huggingface.co/docs/datasets/metrics.html"),f(at,"rel","nofollow"),f(ce,"id","trainer"),f(ce,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),f(ce,"href","#trainer"),f(U,"class","relative group"),f(la,"href","/docs/transformers/pr_15839/en/main_classes/trainer#transformers.Trainer"),f(ia,"href","/docs/transformers/pr_15839/en/main_classes/trainer#transformers.Trainer.train"),f(pa,"id","keras"),f(ge,"id","finetune-with-keras"),f(ge,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),f(ge,"href","#finetune-with-keras"),f(J,"class","relative group"),f(_e,"id","convert-dataset-to-tensorflow-format"),f(_e,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),f(_e,"href","#convert-dataset-to-tensorflow-format"),f(X,"class","relative group"),f(ha,"href","/docs/transformers/pr_15839/en/main_classes/data_collator#transformers.DefaultDataCollator"),f(ct,"href","https://huggingface.co/docs/datasets/package_reference/main_classes.html#datasets.Dataset.to_tf_dataset"),f(ct,"rel","nofollow"),f($e,"id","compile-and-fit"),f($e,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),f($e,"href","#compile-and-fit"),f(Z,"class","relative group"),f(_t,"href","https://keras.io/api/models/model_training_apis/"),f(_t,"rel","nofollow"),f(ca,"id","pytorch_native"),f(ye,"id","finetune-in-native-pytorch"),f(ye,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),f(ye,"href","#finetune-in-native-pytorch"),f(Q,"class","relative group"),f(ua,"href","/docs/transformers/pr_15839/en/main_classes/trainer#transformers.Trainer"),f(ke,"id","dataloader"),f(ke,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),f(ke,"href","#dataloader"),f(V,"class","relative group"),f(Ee,"id","optimizer-and-learning-rate-scheduler"),f(Ee,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),f(Ee,"href","#optimizer-and-learning-rate-scheduler"),f(ee,"class","relative group"),f(St,"href","https://pytorch.org/docs/stable/generated/torch.optim.AdamW.html"),f(St,"rel","nofollow"),f(va,"href","/docs/transformers/pr_15839/en/main_classes/trainer#transformers.Trainer"),f(xe,"id","training-loop"),f(xe,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),f(xe,"href","#training-loop"),f(te,"class","relative group"),f(Ot,"href","https://tqdm.github.io/"),f(Ot,"rel","nofollow"),f(Ce,"id","metrics"),f(Ce,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),f(Ce,"href","#metrics"),f(ae,"class","relative group"),f(wa,"href","/docs/transformers/pr_15839/en/main_classes/trainer#transformers.Trainer"),f(Yt,"href","https://huggingface.co/docs/datasets/package_reference/main_classes.html?highlight=add_batch#datasets.Metric.add_batch"),f(Yt,"rel","nofollow"),f(ya,"id","additional-resources"),f(De,"id","additional-resources"),f(De,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),f(De,"href","#additional-resources"),f(se,"class","relative group"),f(Gt,"href","https://github.com/huggingface/transformers/tree/master/examples"),f(Gt,"rel","nofollow"),f(Ea,"href","notebooks")},m(e,s){a(document.head,$),l(e,k,s),l(e,w,s),a(w,b),a(b,T),d(j,T,null),a(w,F),a(w,P),a(P,E),l(e,A,s),d(z,e,s),l(e,Le,s),l(e,Ut,s),a(Ut,Ro),l(e,Fs,s),l(e,I,s),a(I,Me),a(Me,Go),a(Me,Jt),a(Jt,Ko),a(Me,Uo),a(I,Jo),a(I,Ca),a(Ca,Xo),a(I,Zo),a(I,Da),a(Da,Qo),l(e,Ss,s),l(e,Xt,s),l(e,Ns,s),l(e,W,s),a(W,re),a(re,Fa),d(Oe,Fa,null),a(W,Vo),a(W,Sa),a(Sa,en),l(e,Is,s),d(Be,e,s),l(e,Ls,s),l(e,Zt,s),a(Zt,tn),l(e,Ms,s),l(e,oe,s),a(oe,an),a(oe,He),a(He,sn),a(oe,rn),l(e,Os,s),d(Ye,e,s),l(e,Bs,s),l(e,ne,s),a(ne,on),a(ne,We),a(We,Na),a(Na,nn),a(ne,ln),l(e,Hs,s),d(Re,e,s),l(e,Ys,s),l(e,Qt,s),a(Qt,pn),l(e,Ws,s),d(Ge,e,s),l(e,Rs,s),l(e,Vt,s),l(e,Gs,s),l(e,R,s),a(R,le),a(le,Ia),d(Ke,Ia,null),a(R,fn),a(R,ea),a(ea,hn),a(ea,La),a(La,mn),l(e,Ks,s),d(Ue,e,s),l(e,Us,s),l(e,L,s),a(L,cn),a(L,ta),a(ta,un),a(L,dn),a(L,aa),a(aa,gn),a(L,_n),l(e,Js,s),l(e,ie,s),a(ie,vn),a(ie,Je),a(Je,$n),a(ie,wn),l(e,Xs,s),d(Xe,e,s),l(e,Zs,s),d(pe,e,s),l(e,Qs,s),l(e,G,s),a(G,fe),a(fe,Ma),d(Ze,Ma,null),a(G,yn),a(G,Oa),a(Oa,bn),l(e,Vs,s),l(e,M,s),a(M,kn),a(M,sa),a(sa,jn),a(M,En),a(M,Qe),a(Qe,Tn),a(M,An),l(e,er,s),l(e,ra,s),a(ra,Pn),l(e,tr,s),d(Ve,e,s),l(e,ar,s),l(e,K,s),a(K,he),a(he,Ba),d(et,Ba,null),a(K,qn),a(K,Ha),a(Ha,xn),l(e,sr,s),l(e,q,s),a(q,oa),a(oa,zn),a(q,Cn),a(q,na),a(na,Dn),a(q,Fn),a(q,tt),a(tt,Ya),a(Ya,Sn),a(q,Nn),a(q,Wa),a(Wa,In),a(q,Ln),a(q,at),a(at,Mn),a(q,On),l(e,rr,s),d(st,e,s),l(e,or,s),l(e,C,s),a(C,Bn),a(C,Ra),a(Ra,Hn),a(C,Yn),a(C,Ga),a(Ga,Wn),a(C,Rn),a(C,Ka),a(Ka,Gn),a(C,Kn),l(e,nr,s),d(rt,e,s),l(e,lr,s),l(e,me,s),a(me,Un),a(me,Ua),a(Ua,Jn),a(me,Xn),l(e,ir,s),d(ot,e,s),l(e,pr,s),l(e,U,s),a(U,ce),a(ce,Ja),d(nt,Ja,null),a(U,Zn),a(U,Xa),a(Xa,Qn),l(e,fr,s),l(e,ue,s),a(ue,Vn),a(ue,la),a(la,el),a(ue,tl),l(e,hr,s),d(lt,e,s),l(e,mr,s),l(e,de,s),a(de,al),a(de,ia),a(ia,sl),a(de,rl),l(e,cr,s),d(it,e,s),l(e,ur,s),l(e,pa,s),l(e,dr,s),l(e,J,s),a(J,ge),a(ge,Za),d(pt,Za,null),a(J,ol),a(J,Qa),a(Qa,nl),l(e,gr,s),d(ft,e,s),l(e,_r,s),l(e,fa,s),a(fa,ll),l(e,vr,s),l(e,X,s),a(X,_e),a(_e,Va),d(ht,Va,null),a(X,il),a(X,es),a(es,pl),l(e,$r,s),l(e,O,s),a(O,fl),a(O,ha),a(ha,hl),a(O,ml),a(O,ts),a(ts,cl),a(O,ul),l(e,wr,s),d(mt,e,s),l(e,yr,s),d(ve,e,s),l(e,br,s),l(e,D,s),a(D,dl),a(D,ct),a(ct,as),a(as,gl),a(D,_l),a(D,ss),a(ss,vl),a(D,$l),a(D,rs),a(rs,wl),a(D,yl),l(e,kr,s),d(ut,e,s),l(e,jr,s),l(e,Z,s),a(Z,$e),a($e,os),d(dt,os,null),a(Z,bl),a(Z,ns),a(ns,kl),l(e,Er,s),l(e,ma,s),a(ma,jl),l(e,Tr,s),d(gt,e,s),l(e,Ar,s),l(e,we,s),a(we,El),a(we,_t),a(_t,ls),a(ls,Tl),a(we,Al),l(e,Pr,s),d(vt,e,s),l(e,qr,s),l(e,ca,s),l(e,xr,s),l(e,Q,s),a(Q,ye),a(ye,is),d($t,is,null),a(Q,Pl),a(Q,ps),a(ps,ql),l(e,zr,s),d(wt,e,s),l(e,Cr,s),l(e,yt,s),a(yt,ua),a(ua,xl),a(yt,zl),l(e,Dr,s),l(e,da,s),a(da,Cl),l(e,Fr,s),d(bt,e,s),l(e,Sr,s),l(e,be,s),a(be,Dl),a(be,fs),a(fs,Fl),a(be,Sl),l(e,Nr,s),l(e,B,s),a(B,kt),a(kt,jt),a(jt,Nl),a(jt,hs),a(hs,Il),a(jt,Ll),a(kt,Ml),d(Et,kt,null),a(B,Ol),a(B,Tt),a(Tt,S),a(S,Bl),a(S,ms),a(ms,Hl),a(S,Yl),a(S,cs),a(cs,Wl),a(S,Rl),a(S,us),a(us,Gl),a(S,Kl),a(Tt,Ul),d(At,Tt,null),a(B,Jl),a(B,Pt),a(Pt,ds),a(ds,Xl),a(Pt,Zl),d(qt,Pt,null),l(e,Ir,s),l(e,ga,s),a(ga,Ql),l(e,Lr,s),d(xt,e,s),l(e,Mr,s),l(e,V,s),a(V,ke),a(ke,gs),d(zt,gs,null),a(V,Vl),a(V,_s),a(_s,ei),l(e,Or,s),l(e,je,s),a(je,ti),a(je,vs),a(vs,ai),a(je,si),l(e,Br,s),d(Ct,e,s),l(e,Hr,s),l(e,_a,s),a(_a,ri),l(e,Yr,s),d(Dt,e,s),l(e,Wr,s),l(e,ee,s),a(ee,Ee),a(Ee,$s),d(Ft,$s,null),a(ee,oi),a(ee,ws),a(ws,ni),l(e,Rr,s),l(e,Te,s),a(Te,li),a(Te,St),a(St,ys),a(ys,ii),a(Te,pi),l(e,Gr,s),d(Nt,e,s),l(e,Kr,s),l(e,Ae,s),a(Ae,fi),a(Ae,va),a(va,hi),a(Ae,mi),l(e,Ur,s),d(It,e,s),l(e,Jr,s),l(e,Pe,s),a(Pe,ci),a(Pe,bs),a(bs,ui),a(Pe,di),l(e,Xr,s),d(Lt,e,s),l(e,Zr,s),d(qe,e,s),l(e,Qr,s),l(e,$a,s),a($a,gi),l(e,Vr,s),l(e,te,s),a(te,xe),a(xe,ks),d(Mt,ks,null),a(te,_i),a(te,js),a(js,vi),l(e,eo,s),l(e,ze,s),a(ze,$i),a(ze,Ot),a(Ot,wi),a(ze,yi),l(e,to,s),d(Bt,e,s),l(e,ao,s),l(e,ae,s),a(ae,Ce),a(Ce,Es),d(Ht,Es,null),a(ae,bi),a(ae,Ts),a(Ts,ki),l(e,so,s),l(e,H,s),a(H,ji),a(H,wa),a(wa,Ei),a(H,Ti),a(H,Yt),a(Yt,As),a(As,Ai),a(H,Pi),l(e,ro,s),d(Wt,e,s),l(e,oo,s),l(e,ya,s),l(e,no,s),l(e,se,s),a(se,De),a(De,Ps),d(Rt,Ps,null),a(se,qi),a(se,qs),a(qs,xi),l(e,lo,s),l(e,ba,s),a(ba,zi),l(e,io,s),l(e,Fe,s),a(Fe,xs),a(xs,ka),a(ka,Gt),a(Gt,Ci),a(ka,Di),a(Fe,Fi),a(Fe,zs),a(zs,ja),a(ja,Ea),a(Ea,Si),a(ja,Ni),po=!0},p(e,[s]){const Kt={};s&2&&(Kt.$$scope={dirty:s,ctx:e}),pe.$set(Kt);const Cs={};s&2&&(Cs.$$scope={dirty:s,ctx:e}),ve.$set(Cs);const Ds={};s&2&&(Ds.$$scope={dirty:s,ctx:e}),qe.$set(Ds)},i(e){po||(g(j.$$.fragment,e),g(z.$$.fragment,e),g(Oe.$$.fragment,e),g(Be.$$.fragment,e),g(Ye.$$.fragment,e),g(Re.$$.fragment,e),g(Ge.$$.fragment,e),g(Ke.$$.fragment,e),g(Ue.$$.fragment,e),g(Xe.$$.fragment,e),g(pe.$$.fragment,e),g(Ze.$$.fragment,e),g(Ve.$$.fragment,e),g(et.$$.fragment,e),g(st.$$.fragment,e),g(rt.$$.fragment,e),g(ot.$$.fragment,e),g(nt.$$.fragment,e),g(lt.$$.fragment,e),g(it.$$.fragment,e),g(pt.$$.fragment,e),g(ft.$$.fragment,e),g(ht.$$.fragment,e),g(mt.$$.fragment,e),g(ve.$$.fragment,e),g(ut.$$.fragment,e),g(dt.$$.fragment,e),g(gt.$$.fragment,e),g(vt.$$.fragment,e),g($t.$$.fragment,e),g(wt.$$.fragment,e),g(bt.$$.fragment,e),g(Et.$$.fragment,e),g(At.$$.fragment,e),g(qt.$$.fragment,e),g(xt.$$.fragment,e),g(zt.$$.fragment,e),g(Ct.$$.fragment,e),g(Dt.$$.fragment,e),g(Ft.$$.fragment,e),g(Nt.$$.fragment,e),g(It.$$.fragment,e),g(Lt.$$.fragment,e),g(qe.$$.fragment,e),g(Mt.$$.fragment,e),g(Bt.$$.fragment,e),g(Ht.$$.fragment,e),g(Wt.$$.fragment,e),g(Rt.$$.fragment,e),po=!0)},o(e){_(j.$$.fragment,e),_(z.$$.fragment,e),_(Oe.$$.fragment,e),_(Be.$$.fragment,e),_(Ye.$$.fragment,e),_(Re.$$.fragment,e),_(Ge.$$.fragment,e),_(Ke.$$.fragment,e),_(Ue.$$.fragment,e),_(Xe.$$.fragment,e),_(pe.$$.fragment,e),_(Ze.$$.fragment,e),_(Ve.$$.fragment,e),_(et.$$.fragment,e),_(st.$$.fragment,e),_(rt.$$.fragment,e),_(ot.$$.fragment,e),_(nt.$$.fragment,e),_(lt.$$.fragment,e),_(it.$$.fragment,e),_(pt.$$.fragment,e),_(ft.$$.fragment,e),_(ht.$$.fragment,e),_(mt.$$.fragment,e),_(ve.$$.fragment,e),_(ut.$$.fragment,e),_(dt.$$.fragment,e),_(gt.$$.fragment,e),_(vt.$$.fragment,e),_($t.$$.fragment,e),_(wt.$$.fragment,e),_(bt.$$.fragment,e),_(Et.$$.fragment,e),_(At.$$.fragment,e),_(qt.$$.fragment,e),_(xt.$$.fragment,e),_(zt.$$.fragment,e),_(Ct.$$.fragment,e),_(Dt.$$.fragment,e),_(Ft.$$.fragment,e),_(Nt.$$.fragment,e),_(It.$$.fragment,e),_(Lt.$$.fragment,e),_(qe.$$.fragment,e),_(Mt.$$.fragment,e),_(Bt.$$.fragment,e),_(Ht.$$.fragment,e),_(Wt.$$.fragment,e),_(Rt.$$.fragment,e),po=!1},d(e){t($),e&&t(k),e&&t(w),v(j),e&&t(A),v(z,e),e&&t(Le),e&&t(Ut),e&&t(Fs),e&&t(I),e&&t(Ss),e&&t(Xt),e&&t(Ns),e&&t(W),v(Oe),e&&t(Is),v(Be,e),e&&t(Ls),e&&t(Zt),e&&t(Ms),e&&t(oe),e&&t(Os),v(Ye,e),e&&t(Bs),e&&t(ne),e&&t(Hs),v(Re,e),e&&t(Ys),e&&t(Qt),e&&t(Ws),v(Ge,e),e&&t(Rs),e&&t(Vt),e&&t(Gs),e&&t(R),v(Ke),e&&t(Ks),v(Ue,e),e&&t(Us),e&&t(L),e&&t(Js),e&&t(ie),e&&t(Xs),v(Xe,e),e&&t(Zs),v(pe,e),e&&t(Qs),e&&t(G),v(Ze),e&&t(Vs),e&&t(M),e&&t(er),e&&t(ra),e&&t(tr),v(Ve,e),e&&t(ar),e&&t(K),v(et),e&&t(sr),e&&t(q),e&&t(rr),v(st,e),e&&t(or),e&&t(C),e&&t(nr),v(rt,e),e&&t(lr),e&&t(me),e&&t(ir),v(ot,e),e&&t(pr),e&&t(U),v(nt),e&&t(fr),e&&t(ue),e&&t(hr),v(lt,e),e&&t(mr),e&&t(de),e&&t(cr),v(it,e),e&&t(ur),e&&t(pa),e&&t(dr),e&&t(J),v(pt),e&&t(gr),v(ft,e),e&&t(_r),e&&t(fa),e&&t(vr),e&&t(X),v(ht),e&&t($r),e&&t(O),e&&t(wr),v(mt,e),e&&t(yr),v(ve,e),e&&t(br),e&&t(D),e&&t(kr),v(ut,e),e&&t(jr),e&&t(Z),v(dt),e&&t(Er),e&&t(ma),e&&t(Tr),v(gt,e),e&&t(Ar),e&&t(we),e&&t(Pr),v(vt,e),e&&t(qr),e&&t(ca),e&&t(xr),e&&t(Q),v($t),e&&t(zr),v(wt,e),e&&t(Cr),e&&t(yt),e&&t(Dr),e&&t(da),e&&t(Fr),v(bt,e),e&&t(Sr),e&&t(be),e&&t(Nr),e&&t(B),v(Et),v(At),v(qt),e&&t(Ir),e&&t(ga),e&&t(Lr),v(xt,e),e&&t(Mr),e&&t(V),v(zt),e&&t(Or),e&&t(je),e&&t(Br),v(Ct,e),e&&t(Hr),e&&t(_a),e&&t(Yr),v(Dt,e),e&&t(Wr),e&&t(ee),v(Ft),e&&t(Rr),e&&t(Te),e&&t(Gr),v(Nt,e),e&&t(Kr),e&&t(Ae),e&&t(Ur),v(It,e),e&&t(Jr),e&&t(Pe),e&&t(Xr),v(Lt,e),e&&t(Zr),v(qe,e),e&&t(Qr),e&&t($a),e&&t(Vr),e&&t(te),v(Mt),e&&t(eo),e&&t(ze),e&&t(to),v(Bt,e),e&&t(ao),e&&t(ae),v(Ht),e&&t(so),e&&t(H),e&&t(ro),v(Wt,e),e&&t(oo),e&&t(ya),e&&t(no),e&&t(se),v(Rt),e&&t(lo),e&&t(ba),e&&t(io),e&&t(Fe)}}}const Jf={local:"finetune-a-pretrained-model",sections:[{local:"prepare-a-dataset",title:"Prepare a dataset"},{local:"finetune-with-trainer",sections:[{local:"training-hyperparameters",title:"Training hyperparameters"},{local:"metrics",title:"Metrics"},{local:"trainer",title:"Trainer"}],title:"Fine-tune with `Trainer`"},{local:"finetune-with-keras",sections:[{local:"convert-dataset-to-tensorflow-format",title:"Convert dataset to TensorFlow format"},{local:"compile-and-fit",title:"Compile and fit"}],title:"Fine-tune with Keras"},{local:"finetune-in-native-pytorch",sections:[{local:"dataloader",title:"DataLoader"},{local:"optimizer-and-learning-rate-scheduler",title:"Optimizer and learning rate scheduler"},{local:"training-loop",title:"Training loop"},{local:"metrics",title:"Metrics"}],title:"Fine-tune in native PyTorch"},{local:"additional-resources",title:"Additional resources"}],title:"Fine-tune a pretrained model"};function Xf(Y,$,k){let{fw:w}=$;return Y.$$set=b=>{"fw"in b&&k(0,w=b.fw)},[w]}class rh extends Of{constructor($){super();Bf(this,$,Xf,Uf,Hf,{fw:0})}}export{rh as default,Jf as metadata};
