import{S as Ro,i as Xo,s as Go,e as r,k as f,w as $,t as o,M as Qo,c as l,d as t,m as h,a as i,x as k,h as n,b as u,F as s,g as p,y as w,q as v,o as b,B as q}from"../../chunks/vendor-4833417e.js";import{T as Ts}from"../../chunks/Tip-fffd6df1.js";import{Y as Ko}from"../../chunks/Youtube-27813aed.js";import{I as yt}from"../../chunks/IconCopyLink-4b81c553.js";import{C as A}from"../../chunks/CodeBlock-6a3d1b46.js";import{C as Vo}from"../../chunks/CodeBlockFw-27a176a0.js";import"../../chunks/CopyButton-dacfbfaf.js";function en(C){let m,j,c,_,E;return{c(){m=r("p"),j=o("See the translation "),c=r("a"),_=o("task page"),E=o(" for more information about its associated models, datasets, and metrics."),this.h()},l(d){m=l(d,"P",{});var g=i(m);j=n(g,"See the translation "),c=l(g,"A",{href:!0,rel:!0});var S=i(c);_=n(S,"task page"),S.forEach(t),E=n(g," for more information about its associated models, datasets, and metrics."),g.forEach(t),this.h()},h(){u(c,"href","https://huggingface.co/tasks/translation"),u(c,"rel","nofollow")},m(d,g){p(d,m,g),s(m,j),s(m,c),s(c,_),s(m,E)},d(d){d&&t(m)}}}function tn(C){let m,j,c,_,E,d,g,S;return{c(){m=r("p"),j=o("If you aren\u2019t familiar with fine-tuning a model with the "),c=r("a"),_=o("Trainer"),E=o(", take a look at the basic tutorial "),d=r("a"),g=o("here"),S=o("!"),this.h()},l(T){m=l(T,"P",{});var y=i(m);j=n(y,"If you aren\u2019t familiar with fine-tuning a model with the "),c=l(y,"A",{href:!0});var z=i(c);_=n(z,"Trainer"),z.forEach(t),E=n(y,", take a look at the basic tutorial "),d=l(y,"A",{href:!0});var D=i(d);g=n(D,"here"),D.forEach(t),S=n(y,"!"),y.forEach(t),this.h()},h(){u(c,"href","/docs/transformers/pr_15839/en/main_classes/trainer#transformers.Trainer"),u(d,"href","training#finetune-with-trainer")},m(T,y){p(T,m,y),s(m,j),s(m,c),s(c,_),s(m,E),s(m,d),s(d,g),s(m,S)},d(T){T&&t(m)}}}function sn(C){let m,j,c,_,E;return{c(){m=r("p"),j=o("If you aren\u2019t familiar with fine-tuning a model with Keras, take a look at the basic tutorial "),c=r("a"),_=o("here"),E=o("!"),this.h()},l(d){m=l(d,"P",{});var g=i(m);j=n(g,"If you aren\u2019t familiar with fine-tuning a model with Keras, take a look at the basic tutorial "),c=l(g,"A",{href:!0});var S=i(c);_=n(S,"here"),S.forEach(t),E=n(g,"!"),g.forEach(t),this.h()},h(){u(c,"href","training#finetune-with-keras")},m(d,g){p(d,m,g),s(m,j),s(m,c),s(c,_),s(m,E)},d(d){d&&t(m)}}}function an(C){let m,j,c,_,E,d,g,S;return{c(){m=r("p"),j=o(`For a more in-depth example of how to fine-tune a model for translation, take a look at the corresponding
`),c=r("a"),_=o("PyTorch notebook"),E=o(`
or `),d=r("a"),g=o("TensorFlow notebook"),S=o("."),this.h()},l(T){m=l(T,"P",{});var y=i(m);j=n(y,`For a more in-depth example of how to fine-tune a model for translation, take a look at the corresponding
`),c=l(y,"A",{href:!0,rel:!0});var z=i(c);_=n(z,"PyTorch notebook"),z.forEach(t),E=n(y,`
or `),d=l(y,"A",{href:!0,rel:!0});var D=i(d);g=n(D,"TensorFlow notebook"),D.forEach(t),S=n(y,"."),y.forEach(t),this.h()},h(){u(c,"href","https://colab.research.google.com/github/huggingface/notebooks/blob/master/examples/translation.ipynb"),u(c,"rel","nofollow"),u(d,"href","https://colab.research.google.com/github/huggingface/notebooks/blob/master/examples/translation-tf.ipynb"),u(d,"rel","nofollow")},m(T,y){p(T,m,y),s(m,j),s(m,c),s(c,_),s(m,E),s(m,d),s(d,g),s(m,S)},d(T){T&&t(m)}}}function on(C){let m,j,c,_,E,d,g,S,T,y,z,D,De,xs,Et,L,zs,ne,As,Fs,re,Ps,Cs,St,Y,Tt,N,H,Ve,le,Ds,et,Ls,xt,Le,Ms,zt,ie,At,Me,Os,Ft,pe,Pt,Oe,Is,Ct,fe,Dt,Z,Ns,tt,Us,Bs,Lt,U,J,st,he,Ws,at,Ys,Mt,me,Ot,Ie,Hs,It,ue,Nt,Ne,Zs,Ut,M,ot,Js,Ks,nt,Rs,Xs,ce,Gs,rt,Qs,Vs,Bt,de,Wt,F,ea,_e,lt,ta,sa,it,aa,oa,pt,na,ra,Yt,ge,Ht,x,la,Ue,ia,pa,ft,fa,ha,ht,ma,ua,mt,ca,da,Zt,$e,Jt,B,K,ut,ke,_a,ct,ga,Kt,R,$a,Be,ka,wa,Rt,we,Xt,X,Gt,We,va,Qt,O,ve,ba,Ye,qa,ja,ya,be,Ea,He,Sa,Ta,xa,qe,za,Ze,Aa,Fa,Vt,je,es,W,G,dt,ye,Pa,_t,Ca,ts,Je,Da,ss,Q,as,P,La,gt,Ma,Oa,Ee,$t,Ia,Na,kt,Ua,Ba,os,Se,ns,Ke,Wa,rs,Te,ls,V,Ya,Re,Ha,Za,is,xe,ps,ee,Ja,ze,wt,Ka,Ra,fs,Ae,hs,te,Xa,Fe,vt,Ga,Qa,ms,Pe,us,se,cs;return d=new yt({}),z=new Ko({props:{id:"1JvfrvZgi6c"}}),Y=new Ts({props:{$$slots:{default:[en]},$$scope:{ctx:C}}}),le=new yt({}),ie=new A({props:{code:`from datasets import load_dataset

books = load_dataset("opus_books", "en-fr")`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> datasets <span class="hljs-keyword">import</span> load_dataset

<span class="hljs-meta">&gt;&gt;&gt; </span>books = load_dataset(<span class="hljs-string">&quot;opus_books&quot;</span>, <span class="hljs-string">&quot;en-fr&quot;</span>)`}}),pe=new A({props:{code:'books = books["train"].train_test_split(test_size=0.2)',highlighted:'books = books[<span class="hljs-string">&quot;train&quot;</span>].train_test_split(test_size=<span class="hljs-number">0.2</span>)'}}),fe=new A({props:{code:'books["train"][0]',highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>books[<span class="hljs-string">&quot;train&quot;</span>][<span class="hljs-number">0</span>]
{<span class="hljs-string">&#x27;id&#x27;</span>: <span class="hljs-string">&#x27;90560&#x27;</span>,
 <span class="hljs-string">&#x27;translation&#x27;</span>: {<span class="hljs-string">&#x27;en&#x27;</span>: <span class="hljs-string">&#x27;But this lofty plateau measured only a few fathoms, and soon we reentered Our Element.&#x27;</span>,
  <span class="hljs-string">&#x27;fr&#x27;</span>: <span class="hljs-string">&#x27;Mais ce plateau \xE9lev\xE9 ne mesurait que quelques toises, et bient\xF4t nous f\xFBmes rentr\xE9s dans notre \xE9l\xE9ment.&#x27;</span>}}`}}),he=new yt({}),me=new Ko({props:{id:"XAR8jnZZuUs"}}),ue=new A({props:{code:`from transformers import AutoTokenizer

tokenizer = AutoTokenizer.from_pretrained("t5-small")`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoTokenizer

<span class="hljs-meta">&gt;&gt;&gt; </span>tokenizer = AutoTokenizer.from_pretrained(<span class="hljs-string">&quot;t5-small&quot;</span>)`}}),de=new A({props:{code:`source_lang = "en"
target_lang = "fr"
prefix = "translate English to French: "


def preprocess_function(examples):
    inputs = [prefix + example[source_lang] for example in examples["translation"]]
    targets = [example[target_lang] for example in examples["translation"]]
    model_inputs = tokenizer(inputs, max_length=128, truncation=True)

    with tokenizer.as_target_tokenizer():
        labels = tokenizer(targets, max_length=128, truncation=True)

    model_inputs["labels"] = labels["input_ids"]
    return model_inputs`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>source_lang = <span class="hljs-string">&quot;en&quot;</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>target_lang = <span class="hljs-string">&quot;fr&quot;</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>prefix = <span class="hljs-string">&quot;translate English to French: &quot;</span>


<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">def</span> <span class="hljs-title function_">preprocess_function</span>(<span class="hljs-params">examples</span>):
<span class="hljs-meta">... </span>    inputs = [prefix + example[source_lang] <span class="hljs-keyword">for</span> example <span class="hljs-keyword">in</span> examples[<span class="hljs-string">&quot;translation&quot;</span>]]
<span class="hljs-meta">... </span>    targets = [example[target_lang] <span class="hljs-keyword">for</span> example <span class="hljs-keyword">in</span> examples[<span class="hljs-string">&quot;translation&quot;</span>]]
<span class="hljs-meta">... </span>    model_inputs = tokenizer(inputs, max_length=<span class="hljs-number">128</span>, truncation=<span class="hljs-literal">True</span>)

<span class="hljs-meta">... </span>    <span class="hljs-keyword">with</span> tokenizer.as_target_tokenizer():
<span class="hljs-meta">... </span>        labels = tokenizer(targets, max_length=<span class="hljs-number">128</span>, truncation=<span class="hljs-literal">True</span>)

<span class="hljs-meta">... </span>    model_inputs[<span class="hljs-string">&quot;labels&quot;</span>] = labels[<span class="hljs-string">&quot;input_ids&quot;</span>]
<span class="hljs-meta">... </span>    <span class="hljs-keyword">return</span> model_inputs`}}),ge=new A({props:{code:"tokenized_books = books.map(preprocess_function, batched=True)",highlighted:'<span class="hljs-meta">&gt;&gt;&gt; </span>tokenized_books = books.<span class="hljs-built_in">map</span>(preprocess_function, batched=<span class="hljs-literal">True</span>)'}}),$e=new Vo({props:{group1:{id:"pt",code:`from transformers import DataCollatorForSeq2Seq

data_collator = DataCollatorForSeq2Seq(tokenizer=tokenizer, model=model)`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> DataCollatorForSeq2Seq

<span class="hljs-meta">&gt;&gt;&gt; </span>data_collator = DataCollatorForSeq2Seq(tokenizer=tokenizer, model=model)`},group2:{id:"tf",code:`from transformers import DataCollatorForSeq2Seq

data_collator = DataCollatorForSeq2Seq(tokenizer=tokenizer, model=model, return_tensors="tf")`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> DataCollatorForSeq2Seq

<span class="hljs-meta">&gt;&gt;&gt; </span>data_collator = DataCollatorForSeq2Seq(tokenizer=tokenizer, model=model, return_tensors=<span class="hljs-string">&quot;tf&quot;</span>)`}}}),ke=new yt({}),we=new A({props:{code:`from transformers import AutoModelForSeq2SeqLM, Seq2SeqTrainingArguments, Seq2SeqTrainer

model = AutoModelForSeq2SeqLM.from_pretrained("t5-small")`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoModelForSeq2SeqLM, Seq2SeqTrainingArguments, Seq2SeqTrainer

<span class="hljs-meta">&gt;&gt;&gt; </span>model = AutoModelForSeq2SeqLM.from_pretrained(<span class="hljs-string">&quot;t5-small&quot;</span>)`}}),X=new Ts({props:{$$slots:{default:[tn]},$$scope:{ctx:C}}}),je=new A({props:{code:`training_args = Seq2SeqTrainingArguments(
    output_dir="./results",
    evaluation_strategy="epoch",
    learning_rate=2e-5,
    per_device_train_batch_size=16,
    per_device_eval_batch_size=16,
    weight_decay=0.01,
    save_total_limit=3,
    num_train_epochs=1,
    fp16=True,
)

trainer = Seq2SeqTrainer(
    model=model,
    args=training_args,
    train_dataset=tokenized_books["train"],
    eval_dataset=tokenized_books["test"],
    tokenizer=tokenizer,
    data_collator=data_collator,
)

trainer.train()`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>training_args = Seq2SeqTrainingArguments(
<span class="hljs-meta">... </span>    output_dir=<span class="hljs-string">&quot;./results&quot;</span>,
<span class="hljs-meta">... </span>    evaluation_strategy=<span class="hljs-string">&quot;epoch&quot;</span>,
<span class="hljs-meta">... </span>    learning_rate=<span class="hljs-number">2e-5</span>,
<span class="hljs-meta">... </span>    per_device_train_batch_size=<span class="hljs-number">16</span>,
<span class="hljs-meta">... </span>    per_device_eval_batch_size=<span class="hljs-number">16</span>,
<span class="hljs-meta">... </span>    weight_decay=<span class="hljs-number">0.01</span>,
<span class="hljs-meta">... </span>    save_total_limit=<span class="hljs-number">3</span>,
<span class="hljs-meta">... </span>    num_train_epochs=<span class="hljs-number">1</span>,
<span class="hljs-meta">... </span>    fp16=<span class="hljs-literal">True</span>,
<span class="hljs-meta">... </span>)

<span class="hljs-meta">&gt;&gt;&gt; </span>trainer = Seq2SeqTrainer(
<span class="hljs-meta">... </span>    model=model,
<span class="hljs-meta">... </span>    args=training_args,
<span class="hljs-meta">... </span>    train_dataset=tokenized_books[<span class="hljs-string">&quot;train&quot;</span>],
<span class="hljs-meta">... </span>    eval_dataset=tokenized_books[<span class="hljs-string">&quot;test&quot;</span>],
<span class="hljs-meta">... </span>    tokenizer=tokenizer,
<span class="hljs-meta">... </span>    data_collator=data_collator,
<span class="hljs-meta">... </span>)

<span class="hljs-meta">&gt;&gt;&gt; </span>trainer.train()`}}),ye=new yt({}),Q=new Ts({props:{$$slots:{default:[sn]},$$scope:{ctx:C}}}),Se=new A({props:{code:`tf_train_set = tokenized_books["train"].to_tf_dataset(
    columns=["attention_mask", "input_ids", "labels"],
    shuffle=True,
    batch_size=16,
    collate_fn=data_collator,
)

tf_test_set = tokenized_books["test"].to_tf_dataset(
    columns=["attention_mask", "input_ids", "labels"],
    shuffle=False,
    batch_size=16,
    collate_fn=data_collator,
)`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>tf_train_set = tokenized_books[<span class="hljs-string">&quot;train&quot;</span>].to_tf_dataset(
<span class="hljs-meta">... </span>    columns=[<span class="hljs-string">&quot;attention_mask&quot;</span>, <span class="hljs-string">&quot;input_ids&quot;</span>, <span class="hljs-string">&quot;labels&quot;</span>],
<span class="hljs-meta">... </span>    shuffle=<span class="hljs-literal">True</span>,
<span class="hljs-meta">... </span>    batch_size=<span class="hljs-number">16</span>,
<span class="hljs-meta">... </span>    collate_fn=data_collator,
<span class="hljs-meta">... </span>)

<span class="hljs-meta">&gt;&gt;&gt; </span>tf_test_set = tokenized_books[<span class="hljs-string">&quot;test&quot;</span>].to_tf_dataset(
<span class="hljs-meta">... </span>    columns=[<span class="hljs-string">&quot;attention_mask&quot;</span>, <span class="hljs-string">&quot;input_ids&quot;</span>, <span class="hljs-string">&quot;labels&quot;</span>],
<span class="hljs-meta">... </span>    shuffle=<span class="hljs-literal">False</span>,
<span class="hljs-meta">... </span>    batch_size=<span class="hljs-number">16</span>,
<span class="hljs-meta">... </span>    collate_fn=data_collator,
<span class="hljs-meta">... </span>)`}}),Te=new A({props:{code:`from transformers import create_optimizer, AdamWeightDecay

optimizer = AdamWeightDecay(learning_rate=2e-5, weight_decay_rate=0.01)`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> create_optimizer, AdamWeightDecay

<span class="hljs-meta">&gt;&gt;&gt; </span>optimizer = AdamWeightDecay(learning_rate=<span class="hljs-number">2e-5</span>, weight_decay_rate=<span class="hljs-number">0.01</span>)`}}),xe=new A({props:{code:`from transformers import TFAutoModelForSeq2SeqLM

model = TFAutoModelForSeq2SeqLM.from_pretrained("t5-small")`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> TFAutoModelForSeq2SeqLM

<span class="hljs-meta">&gt;&gt;&gt; </span>model = TFAutoModelForSeq2SeqLM.from_pretrained(<span class="hljs-string">&quot;t5-small&quot;</span>)`}}),Ae=new A({props:{code:"model.compile(optimizer=optimizer)",highlighted:'<span class="hljs-meta">&gt;&gt;&gt; </span>model.<span class="hljs-built_in">compile</span>(optimizer=optimizer)'}}),Pe=new A({props:{code:"model.fit(x=tf_train_set, validation_data=tf_test_set, epochs=3)",highlighted:'<span class="hljs-meta">&gt;&gt;&gt; </span>model.fit(x=tf_train_set, validation_data=tf_test_set, epochs=<span class="hljs-number">3</span>)'}}),se=new Ts({props:{$$slots:{default:[an]},$$scope:{ctx:C}}}),{c(){m=r("meta"),j=f(),c=r("h1"),_=r("a"),E=r("span"),$(d.$$.fragment),g=f(),S=r("span"),T=o("Translation"),y=f(),$(z.$$.fragment),D=f(),De=r("p"),xs=o("Translation converts a sequence of text from one language to another. It is one of several tasks you can formulate as a sequence-to-sequence problem, a powerful framework that extends to vision and audio tasks."),Et=f(),L=r("p"),zs=o("This guide will show you how to fine-tune "),ne=r("a"),As=o("T5"),Fs=o(" on the English-French subset of the "),re=r("a"),Ps=o("OPUS Books"),Cs=o(" dataset to translate English text to French."),St=f(),$(Y.$$.fragment),Tt=f(),N=r("h2"),H=r("a"),Ve=r("span"),$(le.$$.fragment),Ds=f(),et=r("span"),Ls=o("Load OPUS Books dataset"),xt=f(),Le=r("p"),Ms=o("Load the OPUS Books dataset from the \u{1F917} Datasets library:"),zt=f(),$(ie.$$.fragment),At=f(),Me=r("p"),Os=o("Split this dataset into a train and test set:"),Ft=f(),$(pe.$$.fragment),Pt=f(),Oe=r("p"),Is=o("Then take a look at an example:"),Ct=f(),$(fe.$$.fragment),Dt=f(),Z=r("p"),Ns=o("The "),tt=r("code"),Us=o("translation"),Bs=o(" field is a dictionary containing the English and French translations of the text."),Lt=f(),U=r("h2"),J=r("a"),st=r("span"),$(he.$$.fragment),Ws=f(),at=r("span"),Ys=o("Preprocess"),Mt=f(),$(me.$$.fragment),Ot=f(),Ie=r("p"),Hs=o("Load the T5 tokenizer to process the language pairs:"),It=f(),$(ue.$$.fragment),Nt=f(),Ne=r("p"),Zs=o("The preprocessing function needs to:"),Ut=f(),M=r("ol"),ot=r("li"),Js=o("Prefix the input with a prompt so T5 knows this is a translation task. Some models capable of multiple NLP tasks require prompting for specific tasks."),Ks=f(),nt=r("li"),Rs=o("Tokenize the input (English) and target (French) separately. You can\u2019t tokenize French text with a tokenizer pretrained on an English vocabulary. A context manager will help set the tokenizer to French first before tokenizing it."),Xs=f(),ce=r("li"),Gs=o("Truncate sequences to be no longer than the maximum length set by the "),rt=r("code"),Qs=o("max_length"),Vs=o(" parameter."),Bt=f(),$(de.$$.fragment),Wt=f(),F=r("p"),ea=o("Use \u{1F917} Datasets "),_e=r("a"),lt=r("code"),ta=o("map"),sa=o(" function to apply the preprocessing function over the entire dataset. You can speed up the "),it=r("code"),aa=o("map"),oa=o(" function by setting "),pt=r("code"),na=o("batched=True"),ra=o(" to process multiple elements of the dataset at once:"),Yt=f(),$(ge.$$.fragment),Ht=f(),x=r("p"),la=o("Use "),Ue=r("a"),ia=o("DataCollatorForSeq2Seq"),pa=o(" to create a batch of examples. It will also "),ft=r("em"),fa=o("dynamically pad"),ha=o(" your text and labels to the length of the longest element in its batch, so they are a uniform length. While it is possible to pad your text in the "),ht=r("code"),ma=o("tokenizer"),ua=o(" function by setting "),mt=r("code"),ca=o("padding=True"),da=o(", dynamic padding is more efficient."),Zt=f(),$($e.$$.fragment),Jt=f(),B=r("h2"),K=r("a"),ut=r("span"),$(ke.$$.fragment),_a=f(),ct=r("span"),ga=o("Fine-tune with Trainer"),Kt=f(),R=r("p"),$a=o("Load T5 with "),Be=r("a"),ka=o("AutoModelForSeq2SeqLM"),wa=o(":"),Rt=f(),$(we.$$.fragment),Xt=f(),$(X.$$.fragment),Gt=f(),We=r("p"),va=o("At this point, only three steps remain:"),Qt=f(),O=r("ol"),ve=r("li"),ba=o("Define your training hyperparameters in "),Ye=r("a"),qa=o("Seq2SeqTrainingArguments"),ja=o("."),ya=f(),be=r("li"),Ea=o("Pass the training arguments to "),He=r("a"),Sa=o("Seq2SeqTrainer"),Ta=o(" along with the model, dataset, tokenizer, and data collator."),xa=f(),qe=r("li"),za=o("Call "),Ze=r("a"),Aa=o("train()"),Fa=o(" to fine-tune your model."),Vt=f(),$(je.$$.fragment),es=f(),W=r("h2"),G=r("a"),dt=r("span"),$(ye.$$.fragment),Pa=f(),_t=r("span"),Ca=o("Fine-tune with TensorFlow"),ts=f(),Je=r("p"),Da=o("To fine-tune a model in TensorFlow is just as easy, with only a few differences."),ss=f(),$(Q.$$.fragment),as=f(),P=r("p"),La=o("Convert your datasets to the "),gt=r("code"),Ma=o("tf.data.Dataset"),Oa=o(" format with "),Ee=r("a"),$t=r("code"),Ia=o("to_tf_dataset"),Na=o(". Specify inputs and labels in "),kt=r("code"),Ua=o("columns"),Ba=o(", whether to shuffle the dataset order, batch size, and the data collator:"),os=f(),$(Se.$$.fragment),ns=f(),Ke=r("p"),Wa=o("Set up an optimizer function, learning rate schedule, and some training hyperparameters:"),rs=f(),$(Te.$$.fragment),ls=f(),V=r("p"),Ya=o("Load T5 with "),Re=r("a"),Ha=o("TFAutoModelForSeq2SeqLM"),Za=o(":"),is=f(),$(xe.$$.fragment),ps=f(),ee=r("p"),Ja=o("Configure the model for training with "),ze=r("a"),wt=r("code"),Ka=o("compile"),Ra=o(":"),fs=f(),$(Ae.$$.fragment),hs=f(),te=r("p"),Xa=o("Call "),Fe=r("a"),vt=r("code"),Ga=o("fit"),Qa=o(" to fine-tune the model:"),ms=f(),$(Pe.$$.fragment),us=f(),$(se.$$.fragment),this.h()},l(e){const a=Qo('[data-svelte="svelte-1phssyn"]',document.head);m=l(a,"META",{name:!0,content:!0}),a.forEach(t),j=h(e),c=l(e,"H1",{class:!0});var Ce=i(c);_=l(Ce,"A",{id:!0,class:!0,href:!0});var bt=i(_);E=l(bt,"SPAN",{});var qt=i(E);k(d.$$.fragment,qt),qt.forEach(t),bt.forEach(t),g=h(Ce),S=l(Ce,"SPAN",{});var jt=i(S);T=n(jt,"Translation"),jt.forEach(t),Ce.forEach(t),y=h(e),k(z.$$.fragment,e),D=h(e),De=l(e,"P",{});var Va=i(De);xs=n(Va,"Translation converts a sequence of text from one language to another. It is one of several tasks you can formulate as a sequence-to-sequence problem, a powerful framework that extends to vision and audio tasks."),Va.forEach(t),Et=h(e),L=l(e,"P",{});var Xe=i(L);zs=n(Xe,"This guide will show you how to fine-tune "),ne=l(Xe,"A",{href:!0,rel:!0});var eo=i(ne);As=n(eo,"T5"),eo.forEach(t),Fs=n(Xe," on the English-French subset of the "),re=l(Xe,"A",{href:!0,rel:!0});var to=i(re);Ps=n(to,"OPUS Books"),to.forEach(t),Cs=n(Xe," dataset to translate English text to French."),Xe.forEach(t),St=h(e),k(Y.$$.fragment,e),Tt=h(e),N=l(e,"H2",{class:!0});var ds=i(N);H=l(ds,"A",{id:!0,class:!0,href:!0});var so=i(H);Ve=l(so,"SPAN",{});var ao=i(Ve);k(le.$$.fragment,ao),ao.forEach(t),so.forEach(t),Ds=h(ds),et=l(ds,"SPAN",{});var oo=i(et);Ls=n(oo,"Load OPUS Books dataset"),oo.forEach(t),ds.forEach(t),xt=h(e),Le=l(e,"P",{});var no=i(Le);Ms=n(no,"Load the OPUS Books dataset from the \u{1F917} Datasets library:"),no.forEach(t),zt=h(e),k(ie.$$.fragment,e),At=h(e),Me=l(e,"P",{});var ro=i(Me);Os=n(ro,"Split this dataset into a train and test set:"),ro.forEach(t),Ft=h(e),k(pe.$$.fragment,e),Pt=h(e),Oe=l(e,"P",{});var lo=i(Oe);Is=n(lo,"Then take a look at an example:"),lo.forEach(t),Ct=h(e),k(fe.$$.fragment,e),Dt=h(e),Z=l(e,"P",{});var _s=i(Z);Ns=n(_s,"The "),tt=l(_s,"CODE",{});var io=i(tt);Us=n(io,"translation"),io.forEach(t),Bs=n(_s," field is a dictionary containing the English and French translations of the text."),_s.forEach(t),Lt=h(e),U=l(e,"H2",{class:!0});var gs=i(U);J=l(gs,"A",{id:!0,class:!0,href:!0});var po=i(J);st=l(po,"SPAN",{});var fo=i(st);k(he.$$.fragment,fo),fo.forEach(t),po.forEach(t),Ws=h(gs),at=l(gs,"SPAN",{});var ho=i(at);Ys=n(ho,"Preprocess"),ho.forEach(t),gs.forEach(t),Mt=h(e),k(me.$$.fragment,e),Ot=h(e),Ie=l(e,"P",{});var mo=i(Ie);Hs=n(mo,"Load the T5 tokenizer to process the language pairs:"),mo.forEach(t),It=h(e),k(ue.$$.fragment,e),Nt=h(e),Ne=l(e,"P",{});var uo=i(Ne);Zs=n(uo,"The preprocessing function needs to:"),uo.forEach(t),Ut=h(e),M=l(e,"OL",{});var Ge=i(M);ot=l(Ge,"LI",{});var co=i(ot);Js=n(co,"Prefix the input with a prompt so T5 knows this is a translation task. Some models capable of multiple NLP tasks require prompting for specific tasks."),co.forEach(t),Ks=h(Ge),nt=l(Ge,"LI",{});var _o=i(nt);Rs=n(_o,"Tokenize the input (English) and target (French) separately. You can\u2019t tokenize French text with a tokenizer pretrained on an English vocabulary. A context manager will help set the tokenizer to French first before tokenizing it."),_o.forEach(t),Xs=h(Ge),ce=l(Ge,"LI",{});var $s=i(ce);Gs=n($s,"Truncate sequences to be no longer than the maximum length set by the "),rt=l($s,"CODE",{});var go=i(rt);Qs=n(go,"max_length"),go.forEach(t),Vs=n($s," parameter."),$s.forEach(t),Ge.forEach(t),Bt=h(e),k(de.$$.fragment,e),Wt=h(e),F=l(e,"P",{});var ae=i(F);ea=n(ae,"Use \u{1F917} Datasets "),_e=l(ae,"A",{href:!0,rel:!0});var $o=i(_e);lt=l($o,"CODE",{});var ko=i(lt);ta=n(ko,"map"),ko.forEach(t),$o.forEach(t),sa=n(ae," function to apply the preprocessing function over the entire dataset. You can speed up the "),it=l(ae,"CODE",{});var wo=i(it);aa=n(wo,"map"),wo.forEach(t),oa=n(ae," function by setting "),pt=l(ae,"CODE",{});var vo=i(pt);na=n(vo,"batched=True"),vo.forEach(t),ra=n(ae," to process multiple elements of the dataset at once:"),ae.forEach(t),Yt=h(e),k(ge.$$.fragment,e),Ht=h(e),x=l(e,"P",{});var I=i(x);la=n(I,"Use "),Ue=l(I,"A",{href:!0});var bo=i(Ue);ia=n(bo,"DataCollatorForSeq2Seq"),bo.forEach(t),pa=n(I," to create a batch of examples. It will also "),ft=l(I,"EM",{});var qo=i(ft);fa=n(qo,"dynamically pad"),qo.forEach(t),ha=n(I," your text and labels to the length of the longest element in its batch, so they are a uniform length. While it is possible to pad your text in the "),ht=l(I,"CODE",{});var jo=i(ht);ma=n(jo,"tokenizer"),jo.forEach(t),ua=n(I," function by setting "),mt=l(I,"CODE",{});var yo=i(mt);ca=n(yo,"padding=True"),yo.forEach(t),da=n(I,", dynamic padding is more efficient."),I.forEach(t),Zt=h(e),k($e.$$.fragment,e),Jt=h(e),B=l(e,"H2",{class:!0});var ks=i(B);K=l(ks,"A",{id:!0,class:!0,href:!0});var Eo=i(K);ut=l(Eo,"SPAN",{});var So=i(ut);k(ke.$$.fragment,So),So.forEach(t),Eo.forEach(t),_a=h(ks),ct=l(ks,"SPAN",{});var To=i(ct);ga=n(To,"Fine-tune with Trainer"),To.forEach(t),ks.forEach(t),Kt=h(e),R=l(e,"P",{});var ws=i(R);$a=n(ws,"Load T5 with "),Be=l(ws,"A",{href:!0});var xo=i(Be);ka=n(xo,"AutoModelForSeq2SeqLM"),xo.forEach(t),wa=n(ws,":"),ws.forEach(t),Rt=h(e),k(we.$$.fragment,e),Xt=h(e),k(X.$$.fragment,e),Gt=h(e),We=l(e,"P",{});var zo=i(We);va=n(zo,"At this point, only three steps remain:"),zo.forEach(t),Qt=h(e),O=l(e,"OL",{});var Qe=i(O);ve=l(Qe,"LI",{});var vs=i(ve);ba=n(vs,"Define your training hyperparameters in "),Ye=l(vs,"A",{href:!0});var Ao=i(Ye);qa=n(Ao,"Seq2SeqTrainingArguments"),Ao.forEach(t),ja=n(vs,"."),vs.forEach(t),ya=h(Qe),be=l(Qe,"LI",{});var bs=i(be);Ea=n(bs,"Pass the training arguments to "),He=l(bs,"A",{href:!0});var Fo=i(He);Sa=n(Fo,"Seq2SeqTrainer"),Fo.forEach(t),Ta=n(bs," along with the model, dataset, tokenizer, and data collator."),bs.forEach(t),xa=h(Qe),qe=l(Qe,"LI",{});var qs=i(qe);za=n(qs,"Call "),Ze=l(qs,"A",{href:!0});var Po=i(Ze);Aa=n(Po,"train()"),Po.forEach(t),Fa=n(qs," to fine-tune your model."),qs.forEach(t),Qe.forEach(t),Vt=h(e),k(je.$$.fragment,e),es=h(e),W=l(e,"H2",{class:!0});var js=i(W);G=l(js,"A",{id:!0,class:!0,href:!0});var Co=i(G);dt=l(Co,"SPAN",{});var Do=i(dt);k(ye.$$.fragment,Do),Do.forEach(t),Co.forEach(t),Pa=h(js),_t=l(js,"SPAN",{});var Lo=i(_t);Ca=n(Lo,"Fine-tune with TensorFlow"),Lo.forEach(t),js.forEach(t),ts=h(e),Je=l(e,"P",{});var Mo=i(Je);Da=n(Mo,"To fine-tune a model in TensorFlow is just as easy, with only a few differences."),Mo.forEach(t),ss=h(e),k(Q.$$.fragment,e),as=h(e),P=l(e,"P",{});var oe=i(P);La=n(oe,"Convert your datasets to the "),gt=l(oe,"CODE",{});var Oo=i(gt);Ma=n(Oo,"tf.data.Dataset"),Oo.forEach(t),Oa=n(oe," format with "),Ee=l(oe,"A",{href:!0,rel:!0});var Io=i(Ee);$t=l(Io,"CODE",{});var No=i($t);Ia=n(No,"to_tf_dataset"),No.forEach(t),Io.forEach(t),Na=n(oe,". Specify inputs and labels in "),kt=l(oe,"CODE",{});var Uo=i(kt);Ua=n(Uo,"columns"),Uo.forEach(t),Ba=n(oe,", whether to shuffle the dataset order, batch size, and the data collator:"),oe.forEach(t),os=h(e),k(Se.$$.fragment,e),ns=h(e),Ke=l(e,"P",{});var Bo=i(Ke);Wa=n(Bo,"Set up an optimizer function, learning rate schedule, and some training hyperparameters:"),Bo.forEach(t),rs=h(e),k(Te.$$.fragment,e),ls=h(e),V=l(e,"P",{});var ys=i(V);Ya=n(ys,"Load T5 with "),Re=l(ys,"A",{href:!0});var Wo=i(Re);Ha=n(Wo,"TFAutoModelForSeq2SeqLM"),Wo.forEach(t),Za=n(ys,":"),ys.forEach(t),is=h(e),k(xe.$$.fragment,e),ps=h(e),ee=l(e,"P",{});var Es=i(ee);Ja=n(Es,"Configure the model for training with "),ze=l(Es,"A",{href:!0,rel:!0});var Yo=i(ze);wt=l(Yo,"CODE",{});var Ho=i(wt);Ka=n(Ho,"compile"),Ho.forEach(t),Yo.forEach(t),Ra=n(Es,":"),Es.forEach(t),fs=h(e),k(Ae.$$.fragment,e),hs=h(e),te=l(e,"P",{});var Ss=i(te);Xa=n(Ss,"Call "),Fe=l(Ss,"A",{href:!0,rel:!0});var Zo=i(Fe);vt=l(Zo,"CODE",{});var Jo=i(vt);Ga=n(Jo,"fit"),Jo.forEach(t),Zo.forEach(t),Qa=n(Ss," to fine-tune the model:"),Ss.forEach(t),ms=h(e),k(Pe.$$.fragment,e),us=h(e),k(se.$$.fragment,e),this.h()},h(){u(m,"name","hf:doc:metadata"),u(m,"content",JSON.stringify(nn)),u(_,"id","translation"),u(_,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),u(_,"href","#translation"),u(c,"class","relative group"),u(ne,"href","https://huggingface.co/t5-small"),u(ne,"rel","nofollow"),u(re,"href","https://huggingface.co/datasets/opus_books"),u(re,"rel","nofollow"),u(H,"id","load-opus-books-dataset"),u(H,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),u(H,"href","#load-opus-books-dataset"),u(N,"class","relative group"),u(J,"id","preprocess"),u(J,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),u(J,"href","#preprocess"),u(U,"class","relative group"),u(_e,"href","https://huggingface.co/docs/datasets/package_reference/main_classes.html#datasets.Dataset.map"),u(_e,"rel","nofollow"),u(Ue,"href","/docs/transformers/pr_15839/en/main_classes/data_collator#transformers.DataCollatorForSeq2Seq"),u(K,"id","finetune-with-trainer"),u(K,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),u(K,"href","#finetune-with-trainer"),u(B,"class","relative group"),u(Be,"href","/docs/transformers/pr_15839/en/model_doc/auto#transformers.AutoModelForSeq2SeqLM"),u(Ye,"href","/docs/transformers/pr_15839/en/main_classes/trainer#transformers.Seq2SeqTrainingArguments"),u(He,"href","/docs/transformers/pr_15839/en/main_classes/trainer#transformers.Seq2SeqTrainer"),u(Ze,"href","/docs/transformers/pr_15839/en/main_classes/trainer#transformers.Trainer.train"),u(G,"id","finetune-with-tensorflow"),u(G,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),u(G,"href","#finetune-with-tensorflow"),u(W,"class","relative group"),u(Ee,"href","https://huggingface.co/docs/datasets/package_reference/main_classes.html#datasets.Dataset.to_tf_dataset"),u(Ee,"rel","nofollow"),u(Re,"href","/docs/transformers/pr_15839/en/model_doc/auto#transformers.TFAutoModelForSeq2SeqLM"),u(ze,"href","https://keras.io/api/models/model_training_apis/#compile-method"),u(ze,"rel","nofollow"),u(Fe,"href","https://keras.io/api/models/model_training_apis/#fit-method"),u(Fe,"rel","nofollow")},m(e,a){s(document.head,m),p(e,j,a),p(e,c,a),s(c,_),s(_,E),w(d,E,null),s(c,g),s(c,S),s(S,T),p(e,y,a),w(z,e,a),p(e,D,a),p(e,De,a),s(De,xs),p(e,Et,a),p(e,L,a),s(L,zs),s(L,ne),s(ne,As),s(L,Fs),s(L,re),s(re,Ps),s(L,Cs),p(e,St,a),w(Y,e,a),p(e,Tt,a),p(e,N,a),s(N,H),s(H,Ve),w(le,Ve,null),s(N,Ds),s(N,et),s(et,Ls),p(e,xt,a),p(e,Le,a),s(Le,Ms),p(e,zt,a),w(ie,e,a),p(e,At,a),p(e,Me,a),s(Me,Os),p(e,Ft,a),w(pe,e,a),p(e,Pt,a),p(e,Oe,a),s(Oe,Is),p(e,Ct,a),w(fe,e,a),p(e,Dt,a),p(e,Z,a),s(Z,Ns),s(Z,tt),s(tt,Us),s(Z,Bs),p(e,Lt,a),p(e,U,a),s(U,J),s(J,st),w(he,st,null),s(U,Ws),s(U,at),s(at,Ys),p(e,Mt,a),w(me,e,a),p(e,Ot,a),p(e,Ie,a),s(Ie,Hs),p(e,It,a),w(ue,e,a),p(e,Nt,a),p(e,Ne,a),s(Ne,Zs),p(e,Ut,a),p(e,M,a),s(M,ot),s(ot,Js),s(M,Ks),s(M,nt),s(nt,Rs),s(M,Xs),s(M,ce),s(ce,Gs),s(ce,rt),s(rt,Qs),s(ce,Vs),p(e,Bt,a),w(de,e,a),p(e,Wt,a),p(e,F,a),s(F,ea),s(F,_e),s(_e,lt),s(lt,ta),s(F,sa),s(F,it),s(it,aa),s(F,oa),s(F,pt),s(pt,na),s(F,ra),p(e,Yt,a),w(ge,e,a),p(e,Ht,a),p(e,x,a),s(x,la),s(x,Ue),s(Ue,ia),s(x,pa),s(x,ft),s(ft,fa),s(x,ha),s(x,ht),s(ht,ma),s(x,ua),s(x,mt),s(mt,ca),s(x,da),p(e,Zt,a),w($e,e,a),p(e,Jt,a),p(e,B,a),s(B,K),s(K,ut),w(ke,ut,null),s(B,_a),s(B,ct),s(ct,ga),p(e,Kt,a),p(e,R,a),s(R,$a),s(R,Be),s(Be,ka),s(R,wa),p(e,Rt,a),w(we,e,a),p(e,Xt,a),w(X,e,a),p(e,Gt,a),p(e,We,a),s(We,va),p(e,Qt,a),p(e,O,a),s(O,ve),s(ve,ba),s(ve,Ye),s(Ye,qa),s(ve,ja),s(O,ya),s(O,be),s(be,Ea),s(be,He),s(He,Sa),s(be,Ta),s(O,xa),s(O,qe),s(qe,za),s(qe,Ze),s(Ze,Aa),s(qe,Fa),p(e,Vt,a),w(je,e,a),p(e,es,a),p(e,W,a),s(W,G),s(G,dt),w(ye,dt,null),s(W,Pa),s(W,_t),s(_t,Ca),p(e,ts,a),p(e,Je,a),s(Je,Da),p(e,ss,a),w(Q,e,a),p(e,as,a),p(e,P,a),s(P,La),s(P,gt),s(gt,Ma),s(P,Oa),s(P,Ee),s(Ee,$t),s($t,Ia),s(P,Na),s(P,kt),s(kt,Ua),s(P,Ba),p(e,os,a),w(Se,e,a),p(e,ns,a),p(e,Ke,a),s(Ke,Wa),p(e,rs,a),w(Te,e,a),p(e,ls,a),p(e,V,a),s(V,Ya),s(V,Re),s(Re,Ha),s(V,Za),p(e,is,a),w(xe,e,a),p(e,ps,a),p(e,ee,a),s(ee,Ja),s(ee,ze),s(ze,wt),s(wt,Ka),s(ee,Ra),p(e,fs,a),w(Ae,e,a),p(e,hs,a),p(e,te,a),s(te,Xa),s(te,Fe),s(Fe,vt),s(vt,Ga),s(te,Qa),p(e,ms,a),w(Pe,e,a),p(e,us,a),w(se,e,a),cs=!0},p(e,[a]){const Ce={};a&2&&(Ce.$$scope={dirty:a,ctx:e}),Y.$set(Ce);const bt={};a&2&&(bt.$$scope={dirty:a,ctx:e}),X.$set(bt);const qt={};a&2&&(qt.$$scope={dirty:a,ctx:e}),Q.$set(qt);const jt={};a&2&&(jt.$$scope={dirty:a,ctx:e}),se.$set(jt)},i(e){cs||(v(d.$$.fragment,e),v(z.$$.fragment,e),v(Y.$$.fragment,e),v(le.$$.fragment,e),v(ie.$$.fragment,e),v(pe.$$.fragment,e),v(fe.$$.fragment,e),v(he.$$.fragment,e),v(me.$$.fragment,e),v(ue.$$.fragment,e),v(de.$$.fragment,e),v(ge.$$.fragment,e),v($e.$$.fragment,e),v(ke.$$.fragment,e),v(we.$$.fragment,e),v(X.$$.fragment,e),v(je.$$.fragment,e),v(ye.$$.fragment,e),v(Q.$$.fragment,e),v(Se.$$.fragment,e),v(Te.$$.fragment,e),v(xe.$$.fragment,e),v(Ae.$$.fragment,e),v(Pe.$$.fragment,e),v(se.$$.fragment,e),cs=!0)},o(e){b(d.$$.fragment,e),b(z.$$.fragment,e),b(Y.$$.fragment,e),b(le.$$.fragment,e),b(ie.$$.fragment,e),b(pe.$$.fragment,e),b(fe.$$.fragment,e),b(he.$$.fragment,e),b(me.$$.fragment,e),b(ue.$$.fragment,e),b(de.$$.fragment,e),b(ge.$$.fragment,e),b($e.$$.fragment,e),b(ke.$$.fragment,e),b(we.$$.fragment,e),b(X.$$.fragment,e),b(je.$$.fragment,e),b(ye.$$.fragment,e),b(Q.$$.fragment,e),b(Se.$$.fragment,e),b(Te.$$.fragment,e),b(xe.$$.fragment,e),b(Ae.$$.fragment,e),b(Pe.$$.fragment,e),b(se.$$.fragment,e),cs=!1},d(e){t(m),e&&t(j),e&&t(c),q(d),e&&t(y),q(z,e),e&&t(D),e&&t(De),e&&t(Et),e&&t(L),e&&t(St),q(Y,e),e&&t(Tt),e&&t(N),q(le),e&&t(xt),e&&t(Le),e&&t(zt),q(ie,e),e&&t(At),e&&t(Me),e&&t(Ft),q(pe,e),e&&t(Pt),e&&t(Oe),e&&t(Ct),q(fe,e),e&&t(Dt),e&&t(Z),e&&t(Lt),e&&t(U),q(he),e&&t(Mt),q(me,e),e&&t(Ot),e&&t(Ie),e&&t(It),q(ue,e),e&&t(Nt),e&&t(Ne),e&&t(Ut),e&&t(M),e&&t(Bt),q(de,e),e&&t(Wt),e&&t(F),e&&t(Yt),q(ge,e),e&&t(Ht),e&&t(x),e&&t(Zt),q($e,e),e&&t(Jt),e&&t(B),q(ke),e&&t(Kt),e&&t(R),e&&t(Rt),q(we,e),e&&t(Xt),q(X,e),e&&t(Gt),e&&t(We),e&&t(Qt),e&&t(O),e&&t(Vt),q(je,e),e&&t(es),e&&t(W),q(ye),e&&t(ts),e&&t(Je),e&&t(ss),q(Q,e),e&&t(as),e&&t(P),e&&t(os),q(Se,e),e&&t(ns),e&&t(Ke),e&&t(rs),q(Te,e),e&&t(ls),e&&t(V),e&&t(is),q(xe,e),e&&t(ps),e&&t(ee),e&&t(fs),q(Ae,e),e&&t(hs),e&&t(te),e&&t(ms),q(Pe,e),e&&t(us),q(se,e)}}}const nn={local:"translation",sections:[{local:"load-opus-books-dataset",title:"Load OPUS Books dataset"},{local:"preprocess",title:"Preprocess"},{local:"finetune-with-trainer",title:"Fine-tune with Trainer"},{local:"finetune-with-tensorflow",title:"Fine-tune with TensorFlow"}],title:"Translation"};function rn(C,m,j){let{fw:c}=m;return C.$$set=_=>{"fw"in _&&j(0,c=_.fw)},[c]}class dn extends Ro{constructor(m){super();Xo(this,m,rn,on,Go,{fw:0})}}export{dn as default,nn as metadata};
