import{S as Wc,i as Mc,s as Uc,e as l,t as a,k as c,c as r,a as i,h as n,d as t,m as d,b as _,g as o,G as s,w as v,x as y,y as j,q as w,o as b,B as E,M as Bc,N as ti,p as Jn,v as Rc,n as Vn,L as Lc}from"../../chunks/vendor-hf-doc-builder.js";import{T as Ka}from"../../chunks/Tip-hf-doc-builder.js";import{Y as si}from"../../chunks/Youtube-hf-doc-builder.js";import{I as rs}from"../../chunks/IconCopyLink-hf-doc-builder.js";import{C as S}from"../../chunks/CodeBlock-hf-doc-builder.js";import{C as Hc}from"../../chunks/CourseFloatingBanner-hf-doc-builder.js";import{F as Yc}from"../../chunks/FrameworkSwitchCourse-hf-doc-builder.js";function Kc(R){let h,$;return h=new Hc({props:{chapter:7,classNames:"absolute z-10 right-0 top-0",notebooks:[{label:"Google Colab",value:"https://colab.research.google.com/github/huggingface/notebooks/blob/master/course/en/chapter7/section4_tf.ipynb"},{label:"Aws Studio",value:"https://studiolab.sagemaker.aws/import/github/huggingface/notebooks/blob/master/course/en/chapter7/section4_tf.ipynb"}]}}),{c(){v(h.$$.fragment)},l(m){y(h.$$.fragment,m)},m(m,x){j(h,m,x),$=!0},i(m){$||(w(h.$$.fragment,m),$=!0)},o(m){b(h.$$.fragment,m),$=!1},d(m){E(h,m)}}}function Xc(R){let h,$;return h=new Hc({props:{chapter:7,classNames:"absolute z-10 right-0 top-0",notebooks:[{label:"Google Colab",value:"https://colab.research.google.com/github/huggingface/notebooks/blob/master/course/en/chapter7/section4_pt.ipynb"},{label:"Aws Studio",value:"https://studiolab.sagemaker.aws/import/github/huggingface/notebooks/blob/master/course/en/chapter7/section4_pt.ipynb"}]}}),{c(){v(h.$$.fragment)},l(m){y(h.$$.fragment,m)},m(m,x){j(h,m,x),$=!0},i(m){$||(w(h.$$.fragment,m),$=!0)},o(m){b(h.$$.fragment,m),$=!1},d(m){E(h,m)}}}function Gc(R){let h,$,m,x,P;return{c(){h=l("p"),$=a("\u270F\uFE0F "),m=l("strong"),x=a("Your turn!"),P=a(" Another English word that is often used in French is \u201Cemail.\u201D Find the first sample in the training dataset that uses this word. How is it translated? How does the pretrained model translate the same English sentence?")},l(g){h=r(g,"P",{});var D=i(h);$=n(D,"\u270F\uFE0F "),m=r(D,"STRONG",{});var A=i(m);x=n(A,"Your turn!"),A.forEach(t),P=n(D," Another English word that is often used in French is \u201Cemail.\u201D Find the first sample in the training dataset that uses this word. How is it translated? How does the pretrained model translate the same English sentence?"),D.forEach(t)},m(g,D){o(g,h,D),s(h,$),s(h,m),s(m,x),s(h,P)},d(g){g&&t(h)}}}function Zc(R){let h,$,m,x,P,g,D,A;return{c(){h=l("p"),$=a("\u{1F4A1} If you are using a multilingual tokenizer such as mBART, mBART-50, or M2M100, you will need to set the language codes of your inputs and targets in the tokenizer by setting "),m=l("code"),x=a("tokenizer.src_lang"),P=a(" and "),g=l("code"),D=a("tokenizer.tgt_lang"),A=a(" to the right values.")},l(T){h=r(T,"P",{});var z=i(h);$=n(z,"\u{1F4A1} If you are using a multilingual tokenizer such as mBART, mBART-50, or M2M100, you will need to set the language codes of your inputs and targets in the tokenizer by setting "),m=r(z,"CODE",{});var N=i(m);x=n(N,"tokenizer.src_lang"),N.forEach(t),P=n(z," and "),g=r(z,"CODE",{});var q=i(g);D=n(q,"tokenizer.tgt_lang"),q.forEach(t),A=n(z," to the right values."),z.forEach(t)},m(T,z){o(T,h,z),s(h,$),s(h,m),s(m,x),s(h,P),s(h,g),s(g,D),s(h,A)},d(T){T&&t(h)}}}function Jc(R){let h,$,m,x,P,g,D,A;return{c(){h=l("p"),$=a("\u{1F4A1} If you are using a T5 model (more specifically, one of the "),m=l("code"),x=a("t5-xxx"),P=a(" checkpoints), the model will expect the text inputs to have a prefix indicating the task at hand, such as "),g=l("code"),D=a("translate: English to French:"),A=a(".")},l(T){h=r(T,"P",{});var z=i(h);$=n(z,"\u{1F4A1} If you are using a T5 model (more specifically, one of the "),m=r(z,"CODE",{});var N=i(m);x=n(N,"t5-xxx"),N.forEach(t),P=n(z," checkpoints), the model will expect the text inputs to have a prefix indicating the task at hand, such as "),g=r(z,"CODE",{});var q=i(g);D=n(q,"translate: English to French:"),q.forEach(t),A=n(z,"."),z.forEach(t)},m(T,z){o(T,h,z),s(h,$),s(h,m),s(m,x),s(h,P),s(h,g),s(g,D),s(h,A)},d(T){T&&t(h)}}}function Vc(R){let h,$,m,x,P,g,D,A;return{c(){h=l("p"),$=a("\u26A0\uFE0F We don\u2019t pay attention to the attention mask of the targets, as the model won\u2019t expect it. Instead, the labels corresponding to a padding token should be set to "),m=l("code"),x=a("-100"),P=a(" so they are ignored in the loss computation. This will be done by our data collator later on since we are applying dynamic padding, but if you use padding here, you should adapt the preprocessing function to set all labels that correspond to the padding token to "),g=l("code"),D=a("-100"),A=a(".")},l(T){h=r(T,"P",{});var z=i(h);$=n(z,"\u26A0\uFE0F We don\u2019t pay attention to the attention mask of the targets, as the model won\u2019t expect it. Instead, the labels corresponding to a padding token should be set to "),m=r(z,"CODE",{});var N=i(m);x=n(N,"-100"),N.forEach(t),P=n(z," so they are ignored in the loss computation. This will be done by our data collator later on since we are applying dynamic padding, but if you use padding here, you should adapt the preprocessing function to set all labels that correspond to the padding token to "),g=r(z,"CODE",{});var q=i(g);D=n(q,"-100"),q.forEach(t),A=n(z,"."),z.forEach(t)},m(T,z){o(T,h,z),s(h,$),s(h,m),s(m,x),s(h,P),s(h,g),s(g,D),s(h,A)},d(T){T&&t(h)}}}function Qc(R){let h,$,m,x,P,g,D,A,T,z,N,q,F,W,U,L,G,H;return x=new rs({}),U=new S({props:{code:`from transformers import TFAutoModelForSeq2SeqLM

model = TFAutoModelForSeq2SeqLM.from_pretrained(model_checkpoint, from_pt=True)`,highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> TFAutoModelForSeq2SeqLM

model = TFAutoModelForSeq2SeqLM.from_pretrained(model_checkpoint, from_pt=<span class="hljs-literal">True</span>)`}}),G=new Ka({props:{warning:!1,$$slots:{default:[td]},$$scope:{ctx:R}}}),{c(){h=l("h2"),$=l("a"),m=l("span"),v(x.$$.fragment),P=c(),g=l("span"),D=a("Fine-tuning the model with Keras"),A=c(),T=l("p"),z=a("First things first, we need an actual model to fine-tune. We\u2019ll use the usual "),N=l("code"),q=a("AutoModel"),F=a(" API:"),W=c(),v(U.$$.fragment),L=c(),v(G.$$.fragment),this.h()},l(C){h=r(C,"H2",{class:!0});var K=i(h);$=r(K,"A",{id:!0,class:!0,href:!0});var B=i($);m=r(B,"SPAN",{});var Y=i(m);y(x.$$.fragment,Y),Y.forEach(t),B.forEach(t),P=d(K),g=r(K,"SPAN",{});var re=i(g);D=n(re,"Fine-tuning the model with Keras"),re.forEach(t),K.forEach(t),A=d(C),T=r(C,"P",{});var V=i(T);z=n(V,"First things first, we need an actual model to fine-tune. We\u2019ll use the usual "),N=r(V,"CODE",{});var Q=i(N);q=n(Q,"AutoModel"),Q.forEach(t),F=n(V," API:"),V.forEach(t),W=d(C),y(U.$$.fragment,C),L=d(C),y(G.$$.fragment,C),this.h()},h(){_($,"id","finetuning-the-model-with-keras"),_($,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),_($,"href","#finetuning-the-model-with-keras"),_(h,"class","relative group")},m(C,K){o(C,h,K),s(h,$),s($,m),j(x,m,null),s(h,P),s(h,g),s(g,D),o(C,A,K),o(C,T,K),s(T,z),s(T,N),s(N,q),s(T,F),o(C,W,K),j(U,C,K),o(C,L,K),j(G,C,K),H=!0},i(C){H||(w(x.$$.fragment,C),w(U.$$.fragment,C),w(G.$$.fragment,C),H=!0)},o(C){b(x.$$.fragment,C),b(U.$$.fragment,C),b(G.$$.fragment,C),H=!1},d(C){C&&t(h),E(x),C&&t(A),C&&t(T),C&&t(W),E(U,C),C&&t(L),E(G,C)}}}function ed(R){let h,$,m,x,P,g,D,A,T,z,N,q,F,W,U,L,G,H,C,K,B,Y,re,V,Q,ee,Z,oe,$e,se,ae,fe,ke,le,he;return x=new rs({}),le=new S({props:{code:`from transformers import AutoModelForSeq2SeqLM

model = AutoModelForSeq2SeqLM.from_pretrained(model_checkpoint)`,highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoModelForSeq2SeqLM

model = AutoModelForSeq2SeqLM.from_pretrained(model_checkpoint)`}}),{c(){h=l("h2"),$=l("a"),m=l("span"),v(x.$$.fragment),P=c(),g=l("span"),D=a("Fine-tuning the model with the "),A=l("code"),T=a("Trainer"),z=a(" API"),N=c(),q=l("p"),F=a("The actual code using the "),W=l("code"),U=a("Trainer"),L=a(" will be the same as before, with just one little change: we use a "),G=l("a"),H=l("code"),C=a("Seq2SeqTrainer"),K=a(" here, which is a subclass of "),B=l("code"),Y=a("Trainer"),re=a(" that will allow us to properly deal with the evaluation, using the "),V=l("code"),Q=a("generate()"),ee=a(" method to predict outputs from the inputs. We\u2019ll dive into that in more detail when we talk about the metric computation."),Z=c(),oe=l("p"),$e=a("First things first, we need an actual model to fine-tune. We\u2019ll use the usual "),se=l("code"),ae=a("AutoModel"),fe=a(" API:"),ke=c(),v(le.$$.fragment),this.h()},l(M){h=r(M,"H2",{class:!0});var ne=i(h);$=r(ne,"A",{id:!0,class:!0,href:!0});var ze=i($);m=r(ze,"SPAN",{});var me=i(m);y(x.$$.fragment,me),me.forEach(t),ze.forEach(t),P=d(ne),g=r(ne,"SPAN",{});var J=i(g);D=n(J,"Fine-tuning the model with the "),A=r(J,"CODE",{});var ye=i(A);T=n(ye,"Trainer"),ye.forEach(t),z=n(J," API"),J.forEach(t),ne.forEach(t),N=d(M),q=r(M,"P",{});var ie=i(q);F=n(ie,"The actual code using the "),W=r(ie,"CODE",{});var xe=i(W);U=n(xe,"Trainer"),xe.forEach(t),L=n(ie," will be the same as before, with just one little change: we use a "),G=r(ie,"A",{href:!0,rel:!0});var Me=i(G);H=r(Me,"CODE",{});var _e=i(H);C=n(_e,"Seq2SeqTrainer"),_e.forEach(t),Me.forEach(t),K=n(ie," here, which is a subclass of "),B=r(ie,"CODE",{});var qe=i(B);Y=n(qe,"Trainer"),qe.forEach(t),re=n(ie," that will allow us to properly deal with the evaluation, using the "),V=r(ie,"CODE",{});var te=i(V);Q=n(te,"generate()"),te.forEach(t),ee=n(ie," method to predict outputs from the inputs. We\u2019ll dive into that in more detail when we talk about the metric computation."),ie.forEach(t),Z=d(M),oe=r(M,"P",{});var Pe=i(oe);$e=n(Pe,"First things first, we need an actual model to fine-tune. We\u2019ll use the usual "),se=r(Pe,"CODE",{});var ge=i(se);ae=n(ge,"AutoModel"),ge.forEach(t),fe=n(Pe," API:"),Pe.forEach(t),ke=d(M),y(le.$$.fragment,M),this.h()},h(){_($,"id","finetuning-the-model-with-the-trainer-api"),_($,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),_($,"href","#finetuning-the-model-with-the-trainer-api"),_(h,"class","relative group"),_(G,"href","https://huggingface.co/transformers/main_classes/trainer.html#seq2seqtrainer"),_(G,"rel","nofollow")},m(M,ne){o(M,h,ne),s(h,$),s($,m),j(x,m,null),s(h,P),s(h,g),s(g,D),s(g,A),s(A,T),s(g,z),o(M,N,ne),o(M,q,ne),s(q,F),s(q,W),s(W,U),s(q,L),s(q,G),s(G,H),s(H,C),s(q,K),s(q,B),s(B,Y),s(q,re),s(q,V),s(V,Q),s(q,ee),o(M,Z,ne),o(M,oe,ne),s(oe,$e),s(oe,se),s(se,ae),s(oe,fe),o(M,ke,ne),j(le,M,ne),he=!0},i(M){he||(w(x.$$.fragment,M),w(le.$$.fragment,M),he=!0)},o(M){b(x.$$.fragment,M),b(le.$$.fragment,M),he=!1},d(M){M&&t(h),E(x),M&&t(N),M&&t(q),M&&t(Z),M&&t(oe),M&&t(ke),E(le,M)}}}function td(R){let h,$,m,x,P,g,D,A,T,z,N,q,F,W;return{c(){h=l("p"),$=a("\u{1F4A1} The "),m=l("code"),x=a("Helsinki-NLP/opus-mt-en-fr"),P=a(` checkpoint only has PyTorch weights, so
you\u2019ll get an error if you try to load the model without using the
`),g=l("code"),D=a("from_pt=True"),A=a(" argument in the "),T=l("code"),z=a("from_pretrained()"),N=a(` method. When you specify
`),q=l("code"),F=a("from_pt=True"),W=a(`, the library will automatically download and convert the
PyTorch weights for you. As you can see, it is very simple to switch between
frameworks in \u{1F917} Transformers!`)},l(U){h=r(U,"P",{});var L=i(h);$=n(L,"\u{1F4A1} The "),m=r(L,"CODE",{});var G=i(m);x=n(G,"Helsinki-NLP/opus-mt-en-fr"),G.forEach(t),P=n(L,` checkpoint only has PyTorch weights, so
you\u2019ll get an error if you try to load the model without using the
`),g=r(L,"CODE",{});var H=i(g);D=n(H,"from_pt=True"),H.forEach(t),A=n(L," argument in the "),T=r(L,"CODE",{});var C=i(T);z=n(C,"from_pretrained()"),C.forEach(t),N=n(L,` method. When you specify
`),q=r(L,"CODE",{});var K=i(q);F=n(K,"from_pt=True"),K.forEach(t),W=n(L,`, the library will automatically download and convert the
PyTorch weights for you. As you can see, it is very simple to switch between
frameworks in \u{1F917} Transformers!`),L.forEach(t)},m(U,L){o(U,h,L),s(h,$),s(h,m),s(m,x),s(h,P),s(h,g),s(g,D),s(h,A),s(h,T),s(T,z),s(h,N),s(h,q),s(q,F),s(h,W)},d(U){U&&t(h)}}}function sd(R){let h,$;return h=new S({props:{code:`from transformers import DataCollatorForSeq2Seq

data_collator = DataCollatorForSeq2Seq(tokenizer, model=model, return_tensors="tf")`,highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> DataCollatorForSeq2Seq

data_collator = DataCollatorForSeq2Seq(tokenizer, model=model, return_tensors=<span class="hljs-string">&quot;tf&quot;</span>)`}}),{c(){v(h.$$.fragment)},l(m){y(h.$$.fragment,m)},m(m,x){j(h,m,x),$=!0},i(m){$||(w(h.$$.fragment,m),$=!0)},o(m){b(h.$$.fragment,m),$=!1},d(m){E(h,m)}}}function ad(R){let h,$;return h=new S({props:{code:`from transformers import DataCollatorForSeq2Seq

data_collator = DataCollatorForSeq2Seq(tokenizer, model=model)`,highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> DataCollatorForSeq2Seq

data_collator = DataCollatorForSeq2Seq(tokenizer, model=model)`}}),{c(){v(h.$$.fragment)},l(m){y(h.$$.fragment,m)},m(m,x){j(h,m,x),$=!0},i(m){$||(w(h.$$.fragment,m),$=!0)},o(m){b(h.$$.fragment,m),$=!1},d(m){E(h,m)}}}function nd(R){let h,$,m,x,P,g,D,A,T,z,N;return z=new S({props:{code:`tf_train_dataset = model.prepare_tf_dataset(
    tokenized_datasets["train"],
    collate_fn=data_collator,
    shuffle=True,
    batch_size=32,
)
tf_eval_dataset = model.prepare_tf_dataset(
    tokenized_datasets["validation"],
    collate_fn=data_collator,
    shuffle=False,
    batch_size=16,
)`,highlighted:`tf_train_dataset = model.prepare_tf_dataset(
    tokenized_datasets[<span class="hljs-string">&quot;train&quot;</span>],
    collate_fn=data_collator,
    shuffle=<span class="hljs-literal">True</span>,
    batch_size=<span class="hljs-number">32</span>,
)
tf_eval_dataset = model.prepare_tf_dataset(
    tokenized_datasets[<span class="hljs-string">&quot;validation&quot;</span>],
    collate_fn=data_collator,
    shuffle=<span class="hljs-literal">False</span>,
    batch_size=<span class="hljs-number">16</span>,
)`}}),{c(){h=l("p"),$=a("We can now use this "),m=l("code"),x=a("data_collator"),P=a(" to convert each of our datasets to a "),g=l("code"),D=a("tf.data.Dataset"),A=a(", ready for training:"),T=c(),v(z.$$.fragment)},l(q){h=r(q,"P",{});var F=i(h);$=n(F,"We can now use this "),m=r(F,"CODE",{});var W=i(m);x=n(W,"data_collator"),W.forEach(t),P=n(F," to convert each of our datasets to a "),g=r(F,"CODE",{});var U=i(g);D=n(U,"tf.data.Dataset"),U.forEach(t),A=n(F,", ready for training:"),F.forEach(t),T=d(q),y(z.$$.fragment,q)},m(q,F){o(q,h,F),s(h,$),s(h,m),s(m,x),s(h,P),s(h,g),s(g,D),s(h,A),o(q,T,F),j(z,q,F),N=!0},i(q){N||(w(z.$$.fragment,q),N=!0)},o(q){b(z.$$.fragment,q),N=!1},d(q){q&&t(h),q&&t(T),E(z,q)}}}function od(R){let h,$,m,x,P,g,D,A;return{c(){h=l("p"),$=a("We will pass this "),m=l("code"),x=a("data_collator"),P=a(" along to the "),g=l("code"),D=a("Seq2SeqTrainer"),A=a(". Next, let\u2019s have a look at the metric.")},l(T){h=r(T,"P",{});var z=i(h);$=n(z,"We will pass this "),m=r(z,"CODE",{});var N=i(m);x=n(N,"data_collator"),N.forEach(t),P=n(z," along to the "),g=r(z,"CODE",{});var q=i(g);D=n(q,"Seq2SeqTrainer"),q.forEach(t),A=n(z,". Next, let\u2019s have a look at the metric."),z.forEach(t)},m(T,z){o(T,h,z),s(h,$),s(h,m),s(m,x),s(h,P),s(h,g),s(g,D),s(h,A)},i:Lc,o:Lc,d(T){T&&t(h)}}}function Ic(R){let h,$,m,x,P,g,D,A,T,z,N,q,F,W,U,L,G,H,C,K,B,Y,re,V,Q,ee,Z,oe,$e;return{c(){h=l("p"),$=a("The feature that "),m=l("code"),x=a("Seq2SeqTrainer"),P=a(" adds to its superclass "),g=l("code"),D=a("Trainer"),A=a(" is the ability to use the "),T=l("code"),z=a("generate()"),N=a(" method during evaluation or prediction. During training, the model will use the "),q=l("code"),F=a("decoder_input_ids"),W=a(" with an attention mask ensuring it does not use the tokens after the token it\u2019s trying to predict, to speed up training. During inference we won\u2019t be able to use those since we won\u2019t have labels, so it\u2019s a good idea to evaluate our model with the same setup."),U=c(),L=l("p"),G=a("As we saw in "),H=l("a"),C=a("Chapter 1"),K=a(", the decoder performs inference by predicting tokens one by one \u2014 something that\u2019s implemented behind the scenes in \u{1F917} Transformers by the "),B=l("code"),Y=a("generate()"),re=a(" method. The "),V=l("code"),Q=a("Seq2SeqTrainer"),ee=a(" will let us use that method for evaluation if we set "),Z=l("code"),oe=a("predict_with_generate=True"),$e=a("."),this.h()},l(se){h=r(se,"P",{});var ae=i(h);$=n(ae,"The feature that "),m=r(ae,"CODE",{});var fe=i(m);x=n(fe,"Seq2SeqTrainer"),fe.forEach(t),P=n(ae," adds to its superclass "),g=r(ae,"CODE",{});var ke=i(g);D=n(ke,"Trainer"),ke.forEach(t),A=n(ae," is the ability to use the "),T=r(ae,"CODE",{});var le=i(T);z=n(le,"generate()"),le.forEach(t),N=n(ae," method during evaluation or prediction. During training, the model will use the "),q=r(ae,"CODE",{});var he=i(q);F=n(he,"decoder_input_ids"),he.forEach(t),W=n(ae," with an attention mask ensuring it does not use the tokens after the token it\u2019s trying to predict, to speed up training. During inference we won\u2019t be able to use those since we won\u2019t have labels, so it\u2019s a good idea to evaluate our model with the same setup."),ae.forEach(t),U=d(se),L=r(se,"P",{});var M=i(L);G=n(M,"As we saw in "),H=r(M,"A",{href:!0});var ne=i(H);C=n(ne,"Chapter 1"),ne.forEach(t),K=n(M,", the decoder performs inference by predicting tokens one by one \u2014 something that\u2019s implemented behind the scenes in \u{1F917} Transformers by the "),B=r(M,"CODE",{});var ze=i(B);Y=n(ze,"generate()"),ze.forEach(t),re=n(M," method. The "),V=r(M,"CODE",{});var me=i(V);Q=n(me,"Seq2SeqTrainer"),me.forEach(t),ee=n(M," will let us use that method for evaluation if we set "),Z=r(M,"CODE",{});var J=i(Z);oe=n(J,"predict_with_generate=True"),J.forEach(t),$e=n(M,"."),M.forEach(t),this.h()},h(){_(H,"href","/course/chapter1/6")},m(se,ae){o(se,h,ae),s(h,$),s(h,m),s(m,x),s(h,P),s(h,g),s(g,D),s(h,A),s(h,T),s(T,z),s(h,N),s(h,q),s(q,F),s(h,W),o(se,U,ae),o(se,L,ae),s(L,G),s(L,H),s(H,C),s(L,K),s(L,B),s(B,Y),s(L,re),s(L,V),s(V,Q),s(L,ee),s(L,Z),s(Z,oe),s(L,$e)},d(se){se&&t(h),se&&t(U),se&&t(L)}}}function ld(R){let h,$,m,x,P,g,D,A,T,z,N;return z=new S({props:{code:`import numpy as np


def compute_metrics(eval_preds):
    preds, labels = eval_preds
    # In case the model returns more than the prediction logits
    if isinstance(preds, tuple):
        preds = preds[0]

    decoded_preds = tokenizer.batch_decode(preds, skip_special_tokens=True)

    # Replace -100s in the labels as we can't decode them
    labels = np.where(labels != -100, labels, tokenizer.pad_token_id)
    decoded_labels = tokenizer.batch_decode(labels, skip_special_tokens=True)

    # Some simple post-processing
    decoded_preds = [pred.strip() for pred in decoded_preds]
    decoded_labels = [[label.strip()] for label in decoded_labels]

    result = metric.compute(predictions=decoded_preds, references=decoded_labels)
    return {"bleu": result["score"]}`,highlighted:`<span class="hljs-keyword">import</span> numpy <span class="hljs-keyword">as</span> np


<span class="hljs-keyword">def</span> <span class="hljs-title function_">compute_metrics</span>(<span class="hljs-params">eval_preds</span>):
    preds, labels = eval_preds
    <span class="hljs-comment"># In case the model returns more than the prediction logits</span>
    <span class="hljs-keyword">if</span> <span class="hljs-built_in">isinstance</span>(preds, <span class="hljs-built_in">tuple</span>):
        preds = preds[<span class="hljs-number">0</span>]

    decoded_preds = tokenizer.batch_decode(preds, skip_special_tokens=<span class="hljs-literal">True</span>)

    <span class="hljs-comment"># Replace -100s in the labels as we can&#x27;t decode them</span>
    labels = np.where(labels != -<span class="hljs-number">100</span>, labels, tokenizer.pad_token_id)
    decoded_labels = tokenizer.batch_decode(labels, skip_special_tokens=<span class="hljs-literal">True</span>)

    <span class="hljs-comment"># Some simple post-processing</span>
    decoded_preds = [pred.strip() <span class="hljs-keyword">for</span> pred <span class="hljs-keyword">in</span> decoded_preds]
    decoded_labels = [[label.strip()] <span class="hljs-keyword">for</span> label <span class="hljs-keyword">in</span> decoded_labels]

    result = metric.compute(predictions=decoded_preds, references=decoded_labels)
    <span class="hljs-keyword">return</span> {<span class="hljs-string">&quot;bleu&quot;</span>: result[<span class="hljs-string">&quot;score&quot;</span>]}`}}),{c(){h=l("p"),$=a("To get from the model outputs to texts the metric can use, we will use the "),m=l("code"),x=a("tokenizer.batch_decode()"),P=a(" method. We just have to clean up all the "),g=l("code"),D=a("-100"),A=a("s in the labels (the tokenizer will automatically do the same for the padding token):"),T=c(),v(z.$$.fragment)},l(q){h=r(q,"P",{});var F=i(h);$=n(F,"To get from the model outputs to texts the metric can use, we will use the "),m=r(F,"CODE",{});var W=i(m);x=n(W,"tokenizer.batch_decode()"),W.forEach(t),P=n(F," method. We just have to clean up all the "),g=r(F,"CODE",{});var U=i(g);D=n(U,"-100"),U.forEach(t),A=n(F,"s in the labels (the tokenizer will automatically do the same for the padding token):"),F.forEach(t),T=d(q),y(z.$$.fragment,q)},m(q,F){o(q,h,F),s(h,$),s(h,m),s(m,x),s(h,P),s(h,g),s(g,D),s(h,A),o(q,T,F),j(z,q,F),N=!0},i(q){N||(w(z.$$.fragment,q),N=!0)},o(q){b(z.$$.fragment,q),N=!1},d(q){q&&t(h),q&&t(T),E(z,q)}}}function rd(R){let h,$,m,x,P,g,D,A,T,z,N,q,F,W,U,L,G,H,C,K;return C=new S({props:{code:`import numpy as np
import tensorflow as tf
from tqdm import tqdm

generation_data_collator = DataCollatorForSeq2Seq(
    tokenizer, model=model, return_tensors="tf", pad_to_multiple_of=128
)

tf_generate_dataset = model.prepare_tf_dataset(
    tokenized_datasets["validation"],
    collate_fn=generation_data_collator,
    shuffle=False,
    batch_size=8,
)


@tf.function(jit_compile=True)
def generate_with_xla(batch):
    return model.generate(
        input_ids=batch["input_ids"],
        attention_mask=batch["attention_mask"],
        max_new_tokens=128,
    )


def compute_metrics():
    all_preds = []
    all_labels = []

    for batch, labels in tqdm(tf_generate_dataset):
        predictions = generate_with_xla(batch)
        decoded_preds = tokenizer.batch_decode(predictions, skip_special_tokens=True)
        labels = labels.numpy()
        labels = np.where(labels != -100, labels, tokenizer.pad_token_id)
        decoded_labels = tokenizer.batch_decode(labels, skip_special_tokens=True)
        decoded_preds = [pred.strip() for pred in decoded_preds]
        decoded_labels = [[label.strip()] for label in decoded_labels]
        all_preds.extend(decoded_preds)
        all_labels.extend(decoded_labels)

    result = metric.compute(predictions=all_preds, references=all_labels)
    return {"bleu": result["score"]}`,highlighted:`<span class="hljs-keyword">import</span> numpy <span class="hljs-keyword">as</span> np
<span class="hljs-keyword">import</span> tensorflow <span class="hljs-keyword">as</span> tf
<span class="hljs-keyword">from</span> tqdm <span class="hljs-keyword">import</span> tqdm

generation_data_collator = DataCollatorForSeq2Seq(
    tokenizer, model=model, return_tensors=<span class="hljs-string">&quot;tf&quot;</span>, pad_to_multiple_of=<span class="hljs-number">128</span>
)

tf_generate_dataset = model.prepare_tf_dataset(
    tokenized_datasets[<span class="hljs-string">&quot;validation&quot;</span>],
    collate_fn=generation_data_collator,
    shuffle=<span class="hljs-literal">False</span>,
    batch_size=<span class="hljs-number">8</span>,
)


<span class="hljs-meta">@tf.function(<span class="hljs-params">jit_compile=<span class="hljs-literal">True</span></span>)</span>
<span class="hljs-keyword">def</span> <span class="hljs-title function_">generate_with_xla</span>(<span class="hljs-params">batch</span>):
    <span class="hljs-keyword">return</span> model.generate(
        input_ids=batch[<span class="hljs-string">&quot;input_ids&quot;</span>],
        attention_mask=batch[<span class="hljs-string">&quot;attention_mask&quot;</span>],
        max_new_tokens=<span class="hljs-number">128</span>,
    )


<span class="hljs-keyword">def</span> <span class="hljs-title function_">compute_metrics</span>():
    all_preds = []
    all_labels = []

    <span class="hljs-keyword">for</span> batch, labels <span class="hljs-keyword">in</span> tqdm(tf_generate_dataset):
        predictions = generate_with_xla(batch)
        decoded_preds = tokenizer.batch_decode(predictions, skip_special_tokens=<span class="hljs-literal">True</span>)
        labels = labels.numpy()
        labels = np.where(labels != -<span class="hljs-number">100</span>, labels, tokenizer.pad_token_id)
        decoded_labels = tokenizer.batch_decode(labels, skip_special_tokens=<span class="hljs-literal">True</span>)
        decoded_preds = [pred.strip() <span class="hljs-keyword">for</span> pred <span class="hljs-keyword">in</span> decoded_preds]
        decoded_labels = [[label.strip()] <span class="hljs-keyword">for</span> label <span class="hljs-keyword">in</span> decoded_labels]
        all_preds.extend(decoded_preds)
        all_labels.extend(decoded_labels)

    result = metric.compute(predictions=all_preds, references=all_labels)
    <span class="hljs-keyword">return</span> {<span class="hljs-string">&quot;bleu&quot;</span>: result[<span class="hljs-string">&quot;score&quot;</span>]}`}}),{c(){h=l("p"),$=a("To get from the model outputs to texts the metric can use, we will use the "),m=l("code"),x=a("tokenizer.batch_decode()"),P=a(" method. We just have to clean up all the "),g=l("code"),D=a("-100"),A=a("s in the labels; the tokenizer will automatically do the same for the padding token. Let\u2019s define a function that takes our model and a dataset and computes metrics on it. We\u2019re also going to use a trick that dramatically increases performance - compiling our generation code with "),T=l("a"),z=a("XLA"),N=a(", TensorFlow\u2019s accelerated linear algebra compiler. XLA applies various optimizations to the model\u2019s computation graph, and results in significant improvements to speed and memory usage. As described in the Hugging Face "),q=l("a"),F=a("blog"),W=a(", XLA works best when our input shapes don\u2019t vary too much. To handle this, we\u2019ll pad our inputs to multiples of 128, and make a new dataset with the padding collator, and then we\u2019ll apply the "),U=l("code"),L=a("@tf.function(jit_compile=True)"),G=a(" decorator to our generation function, which marks the whole function for compilation with XLA."),H=c(),v(C.$$.fragment),this.h()},l(B){h=r(B,"P",{});var Y=i(h);$=n(Y,"To get from the model outputs to texts the metric can use, we will use the "),m=r(Y,"CODE",{});var re=i(m);x=n(re,"tokenizer.batch_decode()"),re.forEach(t),P=n(Y," method. We just have to clean up all the "),g=r(Y,"CODE",{});var V=i(g);D=n(V,"-100"),V.forEach(t),A=n(Y,"s in the labels; the tokenizer will automatically do the same for the padding token. Let\u2019s define a function that takes our model and a dataset and computes metrics on it. We\u2019re also going to use a trick that dramatically increases performance - compiling our generation code with "),T=r(Y,"A",{href:!0,rel:!0});var Q=i(T);z=n(Q,"XLA"),Q.forEach(t),N=n(Y,", TensorFlow\u2019s accelerated linear algebra compiler. XLA applies various optimizations to the model\u2019s computation graph, and results in significant improvements to speed and memory usage. As described in the Hugging Face "),q=r(Y,"A",{href:!0,rel:!0});var ee=i(q);F=n(ee,"blog"),ee.forEach(t),W=n(Y,", XLA works best when our input shapes don\u2019t vary too much. To handle this, we\u2019ll pad our inputs to multiples of 128, and make a new dataset with the padding collator, and then we\u2019ll apply the "),U=r(Y,"CODE",{});var Z=i(U);L=n(Z,"@tf.function(jit_compile=True)"),Z.forEach(t),G=n(Y," decorator to our generation function, which marks the whole function for compilation with XLA."),Y.forEach(t),H=d(B),y(C.$$.fragment,B),this.h()},h(){_(T,"href","https://www.tensorflow.org/xla"),_(T,"rel","nofollow"),_(q,"href","https://huggingface.co/blog/tf-xla-generate"),_(q,"rel","nofollow")},m(B,Y){o(B,h,Y),s(h,$),s(h,m),s(m,x),s(h,P),s(h,g),s(g,D),s(h,A),s(h,T),s(T,z),s(h,N),s(h,q),s(q,F),s(h,W),s(h,U),s(U,L),s(h,G),o(B,H,Y),j(C,B,Y),K=!0},i(B){K||(w(C.$$.fragment,B),K=!0)},o(B){b(C.$$.fragment,B),K=!1},d(B){B&&t(h),B&&t(H),E(C,B)}}}function id(R){let h,$,m,x,P,g,D,A,T,z,N,q,F,W,U,L,G,H,C,K,B,Y,re,V,Q,ee,Z,oe,$e,se,ae,fe,ke,le,he,M,ne,ze,me,J,ye,ie,xe,Me,_e,qe,te,Pe,ge,at,ft,ve,Ge,Ue,Se,je,mt,we,k,X,Ze,ce,Be,nt,Ce,pe,Ft,Re,Ie,Ne,_t,de,Je,Oe,zt,is,ot,lt,rt,Ve,Dt,ps,Qe,gt,wt,Ee,Xt,Fe,$s,Gt,De,Zt,Ae,Lt,bt,ks,Jt,be,$t,At,kt,it,He,Vt,vt,pt,ht,It,Pt,et,Qt,Le,es,Ye,ts,yt,Nt,hs,ct;return F=new S({props:{code:`from transformers import Seq2SeqTrainingArguments

args = Seq2SeqTrainingArguments(
    f"marian-finetuned-kde4-en-to-fr",
    evaluation_strategy="no",
    save_strategy="epoch",
    learning_rate=2e-5,
    per_device_train_batch_size=32,
    per_device_eval_batch_size=64,
    weight_decay=0.01,
    save_total_limit=3,
    num_train_epochs=3,
    predict_with_generate=True,
    fp16=True,
    push_to_hub=True,
)`,highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> Seq2SeqTrainingArguments

args = Seq2SeqTrainingArguments(
    <span class="hljs-string">f&quot;marian-finetuned-kde4-en-to-fr&quot;</span>,
    evaluation_strategy=<span class="hljs-string">&quot;no&quot;</span>,
    save_strategy=<span class="hljs-string">&quot;epoch&quot;</span>,
    learning_rate=<span class="hljs-number">2e-5</span>,
    per_device_train_batch_size=<span class="hljs-number">32</span>,
    per_device_eval_batch_size=<span class="hljs-number">64</span>,
    weight_decay=<span class="hljs-number">0.01</span>,
    save_total_limit=<span class="hljs-number">3</span>,
    num_train_epochs=<span class="hljs-number">3</span>,
    predict_with_generate=<span class="hljs-literal">True</span>,
    fp16=<span class="hljs-literal">True</span>,
    push_to_hub=<span class="hljs-literal">True</span>,
)`}}),X=new Ka({props:{$$slots:{default:[hd]},$$scope:{ctx:R}}}),Re=new S({props:{code:`from transformers import Seq2SeqTrainer

trainer = Seq2SeqTrainer(
    model,
    args,
    train_dataset=tokenized_datasets["train"],
    eval_dataset=tokenized_datasets["validation"],
    data_collator=data_collator,
    tokenizer=tokenizer,
    compute_metrics=compute_metrics,
)`,highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> Seq2SeqTrainer

trainer = Seq2SeqTrainer(
    model,
    args,
    train_dataset=tokenized_datasets[<span class="hljs-string">&quot;train&quot;</span>],
    eval_dataset=tokenized_datasets[<span class="hljs-string">&quot;validation&quot;</span>],
    data_collator=data_collator,
    tokenizer=tokenizer,
    compute_metrics=compute_metrics,
)`}}),Je=new S({props:{code:"trainer.evaluate(max_length=max_length)",highlighted:"trainer.evaluate(max_length=max_length)"}}),zt=new S({props:{code:`{'eval_loss': 1.6964408159255981,
 'eval_bleu': 39.26865061007616,
 'eval_runtime': 965.8884,
 'eval_samples_per_second': 21.76,
 'eval_steps_per_second': 0.341}`,highlighted:`{<span class="hljs-string">&#x27;eval_loss&#x27;</span>: <span class="hljs-number">1.6964408159255981</span>,
 <span class="hljs-string">&#x27;eval_bleu&#x27;</span>: <span class="hljs-number">39.26865061007616</span>,
 <span class="hljs-string">&#x27;eval_runtime&#x27;</span>: <span class="hljs-number">965.8884</span>,
 <span class="hljs-string">&#x27;eval_samples_per_second&#x27;</span>: <span class="hljs-number">21.76</span>,
 <span class="hljs-string">&#x27;eval_steps_per_second&#x27;</span>: <span class="hljs-number">0.341</span>}`}}),Qe=new S({props:{code:"trainer.train()",highlighted:"trainer.train()"}}),De=new S({props:{code:"trainer.evaluate(max_length=max_length)",highlighted:"trainer.evaluate(max_length=max_length)"}}),Ae=new S({props:{code:`{'eval_loss': 0.8558505773544312,
 'eval_bleu': 52.94161337775576,
 'eval_runtime': 714.2576,
 'eval_samples_per_second': 29.426,
 'eval_steps_per_second': 0.461,
 'epoch': 3.0}`,highlighted:`{<span class="hljs-string">&#x27;eval_loss&#x27;</span>: <span class="hljs-number">0.8558505773544312</span>,
 <span class="hljs-string">&#x27;eval_bleu&#x27;</span>: <span class="hljs-number">52.94161337775576</span>,
 <span class="hljs-string">&#x27;eval_runtime&#x27;</span>: <span class="hljs-number">714.2576</span>,
 <span class="hljs-string">&#x27;eval_samples_per_second&#x27;</span>: <span class="hljs-number">29.426</span>,
 <span class="hljs-string">&#x27;eval_steps_per_second&#x27;</span>: <span class="hljs-number">0.461</span>,
 <span class="hljs-string">&#x27;epoch&#x27;</span>: <span class="hljs-number">3.0</span>}`}}),ht=new S({props:{code:'trainer.push_to_hub(tags="translation", commit_message="Training complete")',highlighted:'trainer.push_to_hub(tags=<span class="hljs-string">&quot;translation&quot;</span>, commit_message=<span class="hljs-string">&quot;Training complete&quot;</span>)'}}),Le=new S({props:{code:"'https://huggingface.co/sgugger/marian-finetuned-kde4-en-to-fr/commit/3601d621e3baae2bc63d3311452535f8f58f6ef3'",highlighted:'<span class="hljs-string">&#x27;https://huggingface.co/sgugger/marian-finetuned-kde4-en-to-fr/commit/3601d621e3baae2bc63d3311452535f8f58f6ef3&#x27;</span>'}}),{c(){h=l("p"),$=a("Once this is done, we can define our "),m=l("code"),x=a("Seq2SeqTrainingArguments"),P=a(". Like for the "),g=l("code"),D=a("Trainer"),A=a(", we use a subclass of "),T=l("code"),z=a("TrainingArguments"),N=a(" that contains a few more fields:"),q=c(),v(F.$$.fragment),W=c(),U=l("p"),L=a("Apart from the usual hyperparameters (like learning rate, number of epochs, batch size, and some weight decay), here are a few changes compared to what we saw in the previous sections:"),G=c(),H=l("ul"),C=l("li"),K=a("We don\u2019t set any regular evaluation, as evaluation takes a while; we will just evaluate our model once before training and after."),B=c(),Y=l("li"),re=a("We set "),V=l("code"),Q=a("fp16=True"),ee=a(", which speeds up training on modern GPUs."),Z=c(),oe=l("li"),$e=a("We set "),se=l("code"),ae=a("predict_with_generate=True"),fe=a(", as discussed above."),ke=c(),le=l("li"),he=a("We use "),M=l("code"),ne=a("push_to_hub=True"),ze=a(" to upload the model to the Hub at the end of each epoch."),me=c(),J=l("p"),ye=a("Note that you can specify the full name of the repository you want to push to with the "),ie=l("code"),xe=a("hub_model_id"),Me=a(" argument (in particular, you will have to use this argument to push to an organization). For instance, when we pushed the model to the "),_e=l("a"),qe=l("code"),te=a("huggingface-course"),Pe=a(" organization"),ge=a(", we added "),at=l("code"),ft=a('hub_model_id="huggingface-course/marian-finetuned-kde4-en-to-fr"'),ve=a(" to "),Ge=l("code"),Ue=a("Seq2SeqTrainingArguments"),Se=a(". By default, the repository used will be in your namespace and named after the output directory you set, so in our case it will be "),je=l("code"),mt=a('"sgugger/marian-finetuned-kde4-en-to-fr"'),we=a(" (which is the model we linked to at the beginning of this section)."),k=c(),v(X.$$.fragment),Ze=c(),ce=l("p"),Be=a("Finally, we just pass everything to the "),nt=l("code"),Ce=a("Seq2SeqTrainer"),pe=a(":"),Ft=c(),v(Re.$$.fragment),Ie=c(),Ne=l("p"),_t=a("Before training, we\u2019ll first look at the score our model gets, to double-check that we\u2019re not making things worse with our fine-tuning. This command will take a bit of time, so you can grab a coffee while it executes:"),de=c(),v(Je.$$.fragment),Oe=c(),v(zt.$$.fragment),is=c(),ot=l("p"),lt=a("A BLEU score of 39 is not too bad, which reflects the fact that our model is already good at translating English sentences to French ones."),rt=c(),Ve=l("p"),Dt=a("Next is the training, which will also take a bit of time:"),ps=c(),v(Qe.$$.fragment),gt=c(),wt=l("p"),Ee=a("Note that while the training happens, each time the model is saved (here, every epoch) it is uploaded to the Hub in the background. This way, you will be able to to resume your training on another machine if necessary."),Xt=c(),Fe=l("p"),$s=a("Once training is done, we evaluate our model again \u2014 hopefully we will see some amelioration in the BLEU score!"),Gt=c(),v(De.$$.fragment),Zt=c(),v(Ae.$$.fragment),Lt=c(),bt=l("p"),ks=a("That\u2019s a nearly 14-point improvement, which is great."),Jt=c(),be=l("p"),$t=a("Finally, we use the "),At=l("code"),kt=a("push_to_hub()"),it=a(" method to make sure we upload the latest version of the model. The "),He=l("code"),Vt=a("Trainer"),vt=a(" also drafts a model card with all the evaluation results and uploads it. This model card contains metadata that helps the Model Hub pick the widget for the inference demo. Usually, there is no need to say anything as it can infer the right widget from the model class, but in this case, the same model class can be used for all kinds of sequence-to-sequence problems, so we specify it\u2019s a translation model:"),pt=c(),v(ht.$$.fragment),It=c(),Pt=l("p"),et=a("This command returns the URL of the commit it just did, if you want to inspect it:"),Qt=c(),v(Le.$$.fragment),es=c(),Ye=l("p"),ts=a("At this stage, you can use the inference widget on the Model Hub to test your model and share it with your friends. You have successfully fine-tuned a model on a translation task \u2014 congratulations!"),yt=c(),Nt=l("p"),hs=a("If you want to dive a bit more deeply into the training loop, we will now show you how to do the same thing using \u{1F917} Accelerate."),this.h()},l(f){h=r(f,"P",{});var O=i(h);$=n(O,"Once this is done, we can define our "),m=r(O,"CODE",{});var vs=i(m);x=n(vs,"Seq2SeqTrainingArguments"),vs.forEach(t),P=n(O,". Like for the "),g=r(O,"CODE",{});var jt=i(g);D=n(jt,"Trainer"),jt.forEach(t),A=n(O,", we use a subclass of "),T=r(O,"CODE",{});var Xs=i(T);z=n(Xs,"TrainingArguments"),Xs.forEach(t),N=n(O," that contains a few more fields:"),O.forEach(t),q=d(f),y(F.$$.fragment,f),W=d(f),U=r(f,"P",{});var ys=i(U);L=n(ys,"Apart from the usual hyperparameters (like learning rate, number of epochs, batch size, and some weight decay), here are a few changes compared to what we saw in the previous sections:"),ys.forEach(t),G=d(f),H=r(f,"UL",{});var Te=i(H);C=r(Te,"LI",{});var js=i(C);K=n(js,"We don\u2019t set any regular evaluation, as evaluation takes a while; we will just evaluate our model once before training and after."),js.forEach(t),B=d(Te),Y=r(Te,"LI",{});var Et=i(Y);re=n(Et,"We set "),V=r(Et,"CODE",{});var cs=i(V);Q=n(cs,"fp16=True"),cs.forEach(t),ee=n(Et,", which speeds up training on modern GPUs."),Et.forEach(t),Z=d(Te),oe=r(Te,"LI",{});var ue=i(oe);$e=n(ue,"We set "),se=r(ue,"CODE",{});var Gs=i(se);ae=n(Gs,"predict_with_generate=True"),Gs.forEach(t),fe=n(ue,", as discussed above."),ue.forEach(t),ke=d(Te),le=r(Te,"LI",{});var dt=i(le);he=n(dt,"We use "),M=r(dt,"CODE",{});var Zs=i(M);ne=n(Zs,"push_to_hub=True"),Zs.forEach(t),ze=n(dt," to upload the model to the Hub at the end of each epoch."),dt.forEach(t),Te.forEach(t),me=d(f),J=r(f,"P",{});var We=i(J);ye=n(We,"Note that you can specify the full name of the repository you want to push to with the "),ie=r(We,"CODE",{});var Ht=i(ie);xe=n(Ht,"hub_model_id"),Ht.forEach(t),Me=n(We," argument (in particular, you will have to use this argument to push to an organization). For instance, when we pushed the model to the "),_e=r(We,"A",{href:!0,rel:!0});var Es=i(_e);qe=r(Es,"CODE",{});var Js=i(qe);te=n(Js,"huggingface-course"),Js.forEach(t),Pe=n(Es," organization"),Es.forEach(t),ge=n(We,", we added "),at=r(We,"CODE",{});var ss=i(at);ft=n(ss,'hub_model_id="huggingface-course/marian-finetuned-kde4-en-to-fr"'),ss.forEach(t),ve=n(We," to "),Ge=r(We,"CODE",{});var Ke=i(Ge);Ue=n(Ke,"Seq2SeqTrainingArguments"),Ke.forEach(t),Se=n(We,". By default, the repository used will be in your namespace and named after the output directory you set, so in our case it will be "),je=r(We,"CODE",{});var Vs=i(je);mt=n(Vs,'"sgugger/marian-finetuned-kde4-en-to-fr"'),Vs.forEach(t),we=n(We," (which is the model we linked to at the beginning of this section)."),We.forEach(t),k=d(f),y(X.$$.fragment,f),Ze=d(f),ce=r(f,"P",{});var xt=i(ce);Be=n(xt,"Finally, we just pass everything to the "),nt=r(xt,"CODE",{});var Wt=i(nt);Ce=n(Wt,"Seq2SeqTrainer"),Wt.forEach(t),pe=n(xt,":"),xt.forEach(t),Ft=d(f),y(Re.$$.fragment,f),Ie=d(f),Ne=r(f,"P",{});var xs=i(Ne);_t=n(xs,"Before training, we\u2019ll first look at the score our model gets, to double-check that we\u2019re not making things worse with our fine-tuning. This command will take a bit of time, so you can grab a coffee while it executes:"),xs.forEach(t),de=d(f),y(Je.$$.fragment,f),Oe=d(f),y(zt.$$.fragment,f),is=d(f),ot=r(f,"P",{});var qt=i(ot);lt=n(qt,"A BLEU score of 39 is not too bad, which reflects the fact that our model is already good at translating English sentences to French ones."),qt.forEach(t),rt=d(f),Ve=r(f,"P",{});var qs=i(Ve);Dt=n(qs,"Next is the training, which will also take a bit of time:"),qs.forEach(t),ps=d(f),y(Qe.$$.fragment,f),gt=d(f),wt=r(f,"P",{});var ut=i(wt);Ee=n(ut,"Note that while the training happens, each time the model is saved (here, every epoch) it is uploaded to the Hub in the background. This way, you will be able to to resume your training on another machine if necessary."),ut.forEach(t),Xt=d(f),Fe=r(f,"P",{});var ds=i(Fe);$s=n(ds,"Once training is done, we evaluate our model again \u2014 hopefully we will see some amelioration in the BLEU score!"),ds.forEach(t),Gt=d(f),y(De.$$.fragment,f),Zt=d(f),y(Ae.$$.fragment,f),Lt=d(f),bt=r(f,"P",{});var us=i(bt);ks=n(us,"That\u2019s a nearly 14-point improvement, which is great."),us.forEach(t),Jt=d(f),be=r(f,"P",{});var Mt=i(be);$t=n(Mt,"Finally, we use the "),At=r(Mt,"CODE",{});var Ts=i(At);kt=n(Ts,"push_to_hub()"),Ts.forEach(t),it=n(Mt," method to make sure we upload the latest version of the model. The "),He=r(Mt,"CODE",{});var Tt=i(He);Vt=n(Tt,"Trainer"),Tt.forEach(t),vt=n(Mt," also drafts a model card with all the evaluation results and uploads it. This model card contains metadata that helps the Model Hub pick the widget for the inference demo. Usually, there is no need to say anything as it can infer the right widget from the model class, but in this case, the same model class can be used for all kinds of sequence-to-sequence problems, so we specify it\u2019s a translation model:"),Mt.forEach(t),pt=d(f),y(ht.$$.fragment,f),It=d(f),Pt=r(f,"P",{});var zs=i(Pt);et=n(zs,"This command returns the URL of the commit it just did, if you want to inspect it:"),zs.forEach(t),Qt=d(f),y(Le.$$.fragment,f),es=d(f),Ye=r(f,"P",{});var Ut=i(Ye);ts=n(Ut,"At this stage, you can use the inference widget on the Model Hub to test your model and share it with your friends. You have successfully fine-tuned a model on a translation task \u2014 congratulations!"),Ut.forEach(t),yt=d(f),Nt=r(f,"P",{});var fs=i(Nt);hs=n(fs,"If you want to dive a bit more deeply into the training loop, we will now show you how to do the same thing using \u{1F917} Accelerate."),fs.forEach(t),this.h()},h(){_(_e,"href","https://huggingface.co/huggingface-course"),_(_e,"rel","nofollow")},m(f,O){o(f,h,O),s(h,$),s(h,m),s(m,x),s(h,P),s(h,g),s(g,D),s(h,A),s(h,T),s(T,z),s(h,N),o(f,q,O),j(F,f,O),o(f,W,O),o(f,U,O),s(U,L),o(f,G,O),o(f,H,O),s(H,C),s(C,K),s(H,B),s(H,Y),s(Y,re),s(Y,V),s(V,Q),s(Y,ee),s(H,Z),s(H,oe),s(oe,$e),s(oe,se),s(se,ae),s(oe,fe),s(H,ke),s(H,le),s(le,he),s(le,M),s(M,ne),s(le,ze),o(f,me,O),o(f,J,O),s(J,ye),s(J,ie),s(ie,xe),s(J,Me),s(J,_e),s(_e,qe),s(qe,te),s(_e,Pe),s(J,ge),s(J,at),s(at,ft),s(J,ve),s(J,Ge),s(Ge,Ue),s(J,Se),s(J,je),s(je,mt),s(J,we),o(f,k,O),j(X,f,O),o(f,Ze,O),o(f,ce,O),s(ce,Be),s(ce,nt),s(nt,Ce),s(ce,pe),o(f,Ft,O),j(Re,f,O),o(f,Ie,O),o(f,Ne,O),s(Ne,_t),o(f,de,O),j(Je,f,O),o(f,Oe,O),j(zt,f,O),o(f,is,O),o(f,ot,O),s(ot,lt),o(f,rt,O),o(f,Ve,O),s(Ve,Dt),o(f,ps,O),j(Qe,f,O),o(f,gt,O),o(f,wt,O),s(wt,Ee),o(f,Xt,O),o(f,Fe,O),s(Fe,$s),o(f,Gt,O),j(De,f,O),o(f,Zt,O),j(Ae,f,O),o(f,Lt,O),o(f,bt,O),s(bt,ks),o(f,Jt,O),o(f,be,O),s(be,$t),s(be,At),s(At,kt),s(be,it),s(be,He),s(He,Vt),s(be,vt),o(f,pt,O),j(ht,f,O),o(f,It,O),o(f,Pt,O),s(Pt,et),o(f,Qt,O),j(Le,f,O),o(f,es,O),o(f,Ye,O),s(Ye,ts),o(f,yt,O),o(f,Nt,O),s(Nt,hs),ct=!0},i(f){ct||(w(F.$$.fragment,f),w(X.$$.fragment,f),w(Re.$$.fragment,f),w(Je.$$.fragment,f),w(zt.$$.fragment,f),w(Qe.$$.fragment,f),w(De.$$.fragment,f),w(Ae.$$.fragment,f),w(ht.$$.fragment,f),w(Le.$$.fragment,f),ct=!0)},o(f){b(F.$$.fragment,f),b(X.$$.fragment,f),b(Re.$$.fragment,f),b(Je.$$.fragment,f),b(zt.$$.fragment,f),b(Qe.$$.fragment,f),b(De.$$.fragment,f),b(Ae.$$.fragment,f),b(ht.$$.fragment,f),b(Le.$$.fragment,f),ct=!1},d(f){f&&t(h),f&&t(q),E(F,f),f&&t(W),f&&t(U),f&&t(G),f&&t(H),f&&t(me),f&&t(J),f&&t(k),E(X,f),f&&t(Ze),f&&t(ce),f&&t(Ft),E(Re,f),f&&t(Ie),f&&t(Ne),f&&t(de),E(Je,f),f&&t(Oe),E(zt,f),f&&t(is),f&&t(ot),f&&t(rt),f&&t(Ve),f&&t(ps),E(Qe,f),f&&t(gt),f&&t(wt),f&&t(Xt),f&&t(Fe),f&&t(Gt),E(De,f),f&&t(Zt),E(Ae,f),f&&t(Lt),f&&t(bt),f&&t(Jt),f&&t(be),f&&t(pt),E(ht,f),f&&t(It),f&&t(Pt),f&&t(Qt),E(Le,f),f&&t(es),f&&t(Ye),f&&t(yt),f&&t(Nt)}}}function pd(R){let h,$,m,x,P,g,D,A,T,z,N,q,F,W,U,L,G,H,C,K,B,Y,re,V,Q,ee,Z,oe,$e,se,ae,fe,ke,le,he,M,ne,ze,me,J,ye,ie,xe,Me,_e,qe,te,Pe,ge,at,ft,ve,Ge,Ue,Se,je,mt,we;return x=new S({props:{code:"print(compute_metrics())",highlighted:'<span class="hljs-built_in">print</span>(compute_metrics())'}}),g=new S({props:{code:"{'bleu': 33.26983701454733}",highlighted:'{&#x27;bleu&#x27;: <span class="hljs-number">33.26983701454733</span>}'}}),W=new S({props:{code:`from transformers import create_optimizer
from transformers.keras_callbacks import PushToHubCallback
import tensorflow as tf

# The number of training steps is the number of samples in the dataset, divided by the batch size then multiplied
# by the total number of epochs. Note that the tf_train_dataset here is a batched tf.data.Dataset,
# not the original Hugging Face Dataset, so its len() is already num_samples // batch_size.
num_epochs = 3
num_train_steps = len(tf_train_dataset) * num_epochs

optimizer, schedule = create_optimizer(
    init_lr=5e-5,
    num_warmup_steps=0,
    num_train_steps=num_train_steps,
    weight_decay_rate=0.01,
)
model.compile(optimizer=optimizer)

# Train in mixed-precision float16
tf.keras.mixed_precision.set_global_policy("mixed_float16")`,highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> create_optimizer
<span class="hljs-keyword">from</span> transformers.keras_callbacks <span class="hljs-keyword">import</span> PushToHubCallback
<span class="hljs-keyword">import</span> tensorflow <span class="hljs-keyword">as</span> tf

<span class="hljs-comment"># The number of training steps is the number of samples in the dataset, divided by the batch size then multiplied</span>
<span class="hljs-comment"># by the total number of epochs. Note that the tf_train_dataset here is a batched tf.data.Dataset,</span>
<span class="hljs-comment"># not the original Hugging Face Dataset, so its len() is already num_samples // batch_size.</span>
num_epochs = <span class="hljs-number">3</span>
num_train_steps = <span class="hljs-built_in">len</span>(tf_train_dataset) * num_epochs

optimizer, schedule = create_optimizer(
    init_lr=<span class="hljs-number">5e-5</span>,
    num_warmup_steps=<span class="hljs-number">0</span>,
    num_train_steps=num_train_steps,
    weight_decay_rate=<span class="hljs-number">0.01</span>,
)
model.<span class="hljs-built_in">compile</span>(optimizer=optimizer)

<span class="hljs-comment"># Train in mixed-precision float16</span>
tf.keras.mixed_precision.set_global_policy(<span class="hljs-string">&quot;mixed_float16&quot;</span>)`}}),Q=new S({props:{code:`from transformers.keras_callbacks import PushToHubCallback

callback = PushToHubCallback(
    output_dir="marian-finetuned-kde4-en-to-fr", tokenizer=tokenizer
)

model.fit(
    tf_train_dataset,
    validation_data=tf_eval_dataset,
    callbacks=[callback],
    epochs=num_epochs,
)`,highlighted:`<span class="hljs-keyword">from</span> transformers.keras_callbacks <span class="hljs-keyword">import</span> PushToHubCallback

callback = PushToHubCallback(
    output_dir=<span class="hljs-string">&quot;marian-finetuned-kde4-en-to-fr&quot;</span>, tokenizer=tokenizer
)

model.fit(
    tf_train_dataset,
    validation_data=tf_eval_dataset,
    callbacks=[callback],
    epochs=num_epochs,
)`}}),te=new Ka({props:{$$slots:{default:[cd]},$$scope:{ctx:R}}}),ve=new S({props:{code:"print(compute_metrics())",highlighted:'<span class="hljs-built_in">print</span>(compute_metrics())'}}),Ue=new S({props:{code:"{'bleu': 57.334066271545865}",highlighted:'{&#x27;bleu&#x27;: <span class="hljs-number">57.334066271545865</span>}'}}),{c(){h=l("p"),$=a("Before we start, let\u2019s see what kind of results we get from our model without any training:"),m=c(),v(x.$$.fragment),P=c(),v(g.$$.fragment),D=c(),A=l("p"),T=a("Once this is done, we can prepare everything we need to compile and train our model. Note the use of "),z=l("code"),N=a('tf.keras.mixed_precision.set_global_policy("mixed_float16")'),q=a(" \u2014 this will tell Keras to train using float16, which can give a significant speedup on GPUs that support it (Nvidia 20xx/V100 or newer)."),F=c(),v(W.$$.fragment),U=c(),L=l("p"),G=a("Next, we define a "),H=l("code"),C=a("PushToHubCallback"),K=a(" to upload our model to the Hub during training, as we saw in "),B=l("a"),Y=a("section 2"),re=a(", and then we simply fit the model with that callback:"),V=c(),v(Q.$$.fragment),ee=c(),Z=l("p"),oe=a("Note that you can specify the name of the repository you want to push to with the "),$e=l("code"),se=a("hub_model_id"),ae=a(" argument (in particular, you will have to use this argument to push to an organization). For instance, when we pushed the model to the "),fe=l("a"),ke=l("code"),le=a("huggingface-course"),he=a(" organization"),M=a(", we added "),ne=l("code"),ze=a('hub_model_id="huggingface-course/marian-finetuned-kde4-en-to-fr"'),me=a(" to "),J=l("code"),ye=a("Seq2SeqTrainingArguments"),ie=a(". By default, the repository used will be in your namespace and named after the output directory you set, so here it will be "),xe=l("code"),Me=a('"sgugger/marian-finetuned-kde4-en-to-fr"'),_e=a(" (which is the model we linked to at the beginning of this section)."),qe=c(),v(te.$$.fragment),Pe=c(),ge=l("p"),at=a("Finally, let\u2019s see what our metrics look like now that training has finished:"),ft=c(),v(ve.$$.fragment),Ge=c(),v(Ue.$$.fragment),Se=c(),je=l("p"),mt=a("At this stage, you can use the inference widget on the Model Hub to test your model and share it with your friends. You have successfully fine-tuned a model on a translation task \u2014 congratulations!"),this.h()},l(k){h=r(k,"P",{});var X=i(h);$=n(X,"Before we start, let\u2019s see what kind of results we get from our model without any training:"),X.forEach(t),m=d(k),y(x.$$.fragment,k),P=d(k),y(g.$$.fragment,k),D=d(k),A=r(k,"P",{});var Ze=i(A);T=n(Ze,"Once this is done, we can prepare everything we need to compile and train our model. Note the use of "),z=r(Ze,"CODE",{});var ce=i(z);N=n(ce,'tf.keras.mixed_precision.set_global_policy("mixed_float16")'),ce.forEach(t),q=n(Ze," \u2014 this will tell Keras to train using float16, which can give a significant speedup on GPUs that support it (Nvidia 20xx/V100 or newer)."),Ze.forEach(t),F=d(k),y(W.$$.fragment,k),U=d(k),L=r(k,"P",{});var Be=i(L);G=n(Be,"Next, we define a "),H=r(Be,"CODE",{});var nt=i(H);C=n(nt,"PushToHubCallback"),nt.forEach(t),K=n(Be," to upload our model to the Hub during training, as we saw in "),B=r(Be,"A",{href:!0});var Ce=i(B);Y=n(Ce,"section 2"),Ce.forEach(t),re=n(Be,", and then we simply fit the model with that callback:"),Be.forEach(t),V=d(k),y(Q.$$.fragment,k),ee=d(k),Z=r(k,"P",{});var pe=i(Z);oe=n(pe,"Note that you can specify the name of the repository you want to push to with the "),$e=r(pe,"CODE",{});var Ft=i($e);se=n(Ft,"hub_model_id"),Ft.forEach(t),ae=n(pe," argument (in particular, you will have to use this argument to push to an organization). For instance, when we pushed the model to the "),fe=r(pe,"A",{href:!0,rel:!0});var Re=i(fe);ke=r(Re,"CODE",{});var Ie=i(ke);le=n(Ie,"huggingface-course"),Ie.forEach(t),he=n(Re," organization"),Re.forEach(t),M=n(pe,", we added "),ne=r(pe,"CODE",{});var Ne=i(ne);ze=n(Ne,'hub_model_id="huggingface-course/marian-finetuned-kde4-en-to-fr"'),Ne.forEach(t),me=n(pe," to "),J=r(pe,"CODE",{});var _t=i(J);ye=n(_t,"Seq2SeqTrainingArguments"),_t.forEach(t),ie=n(pe,". By default, the repository used will be in your namespace and named after the output directory you set, so here it will be "),xe=r(pe,"CODE",{});var de=i(xe);Me=n(de,'"sgugger/marian-finetuned-kde4-en-to-fr"'),de.forEach(t),_e=n(pe," (which is the model we linked to at the beginning of this section)."),pe.forEach(t),qe=d(k),y(te.$$.fragment,k),Pe=d(k),ge=r(k,"P",{});var Je=i(ge);at=n(Je,"Finally, let\u2019s see what our metrics look like now that training has finished:"),Je.forEach(t),ft=d(k),y(ve.$$.fragment,k),Ge=d(k),y(Ue.$$.fragment,k),Se=d(k),je=r(k,"P",{});var Oe=i(je);mt=n(Oe,"At this stage, you can use the inference widget on the Model Hub to test your model and share it with your friends. You have successfully fine-tuned a model on a translation task \u2014 congratulations!"),Oe.forEach(t),this.h()},h(){_(B,"href","(/course/chapter7/2)"),_(fe,"href","https://huggingface.co/huggingface-course"),_(fe,"rel","nofollow")},m(k,X){o(k,h,X),s(h,$),o(k,m,X),j(x,k,X),o(k,P,X),j(g,k,X),o(k,D,X),o(k,A,X),s(A,T),s(A,z),s(z,N),s(A,q),o(k,F,X),j(W,k,X),o(k,U,X),o(k,L,X),s(L,G),s(L,H),s(H,C),s(L,K),s(L,B),s(B,Y),s(L,re),o(k,V,X),j(Q,k,X),o(k,ee,X),o(k,Z,X),s(Z,oe),s(Z,$e),s($e,se),s(Z,ae),s(Z,fe),s(fe,ke),s(ke,le),s(fe,he),s(Z,M),s(Z,ne),s(ne,ze),s(Z,me),s(Z,J),s(J,ye),s(Z,ie),s(Z,xe),s(xe,Me),s(Z,_e),o(k,qe,X),j(te,k,X),o(k,Pe,X),o(k,ge,X),s(ge,at),o(k,ft,X),j(ve,k,X),o(k,Ge,X),j(Ue,k,X),o(k,Se,X),o(k,je,X),s(je,mt),we=!0},i(k){we||(w(x.$$.fragment,k),w(g.$$.fragment,k),w(W.$$.fragment,k),w(Q.$$.fragment,k),w(te.$$.fragment,k),w(ve.$$.fragment,k),w(Ue.$$.fragment,k),we=!0)},o(k){b(x.$$.fragment,k),b(g.$$.fragment,k),b(W.$$.fragment,k),b(Q.$$.fragment,k),b(te.$$.fragment,k),b(ve.$$.fragment,k),b(Ue.$$.fragment,k),we=!1},d(k){k&&t(h),k&&t(m),E(x,k),k&&t(P),E(g,k),k&&t(D),k&&t(A),k&&t(F),E(W,k),k&&t(U),k&&t(L),k&&t(V),E(Q,k),k&&t(ee),k&&t(Z),k&&t(qe),E(te,k),k&&t(Pe),k&&t(ge),k&&t(ft),E(ve,k),k&&t(Ge),E(Ue,k),k&&t(Se),k&&t(je)}}}function hd(R){let h,$,m,x,P;return{c(){h=l("p"),$=a("\u{1F4A1} If the output directory you are using already exists, it needs to be a local clone of the repository you want to push to. If it isn\u2019t, you\u2019ll get an error when defining your "),m=l("code"),x=a("Seq2SeqTrainer"),P=a(" and will need to set a new name.")},l(g){h=r(g,"P",{});var D=i(h);$=n(D,"\u{1F4A1} If the output directory you are using already exists, it needs to be a local clone of the repository you want to push to. If it isn\u2019t, you\u2019ll get an error when defining your "),m=r(D,"CODE",{});var A=i(m);x=n(A,"Seq2SeqTrainer"),A.forEach(t),P=n(D," and will need to set a new name."),D.forEach(t)},m(g,D){o(g,h,D),s(h,$),s(h,m),s(m,x),s(h,P)},d(g){g&&t(h)}}}function cd(R){let h,$,m,x,P;return{c(){h=l("p"),$=a("\u{1F4A1} If the output directory you are using already exists, it needs to be a local clone of the repository you want to push to. If it isn\u2019t, you\u2019ll get an error when calling "),m=l("code"),x=a("model.fit()"),P=a(" and will need to set a new name.")},l(g){h=r(g,"P",{});var D=i(h);$=n(D,"\u{1F4A1} If the output directory you are using already exists, it needs to be a local clone of the repository you want to push to. If it isn\u2019t, you\u2019ll get an error when calling "),m=r(D,"CODE",{});var A=i(m);x=n(A,"model.fit()"),A.forEach(t),P=n(D," and will need to set a new name."),D.forEach(t)},m(g,D){o(g,h,D),s(h,$),s(h,m),s(m,x),s(h,P)},d(g){g&&t(h)}}}function Nc(R){let h,$,m,x,P,g,D,A,T,z,N,q,F,W,U,L,G,H,C,K,B,Y,re,V,Q,ee,Z,oe,$e,se,ae,fe,ke,le,he,M,ne,ze,me,J,ye,ie,xe,Me,_e,qe,te,Pe,ge,at,ft,ve,Ge,Ue,Se,je,mt,we,k,X,Ze,ce,Be,nt,Ce,pe,Ft,Re,Ie,Ne,_t,de,Je,Oe,zt,is,ot,lt,rt,Ve,Dt,ps,Qe,gt,wt,Ee,Xt,Fe,$s,Gt,De,Zt,Ae,Lt,bt,ks,Jt,be,$t,At,kt,it,He,Vt,vt,pt,ht,It,Pt,et,Qt,Le,es,Ye,ts,yt,Nt,hs,ct,f,O,vs,jt,Xs,ys,Te,js,Et,cs,ue,Gs,dt,Zs,We,Ht,Es,Js,ss,Ke,Vs,xt,Wt,xs,qt,qs,ut,ds,us,Mt,Ts,Tt,zs,Ut,fs,tt,Ea,ms,Xa,xa,St,Qs,Ds,qa,As;return x=new rs({}),B=new rs({}),he=new S({props:{code:`from torch.utils.data import DataLoader

tokenized_datasets.set_format("torch")
train_dataloader = DataLoader(
    tokenized_datasets["train"],
    shuffle=True,
    collate_fn=data_collator,
    batch_size=8,
)
eval_dataloader = DataLoader(
    tokenized_datasets["validation"], collate_fn=data_collator, batch_size=8
)`,highlighted:`<span class="hljs-keyword">from</span> torch.utils.data <span class="hljs-keyword">import</span> DataLoader

tokenized_datasets.set_format(<span class="hljs-string">&quot;torch&quot;</span>)
train_dataloader = DataLoader(
    tokenized_datasets[<span class="hljs-string">&quot;train&quot;</span>],
    shuffle=<span class="hljs-literal">True</span>,
    collate_fn=data_collator,
    batch_size=<span class="hljs-number">8</span>,
)
eval_dataloader = DataLoader(
    tokenized_datasets[<span class="hljs-string">&quot;validation&quot;</span>], collate_fn=data_collator, batch_size=<span class="hljs-number">8</span>
)`}}),J=new S({props:{code:"model = AutoModelForSeq2SeqLM.from_pretrained(model_checkpoint)",highlighted:"model = AutoModelForSeq2SeqLM.from_pretrained(model_checkpoint)"}}),_e=new S({props:{code:`from transformers import AdamW

optimizer = AdamW(model.parameters(), lr=2e-5)`,highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AdamW

optimizer = AdamW(model.parameters(), lr=<span class="hljs-number">2e-5</span>)`}}),je=new S({props:{code:`from accelerate import Accelerator

accelerator = Accelerator()
model, optimizer, train_dataloader, eval_dataloader = accelerator.prepare(
    model, optimizer, train_dataloader, eval_dataloader
)`,highlighted:`<span class="hljs-keyword">from</span> accelerate <span class="hljs-keyword">import</span> Accelerator

accelerator = Accelerator()
model, optimizer, train_dataloader, eval_dataloader = accelerator.prepare(
    model, optimizer, train_dataloader, eval_dataloader
)`}}),Ne=new S({props:{code:`from transformers import get_scheduler

num_train_epochs = 3
num_update_steps_per_epoch = len(train_dataloader)
num_training_steps = num_train_epochs * num_update_steps_per_epoch

lr_scheduler = get_scheduler(
    "linear",
    optimizer=optimizer,
    num_warmup_steps=0,
    num_training_steps=num_training_steps,
)`,highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> get_scheduler

num_train_epochs = <span class="hljs-number">3</span>
num_update_steps_per_epoch = <span class="hljs-built_in">len</span>(train_dataloader)
num_training_steps = num_train_epochs * num_update_steps_per_epoch

lr_scheduler = get_scheduler(
    <span class="hljs-string">&quot;linear&quot;</span>,
    optimizer=optimizer,
    num_warmup_steps=<span class="hljs-number">0</span>,
    num_training_steps=num_training_steps,
)`}}),gt=new S({props:{code:`from huggingface_hub import Repository, get_full_repo_name

model_name = "marian-finetuned-kde4-en-to-fr-accelerate"
repo_name = get_full_repo_name(model_name)
repo_name`,highlighted:`<span class="hljs-keyword">from</span> huggingface_hub <span class="hljs-keyword">import</span> Repository, get_full_repo_name

model_name = <span class="hljs-string">&quot;marian-finetuned-kde4-en-to-fr-accelerate&quot;</span>
repo_name = get_full_repo_name(model_name)
repo_name`}}),Ee=new S({props:{code:"'sgugger/marian-finetuned-kde4-en-to-fr-accelerate'",highlighted:'<span class="hljs-string">&#x27;sgugger/marian-finetuned-kde4-en-to-fr-accelerate&#x27;</span>'}}),De=new S({props:{code:`output_dir = "marian-finetuned-kde4-en-to-fr-accelerate"
repo = Repository(output_dir, clone_from=repo_name)`,highlighted:`output_dir = <span class="hljs-string">&quot;marian-finetuned-kde4-en-to-fr-accelerate&quot;</span>
repo = Repository(output_dir, clone_from=repo_name)`}}),vt=new rs({}),ct=new S({props:{code:`def postprocess(predictions, labels):
    predictions = predictions.cpu().numpy()
    labels = labels.cpu().numpy()

    decoded_preds = tokenizer.batch_decode(predictions, skip_special_tokens=True)

    # Replace -100 in the labels as we can't decode them.
    labels = np.where(labels != -100, labels, tokenizer.pad_token_id)
    decoded_labels = tokenizer.batch_decode(labels, skip_special_tokens=True)

    # Some simple post-processing
    decoded_preds = [pred.strip() for pred in decoded_preds]
    decoded_labels = [[label.strip()] for label in decoded_labels]
    return decoded_preds, decoded_labels`,highlighted:`<span class="hljs-keyword">def</span> <span class="hljs-title function_">postprocess</span>(<span class="hljs-params">predictions, labels</span>):
    predictions = predictions.cpu().numpy()
    labels = labels.cpu().numpy()

    decoded_preds = tokenizer.batch_decode(predictions, skip_special_tokens=<span class="hljs-literal">True</span>)

    <span class="hljs-comment"># Replace -100 in the labels as we can&#x27;t decode them.</span>
    labels = np.where(labels != -<span class="hljs-number">100</span>, labels, tokenizer.pad_token_id)
    decoded_labels = tokenizer.batch_decode(labels, skip_special_tokens=<span class="hljs-literal">True</span>)

    <span class="hljs-comment"># Some simple post-processing</span>
    decoded_preds = [pred.strip() <span class="hljs-keyword">for</span> pred <span class="hljs-keyword">in</span> decoded_preds]
    decoded_labels = [[label.strip()] <span class="hljs-keyword">for</span> label <span class="hljs-keyword">in</span> decoded_labels]
    <span class="hljs-keyword">return</span> decoded_preds, decoded_labels`}}),Tt=new S({props:{code:`from tqdm.auto import tqdm
import torch

progress_bar = tqdm(range(num_training_steps))

for epoch in range(num_train_epochs):
    # Training
    model.train()
    for batch in train_dataloader:
        outputs = model(**batch)
        loss = outputs.loss
        accelerator.backward(loss)

        optimizer.step()
        lr_scheduler.step()
        optimizer.zero_grad()
        progress_bar.update(1)

    # Evaluation
    model.eval()
    for batch in tqdm(eval_dataloader):
        with torch.no_grad():
            generated_tokens = accelerator.unwrap_model(model).generate(
                batch["input_ids"],
                attention_mask=batch["attention_mask"],
                max_length=128,
            )
        labels = batch["labels"]

        # Necessary to pad predictions and labels for being gathered
        generated_tokens = accelerator.pad_across_processes(
            generated_tokens, dim=1, pad_index=tokenizer.pad_token_id
        )
        labels = accelerator.pad_across_processes(labels, dim=1, pad_index=-100)

        predictions_gathered = accelerator.gather(generated_tokens)
        labels_gathered = accelerator.gather(labels)

        decoded_preds, decoded_labels = postprocess(predictions_gathered, labels_gathered)
        metric.add_batch(predictions=decoded_preds, references=decoded_labels)

    results = metric.compute()
    print(f"epoch {epoch}, BLEU score: {results['score']:.2f}")

    # Save and upload
    accelerator.wait_for_everyone()
    unwrapped_model = accelerator.unwrap_model(model)
    unwrapped_model.save_pretrained(output_dir, save_function=accelerator.save)
    if accelerator.is_main_process:
        tokenizer.save_pretrained(output_dir)
        repo.push_to_hub(
            commit_message=f"Training in progress epoch {epoch}", blocking=False
        )`,highlighted:`<span class="hljs-keyword">from</span> tqdm.auto <span class="hljs-keyword">import</span> tqdm
<span class="hljs-keyword">import</span> torch

progress_bar = tqdm(<span class="hljs-built_in">range</span>(num_training_steps))

<span class="hljs-keyword">for</span> epoch <span class="hljs-keyword">in</span> <span class="hljs-built_in">range</span>(num_train_epochs):
    <span class="hljs-comment"># Training</span>
    model.train()
    <span class="hljs-keyword">for</span> batch <span class="hljs-keyword">in</span> train_dataloader:
        outputs = model(**batch)
        loss = outputs.loss
        accelerator.backward(loss)

        optimizer.step()
        lr_scheduler.step()
        optimizer.zero_grad()
        progress_bar.update(<span class="hljs-number">1</span>)

    <span class="hljs-comment"># Evaluation</span>
    model.<span class="hljs-built_in">eval</span>()
    <span class="hljs-keyword">for</span> batch <span class="hljs-keyword">in</span> tqdm(eval_dataloader):
        <span class="hljs-keyword">with</span> torch.no_grad():
            generated_tokens = accelerator.unwrap_model(model).generate(
                batch[<span class="hljs-string">&quot;input_ids&quot;</span>],
                attention_mask=batch[<span class="hljs-string">&quot;attention_mask&quot;</span>],
                max_length=<span class="hljs-number">128</span>,
            )
        labels = batch[<span class="hljs-string">&quot;labels&quot;</span>]

        <span class="hljs-comment"># Necessary to pad predictions and labels for being gathered</span>
        generated_tokens = accelerator.pad_across_processes(
            generated_tokens, dim=<span class="hljs-number">1</span>, pad_index=tokenizer.pad_token_id
        )
        labels = accelerator.pad_across_processes(labels, dim=<span class="hljs-number">1</span>, pad_index=-<span class="hljs-number">100</span>)

        predictions_gathered = accelerator.gather(generated_tokens)
        labels_gathered = accelerator.gather(labels)

        decoded_preds, decoded_labels = postprocess(predictions_gathered, labels_gathered)
        metric.add_batch(predictions=decoded_preds, references=decoded_labels)

    results = metric.compute()
    <span class="hljs-built_in">print</span>(<span class="hljs-string">f&quot;epoch <span class="hljs-subst">{epoch}</span>, BLEU score: <span class="hljs-subst">{results[<span class="hljs-string">&#x27;score&#x27;</span>]:<span class="hljs-number">.2</span>f}</span>&quot;</span>)

    <span class="hljs-comment"># Save and upload</span>
    accelerator.wait_for_everyone()
    unwrapped_model = accelerator.unwrap_model(model)
    unwrapped_model.save_pretrained(output_dir, save_function=accelerator.save)
    <span class="hljs-keyword">if</span> accelerator.is_main_process:
        tokenizer.save_pretrained(output_dir)
        repo.push_to_hub(
            commit_message=<span class="hljs-string">f&quot;Training in progress epoch <span class="hljs-subst">{epoch}</span>&quot;</span>, blocking=<span class="hljs-literal">False</span>
        )`}}),Ut=new S({props:{code:`epoch 0, BLEU score: 53.47
epoch 1, BLEU score: 54.24
epoch 2, BLEU score: 54.44`,highlighted:`epoch <span class="hljs-number">0</span>, BLEU score: <span class="hljs-number">53.47</span>
epoch <span class="hljs-number">1</span>, BLEU score: <span class="hljs-number">54.24</span>
epoch <span class="hljs-number">2</span>, BLEU score: <span class="hljs-number">54.44</span>`}}),{c(){h=l("h2"),$=l("a"),m=l("span"),v(x.$$.fragment),P=c(),g=l("span"),D=a("A custom training loop"),A=c(),T=l("p"),z=a("Let\u2019s now take a look at the full training loop, so you can easily customize the parts you need. It will look a lot like what we did in "),N=l("a"),q=a("section 2"),F=a(" and "),W=l("a"),U=a("Chapter 3"),L=a("."),G=c(),H=l("h3"),C=l("a"),K=l("span"),v(B.$$.fragment),Y=c(),re=l("span"),V=a("Preparing everything for training"),Q=c(),ee=l("p"),Z=a("You\u2019ve seen all of this a few times now, so we\u2019ll go through the code quite quickly. First we\u2019ll build the "),oe=l("code"),$e=a("DataLoader"),se=a("s from our datasets, after setting the datasets to the "),ae=l("code"),fe=a('"torch"'),ke=a(" format so we get PyTorch tensors:"),le=c(),v(he.$$.fragment),M=c(),ne=l("p"),ze=a("Next we reinstantiate our model, to make sure we\u2019re not continuing the fine-tuning from before but starting from the pretrained model again:"),me=c(),v(J.$$.fragment),ye=c(),ie=l("p"),xe=a("Then we will need an optimizer:"),Me=c(),v(_e.$$.fragment),qe=c(),te=l("p"),Pe=a("Once we have all those objects, we can send them to the "),ge=l("code"),at=a("accelerator.prepare()"),ft=a(" method. Remember that if you want to train on TPUs in a Colab notebook, you will need to move all of this code into a training function, and that shouldn\u2019t execute any cell that instantiates an "),ve=l("code"),Ge=a("Accelerator"),Ue=a("."),Se=c(),v(je.$$.fragment),mt=c(),we=l("p"),k=a("Now that we have sent our "),X=l("code"),Ze=a("train_dataloader"),ce=a(" to "),Be=l("code"),nt=a("accelerator.prepare()"),Ce=a(", we can use its length to compute the number of training steps. Remember we should always do this after preparing the dataloader, as that method will change the length of the "),pe=l("code"),Ft=a("DataLoader"),Re=a(". We use a classic linear schedule from the learning rate to 0:"),Ie=c(),v(Ne.$$.fragment),_t=c(),de=l("p"),Je=a("Lastly, to push our model to the Hub, we will need to create a "),Oe=l("code"),zt=a("Repository"),is=a(" object in a working folder. First log in to the Hugging Face Hub, if you\u2019re not logged in already. We\u2019ll determine the repository name from the model ID we want to give our model (feel free to replace the "),ot=l("code"),lt=a("repo_name"),rt=a(" with your own choice; it just needs to contain your username, which is what the function "),Ve=l("code"),Dt=a("get_full_repo_name()"),ps=a(" does):"),Qe=c(),v(gt.$$.fragment),wt=c(),v(Ee.$$.fragment),Xt=c(),Fe=l("p"),$s=a("Then we can clone that repository in a local folder. If it already exists, this local folder should be a clone of the repository we are working with:"),Gt=c(),v(De.$$.fragment),Zt=c(),Ae=l("p"),Lt=a("We can now upload anything we save in "),bt=l("code"),ks=a("output_dir"),Jt=a(" by calling the "),be=l("code"),$t=a("repo.push_to_hub()"),At=a(" method. This will help us upload the intermediate models at the end of each epoch."),kt=c(),it=l("h3"),He=l("a"),Vt=l("span"),v(vt.$$.fragment),pt=c(),ht=l("span"),It=a("Training loop"),Pt=c(),et=l("p"),Qt=a("We are now ready to write the full training loop. To simplify its evaluation part, we define this "),Le=l("code"),es=a("postprocess()"),Ye=a(" function that takes predictions and labels and converts them to the lists of strings our "),ts=l("code"),yt=a("metric"),Nt=a(" object will expect:"),hs=c(),v(ct.$$.fragment),f=c(),O=l("p"),vs=a("The training loop looks a lot like the ones in "),jt=l("a"),Xs=a("section 2"),ys=a(" and "),Te=l("a"),js=a("Chapter 3"),Et=a(", with a few differences in the evaluation part \u2014 so let\u2019s focus on that!"),cs=c(),ue=l("p"),Gs=a("The first thing to note is that we use the "),dt=l("code"),Zs=a("generate()"),We=a(" method to compute predictions, but this is a method on our base model, not the wrapped model \u{1F917} Accelerate created in the "),Ht=l("code"),Es=a("prepare()"),Js=a(" method. That\u2019s why we unwrap the model first, then call this method."),ss=c(),Ke=l("p"),Vs=a("The second thing is that, like with "),xt=l("a"),Wt=a("token classification"),xs=a(", two processes may have padded the inputs and labels to different shapes, so we use "),qt=l("code"),qs=a("accelerator.pad_across_processes()"),ut=a(" to make the predictions and labels the same shape before calling the "),ds=l("code"),us=a("gather()"),Mt=a(" method. If we don\u2019t do this, the evaluation will either error out or hang forever."),Ts=c(),v(Tt.$$.fragment),zs=c(),v(Ut.$$.fragment),fs=c(),tt=l("p"),Ea=a("Once this is done, you should have a model that has results pretty similar to the one trained with the "),ms=l("code"),Xa=a("Seq2SeqTrainer"),xa=a(". You can check the one we trained using this code at "),St=l("a"),Qs=l("em"),Ds=a("huggingface-course/marian-finetuned-kde4-en-to-fr-accelerate"),qa=a(". And if you want to test out any tweaks to the training loop, you can directly implement them by editing the code shown above!"),this.h()},l(u){h=r(u,"H2",{class:!0});var I=i(h);$=r(I,"A",{id:!0,class:!0,href:!0});var ea=i($);m=r(ea,"SPAN",{});var Ga=i(m);y(x.$$.fragment,Ga),Ga.forEach(t),ea.forEach(t),P=d(I),g=r(I,"SPAN",{});var ta=i(g);D=n(ta,"A custom training loop"),ta.forEach(t),I.forEach(t),A=d(u),T=r(u,"P",{});var Ps=i(T);z=n(Ps,"Let\u2019s now take a look at the full training loop, so you can easily customize the parts you need. It will look a lot like what we did in "),N=r(Ps,"A",{href:!0});var _a=i(N);q=n(_a,"section 2"),_a.forEach(t),F=n(Ps," and "),W=r(Ps,"A",{href:!0});var Qn=i(W);U=n(Qn,"Chapter 3"),Qn.forEach(t),L=n(Ps,"."),Ps.forEach(t),G=d(u),H=r(u,"H3",{class:!0});var ga=i(H);C=r(ga,"A",{id:!0,class:!0,href:!0});var sa=i(C);K=r(sa,"SPAN",{});var Za=i(K);y(B.$$.fragment,Za),Za.forEach(t),sa.forEach(t),Y=d(ga),re=r(ga,"SPAN",{});var aa=i(re);V=n(aa,"Preparing everything for training"),aa.forEach(t),ga.forEach(t),Q=d(u),ee=r(u,"P",{});var Ss=i(ee);Z=n(Ss,"You\u2019ve seen all of this a few times now, so we\u2019ll go through the code quite quickly. First we\u2019ll build the "),oe=r(Ss,"CODE",{});var wa=i(oe);$e=n(wa,"DataLoader"),wa.forEach(t),se=n(Ss,"s from our datasets, after setting the datasets to the "),ae=r(Ss,"CODE",{});var eo=i(ae);fe=n(eo,'"torch"'),eo.forEach(t),ke=n(Ss," format so we get PyTorch tensors:"),Ss.forEach(t),le=d(u),y(he.$$.fragment,u),M=d(u),ne=r(u,"P",{});var Ja=i(ne);ze=n(Ja,"Next we reinstantiate our model, to make sure we\u2019re not continuing the fine-tuning from before but starting from the pretrained model again:"),Ja.forEach(t),me=d(u),y(J.$$.fragment,u),ye=d(u),ie=r(u,"P",{});var na=i(ie);xe=n(na,"Then we will need an optimizer:"),na.forEach(t),Me=d(u),y(_e.$$.fragment,u),qe=d(u),te=r(u,"P",{});var Cs=i(te);Pe=n(Cs,"Once we have all those objects, we can send them to the "),ge=r(Cs,"CODE",{});var oa=i(ge);at=n(oa,"accelerator.prepare()"),oa.forEach(t),ft=n(Cs," method. Remember that if you want to train on TPUs in a Colab notebook, you will need to move all of this code into a training function, and that shouldn\u2019t execute any cell that instantiates an "),ve=r(Cs,"CODE",{});var Va=i(ve);Ge=n(Va,"Accelerator"),Va.forEach(t),Ue=n(Cs,"."),Cs.forEach(t),Se=d(u),y(je.$$.fragment,u),mt=d(u),we=r(u,"P",{});var Ct=i(we);k=n(Ct,"Now that we have sent our "),X=r(Ct,"CODE",{});var to=i(X);Ze=n(to,"train_dataloader"),to.forEach(t),ce=n(Ct," to "),Be=r(Ct,"CODE",{});var Qa=i(Be);nt=n(Qa,"accelerator.prepare()"),Qa.forEach(t),Ce=n(Ct,", we can use its length to compute the number of training steps. Remember we should always do this after preparing the dataloader, as that method will change the length of the "),pe=r(Ct,"CODE",{});var la=i(pe);Ft=n(la,"DataLoader"),la.forEach(t),Re=n(Ct,". We use a classic linear schedule from the learning rate to 0:"),Ct.forEach(t),Ie=d(u),y(Ne.$$.fragment,u),_t=d(u),de=r(u,"P",{});var as=i(de);Je=n(as,"Lastly, to push our model to the Hub, we will need to create a "),Oe=r(as,"CODE",{});var Os=i(Oe);zt=n(Os,"Repository"),Os.forEach(t),is=n(as," object in a working folder. First log in to the Hugging Face Hub, if you\u2019re not logged in already. We\u2019ll determine the repository name from the model ID we want to give our model (feel free to replace the "),ot=r(as,"CODE",{});var en=i(ot);lt=n(en,"repo_name"),en.forEach(t),rt=n(as," with your own choice; it just needs to contain your username, which is what the function "),Ve=r(as,"CODE",{});var _s=i(Ve);Dt=n(_s,"get_full_repo_name()"),_s.forEach(t),ps=n(as," does):"),as.forEach(t),Qe=d(u),y(gt.$$.fragment,u),wt=d(u),y(Ee.$$.fragment,u),Xt=d(u),Fe=r(u,"P",{});var Fs=i(Fe);$s=n(Fs,"Then we can clone that repository in a local folder. If it already exists, this local folder should be a clone of the repository we are working with:"),Fs.forEach(t),Gt=d(u),y(De.$$.fragment,u),Zt=d(u),Ae=r(u,"P",{});var gs=i(Ae);Lt=n(gs,"We can now upload anything we save in "),bt=r(gs,"CODE",{});var ra=i(bt);ks=n(ra,"output_dir"),ra.forEach(t),Jt=n(gs," by calling the "),be=r(gs,"CODE",{});var so=i(be);$t=n(so,"repo.push_to_hub()"),so.forEach(t),At=n(gs," method. This will help us upload the intermediate models at the end of each epoch."),gs.forEach(t),kt=d(u),it=r(u,"H3",{class:!0});var ia=i(it);He=r(ia,"A",{id:!0,class:!0,href:!0});var ao=i(He);Vt=r(ao,"SPAN",{});var tn=i(Vt);y(vt.$$.fragment,tn),tn.forEach(t),ao.forEach(t),pt=d(ia),ht=r(ia,"SPAN",{});var pa=i(ht);It=n(pa,"Training loop"),pa.forEach(t),ia.forEach(t),Pt=d(u),et=r(u,"P",{});var Ls=i(et);Qt=n(Ls,"We are now ready to write the full training loop. To simplify its evaluation part, we define this "),Le=r(Ls,"CODE",{});var ns=i(Le);es=n(ns,"postprocess()"),ns.forEach(t),Ye=n(Ls," function that takes predictions and labels and converts them to the lists of strings our "),ts=r(Ls,"CODE",{});var no=i(ts);yt=n(no,"metric"),no.forEach(t),Nt=n(Ls," object will expect:"),Ls.forEach(t),hs=d(u),y(ct.$$.fragment,u),f=d(u),O=r(u,"P",{});var ws=i(O);vs=n(ws,"The training loop looks a lot like the ones in "),jt=r(ws,"A",{href:!0});var oo=i(jt);Xs=n(oo,"section 2"),oo.forEach(t),ys=n(ws," and "),Te=r(ws,"A",{href:!0});var lo=i(Te);js=n(lo,"Chapter 3"),lo.forEach(t),Et=n(ws,", with a few differences in the evaluation part \u2014 so let\u2019s focus on that!"),ws.forEach(t),cs=d(u),ue=r(u,"P",{});var Bt=i(ue);Gs=n(Bt,"The first thing to note is that we use the "),dt=r(Bt,"CODE",{});var ro=i(dt);Zs=n(ro,"generate()"),ro.forEach(t),We=n(Bt," method to compute predictions, but this is a method on our base model, not the wrapped model \u{1F917} Accelerate created in the "),Ht=r(Bt,"CODE",{});var io=i(Ht);Es=n(io,"prepare()"),io.forEach(t),Js=n(Bt," method. That\u2019s why we unwrap the model first, then call this method."),Bt.forEach(t),ss=d(u),Ke=r(u,"P",{});var os=i(Ke);Vs=n(os,"The second thing is that, like with "),xt=r(os,"A",{href:!0});var ha=i(xt);Wt=n(ha,"token classification"),ha.forEach(t),xs=n(os,", two processes may have padded the inputs and labels to different shapes, so we use "),qt=r(os,"CODE",{});var sn=i(qt);qs=n(sn,"accelerator.pad_across_processes()"),sn.forEach(t),ut=n(os," to make the predictions and labels the same shape before calling the "),ds=r(os,"CODE",{});var ls=i(ds);us=n(ls,"gather()"),ls.forEach(t),Mt=n(os," method. If we don\u2019t do this, the evaluation will either error out or hang forever."),os.forEach(t),Ts=d(u),y(Tt.$$.fragment,u),zs=d(u),y(Ut.$$.fragment,u),fs=d(u),tt=r(u,"P",{});var ca=i(tt);Ea=n(ca,"Once this is done, you should have a model that has results pretty similar to the one trained with the "),ms=r(ca,"CODE",{});var Ta=i(ms);Xa=n(Ta,"Seq2SeqTrainer"),Ta.forEach(t),xa=n(ca,". You can check the one we trained using this code at "),St=r(ca,"A",{href:!0,rel:!0});var po=i(St);Qs=r(po,"EM",{});var ho=i(Qs);Ds=n(ho,"huggingface-course/marian-finetuned-kde4-en-to-fr-accelerate"),ho.forEach(t),po.forEach(t),qa=n(ca,". And if you want to test out any tweaks to the training loop, you can directly implement them by editing the code shown above!"),ca.forEach(t),this.h()},h(){_($,"id","a-custom-training-loop"),_($,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),_($,"href","#a-custom-training-loop"),_(h,"class","relative group"),_(N,"href","/course/chapter7/2"),_(W,"href","/course/chapter3/4"),_(C,"id","preparing-everything-for-training"),_(C,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),_(C,"href","#preparing-everything-for-training"),_(H,"class","relative group"),_(He,"id","training-loop"),_(He,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),_(He,"href","#training-loop"),_(it,"class","relative group"),_(jt,"href","/course/chapter7/2"),_(Te,"href","/course/chapter3"),_(xt,"href","/course/chapter7/2"),_(St,"href","https://huggingface.co/huggingface-course/marian-finetuned-kde4-en-to-fr-accelerate"),_(St,"rel","nofollow")},m(u,I){o(u,h,I),s(h,$),s($,m),j(x,m,null),s(h,P),s(h,g),s(g,D),o(u,A,I),o(u,T,I),s(T,z),s(T,N),s(N,q),s(T,F),s(T,W),s(W,U),s(T,L),o(u,G,I),o(u,H,I),s(H,C),s(C,K),j(B,K,null),s(H,Y),s(H,re),s(re,V),o(u,Q,I),o(u,ee,I),s(ee,Z),s(ee,oe),s(oe,$e),s(ee,se),s(ee,ae),s(ae,fe),s(ee,ke),o(u,le,I),j(he,u,I),o(u,M,I),o(u,ne,I),s(ne,ze),o(u,me,I),j(J,u,I),o(u,ye,I),o(u,ie,I),s(ie,xe),o(u,Me,I),j(_e,u,I),o(u,qe,I),o(u,te,I),s(te,Pe),s(te,ge),s(ge,at),s(te,ft),s(te,ve),s(ve,Ge),s(te,Ue),o(u,Se,I),j(je,u,I),o(u,mt,I),o(u,we,I),s(we,k),s(we,X),s(X,Ze),s(we,ce),s(we,Be),s(Be,nt),s(we,Ce),s(we,pe),s(pe,Ft),s(we,Re),o(u,Ie,I),j(Ne,u,I),o(u,_t,I),o(u,de,I),s(de,Je),s(de,Oe),s(Oe,zt),s(de,is),s(de,ot),s(ot,lt),s(de,rt),s(de,Ve),s(Ve,Dt),s(de,ps),o(u,Qe,I),j(gt,u,I),o(u,wt,I),j(Ee,u,I),o(u,Xt,I),o(u,Fe,I),s(Fe,$s),o(u,Gt,I),j(De,u,I),o(u,Zt,I),o(u,Ae,I),s(Ae,Lt),s(Ae,bt),s(bt,ks),s(Ae,Jt),s(Ae,be),s(be,$t),s(Ae,At),o(u,kt,I),o(u,it,I),s(it,He),s(He,Vt),j(vt,Vt,null),s(it,pt),s(it,ht),s(ht,It),o(u,Pt,I),o(u,et,I),s(et,Qt),s(et,Le),s(Le,es),s(et,Ye),s(et,ts),s(ts,yt),s(et,Nt),o(u,hs,I),j(ct,u,I),o(u,f,I),o(u,O,I),s(O,vs),s(O,jt),s(jt,Xs),s(O,ys),s(O,Te),s(Te,js),s(O,Et),o(u,cs,I),o(u,ue,I),s(ue,Gs),s(ue,dt),s(dt,Zs),s(ue,We),s(ue,Ht),s(Ht,Es),s(ue,Js),o(u,ss,I),o(u,Ke,I),s(Ke,Vs),s(Ke,xt),s(xt,Wt),s(Ke,xs),s(Ke,qt),s(qt,qs),s(Ke,ut),s(Ke,ds),s(ds,us),s(Ke,Mt),o(u,Ts,I),j(Tt,u,I),o(u,zs,I),j(Ut,u,I),o(u,fs,I),o(u,tt,I),s(tt,Ea),s(tt,ms),s(ms,Xa),s(tt,xa),s(tt,St),s(St,Qs),s(Qs,Ds),s(tt,qa),As=!0},i(u){As||(w(x.$$.fragment,u),w(B.$$.fragment,u),w(he.$$.fragment,u),w(J.$$.fragment,u),w(_e.$$.fragment,u),w(je.$$.fragment,u),w(Ne.$$.fragment,u),w(gt.$$.fragment,u),w(Ee.$$.fragment,u),w(De.$$.fragment,u),w(vt.$$.fragment,u),w(ct.$$.fragment,u),w(Tt.$$.fragment,u),w(Ut.$$.fragment,u),As=!0)},o(u){b(x.$$.fragment,u),b(B.$$.fragment,u),b(he.$$.fragment,u),b(J.$$.fragment,u),b(_e.$$.fragment,u),b(je.$$.fragment,u),b(Ne.$$.fragment,u),b(gt.$$.fragment,u),b(Ee.$$.fragment,u),b(De.$$.fragment,u),b(vt.$$.fragment,u),b(ct.$$.fragment,u),b(Tt.$$.fragment,u),b(Ut.$$.fragment,u),As=!1},d(u){u&&t(h),E(x),u&&t(A),u&&t(T),u&&t(G),u&&t(H),E(B),u&&t(Q),u&&t(ee),u&&t(le),E(he,u),u&&t(M),u&&t(ne),u&&t(me),E(J,u),u&&t(ye),u&&t(ie),u&&t(Me),E(_e,u),u&&t(qe),u&&t(te),u&&t(Se),E(je,u),u&&t(mt),u&&t(we),u&&t(Ie),E(Ne,u),u&&t(_t),u&&t(de),u&&t(Qe),E(gt,u),u&&t(wt),E(Ee,u),u&&t(Xt),u&&t(Fe),u&&t(Gt),E(De,u),u&&t(Zt),u&&t(Ae),u&&t(kt),u&&t(it),E(vt),u&&t(Pt),u&&t(et),u&&t(hs),E(ct,u),u&&t(f),u&&t(O),u&&t(cs),u&&t(ue),u&&t(ss),u&&t(Ke),u&&t(Ts),E(Tt,u),u&&t(zs),E(Ut,u),u&&t(fs),u&&t(tt)}}}function dd(R){let h,$,m,x,P;return{c(){h=l("p"),$=a("\u270F\uFE0F "),m=l("strong"),x=a("Your turn!"),P=a(" What does the model return on the sample with the word \u201Cemail\u201D you identified earlier?")},l(g){h=r(g,"P",{});var D=i(h);$=n(D,"\u270F\uFE0F "),m=r(D,"STRONG",{});var A=i(m);x=n(A,"Your turn!"),A.forEach(t),P=n(D," What does the model return on the sample with the word \u201Cemail\u201D you identified earlier?"),D.forEach(t)},m(g,D){o(g,h,D),s(h,$),s(h,m),s(m,x),s(h,P)},d(g){g&&t(h)}}}function ud(R){let h,$,m,x,P,g,D,A,T,z,N,q,F,W,U,L,G,H,C,K,B,Y,re,V,Q,ee,Z,oe,$e,se,ae,fe,ke,le,he,M,ne,ze,me,J,ye,ie,xe,Me,_e,qe,te,Pe,ge,at,ft,ve,Ge,Ue,Se,je,mt,we,k,X,Ze,ce,Be,nt,Ce,pe,Ft,Re,Ie,Ne,_t,de,Je,Oe,zt,is,ot,lt,rt,Ve,Dt,ps,Qe,gt,wt,Ee,Xt,Fe,$s,Gt,De,Zt,Ae,Lt,bt,ks,Jt,be,$t,At,kt,it,He,Vt,vt,pt,ht,It,Pt,et,Qt,Le,es,Ye,ts,yt,Nt,hs,ct,f,O,vs,jt,Xs,ys,Te,js,Et,cs,ue,Gs,dt,Zs,We,Ht,Es,Js,ss,Ke,Vs,xt,Wt,xs,qt,qs,ut,ds,us,Mt,Ts,Tt,zs,Ut,fs,tt,Ea,ms,Xa,xa,St,Qs,Ds,qa,As,u,I,ea,Ga,ta,Ps,_a,Qn,ga,sa,Za,aa,Ss,wa,eo,Ja,na,Cs,oa,Va,Ct,to,Qa,la,as,Os,en,_s,Fs,gs,ra,so,ia,ao,tn,pa,Ls,ns,no,ws,oo,lo,Bt,ro,io,os,ha,sn,ls,ca,Ta,po,ho,an,ai,ni,El,za,xl,da,oi,Go,li,ri,Zo,ii,pi,ql,co,hi,Tl,nn,zl,on,Dl,Da,ci,Jo,di,ui,Al,ln,Pl,rn,Sl,uo,fi,Cl,Aa,mi,Vo,_i,gi,Ol,pn,Fl,fo,wi,Ll,Pa,Il,Sa,Nl,mo,bi,Hl,hn,Wl,_o,$i,Ml,Is,Ns,go,wo,ki,Ul,ba,Ca,Qo,cn,vi,el,yi,Bl,bs,ji,tl,Ei,xi,bo,qi,Ti,sl,zi,Di,Rl,st,Ai,dn,al,Pi,Si,nl,Ci,Oi,ol,Fi,Li,ll,Ii,Ni,rl,Hi,Wi,il,Mi,Ui,Yl,Hs,Ws,$o,ko,Bi,Kl,un,Xl,fn,Gl,Oa,Ri,pl,Yi,Ki,Zl,mn,Jl,_n,Vl,vo,Xi,Ql,gn,er,wn,tr,yo,Gi,sr,bn,ar,$n,nr,Ms,Us,jo,$a,Fa,hl,kn,Zi,cl,Ji,or,vn,lr,Eo,Rt,Vi,yn,Qi,ep,jn,tp,sp,dl,ap,np,ul,op,lp,rr,La,rp,En,ip,pp,ir,xn,pr,ua,hp,fl,cp,dp,xo,up,fp,hr,qn,cr,qo,mp,dr,To,_p,ur,Tn,fr,zn,mr,Yt,gp,Dn,wp,bp,ml,$p,kp,_l,vp,yp,An,jp,Ep,_r,Pn,gr,Sn,wr,Cn,br,On,$r,zo,xp,kr,Bs,Rs,Do,Ao,qp,vr,ka,Ia,gl,Fn,Tp,wl,zp,yr,Po,Dp,jr,Ln,Er,So,Ap,xr,Co,Pp,qr,In,Tr,Ys,Ks,Oo,Fo,va,Na,bl,Nn,Sp,$l,Cp,zr,Ha,Op,kl,Fp,Lp,Dr,Hn,Ar,Wn,Pr,Lo,Ip,Sr,Mn,Cr,Un,Or,Io,Np,Fr,Wa,Lr;m=new Yc({props:{fw:R[0]}}),A=new rs({});const Hp=[Xc,Kc],Bn=[];function Wp(e,p){return e[0]==="pt"?0:1}F=Wp(R),W=Bn[F]=Hp[F](R),me=new si({props:{id:"1JvfrvZgi6c"}}),Dt=new rs({}),kt=new rs({}),Le=new S({props:{code:`from datasets import load_dataset

raw_datasets = load_dataset("kde4", lang1="en", lang2="fr")`,highlighted:`<span class="hljs-keyword">from</span> datasets <span class="hljs-keyword">import</span> load_dataset

raw_datasets = load_dataset(<span class="hljs-string">&quot;kde4&quot;</span>, lang1=<span class="hljs-string">&quot;en&quot;</span>, lang2=<span class="hljs-string">&quot;fr&quot;</span>)`}}),Te=new S({props:{code:"raw_datasets",highlighted:"raw_datasets"}}),Et=new S({props:{code:`DatasetDict({
    train: Dataset({
        features: ['id', 'translation'],
        num_rows: 210173
    })
})`,highlighted:`DatasetDict({
    train: Dataset({
        features: [<span class="hljs-string">&#x27;id&#x27;</span>, <span class="hljs-string">&#x27;translation&#x27;</span>],
        num_rows: <span class="hljs-number">210173</span>
    })
})`}}),Wt=new S({props:{code:`split_datasets = raw_datasets["train"].train_test_split(train_size=0.9, seed=20)
split_datasets`,highlighted:`split_datasets = raw_datasets[<span class="hljs-string">&quot;train&quot;</span>].train_test_split(train_size=<span class="hljs-number">0.9</span>, seed=<span class="hljs-number">20</span>)
split_datasets`}}),qt=new S({props:{code:`DatasetDict({
    train: Dataset({
        features: ['id', 'translation'],
        num_rows: 189155
    })
    test: Dataset({
        features: ['id', 'translation'],
        num_rows: 21018
    })
})`,highlighted:`DatasetDict({
    train: Dataset({
        features: [<span class="hljs-string">&#x27;id&#x27;</span>, <span class="hljs-string">&#x27;translation&#x27;</span>],
        num_rows: <span class="hljs-number">189155</span>
    })
    test: Dataset({
        features: [<span class="hljs-string">&#x27;id&#x27;</span>, <span class="hljs-string">&#x27;translation&#x27;</span>],
        num_rows: <span class="hljs-number">21018</span>
    })
})`}}),tt=new S({props:{code:'split_datasets["validation"] = split_datasets.pop("test")',highlighted:'split_datasets[<span class="hljs-string">&quot;validation&quot;</span>] = split_datasets.pop(<span class="hljs-string">&quot;test&quot;</span>)'}}),St=new S({props:{code:'split_datasets["train"][1]["translation"]',highlighted:'split_datasets[<span class="hljs-string">&quot;train&quot;</span>][<span class="hljs-number">1</span>][<span class="hljs-string">&quot;translation&quot;</span>]'}}),Ds=new S({props:{code:`{'en': 'Default to expanded threads',
 'fr': 'Par d\xE9faut, d\xE9velopper les fils de discussion'}`,highlighted:`{<span class="hljs-string">&#x27;en&#x27;</span>: <span class="hljs-string">&#x27;Default to expanded threads&#x27;</span>,
 <span class="hljs-string">&#x27;fr&#x27;</span>: <span class="hljs-string">&#x27;Par d\xE9faut, d\xE9velopper les fils de discussion&#x27;</span>}`}}),ea=new S({props:{code:`from transformers import pipeline

model_checkpoint = "Helsinki-NLP/opus-mt-en-fr"
translator = pipeline("translation", model=model_checkpoint)
translator("Default to expanded threads")`,highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> pipeline

model_checkpoint = <span class="hljs-string">&quot;Helsinki-NLP/opus-mt-en-fr&quot;</span>
translator = pipeline(<span class="hljs-string">&quot;translation&quot;</span>, model=model_checkpoint)
translator(<span class="hljs-string">&quot;Default to expanded threads&quot;</span>)`}}),ta=new S({props:{code:"[{'translation_text': 'Par d\xE9faut pour les threads \xE9largis'}]",highlighted:'[{<span class="hljs-string">&#x27;translation_text&#x27;</span>: <span class="hljs-string">&#x27;Par d\xE9faut pour les threads \xE9largis&#x27;</span>}]'}}),sa=new S({props:{code:'split_datasets["train"][172]["translation"]',highlighted:'split_datasets[<span class="hljs-string">&quot;train&quot;</span>][<span class="hljs-number">172</span>][<span class="hljs-string">&quot;translation&quot;</span>]'}}),aa=new S({props:{code:`{'en': 'Unable to import %1 using the OFX importer plugin. This file is not the correct format.',
 'fr': "Impossible d'importer %1 en utilisant le module d'extension d'importation OFX. Ce fichier n'a pas un format correct."}`,highlighted:`{<span class="hljs-string">&#x27;en&#x27;</span>: <span class="hljs-string">&#x27;Unable to import %1 using the OFX importer plugin. This file is not the correct format.&#x27;</span>,
 <span class="hljs-string">&#x27;fr&#x27;</span>: <span class="hljs-string">&quot;Impossible d&#x27;importer %1 en utilisant le module d&#x27;extension d&#x27;importation OFX. Ce fichier n&#x27;a pas un format correct.&quot;</span>}`}}),na=new S({props:{code:`translator(
    "Unable to import %1 using the OFX importer plugin. This file is not the correct format."
)`,highlighted:`translator(
    <span class="hljs-string">&quot;Unable to import %1 using the OFX importer plugin. This file is not the correct format.&quot;</span>
)`}}),oa=new S({props:{code:`[{'translation_text': "Impossible d'importer %1 en utilisant le plugin d'importateur OFX. Ce fichier n'est pas le bon format."}]`,highlighted:'[{<span class="hljs-string">&#x27;translation_text&#x27;</span>: <span class="hljs-string">&quot;Impossible d&#x27;importer %1 en utilisant le plugin d&#x27;importateur OFX. Ce fichier n&#x27;est pas le bon format.&quot;</span>}]'}}),la=new si({props:{id:"0Oxphw4Q9fo"}}),Os=new Ka({props:{$$slots:{default:[Gc]},$$scope:{ctx:R}}}),ra=new rs({}),pa=new si({props:{id:"XAR8jnZZuUs"}}),ha=new S({props:{code:`from transformers import AutoTokenizer

model_checkpoint = "Helsinki-NLP/opus-mt-en-fr"
tokenizer = AutoTokenizer.from_pretrained(model_checkpoint, return_tensors="pt")`,highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoTokenizer

model_checkpoint = <span class="hljs-string">&quot;Helsinki-NLP/opus-mt-en-fr&quot;</span>
tokenizer = AutoTokenizer.from_pretrained(model_checkpoint, return_tensors=<span class="hljs-string">&quot;pt&quot;</span>)`}}),za=new Ka({props:{$$slots:{default:[Zc]},$$scope:{ctx:R}}}),nn=new S({props:{code:`en_sentence = split_datasets["train"][1]["translation"]["en"]
fr_sentence = split_datasets["train"][1]["translation"]["fr"]

inputs = tokenizer(en_sentence, text_target=fr_sentence)
inputs`,highlighted:`en_sentence = split_datasets[<span class="hljs-string">&quot;train&quot;</span>][<span class="hljs-number">1</span>][<span class="hljs-string">&quot;translation&quot;</span>][<span class="hljs-string">&quot;en&quot;</span>]
fr_sentence = split_datasets[<span class="hljs-string">&quot;train&quot;</span>][<span class="hljs-number">1</span>][<span class="hljs-string">&quot;translation&quot;</span>][<span class="hljs-string">&quot;fr&quot;</span>]

inputs = tokenizer(en_sentence, text_target=fr_sentence)
inputs`}}),on=new S({props:{code:"{'input_ids': [47591, 12, 9842, 19634, 9, 0], 'attention_mask': [1, 1, 1, 1, 1, 1], 'labels': [577, 5891, 2, 3184, 16, 2542, 5, 1710, 0]}",highlighted:'{<span class="hljs-string">&#x27;input_ids&#x27;</span>: [<span class="hljs-number">47591</span>, <span class="hljs-number">12</span>, <span class="hljs-number">9842</span>, <span class="hljs-number">19634</span>, <span class="hljs-number">9</span>, <span class="hljs-number">0</span>], <span class="hljs-string">&#x27;attention_mask&#x27;</span>: [<span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>], <span class="hljs-string">&#x27;labels&#x27;</span>: [<span class="hljs-number">577</span>, <span class="hljs-number">5891</span>, <span class="hljs-number">2</span>, <span class="hljs-number">3184</span>, <span class="hljs-number">16</span>, <span class="hljs-number">2542</span>, <span class="hljs-number">5</span>, <span class="hljs-number">1710</span>, <span class="hljs-number">0</span>]}'}}),ln=new S({props:{code:`wrong_targets = tokenizer(fr_sentence)
print(tokenizer.convert_ids_to_tokens(wrong_targets["input_ids"]))
print(tokenizer.convert_ids_to_tokens(inputs["labels"]))`,highlighted:`wrong_targets = tokenizer(fr_sentence)
<span class="hljs-built_in">print</span>(tokenizer.convert_ids_to_tokens(wrong_targets[<span class="hljs-string">&quot;input_ids&quot;</span>]))
<span class="hljs-built_in">print</span>(tokenizer.convert_ids_to_tokens(inputs[<span class="hljs-string">&quot;labels&quot;</span>]))`}}),rn=new S({props:{code:`['\u2581Par', '\u2581d\xE9', 'f', 'aut', ',', '\u2581d\xE9', 've', 'lop', 'per', '\u2581les', '\u2581fil', 's', '\u2581de', '\u2581discussion', '</s>']
['\u2581Par', '\u2581d\xE9faut', ',', '\u2581d\xE9velopper', '\u2581les', '\u2581fils', '\u2581de', '\u2581discussion', '</s>']`,highlighted:`[<span class="hljs-string">&#x27;\u2581Par&#x27;</span>, <span class="hljs-string">&#x27;\u2581d\xE9&#x27;</span>, <span class="hljs-string">&#x27;f&#x27;</span>, <span class="hljs-string">&#x27;aut&#x27;</span>, <span class="hljs-string">&#x27;,&#x27;</span>, <span class="hljs-string">&#x27;\u2581d\xE9&#x27;</span>, <span class="hljs-string">&#x27;ve&#x27;</span>, <span class="hljs-string">&#x27;lop&#x27;</span>, <span class="hljs-string">&#x27;per&#x27;</span>, <span class="hljs-string">&#x27;\u2581les&#x27;</span>, <span class="hljs-string">&#x27;\u2581fil&#x27;</span>, <span class="hljs-string">&#x27;s&#x27;</span>, <span class="hljs-string">&#x27;\u2581de&#x27;</span>, <span class="hljs-string">&#x27;\u2581discussion&#x27;</span>, <span class="hljs-string">&#x27;&lt;/s&gt;&#x27;</span>]
[<span class="hljs-string">&#x27;\u2581Par&#x27;</span>, <span class="hljs-string">&#x27;\u2581d\xE9faut&#x27;</span>, <span class="hljs-string">&#x27;,&#x27;</span>, <span class="hljs-string">&#x27;\u2581d\xE9velopper&#x27;</span>, <span class="hljs-string">&#x27;\u2581les&#x27;</span>, <span class="hljs-string">&#x27;\u2581fils&#x27;</span>, <span class="hljs-string">&#x27;\u2581de&#x27;</span>, <span class="hljs-string">&#x27;\u2581discussion&#x27;</span>, <span class="hljs-string">&#x27;&lt;/s&gt;&#x27;</span>]`}}),pn=new S({props:{code:`max_length = 128


def preprocess_function(examples):
    inputs = [ex["en"] for ex in examples["translation"]]
    targets = [ex["fr"] for ex in examples["translation"]]
    model_inputs = tokenizer(
        inputs, text_target=targets, max_length=max_length, truncation=True
    )
    return model_inputs`,highlighted:`max_length = <span class="hljs-number">128</span>


<span class="hljs-keyword">def</span> <span class="hljs-title function_">preprocess_function</span>(<span class="hljs-params">examples</span>):
    inputs = [ex[<span class="hljs-string">&quot;en&quot;</span>] <span class="hljs-keyword">for</span> ex <span class="hljs-keyword">in</span> examples[<span class="hljs-string">&quot;translation&quot;</span>]]
    targets = [ex[<span class="hljs-string">&quot;fr&quot;</span>] <span class="hljs-keyword">for</span> ex <span class="hljs-keyword">in</span> examples[<span class="hljs-string">&quot;translation&quot;</span>]]
    model_inputs = tokenizer(
        inputs, text_target=targets, max_length=max_length, truncation=<span class="hljs-literal">True</span>
    )
    <span class="hljs-keyword">return</span> model_inputs`}}),Pa=new Ka({props:{$$slots:{default:[Jc]},$$scope:{ctx:R}}}),Sa=new Ka({props:{warning:!0,$$slots:{default:[Vc]},$$scope:{ctx:R}}}),hn=new S({props:{code:`tokenized_datasets = split_datasets.map(
    preprocess_function,
    batched=True,
    remove_columns=split_datasets["train"].column_names,
)`,highlighted:`tokenized_datasets = split_datasets.<span class="hljs-built_in">map</span>(
    preprocess_function,
    batched=<span class="hljs-literal">True</span>,
    remove_columns=split_datasets[<span class="hljs-string">&quot;train&quot;</span>].column_names,
)`}});const Mp=[ed,Qc],Rn=[];function Up(e,p){return e[0]==="pt"?0:1}Is=Up(R),Ns=Rn[Is]=Mp[Is](R),cn=new rs({});const Bp=[ad,sd],Yn=[];function Rp(e,p){return e[0]==="pt"?0:1}Hs=Rp(R),Ws=Yn[Hs]=Bp[Hs](R),un=new S({props:{code:`batch = data_collator([tokenized_datasets["train"][i] for i in range(1, 3)])
batch.keys()`,highlighted:`batch = data_collator([tokenized_datasets[<span class="hljs-string">&quot;train&quot;</span>][i] <span class="hljs-keyword">for</span> i <span class="hljs-keyword">in</span> <span class="hljs-built_in">range</span>(<span class="hljs-number">1</span>, <span class="hljs-number">3</span>)])
batch.keys()`}}),fn=new S({props:{code:"dict_keys(['attention_mask', 'input_ids', 'labels', 'decoder_input_ids'])",highlighted:'dict_keys([<span class="hljs-string">&#x27;attention_mask&#x27;</span>, <span class="hljs-string">&#x27;input_ids&#x27;</span>, <span class="hljs-string">&#x27;labels&#x27;</span>, <span class="hljs-string">&#x27;decoder_input_ids&#x27;</span>])'}}),mn=new S({props:{code:'batch["labels"]',highlighted:'batch[<span class="hljs-string">&quot;labels&quot;</span>]'}}),_n=new S({props:{code:`tensor([[  577,  5891,     2,  3184,    16,  2542,     5,  1710,     0,  -100,
          -100,  -100,  -100,  -100,  -100,  -100],
        [ 1211,     3,    49,  9409,  1211,     3, 29140,   817,  3124,   817,
           550,  7032,  5821,  7907, 12649,     0]])`,highlighted:`tensor([[  <span class="hljs-number">577</span>,  <span class="hljs-number">5891</span>,     <span class="hljs-number">2</span>,  <span class="hljs-number">3184</span>,    <span class="hljs-number">16</span>,  <span class="hljs-number">2542</span>,     <span class="hljs-number">5</span>,  <span class="hljs-number">1710</span>,     <span class="hljs-number">0</span>,  -<span class="hljs-number">100</span>,
          -<span class="hljs-number">100</span>,  -<span class="hljs-number">100</span>,  -<span class="hljs-number">100</span>,  -<span class="hljs-number">100</span>,  -<span class="hljs-number">100</span>,  -<span class="hljs-number">100</span>],
        [ <span class="hljs-number">1211</span>,     <span class="hljs-number">3</span>,    <span class="hljs-number">49</span>,  <span class="hljs-number">9409</span>,  <span class="hljs-number">1211</span>,     <span class="hljs-number">3</span>, <span class="hljs-number">29140</span>,   <span class="hljs-number">817</span>,  <span class="hljs-number">3124</span>,   <span class="hljs-number">817</span>,
           <span class="hljs-number">550</span>,  <span class="hljs-number">7032</span>,  <span class="hljs-number">5821</span>,  <span class="hljs-number">7907</span>, <span class="hljs-number">12649</span>,     <span class="hljs-number">0</span>]])`}}),gn=new S({props:{code:'batch["decoder_input_ids"]',highlighted:'batch[<span class="hljs-string">&quot;decoder_input_ids&quot;</span>]'}}),wn=new S({props:{code:`tensor([[59513,   577,  5891,     2,  3184,    16,  2542,     5,  1710,     0,
         59513, 59513, 59513, 59513, 59513, 59513],
        [59513,  1211,     3,    49,  9409,  1211,     3, 29140,   817,  3124,
           817,   550,  7032,  5821,  7907, 12649]])`,highlighted:`tensor([[<span class="hljs-number">59513</span>,   <span class="hljs-number">577</span>,  <span class="hljs-number">5891</span>,     <span class="hljs-number">2</span>,  <span class="hljs-number">3184</span>,    <span class="hljs-number">16</span>,  <span class="hljs-number">2542</span>,     <span class="hljs-number">5</span>,  <span class="hljs-number">1710</span>,     <span class="hljs-number">0</span>,
         <span class="hljs-number">59513</span>, <span class="hljs-number">59513</span>, <span class="hljs-number">59513</span>, <span class="hljs-number">59513</span>, <span class="hljs-number">59513</span>, <span class="hljs-number">59513</span>],
        [<span class="hljs-number">59513</span>,  <span class="hljs-number">1211</span>,     <span class="hljs-number">3</span>,    <span class="hljs-number">49</span>,  <span class="hljs-number">9409</span>,  <span class="hljs-number">1211</span>,     <span class="hljs-number">3</span>, <span class="hljs-number">29140</span>,   <span class="hljs-number">817</span>,  <span class="hljs-number">3124</span>,
           <span class="hljs-number">817</span>,   <span class="hljs-number">550</span>,  <span class="hljs-number">7032</span>,  <span class="hljs-number">5821</span>,  <span class="hljs-number">7907</span>, <span class="hljs-number">12649</span>]])`}}),bn=new S({props:{code:`for i in range(1, 3):
    print(tokenized_datasets["train"][i]["labels"])`,highlighted:`<span class="hljs-keyword">for</span> i <span class="hljs-keyword">in</span> <span class="hljs-built_in">range</span>(<span class="hljs-number">1</span>, <span class="hljs-number">3</span>):
    <span class="hljs-built_in">print</span>(tokenized_datasets[<span class="hljs-string">&quot;train&quot;</span>][i][<span class="hljs-string">&quot;labels&quot;</span>])`}}),$n=new S({props:{code:`[577, 5891, 2, 3184, 16, 2542, 5, 1710, 0]
[1211, 3, 49, 9409, 1211, 3, 29140, 817, 3124, 817, 550, 7032, 5821, 7907, 12649, 0]`,highlighted:`[<span class="hljs-number">577</span>, <span class="hljs-number">5891</span>, <span class="hljs-number">2</span>, <span class="hljs-number">3184</span>, <span class="hljs-number">16</span>, <span class="hljs-number">2542</span>, <span class="hljs-number">5</span>, <span class="hljs-number">1710</span>, <span class="hljs-number">0</span>]
[<span class="hljs-number">1211</span>, <span class="hljs-number">3</span>, <span class="hljs-number">49</span>, <span class="hljs-number">9409</span>, <span class="hljs-number">1211</span>, <span class="hljs-number">3</span>, <span class="hljs-number">29140</span>, <span class="hljs-number">817</span>, <span class="hljs-number">3124</span>, <span class="hljs-number">817</span>, <span class="hljs-number">550</span>, <span class="hljs-number">7032</span>, <span class="hljs-number">5821</span>, <span class="hljs-number">7907</span>, <span class="hljs-number">12649</span>, <span class="hljs-number">0</span>]`}});const Yp=[od,nd],Kn=[];function Kp(e,p){return e[0]==="pt"?0:1}Ms=Kp(R),Us=Kn[Ms]=Yp[Ms](R),kn=new rs({}),vn=new si({props:{id:"M05L1DhFqcw"}});let Ot=R[0]==="pt"&&Ic();xn=new S({props:{code:"!pip install sacrebleu",highlighted:"!pip install sacrebleu"}}),qn=new S({props:{code:`import evaluate

metric = evaluate.load("sacrebleu")`,highlighted:`<span class="hljs-keyword">import</span> evaluate

metric = evaluate.load(<span class="hljs-string">&quot;sacrebleu&quot;</span>)`}}),Tn=new S({props:{code:`predictions = [
    "This plugin lets you translate web pages between several languages automatically."
]
references = [
    [
        "This plugin allows you to automatically translate web pages between several languages."
    ]
]
metric.compute(predictions=predictions, references=references)`,highlighted:`predictions = [
    <span class="hljs-string">&quot;This plugin lets you translate web pages between several languages automatically.&quot;</span>
]
references = [
    [
        <span class="hljs-string">&quot;This plugin allows you to automatically translate web pages between several languages.&quot;</span>
    ]
]
metric.compute(predictions=predictions, references=references)`}}),zn=new S({props:{code:`{'score': 46.750469682990165,
 'counts': [11, 6, 4, 3],
 'totals': [12, 11, 10, 9],
 'precisions': [91.67, 54.54, 40.0, 33.33],
 'bp': 0.9200444146293233,
 'sys_len': 12,
 'ref_len': 13}`,highlighted:`{<span class="hljs-string">&#x27;score&#x27;</span>: <span class="hljs-number">46.750469682990165</span>,
 <span class="hljs-string">&#x27;counts&#x27;</span>: [<span class="hljs-number">11</span>, <span class="hljs-number">6</span>, <span class="hljs-number">4</span>, <span class="hljs-number">3</span>],
 <span class="hljs-string">&#x27;totals&#x27;</span>: [<span class="hljs-number">12</span>, <span class="hljs-number">11</span>, <span class="hljs-number">10</span>, <span class="hljs-number">9</span>],
 <span class="hljs-string">&#x27;precisions&#x27;</span>: [<span class="hljs-number">91.67</span>, <span class="hljs-number">54.54</span>, <span class="hljs-number">40.0</span>, <span class="hljs-number">33.33</span>],
 <span class="hljs-string">&#x27;bp&#x27;</span>: <span class="hljs-number">0.9200444146293233</span>,
 <span class="hljs-string">&#x27;sys_len&#x27;</span>: <span class="hljs-number">12</span>,
 <span class="hljs-string">&#x27;ref_len&#x27;</span>: <span class="hljs-number">13</span>}`}}),Pn=new S({props:{code:`predictions = ["This This This This"]
references = [
    [
        "This plugin allows you to automatically translate web pages between several languages."
    ]
]
metric.compute(predictions=predictions, references=references)`,highlighted:`predictions = [<span class="hljs-string">&quot;This This This This&quot;</span>]
references = [
    [
        <span class="hljs-string">&quot;This plugin allows you to automatically translate web pages between several languages.&quot;</span>
    ]
]
metric.compute(predictions=predictions, references=references)`}}),Sn=new S({props:{code:`{'score': 1.683602693167689,
 'counts': [1, 0, 0, 0],
 'totals': [4, 3, 2, 1],
 'precisions': [25.0, 16.67, 12.5, 12.5],
 'bp': 0.10539922456186433,
 'sys_len': 4,
 'ref_len': 13}`,highlighted:`{<span class="hljs-string">&#x27;score&#x27;</span>: <span class="hljs-number">1.683602693167689</span>,
 <span class="hljs-string">&#x27;counts&#x27;</span>: [<span class="hljs-number">1</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>],
 <span class="hljs-string">&#x27;totals&#x27;</span>: [<span class="hljs-number">4</span>, <span class="hljs-number">3</span>, <span class="hljs-number">2</span>, <span class="hljs-number">1</span>],
 <span class="hljs-string">&#x27;precisions&#x27;</span>: [<span class="hljs-number">25.0</span>, <span class="hljs-number">16.67</span>, <span class="hljs-number">12.5</span>, <span class="hljs-number">12.5</span>],
 <span class="hljs-string">&#x27;bp&#x27;</span>: <span class="hljs-number">0.10539922456186433</span>,
 <span class="hljs-string">&#x27;sys_len&#x27;</span>: <span class="hljs-number">4</span>,
 <span class="hljs-string">&#x27;ref_len&#x27;</span>: <span class="hljs-number">13</span>}`}}),Cn=new S({props:{code:`predictions = ["This plugin"]
references = [
    [
        "This plugin allows you to automatically translate web pages between several languages."
    ]
]
metric.compute(predictions=predictions, references=references)`,highlighted:`predictions = [<span class="hljs-string">&quot;This plugin&quot;</span>]
references = [
    [
        <span class="hljs-string">&quot;This plugin allows you to automatically translate web pages between several languages.&quot;</span>
    ]
]
metric.compute(predictions=predictions, references=references)`}}),On=new S({props:{code:`{'score': 0.0,
 'counts': [2, 1, 0, 0],
 'totals': [2, 1, 0, 0],
 'precisions': [100.0, 100.0, 0.0, 0.0],
 'bp': 0.004086771438464067,
 'sys_len': 2,
 'ref_len': 13}`,highlighted:`{<span class="hljs-string">&#x27;score&#x27;</span>: <span class="hljs-number">0.0</span>,
 <span class="hljs-string">&#x27;counts&#x27;</span>: [<span class="hljs-number">2</span>, <span class="hljs-number">1</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>],
 <span class="hljs-string">&#x27;totals&#x27;</span>: [<span class="hljs-number">2</span>, <span class="hljs-number">1</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>],
 <span class="hljs-string">&#x27;precisions&#x27;</span>: [<span class="hljs-number">100.0</span>, <span class="hljs-number">100.0</span>, <span class="hljs-number">0.0</span>, <span class="hljs-number">0.0</span>],
 <span class="hljs-string">&#x27;bp&#x27;</span>: <span class="hljs-number">0.004086771438464067</span>,
 <span class="hljs-string">&#x27;sys_len&#x27;</span>: <span class="hljs-number">2</span>,
 <span class="hljs-string">&#x27;ref_len&#x27;</span>: <span class="hljs-number">13</span>}`}});const Xp=[rd,ld],Xn=[];function Gp(e,p){return e[0]==="tf"?0:1}Bs=Gp(R),Rs=Xn[Bs]=Xp[Bs](R),Fn=new rs({}),Ln=new S({props:{code:`from huggingface_hub import notebook_login

notebook_login()`,highlighted:`<span class="hljs-keyword">from</span> huggingface_hub <span class="hljs-keyword">import</span> notebook_login

notebook_login()`}}),In=new S({props:{code:"huggingface-cli login",highlighted:"huggingface-cli login"}});const Zp=[pd,id],Gn=[];function Jp(e,p){return e[0]==="tf"?0:1}Ys=Jp(R),Ks=Gn[Ys]=Zp[Ys](R);let Xe=R[0]==="pt"&&Nc();return Nn=new rs({}),Hn=new S({props:{code:`from transformers import pipeline

# Replace this with your own checkpoint
model_checkpoint = "huggingface-course/marian-finetuned-kde4-en-to-fr"
translator = pipeline("translation", model=model_checkpoint)
translator("Default to expanded threads")`,highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> pipeline

<span class="hljs-comment"># Replace this with your own checkpoint</span>
model_checkpoint = <span class="hljs-string">&quot;huggingface-course/marian-finetuned-kde4-en-to-fr&quot;</span>
translator = pipeline(<span class="hljs-string">&quot;translation&quot;</span>, model=model_checkpoint)
translator(<span class="hljs-string">&quot;Default to expanded threads&quot;</span>)`}}),Wn=new S({props:{code:"[{'translation_text': 'Par d\xE9faut, d\xE9velopper les fils de discussion'}]",highlighted:'[{<span class="hljs-string">&#x27;translation_text&#x27;</span>: <span class="hljs-string">&#x27;Par d\xE9faut, d\xE9velopper les fils de discussion&#x27;</span>}]'}}),Mn=new S({props:{code:`translator(
    "Unable to import %1 using the OFX importer plugin. This file is not the correct format."
)`,highlighted:`translator(
    <span class="hljs-string">&quot;Unable to import %1 using the OFX importer plugin. This file is not the correct format.&quot;</span>
)`}}),Un=new S({props:{code:`[{'translation_text': "Impossible d'importer %1 en utilisant le module externe d'importation OFX. Ce fichier n'est pas le bon format."}]`,highlighted:'[{<span class="hljs-string">&#x27;translation_text&#x27;</span>: <span class="hljs-string">&quot;Impossible d&#x27;importer %1 en utilisant le module externe d&#x27;importation OFX. Ce fichier n&#x27;est pas le bon format.&quot;</span>}]'}}),Wa=new Ka({props:{$$slots:{default:[dd]},$$scope:{ctx:R}}}),{c(){h=l("meta"),$=c(),v(m.$$.fragment),x=c(),P=l("h1"),g=l("a"),D=l("span"),v(A.$$.fragment),T=c(),z=l("span"),N=a("Translation"),q=c(),W.c(),U=c(),L=l("p"),G=a("Let\u2019s now dive into translation. This is another "),H=l("a"),C=a("sequence-to-sequence task"),K=a(", which means it\u2019s a problem that can be formulated as going from one sequence to another. In that sense the problem is pretty close to "),B=l("a"),Y=a("summarization"),re=a(", and you could adapt what we will see here to other sequence-to-sequence problems such as:"),V=c(),Q=l("ul"),ee=l("li"),Z=l("strong"),oe=a("Style transfer"),$e=a(": Creating a model that "),se=l("em"),ae=a("translates"),fe=a(" texts written in a certain style to another (e.g., formal to casual or Shakespearean English to modern English)"),ke=c(),le=l("li"),he=l("strong"),M=a("Generative question answering"),ne=a(": Creating a model that generates answers to questions, given a context"),ze=c(),v(me.$$.fragment),J=c(),ye=l("p"),ie=a("If you have a big enough corpus of texts in two (or more) languages, you can train a new translation model from scratch like we will in the section on "),xe=l("a"),Me=a("causal language modeling"),_e=a(". It will be faster, however, to fine-tune an existing translation model, be it a multilingual one like mT5 or mBART that you want to fine-tune to a specific language pair, or even a model specialized for translation from one language to another that you want to fine-tune to your specific corpus."),qe=c(),te=l("p"),Pe=a("In this section, we will fine-tune a Marian model pretrained to translate from English to French (since a lot of Hugging Face employees speak both those languages) on the "),ge=l("a"),at=a("KDE4 dataset"),ft=a(", which is a dataset of localized files for the "),ve=l("a"),Ge=a("KDE apps"),Ue=a(". The model we will use has been pretrained on a large corpus of French and English texts taken from the "),Se=l("a"),je=a("Opus dataset"),mt=a(", which actually contains the KDE4 dataset. But even if the pretrained model we use has seen that data during its pretraining, we will see that we can get a better version of it after fine-tuning."),we=c(),k=l("p"),X=a("Once we\u2019re finished, we will have a model able to make predictions like this one:"),Ze=c(),ce=l("iframe"),nt=c(),Ce=l("a"),pe=l("img"),Re=c(),Ie=l("img"),_t=c(),de=l("p"),Je=a("As in the previous sections, you can find the actual model that we\u2019ll train and upload to the Hub using the code below and double-check its predictions "),Oe=l("a"),zt=a("here"),is=a("."),ot=c(),lt=l("h2"),rt=l("a"),Ve=l("span"),v(Dt.$$.fragment),ps=c(),Qe=l("span"),gt=a("Preparing the data"),wt=c(),Ee=l("p"),Xt=a("To fine-tune or train a translation model from scratch, we will need a dataset suitable for the task. As mentioned previously, we\u2019ll use the "),Fe=l("a"),$s=a("KDE4 dataset"),Gt=a(" in this section, but you can adapt the code to use your own data quite easily, as long as you have pairs of sentences in the two languages you want to translate from and into. Refer back to "),De=l("a"),Zt=a("Chapter 5"),Ae=a(" if you need a reminder of how to load your custom data in a "),Lt=l("code"),bt=a("Dataset"),ks=a("."),Jt=c(),be=l("h3"),$t=l("a"),At=l("span"),v(kt.$$.fragment),it=c(),He=l("span"),Vt=a("The KDE4 dataset"),vt=c(),pt=l("p"),ht=a("As usual, we download our dataset using the "),It=l("code"),Pt=a("load_dataset()"),et=a(" function:"),Qt=c(),v(Le.$$.fragment),es=c(),Ye=l("p"),ts=a("If you want to work with a different pair of languages, you can specify them by their codes. A total of 92 languages are available for this dataset; you can see them all by expanding the language tags on its "),yt=l("a"),Nt=a("dataset card"),hs=a("."),ct=c(),f=l("img"),vs=c(),jt=l("p"),Xs=a("Let\u2019s have a look at the dataset:"),ys=c(),v(Te.$$.fragment),js=c(),v(Et.$$.fragment),cs=c(),ue=l("p"),Gs=a("We have 210,173 pairs of sentences, but in one single split, so we will need to create our own validation set. As we saw in "),dt=l("a"),Zs=a("Chapter 5"),We=a(", a "),Ht=l("code"),Es=a("Dataset"),Js=a(" has a "),ss=l("code"),Ke=a("train_test_split()"),Vs=a(" method that can help us. We\u2019ll provide a seed for reproducibility:"),xt=c(),v(Wt.$$.fragment),xs=c(),v(qt.$$.fragment),qs=c(),ut=l("p"),ds=a("We can rename the "),us=l("code"),Mt=a('"test"'),Ts=a(" key to "),Tt=l("code"),zs=a('"validation"'),Ut=a(" like this:"),fs=c(),v(tt.$$.fragment),Ea=c(),ms=l("p"),Xa=a("Now let\u2019s take a look at one element of the dataset:"),xa=c(),v(St.$$.fragment),Qs=c(),v(Ds.$$.fragment),qa=c(),As=l("p"),u=a("We get a dictionary with two sentences in the pair of languages we requested. One particularity of this dataset full of technical computer science terms is that they are all fully translated in French. However, French engineers are often lazy and leave most computer science-specific words in English when they talk. Here, for instance, the word \u201Cthreads\u201D might well appear in a French sentence, especially in a technical conversation; but in this dataset it has been translated into the more correct \u201Cfils de discussion.\u201D The pretrained model we use, which has been pretrained on a larger corpus of French and English sentences, takes the easier option of leaving the word as is:"),I=c(),v(ea.$$.fragment),Ga=c(),v(ta.$$.fragment),Ps=c(),_a=l("p"),Qn=a(`Another example of this behavior can be seen with the word \u201Cplugin,\u201D which isn\u2019t officially a French word but which most native speakers will understand and not bother to translate.
In the KDE4 dataset this word has been translated in French into the more official \u201Cmodule d\u2019extension\u201D:`),ga=c(),v(sa.$$.fragment),Za=c(),v(aa.$$.fragment),Ss=c(),wa=l("p"),eo=a("Our pretrained model, however, sticks with the compact and familiar English word:"),Ja=c(),v(na.$$.fragment),Cs=c(),v(oa.$$.fragment),Va=c(),Ct=l("p"),to=a("It will be interesting to see if our fine-tuned model picks up on those particularities of the dataset (spoiler alert: it will)."),Qa=c(),v(la.$$.fragment),as=c(),v(Os.$$.fragment),en=c(),_s=l("h3"),Fs=l("a"),gs=l("span"),v(ra.$$.fragment),so=c(),ia=l("span"),ao=a("Processing the data"),tn=c(),v(pa.$$.fragment),Ls=c(),ns=l("p"),no=a("You should know the drill by now: the texts all need to be converted into sets of token IDs so the model can make sense of them. For this task, we\u2019ll need to tokenize both the inputs and the targets. Our first task is to create our "),ws=l("code"),oo=a("tokenizer"),lo=a(" object. As noted earlier, we\u2019ll be using a Marian English to French pretrained model. If you are trying this code with another pair of languages, make sure to adapt the model checkpoint. The "),Bt=l("a"),ro=a("Helsinki-NLP"),io=a(" organization provides more than a thousand models in multiple languages."),os=c(),v(ha.$$.fragment),sn=c(),ls=l("p"),ca=a("You can also replace the "),Ta=l("code"),po=a("model_checkpoint"),ho=a(" with any other model you prefer from the "),an=l("a"),ai=a("Hub"),ni=a(", or a local folder where you\u2019ve saved a pretrained model and a tokenizer."),El=c(),v(za.$$.fragment),xl=c(),da=l("p"),oi=a("The preparation of our data is pretty straightforward. There\u2019s just one thing to remember; you need to ensure that the tokenizer processes the targets in the output language (here, French). You can do this by passing the targets to the "),Go=l("code"),li=a("text_targets"),ri=a(" argument of the tokenizer\u2019s "),Zo=l("code"),ii=a("__call__"),pi=a(" method."),ql=c(),co=l("p"),hi=a("To see how this works, let\u2019s process one sample of each language in the training set:"),Tl=c(),v(nn.$$.fragment),zl=c(),v(on.$$.fragment),Dl=c(),Da=l("p"),ci=a("As we can see, the output contains the input IDs associated with the English sentence, while the IDs associated with the French one are stored in the "),Jo=l("code"),di=a("labels"),ui=a(" field. If you forget to indicate that you are tokenizing labels, they will be tokenized by the input tokenizer, which in the case of a Marian model is not going to go well at all:"),Al=c(),v(ln.$$.fragment),Pl=c(),v(rn.$$.fragment),Sl=c(),uo=l("p"),fi=a("As we can see, using the English tokenizer to preprocess a French sentence results in a lot more tokens, since the tokenizer doesn\u2019t know any French words (except those that also appear in the English language, like \u201Cdiscussion\u201D)."),Cl=c(),Aa=l("p"),mi=a("Since "),Vo=l("code"),_i=a("inputs"),gi=a(" is a dictionary with our usual keys (input IDs, attention mask, etc.), the last step is to define the preprocessing function we will apply on the datasets:"),Ol=c(),v(pn.$$.fragment),Fl=c(),fo=l("p"),wi=a("Note that we set the same maximum length for our inputs and outputs. Since the texts we\u2019re dealing with seem pretty short, we use 128."),Ll=c(),v(Pa.$$.fragment),Il=c(),v(Sa.$$.fragment),Nl=c(),mo=l("p"),bi=a("We can now apply that preprocessing in one go on all the splits of our dataset:"),Hl=c(),v(hn.$$.fragment),Wl=c(),_o=l("p"),$i=a("Now that the data has been preprocessed, we are ready to fine-tune our pretrained model!"),Ml=c(),Ns.c(),go=c(),wo=l("p"),ki=a("Note that this time we are using a model that was trained on a translation task and can actually be used already, so there is no warning about missing weights or newly initialized ones."),Ul=c(),ba=l("h3"),Ca=l("a"),Qo=l("span"),v(cn.$$.fragment),vi=c(),el=l("span"),yi=a("Data collation"),Bl=c(),bs=l("p"),ji=a("We\u2019ll need a data collator to deal with the padding for dynamic batching. We can\u2019t just use a "),tl=l("code"),Ei=a("DataCollatorWithPadding"),xi=a(" like in "),bo=l("a"),qi=a("Chapter 3"),Ti=a(" in this case, because that only pads the inputs (input IDs, attention mask, and token type IDs). Our labels should also be padded to the maximum length encountered in the labels. And, as mentioned previously, the padding value used to pad the labels should be "),sl=l("code"),zi=a("-100"),Di=a(" and not the padding token of the tokenizer, to make sure those padded values are ignored in the loss computation."),Rl=c(),st=l("p"),Ai=a("This is all done by a "),dn=l("a"),al=l("code"),Pi=a("DataCollatorForSeq2Seq"),Si=a(". Like the "),nl=l("code"),Ci=a("DataCollatorWithPadding"),Oi=a(", it takes the "),ol=l("code"),Fi=a("tokenizer"),Li=a(" used to preprocess the inputs, but it also takes the "),ll=l("code"),Ii=a("model"),Ni=a(". This is because this data collator will also be responsible for preparing the decoder input IDs, which are shifted versions of the labels with a special token at the beginning. Since this shift is done slightly differently for different architectures, the "),rl=l("code"),Hi=a("DataCollatorForSeq2Seq"),Wi=a(" needs to know the "),il=l("code"),Mi=a("model"),Ui=a(" object:"),Yl=c(),Ws.c(),$o=c(),ko=l("p"),Bi=a("To test this on a few samples, we just call it on a list of examples from our tokenized training set:"),Kl=c(),v(un.$$.fragment),Xl=c(),v(fn.$$.fragment),Gl=c(),Oa=l("p"),Ri=a("We can check our labels have been padded to the maximum length of the batch, using "),pl=l("code"),Yi=a("-100"),Ki=a(":"),Zl=c(),v(mn.$$.fragment),Jl=c(),v(_n.$$.fragment),Vl=c(),vo=l("p"),Xi=a("And we can also have a look at the decoder input IDs, to see that they are shifted versions of the labels:"),Ql=c(),v(gn.$$.fragment),er=c(),v(wn.$$.fragment),tr=c(),yo=l("p"),Gi=a("Here are the labels for the first and second elements in our dataset:"),sr=c(),v(bn.$$.fragment),ar=c(),v($n.$$.fragment),nr=c(),Us.c(),jo=c(),$a=l("h3"),Fa=l("a"),hl=l("span"),v(kn.$$.fragment),Zi=c(),cl=l("span"),Ji=a("Metrics"),or=c(),v(vn.$$.fragment),lr=c(),Ot&&Ot.c(),Eo=c(),Rt=l("p"),Vi=a("The traditional metric used for translation is the "),yn=l("a"),Qi=a("BLEU score"),ep=a(", introduced in "),jn=l("a"),tp=a("a 2002 article"),sp=a(" by Kishore Papineni et al. The BLEU score evaluates how close the translations are to their labels. It does not measure the intelligibility or grammatical correctness of the model\u2019s generated outputs, but uses statistical rules to ensure that all the words in the generated outputs also appear in the targets. In addition, there are rules that penalize repetitions of the same words if they are not also repeated in the targets (to avoid the model outputting sentences like "),dl=l("code"),ap=a('"the the the the the"'),np=a(") and output sentences that are shorter than those in the targets (to avoid the model outputting sentences like "),ul=l("code"),op=a('"the"'),lp=a(")."),rr=c(),La=l("p"),rp=a("One weakness with BLEU is that it expects the text to already be tokenized, which makes it difficult to compare scores between models that use different tokenizers. So instead, the most commonly used metric for benchmarking translation models today is "),En=l("a"),ip=a("SacreBLEU"),pp=a(", which addresses this weakness (and others) by standardizing the tokenization step. To use this metric, we first need to install the SacreBLEU library:"),ir=c(),v(xn.$$.fragment),pr=c(),ua=l("p"),hp=a("We can then load it via "),fl=l("code"),cp=a("evaluate.load()"),dp=a(" like we did in "),xo=l("a"),up=a("Chapter 3"),fp=a(":"),hr=c(),v(qn.$$.fragment),cr=c(),qo=l("p"),mp=a("This metric will take texts as inputs and targets. It is designed to accept several acceptable targets, as there are often multiple acceptable translations of the same sentence \u2014 the dataset we\u2019re using only provides one, but it\u2019s not uncommon in NLP to find datasets that give several sentences as labels. So, the predictions should be a list of sentences, but the references should be a list of lists of sentences."),dr=c(),To=l("p"),_p=a("Let\u2019s try an example:"),ur=c(),v(Tn.$$.fragment),fr=c(),v(zn.$$.fragment),mr=c(),Yt=l("p"),gp=a("This gets a BLEU score of 46.75, which is rather good \u2014 for reference, the original Transformer model in the "),Dn=l("a"),wp=a("\u201CAttention Is All You Need\u201D paper"),bp=a(" achieved a BLEU score of 41.8 on a similar translation task between English and French! (For more information about the individual metrics, like "),ml=l("code"),$p=a("counts"),kp=a(" and "),_l=l("code"),vp=a("bp"),yp=a(", see the "),An=l("a"),jp=a("SacreBLEU repository"),Ep=a(".) On the other hand, if we try with the two bad types of predictions (lots of repetitions or too short) that often come out of translation models, we will get rather bad BLEU scores:"),_r=c(),v(Pn.$$.fragment),gr=c(),v(Sn.$$.fragment),wr=c(),v(Cn.$$.fragment),br=c(),v(On.$$.fragment),$r=c(),zo=l("p"),xp=a("The score can go from 0 to 100, and higher is better."),kr=c(),Rs.c(),Do=c(),Ao=l("p"),qp=a("Now that this is done, we are ready to fine-tune our model!"),vr=c(),ka=l("h3"),Ia=l("a"),gl=l("span"),v(Fn.$$.fragment),Tp=c(),wl=l("span"),zp=a("Fine-tuning the model"),yr=c(),Po=l("p"),Dp=a("The first step is to log in to Hugging Face, so you\u2019re able to upload your results to the Model Hub. There\u2019s a convenience function to help you with this in a notebook:"),jr=c(),v(Ln.$$.fragment),Er=c(),So=l("p"),Ap=a("This will display a widget where you can enter your Hugging Face login credentials."),xr=c(),Co=l("p"),Pp=a("If you aren\u2019t working in a notebook, just type the following line in your terminal:"),qr=c(),v(In.$$.fragment),Tr=c(),Ks.c(),Oo=c(),Xe&&Xe.c(),Fo=c(),va=l("h2"),Na=l("a"),bl=l("span"),v(Nn.$$.fragment),Sp=c(),$l=l("span"),Cp=a("Using the fine-tuned model"),zr=c(),Ha=l("p"),Op=a("We\u2019ve already shown you how you can use the model we fine-tuned on the Model Hub with the inference widget. To use it locally in a "),kl=l("code"),Fp=a("pipeline"),Lp=a(", we just have to specify the proper model identifier:"),Dr=c(),v(Hn.$$.fragment),Ar=c(),v(Wn.$$.fragment),Pr=c(),Lo=l("p"),Ip=a("As expected, our pretrained model adapted its knowledge to the corpus we fine-tuned it on, and instead of leaving the English word \u201Cthreads\u201D alone, it now translates it to the French official version. It\u2019s the same for \u201Cplugin\u201D:"),Sr=c(),v(Mn.$$.fragment),Cr=c(),v(Un.$$.fragment),Or=c(),Io=l("p"),Np=a("Another great example of domain adaptation!"),Fr=c(),v(Wa.$$.fragment),this.h()},l(e){const p=Bc('[data-svelte="svelte-1phssyn"]',document.head);h=r(p,"META",{name:!0,content:!0}),p.forEach(t),$=d(e),y(m.$$.fragment,e),x=d(e),P=r(e,"H1",{class:!0});var Zn=i(P);g=r(Zn,"A",{id:!0,class:!0,href:!0});var No=i(g);D=r(No,"SPAN",{});var vl=i(D);y(A.$$.fragment,vl),vl.forEach(t),No.forEach(t),T=d(Zn),z=r(Zn,"SPAN",{});var yl=i(z);N=n(yl,"Translation"),yl.forEach(t),Zn.forEach(t),q=d(e),W.l(e),U=d(e),L=r(e,"P",{});var ya=i(L);G=n(ya,"Let\u2019s now dive into translation. This is another "),H=r(ya,"A",{href:!0});var jl=i(H);C=n(jl,"sequence-to-sequence task"),jl.forEach(t),K=n(ya,", which means it\u2019s a problem that can be formulated as going from one sequence to another. In that sense the problem is pretty close to "),B=r(ya,"A",{href:!0});var Ho=i(B);Y=n(Ho,"summarization"),Ho.forEach(t),re=n(ya,", and you could adapt what we will see here to other sequence-to-sequence problems such as:"),ya.forEach(t),V=d(e),Q=r(e,"UL",{});var Ma=i(Q);ee=r(Ma,"LI",{});var ja=i(ee);Z=r(ja,"STRONG",{});var Wo=i(Z);oe=n(Wo,"Style transfer"),Wo.forEach(t),$e=n(ja,": Creating a model that "),se=r(ja,"EM",{});var Mo=i(se);ae=n(Mo,"translates"),Mo.forEach(t),fe=n(ja," texts written in a certain style to another (e.g., formal to casual or Shakespearean English to modern English)"),ja.forEach(t),ke=d(Ma),le=r(Ma,"LI",{});var Uo=i(le);he=r(Uo,"STRONG",{});var Vp=i(he);M=n(Vp,"Generative question answering"),Vp.forEach(t),ne=n(Uo,": Creating a model that generates answers to questions, given a context"),Uo.forEach(t),Ma.forEach(t),ze=d(e),y(me.$$.fragment,e),J=d(e),ye=r(e,"P",{});var Ir=i(ye);ie=n(Ir,"If you have a big enough corpus of texts in two (or more) languages, you can train a new translation model from scratch like we will in the section on "),xe=r(Ir,"A",{href:!0});var Qp=i(xe);Me=n(Qp,"causal language modeling"),Qp.forEach(t),_e=n(Ir,". It will be faster, however, to fine-tune an existing translation model, be it a multilingual one like mT5 or mBART that you want to fine-tune to a specific language pair, or even a model specialized for translation from one language to another that you want to fine-tune to your specific corpus."),Ir.forEach(t),qe=d(e),te=r(e,"P",{});var Ua=i(te);Pe=n(Ua,"In this section, we will fine-tune a Marian model pretrained to translate from English to French (since a lot of Hugging Face employees speak both those languages) on the "),ge=r(Ua,"A",{href:!0,rel:!0});var eh=i(ge);at=n(eh,"KDE4 dataset"),eh.forEach(t),ft=n(Ua,", which is a dataset of localized files for the "),ve=r(Ua,"A",{href:!0,rel:!0});var th=i(ve);Ge=n(th,"KDE apps"),th.forEach(t),Ue=n(Ua,". The model we will use has been pretrained on a large corpus of French and English texts taken from the "),Se=r(Ua,"A",{href:!0,rel:!0});var sh=i(Se);je=n(sh,"Opus dataset"),sh.forEach(t),mt=n(Ua,", which actually contains the KDE4 dataset. But even if the pretrained model we use has seen that data during its pretraining, we will see that we can get a better version of it after fine-tuning."),Ua.forEach(t),we=d(e),k=r(e,"P",{});var ah=i(k);X=n(ah,"Once we\u2019re finished, we will have a model able to make predictions like this one:"),ah.forEach(t),Ze=d(e),ce=r(e,"IFRAME",{src:!0,frameborder:!0,height:!0,title:!0,class:!0,allow:!0,sandbox:!0}),i(ce).forEach(t),nt=d(e),Ce=r(e,"A",{class:!0,href:!0});var Nr=i(Ce);pe=r(Nr,"IMG",{class:!0,src:!0,alt:!0}),Re=d(Nr),Ie=r(Nr,"IMG",{class:!0,src:!0,alt:!0}),Nr.forEach(t),_t=d(e),de=r(e,"P",{});var Hr=i(de);Je=n(Hr,"As in the previous sections, you can find the actual model that we\u2019ll train and upload to the Hub using the code below and double-check its predictions "),Oe=r(Hr,"A",{href:!0,rel:!0});var nh=i(Oe);zt=n(nh,"here"),nh.forEach(t),is=n(Hr,"."),Hr.forEach(t),ot=d(e),lt=r(e,"H2",{class:!0});var Wr=i(lt);rt=r(Wr,"A",{id:!0,class:!0,href:!0});var oh=i(rt);Ve=r(oh,"SPAN",{});var lh=i(Ve);y(Dt.$$.fragment,lh),lh.forEach(t),oh.forEach(t),ps=d(Wr),Qe=r(Wr,"SPAN",{});var rh=i(Qe);gt=n(rh,"Preparing the data"),rh.forEach(t),Wr.forEach(t),wt=d(e),Ee=r(e,"P",{});var Ba=i(Ee);Xt=n(Ba,"To fine-tune or train a translation model from scratch, we will need a dataset suitable for the task. As mentioned previously, we\u2019ll use the "),Fe=r(Ba,"A",{href:!0,rel:!0});var ih=i(Fe);$s=n(ih,"KDE4 dataset"),ih.forEach(t),Gt=n(Ba," in this section, but you can adapt the code to use your own data quite easily, as long as you have pairs of sentences in the two languages you want to translate from and into. Refer back to "),De=r(Ba,"A",{href:!0});var ph=i(De);Zt=n(ph,"Chapter 5"),ph.forEach(t),Ae=n(Ba," if you need a reminder of how to load your custom data in a "),Lt=r(Ba,"CODE",{});var hh=i(Lt);bt=n(hh,"Dataset"),hh.forEach(t),ks=n(Ba,"."),Ba.forEach(t),Jt=d(e),be=r(e,"H3",{class:!0});var Mr=i(be);$t=r(Mr,"A",{id:!0,class:!0,href:!0});var ch=i($t);At=r(ch,"SPAN",{});var dh=i(At);y(kt.$$.fragment,dh),dh.forEach(t),ch.forEach(t),it=d(Mr),He=r(Mr,"SPAN",{});var uh=i(He);Vt=n(uh,"The KDE4 dataset"),uh.forEach(t),Mr.forEach(t),vt=d(e),pt=r(e,"P",{});var Ur=i(pt);ht=n(Ur,"As usual, we download our dataset using the "),It=r(Ur,"CODE",{});var fh=i(It);Pt=n(fh,"load_dataset()"),fh.forEach(t),et=n(Ur," function:"),Ur.forEach(t),Qt=d(e),y(Le.$$.fragment,e),es=d(e),Ye=r(e,"P",{});var Br=i(Ye);ts=n(Br,"If you want to work with a different pair of languages, you can specify them by their codes. A total of 92 languages are available for this dataset; you can see them all by expanding the language tags on its "),yt=r(Br,"A",{href:!0,rel:!0});var mh=i(yt);Nt=n(mh,"dataset card"),mh.forEach(t),hs=n(Br,"."),Br.forEach(t),ct=d(e),f=r(e,"IMG",{src:!0,alt:!0,width:!0}),vs=d(e),jt=r(e,"P",{});var _h=i(jt);Xs=n(_h,"Let\u2019s have a look at the dataset:"),_h.forEach(t),ys=d(e),y(Te.$$.fragment,e),js=d(e),y(Et.$$.fragment,e),cs=d(e),ue=r(e,"P",{});var Ra=i(ue);Gs=n(Ra,"We have 210,173 pairs of sentences, but in one single split, so we will need to create our own validation set. As we saw in "),dt=r(Ra,"A",{href:!0});var gh=i(dt);Zs=n(gh,"Chapter 5"),gh.forEach(t),We=n(Ra,", a "),Ht=r(Ra,"CODE",{});var wh=i(Ht);Es=n(wh,"Dataset"),wh.forEach(t),Js=n(Ra," has a "),ss=r(Ra,"CODE",{});var bh=i(ss);Ke=n(bh,"train_test_split()"),bh.forEach(t),Vs=n(Ra," method that can help us. We\u2019ll provide a seed for reproducibility:"),Ra.forEach(t),xt=d(e),y(Wt.$$.fragment,e),xs=d(e),y(qt.$$.fragment,e),qs=d(e),ut=r(e,"P",{});var Bo=i(ut);ds=n(Bo,"We can rename the "),us=r(Bo,"CODE",{});var $h=i(us);Mt=n($h,'"test"'),$h.forEach(t),Ts=n(Bo," key to "),Tt=r(Bo,"CODE",{});var kh=i(Tt);zs=n(kh,'"validation"'),kh.forEach(t),Ut=n(Bo," like this:"),Bo.forEach(t),fs=d(e),y(tt.$$.fragment,e),Ea=d(e),ms=r(e,"P",{});var vh=i(ms);Xa=n(vh,"Now let\u2019s take a look at one element of the dataset:"),vh.forEach(t),xa=d(e),y(St.$$.fragment,e),Qs=d(e),y(Ds.$$.fragment,e),qa=d(e),As=r(e,"P",{});var yh=i(As);u=n(yh,"We get a dictionary with two sentences in the pair of languages we requested. One particularity of this dataset full of technical computer science terms is that they are all fully translated in French. However, French engineers are often lazy and leave most computer science-specific words in English when they talk. Here, for instance, the word \u201Cthreads\u201D might well appear in a French sentence, especially in a technical conversation; but in this dataset it has been translated into the more correct \u201Cfils de discussion.\u201D The pretrained model we use, which has been pretrained on a larger corpus of French and English sentences, takes the easier option of leaving the word as is:"),yh.forEach(t),I=d(e),y(ea.$$.fragment,e),Ga=d(e),y(ta.$$.fragment,e),Ps=d(e),_a=r(e,"P",{});var jh=i(_a);Qn=n(jh,`Another example of this behavior can be seen with the word \u201Cplugin,\u201D which isn\u2019t officially a French word but which most native speakers will understand and not bother to translate.
In the KDE4 dataset this word has been translated in French into the more official \u201Cmodule d\u2019extension\u201D:`),jh.forEach(t),ga=d(e),y(sa.$$.fragment,e),Za=d(e),y(aa.$$.fragment,e),Ss=d(e),wa=r(e,"P",{});var Eh=i(wa);eo=n(Eh,"Our pretrained model, however, sticks with the compact and familiar English word:"),Eh.forEach(t),Ja=d(e),y(na.$$.fragment,e),Cs=d(e),y(oa.$$.fragment,e),Va=d(e),Ct=r(e,"P",{});var xh=i(Ct);to=n(xh,"It will be interesting to see if our fine-tuned model picks up on those particularities of the dataset (spoiler alert: it will)."),xh.forEach(t),Qa=d(e),y(la.$$.fragment,e),as=d(e),y(Os.$$.fragment,e),en=d(e),_s=r(e,"H3",{class:!0});var Rr=i(_s);Fs=r(Rr,"A",{id:!0,class:!0,href:!0});var qh=i(Fs);gs=r(qh,"SPAN",{});var Th=i(gs);y(ra.$$.fragment,Th),Th.forEach(t),qh.forEach(t),so=d(Rr),ia=r(Rr,"SPAN",{});var zh=i(ia);ao=n(zh,"Processing the data"),zh.forEach(t),Rr.forEach(t),tn=d(e),y(pa.$$.fragment,e),Ls=d(e),ns=r(e,"P",{});var Ro=i(ns);no=n(Ro,"You should know the drill by now: the texts all need to be converted into sets of token IDs so the model can make sense of them. For this task, we\u2019ll need to tokenize both the inputs and the targets. Our first task is to create our "),ws=r(Ro,"CODE",{});var Dh=i(ws);oo=n(Dh,"tokenizer"),Dh.forEach(t),lo=n(Ro," object. As noted earlier, we\u2019ll be using a Marian English to French pretrained model. If you are trying this code with another pair of languages, make sure to adapt the model checkpoint. The "),Bt=r(Ro,"A",{href:!0,rel:!0});var Ah=i(Bt);ro=n(Ah,"Helsinki-NLP"),Ah.forEach(t),io=n(Ro," organization provides more than a thousand models in multiple languages."),Ro.forEach(t),os=d(e),y(ha.$$.fragment,e),sn=d(e),ls=r(e,"P",{});var Yo=i(ls);ca=n(Yo,"You can also replace the "),Ta=r(Yo,"CODE",{});var Ph=i(Ta);po=n(Ph,"model_checkpoint"),Ph.forEach(t),ho=n(Yo," with any other model you prefer from the "),an=r(Yo,"A",{href:!0,rel:!0});var Sh=i(an);ai=n(Sh,"Hub"),Sh.forEach(t),ni=n(Yo,", or a local folder where you\u2019ve saved a pretrained model and a tokenizer."),Yo.forEach(t),El=d(e),y(za.$$.fragment,e),xl=d(e),da=r(e,"P",{});var Ko=i(da);oi=n(Ko,"The preparation of our data is pretty straightforward. There\u2019s just one thing to remember; you need to ensure that the tokenizer processes the targets in the output language (here, French). You can do this by passing the targets to the "),Go=r(Ko,"CODE",{});var Ch=i(Go);li=n(Ch,"text_targets"),Ch.forEach(t),ri=n(Ko," argument of the tokenizer\u2019s "),Zo=r(Ko,"CODE",{});var Oh=i(Zo);ii=n(Oh,"__call__"),Oh.forEach(t),pi=n(Ko," method."),Ko.forEach(t),ql=d(e),co=r(e,"P",{});var Fh=i(co);hi=n(Fh,"To see how this works, let\u2019s process one sample of each language in the training set:"),Fh.forEach(t),Tl=d(e),y(nn.$$.fragment,e),zl=d(e),y(on.$$.fragment,e),Dl=d(e),Da=r(e,"P",{});var Yr=i(Da);ci=n(Yr,"As we can see, the output contains the input IDs associated with the English sentence, while the IDs associated with the French one are stored in the "),Jo=r(Yr,"CODE",{});var Lh=i(Jo);di=n(Lh,"labels"),Lh.forEach(t),ui=n(Yr," field. If you forget to indicate that you are tokenizing labels, they will be tokenized by the input tokenizer, which in the case of a Marian model is not going to go well at all:"),Yr.forEach(t),Al=d(e),y(ln.$$.fragment,e),Pl=d(e),y(rn.$$.fragment,e),Sl=d(e),uo=r(e,"P",{});var Ih=i(uo);fi=n(Ih,"As we can see, using the English tokenizer to preprocess a French sentence results in a lot more tokens, since the tokenizer doesn\u2019t know any French words (except those that also appear in the English language, like \u201Cdiscussion\u201D)."),Ih.forEach(t),Cl=d(e),Aa=r(e,"P",{});var Kr=i(Aa);mi=n(Kr,"Since "),Vo=r(Kr,"CODE",{});var Nh=i(Vo);_i=n(Nh,"inputs"),Nh.forEach(t),gi=n(Kr," is a dictionary with our usual keys (input IDs, attention mask, etc.), the last step is to define the preprocessing function we will apply on the datasets:"),Kr.forEach(t),Ol=d(e),y(pn.$$.fragment,e),Fl=d(e),fo=r(e,"P",{});var Hh=i(fo);wi=n(Hh,"Note that we set the same maximum length for our inputs and outputs. Since the texts we\u2019re dealing with seem pretty short, we use 128."),Hh.forEach(t),Ll=d(e),y(Pa.$$.fragment,e),Il=d(e),y(Sa.$$.fragment,e),Nl=d(e),mo=r(e,"P",{});var Wh=i(mo);bi=n(Wh,"We can now apply that preprocessing in one go on all the splits of our dataset:"),Wh.forEach(t),Hl=d(e),y(hn.$$.fragment,e),Wl=d(e),_o=r(e,"P",{});var Mh=i(_o);$i=n(Mh,"Now that the data has been preprocessed, we are ready to fine-tune our pretrained model!"),Mh.forEach(t),Ml=d(e),Ns.l(e),go=d(e),wo=r(e,"P",{});var Uh=i(wo);ki=n(Uh,"Note that this time we are using a model that was trained on a translation task and can actually be used already, so there is no warning about missing weights or newly initialized ones."),Uh.forEach(t),Ul=d(e),ba=r(e,"H3",{class:!0});var Xr=i(ba);Ca=r(Xr,"A",{id:!0,class:!0,href:!0});var Bh=i(Ca);Qo=r(Bh,"SPAN",{});var Rh=i(Qo);y(cn.$$.fragment,Rh),Rh.forEach(t),Bh.forEach(t),vi=d(Xr),el=r(Xr,"SPAN",{});var Yh=i(el);yi=n(Yh,"Data collation"),Yh.forEach(t),Xr.forEach(t),Bl=d(e),bs=r(e,"P",{});var Ya=i(bs);ji=n(Ya,"We\u2019ll need a data collator to deal with the padding for dynamic batching. We can\u2019t just use a "),tl=r(Ya,"CODE",{});var Kh=i(tl);Ei=n(Kh,"DataCollatorWithPadding"),Kh.forEach(t),xi=n(Ya," like in "),bo=r(Ya,"A",{href:!0});var Xh=i(bo);qi=n(Xh,"Chapter 3"),Xh.forEach(t),Ti=n(Ya," in this case, because that only pads the inputs (input IDs, attention mask, and token type IDs). Our labels should also be padded to the maximum length encountered in the labels. And, as mentioned previously, the padding value used to pad the labels should be "),sl=r(Ya,"CODE",{});var Gh=i(sl);zi=n(Gh,"-100"),Gh.forEach(t),Di=n(Ya," and not the padding token of the tokenizer, to make sure those padded values are ignored in the loss computation."),Ya.forEach(t),Rl=d(e),st=r(e,"P",{});var Kt=i(st);Ai=n(Kt,"This is all done by a "),dn=r(Kt,"A",{href:!0,rel:!0});var Zh=i(dn);al=r(Zh,"CODE",{});var Jh=i(al);Pi=n(Jh,"DataCollatorForSeq2Seq"),Jh.forEach(t),Zh.forEach(t),Si=n(Kt,". Like the "),nl=r(Kt,"CODE",{});var Vh=i(nl);Ci=n(Vh,"DataCollatorWithPadding"),Vh.forEach(t),Oi=n(Kt,", it takes the "),ol=r(Kt,"CODE",{});var Qh=i(ol);Fi=n(Qh,"tokenizer"),Qh.forEach(t),Li=n(Kt," used to preprocess the inputs, but it also takes the "),ll=r(Kt,"CODE",{});var ec=i(ll);Ii=n(ec,"model"),ec.forEach(t),Ni=n(Kt,". This is because this data collator will also be responsible for preparing the decoder input IDs, which are shifted versions of the labels with a special token at the beginning. Since this shift is done slightly differently for different architectures, the "),rl=r(Kt,"CODE",{});var tc=i(rl);Hi=n(tc,"DataCollatorForSeq2Seq"),tc.forEach(t),Wi=n(Kt," needs to know the "),il=r(Kt,"CODE",{});var sc=i(il);Mi=n(sc,"model"),sc.forEach(t),Ui=n(Kt," object:"),Kt.forEach(t),Yl=d(e),Ws.l(e),$o=d(e),ko=r(e,"P",{});var ac=i(ko);Bi=n(ac,"To test this on a few samples, we just call it on a list of examples from our tokenized training set:"),ac.forEach(t),Kl=d(e),y(un.$$.fragment,e),Xl=d(e),y(fn.$$.fragment,e),Gl=d(e),Oa=r(e,"P",{});var Gr=i(Oa);Ri=n(Gr,"We can check our labels have been padded to the maximum length of the batch, using "),pl=r(Gr,"CODE",{});var nc=i(pl);Yi=n(nc,"-100"),nc.forEach(t),Ki=n(Gr,":"),Gr.forEach(t),Zl=d(e),y(mn.$$.fragment,e),Jl=d(e),y(_n.$$.fragment,e),Vl=d(e),vo=r(e,"P",{});var oc=i(vo);Xi=n(oc,"And we can also have a look at the decoder input IDs, to see that they are shifted versions of the labels:"),oc.forEach(t),Ql=d(e),y(gn.$$.fragment,e),er=d(e),y(wn.$$.fragment,e),tr=d(e),yo=r(e,"P",{});var lc=i(yo);Gi=n(lc,"Here are the labels for the first and second elements in our dataset:"),lc.forEach(t),sr=d(e),y(bn.$$.fragment,e),ar=d(e),y($n.$$.fragment,e),nr=d(e),Us.l(e),jo=d(e),$a=r(e,"H3",{class:!0});var Zr=i($a);Fa=r(Zr,"A",{id:!0,class:!0,href:!0});var rc=i(Fa);hl=r(rc,"SPAN",{});var ic=i(hl);y(kn.$$.fragment,ic),ic.forEach(t),rc.forEach(t),Zi=d(Zr),cl=r(Zr,"SPAN",{});var pc=i(cl);Ji=n(pc,"Metrics"),pc.forEach(t),Zr.forEach(t),or=d(e),y(vn.$$.fragment,e),lr=d(e),Ot&&Ot.l(e),Eo=d(e),Rt=r(e,"P",{});var fa=i(Rt);Vi=n(fa,"The traditional metric used for translation is the "),yn=r(fa,"A",{href:!0,rel:!0});var hc=i(yn);Qi=n(hc,"BLEU score"),hc.forEach(t),ep=n(fa,", introduced in "),jn=r(fa,"A",{href:!0,rel:!0});var cc=i(jn);tp=n(cc,"a 2002 article"),cc.forEach(t),sp=n(fa," by Kishore Papineni et al. The BLEU score evaluates how close the translations are to their labels. It does not measure the intelligibility or grammatical correctness of the model\u2019s generated outputs, but uses statistical rules to ensure that all the words in the generated outputs also appear in the targets. In addition, there are rules that penalize repetitions of the same words if they are not also repeated in the targets (to avoid the model outputting sentences like "),dl=r(fa,"CODE",{});var dc=i(dl);ap=n(dc,'"the the the the the"'),dc.forEach(t),np=n(fa,") and output sentences that are shorter than those in the targets (to avoid the model outputting sentences like "),ul=r(fa,"CODE",{});var uc=i(ul);op=n(uc,'"the"'),uc.forEach(t),lp=n(fa,")."),fa.forEach(t),rr=d(e),La=r(e,"P",{});var Jr=i(La);rp=n(Jr,"One weakness with BLEU is that it expects the text to already be tokenized, which makes it difficult to compare scores between models that use different tokenizers. So instead, the most commonly used metric for benchmarking translation models today is "),En=r(Jr,"A",{href:!0,rel:!0});var fc=i(En);ip=n(fc,"SacreBLEU"),fc.forEach(t),pp=n(Jr,", which addresses this weakness (and others) by standardizing the tokenization step. To use this metric, we first need to install the SacreBLEU library:"),Jr.forEach(t),ir=d(e),y(xn.$$.fragment,e),pr=d(e),ua=r(e,"P",{});var Xo=i(ua);hp=n(Xo,"We can then load it via "),fl=r(Xo,"CODE",{});var mc=i(fl);cp=n(mc,"evaluate.load()"),mc.forEach(t),dp=n(Xo," like we did in "),xo=r(Xo,"A",{href:!0});var _c=i(xo);up=n(_c,"Chapter 3"),_c.forEach(t),fp=n(Xo,":"),Xo.forEach(t),hr=d(e),y(qn.$$.fragment,e),cr=d(e),qo=r(e,"P",{});var gc=i(qo);mp=n(gc,"This metric will take texts as inputs and targets. It is designed to accept several acceptable targets, as there are often multiple acceptable translations of the same sentence \u2014 the dataset we\u2019re using only provides one, but it\u2019s not uncommon in NLP to find datasets that give several sentences as labels. So, the predictions should be a list of sentences, but the references should be a list of lists of sentences."),gc.forEach(t),dr=d(e),To=r(e,"P",{});var wc=i(To);_p=n(wc,"Let\u2019s try an example:"),wc.forEach(t),ur=d(e),y(Tn.$$.fragment,e),fr=d(e),y(zn.$$.fragment,e),mr=d(e),Yt=r(e,"P",{});var ma=i(Yt);gp=n(ma,"This gets a BLEU score of 46.75, which is rather good \u2014 for reference, the original Transformer model in the "),Dn=r(ma,"A",{href:!0,rel:!0});var bc=i(Dn);wp=n(bc,"\u201CAttention Is All You Need\u201D paper"),bc.forEach(t),bp=n(ma," achieved a BLEU score of 41.8 on a similar translation task between English and French! (For more information about the individual metrics, like "),ml=r(ma,"CODE",{});var $c=i(ml);$p=n($c,"counts"),$c.forEach(t),kp=n(ma," and "),_l=r(ma,"CODE",{});var kc=i(_l);vp=n(kc,"bp"),kc.forEach(t),yp=n(ma,", see the "),An=r(ma,"A",{href:!0,rel:!0});var vc=i(An);jp=n(vc,"SacreBLEU repository"),vc.forEach(t),Ep=n(ma,".) On the other hand, if we try with the two bad types of predictions (lots of repetitions or too short) that often come out of translation models, we will get rather bad BLEU scores:"),ma.forEach(t),_r=d(e),y(Pn.$$.fragment,e),gr=d(e),y(Sn.$$.fragment,e),wr=d(e),y(Cn.$$.fragment,e),br=d(e),y(On.$$.fragment,e),$r=d(e),zo=r(e,"P",{});var yc=i(zo);xp=n(yc,"The score can go from 0 to 100, and higher is better."),yc.forEach(t),kr=d(e),Rs.l(e),Do=d(e),Ao=r(e,"P",{});var jc=i(Ao);qp=n(jc,"Now that this is done, we are ready to fine-tune our model!"),jc.forEach(t),vr=d(e),ka=r(e,"H3",{class:!0});var Vr=i(ka);Ia=r(Vr,"A",{id:!0,class:!0,href:!0});var Ec=i(Ia);gl=r(Ec,"SPAN",{});var xc=i(gl);y(Fn.$$.fragment,xc),xc.forEach(t),Ec.forEach(t),Tp=d(Vr),wl=r(Vr,"SPAN",{});var qc=i(wl);zp=n(qc,"Fine-tuning the model"),qc.forEach(t),Vr.forEach(t),yr=d(e),Po=r(e,"P",{});var Tc=i(Po);Dp=n(Tc,"The first step is to log in to Hugging Face, so you\u2019re able to upload your results to the Model Hub. There\u2019s a convenience function to help you with this in a notebook:"),Tc.forEach(t),jr=d(e),y(Ln.$$.fragment,e),Er=d(e),So=r(e,"P",{});var zc=i(So);Ap=n(zc,"This will display a widget where you can enter your Hugging Face login credentials."),zc.forEach(t),xr=d(e),Co=r(e,"P",{});var Dc=i(Co);Pp=n(Dc,"If you aren\u2019t working in a notebook, just type the following line in your terminal:"),Dc.forEach(t),qr=d(e),y(In.$$.fragment,e),Tr=d(e),Ks.l(e),Oo=d(e),Xe&&Xe.l(e),Fo=d(e),va=r(e,"H2",{class:!0});var Qr=i(va);Na=r(Qr,"A",{id:!0,class:!0,href:!0});var Ac=i(Na);bl=r(Ac,"SPAN",{});var Pc=i(bl);y(Nn.$$.fragment,Pc),Pc.forEach(t),Ac.forEach(t),Sp=d(Qr),$l=r(Qr,"SPAN",{});var Sc=i($l);Cp=n(Sc,"Using the fine-tuned model"),Sc.forEach(t),Qr.forEach(t),zr=d(e),Ha=r(e,"P",{});var ei=i(Ha);Op=n(ei,"We\u2019ve already shown you how you can use the model we fine-tuned on the Model Hub with the inference widget. To use it locally in a "),kl=r(ei,"CODE",{});var Cc=i(kl);Fp=n(Cc,"pipeline"),Cc.forEach(t),Lp=n(ei,", we just have to specify the proper model identifier:"),ei.forEach(t),Dr=d(e),y(Hn.$$.fragment,e),Ar=d(e),y(Wn.$$.fragment,e),Pr=d(e),Lo=r(e,"P",{});var Oc=i(Lo);Ip=n(Oc,"As expected, our pretrained model adapted its knowledge to the corpus we fine-tuned it on, and instead of leaving the English word \u201Cthreads\u201D alone, it now translates it to the French official version. It\u2019s the same for \u201Cplugin\u201D:"),Oc.forEach(t),Sr=d(e),y(Mn.$$.fragment,e),Cr=d(e),y(Un.$$.fragment,e),Or=d(e),Io=r(e,"P",{});var Fc=i(Io);Np=n(Fc,"Another great example of domain adaptation!"),Fc.forEach(t),Fr=d(e),y(Wa.$$.fragment,e),this.h()},h(){_(h,"name","hf:doc:metadata"),_(h,"content",JSON.stringify(fd)),_(g,"id","translation"),_(g,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),_(g,"href","#translation"),_(P,"class","relative group"),_(H,"href","/course/chapter1/7"),_(B,"href","/course/chapter7/6"),_(xe,"href","/course/chapter7/6"),_(ge,"href","https://huggingface.co/datasets/kde4"),_(ge,"rel","nofollow"),_(ve,"href","https://apps.kde.org/"),_(ve,"rel","nofollow"),_(Se,"href","https://opus.nlpl.eu/"),_(Se,"rel","nofollow"),ti(ce.src,Be="https://hf.space/gradioiframe/course-demos/marian-finetuned-kde4-en-to-fr/+")||_(ce,"src",Be),_(ce,"frameborder","0"),_(ce,"height","350"),_(ce,"title","Gradio app"),_(ce,"class","block dark:hidden container p-0 flex-grow space-iframe"),_(ce,"allow","accelerometer; ambient-light-sensor; autoplay; battery; camera; document-domain; encrypted-media; fullscreen; geolocation; gyroscope; layout-animations; legacy-image-formats; magnetometer; microphone; midi; oversized-images; payment; picture-in-picture; publickey-credentials-get; sync-xhr; usb; vr ; wake-lock; xr-spatial-tracking"),_(ce,"sandbox","allow-forms allow-modals allow-popups allow-popups-to-escape-sandbox allow-same-origin allow-scripts allow-downloads"),_(pe,"class","block dark:hidden lg:w-3/5"),ti(pe.src,Ft="https://huggingface.co/datasets/huggingface-course/documentation-images/resolve/main/en/chapter7/modeleval-marian-finetuned-kde4-en-to-fr.png")||_(pe,"src",Ft),_(pe,"alt","One-hot encoded labels for question answering."),_(Ie,"class","hidden dark:block lg:w-3/5"),ti(Ie.src,Ne="https://huggingface.co/datasets/huggingface-course/documentation-images/resolve/main/en/chapter7/modeleval-marian-finetuned-kde4-en-to-fr-dark.png")||_(Ie,"src",Ne),_(Ie,"alt","One-hot encoded labels for question answering."),_(Ce,"class","flex justify-center"),_(Ce,"href","/huggingface-course/marian-finetuned-kde4-en-to-fr"),_(Oe,"href","https://huggingface.co/huggingface-course/marian-finetuned-kde4-en-to-fr?text=This+plugin+allows+you+to+automatically+translate+web+pages+between+several+languages."),_(Oe,"rel","nofollow"),_(rt,"id","preparing-the-data"),_(rt,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),_(rt,"href","#preparing-the-data"),_(lt,"class","relative group"),_(Fe,"href","https://huggingface.co/datasets/kde4"),_(Fe,"rel","nofollow"),_(De,"href","/course/chapter5"),_($t,"id","the-kde4-dataset"),_($t,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),_($t,"href","#the-kde4-dataset"),_(be,"class","relative group"),_(yt,"href","https://huggingface.co/datasets/kde4"),_(yt,"rel","nofollow"),ti(f.src,O="https://huggingface.co/datasets/huggingface-course/documentation-images/resolve/main/en/chapter7/language_tags.png")||_(f,"src",O),_(f,"alt","Language available for the KDE4 dataset."),_(f,"width","100%"),_(dt,"href","/course/chapter5"),_(Fs,"id","processing-the-data"),_(Fs,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),_(Fs,"href","#processing-the-data"),_(_s,"class","relative group"),_(Bt,"href","https://huggingface.co/Helsinki-NLP"),_(Bt,"rel","nofollow"),_(an,"href","https://huggingface.co/models"),_(an,"rel","nofollow"),_(Ca,"id","data-collation"),_(Ca,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),_(Ca,"href","#data-collation"),_(ba,"class","relative group"),_(bo,"href","/course/chapter3"),_(dn,"href","https://huggingface.co/transformers/main_classes/data_collator.html#datacollatorforseq2seq"),_(dn,"rel","nofollow"),_(Fa,"id","metrics"),_(Fa,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),_(Fa,"href","#metrics"),_($a,"class","relative group"),_(yn,"href","https://en.wikipedia.org/wiki/BLEU"),_(yn,"rel","nofollow"),_(jn,"href","https://aclanthology.org/P02-1040.pdf"),_(jn,"rel","nofollow"),_(En,"href","https://github.com/mjpost/sacrebleu"),_(En,"rel","nofollow"),_(xo,"href","/course/chapter3"),_(Dn,"href","https://arxiv.org/pdf/1706.03762.pdf"),_(Dn,"rel","nofollow"),_(An,"href","https://github.com/mjpost/sacrebleu/blob/078c440168c6adc89ba75fe6d63f0d922d42bcfe/sacrebleu/metrics/bleu.py#L74"),_(An,"rel","nofollow"),_(Ia,"id","finetuning-the-model"),_(Ia,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),_(Ia,"href","#finetuning-the-model"),_(ka,"class","relative group"),_(Na,"id","using-the-finetuned-model"),_(Na,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),_(Na,"href","#using-the-finetuned-model"),_(va,"class","relative group")},m(e,p){s(document.head,h),o(e,$,p),j(m,e,p),o(e,x,p),o(e,P,p),s(P,g),s(g,D),j(A,D,null),s(P,T),s(P,z),s(z,N),o(e,q,p),Bn[F].m(e,p),o(e,U,p),o(e,L,p),s(L,G),s(L,H),s(H,C),s(L,K),s(L,B),s(B,Y),s(L,re),o(e,V,p),o(e,Q,p),s(Q,ee),s(ee,Z),s(Z,oe),s(ee,$e),s(ee,se),s(se,ae),s(ee,fe),s(Q,ke),s(Q,le),s(le,he),s(he,M),s(le,ne),o(e,ze,p),j(me,e,p),o(e,J,p),o(e,ye,p),s(ye,ie),s(ye,xe),s(xe,Me),s(ye,_e),o(e,qe,p),o(e,te,p),s(te,Pe),s(te,ge),s(ge,at),s(te,ft),s(te,ve),s(ve,Ge),s(te,Ue),s(te,Se),s(Se,je),s(te,mt),o(e,we,p),o(e,k,p),s(k,X),o(e,Ze,p),o(e,ce,p),o(e,nt,p),o(e,Ce,p),s(Ce,pe),s(Ce,Re),s(Ce,Ie),o(e,_t,p),o(e,de,p),s(de,Je),s(de,Oe),s(Oe,zt),s(de,is),o(e,ot,p),o(e,lt,p),s(lt,rt),s(rt,Ve),j(Dt,Ve,null),s(lt,ps),s(lt,Qe),s(Qe,gt),o(e,wt,p),o(e,Ee,p),s(Ee,Xt),s(Ee,Fe),s(Fe,$s),s(Ee,Gt),s(Ee,De),s(De,Zt),s(Ee,Ae),s(Ee,Lt),s(Lt,bt),s(Ee,ks),o(e,Jt,p),o(e,be,p),s(be,$t),s($t,At),j(kt,At,null),s(be,it),s(be,He),s(He,Vt),o(e,vt,p),o(e,pt,p),s(pt,ht),s(pt,It),s(It,Pt),s(pt,et),o(e,Qt,p),j(Le,e,p),o(e,es,p),o(e,Ye,p),s(Ye,ts),s(Ye,yt),s(yt,Nt),s(Ye,hs),o(e,ct,p),o(e,f,p),o(e,vs,p),o(e,jt,p),s(jt,Xs),o(e,ys,p),j(Te,e,p),o(e,js,p),j(Et,e,p),o(e,cs,p),o(e,ue,p),s(ue,Gs),s(ue,dt),s(dt,Zs),s(ue,We),s(ue,Ht),s(Ht,Es),s(ue,Js),s(ue,ss),s(ss,Ke),s(ue,Vs),o(e,xt,p),j(Wt,e,p),o(e,xs,p),j(qt,e,p),o(e,qs,p),o(e,ut,p),s(ut,ds),s(ut,us),s(us,Mt),s(ut,Ts),s(ut,Tt),s(Tt,zs),s(ut,Ut),o(e,fs,p),j(tt,e,p),o(e,Ea,p),o(e,ms,p),s(ms,Xa),o(e,xa,p),j(St,e,p),o(e,Qs,p),j(Ds,e,p),o(e,qa,p),o(e,As,p),s(As,u),o(e,I,p),j(ea,e,p),o(e,Ga,p),j(ta,e,p),o(e,Ps,p),o(e,_a,p),s(_a,Qn),o(e,ga,p),j(sa,e,p),o(e,Za,p),j(aa,e,p),o(e,Ss,p),o(e,wa,p),s(wa,eo),o(e,Ja,p),j(na,e,p),o(e,Cs,p),j(oa,e,p),o(e,Va,p),o(e,Ct,p),s(Ct,to),o(e,Qa,p),j(la,e,p),o(e,as,p),j(Os,e,p),o(e,en,p),o(e,_s,p),s(_s,Fs),s(Fs,gs),j(ra,gs,null),s(_s,so),s(_s,ia),s(ia,ao),o(e,tn,p),j(pa,e,p),o(e,Ls,p),o(e,ns,p),s(ns,no),s(ns,ws),s(ws,oo),s(ns,lo),s(ns,Bt),s(Bt,ro),s(ns,io),o(e,os,p),j(ha,e,p),o(e,sn,p),o(e,ls,p),s(ls,ca),s(ls,Ta),s(Ta,po),s(ls,ho),s(ls,an),s(an,ai),s(ls,ni),o(e,El,p),j(za,e,p),o(e,xl,p),o(e,da,p),s(da,oi),s(da,Go),s(Go,li),s(da,ri),s(da,Zo),s(Zo,ii),s(da,pi),o(e,ql,p),o(e,co,p),s(co,hi),o(e,Tl,p),j(nn,e,p),o(e,zl,p),j(on,e,p),o(e,Dl,p),o(e,Da,p),s(Da,ci),s(Da,Jo),s(Jo,di),s(Da,ui),o(e,Al,p),j(ln,e,p),o(e,Pl,p),j(rn,e,p),o(e,Sl,p),o(e,uo,p),s(uo,fi),o(e,Cl,p),o(e,Aa,p),s(Aa,mi),s(Aa,Vo),s(Vo,_i),s(Aa,gi),o(e,Ol,p),j(pn,e,p),o(e,Fl,p),o(e,fo,p),s(fo,wi),o(e,Ll,p),j(Pa,e,p),o(e,Il,p),j(Sa,e,p),o(e,Nl,p),o(e,mo,p),s(mo,bi),o(e,Hl,p),j(hn,e,p),o(e,Wl,p),o(e,_o,p),s(_o,$i),o(e,Ml,p),Rn[Is].m(e,p),o(e,go,p),o(e,wo,p),s(wo,ki),o(e,Ul,p),o(e,ba,p),s(ba,Ca),s(Ca,Qo),j(cn,Qo,null),s(ba,vi),s(ba,el),s(el,yi),o(e,Bl,p),o(e,bs,p),s(bs,ji),s(bs,tl),s(tl,Ei),s(bs,xi),s(bs,bo),s(bo,qi),s(bs,Ti),s(bs,sl),s(sl,zi),s(bs,Di),o(e,Rl,p),o(e,st,p),s(st,Ai),s(st,dn),s(dn,al),s(al,Pi),s(st,Si),s(st,nl),s(nl,Ci),s(st,Oi),s(st,ol),s(ol,Fi),s(st,Li),s(st,ll),s(ll,Ii),s(st,Ni),s(st,rl),s(rl,Hi),s(st,Wi),s(st,il),s(il,Mi),s(st,Ui),o(e,Yl,p),Yn[Hs].m(e,p),o(e,$o,p),o(e,ko,p),s(ko,Bi),o(e,Kl,p),j(un,e,p),o(e,Xl,p),j(fn,e,p),o(e,Gl,p),o(e,Oa,p),s(Oa,Ri),s(Oa,pl),s(pl,Yi),s(Oa,Ki),o(e,Zl,p),j(mn,e,p),o(e,Jl,p),j(_n,e,p),o(e,Vl,p),o(e,vo,p),s(vo,Xi),o(e,Ql,p),j(gn,e,p),o(e,er,p),j(wn,e,p),o(e,tr,p),o(e,yo,p),s(yo,Gi),o(e,sr,p),j(bn,e,p),o(e,ar,p),j($n,e,p),o(e,nr,p),Kn[Ms].m(e,p),o(e,jo,p),o(e,$a,p),s($a,Fa),s(Fa,hl),j(kn,hl,null),s($a,Zi),s($a,cl),s(cl,Ji),o(e,or,p),j(vn,e,p),o(e,lr,p),Ot&&Ot.m(e,p),o(e,Eo,p),o(e,Rt,p),s(Rt,Vi),s(Rt,yn),s(yn,Qi),s(Rt,ep),s(Rt,jn),s(jn,tp),s(Rt,sp),s(Rt,dl),s(dl,ap),s(Rt,np),s(Rt,ul),s(ul,op),s(Rt,lp),o(e,rr,p),o(e,La,p),s(La,rp),s(La,En),s(En,ip),s(La,pp),o(e,ir,p),j(xn,e,p),o(e,pr,p),o(e,ua,p),s(ua,hp),s(ua,fl),s(fl,cp),s(ua,dp),s(ua,xo),s(xo,up),s(ua,fp),o(e,hr,p),j(qn,e,p),o(e,cr,p),o(e,qo,p),s(qo,mp),o(e,dr,p),o(e,To,p),s(To,_p),o(e,ur,p),j(Tn,e,p),o(e,fr,p),j(zn,e,p),o(e,mr,p),o(e,Yt,p),s(Yt,gp),s(Yt,Dn),s(Dn,wp),s(Yt,bp),s(Yt,ml),s(ml,$p),s(Yt,kp),s(Yt,_l),s(_l,vp),s(Yt,yp),s(Yt,An),s(An,jp),s(Yt,Ep),o(e,_r,p),j(Pn,e,p),o(e,gr,p),j(Sn,e,p),o(e,wr,p),j(Cn,e,p),o(e,br,p),j(On,e,p),o(e,$r,p),o(e,zo,p),s(zo,xp),o(e,kr,p),Xn[Bs].m(e,p),o(e,Do,p),o(e,Ao,p),s(Ao,qp),o(e,vr,p),o(e,ka,p),s(ka,Ia),s(Ia,gl),j(Fn,gl,null),s(ka,Tp),s(ka,wl),s(wl,zp),o(e,yr,p),o(e,Po,p),s(Po,Dp),o(e,jr,p),j(Ln,e,p),o(e,Er,p),o(e,So,p),s(So,Ap),o(e,xr,p),o(e,Co,p),s(Co,Pp),o(e,qr,p),j(In,e,p),o(e,Tr,p),Gn[Ys].m(e,p),o(e,Oo,p),Xe&&Xe.m(e,p),o(e,Fo,p),o(e,va,p),s(va,Na),s(Na,bl),j(Nn,bl,null),s(va,Sp),s(va,$l),s($l,Cp),o(e,zr,p),o(e,Ha,p),s(Ha,Op),s(Ha,kl),s(kl,Fp),s(Ha,Lp),o(e,Dr,p),j(Hn,e,p),o(e,Ar,p),j(Wn,e,p),o(e,Pr,p),o(e,Lo,p),s(Lo,Ip),o(e,Sr,p),j(Mn,e,p),o(e,Cr,p),j(Un,e,p),o(e,Or,p),o(e,Io,p),s(Io,Np),o(e,Fr,p),j(Wa,e,p),Lr=!0},p(e,[p]){const Zn={};p&1&&(Zn.fw=e[0]),m.$set(Zn);let No=F;F=Wp(e),F!==No&&(Vn(),b(Bn[No],1,1,()=>{Bn[No]=null}),Jn(),W=Bn[F],W||(W=Bn[F]=Hp[F](e),W.c()),w(W,1),W.m(U.parentNode,U));const vl={};p&2&&(vl.$$scope={dirty:p,ctx:e}),Os.$set(vl);const yl={};p&2&&(yl.$$scope={dirty:p,ctx:e}),za.$set(yl);const ya={};p&2&&(ya.$$scope={dirty:p,ctx:e}),Pa.$set(ya);const jl={};p&2&&(jl.$$scope={dirty:p,ctx:e}),Sa.$set(jl);let Ho=Is;Is=Up(e),Is!==Ho&&(Vn(),b(Rn[Ho],1,1,()=>{Rn[Ho]=null}),Jn(),Ns=Rn[Is],Ns||(Ns=Rn[Is]=Mp[Is](e),Ns.c()),w(Ns,1),Ns.m(go.parentNode,go));let Ma=Hs;Hs=Rp(e),Hs!==Ma&&(Vn(),b(Yn[Ma],1,1,()=>{Yn[Ma]=null}),Jn(),Ws=Yn[Hs],Ws||(Ws=Yn[Hs]=Bp[Hs](e),Ws.c()),w(Ws,1),Ws.m($o.parentNode,$o));let ja=Ms;Ms=Kp(e),Ms!==ja&&(Vn(),b(Kn[ja],1,1,()=>{Kn[ja]=null}),Jn(),Us=Kn[Ms],Us||(Us=Kn[Ms]=Yp[Ms](e),Us.c()),w(Us,1),Us.m(jo.parentNode,jo)),e[0]==="pt"?Ot||(Ot=Ic(),Ot.c(),Ot.m(Eo.parentNode,Eo)):Ot&&(Ot.d(1),Ot=null);let Wo=Bs;Bs=Gp(e),Bs!==Wo&&(Vn(),b(Xn[Wo],1,1,()=>{Xn[Wo]=null}),Jn(),Rs=Xn[Bs],Rs||(Rs=Xn[Bs]=Xp[Bs](e),Rs.c()),w(Rs,1),Rs.m(Do.parentNode,Do));let Mo=Ys;Ys=Jp(e),Ys!==Mo&&(Vn(),b(Gn[Mo],1,1,()=>{Gn[Mo]=null}),Jn(),Ks=Gn[Ys],Ks||(Ks=Gn[Ys]=Zp[Ys](e),Ks.c()),w(Ks,1),Ks.m(Oo.parentNode,Oo)),e[0]==="pt"?Xe?p&1&&w(Xe,1):(Xe=Nc(),Xe.c(),w(Xe,1),Xe.m(Fo.parentNode,Fo)):Xe&&(Vn(),b(Xe,1,1,()=>{Xe=null}),Jn());const Uo={};p&2&&(Uo.$$scope={dirty:p,ctx:e}),Wa.$set(Uo)},i(e){Lr||(w(m.$$.fragment,e),w(A.$$.fragment,e),w(W),w(me.$$.fragment,e),w(Dt.$$.fragment,e),w(kt.$$.fragment,e),w(Le.$$.fragment,e),w(Te.$$.fragment,e),w(Et.$$.fragment,e),w(Wt.$$.fragment,e),w(qt.$$.fragment,e),w(tt.$$.fragment,e),w(St.$$.fragment,e),w(Ds.$$.fragment,e),w(ea.$$.fragment,e),w(ta.$$.fragment,e),w(sa.$$.fragment,e),w(aa.$$.fragment,e),w(na.$$.fragment,e),w(oa.$$.fragment,e),w(la.$$.fragment,e),w(Os.$$.fragment,e),w(ra.$$.fragment,e),w(pa.$$.fragment,e),w(ha.$$.fragment,e),w(za.$$.fragment,e),w(nn.$$.fragment,e),w(on.$$.fragment,e),w(ln.$$.fragment,e),w(rn.$$.fragment,e),w(pn.$$.fragment,e),w(Pa.$$.fragment,e),w(Sa.$$.fragment,e),w(hn.$$.fragment,e),w(Ns),w(cn.$$.fragment,e),w(Ws),w(un.$$.fragment,e),w(fn.$$.fragment,e),w(mn.$$.fragment,e),w(_n.$$.fragment,e),w(gn.$$.fragment,e),w(wn.$$.fragment,e),w(bn.$$.fragment,e),w($n.$$.fragment,e),w(Us),w(kn.$$.fragment,e),w(vn.$$.fragment,e),w(xn.$$.fragment,e),w(qn.$$.fragment,e),w(Tn.$$.fragment,e),w(zn.$$.fragment,e),w(Pn.$$.fragment,e),w(Sn.$$.fragment,e),w(Cn.$$.fragment,e),w(On.$$.fragment,e),w(Rs),w(Fn.$$.fragment,e),w(Ln.$$.fragment,e),w(In.$$.fragment,e),w(Ks),w(Xe),w(Nn.$$.fragment,e),w(Hn.$$.fragment,e),w(Wn.$$.fragment,e),w(Mn.$$.fragment,e),w(Un.$$.fragment,e),w(Wa.$$.fragment,e),Lr=!0)},o(e){b(m.$$.fragment,e),b(A.$$.fragment,e),b(W),b(me.$$.fragment,e),b(Dt.$$.fragment,e),b(kt.$$.fragment,e),b(Le.$$.fragment,e),b(Te.$$.fragment,e),b(Et.$$.fragment,e),b(Wt.$$.fragment,e),b(qt.$$.fragment,e),b(tt.$$.fragment,e),b(St.$$.fragment,e),b(Ds.$$.fragment,e),b(ea.$$.fragment,e),b(ta.$$.fragment,e),b(sa.$$.fragment,e),b(aa.$$.fragment,e),b(na.$$.fragment,e),b(oa.$$.fragment,e),b(la.$$.fragment,e),b(Os.$$.fragment,e),b(ra.$$.fragment,e),b(pa.$$.fragment,e),b(ha.$$.fragment,e),b(za.$$.fragment,e),b(nn.$$.fragment,e),b(on.$$.fragment,e),b(ln.$$.fragment,e),b(rn.$$.fragment,e),b(pn.$$.fragment,e),b(Pa.$$.fragment,e),b(Sa.$$.fragment,e),b(hn.$$.fragment,e),b(Ns),b(cn.$$.fragment,e),b(Ws),b(un.$$.fragment,e),b(fn.$$.fragment,e),b(mn.$$.fragment,e),b(_n.$$.fragment,e),b(gn.$$.fragment,e),b(wn.$$.fragment,e),b(bn.$$.fragment,e),b($n.$$.fragment,e),b(Us),b(kn.$$.fragment,e),b(vn.$$.fragment,e),b(xn.$$.fragment,e),b(qn.$$.fragment,e),b(Tn.$$.fragment,e),b(zn.$$.fragment,e),b(Pn.$$.fragment,e),b(Sn.$$.fragment,e),b(Cn.$$.fragment,e),b(On.$$.fragment,e),b(Rs),b(Fn.$$.fragment,e),b(Ln.$$.fragment,e),b(In.$$.fragment,e),b(Ks),b(Xe),b(Nn.$$.fragment,e),b(Hn.$$.fragment,e),b(Wn.$$.fragment,e),b(Mn.$$.fragment,e),b(Un.$$.fragment,e),b(Wa.$$.fragment,e),Lr=!1},d(e){t(h),e&&t($),E(m,e),e&&t(x),e&&t(P),E(A),e&&t(q),Bn[F].d(e),e&&t(U),e&&t(L),e&&t(V),e&&t(Q),e&&t(ze),E(me,e),e&&t(J),e&&t(ye),e&&t(qe),e&&t(te),e&&t(we),e&&t(k),e&&t(Ze),e&&t(ce),e&&t(nt),e&&t(Ce),e&&t(_t),e&&t(de),e&&t(ot),e&&t(lt),E(Dt),e&&t(wt),e&&t(Ee),e&&t(Jt),e&&t(be),E(kt),e&&t(vt),e&&t(pt),e&&t(Qt),E(Le,e),e&&t(es),e&&t(Ye),e&&t(ct),e&&t(f),e&&t(vs),e&&t(jt),e&&t(ys),E(Te,e),e&&t(js),E(Et,e),e&&t(cs),e&&t(ue),e&&t(xt),E(Wt,e),e&&t(xs),E(qt,e),e&&t(qs),e&&t(ut),e&&t(fs),E(tt,e),e&&t(Ea),e&&t(ms),e&&t(xa),E(St,e),e&&t(Qs),E(Ds,e),e&&t(qa),e&&t(As),e&&t(I),E(ea,e),e&&t(Ga),E(ta,e),e&&t(Ps),e&&t(_a),e&&t(ga),E(sa,e),e&&t(Za),E(aa,e),e&&t(Ss),e&&t(wa),e&&t(Ja),E(na,e),e&&t(Cs),E(oa,e),e&&t(Va),e&&t(Ct),e&&t(Qa),E(la,e),e&&t(as),E(Os,e),e&&t(en),e&&t(_s),E(ra),e&&t(tn),E(pa,e),e&&t(Ls),e&&t(ns),e&&t(os),E(ha,e),e&&t(sn),e&&t(ls),e&&t(El),E(za,e),e&&t(xl),e&&t(da),e&&t(ql),e&&t(co),e&&t(Tl),E(nn,e),e&&t(zl),E(on,e),e&&t(Dl),e&&t(Da),e&&t(Al),E(ln,e),e&&t(Pl),E(rn,e),e&&t(Sl),e&&t(uo),e&&t(Cl),e&&t(Aa),e&&t(Ol),E(pn,e),e&&t(Fl),e&&t(fo),e&&t(Ll),E(Pa,e),e&&t(Il),E(Sa,e),e&&t(Nl),e&&t(mo),e&&t(Hl),E(hn,e),e&&t(Wl),e&&t(_o),e&&t(Ml),Rn[Is].d(e),e&&t(go),e&&t(wo),e&&t(Ul),e&&t(ba),E(cn),e&&t(Bl),e&&t(bs),e&&t(Rl),e&&t(st),e&&t(Yl),Yn[Hs].d(e),e&&t($o),e&&t(ko),e&&t(Kl),E(un,e),e&&t(Xl),E(fn,e),e&&t(Gl),e&&t(Oa),e&&t(Zl),E(mn,e),e&&t(Jl),E(_n,e),e&&t(Vl),e&&t(vo),e&&t(Ql),E(gn,e),e&&t(er),E(wn,e),e&&t(tr),e&&t(yo),e&&t(sr),E(bn,e),e&&t(ar),E($n,e),e&&t(nr),Kn[Ms].d(e),e&&t(jo),e&&t($a),E(kn),e&&t(or),E(vn,e),e&&t(lr),Ot&&Ot.d(e),e&&t(Eo),e&&t(Rt),e&&t(rr),e&&t(La),e&&t(ir),E(xn,e),e&&t(pr),e&&t(ua),e&&t(hr),E(qn,e),e&&t(cr),e&&t(qo),e&&t(dr),e&&t(To),e&&t(ur),E(Tn,e),e&&t(fr),E(zn,e),e&&t(mr),e&&t(Yt),e&&t(_r),E(Pn,e),e&&t(gr),E(Sn,e),e&&t(wr),E(Cn,e),e&&t(br),E(On,e),e&&t($r),e&&t(zo),e&&t(kr),Xn[Bs].d(e),e&&t(Do),e&&t(Ao),e&&t(vr),e&&t(ka),E(Fn),e&&t(yr),e&&t(Po),e&&t(jr),E(Ln,e),e&&t(Er),e&&t(So),e&&t(xr),e&&t(Co),e&&t(qr),E(In,e),e&&t(Tr),Gn[Ys].d(e),e&&t(Oo),Xe&&Xe.d(e),e&&t(Fo),e&&t(va),E(Nn),e&&t(zr),e&&t(Ha),e&&t(Dr),E(Hn,e),e&&t(Ar),E(Wn,e),e&&t(Pr),e&&t(Lo),e&&t(Sr),E(Mn,e),e&&t(Cr),E(Un,e),e&&t(Or),e&&t(Io),e&&t(Fr),E(Wa,e)}}}const fd={local:"translation",sections:[{local:"preparing-the-data",sections:[{local:"the-kde4-dataset",title:"The KDE4 dataset"},{local:"processing-the-data",title:"Processing the data"}],title:"Preparing the data"},{local:"finetuning-the-model-with-the-trainer-api",title:"Fine-tuning the model with the `Trainer` API"},{local:"finetuning-the-model-with-keras",sections:[{local:"data-collation",title:"Data collation"},{local:"metrics",title:"Metrics"},{local:"finetuning-the-model",title:"Fine-tuning the model"}],title:"Fine-tuning the model with Keras"},{local:"a-custom-training-loop",sections:[{local:"preparing-everything-for-training",title:"Preparing everything for training"},{local:"training-loop",title:"Training loop"}],title:"A custom training loop"},{local:"using-the-finetuned-model",title:"Using the fine-tuned model"}],title:"Translation"};function md(R,h,$){let m="pt";return Rc(()=>{const x=new URLSearchParams(window.location.search);$(0,m=x.get("fw")||"pt")}),[m]}class yd extends Wc{constructor(h){super();Mc(this,h,md,ud,Uc,{})}}export{yd as default,fd as metadata};
