import{S as md,i as _d,s as gd,e as l,t as a,k as c,c as r,a as i,h as n,d as t,m as d,b as _,g as o,G as s,w as y,x as j,y as E,q as w,o as b,B as x,M as wd,Z as ui,p as eo,v as bd,n as to,L as cd}from"../../chunks/vendor-hf-doc-builder.js";import{T as Za}from"../../chunks/Tip-hf-doc-builder.js";import{Y as fi}from"../../chunks/Youtube-hf-doc-builder.js";import{I as rs}from"../../chunks/IconCopyLink-hf-doc-builder.js";import{C}from"../../chunks/CodeBlock-hf-doc-builder.js";import{D as fd}from"../../chunks/DocNotebookDropdown-hf-doc-builder.js";import{F as $d}from"../../chunks/FrameworkSwitchCourse-hf-doc-builder.js";function kd(B){let h,$;return h=new fd({props:{classNames:"absolute z-10 right-0 top-0",options:[{label:"Google Colab",value:"https://colab.research.google.com/github/huggingface/notebooks/blob/master/course/chapter7/section4_tf.ipynb"},{label:"Aws Studio",value:"https://studiolab.sagemaker.aws/import/github/huggingface/notebooks/blob/master/course/chapter7/section4_tf.ipynb"}]}}),{c(){y(h.$$.fragment)},l(m){j(h.$$.fragment,m)},m(m,q){E(h,m,q),$=!0},i(m){$||(w(h.$$.fragment,m),$=!0)},o(m){b(h.$$.fragment,m),$=!1},d(m){x(h,m)}}}function vd(B){let h,$;return h=new fd({props:{classNames:"absolute z-10 right-0 top-0",options:[{label:"Google Colab",value:"https://colab.research.google.com/github/huggingface/notebooks/blob/master/course/chapter7/section4_pt.ipynb"},{label:"Aws Studio",value:"https://studiolab.sagemaker.aws/import/github/huggingface/notebooks/blob/master/course/chapter7/section4_pt.ipynb"}]}}),{c(){y(h.$$.fragment)},l(m){j(h.$$.fragment,m)},m(m,q){E(h,m,q),$=!0},i(m){$||(w(h.$$.fragment,m),$=!0)},o(m){b(h.$$.fragment,m),$=!1},d(m){x(h,m)}}}function yd(B){let h,$,m,q,A;return{c(){h=l("p"),$=a("\u270F\uFE0F "),m=l("strong"),q=a("Your turn!"),A=a(" Another English word that is often used in French is \u201Cemail.\u201D Find the first sample in the training dataset that uses this word. How is it translated? How does the pretrained model translate the same English sentence?")},l(g){h=r(g,"P",{});var z=i(h);$=n(z,"\u270F\uFE0F "),m=r(z,"STRONG",{});var P=i(m);q=n(P,"Your turn!"),P.forEach(t),A=n(z," Another English word that is often used in French is \u201Cemail.\u201D Find the first sample in the training dataset that uses this word. How is it translated? How does the pretrained model translate the same English sentence?"),z.forEach(t)},m(g,z){o(g,h,z),s(h,$),s(h,m),s(m,q),s(h,A)},d(g){g&&t(h)}}}function jd(B){let h,$,m,q,A,g,z,P;return{c(){h=l("p"),$=a("\u{1F4A1} If you are using a multilingual tokenizer such as mBART, mBART-50, or M2M100, you will need to set the language codes of your inputs and targets in the tokenizer by setting "),m=l("code"),q=a("tokenizer.src_lang"),A=a(" and "),g=l("code"),z=a("tokenizer.tgt_lang"),P=a(" to the right values.")},l(D){h=r(D,"P",{});var T=i(h);$=n(T,"\u{1F4A1} If you are using a multilingual tokenizer such as mBART, mBART-50, or M2M100, you will need to set the language codes of your inputs and targets in the tokenizer by setting "),m=r(T,"CODE",{});var N=i(m);q=n(N,"tokenizer.src_lang"),N.forEach(t),A=n(T," and "),g=r(T,"CODE",{});var v=i(g);z=n(v,"tokenizer.tgt_lang"),v.forEach(t),P=n(T," to the right values."),T.forEach(t)},m(D,T){o(D,h,T),s(h,$),s(h,m),s(m,q),s(h,A),s(h,g),s(g,z),s(h,P)},d(D){D&&t(h)}}}function Ed(B){let h,$,m,q,A,g,z,P;return{c(){h=l("p"),$=a("\u{1F4A1} If you are using a T5 model (more specifically, one of the "),m=l("code"),q=a("t5-xxx"),A=a(" checkpoints), the model will expect the text inputs to have a prefix indicating the task at hand, such as "),g=l("code"),z=a("translate: English to French:"),P=a(".")},l(D){h=r(D,"P",{});var T=i(h);$=n(T,"\u{1F4A1} If you are using a T5 model (more specifically, one of the "),m=r(T,"CODE",{});var N=i(m);q=n(N,"t5-xxx"),N.forEach(t),A=n(T," checkpoints), the model will expect the text inputs to have a prefix indicating the task at hand, such as "),g=r(T,"CODE",{});var v=i(g);z=n(v,"translate: English to French:"),v.forEach(t),P=n(T,"."),T.forEach(t)},m(D,T){o(D,h,T),s(h,$),s(h,m),s(m,q),s(h,A),s(h,g),s(g,z),s(h,P)},d(D){D&&t(h)}}}function xd(B){let h,$,m,q,A,g,z,P;return{c(){h=l("p"),$=a("\u26A0\uFE0F We don\u2019t pay attention to the attention mask of the targets, as the model won\u2019t expect it. Instead, the labels corresponding to a padding token should be set to "),m=l("code"),q=a("-100"),A=a(" so they are ignored in the loss computation. This will be done by our data collator later on since we are applying dynamic padding, but if you use padding here, you should adapt the preprocessing function to set all labels that correspond to the padding token to "),g=l("code"),z=a("-100"),P=a(".")},l(D){h=r(D,"P",{});var T=i(h);$=n(T,"\u26A0\uFE0F We don\u2019t pay attention to the attention mask of the targets, as the model won\u2019t expect it. Instead, the labels corresponding to a padding token should be set to "),m=r(T,"CODE",{});var N=i(m);q=n(N,"-100"),N.forEach(t),A=n(T," so they are ignored in the loss computation. This will be done by our data collator later on since we are applying dynamic padding, but if you use padding here, you should adapt the preprocessing function to set all labels that correspond to the padding token to "),g=r(T,"CODE",{});var v=i(g);z=n(v,"-100"),v.forEach(t),P=n(T,"."),T.forEach(t)},m(D,T){o(D,h,T),s(h,$),s(h,m),s(m,q),s(h,A),s(h,g),s(g,z),s(h,P)},d(D){D&&t(h)}}}function qd(B){let h,$,m,q,A,g,z,P,D,T,N,v,S,H,U,F,X,M;return q=new rs({}),U=new C({props:{code:`from transformers import TFAutoModelForSeq2SeqLM

model = TFAutoModelForSeq2SeqLM.from_pretrained(model_checkpoint, from_pt=True)`,highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> TFAutoModelForSeq2SeqLM

model = TFAutoModelForSeq2SeqLM.from_pretrained(model_checkpoint, from_pt=<span class="hljs-literal">True</span>)`}}),X=new Za({props:{warning:!1,$$slots:{default:[zd]},$$scope:{ctx:B}}}),{c(){h=l("h2"),$=l("a"),m=l("span"),y(q.$$.fragment),A=c(),g=l("span"),z=a("Fine-tuning the model with Keras"),P=c(),D=l("p"),T=a("First things first, we need an actual model to fine-tune. We\u2019ll use the usual "),N=l("code"),v=a("AutoModel"),S=a(" API:"),H=c(),y(U.$$.fragment),F=c(),y(X.$$.fragment),this.h()},l(I){h=r(I,"H2",{class:!0});var Y=i(h);$=r(Y,"A",{id:!0,class:!0,href:!0});var Z=i($);m=r(Z,"SPAN",{});var re=i(m);j(q.$$.fragment,re),re.forEach(t),Z.forEach(t),A=d(Y),g=r(Y,"SPAN",{});var de=i(g);z=n(de,"Fine-tuning the model with Keras"),de.forEach(t),Y.forEach(t),P=d(I),D=r(I,"P",{});var Q=i(D);T=n(Q,"First things first, we need an actual model to fine-tune. We\u2019ll use the usual "),N=r(Q,"CODE",{});var ee=i(N);v=n(ee,"AutoModel"),ee.forEach(t),S=n(Q," API:"),Q.forEach(t),H=d(I),j(U.$$.fragment,I),F=d(I),j(X.$$.fragment,I),this.h()},h(){_($,"id","finetuning-the-model-with-keras"),_($,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),_($,"href","#finetuning-the-model-with-keras"),_(h,"class","relative group")},m(I,Y){o(I,h,Y),s(h,$),s($,m),E(q,m,null),s(h,A),s(h,g),s(g,z),o(I,P,Y),o(I,D,Y),s(D,T),s(D,N),s(N,v),s(D,S),o(I,H,Y),E(U,I,Y),o(I,F,Y),E(X,I,Y),M=!0},i(I){M||(w(q.$$.fragment,I),w(U.$$.fragment,I),w(X.$$.fragment,I),M=!0)},o(I){b(q.$$.fragment,I),b(U.$$.fragment,I),b(X.$$.fragment,I),M=!1},d(I){I&&t(h),x(q),I&&t(P),I&&t(D),I&&t(H),x(U,I),I&&t(F),x(X,I)}}}function Td(B){let h,$,m,q,A,g,z,P,D,T,N,v,S,H,U,F,X,M,I,Y,Z,re,de,Q,ee,te,G,ne,$e,V,se,fe,ke,oe,pe;return q=new rs({}),oe=new C({props:{code:`from transformers import AutoModelForSeq2SeqLM

model = AutoModelForSeq2SeqLM.from_pretrained(model_checkpoint)`,highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoModelForSeq2SeqLM

model = AutoModelForSeq2SeqLM.from_pretrained(model_checkpoint)`}}),{c(){h=l("h2"),$=l("a"),m=l("span"),y(q.$$.fragment),A=c(),g=l("span"),z=a("Fine-tuning the model with the "),P=l("code"),D=a("Trainer"),T=a(" API"),N=c(),v=l("p"),S=a("The actual code using the "),H=l("code"),U=a("Trainer"),F=a(" will be the same as before, with just one little change: we use a "),X=l("a"),M=l("code"),I=a("Seq2SeqTrainer"),Y=a(" here, which is a subclass of "),Z=l("code"),re=a("Trainer"),de=a(" that will allow us to properly deal with the evaluation, using the "),Q=l("code"),ee=a("generate()"),te=a(" method to predict outputs from the inputs. We\u2019ll dive into that in more detail when we talk about the metric computation."),G=c(),ne=l("p"),$e=a("First things first, we need an actual model to fine-tune. We\u2019ll use the usual "),V=l("code"),se=a("AutoModel"),fe=a(" API:"),ke=c(),y(oe.$$.fragment),this.h()},l(W){h=r(W,"H2",{class:!0});var ae=i(h);$=r(ae,"A",{id:!0,class:!0,href:!0});var ze=i($);m=r(ze,"SPAN",{});var me=i(m);j(q.$$.fragment,me),me.forEach(t),ze.forEach(t),A=d(ae),g=r(ae,"SPAN",{});var K=i(g);z=n(K,"Fine-tuning the model with the "),P=r(K,"CODE",{});var ye=i(P);D=n(ye,"Trainer"),ye.forEach(t),T=n(K," API"),K.forEach(t),ae.forEach(t),N=d(W),v=r(W,"P",{});var le=i(v);S=n(le,"The actual code using the "),H=r(le,"CODE",{});var xe=i(H);U=n(xe,"Trainer"),xe.forEach(t),F=n(le," will be the same as before, with just one little change: we use a "),X=r(le,"A",{href:!0,rel:!0});var Me=i(X);M=r(Me,"CODE",{});var _e=i(M);I=n(_e,"Seq2SeqTrainer"),_e.forEach(t),Me.forEach(t),Y=n(le," here, which is a subclass of "),Z=r(le,"CODE",{});var qe=i(Z);re=n(qe,"Trainer"),qe.forEach(t),de=n(le," that will allow us to properly deal with the evaluation, using the "),Q=r(le,"CODE",{});var J=i(Q);ee=n(J,"generate()"),J.forEach(t),te=n(le," method to predict outputs from the inputs. We\u2019ll dive into that in more detail when we talk about the metric computation."),le.forEach(t),G=d(W),ne=r(W,"P",{});var Ae=i(ne);$e=n(Ae,"First things first, we need an actual model to fine-tune. We\u2019ll use the usual "),V=r(Ae,"CODE",{});var ge=i(V);se=n(ge,"AutoModel"),ge.forEach(t),fe=n(Ae," API:"),Ae.forEach(t),ke=d(W),j(oe.$$.fragment,W),this.h()},h(){_($,"id","finetuning-the-model-with-the-trainer-api"),_($,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),_($,"href","#finetuning-the-model-with-the-trainer-api"),_(h,"class","relative group"),_(X,"href","https://huggingface.co/transformers/main_classes/trainer.html#seq2seqtrainer"),_(X,"rel","nofollow")},m(W,ae){o(W,h,ae),s(h,$),s($,m),E(q,m,null),s(h,A),s(h,g),s(g,z),s(g,P),s(P,D),s(g,T),o(W,N,ae),o(W,v,ae),s(v,S),s(v,H),s(H,U),s(v,F),s(v,X),s(X,M),s(M,I),s(v,Y),s(v,Z),s(Z,re),s(v,de),s(v,Q),s(Q,ee),s(v,te),o(W,G,ae),o(W,ne,ae),s(ne,$e),s(ne,V),s(V,se),s(ne,fe),o(W,ke,ae),E(oe,W,ae),pe=!0},i(W){pe||(w(q.$$.fragment,W),w(oe.$$.fragment,W),pe=!0)},o(W){b(q.$$.fragment,W),b(oe.$$.fragment,W),pe=!1},d(W){W&&t(h),x(q),W&&t(N),W&&t(v),W&&t(G),W&&t(ne),W&&t(ke),x(oe,W)}}}function zd(B){let h,$,m,q,A,g,z,P,D,T,N,v,S,H;return{c(){h=l("p"),$=a("\u{1F4A1} The "),m=l("code"),q=a("Helsinki-NLP/opus-mt-en-fr"),A=a(` checkpoint only has PyTorch weights, so
you\u2019ll get an error if you try to load the model without using the
`),g=l("code"),z=a("from_pt=True"),P=a(" argument in the "),D=l("code"),T=a("from_pretrained()"),N=a(` method. When you specify
`),v=l("code"),S=a("from_pt=True"),H=a(`, the library will automatically download and convert the
PyTorch weights for you. As you can see, it is very simple to switch between
frameworks in \u{1F917} Transformers!`)},l(U){h=r(U,"P",{});var F=i(h);$=n(F,"\u{1F4A1} The "),m=r(F,"CODE",{});var X=i(m);q=n(X,"Helsinki-NLP/opus-mt-en-fr"),X.forEach(t),A=n(F,` checkpoint only has PyTorch weights, so
you\u2019ll get an error if you try to load the model without using the
`),g=r(F,"CODE",{});var M=i(g);z=n(M,"from_pt=True"),M.forEach(t),P=n(F," argument in the "),D=r(F,"CODE",{});var I=i(D);T=n(I,"from_pretrained()"),I.forEach(t),N=n(F,` method. When you specify
`),v=r(F,"CODE",{});var Y=i(v);S=n(Y,"from_pt=True"),Y.forEach(t),H=n(F,`, the library will automatically download and convert the
PyTorch weights for you. As you can see, it is very simple to switch between
frameworks in \u{1F917} Transformers!`),F.forEach(t)},m(U,F){o(U,h,F),s(h,$),s(h,m),s(m,q),s(h,A),s(h,g),s(g,z),s(h,P),s(h,D),s(D,T),s(h,N),s(h,v),s(v,S),s(h,H)},d(U){U&&t(h)}}}function Dd(B){let h,$;return h=new C({props:{code:`from transformers import DataCollatorForSeq2Seq

data_collator = DataCollatorForSeq2Seq(tokenizer, model=model, return_tensors="tf")`,highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> DataCollatorForSeq2Seq

data_collator = DataCollatorForSeq2Seq(tokenizer, model=model, return_tensors=<span class="hljs-string">&quot;tf&quot;</span>)`}}),{c(){y(h.$$.fragment)},l(m){j(h.$$.fragment,m)},m(m,q){E(h,m,q),$=!0},i(m){$||(w(h.$$.fragment,m),$=!0)},o(m){b(h.$$.fragment,m),$=!1},d(m){x(h,m)}}}function Pd(B){let h,$;return h=new C({props:{code:`from transformers import DataCollatorForSeq2Seq

data_collator = DataCollatorForSeq2Seq(tokenizer, model=model)`,highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> DataCollatorForSeq2Seq

data_collator = DataCollatorForSeq2Seq(tokenizer, model=model)`}}),{c(){y(h.$$.fragment)},l(m){j(h.$$.fragment,m)},m(m,q){E(h,m,q),$=!0},i(m){$||(w(h.$$.fragment,m),$=!0)},o(m){b(h.$$.fragment,m),$=!1},d(m){x(h,m)}}}function Ad(B){let h,$,m,q,A,g,z,P,D,T,N;return T=new C({props:{code:`tf_train_dataset = tokenized_datasets["train"].to_tf_dataset(
    columns=["input_ids", "attention_mask", "labels"],
    collate_fn=data_collator,
    shuffle=True,
    batch_size=32,
)
tf_eval_dataset = tokenized_datasets["validation"].to_tf_dataset(
    columns=["input_ids", "attention_mask", "labels"],
    collate_fn=data_collator,
    shuffle=False,
    batch_size=16,
)`,highlighted:`tf_train_dataset = tokenized_datasets[<span class="hljs-string">&quot;train&quot;</span>].to_tf_dataset(
    columns=[<span class="hljs-string">&quot;input_ids&quot;</span>, <span class="hljs-string">&quot;attention_mask&quot;</span>, <span class="hljs-string">&quot;labels&quot;</span>],
    collate_fn=data_collator,
    shuffle=<span class="hljs-literal">True</span>,
    batch_size=<span class="hljs-number">32</span>,
)
tf_eval_dataset = tokenized_datasets[<span class="hljs-string">&quot;validation&quot;</span>].to_tf_dataset(
    columns=[<span class="hljs-string">&quot;input_ids&quot;</span>, <span class="hljs-string">&quot;attention_mask&quot;</span>, <span class="hljs-string">&quot;labels&quot;</span>],
    collate_fn=data_collator,
    shuffle=<span class="hljs-literal">False</span>,
    batch_size=<span class="hljs-number">16</span>,
)`}}),{c(){h=l("p"),$=a("We can now use this "),m=l("code"),q=a("data_collator"),A=a(" to convert each of our datasets to a "),g=l("code"),z=a("tf.data.Dataset"),P=a(", ready for training:"),D=c(),y(T.$$.fragment)},l(v){h=r(v,"P",{});var S=i(h);$=n(S,"We can now use this "),m=r(S,"CODE",{});var H=i(m);q=n(H,"data_collator"),H.forEach(t),A=n(S," to convert each of our datasets to a "),g=r(S,"CODE",{});var U=i(g);z=n(U,"tf.data.Dataset"),U.forEach(t),P=n(S,", ready for training:"),S.forEach(t),D=d(v),j(T.$$.fragment,v)},m(v,S){o(v,h,S),s(h,$),s(h,m),s(m,q),s(h,A),s(h,g),s(g,z),s(h,P),o(v,D,S),E(T,v,S),N=!0},i(v){N||(w(T.$$.fragment,v),N=!0)},o(v){b(T.$$.fragment,v),N=!1},d(v){v&&t(h),v&&t(D),x(T,v)}}}function Sd(B){let h,$,m,q,A,g,z,P;return{c(){h=l("p"),$=a("We will pass this "),m=l("code"),q=a("data_collator"),A=a(" along to the "),g=l("code"),z=a("Seq2SeqTrainer"),P=a(". Next, let\u2019s have a look at the metric.")},l(D){h=r(D,"P",{});var T=i(h);$=n(T,"We will pass this "),m=r(T,"CODE",{});var N=i(m);q=n(N,"data_collator"),N.forEach(t),A=n(T," along to the "),g=r(T,"CODE",{});var v=i(g);z=n(v,"Seq2SeqTrainer"),v.forEach(t),P=n(T,". Next, let\u2019s have a look at the metric."),T.forEach(t)},m(D,T){o(D,h,T),s(h,$),s(h,m),s(m,q),s(h,A),s(h,g),s(g,z),s(h,P)},i:cd,o:cd,d(D){D&&t(h)}}}function dd(B){let h,$,m,q,A,g,z,P,D,T,N,v,S,H,U,F,X,M,I,Y,Z,re,de,Q,ee,te,G,ne,$e;return{c(){h=l("p"),$=a("The feature that "),m=l("code"),q=a("Seq2SeqTrainer"),A=a(" adds to its superclass "),g=l("code"),z=a("Trainer"),P=a(" is the ability to use the "),D=l("code"),T=a("generate()"),N=a(" method during evaluation or prediction. During training, the model will use the "),v=l("code"),S=a("decoder_input_ids"),H=a(" with an attention mask ensuring it does not use the tokens after the token it\u2019s trying to predict, to speed up training. During inference we won\u2019t be able to use those since we won\u2019t have labels, so it\u2019s a good idea to evaluate our model with the same setup."),U=c(),F=l("p"),X=a("As we saw in "),M=l("a"),I=a("Chapter 1"),Y=a(", the decoder performs inference by predicting tokens one by one \u2014 something that\u2019s implemented behind the scenes in \u{1F917} Transformers by the "),Z=l("code"),re=a("generate()"),de=a(" method. The "),Q=l("code"),ee=a("Seq2SeqTrainer"),te=a(" will let us use that method for evaluation if we set "),G=l("code"),ne=a("predict_with_generate=True"),$e=a("."),this.h()},l(V){h=r(V,"P",{});var se=i(h);$=n(se,"The feature that "),m=r(se,"CODE",{});var fe=i(m);q=n(fe,"Seq2SeqTrainer"),fe.forEach(t),A=n(se," adds to its superclass "),g=r(se,"CODE",{});var ke=i(g);z=n(ke,"Trainer"),ke.forEach(t),P=n(se," is the ability to use the "),D=r(se,"CODE",{});var oe=i(D);T=n(oe,"generate()"),oe.forEach(t),N=n(se," method during evaluation or prediction. During training, the model will use the "),v=r(se,"CODE",{});var pe=i(v);S=n(pe,"decoder_input_ids"),pe.forEach(t),H=n(se," with an attention mask ensuring it does not use the tokens after the token it\u2019s trying to predict, to speed up training. During inference we won\u2019t be able to use those since we won\u2019t have labels, so it\u2019s a good idea to evaluate our model with the same setup."),se.forEach(t),U=d(V),F=r(V,"P",{});var W=i(F);X=n(W,"As we saw in "),M=r(W,"A",{href:!0});var ae=i(M);I=n(ae,"Chapter 1"),ae.forEach(t),Y=n(W,", the decoder performs inference by predicting tokens one by one \u2014 something that\u2019s implemented behind the scenes in \u{1F917} Transformers by the "),Z=r(W,"CODE",{});var ze=i(Z);re=n(ze,"generate()"),ze.forEach(t),de=n(W," method. The "),Q=r(W,"CODE",{});var me=i(Q);ee=n(me,"Seq2SeqTrainer"),me.forEach(t),te=n(W," will let us use that method for evaluation if we set "),G=r(W,"CODE",{});var K=i(G);ne=n(K,"predict_with_generate=True"),K.forEach(t),$e=n(W,"."),W.forEach(t),this.h()},h(){_(M,"href","/course/chapter1/6")},m(V,se){o(V,h,se),s(h,$),s(h,m),s(m,q),s(h,A),s(h,g),s(g,z),s(h,P),s(h,D),s(D,T),s(h,N),s(h,v),s(v,S),s(h,H),o(V,U,se),o(V,F,se),s(F,X),s(F,M),s(M,I),s(F,Y),s(F,Z),s(Z,re),s(F,de),s(F,Q),s(Q,ee),s(F,te),s(F,G),s(G,ne),s(F,$e)},d(V){V&&t(h),V&&t(U),V&&t(F)}}}function Cd(B){let h,$,m,q,A,g,z,P,D,T,N;return T=new C({props:{code:`import numpy as np


def compute_metrics(eval_preds):
    preds, labels = eval_preds
    # In case the model returns more than the prediction logits
    if isinstance(preds, tuple):
        preds = preds[0]

    decoded_preds = tokenizer.batch_decode(preds, skip_special_tokens=True)

    # Replace -100s in the labels as we can't decode them
    labels = np.where(labels != -100, labels, tokenizer.pad_token_id)
    decoded_labels = tokenizer.batch_decode(labels, skip_special_tokens=True)

    # Some simple post-processing
    decoded_preds = [pred.strip() for pred in decoded_preds]
    decoded_labels = [[label.strip()] for label in decoded_labels]

    result = metric.compute(predictions=decoded_preds, references=decoded_labels)
    return {"bleu": result["score"]}`,highlighted:`<span class="hljs-keyword">import</span> numpy <span class="hljs-keyword">as</span> np


<span class="hljs-keyword">def</span> <span class="hljs-title function_">compute_metrics</span>(<span class="hljs-params">eval_preds</span>):
    preds, labels = eval_preds
    <span class="hljs-comment"># In case the model returns more than the prediction logits</span>
    <span class="hljs-keyword">if</span> <span class="hljs-built_in">isinstance</span>(preds, <span class="hljs-built_in">tuple</span>):
        preds = preds[<span class="hljs-number">0</span>]

    decoded_preds = tokenizer.batch_decode(preds, skip_special_tokens=<span class="hljs-literal">True</span>)

    <span class="hljs-comment"># Replace -100s in the labels as we can&#x27;t decode them</span>
    labels = np.where(labels != -<span class="hljs-number">100</span>, labels, tokenizer.pad_token_id)
    decoded_labels = tokenizer.batch_decode(labels, skip_special_tokens=<span class="hljs-literal">True</span>)

    <span class="hljs-comment"># Some simple post-processing</span>
    decoded_preds = [pred.strip() <span class="hljs-keyword">for</span> pred <span class="hljs-keyword">in</span> decoded_preds]
    decoded_labels = [[label.strip()] <span class="hljs-keyword">for</span> label <span class="hljs-keyword">in</span> decoded_labels]

    result = metric.compute(predictions=decoded_preds, references=decoded_labels)
    <span class="hljs-keyword">return</span> {<span class="hljs-string">&quot;bleu&quot;</span>: result[<span class="hljs-string">&quot;score&quot;</span>]}`}}),{c(){h=l("p"),$=a("To get from the model outputs to texts the metric can use, we will use the "),m=l("code"),q=a("tokenizer.batch_decode()"),A=a(" method. We just have to clean up all the "),g=l("code"),z=a("-100"),P=a("s in the labels (the tokenizer will automatically do the same for the padding token):"),D=c(),y(T.$$.fragment)},l(v){h=r(v,"P",{});var S=i(h);$=n(S,"To get from the model outputs to texts the metric can use, we will use the "),m=r(S,"CODE",{});var H=i(m);q=n(H,"tokenizer.batch_decode()"),H.forEach(t),A=n(S," method. We just have to clean up all the "),g=r(S,"CODE",{});var U=i(g);z=n(U,"-100"),U.forEach(t),P=n(S,"s in the labels (the tokenizer will automatically do the same for the padding token):"),S.forEach(t),D=d(v),j(T.$$.fragment,v)},m(v,S){o(v,h,S),s(h,$),s(h,m),s(m,q),s(h,A),s(h,g),s(g,z),s(h,P),o(v,D,S),E(T,v,S),N=!0},i(v){N||(w(T.$$.fragment,v),N=!0)},o(v){b(T.$$.fragment,v),N=!1},d(v){v&&t(h),v&&t(D),x(T,v)}}}function Od(B){let h,$,m,q,A,g,z,P,D,T,N;return T=new C({props:{code:`import numpy as np


def compute_metrics():
    all_preds = []
    all_labels = []
    sampled_dataset = tokenized_datasets["validation"].shuffle().select(range(200))
    tf_generate_dataset = sampled_dataset.to_tf_dataset(
        columns=["input_ids", "attention_mask", "labels"],
        collate_fn=data_collator,
        shuffle=False,
        batch_size=4,
    )
    for batch in tf_generate_dataset:
        predictions = model.generate(
            input_ids=batch["input_ids"], attention_mask=batch["attention_mask"]
        )
        decoded_preds = tokenizer.batch_decode(predictions, skip_special_tokens=True)
        labels = batch["labels"].numpy()
        labels = np.where(labels != -100, labels, tokenizer.pad_token_id)
        decoded_labels = tokenizer.batch_decode(labels, skip_special_tokens=True)
        decoded_preds = [pred.strip() for pred in decoded_preds]
        decoded_labels = [[label.strip()] for label in decoded_labels]
        all_preds.extend(decoded_preds)
        all_labels.extend(decoded_labels)

    result = metric.compute(predictions=all_preds, references=all_labels)
    return {"bleu": result["score"]}`,highlighted:`<span class="hljs-keyword">import</span> numpy <span class="hljs-keyword">as</span> np


<span class="hljs-keyword">def</span> <span class="hljs-title function_">compute_metrics</span>():
    all_preds = []
    all_labels = []
    sampled_dataset = tokenized_datasets[<span class="hljs-string">&quot;validation&quot;</span>].shuffle().select(<span class="hljs-built_in">range</span>(<span class="hljs-number">200</span>))
    tf_generate_dataset = sampled_dataset.to_tf_dataset(
        columns=[<span class="hljs-string">&quot;input_ids&quot;</span>, <span class="hljs-string">&quot;attention_mask&quot;</span>, <span class="hljs-string">&quot;labels&quot;</span>],
        collate_fn=data_collator,
        shuffle=<span class="hljs-literal">False</span>,
        batch_size=<span class="hljs-number">4</span>,
    )
    <span class="hljs-keyword">for</span> batch <span class="hljs-keyword">in</span> tf_generate_dataset:
        predictions = model.generate(
            input_ids=batch[<span class="hljs-string">&quot;input_ids&quot;</span>], attention_mask=batch[<span class="hljs-string">&quot;attention_mask&quot;</span>]
        )
        decoded_preds = tokenizer.batch_decode(predictions, skip_special_tokens=<span class="hljs-literal">True</span>)
        labels = batch[<span class="hljs-string">&quot;labels&quot;</span>].numpy()
        labels = np.where(labels != -<span class="hljs-number">100</span>, labels, tokenizer.pad_token_id)
        decoded_labels = tokenizer.batch_decode(labels, skip_special_tokens=<span class="hljs-literal">True</span>)
        decoded_preds = [pred.strip() <span class="hljs-keyword">for</span> pred <span class="hljs-keyword">in</span> decoded_preds]
        decoded_labels = [[label.strip()] <span class="hljs-keyword">for</span> label <span class="hljs-keyword">in</span> decoded_labels]
        all_preds.extend(decoded_preds)
        all_labels.extend(decoded_labels)

    result = metric.compute(predictions=all_preds, references=all_labels)
    <span class="hljs-keyword">return</span> {<span class="hljs-string">&quot;bleu&quot;</span>: result[<span class="hljs-string">&quot;score&quot;</span>]}`}}),{c(){h=l("p"),$=a("To get from the model outputs to texts the metric can use, we will use the "),m=l("code"),q=a("tokenizer.batch_decode()"),A=a(" method. We just have to clean up all the "),g=l("code"),z=a("-100"),P=a("s in the labels; the tokenizer will automatically do the same for the padding token. Let\u2019s define a function that takes our model and a dataset and computes metrics on it. Because generation of long sequences can be slow, we subsample the validation set to make sure this doesn\u2019t take forever:"),D=c(),y(T.$$.fragment)},l(v){h=r(v,"P",{});var S=i(h);$=n(S,"To get from the model outputs to texts the metric can use, we will use the "),m=r(S,"CODE",{});var H=i(m);q=n(H,"tokenizer.batch_decode()"),H.forEach(t),A=n(S," method. We just have to clean up all the "),g=r(S,"CODE",{});var U=i(g);z=n(U,"-100"),U.forEach(t),P=n(S,"s in the labels; the tokenizer will automatically do the same for the padding token. Let\u2019s define a function that takes our model and a dataset and computes metrics on it. Because generation of long sequences can be slow, we subsample the validation set to make sure this doesn\u2019t take forever:"),S.forEach(t),D=d(v),j(T.$$.fragment,v)},m(v,S){o(v,h,S),s(h,$),s(h,m),s(m,q),s(h,A),s(h,g),s(g,z),s(h,P),o(v,D,S),E(T,v,S),N=!0},i(v){N||(w(T.$$.fragment,v),N=!0)},o(v){b(T.$$.fragment,v),N=!1},d(v){v&&t(h),v&&t(D),x(T,v)}}}function Fd(B){let h,$,m,q,A,g,z,P,D,T,N,v,S,H,U,F,X,M,I,Y,Z,re,de,Q,ee,te,G,ne,$e,V,se,fe,ke,oe,pe,W,ae,ze,me,K,ye,le,xe,Me,_e,qe,J,Ae,ge,at,ft,ve,Xe,Ue,Se,je,mt,we,k,R,Ze,he,Be,nt,Ce,ie,Ft,Re,Le,Ne,_t,ce,Je,Oe,zt,is,ot,lt,rt,Ve,Dt,ps,Qe,gt,wt,Ee,Gt,Fe,ks,Xt,De,Zt,Pe,It,bt,vs,Jt,be,$t,Pt,kt,it,He,Vt,vt,pt,ht,Lt,At,et,Qt,Ie,es,Ke,ts,yt,Nt,hs,ct;return S=new C({props:{code:`from transformers import Seq2SeqTrainingArguments

args = Seq2SeqTrainingArguments(
    f"marian-finetuned-kde4-en-to-fr",
    evaluation_strategy="no",
    save_strategy="epoch",
    learning_rate=2e-5,
    per_device_train_batch_size=32,
    per_device_eval_batch_size=64,
    weight_decay=0.01,
    save_total_limit=3,
    num_train_epochs=3,
    predict_with_generate=True,
    fp16=True,
    push_to_hub=True,
)`,highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> Seq2SeqTrainingArguments

args = Seq2SeqTrainingArguments(
    <span class="hljs-string">f&quot;marian-finetuned-kde4-en-to-fr&quot;</span>,
    evaluation_strategy=<span class="hljs-string">&quot;no&quot;</span>,
    save_strategy=<span class="hljs-string">&quot;epoch&quot;</span>,
    learning_rate=<span class="hljs-number">2e-5</span>,
    per_device_train_batch_size=<span class="hljs-number">32</span>,
    per_device_eval_batch_size=<span class="hljs-number">64</span>,
    weight_decay=<span class="hljs-number">0.01</span>,
    save_total_limit=<span class="hljs-number">3</span>,
    num_train_epochs=<span class="hljs-number">3</span>,
    predict_with_generate=<span class="hljs-literal">True</span>,
    fp16=<span class="hljs-literal">True</span>,
    push_to_hub=<span class="hljs-literal">True</span>,
)`}}),R=new Za({props:{$$slots:{default:[Ld]},$$scope:{ctx:B}}}),Re=new C({props:{code:`from transformers import Seq2SeqTrainer

trainer = Seq2SeqTrainer(
    model,
    args,
    train_dataset=tokenized_datasets["train"],
    eval_dataset=tokenized_datasets["validation"],
    data_collator=data_collator,
    tokenizer=tokenizer,
    compute_metrics=compute_metrics,
)`,highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> Seq2SeqTrainer

trainer = Seq2SeqTrainer(
    model,
    args,
    train_dataset=tokenized_datasets[<span class="hljs-string">&quot;train&quot;</span>],
    eval_dataset=tokenized_datasets[<span class="hljs-string">&quot;validation&quot;</span>],
    data_collator=data_collator,
    tokenizer=tokenizer,
    compute_metrics=compute_metrics,
)`}}),Je=new C({props:{code:"trainer.evaluate(max_length=max_target_length)",highlighted:"trainer.evaluate(max_length=max_target_length)"}}),zt=new C({props:{code:`{'eval_loss': 1.6964408159255981,
 'eval_bleu': 39.26865061007616,
 'eval_runtime': 965.8884,
 'eval_samples_per_second': 21.76,
 'eval_steps_per_second': 0.341}`,highlighted:`{<span class="hljs-string">&#x27;eval_loss&#x27;</span>: <span class="hljs-number">1.6964408159255981</span>,
 <span class="hljs-string">&#x27;eval_bleu&#x27;</span>: <span class="hljs-number">39.26865061007616</span>,
 <span class="hljs-string">&#x27;eval_runtime&#x27;</span>: <span class="hljs-number">965.8884</span>,
 <span class="hljs-string">&#x27;eval_samples_per_second&#x27;</span>: <span class="hljs-number">21.76</span>,
 <span class="hljs-string">&#x27;eval_steps_per_second&#x27;</span>: <span class="hljs-number">0.341</span>}`}}),Qe=new C({props:{code:"trainer.train()",highlighted:"trainer.train()"}}),De=new C({props:{code:"trainer.evaluate(max_length=max_target_length)",highlighted:"trainer.evaluate(max_length=max_target_length)"}}),Pe=new C({props:{code:`{'eval_loss': 0.8558505773544312,
 'eval_bleu': 52.94161337775576,
 'eval_runtime': 714.2576,
 'eval_samples_per_second': 29.426,
 'eval_steps_per_second': 0.461,
 'epoch': 3.0}`,highlighted:`{<span class="hljs-string">&#x27;eval_loss&#x27;</span>: <span class="hljs-number">0.8558505773544312</span>,
 <span class="hljs-string">&#x27;eval_bleu&#x27;</span>: <span class="hljs-number">52.94161337775576</span>,
 <span class="hljs-string">&#x27;eval_runtime&#x27;</span>: <span class="hljs-number">714.2576</span>,
 <span class="hljs-string">&#x27;eval_samples_per_second&#x27;</span>: <span class="hljs-number">29.426</span>,
 <span class="hljs-string">&#x27;eval_steps_per_second&#x27;</span>: <span class="hljs-number">0.461</span>,
 <span class="hljs-string">&#x27;epoch&#x27;</span>: <span class="hljs-number">3.0</span>}`}}),ht=new C({props:{code:'trainer.push_to_hub(tags="translation", commit_message="Training complete")',highlighted:'trainer.push_to_hub(tags=<span class="hljs-string">&quot;translation&quot;</span>, commit_message=<span class="hljs-string">&quot;Training complete&quot;</span>)'}}),Ie=new C({props:{code:"'https://huggingface.co/sgugger/marian-finetuned-kde4-en-to-fr/commit/3601d621e3baae2bc63d3311452535f8f58f6ef3'",highlighted:'<span class="hljs-string">&#x27;https://huggingface.co/sgugger/marian-finetuned-kde4-en-to-fr/commit/3601d621e3baae2bc63d3311452535f8f58f6ef3&#x27;</span>'}}),{c(){h=l("p"),$=a("Once this is done, we can define our "),m=l("code"),q=a("Seq2SeqTrainingArguments"),A=a(". Like for the "),g=l("code"),z=a("Trainer"),P=a(", we use a subclass of "),D=l("code"),T=a("TrainingArguments"),N=a(" that contains a few more fields:"),v=c(),y(S.$$.fragment),H=c(),U=l("p"),F=a("Apart from the usual hyperparameters (like learning rate, number of epochs, batch size, and some weight decay), here are a few changes compared to what we saw in the previous sections:"),X=c(),M=l("ul"),I=l("li"),Y=a("We don\u2019t set any regular evaluation, as evaluation takes a while; we will just evaluate our model once before training and after."),Z=c(),re=l("li"),de=a("We set "),Q=l("code"),ee=a("fp16=True"),te=a(", which speeds up training on modern GPUs."),G=c(),ne=l("li"),$e=a("We set "),V=l("code"),se=a("predict_with_generate=True"),fe=a(", as discussed above."),ke=c(),oe=l("li"),pe=a("We use "),W=l("code"),ae=a("push_to_hub=True"),ze=a(" to upload the model to the Hub at the end of each epoch."),me=c(),K=l("p"),ye=a("Note that you can specify the full name of the repository you want to push to with the "),le=l("code"),xe=a("hub_model_id"),Me=a(" argument (in particular, you will have to use this argument to push to an organization). For instance, when we pushed the model to the "),_e=l("a"),qe=l("code"),J=a("huggingface-course"),Ae=a(" organization"),ge=a(", we added "),at=l("code"),ft=a('hub_model_id="huggingface-course/marian-finetuned-kde4-en-to-fr"'),ve=a(" to "),Xe=l("code"),Ue=a("Seq2SeqTrainingArguments"),Se=a(". By default, the repository used will be in your namespace and named after the output directory you set, so in our case it will be "),je=l("code"),mt=a('"sgugger/marian-finetuned-kde4-en-to-fr"'),we=a(" (which is the model we linked to at the beginning of this section)."),k=c(),y(R.$$.fragment),Ze=c(),he=l("p"),Be=a("Finally, we just pass everything to the "),nt=l("code"),Ce=a("Seq2SeqTrainer"),ie=a(":"),Ft=c(),y(Re.$$.fragment),Le=c(),Ne=l("p"),_t=a("Before training, we\u2019ll first look at the score our model gets, to double-check that we\u2019re not making things worse with our fine-tuning. This command will take a bit of time, so you can grab a coffee while it executes:"),ce=c(),y(Je.$$.fragment),Oe=c(),y(zt.$$.fragment),is=c(),ot=l("p"),lt=a("A BLEU score of 39 is not too bad, which reflects the fact that our model is already good at translating English sentences to French ones."),rt=c(),Ve=l("p"),Dt=a("Next is the training, which will also take a bit of time:"),ps=c(),y(Qe.$$.fragment),gt=c(),wt=l("p"),Ee=a("Note that while the training happens, each time the model is saved (here, every epoch) it is uploaded to the Hub in the background. This way, you will be able to to resume your training on another machine if necessary."),Gt=c(),Fe=l("p"),ks=a("Once training is done, we evaluate our model again \u2014 hopefully we will see some amelioration in the BLEU score!"),Xt=c(),y(De.$$.fragment),Zt=c(),y(Pe.$$.fragment),It=c(),bt=l("p"),vs=a("That\u2019s a nearly 14-point improvement, which is great."),Jt=c(),be=l("p"),$t=a("Finally, we use the "),Pt=l("code"),kt=a("push_to_hub()"),it=a(" method to make sure we upload the latest version of the model. The "),He=l("code"),Vt=a("Trainer"),vt=a(" also drafts a model card with all the evaluation results and uploads it. This model card contains metadata that helps the Model Hub pick the widget for the inference demo. Usually, there is no need to say anything as it can infer the right widget from the model class, but in this case, the same model class can be used for all kinds of sequence-to-sequence problems, so we specify it\u2019s a translation model:"),pt=c(),y(ht.$$.fragment),Lt=c(),At=l("p"),et=a("This command returns the URL of the commit it just did, if you want to inspect it:"),Qt=c(),y(Ie.$$.fragment),es=c(),Ke=l("p"),ts=a("At this stage, you can use the inference widget on the Model Hub to test your model and share it with your friends. You have successfully fine-tuned a model on a translation task \u2014 congratulations!"),yt=c(),Nt=l("p"),hs=a("If you want to dive a bit more deeply into the training loop, we will now show you how to do the same thing using \u{1F917} Accelerate."),this.h()},l(f){h=r(f,"P",{});var O=i(h);$=n(O,"Once this is done, we can define our "),m=r(O,"CODE",{});var ys=i(m);q=n(ys,"Seq2SeqTrainingArguments"),ys.forEach(t),A=n(O,". Like for the "),g=r(O,"CODE",{});var jt=i(g);z=n(jt,"Trainer"),jt.forEach(t),P=n(O,", we use a subclass of "),D=r(O,"CODE",{});var Xs=i(D);T=n(Xs,"TrainingArguments"),Xs.forEach(t),N=n(O," that contains a few more fields:"),O.forEach(t),v=d(f),j(S.$$.fragment,f),H=d(f),U=r(f,"P",{});var js=i(U);F=n(js,"Apart from the usual hyperparameters (like learning rate, number of epochs, batch size, and some weight decay), here are a few changes compared to what we saw in the previous sections:"),js.forEach(t),X=d(f),M=r(f,"UL",{});var Te=i(M);I=r(Te,"LI",{});var Es=i(I);Y=n(Es,"We don\u2019t set any regular evaluation, as evaluation takes a while; we will just evaluate our model once before training and after."),Es.forEach(t),Z=d(Te),re=r(Te,"LI",{});var Et=i(re);de=n(Et,"We set "),Q=r(Et,"CODE",{});var cs=i(Q);ee=n(cs,"fp16=True"),cs.forEach(t),te=n(Et,", which speeds up training on modern GPUs."),Et.forEach(t),G=d(Te),ne=r(Te,"LI",{});var ue=i(ne);$e=n(ue,"We set "),V=r(ue,"CODE",{});var Zs=i(V);se=n(Zs,"predict_with_generate=True"),Zs.forEach(t),fe=n(ue,", as discussed above."),ue.forEach(t),ke=d(Te),oe=r(Te,"LI",{});var dt=i(oe);pe=n(dt,"We use "),W=r(dt,"CODE",{});var Js=i(W);ae=n(Js,"push_to_hub=True"),Js.forEach(t),ze=n(dt," to upload the model to the Hub at the end of each epoch."),dt.forEach(t),Te.forEach(t),me=d(f),K=r(f,"P",{});var We=i(K);ye=n(We,"Note that you can specify the full name of the repository you want to push to with the "),le=r(We,"CODE",{});var Ht=i(le);xe=n(Ht,"hub_model_id"),Ht.forEach(t),Me=n(We," argument (in particular, you will have to use this argument to push to an organization). For instance, when we pushed the model to the "),_e=r(We,"A",{href:!0,rel:!0});var xs=i(_e);qe=r(xs,"CODE",{});var Vs=i(qe);J=n(Vs,"huggingface-course"),Vs.forEach(t),Ae=n(xs," organization"),xs.forEach(t),ge=n(We,", we added "),at=r(We,"CODE",{});var ss=i(at);ft=n(ss,'hub_model_id="huggingface-course/marian-finetuned-kde4-en-to-fr"'),ss.forEach(t),ve=n(We," to "),Xe=r(We,"CODE",{});var Ye=i(Xe);Ue=n(Ye,"Seq2SeqTrainingArguments"),Ye.forEach(t),Se=n(We,". By default, the repository used will be in your namespace and named after the output directory you set, so in our case it will be "),je=r(We,"CODE",{});var Qs=i(je);mt=n(Qs,'"sgugger/marian-finetuned-kde4-en-to-fr"'),Qs.forEach(t),we=n(We," (which is the model we linked to at the beginning of this section)."),We.forEach(t),k=d(f),j(R.$$.fragment,f),Ze=d(f),he=r(f,"P",{});var xt=i(he);Be=n(xt,"Finally, we just pass everything to the "),nt=r(xt,"CODE",{});var Wt=i(nt);Ce=n(Wt,"Seq2SeqTrainer"),Wt.forEach(t),ie=n(xt,":"),xt.forEach(t),Ft=d(f),j(Re.$$.fragment,f),Le=d(f),Ne=r(f,"P",{});var qs=i(Ne);_t=n(qs,"Before training, we\u2019ll first look at the score our model gets, to double-check that we\u2019re not making things worse with our fine-tuning. This command will take a bit of time, so you can grab a coffee while it executes:"),qs.forEach(t),ce=d(f),j(Je.$$.fragment,f),Oe=d(f),j(zt.$$.fragment,f),is=d(f),ot=r(f,"P",{});var qt=i(ot);lt=n(qt,"A BLEU score of 39 is not too bad, which reflects the fact that our model is already good at translating English sentences to French ones."),qt.forEach(t),rt=d(f),Ve=r(f,"P",{});var Ts=i(Ve);Dt=n(Ts,"Next is the training, which will also take a bit of time:"),Ts.forEach(t),ps=d(f),j(Qe.$$.fragment,f),gt=d(f),wt=r(f,"P",{});var ut=i(wt);Ee=n(ut,"Note that while the training happens, each time the model is saved (here, every epoch) it is uploaded to the Hub in the background. This way, you will be able to to resume your training on another machine if necessary."),ut.forEach(t),Gt=d(f),Fe=r(f,"P",{});var ds=i(Fe);ks=n(ds,"Once training is done, we evaluate our model again \u2014 hopefully we will see some amelioration in the BLEU score!"),ds.forEach(t),Xt=d(f),j(De.$$.fragment,f),Zt=d(f),j(Pe.$$.fragment,f),It=d(f),bt=r(f,"P",{});var us=i(bt);vs=n(us,"That\u2019s a nearly 14-point improvement, which is great."),us.forEach(t),Jt=d(f),be=r(f,"P",{});var Mt=i(be);$t=n(Mt,"Finally, we use the "),Pt=r(Mt,"CODE",{});var zs=i(Pt);kt=n(zs,"push_to_hub()"),zs.forEach(t),it=n(Mt," method to make sure we upload the latest version of the model. The "),He=r(Mt,"CODE",{});var Tt=i(He);Vt=n(Tt,"Trainer"),Tt.forEach(t),vt=n(Mt," also drafts a model card with all the evaluation results and uploads it. This model card contains metadata that helps the Model Hub pick the widget for the inference demo. Usually, there is no need to say anything as it can infer the right widget from the model class, but in this case, the same model class can be used for all kinds of sequence-to-sequence problems, so we specify it\u2019s a translation model:"),Mt.forEach(t),pt=d(f),j(ht.$$.fragment,f),Lt=d(f),At=r(f,"P",{});var Ds=i(At);et=n(Ds,"This command returns the URL of the commit it just did, if you want to inspect it:"),Ds.forEach(t),Qt=d(f),j(Ie.$$.fragment,f),es=d(f),Ke=r(f,"P",{});var Ut=i(Ke);ts=n(Ut,"At this stage, you can use the inference widget on the Model Hub to test your model and share it with your friends. You have successfully fine-tuned a model on a translation task \u2014 congratulations!"),Ut.forEach(t),yt=d(f),Nt=r(f,"P",{});var fs=i(Nt);hs=n(fs,"If you want to dive a bit more deeply into the training loop, we will now show you how to do the same thing using \u{1F917} Accelerate."),fs.forEach(t),this.h()},h(){_(_e,"href","https://huggingface.co/huggingface-course"),_(_e,"rel","nofollow")},m(f,O){o(f,h,O),s(h,$),s(h,m),s(m,q),s(h,A),s(h,g),s(g,z),s(h,P),s(h,D),s(D,T),s(h,N),o(f,v,O),E(S,f,O),o(f,H,O),o(f,U,O),s(U,F),o(f,X,O),o(f,M,O),s(M,I),s(I,Y),s(M,Z),s(M,re),s(re,de),s(re,Q),s(Q,ee),s(re,te),s(M,G),s(M,ne),s(ne,$e),s(ne,V),s(V,se),s(ne,fe),s(M,ke),s(M,oe),s(oe,pe),s(oe,W),s(W,ae),s(oe,ze),o(f,me,O),o(f,K,O),s(K,ye),s(K,le),s(le,xe),s(K,Me),s(K,_e),s(_e,qe),s(qe,J),s(_e,Ae),s(K,ge),s(K,at),s(at,ft),s(K,ve),s(K,Xe),s(Xe,Ue),s(K,Se),s(K,je),s(je,mt),s(K,we),o(f,k,O),E(R,f,O),o(f,Ze,O),o(f,he,O),s(he,Be),s(he,nt),s(nt,Ce),s(he,ie),o(f,Ft,O),E(Re,f,O),o(f,Le,O),o(f,Ne,O),s(Ne,_t),o(f,ce,O),E(Je,f,O),o(f,Oe,O),E(zt,f,O),o(f,is,O),o(f,ot,O),s(ot,lt),o(f,rt,O),o(f,Ve,O),s(Ve,Dt),o(f,ps,O),E(Qe,f,O),o(f,gt,O),o(f,wt,O),s(wt,Ee),o(f,Gt,O),o(f,Fe,O),s(Fe,ks),o(f,Xt,O),E(De,f,O),o(f,Zt,O),E(Pe,f,O),o(f,It,O),o(f,bt,O),s(bt,vs),o(f,Jt,O),o(f,be,O),s(be,$t),s(be,Pt),s(Pt,kt),s(be,it),s(be,He),s(He,Vt),s(be,vt),o(f,pt,O),E(ht,f,O),o(f,Lt,O),o(f,At,O),s(At,et),o(f,Qt,O),E(Ie,f,O),o(f,es,O),o(f,Ke,O),s(Ke,ts),o(f,yt,O),o(f,Nt,O),s(Nt,hs),ct=!0},i(f){ct||(w(S.$$.fragment,f),w(R.$$.fragment,f),w(Re.$$.fragment,f),w(Je.$$.fragment,f),w(zt.$$.fragment,f),w(Qe.$$.fragment,f),w(De.$$.fragment,f),w(Pe.$$.fragment,f),w(ht.$$.fragment,f),w(Ie.$$.fragment,f),ct=!0)},o(f){b(S.$$.fragment,f),b(R.$$.fragment,f),b(Re.$$.fragment,f),b(Je.$$.fragment,f),b(zt.$$.fragment,f),b(Qe.$$.fragment,f),b(De.$$.fragment,f),b(Pe.$$.fragment,f),b(ht.$$.fragment,f),b(Ie.$$.fragment,f),ct=!1},d(f){f&&t(h),f&&t(v),x(S,f),f&&t(H),f&&t(U),f&&t(X),f&&t(M),f&&t(me),f&&t(K),f&&t(k),x(R,f),f&&t(Ze),f&&t(he),f&&t(Ft),x(Re,f),f&&t(Le),f&&t(Ne),f&&t(ce),x(Je,f),f&&t(Oe),x(zt,f),f&&t(is),f&&t(ot),f&&t(rt),f&&t(Ve),f&&t(ps),x(Qe,f),f&&t(gt),f&&t(wt),f&&t(Gt),f&&t(Fe),f&&t(Xt),x(De,f),f&&t(Zt),x(Pe,f),f&&t(It),f&&t(bt),f&&t(Jt),f&&t(be),f&&t(pt),x(ht,f),f&&t(Lt),f&&t(At),f&&t(Qt),x(Ie,f),f&&t(es),f&&t(Ke),f&&t(yt),f&&t(Nt)}}}function Id(B){let h,$,m,q,A,g,z,P,D,T,N,v,S,H,U,F,X,M,I,Y,Z,re,de,Q,ee,te,G,ne,$e,V,se,fe,ke,oe,pe,W,ae,ze,me,K,ye,le,xe,Me,_e,qe,J,Ae,ge,at,ft,ve,Xe,Ue,Se,je,mt,we;return q=new C({props:{code:"print(compute_metrics())",highlighted:'<span class="hljs-built_in">print</span>(compute_metrics())'}}),g=new C({props:{code:"{'bleu': 33.26983701454733}",highlighted:'{&#x27;bleu&#x27;: <span class="hljs-number">33.26983701454733</span>}'}}),H=new C({props:{code:`from transformers import create_optimizer
from transformers.keras_callbacks import PushToHubCallback
import tensorflow as tf

# The number of training steps is the number of samples in the dataset, divided by the batch size then multiplied
# by the total number of epochs. Note that the tf_train_dataset here is a batched tf.data.Dataset,
# not the original Hugging Face Dataset, so its len() is already num_samples // batch_size.
num_epochs = 3
num_train_steps = len(tf_train_dataset) * num_epochs

optimizer, schedule = create_optimizer(
    init_lr=5e-5,
    num_warmup_steps=0,
    num_train_steps=num_train_steps,
    weight_decay_rate=0.01,
)
model.compile(optimizer=optimizer)

# Train in mixed-precision float16
tf.keras.mixed_precision.set_global_policy("mixed_float16")`,highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> create_optimizer
<span class="hljs-keyword">from</span> transformers.keras_callbacks <span class="hljs-keyword">import</span> PushToHubCallback
<span class="hljs-keyword">import</span> tensorflow <span class="hljs-keyword">as</span> tf

<span class="hljs-comment"># The number of training steps is the number of samples in the dataset, divided by the batch size then multiplied</span>
<span class="hljs-comment"># by the total number of epochs. Note that the tf_train_dataset here is a batched tf.data.Dataset,</span>
<span class="hljs-comment"># not the original Hugging Face Dataset, so its len() is already num_samples // batch_size.</span>
num_epochs = <span class="hljs-number">3</span>
num_train_steps = <span class="hljs-built_in">len</span>(tf_train_dataset) * num_epochs

optimizer, schedule = create_optimizer(
    init_lr=<span class="hljs-number">5e-5</span>,
    num_warmup_steps=<span class="hljs-number">0</span>,
    num_train_steps=num_train_steps,
    weight_decay_rate=<span class="hljs-number">0.01</span>,
)
model.<span class="hljs-built_in">compile</span>(optimizer=optimizer)

<span class="hljs-comment"># Train in mixed-precision float16</span>
tf.keras.mixed_precision.set_global_policy(<span class="hljs-string">&quot;mixed_float16&quot;</span>)`}}),ee=new C({props:{code:`from transformers.keras_callbacks import PushToHubCallback

callback = PushToHubCallback(
    output_dir="marian-finetuned-kde4-en-to-fr", tokenizer=tokenizer
)

model.fit(
    tf_train_dataset,
    validation_data=tf_eval_dataset,
    callbacks=[callback],
    epochs=num_epochs,
)`,highlighted:`<span class="hljs-keyword">from</span> transformers.keras_callbacks <span class="hljs-keyword">import</span> PushToHubCallback

callback = PushToHubCallback(
    output_dir=<span class="hljs-string">&quot;marian-finetuned-kde4-en-to-fr&quot;</span>, tokenizer=tokenizer
)

model.fit(
    tf_train_dataset,
    validation_data=tf_eval_dataset,
    callbacks=[callback],
    epochs=num_epochs,
)`}}),J=new Za({props:{$$slots:{default:[Nd]},$$scope:{ctx:B}}}),ve=new C({props:{code:"print(compute_metrics())",highlighted:'<span class="hljs-built_in">print</span>(compute_metrics())'}}),Ue=new C({props:{code:"{'bleu': 57.334066271545865}",highlighted:'{&#x27;bleu&#x27;: <span class="hljs-number">57.334066271545865</span>}'}}),{c(){h=l("p"),$=a("Before we start, let\u2019s see what kind of results we get from our model without any training:"),m=c(),y(q.$$.fragment),A=c(),y(g.$$.fragment),z=c(),P=l("p"),D=a("Once this is done, we can prepare everything we need to compile and train our model. Note the use of "),T=l("code"),N=a('tf.keras.mixed_precision.set_global_policy("mixed_float16")'),v=a(" \u2014 this will tell Keras to train using float16, which can give a significant speedup on GPUs that support it (Nvidia 20xx/V100 or newer)."),S=c(),y(H.$$.fragment),U=c(),F=l("p"),X=a("Next, we define a "),M=l("code"),I=a("PushToHubCallback"),Y=a(" to upload our model to the Hub during training, as we saw in "),Z=l("a"),re=a("section 2"),de=a(", and then we simply fit the model with that callback:"),Q=c(),y(ee.$$.fragment),te=c(),G=l("p"),ne=a("Note that you can specify the name of the repository you want to push to with the "),$e=l("code"),V=a("hub_model_id"),se=a(" argument (in particular, you will have to use this argument to push to an organization). For instance, when we pushed the model to the "),fe=l("a"),ke=l("code"),oe=a("huggingface-course"),pe=a(" organization"),W=a(", we added "),ae=l("code"),ze=a('hub_model_id="huggingface-course/marian-finetuned-kde4-en-to-fr"'),me=a(" to "),K=l("code"),ye=a("Seq2SeqTrainingArguments"),le=a(". By default, the repository used will be in your namespace and named after the output directory you set, so here it will be "),xe=l("code"),Me=a('"sgugger/marian-finetuned-kde4-en-to-fr"'),_e=a(" (which is the model we linked to at the beginning of this section)."),qe=c(),y(J.$$.fragment),Ae=c(),ge=l("p"),at=a("Finally, let\u2019s see what our metrics look like now that training has finished:"),ft=c(),y(ve.$$.fragment),Xe=c(),y(Ue.$$.fragment),Se=c(),je=l("p"),mt=a("At this stage, you can use the inference widget on the Model Hub to test your model and share it with your friends. You have successfully fine-tuned a model on a translation task \u2014 congratulations!"),this.h()},l(k){h=r(k,"P",{});var R=i(h);$=n(R,"Before we start, let\u2019s see what kind of results we get from our model without any training:"),R.forEach(t),m=d(k),j(q.$$.fragment,k),A=d(k),j(g.$$.fragment,k),z=d(k),P=r(k,"P",{});var Ze=i(P);D=n(Ze,"Once this is done, we can prepare everything we need to compile and train our model. Note the use of "),T=r(Ze,"CODE",{});var he=i(T);N=n(he,'tf.keras.mixed_precision.set_global_policy("mixed_float16")'),he.forEach(t),v=n(Ze," \u2014 this will tell Keras to train using float16, which can give a significant speedup on GPUs that support it (Nvidia 20xx/V100 or newer)."),Ze.forEach(t),S=d(k),j(H.$$.fragment,k),U=d(k),F=r(k,"P",{});var Be=i(F);X=n(Be,"Next, we define a "),M=r(Be,"CODE",{});var nt=i(M);I=n(nt,"PushToHubCallback"),nt.forEach(t),Y=n(Be," to upload our model to the Hub during training, as we saw in "),Z=r(Be,"A",{href:!0});var Ce=i(Z);re=n(Ce,"section 2"),Ce.forEach(t),de=n(Be,", and then we simply fit the model with that callback:"),Be.forEach(t),Q=d(k),j(ee.$$.fragment,k),te=d(k),G=r(k,"P",{});var ie=i(G);ne=n(ie,"Note that you can specify the name of the repository you want to push to with the "),$e=r(ie,"CODE",{});var Ft=i($e);V=n(Ft,"hub_model_id"),Ft.forEach(t),se=n(ie," argument (in particular, you will have to use this argument to push to an organization). For instance, when we pushed the model to the "),fe=r(ie,"A",{href:!0,rel:!0});var Re=i(fe);ke=r(Re,"CODE",{});var Le=i(ke);oe=n(Le,"huggingface-course"),Le.forEach(t),pe=n(Re," organization"),Re.forEach(t),W=n(ie,", we added "),ae=r(ie,"CODE",{});var Ne=i(ae);ze=n(Ne,'hub_model_id="huggingface-course/marian-finetuned-kde4-en-to-fr"'),Ne.forEach(t),me=n(ie," to "),K=r(ie,"CODE",{});var _t=i(K);ye=n(_t,"Seq2SeqTrainingArguments"),_t.forEach(t),le=n(ie,". By default, the repository used will be in your namespace and named after the output directory you set, so here it will be "),xe=r(ie,"CODE",{});var ce=i(xe);Me=n(ce,'"sgugger/marian-finetuned-kde4-en-to-fr"'),ce.forEach(t),_e=n(ie," (which is the model we linked to at the beginning of this section)."),ie.forEach(t),qe=d(k),j(J.$$.fragment,k),Ae=d(k),ge=r(k,"P",{});var Je=i(ge);at=n(Je,"Finally, let\u2019s see what our metrics look like now that training has finished:"),Je.forEach(t),ft=d(k),j(ve.$$.fragment,k),Xe=d(k),j(Ue.$$.fragment,k),Se=d(k),je=r(k,"P",{});var Oe=i(je);mt=n(Oe,"At this stage, you can use the inference widget on the Model Hub to test your model and share it with your friends. You have successfully fine-tuned a model on a translation task \u2014 congratulations!"),Oe.forEach(t),this.h()},h(){_(Z,"href","(/course/chapter7/2)"),_(fe,"href","https://huggingface.co/huggingface-course"),_(fe,"rel","nofollow")},m(k,R){o(k,h,R),s(h,$),o(k,m,R),E(q,k,R),o(k,A,R),E(g,k,R),o(k,z,R),o(k,P,R),s(P,D),s(P,T),s(T,N),s(P,v),o(k,S,R),E(H,k,R),o(k,U,R),o(k,F,R),s(F,X),s(F,M),s(M,I),s(F,Y),s(F,Z),s(Z,re),s(F,de),o(k,Q,R),E(ee,k,R),o(k,te,R),o(k,G,R),s(G,ne),s(G,$e),s($e,V),s(G,se),s(G,fe),s(fe,ke),s(ke,oe),s(fe,pe),s(G,W),s(G,ae),s(ae,ze),s(G,me),s(G,K),s(K,ye),s(G,le),s(G,xe),s(xe,Me),s(G,_e),o(k,qe,R),E(J,k,R),o(k,Ae,R),o(k,ge,R),s(ge,at),o(k,ft,R),E(ve,k,R),o(k,Xe,R),E(Ue,k,R),o(k,Se,R),o(k,je,R),s(je,mt),we=!0},i(k){we||(w(q.$$.fragment,k),w(g.$$.fragment,k),w(H.$$.fragment,k),w(ee.$$.fragment,k),w(J.$$.fragment,k),w(ve.$$.fragment,k),w(Ue.$$.fragment,k),we=!0)},o(k){b(q.$$.fragment,k),b(g.$$.fragment,k),b(H.$$.fragment,k),b(ee.$$.fragment,k),b(J.$$.fragment,k),b(ve.$$.fragment,k),b(Ue.$$.fragment,k),we=!1},d(k){k&&t(h),k&&t(m),x(q,k),k&&t(A),x(g,k),k&&t(z),k&&t(P),k&&t(S),x(H,k),k&&t(U),k&&t(F),k&&t(Q),x(ee,k),k&&t(te),k&&t(G),k&&t(qe),x(J,k),k&&t(Ae),k&&t(ge),k&&t(ft),x(ve,k),k&&t(Xe),x(Ue,k),k&&t(Se),k&&t(je)}}}function Ld(B){let h,$,m,q,A;return{c(){h=l("p"),$=a("\u{1F4A1} If the output directory you are using already exists, it needs to be a local clone of the repository you want to push to. If it isn\u2019t, you\u2019ll get an error when defining your "),m=l("code"),q=a("Seq2SeqTrainer"),A=a(" and will need to set a new name.")},l(g){h=r(g,"P",{});var z=i(h);$=n(z,"\u{1F4A1} If the output directory you are using already exists, it needs to be a local clone of the repository you want to push to. If it isn\u2019t, you\u2019ll get an error when defining your "),m=r(z,"CODE",{});var P=i(m);q=n(P,"Seq2SeqTrainer"),P.forEach(t),A=n(z," and will need to set a new name."),z.forEach(t)},m(g,z){o(g,h,z),s(h,$),s(h,m),s(m,q),s(h,A)},d(g){g&&t(h)}}}function Nd(B){let h,$,m,q,A;return{c(){h=l("p"),$=a("\u{1F4A1} If the output directory you are using already exists, it needs to be a local clone of the repository you want to push to. If it isn\u2019t, you\u2019ll get an error when calling "),m=l("code"),q=a("model.fit()"),A=a(" and will need to set a new name.")},l(g){h=r(g,"P",{});var z=i(h);$=n(z,"\u{1F4A1} If the output directory you are using already exists, it needs to be a local clone of the repository you want to push to. If it isn\u2019t, you\u2019ll get an error when calling "),m=r(z,"CODE",{});var P=i(m);q=n(P,"model.fit()"),P.forEach(t),A=n(z," and will need to set a new name."),z.forEach(t)},m(g,z){o(g,h,z),s(h,$),s(h,m),s(m,q),s(h,A)},d(g){g&&t(h)}}}function ud(B){let h,$,m,q,A,g,z,P,D,T,N,v,S,H,U,F,X,M,I,Y,Z,re,de,Q,ee,te,G,ne,$e,V,se,fe,ke,oe,pe,W,ae,ze,me,K,ye,le,xe,Me,_e,qe,J,Ae,ge,at,ft,ve,Xe,Ue,Se,je,mt,we,k,R,Ze,he,Be,nt,Ce,ie,Ft,Re,Le,Ne,_t,ce,Je,Oe,zt,is,ot,lt,rt,Ve,Dt,ps,Qe,gt,wt,Ee,Gt,Fe,ks,Xt,De,Zt,Pe,It,bt,vs,Jt,be,$t,Pt,kt,it,He,Vt,vt,pt,ht,Lt,At,et,Qt,Ie,es,Ke,ts,yt,Nt,hs,ct,f,O,ys,jt,Xs,js,Te,Es,Et,cs,ue,Zs,dt,Js,We,Ht,xs,Vs,ss,Ye,Qs,xt,Wt,qs,qt,Ts,ut,ds,us,Mt,zs,Tt,Ds,Ut,fs,tt,xa,ms,Ja,qa,St,ea,Ps,Ta,As;return q=new rs({}),Z=new rs({}),pe=new C({props:{code:`from torch.utils.data import DataLoader

tokenized_datasets.set_format("torch")
train_dataloader = DataLoader(
    tokenized_datasets["train"],
    shuffle=True,
    collate_fn=data_collator,
    batch_size=8,
)
eval_dataloader = DataLoader(
    tokenized_datasets["validation"], collate_fn=data_collator, batch_size=8
)`,highlighted:`<span class="hljs-keyword">from</span> torch.utils.data <span class="hljs-keyword">import</span> DataLoader

tokenized_datasets.set_format(<span class="hljs-string">&quot;torch&quot;</span>)
train_dataloader = DataLoader(
    tokenized_datasets[<span class="hljs-string">&quot;train&quot;</span>],
    shuffle=<span class="hljs-literal">True</span>,
    collate_fn=data_collator,
    batch_size=<span class="hljs-number">8</span>,
)
eval_dataloader = DataLoader(
    tokenized_datasets[<span class="hljs-string">&quot;validation&quot;</span>], collate_fn=data_collator, batch_size=<span class="hljs-number">8</span>
)`}}),K=new C({props:{code:"model = AutoModelForSeq2SeqLM.from_pretrained(model_checkpoint)",highlighted:"model = AutoModelForSeq2SeqLM.from_pretrained(model_checkpoint)"}}),_e=new C({props:{code:`from transformers import AdamW

optimizer = AdamW(model.parameters(), lr=2e-5)`,highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AdamW

optimizer = AdamW(model.parameters(), lr=<span class="hljs-number">2e-5</span>)`}}),je=new C({props:{code:`from accelerate import Accelerator

accelerator = Accelerator()
model, optimizer, train_dataloader, eval_dataloader = accelerator.prepare(
    model, optimizer, train_dataloader, eval_dataloader
)`,highlighted:`<span class="hljs-keyword">from</span> accelerate <span class="hljs-keyword">import</span> Accelerator

accelerator = Accelerator()
model, optimizer, train_dataloader, eval_dataloader = accelerator.prepare(
    model, optimizer, train_dataloader, eval_dataloader
)`}}),Ne=new C({props:{code:`from transformers import get_scheduler

num_train_epochs = 3
num_update_steps_per_epoch = len(train_dataloader)
num_training_steps = num_train_epochs * num_update_steps_per_epoch

lr_scheduler = get_scheduler(
    "linear",
    optimizer=optimizer,
    num_warmup_steps=0,
    num_training_steps=num_training_steps,
)`,highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> get_scheduler

num_train_epochs = <span class="hljs-number">3</span>
num_update_steps_per_epoch = <span class="hljs-built_in">len</span>(train_dataloader)
num_training_steps = num_train_epochs * num_update_steps_per_epoch

lr_scheduler = get_scheduler(
    <span class="hljs-string">&quot;linear&quot;</span>,
    optimizer=optimizer,
    num_warmup_steps=<span class="hljs-number">0</span>,
    num_training_steps=num_training_steps,
)`}}),gt=new C({props:{code:`from huggingface_hub import Repository, get_full_repo_name

model_name = "marian-finetuned-kde4-en-to-fr-accelerate"
repo_name = get_full_repo_name(model_name)
repo_name`,highlighted:`<span class="hljs-keyword">from</span> huggingface_hub <span class="hljs-keyword">import</span> Repository, get_full_repo_name

model_name = <span class="hljs-string">&quot;marian-finetuned-kde4-en-to-fr-accelerate&quot;</span>
repo_name = get_full_repo_name(model_name)
repo_name`}}),Ee=new C({props:{code:"'sgugger/marian-finetuned-kde4-en-to-fr-accelerate'",highlighted:'<span class="hljs-string">&#x27;sgugger/marian-finetuned-kde4-en-to-fr-accelerate&#x27;</span>'}}),De=new C({props:{code:`output_dir = "marian-finetuned-kde4-en-to-fr-accelerate"
repo = Repository(output_dir, clone_from=repo_name)`,highlighted:`output_dir = <span class="hljs-string">&quot;marian-finetuned-kde4-en-to-fr-accelerate&quot;</span>
repo = Repository(output_dir, clone_from=repo_name)`}}),vt=new rs({}),ct=new C({props:{code:`def postprocess(predictions, labels):
    predictions = predictions.cpu().numpy()
    labels = labels.cpu().numpy()

    decoded_preds = tokenizer.batch_decode(predictions, skip_special_tokens=True)

    # Replace -100 in the labels as we can't decode them.
    labels = np.where(labels != -100, labels, tokenizer.pad_token_id)
    decoded_labels = tokenizer.batch_decode(labels, skip_special_tokens=True)

    # Some simple post-processing
    decoded_preds = [pred.strip() for pred in decoded_preds]
    decoded_labels = [[label.strip()] for label in decoded_labels]
    return decoded_preds, decoded_labels`,highlighted:`<span class="hljs-keyword">def</span> <span class="hljs-title function_">postprocess</span>(<span class="hljs-params">predictions, labels</span>):
    predictions = predictions.cpu().numpy()
    labels = labels.cpu().numpy()

    decoded_preds = tokenizer.batch_decode(predictions, skip_special_tokens=<span class="hljs-literal">True</span>)

    <span class="hljs-comment"># Replace -100 in the labels as we can&#x27;t decode them.</span>
    labels = np.where(labels != -<span class="hljs-number">100</span>, labels, tokenizer.pad_token_id)
    decoded_labels = tokenizer.batch_decode(labels, skip_special_tokens=<span class="hljs-literal">True</span>)

    <span class="hljs-comment"># Some simple post-processing</span>
    decoded_preds = [pred.strip() <span class="hljs-keyword">for</span> pred <span class="hljs-keyword">in</span> decoded_preds]
    decoded_labels = [[label.strip()] <span class="hljs-keyword">for</span> label <span class="hljs-keyword">in</span> decoded_labels]
    <span class="hljs-keyword">return</span> decoded_preds, decoded_labels`}}),Tt=new C({props:{code:`from tqdm.auto import tqdm
import torch

progress_bar = tqdm(range(num_training_steps))

for epoch in range(num_train_epochs):
    # Training
    model.train()
    for batch in train_dataloader:
        outputs = model(**batch)
        loss = outputs.loss
        accelerator.backward(loss)

        optimizer.step()
        lr_scheduler.step()
        optimizer.zero_grad()
        progress_bar.update(1)

    # Evaluation
    model.eval()
    for batch in tqdm(eval_dataloader):
        with torch.no_grad():
            generated_tokens = accelerator.unwrap_model(model).generate(
                batch["input_ids"],
                attention_mask=batch["attention_mask"],
                max_length=128,
            )
        labels = batch["labels"]

        # Necessary to pad predictions and labels for being gathered
        generated_tokens = accelerator.pad_across_processes(
            generated_tokens, dim=1, pad_index=tokenizer.pad_token_id
        )
        labels = accelerator.pad_across_processes(labels, dim=1, pad_index=-100)

        predictions_gathered = accelerator.gather(generated_tokens)
        labels_gathered = accelerator.gather(labels)

        decoded_preds, decoded_labels = postprocess(predictions_gathered, labels_gathered)
        metric.add_batch(predictions=decoded_preds, references=decoded_labels)

    results = metric.compute()
    print(f"epoch {epoch}, BLEU score: {results['score']:.2f}")

    # Save and upload
    accelerator.wait_for_everyone()
    unwrapped_model = accelerator.unwrap_model(model)
    unwrapped_model.save_pretrained(output_dir, save_function=accelerator.save)
    if accelerator.is_main_process:
        tokenizer.save_pretrained(output_dir)
        repo.push_to_hub(
            commit_message=f"Training in progress epoch {epoch}", blocking=False
        )`,highlighted:`<span class="hljs-keyword">from</span> tqdm.auto <span class="hljs-keyword">import</span> tqdm
<span class="hljs-keyword">import</span> torch

progress_bar = tqdm(<span class="hljs-built_in">range</span>(num_training_steps))

<span class="hljs-keyword">for</span> epoch <span class="hljs-keyword">in</span> <span class="hljs-built_in">range</span>(num_train_epochs):
    <span class="hljs-comment"># Training</span>
    model.train()
    <span class="hljs-keyword">for</span> batch <span class="hljs-keyword">in</span> train_dataloader:
        outputs = model(**batch)
        loss = outputs.loss
        accelerator.backward(loss)

        optimizer.step()
        lr_scheduler.step()
        optimizer.zero_grad()
        progress_bar.update(<span class="hljs-number">1</span>)

    <span class="hljs-comment"># Evaluation</span>
    model.<span class="hljs-built_in">eval</span>()
    <span class="hljs-keyword">for</span> batch <span class="hljs-keyword">in</span> tqdm(eval_dataloader):
        <span class="hljs-keyword">with</span> torch.no_grad():
            generated_tokens = accelerator.unwrap_model(model).generate(
                batch[<span class="hljs-string">&quot;input_ids&quot;</span>],
                attention_mask=batch[<span class="hljs-string">&quot;attention_mask&quot;</span>],
                max_length=<span class="hljs-number">128</span>,
            )
        labels = batch[<span class="hljs-string">&quot;labels&quot;</span>]

        <span class="hljs-comment"># Necessary to pad predictions and labels for being gathered</span>
        generated_tokens = accelerator.pad_across_processes(
            generated_tokens, dim=<span class="hljs-number">1</span>, pad_index=tokenizer.pad_token_id
        )
        labels = accelerator.pad_across_processes(labels, dim=<span class="hljs-number">1</span>, pad_index=-<span class="hljs-number">100</span>)

        predictions_gathered = accelerator.gather(generated_tokens)
        labels_gathered = accelerator.gather(labels)

        decoded_preds, decoded_labels = postprocess(predictions_gathered, labels_gathered)
        metric.add_batch(predictions=decoded_preds, references=decoded_labels)

    results = metric.compute()
    <span class="hljs-built_in">print</span>(<span class="hljs-string">f&quot;epoch <span class="hljs-subst">{epoch}</span>, BLEU score: <span class="hljs-subst">{results[<span class="hljs-string">&#x27;score&#x27;</span>]:<span class="hljs-number">.2</span>f}</span>&quot;</span>)

    <span class="hljs-comment"># Save and upload</span>
    accelerator.wait_for_everyone()
    unwrapped_model = accelerator.unwrap_model(model)
    unwrapped_model.save_pretrained(output_dir, save_function=accelerator.save)
    <span class="hljs-keyword">if</span> accelerator.is_main_process:
        tokenizer.save_pretrained(output_dir)
        repo.push_to_hub(
            commit_message=<span class="hljs-string">f&quot;Training in progress epoch <span class="hljs-subst">{epoch}</span>&quot;</span>, blocking=<span class="hljs-literal">False</span>
        )`}}),Ut=new C({props:{code:`epoch 0, BLEU score: 53.47
epoch 1, BLEU score: 54.24
epoch 2, BLEU score: 54.44`,highlighted:`epoch <span class="hljs-number">0</span>, BLEU score: <span class="hljs-number">53.47</span>
epoch <span class="hljs-number">1</span>, BLEU score: <span class="hljs-number">54.24</span>
epoch <span class="hljs-number">2</span>, BLEU score: <span class="hljs-number">54.44</span>`}}),{c(){h=l("h2"),$=l("a"),m=l("span"),y(q.$$.fragment),A=c(),g=l("span"),z=a("A custom training loop"),P=c(),D=l("p"),T=a("Let\u2019s now take a look at the full training loop, so you can easily customize the parts you need. It will look a lot like what we did in "),N=l("a"),v=a("section 2"),S=a(" and "),H=l("a"),U=a("Chapter 3"),F=a("."),X=c(),M=l("h3"),I=l("a"),Y=l("span"),y(Z.$$.fragment),re=c(),de=l("span"),Q=a("Preparing everything for training"),ee=c(),te=l("p"),G=a("You\u2019ve seen all of this a few times now, so we\u2019ll go through the code quite quickly. First we\u2019ll build the "),ne=l("code"),$e=a("DataLoader"),V=a("s from our datasets, after setting the datasets to the "),se=l("code"),fe=a('"torch"'),ke=a(" format so we get PyTorch tensors:"),oe=c(),y(pe.$$.fragment),W=c(),ae=l("p"),ze=a("Next we reinstantiate our model, to make sure we\u2019re not continuing the fine-tuning from before but starting from the pretrained model again:"),me=c(),y(K.$$.fragment),ye=c(),le=l("p"),xe=a("Then we will need an optimizer:"),Me=c(),y(_e.$$.fragment),qe=c(),J=l("p"),Ae=a("Once we have all those objects, we can send them to the "),ge=l("code"),at=a("accelerator.prepare()"),ft=a(" method. Remember that if you want to train on TPUs in a Colab notebook, you will need to move all of this code into a training function, and that shouldn\u2019t execute any cell that instantiates an "),ve=l("code"),Xe=a("Accelerator"),Ue=a("."),Se=c(),y(je.$$.fragment),mt=c(),we=l("p"),k=a("Now that we have sent our "),R=l("code"),Ze=a("train_dataloader"),he=a(" to "),Be=l("code"),nt=a("accelerator.prepare()"),Ce=a(", we can use its length to compute the number of training steps. Remember we should always do this after preparing the dataloader, as that method will change the length of the "),ie=l("code"),Ft=a("DataLoader"),Re=a(". We use a classic linear schedule from the learning rate to 0:"),Le=c(),y(Ne.$$.fragment),_t=c(),ce=l("p"),Je=a("Lastly, to push our model to the Hub, we will need to create a "),Oe=l("code"),zt=a("Repository"),is=a(" object in a working folder. First log in to the Hugging Face Hub, if you\u2019re not logged in already. We\u2019ll determine the repository name from the model ID we want to give our model (feel free to replace the "),ot=l("code"),lt=a("repo_name"),rt=a(" with your own choice; it just needs to contain your username, which is what the function "),Ve=l("code"),Dt=a("get_full_repo_name()"),ps=a(" does):"),Qe=c(),y(gt.$$.fragment),wt=c(),y(Ee.$$.fragment),Gt=c(),Fe=l("p"),ks=a("Then we can clone that repository in a local folder. If it already exists, this local folder should be a clone of the repository we are working with:"),Xt=c(),y(De.$$.fragment),Zt=c(),Pe=l("p"),It=a("We can now upload anything we save in "),bt=l("code"),vs=a("output_dir"),Jt=a(" by calling the "),be=l("code"),$t=a("repo.push_to_hub()"),Pt=a(" method. This will help us upload the intermediate models at the end of each epoch."),kt=c(),it=l("h3"),He=l("a"),Vt=l("span"),y(vt.$$.fragment),pt=c(),ht=l("span"),Lt=a("Training loop"),At=c(),et=l("p"),Qt=a("We are now ready to write the full training loop. To simplify its evaluation part, we define this "),Ie=l("code"),es=a("postprocess()"),Ke=a(" function that takes predictions and labels and converts them to the lists of strings our "),ts=l("code"),yt=a("metric"),Nt=a(" object will expect:"),hs=c(),y(ct.$$.fragment),f=c(),O=l("p"),ys=a("The training loop looks a lot like the ones in "),jt=l("a"),Xs=a("section 2"),js=a(" and "),Te=l("a"),Es=a("Chapter 3"),Et=a(", with a few differences in the evaluation part \u2014 so let\u2019s focus on that!"),cs=c(),ue=l("p"),Zs=a("The first thing to note is that we use the "),dt=l("code"),Js=a("generate()"),We=a(" method to compute predictions, but this is a method on our base model, not the wrapped model \u{1F917} Accelerate created in the "),Ht=l("code"),xs=a("prepare()"),Vs=a(" method. That\u2019s why we unwrap the model first, then call this method."),ss=c(),Ye=l("p"),Qs=a("The second thing is that, like with "),xt=l("a"),Wt=a("token classification"),qs=a(", two processes may have padded the inputs and labels to different shapes, so we use "),qt=l("code"),Ts=a("accelerator.pad_across_processes()"),ut=a(" to make the predictions and labels the same shape before calling the "),ds=l("code"),us=a("gather()"),Mt=a(" method. If we don\u2019t do this, the evaluation will either error out or hang forever."),zs=c(),y(Tt.$$.fragment),Ds=c(),y(Ut.$$.fragment),fs=c(),tt=l("p"),xa=a("Once this is done, you should have a model that has results pretty similar to the one trained with the "),ms=l("code"),Ja=a("Seq2SeqTrainer"),qa=a(". You can check the one we trained using this code at "),St=l("a"),ea=l("em"),Ps=a("huggingface-course/marian-finetuned-kde4-en-to-fr-accelerate"),Ta=a(". And if you want to test out any tweaks to the training loop, you can directly implement them by editing the code shown above!"),this.h()},l(u){h=r(u,"H2",{class:!0});var L=i(h);$=r(L,"A",{id:!0,class:!0,href:!0});var ta=i($);m=r(ta,"SPAN",{});var Va=i(m);j(q.$$.fragment,Va),Va.forEach(t),ta.forEach(t),A=d(L),g=r(L,"SPAN",{});var sa=i(g);z=n(sa,"A custom training loop"),sa.forEach(t),L.forEach(t),P=d(u),D=r(u,"P",{});var Ss=i(D);T=n(Ss,"Let\u2019s now take a look at the full training loop, so you can easily customize the parts you need. It will look a lot like what we did in "),N=r(Ss,"A",{href:!0});var ga=i(N);v=n(ga,"section 2"),ga.forEach(t),S=n(Ss," and "),H=r(Ss,"A",{href:!0});var so=i(H);U=n(so,"Chapter 3"),so.forEach(t),F=n(Ss,"."),Ss.forEach(t),X=d(u),M=r(u,"H3",{class:!0});var wa=i(M);I=r(wa,"A",{id:!0,class:!0,href:!0});var aa=i(I);Y=r(aa,"SPAN",{});var Qa=i(Y);j(Z.$$.fragment,Qa),Qa.forEach(t),aa.forEach(t),re=d(wa),de=r(wa,"SPAN",{});var na=i(de);Q=n(na,"Preparing everything for training"),na.forEach(t),wa.forEach(t),ee=d(u),te=r(u,"P",{});var Cs=i(te);G=n(Cs,"You\u2019ve seen all of this a few times now, so we\u2019ll go through the code quite quickly. First we\u2019ll build the "),ne=r(Cs,"CODE",{});var ba=i(ne);$e=n(ba,"DataLoader"),ba.forEach(t),V=n(Cs,"s from our datasets, after setting the datasets to the "),se=r(Cs,"CODE",{});var ao=i(se);fe=n(ao,'"torch"'),ao.forEach(t),ke=n(Cs," format so we get PyTorch tensors:"),Cs.forEach(t),oe=d(u),j(pe.$$.fragment,u),W=d(u),ae=r(u,"P",{});var en=i(ae);ze=n(en,"Next we reinstantiate our model, to make sure we\u2019re not continuing the fine-tuning from before but starting from the pretrained model again:"),en.forEach(t),me=d(u),j(K.$$.fragment,u),ye=d(u),le=r(u,"P",{});var oa=i(le);xe=n(oa,"Then we will need an optimizer:"),oa.forEach(t),Me=d(u),j(_e.$$.fragment,u),qe=d(u),J=r(u,"P",{});var Os=i(J);Ae=n(Os,"Once we have all those objects, we can send them to the "),ge=r(Os,"CODE",{});var la=i(ge);at=n(la,"accelerator.prepare()"),la.forEach(t),ft=n(Os," method. Remember that if you want to train on TPUs in a Colab notebook, you will need to move all of this code into a training function, and that shouldn\u2019t execute any cell that instantiates an "),ve=r(Os,"CODE",{});var tn=i(ve);Xe=n(tn,"Accelerator"),tn.forEach(t),Ue=n(Os,"."),Os.forEach(t),Se=d(u),j(je.$$.fragment,u),mt=d(u),we=r(u,"P",{});var Ct=i(we);k=n(Ct,"Now that we have sent our "),R=r(Ct,"CODE",{});var no=i(R);Ze=n(no,"train_dataloader"),no.forEach(t),he=n(Ct," to "),Be=r(Ct,"CODE",{});var sn=i(Be);nt=n(sn,"accelerator.prepare()"),sn.forEach(t),Ce=n(Ct,", we can use its length to compute the number of training steps. Remember we should always do this after preparing the dataloader, as that method will change the length of the "),ie=r(Ct,"CODE",{});var ra=i(ie);Ft=n(ra,"DataLoader"),ra.forEach(t),Re=n(Ct,". We use a classic linear schedule from the learning rate to 0:"),Ct.forEach(t),Le=d(u),j(Ne.$$.fragment,u),_t=d(u),ce=r(u,"P",{});var as=i(ce);Je=n(as,"Lastly, to push our model to the Hub, we will need to create a "),Oe=r(as,"CODE",{});var Fs=i(Oe);zt=n(Fs,"Repository"),Fs.forEach(t),is=n(as," object in a working folder. First log in to the Hugging Face Hub, if you\u2019re not logged in already. We\u2019ll determine the repository name from the model ID we want to give our model (feel free to replace the "),ot=r(as,"CODE",{});var an=i(ot);lt=n(an,"repo_name"),an.forEach(t),rt=n(as," with your own choice; it just needs to contain your username, which is what the function "),Ve=r(as,"CODE",{});var _s=i(Ve);Dt=n(_s,"get_full_repo_name()"),_s.forEach(t),ps=n(as," does):"),as.forEach(t),Qe=d(u),j(gt.$$.fragment,u),wt=d(u),j(Ee.$$.fragment,u),Gt=d(u),Fe=r(u,"P",{});var Is=i(Fe);ks=n(Is,"Then we can clone that repository in a local folder. If it already exists, this local folder should be a clone of the repository we are working with:"),Is.forEach(t),Xt=d(u),j(De.$$.fragment,u),Zt=d(u),Pe=r(u,"P",{});var gs=i(Pe);It=n(gs,"We can now upload anything we save in "),bt=r(gs,"CODE",{});var ia=i(bt);vs=n(ia,"output_dir"),ia.forEach(t),Jt=n(gs," by calling the "),be=r(gs,"CODE",{});var oo=i(be);$t=n(oo,"repo.push_to_hub()"),oo.forEach(t),Pt=n(gs," method. This will help us upload the intermediate models at the end of each epoch."),gs.forEach(t),kt=d(u),it=r(u,"H3",{class:!0});var pa=i(it);He=r(pa,"A",{id:!0,class:!0,href:!0});var lo=i(He);Vt=r(lo,"SPAN",{});var nn=i(Vt);j(vt.$$.fragment,nn),nn.forEach(t),lo.forEach(t),pt=d(pa),ht=r(pa,"SPAN",{});var ha=i(ht);Lt=n(ha,"Training loop"),ha.forEach(t),pa.forEach(t),At=d(u),et=r(u,"P",{});var Ls=i(et);Qt=n(Ls,"We are now ready to write the full training loop. To simplify its evaluation part, we define this "),Ie=r(Ls,"CODE",{});var ns=i(Ie);es=n(ns,"postprocess()"),ns.forEach(t),Ke=n(Ls," function that takes predictions and labels and converts them to the lists of strings our "),ts=r(Ls,"CODE",{});var ro=i(ts);yt=n(ro,"metric"),ro.forEach(t),Nt=n(Ls," object will expect:"),Ls.forEach(t),hs=d(u),j(ct.$$.fragment,u),f=d(u),O=r(u,"P",{});var ws=i(O);ys=n(ws,"The training loop looks a lot like the ones in "),jt=r(ws,"A",{href:!0});var io=i(jt);Xs=n(io,"section 2"),io.forEach(t),js=n(ws," and "),Te=r(ws,"A",{href:!0});var po=i(Te);Es=n(po,"Chapter 3"),po.forEach(t),Et=n(ws,", with a few differences in the evaluation part \u2014 so let\u2019s focus on that!"),ws.forEach(t),cs=d(u),ue=r(u,"P",{});var Bt=i(ue);Zs=n(Bt,"The first thing to note is that we use the "),dt=r(Bt,"CODE",{});var ho=i(dt);Js=n(ho,"generate()"),ho.forEach(t),We=n(Bt," method to compute predictions, but this is a method on our base model, not the wrapped model \u{1F917} Accelerate created in the "),Ht=r(Bt,"CODE",{});var co=i(Ht);xs=n(co,"prepare()"),co.forEach(t),Vs=n(Bt," method. That\u2019s why we unwrap the model first, then call this method."),Bt.forEach(t),ss=d(u),Ye=r(u,"P",{});var os=i(Ye);Qs=n(os,"The second thing is that, like with "),xt=r(os,"A",{href:!0});var ca=i(xt);Wt=n(ca,"token classification"),ca.forEach(t),qs=n(os,", two processes may have padded the inputs and labels to different shapes, so we use "),qt=r(os,"CODE",{});var on=i(qt);Ts=n(on,"accelerator.pad_across_processes()"),on.forEach(t),ut=n(os," to make the predictions and labels the same shape before calling the "),ds=r(os,"CODE",{});var ls=i(ds);us=n(ls,"gather()"),ls.forEach(t),Mt=n(os," method. If we don\u2019t do this, the evaluation will either error out or hang forever."),os.forEach(t),zs=d(u),j(Tt.$$.fragment,u),Ds=d(u),j(Ut.$$.fragment,u),fs=d(u),tt=r(u,"P",{});var da=i(tt);xa=n(da,"Once this is done, you should have a model that has results pretty similar to the one trained with the "),ms=r(da,"CODE",{});var za=i(ms);Ja=n(za,"Seq2SeqTrainer"),za.forEach(t),qa=n(da,". You can check the one we trained using this code at "),St=r(da,"A",{href:!0,rel:!0});var uo=i(St);ea=r(uo,"EM",{});var fo=i(ea);Ps=n(fo,"huggingface-course/marian-finetuned-kde4-en-to-fr-accelerate"),fo.forEach(t),uo.forEach(t),Ta=n(da,". And if you want to test out any tweaks to the training loop, you can directly implement them by editing the code shown above!"),da.forEach(t),this.h()},h(){_($,"id","a-custom-training-loop"),_($,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),_($,"href","#a-custom-training-loop"),_(h,"class","relative group"),_(N,"href","/course/chapter7/2"),_(H,"href","/course/chapter3/4"),_(I,"id","preparing-everything-for-training"),_(I,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),_(I,"href","#preparing-everything-for-training"),_(M,"class","relative group"),_(He,"id","training-loop"),_(He,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),_(He,"href","#training-loop"),_(it,"class","relative group"),_(jt,"href","/course/chapter7/2"),_(Te,"href","/course/chapter3"),_(xt,"href","/course/chapter7/2"),_(St,"href","https://huggingface.co/huggingface-course/marian-finetuned-kde4-en-to-fr-accelerate"),_(St,"rel","nofollow")},m(u,L){o(u,h,L),s(h,$),s($,m),E(q,m,null),s(h,A),s(h,g),s(g,z),o(u,P,L),o(u,D,L),s(D,T),s(D,N),s(N,v),s(D,S),s(D,H),s(H,U),s(D,F),o(u,X,L),o(u,M,L),s(M,I),s(I,Y),E(Z,Y,null),s(M,re),s(M,de),s(de,Q),o(u,ee,L),o(u,te,L),s(te,G),s(te,ne),s(ne,$e),s(te,V),s(te,se),s(se,fe),s(te,ke),o(u,oe,L),E(pe,u,L),o(u,W,L),o(u,ae,L),s(ae,ze),o(u,me,L),E(K,u,L),o(u,ye,L),o(u,le,L),s(le,xe),o(u,Me,L),E(_e,u,L),o(u,qe,L),o(u,J,L),s(J,Ae),s(J,ge),s(ge,at),s(J,ft),s(J,ve),s(ve,Xe),s(J,Ue),o(u,Se,L),E(je,u,L),o(u,mt,L),o(u,we,L),s(we,k),s(we,R),s(R,Ze),s(we,he),s(we,Be),s(Be,nt),s(we,Ce),s(we,ie),s(ie,Ft),s(we,Re),o(u,Le,L),E(Ne,u,L),o(u,_t,L),o(u,ce,L),s(ce,Je),s(ce,Oe),s(Oe,zt),s(ce,is),s(ce,ot),s(ot,lt),s(ce,rt),s(ce,Ve),s(Ve,Dt),s(ce,ps),o(u,Qe,L),E(gt,u,L),o(u,wt,L),E(Ee,u,L),o(u,Gt,L),o(u,Fe,L),s(Fe,ks),o(u,Xt,L),E(De,u,L),o(u,Zt,L),o(u,Pe,L),s(Pe,It),s(Pe,bt),s(bt,vs),s(Pe,Jt),s(Pe,be),s(be,$t),s(Pe,Pt),o(u,kt,L),o(u,it,L),s(it,He),s(He,Vt),E(vt,Vt,null),s(it,pt),s(it,ht),s(ht,Lt),o(u,At,L),o(u,et,L),s(et,Qt),s(et,Ie),s(Ie,es),s(et,Ke),s(et,ts),s(ts,yt),s(et,Nt),o(u,hs,L),E(ct,u,L),o(u,f,L),o(u,O,L),s(O,ys),s(O,jt),s(jt,Xs),s(O,js),s(O,Te),s(Te,Es),s(O,Et),o(u,cs,L),o(u,ue,L),s(ue,Zs),s(ue,dt),s(dt,Js),s(ue,We),s(ue,Ht),s(Ht,xs),s(ue,Vs),o(u,ss,L),o(u,Ye,L),s(Ye,Qs),s(Ye,xt),s(xt,Wt),s(Ye,qs),s(Ye,qt),s(qt,Ts),s(Ye,ut),s(Ye,ds),s(ds,us),s(Ye,Mt),o(u,zs,L),E(Tt,u,L),o(u,Ds,L),E(Ut,u,L),o(u,fs,L),o(u,tt,L),s(tt,xa),s(tt,ms),s(ms,Ja),s(tt,qa),s(tt,St),s(St,ea),s(ea,Ps),s(tt,Ta),As=!0},i(u){As||(w(q.$$.fragment,u),w(Z.$$.fragment,u),w(pe.$$.fragment,u),w(K.$$.fragment,u),w(_e.$$.fragment,u),w(je.$$.fragment,u),w(Ne.$$.fragment,u),w(gt.$$.fragment,u),w(Ee.$$.fragment,u),w(De.$$.fragment,u),w(vt.$$.fragment,u),w(ct.$$.fragment,u),w(Tt.$$.fragment,u),w(Ut.$$.fragment,u),As=!0)},o(u){b(q.$$.fragment,u),b(Z.$$.fragment,u),b(pe.$$.fragment,u),b(K.$$.fragment,u),b(_e.$$.fragment,u),b(je.$$.fragment,u),b(Ne.$$.fragment,u),b(gt.$$.fragment,u),b(Ee.$$.fragment,u),b(De.$$.fragment,u),b(vt.$$.fragment,u),b(ct.$$.fragment,u),b(Tt.$$.fragment,u),b(Ut.$$.fragment,u),As=!1},d(u){u&&t(h),x(q),u&&t(P),u&&t(D),u&&t(X),u&&t(M),x(Z),u&&t(ee),u&&t(te),u&&t(oe),x(pe,u),u&&t(W),u&&t(ae),u&&t(me),x(K,u),u&&t(ye),u&&t(le),u&&t(Me),x(_e,u),u&&t(qe),u&&t(J),u&&t(Se),x(je,u),u&&t(mt),u&&t(we),u&&t(Le),x(Ne,u),u&&t(_t),u&&t(ce),u&&t(Qe),x(gt,u),u&&t(wt),x(Ee,u),u&&t(Gt),u&&t(Fe),u&&t(Xt),x(De,u),u&&t(Zt),u&&t(Pe),u&&t(kt),u&&t(it),x(vt),u&&t(At),u&&t(et),u&&t(hs),x(ct,u),u&&t(f),u&&t(O),u&&t(cs),u&&t(ue),u&&t(ss),u&&t(Ye),u&&t(zs),x(Tt,u),u&&t(Ds),x(Ut,u),u&&t(fs),u&&t(tt)}}}function Hd(B){let h,$,m,q,A;return{c(){h=l("p"),$=a("\u270F\uFE0F "),m=l("strong"),q=a("Your turn!"),A=a(" What does the model return on the sample with the word \u201Cemail\u201D you identified earlier?")},l(g){h=r(g,"P",{});var z=i(h);$=n(z,"\u270F\uFE0F "),m=r(z,"STRONG",{});var P=i(m);q=n(P,"Your turn!"),P.forEach(t),A=n(z," What does the model return on the sample with the word \u201Cemail\u201D you identified earlier?"),z.forEach(t)},m(g,z){o(g,h,z),s(h,$),s(h,m),s(m,q),s(h,A)},d(g){g&&t(h)}}}function Wd(B){let h,$,m,q,A,g,z,P,D,T,N,v,S,H,U,F,X,M,I,Y,Z,re,de,Q,ee,te,G,ne,$e,V,se,fe,ke,oe,pe,W,ae,ze,me,K,ye,le,xe,Me,_e,qe,J,Ae,ge,at,ft,ve,Xe,Ue,Se,je,mt,we,k,R,Ze,he,Be,nt,Ce,ie,Ft,Re,Le,Ne,_t,ce,Je,Oe,zt,is,ot,lt,rt,Ve,Dt,ps,Qe,gt,wt,Ee,Gt,Fe,ks,Xt,De,Zt,Pe,It,bt,vs,Jt,be,$t,Pt,kt,it,He,Vt,vt,pt,ht,Lt,At,et,Qt,Ie,es,Ke,ts,yt,Nt,hs,ct,f,O,ys,jt,Xs,js,Te,Es,Et,cs,ue,Zs,dt,Js,We,Ht,xs,Vs,ss,Ye,Qs,xt,Wt,qs,qt,Ts,ut,ds,us,Mt,zs,Tt,Ds,Ut,fs,tt,xa,ms,Ja,qa,St,ea,Ps,Ta,As,u,L,ta,Va,sa,Ss,ga,so,wa,aa,Qa,na,Cs,ba,ao,en,oa,Os,la,tn,Ct,no,sn,ra,as,Fs,an,_s,Is,gs,ia,oo,pa,lo,nn,ha,Ls,ns,ro,ws,io,po,Bt,ho,co,os,ca,on,ls,da,za,uo,fo,ln,mi,_i,Sl,Da,Cl,Pa,gi,Qo,wi,bi,Ol,Aa,$i,el,ki,vi,Fl,rn,Il,ua,yi,tl,ji,Ei,sl,xi,qi,Ll,Sa,Ti,al,zi,Di,Nl,mo,Pi,Hl,pn,Wl,_o,Ai,Ml,hn,Ul,cn,Bl,go,Si,Rl,bs,Ci,nl,Oi,Fi,ol,Ii,Li,ll,Ni,Hi,Kl,dn,Yl,wo,Wi,Gl,Ca,Xl,Oa,Zl,bo,Mi,Jl,un,Vl,$o,Ui,Ql,Ns,Hs,ko,vo,Bi,er,$a,Fa,rl,fn,Ri,il,Ki,tr,$s,Yi,pl,Gi,Xi,yo,Zi,Ji,hl,Vi,Qi,sr,st,ep,mn,cl,tp,sp,dl,ap,np,ul,op,lp,fl,rp,ip,ml,pp,hp,_l,cp,dp,ar,Ws,Ms,jo,Eo,up,nr,_n,or,gn,lr,Ia,fp,gl,mp,_p,rr,wn,ir,bn,pr,xo,gp,hr,$n,cr,kn,dr,qo,wp,ur,vn,fr,yn,mr,Us,Bs,To,ka,La,wl,jn,bp,bl,$p,_r,En,gr,zo,Rt,kp,xn,vp,yp,qn,jp,Ep,$l,xp,qp,kl,Tp,zp,wr,Na,Dp,Tn,Pp,Ap,br,zn,$r,fa,Sp,vl,Cp,Op,Do,Fp,Ip,kr,Dn,vr,Po,Lp,yr,Ao,Np,jr,Pn,Er,An,xr,Kt,Hp,Sn,Wp,Mp,yl,Up,Bp,jl,Rp,Kp,Cn,Yp,Gp,qr,On,Tr,Fn,zr,In,Dr,Ln,Pr,So,Xp,Ar,Rs,Ks,Co,Oo,Zp,Sr,va,Ha,El,Nn,Jp,xl,Vp,Cr,Fo,Qp,Or,Hn,Fr,Io,eh,Ir,Lo,th,Lr,Wn,Nr,Ys,Gs,No,Ho,ya,Wa,ql,Mn,sh,Tl,ah,Hr,Ma,nh,zl,oh,lh,Wr,Un,Mr,Bn,Ur,Wo,rh,Br,Rn,Rr,Kn,Kr,Mo,ih,Yr,Ua,Gr;m=new $d({props:{fw:B[0]}}),P=new rs({});const ph=[vd,kd],Yn=[];function hh(e,p){return e[0]==="pt"?0:1}S=hh(B),H=Yn[S]=ph[S](B),me=new fi({props:{id:"1JvfrvZgi6c"}}),Dt=new rs({}),kt=new rs({}),Ie=new C({props:{code:`from datasets import load_dataset, load_metric

raw_datasets = load_dataset("kde4", lang1="en", lang2="fr")`,highlighted:`<span class="hljs-keyword">from</span> datasets <span class="hljs-keyword">import</span> load_dataset, load_metric

raw_datasets = load_dataset(<span class="hljs-string">&quot;kde4&quot;</span>, lang1=<span class="hljs-string">&quot;en&quot;</span>, lang2=<span class="hljs-string">&quot;fr&quot;</span>)`}}),Te=new C({props:{code:"raw_datasets",highlighted:"raw_datasets"}}),Et=new C({props:{code:`DatasetDict({
    train: Dataset({
        features: ['id', 'translation'],
        num_rows: 210173
    })
})`,highlighted:`DatasetDict({
    train: Dataset({
        features: [<span class="hljs-string">&#x27;id&#x27;</span>, <span class="hljs-string">&#x27;translation&#x27;</span>],
        num_rows: <span class="hljs-number">210173</span>
    })
})`}}),Wt=new C({props:{code:`split_datasets = raw_datasets["train"].train_test_split(train_size=0.9, seed=20)
split_datasets`,highlighted:`split_datasets = raw_datasets[<span class="hljs-string">&quot;train&quot;</span>].train_test_split(train_size=<span class="hljs-number">0.9</span>, seed=<span class="hljs-number">20</span>)
split_datasets`}}),qt=new C({props:{code:`DatasetDict({
    train: Dataset({
        features: ['id', 'translation'],
        num_rows: 189155
    })
    test: Dataset({
        features: ['id', 'translation'],
        num_rows: 21018
    })
})`,highlighted:`DatasetDict({
    train: Dataset({
        features: [<span class="hljs-string">&#x27;id&#x27;</span>, <span class="hljs-string">&#x27;translation&#x27;</span>],
        num_rows: <span class="hljs-number">189155</span>
    })
    test: Dataset({
        features: [<span class="hljs-string">&#x27;id&#x27;</span>, <span class="hljs-string">&#x27;translation&#x27;</span>],
        num_rows: <span class="hljs-number">21018</span>
    })
})`}}),tt=new C({props:{code:'split_datasets["validation"] = split_datasets.pop("test")',highlighted:'split_datasets[<span class="hljs-string">&quot;validation&quot;</span>] = split_datasets.pop(<span class="hljs-string">&quot;test&quot;</span>)'}}),St=new C({props:{code:'split_datasets["train"][1]["translation"]',highlighted:'split_datasets[<span class="hljs-string">&quot;train&quot;</span>][<span class="hljs-number">1</span>][<span class="hljs-string">&quot;translation&quot;</span>]'}}),Ps=new C({props:{code:`{'en': 'Default to expanded threads',
 'fr': 'Par d\xE9faut, d\xE9velopper les fils de discussion'}`,highlighted:`{<span class="hljs-string">&#x27;en&#x27;</span>: <span class="hljs-string">&#x27;Default to expanded threads&#x27;</span>,
 <span class="hljs-string">&#x27;fr&#x27;</span>: <span class="hljs-string">&#x27;Par d\xE9faut, d\xE9velopper les fils de discussion&#x27;</span>}`}}),ta=new C({props:{code:`from transformers import pipeline

model_checkpoint = "Helsinki-NLP/opus-mt-en-fr"
translator = pipeline("translation", model=model_checkpoint)
translator("Default to expanded threads")`,highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> pipeline

model_checkpoint = <span class="hljs-string">&quot;Helsinki-NLP/opus-mt-en-fr&quot;</span>
translator = pipeline(<span class="hljs-string">&quot;translation&quot;</span>, model=model_checkpoint)
translator(<span class="hljs-string">&quot;Default to expanded threads&quot;</span>)`}}),sa=new C({props:{code:"[{'translation_text': 'Par d\xE9faut pour les threads \xE9largis'}]",highlighted:'[{<span class="hljs-string">&#x27;translation_text&#x27;</span>: <span class="hljs-string">&#x27;Par d\xE9faut pour les threads \xE9largis&#x27;</span>}]'}}),aa=new C({props:{code:'split_datasets["train"][172]["translation"]',highlighted:'split_datasets[<span class="hljs-string">&quot;train&quot;</span>][<span class="hljs-number">172</span>][<span class="hljs-string">&quot;translation&quot;</span>]'}}),na=new C({props:{code:`{'en': 'Unable to import %1 using the OFX importer plugin. This file is not the correct format.',
 'fr': "Impossible d'importer %1 en utilisant le module d'extension d'importation OFX. Ce fichier n'a pas un format correct."}`,highlighted:`{<span class="hljs-string">&#x27;en&#x27;</span>: <span class="hljs-string">&#x27;Unable to import %1 using the OFX importer plugin. This file is not the correct format.&#x27;</span>,
 <span class="hljs-string">&#x27;fr&#x27;</span>: <span class="hljs-string">&quot;Impossible d&#x27;importer %1 en utilisant le module d&#x27;extension d&#x27;importation OFX. Ce fichier n&#x27;a pas un format correct.&quot;</span>}`}}),oa=new C({props:{code:`translator(
    "Unable to import %1 using the OFX importer plugin. This file is not the correct format."
)`,highlighted:`translator(
    <span class="hljs-string">&quot;Unable to import %1 using the OFX importer plugin. This file is not the correct format.&quot;</span>
)`}}),la=new C({props:{code:`[{'translation_text': "Impossible d'importer %1 en utilisant le plugin d'importateur OFX. Ce fichier n'est pas le bon format."}]`,highlighted:'[{<span class="hljs-string">&#x27;translation_text&#x27;</span>: <span class="hljs-string">&quot;Impossible d&#x27;importer %1 en utilisant le plugin d&#x27;importateur OFX. Ce fichier n&#x27;est pas le bon format.&quot;</span>}]'}}),ra=new fi({props:{id:"0Oxphw4Q9fo"}}),Fs=new Za({props:{$$slots:{default:[yd]},$$scope:{ctx:B}}}),ia=new rs({}),ha=new fi({props:{id:"XAR8jnZZuUs"}}),ca=new C({props:{code:`from transformers import AutoTokenizer

model_checkpoint = "Helsinki-NLP/opus-mt-en-fr"
tokenizer = AutoTokenizer.from_pretrained(model_checkpoint, return_tensors="tf")`,highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoTokenizer

model_checkpoint = <span class="hljs-string">&quot;Helsinki-NLP/opus-mt-en-fr&quot;</span>
tokenizer = AutoTokenizer.from_pretrained(model_checkpoint, return_tensors=<span class="hljs-string">&quot;tf&quot;</span>)`}}),Da=new Za({props:{$$slots:{default:[jd]},$$scope:{ctx:B}}}),rn=new C({props:{code:`with open(file_path) as f:
    content = f.read()`,highlighted:`<span class="hljs-keyword">with</span> <span class="hljs-built_in">open</span>(file_path) <span class="hljs-keyword">as</span> f:
    content = f.<span class="hljs-built_in">read</span>()`}}),pn=new C({props:{code:`en_sentence = split_datasets["train"][1]["translation"]["en"]
fr_sentence = split_datasets["train"][1]["translation"]["fr"]

inputs = tokenizer(en_sentence)
with tokenizer.as_target_tokenizer():
    targets = tokenizer(fr_sentence)`,highlighted:`en_sentence = split_datasets[<span class="hljs-string">&quot;train&quot;</span>][<span class="hljs-number">1</span>][<span class="hljs-string">&quot;translation&quot;</span>][<span class="hljs-string">&quot;en&quot;</span>]
fr_sentence = split_datasets[<span class="hljs-string">&quot;train&quot;</span>][<span class="hljs-number">1</span>][<span class="hljs-string">&quot;translation&quot;</span>][<span class="hljs-string">&quot;fr&quot;</span>]

inputs = tokenizer(en_sentence)
<span class="hljs-keyword">with</span> tokenizer.as_target_tokenizer():
    targets = tokenizer(fr_sentence)`}}),hn=new C({props:{code:`wrong_targets = tokenizer(fr_sentence)
print(tokenizer.convert_ids_to_tokens(wrong_targets["input_ids"]))
print(tokenizer.convert_ids_to_tokens(targets["input_ids"]))`,highlighted:`wrong_targets = tokenizer(fr_sentence)
<span class="hljs-built_in">print</span>(tokenizer.convert_ids_to_tokens(wrong_targets[<span class="hljs-string">&quot;input_ids&quot;</span>]))
<span class="hljs-built_in">print</span>(tokenizer.convert_ids_to_tokens(targets[<span class="hljs-string">&quot;input_ids&quot;</span>]))`}}),cn=new C({props:{code:`['\u2581Par', '\u2581d\xE9', 'f', 'aut', ',', '\u2581d\xE9', 've', 'lop', 'per', '\u2581les', '\u2581fil', 's', '\u2581de', '\u2581discussion', '</s>']
['\u2581Par', '\u2581d\xE9faut', ',', '\u2581d\xE9velopper', '\u2581les', '\u2581fils', '\u2581de', '\u2581discussion', '</s>']`,highlighted:`[<span class="hljs-string">&#x27;\u2581Par&#x27;</span>, <span class="hljs-string">&#x27;\u2581d\xE9&#x27;</span>, <span class="hljs-string">&#x27;f&#x27;</span>, <span class="hljs-string">&#x27;aut&#x27;</span>, <span class="hljs-string">&#x27;,&#x27;</span>, <span class="hljs-string">&#x27;\u2581d\xE9&#x27;</span>, <span class="hljs-string">&#x27;ve&#x27;</span>, <span class="hljs-string">&#x27;lop&#x27;</span>, <span class="hljs-string">&#x27;per&#x27;</span>, <span class="hljs-string">&#x27;\u2581les&#x27;</span>, <span class="hljs-string">&#x27;\u2581fil&#x27;</span>, <span class="hljs-string">&#x27;s&#x27;</span>, <span class="hljs-string">&#x27;\u2581de&#x27;</span>, <span class="hljs-string">&#x27;\u2581discussion&#x27;</span>, <span class="hljs-string">&#x27;&lt;/s&gt;&#x27;</span>]
[<span class="hljs-string">&#x27;\u2581Par&#x27;</span>, <span class="hljs-string">&#x27;\u2581d\xE9faut&#x27;</span>, <span class="hljs-string">&#x27;,&#x27;</span>, <span class="hljs-string">&#x27;\u2581d\xE9velopper&#x27;</span>, <span class="hljs-string">&#x27;\u2581les&#x27;</span>, <span class="hljs-string">&#x27;\u2581fils&#x27;</span>, <span class="hljs-string">&#x27;\u2581de&#x27;</span>, <span class="hljs-string">&#x27;\u2581discussion&#x27;</span>, <span class="hljs-string">&#x27;&lt;/s&gt;&#x27;</span>]`}}),dn=new C({props:{code:`max_input_length = 128
max_target_length = 128


def preprocess_function(examples):
    inputs = [ex["en"] for ex in examples["translation"]]
    targets = [ex["fr"] for ex in examples["translation"]]
    model_inputs = tokenizer(inputs, max_length=max_input_length, truncation=True)

    # Set up the tokenizer for targets
    with tokenizer.as_target_tokenizer():
        labels = tokenizer(targets, max_length=max_target_length, truncation=True)

    model_inputs["labels"] = labels["input_ids"]
    return model_inputs`,highlighted:`max_input_length = <span class="hljs-number">128</span>
max_target_length = <span class="hljs-number">128</span>


<span class="hljs-keyword">def</span> <span class="hljs-title function_">preprocess_function</span>(<span class="hljs-params">examples</span>):
    inputs = [ex[<span class="hljs-string">&quot;en&quot;</span>] <span class="hljs-keyword">for</span> ex <span class="hljs-keyword">in</span> examples[<span class="hljs-string">&quot;translation&quot;</span>]]
    targets = [ex[<span class="hljs-string">&quot;fr&quot;</span>] <span class="hljs-keyword">for</span> ex <span class="hljs-keyword">in</span> examples[<span class="hljs-string">&quot;translation&quot;</span>]]
    model_inputs = tokenizer(inputs, max_length=max_input_length, truncation=<span class="hljs-literal">True</span>)

    <span class="hljs-comment"># Set up the tokenizer for targets</span>
    <span class="hljs-keyword">with</span> tokenizer.as_target_tokenizer():
        labels = tokenizer(targets, max_length=max_target_length, truncation=<span class="hljs-literal">True</span>)

    model_inputs[<span class="hljs-string">&quot;labels&quot;</span>] = labels[<span class="hljs-string">&quot;input_ids&quot;</span>]
    <span class="hljs-keyword">return</span> model_inputs`}}),Ca=new Za({props:{$$slots:{default:[Ed]},$$scope:{ctx:B}}}),Oa=new Za({props:{warning:!0,$$slots:{default:[xd]},$$scope:{ctx:B}}}),un=new C({props:{code:`tokenized_datasets = split_datasets.map(
    preprocess_function,
    batched=True,
    remove_columns=split_datasets["train"].column_names,
)`,highlighted:`tokenized_datasets = split_datasets.<span class="hljs-built_in">map</span>(
    preprocess_function,
    batched=<span class="hljs-literal">True</span>,
    remove_columns=split_datasets[<span class="hljs-string">&quot;train&quot;</span>].column_names,
)`}});const ch=[Td,qd],Gn=[];function dh(e,p){return e[0]==="pt"?0:1}Ns=dh(B),Hs=Gn[Ns]=ch[Ns](B),fn=new rs({});const uh=[Pd,Dd],Xn=[];function fh(e,p){return e[0]==="pt"?0:1}Ws=fh(B),Ms=Xn[Ws]=uh[Ws](B),_n=new C({props:{code:`batch = data_collator([tokenized_datasets["train"][i] for i in range(1, 3)])
batch.keys()`,highlighted:`batch = data_collator([tokenized_datasets[<span class="hljs-string">&quot;train&quot;</span>][i] <span class="hljs-keyword">for</span> i <span class="hljs-keyword">in</span> <span class="hljs-built_in">range</span>(<span class="hljs-number">1</span>, <span class="hljs-number">3</span>)])
batch.keys()`}}),gn=new C({props:{code:"dict_keys(['attention_mask', 'input_ids', 'labels', 'decoder_input_ids'])",highlighted:'dict_keys([<span class="hljs-string">&#x27;attention_mask&#x27;</span>, <span class="hljs-string">&#x27;input_ids&#x27;</span>, <span class="hljs-string">&#x27;labels&#x27;</span>, <span class="hljs-string">&#x27;decoder_input_ids&#x27;</span>])'}}),wn=new C({props:{code:'batch["labels"]',highlighted:'batch[<span class="hljs-string">&quot;labels&quot;</span>]'}}),bn=new C({props:{code:`tensor([[  577,  5891,     2,  3184,    16,  2542,     5,  1710,     0,  -100,
          -100,  -100,  -100,  -100,  -100,  -100],
        [ 1211,     3,    49,  9409,  1211,     3, 29140,   817,  3124,   817,
           550,  7032,  5821,  7907, 12649,     0]])`,highlighted:`tensor([[  <span class="hljs-number">577</span>,  <span class="hljs-number">5891</span>,     <span class="hljs-number">2</span>,  <span class="hljs-number">3184</span>,    <span class="hljs-number">16</span>,  <span class="hljs-number">2542</span>,     <span class="hljs-number">5</span>,  <span class="hljs-number">1710</span>,     <span class="hljs-number">0</span>,  -<span class="hljs-number">100</span>,
          -<span class="hljs-number">100</span>,  -<span class="hljs-number">100</span>,  -<span class="hljs-number">100</span>,  -<span class="hljs-number">100</span>,  -<span class="hljs-number">100</span>,  -<span class="hljs-number">100</span>],
        [ <span class="hljs-number">1211</span>,     <span class="hljs-number">3</span>,    <span class="hljs-number">49</span>,  <span class="hljs-number">9409</span>,  <span class="hljs-number">1211</span>,     <span class="hljs-number">3</span>, <span class="hljs-number">29140</span>,   <span class="hljs-number">817</span>,  <span class="hljs-number">3124</span>,   <span class="hljs-number">817</span>,
           <span class="hljs-number">550</span>,  <span class="hljs-number">7032</span>,  <span class="hljs-number">5821</span>,  <span class="hljs-number">7907</span>, <span class="hljs-number">12649</span>,     <span class="hljs-number">0</span>]])`}}),$n=new C({props:{code:'batch["decoder_input_ids"]',highlighted:'batch[<span class="hljs-string">&quot;decoder_input_ids&quot;</span>]'}}),kn=new C({props:{code:`tensor([[59513,   577,  5891,     2,  3184,    16,  2542,     5,  1710,     0,
         59513, 59513, 59513, 59513, 59513, 59513],
        [59513,  1211,     3,    49,  9409,  1211,     3, 29140,   817,  3124,
           817,   550,  7032,  5821,  7907, 12649]])`,highlighted:`tensor([[<span class="hljs-number">59513</span>,   <span class="hljs-number">577</span>,  <span class="hljs-number">5891</span>,     <span class="hljs-number">2</span>,  <span class="hljs-number">3184</span>,    <span class="hljs-number">16</span>,  <span class="hljs-number">2542</span>,     <span class="hljs-number">5</span>,  <span class="hljs-number">1710</span>,     <span class="hljs-number">0</span>,
         <span class="hljs-number">59513</span>, <span class="hljs-number">59513</span>, <span class="hljs-number">59513</span>, <span class="hljs-number">59513</span>, <span class="hljs-number">59513</span>, <span class="hljs-number">59513</span>],
        [<span class="hljs-number">59513</span>,  <span class="hljs-number">1211</span>,     <span class="hljs-number">3</span>,    <span class="hljs-number">49</span>,  <span class="hljs-number">9409</span>,  <span class="hljs-number">1211</span>,     <span class="hljs-number">3</span>, <span class="hljs-number">29140</span>,   <span class="hljs-number">817</span>,  <span class="hljs-number">3124</span>,
           <span class="hljs-number">817</span>,   <span class="hljs-number">550</span>,  <span class="hljs-number">7032</span>,  <span class="hljs-number">5821</span>,  <span class="hljs-number">7907</span>, <span class="hljs-number">12649</span>]])`}}),vn=new C({props:{code:`for i in range(1, 3):
    print(tokenized_datasets["train"][i]["labels"])`,highlighted:`<span class="hljs-keyword">for</span> i <span class="hljs-keyword">in</span> <span class="hljs-built_in">range</span>(<span class="hljs-number">1</span>, <span class="hljs-number">3</span>):
    <span class="hljs-built_in">print</span>(tokenized_datasets[<span class="hljs-string">&quot;train&quot;</span>][i][<span class="hljs-string">&quot;labels&quot;</span>])`}}),yn=new C({props:{code:`[577, 5891, 2, 3184, 16, 2542, 5, 1710, 0]
[1211, 3, 49, 9409, 1211, 3, 29140, 817, 3124, 817, 550, 7032, 5821, 7907, 12649, 0]`,highlighted:`[<span class="hljs-number">577</span>, <span class="hljs-number">5891</span>, <span class="hljs-number">2</span>, <span class="hljs-number">3184</span>, <span class="hljs-number">16</span>, <span class="hljs-number">2542</span>, <span class="hljs-number">5</span>, <span class="hljs-number">1710</span>, <span class="hljs-number">0</span>]
[<span class="hljs-number">1211</span>, <span class="hljs-number">3</span>, <span class="hljs-number">49</span>, <span class="hljs-number">9409</span>, <span class="hljs-number">1211</span>, <span class="hljs-number">3</span>, <span class="hljs-number">29140</span>, <span class="hljs-number">817</span>, <span class="hljs-number">3124</span>, <span class="hljs-number">817</span>, <span class="hljs-number">550</span>, <span class="hljs-number">7032</span>, <span class="hljs-number">5821</span>, <span class="hljs-number">7907</span>, <span class="hljs-number">12649</span>, <span class="hljs-number">0</span>]`}});const mh=[Sd,Ad],Zn=[];function _h(e,p){return e[0]==="pt"?0:1}Us=_h(B),Bs=Zn[Us]=mh[Us](B),jn=new rs({}),En=new fi({props:{id:"M05L1DhFqcw"}});let Ot=B[0]==="pt"&&dd();zn=new C({props:{code:"!pip install sacrebleu",highlighted:"!pip install sacrebleu"}}),Dn=new C({props:{code:`from datasets import load_metric

metric = load_metric("sacrebleu")`,highlighted:`<span class="hljs-keyword">from</span> datasets <span class="hljs-keyword">import</span> load_metric

metric = load_metric(<span class="hljs-string">&quot;sacrebleu&quot;</span>)`}}),Pn=new C({props:{code:`predictions = [
    "This plugin lets you translate web pages between several languages automatically."
]
references = [
    [
        "This plugin allows you to automatically translate web pages between several languages."
    ]
]
metric.compute(predictions=predictions, references=references)`,highlighted:`predictions = [
    <span class="hljs-string">&quot;This plugin lets you translate web pages between several languages automatically.&quot;</span>
]
references = [
    [
        <span class="hljs-string">&quot;This plugin allows you to automatically translate web pages between several languages.&quot;</span>
    ]
]
metric.compute(predictions=predictions, references=references)`}}),An=new C({props:{code:`{'score': 46.750469682990165,
 'counts': [11, 6, 4, 3],
 'totals': [12, 11, 10, 9],
 'precisions': [91.67, 54.54, 40.0, 33.33],
 'bp': 0.9200444146293233,
 'sys_len': 12,
 'ref_len': 13}`,highlighted:`{<span class="hljs-string">&#x27;score&#x27;</span>: <span class="hljs-number">46.750469682990165</span>,
 <span class="hljs-string">&#x27;counts&#x27;</span>: [<span class="hljs-number">11</span>, <span class="hljs-number">6</span>, <span class="hljs-number">4</span>, <span class="hljs-number">3</span>],
 <span class="hljs-string">&#x27;totals&#x27;</span>: [<span class="hljs-number">12</span>, <span class="hljs-number">11</span>, <span class="hljs-number">10</span>, <span class="hljs-number">9</span>],
 <span class="hljs-string">&#x27;precisions&#x27;</span>: [<span class="hljs-number">91.67</span>, <span class="hljs-number">54.54</span>, <span class="hljs-number">40.0</span>, <span class="hljs-number">33.33</span>],
 <span class="hljs-string">&#x27;bp&#x27;</span>: <span class="hljs-number">0.9200444146293233</span>,
 <span class="hljs-string">&#x27;sys_len&#x27;</span>: <span class="hljs-number">12</span>,
 <span class="hljs-string">&#x27;ref_len&#x27;</span>: <span class="hljs-number">13</span>}`}}),On=new C({props:{code:`predictions = ["This This This This"]
references = [
    [
        "This plugin allows you to automatically translate web pages between several languages."
    ]
]
metric.compute(predictions=predictions, references=references)`,highlighted:`predictions = [<span class="hljs-string">&quot;This This This This&quot;</span>]
references = [
    [
        <span class="hljs-string">&quot;This plugin allows you to automatically translate web pages between several languages.&quot;</span>
    ]
]
metric.compute(predictions=predictions, references=references)`}}),Fn=new C({props:{code:`{'score': 1.683602693167689,
 'counts': [1, 0, 0, 0],
 'totals': [4, 3, 2, 1],
 'precisions': [25.0, 16.67, 12.5, 12.5],
 'bp': 0.10539922456186433,
 'sys_len': 4,
 'ref_len': 13}`,highlighted:`{<span class="hljs-string">&#x27;score&#x27;</span>: <span class="hljs-number">1.683602693167689</span>,
 <span class="hljs-string">&#x27;counts&#x27;</span>: [<span class="hljs-number">1</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>],
 <span class="hljs-string">&#x27;totals&#x27;</span>: [<span class="hljs-number">4</span>, <span class="hljs-number">3</span>, <span class="hljs-number">2</span>, <span class="hljs-number">1</span>],
 <span class="hljs-string">&#x27;precisions&#x27;</span>: [<span class="hljs-number">25.0</span>, <span class="hljs-number">16.67</span>, <span class="hljs-number">12.5</span>, <span class="hljs-number">12.5</span>],
 <span class="hljs-string">&#x27;bp&#x27;</span>: <span class="hljs-number">0.10539922456186433</span>,
 <span class="hljs-string">&#x27;sys_len&#x27;</span>: <span class="hljs-number">4</span>,
 <span class="hljs-string">&#x27;ref_len&#x27;</span>: <span class="hljs-number">13</span>}`}}),In=new C({props:{code:`predictions = ["This plugin"]
references = [
    [
        "This plugin allows you to automatically translate web pages between several languages."
    ]
]
metric.compute(predictions=predictions, references=references)`,highlighted:`predictions = [<span class="hljs-string">&quot;This plugin&quot;</span>]
references = [
    [
        <span class="hljs-string">&quot;This plugin allows you to automatically translate web pages between several languages.&quot;</span>
    ]
]
metric.compute(predictions=predictions, references=references)`}}),Ln=new C({props:{code:`{'score': 0.0,
 'counts': [2, 1, 0, 0],
 'totals': [2, 1, 0, 0],
 'precisions': [100.0, 100.0, 0.0, 0.0],
 'bp': 0.004086771438464067,
 'sys_len': 2,
 'ref_len': 13}`,highlighted:`{<span class="hljs-string">&#x27;score&#x27;</span>: <span class="hljs-number">0.0</span>,
 <span class="hljs-string">&#x27;counts&#x27;</span>: [<span class="hljs-number">2</span>, <span class="hljs-number">1</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>],
 <span class="hljs-string">&#x27;totals&#x27;</span>: [<span class="hljs-number">2</span>, <span class="hljs-number">1</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>],
 <span class="hljs-string">&#x27;precisions&#x27;</span>: [<span class="hljs-number">100.0</span>, <span class="hljs-number">100.0</span>, <span class="hljs-number">0.0</span>, <span class="hljs-number">0.0</span>],
 <span class="hljs-string">&#x27;bp&#x27;</span>: <span class="hljs-number">0.004086771438464067</span>,
 <span class="hljs-string">&#x27;sys_len&#x27;</span>: <span class="hljs-number">2</span>,
 <span class="hljs-string">&#x27;ref_len&#x27;</span>: <span class="hljs-number">13</span>}`}});const gh=[Od,Cd],Jn=[];function wh(e,p){return e[0]==="tf"?0:1}Rs=wh(B),Ks=Jn[Rs]=gh[Rs](B),Nn=new rs({}),Hn=new C({props:{code:`from huggingface_hub import notebook_login

notebook_login()`,highlighted:`<span class="hljs-keyword">from</span> huggingface_hub <span class="hljs-keyword">import</span> notebook_login

notebook_login()`}}),Wn=new C({props:{code:"huggingface-cli login",highlighted:"huggingface-cli login"}});const bh=[Id,Fd],Vn=[];function $h(e,p){return e[0]==="tf"?0:1}Ys=$h(B),Gs=Vn[Ys]=bh[Ys](B);let Ge=B[0]==="pt"&&ud();return Mn=new rs({}),Un=new C({props:{code:`from transformers import pipeline

# Replace this with your own checkpoint
model_checkpoint = "huggingface-course/marian-finetuned-kde4-en-to-fr"
translator = pipeline("translation", model=model_checkpoint)
translator("Default to expanded threads")`,highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> pipeline

<span class="hljs-comment"># Replace this with your own checkpoint</span>
model_checkpoint = <span class="hljs-string">&quot;huggingface-course/marian-finetuned-kde4-en-to-fr&quot;</span>
translator = pipeline(<span class="hljs-string">&quot;translation&quot;</span>, model=model_checkpoint)
translator(<span class="hljs-string">&quot;Default to expanded threads&quot;</span>)`}}),Bn=new C({props:{code:"[{'translation_text': 'Par d\xE9faut, d\xE9velopper les fils de discussion'}]",highlighted:'[{<span class="hljs-string">&#x27;translation_text&#x27;</span>: <span class="hljs-string">&#x27;Par d\xE9faut, d\xE9velopper les fils de discussion&#x27;</span>}]'}}),Rn=new C({props:{code:`translator(
    "Unable to import %1 using the OFX importer plugin. This file is not the correct format."
)`,highlighted:`translator(
    <span class="hljs-string">&quot;Unable to import %1 using the OFX importer plugin. This file is not the correct format.&quot;</span>
)`}}),Kn=new C({props:{code:`[{'translation_text': "Impossible d'importer %1 en utilisant le module externe d'importation OFX. Ce fichier n'est pas le bon format."}]`,highlighted:'[{<span class="hljs-string">&#x27;translation_text&#x27;</span>: <span class="hljs-string">&quot;Impossible d&#x27;importer %1 en utilisant le module externe d&#x27;importation OFX. Ce fichier n&#x27;est pas le bon format.&quot;</span>}]'}}),Ua=new Za({props:{$$slots:{default:[Hd]},$$scope:{ctx:B}}}),{c(){h=l("meta"),$=c(),y(m.$$.fragment),q=c(),A=l("h1"),g=l("a"),z=l("span"),y(P.$$.fragment),D=c(),T=l("span"),N=a("Translation"),v=c(),H.c(),U=c(),F=l("p"),X=a("Let\u2019s now dive into translation. This is another "),M=l("a"),I=a("sequence-to-sequence task"),Y=a(", which means it\u2019s a problem that can be formulated as going from one sequence to another. In that sense the problem is pretty close to "),Z=l("a"),re=a("summarization"),de=a(", and you could adapt what we will see here to other sequence-to-sequence problems such as:"),Q=c(),ee=l("ul"),te=l("li"),G=l("strong"),ne=a("Style transfer"),$e=a(": Creating a model that "),V=l("em"),se=a("translates"),fe=a(" texts written in a certain style to another (e.g., formal to casual or Shakespearean English to modern English)"),ke=c(),oe=l("li"),pe=l("strong"),W=a("Generative question answering"),ae=a(": Creating a model that generates answers to questions, given a context"),ze=c(),y(me.$$.fragment),K=c(),ye=l("p"),le=a("If you have a big enough corpus of texts in two (or more) languages, you can train a new translation model from scratch like we will in the section on "),xe=l("a"),Me=a("causal language modeling"),_e=a(". It will be faster, however, to fine-tune an existing translation model, be it a multilingual one like mT5 or mBART that you want to fine-tune to a specific language pair, or even a model specialized for translation from one language to another that you want to fine-tune to your specific corpus."),qe=c(),J=l("p"),Ae=a("In this section, we will fine-tune a Marian model pretrained to translate from English to French (since a lot of Hugging Face employees speak both those languages) on the "),ge=l("a"),at=a("KDE4 dataset"),ft=a(", which is a dataset of localized files for the "),ve=l("a"),Xe=a("KDE apps"),Ue=a(". The model we will use has been pretrained on a large corpus of French and English texts taken from the "),Se=l("a"),je=a("Opus dataset"),mt=a(", which actually contains the KDE4 dataset. But even if the pretrained model we use has seen that data during its pretraining, we will see that we can get a better version of it after fine-tuning."),we=c(),k=l("p"),R=a("Once we\u2019re finished, we will have a model able to make predictions like this one:"),Ze=c(),he=l("iframe"),nt=c(),Ce=l("a"),ie=l("img"),Re=c(),Le=l("img"),_t=c(),ce=l("p"),Je=a("As in the previous sections, you can find the actual model that we\u2019ll train and upload to the Hub using the code below and double-check its predictions "),Oe=l("a"),zt=a("here"),is=a("."),ot=c(),lt=l("h2"),rt=l("a"),Ve=l("span"),y(Dt.$$.fragment),ps=c(),Qe=l("span"),gt=a("Preparing the data"),wt=c(),Ee=l("p"),Gt=a("To fine-tune or train a translation model from scratch, we will need a dataset suitable for the task. As mentioned previously, we\u2019ll use the "),Fe=l("a"),ks=a("KDE4 dataset"),Xt=a(" in this section, but you can adapt the code to use your own data quite easily, as long as you have pairs of sentences in the two languages you want to translate from and into. Refer back to "),De=l("a"),Zt=a("Chapter 5"),Pe=a(" if you need a reminder of how to load your custom data in a "),It=l("code"),bt=a("Dataset"),vs=a("."),Jt=c(),be=l("h3"),$t=l("a"),Pt=l("span"),y(kt.$$.fragment),it=c(),He=l("span"),Vt=a("The KDE4 dataset"),vt=c(),pt=l("p"),ht=a("As usual, we download our dataset using the "),Lt=l("code"),At=a("load_dataset()"),et=a(" function:"),Qt=c(),y(Ie.$$.fragment),es=c(),Ke=l("p"),ts=a("If you want to work with a different pair of languages, you can specify them by their codes. A total of 92 languages are available for this dataset; you can see them all by expanding the language tags on its "),yt=l("a"),Nt=a("dataset card"),hs=a("."),ct=c(),f=l("img"),ys=c(),jt=l("p"),Xs=a("Let\u2019s have a look at the dataset:"),js=c(),y(Te.$$.fragment),Es=c(),y(Et.$$.fragment),cs=c(),ue=l("p"),Zs=a("We have 210,173 pairs of sentences, but in one single split, so we will need to create our own validation set. As we saw in "),dt=l("a"),Js=a("Chapter 5"),We=a(", a "),Ht=l("code"),xs=a("Dataset"),Vs=a(" has a "),ss=l("code"),Ye=a("train_test_split()"),Qs=a(" method that can help us. We\u2019ll provide a seed for reproducibility:"),xt=c(),y(Wt.$$.fragment),qs=c(),y(qt.$$.fragment),Ts=c(),ut=l("p"),ds=a("We can rename the "),us=l("code"),Mt=a('"test"'),zs=a(" key to "),Tt=l("code"),Ds=a('"validation"'),Ut=a(" like this:"),fs=c(),y(tt.$$.fragment),xa=c(),ms=l("p"),Ja=a("Now let\u2019s take a look at one element of the dataset:"),qa=c(),y(St.$$.fragment),ea=c(),y(Ps.$$.fragment),Ta=c(),As=l("p"),u=a("We get a dictionary with two sentences in the pair of languages we requested. One particularity of this dataset full of technical computer science terms is that they are all fully translated in French. However, French engineers are often lazy and leave most computer science-specific words in English when they talk. Here, for instance, the word \u201Cthreads\u201D might well appear in a French sentence, especially in a technical conversation; but in this dataset it has been translated into the more correct \u201Cfils de discussion.\u201D The pretrained model we use, which has been pretrained on a larger corpus of French and English sentences, takes the easier option of leaving the word as is:"),L=c(),y(ta.$$.fragment),Va=c(),y(sa.$$.fragment),Ss=c(),ga=l("p"),so=a(`Another example of this behavior can be seen with the word \u201Cplugin,\u201D which isn\u2019t officially a French word but which most native speakers will understand and not bother to translate.
In the KDE4 dataset this word has been translated in French into the more official \u201Cmodule d\u2019extension\u201D:`),wa=c(),y(aa.$$.fragment),Qa=c(),y(na.$$.fragment),Cs=c(),ba=l("p"),ao=a("Our pretrained model, however, sticks with the compact and familiar English word:"),en=c(),y(oa.$$.fragment),Os=c(),y(la.$$.fragment),tn=c(),Ct=l("p"),no=a("It will be interesting to see if our fine-tuned model picks up on those particularities of the dataset (spoiler alert: it will)."),sn=c(),y(ra.$$.fragment),as=c(),y(Fs.$$.fragment),an=c(),_s=l("h3"),Is=l("a"),gs=l("span"),y(ia.$$.fragment),oo=c(),pa=l("span"),lo=a("Processing the data"),nn=c(),y(ha.$$.fragment),Ls=c(),ns=l("p"),ro=a("You should know the drill by now: the texts all need to be converted into sets of token IDs so the model can make sense of them. For this task, we\u2019ll need to tokenize both the inputs and the targets. Our first task is to create our "),ws=l("code"),io=a("tokenizer"),po=a(" object. As noted earlier, we\u2019ll be using a Marian English to French pretrained model. If you are trying this code with another pair of languages, make sure to adapt the model checkpoint. The "),Bt=l("a"),ho=a("Helsinki-NLP"),co=a(" organization provides more than a thousand models in multiple languages."),os=c(),y(ca.$$.fragment),on=c(),ls=l("p"),da=a("You can also replace the "),za=l("code"),uo=a("model_checkpoint"),fo=a(" with any other model you prefer from the "),ln=l("a"),mi=a("Hub"),_i=a(", or a local folder where you\u2019ve saved a pretrained model and a tokenizer."),Sl=c(),y(Da.$$.fragment),Cl=c(),Pa=l("p"),gi=a("The preparation of our data is pretty straightforward. There\u2019s just one thing to remember: you process the inputs as usual, but for the targets, you need to wrap the tokenizer inside the context manager "),Qo=l("code"),wi=a("as_target_tokenizer()"),bi=a("."),Ol=c(),Aa=l("p"),$i=a("A context manager in Python is introduced with the "),el=l("code"),ki=a("with"),vi=a(" statement and is useful when you have two related operations to execute as a pair. The most common example of this is when you write or read a file, which is often done inside an instruction like:"),Fl=c(),y(rn.$$.fragment),Il=c(),ua=l("p"),yi=a("Here the two related operations that are executed as a pair are the actions of opening and closing the file. The object corresponding to the opened file "),tl=l("code"),ji=a("f"),Ei=a(" only exists inside the indented block under the "),sl=l("code"),xi=a("with"),qi=a("; the opening happens before that block and the closing at the end of the block."),Ll=c(),Sa=l("p"),Ti=a("In the case at hand, the context manager "),al=l("code"),zi=a("as_target_tokenizer()"),Di=a(" will set the tokenizer in the output language (here, French) before the indented block is executed, then set it back in the input language (here, English)."),Nl=c(),mo=l("p"),Pi=a("So, preprocessing one sample looks like this:"),Hl=c(),y(pn.$$.fragment),Wl=c(),_o=l("p"),Ai=a("If we forget to tokenize the targets inside the context manager, they will be tokenized by the input tokenizer, which in the case of a Marian model is not going to go well at all:"),Ml=c(),y(hn.$$.fragment),Ul=c(),y(cn.$$.fragment),Bl=c(),go=l("p"),Si=a("As we can see, using the English tokenizer to preprocess a French sentence results in a lot more tokens, since the tokenizer doesn\u2019t know any French words (except those that also appear in the English language, like \u201Cdiscussion\u201D)."),Rl=c(),bs=l("p"),Ci=a("Both "),nl=l("code"),Oi=a("inputs"),Fi=a(" and "),ol=l("code"),Ii=a("targets"),Li=a(" are dictionaries with our usual keys (input IDs, attention mask, etc.), so the last step is to set a "),ll=l("code"),Ni=a('"labels"'),Hi=a(" key inside the inputs. We do this in the preprocessing function we will apply on the datasets:"),Kl=c(),y(dn.$$.fragment),Yl=c(),wo=l("p"),Wi=a("Note that we set similar maximum lengths for our inputs and outputs. Since the texts we\u2019re dealing with seem pretty short, we use 128."),Gl=c(),y(Ca.$$.fragment),Xl=c(),y(Oa.$$.fragment),Zl=c(),bo=l("p"),Mi=a("We can now apply that preprocessing in one go on all the splits of our dataset:"),Jl=c(),y(un.$$.fragment),Vl=c(),$o=l("p"),Ui=a("Now that the data has been preprocessed, we are ready to fine-tune our pretrained model!"),Ql=c(),Hs.c(),ko=c(),vo=l("p"),Bi=a("Note that this time we are using a model that was trained on a translation task and can actually be used already, so there is no warning about missing weights or newly initialized ones."),er=c(),$a=l("h3"),Fa=l("a"),rl=l("span"),y(fn.$$.fragment),Ri=c(),il=l("span"),Ki=a("Data collation"),tr=c(),$s=l("p"),Yi=a("We\u2019ll need a data collator to deal with the padding for dynamic batching. We can\u2019t just use a "),pl=l("code"),Gi=a("DataCollatorWithPadding"),Xi=a(" like in "),yo=l("a"),Zi=a("Chapter 3"),Ji=a(" in this case, because that only pads the inputs (input IDs, attention mask, and token type IDs). Our labels should also be padded to the maximum length encountered in the labels. And, as mentioned previously, the padding value used to pad the labels should be "),hl=l("code"),Vi=a("-100"),Qi=a(" and not the padding token of the tokenizer, to make sure those padded values are ignored in the loss computation."),sr=c(),st=l("p"),ep=a("This is all done by a "),mn=l("a"),cl=l("code"),tp=a("DataCollatorForSeq2Seq"),sp=a(". Like the "),dl=l("code"),ap=a("DataCollatorWithPadding"),np=a(", it takes the "),ul=l("code"),op=a("tokenizer"),lp=a(" used to preprocess the inputs, but it also takes the "),fl=l("code"),rp=a("model"),ip=a(". This is because this data collator will also be responsible for preparing the decoder input IDs, which are shifted versions of the labels with a special token at the beginning. Since this shift is done slightly differently for different architectures, the "),ml=l("code"),pp=a("DataCollatorForSeq2Seq"),hp=a(" needs to know the "),_l=l("code"),cp=a("model"),dp=a(" object:"),ar=c(),Ms.c(),jo=c(),Eo=l("p"),up=a("To test this on a few samples, we just call it on a list of examples from our tokenized training set:"),nr=c(),y(_n.$$.fragment),or=c(),y(gn.$$.fragment),lr=c(),Ia=l("p"),fp=a("We can check our labels have been padded to the maximum length of the batch, using "),gl=l("code"),mp=a("-100"),_p=a(":"),rr=c(),y(wn.$$.fragment),ir=c(),y(bn.$$.fragment),pr=c(),xo=l("p"),gp=a("And we can also have a look at the decoder input IDs, to see that they are shifted versions of the labels:"),hr=c(),y($n.$$.fragment),cr=c(),y(kn.$$.fragment),dr=c(),qo=l("p"),wp=a("Here are the labels for the first and second elements in our dataset:"),ur=c(),y(vn.$$.fragment),fr=c(),y(yn.$$.fragment),mr=c(),Bs.c(),To=c(),ka=l("h3"),La=l("a"),wl=l("span"),y(jn.$$.fragment),bp=c(),bl=l("span"),$p=a("Metrics"),_r=c(),y(En.$$.fragment),gr=c(),Ot&&Ot.c(),zo=c(),Rt=l("p"),kp=a("The traditional metric used for translation is the "),xn=l("a"),vp=a("BLEU score"),yp=a(", introduced in "),qn=l("a"),jp=a("a 2002 article"),Ep=a(" by Kishore Papineni et al. The BLEU score evaluates how close the translations are to their labels. It does not measure the intelligibility or grammatical correctness of the model\u2019s generated outputs, but uses statistical rules to ensure that all the words in the generated outputs also appear in the targets. In addition, there are rules that penalize repetitions of the same words if they are not also repeated in the targets (to avoid the model outputting sentences like "),$l=l("code"),xp=a('"the the the the the"'),qp=a(") and output sentences that are shorter than those in the targets (to avoid the model outputting sentences like "),kl=l("code"),Tp=a('"the"'),zp=a(")."),wr=c(),Na=l("p"),Dp=a("One weakness with BLEU is that it expects the text to already be tokenized, which makes it difficult to compare scores between models that use different tokenizers. So instead, the most commonly used metric for benchmarking translation models today is "),Tn=l("a"),Pp=a("SacreBLEU"),Ap=a(", which addresses this weakness (and others) by standardizing the tokenization step. To use this metric, we first need to install the SacreBLEU library:"),br=c(),y(zn.$$.fragment),$r=c(),fa=l("p"),Sp=a("We can then load it via "),vl=l("code"),Cp=a("load_metric()"),Op=a(" like we did in "),Do=l("a"),Fp=a("Chapter 3"),Ip=a(":"),kr=c(),y(Dn.$$.fragment),vr=c(),Po=l("p"),Lp=a("This metric will take texts as inputs and targets. It is designed to accept several acceptable targets, as there are often multiple acceptable translations of the same sentence \u2014 the dataset we\u2019re using only provides one, but it\u2019s not uncommon in NLP to find datasets that give several sentences as labels. So, the predictions should be a list of sentences, but the references should be a list of lists of sentences."),yr=c(),Ao=l("p"),Np=a("Let\u2019s try an example:"),jr=c(),y(Pn.$$.fragment),Er=c(),y(An.$$.fragment),xr=c(),Kt=l("p"),Hp=a("This gets a BLEU score of 46.75, which is rather good \u2014 for reference, the original Transformer model in the "),Sn=l("a"),Wp=a("\u201CAttention Is All You Need\u201D paper"),Mp=a(" achieved a BLEU score of 41.8 on a similar translation task between English and French! (For more information about the individual metrics, like "),yl=l("code"),Up=a("counts"),Bp=a(" and "),jl=l("code"),Rp=a("bp"),Kp=a(", see the "),Cn=l("a"),Yp=a("SacreBLEU repository"),Gp=a(".) On the other hand, if we try with the two bad types of predictions (lots of repetitions or too short) that often come out of translation models, we will get rather bad BLEU scores:"),qr=c(),y(On.$$.fragment),Tr=c(),y(Fn.$$.fragment),zr=c(),y(In.$$.fragment),Dr=c(),y(Ln.$$.fragment),Pr=c(),So=l("p"),Xp=a("The score can go from 0 to 100, and higher is better."),Ar=c(),Ks.c(),Co=c(),Oo=l("p"),Zp=a("Now that this is done, we are ready to fine-tune our model!"),Sr=c(),va=l("h3"),Ha=l("a"),El=l("span"),y(Nn.$$.fragment),Jp=c(),xl=l("span"),Vp=a("Fine-tuning the model"),Cr=c(),Fo=l("p"),Qp=a("The first step is to log in to Hugging Face, so you\u2019re able to upload your results to the Model Hub. There\u2019s a convenience function to help you with this in a notebook:"),Or=c(),y(Hn.$$.fragment),Fr=c(),Io=l("p"),eh=a("This will display a widget where you can enter your Hugging Face login credentials."),Ir=c(),Lo=l("p"),th=a("If you aren\u2019t working in a notebook, just type the following line in your terminal:"),Lr=c(),y(Wn.$$.fragment),Nr=c(),Gs.c(),No=c(),Ge&&Ge.c(),Ho=c(),ya=l("h2"),Wa=l("a"),ql=l("span"),y(Mn.$$.fragment),sh=c(),Tl=l("span"),ah=a("Using the fine-tuned model"),Hr=c(),Ma=l("p"),nh=a("We\u2019ve already shown you how you can use the model we fine-tuned on the Model Hub with the inference widget. To use it locally in a "),zl=l("code"),oh=a("pipeline"),lh=a(", we just have to specify the proper model identifier:"),Wr=c(),y(Un.$$.fragment),Mr=c(),y(Bn.$$.fragment),Ur=c(),Wo=l("p"),rh=a("As expected, our pretrained model adapted its knowledge to the corpus we fine-tuned it on, and instead of leaving the English word \u201Cthreads\u201D alone, it now translates it to the French official version. It\u2019s the same for \u201Cplugin\u201D:"),Br=c(),y(Rn.$$.fragment),Rr=c(),y(Kn.$$.fragment),Kr=c(),Mo=l("p"),ih=a("Another great example of domain adaptation!"),Yr=c(),y(Ua.$$.fragment),this.h()},l(e){const p=wd('[data-svelte="svelte-1phssyn"]',document.head);h=r(p,"META",{name:!0,content:!0}),p.forEach(t),$=d(e),j(m.$$.fragment,e),q=d(e),A=r(e,"H1",{class:!0});var Qn=i(A);g=r(Qn,"A",{id:!0,class:!0,href:!0});var Uo=i(g);z=r(Uo,"SPAN",{});var Dl=i(z);j(P.$$.fragment,Dl),Dl.forEach(t),Uo.forEach(t),D=d(Qn),T=r(Qn,"SPAN",{});var Pl=i(T);N=n(Pl,"Translation"),Pl.forEach(t),Qn.forEach(t),v=d(e),H.l(e),U=d(e),F=r(e,"P",{});var ja=i(F);X=n(ja,"Let\u2019s now dive into translation. This is another "),M=r(ja,"A",{href:!0});var Al=i(M);I=n(Al,"sequence-to-sequence task"),Al.forEach(t),Y=n(ja,", which means it\u2019s a problem that can be formulated as going from one sequence to another. In that sense the problem is pretty close to "),Z=r(ja,"A",{href:!0});var Bo=i(Z);re=n(Bo,"summarization"),Bo.forEach(t),de=n(ja,", and you could adapt what we will see here to other sequence-to-sequence problems such as:"),ja.forEach(t),Q=d(e),ee=r(e,"UL",{});var Ba=i(ee);te=r(Ba,"LI",{});var Ea=i(te);G=r(Ea,"STRONG",{});var Ro=i(G);ne=n(Ro,"Style transfer"),Ro.forEach(t),$e=n(Ea,": Creating a model that "),V=r(Ea,"EM",{});var Ko=i(V);se=n(Ko,"translates"),Ko.forEach(t),fe=n(Ea," texts written in a certain style to another (e.g., formal to casual or Shakespearean English to modern English)"),Ea.forEach(t),ke=d(Ba),oe=r(Ba,"LI",{});var Yo=i(oe);pe=r(Yo,"STRONG",{});var kh=i(pe);W=n(kh,"Generative question answering"),kh.forEach(t),ae=n(Yo,": Creating a model that generates answers to questions, given a context"),Yo.forEach(t),Ba.forEach(t),ze=d(e),j(me.$$.fragment,e),K=d(e),ye=r(e,"P",{});var Xr=i(ye);le=n(Xr,"If you have a big enough corpus of texts in two (or more) languages, you can train a new translation model from scratch like we will in the section on "),xe=r(Xr,"A",{href:!0});var vh=i(xe);Me=n(vh,"causal language modeling"),vh.forEach(t),_e=n(Xr,". It will be faster, however, to fine-tune an existing translation model, be it a multilingual one like mT5 or mBART that you want to fine-tune to a specific language pair, or even a model specialized for translation from one language to another that you want to fine-tune to your specific corpus."),Xr.forEach(t),qe=d(e),J=r(e,"P",{});var Ra=i(J);Ae=n(Ra,"In this section, we will fine-tune a Marian model pretrained to translate from English to French (since a lot of Hugging Face employees speak both those languages) on the "),ge=r(Ra,"A",{href:!0,rel:!0});var yh=i(ge);at=n(yh,"KDE4 dataset"),yh.forEach(t),ft=n(Ra,", which is a dataset of localized files for the "),ve=r(Ra,"A",{href:!0,rel:!0});var jh=i(ve);Xe=n(jh,"KDE apps"),jh.forEach(t),Ue=n(Ra,". The model we will use has been pretrained on a large corpus of French and English texts taken from the "),Se=r(Ra,"A",{href:!0,rel:!0});var Eh=i(Se);je=n(Eh,"Opus dataset"),Eh.forEach(t),mt=n(Ra,", which actually contains the KDE4 dataset. But even if the pretrained model we use has seen that data during its pretraining, we will see that we can get a better version of it after fine-tuning."),Ra.forEach(t),we=d(e),k=r(e,"P",{});var xh=i(k);R=n(xh,"Once we\u2019re finished, we will have a model able to make predictions like this one:"),xh.forEach(t),Ze=d(e),he=r(e,"IFRAME",{src:!0,frameborder:!0,height:!0,title:!0,class:!0,allow:!0,sandbox:!0}),i(he).forEach(t),nt=d(e),Ce=r(e,"A",{class:!0,href:!0});var Zr=i(Ce);ie=r(Zr,"IMG",{class:!0,src:!0,alt:!0}),Re=d(Zr),Le=r(Zr,"IMG",{class:!0,src:!0,alt:!0}),Zr.forEach(t),_t=d(e),ce=r(e,"P",{});var Jr=i(ce);Je=n(Jr,"As in the previous sections, you can find the actual model that we\u2019ll train and upload to the Hub using the code below and double-check its predictions "),Oe=r(Jr,"A",{href:!0,rel:!0});var qh=i(Oe);zt=n(qh,"here"),qh.forEach(t),is=n(Jr,"."),Jr.forEach(t),ot=d(e),lt=r(e,"H2",{class:!0});var Vr=i(lt);rt=r(Vr,"A",{id:!0,class:!0,href:!0});var Th=i(rt);Ve=r(Th,"SPAN",{});var zh=i(Ve);j(Dt.$$.fragment,zh),zh.forEach(t),Th.forEach(t),ps=d(Vr),Qe=r(Vr,"SPAN",{});var Dh=i(Qe);gt=n(Dh,"Preparing the data"),Dh.forEach(t),Vr.forEach(t),wt=d(e),Ee=r(e,"P",{});var Ka=i(Ee);Gt=n(Ka,"To fine-tune or train a translation model from scratch, we will need a dataset suitable for the task. As mentioned previously, we\u2019ll use the "),Fe=r(Ka,"A",{href:!0,rel:!0});var Ph=i(Fe);ks=n(Ph,"KDE4 dataset"),Ph.forEach(t),Xt=n(Ka," in this section, but you can adapt the code to use your own data quite easily, as long as you have pairs of sentences in the two languages you want to translate from and into. Refer back to "),De=r(Ka,"A",{href:!0});var Ah=i(De);Zt=n(Ah,"Chapter 5"),Ah.forEach(t),Pe=n(Ka," if you need a reminder of how to load your custom data in a "),It=r(Ka,"CODE",{});var Sh=i(It);bt=n(Sh,"Dataset"),Sh.forEach(t),vs=n(Ka,"."),Ka.forEach(t),Jt=d(e),be=r(e,"H3",{class:!0});var Qr=i(be);$t=r(Qr,"A",{id:!0,class:!0,href:!0});var Ch=i($t);Pt=r(Ch,"SPAN",{});var Oh=i(Pt);j(kt.$$.fragment,Oh),Oh.forEach(t),Ch.forEach(t),it=d(Qr),He=r(Qr,"SPAN",{});var Fh=i(He);Vt=n(Fh,"The KDE4 dataset"),Fh.forEach(t),Qr.forEach(t),vt=d(e),pt=r(e,"P",{});var ei=i(pt);ht=n(ei,"As usual, we download our dataset using the "),Lt=r(ei,"CODE",{});var Ih=i(Lt);At=n(Ih,"load_dataset()"),Ih.forEach(t),et=n(ei," function:"),ei.forEach(t),Qt=d(e),j(Ie.$$.fragment,e),es=d(e),Ke=r(e,"P",{});var ti=i(Ke);ts=n(ti,"If you want to work with a different pair of languages, you can specify them by their codes. A total of 92 languages are available for this dataset; you can see them all by expanding the language tags on its "),yt=r(ti,"A",{href:!0,rel:!0});var Lh=i(yt);Nt=n(Lh,"dataset card"),Lh.forEach(t),hs=n(ti,"."),ti.forEach(t),ct=d(e),f=r(e,"IMG",{src:!0,alt:!0,width:!0}),ys=d(e),jt=r(e,"P",{});var Nh=i(jt);Xs=n(Nh,"Let\u2019s have a look at the dataset:"),Nh.forEach(t),js=d(e),j(Te.$$.fragment,e),Es=d(e),j(Et.$$.fragment,e),cs=d(e),ue=r(e,"P",{});var Ya=i(ue);Zs=n(Ya,"We have 210,173 pairs of sentences, but in one single split, so we will need to create our own validation set. As we saw in "),dt=r(Ya,"A",{href:!0});var Hh=i(dt);Js=n(Hh,"Chapter 5"),Hh.forEach(t),We=n(Ya,", a "),Ht=r(Ya,"CODE",{});var Wh=i(Ht);xs=n(Wh,"Dataset"),Wh.forEach(t),Vs=n(Ya," has a "),ss=r(Ya,"CODE",{});var Mh=i(ss);Ye=n(Mh,"train_test_split()"),Mh.forEach(t),Qs=n(Ya," method that can help us. We\u2019ll provide a seed for reproducibility:"),Ya.forEach(t),xt=d(e),j(Wt.$$.fragment,e),qs=d(e),j(qt.$$.fragment,e),Ts=d(e),ut=r(e,"P",{});var Go=i(ut);ds=n(Go,"We can rename the "),us=r(Go,"CODE",{});var Uh=i(us);Mt=n(Uh,'"test"'),Uh.forEach(t),zs=n(Go," key to "),Tt=r(Go,"CODE",{});var Bh=i(Tt);Ds=n(Bh,'"validation"'),Bh.forEach(t),Ut=n(Go," like this:"),Go.forEach(t),fs=d(e),j(tt.$$.fragment,e),xa=d(e),ms=r(e,"P",{});var Rh=i(ms);Ja=n(Rh,"Now let\u2019s take a look at one element of the dataset:"),Rh.forEach(t),qa=d(e),j(St.$$.fragment,e),ea=d(e),j(Ps.$$.fragment,e),Ta=d(e),As=r(e,"P",{});var Kh=i(As);u=n(Kh,"We get a dictionary with two sentences in the pair of languages we requested. One particularity of this dataset full of technical computer science terms is that they are all fully translated in French. However, French engineers are often lazy and leave most computer science-specific words in English when they talk. Here, for instance, the word \u201Cthreads\u201D might well appear in a French sentence, especially in a technical conversation; but in this dataset it has been translated into the more correct \u201Cfils de discussion.\u201D The pretrained model we use, which has been pretrained on a larger corpus of French and English sentences, takes the easier option of leaving the word as is:"),Kh.forEach(t),L=d(e),j(ta.$$.fragment,e),Va=d(e),j(sa.$$.fragment,e),Ss=d(e),ga=r(e,"P",{});var Yh=i(ga);so=n(Yh,`Another example of this behavior can be seen with the word \u201Cplugin,\u201D which isn\u2019t officially a French word but which most native speakers will understand and not bother to translate.
In the KDE4 dataset this word has been translated in French into the more official \u201Cmodule d\u2019extension\u201D:`),Yh.forEach(t),wa=d(e),j(aa.$$.fragment,e),Qa=d(e),j(na.$$.fragment,e),Cs=d(e),ba=r(e,"P",{});var Gh=i(ba);ao=n(Gh,"Our pretrained model, however, sticks with the compact and familiar English word:"),Gh.forEach(t),en=d(e),j(oa.$$.fragment,e),Os=d(e),j(la.$$.fragment,e),tn=d(e),Ct=r(e,"P",{});var Xh=i(Ct);no=n(Xh,"It will be interesting to see if our fine-tuned model picks up on those particularities of the dataset (spoiler alert: it will)."),Xh.forEach(t),sn=d(e),j(ra.$$.fragment,e),as=d(e),j(Fs.$$.fragment,e),an=d(e),_s=r(e,"H3",{class:!0});var si=i(_s);Is=r(si,"A",{id:!0,class:!0,href:!0});var Zh=i(Is);gs=r(Zh,"SPAN",{});var Jh=i(gs);j(ia.$$.fragment,Jh),Jh.forEach(t),Zh.forEach(t),oo=d(si),pa=r(si,"SPAN",{});var Vh=i(pa);lo=n(Vh,"Processing the data"),Vh.forEach(t),si.forEach(t),nn=d(e),j(ha.$$.fragment,e),Ls=d(e),ns=r(e,"P",{});var Xo=i(ns);ro=n(Xo,"You should know the drill by now: the texts all need to be converted into sets of token IDs so the model can make sense of them. For this task, we\u2019ll need to tokenize both the inputs and the targets. Our first task is to create our "),ws=r(Xo,"CODE",{});var Qh=i(ws);io=n(Qh,"tokenizer"),Qh.forEach(t),po=n(Xo," object. As noted earlier, we\u2019ll be using a Marian English to French pretrained model. If you are trying this code with another pair of languages, make sure to adapt the model checkpoint. The "),Bt=r(Xo,"A",{href:!0,rel:!0});var ec=i(Bt);ho=n(ec,"Helsinki-NLP"),ec.forEach(t),co=n(Xo," organization provides more than a thousand models in multiple languages."),Xo.forEach(t),os=d(e),j(ca.$$.fragment,e),on=d(e),ls=r(e,"P",{});var Zo=i(ls);da=n(Zo,"You can also replace the "),za=r(Zo,"CODE",{});var tc=i(za);uo=n(tc,"model_checkpoint"),tc.forEach(t),fo=n(Zo," with any other model you prefer from the "),ln=r(Zo,"A",{href:!0,rel:!0});var sc=i(ln);mi=n(sc,"Hub"),sc.forEach(t),_i=n(Zo,", or a local folder where you\u2019ve saved a pretrained model and a tokenizer."),Zo.forEach(t),Sl=d(e),j(Da.$$.fragment,e),Cl=d(e),Pa=r(e,"P",{});var ai=i(Pa);gi=n(ai,"The preparation of our data is pretty straightforward. There\u2019s just one thing to remember: you process the inputs as usual, but for the targets, you need to wrap the tokenizer inside the context manager "),Qo=r(ai,"CODE",{});var ac=i(Qo);wi=n(ac,"as_target_tokenizer()"),ac.forEach(t),bi=n(ai,"."),ai.forEach(t),Ol=d(e),Aa=r(e,"P",{});var ni=i(Aa);$i=n(ni,"A context manager in Python is introduced with the "),el=r(ni,"CODE",{});var nc=i(el);ki=n(nc,"with"),nc.forEach(t),vi=n(ni," statement and is useful when you have two related operations to execute as a pair. The most common example of this is when you write or read a file, which is often done inside an instruction like:"),ni.forEach(t),Fl=d(e),j(rn.$$.fragment,e),Il=d(e),ua=r(e,"P",{});var Jo=i(ua);yi=n(Jo,"Here the two related operations that are executed as a pair are the actions of opening and closing the file. The object corresponding to the opened file "),tl=r(Jo,"CODE",{});var oc=i(tl);ji=n(oc,"f"),oc.forEach(t),Ei=n(Jo," only exists inside the indented block under the "),sl=r(Jo,"CODE",{});var lc=i(sl);xi=n(lc,"with"),lc.forEach(t),qi=n(Jo,"; the opening happens before that block and the closing at the end of the block."),Jo.forEach(t),Ll=d(e),Sa=r(e,"P",{});var oi=i(Sa);Ti=n(oi,"In the case at hand, the context manager "),al=r(oi,"CODE",{});var rc=i(al);zi=n(rc,"as_target_tokenizer()"),rc.forEach(t),Di=n(oi," will set the tokenizer in the output language (here, French) before the indented block is executed, then set it back in the input language (here, English)."),oi.forEach(t),Nl=d(e),mo=r(e,"P",{});var ic=i(mo);Pi=n(ic,"So, preprocessing one sample looks like this:"),ic.forEach(t),Hl=d(e),j(pn.$$.fragment,e),Wl=d(e),_o=r(e,"P",{});var pc=i(_o);Ai=n(pc,"If we forget to tokenize the targets inside the context manager, they will be tokenized by the input tokenizer, which in the case of a Marian model is not going to go well at all:"),pc.forEach(t),Ml=d(e),j(hn.$$.fragment,e),Ul=d(e),j(cn.$$.fragment,e),Bl=d(e),go=r(e,"P",{});var hc=i(go);Si=n(hc,"As we can see, using the English tokenizer to preprocess a French sentence results in a lot more tokens, since the tokenizer doesn\u2019t know any French words (except those that also appear in the English language, like \u201Cdiscussion\u201D)."),hc.forEach(t),Rl=d(e),bs=r(e,"P",{});var Ga=i(bs);Ci=n(Ga,"Both "),nl=r(Ga,"CODE",{});var cc=i(nl);Oi=n(cc,"inputs"),cc.forEach(t),Fi=n(Ga," and "),ol=r(Ga,"CODE",{});var dc=i(ol);Ii=n(dc,"targets"),dc.forEach(t),Li=n(Ga," are dictionaries with our usual keys (input IDs, attention mask, etc.), so the last step is to set a "),ll=r(Ga,"CODE",{});var uc=i(ll);Ni=n(uc,'"labels"'),uc.forEach(t),Hi=n(Ga," key inside the inputs. We do this in the preprocessing function we will apply on the datasets:"),Ga.forEach(t),Kl=d(e),j(dn.$$.fragment,e),Yl=d(e),wo=r(e,"P",{});var fc=i(wo);Wi=n(fc,"Note that we set similar maximum lengths for our inputs and outputs. Since the texts we\u2019re dealing with seem pretty short, we use 128."),fc.forEach(t),Gl=d(e),j(Ca.$$.fragment,e),Xl=d(e),j(Oa.$$.fragment,e),Zl=d(e),bo=r(e,"P",{});var mc=i(bo);Mi=n(mc,"We can now apply that preprocessing in one go on all the splits of our dataset:"),mc.forEach(t),Jl=d(e),j(un.$$.fragment,e),Vl=d(e),$o=r(e,"P",{});var _c=i($o);Ui=n(_c,"Now that the data has been preprocessed, we are ready to fine-tune our pretrained model!"),_c.forEach(t),Ql=d(e),Hs.l(e),ko=d(e),vo=r(e,"P",{});var gc=i(vo);Bi=n(gc,"Note that this time we are using a model that was trained on a translation task and can actually be used already, so there is no warning about missing weights or newly initialized ones."),gc.forEach(t),er=d(e),$a=r(e,"H3",{class:!0});var li=i($a);Fa=r(li,"A",{id:!0,class:!0,href:!0});var wc=i(Fa);rl=r(wc,"SPAN",{});var bc=i(rl);j(fn.$$.fragment,bc),bc.forEach(t),wc.forEach(t),Ri=d(li),il=r(li,"SPAN",{});var $c=i(il);Ki=n($c,"Data collation"),$c.forEach(t),li.forEach(t),tr=d(e),$s=r(e,"P",{});var Xa=i($s);Yi=n(Xa,"We\u2019ll need a data collator to deal with the padding for dynamic batching. We can\u2019t just use a "),pl=r(Xa,"CODE",{});var kc=i(pl);Gi=n(kc,"DataCollatorWithPadding"),kc.forEach(t),Xi=n(Xa," like in "),yo=r(Xa,"A",{href:!0});var vc=i(yo);Zi=n(vc,"Chapter 3"),vc.forEach(t),Ji=n(Xa," in this case, because that only pads the inputs (input IDs, attention mask, and token type IDs). Our labels should also be padded to the maximum length encountered in the labels. And, as mentioned previously, the padding value used to pad the labels should be "),hl=r(Xa,"CODE",{});var yc=i(hl);Vi=n(yc,"-100"),yc.forEach(t),Qi=n(Xa," and not the padding token of the tokenizer, to make sure those padded values are ignored in the loss computation."),Xa.forEach(t),sr=d(e),st=r(e,"P",{});var Yt=i(st);ep=n(Yt,"This is all done by a "),mn=r(Yt,"A",{href:!0,rel:!0});var jc=i(mn);cl=r(jc,"CODE",{});var Ec=i(cl);tp=n(Ec,"DataCollatorForSeq2Seq"),Ec.forEach(t),jc.forEach(t),sp=n(Yt,". Like the "),dl=r(Yt,"CODE",{});var xc=i(dl);ap=n(xc,"DataCollatorWithPadding"),xc.forEach(t),np=n(Yt,", it takes the "),ul=r(Yt,"CODE",{});var qc=i(ul);op=n(qc,"tokenizer"),qc.forEach(t),lp=n(Yt," used to preprocess the inputs, but it also takes the "),fl=r(Yt,"CODE",{});var Tc=i(fl);rp=n(Tc,"model"),Tc.forEach(t),ip=n(Yt,". This is because this data collator will also be responsible for preparing the decoder input IDs, which are shifted versions of the labels with a special token at the beginning. Since this shift is done slightly differently for different architectures, the "),ml=r(Yt,"CODE",{});var zc=i(ml);pp=n(zc,"DataCollatorForSeq2Seq"),zc.forEach(t),hp=n(Yt," needs to know the "),_l=r(Yt,"CODE",{});var Dc=i(_l);cp=n(Dc,"model"),Dc.forEach(t),dp=n(Yt," object:"),Yt.forEach(t),ar=d(e),Ms.l(e),jo=d(e),Eo=r(e,"P",{});var Pc=i(Eo);up=n(Pc,"To test this on a few samples, we just call it on a list of examples from our tokenized training set:"),Pc.forEach(t),nr=d(e),j(_n.$$.fragment,e),or=d(e),j(gn.$$.fragment,e),lr=d(e),Ia=r(e,"P",{});var ri=i(Ia);fp=n(ri,"We can check our labels have been padded to the maximum length of the batch, using "),gl=r(ri,"CODE",{});var Ac=i(gl);mp=n(Ac,"-100"),Ac.forEach(t),_p=n(ri,":"),ri.forEach(t),rr=d(e),j(wn.$$.fragment,e),ir=d(e),j(bn.$$.fragment,e),pr=d(e),xo=r(e,"P",{});var Sc=i(xo);gp=n(Sc,"And we can also have a look at the decoder input IDs, to see that they are shifted versions of the labels:"),Sc.forEach(t),hr=d(e),j($n.$$.fragment,e),cr=d(e),j(kn.$$.fragment,e),dr=d(e),qo=r(e,"P",{});var Cc=i(qo);wp=n(Cc,"Here are the labels for the first and second elements in our dataset:"),Cc.forEach(t),ur=d(e),j(vn.$$.fragment,e),fr=d(e),j(yn.$$.fragment,e),mr=d(e),Bs.l(e),To=d(e),ka=r(e,"H3",{class:!0});var ii=i(ka);La=r(ii,"A",{id:!0,class:!0,href:!0});var Oc=i(La);wl=r(Oc,"SPAN",{});var Fc=i(wl);j(jn.$$.fragment,Fc),Fc.forEach(t),Oc.forEach(t),bp=d(ii),bl=r(ii,"SPAN",{});var Ic=i(bl);$p=n(Ic,"Metrics"),Ic.forEach(t),ii.forEach(t),_r=d(e),j(En.$$.fragment,e),gr=d(e),Ot&&Ot.l(e),zo=d(e),Rt=r(e,"P",{});var ma=i(Rt);kp=n(ma,"The traditional metric used for translation is the "),xn=r(ma,"A",{href:!0,rel:!0});var Lc=i(xn);vp=n(Lc,"BLEU score"),Lc.forEach(t),yp=n(ma,", introduced in "),qn=r(ma,"A",{href:!0,rel:!0});var Nc=i(qn);jp=n(Nc,"a 2002 article"),Nc.forEach(t),Ep=n(ma," by Kishore Papineni et al. The BLEU score evaluates how close the translations are to their labels. It does not measure the intelligibility or grammatical correctness of the model\u2019s generated outputs, but uses statistical rules to ensure that all the words in the generated outputs also appear in the targets. In addition, there are rules that penalize repetitions of the same words if they are not also repeated in the targets (to avoid the model outputting sentences like "),$l=r(ma,"CODE",{});var Hc=i($l);xp=n(Hc,'"the the the the the"'),Hc.forEach(t),qp=n(ma,") and output sentences that are shorter than those in the targets (to avoid the model outputting sentences like "),kl=r(ma,"CODE",{});var Wc=i(kl);Tp=n(Wc,'"the"'),Wc.forEach(t),zp=n(ma,")."),ma.forEach(t),wr=d(e),Na=r(e,"P",{});var pi=i(Na);Dp=n(pi,"One weakness with BLEU is that it expects the text to already be tokenized, which makes it difficult to compare scores between models that use different tokenizers. So instead, the most commonly used metric for benchmarking translation models today is "),Tn=r(pi,"A",{href:!0,rel:!0});var Mc=i(Tn);Pp=n(Mc,"SacreBLEU"),Mc.forEach(t),Ap=n(pi,", which addresses this weakness (and others) by standardizing the tokenization step. To use this metric, we first need to install the SacreBLEU library:"),pi.forEach(t),br=d(e),j(zn.$$.fragment,e),$r=d(e),fa=r(e,"P",{});var Vo=i(fa);Sp=n(Vo,"We can then load it via "),vl=r(Vo,"CODE",{});var Uc=i(vl);Cp=n(Uc,"load_metric()"),Uc.forEach(t),Op=n(Vo," like we did in "),Do=r(Vo,"A",{href:!0});var Bc=i(Do);Fp=n(Bc,"Chapter 3"),Bc.forEach(t),Ip=n(Vo,":"),Vo.forEach(t),kr=d(e),j(Dn.$$.fragment,e),vr=d(e),Po=r(e,"P",{});var Rc=i(Po);Lp=n(Rc,"This metric will take texts as inputs and targets. It is designed to accept several acceptable targets, as there are often multiple acceptable translations of the same sentence \u2014 the dataset we\u2019re using only provides one, but it\u2019s not uncommon in NLP to find datasets that give several sentences as labels. So, the predictions should be a list of sentences, but the references should be a list of lists of sentences."),Rc.forEach(t),yr=d(e),Ao=r(e,"P",{});var Kc=i(Ao);Np=n(Kc,"Let\u2019s try an example:"),Kc.forEach(t),jr=d(e),j(Pn.$$.fragment,e),Er=d(e),j(An.$$.fragment,e),xr=d(e),Kt=r(e,"P",{});var _a=i(Kt);Hp=n(_a,"This gets a BLEU score of 46.75, which is rather good \u2014 for reference, the original Transformer model in the "),Sn=r(_a,"A",{href:!0,rel:!0});var Yc=i(Sn);Wp=n(Yc,"\u201CAttention Is All You Need\u201D paper"),Yc.forEach(t),Mp=n(_a," achieved a BLEU score of 41.8 on a similar translation task between English and French! (For more information about the individual metrics, like "),yl=r(_a,"CODE",{});var Gc=i(yl);Up=n(Gc,"counts"),Gc.forEach(t),Bp=n(_a," and "),jl=r(_a,"CODE",{});var Xc=i(jl);Rp=n(Xc,"bp"),Xc.forEach(t),Kp=n(_a,", see the "),Cn=r(_a,"A",{href:!0,rel:!0});var Zc=i(Cn);Yp=n(Zc,"SacreBLEU repository"),Zc.forEach(t),Gp=n(_a,".) On the other hand, if we try with the two bad types of predictions (lots of repetitions or too short) that often come out of translation models, we will get rather bad BLEU scores:"),_a.forEach(t),qr=d(e),j(On.$$.fragment,e),Tr=d(e),j(Fn.$$.fragment,e),zr=d(e),j(In.$$.fragment,e),Dr=d(e),j(Ln.$$.fragment,e),Pr=d(e),So=r(e,"P",{});var Jc=i(So);Xp=n(Jc,"The score can go from 0 to 100, and higher is better."),Jc.forEach(t),Ar=d(e),Ks.l(e),Co=d(e),Oo=r(e,"P",{});var Vc=i(Oo);Zp=n(Vc,"Now that this is done, we are ready to fine-tune our model!"),Vc.forEach(t),Sr=d(e),va=r(e,"H3",{class:!0});var hi=i(va);Ha=r(hi,"A",{id:!0,class:!0,href:!0});var Qc=i(Ha);El=r(Qc,"SPAN",{});var ed=i(El);j(Nn.$$.fragment,ed),ed.forEach(t),Qc.forEach(t),Jp=d(hi),xl=r(hi,"SPAN",{});var td=i(xl);Vp=n(td,"Fine-tuning the model"),td.forEach(t),hi.forEach(t),Cr=d(e),Fo=r(e,"P",{});var sd=i(Fo);Qp=n(sd,"The first step is to log in to Hugging Face, so you\u2019re able to upload your results to the Model Hub. There\u2019s a convenience function to help you with this in a notebook:"),sd.forEach(t),Or=d(e),j(Hn.$$.fragment,e),Fr=d(e),Io=r(e,"P",{});var ad=i(Io);eh=n(ad,"This will display a widget where you can enter your Hugging Face login credentials."),ad.forEach(t),Ir=d(e),Lo=r(e,"P",{});var nd=i(Lo);th=n(nd,"If you aren\u2019t working in a notebook, just type the following line in your terminal:"),nd.forEach(t),Lr=d(e),j(Wn.$$.fragment,e),Nr=d(e),Gs.l(e),No=d(e),Ge&&Ge.l(e),Ho=d(e),ya=r(e,"H2",{class:!0});var ci=i(ya);Wa=r(ci,"A",{id:!0,class:!0,href:!0});var od=i(Wa);ql=r(od,"SPAN",{});var ld=i(ql);j(Mn.$$.fragment,ld),ld.forEach(t),od.forEach(t),sh=d(ci),Tl=r(ci,"SPAN",{});var rd=i(Tl);ah=n(rd,"Using the fine-tuned model"),rd.forEach(t),ci.forEach(t),Hr=d(e),Ma=r(e,"P",{});var di=i(Ma);nh=n(di,"We\u2019ve already shown you how you can use the model we fine-tuned on the Model Hub with the inference widget. To use it locally in a "),zl=r(di,"CODE",{});var id=i(zl);oh=n(id,"pipeline"),id.forEach(t),lh=n(di,", we just have to specify the proper model identifier:"),di.forEach(t),Wr=d(e),j(Un.$$.fragment,e),Mr=d(e),j(Bn.$$.fragment,e),Ur=d(e),Wo=r(e,"P",{});var pd=i(Wo);rh=n(pd,"As expected, our pretrained model adapted its knowledge to the corpus we fine-tuned it on, and instead of leaving the English word \u201Cthreads\u201D alone, it now translates it to the French official version. It\u2019s the same for \u201Cplugin\u201D:"),pd.forEach(t),Br=d(e),j(Rn.$$.fragment,e),Rr=d(e),j(Kn.$$.fragment,e),Kr=d(e),Mo=r(e,"P",{});var hd=i(Mo);ih=n(hd,"Another great example of domain adaptation!"),hd.forEach(t),Yr=d(e),j(Ua.$$.fragment,e),this.h()},h(){_(h,"name","hf:doc:metadata"),_(h,"content",JSON.stringify(Md)),_(g,"id","translation"),_(g,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),_(g,"href","#translation"),_(A,"class","relative group"),_(M,"href","/course/chapter1/7"),_(Z,"href","/course/chapter7/6"),_(xe,"href","/course/chapter7/6"),_(ge,"href","https://huggingface.co/datasets/kde4"),_(ge,"rel","nofollow"),_(ve,"href","https://apps.kde.org/"),_(ve,"rel","nofollow"),_(Se,"href","https://opus.nlpl.eu/"),_(Se,"rel","nofollow"),ui(he.src,Be="https://hf.space/gradioiframe/course-demos/marian-finetuned-kde4-en-to-fr/+")||_(he,"src",Be),_(he,"frameborder","0"),_(he,"height","350"),_(he,"title","Gradio app"),_(he,"class","block dark:hidden container p-0 flex-grow space-iframe"),_(he,"allow","accelerometer; ambient-light-sensor; autoplay; battery; camera; document-domain; encrypted-media; fullscreen; geolocation; gyroscope; layout-animations; legacy-image-formats; magnetometer; microphone; midi; oversized-images; payment; picture-in-picture; publickey-credentials-get; sync-xhr; usb; vr ; wake-lock; xr-spatial-tracking"),_(he,"sandbox","allow-forms allow-modals allow-popups allow-popups-to-escape-sandbox allow-same-origin allow-scripts allow-downloads"),_(ie,"class","block dark:hidden lg:w-3/5"),ui(ie.src,Ft="https://huggingface.co/datasets/huggingface-course/documentation-images/resolve/main/en/chapter7/modeleval-marian-finetuned-kde4-en-to-fr.png")||_(ie,"src",Ft),_(ie,"alt","One-hot encoded labels for question answering."),_(Le,"class","hidden dark:block lg:w-3/5"),ui(Le.src,Ne="https://huggingface.co/datasets/huggingface-course/documentation-images/resolve/main/en/chapter7/modeleval-marian-finetuned-kde4-en-to-fr-dark.png")||_(Le,"src",Ne),_(Le,"alt","One-hot encoded labels for question answering."),_(Ce,"class","flex justify-center"),_(Ce,"href","/huggingface-course/marian-finetuned-kde4-en-to-fr"),_(Oe,"href","https://huggingface.co/huggingface-course/marian-finetuned-kde4-en-to-fr?text=This+plugin+allows+you+to+automatically+translate+web+pages+between+several+languages."),_(Oe,"rel","nofollow"),_(rt,"id","preparing-the-data"),_(rt,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),_(rt,"href","#preparing-the-data"),_(lt,"class","relative group"),_(Fe,"href","https://huggingface.co/datasets/kde4"),_(Fe,"rel","nofollow"),_(De,"href","/course/chapter5"),_($t,"id","the-kde4-dataset"),_($t,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),_($t,"href","#the-kde4-dataset"),_(be,"class","relative group"),_(yt,"href","https://huggingface.co/datasets/kde4"),_(yt,"rel","nofollow"),ui(f.src,O="https://huggingface.co/datasets/huggingface-course/documentation-images/resolve/main/en/chapter7/language_tags.png")||_(f,"src",O),_(f,"alt","Language available for the KDE4 dataset."),_(f,"width","100%"),_(dt,"href","/course/chapter5"),_(Is,"id","processing-the-data"),_(Is,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),_(Is,"href","#processing-the-data"),_(_s,"class","relative group"),_(Bt,"href","https://huggingface.co/Helsinki-NLP"),_(Bt,"rel","nofollow"),_(ln,"href","https://huggingface.co/models"),_(ln,"rel","nofollow"),_(Fa,"id","data-collation"),_(Fa,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),_(Fa,"href","#data-collation"),_($a,"class","relative group"),_(yo,"href","/course/chapter3"),_(mn,"href","https://huggingface.co/transformers/main_classes/data_collator.html#datacollatorforseq2seq"),_(mn,"rel","nofollow"),_(La,"id","metrics"),_(La,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),_(La,"href","#metrics"),_(ka,"class","relative group"),_(xn,"href","https://en.wikipedia.org/wiki/BLEU"),_(xn,"rel","nofollow"),_(qn,"href","https://aclanthology.org/P02-1040.pdf"),_(qn,"rel","nofollow"),_(Tn,"href","https://github.com/mjpost/sacrebleu"),_(Tn,"rel","nofollow"),_(Do,"href","/course/chapter3"),_(Sn,"href","https://arxiv.org/pdf/1706.03762.pdf"),_(Sn,"rel","nofollow"),_(Cn,"href","https://github.com/mjpost/sacrebleu/blob/078c440168c6adc89ba75fe6d63f0d922d42bcfe/sacrebleu/metrics/bleu.py#L74"),_(Cn,"rel","nofollow"),_(Ha,"id","finetuning-the-model"),_(Ha,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),_(Ha,"href","#finetuning-the-model"),_(va,"class","relative group"),_(Wa,"id","using-the-finetuned-model"),_(Wa,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),_(Wa,"href","#using-the-finetuned-model"),_(ya,"class","relative group")},m(e,p){s(document.head,h),o(e,$,p),E(m,e,p),o(e,q,p),o(e,A,p),s(A,g),s(g,z),E(P,z,null),s(A,D),s(A,T),s(T,N),o(e,v,p),Yn[S].m(e,p),o(e,U,p),o(e,F,p),s(F,X),s(F,M),s(M,I),s(F,Y),s(F,Z),s(Z,re),s(F,de),o(e,Q,p),o(e,ee,p),s(ee,te),s(te,G),s(G,ne),s(te,$e),s(te,V),s(V,se),s(te,fe),s(ee,ke),s(ee,oe),s(oe,pe),s(pe,W),s(oe,ae),o(e,ze,p),E(me,e,p),o(e,K,p),o(e,ye,p),s(ye,le),s(ye,xe),s(xe,Me),s(ye,_e),o(e,qe,p),o(e,J,p),s(J,Ae),s(J,ge),s(ge,at),s(J,ft),s(J,ve),s(ve,Xe),s(J,Ue),s(J,Se),s(Se,je),s(J,mt),o(e,we,p),o(e,k,p),s(k,R),o(e,Ze,p),o(e,he,p),o(e,nt,p),o(e,Ce,p),s(Ce,ie),s(Ce,Re),s(Ce,Le),o(e,_t,p),o(e,ce,p),s(ce,Je),s(ce,Oe),s(Oe,zt),s(ce,is),o(e,ot,p),o(e,lt,p),s(lt,rt),s(rt,Ve),E(Dt,Ve,null),s(lt,ps),s(lt,Qe),s(Qe,gt),o(e,wt,p),o(e,Ee,p),s(Ee,Gt),s(Ee,Fe),s(Fe,ks),s(Ee,Xt),s(Ee,De),s(De,Zt),s(Ee,Pe),s(Ee,It),s(It,bt),s(Ee,vs),o(e,Jt,p),o(e,be,p),s(be,$t),s($t,Pt),E(kt,Pt,null),s(be,it),s(be,He),s(He,Vt),o(e,vt,p),o(e,pt,p),s(pt,ht),s(pt,Lt),s(Lt,At),s(pt,et),o(e,Qt,p),E(Ie,e,p),o(e,es,p),o(e,Ke,p),s(Ke,ts),s(Ke,yt),s(yt,Nt),s(Ke,hs),o(e,ct,p),o(e,f,p),o(e,ys,p),o(e,jt,p),s(jt,Xs),o(e,js,p),E(Te,e,p),o(e,Es,p),E(Et,e,p),o(e,cs,p),o(e,ue,p),s(ue,Zs),s(ue,dt),s(dt,Js),s(ue,We),s(ue,Ht),s(Ht,xs),s(ue,Vs),s(ue,ss),s(ss,Ye),s(ue,Qs),o(e,xt,p),E(Wt,e,p),o(e,qs,p),E(qt,e,p),o(e,Ts,p),o(e,ut,p),s(ut,ds),s(ut,us),s(us,Mt),s(ut,zs),s(ut,Tt),s(Tt,Ds),s(ut,Ut),o(e,fs,p),E(tt,e,p),o(e,xa,p),o(e,ms,p),s(ms,Ja),o(e,qa,p),E(St,e,p),o(e,ea,p),E(Ps,e,p),o(e,Ta,p),o(e,As,p),s(As,u),o(e,L,p),E(ta,e,p),o(e,Va,p),E(sa,e,p),o(e,Ss,p),o(e,ga,p),s(ga,so),o(e,wa,p),E(aa,e,p),o(e,Qa,p),E(na,e,p),o(e,Cs,p),o(e,ba,p),s(ba,ao),o(e,en,p),E(oa,e,p),o(e,Os,p),E(la,e,p),o(e,tn,p),o(e,Ct,p),s(Ct,no),o(e,sn,p),E(ra,e,p),o(e,as,p),E(Fs,e,p),o(e,an,p),o(e,_s,p),s(_s,Is),s(Is,gs),E(ia,gs,null),s(_s,oo),s(_s,pa),s(pa,lo),o(e,nn,p),E(ha,e,p),o(e,Ls,p),o(e,ns,p),s(ns,ro),s(ns,ws),s(ws,io),s(ns,po),s(ns,Bt),s(Bt,ho),s(ns,co),o(e,os,p),E(ca,e,p),o(e,on,p),o(e,ls,p),s(ls,da),s(ls,za),s(za,uo),s(ls,fo),s(ls,ln),s(ln,mi),s(ls,_i),o(e,Sl,p),E(Da,e,p),o(e,Cl,p),o(e,Pa,p),s(Pa,gi),s(Pa,Qo),s(Qo,wi),s(Pa,bi),o(e,Ol,p),o(e,Aa,p),s(Aa,$i),s(Aa,el),s(el,ki),s(Aa,vi),o(e,Fl,p),E(rn,e,p),o(e,Il,p),o(e,ua,p),s(ua,yi),s(ua,tl),s(tl,ji),s(ua,Ei),s(ua,sl),s(sl,xi),s(ua,qi),o(e,Ll,p),o(e,Sa,p),s(Sa,Ti),s(Sa,al),s(al,zi),s(Sa,Di),o(e,Nl,p),o(e,mo,p),s(mo,Pi),o(e,Hl,p),E(pn,e,p),o(e,Wl,p),o(e,_o,p),s(_o,Ai),o(e,Ml,p),E(hn,e,p),o(e,Ul,p),E(cn,e,p),o(e,Bl,p),o(e,go,p),s(go,Si),o(e,Rl,p),o(e,bs,p),s(bs,Ci),s(bs,nl),s(nl,Oi),s(bs,Fi),s(bs,ol),s(ol,Ii),s(bs,Li),s(bs,ll),s(ll,Ni),s(bs,Hi),o(e,Kl,p),E(dn,e,p),o(e,Yl,p),o(e,wo,p),s(wo,Wi),o(e,Gl,p),E(Ca,e,p),o(e,Xl,p),E(Oa,e,p),o(e,Zl,p),o(e,bo,p),s(bo,Mi),o(e,Jl,p),E(un,e,p),o(e,Vl,p),o(e,$o,p),s($o,Ui),o(e,Ql,p),Gn[Ns].m(e,p),o(e,ko,p),o(e,vo,p),s(vo,Bi),o(e,er,p),o(e,$a,p),s($a,Fa),s(Fa,rl),E(fn,rl,null),s($a,Ri),s($a,il),s(il,Ki),o(e,tr,p),o(e,$s,p),s($s,Yi),s($s,pl),s(pl,Gi),s($s,Xi),s($s,yo),s(yo,Zi),s($s,Ji),s($s,hl),s(hl,Vi),s($s,Qi),o(e,sr,p),o(e,st,p),s(st,ep),s(st,mn),s(mn,cl),s(cl,tp),s(st,sp),s(st,dl),s(dl,ap),s(st,np),s(st,ul),s(ul,op),s(st,lp),s(st,fl),s(fl,rp),s(st,ip),s(st,ml),s(ml,pp),s(st,hp),s(st,_l),s(_l,cp),s(st,dp),o(e,ar,p),Xn[Ws].m(e,p),o(e,jo,p),o(e,Eo,p),s(Eo,up),o(e,nr,p),E(_n,e,p),o(e,or,p),E(gn,e,p),o(e,lr,p),o(e,Ia,p),s(Ia,fp),s(Ia,gl),s(gl,mp),s(Ia,_p),o(e,rr,p),E(wn,e,p),o(e,ir,p),E(bn,e,p),o(e,pr,p),o(e,xo,p),s(xo,gp),o(e,hr,p),E($n,e,p),o(e,cr,p),E(kn,e,p),o(e,dr,p),o(e,qo,p),s(qo,wp),o(e,ur,p),E(vn,e,p),o(e,fr,p),E(yn,e,p),o(e,mr,p),Zn[Us].m(e,p),o(e,To,p),o(e,ka,p),s(ka,La),s(La,wl),E(jn,wl,null),s(ka,bp),s(ka,bl),s(bl,$p),o(e,_r,p),E(En,e,p),o(e,gr,p),Ot&&Ot.m(e,p),o(e,zo,p),o(e,Rt,p),s(Rt,kp),s(Rt,xn),s(xn,vp),s(Rt,yp),s(Rt,qn),s(qn,jp),s(Rt,Ep),s(Rt,$l),s($l,xp),s(Rt,qp),s(Rt,kl),s(kl,Tp),s(Rt,zp),o(e,wr,p),o(e,Na,p),s(Na,Dp),s(Na,Tn),s(Tn,Pp),s(Na,Ap),o(e,br,p),E(zn,e,p),o(e,$r,p),o(e,fa,p),s(fa,Sp),s(fa,vl),s(vl,Cp),s(fa,Op),s(fa,Do),s(Do,Fp),s(fa,Ip),o(e,kr,p),E(Dn,e,p),o(e,vr,p),o(e,Po,p),s(Po,Lp),o(e,yr,p),o(e,Ao,p),s(Ao,Np),o(e,jr,p),E(Pn,e,p),o(e,Er,p),E(An,e,p),o(e,xr,p),o(e,Kt,p),s(Kt,Hp),s(Kt,Sn),s(Sn,Wp),s(Kt,Mp),s(Kt,yl),s(yl,Up),s(Kt,Bp),s(Kt,jl),s(jl,Rp),s(Kt,Kp),s(Kt,Cn),s(Cn,Yp),s(Kt,Gp),o(e,qr,p),E(On,e,p),o(e,Tr,p),E(Fn,e,p),o(e,zr,p),E(In,e,p),o(e,Dr,p),E(Ln,e,p),o(e,Pr,p),o(e,So,p),s(So,Xp),o(e,Ar,p),Jn[Rs].m(e,p),o(e,Co,p),o(e,Oo,p),s(Oo,Zp),o(e,Sr,p),o(e,va,p),s(va,Ha),s(Ha,El),E(Nn,El,null),s(va,Jp),s(va,xl),s(xl,Vp),o(e,Cr,p),o(e,Fo,p),s(Fo,Qp),o(e,Or,p),E(Hn,e,p),o(e,Fr,p),o(e,Io,p),s(Io,eh),o(e,Ir,p),o(e,Lo,p),s(Lo,th),o(e,Lr,p),E(Wn,e,p),o(e,Nr,p),Vn[Ys].m(e,p),o(e,No,p),Ge&&Ge.m(e,p),o(e,Ho,p),o(e,ya,p),s(ya,Wa),s(Wa,ql),E(Mn,ql,null),s(ya,sh),s(ya,Tl),s(Tl,ah),o(e,Hr,p),o(e,Ma,p),s(Ma,nh),s(Ma,zl),s(zl,oh),s(Ma,lh),o(e,Wr,p),E(Un,e,p),o(e,Mr,p),E(Bn,e,p),o(e,Ur,p),o(e,Wo,p),s(Wo,rh),o(e,Br,p),E(Rn,e,p),o(e,Rr,p),E(Kn,e,p),o(e,Kr,p),o(e,Mo,p),s(Mo,ih),o(e,Yr,p),E(Ua,e,p),Gr=!0},p(e,[p]){const Qn={};p&1&&(Qn.fw=e[0]),m.$set(Qn);let Uo=S;S=hh(e),S!==Uo&&(to(),b(Yn[Uo],1,1,()=>{Yn[Uo]=null}),eo(),H=Yn[S],H||(H=Yn[S]=ph[S](e),H.c()),w(H,1),H.m(U.parentNode,U));const Dl={};p&2&&(Dl.$$scope={dirty:p,ctx:e}),Fs.$set(Dl);const Pl={};p&2&&(Pl.$$scope={dirty:p,ctx:e}),Da.$set(Pl);const ja={};p&2&&(ja.$$scope={dirty:p,ctx:e}),Ca.$set(ja);const Al={};p&2&&(Al.$$scope={dirty:p,ctx:e}),Oa.$set(Al);let Bo=Ns;Ns=dh(e),Ns!==Bo&&(to(),b(Gn[Bo],1,1,()=>{Gn[Bo]=null}),eo(),Hs=Gn[Ns],Hs||(Hs=Gn[Ns]=ch[Ns](e),Hs.c()),w(Hs,1),Hs.m(ko.parentNode,ko));let Ba=Ws;Ws=fh(e),Ws!==Ba&&(to(),b(Xn[Ba],1,1,()=>{Xn[Ba]=null}),eo(),Ms=Xn[Ws],Ms||(Ms=Xn[Ws]=uh[Ws](e),Ms.c()),w(Ms,1),Ms.m(jo.parentNode,jo));let Ea=Us;Us=_h(e),Us!==Ea&&(to(),b(Zn[Ea],1,1,()=>{Zn[Ea]=null}),eo(),Bs=Zn[Us],Bs||(Bs=Zn[Us]=mh[Us](e),Bs.c()),w(Bs,1),Bs.m(To.parentNode,To)),e[0]==="pt"?Ot||(Ot=dd(),Ot.c(),Ot.m(zo.parentNode,zo)):Ot&&(Ot.d(1),Ot=null);let Ro=Rs;Rs=wh(e),Rs!==Ro&&(to(),b(Jn[Ro],1,1,()=>{Jn[Ro]=null}),eo(),Ks=Jn[Rs],Ks||(Ks=Jn[Rs]=gh[Rs](e),Ks.c()),w(Ks,1),Ks.m(Co.parentNode,Co));let Ko=Ys;Ys=$h(e),Ys!==Ko&&(to(),b(Vn[Ko],1,1,()=>{Vn[Ko]=null}),eo(),Gs=Vn[Ys],Gs||(Gs=Vn[Ys]=bh[Ys](e),Gs.c()),w(Gs,1),Gs.m(No.parentNode,No)),e[0]==="pt"?Ge?p&1&&w(Ge,1):(Ge=ud(),Ge.c(),w(Ge,1),Ge.m(Ho.parentNode,Ho)):Ge&&(to(),b(Ge,1,1,()=>{Ge=null}),eo());const Yo={};p&2&&(Yo.$$scope={dirty:p,ctx:e}),Ua.$set(Yo)},i(e){Gr||(w(m.$$.fragment,e),w(P.$$.fragment,e),w(H),w(me.$$.fragment,e),w(Dt.$$.fragment,e),w(kt.$$.fragment,e),w(Ie.$$.fragment,e),w(Te.$$.fragment,e),w(Et.$$.fragment,e),w(Wt.$$.fragment,e),w(qt.$$.fragment,e),w(tt.$$.fragment,e),w(St.$$.fragment,e),w(Ps.$$.fragment,e),w(ta.$$.fragment,e),w(sa.$$.fragment,e),w(aa.$$.fragment,e),w(na.$$.fragment,e),w(oa.$$.fragment,e),w(la.$$.fragment,e),w(ra.$$.fragment,e),w(Fs.$$.fragment,e),w(ia.$$.fragment,e),w(ha.$$.fragment,e),w(ca.$$.fragment,e),w(Da.$$.fragment,e),w(rn.$$.fragment,e),w(pn.$$.fragment,e),w(hn.$$.fragment,e),w(cn.$$.fragment,e),w(dn.$$.fragment,e),w(Ca.$$.fragment,e),w(Oa.$$.fragment,e),w(un.$$.fragment,e),w(Hs),w(fn.$$.fragment,e),w(Ms),w(_n.$$.fragment,e),w(gn.$$.fragment,e),w(wn.$$.fragment,e),w(bn.$$.fragment,e),w($n.$$.fragment,e),w(kn.$$.fragment,e),w(vn.$$.fragment,e),w(yn.$$.fragment,e),w(Bs),w(jn.$$.fragment,e),w(En.$$.fragment,e),w(zn.$$.fragment,e),w(Dn.$$.fragment,e),w(Pn.$$.fragment,e),w(An.$$.fragment,e),w(On.$$.fragment,e),w(Fn.$$.fragment,e),w(In.$$.fragment,e),w(Ln.$$.fragment,e),w(Ks),w(Nn.$$.fragment,e),w(Hn.$$.fragment,e),w(Wn.$$.fragment,e),w(Gs),w(Ge),w(Mn.$$.fragment,e),w(Un.$$.fragment,e),w(Bn.$$.fragment,e),w(Rn.$$.fragment,e),w(Kn.$$.fragment,e),w(Ua.$$.fragment,e),Gr=!0)},o(e){b(m.$$.fragment,e),b(P.$$.fragment,e),b(H),b(me.$$.fragment,e),b(Dt.$$.fragment,e),b(kt.$$.fragment,e),b(Ie.$$.fragment,e),b(Te.$$.fragment,e),b(Et.$$.fragment,e),b(Wt.$$.fragment,e),b(qt.$$.fragment,e),b(tt.$$.fragment,e),b(St.$$.fragment,e),b(Ps.$$.fragment,e),b(ta.$$.fragment,e),b(sa.$$.fragment,e),b(aa.$$.fragment,e),b(na.$$.fragment,e),b(oa.$$.fragment,e),b(la.$$.fragment,e),b(ra.$$.fragment,e),b(Fs.$$.fragment,e),b(ia.$$.fragment,e),b(ha.$$.fragment,e),b(ca.$$.fragment,e),b(Da.$$.fragment,e),b(rn.$$.fragment,e),b(pn.$$.fragment,e),b(hn.$$.fragment,e),b(cn.$$.fragment,e),b(dn.$$.fragment,e),b(Ca.$$.fragment,e),b(Oa.$$.fragment,e),b(un.$$.fragment,e),b(Hs),b(fn.$$.fragment,e),b(Ms),b(_n.$$.fragment,e),b(gn.$$.fragment,e),b(wn.$$.fragment,e),b(bn.$$.fragment,e),b($n.$$.fragment,e),b(kn.$$.fragment,e),b(vn.$$.fragment,e),b(yn.$$.fragment,e),b(Bs),b(jn.$$.fragment,e),b(En.$$.fragment,e),b(zn.$$.fragment,e),b(Dn.$$.fragment,e),b(Pn.$$.fragment,e),b(An.$$.fragment,e),b(On.$$.fragment,e),b(Fn.$$.fragment,e),b(In.$$.fragment,e),b(Ln.$$.fragment,e),b(Ks),b(Nn.$$.fragment,e),b(Hn.$$.fragment,e),b(Wn.$$.fragment,e),b(Gs),b(Ge),b(Mn.$$.fragment,e),b(Un.$$.fragment,e),b(Bn.$$.fragment,e),b(Rn.$$.fragment,e),b(Kn.$$.fragment,e),b(Ua.$$.fragment,e),Gr=!1},d(e){t(h),e&&t($),x(m,e),e&&t(q),e&&t(A),x(P),e&&t(v),Yn[S].d(e),e&&t(U),e&&t(F),e&&t(Q),e&&t(ee),e&&t(ze),x(me,e),e&&t(K),e&&t(ye),e&&t(qe),e&&t(J),e&&t(we),e&&t(k),e&&t(Ze),e&&t(he),e&&t(nt),e&&t(Ce),e&&t(_t),e&&t(ce),e&&t(ot),e&&t(lt),x(Dt),e&&t(wt),e&&t(Ee),e&&t(Jt),e&&t(be),x(kt),e&&t(vt),e&&t(pt),e&&t(Qt),x(Ie,e),e&&t(es),e&&t(Ke),e&&t(ct),e&&t(f),e&&t(ys),e&&t(jt),e&&t(js),x(Te,e),e&&t(Es),x(Et,e),e&&t(cs),e&&t(ue),e&&t(xt),x(Wt,e),e&&t(qs),x(qt,e),e&&t(Ts),e&&t(ut),e&&t(fs),x(tt,e),e&&t(xa),e&&t(ms),e&&t(qa),x(St,e),e&&t(ea),x(Ps,e),e&&t(Ta),e&&t(As),e&&t(L),x(ta,e),e&&t(Va),x(sa,e),e&&t(Ss),e&&t(ga),e&&t(wa),x(aa,e),e&&t(Qa),x(na,e),e&&t(Cs),e&&t(ba),e&&t(en),x(oa,e),e&&t(Os),x(la,e),e&&t(tn),e&&t(Ct),e&&t(sn),x(ra,e),e&&t(as),x(Fs,e),e&&t(an),e&&t(_s),x(ia),e&&t(nn),x(ha,e),e&&t(Ls),e&&t(ns),e&&t(os),x(ca,e),e&&t(on),e&&t(ls),e&&t(Sl),x(Da,e),e&&t(Cl),e&&t(Pa),e&&t(Ol),e&&t(Aa),e&&t(Fl),x(rn,e),e&&t(Il),e&&t(ua),e&&t(Ll),e&&t(Sa),e&&t(Nl),e&&t(mo),e&&t(Hl),x(pn,e),e&&t(Wl),e&&t(_o),e&&t(Ml),x(hn,e),e&&t(Ul),x(cn,e),e&&t(Bl),e&&t(go),e&&t(Rl),e&&t(bs),e&&t(Kl),x(dn,e),e&&t(Yl),e&&t(wo),e&&t(Gl),x(Ca,e),e&&t(Xl),x(Oa,e),e&&t(Zl),e&&t(bo),e&&t(Jl),x(un,e),e&&t(Vl),e&&t($o),e&&t(Ql),Gn[Ns].d(e),e&&t(ko),e&&t(vo),e&&t(er),e&&t($a),x(fn),e&&t(tr),e&&t($s),e&&t(sr),e&&t(st),e&&t(ar),Xn[Ws].d(e),e&&t(jo),e&&t(Eo),e&&t(nr),x(_n,e),e&&t(or),x(gn,e),e&&t(lr),e&&t(Ia),e&&t(rr),x(wn,e),e&&t(ir),x(bn,e),e&&t(pr),e&&t(xo),e&&t(hr),x($n,e),e&&t(cr),x(kn,e),e&&t(dr),e&&t(qo),e&&t(ur),x(vn,e),e&&t(fr),x(yn,e),e&&t(mr),Zn[Us].d(e),e&&t(To),e&&t(ka),x(jn),e&&t(_r),x(En,e),e&&t(gr),Ot&&Ot.d(e),e&&t(zo),e&&t(Rt),e&&t(wr),e&&t(Na),e&&t(br),x(zn,e),e&&t($r),e&&t(fa),e&&t(kr),x(Dn,e),e&&t(vr),e&&t(Po),e&&t(yr),e&&t(Ao),e&&t(jr),x(Pn,e),e&&t(Er),x(An,e),e&&t(xr),e&&t(Kt),e&&t(qr),x(On,e),e&&t(Tr),x(Fn,e),e&&t(zr),x(In,e),e&&t(Dr),x(Ln,e),e&&t(Pr),e&&t(So),e&&t(Ar),Jn[Rs].d(e),e&&t(Co),e&&t(Oo),e&&t(Sr),e&&t(va),x(Nn),e&&t(Cr),e&&t(Fo),e&&t(Or),x(Hn,e),e&&t(Fr),e&&t(Io),e&&t(Ir),e&&t(Lo),e&&t(Lr),x(Wn,e),e&&t(Nr),Vn[Ys].d(e),e&&t(No),Ge&&Ge.d(e),e&&t(Ho),e&&t(ya),x(Mn),e&&t(Hr),e&&t(Ma),e&&t(Wr),x(Un,e),e&&t(Mr),x(Bn,e),e&&t(Ur),e&&t(Wo),e&&t(Br),x(Rn,e),e&&t(Rr),x(Kn,e),e&&t(Kr),e&&t(Mo),e&&t(Yr),x(Ua,e)}}}const Md={local:"translation",sections:[{local:"preparing-the-data",sections:[{local:"the-kde4-dataset",title:"The KDE4 dataset"},{local:"processing-the-data",title:"Processing the data"}],title:"Preparing the data"},{local:"finetuning-the-model-with-the-trainer-api",title:"Fine-tuning the model with the `Trainer` API"},{local:"finetuning-the-model-with-keras",sections:[{local:"data-collation",title:"Data collation"},{local:"metrics",title:"Metrics"},{local:"finetuning-the-model",title:"Fine-tuning the model"}],title:"Fine-tuning the model with Keras"},{local:"a-custom-training-loop",sections:[{local:"preparing-everything-for-training",title:"Preparing everything for training"},{local:"training-loop",title:"Training loop"}],title:"A custom training loop"},{local:"using-the-finetuned-model",title:"Using the fine-tuned model"}],title:"Translation"};function Ud(B,h,$){let m="pt";return bd(()=>{const q=new URLSearchParams(window.location.search);$(0,m=q.get("fw")||"pt")}),[m]}class Jd extends md{constructor(h){super();_d(this,h,Ud,Wd,gd,{})}}export{Jd as default,Md as metadata};
