import{S as wg,i as bg,s as vg,e as o,t as a,k as h,w as v,c as r,a as l,h as n,d as t,m as c,x as y,b as f,g as i,G as s,y as k,q as g,o as _,B as $,T as hg,M as yg,U as cg,Z as Bl,p as Ar,v as kg,n as Or}from"../../chunks/vendor-hf-doc-builder.js";import{T as pn}from"../../chunks/Tip-hf-doc-builder.js";import{Y as yu}from"../../chunks/Youtube-hf-doc-builder.js";import{I as Ks}from"../../chunks/IconCopyLink-hf-doc-builder.js";import{C as z}from"../../chunks/CodeBlock-hf-doc-builder.js";import{D as fg}from"../../chunks/DocNotebookDropdown-hf-doc-builder.js";import{F as $g}from"../../chunks/FrameworkSwitchCourse-hf-doc-builder.js";function jg(Y){let u,j;return u=new fg({props:{classNames:"absolute z-10 right-0 top-0",options:[{label:"Google Colab",value:"https://colab.research.google.com/github/huggingface/notebooks/blob/master/course/chapter7/section5_tf.ipynb"},{label:"Aws Studio",value:"https://studiolab.sagemaker.aws/import/github/huggingface/notebooks/blob/master/course/chapter7/section5_tf.ipynb"}]}}),{c(){v(u.$$.fragment)},l(d){y(u.$$.fragment,d)},m(d,E){k(u,d,E),j=!0},i(d){j||(g(u.$$.fragment,d),j=!0)},o(d){_(u.$$.fragment,d),j=!1},d(d){$(u,d)}}}function xg(Y){let u,j;return u=new fg({props:{classNames:"absolute z-10 right-0 top-0",options:[{label:"Google Colab",value:"https://colab.research.google.com/github/huggingface/notebooks/blob/master/course/chapter7/section5_pt.ipynb"},{label:"Aws Studio",value:"https://studiolab.sagemaker.aws/import/github/huggingface/notebooks/blob/master/course/chapter7/section5_pt.ipynb"}]}}),{c(){v(u.$$.fragment)},l(d){y(u.$$.fragment,d)},m(d,E){k(u,d,E),j=!0},i(d){j||(g(u.$$.fragment,d),j=!0)},o(d){_(u.$$.fragment,d),j=!1},d(d){$(u,d)}}}function Eg(Y){let u,j,d,E,C,x,A,P,T,S,I;return{c(){u=o("p"),j=a("\u270F\uFE0F "),d=o("strong"),E=a("Try it out!"),C=a(" Change the random seed in the "),x=o("code"),A=a("Dataset.shuffle()"),P=a(" command to explore other reviews in the corpus. If you\u2019re a Spanish speaker, take a look at some of the reviews in "),T=o("code"),S=a("spanish_dataset"),I=a(" to see if the titles also seem like reasonable summaries.")},l(D){u=r(D,"P",{});var O=l(u);j=n(O,"\u270F\uFE0F "),d=r(O,"STRONG",{});var G=l(d);E=n(G,"Try it out!"),G.forEach(t),C=n(O," Change the random seed in the "),x=r(O,"CODE",{});var J=l(x);A=n(J,"Dataset.shuffle()"),J.forEach(t),P=n(O," command to explore other reviews in the corpus. If you\u2019re a Spanish speaker, take a look at some of the reviews in "),T=r(O,"CODE",{});var X=l(T);S=n(X,"spanish_dataset"),X.forEach(t),I=n(O," to see if the titles also seem like reasonable summaries."),O.forEach(t)},m(D,O){i(D,u,O),s(u,j),s(u,d),s(d,E),s(u,C),s(u,x),s(x,A),s(u,P),s(u,T),s(T,S),s(u,I)},d(D){D&&t(u)}}}function Tg(Y){let u,j,d,E,C,x,A,P;return{c(){u=o("p"),j=a("\u270F\uFE0F "),d=o("strong"),E=a("Try it out!"),C=a(" Once you\u2019ve worked through this section, see how well mT5 compares to mBART by fine-tuning the latter with the same techniques. For bonus points, you can also try fine-tuning T5 on just the English reviews. Since T5 has a special prefix prompt, you\u2019ll need to prepend "),x=o("code"),A=a("summarize:"),P=a(" to the input examples in the preprocessing steps below.")},l(T){u=r(T,"P",{});var S=l(u);j=n(S,"\u270F\uFE0F "),d=r(S,"STRONG",{});var I=l(d);E=n(I,"Try it out!"),I.forEach(t),C=n(S," Once you\u2019ve worked through this section, see how well mT5 compares to mBART by fine-tuning the latter with the same techniques. For bonus points, you can also try fine-tuning T5 on just the English reviews. Since T5 has a special prefix prompt, you\u2019ll need to prepend "),x=r(S,"CODE",{});var D=l(x);A=n(D,"summarize:"),D.forEach(t),P=n(S," to the input examples in the preprocessing steps below."),S.forEach(t)},m(T,S){i(T,u,S),s(u,j),s(u,d),s(d,E),s(u,C),s(u,x),s(x,A),s(u,P)},d(T){T&&t(u)}}}function qg(Y){let u,j;return{c(){u=o("p"),j=a("\u{1F4A1} In the early stages of your NLP projects, a good practice is to train a class of \u201Csmall\u201D models on a small sample of data. This allows you to debug and iterate faster toward an end-to-end workflow. Once you are confident in the results, you can always scale up the model by simply changing the model checkpoint!")},l(d){u=r(d,"P",{});var E=l(u);j=n(E,"\u{1F4A1} In the early stages of your NLP projects, a good practice is to train a class of \u201Csmall\u201D models on a small sample of data. This allows you to debug and iterate faster toward an end-to-end workflow. Once you are confident in the results, you can always scale up the model by simply changing the model checkpoint!"),E.forEach(t)},m(d,E){i(d,u,E),s(u,j)},d(d){d&&t(u)}}}function zg(Y){let u,j,d,E,C,x,A,P,T,S,I;return{c(){u=o("p"),j=a("\u{1F4A1} You may have noticed that we used "),d=o("code"),E=a("batched=True"),C=a(" in our "),x=o("code"),A=a("Dataset.map()"),P=a(" function above. This encodes the examples in batches of 1,000 (the default) and allows you to make use of the multithreading capabilities of the fast tokenizers in \u{1F917} Transformers. Where possible, try using "),T=o("code"),S=a("batched=True"),I=a(" to get the most out of your preprocessing!")},l(D){u=r(D,"P",{});var O=l(u);j=n(O,"\u{1F4A1} You may have noticed that we used "),d=r(O,"CODE",{});var G=l(d);E=n(G,"batched=True"),G.forEach(t),C=n(O," in our "),x=r(O,"CODE",{});var J=l(x);A=n(J,"Dataset.map()"),J.forEach(t),P=n(O," function above. This encodes the examples in batches of 1,000 (the default) and allows you to make use of the multithreading capabilities of the fast tokenizers in \u{1F917} Transformers. Where possible, try using "),T=r(O,"CODE",{});var X=l(T);S=n(X,"batched=True"),X.forEach(t),I=n(O," to get the most out of your preprocessing!"),O.forEach(t)},m(D,O){i(D,u,O),s(u,j),s(u,d),s(d,E),s(u,C),s(u,x),s(x,A),s(u,P),s(u,T),s(T,S),s(u,I)},d(D){D&&t(u)}}}function Dg(Y){let u,j,d,E,C,x,A,P;return{c(){u=o("p"),j=a("\u{1F64B} Don\u2019t worry if this is the first time you\u2019ve heard of precision and recall \u2014 we\u2019ll go through some explicit examples together to make it all clear. These metrics are usually encountered in classification tasks, so if you want to understand how precision and recall are defined in that context, we recommend checking out the "),d=o("code"),E=a("scikit-learn"),C=h(),x=o("a"),A=a("guides"),P=a("."),this.h()},l(T){u=r(T,"P",{});var S=l(u);j=n(S,"\u{1F64B} Don\u2019t worry if this is the first time you\u2019ve heard of precision and recall \u2014 we\u2019ll go through some explicit examples together to make it all clear. These metrics are usually encountered in classification tasks, so if you want to understand how precision and recall are defined in that context, we recommend checking out the "),d=r(S,"CODE",{});var I=l(d);E=n(I,"scikit-learn"),I.forEach(t),C=c(S),x=r(S,"A",{href:!0,rel:!0});var D=l(x);A=n(D,"guides"),D.forEach(t),P=n(S,"."),S.forEach(t),this.h()},h(){f(x,"href","https://scikit-learn.org/stable/auto_examples/model_selection/plot_precision_recall.html"),f(x,"rel","nofollow")},m(T,S){i(T,u,S),s(u,j),s(u,d),s(d,E),s(u,C),s(u,x),s(x,A),s(u,P)},d(T){T&&t(u)}}}function Sg(Y){let u,j,d,E,C,x,A,P;return{c(){u=o("p"),j=a("\u270F\uFE0F "),d=o("strong"),E=a("Try it out!"),C=a(" Create your own example of a generated and reference summary and see if the resulting ROUGE scores agree with a manual calculation based on the formulas for precision and recall. For bonus points, split the text into bigrams and compare the precision and recall for the "),x=o("code"),A=a("rouge2"),P=a(" metric.")},l(T){u=r(T,"P",{});var S=l(u);j=n(S,"\u270F\uFE0F "),d=r(S,"STRONG",{});var I=l(d);E=n(I,"Try it out!"),I.forEach(t),C=n(S," Create your own example of a generated and reference summary and see if the resulting ROUGE scores agree with a manual calculation based on the formulas for precision and recall. For bonus points, split the text into bigrams and compare the precision and recall for the "),x=r(S,"CODE",{});var D=l(x);A=n(D,"rouge2"),D.forEach(t),P=n(S," metric."),S.forEach(t)},m(T,S){i(T,u,S),s(u,j),s(u,d),s(d,E),s(u,C),s(u,x),s(x,A),s(u,P)},d(T){T&&t(u)}}}function Ag(Y){let u,j,d,E,C,x,A,P,T,S,I,D,O,G,J,X,Z,B,Q;return E=new Ks({}),B=new z({props:{code:`from transformers import TFAutoModelForSeq2SeqLM

model = TFAutoModelForSeq2SeqLM.from_pretrained(model_checkpoint)`,highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> TFAutoModelForSeq2SeqLM

model = TFAutoModelForSeq2SeqLM.from_pretrained(model_checkpoint)`}}),{c(){u=o("h2"),j=o("a"),d=o("span"),v(E.$$.fragment),C=h(),x=o("span"),A=a("Fine-tuning mT5 with Keras"),P=h(),T=o("p"),S=a("Fine-tuning a model for summarization is very similar to the other tasks we\u2019ve covered in this chapter. The first thing we need to do is load the pretrained model from the "),I=o("code"),D=a("mt5-small"),O=a(" checkpoint. Since summarization is a sequence-to-sequence task, we can load the model with the "),G=o("code"),J=a("AutoModelForSeq2SeqLM"),X=a(" class, which will automatically download and cache the weights:"),Z=h(),v(B.$$.fragment),this.h()},l(L){u=r(L,"H2",{class:!0});var M=l(u);j=r(M,"A",{id:!0,class:!0,href:!0});var ee=l(j);d=r(ee,"SPAN",{});var F=l(d);y(E.$$.fragment,F),F.forEach(t),ee.forEach(t),C=c(M),x=r(M,"SPAN",{});var K=l(x);A=n(K,"Fine-tuning mT5 with Keras"),K.forEach(t),M.forEach(t),P=c(L),T=r(L,"P",{});var te=l(T);S=n(te,"Fine-tuning a model for summarization is very similar to the other tasks we\u2019ve covered in this chapter. The first thing we need to do is load the pretrained model from the "),I=r(te,"CODE",{});var W=l(I);D=n(W,"mt5-small"),W.forEach(t),O=n(te," checkpoint. Since summarization is a sequence-to-sequence task, we can load the model with the "),G=r(te,"CODE",{});var ie=l(G);J=n(ie,"AutoModelForSeq2SeqLM"),ie.forEach(t),X=n(te," class, which will automatically download and cache the weights:"),te.forEach(t),Z=c(L),y(B.$$.fragment,L),this.h()},h(){f(j,"id","finetuning-mt5-with-keras"),f(j,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),f(j,"href","#finetuning-mt5-with-keras"),f(u,"class","relative group")},m(L,M){i(L,u,M),s(u,j),s(j,d),k(E,d,null),s(u,C),s(u,x),s(x,A),i(L,P,M),i(L,T,M),s(T,S),s(T,I),s(I,D),s(T,O),s(T,G),s(G,J),s(T,X),i(L,Z,M),k(B,L,M),Q=!0},i(L){Q||(g(E.$$.fragment,L),g(B.$$.fragment,L),Q=!0)},o(L){_(E.$$.fragment,L),_(B.$$.fragment,L),Q=!1},d(L){L&&t(u),$(E),L&&t(P),L&&t(T),L&&t(Z),$(B,L)}}}function Og(Y){let u,j,d,E,C,x,A,P,T,S,I,D,O,G,J,X,Z,B,Q,L,M,ee;return E=new Ks({}),M=new z({props:{code:`from transformers import AutoModelForSeq2SeqLM

model = AutoModelForSeq2SeqLM.from_pretrained(model_checkpoint)`,highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoModelForSeq2SeqLM

model = AutoModelForSeq2SeqLM.from_pretrained(model_checkpoint)`}}),{c(){u=o("h2"),j=o("a"),d=o("span"),v(E.$$.fragment),C=h(),x=o("span"),A=a("Fine-tuning mT5 with the "),P=o("code"),T=a("Trainer"),S=a(" API"),I=h(),D=o("p"),O=a("Fine-tuning a model for summarization is very similar to the other tasks we\u2019ve covered in this chapter. The first thing we need to do is load the pretrained model from the "),G=o("code"),J=a("mt5-small"),X=a(" checkpoint. Since summarization is a sequence-to-sequence task, we can load the model with the "),Z=o("code"),B=a("AutoModelForSeq2SeqLM"),Q=a(" class, which will automatically download and cache the weights:"),L=h(),v(M.$$.fragment),this.h()},l(F){u=r(F,"H2",{class:!0});var K=l(u);j=r(K,"A",{id:!0,class:!0,href:!0});var te=l(j);d=r(te,"SPAN",{});var W=l(d);y(E.$$.fragment,W),W.forEach(t),te.forEach(t),C=c(K),x=r(K,"SPAN",{});var ie=l(x);A=n(ie,"Fine-tuning mT5 with the "),P=r(ie,"CODE",{});var V=l(P);T=n(V,"Trainer"),V.forEach(t),S=n(ie," API"),ie.forEach(t),K.forEach(t),I=c(F),D=r(F,"P",{});var ae=l(D);O=n(ae,"Fine-tuning a model for summarization is very similar to the other tasks we\u2019ve covered in this chapter. The first thing we need to do is load the pretrained model from the "),G=r(ae,"CODE",{});var de=l(G);J=n(de,"mt5-small"),de.forEach(t),X=n(ae," checkpoint. Since summarization is a sequence-to-sequence task, we can load the model with the "),Z=r(ae,"CODE",{});var _e=l(Z);B=n(_e,"AutoModelForSeq2SeqLM"),_e.forEach(t),Q=n(ae," class, which will automatically download and cache the weights:"),ae.forEach(t),L=c(F),y(M.$$.fragment,F),this.h()},h(){f(j,"id","finetuning-mt5-with-the-trainer-api"),f(j,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),f(j,"href","#finetuning-mt5-with-the-trainer-api"),f(u,"class","relative group")},m(F,K){i(F,u,K),s(u,j),s(j,d),k(E,d,null),s(u,C),s(u,x),s(x,A),s(x,P),s(P,T),s(x,S),i(F,I,K),i(F,D,K),s(D,O),s(D,G),s(G,J),s(D,X),s(D,Z),s(Z,B),s(D,Q),i(F,L,K),k(M,F,K),ee=!0},i(F){ee||(g(E.$$.fragment,F),g(M.$$.fragment,F),ee=!0)},o(F){_(E.$$.fragment,F),_(M.$$.fragment,F),ee=!1},d(F){F&&t(u),$(E),F&&t(I),F&&t(D),F&&t(L),$(M,F)}}}function Cg(Y){let u,j,d,E,C;return{c(){u=o("p"),j=a("\u{1F4A1} If you\u2019re wondering why you don\u2019t see any warnings about fine-tuning the model on a downstream task, that\u2019s because for sequence-to-sequence tasks we keep all the weights of the network. Compare this to our text classification model in "),d=o("a"),E=a("Chapter 3"),C=a(", where the head of the pretrained model was replaced with a randomly initialized network."),this.h()},l(x){u=r(x,"P",{});var A=l(u);j=n(A,"\u{1F4A1} If you\u2019re wondering why you don\u2019t see any warnings about fine-tuning the model on a downstream task, that\u2019s because for sequence-to-sequence tasks we keep all the weights of the network. Compare this to our text classification model in "),d=r(A,"A",{href:!0});var P=l(d);E=n(P,"Chapter 3"),P.forEach(t),C=n(A,", where the head of the pretrained model was replaced with a randomly initialized network."),A.forEach(t),this.h()},h(){f(d,"href","/course/chapter3")},m(x,A){i(x,u,A),s(u,j),s(u,d),s(d,E),s(u,C)},d(x){x&&t(u)}}}function ug(Y){let u,j,d,E,C,x,A,P,T,S,I,D,O,G,J,X,Z,B,Q,L,M,ee,F,K,te,W,ie,V,ae,de,_e,Te,H,ye,me,oe,we,ke,se,ge,he,fe,w,R,be,b,U,qe,pe,$e,Ce,re,ze,Qe,Ae,ne,le,Ie,ws,bs,vs,ys,zs,He,rs,es,ls,Ue,Ys,is,Oe,ks,Ds,Re,ps;return S=new z({props:{code:`from transformers import Seq2SeqTrainingArguments

batch_size = 8
num_train_epochs = 8
# Show the training loss with every epoch
logging_steps = len(tokenized_datasets["train"]) // batch_size
model_name = model_checkpoint.split("/")[-1]

args = Seq2SeqTrainingArguments(
    output_dir=f"{model_name}-finetuned-amazon-en-es",
    evaluation_strategy="epoch",
    learning_rate=5.6e-5,
    per_device_train_batch_size=batch_size,
    per_device_eval_batch_size=batch_size,
    weight_decay=0.01,
    save_total_limit=3,
    num_train_epochs=num_train_epochs,
    predict_with_generate=True,
    logging_steps=logging_steps,
    push_to_hub=True,
)`,highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> Seq2SeqTrainingArguments

batch_size = <span class="hljs-number">8</span>
num_train_epochs = <span class="hljs-number">8</span>
<span class="hljs-comment"># Show the training loss with every epoch</span>
logging_steps = <span class="hljs-built_in">len</span>(tokenized_datasets[<span class="hljs-string">&quot;train&quot;</span>]) // batch_size
model_name = model_checkpoint.split(<span class="hljs-string">&quot;/&quot;</span>)[-<span class="hljs-number">1</span>]

args = Seq2SeqTrainingArguments(
    output_dir=<span class="hljs-string">f&quot;<span class="hljs-subst">{model_name}</span>-finetuned-amazon-en-es&quot;</span>,
    evaluation_strategy=<span class="hljs-string">&quot;epoch&quot;</span>,
    learning_rate=<span class="hljs-number">5.6e-5</span>,
    per_device_train_batch_size=batch_size,
    per_device_eval_batch_size=batch_size,
    weight_decay=<span class="hljs-number">0.01</span>,
    save_total_limit=<span class="hljs-number">3</span>,
    num_train_epochs=num_train_epochs,
    predict_with_generate=<span class="hljs-literal">True</span>,
    logging_steps=logging_steps,
    push_to_hub=<span class="hljs-literal">True</span>,
)`}}),Re=new z({props:{code:`import numpy as np


def compute_metrics(eval_pred):
    predictions, labels = eval_pred
    # Decode generated summaries into text
    decoded_preds = tokenizer.batch_decode(predictions, skip_special_tokens=True)
    # Replace -100 in the labels as we can't decode them
    labels = np.where(labels != -100, labels, tokenizer.pad_token_id)
    # Decode reference summaries into text
    decoded_labels = tokenizer.batch_decode(labels, skip_special_tokens=True)
    # ROUGE expects a newline after each sentence
    decoded_preds = ["\\n".join(sent_tokenize(pred.strip())) for pred in decoded_preds]
    decoded_labels = ["\\n".join(sent_tokenize(label.strip())) for label in decoded_labels]
    # Compute ROUGE scores
    result = rouge_score.compute(
        predictions=decoded_preds, references=decoded_labels, use_stemmer=True
    )
    # Extract the median scores
    result = {key: value.mid.fmeasure * 100 for key, value in result.items()}
    return {k: round(v, 4) for k, v in result.items()}`,highlighted:`<span class="hljs-keyword">import</span> numpy <span class="hljs-keyword">as</span> np


<span class="hljs-keyword">def</span> <span class="hljs-title function_">compute_metrics</span>(<span class="hljs-params">eval_pred</span>):
    predictions, labels = eval_pred
    <span class="hljs-comment"># Decode generated summaries into text</span>
    decoded_preds = tokenizer.batch_decode(predictions, skip_special_tokens=<span class="hljs-literal">True</span>)
    <span class="hljs-comment"># Replace -100 in the labels as we can&#x27;t decode them</span>
    labels = np.where(labels != -<span class="hljs-number">100</span>, labels, tokenizer.pad_token_id)
    <span class="hljs-comment"># Decode reference summaries into text</span>
    decoded_labels = tokenizer.batch_decode(labels, skip_special_tokens=<span class="hljs-literal">True</span>)
    <span class="hljs-comment"># ROUGE expects a newline after each sentence</span>
    decoded_preds = [<span class="hljs-string">&quot;\\n&quot;</span>.join(sent_tokenize(pred.strip())) <span class="hljs-keyword">for</span> pred <span class="hljs-keyword">in</span> decoded_preds]
    decoded_labels = [<span class="hljs-string">&quot;\\n&quot;</span>.join(sent_tokenize(label.strip())) <span class="hljs-keyword">for</span> label <span class="hljs-keyword">in</span> decoded_labels]
    <span class="hljs-comment"># Compute ROUGE scores</span>
    result = rouge_score.compute(
        predictions=decoded_preds, references=decoded_labels, use_stemmer=<span class="hljs-literal">True</span>
    )
    <span class="hljs-comment"># Extract the median scores</span>
    result = {key: value.mid.fmeasure * <span class="hljs-number">100</span> <span class="hljs-keyword">for</span> key, value <span class="hljs-keyword">in</span> result.items()}
    <span class="hljs-keyword">return</span> {k: <span class="hljs-built_in">round</span>(v, <span class="hljs-number">4</span>) <span class="hljs-keyword">for</span> k, v <span class="hljs-keyword">in</span> result.items()}`}}),{c(){u=o("p"),j=a("We\u2019ll need to generate summaries in order to compute ROUGE scores during training. Fortunately, \u{1F917} Transformers provides dedicated "),d=o("code"),E=a("Seq2SeqTrainingArguments"),C=a(" and "),x=o("code"),A=a("Seq2SeqTrainer"),P=a(" classes that can do this for us automatically! To see how this works, let\u2019s first define the hyperparameters and other arguments for our experiments:"),T=h(),v(S.$$.fragment),I=h(),D=o("p"),O=a("Here, the "),G=o("code"),J=a("predict_with_generate"),X=a(" argument has been set to indicate that we should generate summaries during evaluation so that we can compute ROUGE scores for each epoch. As discussed in "),Z=o("a"),B=a("Chapter 1"),Q=a(", the decoder performs inference by predicting tokens one by one, and this is implemented by the model\u2019s "),L=o("code"),M=a("generate()"),ee=a(" method. Setting "),F=o("code"),K=a("predict_with_generate=True"),te=a(" tells the "),W=o("code"),ie=a("Seq2SeqTrainer"),V=a(" to use that method for evaluation. We\u2019ve also adjusted some of the default hyperparameters, like the learning rate, number of epochs, and weight decay, and we\u2019ve set the "),ae=o("code"),de=a("save_total_limit"),_e=a(" option to only save up to 3 checkpoints during training \u2014 this is because even the \u201Csmall\u201D version of mT5 uses around a GB of hard drive space, and we can save a bit of room by limiting the number of copies we save."),Te=h(),H=o("p"),ye=a("The "),me=o("code"),oe=a("push_to_hub=True"),we=a(" argument will allow us to push the model to the Hub after training; you\u2019ll find the repository under your user profile in the location defined by "),ke=o("code"),se=a("output_dir"),ge=a(". Note that you can specify the name of the repository you want to push to with the "),he=o("code"),fe=a("hub_model_id"),w=a(" argument (in particular, you will have to use this argument to push to an organization). For instance, when we pushed the model to the "),R=o("a"),be=o("code"),b=a("huggingface-course"),U=a(" organization"),qe=a(", we added "),pe=o("code"),$e=a('hub_model_id="huggingface-course/mt5-finetuned-amazon-en-es"'),Ce=a(" to "),re=o("code"),ze=a("Seq2SeqTrainingArguments"),Qe=a("."),Ae=h(),ne=o("p"),le=a("The next thing we need to do is provide the trainer with a "),Ie=o("code"),ws=a("compute_metrics()"),bs=a(" function so that we can evaluate our model during training. For summarization this is a bit more involved than simply calling "),vs=o("code"),ys=a("rouge_score.compute()"),zs=a(" on the model\u2019s predictions, since we need to "),He=o("em"),rs=a("decode"),es=a(" the outputs and labels into text before we can compute the ROUGE scores. The following function does exactly that, and also makes use of the "),ls=o("code"),Ue=a("sent_tokenize()"),Ys=a(" function from "),is=o("code"),Oe=a("nltk"),ks=a(" to separate the summary sentences with newlines:"),Ds=h(),v(Re.$$.fragment),this.h()},l(N){u=r(N,"P",{});var ce=l(u);j=n(ce,"We\u2019ll need to generate summaries in order to compute ROUGE scores during training. Fortunately, \u{1F917} Transformers provides dedicated "),d=r(ce,"CODE",{});var Me=l(d);E=n(Me,"Seq2SeqTrainingArguments"),Me.forEach(t),C=n(ce," and "),x=r(ce,"CODE",{});var Vs=l(x);A=n(Vs,"Seq2SeqTrainer"),Vs.forEach(t),P=n(ce," classes that can do this for us automatically! To see how this works, let\u2019s first define the hyperparameters and other arguments for our experiments:"),ce.forEach(t),T=c(N),y(S.$$.fragment,N),I=c(N),D=r(N,"P",{});var ue=l(D);O=n(ue,"Here, the "),G=r(ue,"CODE",{});var ms=l(G);J=n(ms,"predict_with_generate"),ms.forEach(t),X=n(ue," argument has been set to indicate that we should generate summaries during evaluation so that we can compute ROUGE scores for each epoch. As discussed in "),Z=r(ue,"A",{href:!0});var ss=l(Z);B=n(ss,"Chapter 1"),ss.forEach(t),Q=n(ue,", the decoder performs inference by predicting tokens one by one, and this is implemented by the model\u2019s "),L=r(ue,"CODE",{});var Ss=l(L);M=n(Ss,"generate()"),Ss.forEach(t),ee=n(ue," method. Setting "),F=r(ue,"CODE",{});var De=l(F);K=n(De,"predict_with_generate=True"),De.forEach(t),te=n(ue," tells the "),W=r(ue,"CODE",{});var Js=l(W);ie=n(Js,"Seq2SeqTrainer"),Js.forEach(t),V=n(ue," to use that method for evaluation. We\u2019ve also adjusted some of the default hyperparameters, like the learning rate, number of epochs, and weight decay, and we\u2019ve set the "),ae=r(ue,"CODE",{});var ts=l(ae);de=n(ts,"save_total_limit"),ts.forEach(t),_e=n(ue," option to only save up to 3 checkpoints during training \u2014 this is because even the \u201Csmall\u201D version of mT5 uses around a GB of hard drive space, and we can save a bit of room by limiting the number of copies we save."),ue.forEach(t),Te=c(N),H=r(N,"P",{});var je=l(H);ye=n(je,"The "),me=r(je,"CODE",{});var Xs=l(me);oe=n(Xs,"push_to_hub=True"),Xs.forEach(t),we=n(je," argument will allow us to push the model to the Hub after training; you\u2019ll find the repository under your user profile in the location defined by "),ke=r(je,"CODE",{});var We=l(ke);se=n(We,"output_dir"),We.forEach(t),ge=n(je,". Note that you can specify the name of the repository you want to push to with the "),he=r(je,"CODE",{});var Zs=l(he);fe=n(Zs,"hub_model_id"),Zs.forEach(t),w=n(je," argument (in particular, you will have to use this argument to push to an organization). For instance, when we pushed the model to the "),R=r(je,"A",{href:!0,rel:!0});var Fe=l(R);be=r(Fe,"CODE",{});var Qs=l(be);b=n(Qs,"huggingface-course"),Qs.forEach(t),U=n(Fe," organization"),Fe.forEach(t),qe=n(je,", we added "),pe=r(je,"CODE",{});var Be=l(pe);$e=n(Be,'hub_model_id="huggingface-course/mt5-finetuned-amazon-en-es"'),Be.forEach(t),Ce=n(je," to "),re=r(je,"CODE",{});var et=l(re);ze=n(et,"Seq2SeqTrainingArguments"),et.forEach(t),Qe=n(je,"."),je.forEach(t),Ae=c(N),ne=r(N,"P",{});var ve=l(ne);le=n(ve,"The next thing we need to do is provide the trainer with a "),Ie=r(ve,"CODE",{});var $s=l(Ie);ws=n($s,"compute_metrics()"),$s.forEach(t),bs=n(ve," function so that we can evaluate our model during training. For summarization this is a bit more involved than simply calling "),vs=r(ve,"CODE",{});var Se=l(vs);ys=n(Se,"rouge_score.compute()"),Se.forEach(t),zs=n(ve," on the model\u2019s predictions, since we need to "),He=r(ve,"EM",{});var vt=l(He);rs=n(vt,"decode"),vt.forEach(t),es=n(ve," the outputs and labels into text before we can compute the ROUGE scores. The following function does exactly that, and also makes use of the "),ls=r(ve,"CODE",{});var js=l(ls);Ue=n(js,"sent_tokenize()"),js.forEach(t),Ys=n(ve," function from "),is=r(ve,"CODE",{});var xs=l(is);Oe=n(xs,"nltk"),xs.forEach(t),ks=n(ve," to separate the summary sentences with newlines:"),ve.forEach(t),Ds=c(N),y(Re.$$.fragment,N),this.h()},h(){f(Z,"href","/course/chapter1"),f(R,"href","https://huggingface.co/huggingface-course"),f(R,"rel","nofollow")},m(N,ce){i(N,u,ce),s(u,j),s(u,d),s(d,E),s(u,C),s(u,x),s(x,A),s(u,P),i(N,T,ce),k(S,N,ce),i(N,I,ce),i(N,D,ce),s(D,O),s(D,G),s(G,J),s(D,X),s(D,Z),s(Z,B),s(D,Q),s(D,L),s(L,M),s(D,ee),s(D,F),s(F,K),s(D,te),s(D,W),s(W,ie),s(D,V),s(D,ae),s(ae,de),s(D,_e),i(N,Te,ce),i(N,H,ce),s(H,ye),s(H,me),s(me,oe),s(H,we),s(H,ke),s(ke,se),s(H,ge),s(H,he),s(he,fe),s(H,w),s(H,R),s(R,be),s(be,b),s(R,U),s(H,qe),s(H,pe),s(pe,$e),s(H,Ce),s(H,re),s(re,ze),s(H,Qe),i(N,Ae,ce),i(N,ne,ce),s(ne,le),s(ne,Ie),s(Ie,ws),s(ne,bs),s(ne,vs),s(vs,ys),s(ne,zs),s(ne,He),s(He,rs),s(ne,es),s(ne,ls),s(ls,Ue),s(ne,Ys),s(ne,is),s(is,Oe),s(ne,ks),i(N,Ds,ce),k(Re,N,ce),ps=!0},i(N){ps||(g(S.$$.fragment,N),g(Re.$$.fragment,N),ps=!0)},o(N){_(S.$$.fragment,N),_(Re.$$.fragment,N),ps=!1},d(N){N&&t(u),N&&t(T),$(S,N),N&&t(I),N&&t(D),N&&t(Te),N&&t(H),N&&t(Ae),N&&t(ne),N&&t(Ds),$(Re,N)}}}function Pg(Y){let u,j;return u=new z({props:{code:`from transformers import DataCollatorForSeq2Seq

data_collator = DataCollatorForSeq2Seq(tokenizer, model=model, return_tensors="tf")`,highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> DataCollatorForSeq2Seq

data_collator = DataCollatorForSeq2Seq(tokenizer, model=model, return_tensors=<span class="hljs-string">&quot;tf&quot;</span>)`}}),{c(){v(u.$$.fragment)},l(d){y(u.$$.fragment,d)},m(d,E){k(u,d,E),j=!0},i(d){j||(g(u.$$.fragment,d),j=!0)},o(d){_(u.$$.fragment,d),j=!1},d(d){$(u,d)}}}function Lg(Y){let u,j;return u=new z({props:{code:`from transformers import DataCollatorForSeq2Seq

data_collator = DataCollatorForSeq2Seq(tokenizer, model=model)`,highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> DataCollatorForSeq2Seq

data_collator = DataCollatorForSeq2Seq(tokenizer, model=model)`}}),{c(){v(u.$$.fragment)},l(d){y(u.$$.fragment,d)},m(d,E){k(u,d,E),j=!0},i(d){j||(g(u.$$.fragment,d),j=!0)},o(d){_(u.$$.fragment,d),j=!1},d(d){$(u,d)}}}function Ig(Y){let u,j,d,E,C,x,A,P,T,S,I,D,O,G,J,X,Z,B,Q,L,M,ee,F,K,te,W,ie,V,ae,de,_e,Te,H,ye,me,oe,we,ke,se,ge,he,fe;return O=new z({props:{code:`tf_train_dataset = tokenized_datasets["train"].to_tf_dataset(
    columns=["input_ids", "attention_mask", "labels"],
    collate_fn=data_collator,
    shuffle=True,
    batch_size=8,
)
tf_eval_dataset = tokenized_datasets["validation"].to_tf_dataset(
    columns=["input_ids", "attention_mask", "labels"],
    collate_fn=data_collator,
    shuffle=False,
    batch_size=8,
)`,highlighted:`tf_train_dataset = tokenized_datasets[<span class="hljs-string">&quot;train&quot;</span>].to_tf_dataset(
    columns=[<span class="hljs-string">&quot;input_ids&quot;</span>, <span class="hljs-string">&quot;attention_mask&quot;</span>, <span class="hljs-string">&quot;labels&quot;</span>],
    collate_fn=data_collator,
    shuffle=<span class="hljs-literal">True</span>,
    batch_size=<span class="hljs-number">8</span>,
)
tf_eval_dataset = tokenized_datasets[<span class="hljs-string">&quot;validation&quot;</span>].to_tf_dataset(
    columns=[<span class="hljs-string">&quot;input_ids&quot;</span>, <span class="hljs-string">&quot;attention_mask&quot;</span>, <span class="hljs-string">&quot;labels&quot;</span>],
    collate_fn=data_collator,
    shuffle=<span class="hljs-literal">False</span>,
    batch_size=<span class="hljs-number">8</span>,
)`}}),B=new z({props:{code:`from transformers import create_optimizer
import tensorflow as tf

# The number of training steps is the number of samples in the dataset, divided by the batch size then multiplied
# by the total number of epochs. Note that the tf_train_dataset here is a batched tf.data.Dataset,
# not the original Hugging Face Dataset, so its len() is already num_samples // batch_size.
num_train_epochs = 8
num_train_steps = len(tf_train_dataset) * num_train_epochs
model_name = model_checkpoint.split("/")[-1]

optimizer, schedule = create_optimizer(
    init_lr=5.6e-5,
    num_warmup_steps=0,
    num_train_steps=num_train_steps,
    weight_decay_rate=0.01,
)

model.compile(optimizer=optimizer)

# Train in mixed-precision float16
tf.keras.mixed_precision.set_global_policy("mixed_float16")`,highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> create_optimizer
<span class="hljs-keyword">import</span> tensorflow <span class="hljs-keyword">as</span> tf

<span class="hljs-comment"># The number of training steps is the number of samples in the dataset, divided by the batch size then multiplied</span>
<span class="hljs-comment"># by the total number of epochs. Note that the tf_train_dataset here is a batched tf.data.Dataset,</span>
<span class="hljs-comment"># not the original Hugging Face Dataset, so its len() is already num_samples // batch_size.</span>
num_train_epochs = <span class="hljs-number">8</span>
num_train_steps = <span class="hljs-built_in">len</span>(tf_train_dataset) * num_train_epochs
model_name = model_checkpoint.split(<span class="hljs-string">&quot;/&quot;</span>)[-<span class="hljs-number">1</span>]

optimizer, schedule = create_optimizer(
    init_lr=<span class="hljs-number">5.6e-5</span>,
    num_warmup_steps=<span class="hljs-number">0</span>,
    num_train_steps=num_train_steps,
    weight_decay_rate=<span class="hljs-number">0.01</span>,
)

model.<span class="hljs-built_in">compile</span>(optimizer=optimizer)

<span class="hljs-comment"># Train in mixed-precision float16</span>
tf.keras.mixed_precision.set_global_policy(<span class="hljs-string">&quot;mixed_float16&quot;</span>)`}}),W=new z({props:{code:`from transformers.keras_callbacks import PushToHubCallback

callback = PushToHubCallback(
    output_dir=f"{model_name}-finetuned-amazon-en-es", tokenizer=tokenizer
)

model.fit(
    tf_train_dataset, validation_data=tf_eval_dataset, callbacks=[callback], epochs=8
)`,highlighted:`<span class="hljs-keyword">from</span> transformers.keras_callbacks <span class="hljs-keyword">import</span> PushToHubCallback

callback = PushToHubCallback(
    output_dir=<span class="hljs-string">f&quot;<span class="hljs-subst">{model_name}</span>-finetuned-amazon-en-es&quot;</span>, tokenizer=tokenizer
)

model.fit(
    tf_train_dataset, validation_data=tf_eval_dataset, callbacks=[callback], epochs=<span class="hljs-number">8</span>
)`}}),ye=new z({props:{code:`from tqdm import tqdm
import numpy as np

all_preds = []
all_labels = []
for batch in tqdm(tf_eval_dataset):
    predictions = model.generate(**batch)
    decoded_preds = tokenizer.batch_decode(predictions, skip_special_tokens=True)
    labels = batch["labels"].numpy()
    labels = np.where(labels != -100, labels, tokenizer.pad_token_id)
    decoded_labels = tokenizer.batch_decode(labels, skip_special_tokens=True)
    decoded_preds = ["\\n".join(sent_tokenize(pred.strip())) for pred in decoded_preds]
    decoded_labels = ["\\n".join(sent_tokenize(label.strip())) for label in decoded_labels]
    all_preds.extend(decoded_preds)
    all_labels.extend(decoded_labels)`,highlighted:`<span class="hljs-keyword">from</span> tqdm <span class="hljs-keyword">import</span> tqdm
<span class="hljs-keyword">import</span> numpy <span class="hljs-keyword">as</span> np

all_preds = []
all_labels = []
<span class="hljs-keyword">for</span> batch <span class="hljs-keyword">in</span> tqdm(tf_eval_dataset):
    predictions = model.generate(**batch)
    decoded_preds = tokenizer.batch_decode(predictions, skip_special_tokens=<span class="hljs-literal">True</span>)
    labels = batch[<span class="hljs-string">&quot;labels&quot;</span>].numpy()
    labels = np.where(labels != -<span class="hljs-number">100</span>, labels, tokenizer.pad_token_id)
    decoded_labels = tokenizer.batch_decode(labels, skip_special_tokens=<span class="hljs-literal">True</span>)
    decoded_preds = [<span class="hljs-string">&quot;\\n&quot;</span>.join(sent_tokenize(pred.strip())) <span class="hljs-keyword">for</span> pred <span class="hljs-keyword">in</span> decoded_preds]
    decoded_labels = [<span class="hljs-string">&quot;\\n&quot;</span>.join(sent_tokenize(label.strip())) <span class="hljs-keyword">for</span> label <span class="hljs-keyword">in</span> decoded_labels]
    all_preds.extend(decoded_preds)
    all_labels.extend(decoded_labels)`}}),se=new z({props:{code:`result = rouge_score.compute(
    predictions=decoded_preds, references=decoded_labels, use_stemmer=True
)
result = {key: value.mid.fmeasure * 100 for key, value in result.items()}
{k: round(v, 4) for k, v in result.items()}`,highlighted:`result = rouge_score.compute(
    predictions=decoded_preds, references=decoded_labels, use_stemmer=<span class="hljs-literal">True</span>
)
result = {key: value.mid.fmeasure * <span class="hljs-number">100</span> <span class="hljs-keyword">for</span> key, value <span class="hljs-keyword">in</span> result.items()}
{k: <span class="hljs-built_in">round</span>(v, <span class="hljs-number">4</span>) <span class="hljs-keyword">for</span> k, v <span class="hljs-keyword">in</span> result.items()}`}}),he=new z({props:{code:"{'rouge1': 31.4815, 'rouge2': 25.4386, 'rougeL': 31.4815, 'rougeLsum': 31.4815}",highlighted:'{&#x27;rouge1&#x27;: <span class="hljs-number">31.4815</span>, &#x27;rouge2&#x27;: <span class="hljs-number">25.4386</span>, &#x27;rougeL&#x27;: <span class="hljs-number">31.4815</span>, &#x27;rougeLsum&#x27;: <span class="hljs-number">31.4815</span>}'}}),{c(){u=o("p"),j=a("We\u2019re almost ready to train! We just need to convert our datasets to "),d=o("code"),E=a("tf.data.Dataset"),C=a("s using the data collator we defined above, and then "),x=o("code"),A=a("compile()"),P=a(" and "),T=o("code"),S=a("fit()"),I=a(" the model. First, the datasets:"),D=h(),v(O.$$.fragment),G=h(),J=o("p"),X=a("Now, we define our training hyperparameters and compile:"),Z=h(),v(B.$$.fragment),Q=h(),L=o("p"),M=a("And finally, we fit the model. We use a "),ee=o("code"),F=a("PushToHubCallback"),K=a(" to save the model to the Hub after each epoch, which will allow us to use it for inference later:"),te=h(),v(W.$$.fragment),ie=h(),V=o("p"),ae=a("We got some loss values during training, but really we\u2019d like to see the ROUGE metrics we computed earlier. To get those metrics, we\u2019ll need to generate outputs from the model and convert them to strings. Let\u2019s build some lists of labels and predictions for the ROUGE metric to compare (note that if you get import errors for this section, you may need to"),de=o("code"),_e=a("!pip install tqdm"),Te=a("):"),H=h(),v(ye.$$.fragment),me=h(),oe=o("p"),we=a("Once we have our lists of label and prediction strings, computing the ROUGE score is easy:"),ke=h(),v(se.$$.fragment),ge=h(),v(he.$$.fragment)},l(w){u=r(w,"P",{});var R=l(u);j=n(R,"We\u2019re almost ready to train! We just need to convert our datasets to "),d=r(R,"CODE",{});var be=l(d);E=n(be,"tf.data.Dataset"),be.forEach(t),C=n(R,"s using the data collator we defined above, and then "),x=r(R,"CODE",{});var b=l(x);A=n(b,"compile()"),b.forEach(t),P=n(R," and "),T=r(R,"CODE",{});var U=l(T);S=n(U,"fit()"),U.forEach(t),I=n(R," the model. First, the datasets:"),R.forEach(t),D=c(w),y(O.$$.fragment,w),G=c(w),J=r(w,"P",{});var qe=l(J);X=n(qe,"Now, we define our training hyperparameters and compile:"),qe.forEach(t),Z=c(w),y(B.$$.fragment,w),Q=c(w),L=r(w,"P",{});var pe=l(L);M=n(pe,"And finally, we fit the model. We use a "),ee=r(pe,"CODE",{});var $e=l(ee);F=n($e,"PushToHubCallback"),$e.forEach(t),K=n(pe," to save the model to the Hub after each epoch, which will allow us to use it for inference later:"),pe.forEach(t),te=c(w),y(W.$$.fragment,w),ie=c(w),V=r(w,"P",{});var Ce=l(V);ae=n(Ce,"We got some loss values during training, but really we\u2019d like to see the ROUGE metrics we computed earlier. To get those metrics, we\u2019ll need to generate outputs from the model and convert them to strings. Let\u2019s build some lists of labels and predictions for the ROUGE metric to compare (note that if you get import errors for this section, you may need to"),de=r(Ce,"CODE",{});var re=l(de);_e=n(re,"!pip install tqdm"),re.forEach(t),Te=n(Ce,"):"),Ce.forEach(t),H=c(w),y(ye.$$.fragment,w),me=c(w),oe=r(w,"P",{});var ze=l(oe);we=n(ze,"Once we have our lists of label and prediction strings, computing the ROUGE score is easy:"),ze.forEach(t),ke=c(w),y(se.$$.fragment,w),ge=c(w),y(he.$$.fragment,w)},m(w,R){i(w,u,R),s(u,j),s(u,d),s(d,E),s(u,C),s(u,x),s(x,A),s(u,P),s(u,T),s(T,S),s(u,I),i(w,D,R),k(O,w,R),i(w,G,R),i(w,J,R),s(J,X),i(w,Z,R),k(B,w,R),i(w,Q,R),i(w,L,R),s(L,M),s(L,ee),s(ee,F),s(L,K),i(w,te,R),k(W,w,R),i(w,ie,R),i(w,V,R),s(V,ae),s(V,de),s(de,_e),s(V,Te),i(w,H,R),k(ye,w,R),i(w,me,R),i(w,oe,R),s(oe,we),i(w,ke,R),k(se,w,R),i(w,ge,R),k(he,w,R),fe=!0},i(w){fe||(g(O.$$.fragment,w),g(B.$$.fragment,w),g(W.$$.fragment,w),g(ye.$$.fragment,w),g(se.$$.fragment,w),g(he.$$.fragment,w),fe=!0)},o(w){_(O.$$.fragment,w),_(B.$$.fragment,w),_(W.$$.fragment,w),_(ye.$$.fragment,w),_(se.$$.fragment,w),_(he.$$.fragment,w),fe=!1},d(w){w&&t(u),w&&t(D),$(O,w),w&&t(G),w&&t(J),w&&t(Z),$(B,w),w&&t(Q),w&&t(L),w&&t(te),$(W,w),w&&t(ie),w&&t(V),w&&t(H),$(ye,w),w&&t(me),w&&t(oe),w&&t(ke),$(se,w),w&&t(ge),$(he,w)}}}function Rg(Y){let u,j,d,E,C,x,A,P,T,S,I,D,O,G,J,X,Z,B,Q,L,M,ee,F,K,te,W,ie,V,ae,de,_e,Te,H,ye,me,oe,we,ke,se,ge,he,fe,w,R,be;return E=new z({props:{code:`from transformers import Seq2SeqTrainer

trainer = Seq2SeqTrainer(
    model,
    args,
    train_dataset=tokenized_datasets["train"],
    eval_dataset=tokenized_datasets["validation"],
    data_collator=data_collator,
    tokenizer=tokenizer,
    compute_metrics=compute_metrics,
)`,highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> Seq2SeqTrainer

trainer = Seq2SeqTrainer(
    model,
    args,
    train_dataset=tokenized_datasets[<span class="hljs-string">&quot;train&quot;</span>],
    eval_dataset=tokenized_datasets[<span class="hljs-string">&quot;validation&quot;</span>],
    data_collator=data_collator,
    tokenizer=tokenizer,
    compute_metrics=compute_metrics,
)`}}),T=new z({props:{code:"trainer.train()",highlighted:"trainer.train()"}}),Z=new z({props:{code:"trainer.evaluate()",highlighted:"trainer.evaluate()"}}),Q=new z({props:{code:`{'eval_loss': 3.028524398803711,
 'eval_rouge1': 16.9728,
 'eval_rouge2': 8.2969,
 'eval_rougeL': 16.8366,
 'eval_rougeLsum': 16.851,
 'eval_gen_len': 10.1597,
 'eval_runtime': 6.1054,
 'eval_samples_per_second': 38.982,
 'eval_steps_per_second': 4.914}`,highlighted:`{<span class="hljs-string">&#x27;eval_loss&#x27;</span>: <span class="hljs-number">3.028524398803711</span>,
 <span class="hljs-string">&#x27;eval_rouge1&#x27;</span>: <span class="hljs-number">16.9728</span>,
 <span class="hljs-string">&#x27;eval_rouge2&#x27;</span>: <span class="hljs-number">8.2969</span>,
 <span class="hljs-string">&#x27;eval_rougeL&#x27;</span>: <span class="hljs-number">16.8366</span>,
 <span class="hljs-string">&#x27;eval_rougeLsum&#x27;</span>: <span class="hljs-number">16.851</span>,
 <span class="hljs-string">&#x27;eval_gen_len&#x27;</span>: <span class="hljs-number">10.1597</span>,
 <span class="hljs-string">&#x27;eval_runtime&#x27;</span>: <span class="hljs-number">6.1054</span>,
 <span class="hljs-string">&#x27;eval_samples_per_second&#x27;</span>: <span class="hljs-number">38.982</span>,
 <span class="hljs-string">&#x27;eval_steps_per_second&#x27;</span>: <span class="hljs-number">4.914</span>}`}}),K=new z({props:{code:'trainer.push_to_hub(commit_message="Training complete", tags="summarization")',highlighted:'trainer.push_to_hub(<span class="hljs-attribute">commit_message</span>=<span class="hljs-string">&quot;Training complete&quot;</span>, <span class="hljs-attribute">tags</span>=<span class="hljs-string">&quot;summarization&quot;</span>)'}}),W=new z({props:{code:"'https://huggingface.co/huggingface-course/mt5-finetuned-amazon-en-es/commit/aa0536b829b28e73e1e4b94b8a5aacec420d40e0'",highlighted:'<span class="hljs-string">&#x27;https://huggingface.co/huggingface-course/mt5-finetuned-amazon-en-es/commit/aa0536b829b28e73e1e4b94b8a5aacec420d40e0&#x27;</span>'}}),{c(){u=o("p"),j=a("We finally have all the ingredients we need to train with! We now simply need to instantiate the trainer with the standard arguments:"),d=h(),v(E.$$.fragment),C=h(),x=o("p"),A=a("and launch our training run:"),P=h(),v(T.$$.fragment),S=h(),I=o("p"),D=a("During training, you should see the training loss decrease and the ROUGE scores increase with each epoch. Once the training is complete, you can see the final ROUGE scores by running "),O=o("code"),G=a("Trainer.evaluate()"),J=a(":"),X=h(),v(Z.$$.fragment),B=h(),v(Q.$$.fragment),L=h(),M=o("p"),ee=a("From the scores we can see that our model has handily outperformed our lead-3 baseline \u2014 nice! The final thing to do is push the model weights to the Hub, as follows:"),F=h(),v(K.$$.fragment),te=h(),v(W.$$.fragment),ie=h(),V=o("p"),ae=a("This will save the checkpoint and configuration files to "),de=o("code"),_e=a("output_dir"),Te=a(", before uploading all the files to the Hub. By specifying the "),H=o("code"),ye=a("tags"),me=a(" argument, we also ensure that the widget on the Hub will be one for a summarization pipeline instead of the default text generation one associated with the mT5 architecture (for more information about model tags, see the "),oe=o("a"),we=a("\u{1F917} Hub documentation"),ke=a("). The output from "),se=o("code"),ge=a("trainer.push_to_hub()"),he=a(" is a URL to the Git commit hash, so you can easily see the changes that were made to the model repository!"),fe=h(),w=o("p"),R=a("To wrap up this section, let\u2019s take a look at how we can also fine-tune mT5 using the low-level features provided by \u{1F917} Accelerate."),this.h()},l(b){u=r(b,"P",{});var U=l(u);j=n(U,"We finally have all the ingredients we need to train with! We now simply need to instantiate the trainer with the standard arguments:"),U.forEach(t),d=c(b),y(E.$$.fragment,b),C=c(b),x=r(b,"P",{});var qe=l(x);A=n(qe,"and launch our training run:"),qe.forEach(t),P=c(b),y(T.$$.fragment,b),S=c(b),I=r(b,"P",{});var pe=l(I);D=n(pe,"During training, you should see the training loss decrease and the ROUGE scores increase with each epoch. Once the training is complete, you can see the final ROUGE scores by running "),O=r(pe,"CODE",{});var $e=l(O);G=n($e,"Trainer.evaluate()"),$e.forEach(t),J=n(pe,":"),pe.forEach(t),X=c(b),y(Z.$$.fragment,b),B=c(b),y(Q.$$.fragment,b),L=c(b),M=r(b,"P",{});var Ce=l(M);ee=n(Ce,"From the scores we can see that our model has handily outperformed our lead-3 baseline \u2014 nice! The final thing to do is push the model weights to the Hub, as follows:"),Ce.forEach(t),F=c(b),y(K.$$.fragment,b),te=c(b),y(W.$$.fragment,b),ie=c(b),V=r(b,"P",{});var re=l(V);ae=n(re,"This will save the checkpoint and configuration files to "),de=r(re,"CODE",{});var ze=l(de);_e=n(ze,"output_dir"),ze.forEach(t),Te=n(re,", before uploading all the files to the Hub. By specifying the "),H=r(re,"CODE",{});var Qe=l(H);ye=n(Qe,"tags"),Qe.forEach(t),me=n(re," argument, we also ensure that the widget on the Hub will be one for a summarization pipeline instead of the default text generation one associated with the mT5 architecture (for more information about model tags, see the "),oe=r(re,"A",{href:!0,rel:!0});var Ae=l(oe);we=n(Ae,"\u{1F917} Hub documentation"),Ae.forEach(t),ke=n(re,"). The output from "),se=r(re,"CODE",{});var ne=l(se);ge=n(ne,"trainer.push_to_hub()"),ne.forEach(t),he=n(re," is a URL to the Git commit hash, so you can easily see the changes that were made to the model repository!"),re.forEach(t),fe=c(b),w=r(b,"P",{});var le=l(w);R=n(le,"To wrap up this section, let\u2019s take a look at how we can also fine-tune mT5 using the low-level features provided by \u{1F917} Accelerate."),le.forEach(t),this.h()},h(){f(oe,"href","https://huggingface.co/docs/hub/main#how-is-a-models-type-of-inference-api-and-widget-determined"),f(oe,"rel","nofollow")},m(b,U){i(b,u,U),s(u,j),i(b,d,U),k(E,b,U),i(b,C,U),i(b,x,U),s(x,A),i(b,P,U),k(T,b,U),i(b,S,U),i(b,I,U),s(I,D),s(I,O),s(O,G),s(I,J),i(b,X,U),k(Z,b,U),i(b,B,U),k(Q,b,U),i(b,L,U),i(b,M,U),s(M,ee),i(b,F,U),k(K,b,U),i(b,te,U),k(W,b,U),i(b,ie,U),i(b,V,U),s(V,ae),s(V,de),s(de,_e),s(V,Te),s(V,H),s(H,ye),s(V,me),s(V,oe),s(oe,we),s(V,ke),s(V,se),s(se,ge),s(V,he),i(b,fe,U),i(b,w,U),s(w,R),be=!0},i(b){be||(g(E.$$.fragment,b),g(T.$$.fragment,b),g(Z.$$.fragment,b),g(Q.$$.fragment,b),g(K.$$.fragment,b),g(W.$$.fragment,b),be=!0)},o(b){_(E.$$.fragment,b),_(T.$$.fragment,b),_(Z.$$.fragment,b),_(Q.$$.fragment,b),_(K.$$.fragment,b),_(W.$$.fragment,b),be=!1},d(b){b&&t(u),b&&t(d),$(E,b),b&&t(C),b&&t(x),b&&t(P),$(T,b),b&&t(S),b&&t(I),b&&t(X),$(Z,b),b&&t(B),$(Q,b),b&&t(L),b&&t(M),b&&t(F),$(K,b),b&&t(te),$(W,b),b&&t(ie),b&&t(V),b&&t(fe),b&&t(w)}}}function dg(Y){let u,j,d,E,C,x,A,P,T,S,I,D,O,G,J,X,Z,B,Q,L,M,ee,F,K,te,W,ie,V,ae,de,_e,Te,H,ye,me,oe,we,ke,se,ge,he,fe,w,R,be,b,U,qe,pe,$e,Ce,re,ze,Qe,Ae,ne,le,Ie,ws,bs,vs,ys,zs,He,rs,es,ls,Ue,Ys,is,Oe,ks,Ds,Re,ps,N,ce,Me,Vs,ue,ms,ss,Ss,De,Js,ts,je,Xs,We,Zs,Fe,Qs,Be,et,ve,$s,Se,vt,js,xs,mn,yt,st,fa,hs,ga,kt,Ft,_a,tt,Gt,Ke,wa,xe,hn,Nt,ba,Es,cs,Ht,at,Ut,Mt,cn,$t,jt,un,xt,Ye,nt,va,Ts,ya,Et,dn,Tt,ot,ka,qs,$a,Ve,as,Wt,Bt,fn,Kt,Yt,gn,Vt,Jt,_n,ja,As,rt,lt,Xt,us,xa,Os,ds,Je,ko,Zt,it,$o,Qt;return E=new Ks({}),M=new Ks({}),me=new z({props:{code:'tokenized_datasets.set_format("torch")',highlighted:'tokenized_datasets.set_format(<span class="hljs-string">&quot;torch&quot;</span>)'}}),w=new z({props:{code:"model = AutoModelForSeq2SeqLM.from_pretrained(model_checkpoint)",highlighted:"model = AutoModelForSeq2SeqLM.from_pretrained(model_checkpoint)"}}),qe=new z({props:{code:`from torch.utils.data import DataLoader

batch_size = 8
train_dataloader = DataLoader(
    tokenized_datasets["train"],
    shuffle=True,
    collate_fn=data_collator,
    batch_size=batch_size,
)
eval_dataloader = DataLoader(
    tokenized_datasets["validation"], collate_fn=data_collator, batch_size=batch_size
)`,highlighted:`<span class="hljs-keyword">from</span> torch.utils.data <span class="hljs-keyword">import</span> DataLoader

batch_size = <span class="hljs-number">8</span>
train_dataloader = DataLoader(
    tokenized_datasets[<span class="hljs-string">&quot;train&quot;</span>],
    shuffle=<span class="hljs-literal">True</span>,
    collate_fn=data_collator,
    batch_size=batch_size,
)
eval_dataloader = DataLoader(
    tokenized_datasets[<span class="hljs-string">&quot;validation&quot;</span>], collate_fn=data_collator, batch_size=batch_size
)`}}),ne=new z({props:{code:`from torch.optim import AdamW

optimizer = AdamW(model.parameters(), lr=2e-5)`,highlighted:`<span class="hljs-keyword">from</span> torch.optim <span class="hljs-keyword">import</span> AdamW

optimizer = AdamW(model.parameters(), lr=<span class="hljs-number">2e-5</span>)`}}),He=new z({props:{code:`from accelerate import Accelerator

accelerator = Accelerator()
model, optimizer, train_dataloader, eval_dataloader = accelerator.prepare(
    model, optimizer, train_dataloader, eval_dataloader
)`,highlighted:`<span class="hljs-keyword">from</span> accelerate <span class="hljs-keyword">import</span> Accelerator

accelerator = Accelerator()
model, optimizer, train_dataloader, eval_dataloader = accelerator.prepare(
    model, optimizer, train_dataloader, eval_dataloader
)`}}),es=new pn({props:{$$slots:{default:[Fg]},$$scope:{ctx:Y}}}),De=new z({props:{code:`from transformers import get_scheduler

num_train_epochs = 10
num_update_steps_per_epoch = len(train_dataloader)
num_training_steps = num_train_epochs * num_update_steps_per_epoch

lr_scheduler = get_scheduler(
    "linear",
    optimizer=optimizer,
    num_warmup_steps=0,
    num_training_steps=num_training_steps,
)`,highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> get_scheduler

num_train_epochs = <span class="hljs-number">10</span>
num_update_steps_per_epoch = <span class="hljs-built_in">len</span>(train_dataloader)
num_training_steps = num_train_epochs * num_update_steps_per_epoch

lr_scheduler = get_scheduler(
    <span class="hljs-string">&quot;linear&quot;</span>,
    optimizer=optimizer,
    num_warmup_steps=<span class="hljs-number">0</span>,
    num_training_steps=num_training_steps,
)`}}),We=new z({props:{code:`def postprocess_text(preds, labels):
    preds = [pred.strip() for pred in preds]
    labels = [label.strip() for label in labels]

    # ROUGE expects a newline after each sentence
    preds = ["\\n".join(nltk.sent_tokenize(pred)) for pred in preds]
    labels = ["\\n".join(nltk.sent_tokenize(label)) for label in labels]

    return preds, labels`,highlighted:`<span class="hljs-keyword">def</span> <span class="hljs-title function_">postprocess_text</span>(<span class="hljs-params">preds, labels</span>):
    preds = [pred.strip() <span class="hljs-keyword">for</span> pred <span class="hljs-keyword">in</span> preds]
    labels = [label.strip() <span class="hljs-keyword">for</span> label <span class="hljs-keyword">in</span> labels]

    <span class="hljs-comment"># ROUGE expects a newline after each sentence</span>
    preds = [<span class="hljs-string">&quot;\\n&quot;</span>.join(nltk.sent_tokenize(pred)) <span class="hljs-keyword">for</span> pred <span class="hljs-keyword">in</span> preds]
    labels = [<span class="hljs-string">&quot;\\n&quot;</span>.join(nltk.sent_tokenize(label)) <span class="hljs-keyword">for</span> label <span class="hljs-keyword">in</span> labels]

    <span class="hljs-keyword">return</span> preds, labels`}}),st=new z({props:{code:`from huggingface_hub import get_full_repo_name

model_name = "test-bert-finetuned-squad-accelerate"
repo_name = get_full_repo_name(model_name)
repo_name`,highlighted:`<span class="hljs-keyword">from</span> huggingface_hub <span class="hljs-keyword">import</span> get_full_repo_name

model_name = <span class="hljs-string">&quot;test-bert-finetuned-squad-accelerate&quot;</span>
repo_name = get_full_repo_name(model_name)
repo_name`}}),hs=new z({props:{code:"'lewtun/mt5-finetuned-amazon-en-es-accelerate'",highlighted:'<span class="hljs-string">&#x27;lewtun/mt5-finetuned-amazon-en-es-accelerate&#x27;</span>'}}),tt=new z({props:{code:`from huggingface_hub import Repository

output_dir = "results-mt5-finetuned-squad-accelerate"
repo = Repository(output_dir, clone_from=repo_name)`,highlighted:`<span class="hljs-keyword">from</span> huggingface_hub <span class="hljs-keyword">import</span> Repository

output_dir = <span class="hljs-string">&quot;results-mt5-finetuned-squad-accelerate&quot;</span>
repo = Repository(output_dir, clone_from=repo_name)`}}),at=new Ks({}),us=new z({props:{code:`from tqdm.auto import tqdm
import torch
import numpy as np

progress_bar = tqdm(range(num_training_steps))

for epoch in range(num_train_epochs):
    # Training
    model.train()
    for step, batch in enumerate(train_dataloader):
        outputs = model(**batch)
        loss = outputs.loss
        accelerator.backward(loss)

        optimizer.step()
        lr_scheduler.step()
        optimizer.zero_grad()
        progress_bar.update(1)

    # Evaluation
    model.eval()
    for step, batch in enumerate(eval_dataloader):
        with torch.no_grad():
            generated_tokens = accelerator.unwrap_model(model).generate(
                batch["input_ids"],
                attention_mask=batch["attention_mask"],
            )

            generated_tokens = accelerator.pad_across_processes(
                generated_tokens, dim=1, pad_index=tokenizer.pad_token_id
            )
            labels = batch["labels"]

            # If we did not pad to max length, we need to pad the labels too
            labels = accelerator.pad_across_processes(
                batch["labels"], dim=1, pad_index=tokenizer.pad_token_id
            )

            generated_tokens = accelerator.gather(generated_tokens).cpu().numpy()
            labels = accelerator.gather(labels).cpu().numpy()

            # Replace -100 in the labels as we can't decode them
            labels = np.where(labels != -100, labels, tokenizer.pad_token_id)
            if isinstance(generated_tokens, tuple):
                generated_tokens = generated_tokens[0]
            decoded_preds = tokenizer.batch_decode(
                generated_tokens, skip_special_tokens=True
            )
            decoded_labels = tokenizer.batch_decode(labels, skip_special_tokens=True)

            decoded_preds, decoded_labels = postprocess_text(
                decoded_preds, decoded_labels
            )

            rouge_score.add_batch(predictions=decoded_preds, references=decoded_labels)

    # Compute metrics
    result = rouge_score.compute()
    # Extract the median ROUGE scores
    result = {key: value.mid.fmeasure * 100 for key, value in result.items()}
    result = {k: round(v, 4) for k, v in result.items()}
    print(f"Epoch {epoch}:", result)

    # Save and upload
    accelerator.wait_for_everyone()
    unwrapped_model = accelerator.unwrap_model(model)
    unwrapped_model.save_pretrained(output_dir, save_function=accelerator.save)
    if accelerator.is_main_process:
        tokenizer.save_pretrained(output_dir)
        repo.push_to_hub(
            commit_message=f"Training in progress epoch {epoch}", blocking=False
        )`,highlighted:`<span class="hljs-keyword">from</span> tqdm.auto <span class="hljs-keyword">import</span> tqdm
<span class="hljs-keyword">import</span> torch
<span class="hljs-keyword">import</span> numpy <span class="hljs-keyword">as</span> np

progress_bar = tqdm(<span class="hljs-built_in">range</span>(num_training_steps))

<span class="hljs-keyword">for</span> epoch <span class="hljs-keyword">in</span> <span class="hljs-built_in">range</span>(num_train_epochs):
    <span class="hljs-comment"># Training</span>
    model.train()
    <span class="hljs-keyword">for</span> step, batch <span class="hljs-keyword">in</span> <span class="hljs-built_in">enumerate</span>(train_dataloader):
        outputs = model(**batch)
        loss = outputs.loss
        accelerator.backward(loss)

        optimizer.step()
        lr_scheduler.step()
        optimizer.zero_grad()
        progress_bar.update(<span class="hljs-number">1</span>)

    <span class="hljs-comment"># Evaluation</span>
    model.<span class="hljs-built_in">eval</span>()
    <span class="hljs-keyword">for</span> step, batch <span class="hljs-keyword">in</span> <span class="hljs-built_in">enumerate</span>(eval_dataloader):
        <span class="hljs-keyword">with</span> torch.no_grad():
            generated_tokens = accelerator.unwrap_model(model).generate(
                batch[<span class="hljs-string">&quot;input_ids&quot;</span>],
                attention_mask=batch[<span class="hljs-string">&quot;attention_mask&quot;</span>],
            )

            generated_tokens = accelerator.pad_across_processes(
                generated_tokens, dim=<span class="hljs-number">1</span>, pad_index=tokenizer.pad_token_id
            )
            labels = batch[<span class="hljs-string">&quot;labels&quot;</span>]

            <span class="hljs-comment"># If we did not pad to max length, we need to pad the labels too</span>
            labels = accelerator.pad_across_processes(
                batch[<span class="hljs-string">&quot;labels&quot;</span>], dim=<span class="hljs-number">1</span>, pad_index=tokenizer.pad_token_id
            )

            generated_tokens = accelerator.gather(generated_tokens).cpu().numpy()
            labels = accelerator.gather(labels).cpu().numpy()

            <span class="hljs-comment"># Replace -100 in the labels as we can&#x27;t decode them</span>
            labels = np.where(labels != -<span class="hljs-number">100</span>, labels, tokenizer.pad_token_id)
            <span class="hljs-keyword">if</span> <span class="hljs-built_in">isinstance</span>(generated_tokens, <span class="hljs-built_in">tuple</span>):
                generated_tokens = generated_tokens[<span class="hljs-number">0</span>]
            decoded_preds = tokenizer.batch_decode(
                generated_tokens, skip_special_tokens=<span class="hljs-literal">True</span>
            )
            decoded_labels = tokenizer.batch_decode(labels, skip_special_tokens=<span class="hljs-literal">True</span>)

            decoded_preds, decoded_labels = postprocess_text(
                decoded_preds, decoded_labels
            )

            rouge_score.add_batch(predictions=decoded_preds, references=decoded_labels)

    <span class="hljs-comment"># Compute metrics</span>
    result = rouge_score.compute()
    <span class="hljs-comment"># Extract the median ROUGE scores</span>
    result = {key: value.mid.fmeasure * <span class="hljs-number">100</span> <span class="hljs-keyword">for</span> key, value <span class="hljs-keyword">in</span> result.items()}
    result = {k: <span class="hljs-built_in">round</span>(v, <span class="hljs-number">4</span>) <span class="hljs-keyword">for</span> k, v <span class="hljs-keyword">in</span> result.items()}
    <span class="hljs-built_in">print</span>(<span class="hljs-string">f&quot;Epoch <span class="hljs-subst">{epoch}</span>:&quot;</span>, result)

    <span class="hljs-comment"># Save and upload</span>
    accelerator.wait_for_everyone()
    unwrapped_model = accelerator.unwrap_model(model)
    unwrapped_model.save_pretrained(output_dir, save_function=accelerator.save)
    <span class="hljs-keyword">if</span> accelerator.is_main_process:
        tokenizer.save_pretrained(output_dir)
        repo.push_to_hub(
            commit_message=<span class="hljs-string">f&quot;Training in progress epoch <span class="hljs-subst">{epoch}</span>&quot;</span>, blocking=<span class="hljs-literal">False</span>
        )`}}),Os=new z({props:{code:`Epoch 0: {'rouge1': 5.6351, 'rouge2': 1.1625, 'rougeL': 5.4866, 'rougeLsum': 5.5005}
Epoch 1: {'rouge1': 9.8646, 'rouge2': 3.4106, 'rougeL': 9.9439, 'rougeLsum': 9.9306}
Epoch 2: {'rouge1': 11.0872, 'rouge2': 3.3273, 'rougeL': 11.0508, 'rougeLsum': 10.9468}
Epoch 3: {'rouge1': 11.8587, 'rouge2': 4.8167, 'rougeL': 11.7986, 'rougeLsum': 11.7518}
Epoch 4: {'rouge1': 12.9842, 'rouge2': 5.5887, 'rougeL': 12.7546, 'rougeLsum': 12.7029}
Epoch 5: {'rouge1': 13.4628, 'rouge2': 6.4598, 'rougeL': 13.312, 'rougeLsum': 13.2913}
Epoch 6: {'rouge1': 12.9131, 'rouge2': 5.8914, 'rougeL': 12.6896, 'rougeLsum': 12.5701}
Epoch 7: {'rouge1': 13.3079, 'rouge2': 6.2994, 'rougeL': 13.1536, 'rougeLsum': 13.1194}
Epoch 8: {'rouge1': 13.96, 'rouge2': 6.5998, 'rougeL': 13.9123, 'rougeLsum': 13.7744}
Epoch 9: {'rouge1': 14.1192, 'rouge2': 7.0059, 'rougeL': 14.1172, 'rougeLsum': 13.9509}`,highlighted:`Epoch <span class="hljs-number">0</span>: {<span class="hljs-string">&#x27;rouge1&#x27;</span>: <span class="hljs-number">5.6351</span>, <span class="hljs-string">&#x27;rouge2&#x27;</span>: <span class="hljs-number">1.1625</span>, <span class="hljs-string">&#x27;rougeL&#x27;</span>: <span class="hljs-number">5.4866</span>, <span class="hljs-string">&#x27;rougeLsum&#x27;</span>: <span class="hljs-number">5.5005</span>}
Epoch <span class="hljs-number">1</span>: {<span class="hljs-string">&#x27;rouge1&#x27;</span>: <span class="hljs-number">9.8646</span>, <span class="hljs-string">&#x27;rouge2&#x27;</span>: <span class="hljs-number">3.4106</span>, <span class="hljs-string">&#x27;rougeL&#x27;</span>: <span class="hljs-number">9.9439</span>, <span class="hljs-string">&#x27;rougeLsum&#x27;</span>: <span class="hljs-number">9.9306</span>}
Epoch <span class="hljs-number">2</span>: {<span class="hljs-string">&#x27;rouge1&#x27;</span>: <span class="hljs-number">11.0872</span>, <span class="hljs-string">&#x27;rouge2&#x27;</span>: <span class="hljs-number">3.3273</span>, <span class="hljs-string">&#x27;rougeL&#x27;</span>: <span class="hljs-number">11.0508</span>, <span class="hljs-string">&#x27;rougeLsum&#x27;</span>: <span class="hljs-number">10.9468</span>}
Epoch <span class="hljs-number">3</span>: {<span class="hljs-string">&#x27;rouge1&#x27;</span>: <span class="hljs-number">11.8587</span>, <span class="hljs-string">&#x27;rouge2&#x27;</span>: <span class="hljs-number">4.8167</span>, <span class="hljs-string">&#x27;rougeL&#x27;</span>: <span class="hljs-number">11.7986</span>, <span class="hljs-string">&#x27;rougeLsum&#x27;</span>: <span class="hljs-number">11.7518</span>}
Epoch <span class="hljs-number">4</span>: {<span class="hljs-string">&#x27;rouge1&#x27;</span>: <span class="hljs-number">12.9842</span>, <span class="hljs-string">&#x27;rouge2&#x27;</span>: <span class="hljs-number">5.5887</span>, <span class="hljs-string">&#x27;rougeL&#x27;</span>: <span class="hljs-number">12.7546</span>, <span class="hljs-string">&#x27;rougeLsum&#x27;</span>: <span class="hljs-number">12.7029</span>}
Epoch <span class="hljs-number">5</span>: {<span class="hljs-string">&#x27;rouge1&#x27;</span>: <span class="hljs-number">13.4628</span>, <span class="hljs-string">&#x27;rouge2&#x27;</span>: <span class="hljs-number">6.4598</span>, <span class="hljs-string">&#x27;rougeL&#x27;</span>: <span class="hljs-number">13.312</span>, <span class="hljs-string">&#x27;rougeLsum&#x27;</span>: <span class="hljs-number">13.2913</span>}
Epoch <span class="hljs-number">6</span>: {<span class="hljs-string">&#x27;rouge1&#x27;</span>: <span class="hljs-number">12.9131</span>, <span class="hljs-string">&#x27;rouge2&#x27;</span>: <span class="hljs-number">5.8914</span>, <span class="hljs-string">&#x27;rougeL&#x27;</span>: <span class="hljs-number">12.6896</span>, <span class="hljs-string">&#x27;rougeLsum&#x27;</span>: <span class="hljs-number">12.5701</span>}
Epoch <span class="hljs-number">7</span>: {<span class="hljs-string">&#x27;rouge1&#x27;</span>: <span class="hljs-number">13.3079</span>, <span class="hljs-string">&#x27;rouge2&#x27;</span>: <span class="hljs-number">6.2994</span>, <span class="hljs-string">&#x27;rougeL&#x27;</span>: <span class="hljs-number">13.1536</span>, <span class="hljs-string">&#x27;rougeLsum&#x27;</span>: <span class="hljs-number">13.1194</span>}
Epoch <span class="hljs-number">8</span>: {<span class="hljs-string">&#x27;rouge1&#x27;</span>: <span class="hljs-number">13.96</span>, <span class="hljs-string">&#x27;rouge2&#x27;</span>: <span class="hljs-number">6.5998</span>, <span class="hljs-string">&#x27;rougeL&#x27;</span>: <span class="hljs-number">13.9123</span>, <span class="hljs-string">&#x27;rougeLsum&#x27;</span>: <span class="hljs-number">13.7744</span>}
Epoch <span class="hljs-number">9</span>: {<span class="hljs-string">&#x27;rouge1&#x27;</span>: <span class="hljs-number">14.1192</span>, <span class="hljs-string">&#x27;rouge2&#x27;</span>: <span class="hljs-number">7.0059</span>, <span class="hljs-string">&#x27;rougeL&#x27;</span>: <span class="hljs-number">14.1172</span>, <span class="hljs-string">&#x27;rougeLsum&#x27;</span>: <span class="hljs-number">13.9509</span>}`}}),{c(){u=o("h2"),j=o("a"),d=o("span"),v(E.$$.fragment),C=h(),x=o("span"),A=a("Fine-tuning mT5 with \u{1F917} Accelerate"),P=h(),T=o("p"),S=a("Fine-tuning our model with \u{1F917} Accelerate is very similar to the text classification example we encountered in "),I=o("a"),D=a("Chapter 3"),O=a(". The main differences will be the need to explicitly generate our summaries during training and define how we compute the ROUGE scores (recall that the "),G=o("code"),J=a("Seq2SeqTrainer"),X=a(" took care of the generation for us). Let\u2019s take a look how we can implement these two requirements within \u{1F917} Accelerate!"),Z=h(),B=o("h3"),Q=o("a"),L=o("span"),v(M.$$.fragment),ee=h(),F=o("span"),K=a("Preparing everything for training"),te=h(),W=o("p"),ie=a("The first thing we need to do is create a "),V=o("code"),ae=a("DataLoader"),de=a(" for each of our splits. Since the PyTorch dataloaders expect batches of tensors, we need to set the format to "),_e=o("code"),Te=a('"torch"'),H=a(" in our datasets:"),ye=h(),v(me.$$.fragment),oe=h(),we=o("p"),ke=a("Now that we\u2019ve got datasets consisting of just tensors, the next thing to do is instantiate the "),se=o("code"),ge=a("DataCollatorForSeq2Seq"),he=a(" again. For this we need to provide a fresh version of the model, so let\u2019s load it again from our cache:"),fe=h(),v(w.$$.fragment),R=h(),be=o("p"),b=a("We can then instantiate the data collator and use this to define our dataloaders:"),U=h(),v(qe.$$.fragment),pe=h(),$e=o("p"),Ce=a("The next thing to do is define the optimizer we want to use. As in our other examples, we\u2019ll use "),re=o("code"),ze=a("AdamW"),Qe=a(", which works well for most problems:"),Ae=h(),v(ne.$$.fragment),le=h(),Ie=o("p"),ws=a("Finally, we feed our model, optimizer, and dataloaders to the "),bs=o("code"),vs=a("accelerator.prepare()"),ys=a(" method:"),zs=h(),v(He.$$.fragment),rs=h(),v(es.$$.fragment),ls=h(),Ue=o("p"),Ys=a("Now that we\u2019ve prepared our objects, there are three remaining things to do:"),is=h(),Oe=o("ul"),ks=o("li"),Ds=a("Define the learning rate schedule."),Re=h(),ps=o("li"),N=a("Implement a function to post-process the summaries for evaluation."),ce=h(),Me=o("li"),Vs=a("Create a repository on the Hub that we can push our model to."),ue=h(),ms=o("p"),ss=a("For the learning rate schedule, we\u2019ll use the standard linear one from previous sections:"),Ss=h(),v(De.$$.fragment),Js=h(),ts=o("p"),je=a("For post-processing, we need a function that splits the generated summaries into sentences that are separated by newlines. This is the format the ROUGE metric expects, and we can achieve this with the following snippet of code:"),Xs=h(),v(We.$$.fragment),Zs=h(),Fe=o("p"),Qs=a("This should look familiar to you if you recall how we defined the "),Be=o("code"),et=a("compute_metrics()"),ve=a(" function of the "),$s=o("code"),Se=a("Seq2SeqTrainer"),vt=a("."),js=h(),xs=o("p"),mn=a("Finally, we need to create a model repository on the Hugging Face Hub. For this, we can use the appropriately titled \u{1F917} Hub library. We just need to define a name for our repository, and the library has a utility function to combine the repository ID with the user profile:"),yt=h(),v(st.$$.fragment),fa=h(),v(hs.$$.fragment),ga=h(),kt=o("p"),Ft=a("Now we can use this repository name to clone a local version to our results directory that will store the training artifacts:"),_a=h(),v(tt.$$.fragment),Gt=h(),Ke=o("p"),wa=a("This will allow us to push the artifacts back to the Hub by calling the "),xe=o("code"),hn=a("repo.push_to_hub()"),Nt=a(" method during training! Let\u2019s now wrap up our analysis by writing out the training loop."),ba=h(),Es=o("h3"),cs=o("a"),Ht=o("span"),v(at.$$.fragment),Ut=h(),Mt=o("span"),cn=a("Training loop"),$t=h(),jt=o("p"),un=a("The training loop for summarization is quite similar to the other \u{1F917} Accelerate examples that we\u2019ve encountered and is roughly split into four main steps:"),xt=h(),Ye=o("ol"),nt=o("li"),va=a("Train the model by iterating over all the examples in "),Ts=o("code"),ya=a("train_dataloader"),Et=a(" for each epoch."),dn=h(),Tt=o("li"),ot=a("Generate model summaries at the end of each epoch, by first generating the tokens and then decoding them (and the reference summaries) into text."),ka=h(),qs=o("li"),$a=a("Compute the ROUGE scores using the same techniques we saw earlier."),Ve=h(),as=o("li"),Wt=a("Save the checkpoints and push everything to the Hub. Here we rely on the nifty "),Bt=o("code"),fn=a("blocking=False"),Kt=a(" argument of the "),Yt=o("code"),gn=a("Repository"),Vt=a(" object so that we can push the checkpoints per epoch "),Jt=o("em"),_n=a("asynchronously"),ja=a(". This allows us to continue training without having to wait for the somewhat slow upload associated with a GB-sized model!"),As=h(),rt=o("p"),lt=a("These steps can be seen in the following block of code:"),Xt=h(),v(us.$$.fragment),xa=h(),v(Os.$$.fragment),ds=h(),Je=o("p"),ko=a("And that\u2019s it! Once you run this, you\u2019ll have a model and results that are pretty similar to the ones we obtained with the "),Zt=o("code"),it=a("Trainer"),$o=a("."),this.h()},l(m){u=r(m,"H2",{class:!0});var q=l(u);j=r(q,"A",{id:!0,class:!0,href:!0});var Ea=l(j);d=r(Ea,"SPAN",{});var jo=l(d);y(E.$$.fragment,jo),jo.forEach(t),Ea.forEach(t),C=c(q),x=r(q,"SPAN",{});var xo=l(x);A=n(xo,"Fine-tuning mT5 with \u{1F917} Accelerate"),xo.forEach(t),q.forEach(t),P=c(m),T=r(m,"P",{});var pt=l(T);S=n(pt,"Fine-tuning our model with \u{1F917} Accelerate is very similar to the text classification example we encountered in "),I=r(pt,"A",{href:!0});var qt=l(I);D=n(qt,"Chapter 3"),qt.forEach(t),O=n(pt,". The main differences will be the need to explicitly generate our summaries during training and define how we compute the ROUGE scores (recall that the "),G=r(pt,"CODE",{});var wn=l(G);J=n(wn,"Seq2SeqTrainer"),wn.forEach(t),X=n(pt," took care of the generation for us). Let\u2019s take a look how we can implement these two requirements within \u{1F917} Accelerate!"),pt.forEach(t),Z=c(m),B=r(m,"H3",{class:!0});var mt=l(B);Q=r(mt,"A",{id:!0,class:!0,href:!0});var Eo=l(Q);L=r(Eo,"SPAN",{});var bn=l(L);y(M.$$.fragment,bn),bn.forEach(t),Eo.forEach(t),ee=c(mt),F=r(mt,"SPAN",{});var Cs=l(F);K=n(Cs,"Preparing everything for training"),Cs.forEach(t),mt.forEach(t),te=c(m),W=r(m,"P",{});var ns=l(W);ie=n(ns,"The first thing we need to do is create a "),V=r(ns,"CODE",{});var Ta=l(V);ae=n(Ta,"DataLoader"),Ta.forEach(t),de=n(ns," for each of our splits. Since the PyTorch dataloaders expect batches of tensors, we need to set the format to "),_e=r(ns,"CODE",{});var zt=l(_e);Te=n(zt,'"torch"'),zt.forEach(t),H=n(ns," in our datasets:"),ns.forEach(t),ye=c(m),y(me.$$.fragment,m),oe=c(m),we=r(m,"P",{});var qa=l(we);ke=n(qa,"Now that we\u2019ve got datasets consisting of just tensors, the next thing to do is instantiate the "),se=r(qa,"CODE",{});var za=l(se);ge=n(za,"DataCollatorForSeq2Seq"),za.forEach(t),he=n(qa," again. For this we need to provide a fresh version of the model, so let\u2019s load it again from our cache:"),qa.forEach(t),fe=c(m),y(w.$$.fragment,m),R=c(m),be=r(m,"P",{});var To=l(be);b=n(To,"We can then instantiate the data collator and use this to define our dataloaders:"),To.forEach(t),U=c(m),y(qe.$$.fragment,m),pe=c(m),$e=r(m,"P",{});var ea=l($e);Ce=n(ea,"The next thing to do is define the optimizer we want to use. As in our other examples, we\u2019ll use "),re=r(ea,"CODE",{});var ht=l(re);ze=n(ht,"AdamW"),ht.forEach(t),Qe=n(ea,", which works well for most problems:"),ea.forEach(t),Ae=c(m),y(ne.$$.fragment,m),le=c(m),Ie=r(m,"P",{});var Da=l(Ie);ws=n(Da,"Finally, we feed our model, optimizer, and dataloaders to the "),bs=r(Da,"CODE",{});var sa=l(bs);vs=n(sa,"accelerator.prepare()"),sa.forEach(t),ys=n(Da," method:"),Da.forEach(t),zs=c(m),y(He.$$.fragment,m),rs=c(m),y(es.$$.fragment,m),ls=c(m),Ue=r(m,"P",{});var qo=l(Ue);Ys=n(qo,"Now that we\u2019ve prepared our objects, there are three remaining things to do:"),qo.forEach(t),is=c(m),Oe=r(m,"UL",{});var Dt=l(Oe);ks=r(Dt,"LI",{});var vn=l(ks);Ds=n(vn,"Define the learning rate schedule."),vn.forEach(t),Re=c(Dt),ps=r(Dt,"LI",{});var ct=l(ps);N=n(ct,"Implement a function to post-process the summaries for evaluation."),ct.forEach(t),ce=c(Dt),Me=r(Dt,"LI",{});var Sa=l(Me);Vs=n(Sa,"Create a repository on the Hub that we can push our model to."),Sa.forEach(t),Dt.forEach(t),ue=c(m),ms=r(m,"P",{});var Ps=l(ms);ss=n(Ps,"For the learning rate schedule, we\u2019ll use the standard linear one from previous sections:"),Ps.forEach(t),Ss=c(m),y(De.$$.fragment,m),Js=c(m),ts=r(m,"P",{});var ta=l(ts);je=n(ta,"For post-processing, we need a function that splits the generated summaries into sentences that are separated by newlines. This is the format the ROUGE metric expects, and we can achieve this with the following snippet of code:"),ta.forEach(t),Xs=c(m),y(We.$$.fragment,m),Zs=c(m),Fe=r(m,"P",{});var St=l(Fe);Qs=n(St,"This should look familiar to you if you recall how we defined the "),Be=r(St,"CODE",{});var zo=l(Be);et=n(zo,"compute_metrics()"),zo.forEach(t),ve=n(St," function of the "),$s=r(St,"CODE",{});var Aa=l($s);Se=n(Aa,"Seq2SeqTrainer"),Aa.forEach(t),vt=n(St,"."),St.forEach(t),js=c(m),xs=r(m,"P",{});var Do=l(xs);mn=n(Do,"Finally, we need to create a model repository on the Hugging Face Hub. For this, we can use the appropriately titled \u{1F917} Hub library. We just need to define a name for our repository, and the library has a utility function to combine the repository ID with the user profile:"),Do.forEach(t),yt=c(m),y(st.$$.fragment,m),fa=c(m),y(hs.$$.fragment,m),ga=c(m),kt=r(m,"P",{});var So=l(kt);Ft=n(So,"Now we can use this repository name to clone a local version to our results directory that will store the training artifacts:"),So.forEach(t),_a=c(m),y(tt.$$.fragment,m),Gt=c(m),Ke=r(m,"P",{});var ut=l(Ke);wa=n(ut,"This will allow us to push the artifacts back to the Hub by calling the "),xe=r(ut,"CODE",{});var Ao=l(xe);hn=n(Ao,"repo.push_to_hub()"),Ao.forEach(t),Nt=n(ut," method during training! Let\u2019s now wrap up our analysis by writing out the training loop."),ut.forEach(t),ba=c(m),Es=r(m,"H3",{class:!0});var Oa=l(Es);cs=r(Oa,"A",{id:!0,class:!0,href:!0});var Ge=l(cs);Ht=r(Ge,"SPAN",{});var Ls=l(Ht);y(at.$$.fragment,Ls),Ls.forEach(t),Ge.forEach(t),Ut=c(Oa),Mt=r(Oa,"SPAN",{});var aa=l(Mt);cn=n(aa,"Training loop"),aa.forEach(t),Oa.forEach(t),$t=c(m),jt=r(m,"P",{});var At=l(jt);un=n(At,"The training loop for summarization is quite similar to the other \u{1F917} Accelerate examples that we\u2019ve encountered and is roughly split into four main steps:"),At.forEach(t),xt=c(m),Ye=r(m,"OL",{});var Is=l(Ye);nt=r(Is,"LI",{});var Ca=l(nt);va=n(Ca,"Train the model by iterating over all the examples in "),Ts=r(Ca,"CODE",{});var Pa=l(Ts);ya=n(Pa,"train_dataloader"),Pa.forEach(t),Et=n(Ca," for each epoch."),Ca.forEach(t),dn=c(Is),Tt=r(Is,"LI",{});var Oo=l(Tt);ot=n(Oo,"Generate model summaries at the end of each epoch, by first generating the tokens and then decoding them (and the reference summaries) into text."),Oo.forEach(t),ka=c(Is),qs=r(Is,"LI",{});var Co=l(qs);$a=n(Co,"Compute the ROUGE scores using the same techniques we saw earlier."),Co.forEach(t),Ve=c(Is),as=r(Is,"LI",{});var os=l(as);Wt=n(os,"Save the checkpoints and push everything to the Hub. Here we rely on the nifty "),Bt=r(os,"CODE",{});var Po=l(Bt);fn=n(Po,"blocking=False"),Po.forEach(t),Kt=n(os," argument of the "),Yt=r(os,"CODE",{});var Lo=l(Yt);gn=n(Lo,"Repository"),Lo.forEach(t),Vt=n(os," object so that we can push the checkpoints per epoch "),Jt=r(os,"EM",{});var Rs=l(Jt);_n=n(Rs,"asynchronously"),Rs.forEach(t),ja=n(os,". This allows us to continue training without having to wait for the somewhat slow upload associated with a GB-sized model!"),os.forEach(t),Is.forEach(t),As=c(m),rt=r(m,"P",{});var na=l(rt);lt=n(na,"These steps can be seen in the following block of code:"),na.forEach(t),Xt=c(m),y(us.$$.fragment,m),xa=c(m),y(Os.$$.fragment,m),ds=c(m),Je=r(m,"P",{});var Fs=l(Je);ko=n(Fs,"And that\u2019s it! Once you run this, you\u2019ll have a model and results that are pretty similar to the ones we obtained with the "),Zt=r(Fs,"CODE",{});var Io=l(Zt);it=n(Io,"Trainer"),Io.forEach(t),$o=n(Fs,"."),Fs.forEach(t),this.h()},h(){f(j,"id","finetuning-mt5-with-accelerate"),f(j,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),f(j,"href","#finetuning-mt5-with-accelerate"),f(u,"class","relative group"),f(I,"href","/course/chapter3"),f(Q,"id","preparing-everything-for-training"),f(Q,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),f(Q,"href","#preparing-everything-for-training"),f(B,"class","relative group"),f(cs,"id","training-loop"),f(cs,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),f(cs,"href","#training-loop"),f(Es,"class","relative group")},m(m,q){i(m,u,q),s(u,j),s(j,d),k(E,d,null),s(u,C),s(u,x),s(x,A),i(m,P,q),i(m,T,q),s(T,S),s(T,I),s(I,D),s(T,O),s(T,G),s(G,J),s(T,X),i(m,Z,q),i(m,B,q),s(B,Q),s(Q,L),k(M,L,null),s(B,ee),s(B,F),s(F,K),i(m,te,q),i(m,W,q),s(W,ie),s(W,V),s(V,ae),s(W,de),s(W,_e),s(_e,Te),s(W,H),i(m,ye,q),k(me,m,q),i(m,oe,q),i(m,we,q),s(we,ke),s(we,se),s(se,ge),s(we,he),i(m,fe,q),k(w,m,q),i(m,R,q),i(m,be,q),s(be,b),i(m,U,q),k(qe,m,q),i(m,pe,q),i(m,$e,q),s($e,Ce),s($e,re),s(re,ze),s($e,Qe),i(m,Ae,q),k(ne,m,q),i(m,le,q),i(m,Ie,q),s(Ie,ws),s(Ie,bs),s(bs,vs),s(Ie,ys),i(m,zs,q),k(He,m,q),i(m,rs,q),k(es,m,q),i(m,ls,q),i(m,Ue,q),s(Ue,Ys),i(m,is,q),i(m,Oe,q),s(Oe,ks),s(ks,Ds),s(Oe,Re),s(Oe,ps),s(ps,N),s(Oe,ce),s(Oe,Me),s(Me,Vs),i(m,ue,q),i(m,ms,q),s(ms,ss),i(m,Ss,q),k(De,m,q),i(m,Js,q),i(m,ts,q),s(ts,je),i(m,Xs,q),k(We,m,q),i(m,Zs,q),i(m,Fe,q),s(Fe,Qs),s(Fe,Be),s(Be,et),s(Fe,ve),s(Fe,$s),s($s,Se),s(Fe,vt),i(m,js,q),i(m,xs,q),s(xs,mn),i(m,yt,q),k(st,m,q),i(m,fa,q),k(hs,m,q),i(m,ga,q),i(m,kt,q),s(kt,Ft),i(m,_a,q),k(tt,m,q),i(m,Gt,q),i(m,Ke,q),s(Ke,wa),s(Ke,xe),s(xe,hn),s(Ke,Nt),i(m,ba,q),i(m,Es,q),s(Es,cs),s(cs,Ht),k(at,Ht,null),s(Es,Ut),s(Es,Mt),s(Mt,cn),i(m,$t,q),i(m,jt,q),s(jt,un),i(m,xt,q),i(m,Ye,q),s(Ye,nt),s(nt,va),s(nt,Ts),s(Ts,ya),s(nt,Et),s(Ye,dn),s(Ye,Tt),s(Tt,ot),s(Ye,ka),s(Ye,qs),s(qs,$a),s(Ye,Ve),s(Ye,as),s(as,Wt),s(as,Bt),s(Bt,fn),s(as,Kt),s(as,Yt),s(Yt,gn),s(as,Vt),s(as,Jt),s(Jt,_n),s(as,ja),i(m,As,q),i(m,rt,q),s(rt,lt),i(m,Xt,q),k(us,m,q),i(m,xa,q),k(Os,m,q),i(m,ds,q),i(m,Je,q),s(Je,ko),s(Je,Zt),s(Zt,it),s(Je,$o),Qt=!0},i(m){Qt||(g(E.$$.fragment,m),g(M.$$.fragment,m),g(me.$$.fragment,m),g(w.$$.fragment,m),g(qe.$$.fragment,m),g(ne.$$.fragment,m),g(He.$$.fragment,m),g(es.$$.fragment,m),g(De.$$.fragment,m),g(We.$$.fragment,m),g(st.$$.fragment,m),g(hs.$$.fragment,m),g(tt.$$.fragment,m),g(at.$$.fragment,m),g(us.$$.fragment,m),g(Os.$$.fragment,m),Qt=!0)},o(m){_(E.$$.fragment,m),_(M.$$.fragment,m),_(me.$$.fragment,m),_(w.$$.fragment,m),_(qe.$$.fragment,m),_(ne.$$.fragment,m),_(He.$$.fragment,m),_(es.$$.fragment,m),_(De.$$.fragment,m),_(We.$$.fragment,m),_(st.$$.fragment,m),_(hs.$$.fragment,m),_(tt.$$.fragment,m),_(at.$$.fragment,m),_(us.$$.fragment,m),_(Os.$$.fragment,m),Qt=!1},d(m){m&&t(u),$(E),m&&t(P),m&&t(T),m&&t(Z),m&&t(B),$(M),m&&t(te),m&&t(W),m&&t(ye),$(me,m),m&&t(oe),m&&t(we),m&&t(fe),$(w,m),m&&t(R),m&&t(be),m&&t(U),$(qe,m),m&&t(pe),m&&t($e),m&&t(Ae),$(ne,m),m&&t(le),m&&t(Ie),m&&t(zs),$(He,m),m&&t(rs),$(es,m),m&&t(ls),m&&t(Ue),m&&t(is),m&&t(Oe),m&&t(ue),m&&t(ms),m&&t(Ss),$(De,m),m&&t(Js),m&&t(ts),m&&t(Xs),$(We,m),m&&t(Zs),m&&t(Fe),m&&t(js),m&&t(xs),m&&t(yt),$(st,m),m&&t(fa),$(hs,m),m&&t(ga),m&&t(kt),m&&t(_a),$(tt,m),m&&t(Gt),m&&t(Ke),m&&t(ba),m&&t(Es),$(at),m&&t($t),m&&t(jt),m&&t(xt),m&&t(Ye),m&&t(As),m&&t(rt),m&&t(Xt),$(us,m),m&&t(xa),$(Os,m),m&&t(ds),m&&t(Je)}}}function Fg(Y){let u,j,d,E,C;return{c(){u=o("p"),j=a("\u{1F6A8} If you\u2019re training on a TPU, you\u2019ll need to move all the code above into a dedicated training function. See "),d=o("a"),E=a("Chapter 3"),C=a(" for more details."),this.h()},l(x){u=r(x,"P",{});var A=l(u);j=n(A,"\u{1F6A8} If you\u2019re training on a TPU, you\u2019ll need to move all the code above into a dedicated training function. See "),d=r(A,"A",{href:!0});var P=l(d);E=n(P,"Chapter 3"),P.forEach(t),C=n(A," for more details."),A.forEach(t),this.h()},h(){f(d,"href","/course/chapter3")},m(x,A){i(x,u,A),s(u,j),s(u,d),s(d,E),s(u,C)},d(x){x&&t(u)}}}function Gg(Y){let u,j,d,E,C,x,A,P,T,S,I,D,O,G,J,X,Z,B,Q,L,M,ee,F,K,te,W,ie,V,ae,de,_e,Te,H,ye,me,oe,we,ke,se,ge,he,fe,w,R,be,b,U,qe,pe,$e,Ce,re,ze,Qe,Ae,ne,le,Ie,ws,bs,vs,ys,zs,He,rs,es,ls,Ue,Ys,is,Oe,ks,Ds,Re,ps,N,ce,Me,Vs,ue,ms,ss,Ss,De,Js,ts,je,Xs,We,Zs,Fe,Qs,Be,et,ve,$s,Se,vt,js,xs,mn,yt,st,fa,hs,ga,kt,Ft,_a,tt,Gt,Ke,wa,xe,hn,Nt,ba,Es,cs,Ht,at,Ut,Mt,cn,$t,jt,un,xt,Ye,nt,va,Ts,ya,Et,dn,Tt,ot,ka,qs,$a,Ve,as,Wt,Bt,fn,Kt,Yt,gn,Vt,Jt,_n,ja,As,rt,lt,Xt,us,xa,Os,ds,Je,ko,Zt,it,$o,Qt,m,q,Ea,jo,xo,pt,qt,wn,mt,Eo,bn,Cs,ns,Ta,zt,qa,za,To,ea,ht,Da,sa,qo,Dt,vn,ct,Sa,Ps,ta,St,zo,Aa,Do,So,ut,Ao,Oa,Ge,Ls,aa,At,Is,Ca,Pa,Oo,Co,os,Po,Lo,Rs,na,Fs,Io,Xp,Cr,Zp,Qp,Ro,em,sm,oa,Fo,yn,tm,am,kn,nm,Pr,om,rm,lm,Go,im,pm,ra,No,$n,mm,hm,Lr,cm,um,Ho,dm,fm,la,Uo,jn,gm,_m,Ir,wm,bm,Mo,vm,ym,ia,Wo,xn,km,$m,Rr,jm,xm,Bo,Em,Kl,Ko,Tm,Yl,La,qm,Fr,zm,Dm,Vl,pa,En,ku,Sm,Tn,$u,Jl,Yo,Am,Xl,Ia,Zl,ma,Ra,Gr,qn,Om,Nr,Cm,Ql,zn,ei,Fa,Pm,Hr,Lm,Im,si,Dn,ti,Ga,ai,Vo,Rm,ni,Sn,oi,An,ri,fs,Fm,Ur,Gm,Nm,Mr,Hm,Um,Jo,Mm,Wm,Wr,Bm,Km,li,On,ii,Cn,pi,Gs,Ym,Br,Vm,Jm,Kr,Xm,Zm,Xo,Qm,eh,mi,Na,sh,Yr,th,ah,hi,Pn,ci,gs,nh,Vr,oh,rh,Jr,lh,ih,Xr,ph,mh,Zr,hh,ch,ui,Ot,uh,Qr,dh,fh,el,gh,_h,di,Ln,fi,Zo,wh,gi,Ha,_i,ha,Ua,sl,In,bh,tl,vh,wi,Rn,bi,Qo,yh,vi,Ma,kh,Fn,$h,jh,yi,Gn,ki,Ct,xh,al,Eh,Th,nl,qh,zh,$i,Wa,ji,Nn,Dh,xi,gg='<span class="katex-display"><span class="katex"><span class="katex-mathml"><math xmlns="http://www.w3.org/1998/Math/MathML" display="block"><semantics><mrow><mrow><mi mathvariant="normal">R</mi><mi mathvariant="normal">e</mi><mi mathvariant="normal">c</mi><mi mathvariant="normal">a</mi><mi mathvariant="normal">l</mi><mi mathvariant="normal">l</mi></mrow><mo>=</mo><mfrac><mrow><mi mathvariant="normal">N</mi><mi mathvariant="normal">u</mi><mi mathvariant="normal">m</mi><mi mathvariant="normal">b</mi><mi mathvariant="normal">e</mi><mi mathvariant="normal">r</mi><mtext>\u2009</mtext><mi mathvariant="normal">o</mi><mi mathvariant="normal">f</mi><mtext>\u2009</mtext><mi mathvariant="normal">o</mi><mi mathvariant="normal">v</mi><mi mathvariant="normal">e</mi><mi mathvariant="normal">r</mi><mi mathvariant="normal">l</mi><mi mathvariant="normal">a</mi><mi mathvariant="normal">p</mi><mi mathvariant="normal">p</mi><mi mathvariant="normal">i</mi><mi mathvariant="normal">n</mi><mi mathvariant="normal">g</mi><mtext>\u2009</mtext><mi mathvariant="normal">w</mi><mi mathvariant="normal">o</mi><mi mathvariant="normal">r</mi><mi mathvariant="normal">d</mi><mi mathvariant="normal">s</mi></mrow><mrow><mi mathvariant="normal">T</mi><mi mathvariant="normal">o</mi><mi mathvariant="normal">t</mi><mi mathvariant="normal">a</mi><mi mathvariant="normal">l</mi><mtext>\u2009</mtext><mi mathvariant="normal">n</mi><mi mathvariant="normal">u</mi><mi mathvariant="normal">m</mi><mi mathvariant="normal">b</mi><mi mathvariant="normal">e</mi><mi mathvariant="normal">r</mi><mtext>\u2009</mtext><mi mathvariant="normal">o</mi><mi mathvariant="normal">f</mi><mtext>\u2009</mtext><mi mathvariant="normal">w</mi><mi mathvariant="normal">o</mi><mi mathvariant="normal">r</mi><mi mathvariant="normal">d</mi><mi mathvariant="normal">s</mi><mtext>\u2009</mtext><mi mathvariant="normal">i</mi><mi mathvariant="normal">n</mi><mtext>\u2009</mtext><mi mathvariant="normal">r</mi><mi mathvariant="normal">e</mi><mi mathvariant="normal">f</mi><mi mathvariant="normal">e</mi><mi mathvariant="normal">r</mi><mi mathvariant="normal">e</mi><mi mathvariant="normal">n</mi><mi mathvariant="normal">c</mi><mi mathvariant="normal">e</mi><mtext>\u2009</mtext><mi mathvariant="normal">s</mi><mi mathvariant="normal">u</mi><mi mathvariant="normal">m</mi><mi mathvariant="normal">m</mi><mi mathvariant="normal">a</mi><mi mathvariant="normal">r</mi><mi mathvariant="normal">y</mi></mrow></mfrac></mrow><annotation encoding="application/x-tex"> \\mathrm{Recall} = \\frac{\\mathrm{Number\\,of\\,overlapping\\, words}}{\\mathrm{Total\\, number\\, of\\, words\\, in\\, reference\\, summary}} </annotation></semantics></math></span><span class="katex-html" aria-hidden="true"><span class="base"><span class="strut" style="height:0.6944em;"></span><span class="mord"><span class="mord mathrm">Recall</span></span><span class="mspace" style="margin-right:0.2778em;"></span><span class="mrel">=</span><span class="mspace" style="margin-right:0.2778em;"></span></span><span class="base"><span class="strut" style="height:2.2519em;vertical-align:-0.8804em;"></span><span class="mord"><span class="mopen nulldelimiter"></span><span class="mfrac"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:1.3714em;"><span style="top:-2.314em;"><span class="pstrut" style="height:3em;"></span><span class="mord"><span class="mord"><span class="mord mathrm">Total</span><span class="mspace" style="margin-right:0.1667em;"></span><span class="mord mathrm">number</span><span class="mspace" style="margin-right:0.1667em;"></span><span class="mord mathrm" style="margin-right:0.07778em;">of</span><span class="mspace" style="margin-right:0.1667em;"></span><span class="mord mathrm">words</span><span class="mspace" style="margin-right:0.1667em;"></span><span class="mord mathrm">in</span><span class="mspace" style="margin-right:0.1667em;"></span><span class="mord mathrm">reference</span><span class="mspace" style="margin-right:0.1667em;"></span><span class="mord mathrm" style="margin-right:0.01389em;">summary</span></span></span></span><span style="top:-3.23em;"><span class="pstrut" style="height:3em;"></span><span class="frac-line" style="border-bottom-width:0.04em;"></span></span><span style="top:-3.677em;"><span class="pstrut" style="height:3em;"></span><span class="mord"><span class="mord"><span class="mord mathrm">Number</span><span class="mspace" style="margin-right:0.1667em;"></span><span class="mord mathrm" style="margin-right:0.07778em;">of</span><span class="mspace" style="margin-right:0.1667em;"></span><span class="mord mathrm" style="margin-right:0.01389em;">overlapping</span><span class="mspace" style="margin-right:0.1667em;"></span><span class="mord mathrm">words</span></span></span></span></span><span class="vlist-s">\u200B</span></span><span class="vlist-r"><span class="vlist" style="height:0.8804em;"><span></span></span></span></span></span><span class="mclose nulldelimiter"></span></span></span></span></span></span>',Ei,Hn,Sh,Ti,_g='<span class="katex-display"><span class="katex"><span class="katex-mathml"><math xmlns="http://www.w3.org/1998/Math/MathML" display="block"><semantics><mrow><mrow><mi mathvariant="normal">P</mi><mi mathvariant="normal">r</mi><mi mathvariant="normal">e</mi><mi mathvariant="normal">c</mi><mi mathvariant="normal">i</mi><mi mathvariant="normal">s</mi><mi mathvariant="normal">i</mi><mi mathvariant="normal">o</mi><mi mathvariant="normal">n</mi></mrow><mo>=</mo><mfrac><mrow><mi mathvariant="normal">N</mi><mi mathvariant="normal">u</mi><mi mathvariant="normal">m</mi><mi mathvariant="normal">b</mi><mi mathvariant="normal">e</mi><mi mathvariant="normal">r</mi><mtext>\u2009</mtext><mi mathvariant="normal">o</mi><mi mathvariant="normal">f</mi><mtext>\u2009</mtext><mi mathvariant="normal">o</mi><mi mathvariant="normal">v</mi><mi mathvariant="normal">e</mi><mi mathvariant="normal">r</mi><mi mathvariant="normal">l</mi><mi mathvariant="normal">a</mi><mi mathvariant="normal">p</mi><mi mathvariant="normal">p</mi><mi mathvariant="normal">i</mi><mi mathvariant="normal">n</mi><mi mathvariant="normal">g</mi><mtext>\u2009</mtext><mi mathvariant="normal">w</mi><mi mathvariant="normal">o</mi><mi mathvariant="normal">r</mi><mi mathvariant="normal">d</mi><mi mathvariant="normal">s</mi></mrow><mrow><mi mathvariant="normal">T</mi><mi mathvariant="normal">o</mi><mi mathvariant="normal">t</mi><mi mathvariant="normal">a</mi><mi mathvariant="normal">l</mi><mtext>\u2009</mtext><mi mathvariant="normal">n</mi><mi mathvariant="normal">u</mi><mi mathvariant="normal">m</mi><mi mathvariant="normal">b</mi><mi mathvariant="normal">e</mi><mi mathvariant="normal">r</mi><mtext>\u2009</mtext><mi mathvariant="normal">o</mi><mi mathvariant="normal">f</mi><mtext>\u2009</mtext><mi mathvariant="normal">w</mi><mi mathvariant="normal">o</mi><mi mathvariant="normal">r</mi><mi mathvariant="normal">d</mi><mi mathvariant="normal">s</mi><mtext>\u2009</mtext><mi mathvariant="normal">i</mi><mi mathvariant="normal">n</mi><mtext>\u2009</mtext><mi mathvariant="normal">g</mi><mi mathvariant="normal">e</mi><mi mathvariant="normal">n</mi><mi mathvariant="normal">e</mi><mi mathvariant="normal">r</mi><mi mathvariant="normal">a</mi><mi mathvariant="normal">t</mi><mi mathvariant="normal">e</mi><mi mathvariant="normal">d</mi><mtext>\u2009</mtext><mi mathvariant="normal">s</mi><mi mathvariant="normal">u</mi><mi mathvariant="normal">m</mi><mi mathvariant="normal">m</mi><mi mathvariant="normal">a</mi><mi mathvariant="normal">r</mi><mi mathvariant="normal">y</mi></mrow></mfrac></mrow><annotation encoding="application/x-tex"> \\mathrm{Precision} = \\frac{\\mathrm{Number\\,of\\,overlapping\\, words}}{\\mathrm{Total\\, number\\, of\\, words\\, in\\, generated\\, summary}} </annotation></semantics></math></span><span class="katex-html" aria-hidden="true"><span class="base"><span class="strut" style="height:0.6833em;"></span><span class="mord"><span class="mord mathrm">Precision</span></span><span class="mspace" style="margin-right:0.2778em;"></span><span class="mrel">=</span><span class="mspace" style="margin-right:0.2778em;"></span></span><span class="base"><span class="strut" style="height:2.2519em;vertical-align:-0.8804em;"></span><span class="mord"><span class="mopen nulldelimiter"></span><span class="mfrac"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:1.3714em;"><span style="top:-2.314em;"><span class="pstrut" style="height:3em;"></span><span class="mord"><span class="mord"><span class="mord mathrm">Total</span><span class="mspace" style="margin-right:0.1667em;"></span><span class="mord mathrm">number</span><span class="mspace" style="margin-right:0.1667em;"></span><span class="mord mathrm" style="margin-right:0.07778em;">of</span><span class="mspace" style="margin-right:0.1667em;"></span><span class="mord mathrm">words</span><span class="mspace" style="margin-right:0.1667em;"></span><span class="mord mathrm">in</span><span class="mspace" style="margin-right:0.1667em;"></span><span class="mord mathrm">generated</span><span class="mspace" style="margin-right:0.1667em;"></span><span class="mord mathrm" style="margin-right:0.01389em;">summary</span></span></span></span><span style="top:-3.23em;"><span class="pstrut" style="height:3em;"></span><span class="frac-line" style="border-bottom-width:0.04em;"></span></span><span style="top:-3.677em;"><span class="pstrut" style="height:3em;"></span><span class="mord"><span class="mord"><span class="mord mathrm">Number</span><span class="mspace" style="margin-right:0.1667em;"></span><span class="mord mathrm" style="margin-right:0.07778em;">of</span><span class="mspace" style="margin-right:0.1667em;"></span><span class="mord mathrm" style="margin-right:0.01389em;">overlapping</span><span class="mspace" style="margin-right:0.1667em;"></span><span class="mord mathrm">words</span></span></span></span></span><span class="vlist-s">\u200B</span></span><span class="vlist-r"><span class="vlist" style="height:0.8804em;"><span></span></span></span></span></span><span class="mclose nulldelimiter"></span></span></span></span></span></span>',qi,Ba,Ah,ol,Oh,Ch,zi,Un,Di,er,Ph,Si,Mn,Ai,Ka,Lh,rl,Ih,Rh,Oi,Wn,Ci,Bn,Pi,Xe,Fh,ll,Gh,Nh,il,Hh,Uh,pl,Mh,Wh,ml,Bh,Kh,hl,Yh,Vh,Li,Kn,Ii,Yn,Ri,Ze,Jh,cl,Xh,Zh,ul,Qh,ec,dl,sc,tc,fl,ac,nc,gl,oc,rc,Fi,Ya,Gi,sr,lc,Ni,ca,Va,_l,Vn,ic,wl,pc,Hi,Ns,mc,bl,hc,cc,vl,uc,dc,yl,fc,gc,Ui,Jn,Mi,tr,_c,Wi,Xn,Bi,Ja,wc,kl,bc,vc,Ki,Zn,Yi,Qn,Vi,ar,yc,Ji,eo,Xi,nr,kc,Zi,so,Qi,to,ep,Xa,$c,$l,jc,xc,sp,dt,ft,or,Za,tp,rr,Ec,ap,ao,np,lr,Tc,op,no,rp,ir,Qa,qc,pr,zc,Dc,lp,Hs,Sc,jl,Ac,Oc,xl,Cc,Pc,El,Lc,Ic,ip,gt,_t,mr,hr,Rc,pp,oo,mp,Pt,Fc,Tl,Gc,Nc,ql,Hc,Uc,hp,ro,cp,lo,up,Ee,Mc,zl,Wc,Bc,Dl,Kc,Yc,Sl,Vc,Jc,Al,Xc,Zc,Ol,Qc,eu,Cl,su,tu,Pl,au,nu,Ll,ou,ru,dp,wt,bt,cr,ur,ua,en,Il,io,lu,Rl,iu,fp,sn,pu,Fl,mu,hu,gp,po,_p,dr,cu,wp,mo,bp,fr,uu,vp,ho,yp,co,kp,tn,du,Gl,fu,gu,$p,uo,jp,fo,xp,gr,_u,Ep,_r,wu,Tp;d=new $g({props:{fw:Y[0]}}),P=new Ks({});const ju=[xg,jg],go=[];function xu(e,p){return e[0]==="pt"?0:1}O=xu(Y),G=go[O]=ju[O](Y),ee=new yu({props:{id:"yHnr5Dk2zCI"}}),fe=new Ks({}),ze=new z({props:{code:`from datasets import load_dataset

spanish_dataset = load_dataset("amazon_reviews_multi", "es")
english_dataset = load_dataset("amazon_reviews_multi", "en")
english_dataset`,highlighted:`<span class="hljs-keyword">from</span> datasets <span class="hljs-keyword">import</span> load_dataset

spanish_dataset = load_dataset(<span class="hljs-string">&quot;amazon_reviews_multi&quot;</span>, <span class="hljs-string">&quot;es&quot;</span>)
english_dataset = load_dataset(<span class="hljs-string">&quot;amazon_reviews_multi&quot;</span>, <span class="hljs-string">&quot;en&quot;</span>)
english_dataset`}}),Ae=new z({props:{code:`DatasetDict({
    train: Dataset({
        features: ['review_id', 'product_id', 'reviewer_id', 'stars', 'review_body', 'review_title', 'language', 'product_category'],
        num_rows: 200000
    })
    validation: Dataset({
        features: ['review_id', 'product_id', 'reviewer_id', 'stars', 'review_body', 'review_title', 'language', 'product_category'],
        num_rows: 5000
    })
    test: Dataset({
        features: ['review_id', 'product_id', 'reviewer_id', 'stars', 'review_body', 'review_title', 'language', 'product_category'],
        num_rows: 5000
    })
})`,highlighted:`DatasetDict({
    train: Dataset({
        features: [<span class="hljs-string">&#x27;review_id&#x27;</span>, <span class="hljs-string">&#x27;product_id&#x27;</span>, <span class="hljs-string">&#x27;reviewer_id&#x27;</span>, <span class="hljs-string">&#x27;stars&#x27;</span>, <span class="hljs-string">&#x27;review_body&#x27;</span>, <span class="hljs-string">&#x27;review_title&#x27;</span>, <span class="hljs-string">&#x27;language&#x27;</span>, <span class="hljs-string">&#x27;product_category&#x27;</span>],
        num_rows: <span class="hljs-number">200000</span>
    })
    validation: Dataset({
        features: [<span class="hljs-string">&#x27;review_id&#x27;</span>, <span class="hljs-string">&#x27;product_id&#x27;</span>, <span class="hljs-string">&#x27;reviewer_id&#x27;</span>, <span class="hljs-string">&#x27;stars&#x27;</span>, <span class="hljs-string">&#x27;review_body&#x27;</span>, <span class="hljs-string">&#x27;review_title&#x27;</span>, <span class="hljs-string">&#x27;language&#x27;</span>, <span class="hljs-string">&#x27;product_category&#x27;</span>],
        num_rows: <span class="hljs-number">5000</span>
    })
    test: Dataset({
        features: [<span class="hljs-string">&#x27;review_id&#x27;</span>, <span class="hljs-string">&#x27;product_id&#x27;</span>, <span class="hljs-string">&#x27;reviewer_id&#x27;</span>, <span class="hljs-string">&#x27;stars&#x27;</span>, <span class="hljs-string">&#x27;review_body&#x27;</span>, <span class="hljs-string">&#x27;review_title&#x27;</span>, <span class="hljs-string">&#x27;language&#x27;</span>, <span class="hljs-string">&#x27;product_category&#x27;</span>],
        num_rows: <span class="hljs-number">5000</span>
    })
})`}}),Me=new z({props:{code:`def show_samples(dataset, num_samples=3, seed=42):
    sample = dataset["train"].shuffle(seed=seed).select(range(num_samples))
    for example in sample:
        print(f"\\n'>> Title: {example['review_title']}'")
        print(f"'>> Review: {example['review_body']}'")


show_samples(english_dataset)`,highlighted:`<span class="hljs-keyword">def</span> <span class="hljs-title function_">show_samples</span>(<span class="hljs-params">dataset, num_samples=<span class="hljs-number">3</span>, seed=<span class="hljs-number">42</span></span>):
    sample = dataset[<span class="hljs-string">&quot;train&quot;</span>].shuffle(seed=seed).select(<span class="hljs-built_in">range</span>(num_samples))
    <span class="hljs-keyword">for</span> example <span class="hljs-keyword">in</span> sample:
        <span class="hljs-built_in">print</span>(<span class="hljs-string">f&quot;\\n&#x27;&gt;&gt; Title: <span class="hljs-subst">{example[<span class="hljs-string">&#x27;review_title&#x27;</span>]}</span>&#x27;&quot;</span>)
        <span class="hljs-built_in">print</span>(<span class="hljs-string">f&quot;&#x27;&gt;&gt; Review: <span class="hljs-subst">{example[<span class="hljs-string">&#x27;review_body&#x27;</span>]}</span>&#x27;&quot;</span>)


show_samples(english_dataset)`}}),ue=new z({props:{code:`'>> Title: Worked in front position, not rear'
'>> Review: 3 stars because these are not rear brakes as stated in the item description. At least the mount adapter only worked on the front fork of the bike that I got it for.'

'>> Title: meh'
'>> Review: Does it\u2019s job and it\u2019s gorgeous but mine is falling apart, I had to basically put it together again with hot glue'

'>> Title: Can\\'t beat these for the money'
'>> Review: Bought this for handling miscellaneous aircraft parts and hanger "stuff" that I needed to organize; it really fit the bill. The unit arrived quickly, was well packaged and arrived intact (always a good sign). There are five wall mounts-- three on the top and two on the bottom. I wanted to mount it on the wall, so all I had to do was to remove the top two layers of plastic drawers, as well as the bottom corner drawers, place it when I wanted and mark it; I then used some of the new plastic screw in wall anchors (the 50 pound variety) and it easily mounted to the wall. Some have remarked that they wanted dividers for the drawers, and that they made those. Good idea. My application was that I needed something that I can see the contents at about eye level, so I wanted the fuller-sized drawers. I also like that these are the new plastic that doesn\\'t get brittle and split like my older plastic drawers did. I like the all-plastic construction. It\\'s heavy duty enough to hold metal parts, but being made of plastic it\\'s not as heavy as a metal frame, so you can easily mount it to the wall and still load it up with heavy stuff, or light stuff. No problem there. For the money, you can\\'t beat it. Best one of these I\\'ve bought to date-- and I\\'ve been using some version of these for over forty years.'`,highlighted:`<span class="hljs-string">&#x27;&gt;&gt; Title: Worked in front position, not rear&#x27;</span>
<span class="hljs-string">&#x27;&gt;&gt; Review: 3 stars because these are not rear brakes as stated in the item description. At least the mount adapter only worked on the front fork of the bike that I got it for.&#x27;</span>

<span class="hljs-string">&#x27;&gt;&gt; Title: meh&#x27;</span>
<span class="hljs-string">&#x27;&gt;&gt; Review: Does it\u2019s job and it\u2019s gorgeous but mine is falling apart, I had to basically put it together again with hot glue&#x27;</span>

<span class="hljs-string">&#x27;&gt;&gt; Title: Can\\&#x27;t beat these for the money&#x27;</span>
<span class="hljs-string">&#x27;&gt;&gt; Review: Bought this for handling miscellaneous aircraft parts and hanger &quot;stuff&quot; that I needed to organize; it really fit the bill. The unit arrived quickly, was well packaged and arrived intact (always a good sign). There are five wall mounts-- three on the top and two on the bottom. I wanted to mount it on the wall, so all I had to do was to remove the top two layers of plastic drawers, as well as the bottom corner drawers, place it when I wanted and mark it; I then used some of the new plastic screw in wall anchors (the 50 pound variety) and it easily mounted to the wall. Some have remarked that they wanted dividers for the drawers, and that they made those. Good idea. My application was that I needed something that I can see the contents at about eye level, so I wanted the fuller-sized drawers. I also like that these are the new plastic that doesn\\&#x27;t get brittle and split like my older plastic drawers did. I like the all-plastic construction. It\\&#x27;s heavy duty enough to hold metal parts, but being made of plastic it\\&#x27;s not as heavy as a metal frame, so you can easily mount it to the wall and still load it up with heavy stuff, or light stuff. No problem there. For the money, you can\\&#x27;t beat it. Best one of these I\\&#x27;ve bought to date-- and I\\&#x27;ve been using some version of these for over forty years.&#x27;</span>`}}),ss=new pn({props:{$$slots:{default:[Eg]},$$scope:{ctx:Y}}}),Be=new z({props:{code:`english_dataset.set_format("pandas")
english_df = english_dataset["train"][:]
# Show counts for top 20 products
english_df["product_category"].value_counts()[:20]`,highlighted:`english_dataset.set_format(<span class="hljs-string">&quot;pandas&quot;</span>)
english_df = english_dataset[<span class="hljs-string">&quot;train&quot;</span>][:]
<span class="hljs-comment"># Show counts for top 20 products</span>
english_df[<span class="hljs-string">&quot;product_category&quot;</span>].value_counts()[:<span class="hljs-number">20</span>]`}}),ve=new z({props:{code:`home                      17679
apparel                   15951
wireless                  15717
other                     13418
beauty                    12091
drugstore                 11730
kitchen                   10382
toy                        8745
sports                     8277
automotive                 7506
lawn_and_garden            7327
home_improvement           7136
pet_products               7082
digital_ebook_purchase     6749
pc                         6401
electronics                6186
office_product             5521
shoes                      5197
grocery                    4730
book                       3756
Name: product_category, dtype: int64`,highlighted:`home                      <span class="hljs-number">17679</span>
apparel                   <span class="hljs-number">15951</span>
wireless                  <span class="hljs-number">15717</span>
other                     <span class="hljs-number">13418</span>
beauty                    <span class="hljs-number">12091</span>
drugstore                 <span class="hljs-number">11730</span>
kitchen                   <span class="hljs-number">10382</span>
toy                        <span class="hljs-number">8745</span>
sports                     <span class="hljs-number">8277</span>
automotive                 <span class="hljs-number">7506</span>
lawn_and_garden            <span class="hljs-number">7327</span>
home_improvement           <span class="hljs-number">7136</span>
pet_products               <span class="hljs-number">7082</span>
digital_ebook_purchase     <span class="hljs-number">6749</span>
pc                         <span class="hljs-number">6401</span>
electronics                <span class="hljs-number">6186</span>
office_product             <span class="hljs-number">5521</span>
shoes                      <span class="hljs-number">5197</span>
grocery                    <span class="hljs-number">4730</span>
book                       <span class="hljs-number">3756</span>
Name: product_category, dtype: int64`}}),Ke=new z({props:{code:`def filter_books(example):
    return (
        example["product_category"] == "book"
        or example["product_category"] == "digital_ebook_purchase"
    )`,highlighted:`<span class="hljs-keyword">def</span> <span class="hljs-title function_">filter_books</span>(<span class="hljs-params">example</span>):
    <span class="hljs-keyword">return</span> (
        example[<span class="hljs-string">&quot;product_category&quot;</span>] == <span class="hljs-string">&quot;book&quot;</span>
        <span class="hljs-keyword">or</span> example[<span class="hljs-string">&quot;product_category&quot;</span>] == <span class="hljs-string">&quot;digital_ebook_purchase&quot;</span>
    )`}}),Ts=new z({props:{code:"english_dataset.reset_format()",highlighted:"english_dataset.reset_format()"}}),ot=new z({props:{code:`spanish_books = spanish_dataset.filter(filter_books)
english_books = english_dataset.filter(filter_books)
show_samples(english_books)`,highlighted:`spanish_books = spanish_dataset.<span class="hljs-built_in">filter</span>(filter_books)
english_books = english_dataset.<span class="hljs-built_in">filter</span>(filter_books)
show_samples(english_books)`}}),qs=new z({props:{code:`'>> Title: I\\'m dissapointed.'
'>> Review: I guess I had higher expectations for this book from the reviews. I really thought I\\'d at least like it. The plot idea was great. I loved Ash but, it just didnt go anywhere. Most of the book was about their radio show and talking to callers. I wanted the author to dig deeper so we could really get to know the characters. All we know about Grace is that she is attractive looking, Latino and is kind of a brat. I\\'m dissapointed.'

'>> Title: Good art, good price, poor design'
'>> Review: I had gotten the DC Vintage calendar the past two years, but it was on backorder forever this year and I saw they had shrunk the dimensions for no good reason. This one has good art choices but the design has the fold going through the picture, so it\\'s less aesthetically pleasing, especially if you want to keep a picture to hang. For the price, a good calendar'

'>> Title: Helpful'
'>> Review: Nearly all the tips useful and. I consider myself an intermediate to advanced user of OneNote. I would highly recommend.'`,highlighted:`<span class="hljs-string">&#x27;&gt;&gt; Title: I\\&#x27;m dissapointed.&#x27;</span>
<span class="hljs-string">&#x27;&gt;&gt; Review: I guess I had higher expectations for this book from the reviews. I really thought I\\&#x27;d at least like it. The plot idea was great. I loved Ash but, it just didnt go anywhere. Most of the book was about their radio show and talking to callers. I wanted the author to dig deeper so we could really get to know the characters. All we know about Grace is that she is attractive looking, Latino and is kind of a brat. I\\&#x27;m dissapointed.&#x27;</span>

<span class="hljs-string">&#x27;&gt;&gt; Title: Good art, good price, poor design&#x27;</span>
<span class="hljs-string">&#x27;&gt;&gt; Review: I had gotten the DC Vintage calendar the past two years, but it was on backorder forever this year and I saw they had shrunk the dimensions for no good reason. This one has good art choices but the design has the fold going through the picture, so it\\&#x27;s less aesthetically pleasing, especially if you want to keep a picture to hang. For the price, a good calendar&#x27;</span>

<span class="hljs-string">&#x27;&gt;&gt; Title: Helpful&#x27;</span>
<span class="hljs-string">&#x27;&gt;&gt; Review: Nearly all the tips useful and. I consider myself an intermediate to advanced user of OneNote. I would highly recommend.&#x27;</span>`}}),As=new z({props:{code:`from datasets import concatenate_datasets, DatasetDict

books_dataset = DatasetDict()

for split in english_books.keys():
    books_dataset[split] = concatenate_datasets(
        [english_books[split], spanish_books[split]]
    )
    books_dataset[split] = books_dataset[split].shuffle(seed=42)

# Peek at a few examples
show_samples(books_dataset)`,highlighted:`<span class="hljs-keyword">from</span> datasets <span class="hljs-keyword">import</span> concatenate_datasets, DatasetDict

books_dataset = DatasetDict()

<span class="hljs-keyword">for</span> split <span class="hljs-keyword">in</span> english_books.keys():
    books_dataset[split] = concatenate_datasets(
        [english_books[split], spanish_books[split]]
    )
    books_dataset[split] = books_dataset[split].shuffle(seed=<span class="hljs-number">42</span>)

<span class="hljs-comment"># Peek at a few examples</span>
show_samples(books_dataset)`}}),lt=new z({props:{code:`'>> Title: Easy to follow!!!!'
'>> Review: I loved The dash diet weight loss Solution. Never hungry. I would recommend this diet. Also the menus are well rounded. Try it. Has lots of the information need thanks.'

'>> Title: PARCIALMENTE DA\xD1ADO'
'>> Review: Me lleg\xF3 el d\xEDa que tocaba, junto a otros libros que ped\xED, pero la caja lleg\xF3 en mal estado lo cual da\xF1\xF3 las esquinas de los libros porque ven\xEDan sin protecci\xF3n (forro).'

'>> Title: no lo he podido descargar'
'>> Review: igual que el anterior'`,highlighted:`<span class="hljs-string">&#x27;&gt;&gt; Title: Easy to follow!!!!&#x27;</span>
<span class="hljs-string">&#x27;&gt;&gt; Review: I loved The dash diet weight loss Solution. Never hungry. I would recommend this diet. Also the menus are well rounded. Try it. Has lots of the information need thanks.&#x27;</span>

<span class="hljs-string">&#x27;&gt;&gt; Title: PARCIALMENTE DA\xD1ADO&#x27;</span>
<span class="hljs-string">&#x27;&gt;&gt; Review: Me lleg\xF3 el d\xEDa que tocaba, junto a otros libros que ped\xED, pero la caja lleg\xF3 en mal estado lo cual da\xF1\xF3 las esquinas de los libros porque ven\xEDan sin protecci\xF3n (forro).&#x27;</span>

<span class="hljs-string">&#x27;&gt;&gt; Title: no lo he podido descargar&#x27;</span>
<span class="hljs-string">&#x27;&gt;&gt; Review: igual que el anterior&#x27;</span>`}}),qt=new z({props:{code:'books_dataset = books_dataset.filter(lambda x: len(x["review_title"].split()) > 2)',highlighted:'books_dataset = books_dataset.<span class="hljs-built_in">filter</span>(<span class="hljs-keyword">lambda</span> x: <span class="hljs-built_in">len</span>(x[<span class="hljs-string">&quot;review_title&quot;</span>].split()) &gt; <span class="hljs-number">2</span>)'}}),zt=new Ks({}),Ia=new pn({props:{$$slots:{default:[Tg]},$$scope:{ctx:Y}}}),qn=new Ks({}),zn=new yu({props:{id:"1m7BerpSq8A"}}),Dn=new z({props:{code:`from transformers import AutoTokenizer

model_checkpoint = "google/mt5-small"
tokenizer = AutoTokenizer.from_pretrained(model_checkpoint)`,highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoTokenizer

model_checkpoint = <span class="hljs-string">&quot;google/mt5-small&quot;</span>
tokenizer = AutoTokenizer.from_pretrained(model_checkpoint)`}}),Ga=new pn({props:{$$slots:{default:[qg]},$$scope:{ctx:Y}}}),Sn=new z({props:{code:`inputs = tokenizer("I loved reading the Hunger Games!")
inputs`,highlighted:`inputs = tokenizer(<span class="hljs-string">&quot;I loved reading the Hunger Games!&quot;</span>)
inputs`}}),An=new z({props:{code:"{'input_ids': [336, 259, 28387, 11807, 287, 62893, 295, 12507, 1], 'attention_mask': [1, 1, 1, 1, 1, 1, 1, 1, 1]}",highlighted:'{<span class="hljs-string">&#x27;input_ids&#x27;</span>: [<span class="hljs-number">336</span>, <span class="hljs-number">259</span>, <span class="hljs-number">28387</span>, <span class="hljs-number">11807</span>, <span class="hljs-number">287</span>, <span class="hljs-number">62893</span>, <span class="hljs-number">295</span>, <span class="hljs-number">12507</span>, <span class="hljs-number">1</span>], <span class="hljs-string">&#x27;attention_mask&#x27;</span>: [<span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>]}'}}),On=new z({props:{code:"tokenizer.convert_ids_to_tokens(inputs.input_ids)",highlighted:"tokenizer.convert_ids_to_tokens(inputs.input_ids)"}}),Cn=new z({props:{code:"['\u2581I', '\u2581', 'loved', '\u2581reading', '\u2581the', '\u2581Hung', 'er', '\u2581Games', '</s>']",highlighted:'[<span class="hljs-string">&#x27;\u2581I&#x27;</span>, <span class="hljs-string">&#x27;\u2581&#x27;</span>, <span class="hljs-string">&#x27;loved&#x27;</span>, <span class="hljs-string">&#x27;\u2581reading&#x27;</span>, <span class="hljs-string">&#x27;\u2581the&#x27;</span>, <span class="hljs-string">&#x27;\u2581Hung&#x27;</span>, <span class="hljs-string">&#x27;er&#x27;</span>, <span class="hljs-string">&#x27;\u2581Games&#x27;</span>, <span class="hljs-string">&#x27;&lt;/s&gt;&#x27;</span>]'}}),Pn=new z({props:{code:`max_input_length = 512
max_target_length = 30


def preprocess_function(examples):
    model_inputs = tokenizer(
        examples["review_body"], max_length=max_input_length, truncation=True
    )
    # Set up the tokenizer for targets
    with tokenizer.as_target_tokenizer():
        labels = tokenizer(
            examples["review_title"], max_length=max_target_length, truncation=True
        )

    model_inputs["labels"] = labels["input_ids"]
    return model_inputs`,highlighted:`max_input_length = <span class="hljs-number">512</span>
max_target_length = <span class="hljs-number">30</span>


<span class="hljs-keyword">def</span> <span class="hljs-title function_">preprocess_function</span>(<span class="hljs-params">examples</span>):
    model_inputs = tokenizer(
        examples[<span class="hljs-string">&quot;review_body&quot;</span>], max_length=max_input_length, truncation=<span class="hljs-literal">True</span>
    )
    <span class="hljs-comment"># Set up the tokenizer for targets</span>
    <span class="hljs-keyword">with</span> tokenizer.as_target_tokenizer():
        labels = tokenizer(
            examples[<span class="hljs-string">&quot;review_title&quot;</span>], max_length=max_target_length, truncation=<span class="hljs-literal">True</span>
        )

    model_inputs[<span class="hljs-string">&quot;labels&quot;</span>] = labels[<span class="hljs-string">&quot;input_ids&quot;</span>]
    <span class="hljs-keyword">return</span> model_inputs`}}),Ln=new z({props:{code:"tokenized_datasets = books_dataset.map(preprocess_function, batched=True)",highlighted:'tokenized_datasets = books_dataset.<span class="hljs-built_in">map</span>(preprocess_function, batched=<span class="hljs-literal">True</span>)'}}),Ha=new pn({props:{$$slots:{default:[zg]},$$scope:{ctx:Y}}}),In=new Ks({}),Rn=new yu({props:{id:"TMshhnrEXlg"}}),Gn=new z({props:{code:`generated_summary = "I absolutely loved reading the Hunger Games"
reference_summary = "I loved reading the Hunger Games"`,highlighted:`generated_summary = <span class="hljs-string">&quot;I absolutely loved reading the Hunger Games&quot;</span>
reference_summary = <span class="hljs-string">&quot;I loved reading the Hunger Games&quot;</span>`}}),Wa=new pn({props:{$$slots:{default:[Dg]},$$scope:{ctx:Y}}}),Un=new z({props:{code:"!pip install rouge_score",highlighted:"!pip install rouge_score"}}),Mn=new z({props:{code:`from datasets import load_metric

rouge_score = load_metric("rouge")`,highlighted:`<span class="hljs-keyword">from</span> datasets <span class="hljs-keyword">import</span> load_metric

rouge_score = load_metric(<span class="hljs-string">&quot;rouge&quot;</span>)`}}),Wn=new z({props:{code:`scores = rouge_score.compute(
    predictions=[generated_summary], references=[reference_summary]
)
scores`,highlighted:`scores = rouge_score.compute(
    predictions=[generated_summary], references=[reference_summary]
)
scores`}}),Bn=new z({props:{code:`{'rouge1': AggregateScore(low=Score(precision=0.86, recall=1.0, fmeasure=0.92), mid=Score(precision=0.86, recall=1.0, fmeasure=0.92), high=Score(precision=0.86, recall=1.0, fmeasure=0.92)),
 'rouge2': AggregateScore(low=Score(precision=0.67, recall=0.8, fmeasure=0.73), mid=Score(precision=0.67, recall=0.8, fmeasure=0.73), high=Score(precision=0.67, recall=0.8, fmeasure=0.73)),
 'rougeL': AggregateScore(low=Score(precision=0.86, recall=1.0, fmeasure=0.92), mid=Score(precision=0.86, recall=1.0, fmeasure=0.92), high=Score(precision=0.86, recall=1.0, fmeasure=0.92)),
 'rougeLsum': AggregateScore(low=Score(precision=0.86, recall=1.0, fmeasure=0.92), mid=Score(precision=0.86, recall=1.0, fmeasure=0.92), high=Score(precision=0.86, recall=1.0, fmeasure=0.92))}`,highlighted:`{<span class="hljs-string">&#x27;rouge1&#x27;</span>: AggregateScore(low=Score(precision=<span class="hljs-number">0.86</span>, recall=<span class="hljs-number">1.0</span>, fmeasure=<span class="hljs-number">0.92</span>), mid=Score(precision=<span class="hljs-number">0.86</span>, recall=<span class="hljs-number">1.0</span>, fmeasure=<span class="hljs-number">0.92</span>), high=Score(precision=<span class="hljs-number">0.86</span>, recall=<span class="hljs-number">1.0</span>, fmeasure=<span class="hljs-number">0.92</span>)),
 <span class="hljs-string">&#x27;rouge2&#x27;</span>: AggregateScore(low=Score(precision=<span class="hljs-number">0.67</span>, recall=<span class="hljs-number">0.8</span>, fmeasure=<span class="hljs-number">0.73</span>), mid=Score(precision=<span class="hljs-number">0.67</span>, recall=<span class="hljs-number">0.8</span>, fmeasure=<span class="hljs-number">0.73</span>), high=Score(precision=<span class="hljs-number">0.67</span>, recall=<span class="hljs-number">0.8</span>, fmeasure=<span class="hljs-number">0.73</span>)),
 <span class="hljs-string">&#x27;rougeL&#x27;</span>: AggregateScore(low=Score(precision=<span class="hljs-number">0.86</span>, recall=<span class="hljs-number">1.0</span>, fmeasure=<span class="hljs-number">0.92</span>), mid=Score(precision=<span class="hljs-number">0.86</span>, recall=<span class="hljs-number">1.0</span>, fmeasure=<span class="hljs-number">0.92</span>), high=Score(precision=<span class="hljs-number">0.86</span>, recall=<span class="hljs-number">1.0</span>, fmeasure=<span class="hljs-number">0.92</span>)),
 <span class="hljs-string">&#x27;rougeLsum&#x27;</span>: AggregateScore(low=Score(precision=<span class="hljs-number">0.86</span>, recall=<span class="hljs-number">1.0</span>, fmeasure=<span class="hljs-number">0.92</span>), mid=Score(precision=<span class="hljs-number">0.86</span>, recall=<span class="hljs-number">1.0</span>, fmeasure=<span class="hljs-number">0.92</span>), high=Score(precision=<span class="hljs-number">0.86</span>, recall=<span class="hljs-number">1.0</span>, fmeasure=<span class="hljs-number">0.92</span>))}`}}),Kn=new z({props:{code:'scores["rouge1"].mid',highlighted:'scores[<span class="hljs-string">&quot;rouge1&quot;</span>].mid'}}),Yn=new z({props:{code:"Score(precision=0.86, recall=1.0, fmeasure=0.92)",highlighted:'Score(precision=<span class="hljs-number">0.86</span>, recall=<span class="hljs-number">1.0</span>, fmeasure=<span class="hljs-number">0.92</span>)'}}),Ya=new pn({props:{$$slots:{default:[Sg]},$$scope:{ctx:Y}}}),Vn=new Ks({}),Jn=new z({props:{code:"!pip install nltk",highlighted:"!pip install nltk"}}),Xn=new z({props:{code:`import nltk

nltk.download("punkt")`,highlighted:`<span class="hljs-keyword">import</span> nltk

nltk.download(<span class="hljs-string">&quot;punkt&quot;</span>)`}}),Zn=new z({props:{code:`from nltk.tokenize import sent_tokenize


def three_sentence_summary(text):
    return "\\n".join(sent_tokenize(text)[:3])


print(three_sentence_summary(books_dataset["train"][1]["review_body"]))`,highlighted:`<span class="hljs-keyword">from</span> nltk.tokenize <span class="hljs-keyword">import</span> sent_tokenize


<span class="hljs-keyword">def</span> <span class="hljs-title function_">three_sentence_summary</span>(<span class="hljs-params">text</span>):
    <span class="hljs-keyword">return</span> <span class="hljs-string">&quot;\\n&quot;</span>.join(sent_tokenize(text)[:<span class="hljs-number">3</span>])


<span class="hljs-built_in">print</span>(three_sentence_summary(books_dataset[<span class="hljs-string">&quot;train&quot;</span>][<span class="hljs-number">1</span>][<span class="hljs-string">&quot;review_body&quot;</span>]))`}}),Qn=new z({props:{code:`'I grew up reading Koontz, and years ago, I stopped,convinced i had "outgrown" him.'
'Still,when a friend was looking for something suspenseful too read, I suggested Koontz.'
'She found Strangers.'`,highlighted:`<span class="hljs-string">&#x27;I grew up reading Koontz, and years ago, I stopped,convinced i had &quot;outgrown&quot; him.&#x27;</span>
<span class="hljs-string">&#x27;Still,when a friend was looking for something suspenseful too read, I suggested Koontz.&#x27;</span>
<span class="hljs-string">&#x27;She found Strangers.&#x27;</span>`}}),eo=new z({props:{code:`def evaluate_baseline(dataset, metric):
    summaries = [three_sentence_summary(text) for text in dataset["review_body"]]
    return metric.compute(predictions=summaries, references=dataset["review_title"])`,highlighted:`<span class="hljs-keyword">def</span> <span class="hljs-title function_">evaluate_baseline</span>(<span class="hljs-params">dataset, metric</span>):
    summaries = [three_sentence_summary(text) <span class="hljs-keyword">for</span> text <span class="hljs-keyword">in</span> dataset[<span class="hljs-string">&quot;review_body&quot;</span>]]
    <span class="hljs-keyword">return</span> metric.compute(predictions=summaries, references=dataset[<span class="hljs-string">&quot;review_title&quot;</span>])`}}),so=new z({props:{code:`import pandas as pd

score = evaluate_baseline(books_dataset["validation"], rouge_score)
rouge_names = ["rouge1", "rouge2", "rougeL", "rougeLsum"]
rouge_dict = dict((rn, round(score[rn].mid.fmeasure * 100, 2)) for rn in rouge_names)
rouge_dict`,highlighted:`<span class="hljs-keyword">import</span> pandas <span class="hljs-keyword">as</span> pd

score = evaluate_baseline(books_dataset[<span class="hljs-string">&quot;validation&quot;</span>], rouge_score)
rouge_names = [<span class="hljs-string">&quot;rouge1&quot;</span>, <span class="hljs-string">&quot;rouge2&quot;</span>, <span class="hljs-string">&quot;rougeL&quot;</span>, <span class="hljs-string">&quot;rougeLsum&quot;</span>]
rouge_dict = <span class="hljs-built_in">dict</span>((rn, <span class="hljs-built_in">round</span>(score[rn].mid.fmeasure * <span class="hljs-number">100</span>, <span class="hljs-number">2</span>)) <span class="hljs-keyword">for</span> rn <span class="hljs-keyword">in</span> rouge_names)
rouge_dict`}}),to=new z({props:{code:"{'rouge1': 16.74, 'rouge2': 8.83, 'rougeL': 15.6, 'rougeLsum': 15.96}",highlighted:'{<span class="hljs-string">&#x27;rouge1&#x27;</span>: <span class="hljs-number">16.74</span>, <span class="hljs-string">&#x27;rouge2&#x27;</span>: <span class="hljs-number">8.83</span>, <span class="hljs-string">&#x27;rougeL&#x27;</span>: <span class="hljs-number">15.6</span>, <span class="hljs-string">&#x27;rougeLsum&#x27;</span>: <span class="hljs-number">15.96</span>}'}});const Eu=[Og,Ag],_o=[];function Tu(e,p){return e[0]==="pt"?0:1}dt=Tu(Y),ft=_o[dt]=Eu[dt](Y),Za=new pn({props:{$$slots:{default:[Cg]},$$scope:{ctx:Y}}}),ao=new z({props:{code:`from huggingface_hub import notebook_login

notebook_login()`,highlighted:`<span class="hljs-keyword">from</span> huggingface_hub <span class="hljs-keyword">import</span> notebook_login

notebook_login()`}}),no=new z({props:{code:"huggingface-cli login",highlighted:'huggingface-<span class="hljs-keyword">cli</span> login'}});let Pe=Y[0]==="pt"&&ug();const qu=[Lg,Pg],wo=[];function zu(e,p){return e[0]==="pt"?0:1}gt=zu(Y),_t=wo[gt]=qu[gt](Y),oo=new z({props:{code:`tokenized_datasets = tokenized_datasets.remove_columns(
    books_dataset["train"].column_names
)`,highlighted:`tokenized_datasets = tokenized_datasets.remove_columns(
    books_dataset[<span class="hljs-string">&quot;train&quot;</span>].column_names
)`}}),ro=new z({props:{code:`features = [tokenized_datasets["train"][i] for i in range(2)]
data_collator(features)`,highlighted:`features = [tokenized_datasets[<span class="hljs-string">&quot;train&quot;</span>][i] <span class="hljs-keyword">for</span> i <span class="hljs-keyword">in</span> <span class="hljs-built_in">range</span>(<span class="hljs-number">2</span>)]
data_collator(features)`}}),lo=new z({props:{code:`{'attention_mask': tensor([[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
         1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0],
        [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
         1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]]), 'input_ids': tensor([[  1494,    259,   8622,    390,    259,    262,   2316,   3435,    955,
            772,    281,    772,   1617,    263,    305,  14701,    260,   1385,
           3031,    259,  24146,    332,   1037,    259,  43906,    305,    336,
            260,      1,      0,      0,      0,      0,      0,      0],
        [   259,  27531,  13483,    259,   7505,    260, 112240,  15192,    305,
          53198,    276,    259,  74060,    263,    260,    459,  25640,    776,
           2119,    336,    259,   2220,    259,  18896,    288,   4906,    288,
           1037,   3931,    260,   7083, 101476,   1143,    260,      1]]), 'labels': tensor([[ 7483,   259,  2364, 15695,     1,  -100],
        [  259, 27531, 13483,   259,  7505,     1]]), 'decoder_input_ids': tensor([[    0,  7483,   259,  2364, 15695,     1],
        [    0,   259, 27531, 13483,   259,  7505]])}`,highlighted:`{<span class="hljs-string">&#x27;attention_mask&#x27;</span>: tensor([[<span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>,
         <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>],
        [<span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>,
         <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>]]), <span class="hljs-string">&#x27;input_ids&#x27;</span>: tensor([[  <span class="hljs-number">1494</span>,    <span class="hljs-number">259</span>,   <span class="hljs-number">8622</span>,    <span class="hljs-number">390</span>,    <span class="hljs-number">259</span>,    <span class="hljs-number">262</span>,   <span class="hljs-number">2316</span>,   <span class="hljs-number">3435</span>,    <span class="hljs-number">955</span>,
            <span class="hljs-number">772</span>,    <span class="hljs-number">281</span>,    <span class="hljs-number">772</span>,   <span class="hljs-number">1617</span>,    <span class="hljs-number">263</span>,    <span class="hljs-number">305</span>,  <span class="hljs-number">14701</span>,    <span class="hljs-number">260</span>,   <span class="hljs-number">1385</span>,
           <span class="hljs-number">3031</span>,    <span class="hljs-number">259</span>,  <span class="hljs-number">24146</span>,    <span class="hljs-number">332</span>,   <span class="hljs-number">1037</span>,    <span class="hljs-number">259</span>,  <span class="hljs-number">43906</span>,    <span class="hljs-number">305</span>,    <span class="hljs-number">336</span>,
            <span class="hljs-number">260</span>,      <span class="hljs-number">1</span>,      <span class="hljs-number">0</span>,      <span class="hljs-number">0</span>,      <span class="hljs-number">0</span>,      <span class="hljs-number">0</span>,      <span class="hljs-number">0</span>,      <span class="hljs-number">0</span>],
        [   <span class="hljs-number">259</span>,  <span class="hljs-number">27531</span>,  <span class="hljs-number">13483</span>,    <span class="hljs-number">259</span>,   <span class="hljs-number">7505</span>,    <span class="hljs-number">260</span>, <span class="hljs-number">112240</span>,  <span class="hljs-number">15192</span>,    <span class="hljs-number">305</span>,
          <span class="hljs-number">53198</span>,    <span class="hljs-number">276</span>,    <span class="hljs-number">259</span>,  <span class="hljs-number">74060</span>,    <span class="hljs-number">263</span>,    <span class="hljs-number">260</span>,    <span class="hljs-number">459</span>,  <span class="hljs-number">25640</span>,    <span class="hljs-number">776</span>,
           <span class="hljs-number">2119</span>,    <span class="hljs-number">336</span>,    <span class="hljs-number">259</span>,   <span class="hljs-number">2220</span>,    <span class="hljs-number">259</span>,  <span class="hljs-number">18896</span>,    <span class="hljs-number">288</span>,   <span class="hljs-number">4906</span>,    <span class="hljs-number">288</span>,
           <span class="hljs-number">1037</span>,   <span class="hljs-number">3931</span>,    <span class="hljs-number">260</span>,   <span class="hljs-number">7083</span>, <span class="hljs-number">101476</span>,   <span class="hljs-number">1143</span>,    <span class="hljs-number">260</span>,      <span class="hljs-number">1</span>]]), <span class="hljs-string">&#x27;labels&#x27;</span>: tensor([[ <span class="hljs-number">7483</span>,   <span class="hljs-number">259</span>,  <span class="hljs-number">2364</span>, <span class="hljs-number">15695</span>,     <span class="hljs-number">1</span>,  -<span class="hljs-number">100</span>],
        [  <span class="hljs-number">259</span>, <span class="hljs-number">27531</span>, <span class="hljs-number">13483</span>,   <span class="hljs-number">259</span>,  <span class="hljs-number">7505</span>,     <span class="hljs-number">1</span>]]), <span class="hljs-string">&#x27;decoder_input_ids&#x27;</span>: tensor([[    <span class="hljs-number">0</span>,  <span class="hljs-number">7483</span>,   <span class="hljs-number">259</span>,  <span class="hljs-number">2364</span>, <span class="hljs-number">15695</span>,     <span class="hljs-number">1</span>],
        [    <span class="hljs-number">0</span>,   <span class="hljs-number">259</span>, <span class="hljs-number">27531</span>, <span class="hljs-number">13483</span>,   <span class="hljs-number">259</span>,  <span class="hljs-number">7505</span>]])}`}});const Du=[Rg,Ig],bo=[];function Su(e,p){return e[0]==="pt"?0:1}wt=Su(Y),bt=bo[wt]=Du[wt](Y);let Le=Y[0]==="pt"&&dg(Y);return io=new Ks({}),po=new z({props:{code:`from transformers import pipeline

hub_model_id = "huggingface-course/mt5-small-finetuned-amazon-en-es"
summarizer = pipeline("summarization", model=hub_model_id)`,highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> pipeline

hub_model_id = <span class="hljs-string">&quot;huggingface-course/mt5-small-finetuned-amazon-en-es&quot;</span>
summarizer = pipeline(<span class="hljs-string">&quot;summarization&quot;</span>, model=hub_model_id)`}}),mo=new z({props:{code:`def print_summary(idx):
    review = books_dataset["test"][idx]["review_body"]
    title = books_dataset["test"][idx]["review_title"]
    summary = summarizer(books_dataset["test"][idx]["review_body"])[0]["summary_text"]
    print(f"'>>> Review: {review}'")
    print(f"\\n'>>> Title: {title}'")
    print(f"\\n'>>> Summary: {summary}'")`,highlighted:`<span class="hljs-keyword">def</span> <span class="hljs-title function_">print_summary</span>(<span class="hljs-params">idx</span>):
    review = books_dataset[<span class="hljs-string">&quot;test&quot;</span>][idx][<span class="hljs-string">&quot;review_body&quot;</span>]
    title = books_dataset[<span class="hljs-string">&quot;test&quot;</span>][idx][<span class="hljs-string">&quot;review_title&quot;</span>]
    summary = summarizer(books_dataset[<span class="hljs-string">&quot;test&quot;</span>][idx][<span class="hljs-string">&quot;review_body&quot;</span>])[<span class="hljs-number">0</span>][<span class="hljs-string">&quot;summary_text&quot;</span>]
    <span class="hljs-built_in">print</span>(<span class="hljs-string">f&quot;&#x27;&gt;&gt;&gt; Review: <span class="hljs-subst">{review}</span>&#x27;&quot;</span>)
    <span class="hljs-built_in">print</span>(<span class="hljs-string">f&quot;\\n&#x27;&gt;&gt;&gt; Title: <span class="hljs-subst">{title}</span>&#x27;&quot;</span>)
    <span class="hljs-built_in">print</span>(<span class="hljs-string">f&quot;\\n&#x27;&gt;&gt;&gt; Summary: <span class="hljs-subst">{summary}</span>&#x27;&quot;</span>)`}}),ho=new z({props:{code:"print_summary(100)",highlighted:'print_summary(<span class="hljs-number">100</span>)'}}),co=new z({props:{code:`'>>> Review: Nothing special at all about this product... the book is too small and stiff and hard to write in. The huge sticker on the back doesn\u2019t come off and looks super tacky. I would not purchase this again. I could have just bought a journal from the dollar store and it would be basically the same thing. It\u2019s also really expensive for what it is.'

'>>> Title: Not impressed at all... buy something else'

'>>> Summary: Nothing special at all about this product'`,highlighted:`<span class="hljs-string">&#x27;&gt;&gt;&gt; Review: Nothing special at all about this product... the book is too small and stiff and hard to write in. The huge sticker on the back doesn\u2019t come off and looks super tacky. I would not purchase this again. I could have just bought a journal from the dollar store and it would be basically the same thing. It\u2019s also really expensive for what it is.&#x27;</span>

<span class="hljs-string">&#x27;&gt;&gt;&gt; Title: Not impressed at all... buy something else&#x27;</span>

<span class="hljs-string">&#x27;&gt;&gt;&gt; Summary: Nothing special at all about this product&#x27;</span>`}}),uo=new z({props:{code:"print_summary(0)",highlighted:'print_summary(<span class="hljs-number">0</span>)'}}),fo=new z({props:{code:`'>>> Review: Es una trilogia que se hace muy facil de leer. Me ha gustado, no me esperaba el final para nada'

'>>> Title: Buena literatura para adolescentes'

'>>> Summary: Muy facil de leer'`,highlighted:`<span class="hljs-string">&#x27;&gt;&gt;&gt; Review: Es una trilogia que se hace muy facil de leer. Me ha gustado, no me esperaba el final para nada&#x27;</span>

<span class="hljs-string">&#x27;&gt;&gt;&gt; Title: Buena literatura para adolescentes&#x27;</span>

<span class="hljs-string">&#x27;&gt;&gt;&gt; Summary: Muy facil de leer&#x27;</span>`}}),{c(){u=o("meta"),j=h(),v(d.$$.fragment),E=h(),C=o("h1"),x=o("a"),A=o("span"),v(P.$$.fragment),T=h(),S=o("span"),I=a("Summarization"),D=h(),G.c(),J=h(),X=o("p"),Z=a("In this section we\u2019ll take a look at how Transformer models can be used to condense long documents into summaries, a task known as "),B=o("em"),Q=a("text summarization"),L=a(". This is one of the most challenging NLP tasks as it requires a range of abilities, such as understanding long passages and generating coherent text that captures the main topics in a document. However, when done well, text summarization is a powerful tool that can speed up various business processes by relieving the burden of domain experts to read long documents in detail."),M=h(),v(ee.$$.fragment),F=h(),K=o("p"),te=a("Although there already exist various fine-tuned models for summarization on the "),W=o("a"),ie=a("Hugging Face Hub"),V=a(", almost all of these are only suitable for English documents. So, to add a twist in this section, we\u2019ll train a bilingual model for English and Spanish. By the end of this section, you\u2019ll have a "),ae=o("a"),de=a("model"),_e=a(" that can summarize customer reviews like the one shown here:"),Te=h(),H=o("iframe"),me=h(),oe=o("p"),we=a("As we\u2019ll see, these summaries are concise because they\u2019re learned from the titles that customers provide in their product reviews. Let\u2019s start by putting together a suitable bilingual corpus for this task."),ke=h(),se=o("h2"),ge=o("a"),he=o("span"),v(fe.$$.fragment),w=h(),R=o("span"),be=a("Preparing a multilingual corpus"),b=h(),U=o("p"),qe=a("We\u2019ll use the "),pe=o("a"),$e=a("Multilingual Amazon Reviews Corpus"),Ce=a(" to create our bilingual summarizer. This corpus consists of Amazon product reviews in six languages and is typically used to benchmark multilingual classifiers. However, since each review is accompanied by a short title, we can use the titles as the target summaries for our model to learn from! To get started, let\u2019s download the English and Spanish subsets from the Hugging Face Hub:"),re=h(),v(ze.$$.fragment),Qe=h(),v(Ae.$$.fragment),ne=h(),le=o("p"),Ie=a("As you can see, for each language there are 200,000 reviews for the "),ws=o("code"),bs=a("train"),vs=a(" split, and 5,000 reviews for each of the "),ys=o("code"),zs=a("validation"),He=a(" and "),rs=o("code"),es=a("test"),ls=a(" splits. The review information we are interested in is contained in the "),Ue=o("code"),Ys=a("review_body"),is=a(" and "),Oe=o("code"),ks=a("review_title"),Ds=a(" columns. Let\u2019s take a look at a few examples by creating a simple function that takes a random sample from the training set with the techniques we learned in "),Re=o("a"),ps=a("Chapter 5"),N=a(":"),ce=h(),v(Me.$$.fragment),Vs=h(),v(ue.$$.fragment),ms=h(),v(ss.$$.fragment),Ss=h(),De=o("p"),Js=a("This sample shows the diversity of reviews one typically finds online, ranging from positive to negative (and everything in between!). Although the example with the \u201Cmeh\u201D title is not very informative, the other titles look like decent summaries of the reviews themselves. Training a summarization model on all 400,000 reviews would take far too long on a single GPU, so instead we\u2019ll focus on generating summaries for a single domain of products. To get a feel for what domains we can choose from, let\u2019s convert "),ts=o("code"),je=a("english_dataset"),Xs=a(" to a "),We=o("code"),Zs=a("pandas.DataFrame"),Fe=a(" and compute the number of reviews per product category:"),Qs=h(),v(Be.$$.fragment),et=h(),v(ve.$$.fragment),$s=h(),Se=o("p"),vt=a("The most popular products in the English dataset are about household items, clothing, and wireless electronics. To stick with the Amazon theme, though, let\u2019s focus on summarizing book reviews \u2014 after all, this is what the company was founded on! We can see two product categories that fit the bill ("),js=o("code"),xs=a("book"),mn=a(" and "),yt=o("code"),st=a("digital_ebook_purchase"),fa=a("), so let\u2019s filter the datasets in both languages for just these products. As we saw in "),hs=o("a"),ga=a("Chapter 5"),kt=a(", the "),Ft=o("code"),_a=a("Dataset.filter()"),tt=a(" function allows us to slice a dataset very efficiently, so we can define a simple function to do this:"),Gt=h(),v(Ke.$$.fragment),wa=h(),xe=o("p"),hn=a("Now when we apply this function to "),Nt=o("code"),ba=a("english_dataset"),Es=a(" and "),cs=o("code"),Ht=a("spanish_dataset"),at=a(", the result will contain just those rows involving the book categories. Before applying the filter, let\u2019s switch the format of "),Ut=o("code"),Mt=a("english_dataset"),cn=a(" from "),$t=o("code"),jt=a('"pandas"'),un=a(" back to "),xt=o("code"),Ye=a('"arrow"'),nt=a(":"),va=h(),v(Ts.$$.fragment),ya=h(),Et=o("p"),dn=a("We can then apply the filter function, and as a sanity check let\u2019s inspect a sample of reviews to see if they are indeed about books:"),Tt=h(),v(ot.$$.fragment),ka=h(),v(qs.$$.fragment),$a=h(),Ve=o("p"),as=a("Okay, we can see that the reviews are not strictly about books and might refer to things like calendars and electronic applications such as OneNote. Nevertheless, the domain seems about right to train a summarization model on. Before we look at various models that are suitable for this task, we have one last bit of data preparation to do: combining the English and Spanish reviews as a single "),Wt=o("code"),Bt=a("DatasetDict"),fn=a(" object. \u{1F917} Datasets provides a handy "),Kt=o("code"),Yt=a("concatenate_datasets()"),gn=a(" function that (as the name suggests) will stack two "),Vt=o("code"),Jt=a("Dataset"),_n=a(" objects on top of each other. So, to create our bilingual dataset, we\u2019ll loop over each split, concatenate the datasets for that split, and shuffle the result to ensure our model doesn\u2019t overfit to a single language:"),ja=h(),v(As.$$.fragment),rt=h(),v(lt.$$.fragment),Xt=h(),us=o("p"),xa=a("This certainly looks like a mix of English and Spanish reviews! Now that we have a training corpus, one final thing to check is the distribution of words in the reviews and their titles. This is especially important for summarization tasks, where short reference summaries in the data can bias the model to only output one or two words in the generated summaries. The plots below show the word distributions, and we can see that the titles are heavily skewed toward just 1-2 words:"),Os=h(),ds=o("div"),Je=o("img"),Zt=h(),it=o("img"),Qt=h(),m=o("p"),q=a("To deal with this, we\u2019ll filter out the examples with very short titles so that our model can produce more interesting summaries. Since we\u2019re dealing with English and Spanish texts, we can use a rough heuristic to split the titles on whitespace and then use our trusty "),Ea=o("code"),jo=a("Dataset.filter()"),xo=a(" method as follows:"),pt=h(),v(qt.$$.fragment),wn=h(),mt=o("p"),Eo=a("Now that we\u2019ve prepared our corpus, let\u2019s take a look at a few possible Transformer models that one might fine-tune on it!"),bn=h(),Cs=o("h2"),ns=o("a"),Ta=o("span"),v(zt.$$.fragment),qa=h(),za=o("span"),To=a("Models for text summarization"),ea=h(),ht=o("p"),Da=a("If you think about it, text summarization is a similar sort of task to machine translation: we have a body of text like a review that we\u2019d like to \u201Ctranslate\u201D into a shorter version that captures the salient features of the input. Accordingly, most Transformer models for summarization adopt the encoder-decoder architecture that we first encountered in "),sa=o("a"),qo=a("Chapter 1"),Dt=a(", although there are some exceptions like the GPT family of models which can also be used for summarization in few-shot settings. The following table lists some popular pretrained models that can be fine-tuned for summarization."),vn=h(),ct=o("table"),Sa=o("thead"),Ps=o("tr"),ta=o("th"),St=a("Transformer model"),zo=h(),Aa=o("th"),Do=a("Description"),So=h(),ut=o("th"),Ao=a("Multilingual?"),Oa=h(),Ge=o("tbody"),Ls=o("tr"),aa=o("td"),At=o("a"),Is=a("GPT-2"),Ca=h(),Pa=o("td"),Oo=a("Although trained as an auto-regressive language model, you can make GPT-2 generate summaries by appending \u201CTL;DR\u201D at the end of the input text."),Co=h(),os=o("td"),Po=a("\u274C"),Lo=h(),Rs=o("tr"),na=o("td"),Fs=o("a"),Io=a("PEGASUS"),Xp=h(),Cr=o("td"),Zp=a("Uses a pretraining objective to predict masked sentences in multi-sentence texts. This pretraining objective is closer to summarization than vanilla language modeling and scores highly on popular benchmarks."),Qp=h(),Ro=o("td"),em=a("\u274C"),sm=h(),oa=o("tr"),Fo=o("td"),yn=o("a"),tm=a("T5"),am=h(),kn=o("td"),nm=a("A universal Transformer architecture that formulates all tasks in a text-to-text framework; e.g., the input format for the model to summarize a document is "),Pr=o("code"),om=a("summarize: ARTICLE"),rm=a("."),lm=h(),Go=o("td"),im=a("\u274C"),pm=h(),ra=o("tr"),No=o("td"),$n=o("a"),mm=a("mT5"),hm=h(),Lr=o("td"),cm=a("A multilingual version of T5, pretrained on the multilingual Common Crawl corpus (mC4), covering 101 languages."),um=h(),Ho=o("td"),dm=a("\u2705"),fm=h(),la=o("tr"),Uo=o("td"),jn=o("a"),gm=a("BART"),_m=h(),Ir=o("td"),wm=a("A novel Transformer architecture with both an encoder and a decoder stack trained to reconstruct corrupted input that combines the pretraining schemes of BERT and GPT-2."),bm=h(),Mo=o("td"),vm=a("\u274C"),ym=h(),ia=o("tr"),Wo=o("td"),xn=o("a"),km=a("mBART-50"),$m=h(),Rr=o("td"),jm=a("A multilingual version of BART, pretrained on 50 languages."),xm=h(),Bo=o("td"),Em=a("\u2705"),Kl=h(),Ko=o("p"),Tm=a("As you can see from this table, the majority of Transformer models for summarization (and indeed most NLP tasks) are monolingual. This is great if your task is in a \u201Chigh-resource\u201D language like English or German, but less so for the thousands of other languages in use across the world. Fortunately, there is a class of multilingual Transformer models, like mT5 and mBART, that come to the rescue. These models are pretrained using language modeling, but with a twist: instead of training on a corpus of one language, they are trained jointly on texts in over 50 languages at once!"),Yl=h(),La=o("p"),qm=a("We\u2019ll focus on mT5, an interesting architecture based on T5 that was pretrained in a text-to-text framework. In T5, every NLP task is formulated in terms of a prompt prefix like "),Fr=o("code"),zm=a("summarize:"),Dm=a(" which conditions the model to adapt the generated text to the prompt. As shown in the figure below, this makes T5 extremely versatile, as you can solve many tasks with a single model!"),Vl=h(),pa=o("div"),En=o("img"),Sm=h(),Tn=o("img"),Jl=h(),Yo=o("p"),Am=a("mT5 doesn\u2019t use prefixes, but shares much of the versatility of T5 and has the advantage of being multilingual. Now that we\u2019ve picked a model, let\u2019s take a look at preparing our data for training."),Xl=h(),v(Ia.$$.fragment),Zl=h(),ma=o("h2"),Ra=o("a"),Gr=o("span"),v(qn.$$.fragment),Om=h(),Nr=o("span"),Cm=a("Preprocessing the data"),Ql=h(),v(zn.$$.fragment),ei=h(),Fa=o("p"),Pm=a("Our next task is to tokenize and encode our reviews and their titles. As usual, we begin by loading the tokenizer associated with the pretrained model checkpoint. We\u2019ll use "),Hr=o("code"),Lm=a("mt5-small"),Im=a(" as our checkpoint so we can fine-tune the model in a reasonable amount of time:"),si=h(),v(Dn.$$.fragment),ti=h(),v(Ga.$$.fragment),ai=h(),Vo=o("p"),Rm=a("Let\u2019s test out the mT5 tokenizer on a small example:"),ni=h(),v(Sn.$$.fragment),oi=h(),v(An.$$.fragment),ri=h(),fs=o("p"),Fm=a("Here we can see the familiar "),Ur=o("code"),Gm=a("input_ids"),Nm=a(" and "),Mr=o("code"),Hm=a("attention_mask"),Um=a(" that we encountered in our first fine-tuning experiments back in "),Jo=o("a"),Mm=a("Chapter 3"),Wm=a(". Let\u2019s decode these input IDs with the tokenizer\u2019s "),Wr=o("code"),Bm=a("convert_ids_to_tokens()"),Km=a(" function to see what kind of tokenizer we\u2019re dealing with:"),li=h(),v(On.$$.fragment),ii=h(),v(Cn.$$.fragment),pi=h(),Gs=o("p"),Ym=a("The special Unicode character "),Br=o("code"),Vm=a("\u2581"),Jm=a(" and end-of-sequence token "),Kr=o("code"),Xm=a("</s>"),Zm=a(" indicate that we\u2019re dealing with the SentencePiece tokenizer, which is based on the Unigram segmentation algorithm discussed in "),Xo=o("a"),Qm=a("Chapter 6"),eh=a(". Unigram is especially useful for multilingual corpora since it allows SentencePiece to be agnostic about accents, punctuation, and the fact that many languages, like Japanese, do not have whitespace characters."),mi=h(),Na=o("p"),sh=a("To tokenize our corpus, we have to deal with a subtlety associated with summarization: because our labels are also text, it is possible that they exceed the model\u2019s maximum context size. This means we need to apply truncation to both the reviews and their titles to ensure we don\u2019t pass excessively long inputs to our model. The tokenizers in \u{1F917} Transformers provide a nifty "),Yr=o("code"),th=a("as_target_tokenizer()"),ah=a(" function that allows you to tokenize the labels in parallel to the inputs. This is typically done using a context manager inside a preprocessing function that first encodes the inputs, and then encodes the labels as a separate column. Here is an example of such a function for mT5:"),hi=h(),v(Pn.$$.fragment),ci=h(),gs=o("p"),nh=a("Let\u2019s walk through this code to understand what\u2019s happening. The first thing we\u2019ve done is define values for "),Vr=o("code"),oh=a("max_input_length"),rh=a(" and "),Jr=o("code"),lh=a("max_target_length"),ih=a(", which set the upper limits for how long our reviews and titles can be. Since the review body is typically much larger than the title, we\u2019ve scaled these values accordingly. Then, in the  "),Xr=o("code"),ph=a("preprocess_function()"),mh=a(" itself we can see the reviews are first tokenized, followed by the titles with "),Zr=o("code"),hh=a("as_target_tokenizer()"),ch=a("."),ui=h(),Ot=o("p"),uh=a("With "),Qr=o("code"),dh=a("preprocess_function()"),fh=a(", it is then a simple matter to tokenize the whole corpus using the handy "),el=o("code"),gh=a("Dataset.map()"),_h=a(" function we\u2019ve used extensively throughout this course:"),di=h(),v(Ln.$$.fragment),fi=h(),Zo=o("p"),wh=a("Now that the corpus has been preprocessed, let\u2019s take a look at some metrics that are commonly used for summarization. As we\u2019ll see, there is no silver bullet when it comes to measuring the quality of machine-generated text."),gi=h(),v(Ha.$$.fragment),_i=h(),ha=o("h2"),Ua=o("a"),sl=o("span"),v(In.$$.fragment),bh=h(),tl=o("span"),vh=a("Metrics for text summarization"),wi=h(),v(Rn.$$.fragment),bi=h(),Qo=o("p"),yh=a("In comparison to most of the other tasks we\u2019ve covered in this course, measuring the performance of text generation tasks like summarization or translation is not as straightforward. For example, given a review like \u201CI loved reading the Hunger Games\u201D, there are multiple valid summaries, like \u201CI loved the Hunger Games\u201D or \u201CHunger Games is a great read\u201D. Clearly, applying some sort of exact match between the generated summary and the label is not a good solution \u2014 even humans would fare poorly under such a metric, because we all have our own writing style."),vi=h(),Ma=o("p"),kh=a("For summarization, one of the most commonly used metrics is the "),Fn=o("a"),$h=a("ROUGE score"),jh=a(" (short for Recall-Oriented Understudy for Gisting Evaluation). The basic idea behind this metric is to compare a generated summary against a set of reference summaries that are typically created by humans. To make this more precise, suppose we want to compare the following two summaries:"),yi=h(),v(Gn.$$.fragment),ki=h(),Ct=o("p"),xh=a("One way to compare them could be to count the number of overlapping words, which in this case would be 6. However, this is a bit crude, so instead ROUGE is based on computing the "),al=o("em"),Eh=a("precision"),Th=a(" and "),nl=o("em"),qh=a("recall"),zh=a(" scores for the overlap."),$i=h(),v(Wa.$$.fragment),ji=h(),Nn=o("p"),Dh=a(`For ROUGE, recall measures how much of the reference summary is captured by the generated one. If we are just comparing words, recall can be calculated according to the following formula:
`),xi=new hg,Ei=h(),Hn=o("p"),Sh=a(`For our simple example above, this formula gives a perfect recall of 6/6 = 1; i.e., all the words in the reference summary have been produced by the model. This may sound great, but imagine if our generated summary had been \u201CI really really loved reading the Hunger Games all night\u201D. This would also have perfect recall, but is arguably a worse summary since it is verbose. To deal with these scenarios we also compute the precision, which in the ROUGE context measures how much of the generated summary was relevant:
`),Ti=new hg,qi=h(),Ba=o("p"),Ah=a("Applying this to our verbose summary gives a precision of 6/10  = 0.6, which is considerably worse than the precision of 6/7 = 0.86 obtained by our shorter one. In practice, both precision and recall are usually computed, and then the F1-score (the harmonic mean of precision and recall) is reported. We can do this easily in \u{1F917} Datasets by first installing the "),ol=o("code"),Oh=a("rouge_score"),Ch=a(" package:"),zi=h(),v(Un.$$.fragment),Di=h(),er=o("p"),Ph=a("and then loading the ROUGE metric as follows:"),Si=h(),v(Mn.$$.fragment),Ai=h(),Ka=o("p"),Lh=a("Then we can use the "),rl=o("code"),Ih=a("rouge_score.compute()"),Rh=a(" function to calculate all the metrics at once:"),Oi=h(),v(Wn.$$.fragment),Ci=h(),v(Bn.$$.fragment),Pi=h(),Xe=o("p"),Fh=a("Whoa, there\u2019s a lot of information in that output \u2014 what does it all mean? First, \u{1F917} Datasets actually computes confidence intervals for precision, recall, and F1-score; these are the "),ll=o("code"),Gh=a("low"),Nh=a(", "),il=o("code"),Hh=a("mid"),Uh=a(", and "),pl=o("code"),Mh=a("high"),Wh=a(" attributes you can see here. Moreover, \u{1F917} Datasets computes a variety of ROUGE scores which are based on different types of text granularity when comparing the generated and reference summaries. The "),ml=o("code"),Bh=a("rouge1"),Kh=a(" variant is the overlap of unigrams \u2014 this is just a fancy way of saying the overlap of words and is exactly the metric we\u2019ve discussed above. To verify this, let\u2019s pull out the "),hl=o("code"),Yh=a("mid"),Vh=a(" value of our scores:"),Li=h(),v(Kn.$$.fragment),Ii=h(),v(Yn.$$.fragment),Ri=h(),Ze=o("p"),Jh=a("Great, the precision and recall numbers match up! Now what about those other ROUGE scores? "),cl=o("code"),Xh=a("rouge2"),Zh=a(" measures the overlap between bigrams (think the overlap of pairs of words), while "),ul=o("code"),Qh=a("rougeL"),ec=a(" and "),dl=o("code"),sc=a("rougeLsum"),tc=a(" measure the longest matching sequences of words by looking for the longest common substrings in the generated and reference summaries. The \u201Csum\u201D in "),fl=o("code"),ac=a("rougeLsum"),nc=a(" refers to the fact that this metric is computed over a whole summary, while "),gl=o("code"),oc=a("rougeL"),rc=a(" is computed as the average over individual sentences."),Fi=h(),v(Ya.$$.fragment),Gi=h(),sr=o("p"),lc=a("We\u2019ll use these ROUGE scores to track the performance of our model, but before doing that let\u2019s do something every good NLP practitioner should do: create a strong, yet simple baseline!"),Ni=h(),ca=o("h3"),Va=o("a"),_l=o("span"),v(Vn.$$.fragment),ic=h(),wl=o("span"),pc=a("Creating a strong baseline"),Hi=h(),Ns=o("p"),mc=a("A common baseline for text summarization is to simply take the first three sentences of an article, often called the "),bl=o("em"),hc=a("lead-3"),cc=a(" baseline. We could use full stops to track the sentence boundaries, but this will fail on acronyms like \u201CU.S.\u201D or \u201CU.N.\u201D \u2014 so instead we\u2019ll use the "),vl=o("code"),uc=a("nltk"),dc=a(" library, which includes a better algorithm to handle these cases. You can install the package using "),yl=o("code"),fc=a("pip"),gc=a(" as follows:"),Ui=h(),v(Jn.$$.fragment),Mi=h(),tr=o("p"),_c=a("and then download the punctuation rules:"),Wi=h(),v(Xn.$$.fragment),Bi=h(),Ja=o("p"),wc=a("Next, we import the sentence tokenizer from "),kl=o("code"),bc=a("nltk"),vc=a(" and create a simple function to extract the first three sentences in a review. The convention in text summarization is to separate each summary with a newline, so let\u2019s also include this and test it on a training example:"),Ki=h(),v(Zn.$$.fragment),Yi=h(),v(Qn.$$.fragment),Vi=h(),ar=o("p"),yc=a("This seems to work, so let\u2019s now implement a function that extracts these \u201Csummaries\u201D from a dataset and computes the ROUGE scores for the baseline:"),Ji=h(),v(eo.$$.fragment),Xi=h(),nr=o("p"),kc=a("We can then use this function to compute the ROUGE scores over the validation set and prettify them a bit using Pandas:"),Zi=h(),v(so.$$.fragment),Qi=h(),v(to.$$.fragment),ep=h(),Xa=o("p"),$c=a("We can see that the "),$l=o("code"),jc=a("rouge2"),xc=a(" score is significantly lower than the rest; this likely reflects the fact that review titles are typically concise and so the lead-3 baseline is too verbose. Now that we have a good baseline to work from, let\u2019s turn our attention toward fine-tuning mT5!"),sp=h(),ft.c(),or=h(),v(Za.$$.fragment),tp=h(),rr=o("p"),Ec=a("The next thing we need to do is log in to the Hugging Face Hub. If you\u2019re running this code in a notebook, you can do so with the following utility function:"),ap=h(),v(ao.$$.fragment),np=h(),lr=o("p"),Tc=a("which will display a widget where you can enter your credentials. Alternatively, you can run this command in your terminal and log in there:"),op=h(),v(no.$$.fragment),rp=h(),Pe&&Pe.c(),ir=h(),Qa=o("p"),qc=a("Next, we need to define a data collator for our sequence-to-sequence task. Since mT5 is an encoder-decoder Transformer model, one subtlety with preparing our batches is that during decoding we need to shift the labels to the right by one. This is required to ensure that the decoder only sees the previous ground truth labels and not the current or future ones, which would be easy for the model to memorize. This is similar to how masked self-attention is applied to the inputs in a task like "),pr=o("a"),zc=a("causal language modeling"),Dc=a("."),lp=h(),Hs=o("p"),Sc=a("Luckily, \u{1F917} Transformers provides a "),jl=o("code"),Ac=a("DataCollatorForSeq2Seq"),Oc=a(" collator that will dynamically pad the inputs and the labels for us. To instantiate this collator, we simply need to provide the "),xl=o("code"),Cc=a("tokenizer"),Pc=a(" and "),El=o("code"),Lc=a("model"),Ic=a(":"),ip=h(),_t.c(),mr=h(),hr=o("p"),Rc=a("Let\u2019s see what this collator produces when fed a small batch of examples. First, we need to remove the columns with strings because the collator won\u2019t know how to pad these elements:"),pp=h(),v(oo.$$.fragment),mp=h(),Pt=o("p"),Fc=a("Since the collator expects a list of "),Tl=o("code"),Gc=a("dict"),Nc=a("s, where each "),ql=o("code"),Hc=a("dict"),Uc=a(" represents a single example in the dataset, we also need to wrangle the data into the expected format before passing it to the data collator:"),hp=h(),v(ro.$$.fragment),cp=h(),v(lo.$$.fragment),up=h(),Ee=o("p"),Mc=a("The main thing to notice here is that the first example is longer than the second one, so the "),zl=o("code"),Wc=a("input_ids"),Bc=a(" and "),Dl=o("code"),Kc=a("attention_mask"),Yc=a(" of the second example have been padded on the right with a "),Sl=o("code"),Vc=a("[PAD]"),Jc=a(" token (whose ID is "),Al=o("code"),Xc=a("0"),Zc=a("). Similarly, we can see that the "),Ol=o("code"),Qc=a("labels"),eu=a(" have been padded with "),Cl=o("code"),su=a("-100"),tu=a("s, to make sure the padding tokens are ignored by the loss function. And finally, we can see a new "),Pl=o("code"),au=a("decoder_input_ids"),nu=a(" which has shifted the labels to the right by inserting a "),Ll=o("code"),ou=a("[PAD]"),ru=a(" token in the first entry."),dp=h(),bt.c(),cr=h(),Le&&Le.c(),ur=h(),ua=o("h2"),en=o("a"),Il=o("span"),v(io.$$.fragment),lu=h(),Rl=o("span"),iu=a("Using your fine-tuned model"),fp=h(),sn=o("p"),pu=a("Once you\u2019ve pushed the model to the Hub, you can play with it either via the inference widget or with a "),Fl=o("code"),mu=a("pipeline"),hu=a(" object, as follows:"),gp=h(),v(po.$$.fragment),_p=h(),dr=o("p"),cu=a("We can feed some examples from the test set (which the model has not seen) to our pipeline to get a feel for the quality of the summaries. First let\u2019s implement a simple function to show the review, title, and generated summary together:"),wp=h(),v(mo.$$.fragment),bp=h(),fr=o("p"),uu=a("Let\u2019s take a look at one of the English examples we get:"),vp=h(),v(ho.$$.fragment),yp=h(),v(co.$$.fragment),kp=h(),tn=o("p"),du=a("This is not too bad! We can see that our model has actually been able to perform "),Gl=o("em"),fu=a("abstractive"),gu=a(" summarization by augmenting parts of the review with new words. And perhaps the coolest aspect of our model is that it is bilingual, so we can also generate summaries of Spanish reviews:"),$p=h(),v(uo.$$.fragment),jp=h(),v(fo.$$.fragment),xp=h(),gr=o("p"),_u=a("The summary translates into \u201CVery easy to read\u201D in English, which we can see in this case was extracted directly from the review. Nevertheless, this shows the versatility of the mT5 model and has given you a taste of what it\u2019s like to deal with a multilingual corpus!"),Ep=h(),_r=o("p"),wu=a("Next, we\u2019ll turn our attention to a slightly more complex task: training a language model from scratch."),this.h()},l(e){const p=yg('[data-svelte="svelte-1phssyn"]',document.head);u=r(p,"META",{name:!0,content:!0}),p.forEach(t),j=c(e),y(d.$$.fragment,e),E=c(e),C=r(e,"H1",{class:!0});var vo=l(C);x=r(vo,"A",{id:!0,class:!0,href:!0});var wr=l(x);A=r(wr,"SPAN",{});var Nl=l(A);y(P.$$.fragment,Nl),Nl.forEach(t),wr.forEach(t),T=c(vo),S=r(vo,"SPAN",{});var Hl=l(S);I=n(Hl,"Summarization"),Hl.forEach(t),vo.forEach(t),D=c(e),G.l(e),J=c(e),X=r(e,"P",{});var yo=l(X);Z=n(yo,"In this section we\u2019ll take a look at how Transformer models can be used to condense long documents into summaries, a task known as "),B=r(yo,"EM",{});var Ul=l(B);Q=n(Ul,"text summarization"),Ul.forEach(t),L=n(yo,". This is one of the most challenging NLP tasks as it requires a range of abilities, such as understanding long passages and generating coherent text that captures the main topics in a document. However, when done well, text summarization is a powerful tool that can speed up various business processes by relieving the burden of domain experts to read long documents in detail."),yo.forEach(t),M=c(e),y(ee.$$.fragment,e),F=c(e),K=r(e,"P",{});var da=l(K);te=n(da,"Although there already exist various fine-tuned models for summarization on the "),W=r(da,"A",{href:!0,rel:!0});var Ml=l(W);ie=n(Ml,"Hugging Face Hub"),Ml.forEach(t),V=n(da,", almost all of these are only suitable for English documents. So, to add a twist in this section, we\u2019ll train a bilingual model for English and Spanish. By the end of this section, you\u2019ll have a "),ae=r(da,"A",{href:!0,rel:!0});var br=l(ae);de=n(br,"model"),br.forEach(t),_e=n(da," that can summarize customer reviews like the one shown here:"),da.forEach(t),Te=c(e),H=r(e,"IFRAME",{src:!0,frameborder:!0,height:!0,title:!0,class:!0,allow:!0,sandbox:!0}),l(H).forEach(t),me=c(e),oe=r(e,"P",{});var Wl=l(oe);we=n(Wl,"As we\u2019ll see, these summaries are concise because they\u2019re learned from the titles that customers provide in their product reviews. Let\u2019s start by putting together a suitable bilingual corpus for this task."),Wl.forEach(t),ke=c(e),se=r(e,"H2",{class:!0});var an=l(se);ge=r(an,"A",{id:!0,class:!0,href:!0});var vr=l(ge);he=r(vr,"SPAN",{});var Au=l(he);y(fe.$$.fragment,Au),Au.forEach(t),vr.forEach(t),w=c(an),R=r(an,"SPAN",{});var Ou=l(R);be=n(Ou,"Preparing a multilingual corpus"),Ou.forEach(t),an.forEach(t),b=c(e),U=r(e,"P",{});var qp=l(U);qe=n(qp,"We\u2019ll use the "),pe=r(qp,"A",{href:!0,rel:!0});var Cu=l(pe);$e=n(Cu,"Multilingual Amazon Reviews Corpus"),Cu.forEach(t),Ce=n(qp," to create our bilingual summarizer. This corpus consists of Amazon product reviews in six languages and is typically used to benchmark multilingual classifiers. However, since each review is accompanied by a short title, we can use the titles as the target summaries for our model to learn from! To get started, let\u2019s download the English and Spanish subsets from the Hugging Face Hub:"),qp.forEach(t),re=c(e),y(ze.$$.fragment,e),Qe=c(e),y(Ae.$$.fragment,e),ne=c(e),le=r(e,"P",{});var _s=l(le);Ie=n(_s,"As you can see, for each language there are 200,000 reviews for the "),ws=r(_s,"CODE",{});var Pu=l(ws);bs=n(Pu,"train"),Pu.forEach(t),vs=n(_s," split, and 5,000 reviews for each of the "),ys=r(_s,"CODE",{});var Lu=l(ys);zs=n(Lu,"validation"),Lu.forEach(t),He=n(_s," and "),rs=r(_s,"CODE",{});var Iu=l(rs);es=n(Iu,"test"),Iu.forEach(t),ls=n(_s," splits. The review information we are interested in is contained in the "),Ue=r(_s,"CODE",{});var Ru=l(Ue);Ys=n(Ru,"review_body"),Ru.forEach(t),is=n(_s," and "),Oe=r(_s,"CODE",{});var Fu=l(Oe);ks=n(Fu,"review_title"),Fu.forEach(t),Ds=n(_s," columns. Let\u2019s take a look at a few examples by creating a simple function that takes a random sample from the training set with the techniques we learned in "),Re=r(_s,"A",{href:!0});var Gu=l(Re);ps=n(Gu,"Chapter 5"),Gu.forEach(t),N=n(_s,":"),_s.forEach(t),ce=c(e),y(Me.$$.fragment,e),Vs=c(e),y(ue.$$.fragment,e),ms=c(e),y(ss.$$.fragment,e),Ss=c(e),De=r(e,"P",{});var yr=l(De);Js=n(yr,"This sample shows the diversity of reviews one typically finds online, ranging from positive to negative (and everything in between!). Although the example with the \u201Cmeh\u201D title is not very informative, the other titles look like decent summaries of the reviews themselves. Training a summarization model on all 400,000 reviews would take far too long on a single GPU, so instead we\u2019ll focus on generating summaries for a single domain of products. To get a feel for what domains we can choose from, let\u2019s convert "),ts=r(yr,"CODE",{});var Nu=l(ts);je=n(Nu,"english_dataset"),Nu.forEach(t),Xs=n(yr," to a "),We=r(yr,"CODE",{});var Hu=l(We);Zs=n(Hu,"pandas.DataFrame"),Hu.forEach(t),Fe=n(yr," and compute the number of reviews per product category:"),yr.forEach(t),Qs=c(e),y(Be.$$.fragment,e),et=c(e),y(ve.$$.fragment,e),$s=c(e),Se=r(e,"P",{});var Lt=l(Se);vt=n(Lt,"The most popular products in the English dataset are about household items, clothing, and wireless electronics. To stick with the Amazon theme, though, let\u2019s focus on summarizing book reviews \u2014 after all, this is what the company was founded on! We can see two product categories that fit the bill ("),js=r(Lt,"CODE",{});var Uu=l(js);xs=n(Uu,"book"),Uu.forEach(t),mn=n(Lt," and "),yt=r(Lt,"CODE",{});var Mu=l(yt);st=n(Mu,"digital_ebook_purchase"),Mu.forEach(t),fa=n(Lt,"), so let\u2019s filter the datasets in both languages for just these products. As we saw in "),hs=r(Lt,"A",{href:!0});var Wu=l(hs);ga=n(Wu,"Chapter 5"),Wu.forEach(t),kt=n(Lt,", the "),Ft=r(Lt,"CODE",{});var Bu=l(Ft);_a=n(Bu,"Dataset.filter()"),Bu.forEach(t),tt=n(Lt," function allows us to slice a dataset very efficiently, so we can define a simple function to do this:"),Lt.forEach(t),Gt=c(e),y(Ke.$$.fragment,e),wa=c(e),xe=r(e,"P",{});var Us=l(xe);hn=n(Us,"Now when we apply this function to "),Nt=r(Us,"CODE",{});var Ku=l(Nt);ba=n(Ku,"english_dataset"),Ku.forEach(t),Es=n(Us," and "),cs=r(Us,"CODE",{});var Yu=l(cs);Ht=n(Yu,"spanish_dataset"),Yu.forEach(t),at=n(Us,", the result will contain just those rows involving the book categories. Before applying the filter, let\u2019s switch the format of "),Ut=r(Us,"CODE",{});var Vu=l(Ut);Mt=n(Vu,"english_dataset"),Vu.forEach(t),cn=n(Us," from "),$t=r(Us,"CODE",{});var Ju=l($t);jt=n(Ju,'"pandas"'),Ju.forEach(t),un=n(Us," back to "),xt=r(Us,"CODE",{});var Xu=l(xt);Ye=n(Xu,'"arrow"'),Xu.forEach(t),nt=n(Us,":"),Us.forEach(t),va=c(e),y(Ts.$$.fragment,e),ya=c(e),Et=r(e,"P",{});var Zu=l(Et);dn=n(Zu,"We can then apply the filter function, and as a sanity check let\u2019s inspect a sample of reviews to see if they are indeed about books:"),Zu.forEach(t),Tt=c(e),y(ot.$$.fragment,e),ka=c(e),y(qs.$$.fragment,e),$a=c(e),Ve=r(e,"P",{});var nn=l(Ve);as=n(nn,"Okay, we can see that the reviews are not strictly about books and might refer to things like calendars and electronic applications such as OneNote. Nevertheless, the domain seems about right to train a summarization model on. Before we look at various models that are suitable for this task, we have one last bit of data preparation to do: combining the English and Spanish reviews as a single "),Wt=r(nn,"CODE",{});var Qu=l(Wt);Bt=n(Qu,"DatasetDict"),Qu.forEach(t),fn=n(nn," object. \u{1F917} Datasets provides a handy "),Kt=r(nn,"CODE",{});var ed=l(Kt);Yt=n(ed,"concatenate_datasets()"),ed.forEach(t),gn=n(nn," function that (as the name suggests) will stack two "),Vt=r(nn,"CODE",{});var sd=l(Vt);Jt=n(sd,"Dataset"),sd.forEach(t),_n=n(nn," objects on top of each other. So, to create our bilingual dataset, we\u2019ll loop over each split, concatenate the datasets for that split, and shuffle the result to ensure our model doesn\u2019t overfit to a single language:"),nn.forEach(t),ja=c(e),y(As.$$.fragment,e),rt=c(e),y(lt.$$.fragment,e),Xt=c(e),us=r(e,"P",{});var td=l(us);xa=n(td,"This certainly looks like a mix of English and Spanish reviews! Now that we have a training corpus, one final thing to check is the distribution of words in the reviews and their titles. This is especially important for summarization tasks, where short reference summaries in the data can bias the model to only output one or two words in the generated summaries. The plots below show the word distributions, and we can see that the titles are heavily skewed toward just 1-2 words:"),td.forEach(t),Os=c(e),ds=r(e,"DIV",{class:!0});var zp=l(ds);Je=r(zp,"IMG",{class:!0,src:!0,alt:!0}),Zt=c(zp),it=r(zp,"IMG",{class:!0,src:!0,alt:!0}),zp.forEach(t),Qt=c(e),m=r(e,"P",{});var Dp=l(m);q=n(Dp,"To deal with this, we\u2019ll filter out the examples with very short titles so that our model can produce more interesting summaries. Since we\u2019re dealing with English and Spanish texts, we can use a rough heuristic to split the titles on whitespace and then use our trusty "),Ea=r(Dp,"CODE",{});var ad=l(Ea);jo=n(ad,"Dataset.filter()"),ad.forEach(t),xo=n(Dp," method as follows:"),Dp.forEach(t),pt=c(e),y(qt.$$.fragment,e),wn=c(e),mt=r(e,"P",{});var nd=l(mt);Eo=n(nd,"Now that we\u2019ve prepared our corpus, let\u2019s take a look at a few possible Transformer models that one might fine-tune on it!"),nd.forEach(t),bn=c(e),Cs=r(e,"H2",{class:!0});var Sp=l(Cs);ns=r(Sp,"A",{id:!0,class:!0,href:!0});var od=l(ns);Ta=r(od,"SPAN",{});var rd=l(Ta);y(zt.$$.fragment,rd),rd.forEach(t),od.forEach(t),qa=c(Sp),za=r(Sp,"SPAN",{});var ld=l(za);To=n(ld,"Models for text summarization"),ld.forEach(t),Sp.forEach(t),ea=c(e),ht=r(e,"P",{});var Ap=l(ht);Da=n(Ap,"If you think about it, text summarization is a similar sort of task to machine translation: we have a body of text like a review that we\u2019d like to \u201Ctranslate\u201D into a shorter version that captures the salient features of the input. Accordingly, most Transformer models for summarization adopt the encoder-decoder architecture that we first encountered in "),sa=r(Ap,"A",{href:!0});var id=l(sa);qo=n(id,"Chapter 1"),id.forEach(t),Dt=n(Ap,", although there are some exceptions like the GPT family of models which can also be used for summarization in few-shot settings. The following table lists some popular pretrained models that can be fine-tuned for summarization."),Ap.forEach(t),vn=c(e),ct=r(e,"TABLE",{});var Op=l(ct);Sa=r(Op,"THEAD",{});var pd=l(Sa);Ps=r(pd,"TR",{});var kr=l(Ps);ta=r(kr,"TH",{align:!0});var md=l(ta);St=n(md,"Transformer model"),md.forEach(t),zo=c(kr),Aa=r(kr,"TH",{});var hd=l(Aa);Do=n(hd,"Description"),hd.forEach(t),So=c(kr),ut=r(kr,"TH",{align:!0});var cd=l(ut);Ao=n(cd,"Multilingual?"),cd.forEach(t),kr.forEach(t),pd.forEach(t),Oa=c(Op),Ge=r(Op,"TBODY",{});var Ms=l(Ge);Ls=r(Ms,"TR",{});var $r=l(Ls);aa=r($r,"TD",{align:!0});var ud=l(aa);At=r(ud,"A",{href:!0,rel:!0});var dd=l(At);Is=n(dd,"GPT-2"),dd.forEach(t),ud.forEach(t),Ca=c($r),Pa=r($r,"TD",{});var fd=l(Pa);Oo=n(fd,"Although trained as an auto-regressive language model, you can make GPT-2 generate summaries by appending \u201CTL;DR\u201D at the end of the input text."),fd.forEach(t),Co=c($r),os=r($r,"TD",{align:!0});var gd=l(os);Po=n(gd,"\u274C"),gd.forEach(t),$r.forEach(t),Lo=c(Ms),Rs=r(Ms,"TR",{});var jr=l(Rs);na=r(jr,"TD",{align:!0});var _d=l(na);Fs=r(_d,"A",{href:!0,rel:!0});var wd=l(Fs);Io=n(wd,"PEGASUS"),wd.forEach(t),_d.forEach(t),Xp=c(jr),Cr=r(jr,"TD",{});var bd=l(Cr);Zp=n(bd,"Uses a pretraining objective to predict masked sentences in multi-sentence texts. This pretraining objective is closer to summarization than vanilla language modeling and scores highly on popular benchmarks."),bd.forEach(t),Qp=c(jr),Ro=r(jr,"TD",{align:!0});var vd=l(Ro);em=n(vd,"\u274C"),vd.forEach(t),jr.forEach(t),sm=c(Ms),oa=r(Ms,"TR",{});var xr=l(oa);Fo=r(xr,"TD",{align:!0});var yd=l(Fo);yn=r(yd,"A",{href:!0,rel:!0});var kd=l(yn);tm=n(kd,"T5"),kd.forEach(t),yd.forEach(t),am=c(xr),kn=r(xr,"TD",{});var Cp=l(kn);nm=n(Cp,"A universal Transformer architecture that formulates all tasks in a text-to-text framework; e.g., the input format for the model to summarize a document is "),Pr=r(Cp,"CODE",{});var $d=l(Pr);om=n($d,"summarize: ARTICLE"),$d.forEach(t),rm=n(Cp,"."),Cp.forEach(t),lm=c(xr),Go=r(xr,"TD",{align:!0});var jd=l(Go);im=n(jd,"\u274C"),jd.forEach(t),xr.forEach(t),pm=c(Ms),ra=r(Ms,"TR",{});var Er=l(ra);No=r(Er,"TD",{align:!0});var xd=l(No);$n=r(xd,"A",{href:!0,rel:!0});var Ed=l($n);mm=n(Ed,"mT5"),Ed.forEach(t),xd.forEach(t),hm=c(Er),Lr=r(Er,"TD",{});var Td=l(Lr);cm=n(Td,"A multilingual version of T5, pretrained on the multilingual Common Crawl corpus (mC4), covering 101 languages."),Td.forEach(t),um=c(Er),Ho=r(Er,"TD",{align:!0});var qd=l(Ho);dm=n(qd,"\u2705"),qd.forEach(t),Er.forEach(t),fm=c(Ms),la=r(Ms,"TR",{});var Tr=l(la);Uo=r(Tr,"TD",{align:!0});var zd=l(Uo);jn=r(zd,"A",{href:!0,rel:!0});var Dd=l(jn);gm=n(Dd,"BART"),Dd.forEach(t),zd.forEach(t),_m=c(Tr),Ir=r(Tr,"TD",{});var Sd=l(Ir);wm=n(Sd,"A novel Transformer architecture with both an encoder and a decoder stack trained to reconstruct corrupted input that combines the pretraining schemes of BERT and GPT-2."),Sd.forEach(t),bm=c(Tr),Mo=r(Tr,"TD",{align:!0});var Ad=l(Mo);vm=n(Ad,"\u274C"),Ad.forEach(t),Tr.forEach(t),ym=c(Ms),ia=r(Ms,"TR",{});var qr=l(ia);Wo=r(qr,"TD",{align:!0});var Od=l(Wo);xn=r(Od,"A",{href:!0,rel:!0});var Cd=l(xn);km=n(Cd,"mBART-50"),Cd.forEach(t),Od.forEach(t),$m=c(qr),Rr=r(qr,"TD",{});var Pd=l(Rr);jm=n(Pd,"A multilingual version of BART, pretrained on 50 languages."),Pd.forEach(t),xm=c(qr),Bo=r(qr,"TD",{align:!0});var Ld=l(Bo);Em=n(Ld,"\u2705"),Ld.forEach(t),qr.forEach(t),Ms.forEach(t),Op.forEach(t),Kl=c(e),Ko=r(e,"P",{});var Id=l(Ko);Tm=n(Id,"As you can see from this table, the majority of Transformer models for summarization (and indeed most NLP tasks) are monolingual. This is great if your task is in a \u201Chigh-resource\u201D language like English or German, but less so for the thousands of other languages in use across the world. Fortunately, there is a class of multilingual Transformer models, like mT5 and mBART, that come to the rescue. These models are pretrained using language modeling, but with a twist: instead of training on a corpus of one language, they are trained jointly on texts in over 50 languages at once!"),Id.forEach(t),Yl=c(e),La=r(e,"P",{});var Pp=l(La);qm=n(Pp,"We\u2019ll focus on mT5, an interesting architecture based on T5 that was pretrained in a text-to-text framework. In T5, every NLP task is formulated in terms of a prompt prefix like "),Fr=r(Pp,"CODE",{});var Rd=l(Fr);zm=n(Rd,"summarize:"),Rd.forEach(t),Dm=n(Pp," which conditions the model to adapt the generated text to the prompt. As shown in the figure below, this makes T5 extremely versatile, as you can solve many tasks with a single model!"),Pp.forEach(t),Vl=c(e),pa=r(e,"DIV",{class:!0});var Lp=l(pa);En=r(Lp,"IMG",{class:!0,src:!0,alt:!0}),Sm=c(Lp),Tn=r(Lp,"IMG",{class:!0,src:!0,alt:!0}),Lp.forEach(t),Jl=c(e),Yo=r(e,"P",{});var Fd=l(Yo);Am=n(Fd,"mT5 doesn\u2019t use prefixes, but shares much of the versatility of T5 and has the advantage of being multilingual. Now that we\u2019ve picked a model, let\u2019s take a look at preparing our data for training."),Fd.forEach(t),Xl=c(e),y(Ia.$$.fragment,e),Zl=c(e),ma=r(e,"H2",{class:!0});var Ip=l(ma);Ra=r(Ip,"A",{id:!0,class:!0,href:!0});var Gd=l(Ra);Gr=r(Gd,"SPAN",{});var Nd=l(Gr);y(qn.$$.fragment,Nd),Nd.forEach(t),Gd.forEach(t),Om=c(Ip),Nr=r(Ip,"SPAN",{});var Hd=l(Nr);Cm=n(Hd,"Preprocessing the data"),Hd.forEach(t),Ip.forEach(t),Ql=c(e),y(zn.$$.fragment,e),ei=c(e),Fa=r(e,"P",{});var Rp=l(Fa);Pm=n(Rp,"Our next task is to tokenize and encode our reviews and their titles. As usual, we begin by loading the tokenizer associated with the pretrained model checkpoint. We\u2019ll use "),Hr=r(Rp,"CODE",{});var Ud=l(Hr);Lm=n(Ud,"mt5-small"),Ud.forEach(t),Im=n(Rp," as our checkpoint so we can fine-tune the model in a reasonable amount of time:"),Rp.forEach(t),si=c(e),y(Dn.$$.fragment,e),ti=c(e),y(Ga.$$.fragment,e),ai=c(e),Vo=r(e,"P",{});var Md=l(Vo);Rm=n(Md,"Let\u2019s test out the mT5 tokenizer on a small example:"),Md.forEach(t),ni=c(e),y(Sn.$$.fragment,e),oi=c(e),y(An.$$.fragment,e),ri=c(e),fs=r(e,"P",{});var It=l(fs);Fm=n(It,"Here we can see the familiar "),Ur=r(It,"CODE",{});var Wd=l(Ur);Gm=n(Wd,"input_ids"),Wd.forEach(t),Nm=n(It," and "),Mr=r(It,"CODE",{});var Bd=l(Mr);Hm=n(Bd,"attention_mask"),Bd.forEach(t),Um=n(It," that we encountered in our first fine-tuning experiments back in "),Jo=r(It,"A",{href:!0});var Kd=l(Jo);Mm=n(Kd,"Chapter 3"),Kd.forEach(t),Wm=n(It,". Let\u2019s decode these input IDs with the tokenizer\u2019s "),Wr=r(It,"CODE",{});var Yd=l(Wr);Bm=n(Yd,"convert_ids_to_tokens()"),Yd.forEach(t),Km=n(It," function to see what kind of tokenizer we\u2019re dealing with:"),It.forEach(t),li=c(e),y(On.$$.fragment,e),ii=c(e),y(Cn.$$.fragment,e),pi=c(e),Gs=r(e,"P",{});var on=l(Gs);Ym=n(on,"The special Unicode character "),Br=r(on,"CODE",{});var Vd=l(Br);Vm=n(Vd,"\u2581"),Vd.forEach(t),Jm=n(on," and end-of-sequence token "),Kr=r(on,"CODE",{});var Jd=l(Kr);Xm=n(Jd,"</s>"),Jd.forEach(t),Zm=n(on," indicate that we\u2019re dealing with the SentencePiece tokenizer, which is based on the Unigram segmentation algorithm discussed in "),Xo=r(on,"A",{href:!0});var Xd=l(Xo);Qm=n(Xd,"Chapter 6"),Xd.forEach(t),eh=n(on,". Unigram is especially useful for multilingual corpora since it allows SentencePiece to be agnostic about accents, punctuation, and the fact that many languages, like Japanese, do not have whitespace characters."),on.forEach(t),mi=c(e),Na=r(e,"P",{});var Fp=l(Na);sh=n(Fp,"To tokenize our corpus, we have to deal with a subtlety associated with summarization: because our labels are also text, it is possible that they exceed the model\u2019s maximum context size. This means we need to apply truncation to both the reviews and their titles to ensure we don\u2019t pass excessively long inputs to our model. The tokenizers in \u{1F917} Transformers provide a nifty "),Yr=r(Fp,"CODE",{});var Zd=l(Yr);th=n(Zd,"as_target_tokenizer()"),Zd.forEach(t),ah=n(Fp," function that allows you to tokenize the labels in parallel to the inputs. This is typically done using a context manager inside a preprocessing function that first encodes the inputs, and then encodes the labels as a separate column. Here is an example of such a function for mT5:"),Fp.forEach(t),hi=c(e),y(Pn.$$.fragment,e),ci=c(e),gs=r(e,"P",{});var Rt=l(gs);nh=n(Rt,"Let\u2019s walk through this code to understand what\u2019s happening. The first thing we\u2019ve done is define values for "),Vr=r(Rt,"CODE",{});var Qd=l(Vr);oh=n(Qd,"max_input_length"),Qd.forEach(t),rh=n(Rt," and "),Jr=r(Rt,"CODE",{});var ef=l(Jr);lh=n(ef,"max_target_length"),ef.forEach(t),ih=n(Rt,", which set the upper limits for how long our reviews and titles can be. Since the review body is typically much larger than the title, we\u2019ve scaled these values accordingly. Then, in the  "),Xr=r(Rt,"CODE",{});var sf=l(Xr);ph=n(sf,"preprocess_function()"),sf.forEach(t),mh=n(Rt," itself we can see the reviews are first tokenized, followed by the titles with "),Zr=r(Rt,"CODE",{});var tf=l(Zr);hh=n(tf,"as_target_tokenizer()"),tf.forEach(t),ch=n(Rt,"."),Rt.forEach(t),ui=c(e),Ot=r(e,"P",{});var zr=l(Ot);uh=n(zr,"With "),Qr=r(zr,"CODE",{});var af=l(Qr);dh=n(af,"preprocess_function()"),af.forEach(t),fh=n(zr,", it is then a simple matter to tokenize the whole corpus using the handy "),el=r(zr,"CODE",{});var nf=l(el);gh=n(nf,"Dataset.map()"),nf.forEach(t),_h=n(zr," function we\u2019ve used extensively throughout this course:"),zr.forEach(t),di=c(e),y(Ln.$$.fragment,e),fi=c(e),Zo=r(e,"P",{});var of=l(Zo);wh=n(of,"Now that the corpus has been preprocessed, let\u2019s take a look at some metrics that are commonly used for summarization. As we\u2019ll see, there is no silver bullet when it comes to measuring the quality of machine-generated text."),of.forEach(t),gi=c(e),y(Ha.$$.fragment,e),_i=c(e),ha=r(e,"H2",{class:!0});var Gp=l(ha);Ua=r(Gp,"A",{id:!0,class:!0,href:!0});var rf=l(Ua);sl=r(rf,"SPAN",{});var lf=l(sl);y(In.$$.fragment,lf),lf.forEach(t),rf.forEach(t),bh=c(Gp),tl=r(Gp,"SPAN",{});var pf=l(tl);vh=n(pf,"Metrics for text summarization"),pf.forEach(t),Gp.forEach(t),wi=c(e),y(Rn.$$.fragment,e),bi=c(e),Qo=r(e,"P",{});var mf=l(Qo);yh=n(mf,"In comparison to most of the other tasks we\u2019ve covered in this course, measuring the performance of text generation tasks like summarization or translation is not as straightforward. For example, given a review like \u201CI loved reading the Hunger Games\u201D, there are multiple valid summaries, like \u201CI loved the Hunger Games\u201D or \u201CHunger Games is a great read\u201D. Clearly, applying some sort of exact match between the generated summary and the label is not a good solution \u2014 even humans would fare poorly under such a metric, because we all have our own writing style."),mf.forEach(t),vi=c(e),Ma=r(e,"P",{});var Np=l(Ma);kh=n(Np,"For summarization, one of the most commonly used metrics is the "),Fn=r(Np,"A",{href:!0,rel:!0});var hf=l(Fn);$h=n(hf,"ROUGE score"),hf.forEach(t),jh=n(Np," (short for Recall-Oriented Understudy for Gisting Evaluation). The basic idea behind this metric is to compare a generated summary against a set of reference summaries that are typically created by humans. To make this more precise, suppose we want to compare the following two summaries:"),Np.forEach(t),yi=c(e),y(Gn.$$.fragment,e),ki=c(e),Ct=r(e,"P",{});var Dr=l(Ct);xh=n(Dr,"One way to compare them could be to count the number of overlapping words, which in this case would be 6. However, this is a bit crude, so instead ROUGE is based on computing the "),al=r(Dr,"EM",{});var cf=l(al);Eh=n(cf,"precision"),cf.forEach(t),Th=n(Dr," and "),nl=r(Dr,"EM",{});var uf=l(nl);qh=n(uf,"recall"),uf.forEach(t),zh=n(Dr," scores for the overlap."),Dr.forEach(t),$i=c(e),y(Wa.$$.fragment,e),ji=c(e),Nn=r(e,"P",{});var bu=l(Nn);Dh=n(bu,`For ROUGE, recall measures how much of the reference summary is captured by the generated one. If we are just comparing words, recall can be calculated according to the following formula:
`),xi=cg(bu),bu.forEach(t),Ei=c(e),Hn=r(e,"P",{});var vu=l(Hn);Sh=n(vu,`For our simple example above, this formula gives a perfect recall of 6/6 = 1; i.e., all the words in the reference summary have been produced by the model. This may sound great, but imagine if our generated summary had been \u201CI really really loved reading the Hunger Games all night\u201D. This would also have perfect recall, but is arguably a worse summary since it is verbose. To deal with these scenarios we also compute the precision, which in the ROUGE context measures how much of the generated summary was relevant:
`),Ti=cg(vu),vu.forEach(t),qi=c(e),Ba=r(e,"P",{});var Hp=l(Ba);Ah=n(Hp,"Applying this to our verbose summary gives a precision of 6/10  = 0.6, which is considerably worse than the precision of 6/7 = 0.86 obtained by our shorter one. In practice, both precision and recall are usually computed, and then the F1-score (the harmonic mean of precision and recall) is reported. We can do this easily in \u{1F917} Datasets by first installing the "),ol=r(Hp,"CODE",{});var df=l(ol);Oh=n(df,"rouge_score"),df.forEach(t),Ch=n(Hp," package:"),Hp.forEach(t),zi=c(e),y(Un.$$.fragment,e),Di=c(e),er=r(e,"P",{});var ff=l(er);Ph=n(ff,"and then loading the ROUGE metric as follows:"),ff.forEach(t),Si=c(e),y(Mn.$$.fragment,e),Ai=c(e),Ka=r(e,"P",{});var Up=l(Ka);Lh=n(Up,"Then we can use the "),rl=r(Up,"CODE",{});var gf=l(rl);Ih=n(gf,"rouge_score.compute()"),gf.forEach(t),Rh=n(Up," function to calculate all the metrics at once:"),Up.forEach(t),Oi=c(e),y(Wn.$$.fragment,e),Ci=c(e),y(Bn.$$.fragment,e),Pi=c(e),Xe=r(e,"P",{});var Ws=l(Xe);Fh=n(Ws,"Whoa, there\u2019s a lot of information in that output \u2014 what does it all mean? First, \u{1F917} Datasets actually computes confidence intervals for precision, recall, and F1-score; these are the "),ll=r(Ws,"CODE",{});var _f=l(ll);Gh=n(_f,"low"),_f.forEach(t),Nh=n(Ws,", "),il=r(Ws,"CODE",{});var wf=l(il);Hh=n(wf,"mid"),wf.forEach(t),Uh=n(Ws,", and "),pl=r(Ws,"CODE",{});var bf=l(pl);Mh=n(bf,"high"),bf.forEach(t),Wh=n(Ws," attributes you can see here. Moreover, \u{1F917} Datasets computes a variety of ROUGE scores which are based on different types of text granularity when comparing the generated and reference summaries. The "),ml=r(Ws,"CODE",{});var vf=l(ml);Bh=n(vf,"rouge1"),vf.forEach(t),Kh=n(Ws," variant is the overlap of unigrams \u2014 this is just a fancy way of saying the overlap of words and is exactly the metric we\u2019ve discussed above. To verify this, let\u2019s pull out the "),hl=r(Ws,"CODE",{});var yf=l(hl);Yh=n(yf,"mid"),yf.forEach(t),Vh=n(Ws," value of our scores:"),Ws.forEach(t),Li=c(e),y(Kn.$$.fragment,e),Ii=c(e),y(Yn.$$.fragment,e),Ri=c(e),Ze=r(e,"P",{});var Bs=l(Ze);Jh=n(Bs,"Great, the precision and recall numbers match up! Now what about those other ROUGE scores? "),cl=r(Bs,"CODE",{});var kf=l(cl);Xh=n(kf,"rouge2"),kf.forEach(t),Zh=n(Bs," measures the overlap between bigrams (think the overlap of pairs of words), while "),ul=r(Bs,"CODE",{});var $f=l(ul);Qh=n($f,"rougeL"),$f.forEach(t),ec=n(Bs," and "),dl=r(Bs,"CODE",{});var jf=l(dl);sc=n(jf,"rougeLsum"),jf.forEach(t),tc=n(Bs," measure the longest matching sequences of words by looking for the longest common substrings in the generated and reference summaries. The \u201Csum\u201D in "),fl=r(Bs,"CODE",{});var xf=l(fl);ac=n(xf,"rougeLsum"),xf.forEach(t),nc=n(Bs," refers to the fact that this metric is computed over a whole summary, while "),gl=r(Bs,"CODE",{});var Ef=l(gl);oc=n(Ef,"rougeL"),Ef.forEach(t),rc=n(Bs," is computed as the average over individual sentences."),Bs.forEach(t),Fi=c(e),y(Ya.$$.fragment,e),Gi=c(e),sr=r(e,"P",{});var Tf=l(sr);lc=n(Tf,"We\u2019ll use these ROUGE scores to track the performance of our model, but before doing that let\u2019s do something every good NLP practitioner should do: create a strong, yet simple baseline!"),Tf.forEach(t),Ni=c(e),ca=r(e,"H3",{class:!0});var Mp=l(ca);Va=r(Mp,"A",{id:!0,class:!0,href:!0});var qf=l(Va);_l=r(qf,"SPAN",{});var zf=l(_l);y(Vn.$$.fragment,zf),zf.forEach(t),qf.forEach(t),ic=c(Mp),wl=r(Mp,"SPAN",{});var Df=l(wl);pc=n(Df,"Creating a strong baseline"),Df.forEach(t),Mp.forEach(t),Hi=c(e),Ns=r(e,"P",{});var rn=l(Ns);mc=n(rn,"A common baseline for text summarization is to simply take the first three sentences of an article, often called the "),bl=r(rn,"EM",{});var Sf=l(bl);hc=n(Sf,"lead-3"),Sf.forEach(t),cc=n(rn," baseline. We could use full stops to track the sentence boundaries, but this will fail on acronyms like \u201CU.S.\u201D or \u201CU.N.\u201D \u2014 so instead we\u2019ll use the "),vl=r(rn,"CODE",{});var Af=l(vl);uc=n(Af,"nltk"),Af.forEach(t),dc=n(rn," library, which includes a better algorithm to handle these cases. You can install the package using "),yl=r(rn,"CODE",{});var Of=l(yl);fc=n(Of,"pip"),Of.forEach(t),gc=n(rn," as follows:"),rn.forEach(t),Ui=c(e),y(Jn.$$.fragment,e),Mi=c(e),tr=r(e,"P",{});var Cf=l(tr);_c=n(Cf,"and then download the punctuation rules:"),Cf.forEach(t),Wi=c(e),y(Xn.$$.fragment,e),Bi=c(e),Ja=r(e,"P",{});var Wp=l(Ja);wc=n(Wp,"Next, we import the sentence tokenizer from "),kl=r(Wp,"CODE",{});var Pf=l(kl);bc=n(Pf,"nltk"),Pf.forEach(t),vc=n(Wp," and create a simple function to extract the first three sentences in a review. The convention in text summarization is to separate each summary with a newline, so let\u2019s also include this and test it on a training example:"),Wp.forEach(t),Ki=c(e),y(Zn.$$.fragment,e),Yi=c(e),y(Qn.$$.fragment,e),Vi=c(e),ar=r(e,"P",{});var Lf=l(ar);yc=n(Lf,"This seems to work, so let\u2019s now implement a function that extracts these \u201Csummaries\u201D from a dataset and computes the ROUGE scores for the baseline:"),Lf.forEach(t),Ji=c(e),y(eo.$$.fragment,e),Xi=c(e),nr=r(e,"P",{});var If=l(nr);kc=n(If,"We can then use this function to compute the ROUGE scores over the validation set and prettify them a bit using Pandas:"),If.forEach(t),Zi=c(e),y(so.$$.fragment,e),Qi=c(e),y(to.$$.fragment,e),ep=c(e),Xa=r(e,"P",{});var Bp=l(Xa);$c=n(Bp,"We can see that the "),$l=r(Bp,"CODE",{});var Rf=l($l);jc=n(Rf,"rouge2"),Rf.forEach(t),xc=n(Bp," score is significantly lower than the rest; this likely reflects the fact that review titles are typically concise and so the lead-3 baseline is too verbose. Now that we have a good baseline to work from, let\u2019s turn our attention toward fine-tuning mT5!"),Bp.forEach(t),sp=c(e),ft.l(e),or=c(e),y(Za.$$.fragment,e),tp=c(e),rr=r(e,"P",{});var Ff=l(rr);Ec=n(Ff,"The next thing we need to do is log in to the Hugging Face Hub. If you\u2019re running this code in a notebook, you can do so with the following utility function:"),Ff.forEach(t),ap=c(e),y(ao.$$.fragment,e),np=c(e),lr=r(e,"P",{});var Gf=l(lr);Tc=n(Gf,"which will display a widget where you can enter your credentials. Alternatively, you can run this command in your terminal and log in there:"),Gf.forEach(t),op=c(e),y(no.$$.fragment,e),rp=c(e),Pe&&Pe.l(e),ir=c(e),Qa=r(e,"P",{});var Kp=l(Qa);qc=n(Kp,"Next, we need to define a data collator for our sequence-to-sequence task. Since mT5 is an encoder-decoder Transformer model, one subtlety with preparing our batches is that during decoding we need to shift the labels to the right by one. This is required to ensure that the decoder only sees the previous ground truth labels and not the current or future ones, which would be easy for the model to memorize. This is similar to how masked self-attention is applied to the inputs in a task like "),pr=r(Kp,"A",{href:!0});var Nf=l(pr);zc=n(Nf,"causal language modeling"),Nf.forEach(t),Dc=n(Kp,"."),Kp.forEach(t),lp=c(e),Hs=r(e,"P",{});var ln=l(Hs);Sc=n(ln,"Luckily, \u{1F917} Transformers provides a "),jl=r(ln,"CODE",{});var Hf=l(jl);Ac=n(Hf,"DataCollatorForSeq2Seq"),Hf.forEach(t),Oc=n(ln," collator that will dynamically pad the inputs and the labels for us. To instantiate this collator, we simply need to provide the "),xl=r(ln,"CODE",{});var Uf=l(xl);Cc=n(Uf,"tokenizer"),Uf.forEach(t),Pc=n(ln," and "),El=r(ln,"CODE",{});var Mf=l(El);Lc=n(Mf,"model"),Mf.forEach(t),Ic=n(ln,":"),ln.forEach(t),ip=c(e),_t.l(e),mr=c(e),hr=r(e,"P",{});var Wf=l(hr);Rc=n(Wf,"Let\u2019s see what this collator produces when fed a small batch of examples. First, we need to remove the columns with strings because the collator won\u2019t know how to pad these elements:"),Wf.forEach(t),pp=c(e),y(oo.$$.fragment,e),mp=c(e),Pt=r(e,"P",{});var Sr=l(Pt);Fc=n(Sr,"Since the collator expects a list of "),Tl=r(Sr,"CODE",{});var Bf=l(Tl);Gc=n(Bf,"dict"),Bf.forEach(t),Nc=n(Sr,"s, where each "),ql=r(Sr,"CODE",{});var Kf=l(ql);Hc=n(Kf,"dict"),Kf.forEach(t),Uc=n(Sr," represents a single example in the dataset, we also need to wrangle the data into the expected format before passing it to the data collator:"),Sr.forEach(t),hp=c(e),y(ro.$$.fragment,e),cp=c(e),y(lo.$$.fragment,e),up=c(e),Ee=r(e,"P",{});var Ne=l(Ee);Mc=n(Ne,"The main thing to notice here is that the first example is longer than the second one, so the "),zl=r(Ne,"CODE",{});var Yf=l(zl);Wc=n(Yf,"input_ids"),Yf.forEach(t),Bc=n(Ne," and "),Dl=r(Ne,"CODE",{});var Vf=l(Dl);Kc=n(Vf,"attention_mask"),Vf.forEach(t),Yc=n(Ne," of the second example have been padded on the right with a "),Sl=r(Ne,"CODE",{});var Jf=l(Sl);Vc=n(Jf,"[PAD]"),Jf.forEach(t),Jc=n(Ne," token (whose ID is "),Al=r(Ne,"CODE",{});var Xf=l(Al);Xc=n(Xf,"0"),Xf.forEach(t),Zc=n(Ne,"). Similarly, we can see that the "),Ol=r(Ne,"CODE",{});var Zf=l(Ol);Qc=n(Zf,"labels"),Zf.forEach(t),eu=n(Ne," have been padded with "),Cl=r(Ne,"CODE",{});var Qf=l(Cl);su=n(Qf,"-100"),Qf.forEach(t),tu=n(Ne,"s, to make sure the padding tokens are ignored by the loss function. And finally, we can see a new "),Pl=r(Ne,"CODE",{});var eg=l(Pl);au=n(eg,"decoder_input_ids"),eg.forEach(t),nu=n(Ne," which has shifted the labels to the right by inserting a "),Ll=r(Ne,"CODE",{});var sg=l(Ll);ou=n(sg,"[PAD]"),sg.forEach(t),ru=n(Ne," token in the first entry."),Ne.forEach(t),dp=c(e),bt.l(e),cr=c(e),Le&&Le.l(e),ur=c(e),ua=r(e,"H2",{class:!0});var Yp=l(ua);en=r(Yp,"A",{id:!0,class:!0,href:!0});var tg=l(en);Il=r(tg,"SPAN",{});var ag=l(Il);y(io.$$.fragment,ag),ag.forEach(t),tg.forEach(t),lu=c(Yp),Rl=r(Yp,"SPAN",{});var ng=l(Rl);iu=n(ng,"Using your fine-tuned model"),ng.forEach(t),Yp.forEach(t),fp=c(e),sn=r(e,"P",{});var Vp=l(sn);pu=n(Vp,"Once you\u2019ve pushed the model to the Hub, you can play with it either via the inference widget or with a "),Fl=r(Vp,"CODE",{});var og=l(Fl);mu=n(og,"pipeline"),og.forEach(t),hu=n(Vp," object, as follows:"),Vp.forEach(t),gp=c(e),y(po.$$.fragment,e),_p=c(e),dr=r(e,"P",{});var rg=l(dr);cu=n(rg,"We can feed some examples from the test set (which the model has not seen) to our pipeline to get a feel for the quality of the summaries. First let\u2019s implement a simple function to show the review, title, and generated summary together:"),rg.forEach(t),wp=c(e),y(mo.$$.fragment,e),bp=c(e),fr=r(e,"P",{});var lg=l(fr);uu=n(lg,"Let\u2019s take a look at one of the English examples we get:"),lg.forEach(t),vp=c(e),y(ho.$$.fragment,e),yp=c(e),y(co.$$.fragment,e),kp=c(e),tn=r(e,"P",{});var Jp=l(tn);du=n(Jp,"This is not too bad! We can see that our model has actually been able to perform "),Gl=r(Jp,"EM",{});var ig=l(Gl);fu=n(ig,"abstractive"),ig.forEach(t),gu=n(Jp," summarization by augmenting parts of the review with new words. And perhaps the coolest aspect of our model is that it is bilingual, so we can also generate summaries of Spanish reviews:"),Jp.forEach(t),$p=c(e),y(uo.$$.fragment,e),jp=c(e),y(fo.$$.fragment,e),xp=c(e),gr=r(e,"P",{});var pg=l(gr);_u=n(pg,"The summary translates into \u201CVery easy to read\u201D in English, which we can see in this case was extracted directly from the review. Nevertheless, this shows the versatility of the mT5 model and has given you a taste of what it\u2019s like to deal with a multilingual corpus!"),pg.forEach(t),Ep=c(e),_r=r(e,"P",{});var mg=l(_r);wu=n(mg,"Next, we\u2019ll turn our attention to a slightly more complex task: training a language model from scratch."),mg.forEach(t),this.h()},h(){f(u,"name","hf:doc:metadata"),f(u,"content",JSON.stringify(Ng)),f(x,"id","summarization"),f(x,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),f(x,"href","#summarization"),f(C,"class","relative group"),f(W,"href","https://huggingface.co/models?pipeline_tag=summarization&sort=downloads"),f(W,"rel","nofollow"),f(ae,"href","https://huggingface.co/huggingface-course/mt5-small-finetuned-amazon-en-es"),f(ae,"rel","nofollow"),Bl(H.src,ye="https://hf.space/gradioiframe/course-demos/mt5-small-finetuned-amazon-en-es/+")||f(H,"src",ye),f(H,"frameborder","0"),f(H,"height","400"),f(H,"title","Gradio app"),f(H,"class","block dark:hidden container p-0 flex-grow space-iframe"),f(H,"allow","accelerometer; ambient-light-sensor; autoplay; battery; camera; document-domain; encrypted-media; fullscreen; geolocation; gyroscope; layout-animations; legacy-image-formats; magnetometer; microphone; midi; oversized-images; payment; picture-in-picture; publickey-credentials-get; sync-xhr; usb; vr ; wake-lock; xr-spatial-tracking"),f(H,"sandbox","allow-forms allow-modals allow-popups allow-popups-to-escape-sandbox allow-same-origin allow-scripts allow-downloads"),f(ge,"id","preparing-a-multilingual-corpus"),f(ge,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),f(ge,"href","#preparing-a-multilingual-corpus"),f(se,"class","relative group"),f(pe,"href","https://huggingface.co/datasets/amazon_reviews_multi"),f(pe,"rel","nofollow"),f(Re,"href","/course/chapter5"),f(hs,"href","/course/chapter5"),f(Je,"class","block dark:hidden"),Bl(Je.src,ko="https://huggingface.co/datasets/huggingface-course/documentation-images/resolve/main/en/chapter7/review-lengths.svg")||f(Je,"src",ko),f(Je,"alt","Word count distributions for the review titles and texts."),f(it,"class","hidden dark:block"),Bl(it.src,$o="https://huggingface.co/datasets/huggingface-course/documentation-images/resolve/main/en/chapter7/review-lengths-dark.svg")||f(it,"src",$o),f(it,"alt","Word count distributions for the review titles and texts."),f(ds,"class","flex justify-center"),f(ns,"id","models-for-text-summarization"),f(ns,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),f(ns,"href","#models-for-text-summarization"),f(Cs,"class","relative group"),f(sa,"href","/course/chapter1"),f(ta,"align","center"),f(ut,"align","center"),f(At,"href","https://huggingface.co/gpt2-xl"),f(At,"rel","nofollow"),f(aa,"align","center"),f(os,"align","center"),f(Fs,"href","https://huggingface.co/google/pegasus-large"),f(Fs,"rel","nofollow"),f(na,"align","center"),f(Ro,"align","center"),f(yn,"href","https://huggingface.co/t5-base"),f(yn,"rel","nofollow"),f(Fo,"align","center"),f(Go,"align","center"),f($n,"href","https://huggingface.co/google/mt5-base"),f($n,"rel","nofollow"),f(No,"align","center"),f(Ho,"align","center"),f(jn,"href","https://huggingface.co/facebook/bart-base"),f(jn,"rel","nofollow"),f(Uo,"align","center"),f(Mo,"align","center"),f(xn,"href","https://huggingface.co/facebook/mbart-large-50"),f(xn,"rel","nofollow"),f(Wo,"align","center"),f(Bo,"align","center"),f(En,"class","block dark:hidden"),Bl(En.src,ku="https://huggingface.co/datasets/huggingface-course/documentation-images/resolve/main/en/chapter7/t5.svg")||f(En,"src",ku),f(En,"alt","Different tasks performed by the T5 architecture."),f(Tn,"class","hidden dark:block"),Bl(Tn.src,$u="https://huggingface.co/datasets/huggingface-course/documentation-images/resolve/main/en/chapter7/t5-dark.svg")||f(Tn,"src",$u),f(Tn,"alt","Different tasks performed by the T5 architecture."),f(pa,"class","flex justify-center"),f(Ra,"id","preprocessing-the-data"),f(Ra,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),f(Ra,"href","#preprocessing-the-data"),f(ma,"class","relative group"),f(Jo,"href","/course/chapter3"),f(Xo,"href","/course/chapter6"),f(Ua,"id","metrics-for-text-summarization"),f(Ua,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),f(Ua,"href","#metrics-for-text-summarization"),f(ha,"class","relative group"),f(Fn,"href","https://en.wikipedia.org/wiki/ROUGE_(metric)"),f(Fn,"rel","nofollow"),xi.a=null,Ti.a=null,f(Va,"id","creating-a-strong-baseline"),f(Va,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),f(Va,"href","#creating-a-strong-baseline"),f(ca,"class","relative group"),f(pr,"href","/course/chapter7/6"),f(en,"id","using-your-finetuned-model"),f(en,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),f(en,"href","#using-your-finetuned-model"),f(ua,"class","relative group")},m(e,p){s(document.head,u),i(e,j,p),k(d,e,p),i(e,E,p),i(e,C,p),s(C,x),s(x,A),k(P,A,null),s(C,T),s(C,S),s(S,I),i(e,D,p),go[O].m(e,p),i(e,J,p),i(e,X,p),s(X,Z),s(X,B),s(B,Q),s(X,L),i(e,M,p),k(ee,e,p),i(e,F,p),i(e,K,p),s(K,te),s(K,W),s(W,ie),s(K,V),s(K,ae),s(ae,de),s(K,_e),i(e,Te,p),i(e,H,p),i(e,me,p),i(e,oe,p),s(oe,we),i(e,ke,p),i(e,se,p),s(se,ge),s(ge,he),k(fe,he,null),s(se,w),s(se,R),s(R,be),i(e,b,p),i(e,U,p),s(U,qe),s(U,pe),s(pe,$e),s(U,Ce),i(e,re,p),k(ze,e,p),i(e,Qe,p),k(Ae,e,p),i(e,ne,p),i(e,le,p),s(le,Ie),s(le,ws),s(ws,bs),s(le,vs),s(le,ys),s(ys,zs),s(le,He),s(le,rs),s(rs,es),s(le,ls),s(le,Ue),s(Ue,Ys),s(le,is),s(le,Oe),s(Oe,ks),s(le,Ds),s(le,Re),s(Re,ps),s(le,N),i(e,ce,p),k(Me,e,p),i(e,Vs,p),k(ue,e,p),i(e,ms,p),k(ss,e,p),i(e,Ss,p),i(e,De,p),s(De,Js),s(De,ts),s(ts,je),s(De,Xs),s(De,We),s(We,Zs),s(De,Fe),i(e,Qs,p),k(Be,e,p),i(e,et,p),k(ve,e,p),i(e,$s,p),i(e,Se,p),s(Se,vt),s(Se,js),s(js,xs),s(Se,mn),s(Se,yt),s(yt,st),s(Se,fa),s(Se,hs),s(hs,ga),s(Se,kt),s(Se,Ft),s(Ft,_a),s(Se,tt),i(e,Gt,p),k(Ke,e,p),i(e,wa,p),i(e,xe,p),s(xe,hn),s(xe,Nt),s(Nt,ba),s(xe,Es),s(xe,cs),s(cs,Ht),s(xe,at),s(xe,Ut),s(Ut,Mt),s(xe,cn),s(xe,$t),s($t,jt),s(xe,un),s(xe,xt),s(xt,Ye),s(xe,nt),i(e,va,p),k(Ts,e,p),i(e,ya,p),i(e,Et,p),s(Et,dn),i(e,Tt,p),k(ot,e,p),i(e,ka,p),k(qs,e,p),i(e,$a,p),i(e,Ve,p),s(Ve,as),s(Ve,Wt),s(Wt,Bt),s(Ve,fn),s(Ve,Kt),s(Kt,Yt),s(Ve,gn),s(Ve,Vt),s(Vt,Jt),s(Ve,_n),i(e,ja,p),k(As,e,p),i(e,rt,p),k(lt,e,p),i(e,Xt,p),i(e,us,p),s(us,xa),i(e,Os,p),i(e,ds,p),s(ds,Je),s(ds,Zt),s(ds,it),i(e,Qt,p),i(e,m,p),s(m,q),s(m,Ea),s(Ea,jo),s(m,xo),i(e,pt,p),k(qt,e,p),i(e,wn,p),i(e,mt,p),s(mt,Eo),i(e,bn,p),i(e,Cs,p),s(Cs,ns),s(ns,Ta),k(zt,Ta,null),s(Cs,qa),s(Cs,za),s(za,To),i(e,ea,p),i(e,ht,p),s(ht,Da),s(ht,sa),s(sa,qo),s(ht,Dt),i(e,vn,p),i(e,ct,p),s(ct,Sa),s(Sa,Ps),s(Ps,ta),s(ta,St),s(Ps,zo),s(Ps,Aa),s(Aa,Do),s(Ps,So),s(Ps,ut),s(ut,Ao),s(ct,Oa),s(ct,Ge),s(Ge,Ls),s(Ls,aa),s(aa,At),s(At,Is),s(Ls,Ca),s(Ls,Pa),s(Pa,Oo),s(Ls,Co),s(Ls,os),s(os,Po),s(Ge,Lo),s(Ge,Rs),s(Rs,na),s(na,Fs),s(Fs,Io),s(Rs,Xp),s(Rs,Cr),s(Cr,Zp),s(Rs,Qp),s(Rs,Ro),s(Ro,em),s(Ge,sm),s(Ge,oa),s(oa,Fo),s(Fo,yn),s(yn,tm),s(oa,am),s(oa,kn),s(kn,nm),s(kn,Pr),s(Pr,om),s(kn,rm),s(oa,lm),s(oa,Go),s(Go,im),s(Ge,pm),s(Ge,ra),s(ra,No),s(No,$n),s($n,mm),s(ra,hm),s(ra,Lr),s(Lr,cm),s(ra,um),s(ra,Ho),s(Ho,dm),s(Ge,fm),s(Ge,la),s(la,Uo),s(Uo,jn),s(jn,gm),s(la,_m),s(la,Ir),s(Ir,wm),s(la,bm),s(la,Mo),s(Mo,vm),s(Ge,ym),s(Ge,ia),s(ia,Wo),s(Wo,xn),s(xn,km),s(ia,$m),s(ia,Rr),s(Rr,jm),s(ia,xm),s(ia,Bo),s(Bo,Em),i(e,Kl,p),i(e,Ko,p),s(Ko,Tm),i(e,Yl,p),i(e,La,p),s(La,qm),s(La,Fr),s(Fr,zm),s(La,Dm),i(e,Vl,p),i(e,pa,p),s(pa,En),s(pa,Sm),s(pa,Tn),i(e,Jl,p),i(e,Yo,p),s(Yo,Am),i(e,Xl,p),k(Ia,e,p),i(e,Zl,p),i(e,ma,p),s(ma,Ra),s(Ra,Gr),k(qn,Gr,null),s(ma,Om),s(ma,Nr),s(Nr,Cm),i(e,Ql,p),k(zn,e,p),i(e,ei,p),i(e,Fa,p),s(Fa,Pm),s(Fa,Hr),s(Hr,Lm),s(Fa,Im),i(e,si,p),k(Dn,e,p),i(e,ti,p),k(Ga,e,p),i(e,ai,p),i(e,Vo,p),s(Vo,Rm),i(e,ni,p),k(Sn,e,p),i(e,oi,p),k(An,e,p),i(e,ri,p),i(e,fs,p),s(fs,Fm),s(fs,Ur),s(Ur,Gm),s(fs,Nm),s(fs,Mr),s(Mr,Hm),s(fs,Um),s(fs,Jo),s(Jo,Mm),s(fs,Wm),s(fs,Wr),s(Wr,Bm),s(fs,Km),i(e,li,p),k(On,e,p),i(e,ii,p),k(Cn,e,p),i(e,pi,p),i(e,Gs,p),s(Gs,Ym),s(Gs,Br),s(Br,Vm),s(Gs,Jm),s(Gs,Kr),s(Kr,Xm),s(Gs,Zm),s(Gs,Xo),s(Xo,Qm),s(Gs,eh),i(e,mi,p),i(e,Na,p),s(Na,sh),s(Na,Yr),s(Yr,th),s(Na,ah),i(e,hi,p),k(Pn,e,p),i(e,ci,p),i(e,gs,p),s(gs,nh),s(gs,Vr),s(Vr,oh),s(gs,rh),s(gs,Jr),s(Jr,lh),s(gs,ih),s(gs,Xr),s(Xr,ph),s(gs,mh),s(gs,Zr),s(Zr,hh),s(gs,ch),i(e,ui,p),i(e,Ot,p),s(Ot,uh),s(Ot,Qr),s(Qr,dh),s(Ot,fh),s(Ot,el),s(el,gh),s(Ot,_h),i(e,di,p),k(Ln,e,p),i(e,fi,p),i(e,Zo,p),s(Zo,wh),i(e,gi,p),k(Ha,e,p),i(e,_i,p),i(e,ha,p),s(ha,Ua),s(Ua,sl),k(In,sl,null),s(ha,bh),s(ha,tl),s(tl,vh),i(e,wi,p),k(Rn,e,p),i(e,bi,p),i(e,Qo,p),s(Qo,yh),i(e,vi,p),i(e,Ma,p),s(Ma,kh),s(Ma,Fn),s(Fn,$h),s(Ma,jh),i(e,yi,p),k(Gn,e,p),i(e,ki,p),i(e,Ct,p),s(Ct,xh),s(Ct,al),s(al,Eh),s(Ct,Th),s(Ct,nl),s(nl,qh),s(Ct,zh),i(e,$i,p),k(Wa,e,p),i(e,ji,p),i(e,Nn,p),s(Nn,Dh),xi.m(gg,Nn),i(e,Ei,p),i(e,Hn,p),s(Hn,Sh),Ti.m(_g,Hn),i(e,qi,p),i(e,Ba,p),s(Ba,Ah),s(Ba,ol),s(ol,Oh),s(Ba,Ch),i(e,zi,p),k(Un,e,p),i(e,Di,p),i(e,er,p),s(er,Ph),i(e,Si,p),k(Mn,e,p),i(e,Ai,p),i(e,Ka,p),s(Ka,Lh),s(Ka,rl),s(rl,Ih),s(Ka,Rh),i(e,Oi,p),k(Wn,e,p),i(e,Ci,p),k(Bn,e,p),i(e,Pi,p),i(e,Xe,p),s(Xe,Fh),s(Xe,ll),s(ll,Gh),s(Xe,Nh),s(Xe,il),s(il,Hh),s(Xe,Uh),s(Xe,pl),s(pl,Mh),s(Xe,Wh),s(Xe,ml),s(ml,Bh),s(Xe,Kh),s(Xe,hl),s(hl,Yh),s(Xe,Vh),i(e,Li,p),k(Kn,e,p),i(e,Ii,p),k(Yn,e,p),i(e,Ri,p),i(e,Ze,p),s(Ze,Jh),s(Ze,cl),s(cl,Xh),s(Ze,Zh),s(Ze,ul),s(ul,Qh),s(Ze,ec),s(Ze,dl),s(dl,sc),s(Ze,tc),s(Ze,fl),s(fl,ac),s(Ze,nc),s(Ze,gl),s(gl,oc),s(Ze,rc),i(e,Fi,p),k(Ya,e,p),i(e,Gi,p),i(e,sr,p),s(sr,lc),i(e,Ni,p),i(e,ca,p),s(ca,Va),s(Va,_l),k(Vn,_l,null),s(ca,ic),s(ca,wl),s(wl,pc),i(e,Hi,p),i(e,Ns,p),s(Ns,mc),s(Ns,bl),s(bl,hc),s(Ns,cc),s(Ns,vl),s(vl,uc),s(Ns,dc),s(Ns,yl),s(yl,fc),s(Ns,gc),i(e,Ui,p),k(Jn,e,p),i(e,Mi,p),i(e,tr,p),s(tr,_c),i(e,Wi,p),k(Xn,e,p),i(e,Bi,p),i(e,Ja,p),s(Ja,wc),s(Ja,kl),s(kl,bc),s(Ja,vc),i(e,Ki,p),k(Zn,e,p),i(e,Yi,p),k(Qn,e,p),i(e,Vi,p),i(e,ar,p),s(ar,yc),i(e,Ji,p),k(eo,e,p),i(e,Xi,p),i(e,nr,p),s(nr,kc),i(e,Zi,p),k(so,e,p),i(e,Qi,p),k(to,e,p),i(e,ep,p),i(e,Xa,p),s(Xa,$c),s(Xa,$l),s($l,jc),s(Xa,xc),i(e,sp,p),_o[dt].m(e,p),i(e,or,p),k(Za,e,p),i(e,tp,p),i(e,rr,p),s(rr,Ec),i(e,ap,p),k(ao,e,p),i(e,np,p),i(e,lr,p),s(lr,Tc),i(e,op,p),k(no,e,p),i(e,rp,p),Pe&&Pe.m(e,p),i(e,ir,p),i(e,Qa,p),s(Qa,qc),s(Qa,pr),s(pr,zc),s(Qa,Dc),i(e,lp,p),i(e,Hs,p),s(Hs,Sc),s(Hs,jl),s(jl,Ac),s(Hs,Oc),s(Hs,xl),s(xl,Cc),s(Hs,Pc),s(Hs,El),s(El,Lc),s(Hs,Ic),i(e,ip,p),wo[gt].m(e,p),i(e,mr,p),i(e,hr,p),s(hr,Rc),i(e,pp,p),k(oo,e,p),i(e,mp,p),i(e,Pt,p),s(Pt,Fc),s(Pt,Tl),s(Tl,Gc),s(Pt,Nc),s(Pt,ql),s(ql,Hc),s(Pt,Uc),i(e,hp,p),k(ro,e,p),i(e,cp,p),k(lo,e,p),i(e,up,p),i(e,Ee,p),s(Ee,Mc),s(Ee,zl),s(zl,Wc),s(Ee,Bc),s(Ee,Dl),s(Dl,Kc),s(Ee,Yc),s(Ee,Sl),s(Sl,Vc),s(Ee,Jc),s(Ee,Al),s(Al,Xc),s(Ee,Zc),s(Ee,Ol),s(Ol,Qc),s(Ee,eu),s(Ee,Cl),s(Cl,su),s(Ee,tu),s(Ee,Pl),s(Pl,au),s(Ee,nu),s(Ee,Ll),s(Ll,ou),s(Ee,ru),i(e,dp,p),bo[wt].m(e,p),i(e,cr,p),Le&&Le.m(e,p),i(e,ur,p),i(e,ua,p),s(ua,en),s(en,Il),k(io,Il,null),s(ua,lu),s(ua,Rl),s(Rl,iu),i(e,fp,p),i(e,sn,p),s(sn,pu),s(sn,Fl),s(Fl,mu),s(sn,hu),i(e,gp,p),k(po,e,p),i(e,_p,p),i(e,dr,p),s(dr,cu),i(e,wp,p),k(mo,e,p),i(e,bp,p),i(e,fr,p),s(fr,uu),i(e,vp,p),k(ho,e,p),i(e,yp,p),k(co,e,p),i(e,kp,p),i(e,tn,p),s(tn,du),s(tn,Gl),s(Gl,fu),s(tn,gu),i(e,$p,p),k(uo,e,p),i(e,jp,p),k(fo,e,p),i(e,xp,p),i(e,gr,p),s(gr,_u),i(e,Ep,p),i(e,_r,p),s(_r,wu),Tp=!0},p(e,[p]){const vo={};p&1&&(vo.fw=e[0]),d.$set(vo);let wr=O;O=xu(e),O!==wr&&(Or(),_(go[wr],1,1,()=>{go[wr]=null}),Ar(),G=go[O],G||(G=go[O]=ju[O](e),G.c()),g(G,1),G.m(J.parentNode,J));const Nl={};p&2&&(Nl.$$scope={dirty:p,ctx:e}),ss.$set(Nl);const Hl={};p&2&&(Hl.$$scope={dirty:p,ctx:e}),Ia.$set(Hl);const yo={};p&2&&(yo.$$scope={dirty:p,ctx:e}),Ga.$set(yo);const Ul={};p&2&&(Ul.$$scope={dirty:p,ctx:e}),Ha.$set(Ul);const da={};p&2&&(da.$$scope={dirty:p,ctx:e}),Wa.$set(da);const Ml={};p&2&&(Ml.$$scope={dirty:p,ctx:e}),Ya.$set(Ml);let br=dt;dt=Tu(e),dt!==br&&(Or(),_(_o[br],1,1,()=>{_o[br]=null}),Ar(),ft=_o[dt],ft||(ft=_o[dt]=Eu[dt](e),ft.c()),g(ft,1),ft.m(or.parentNode,or));const Wl={};p&2&&(Wl.$$scope={dirty:p,ctx:e}),Za.$set(Wl),e[0]==="pt"?Pe?p&1&&g(Pe,1):(Pe=ug(),Pe.c(),g(Pe,1),Pe.m(ir.parentNode,ir)):Pe&&(Or(),_(Pe,1,1,()=>{Pe=null}),Ar());let an=gt;gt=zu(e),gt!==an&&(Or(),_(wo[an],1,1,()=>{wo[an]=null}),Ar(),_t=wo[gt],_t||(_t=wo[gt]=qu[gt](e),_t.c()),g(_t,1),_t.m(mr.parentNode,mr));let vr=wt;wt=Su(e),wt!==vr&&(Or(),_(bo[vr],1,1,()=>{bo[vr]=null}),Ar(),bt=bo[wt],bt||(bt=bo[wt]=Du[wt](e),bt.c()),g(bt,1),bt.m(cr.parentNode,cr)),e[0]==="pt"?Le?p&1&&g(Le,1):(Le=dg(e),Le.c(),g(Le,1),Le.m(ur.parentNode,ur)):Le&&(Or(),_(Le,1,1,()=>{Le=null}),Ar())},i(e){Tp||(g(d.$$.fragment,e),g(P.$$.fragment,e),g(G),g(ee.$$.fragment,e),g(fe.$$.fragment,e),g(ze.$$.fragment,e),g(Ae.$$.fragment,e),g(Me.$$.fragment,e),g(ue.$$.fragment,e),g(ss.$$.fragment,e),g(Be.$$.fragment,e),g(ve.$$.fragment,e),g(Ke.$$.fragment,e),g(Ts.$$.fragment,e),g(ot.$$.fragment,e),g(qs.$$.fragment,e),g(As.$$.fragment,e),g(lt.$$.fragment,e),g(qt.$$.fragment,e),g(zt.$$.fragment,e),g(Ia.$$.fragment,e),g(qn.$$.fragment,e),g(zn.$$.fragment,e),g(Dn.$$.fragment,e),g(Ga.$$.fragment,e),g(Sn.$$.fragment,e),g(An.$$.fragment,e),g(On.$$.fragment,e),g(Cn.$$.fragment,e),g(Pn.$$.fragment,e),g(Ln.$$.fragment,e),g(Ha.$$.fragment,e),g(In.$$.fragment,e),g(Rn.$$.fragment,e),g(Gn.$$.fragment,e),g(Wa.$$.fragment,e),g(Un.$$.fragment,e),g(Mn.$$.fragment,e),g(Wn.$$.fragment,e),g(Bn.$$.fragment,e),g(Kn.$$.fragment,e),g(Yn.$$.fragment,e),g(Ya.$$.fragment,e),g(Vn.$$.fragment,e),g(Jn.$$.fragment,e),g(Xn.$$.fragment,e),g(Zn.$$.fragment,e),g(Qn.$$.fragment,e),g(eo.$$.fragment,e),g(so.$$.fragment,e),g(to.$$.fragment,e),g(ft),g(Za.$$.fragment,e),g(ao.$$.fragment,e),g(no.$$.fragment,e),g(Pe),g(_t),g(oo.$$.fragment,e),g(ro.$$.fragment,e),g(lo.$$.fragment,e),g(bt),g(Le),g(io.$$.fragment,e),g(po.$$.fragment,e),g(mo.$$.fragment,e),g(ho.$$.fragment,e),g(co.$$.fragment,e),g(uo.$$.fragment,e),g(fo.$$.fragment,e),Tp=!0)},o(e){_(d.$$.fragment,e),_(P.$$.fragment,e),_(G),_(ee.$$.fragment,e),_(fe.$$.fragment,e),_(ze.$$.fragment,e),_(Ae.$$.fragment,e),_(Me.$$.fragment,e),_(ue.$$.fragment,e),_(ss.$$.fragment,e),_(Be.$$.fragment,e),_(ve.$$.fragment,e),_(Ke.$$.fragment,e),_(Ts.$$.fragment,e),_(ot.$$.fragment,e),_(qs.$$.fragment,e),_(As.$$.fragment,e),_(lt.$$.fragment,e),_(qt.$$.fragment,e),_(zt.$$.fragment,e),_(Ia.$$.fragment,e),_(qn.$$.fragment,e),_(zn.$$.fragment,e),_(Dn.$$.fragment,e),_(Ga.$$.fragment,e),_(Sn.$$.fragment,e),_(An.$$.fragment,e),_(On.$$.fragment,e),_(Cn.$$.fragment,e),_(Pn.$$.fragment,e),_(Ln.$$.fragment,e),_(Ha.$$.fragment,e),_(In.$$.fragment,e),_(Rn.$$.fragment,e),_(Gn.$$.fragment,e),_(Wa.$$.fragment,e),_(Un.$$.fragment,e),_(Mn.$$.fragment,e),_(Wn.$$.fragment,e),_(Bn.$$.fragment,e),_(Kn.$$.fragment,e),_(Yn.$$.fragment,e),_(Ya.$$.fragment,e),_(Vn.$$.fragment,e),_(Jn.$$.fragment,e),_(Xn.$$.fragment,e),_(Zn.$$.fragment,e),_(Qn.$$.fragment,e),_(eo.$$.fragment,e),_(so.$$.fragment,e),_(to.$$.fragment,e),_(ft),_(Za.$$.fragment,e),_(ao.$$.fragment,e),_(no.$$.fragment,e),_(Pe),_(_t),_(oo.$$.fragment,e),_(ro.$$.fragment,e),_(lo.$$.fragment,e),_(bt),_(Le),_(io.$$.fragment,e),_(po.$$.fragment,e),_(mo.$$.fragment,e),_(ho.$$.fragment,e),_(co.$$.fragment,e),_(uo.$$.fragment,e),_(fo.$$.fragment,e),Tp=!1},d(e){t(u),e&&t(j),$(d,e),e&&t(E),e&&t(C),$(P),e&&t(D),go[O].d(e),e&&t(J),e&&t(X),e&&t(M),$(ee,e),e&&t(F),e&&t(K),e&&t(Te),e&&t(H),e&&t(me),e&&t(oe),e&&t(ke),e&&t(se),$(fe),e&&t(b),e&&t(U),e&&t(re),$(ze,e),e&&t(Qe),$(Ae,e),e&&t(ne),e&&t(le),e&&t(ce),$(Me,e),e&&t(Vs),$(ue,e),e&&t(ms),$(ss,e),e&&t(Ss),e&&t(De),e&&t(Qs),$(Be,e),e&&t(et),$(ve,e),e&&t($s),e&&t(Se),e&&t(Gt),$(Ke,e),e&&t(wa),e&&t(xe),e&&t(va),$(Ts,e),e&&t(ya),e&&t(Et),e&&t(Tt),$(ot,e),e&&t(ka),$(qs,e),e&&t($a),e&&t(Ve),e&&t(ja),$(As,e),e&&t(rt),$(lt,e),e&&t(Xt),e&&t(us),e&&t(Os),e&&t(ds),e&&t(Qt),e&&t(m),e&&t(pt),$(qt,e),e&&t(wn),e&&t(mt),e&&t(bn),e&&t(Cs),$(zt),e&&t(ea),e&&t(ht),e&&t(vn),e&&t(ct),e&&t(Kl),e&&t(Ko),e&&t(Yl),e&&t(La),e&&t(Vl),e&&t(pa),e&&t(Jl),e&&t(Yo),e&&t(Xl),$(Ia,e),e&&t(Zl),e&&t(ma),$(qn),e&&t(Ql),$(zn,e),e&&t(ei),e&&t(Fa),e&&t(si),$(Dn,e),e&&t(ti),$(Ga,e),e&&t(ai),e&&t(Vo),e&&t(ni),$(Sn,e),e&&t(oi),$(An,e),e&&t(ri),e&&t(fs),e&&t(li),$(On,e),e&&t(ii),$(Cn,e),e&&t(pi),e&&t(Gs),e&&t(mi),e&&t(Na),e&&t(hi),$(Pn,e),e&&t(ci),e&&t(gs),e&&t(ui),e&&t(Ot),e&&t(di),$(Ln,e),e&&t(fi),e&&t(Zo),e&&t(gi),$(Ha,e),e&&t(_i),e&&t(ha),$(In),e&&t(wi),$(Rn,e),e&&t(bi),e&&t(Qo),e&&t(vi),e&&t(Ma),e&&t(yi),$(Gn,e),e&&t(ki),e&&t(Ct),e&&t($i),$(Wa,e),e&&t(ji),e&&t(Nn),e&&t(Ei),e&&t(Hn),e&&t(qi),e&&t(Ba),e&&t(zi),$(Un,e),e&&t(Di),e&&t(er),e&&t(Si),$(Mn,e),e&&t(Ai),e&&t(Ka),e&&t(Oi),$(Wn,e),e&&t(Ci),$(Bn,e),e&&t(Pi),e&&t(Xe),e&&t(Li),$(Kn,e),e&&t(Ii),$(Yn,e),e&&t(Ri),e&&t(Ze),e&&t(Fi),$(Ya,e),e&&t(Gi),e&&t(sr),e&&t(Ni),e&&t(ca),$(Vn),e&&t(Hi),e&&t(Ns),e&&t(Ui),$(Jn,e),e&&t(Mi),e&&t(tr),e&&t(Wi),$(Xn,e),e&&t(Bi),e&&t(Ja),e&&t(Ki),$(Zn,e),e&&t(Yi),$(Qn,e),e&&t(Vi),e&&t(ar),e&&t(Ji),$(eo,e),e&&t(Xi),e&&t(nr),e&&t(Zi),$(so,e),e&&t(Qi),$(to,e),e&&t(ep),e&&t(Xa),e&&t(sp),_o[dt].d(e),e&&t(or),$(Za,e),e&&t(tp),e&&t(rr),e&&t(ap),$(ao,e),e&&t(np),e&&t(lr),e&&t(op),$(no,e),e&&t(rp),Pe&&Pe.d(e),e&&t(ir),e&&t(Qa),e&&t(lp),e&&t(Hs),e&&t(ip),wo[gt].d(e),e&&t(mr),e&&t(hr),e&&t(pp),$(oo,e),e&&t(mp),e&&t(Pt),e&&t(hp),$(ro,e),e&&t(cp),$(lo,e),e&&t(up),e&&t(Ee),e&&t(dp),bo[wt].d(e),e&&t(cr),Le&&Le.d(e),e&&t(ur),e&&t(ua),$(io),e&&t(fp),e&&t(sn),e&&t(gp),$(po,e),e&&t(_p),e&&t(dr),e&&t(wp),$(mo,e),e&&t(bp),e&&t(fr),e&&t(vp),$(ho,e),e&&t(yp),$(co,e),e&&t(kp),e&&t(tn),e&&t($p),$(uo,e),e&&t(jp),$(fo,e),e&&t(xp),e&&t(gr),e&&t(Ep),e&&t(_r)}}}const Ng={local:"summarization",sections:[{local:"preparing-a-multilingual-corpus",title:"Preparing a multilingual corpus"},{local:"models-for-text-summarization",title:"Models for text summarization"},{local:"preprocessing-the-data",title:"Preprocessing the data"},{local:"metrics-for-text-summarization",sections:[{local:"creating-a-strong-baseline",title:"Creating a strong baseline"}],title:"Metrics for text summarization"},{local:"finetuning-mt5-with-the-trainer-api",title:"Fine-tuning mT5 with the `Trainer` API"},{local:"finetuning-mt5-with-keras",title:"Fine-tuning mT5 with Keras"},{local:"finetuning-mt5-with-accelerate",sections:[{local:"preparing-everything-for-training",title:"Preparing everything for training"},{local:"training-loop",title:"Training loop"}],title:"Fine-tuning mT5 with \u{1F917} Accelerate"},{local:"using-your-finetuned-model",title:"Using your fine-tuned model"}],title:"Summarization"};function Hg(Y,u,j){let d="pt";return kg(()=>{const E=new URLSearchParams(window.location.search);j(0,d=E.get("fw")||"pt")}),[d]}class Jg extends wg{constructor(u){super();bg(this,u,Hg,Gg,vg,{})}}export{Jg as default,Ng as metadata};
