import{S as cn,i as dn,s as gn,e as i,k as c,w as M,t as l,M as _n,c as p,d as s,m as d,a as f,x as D,h as n,b as g,G as t,g as m,y as F,q as C,o as z,B as L,v as $n,L as un}from"../../chunks/vendor-hf-doc-builder.js";import{T as Dt}from"../../chunks/Tip-hf-doc-builder.js";import{Y as ea}from"../../chunks/Youtube-hf-doc-builder.js";import{I as vt}from"../../chunks/IconCopyLink-hf-doc-builder.js";import{C as ae}from"../../chunks/CodeBlock-hf-doc-builder.js";import{F as hl,M as Ft}from"../../chunks/Markdown-hf-doc-builder.js";function kn(J){let a,w,o,_,v,k,j,A,y,x,q,G,N,T,S,R,B,O,Y,$;return{c(){a=i("p"),w=l("You can fine-tune other architectures for language modeling such as "),o=i("a"),_=l("GPT-Neo"),v=l(", "),k=i("a"),j=l("GPT-J"),A=l(", and "),y=i("a"),x=l("BERT"),q=l(", following the same steps presented in this guide!"),G=c(),N=i("p"),T=l("See the text generation "),S=i("a"),R=l("task page"),B=l(" and fill mask "),O=i("a"),Y=l("task page"),$=l(" for more information about their associated models, datasets, and metrics."),this.h()},l(b){a=p(b,"P",{});var W=f(a);w=n(W,"You can fine-tune other architectures for language modeling such as "),o=p(W,"A",{href:!0,rel:!0});var P=f(o);_=n(P,"GPT-Neo"),P.forEach(s),v=n(W,", "),k=p(W,"A",{href:!0,rel:!0});var H=f(k);j=n(H,"GPT-J"),H.forEach(s),A=n(W,", and "),y=p(W,"A",{href:!0,rel:!0});var Q=f(y);x=n(Q,"BERT"),Q.forEach(s),q=n(W,", following the same steps presented in this guide!"),W.forEach(s),G=d(b),N=p(b,"P",{});var ee=f(N);T=n(ee,"See the text generation "),S=p(ee,"A",{href:!0,rel:!0});var te=f(S);R=n(te,"task page"),te.forEach(s),B=n(ee," and fill mask "),O=p(ee,"A",{href:!0,rel:!0});var pe=f(O);Y=n(pe,"task page"),pe.forEach(s),$=n(ee," for more information about their associated models, datasets, and metrics."),ee.forEach(s),this.h()},h(){g(o,"href","https://huggingface.co/EleutherAI/gpt-neo-125M"),g(o,"rel","nofollow"),g(k,"href","https://huggingface.co/EleutherAI/gpt-j-6B"),g(k,"rel","nofollow"),g(y,"href","https://huggingface.co/bert-base-uncased"),g(y,"rel","nofollow"),g(S,"href","https://huggingface.co/tasks/text-generation"),g(S,"rel","nofollow"),g(O,"href","https://huggingface.co/tasks/fill-mask"),g(O,"rel","nofollow")},m(b,W){m(b,a,W),t(a,w),t(a,o),t(o,_),t(a,v),t(a,k),t(k,j),t(a,A),t(a,y),t(y,x),t(a,q),m(b,G,W),m(b,N,W),t(N,T),t(N,S),t(S,R),t(N,B),t(N,O),t(O,Y),t(N,$)},d(b){b&&s(a),b&&s(G),b&&s(N)}}}function wn(J){let a,w,o,_,v,k,j,A,y,x,q,G,N,T,S,R,B,O,Y;return j=new ae({props:{code:`from transformers import DataCollatorForLanguageModeling

tokenizer.pad_token = tokenizer.eos_token
data_collator = DataCollatorForLanguageModeling(tokenizer=tokenizer, mlm=False)`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> DataCollatorForLanguageModeling

<span class="hljs-meta">&gt;&gt;&gt; </span>tokenizer.pad_token = tokenizer.eos_token
<span class="hljs-meta">&gt;&gt;&gt; </span>data_collator = DataCollatorForLanguageModeling(tokenizer=tokenizer, mlm=<span class="hljs-literal">False</span>)`}}),O=new ae({props:{code:`from transformers import DataCollatorForLanguageModeling

tokenizer.pad_token = tokenizer.eos_token
data_collator = DataCollatorForLanguageModeling(tokenizer=tokenizer, mlm_probability=0.15)`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> DataCollatorForLanguageModeling

<span class="hljs-meta">&gt;&gt;&gt; </span>tokenizer.pad_token = tokenizer.eos_token
<span class="hljs-meta">&gt;&gt;&gt; </span>data_collator = DataCollatorForLanguageModeling(tokenizer=tokenizer, mlm_probability=<span class="hljs-number">0.15</span>)`}}),{c(){a=i("p"),w=l("You can use the end of sequence token as the padding token, and set "),o=i("code"),_=l("mlm=False"),v=l(". This will use the inputs as labels shifted to the right by one element:"),k=c(),M(j.$$.fragment),A=c(),y=i("p"),x=l("For masked language modeling, use the same "),q=i("a"),G=l("DataCollatorForLanguageModeling"),N=l(" except you should specify "),T=i("code"),S=l("mlm_probability"),R=l(" to randomly mask tokens each time you iterate over the data."),B=c(),M(O.$$.fragment),this.h()},l($){a=p($,"P",{});var b=f(a);w=n(b,"You can use the end of sequence token as the padding token, and set "),o=p(b,"CODE",{});var W=f(o);_=n(W,"mlm=False"),W.forEach(s),v=n(b,". This will use the inputs as labels shifted to the right by one element:"),b.forEach(s),k=d($),D(j.$$.fragment,$),A=d($),y=p($,"P",{});var P=f(y);x=n(P,"For masked language modeling, use the same "),q=p(P,"A",{href:!0});var H=f(q);G=n(H,"DataCollatorForLanguageModeling"),H.forEach(s),N=n(P," except you should specify "),T=p(P,"CODE",{});var Q=f(T);S=n(Q,"mlm_probability"),Q.forEach(s),R=n(P," to randomly mask tokens each time you iterate over the data."),P.forEach(s),B=d($),D(O.$$.fragment,$),this.h()},h(){g(q,"href","/docs/transformers/pr_17548/en/main_classes/data_collator#transformers.DataCollatorForLanguageModeling")},m($,b){m($,a,b),t(a,w),t(a,o),t(o,_),t(a,v),m($,k,b),F(j,$,b),m($,A,b),m($,y,b),t(y,x),t(y,q),t(q,G),t(y,N),t(y,T),t(T,S),t(y,R),m($,B,b),F(O,$,b),Y=!0},p:un,i($){Y||(C(j.$$.fragment,$),C(O.$$.fragment,$),Y=!0)},o($){z(j.$$.fragment,$),z(O.$$.fragment,$),Y=!1},d($){$&&s(a),$&&s(k),L(j,$),$&&s(A),$&&s(y),$&&s(B),L(O,$)}}}function jn(J){let a,w;return a=new Ft({props:{$$slots:{default:[wn]},$$scope:{ctx:J}}}),{c(){M(a.$$.fragment)},l(o){D(a.$$.fragment,o)},m(o,_){F(a,o,_),w=!0},p(o,_){const v={};_&2&&(v.$$scope={dirty:_,ctx:o}),a.$set(v)},i(o){w||(C(a.$$.fragment,o),w=!0)},o(o){z(a.$$.fragment,o),w=!1},d(o){L(a,o)}}}function yn(J){let a,w,o,_,v,k,j,A,y,x,q,G,N,T,S,R,B,O,Y;return j=new ae({props:{code:`from transformers import DataCollatorForLanguageModeling

data_collator = DataCollatorForLanguageModeling(tokenizer=tokenizer, mlm=False, return_tensors="tf")`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> DataCollatorForLanguageModeling

<span class="hljs-meta">&gt;&gt;&gt; </span>data_collator = DataCollatorForLanguageModeling(tokenizer=tokenizer, mlm=<span class="hljs-literal">False</span>, return_tensors=<span class="hljs-string">&quot;tf&quot;</span>)`}}),O=new ae({props:{code:`from transformers import DataCollatorForLanguageModeling

data_collator = DataCollatorForLanguageModeling(tokenizer=tokenizer, mlm=False, return_tensors="tf")`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> DataCollatorForLanguageModeling

<span class="hljs-meta">&gt;&gt;&gt; </span>data_collator = DataCollatorForLanguageModeling(tokenizer=tokenizer, mlm=<span class="hljs-literal">False</span>, return_tensors=<span class="hljs-string">&quot;tf&quot;</span>)`}}),{c(){a=i("p"),w=l("You can use the end of sequence token as the padding token, and set "),o=i("code"),_=l("mlm=False"),v=l(". This will use the inputs as labels shifted to the right by one element:"),k=c(),M(j.$$.fragment),A=c(),y=i("p"),x=l("For masked language modeling, use the same "),q=i("a"),G=l("DataCollatorForLanguageModeling"),N=l(" except you should specify "),T=i("code"),S=l("mlm_probability"),R=l(" to randomly mask tokens each time you iterate over the data."),B=c(),M(O.$$.fragment),this.h()},l($){a=p($,"P",{});var b=f(a);w=n(b,"You can use the end of sequence token as the padding token, and set "),o=p(b,"CODE",{});var W=f(o);_=n(W,"mlm=False"),W.forEach(s),v=n(b,". This will use the inputs as labels shifted to the right by one element:"),b.forEach(s),k=d($),D(j.$$.fragment,$),A=d($),y=p($,"P",{});var P=f(y);x=n(P,"For masked language modeling, use the same "),q=p(P,"A",{href:!0});var H=f(q);G=n(H,"DataCollatorForLanguageModeling"),H.forEach(s),N=n(P," except you should specify "),T=p(P,"CODE",{});var Q=f(T);S=n(Q,"mlm_probability"),Q.forEach(s),R=n(P," to randomly mask tokens each time you iterate over the data."),P.forEach(s),B=d($),D(O.$$.fragment,$),this.h()},h(){g(q,"href","/docs/transformers/pr_17548/en/main_classes/data_collator#transformers.DataCollatorForLanguageModeling")},m($,b){m($,a,b),t(a,w),t(a,o),t(o,_),t(a,v),m($,k,b),F(j,$,b),m($,A,b),m($,y,b),t(y,x),t(y,q),t(q,G),t(y,N),t(y,T),t(T,S),t(y,R),m($,B,b),F(O,$,b),Y=!0},p:un,i($){Y||(C(j.$$.fragment,$),C(O.$$.fragment,$),Y=!0)},o($){z(j.$$.fragment,$),z(O.$$.fragment,$),Y=!1},d($){$&&s(a),$&&s(k),L(j,$),$&&s(A),$&&s(y),$&&s(B),L(O,$)}}}function bn(J){let a,w;return a=new Ft({props:{$$slots:{default:[yn]},$$scope:{ctx:J}}}),{c(){M(a.$$.fragment)},l(o){D(a.$$.fragment,o)},m(o,_){F(a,o,_),w=!0},p(o,_){const v={};_&2&&(v.$$scope={dirty:_,ctx:o}),a.$set(v)},i(o){w||(C(a.$$.fragment,o),w=!0)},o(o){z(a.$$.fragment,o),w=!1},d(o){L(a,o)}}}function vn(J){let a,w,o,_,v,k,j,A;return{c(){a=i("p"),w=l("If you aren\u2019t familiar with fine-tuning a model with the "),o=i("a"),_=l("Trainer"),v=l(", take a look at the basic tutorial "),k=i("a"),j=l("here"),A=l("!"),this.h()},l(y){a=p(y,"P",{});var x=f(a);w=n(x,"If you aren\u2019t familiar with fine-tuning a model with the "),o=p(x,"A",{href:!0});var q=f(o);_=n(q,"Trainer"),q.forEach(s),v=n(x,", take a look at the basic tutorial "),k=p(x,"A",{href:!0});var G=f(k);j=n(G,"here"),G.forEach(s),A=n(x,"!"),x.forEach(s),this.h()},h(){g(o,"href","/docs/transformers/pr_17548/en/main_classes/trainer#transformers.Trainer"),g(k,"href","../training#finetune-with-trainer")},m(y,x){m(y,a,x),t(a,w),t(a,o),t(o,_),t(a,v),t(a,k),t(k,j),t(a,A)},d(y){y&&s(a)}}}function En(J){let a,w,o,_,v,k,j,A,y,x,q,G,N,T,S,R,B,O,Y,$,b,W,P,H,Q,ee,te,pe,U,he,se,ne,K,re;return j=new ae({props:{code:`from transformers import AutoModelForCausalLM, TrainingArguments, Trainer

model = AutoModelForCausalLM.from_pretrained("distilgpt2")`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoModelForCausalLM, TrainingArguments, Trainer

<span class="hljs-meta">&gt;&gt;&gt; </span>model = AutoModelForCausalLM.from_pretrained(<span class="hljs-string">&quot;distilgpt2&quot;</span>)`}}),y=new Dt({props:{$$slots:{default:[vn]},$$scope:{ctx:J}}}),K=new ae({props:{code:`training_args = TrainingArguments(
    output_dir="./results",
    evaluation_strategy="epoch",
    learning_rate=2e-5,
    weight_decay=0.01,
)

trainer = Trainer(
    model=model,
    args=training_args,
    train_dataset=lm_dataset["train"],
    eval_dataset=lm_dataset["test"],
    data_collator=data_collator,
)

trainer.train()`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>training_args = TrainingArguments(
<span class="hljs-meta">... </span>    output_dir=<span class="hljs-string">&quot;./results&quot;</span>,
<span class="hljs-meta">... </span>    evaluation_strategy=<span class="hljs-string">&quot;epoch&quot;</span>,
<span class="hljs-meta">... </span>    learning_rate=<span class="hljs-number">2e-5</span>,
<span class="hljs-meta">... </span>    weight_decay=<span class="hljs-number">0.01</span>,
<span class="hljs-meta">... </span>)

<span class="hljs-meta">&gt;&gt;&gt; </span>trainer = Trainer(
<span class="hljs-meta">... </span>    model=model,
<span class="hljs-meta">... </span>    args=training_args,
<span class="hljs-meta">... </span>    train_dataset=lm_dataset[<span class="hljs-string">&quot;train&quot;</span>],
<span class="hljs-meta">... </span>    eval_dataset=lm_dataset[<span class="hljs-string">&quot;test&quot;</span>],
<span class="hljs-meta">... </span>    data_collator=data_collator,
<span class="hljs-meta">... </span>)

<span class="hljs-meta">&gt;&gt;&gt; </span>trainer.train()`}}),{c(){a=i("p"),w=l("Load DistilGPT2 with "),o=i("a"),_=l("AutoModelForCausalLM"),v=l(":"),k=c(),M(j.$$.fragment),A=c(),M(y.$$.fragment),x=c(),q=i("p"),G=l("At this point, only three steps remain:"),N=c(),T=i("ol"),S=i("li"),R=l("Define your training hyperparameters in "),B=i("a"),O=l("TrainingArguments"),Y=l("."),$=c(),b=i("li"),W=l("Pass the training arguments to "),P=i("a"),H=l("Trainer"),Q=l(" along with the model, datasets, and data collator."),ee=c(),te=i("li"),pe=l("Call "),U=i("a"),he=l("train()"),se=l(" to fine-tune your model."),ne=c(),M(K.$$.fragment),this.h()},l(h){a=p(h,"P",{});var I=f(a);w=n(I,"Load DistilGPT2 with "),o=p(I,"A",{href:!0});var oe=f(o);_=n(oe,"AutoModelForCausalLM"),oe.forEach(s),v=n(I,":"),I.forEach(s),k=d(h),D(j.$$.fragment,h),A=d(h),D(y.$$.fragment,h),x=d(h),q=p(h,"P",{});var le=f(q);G=n(le,"At this point, only three steps remain:"),le.forEach(s),N=d(h),T=p(h,"OL",{});var X=f(T);S=p(X,"LI",{});var Z=f(S);R=n(Z,"Define your training hyperparameters in "),B=p(Z,"A",{href:!0});var fe=f(B);O=n(fe,"TrainingArguments"),fe.forEach(s),Y=n(Z,"."),Z.forEach(s),$=d(X),b=p(X,"LI",{});var V=f(b);W=n(V,"Pass the training arguments to "),P=p(V,"A",{href:!0});var me=f(P);H=n(me,"Trainer"),me.forEach(s),Q=n(V," along with the model, datasets, and data collator."),V.forEach(s),ee=d(X),te=p(X,"LI",{});var ie=f(te);pe=n(ie,"Call "),U=p(ie,"A",{href:!0});var ce=f(U);he=n(ce,"train()"),ce.forEach(s),se=n(ie," to fine-tune your model."),ie.forEach(s),X.forEach(s),ne=d(h),D(K.$$.fragment,h),this.h()},h(){g(o,"href","/docs/transformers/pr_17548/en/model_doc/auto#transformers.AutoModelForCausalLM"),g(B,"href","/docs/transformers/pr_17548/en/main_classes/trainer#transformers.TrainingArguments"),g(P,"href","/docs/transformers/pr_17548/en/main_classes/trainer#transformers.Trainer"),g(U,"href","/docs/transformers/pr_17548/en/main_classes/trainer#transformers.Trainer.train")},m(h,I){m(h,a,I),t(a,w),t(a,o),t(o,_),t(a,v),m(h,k,I),F(j,h,I),m(h,A,I),F(y,h,I),m(h,x,I),m(h,q,I),t(q,G),m(h,N,I),m(h,T,I),t(T,S),t(S,R),t(S,B),t(B,O),t(S,Y),t(T,$),t(T,b),t(b,W),t(b,P),t(P,H),t(b,Q),t(T,ee),t(T,te),t(te,pe),t(te,U),t(U,he),t(te,se),m(h,ne,I),F(K,h,I),re=!0},p(h,I){const oe={};I&2&&(oe.$$scope={dirty:I,ctx:h}),y.$set(oe)},i(h){re||(C(j.$$.fragment,h),C(y.$$.fragment,h),C(K.$$.fragment,h),re=!0)},o(h){z(j.$$.fragment,h),z(y.$$.fragment,h),z(K.$$.fragment,h),re=!1},d(h){h&&s(a),h&&s(k),L(j,h),h&&s(A),L(y,h),h&&s(x),h&&s(q),h&&s(N),h&&s(T),h&&s(ne),L(K,h)}}}function xn(J){let a,w;return a=new Ft({props:{$$slots:{default:[En]},$$scope:{ctx:J}}}),{c(){M(a.$$.fragment)},l(o){D(a.$$.fragment,o)},m(o,_){F(a,o,_),w=!0},p(o,_){const v={};_&2&&(v.$$scope={dirty:_,ctx:o}),a.$set(v)},i(o){w||(C(a.$$.fragment,o),w=!0)},o(o){z(a.$$.fragment,o),w=!1},d(o){L(a,o)}}}function Tn(J){let a,w,o,_,v;return{c(){a=i("p"),w=l("If you aren\u2019t familiar with fine-tuning a model with Keras, take a look at the basic tutorial "),o=i("a"),_=l("here"),v=l("!"),this.h()},l(k){a=p(k,"P",{});var j=f(a);w=n(j,"If you aren\u2019t familiar with fine-tuning a model with Keras, take a look at the basic tutorial "),o=p(j,"A",{href:!0});var A=f(o);_=n(A,"here"),A.forEach(s),v=n(j,"!"),j.forEach(s),this.h()},h(){g(o,"href","training#finetune-with-keras")},m(k,j){m(k,a,j),t(a,w),t(a,o),t(o,_),t(a,v)},d(k){k&&s(a)}}}function An(J){let a,w,o,_,v,k,j,A,y,x,q,G,N,T,S,R,B,O,Y,$,b,W,P,H,Q,ee,te,pe,U,he,se,ne,K,re,h,I,oe,le,X,Z,fe,V,me,ie,ce,ge,ue,_e;return T=new ae({props:{code:`tf_train_set = lm_dataset["train"].to_tf_dataset(
    columns=["attention_mask", "input_ids", "labels"],
    dummy_labels=True,
    shuffle=True,
    batch_size=16,
    collate_fn=data_collator,
)

tf_test_set = lm_dataset["test"].to_tf_dataset(
    columns=["attention_mask", "input_ids", "labels"],
    dummy_labels=True,
    shuffle=False,
    batch_size=16,
    collate_fn=data_collator,
)`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>tf_train_set = lm_dataset[<span class="hljs-string">&quot;train&quot;</span>].to_tf_dataset(
<span class="hljs-meta">... </span>    columns=[<span class="hljs-string">&quot;attention_mask&quot;</span>, <span class="hljs-string">&quot;input_ids&quot;</span>, <span class="hljs-string">&quot;labels&quot;</span>],
<span class="hljs-meta">... </span>    dummy_labels=<span class="hljs-literal">True</span>,
<span class="hljs-meta">... </span>    shuffle=<span class="hljs-literal">True</span>,
<span class="hljs-meta">... </span>    batch_size=<span class="hljs-number">16</span>,
<span class="hljs-meta">... </span>    collate_fn=data_collator,
<span class="hljs-meta">... </span>)

<span class="hljs-meta">&gt;&gt;&gt; </span>tf_test_set = lm_dataset[<span class="hljs-string">&quot;test&quot;</span>].to_tf_dataset(
<span class="hljs-meta">... </span>    columns=[<span class="hljs-string">&quot;attention_mask&quot;</span>, <span class="hljs-string">&quot;input_ids&quot;</span>, <span class="hljs-string">&quot;labels&quot;</span>],
<span class="hljs-meta">... </span>    dummy_labels=<span class="hljs-literal">True</span>,
<span class="hljs-meta">... </span>    shuffle=<span class="hljs-literal">False</span>,
<span class="hljs-meta">... </span>    batch_size=<span class="hljs-number">16</span>,
<span class="hljs-meta">... </span>    collate_fn=data_collator,
<span class="hljs-meta">... </span>)`}}),R=new Dt({props:{$$slots:{default:[Tn]},$$scope:{ctx:J}}}),b=new ae({props:{code:`from transformers import create_optimizer, AdamWeightDecay

optimizer = AdamWeightDecay(learning_rate=2e-5, weight_decay_rate=0.01)`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> create_optimizer, AdamWeightDecay

<span class="hljs-meta">&gt;&gt;&gt; </span>optimizer = AdamWeightDecay(learning_rate=<span class="hljs-number">2e-5</span>, weight_decay_rate=<span class="hljs-number">0.01</span>)`}}),U=new ae({props:{code:`from transformers import TFAutoModelForCausalLM

model = TFAutoModelForCausalLM.from_pretrained("distilgpt2")`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> TFAutoModelForCausalLM

<span class="hljs-meta">&gt;&gt;&gt; </span>model = TFAutoModelForCausalLM.from_pretrained(<span class="hljs-string">&quot;distilgpt2&quot;</span>)`}}),le=new ae({props:{code:`import tensorflow as tf

model.compile(optimizer=optimizer)`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">import</span> tensorflow <span class="hljs-keyword">as</span> tf

<span class="hljs-meta">&gt;&gt;&gt; </span>model.<span class="hljs-built_in">compile</span>(optimizer=optimizer)`}}),ue=new ae({props:{code:"model.fit(x=tf_train_set, validation_data=tf_test_set, epochs=3)",highlighted:'<span class="hljs-meta">&gt;&gt;&gt; </span>model.fit(x=tf_train_set, validation_data=tf_test_set, epochs=<span class="hljs-number">3</span>)'}}),{c(){a=i("p"),w=l("To fine-tune a model in TensorFlow, start by converting your datasets to the "),o=i("code"),_=l("tf.data.Dataset"),v=l(" format with "),k=i("a"),j=i("code"),A=l("to_tf_dataset"),y=l(". Specify inputs and labels in "),x=i("code"),q=l("columns"),G=l(", whether to shuffle the dataset order, batch size, and the data collator:"),N=c(),M(T.$$.fragment),S=c(),M(R.$$.fragment),B=c(),O=i("p"),Y=l("Set up an optimizer function, learning rate, and some training hyperparameters:"),$=c(),M(b.$$.fragment),W=c(),P=i("p"),H=l("Load DistilGPT2 with "),Q=i("a"),ee=l("TFAutoModelForCausalLM"),te=l(":"),pe=c(),M(U.$$.fragment),he=c(),se=i("p"),ne=l("Configure the model for training with "),K=i("a"),re=i("code"),h=l("compile"),I=l(":"),oe=c(),M(le.$$.fragment),X=c(),Z=i("p"),fe=l("Call "),V=i("a"),me=i("code"),ie=l("fit"),ce=l(" to fine-tune the model:"),ge=c(),M(ue.$$.fragment),this.h()},l(r){a=p(r,"P",{});var E=f(a);w=n(E,"To fine-tune a model in TensorFlow, start by converting your datasets to the "),o=p(E,"CODE",{});var de=f(o);_=n(de,"tf.data.Dataset"),de.forEach(s),v=n(E," format with "),k=p(E,"A",{href:!0,rel:!0});var be=f(k);j=p(be,"CODE",{});var qe=f(j);A=n(qe,"to_tf_dataset"),qe.forEach(s),be.forEach(s),y=n(E,". Specify inputs and labels in "),x=p(E,"CODE",{});var xe=f(x);q=n(xe,"columns"),xe.forEach(s),G=n(E,", whether to shuffle the dataset order, batch size, and the data collator:"),E.forEach(s),N=d(r),D(T.$$.fragment,r),S=d(r),D(R.$$.fragment,r),B=d(r),O=p(r,"P",{});var Pe=f(O);Y=n(Pe,"Set up an optimizer function, learning rate, and some training hyperparameters:"),Pe.forEach(s),$=d(r),D(b.$$.fragment,r),W=d(r),P=p(r,"P",{});var $e=f(P);H=n($e,"Load DistilGPT2 with "),Q=p($e,"A",{href:!0});var ve=f(Q);ee=n(ve,"TFAutoModelForCausalLM"),ve.forEach(s),te=n($e,":"),$e.forEach(s),pe=d(r),D(U.$$.fragment,r),he=d(r),se=p(r,"P",{});var ke=f(se);ne=n(ke,"Configure the model for training with "),K=p(ke,"A",{href:!0,rel:!0});var Te=f(K);re=p(Te,"CODE",{});var Ie=f(re);h=n(Ie,"compile"),Ie.forEach(s),Te.forEach(s),I=n(ke,":"),ke.forEach(s),oe=d(r),D(le.$$.fragment,r),X=d(r),Z=p(r,"P",{});var we=f(Z);fe=n(we,"Call "),V=p(we,"A",{href:!0,rel:!0});var Ee=f(V);me=p(Ee,"CODE",{});var Me=f(me);ie=n(Me,"fit"),Me.forEach(s),Ee.forEach(s),ce=n(we," to fine-tune the model:"),we.forEach(s),ge=d(r),D(ue.$$.fragment,r),this.h()},h(){g(k,"href","https://huggingface.co/docs/datasets/package_reference/main_classes.html#datasets.Dataset.to_tf_dataset"),g(k,"rel","nofollow"),g(Q,"href","/docs/transformers/pr_17548/en/model_doc/auto#transformers.TFAutoModelForCausalLM"),g(K,"href","https://keras.io/api/models/model_training_apis/#compile-method"),g(K,"rel","nofollow"),g(V,"href","https://keras.io/api/models/model_training_apis/#fit-method"),g(V,"rel","nofollow")},m(r,E){m(r,a,E),t(a,w),t(a,o),t(o,_),t(a,v),t(a,k),t(k,j),t(j,A),t(a,y),t(a,x),t(x,q),t(a,G),m(r,N,E),F(T,r,E),m(r,S,E),F(R,r,E),m(r,B,E),m(r,O,E),t(O,Y),m(r,$,E),F(b,r,E),m(r,W,E),m(r,P,E),t(P,H),t(P,Q),t(Q,ee),t(P,te),m(r,pe,E),F(U,r,E),m(r,he,E),m(r,se,E),t(se,ne),t(se,K),t(K,re),t(re,h),t(se,I),m(r,oe,E),F(le,r,E),m(r,X,E),m(r,Z,E),t(Z,fe),t(Z,V),t(V,me),t(me,ie),t(Z,ce),m(r,ge,E),F(ue,r,E),_e=!0},p(r,E){const de={};E&2&&(de.$$scope={dirty:E,ctx:r}),R.$set(de)},i(r){_e||(C(T.$$.fragment,r),C(R.$$.fragment,r),C(b.$$.fragment,r),C(U.$$.fragment,r),C(le.$$.fragment,r),C(ue.$$.fragment,r),_e=!0)},o(r){z(T.$$.fragment,r),z(R.$$.fragment,r),z(b.$$.fragment,r),z(U.$$.fragment,r),z(le.$$.fragment,r),z(ue.$$.fragment,r),_e=!1},d(r){r&&s(a),r&&s(N),L(T,r),r&&s(S),L(R,r),r&&s(B),r&&s(O),r&&s($),L(b,r),r&&s(W),r&&s(P),r&&s(pe),L(U,r),r&&s(he),r&&s(se),r&&s(oe),L(le,r),r&&s(X),r&&s(Z),r&&s(ge),L(ue,r)}}}function qn(J){let a,w;return a=new Ft({props:{$$slots:{default:[An]},$$scope:{ctx:J}}}),{c(){M(a.$$.fragment)},l(o){D(a.$$.fragment,o)},m(o,_){F(a,o,_),w=!0},p(o,_){const v={};_&2&&(v.$$scope={dirty:_,ctx:o}),a.$set(v)},i(o){w||(C(a.$$.fragment,o),w=!0)},o(o){z(a.$$.fragment,o),w=!1},d(o){L(a,o)}}}function Mn(J){let a,w,o,_,v,k,j,A;return{c(){a=i("p"),w=l("If you aren\u2019t familiar with fine-tuning a model with the "),o=i("a"),_=l("Trainer"),v=l(", take a look at the basic tutorial "),k=i("a"),j=l("here"),A=l("!"),this.h()},l(y){a=p(y,"P",{});var x=f(a);w=n(x,"If you aren\u2019t familiar with fine-tuning a model with the "),o=p(x,"A",{href:!0});var q=f(o);_=n(q,"Trainer"),q.forEach(s),v=n(x,", take a look at the basic tutorial "),k=p(x,"A",{href:!0});var G=f(k);j=n(G,"here"),G.forEach(s),A=n(x,"!"),x.forEach(s),this.h()},h(){g(o,"href","/docs/transformers/pr_17548/en/main_classes/trainer#transformers.Trainer"),g(k,"href","../training#finetune-with-trainer")},m(y,x){m(y,a,x),t(a,w),t(a,o),t(o,_),t(a,v),t(a,k),t(k,j),t(a,A)},d(y){y&&s(a)}}}function Dn(J){let a,w,o,_,v,k,j,A,y,x,q,G,N,T,S,R,B,O,Y,$,b,W,P,H,Q,ee,te,pe,U,he,se,ne,K,re;return j=new ae({props:{code:`from transformers import AutoModelForMaskedLM

model = AutoModelForMaskedLM.from_pretrained("distilroberta-base")`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoModelForMaskedLM

<span class="hljs-meta">&gt;&gt;&gt; </span>model = AutoModelForMaskedLM.from_pretrained(<span class="hljs-string">&quot;distilroberta-base&quot;</span>)`}}),y=new Dt({props:{$$slots:{default:[Mn]},$$scope:{ctx:J}}}),K=new ae({props:{code:`training_args = TrainingArguments(
    output_dir="./results",
    evaluation_strategy="epoch",
    learning_rate=2e-5,
    num_train_epochs=3,
    weight_decay=0.01,
)

trainer = Trainer(
    model=model,
    args=training_args,
    train_dataset=lm_dataset["train"],
    eval_dataset=lm_dataset["test"],
    data_collator=data_collator,
)

trainer.train()`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>training_args = TrainingArguments(
<span class="hljs-meta">... </span>    output_dir=<span class="hljs-string">&quot;./results&quot;</span>,
<span class="hljs-meta">... </span>    evaluation_strategy=<span class="hljs-string">&quot;epoch&quot;</span>,
<span class="hljs-meta">... </span>    learning_rate=<span class="hljs-number">2e-5</span>,
<span class="hljs-meta">... </span>    num_train_epochs=<span class="hljs-number">3</span>,
<span class="hljs-meta">... </span>    weight_decay=<span class="hljs-number">0.01</span>,
<span class="hljs-meta">... </span>)

<span class="hljs-meta">&gt;&gt;&gt; </span>trainer = Trainer(
<span class="hljs-meta">... </span>    model=model,
<span class="hljs-meta">... </span>    args=training_args,
<span class="hljs-meta">... </span>    train_dataset=lm_dataset[<span class="hljs-string">&quot;train&quot;</span>],
<span class="hljs-meta">... </span>    eval_dataset=lm_dataset[<span class="hljs-string">&quot;test&quot;</span>],
<span class="hljs-meta">... </span>    data_collator=data_collator,
<span class="hljs-meta">... </span>)

<span class="hljs-meta">&gt;&gt;&gt; </span>trainer.train()`}}),{c(){a=i("p"),w=l("Load DistilRoBERTa with "),o=i("code"),_=l("AutoModelForMaskedlM"),v=l(":"),k=c(),M(j.$$.fragment),A=c(),M(y.$$.fragment),x=c(),q=i("p"),G=l("At this point, only three steps remain:"),N=c(),T=i("ol"),S=i("li"),R=l("Define your training hyperparameters in "),B=i("a"),O=l("TrainingArguments"),Y=l("."),$=c(),b=i("li"),W=l("Pass the training arguments to "),P=i("a"),H=l("Trainer"),Q=l(" along with the model, datasets, and data collator."),ee=c(),te=i("li"),pe=l("Call "),U=i("a"),he=l("train()"),se=l(" to fine-tune your model."),ne=c(),M(K.$$.fragment),this.h()},l(h){a=p(h,"P",{});var I=f(a);w=n(I,"Load DistilRoBERTa with "),o=p(I,"CODE",{});var oe=f(o);_=n(oe,"AutoModelForMaskedlM"),oe.forEach(s),v=n(I,":"),I.forEach(s),k=d(h),D(j.$$.fragment,h),A=d(h),D(y.$$.fragment,h),x=d(h),q=p(h,"P",{});var le=f(q);G=n(le,"At this point, only three steps remain:"),le.forEach(s),N=d(h),T=p(h,"OL",{});var X=f(T);S=p(X,"LI",{});var Z=f(S);R=n(Z,"Define your training hyperparameters in "),B=p(Z,"A",{href:!0});var fe=f(B);O=n(fe,"TrainingArguments"),fe.forEach(s),Y=n(Z,"."),Z.forEach(s),$=d(X),b=p(X,"LI",{});var V=f(b);W=n(V,"Pass the training arguments to "),P=p(V,"A",{href:!0});var me=f(P);H=n(me,"Trainer"),me.forEach(s),Q=n(V," along with the model, datasets, and data collator."),V.forEach(s),ee=d(X),te=p(X,"LI",{});var ie=f(te);pe=n(ie,"Call "),U=p(ie,"A",{href:!0});var ce=f(U);he=n(ce,"train()"),ce.forEach(s),se=n(ie," to fine-tune your model."),ie.forEach(s),X.forEach(s),ne=d(h),D(K.$$.fragment,h),this.h()},h(){g(B,"href","/docs/transformers/pr_17548/en/main_classes/trainer#transformers.TrainingArguments"),g(P,"href","/docs/transformers/pr_17548/en/main_classes/trainer#transformers.Trainer"),g(U,"href","/docs/transformers/pr_17548/en/main_classes/trainer#transformers.Trainer.train")},m(h,I){m(h,a,I),t(a,w),t(a,o),t(o,_),t(a,v),m(h,k,I),F(j,h,I),m(h,A,I),F(y,h,I),m(h,x,I),m(h,q,I),t(q,G),m(h,N,I),m(h,T,I),t(T,S),t(S,R),t(S,B),t(B,O),t(S,Y),t(T,$),t(T,b),t(b,W),t(b,P),t(P,H),t(b,Q),t(T,ee),t(T,te),t(te,pe),t(te,U),t(U,he),t(te,se),m(h,ne,I),F(K,h,I),re=!0},p(h,I){const oe={};I&2&&(oe.$$scope={dirty:I,ctx:h}),y.$set(oe)},i(h){re||(C(j.$$.fragment,h),C(y.$$.fragment,h),C(K.$$.fragment,h),re=!0)},o(h){z(j.$$.fragment,h),z(y.$$.fragment,h),z(K.$$.fragment,h),re=!1},d(h){h&&s(a),h&&s(k),L(j,h),h&&s(A),L(y,h),h&&s(x),h&&s(q),h&&s(N),h&&s(T),h&&s(ne),L(K,h)}}}function Fn(J){let a,w;return a=new Ft({props:{$$slots:{default:[Dn]},$$scope:{ctx:J}}}),{c(){M(a.$$.fragment)},l(o){D(a.$$.fragment,o)},m(o,_){F(a,o,_),w=!0},p(o,_){const v={};_&2&&(v.$$scope={dirty:_,ctx:o}),a.$set(v)},i(o){w||(C(a.$$.fragment,o),w=!0)},o(o){z(a.$$.fragment,o),w=!1},d(o){L(a,o)}}}function Cn(J){let a,w,o,_,v;return{c(){a=i("p"),w=l("If you aren\u2019t familiar with fine-tuning a model with Keras, take a look at the basic tutorial "),o=i("a"),_=l("here"),v=l("!"),this.h()},l(k){a=p(k,"P",{});var j=f(a);w=n(j,"If you aren\u2019t familiar with fine-tuning a model with Keras, take a look at the basic tutorial "),o=p(j,"A",{href:!0});var A=f(o);_=n(A,"here"),A.forEach(s),v=n(j,"!"),j.forEach(s),this.h()},h(){g(o,"href","training#finetune-with-keras")},m(k,j){m(k,a,j),t(a,w),t(a,o),t(o,_),t(a,v)},d(k){k&&s(a)}}}function zn(J){let a,w,o,_,v,k,j,A,y,x,q,G,N,T,S,R,B,O,Y,$,b,W,P,H,Q,ee,te,pe,U,he,se,ne,K,re,h,I,oe,le,X,Z,fe,V,me,ie,ce,ge,ue,_e;return T=new ae({props:{code:`tf_train_set = lm_dataset["train"].to_tf_dataset(
    columns=["attention_mask", "input_ids", "labels"],
    dummy_labels=True,
    shuffle=True,
    batch_size=16,
    collate_fn=data_collator,
)

tf_test_set = lm_dataset["test"].to_tf_dataset(
    columns=["attention_mask", "input_ids", "labels"],
    dummy_labels=True,
    shuffle=False,
    batch_size=16,
    collate_fn=data_collator,
)`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>tf_train_set = lm_dataset[<span class="hljs-string">&quot;train&quot;</span>].to_tf_dataset(
<span class="hljs-meta">... </span>    columns=[<span class="hljs-string">&quot;attention_mask&quot;</span>, <span class="hljs-string">&quot;input_ids&quot;</span>, <span class="hljs-string">&quot;labels&quot;</span>],
<span class="hljs-meta">... </span>    dummy_labels=<span class="hljs-literal">True</span>,
<span class="hljs-meta">... </span>    shuffle=<span class="hljs-literal">True</span>,
<span class="hljs-meta">... </span>    batch_size=<span class="hljs-number">16</span>,
<span class="hljs-meta">... </span>    collate_fn=data_collator,
<span class="hljs-meta">... </span>)

<span class="hljs-meta">&gt;&gt;&gt; </span>tf_test_set = lm_dataset[<span class="hljs-string">&quot;test&quot;</span>].to_tf_dataset(
<span class="hljs-meta">... </span>    columns=[<span class="hljs-string">&quot;attention_mask&quot;</span>, <span class="hljs-string">&quot;input_ids&quot;</span>, <span class="hljs-string">&quot;labels&quot;</span>],
<span class="hljs-meta">... </span>    dummy_labels=<span class="hljs-literal">True</span>,
<span class="hljs-meta">... </span>    shuffle=<span class="hljs-literal">False</span>,
<span class="hljs-meta">... </span>    batch_size=<span class="hljs-number">16</span>,
<span class="hljs-meta">... </span>    collate_fn=data_collator,
<span class="hljs-meta">... </span>)`}}),R=new Dt({props:{$$slots:{default:[Cn]},$$scope:{ctx:J}}}),b=new ae({props:{code:`from transformers import create_optimizer, AdamWeightDecay

optimizer = AdamWeightDecay(learning_rate=2e-5, weight_decay_rate=0.01)`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> create_optimizer, AdamWeightDecay

<span class="hljs-meta">&gt;&gt;&gt; </span>optimizer = AdamWeightDecay(learning_rate=<span class="hljs-number">2e-5</span>, weight_decay_rate=<span class="hljs-number">0.01</span>)`}}),U=new ae({props:{code:`from transformers import TFAutoModelForMaskedLM

model = TFAutoModelForCausalLM.from_pretrained("distilroberta-base")`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> TFAutoModelForMaskedLM

<span class="hljs-meta">&gt;&gt;&gt; </span>model = TFAutoModelForCausalLM.from_pretrained(<span class="hljs-string">&quot;distilroberta-base&quot;</span>)`}}),le=new ae({props:{code:`import tensorflow as tf

model.compile(optimizer=optimizer)`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">import</span> tensorflow <span class="hljs-keyword">as</span> tf

<span class="hljs-meta">&gt;&gt;&gt; </span>model.<span class="hljs-built_in">compile</span>(optimizer=optimizer)`}}),ue=new ae({props:{code:"model.fit(x=tf_train_set, validation_data=tf_test_set, epochs=3)",highlighted:'<span class="hljs-meta">&gt;&gt;&gt; </span>model.fit(x=tf_train_set, validation_data=tf_test_set, epochs=<span class="hljs-number">3</span>)'}}),{c(){a=i("p"),w=l("To fine-tune a model in TensorFlow, start by converting your datasets to the "),o=i("code"),_=l("tf.data.Dataset"),v=l(" format with "),k=i("a"),j=i("code"),A=l("to_tf_dataset"),y=l(". Specify inputs and labels in "),x=i("code"),q=l("columns"),G=l(", whether to shuffle the dataset order, batch size, and the data collator:"),N=c(),M(T.$$.fragment),S=c(),M(R.$$.fragment),B=c(),O=i("p"),Y=l("Set up an optimizer function, learning rate, and some training hyperparameters:"),$=c(),M(b.$$.fragment),W=c(),P=i("p"),H=l("Load DistilRoBERTa with "),Q=i("a"),ee=l("TFAutoModelForMaskedLM"),te=l(":"),pe=c(),M(U.$$.fragment),he=c(),se=i("p"),ne=l("Configure the model for training with "),K=i("a"),re=i("code"),h=l("compile"),I=l(":"),oe=c(),M(le.$$.fragment),X=c(),Z=i("p"),fe=l("Call "),V=i("a"),me=i("code"),ie=l("fit"),ce=l(" to fine-tune the model:"),ge=c(),M(ue.$$.fragment),this.h()},l(r){a=p(r,"P",{});var E=f(a);w=n(E,"To fine-tune a model in TensorFlow, start by converting your datasets to the "),o=p(E,"CODE",{});var de=f(o);_=n(de,"tf.data.Dataset"),de.forEach(s),v=n(E," format with "),k=p(E,"A",{href:!0,rel:!0});var be=f(k);j=p(be,"CODE",{});var qe=f(j);A=n(qe,"to_tf_dataset"),qe.forEach(s),be.forEach(s),y=n(E,". Specify inputs and labels in "),x=p(E,"CODE",{});var xe=f(x);q=n(xe,"columns"),xe.forEach(s),G=n(E,", whether to shuffle the dataset order, batch size, and the data collator:"),E.forEach(s),N=d(r),D(T.$$.fragment,r),S=d(r),D(R.$$.fragment,r),B=d(r),O=p(r,"P",{});var Pe=f(O);Y=n(Pe,"Set up an optimizer function, learning rate, and some training hyperparameters:"),Pe.forEach(s),$=d(r),D(b.$$.fragment,r),W=d(r),P=p(r,"P",{});var $e=f(P);H=n($e,"Load DistilRoBERTa with "),Q=p($e,"A",{href:!0});var ve=f(Q);ee=n(ve,"TFAutoModelForMaskedLM"),ve.forEach(s),te=n($e,":"),$e.forEach(s),pe=d(r),D(U.$$.fragment,r),he=d(r),se=p(r,"P",{});var ke=f(se);ne=n(ke,"Configure the model for training with "),K=p(ke,"A",{href:!0,rel:!0});var Te=f(K);re=p(Te,"CODE",{});var Ie=f(re);h=n(Ie,"compile"),Ie.forEach(s),Te.forEach(s),I=n(ke,":"),ke.forEach(s),oe=d(r),D(le.$$.fragment,r),X=d(r),Z=p(r,"P",{});var we=f(Z);fe=n(we,"Call "),V=p(we,"A",{href:!0,rel:!0});var Ee=f(V);me=p(Ee,"CODE",{});var Me=f(me);ie=n(Me,"fit"),Me.forEach(s),Ee.forEach(s),ce=n(we," to fine-tune the model:"),we.forEach(s),ge=d(r),D(ue.$$.fragment,r),this.h()},h(){g(k,"href","https://huggingface.co/docs/datasets/package_reference/main_classes.html#datasets.Dataset.to_tf_dataset"),g(k,"rel","nofollow"),g(Q,"href","/docs/transformers/pr_17548/en/model_doc/auto#transformers.TFAutoModelForMaskedLM"),g(K,"href","https://keras.io/api/models/model_training_apis/#compile-method"),g(K,"rel","nofollow"),g(V,"href","https://keras.io/api/models/model_training_apis/#fit-method"),g(V,"rel","nofollow")},m(r,E){m(r,a,E),t(a,w),t(a,o),t(o,_),t(a,v),t(a,k),t(k,j),t(j,A),t(a,y),t(a,x),t(x,q),t(a,G),m(r,N,E),F(T,r,E),m(r,S,E),F(R,r,E),m(r,B,E),m(r,O,E),t(O,Y),m(r,$,E),F(b,r,E),m(r,W,E),m(r,P,E),t(P,H),t(P,Q),t(Q,ee),t(P,te),m(r,pe,E),F(U,r,E),m(r,he,E),m(r,se,E),t(se,ne),t(se,K),t(K,re),t(re,h),t(se,I),m(r,oe,E),F(le,r,E),m(r,X,E),m(r,Z,E),t(Z,fe),t(Z,V),t(V,me),t(me,ie),t(Z,ce),m(r,ge,E),F(ue,r,E),_e=!0},p(r,E){const de={};E&2&&(de.$$scope={dirty:E,ctx:r}),R.$set(de)},i(r){_e||(C(T.$$.fragment,r),C(R.$$.fragment,r),C(b.$$.fragment,r),C(U.$$.fragment,r),C(le.$$.fragment,r),C(ue.$$.fragment,r),_e=!0)},o(r){z(T.$$.fragment,r),z(R.$$.fragment,r),z(b.$$.fragment,r),z(U.$$.fragment,r),z(le.$$.fragment,r),z(ue.$$.fragment,r),_e=!1},d(r){r&&s(a),r&&s(N),L(T,r),r&&s(S),L(R,r),r&&s(B),r&&s(O),r&&s($),L(b,r),r&&s(W),r&&s(P),r&&s(pe),L(U,r),r&&s(he),r&&s(se),r&&s(oe),L(le,r),r&&s(X),r&&s(Z),r&&s(ge),L(ue,r)}}}function Ln(J){let a,w;return a=new Ft({props:{$$slots:{default:[zn]},$$scope:{ctx:J}}}),{c(){M(a.$$.fragment)},l(o){D(a.$$.fragment,o)},m(o,_){F(a,o,_),w=!0},p(o,_){const v={};_&2&&(v.$$scope={dirty:_,ctx:o}),a.$set(v)},i(o){w||(C(a.$$.fragment,o),w=!0)},o(o){z(a.$$.fragment,o),w=!1},d(o){L(a,o)}}}function Pn(J){let a,w,o,_,v,k,j,A;return{c(){a=i("p"),w=l(`For a more in-depth example of how to fine-tune a model for causal language modeling, take a look at the corresponding
`),o=i("a"),_=l("PyTorch notebook"),v=l(`
or `),k=i("a"),j=l("TensorFlow notebook"),A=l("."),this.h()},l(y){a=p(y,"P",{});var x=f(a);w=n(x,`For a more in-depth example of how to fine-tune a model for causal language modeling, take a look at the corresponding
`),o=p(x,"A",{href:!0,rel:!0});var q=f(o);_=n(q,"PyTorch notebook"),q.forEach(s),v=n(x,`
or `),k=p(x,"A",{href:!0,rel:!0});var G=f(k);j=n(G,"TensorFlow notebook"),G.forEach(s),A=n(x,"."),x.forEach(s),this.h()},h(){g(o,"href","https://colab.research.google.com/github/huggingface/notebooks/blob/main/examples/language_modeling.ipynb"),g(o,"rel","nofollow"),g(k,"href","https://colab.research.google.com/github/huggingface/notebooks/blob/main/examples/language_modeling-tf.ipynb"),g(k,"rel","nofollow")},m(y,x){m(y,a,x),t(a,w),t(a,o),t(o,_),t(a,v),t(a,k),t(k,j),t(a,A)},d(y){y&&s(a)}}}function In(J){let a,w,o,_,v,k,j,A,y,x,q,G,N,T,S,R,B,O,Y,$,b,W,P,H,Q,ee,te,pe,U,he,se,ne,K,re,h,I,oe,le,X,Z,fe,V,me,ie,ce,ge,ue,_e,r,E,de,be,qe,xe,Pe,$e,ve,ke,Te,Ie,we,Ee,Me,Ae,ta,Ct,sa,aa,zt,la,na,Lt,ra,oa,ms,Oe,Be,Pt,lt,ia,It,pa,hs,nt,us,We,fa,Ot,ma,ha,cs,rt,ds,ot,gs,Et,ua,_s,it,$s,De,ca,St,da,ga,pt,Rt,_a,$a,ks,ft,ws,Fe,ka,Nt,wa,ja,Gt,ya,ba,js,xt,va,ys,mt,bs,je,Ea,ht,Bt,xa,Ta,Wt,Aa,qa,Ht,Ma,Da,Yt,Fa,Ca,vs,ut,Es,Tt,za,xs,He,Ut,La,Pa,ct,Ia,Jt,Oa,Sa,Ts,dt,As,Ye,Ra,Kt,Na,Ga,qs,gt,Ms,ye,Ba,At,Wa,Ha,Qt,Ya,Ua,Vt,Ja,Ka,Xt,Qa,Va,Ds,Ue,Fs,Se,Je,Zt,_t,Xa,es,Za,Cs,Ke,el,$t,tl,sl,zs,Re,Qe,ts,kt,al,ss,ll,Ls,Ve,Ps,Ne,Xe,as,wt,nl,ls,rl,Is,Ze,ol,jt,il,pl,Os,Ge,et,ns,yt,fl,rs,ml,Ss,tt,Rs,st,Ns;return k=new vt({}),T=new ea({props:{id:"Vpjb1lu0MDk"}}),Y=new ea({props:{id:"mqElG5QJWUg"}}),X=new Dt({props:{$$slots:{default:[kn]},$$scope:{ctx:J}}}),ie=new vt({}),be=new ae({props:{code:`from datasets import load_dataset

eli5 = load_dataset("eli5", split="train_asks[:5000]")`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> datasets <span class="hljs-keyword">import</span> load_dataset

<span class="hljs-meta">&gt;&gt;&gt; </span>eli5 = load_dataset(<span class="hljs-string">&quot;eli5&quot;</span>, split=<span class="hljs-string">&quot;train_asks[:5000]&quot;</span>)`}}),ve=new ae({props:{code:"eli5 = eli5.train_test_split(test_size=0.2)",highlighted:'eli5 = eli5.train_test_split(test_size=<span class="hljs-number">0.2</span>)'}}),Ee=new ae({props:{code:'eli5["train"][0]',highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>eli5[<span class="hljs-string">&quot;train&quot;</span>][<span class="hljs-number">0</span>]
{<span class="hljs-string">&#x27;answers&#x27;</span>: {<span class="hljs-string">&#x27;a_id&#x27;</span>: [<span class="hljs-string">&#x27;c3d1aib&#x27;</span>, <span class="hljs-string">&#x27;c3d4lya&#x27;</span>],
  <span class="hljs-string">&#x27;score&#x27;</span>: [<span class="hljs-number">6</span>, <span class="hljs-number">3</span>],
  <span class="hljs-string">&#x27;text&#x27;</span>: [<span class="hljs-string">&quot;The velocity needed to remain in orbit is equal to the square root of Newton&#x27;s constant times the mass of earth divided by the distance from the center of the earth. I don&#x27;t know the altitude of that specific mission, but they&#x27;re usually around 300 km. That means he&#x27;s going 7-8 km/s.\\n\\nIn space there are no other forces acting on either the shuttle or the guy, so they stay in the same position relative to each other. If he were to become unable to return to the ship, he would presumably run out of oxygen, or slowly fall into the atmosphere and burn up.&quot;</span>,
   <span class="hljs-string">&quot;Hope you don&#x27;t mind me asking another question, but why aren&#x27;t there any stars visible in this photo?&quot;</span>]},
 <span class="hljs-string">&#x27;answers_urls&#x27;</span>: {<span class="hljs-string">&#x27;url&#x27;</span>: []},
 <span class="hljs-string">&#x27;document&#x27;</span>: <span class="hljs-string">&#x27;&#x27;</span>,
 <span class="hljs-string">&#x27;q_id&#x27;</span>: <span class="hljs-string">&#x27;nyxfp&#x27;</span>,
 <span class="hljs-string">&#x27;selftext&#x27;</span>: <span class="hljs-string">&#x27;_URL_0_\\n\\nThis was on the front page earlier and I have a few questions about it. Is it possible to calculate how fast the astronaut would be orbiting the earth? Also how does he stay close to the shuttle so that he can return safely, i.e is he orbiting at the same speed and can therefore stay next to it? And finally if his propulsion system failed, would he eventually re-enter the atmosphere and presumably die?&#x27;</span>,
 <span class="hljs-string">&#x27;selftext_urls&#x27;</span>: {<span class="hljs-string">&#x27;url&#x27;</span>: [<span class="hljs-string">&#x27;http://apod.nasa.gov/apod/image/1201/freeflyer_nasa_3000.jpg&#x27;</span>]},
 <span class="hljs-string">&#x27;subreddit&#x27;</span>: <span class="hljs-string">&#x27;askscience&#x27;</span>,
 <span class="hljs-string">&#x27;title&#x27;</span>: <span class="hljs-string">&#x27;Few questions about this space walk photograph.&#x27;</span>,
 <span class="hljs-string">&#x27;title_urls&#x27;</span>: {<span class="hljs-string">&#x27;url&#x27;</span>: []}}`}}),lt=new vt({}),nt=new ea({props:{id:"ma1TrR7gE7I"}}),rt=new ae({props:{code:`from transformers import AutoTokenizer

tokenizer = AutoTokenizer.from_pretrained("distilgpt2")`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoTokenizer

<span class="hljs-meta">&gt;&gt;&gt; </span>tokenizer = AutoTokenizer.from_pretrained(<span class="hljs-string">&quot;distilgpt2&quot;</span>)`}}),ot=new ea({props:{id:"8PmhEIXhBvI"}}),it=new ae({props:{code:`from transformers import AutoTokenizer

tokenizer = AutoTokenizer.from_pretrained("distilroberta-base")`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoTokenizer

<span class="hljs-meta">&gt;&gt;&gt; </span>tokenizer = AutoTokenizer.from_pretrained(<span class="hljs-string">&quot;distilroberta-base&quot;</span>)`}}),ft=new ae({props:{code:`eli5 = eli5.flatten()
eli5["train"][0]`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>eli5 = eli5.flatten()
<span class="hljs-meta">&gt;&gt;&gt; </span>eli5[<span class="hljs-string">&quot;train&quot;</span>][<span class="hljs-number">0</span>]
{<span class="hljs-string">&#x27;answers.a_id&#x27;</span>: [<span class="hljs-string">&#x27;c3d1aib&#x27;</span>, <span class="hljs-string">&#x27;c3d4lya&#x27;</span>],
 <span class="hljs-string">&#x27;answers.score&#x27;</span>: [<span class="hljs-number">6</span>, <span class="hljs-number">3</span>],
 <span class="hljs-string">&#x27;answers.text&#x27;</span>: [<span class="hljs-string">&quot;The velocity needed to remain in orbit is equal to the square root of Newton&#x27;s constant times the mass of earth divided by the distance from the center of the earth. I don&#x27;t know the altitude of that specific mission, but they&#x27;re usually around 300 km. That means he&#x27;s going 7-8 km/s.\\n\\nIn space there are no other forces acting on either the shuttle or the guy, so they stay in the same position relative to each other. If he were to become unable to return to the ship, he would presumably run out of oxygen, or slowly fall into the atmosphere and burn up.&quot;</span>,
  <span class="hljs-string">&quot;Hope you don&#x27;t mind me asking another question, but why aren&#x27;t there any stars visible in this photo?&quot;</span>],
 <span class="hljs-string">&#x27;answers_urls.url&#x27;</span>: [],
 <span class="hljs-string">&#x27;document&#x27;</span>: <span class="hljs-string">&#x27;&#x27;</span>,
 <span class="hljs-string">&#x27;q_id&#x27;</span>: <span class="hljs-string">&#x27;nyxfp&#x27;</span>,
 <span class="hljs-string">&#x27;selftext&#x27;</span>: <span class="hljs-string">&#x27;_URL_0_\\n\\nThis was on the front page earlier and I have a few questions about it. Is it possible to calculate how fast the astronaut would be orbiting the earth? Also how does he stay close to the shuttle so that he can return safely, i.e is he orbiting at the same speed and can therefore stay next to it? And finally if his propulsion system failed, would he eventually re-enter the atmosphere and presumably die?&#x27;</span>,
 <span class="hljs-string">&#x27;selftext_urls.url&#x27;</span>: [<span class="hljs-string">&#x27;http://apod.nasa.gov/apod/image/1201/freeflyer_nasa_3000.jpg&#x27;</span>],
 <span class="hljs-string">&#x27;subreddit&#x27;</span>: <span class="hljs-string">&#x27;askscience&#x27;</span>,
 <span class="hljs-string">&#x27;title&#x27;</span>: <span class="hljs-string">&#x27;Few questions about this space walk photograph.&#x27;</span>,
 <span class="hljs-string">&#x27;title_urls.url&#x27;</span>: []}`}}),mt=new ae({props:{code:`def preprocess_function(examples):
    return tokenizer([" ".join(x) for x in examples["answers.text"]], truncation=True)`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">def</span> <span class="hljs-title function_">preprocess_function</span>(<span class="hljs-params">examples</span>):
<span class="hljs-meta">... </span>    <span class="hljs-keyword">return</span> tokenizer([<span class="hljs-string">&quot; &quot;</span>.join(x) <span class="hljs-keyword">for</span> x <span class="hljs-keyword">in</span> examples[<span class="hljs-string">&quot;answers.text&quot;</span>]], truncation=<span class="hljs-literal">True</span>)`}}),ut=new ae({props:{code:`tokenized_eli5 = eli5.map(
    preprocess_function,
    batched=True,
    num_proc=4,
    remove_columns=eli5["train"].column_names,
)`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>tokenized_eli5 = eli5.<span class="hljs-built_in">map</span>(
<span class="hljs-meta">... </span>    preprocess_function,
<span class="hljs-meta">... </span>    batched=<span class="hljs-literal">True</span>,
<span class="hljs-meta">... </span>    num_proc=<span class="hljs-number">4</span>,
<span class="hljs-meta">... </span>    remove_columns=eli5[<span class="hljs-string">&quot;train&quot;</span>].column_names,
<span class="hljs-meta">... </span>)`}}),dt=new ae({props:{code:`block_size = 128


def group_texts(examples):
    concatenated_examples = {k: sum(examples[k], []) for k in examples.keys()}
    total_length = len(concatenated_examples[list(examples.keys())[0]])
    result = {
        k: [t[i : i + block_size] for i in range(0, total_length, block_size)]
        for k, t in concatenated_examples.items()
    }
    result["labels"] = result["input_ids"].copy()
    return result`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>block_size = <span class="hljs-number">128</span>


<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">def</span> <span class="hljs-title function_">group_texts</span>(<span class="hljs-params">examples</span>):
<span class="hljs-meta">... </span>    concatenated_examples = {k: <span class="hljs-built_in">sum</span>(examples[k], []) <span class="hljs-keyword">for</span> k <span class="hljs-keyword">in</span> examples.keys()}
<span class="hljs-meta">... </span>    total_length = <span class="hljs-built_in">len</span>(concatenated_examples[<span class="hljs-built_in">list</span>(examples.keys())[<span class="hljs-number">0</span>]])
<span class="hljs-meta">... </span>    result = {
<span class="hljs-meta">... </span>        k: [t[i : i + block_size] <span class="hljs-keyword">for</span> i <span class="hljs-keyword">in</span> <span class="hljs-built_in">range</span>(<span class="hljs-number">0</span>, total_length, block_size)]
<span class="hljs-meta">... </span>        <span class="hljs-keyword">for</span> k, t <span class="hljs-keyword">in</span> concatenated_examples.items()
<span class="hljs-meta">... </span>    }
<span class="hljs-meta">... </span>    result[<span class="hljs-string">&quot;labels&quot;</span>] = result[<span class="hljs-string">&quot;input_ids&quot;</span>].copy()
<span class="hljs-meta">... </span>    <span class="hljs-keyword">return</span> result`}}),gt=new ae({props:{code:"lm_dataset = tokenized_eli5.map(group_texts, batched=True, num_proc=4)",highlighted:'<span class="hljs-meta">&gt;&gt;&gt; </span>lm_dataset = tokenized_eli5.<span class="hljs-built_in">map</span>(group_texts, batched=<span class="hljs-literal">True</span>, num_proc=<span class="hljs-number">4</span>)'}}),Ue=new hl({props:{pytorch:!0,tensorflow:!0,jax:!1,$$slots:{tensorflow:[bn],pytorch:[jn]},$$scope:{ctx:J}}}),_t=new vt({}),kt=new vt({}),Ve=new hl({props:{pytorch:!0,tensorflow:!0,jax:!1,$$slots:{tensorflow:[qn],pytorch:[xn]},$$scope:{ctx:J}}}),wt=new vt({}),yt=new vt({}),tt=new hl({props:{pytorch:!0,tensorflow:!0,jax:!1,$$slots:{tensorflow:[Ln],pytorch:[Fn]},$$scope:{ctx:J}}}),st=new Dt({props:{$$slots:{default:[Pn]},$$scope:{ctx:J}}}),{c(){a=i("meta"),w=c(),o=i("h1"),_=i("a"),v=i("span"),M(k.$$.fragment),j=c(),A=i("span"),y=l("Language modeling"),x=c(),q=i("p"),G=l("Language modeling predicts words in a sentence. There are two forms of language modeling."),N=c(),M(T.$$.fragment),S=c(),R=i("p"),B=l("Causal language modeling predicts the next token in a sequence of tokens, and the model can only attend to tokens on the left."),O=c(),M(Y.$$.fragment),$=c(),b=i("p"),W=l("Masked language modeling predicts a masked token in a sequence, and the model can attend to tokens bidirectionally."),P=c(),H=i("p"),Q=l("This guide will show you how to fine-tune "),ee=i("a"),te=l("DistilGPT2"),pe=l(" for causal language modeling and "),U=i("a"),he=l("DistilRoBERTa"),se=l(" for masked language modeling on the "),ne=i("a"),K=l("r/askscience"),re=l(" subset of the "),h=i("a"),I=l("ELI5"),oe=l(" dataset."),le=c(),M(X.$$.fragment),Z=c(),fe=i("h2"),V=i("a"),me=i("span"),M(ie.$$.fragment),ce=c(),ge=i("span"),ue=l("Load ELI5 dataset"),_e=c(),r=i("p"),E=l("Load only the first 5000 rows of the ELI5 dataset from the \u{1F917} Datasets library since it is pretty large:"),de=c(),M(be.$$.fragment),qe=c(),xe=i("p"),Pe=l("Split this dataset into a train and test set:"),$e=c(),M(ve.$$.fragment),ke=c(),Te=i("p"),Ie=l("Then take a look at an example:"),we=c(),M(Ee.$$.fragment),Me=c(),Ae=i("p"),ta=l("Notice "),Ct=i("code"),sa=l("text"),aa=l(" is a subfield nested inside the "),zt=i("code"),la=l("answers"),na=l(" dictionary. When you preprocess the dataset, you will need to extract the "),Lt=i("code"),ra=l("text"),oa=l(" subfield into a separate column."),ms=c(),Oe=i("h2"),Be=i("a"),Pt=i("span"),M(lt.$$.fragment),ia=c(),It=i("span"),pa=l("Preprocess"),hs=c(),M(nt.$$.fragment),us=c(),We=i("p"),fa=l("For causal language modeling, load the DistilGPT2 tokenizer to process the "),Ot=i("code"),ma=l("text"),ha=l(" subfield:"),cs=c(),M(rt.$$.fragment),ds=c(),M(ot.$$.fragment),gs=c(),Et=i("p"),ua=l("For masked language modeling, load the DistilRoBERTa tokenizer instead:"),_s=c(),M(it.$$.fragment),$s=c(),De=i("p"),ca=l("Extract the "),St=i("code"),da=l("text"),ga=l(" subfield from its nested structure with the "),pt=i("a"),Rt=i("code"),_a=l("flatten"),$a=l(" method:"),ks=c(),M(ft.$$.fragment),ws=c(),Fe=i("p"),ka=l("Each subfield is now a separate column as indicated by the "),Nt=i("code"),wa=l("answers"),ja=l(" prefix. Notice that "),Gt=i("code"),ya=l("answers.text"),ba=l(" is a list. Instead of tokenizing each sentence separately, convert the list to a string to jointly tokenize them."),js=c(),xt=i("p"),va=l("Here is how you can create a preprocessing function to convert the list to a string and truncate sequences to be no longer than DistilGPT2\u2019s maximum input length:"),ys=c(),M(mt.$$.fragment),bs=c(),je=i("p"),Ea=l("Use \u{1F917} Datasets "),ht=i("a"),Bt=i("code"),xa=l("map"),Ta=l(" function to apply the preprocessing function over the entire dataset. You can speed up the "),Wt=i("code"),Aa=l("map"),qa=l(" function by setting "),Ht=i("code"),Ma=l("batched=True"),Da=l(" to process multiple elements of the dataset at once and increasing the number of processes with "),Yt=i("code"),Fa=l("num_proc"),Ca=l(". Remove the columns you don\u2019t need:"),vs=c(),M(ut.$$.fragment),Es=c(),Tt=i("p"),za=l("Now you need a second preprocessing function to capture text truncated from any lengthy examples to prevent loss of information. This preprocessing function should:"),xs=c(),He=i("ul"),Ut=i("li"),La=l("Concatenate all the text."),Pa=c(),ct=i("li"),Ia=l("Split the concatenated text into smaller chunks defined by "),Jt=i("code"),Oa=l("block_size"),Sa=l("."),Ts=c(),M(dt.$$.fragment),As=c(),Ye=i("p"),Ra=l("Apply the "),Kt=i("code"),Na=l("group_texts"),Ga=l(" function over the entire dataset:"),qs=c(),M(gt.$$.fragment),Ms=c(),ye=i("p"),Ba=l("For causal language modeling, use "),At=i("a"),Wa=l("DataCollatorForLanguageModeling"),Ha=l(" to create a batch of examples. It will also "),Qt=i("em"),Ya=l("dynamically pad"),Ua=l(" your text to the length of the longest element in its batch, so they are a uniform length. While it is possible to pad your text in the "),Vt=i("code"),Ja=l("tokenizer"),Ka=l(" function by setting "),Xt=i("code"),Qa=l("padding=True"),Va=l(", dynamic padding is more efficient."),Ds=c(),M(Ue.$$.fragment),Fs=c(),Se=i("h2"),Je=i("a"),Zt=i("span"),M(_t.$$.fragment),Xa=c(),es=i("span"),Za=l("Causal language modeling"),Cs=c(),Ke=i("p"),el=l("Causal language modeling is frequently used for text generation. This section shows you how to fine-tune "),$t=i("a"),tl=l("DistilGPT2"),sl=l(" to generate new text."),zs=c(),Re=i("h3"),Qe=i("a"),ts=i("span"),M(kt.$$.fragment),al=c(),ss=i("span"),ll=l("Train"),Ls=c(),M(Ve.$$.fragment),Ps=c(),Ne=i("h2"),Xe=i("a"),as=i("span"),M(wt.$$.fragment),nl=c(),ls=i("span"),rl=l("Masked language modeling"),Is=c(),Ze=i("p"),ol=l("Masked language modeling is also known as a fill-mask task because it predicts a masked token in a sequence. Models for masked language modeling require a good contextual understanding of an entire sequence instead of only the left context. This section shows you how to fine-tune "),jt=i("a"),il=l("DistilRoBERTa"),pl=l(" to predict a masked word."),Os=c(),Ge=i("h3"),et=i("a"),ns=i("span"),M(yt.$$.fragment),fl=c(),rs=i("span"),ml=l("Train"),Ss=c(),M(tt.$$.fragment),Rs=c(),M(st.$$.fragment),this.h()},l(e){const u=_n('[data-svelte="svelte-1phssyn"]',document.head);a=p(u,"META",{name:!0,content:!0}),u.forEach(s),w=d(e),o=p(e,"H1",{class:!0});var bt=f(o);_=p(bt,"A",{id:!0,class:!0,href:!0});var os=f(_);v=p(os,"SPAN",{});var is=f(v);D(k.$$.fragment,is),is.forEach(s),os.forEach(s),j=d(bt),A=p(bt,"SPAN",{});var ps=f(A);y=n(ps,"Language modeling"),ps.forEach(s),bt.forEach(s),x=d(e),q=p(e,"P",{});var fs=f(q);G=n(fs,"Language modeling predicts words in a sentence. There are two forms of language modeling."),fs.forEach(s),N=d(e),D(T.$$.fragment,e),S=d(e),R=p(e,"P",{});var ul=f(R);B=n(ul,"Causal language modeling predicts the next token in a sequence of tokens, and the model can only attend to tokens on the left."),ul.forEach(s),O=d(e),D(Y.$$.fragment,e),$=d(e),b=p(e,"P",{});var cl=f(b);W=n(cl,"Masked language modeling predicts a masked token in a sequence, and the model can attend to tokens bidirectionally."),cl.forEach(s),P=d(e),H=p(e,"P",{});var Ce=f(H);Q=n(Ce,"This guide will show you how to fine-tune "),ee=p(Ce,"A",{href:!0,rel:!0});var dl=f(ee);te=n(dl,"DistilGPT2"),dl.forEach(s),pe=n(Ce," for causal language modeling and "),U=p(Ce,"A",{href:!0,rel:!0});var gl=f(U);he=n(gl,"DistilRoBERTa"),gl.forEach(s),se=n(Ce," for masked language modeling on the "),ne=p(Ce,"A",{href:!0,rel:!0});var _l=f(ne);K=n(_l,"r/askscience"),_l.forEach(s),re=n(Ce," subset of the "),h=p(Ce,"A",{href:!0,rel:!0});var $l=f(h);I=n($l,"ELI5"),$l.forEach(s),oe=n(Ce," dataset."),Ce.forEach(s),le=d(e),D(X.$$.fragment,e),Z=d(e),fe=p(e,"H2",{class:!0});var Gs=f(fe);V=p(Gs,"A",{id:!0,class:!0,href:!0});var kl=f(V);me=p(kl,"SPAN",{});var wl=f(me);D(ie.$$.fragment,wl),wl.forEach(s),kl.forEach(s),ce=d(Gs),ge=p(Gs,"SPAN",{});var jl=f(ge);ue=n(jl,"Load ELI5 dataset"),jl.forEach(s),Gs.forEach(s),_e=d(e),r=p(e,"P",{});var yl=f(r);E=n(yl,"Load only the first 5000 rows of the ELI5 dataset from the \u{1F917} Datasets library since it is pretty large:"),yl.forEach(s),de=d(e),D(be.$$.fragment,e),qe=d(e),xe=p(e,"P",{});var bl=f(xe);Pe=n(bl,"Split this dataset into a train and test set:"),bl.forEach(s),$e=d(e),D(ve.$$.fragment,e),ke=d(e),Te=p(e,"P",{});var vl=f(Te);Ie=n(vl,"Then take a look at an example:"),vl.forEach(s),we=d(e),D(Ee.$$.fragment,e),Me=d(e),Ae=p(e,"P",{});var at=f(Ae);ta=n(at,"Notice "),Ct=p(at,"CODE",{});var El=f(Ct);sa=n(El,"text"),El.forEach(s),aa=n(at," is a subfield nested inside the "),zt=p(at,"CODE",{});var xl=f(zt);la=n(xl,"answers"),xl.forEach(s),na=n(at," dictionary. When you preprocess the dataset, you will need to extract the "),Lt=p(at,"CODE",{});var Tl=f(Lt);ra=n(Tl,"text"),Tl.forEach(s),oa=n(at," subfield into a separate column."),at.forEach(s),ms=d(e),Oe=p(e,"H2",{class:!0});var Bs=f(Oe);Be=p(Bs,"A",{id:!0,class:!0,href:!0});var Al=f(Be);Pt=p(Al,"SPAN",{});var ql=f(Pt);D(lt.$$.fragment,ql),ql.forEach(s),Al.forEach(s),ia=d(Bs),It=p(Bs,"SPAN",{});var Ml=f(It);pa=n(Ml,"Preprocess"),Ml.forEach(s),Bs.forEach(s),hs=d(e),D(nt.$$.fragment,e),us=d(e),We=p(e,"P",{});var Ws=f(We);fa=n(Ws,"For causal language modeling, load the DistilGPT2 tokenizer to process the "),Ot=p(Ws,"CODE",{});var Dl=f(Ot);ma=n(Dl,"text"),Dl.forEach(s),ha=n(Ws," subfield:"),Ws.forEach(s),cs=d(e),D(rt.$$.fragment,e),ds=d(e),D(ot.$$.fragment,e),gs=d(e),Et=p(e,"P",{});var Fl=f(Et);ua=n(Fl,"For masked language modeling, load the DistilRoBERTa tokenizer instead:"),Fl.forEach(s),_s=d(e),D(it.$$.fragment,e),$s=d(e),De=p(e,"P",{});var qt=f(De);ca=n(qt,"Extract the "),St=p(qt,"CODE",{});var Cl=f(St);da=n(Cl,"text"),Cl.forEach(s),ga=n(qt," subfield from its nested structure with the "),pt=p(qt,"A",{href:!0,rel:!0});var zl=f(pt);Rt=p(zl,"CODE",{});var Ll=f(Rt);_a=n(Ll,"flatten"),Ll.forEach(s),zl.forEach(s),$a=n(qt," method:"),qt.forEach(s),ks=d(e),D(ft.$$.fragment,e),ws=d(e),Fe=p(e,"P",{});var Mt=f(Fe);ka=n(Mt,"Each subfield is now a separate column as indicated by the "),Nt=p(Mt,"CODE",{});var Pl=f(Nt);wa=n(Pl,"answers"),Pl.forEach(s),ja=n(Mt," prefix. Notice that "),Gt=p(Mt,"CODE",{});var Il=f(Gt);ya=n(Il,"answers.text"),Il.forEach(s),ba=n(Mt," is a list. Instead of tokenizing each sentence separately, convert the list to a string to jointly tokenize them."),Mt.forEach(s),js=d(e),xt=p(e,"P",{});var Ol=f(xt);va=n(Ol,"Here is how you can create a preprocessing function to convert the list to a string and truncate sequences to be no longer than DistilGPT2\u2019s maximum input length:"),Ol.forEach(s),ys=d(e),D(mt.$$.fragment,e),bs=d(e),je=p(e,"P",{});var ze=f(je);Ea=n(ze,"Use \u{1F917} Datasets "),ht=p(ze,"A",{href:!0,rel:!0});var Sl=f(ht);Bt=p(Sl,"CODE",{});var Rl=f(Bt);xa=n(Rl,"map"),Rl.forEach(s),Sl.forEach(s),Ta=n(ze," function to apply the preprocessing function over the entire dataset. You can speed up the "),Wt=p(ze,"CODE",{});var Nl=f(Wt);Aa=n(Nl,"map"),Nl.forEach(s),qa=n(ze," function by setting "),Ht=p(ze,"CODE",{});var Gl=f(Ht);Ma=n(Gl,"batched=True"),Gl.forEach(s),Da=n(ze," to process multiple elements of the dataset at once and increasing the number of processes with "),Yt=p(ze,"CODE",{});var Bl=f(Yt);Fa=n(Bl,"num_proc"),Bl.forEach(s),Ca=n(ze,". Remove the columns you don\u2019t need:"),ze.forEach(s),vs=d(e),D(ut.$$.fragment,e),Es=d(e),Tt=p(e,"P",{});var Wl=f(Tt);za=n(Wl,"Now you need a second preprocessing function to capture text truncated from any lengthy examples to prevent loss of information. This preprocessing function should:"),Wl.forEach(s),xs=d(e),He=p(e,"UL",{});var Hs=f(He);Ut=p(Hs,"LI",{});var Hl=f(Ut);La=n(Hl,"Concatenate all the text."),Hl.forEach(s),Pa=d(Hs),ct=p(Hs,"LI",{});var Ys=f(ct);Ia=n(Ys,"Split the concatenated text into smaller chunks defined by "),Jt=p(Ys,"CODE",{});var Yl=f(Jt);Oa=n(Yl,"block_size"),Yl.forEach(s),Sa=n(Ys,"."),Ys.forEach(s),Hs.forEach(s),Ts=d(e),D(dt.$$.fragment,e),As=d(e),Ye=p(e,"P",{});var Us=f(Ye);Ra=n(Us,"Apply the "),Kt=p(Us,"CODE",{});var Ul=f(Kt);Na=n(Ul,"group_texts"),Ul.forEach(s),Ga=n(Us," function over the entire dataset:"),Us.forEach(s),qs=d(e),D(gt.$$.fragment,e),Ms=d(e),ye=p(e,"P",{});var Le=f(ye);Ba=n(Le,"For causal language modeling, use "),At=p(Le,"A",{href:!0});var Jl=f(At);Wa=n(Jl,"DataCollatorForLanguageModeling"),Jl.forEach(s),Ha=n(Le," to create a batch of examples. It will also "),Qt=p(Le,"EM",{});var Kl=f(Qt);Ya=n(Kl,"dynamically pad"),Kl.forEach(s),Ua=n(Le," your text to the length of the longest element in its batch, so they are a uniform length. While it is possible to pad your text in the "),Vt=p(Le,"CODE",{});var Ql=f(Vt);Ja=n(Ql,"tokenizer"),Ql.forEach(s),Ka=n(Le," function by setting "),Xt=p(Le,"CODE",{});var Vl=f(Xt);Qa=n(Vl,"padding=True"),Vl.forEach(s),Va=n(Le,", dynamic padding is more efficient."),Le.forEach(s),Ds=d(e),D(Ue.$$.fragment,e),Fs=d(e),Se=p(e,"H2",{class:!0});var Js=f(Se);Je=p(Js,"A",{id:!0,class:!0,href:!0});var Xl=f(Je);Zt=p(Xl,"SPAN",{});var Zl=f(Zt);D(_t.$$.fragment,Zl),Zl.forEach(s),Xl.forEach(s),Xa=d(Js),es=p(Js,"SPAN",{});var en=f(es);Za=n(en,"Causal language modeling"),en.forEach(s),Js.forEach(s),Cs=d(e),Ke=p(e,"P",{});var Ks=f(Ke);el=n(Ks,"Causal language modeling is frequently used for text generation. This section shows you how to fine-tune "),$t=p(Ks,"A",{href:!0,rel:!0});var tn=f($t);tl=n(tn,"DistilGPT2"),tn.forEach(s),sl=n(Ks," to generate new text."),Ks.forEach(s),zs=d(e),Re=p(e,"H3",{class:!0});var Qs=f(Re);Qe=p(Qs,"A",{id:!0,class:!0,href:!0});var sn=f(Qe);ts=p(sn,"SPAN",{});var an=f(ts);D(kt.$$.fragment,an),an.forEach(s),sn.forEach(s),al=d(Qs),ss=p(Qs,"SPAN",{});var ln=f(ss);ll=n(ln,"Train"),ln.forEach(s),Qs.forEach(s),Ls=d(e),D(Ve.$$.fragment,e),Ps=d(e),Ne=p(e,"H2",{class:!0});var Vs=f(Ne);Xe=p(Vs,"A",{id:!0,class:!0,href:!0});var nn=f(Xe);as=p(nn,"SPAN",{});var rn=f(as);D(wt.$$.fragment,rn),rn.forEach(s),nn.forEach(s),nl=d(Vs),ls=p(Vs,"SPAN",{});var on=f(ls);rl=n(on,"Masked language modeling"),on.forEach(s),Vs.forEach(s),Is=d(e),Ze=p(e,"P",{});var Xs=f(Ze);ol=n(Xs,"Masked language modeling is also known as a fill-mask task because it predicts a masked token in a sequence. Models for masked language modeling require a good contextual understanding of an entire sequence instead of only the left context. This section shows you how to fine-tune "),jt=p(Xs,"A",{href:!0,rel:!0});var pn=f(jt);il=n(pn,"DistilRoBERTa"),pn.forEach(s),pl=n(Xs," to predict a masked word."),Xs.forEach(s),Os=d(e),Ge=p(e,"H3",{class:!0});var Zs=f(Ge);et=p(Zs,"A",{id:!0,class:!0,href:!0});var fn=f(et);ns=p(fn,"SPAN",{});var mn=f(ns);D(yt.$$.fragment,mn),mn.forEach(s),fn.forEach(s),fl=d(Zs),rs=p(Zs,"SPAN",{});var hn=f(rs);ml=n(hn,"Train"),hn.forEach(s),Zs.forEach(s),Ss=d(e),D(tt.$$.fragment,e),Rs=d(e),D(st.$$.fragment,e),this.h()},h(){g(a,"name","hf:doc:metadata"),g(a,"content",JSON.stringify(On)),g(_,"id","language-modeling"),g(_,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),g(_,"href","#language-modeling"),g(o,"class","relative group"),g(ee,"href","https://huggingface.co/distilgpt2"),g(ee,"rel","nofollow"),g(U,"href","https://huggingface.co/distilroberta-base"),g(U,"rel","nofollow"),g(ne,"href","https://www.reddit.com/r/askscience/"),g(ne,"rel","nofollow"),g(h,"href","https://huggingface.co/datasets/eli5"),g(h,"rel","nofollow"),g(V,"id","load-eli5-dataset"),g(V,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),g(V,"href","#load-eli5-dataset"),g(fe,"class","relative group"),g(Be,"id","preprocess"),g(Be,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),g(Be,"href","#preprocess"),g(Oe,"class","relative group"),g(pt,"href","https://huggingface.co/docs/datasets/process.html#flatten"),g(pt,"rel","nofollow"),g(ht,"href","https://huggingface.co/docs/datasets/package_reference/main_classes.html#datasets.Dataset.map"),g(ht,"rel","nofollow"),g(At,"href","/docs/transformers/pr_17548/en/main_classes/data_collator#transformers.DataCollatorForLanguageModeling"),g(Je,"id","causal-language-modeling"),g(Je,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),g(Je,"href","#causal-language-modeling"),g(Se,"class","relative group"),g($t,"href","https://huggingface.co/distilgpt2"),g($t,"rel","nofollow"),g(Qe,"id","train"),g(Qe,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),g(Qe,"href","#train"),g(Re,"class","relative group"),g(Xe,"id","masked-language-modeling"),g(Xe,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),g(Xe,"href","#masked-language-modeling"),g(Ne,"class","relative group"),g(jt,"href","https://huggingface.co/distilroberta-base"),g(jt,"rel","nofollow"),g(et,"id","train"),g(et,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),g(et,"href","#train"),g(Ge,"class","relative group")},m(e,u){t(document.head,a),m(e,w,u),m(e,o,u),t(o,_),t(_,v),F(k,v,null),t(o,j),t(o,A),t(A,y),m(e,x,u),m(e,q,u),t(q,G),m(e,N,u),F(T,e,u),m(e,S,u),m(e,R,u),t(R,B),m(e,O,u),F(Y,e,u),m(e,$,u),m(e,b,u),t(b,W),m(e,P,u),m(e,H,u),t(H,Q),t(H,ee),t(ee,te),t(H,pe),t(H,U),t(U,he),t(H,se),t(H,ne),t(ne,K),t(H,re),t(H,h),t(h,I),t(H,oe),m(e,le,u),F(X,e,u),m(e,Z,u),m(e,fe,u),t(fe,V),t(V,me),F(ie,me,null),t(fe,ce),t(fe,ge),t(ge,ue),m(e,_e,u),m(e,r,u),t(r,E),m(e,de,u),F(be,e,u),m(e,qe,u),m(e,xe,u),t(xe,Pe),m(e,$e,u),F(ve,e,u),m(e,ke,u),m(e,Te,u),t(Te,Ie),m(e,we,u),F(Ee,e,u),m(e,Me,u),m(e,Ae,u),t(Ae,ta),t(Ae,Ct),t(Ct,sa),t(Ae,aa),t(Ae,zt),t(zt,la),t(Ae,na),t(Ae,Lt),t(Lt,ra),t(Ae,oa),m(e,ms,u),m(e,Oe,u),t(Oe,Be),t(Be,Pt),F(lt,Pt,null),t(Oe,ia),t(Oe,It),t(It,pa),m(e,hs,u),F(nt,e,u),m(e,us,u),m(e,We,u),t(We,fa),t(We,Ot),t(Ot,ma),t(We,ha),m(e,cs,u),F(rt,e,u),m(e,ds,u),F(ot,e,u),m(e,gs,u),m(e,Et,u),t(Et,ua),m(e,_s,u),F(it,e,u),m(e,$s,u),m(e,De,u),t(De,ca),t(De,St),t(St,da),t(De,ga),t(De,pt),t(pt,Rt),t(Rt,_a),t(De,$a),m(e,ks,u),F(ft,e,u),m(e,ws,u),m(e,Fe,u),t(Fe,ka),t(Fe,Nt),t(Nt,wa),t(Fe,ja),t(Fe,Gt),t(Gt,ya),t(Fe,ba),m(e,js,u),m(e,xt,u),t(xt,va),m(e,ys,u),F(mt,e,u),m(e,bs,u),m(e,je,u),t(je,Ea),t(je,ht),t(ht,Bt),t(Bt,xa),t(je,Ta),t(je,Wt),t(Wt,Aa),t(je,qa),t(je,Ht),t(Ht,Ma),t(je,Da),t(je,Yt),t(Yt,Fa),t(je,Ca),m(e,vs,u),F(ut,e,u),m(e,Es,u),m(e,Tt,u),t(Tt,za),m(e,xs,u),m(e,He,u),t(He,Ut),t(Ut,La),t(He,Pa),t(He,ct),t(ct,Ia),t(ct,Jt),t(Jt,Oa),t(ct,Sa),m(e,Ts,u),F(dt,e,u),m(e,As,u),m(e,Ye,u),t(Ye,Ra),t(Ye,Kt),t(Kt,Na),t(Ye,Ga),m(e,qs,u),F(gt,e,u),m(e,Ms,u),m(e,ye,u),t(ye,Ba),t(ye,At),t(At,Wa),t(ye,Ha),t(ye,Qt),t(Qt,Ya),t(ye,Ua),t(ye,Vt),t(Vt,Ja),t(ye,Ka),t(ye,Xt),t(Xt,Qa),t(ye,Va),m(e,Ds,u),F(Ue,e,u),m(e,Fs,u),m(e,Se,u),t(Se,Je),t(Je,Zt),F(_t,Zt,null),t(Se,Xa),t(Se,es),t(es,Za),m(e,Cs,u),m(e,Ke,u),t(Ke,el),t(Ke,$t),t($t,tl),t(Ke,sl),m(e,zs,u),m(e,Re,u),t(Re,Qe),t(Qe,ts),F(kt,ts,null),t(Re,al),t(Re,ss),t(ss,ll),m(e,Ls,u),F(Ve,e,u),m(e,Ps,u),m(e,Ne,u),t(Ne,Xe),t(Xe,as),F(wt,as,null),t(Ne,nl),t(Ne,ls),t(ls,rl),m(e,Is,u),m(e,Ze,u),t(Ze,ol),t(Ze,jt),t(jt,il),t(Ze,pl),m(e,Os,u),m(e,Ge,u),t(Ge,et),t(et,ns),F(yt,ns,null),t(Ge,fl),t(Ge,rs),t(rs,ml),m(e,Ss,u),F(tt,e,u),m(e,Rs,u),F(st,e,u),Ns=!0},p(e,[u]){const bt={};u&2&&(bt.$$scope={dirty:u,ctx:e}),X.$set(bt);const os={};u&2&&(os.$$scope={dirty:u,ctx:e}),Ue.$set(os);const is={};u&2&&(is.$$scope={dirty:u,ctx:e}),Ve.$set(is);const ps={};u&2&&(ps.$$scope={dirty:u,ctx:e}),tt.$set(ps);const fs={};u&2&&(fs.$$scope={dirty:u,ctx:e}),st.$set(fs)},i(e){Ns||(C(k.$$.fragment,e),C(T.$$.fragment,e),C(Y.$$.fragment,e),C(X.$$.fragment,e),C(ie.$$.fragment,e),C(be.$$.fragment,e),C(ve.$$.fragment,e),C(Ee.$$.fragment,e),C(lt.$$.fragment,e),C(nt.$$.fragment,e),C(rt.$$.fragment,e),C(ot.$$.fragment,e),C(it.$$.fragment,e),C(ft.$$.fragment,e),C(mt.$$.fragment,e),C(ut.$$.fragment,e),C(dt.$$.fragment,e),C(gt.$$.fragment,e),C(Ue.$$.fragment,e),C(_t.$$.fragment,e),C(kt.$$.fragment,e),C(Ve.$$.fragment,e),C(wt.$$.fragment,e),C(yt.$$.fragment,e),C(tt.$$.fragment,e),C(st.$$.fragment,e),Ns=!0)},o(e){z(k.$$.fragment,e),z(T.$$.fragment,e),z(Y.$$.fragment,e),z(X.$$.fragment,e),z(ie.$$.fragment,e),z(be.$$.fragment,e),z(ve.$$.fragment,e),z(Ee.$$.fragment,e),z(lt.$$.fragment,e),z(nt.$$.fragment,e),z(rt.$$.fragment,e),z(ot.$$.fragment,e),z(it.$$.fragment,e),z(ft.$$.fragment,e),z(mt.$$.fragment,e),z(ut.$$.fragment,e),z(dt.$$.fragment,e),z(gt.$$.fragment,e),z(Ue.$$.fragment,e),z(_t.$$.fragment,e),z(kt.$$.fragment,e),z(Ve.$$.fragment,e),z(wt.$$.fragment,e),z(yt.$$.fragment,e),z(tt.$$.fragment,e),z(st.$$.fragment,e),Ns=!1},d(e){s(a),e&&s(w),e&&s(o),L(k),e&&s(x),e&&s(q),e&&s(N),L(T,e),e&&s(S),e&&s(R),e&&s(O),L(Y,e),e&&s($),e&&s(b),e&&s(P),e&&s(H),e&&s(le),L(X,e),e&&s(Z),e&&s(fe),L(ie),e&&s(_e),e&&s(r),e&&s(de),L(be,e),e&&s(qe),e&&s(xe),e&&s($e),L(ve,e),e&&s(ke),e&&s(Te),e&&s(we),L(Ee,e),e&&s(Me),e&&s(Ae),e&&s(ms),e&&s(Oe),L(lt),e&&s(hs),L(nt,e),e&&s(us),e&&s(We),e&&s(cs),L(rt,e),e&&s(ds),L(ot,e),e&&s(gs),e&&s(Et),e&&s(_s),L(it,e),e&&s($s),e&&s(De),e&&s(ks),L(ft,e),e&&s(ws),e&&s(Fe),e&&s(js),e&&s(xt),e&&s(ys),L(mt,e),e&&s(bs),e&&s(je),e&&s(vs),L(ut,e),e&&s(Es),e&&s(Tt),e&&s(xs),e&&s(He),e&&s(Ts),L(dt,e),e&&s(As),e&&s(Ye),e&&s(qs),L(gt,e),e&&s(Ms),e&&s(ye),e&&s(Ds),L(Ue,e),e&&s(Fs),e&&s(Se),L(_t),e&&s(Cs),e&&s(Ke),e&&s(zs),e&&s(Re),L(kt),e&&s(Ls),L(Ve,e),e&&s(Ps),e&&s(Ne),L(wt),e&&s(Is),e&&s(Ze),e&&s(Os),e&&s(Ge),L(yt),e&&s(Ss),L(tt,e),e&&s(Rs),L(st,e)}}}const On={local:"language-modeling",sections:[{local:"load-eli5-dataset",title:"Load ELI5 dataset"},{local:"preprocess",title:"Preprocess"},{local:"causal-language-modeling",sections:[{local:"train",title:"Train"}],title:"Causal language modeling"},{local:"masked-language-modeling",sections:[{local:"train",title:"Train"}],title:"Masked language modeling"}],title:"Language modeling"};function Sn(J){return $n(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class Yn extends cn{constructor(a){super();dn(this,a,Sn,In,gn,{})}}export{Yn as default,On as metadata};
