import{S as hn,i as un,s as cn,e as i,k as c,w as q,t as l,M as dn,c as p,d as a,m as d,a as f,x as D,h as n,b as g,G as t,g as m,y as z,q as C,o as L,B as P,v as gn,L as mn}from"../../chunks/vendor-hf-doc-builder.js";import{T as Tt}from"../../chunks/Tip-hf-doc-builder.js";import{Y as Xa}from"../../chunks/Youtube-hf-doc-builder.js";import{I as kt}from"../../chunks/IconCopyLink-hf-doc-builder.js";import{C as ae}from"../../chunks/CodeBlock-hf-doc-builder.js";import{F as ml,M as At}from"../../chunks/Markdown-hf-doc-builder.js";function _n(Q){let s,w,o,_,b,k,v,F,j,x,M,I,G,A,N,J,O,S,R,$;return{c(){s=i("p"),w=l("You can fine-tune other architectures for language modeling such as "),o=i("a"),_=l("GPT-Neo"),b=l(", "),k=i("a"),v=l("GPT-J"),F=l(", and "),j=i("a"),x=l("BERT"),M=l(", following the same steps presented in this guide!"),I=c(),G=i("p"),A=l("See the text generation "),N=i("a"),J=l("task page"),O=l(" and fill mask "),S=i("a"),R=l("task page"),$=l(" for more information about their associated models, datasets, and metrics."),this.h()},l(y){s=p(y,"P",{});var Y=f(s);w=n(Y,"You can fine-tune other architectures for language modeling such as "),o=p(Y,"A",{href:!0,rel:!0});var B=f(o);_=n(B,"GPT-Neo"),B.forEach(a),b=n(Y,", "),k=p(Y,"A",{href:!0,rel:!0});var W=f(k);v=n(W,"GPT-J"),W.forEach(a),F=n(Y,", and "),j=p(Y,"A",{href:!0,rel:!0});var K=f(j);x=n(K,"BERT"),K.forEach(a),M=n(Y,", following the same steps presented in this guide!"),Y.forEach(a),I=d(y),G=p(y,"P",{});var X=f(G);A=n(X,"See the text generation "),N=p(X,"A",{href:!0,rel:!0});var H=f(N);J=n(H,"task page"),H.forEach(a),O=n(X," and fill mask "),S=p(X,"A",{href:!0,rel:!0});var fe=f(S);R=n(fe,"task page"),fe.forEach(a),$=n(X," for more information about their associated models, datasets, and metrics."),X.forEach(a),this.h()},h(){g(o,"href","https://huggingface.co/EleutherAI/gpt-neo-125M"),g(o,"rel","nofollow"),g(k,"href","https://huggingface.co/EleutherAI/gpt-j-6B"),g(k,"rel","nofollow"),g(j,"href","https://huggingface.co/bert-base-uncased"),g(j,"rel","nofollow"),g(N,"href","https://huggingface.co/tasks/text-generation"),g(N,"rel","nofollow"),g(S,"href","https://huggingface.co/tasks/fill-mask"),g(S,"rel","nofollow")},m(y,Y){m(y,s,Y),t(s,w),t(s,o),t(o,_),t(s,b),t(s,k),t(k,v),t(s,F),t(s,j),t(j,x),t(s,M),m(y,I,Y),m(y,G,Y),t(G,A),t(G,N),t(N,J),t(G,O),t(G,S),t(S,R),t(G,$)},d(y){y&&a(s),y&&a(I),y&&a(G)}}}function $n(Q){let s,w,o,_,b,k,v,F,j,x,M,I,G,A,N,J,O,S,R;return v=new ae({props:{code:`from transformers import DataCollatorForLanguageModeling

tokenizer.pad_token = tokenizer.eos_token
data_collator = DataCollatorForLanguageModeling(tokenizer=tokenizer, mlm=False)`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> DataCollatorForLanguageModeling

<span class="hljs-meta">&gt;&gt;&gt; </span>tokenizer.pad_token = tokenizer.eos_token
<span class="hljs-meta">&gt;&gt;&gt; </span>data_collator = DataCollatorForLanguageModeling(tokenizer=tokenizer, mlm=<span class="hljs-literal">False</span>)`}}),S=new ae({props:{code:`from transformers import DataCollatorForLanguageModeling

tokenizer.pad_token = tokenizer.eos_token
data_collator = DataCollatorForLanguageModeling(tokenizer=tokenizer, mlm_probability=0.15)`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> DataCollatorForLanguageModeling

<span class="hljs-meta">&gt;&gt;&gt; </span>tokenizer.pad_token = tokenizer.eos_token
<span class="hljs-meta">&gt;&gt;&gt; </span>data_collator = DataCollatorForLanguageModeling(tokenizer=tokenizer, mlm_probability=<span class="hljs-number">0.15</span>)`}}),{c(){s=i("p"),w=l("You can use the end of sequence token as the padding token, and set "),o=i("code"),_=l("mlm=False"),b=l(". This will use the inputs as labels shifted to the right by one element:"),k=c(),q(v.$$.fragment),F=c(),j=i("p"),x=l("For masked language modeling, use the same "),M=i("a"),I=l("DataCollatorForLanguageModeling"),G=l(" except you should specify "),A=i("code"),N=l("mlm_probability"),J=l(" to randomly mask tokens each time you iterate over the data."),O=c(),q(S.$$.fragment),this.h()},l($){s=p($,"P",{});var y=f(s);w=n(y,"You can use the end of sequence token as the padding token, and set "),o=p(y,"CODE",{});var Y=f(o);_=n(Y,"mlm=False"),Y.forEach(a),b=n(y,". This will use the inputs as labels shifted to the right by one element:"),y.forEach(a),k=d($),D(v.$$.fragment,$),F=d($),j=p($,"P",{});var B=f(j);x=n(B,"For masked language modeling, use the same "),M=p(B,"A",{href:!0});var W=f(M);I=n(W,"DataCollatorForLanguageModeling"),W.forEach(a),G=n(B," except you should specify "),A=p(B,"CODE",{});var K=f(A);N=n(K,"mlm_probability"),K.forEach(a),J=n(B," to randomly mask tokens each time you iterate over the data."),B.forEach(a),O=d($),D(S.$$.fragment,$),this.h()},h(){g(M,"href","/docs/transformers/pr_19907/en/main_classes/data_collator#transformers.DataCollatorForLanguageModeling")},m($,y){m($,s,y),t(s,w),t(s,o),t(o,_),t(s,b),m($,k,y),z(v,$,y),m($,F,y),m($,j,y),t(j,x),t(j,M),t(M,I),t(j,G),t(j,A),t(A,N),t(j,J),m($,O,y),z(S,$,y),R=!0},p:mn,i($){R||(C(v.$$.fragment,$),C(S.$$.fragment,$),R=!0)},o($){L(v.$$.fragment,$),L(S.$$.fragment,$),R=!1},d($){$&&a(s),$&&a(k),P(v,$),$&&a(F),$&&a(j),$&&a(O),P(S,$)}}}function kn(Q){let s,w;return s=new At({props:{$$slots:{default:[$n]},$$scope:{ctx:Q}}}),{c(){q(s.$$.fragment)},l(o){D(s.$$.fragment,o)},m(o,_){z(s,o,_),w=!0},p(o,_){const b={};_&2&&(b.$$scope={dirty:_,ctx:o}),s.$set(b)},i(o){w||(C(s.$$.fragment,o),w=!0)},o(o){L(s.$$.fragment,o),w=!1},d(o){P(s,o)}}}function wn(Q){let s,w,o,_,b,k,v,F,j,x,M,I,G,A,N,J,O,S,R;return v=new ae({props:{code:`from transformers import DataCollatorForLanguageModeling

data_collator = DataCollatorForLanguageModeling(tokenizer=tokenizer, mlm=False, return_tensors="tf")`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> DataCollatorForLanguageModeling

<span class="hljs-meta">&gt;&gt;&gt; </span>data_collator = DataCollatorForLanguageModeling(tokenizer=tokenizer, mlm=<span class="hljs-literal">False</span>, return_tensors=<span class="hljs-string">&quot;tf&quot;</span>)`}}),S=new ae({props:{code:`from transformers import DataCollatorForLanguageModeling

data_collator = DataCollatorForLanguageModeling(tokenizer=tokenizer, mlm=False, return_tensors="tf")`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> DataCollatorForLanguageModeling

<span class="hljs-meta">&gt;&gt;&gt; </span>data_collator = DataCollatorForLanguageModeling(tokenizer=tokenizer, mlm=<span class="hljs-literal">False</span>, return_tensors=<span class="hljs-string">&quot;tf&quot;</span>)`}}),{c(){s=i("p"),w=l("You can use the end of sequence token as the padding token, and set "),o=i("code"),_=l("mlm=False"),b=l(". This will use the inputs as labels shifted to the right by one element:"),k=c(),q(v.$$.fragment),F=c(),j=i("p"),x=l("For masked language modeling, use the same "),M=i("a"),I=l("DataCollatorForLanguageModeling"),G=l(" except you should specify "),A=i("code"),N=l("mlm_probability"),J=l(" to randomly mask tokens each time you iterate over the data."),O=c(),q(S.$$.fragment),this.h()},l($){s=p($,"P",{});var y=f(s);w=n(y,"You can use the end of sequence token as the padding token, and set "),o=p(y,"CODE",{});var Y=f(o);_=n(Y,"mlm=False"),Y.forEach(a),b=n(y,". This will use the inputs as labels shifted to the right by one element:"),y.forEach(a),k=d($),D(v.$$.fragment,$),F=d($),j=p($,"P",{});var B=f(j);x=n(B,"For masked language modeling, use the same "),M=p(B,"A",{href:!0});var W=f(M);I=n(W,"DataCollatorForLanguageModeling"),W.forEach(a),G=n(B," except you should specify "),A=p(B,"CODE",{});var K=f(A);N=n(K,"mlm_probability"),K.forEach(a),J=n(B," to randomly mask tokens each time you iterate over the data."),B.forEach(a),O=d($),D(S.$$.fragment,$),this.h()},h(){g(M,"href","/docs/transformers/pr_19907/en/main_classes/data_collator#transformers.DataCollatorForLanguageModeling")},m($,y){m($,s,y),t(s,w),t(s,o),t(o,_),t(s,b),m($,k,y),z(v,$,y),m($,F,y),m($,j,y),t(j,x),t(j,M),t(M,I),t(j,G),t(j,A),t(A,N),t(j,J),m($,O,y),z(S,$,y),R=!0},p:mn,i($){R||(C(v.$$.fragment,$),C(S.$$.fragment,$),R=!0)},o($){L(v.$$.fragment,$),L(S.$$.fragment,$),R=!1},d($){$&&a(s),$&&a(k),P(v,$),$&&a(F),$&&a(j),$&&a(O),P(S,$)}}}function jn(Q){let s,w;return s=new At({props:{$$slots:{default:[wn]},$$scope:{ctx:Q}}}),{c(){q(s.$$.fragment)},l(o){D(s.$$.fragment,o)},m(o,_){z(s,o,_),w=!0},p(o,_){const b={};_&2&&(b.$$scope={dirty:_,ctx:o}),s.$set(b)},i(o){w||(C(s.$$.fragment,o),w=!0)},o(o){L(s.$$.fragment,o),w=!1},d(o){P(s,o)}}}function yn(Q){let s,w,o,_,b,k,v,F;return{c(){s=i("p"),w=l("If you aren\u2019t familiar with fine-tuning a model with the "),o=i("a"),_=l("Trainer"),b=l(", take a look at the basic tutorial "),k=i("a"),v=l("here"),F=l("!"),this.h()},l(j){s=p(j,"P",{});var x=f(s);w=n(x,"If you aren\u2019t familiar with fine-tuning a model with the "),o=p(x,"A",{href:!0});var M=f(o);_=n(M,"Trainer"),M.forEach(a),b=n(x,", take a look at the basic tutorial "),k=p(x,"A",{href:!0});var I=f(k);v=n(I,"here"),I.forEach(a),F=n(x,"!"),x.forEach(a),this.h()},h(){g(o,"href","/docs/transformers/pr_19907/en/main_classes/trainer#transformers.Trainer"),g(k,"href","../training#finetune-with-trainer")},m(j,x){m(j,s,x),t(s,w),t(s,o),t(o,_),t(s,b),t(s,k),t(k,v),t(s,F)},d(j){j&&a(s)}}}function vn(Q){let s,w,o,_,b,k,v,F,j,x,M,I,G,A,N,J,O,S,R,$,y,Y,B,W,K,X,H,fe,U,oe,he,re,Z,ee;return v=new ae({props:{code:`from transformers import AutoModelForCausalLM, TrainingArguments, Trainer

model = AutoModelForCausalLM.from_pretrained("distilgpt2")`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoModelForCausalLM, TrainingArguments, Trainer

<span class="hljs-meta">&gt;&gt;&gt; </span>model = AutoModelForCausalLM.from_pretrained(<span class="hljs-string">&quot;distilgpt2&quot;</span>)`}}),j=new Tt({props:{$$slots:{default:[yn]},$$scope:{ctx:Q}}}),Z=new ae({props:{code:`training_args = TrainingArguments(
    output_dir="./results",
    evaluation_strategy="epoch",
    learning_rate=2e-5,
    weight_decay=0.01,
)

trainer = Trainer(
    model=model,
    args=training_args,
    train_dataset=lm_dataset["train"],
    eval_dataset=lm_dataset["test"],
    data_collator=data_collator,
)

trainer.train()`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>training_args = TrainingArguments(
<span class="hljs-meta">... </span>    output_dir=<span class="hljs-string">&quot;./results&quot;</span>,
<span class="hljs-meta">... </span>    evaluation_strategy=<span class="hljs-string">&quot;epoch&quot;</span>,
<span class="hljs-meta">... </span>    learning_rate=<span class="hljs-number">2e-5</span>,
<span class="hljs-meta">... </span>    weight_decay=<span class="hljs-number">0.01</span>,
<span class="hljs-meta">... </span>)

<span class="hljs-meta">&gt;&gt;&gt; </span>trainer = Trainer(
<span class="hljs-meta">... </span>    model=model,
<span class="hljs-meta">... </span>    args=training_args,
<span class="hljs-meta">... </span>    train_dataset=lm_dataset[<span class="hljs-string">&quot;train&quot;</span>],
<span class="hljs-meta">... </span>    eval_dataset=lm_dataset[<span class="hljs-string">&quot;test&quot;</span>],
<span class="hljs-meta">... </span>    data_collator=data_collator,
<span class="hljs-meta">... </span>)

<span class="hljs-meta">&gt;&gt;&gt; </span>trainer.train()`}}),{c(){s=i("p"),w=l("Load DistilGPT2 with "),o=i("a"),_=l("AutoModelForCausalLM"),b=l(":"),k=c(),q(v.$$.fragment),F=c(),q(j.$$.fragment),x=c(),M=i("p"),I=l("At this point, only three steps remain:"),G=c(),A=i("ol"),N=i("li"),J=l("Define your training hyperparameters in "),O=i("a"),S=l("TrainingArguments"),R=l("."),$=c(),y=i("li"),Y=l("Pass the training arguments to "),B=i("a"),W=l("Trainer"),K=l(" along with the model, datasets, and data collator."),X=c(),H=i("li"),fe=l("Call "),U=i("a"),oe=l("train()"),he=l(" to fine-tune your model."),re=c(),q(Z.$$.fragment),this.h()},l(h){s=p(h,"P",{});var T=f(s);w=n(T,"Load DistilGPT2 with "),o=p(T,"A",{href:!0});var ie=f(o);_=n(ie,"AutoModelForCausalLM"),ie.forEach(a),b=n(T,":"),T.forEach(a),k=d(h),D(v.$$.fragment,h),F=d(h),D(j.$$.fragment,h),x=d(h),M=p(h,"P",{});var ne=f(M);I=n(ne,"At this point, only three steps remain:"),ne.forEach(a),G=d(h),A=p(h,"OL",{});var V=f(A);N=p(V,"LI",{});var me=f(N);J=n(me,"Define your training hyperparameters in "),O=p(me,"A",{href:!0});var pe=f(O);S=n(pe,"TrainingArguments"),pe.forEach(a),R=n(me,"."),me.forEach(a),$=d(V),y=p(V,"LI",{});var te=f(y);Y=n(te,"Pass the training arguments to "),B=p(te,"A",{href:!0});var se=f(B);W=n(se,"Trainer"),se.forEach(a),K=n(te," along with the model, datasets, and data collator."),te.forEach(a),X=d(V),H=p(V,"LI",{});var le=f(H);fe=n(le,"Call "),U=p(le,"A",{href:!0});var r=f(U);oe=n(r,"train()"),r.forEach(a),he=n(le," to fine-tune your model."),le.forEach(a),V.forEach(a),re=d(h),D(Z.$$.fragment,h),this.h()},h(){g(o,"href","/docs/transformers/pr_19907/en/model_doc/auto#transformers.AutoModelForCausalLM"),g(O,"href","/docs/transformers/pr_19907/en/main_classes/trainer#transformers.TrainingArguments"),g(B,"href","/docs/transformers/pr_19907/en/main_classes/trainer#transformers.Trainer"),g(U,"href","/docs/transformers/pr_19907/en/main_classes/trainer#transformers.Trainer.train")},m(h,T){m(h,s,T),t(s,w),t(s,o),t(o,_),t(s,b),m(h,k,T),z(v,h,T),m(h,F,T),z(j,h,T),m(h,x,T),m(h,M,T),t(M,I),m(h,G,T),m(h,A,T),t(A,N),t(N,J),t(N,O),t(O,S),t(N,R),t(A,$),t(A,y),t(y,Y),t(y,B),t(B,W),t(y,K),t(A,X),t(A,H),t(H,fe),t(H,U),t(U,oe),t(H,he),m(h,re,T),z(Z,h,T),ee=!0},p(h,T){const ie={};T&2&&(ie.$$scope={dirty:T,ctx:h}),j.$set(ie)},i(h){ee||(C(v.$$.fragment,h),C(j.$$.fragment,h),C(Z.$$.fragment,h),ee=!0)},o(h){L(v.$$.fragment,h),L(j.$$.fragment,h),L(Z.$$.fragment,h),ee=!1},d(h){h&&a(s),h&&a(k),P(v,h),h&&a(F),P(j,h),h&&a(x),h&&a(M),h&&a(G),h&&a(A),h&&a(re),P(Z,h)}}}function bn(Q){let s,w;return s=new At({props:{$$slots:{default:[vn]},$$scope:{ctx:Q}}}),{c(){q(s.$$.fragment)},l(o){D(s.$$.fragment,o)},m(o,_){z(s,o,_),w=!0},p(o,_){const b={};_&2&&(b.$$scope={dirty:_,ctx:o}),s.$set(b)},i(o){w||(C(s.$$.fragment,o),w=!0)},o(o){L(s.$$.fragment,o),w=!1},d(o){P(s,o)}}}function xn(Q){let s,w,o,_,b;return{c(){s=i("p"),w=l("If you aren\u2019t familiar with fine-tuning a model with Keras, take a look at the basic tutorial "),o=i("a"),_=l("here"),b=l("!"),this.h()},l(k){s=p(k,"P",{});var v=f(s);w=n(v,"If you aren\u2019t familiar with fine-tuning a model with Keras, take a look at the basic tutorial "),o=p(v,"A",{href:!0});var F=f(o);_=n(F,"here"),F.forEach(a),b=n(v,"!"),v.forEach(a),this.h()},h(){g(o,"href","training#finetune-with-keras")},m(k,v){m(k,s,v),t(s,w),t(s,o),t(o,_),t(s,b)},d(k){k&&a(s)}}}function En(Q){let s,w,o,_,b,k,v,F,j,x,M,I,G,A,N,J,O,S,R,$,y,Y,B,W,K,X,H,fe,U,oe,he,re,Z,ee,h,T,ie,ne,V,me,pe,te,se,le;return x=new ae({props:{code:`tf_train_set = model.prepare_tf_dataset(
    lm_dataset["train"],
    shuffle=True,
    batch_size=16,
    collate_fn=data_collator,
)

tf_test_set = model.prepare_tf_dataset(
    lm_dataset["test"],
    shuffle=False,
    batch_size=16,
    collate_fn=data_collator,
)`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>tf_train_set = model.prepare_tf_dataset(
<span class="hljs-meta">... </span>    lm_dataset[<span class="hljs-string">&quot;train&quot;</span>],
<span class="hljs-meta">... </span>    shuffle=<span class="hljs-literal">True</span>,
<span class="hljs-meta">... </span>    batch_size=<span class="hljs-number">16</span>,
<span class="hljs-meta">... </span>    collate_fn=data_collator,
<span class="hljs-meta">... </span>)

<span class="hljs-meta">&gt;&gt;&gt; </span>tf_test_set = model.prepare_tf_dataset(
<span class="hljs-meta">... </span>    lm_dataset[<span class="hljs-string">&quot;test&quot;</span>],
<span class="hljs-meta">... </span>    shuffle=<span class="hljs-literal">False</span>,
<span class="hljs-meta">... </span>    batch_size=<span class="hljs-number">16</span>,
<span class="hljs-meta">... </span>    collate_fn=data_collator,
<span class="hljs-meta">... </span>)`}}),I=new Tt({props:{$$slots:{default:[xn]},$$scope:{ctx:Q}}}),O=new ae({props:{code:`from transformers import create_optimizer, AdamWeightDecay

optimizer = AdamWeightDecay(learning_rate=2e-5, weight_decay_rate=0.01)`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> create_optimizer, AdamWeightDecay

<span class="hljs-meta">&gt;&gt;&gt; </span>optimizer = AdamWeightDecay(learning_rate=<span class="hljs-number">2e-5</span>, weight_decay_rate=<span class="hljs-number">0.01</span>)`}}),K=new ae({props:{code:`from transformers import TFAutoModelForCausalLM

model = TFAutoModelForCausalLM.from_pretrained("distilgpt2")`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> TFAutoModelForCausalLM

<span class="hljs-meta">&gt;&gt;&gt; </span>model = TFAutoModelForCausalLM.from_pretrained(<span class="hljs-string">&quot;distilgpt2&quot;</span>)`}}),ee=new ae({props:{code:`import tensorflow as tf

model.compile(optimizer=optimizer)`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">import</span> tensorflow <span class="hljs-keyword">as</span> tf

<span class="hljs-meta">&gt;&gt;&gt; </span>model.<span class="hljs-built_in">compile</span>(optimizer=optimizer)`}}),se=new ae({props:{code:"model.fit(x=tf_train_set, validation_data=tf_test_set, epochs=3)",highlighted:'<span class="hljs-meta">&gt;&gt;&gt; </span>model.fit(x=tf_train_set, validation_data=tf_test_set, epochs=<span class="hljs-number">3</span>)'}}),{c(){s=i("p"),w=l("To fine-tune a model in TensorFlow, start by converting your datasets to the "),o=i("code"),_=l("tf.data.Dataset"),b=l(" format with "),k=i("a"),v=l("prepare_tf_dataset()"),F=l("."),j=c(),q(x.$$.fragment),M=c(),q(I.$$.fragment),G=c(),A=i("p"),N=l("Set up an optimizer function, learning rate, and some training hyperparameters:"),J=c(),q(O.$$.fragment),S=c(),R=i("p"),$=l("Load DistilGPT2 with "),y=i("a"),Y=l("TFAutoModelForCausalLM"),B=l(":"),W=c(),q(K.$$.fragment),X=c(),H=i("p"),fe=l("Configure the model for training with "),U=i("a"),oe=i("code"),he=l("compile"),re=l(":"),Z=c(),q(ee.$$.fragment),h=c(),T=i("p"),ie=l("Call "),ne=i("a"),V=i("code"),me=l("fit"),pe=l(" to fine-tune the model:"),te=c(),q(se.$$.fragment),this.h()},l(r){s=p(r,"P",{});var E=f(s);w=n(E,"To fine-tune a model in TensorFlow, start by converting your datasets to the "),o=p(E,"CODE",{});var ce=f(o);_=n(ce,"tf.data.Dataset"),ce.forEach(a),b=n(E," format with "),k=p(E,"A",{href:!0});var ve=f(k);v=n(ve,"prepare_tf_dataset()"),ve.forEach(a),F=n(E,"."),E.forEach(a),j=d(r),D(x.$$.fragment,r),M=d(r),D(I.$$.fragment,r),G=d(r),A=p(r,"P",{});var we=f(A);N=n(we,"Set up an optimizer function, learning rate, and some training hyperparameters:"),we.forEach(a),J=d(r),D(O.$$.fragment,r),S=d(r),R=p(r,"P",{});var _e=f(R);$=n(_e,"Load DistilGPT2 with "),y=p(_e,"A",{href:!0});var be=f(y);Y=n(be,"TFAutoModelForCausalLM"),be.forEach(a),B=n(_e,":"),_e.forEach(a),W=d(r),D(K.$$.fragment,r),X=d(r),H=p(r,"P",{});var ue=f(H);fe=n(ue,"Configure the model for training with "),U=p(ue,"A",{href:!0,rel:!0});var xe=f(U);oe=p(xe,"CODE",{});var je=f(oe);he=n(je,"compile"),je.forEach(a),xe.forEach(a),re=n(ue,":"),ue.forEach(a),Z=d(r),D(ee.$$.fragment,r),h=d(r),T=p(r,"P",{});var $e=f(T);ie=n($e,"Call "),ne=p($e,"A",{href:!0,rel:!0});var Ee=f(ne);V=p(Ee,"CODE",{});var ke=f(V);me=n(ke,"fit"),ke.forEach(a),Ee.forEach(a),pe=n($e," to fine-tune the model:"),$e.forEach(a),te=d(r),D(se.$$.fragment,r),this.h()},h(){g(k,"href","/docs/transformers/pr_19907/en/main_classes/model#transformers.TFPreTrainedModel.prepare_tf_dataset"),g(y,"href","/docs/transformers/pr_19907/en/model_doc/auto#transformers.TFAutoModelForCausalLM"),g(U,"href","https://keras.io/api/models/model_training_apis/#compile-method"),g(U,"rel","nofollow"),g(ne,"href","https://keras.io/api/models/model_training_apis/#fit-method"),g(ne,"rel","nofollow")},m(r,E){m(r,s,E),t(s,w),t(s,o),t(o,_),t(s,b),t(s,k),t(k,v),t(s,F),m(r,j,E),z(x,r,E),m(r,M,E),z(I,r,E),m(r,G,E),m(r,A,E),t(A,N),m(r,J,E),z(O,r,E),m(r,S,E),m(r,R,E),t(R,$),t(R,y),t(y,Y),t(R,B),m(r,W,E),z(K,r,E),m(r,X,E),m(r,H,E),t(H,fe),t(H,U),t(U,oe),t(oe,he),t(H,re),m(r,Z,E),z(ee,r,E),m(r,h,E),m(r,T,E),t(T,ie),t(T,ne),t(ne,V),t(V,me),t(T,pe),m(r,te,E),z(se,r,E),le=!0},p(r,E){const ce={};E&2&&(ce.$$scope={dirty:E,ctx:r}),I.$set(ce)},i(r){le||(C(x.$$.fragment,r),C(I.$$.fragment,r),C(O.$$.fragment,r),C(K.$$.fragment,r),C(ee.$$.fragment,r),C(se.$$.fragment,r),le=!0)},o(r){L(x.$$.fragment,r),L(I.$$.fragment,r),L(O.$$.fragment,r),L(K.$$.fragment,r),L(ee.$$.fragment,r),L(se.$$.fragment,r),le=!1},d(r){r&&a(s),r&&a(j),P(x,r),r&&a(M),P(I,r),r&&a(G),r&&a(A),r&&a(J),P(O,r),r&&a(S),r&&a(R),r&&a(W),P(K,r),r&&a(X),r&&a(H),r&&a(Z),P(ee,r),r&&a(h),r&&a(T),r&&a(te),P(se,r)}}}function Tn(Q){let s,w;return s=new At({props:{$$slots:{default:[En]},$$scope:{ctx:Q}}}),{c(){q(s.$$.fragment)},l(o){D(s.$$.fragment,o)},m(o,_){z(s,o,_),w=!0},p(o,_){const b={};_&2&&(b.$$scope={dirty:_,ctx:o}),s.$set(b)},i(o){w||(C(s.$$.fragment,o),w=!0)},o(o){L(s.$$.fragment,o),w=!1},d(o){P(s,o)}}}function An(Q){let s,w,o,_,b,k,v,F;return{c(){s=i("p"),w=l("If you aren\u2019t familiar with fine-tuning a model with the "),o=i("a"),_=l("Trainer"),b=l(", take a look at the basic tutorial "),k=i("a"),v=l("here"),F=l("!"),this.h()},l(j){s=p(j,"P",{});var x=f(s);w=n(x,"If you aren\u2019t familiar with fine-tuning a model with the "),o=p(x,"A",{href:!0});var M=f(o);_=n(M,"Trainer"),M.forEach(a),b=n(x,", take a look at the basic tutorial "),k=p(x,"A",{href:!0});var I=f(k);v=n(I,"here"),I.forEach(a),F=n(x,"!"),x.forEach(a),this.h()},h(){g(o,"href","/docs/transformers/pr_19907/en/main_classes/trainer#transformers.Trainer"),g(k,"href","../training#finetune-with-trainer")},m(j,x){m(j,s,x),t(s,w),t(s,o),t(o,_),t(s,b),t(s,k),t(k,v),t(s,F)},d(j){j&&a(s)}}}function Mn(Q){let s,w,o,_,b,k,v,F,j,x,M,I,G,A,N,J,O,S,R,$,y,Y,B,W,K,X,H,fe,U,oe,he,re,Z,ee;return v=new ae({props:{code:`from transformers import AutoModelForMaskedLM

model = AutoModelForMaskedLM.from_pretrained("distilroberta-base")`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoModelForMaskedLM

<span class="hljs-meta">&gt;&gt;&gt; </span>model = AutoModelForMaskedLM.from_pretrained(<span class="hljs-string">&quot;distilroberta-base&quot;</span>)`}}),j=new Tt({props:{$$slots:{default:[An]},$$scope:{ctx:Q}}}),Z=new ae({props:{code:`training_args = TrainingArguments(
    output_dir="./results",
    evaluation_strategy="epoch",
    learning_rate=2e-5,
    num_train_epochs=3,
    weight_decay=0.01,
)

trainer = Trainer(
    model=model,
    args=training_args,
    train_dataset=lm_dataset["train"],
    eval_dataset=lm_dataset["test"],
    data_collator=data_collator,
)

trainer.train()`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>training_args = TrainingArguments(
<span class="hljs-meta">... </span>    output_dir=<span class="hljs-string">&quot;./results&quot;</span>,
<span class="hljs-meta">... </span>    evaluation_strategy=<span class="hljs-string">&quot;epoch&quot;</span>,
<span class="hljs-meta">... </span>    learning_rate=<span class="hljs-number">2e-5</span>,
<span class="hljs-meta">... </span>    num_train_epochs=<span class="hljs-number">3</span>,
<span class="hljs-meta">... </span>    weight_decay=<span class="hljs-number">0.01</span>,
<span class="hljs-meta">... </span>)

<span class="hljs-meta">&gt;&gt;&gt; </span>trainer = Trainer(
<span class="hljs-meta">... </span>    model=model,
<span class="hljs-meta">... </span>    args=training_args,
<span class="hljs-meta">... </span>    train_dataset=lm_dataset[<span class="hljs-string">&quot;train&quot;</span>],
<span class="hljs-meta">... </span>    eval_dataset=lm_dataset[<span class="hljs-string">&quot;test&quot;</span>],
<span class="hljs-meta">... </span>    data_collator=data_collator,
<span class="hljs-meta">... </span>)

<span class="hljs-meta">&gt;&gt;&gt; </span>trainer.train()`}}),{c(){s=i("p"),w=l("Load DistilRoBERTa with "),o=i("code"),_=l("AutoModelForMaskedlM"),b=l(":"),k=c(),q(v.$$.fragment),F=c(),q(j.$$.fragment),x=c(),M=i("p"),I=l("At this point, only three steps remain:"),G=c(),A=i("ol"),N=i("li"),J=l("Define your training hyperparameters in "),O=i("a"),S=l("TrainingArguments"),R=l("."),$=c(),y=i("li"),Y=l("Pass the training arguments to "),B=i("a"),W=l("Trainer"),K=l(" along with the model, datasets, and data collator."),X=c(),H=i("li"),fe=l("Call "),U=i("a"),oe=l("train()"),he=l(" to fine-tune your model."),re=c(),q(Z.$$.fragment),this.h()},l(h){s=p(h,"P",{});var T=f(s);w=n(T,"Load DistilRoBERTa with "),o=p(T,"CODE",{});var ie=f(o);_=n(ie,"AutoModelForMaskedlM"),ie.forEach(a),b=n(T,":"),T.forEach(a),k=d(h),D(v.$$.fragment,h),F=d(h),D(j.$$.fragment,h),x=d(h),M=p(h,"P",{});var ne=f(M);I=n(ne,"At this point, only three steps remain:"),ne.forEach(a),G=d(h),A=p(h,"OL",{});var V=f(A);N=p(V,"LI",{});var me=f(N);J=n(me,"Define your training hyperparameters in "),O=p(me,"A",{href:!0});var pe=f(O);S=n(pe,"TrainingArguments"),pe.forEach(a),R=n(me,"."),me.forEach(a),$=d(V),y=p(V,"LI",{});var te=f(y);Y=n(te,"Pass the training arguments to "),B=p(te,"A",{href:!0});var se=f(B);W=n(se,"Trainer"),se.forEach(a),K=n(te," along with the model, datasets, and data collator."),te.forEach(a),X=d(V),H=p(V,"LI",{});var le=f(H);fe=n(le,"Call "),U=p(le,"A",{href:!0});var r=f(U);oe=n(r,"train()"),r.forEach(a),he=n(le," to fine-tune your model."),le.forEach(a),V.forEach(a),re=d(h),D(Z.$$.fragment,h),this.h()},h(){g(O,"href","/docs/transformers/pr_19907/en/main_classes/trainer#transformers.TrainingArguments"),g(B,"href","/docs/transformers/pr_19907/en/main_classes/trainer#transformers.Trainer"),g(U,"href","/docs/transformers/pr_19907/en/main_classes/trainer#transformers.Trainer.train")},m(h,T){m(h,s,T),t(s,w),t(s,o),t(o,_),t(s,b),m(h,k,T),z(v,h,T),m(h,F,T),z(j,h,T),m(h,x,T),m(h,M,T),t(M,I),m(h,G,T),m(h,A,T),t(A,N),t(N,J),t(N,O),t(O,S),t(N,R),t(A,$),t(A,y),t(y,Y),t(y,B),t(B,W),t(y,K),t(A,X),t(A,H),t(H,fe),t(H,U),t(U,oe),t(H,he),m(h,re,T),z(Z,h,T),ee=!0},p(h,T){const ie={};T&2&&(ie.$$scope={dirty:T,ctx:h}),j.$set(ie)},i(h){ee||(C(v.$$.fragment,h),C(j.$$.fragment,h),C(Z.$$.fragment,h),ee=!0)},o(h){L(v.$$.fragment,h),L(j.$$.fragment,h),L(Z.$$.fragment,h),ee=!1},d(h){h&&a(s),h&&a(k),P(v,h),h&&a(F),P(j,h),h&&a(x),h&&a(M),h&&a(G),h&&a(A),h&&a(re),P(Z,h)}}}function Fn(Q){let s,w;return s=new At({props:{$$slots:{default:[Mn]},$$scope:{ctx:Q}}}),{c(){q(s.$$.fragment)},l(o){D(s.$$.fragment,o)},m(o,_){z(s,o,_),w=!0},p(o,_){const b={};_&2&&(b.$$scope={dirty:_,ctx:o}),s.$set(b)},i(o){w||(C(s.$$.fragment,o),w=!0)},o(o){L(s.$$.fragment,o),w=!1},d(o){P(s,o)}}}function qn(Q){let s,w,o,_,b;return{c(){s=i("p"),w=l("If you aren\u2019t familiar with fine-tuning a model with Keras, take a look at the basic tutorial "),o=i("a"),_=l("here"),b=l("!"),this.h()},l(k){s=p(k,"P",{});var v=f(s);w=n(v,"If you aren\u2019t familiar with fine-tuning a model with Keras, take a look at the basic tutorial "),o=p(v,"A",{href:!0});var F=f(o);_=n(F,"here"),F.forEach(a),b=n(v,"!"),v.forEach(a),this.h()},h(){g(o,"href","training#finetune-with-keras")},m(k,v){m(k,s,v),t(s,w),t(s,o),t(o,_),t(s,b)},d(k){k&&a(s)}}}function Dn(Q){let s,w,o,_,b,k,v,F,j,x,M,I,G,A,N,J,O,S,R,$,y,Y,B,W,K,X,H,fe,U,oe,he,re,Z,ee,h,T,ie,ne,V,me,pe,te,se,le;return x=new ae({props:{code:`tf_train_set = model.prepare_tf_dataset(
    lm_dataset["train"],
    shuffle=True,
    batch_size=16,
    collate_fn=data_collator,
)

tf_test_set = model.prepare_tf_dataset(
    lm_dataset["test"],
    shuffle=False,
    batch_size=16,
    collate_fn=data_collator,
)`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>tf_train_set = model.prepare_tf_dataset(
<span class="hljs-meta">... </span>    lm_dataset[<span class="hljs-string">&quot;train&quot;</span>],
<span class="hljs-meta">... </span>    shuffle=<span class="hljs-literal">True</span>,
<span class="hljs-meta">... </span>    batch_size=<span class="hljs-number">16</span>,
<span class="hljs-meta">... </span>    collate_fn=data_collator,
<span class="hljs-meta">... </span>)

<span class="hljs-meta">&gt;&gt;&gt; </span>tf_test_set = model.prepare_tf_dataset(
<span class="hljs-meta">... </span>    lm_dataset[<span class="hljs-string">&quot;test&quot;</span>],
<span class="hljs-meta">... </span>    shuffle=<span class="hljs-literal">False</span>,
<span class="hljs-meta">... </span>    batch_size=<span class="hljs-number">16</span>,
<span class="hljs-meta">... </span>    collate_fn=data_collator,
<span class="hljs-meta">... </span>)`}}),I=new Tt({props:{$$slots:{default:[qn]},$$scope:{ctx:Q}}}),O=new ae({props:{code:`from transformers import create_optimizer, AdamWeightDecay

optimizer = AdamWeightDecay(learning_rate=2e-5, weight_decay_rate=0.01)`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> create_optimizer, AdamWeightDecay

<span class="hljs-meta">&gt;&gt;&gt; </span>optimizer = AdamWeightDecay(learning_rate=<span class="hljs-number">2e-5</span>, weight_decay_rate=<span class="hljs-number">0.01</span>)`}}),K=new ae({props:{code:`from transformers import TFAutoModelForMaskedLM

model = TFAutoModelForCausalLM.from_pretrained("distilroberta-base")`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> TFAutoModelForMaskedLM

<span class="hljs-meta">&gt;&gt;&gt; </span>model = TFAutoModelForCausalLM.from_pretrained(<span class="hljs-string">&quot;distilroberta-base&quot;</span>)`}}),ee=new ae({props:{code:`import tensorflow as tf

model.compile(optimizer=optimizer)`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">import</span> tensorflow <span class="hljs-keyword">as</span> tf

<span class="hljs-meta">&gt;&gt;&gt; </span>model.<span class="hljs-built_in">compile</span>(optimizer=optimizer)`}}),se=new ae({props:{code:"model.fit(x=tf_train_set, validation_data=tf_test_set, epochs=3)",highlighted:'<span class="hljs-meta">&gt;&gt;&gt; </span>model.fit(x=tf_train_set, validation_data=tf_test_set, epochs=<span class="hljs-number">3</span>)'}}),{c(){s=i("p"),w=l("To fine-tune a model in TensorFlow, start by converting your datasets to the "),o=i("code"),_=l("tf.data.Dataset"),b=l(" format with "),k=i("a"),v=l("prepare_tf_dataset()"),F=l("."),j=c(),q(x.$$.fragment),M=c(),q(I.$$.fragment),G=c(),A=i("p"),N=l("Set up an optimizer function, learning rate, and some training hyperparameters:"),J=c(),q(O.$$.fragment),S=c(),R=i("p"),$=l("Load DistilRoBERTa with "),y=i("a"),Y=l("TFAutoModelForMaskedLM"),B=l(":"),W=c(),q(K.$$.fragment),X=c(),H=i("p"),fe=l("Configure the model for training with "),U=i("a"),oe=i("code"),he=l("compile"),re=l(":"),Z=c(),q(ee.$$.fragment),h=c(),T=i("p"),ie=l("Call "),ne=i("a"),V=i("code"),me=l("fit"),pe=l(" to fine-tune the model:"),te=c(),q(se.$$.fragment),this.h()},l(r){s=p(r,"P",{});var E=f(s);w=n(E,"To fine-tune a model in TensorFlow, start by converting your datasets to the "),o=p(E,"CODE",{});var ce=f(o);_=n(ce,"tf.data.Dataset"),ce.forEach(a),b=n(E," format with "),k=p(E,"A",{href:!0});var ve=f(k);v=n(ve,"prepare_tf_dataset()"),ve.forEach(a),F=n(E,"."),E.forEach(a),j=d(r),D(x.$$.fragment,r),M=d(r),D(I.$$.fragment,r),G=d(r),A=p(r,"P",{});var we=f(A);N=n(we,"Set up an optimizer function, learning rate, and some training hyperparameters:"),we.forEach(a),J=d(r),D(O.$$.fragment,r),S=d(r),R=p(r,"P",{});var _e=f(R);$=n(_e,"Load DistilRoBERTa with "),y=p(_e,"A",{href:!0});var be=f(y);Y=n(be,"TFAutoModelForMaskedLM"),be.forEach(a),B=n(_e,":"),_e.forEach(a),W=d(r),D(K.$$.fragment,r),X=d(r),H=p(r,"P",{});var ue=f(H);fe=n(ue,"Configure the model for training with "),U=p(ue,"A",{href:!0,rel:!0});var xe=f(U);oe=p(xe,"CODE",{});var je=f(oe);he=n(je,"compile"),je.forEach(a),xe.forEach(a),re=n(ue,":"),ue.forEach(a),Z=d(r),D(ee.$$.fragment,r),h=d(r),T=p(r,"P",{});var $e=f(T);ie=n($e,"Call "),ne=p($e,"A",{href:!0,rel:!0});var Ee=f(ne);V=p(Ee,"CODE",{});var ke=f(V);me=n(ke,"fit"),ke.forEach(a),Ee.forEach(a),pe=n($e," to fine-tune the model:"),$e.forEach(a),te=d(r),D(se.$$.fragment,r),this.h()},h(){g(k,"href","/docs/transformers/pr_19907/en/main_classes/model#transformers.TFPreTrainedModel.prepare_tf_dataset"),g(y,"href","/docs/transformers/pr_19907/en/model_doc/auto#transformers.TFAutoModelForMaskedLM"),g(U,"href","https://keras.io/api/models/model_training_apis/#compile-method"),g(U,"rel","nofollow"),g(ne,"href","https://keras.io/api/models/model_training_apis/#fit-method"),g(ne,"rel","nofollow")},m(r,E){m(r,s,E),t(s,w),t(s,o),t(o,_),t(s,b),t(s,k),t(k,v),t(s,F),m(r,j,E),z(x,r,E),m(r,M,E),z(I,r,E),m(r,G,E),m(r,A,E),t(A,N),m(r,J,E),z(O,r,E),m(r,S,E),m(r,R,E),t(R,$),t(R,y),t(y,Y),t(R,B),m(r,W,E),z(K,r,E),m(r,X,E),m(r,H,E),t(H,fe),t(H,U),t(U,oe),t(oe,he),t(H,re),m(r,Z,E),z(ee,r,E),m(r,h,E),m(r,T,E),t(T,ie),t(T,ne),t(ne,V),t(V,me),t(T,pe),m(r,te,E),z(se,r,E),le=!0},p(r,E){const ce={};E&2&&(ce.$$scope={dirty:E,ctx:r}),I.$set(ce)},i(r){le||(C(x.$$.fragment,r),C(I.$$.fragment,r),C(O.$$.fragment,r),C(K.$$.fragment,r),C(ee.$$.fragment,r),C(se.$$.fragment,r),le=!0)},o(r){L(x.$$.fragment,r),L(I.$$.fragment,r),L(O.$$.fragment,r),L(K.$$.fragment,r),L(ee.$$.fragment,r),L(se.$$.fragment,r),le=!1},d(r){r&&a(s),r&&a(j),P(x,r),r&&a(M),P(I,r),r&&a(G),r&&a(A),r&&a(J),P(O,r),r&&a(S),r&&a(R),r&&a(W),P(K,r),r&&a(X),r&&a(H),r&&a(Z),P(ee,r),r&&a(h),r&&a(T),r&&a(te),P(se,r)}}}function zn(Q){let s,w;return s=new At({props:{$$slots:{default:[Dn]},$$scope:{ctx:Q}}}),{c(){q(s.$$.fragment)},l(o){D(s.$$.fragment,o)},m(o,_){z(s,o,_),w=!0},p(o,_){const b={};_&2&&(b.$$scope={dirty:_,ctx:o}),s.$set(b)},i(o){w||(C(s.$$.fragment,o),w=!0)},o(o){L(s.$$.fragment,o),w=!1},d(o){P(s,o)}}}function Cn(Q){let s,w,o,_,b,k,v,F;return{c(){s=i("p"),w=l(`For a more in-depth example of how to fine-tune a model for causal language modeling, take a look at the corresponding
`),o=i("a"),_=l("PyTorch notebook"),b=l(`
or `),k=i("a"),v=l("TensorFlow notebook"),F=l("."),this.h()},l(j){s=p(j,"P",{});var x=f(s);w=n(x,`For a more in-depth example of how to fine-tune a model for causal language modeling, take a look at the corresponding
`),o=p(x,"A",{href:!0,rel:!0});var M=f(o);_=n(M,"PyTorch notebook"),M.forEach(a),b=n(x,`
or `),k=p(x,"A",{href:!0,rel:!0});var I=f(k);v=n(I,"TensorFlow notebook"),I.forEach(a),F=n(x,"."),x.forEach(a),this.h()},h(){g(o,"href","https://colab.research.google.com/github/huggingface/notebooks/blob/main/examples/language_modeling.ipynb"),g(o,"rel","nofollow"),g(k,"href","https://colab.research.google.com/github/huggingface/notebooks/blob/main/examples/language_modeling-tf.ipynb"),g(k,"rel","nofollow")},m(j,x){m(j,s,x),t(s,w),t(s,o),t(o,_),t(s,b),t(s,k),t(k,v),t(s,F)},d(j){j&&a(s)}}}function Ln(Q){let s,w,o,_,b,k,v,F,j,x,M,I,G,A,N,J,O,S,R,$,y,Y,B,W,K,X,H,fe,U,oe,he,re,Z,ee,h,T,ie,ne,V,me,pe,te,se,le,r,E,ce,ve,we,_e,be,ue,xe,je,$e,Ee,ke,ra,wt,Za,oa,Xe,ia,ye,es,Mt,ts,as,Ft,ss,ls,qt,ns,rs,pa,De,Ie,Dt,Ze,os,zt,is,fa,et,ma,Oe,ps,Ct,fs,ms,ha,tt,ua,at,ca,jt,hs,da,st,ga,Te,us,Lt,cs,ds,lt,Pt,gs,_s,_a,nt,$a,Ae,$s,It,ks,ws,Ot,js,ys,ka,yt,vs,wa,rt,ja,de,bs,ot,xs,Es,Rt,Ts,As,St,Ms,Fs,Nt,qs,Ds,ya,it,va,vt,zs,ba,Re,Gt,Cs,Ls,pt,Ps,Bt,Is,Os,xa,ft,Ea,Se,Rs,Wt,Ss,Ns,Ta,mt,Aa,ge,Gs,bt,Bs,Ws,Ht,Hs,Ys,Yt,Us,Js,Ut,Ks,Qs,Ma,Ne,Fa,ze,Ge,Jt,ht,Vs,Kt,Xs,qa,Be,Zs,ut,el,tl,Da,Ce,We,Qt,ct,al,Vt,sl,za,He,Ca,Le,Ye,Xt,dt,ll,Zt,nl,La,Ue,rl,gt,ol,il,Pa,Pe,Je,ea,_t,pl,ta,fl,Ia,Ke,Oa,Qe,Ra;return k=new kt({}),A=new Xa({props:{id:"Vpjb1lu0MDk"}}),R=new Xa({props:{id:"mqElG5QJWUg"}}),V=new Tt({props:{$$slots:{default:[_n]},$$scope:{ctx:Q}}}),le=new kt({}),ue=new ae({props:{code:`from datasets import load_dataset

eli5 = load_dataset("eli5", split="train_asks[:5000]")`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> datasets <span class="hljs-keyword">import</span> load_dataset

<span class="hljs-meta">&gt;&gt;&gt; </span>eli5 = load_dataset(<span class="hljs-string">&quot;eli5&quot;</span>, split=<span class="hljs-string">&quot;train_asks[:5000]&quot;</span>)`}}),ke=new ae({props:{code:"eli5 = eli5.train_test_split(test_size=0.2)",highlighted:'eli5 = eli5.train_test_split(test_size=<span class="hljs-number">0.2</span>)'}}),Xe=new ae({props:{code:'eli5["train"][0]',highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>eli5[<span class="hljs-string">&quot;train&quot;</span>][<span class="hljs-number">0</span>]
{<span class="hljs-string">&#x27;answers&#x27;</span>: {<span class="hljs-string">&#x27;a_id&#x27;</span>: [<span class="hljs-string">&#x27;c3d1aib&#x27;</span>, <span class="hljs-string">&#x27;c3d4lya&#x27;</span>],
  <span class="hljs-string">&#x27;score&#x27;</span>: [<span class="hljs-number">6</span>, <span class="hljs-number">3</span>],
  <span class="hljs-string">&#x27;text&#x27;</span>: [<span class="hljs-string">&quot;The velocity needed to remain in orbit is equal to the square root of Newton&#x27;s constant times the mass of earth divided by the distance from the center of the earth. I don&#x27;t know the altitude of that specific mission, but they&#x27;re usually around 300 km. That means he&#x27;s going 7-8 km/s.\\n\\nIn space there are no other forces acting on either the shuttle or the guy, so they stay in the same position relative to each other. If he were to become unable to return to the ship, he would presumably run out of oxygen, or slowly fall into the atmosphere and burn up.&quot;</span>,
   <span class="hljs-string">&quot;Hope you don&#x27;t mind me asking another question, but why aren&#x27;t there any stars visible in this photo?&quot;</span>]},
 <span class="hljs-string">&#x27;answers_urls&#x27;</span>: {<span class="hljs-string">&#x27;url&#x27;</span>: []},
 <span class="hljs-string">&#x27;document&#x27;</span>: <span class="hljs-string">&#x27;&#x27;</span>,
 <span class="hljs-string">&#x27;q_id&#x27;</span>: <span class="hljs-string">&#x27;nyxfp&#x27;</span>,
 <span class="hljs-string">&#x27;selftext&#x27;</span>: <span class="hljs-string">&#x27;_URL_0_\\n\\nThis was on the front page earlier and I have a few questions about it. Is it possible to calculate how fast the astronaut would be orbiting the earth? Also how does he stay close to the shuttle so that he can return safely, i.e is he orbiting at the same speed and can therefore stay next to it? And finally if his propulsion system failed, would he eventually re-enter the atmosphere and presumably die?&#x27;</span>,
 <span class="hljs-string">&#x27;selftext_urls&#x27;</span>: {<span class="hljs-string">&#x27;url&#x27;</span>: [<span class="hljs-string">&#x27;http://apod.nasa.gov/apod/image/1201/freeflyer_nasa_3000.jpg&#x27;</span>]},
 <span class="hljs-string">&#x27;subreddit&#x27;</span>: <span class="hljs-string">&#x27;askscience&#x27;</span>,
 <span class="hljs-string">&#x27;title&#x27;</span>: <span class="hljs-string">&#x27;Few questions about this space walk photograph.&#x27;</span>,
 <span class="hljs-string">&#x27;title_urls&#x27;</span>: {<span class="hljs-string">&#x27;url&#x27;</span>: []}}`}}),Ze=new kt({}),et=new Xa({props:{id:"ma1TrR7gE7I"}}),tt=new ae({props:{code:`from transformers import AutoTokenizer

tokenizer = AutoTokenizer.from_pretrained("distilgpt2")`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoTokenizer

<span class="hljs-meta">&gt;&gt;&gt; </span>tokenizer = AutoTokenizer.from_pretrained(<span class="hljs-string">&quot;distilgpt2&quot;</span>)`}}),at=new Xa({props:{id:"8PmhEIXhBvI"}}),st=new ae({props:{code:`from transformers import AutoTokenizer

tokenizer = AutoTokenizer.from_pretrained("distilroberta-base")`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoTokenizer

<span class="hljs-meta">&gt;&gt;&gt; </span>tokenizer = AutoTokenizer.from_pretrained(<span class="hljs-string">&quot;distilroberta-base&quot;</span>)`}}),nt=new ae({props:{code:`eli5 = eli5.flatten()
eli5["train"][0]`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>eli5 = eli5.flatten()
<span class="hljs-meta">&gt;&gt;&gt; </span>eli5[<span class="hljs-string">&quot;train&quot;</span>][<span class="hljs-number">0</span>]
{<span class="hljs-string">&#x27;answers.a_id&#x27;</span>: [<span class="hljs-string">&#x27;c3d1aib&#x27;</span>, <span class="hljs-string">&#x27;c3d4lya&#x27;</span>],
 <span class="hljs-string">&#x27;answers.score&#x27;</span>: [<span class="hljs-number">6</span>, <span class="hljs-number">3</span>],
 <span class="hljs-string">&#x27;answers.text&#x27;</span>: [<span class="hljs-string">&quot;The velocity needed to remain in orbit is equal to the square root of Newton&#x27;s constant times the mass of earth divided by the distance from the center of the earth. I don&#x27;t know the altitude of that specific mission, but they&#x27;re usually around 300 km. That means he&#x27;s going 7-8 km/s.\\n\\nIn space there are no other forces acting on either the shuttle or the guy, so they stay in the same position relative to each other. If he were to become unable to return to the ship, he would presumably run out of oxygen, or slowly fall into the atmosphere and burn up.&quot;</span>,
  <span class="hljs-string">&quot;Hope you don&#x27;t mind me asking another question, but why aren&#x27;t there any stars visible in this photo?&quot;</span>],
 <span class="hljs-string">&#x27;answers_urls.url&#x27;</span>: [],
 <span class="hljs-string">&#x27;document&#x27;</span>: <span class="hljs-string">&#x27;&#x27;</span>,
 <span class="hljs-string">&#x27;q_id&#x27;</span>: <span class="hljs-string">&#x27;nyxfp&#x27;</span>,
 <span class="hljs-string">&#x27;selftext&#x27;</span>: <span class="hljs-string">&#x27;_URL_0_\\n\\nThis was on the front page earlier and I have a few questions about it. Is it possible to calculate how fast the astronaut would be orbiting the earth? Also how does he stay close to the shuttle so that he can return safely, i.e is he orbiting at the same speed and can therefore stay next to it? And finally if his propulsion system failed, would he eventually re-enter the atmosphere and presumably die?&#x27;</span>,
 <span class="hljs-string">&#x27;selftext_urls.url&#x27;</span>: [<span class="hljs-string">&#x27;http://apod.nasa.gov/apod/image/1201/freeflyer_nasa_3000.jpg&#x27;</span>],
 <span class="hljs-string">&#x27;subreddit&#x27;</span>: <span class="hljs-string">&#x27;askscience&#x27;</span>,
 <span class="hljs-string">&#x27;title&#x27;</span>: <span class="hljs-string">&#x27;Few questions about this space walk photograph.&#x27;</span>,
 <span class="hljs-string">&#x27;title_urls.url&#x27;</span>: []}`}}),rt=new ae({props:{code:`def preprocess_function(examples):
    return tokenizer([" ".join(x) for x in examples["answers.text"]], truncation=True)`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">def</span> <span class="hljs-title function_">preprocess_function</span>(<span class="hljs-params">examples</span>):
<span class="hljs-meta">... </span>    <span class="hljs-keyword">return</span> tokenizer([<span class="hljs-string">&quot; &quot;</span>.join(x) <span class="hljs-keyword">for</span> x <span class="hljs-keyword">in</span> examples[<span class="hljs-string">&quot;answers.text&quot;</span>]], truncation=<span class="hljs-literal">True</span>)`}}),it=new ae({props:{code:`tokenized_eli5 = eli5.map(
    preprocess_function,
    batched=True,
    num_proc=4,
    remove_columns=eli5["train"].column_names,
)`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>tokenized_eli5 = eli5.<span class="hljs-built_in">map</span>(
<span class="hljs-meta">... </span>    preprocess_function,
<span class="hljs-meta">... </span>    batched=<span class="hljs-literal">True</span>,
<span class="hljs-meta">... </span>    num_proc=<span class="hljs-number">4</span>,
<span class="hljs-meta">... </span>    remove_columns=eli5[<span class="hljs-string">&quot;train&quot;</span>].column_names,
<span class="hljs-meta">... </span>)`}}),ft=new ae({props:{code:`block_size = 128


def group_texts(examples):
    concatenated_examples = {k: sum(examples[k], []) for k in examples.keys()}
    total_length = len(concatenated_examples[list(examples.keys())[0]])
    total_length = (total_length // block_size) * block_size
    result = {
        k: [t[i : i + block_size] for i in range(0, total_length, block_size)]
        for k, t in concatenated_examples.items()
    }
    result["labels"] = result["input_ids"].copy()
    return result`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>block_size = <span class="hljs-number">128</span>


<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">def</span> <span class="hljs-title function_">group_texts</span>(<span class="hljs-params">examples</span>):
<span class="hljs-meta">... </span>    concatenated_examples = {k: <span class="hljs-built_in">sum</span>(examples[k], []) <span class="hljs-keyword">for</span> k <span class="hljs-keyword">in</span> examples.keys()}
<span class="hljs-meta">... </span>    total_length = <span class="hljs-built_in">len</span>(concatenated_examples[<span class="hljs-built_in">list</span>(examples.keys())[<span class="hljs-number">0</span>]])
<span class="hljs-meta">... </span>    total_length = (total_length // block_size) * block_size
<span class="hljs-meta">... </span>    result = {
<span class="hljs-meta">... </span>        k: [t[i : i + block_size] <span class="hljs-keyword">for</span> i <span class="hljs-keyword">in</span> <span class="hljs-built_in">range</span>(<span class="hljs-number">0</span>, total_length, block_size)]
<span class="hljs-meta">... </span>        <span class="hljs-keyword">for</span> k, t <span class="hljs-keyword">in</span> concatenated_examples.items()
<span class="hljs-meta">... </span>    }
<span class="hljs-meta">... </span>    result[<span class="hljs-string">&quot;labels&quot;</span>] = result[<span class="hljs-string">&quot;input_ids&quot;</span>].copy()
<span class="hljs-meta">... </span>    <span class="hljs-keyword">return</span> result`}}),mt=new ae({props:{code:"lm_dataset = tokenized_eli5.map(group_texts, batched=True, num_proc=4)",highlighted:'<span class="hljs-meta">&gt;&gt;&gt; </span>lm_dataset = tokenized_eli5.<span class="hljs-built_in">map</span>(group_texts, batched=<span class="hljs-literal">True</span>, num_proc=<span class="hljs-number">4</span>)'}}),Ne=new ml({props:{pytorch:!0,tensorflow:!0,jax:!1,$$slots:{tensorflow:[jn],pytorch:[kn]},$$scope:{ctx:Q}}}),ht=new kt({}),ct=new kt({}),He=new ml({props:{pytorch:!0,tensorflow:!0,jax:!1,$$slots:{tensorflow:[Tn],pytorch:[bn]},$$scope:{ctx:Q}}}),dt=new kt({}),_t=new kt({}),Ke=new ml({props:{pytorch:!0,tensorflow:!0,jax:!1,$$slots:{tensorflow:[zn],pytorch:[Fn]},$$scope:{ctx:Q}}}),Qe=new Tt({props:{$$slots:{default:[Cn]},$$scope:{ctx:Q}}}),{c(){s=i("meta"),w=c(),o=i("h1"),_=i("a"),b=i("span"),q(k.$$.fragment),v=c(),F=i("span"),j=l("Language modeling"),x=c(),M=i("p"),I=l("Language modeling predicts words in a sentence. There are two forms of language modeling."),G=c(),q(A.$$.fragment),N=c(),J=i("p"),O=l("Causal language modeling predicts the next token in a sequence of tokens, and the model can only attend to tokens on the left."),S=c(),q(R.$$.fragment),$=c(),y=i("p"),Y=l("Masked language modeling predicts a masked token in a sequence, and the model can attend to tokens bidirectionally."),B=c(),W=i("p"),K=l("This guide will show you how to fine-tune "),X=i("a"),H=l("DistilGPT2"),fe=l(" for causal language modeling and "),U=i("a"),oe=l("DistilRoBERTa"),he=l(" for masked language modeling on the "),re=i("a"),Z=l("r/askscience"),ee=l(" subset of the "),h=i("a"),T=l("ELI5"),ie=l(" dataset."),ne=c(),q(V.$$.fragment),me=c(),pe=i("h2"),te=i("a"),se=i("span"),q(le.$$.fragment),r=c(),E=i("span"),ce=l("Load ELI5 dataset"),ve=c(),we=i("p"),_e=l("Load only the first 5000 rows of the ELI5 dataset from the \u{1F917} Datasets library since it is pretty large:"),be=c(),q(ue.$$.fragment),xe=c(),je=i("p"),$e=l("Split this dataset into a train and test set:"),Ee=c(),q(ke.$$.fragment),ra=c(),wt=i("p"),Za=l("Then take a look at an example:"),oa=c(),q(Xe.$$.fragment),ia=c(),ye=i("p"),es=l("Notice "),Mt=i("code"),ts=l("text"),as=l(" is a subfield nested inside the "),Ft=i("code"),ss=l("answers"),ls=l(" dictionary. When you preprocess the dataset, you will need to extract the "),qt=i("code"),ns=l("text"),rs=l(" subfield into a separate column."),pa=c(),De=i("h2"),Ie=i("a"),Dt=i("span"),q(Ze.$$.fragment),os=c(),zt=i("span"),is=l("Preprocess"),fa=c(),q(et.$$.fragment),ma=c(),Oe=i("p"),ps=l("For causal language modeling, load the DistilGPT2 tokenizer to process the "),Ct=i("code"),fs=l("text"),ms=l(" subfield:"),ha=c(),q(tt.$$.fragment),ua=c(),q(at.$$.fragment),ca=c(),jt=i("p"),hs=l("For masked language modeling, load the DistilRoBERTa tokenizer instead:"),da=c(),q(st.$$.fragment),ga=c(),Te=i("p"),us=l("Extract the "),Lt=i("code"),cs=l("text"),ds=l(" subfield from its nested structure with the "),lt=i("a"),Pt=i("code"),gs=l("flatten"),_s=l(" method:"),_a=c(),q(nt.$$.fragment),$a=c(),Ae=i("p"),$s=l("Each subfield is now a separate column as indicated by the "),It=i("code"),ks=l("answers"),ws=l(" prefix. Notice that "),Ot=i("code"),js=l("answers.text"),ys=l(" is a list. Instead of tokenizing each sentence separately, convert the list to a string to jointly tokenize them."),ka=c(),yt=i("p"),vs=l("Here is how you can create a preprocessing function to convert the list to a string and truncate sequences to be no longer than DistilGPT2\u2019s maximum input length:"),wa=c(),q(rt.$$.fragment),ja=c(),de=i("p"),bs=l("Use \u{1F917} Datasets "),ot=i("a"),xs=l("map"),Es=l(" function to apply the preprocessing function over the entire dataset. You can speed up the "),Rt=i("code"),Ts=l("map"),As=l(" function by setting "),St=i("code"),Ms=l("batched=True"),Fs=l(" to process multiple elements of the dataset at once and increasing the number of processes with "),Nt=i("code"),qs=l("num_proc"),Ds=l(". Remove the columns you don\u2019t need:"),ya=c(),q(it.$$.fragment),va=c(),vt=i("p"),zs=l("Now you need a second preprocessing function to capture text truncated from any lengthy examples to prevent loss of information. This preprocessing function should:"),ba=c(),Re=i("ul"),Gt=i("li"),Cs=l("Concatenate all the text."),Ls=c(),pt=i("li"),Ps=l("Split the concatenated text into smaller chunks defined by "),Bt=i("code"),Is=l("block_size"),Os=l("."),xa=c(),q(ft.$$.fragment),Ea=c(),Se=i("p"),Rs=l("Apply the "),Wt=i("code"),Ss=l("group_texts"),Ns=l(" function over the entire dataset:"),Ta=c(),q(mt.$$.fragment),Aa=c(),ge=i("p"),Gs=l("For causal language modeling, use "),bt=i("a"),Bs=l("DataCollatorForLanguageModeling"),Ws=l(" to create a batch of examples. It will also "),Ht=i("em"),Hs=l("dynamically pad"),Ys=l(" your text to the length of the longest element in its batch, so they are a uniform length. While it is possible to pad your text in the "),Yt=i("code"),Us=l("tokenizer"),Js=l(" function by setting "),Ut=i("code"),Ks=l("padding=True"),Qs=l(", dynamic padding is more efficient."),Ma=c(),q(Ne.$$.fragment),Fa=c(),ze=i("h2"),Ge=i("a"),Jt=i("span"),q(ht.$$.fragment),Vs=c(),Kt=i("span"),Xs=l("Causal language modeling"),qa=c(),Be=i("p"),Zs=l("Causal language modeling is frequently used for text generation. This section shows you how to fine-tune "),ut=i("a"),el=l("DistilGPT2"),tl=l(" to generate new text."),Da=c(),Ce=i("h3"),We=i("a"),Qt=i("span"),q(ct.$$.fragment),al=c(),Vt=i("span"),sl=l("Train"),za=c(),q(He.$$.fragment),Ca=c(),Le=i("h2"),Ye=i("a"),Xt=i("span"),q(dt.$$.fragment),ll=c(),Zt=i("span"),nl=l("Masked language modeling"),La=c(),Ue=i("p"),rl=l("Masked language modeling is also known as a fill-mask task because it predicts a masked token in a sequence. Models for masked language modeling require a good contextual understanding of an entire sequence instead of only the left context. This section shows you how to fine-tune "),gt=i("a"),ol=l("DistilRoBERTa"),il=l(" to predict a masked word."),Pa=c(),Pe=i("h3"),Je=i("a"),ea=i("span"),q(_t.$$.fragment),pl=c(),ta=i("span"),fl=l("Train"),Ia=c(),q(Ke.$$.fragment),Oa=c(),q(Qe.$$.fragment),this.h()},l(e){const u=dn('[data-svelte="svelte-1phssyn"]',document.head);s=p(u,"META",{name:!0,content:!0}),u.forEach(a),w=d(e),o=p(e,"H1",{class:!0});var $t=f(o);_=p($t,"A",{id:!0,class:!0,href:!0});var aa=f(_);b=p(aa,"SPAN",{});var sa=f(b);D(k.$$.fragment,sa),sa.forEach(a),aa.forEach(a),v=d($t),F=p($t,"SPAN",{});var la=f(F);j=n(la,"Language modeling"),la.forEach(a),$t.forEach(a),x=d(e),M=p(e,"P",{});var na=f(M);I=n(na,"Language modeling predicts words in a sentence. There are two forms of language modeling."),na.forEach(a),G=d(e),D(A.$$.fragment,e),N=d(e),J=p(e,"P",{});var hl=f(J);O=n(hl,"Causal language modeling predicts the next token in a sequence of tokens, and the model can only attend to tokens on the left."),hl.forEach(a),S=d(e),D(R.$$.fragment,e),$=d(e),y=p(e,"P",{});var ul=f(y);Y=n(ul,"Masked language modeling predicts a masked token in a sequence, and the model can attend to tokens bidirectionally."),ul.forEach(a),B=d(e),W=p(e,"P",{});var Me=f(W);K=n(Me,"This guide will show you how to fine-tune "),X=p(Me,"A",{href:!0,rel:!0});var cl=f(X);H=n(cl,"DistilGPT2"),cl.forEach(a),fe=n(Me," for causal language modeling and "),U=p(Me,"A",{href:!0,rel:!0});var dl=f(U);oe=n(dl,"DistilRoBERTa"),dl.forEach(a),he=n(Me," for masked language modeling on the "),re=p(Me,"A",{href:!0,rel:!0});var gl=f(re);Z=n(gl,"r/askscience"),gl.forEach(a),ee=n(Me," subset of the "),h=p(Me,"A",{href:!0,rel:!0});var _l=f(h);T=n(_l,"ELI5"),_l.forEach(a),ie=n(Me," dataset."),Me.forEach(a),ne=d(e),D(V.$$.fragment,e),me=d(e),pe=p(e,"H2",{class:!0});var Sa=f(pe);te=p(Sa,"A",{id:!0,class:!0,href:!0});var $l=f(te);se=p($l,"SPAN",{});var kl=f(se);D(le.$$.fragment,kl),kl.forEach(a),$l.forEach(a),r=d(Sa),E=p(Sa,"SPAN",{});var wl=f(E);ce=n(wl,"Load ELI5 dataset"),wl.forEach(a),Sa.forEach(a),ve=d(e),we=p(e,"P",{});var jl=f(we);_e=n(jl,"Load only the first 5000 rows of the ELI5 dataset from the \u{1F917} Datasets library since it is pretty large:"),jl.forEach(a),be=d(e),D(ue.$$.fragment,e),xe=d(e),je=p(e,"P",{});var yl=f(je);$e=n(yl,"Split this dataset into a train and test set:"),yl.forEach(a),Ee=d(e),D(ke.$$.fragment,e),ra=d(e),wt=p(e,"P",{});var vl=f(wt);Za=n(vl,"Then take a look at an example:"),vl.forEach(a),oa=d(e),D(Xe.$$.fragment,e),ia=d(e),ye=p(e,"P",{});var Ve=f(ye);es=n(Ve,"Notice "),Mt=p(Ve,"CODE",{});var bl=f(Mt);ts=n(bl,"text"),bl.forEach(a),as=n(Ve," is a subfield nested inside the "),Ft=p(Ve,"CODE",{});var xl=f(Ft);ss=n(xl,"answers"),xl.forEach(a),ls=n(Ve," dictionary. When you preprocess the dataset, you will need to extract the "),qt=p(Ve,"CODE",{});var El=f(qt);ns=n(El,"text"),El.forEach(a),rs=n(Ve," subfield into a separate column."),Ve.forEach(a),pa=d(e),De=p(e,"H2",{class:!0});var Na=f(De);Ie=p(Na,"A",{id:!0,class:!0,href:!0});var Tl=f(Ie);Dt=p(Tl,"SPAN",{});var Al=f(Dt);D(Ze.$$.fragment,Al),Al.forEach(a),Tl.forEach(a),os=d(Na),zt=p(Na,"SPAN",{});var Ml=f(zt);is=n(Ml,"Preprocess"),Ml.forEach(a),Na.forEach(a),fa=d(e),D(et.$$.fragment,e),ma=d(e),Oe=p(e,"P",{});var Ga=f(Oe);ps=n(Ga,"For causal language modeling, load the DistilGPT2 tokenizer to process the "),Ct=p(Ga,"CODE",{});var Fl=f(Ct);fs=n(Fl,"text"),Fl.forEach(a),ms=n(Ga," subfield:"),Ga.forEach(a),ha=d(e),D(tt.$$.fragment,e),ua=d(e),D(at.$$.fragment,e),ca=d(e),jt=p(e,"P",{});var ql=f(jt);hs=n(ql,"For masked language modeling, load the DistilRoBERTa tokenizer instead:"),ql.forEach(a),da=d(e),D(st.$$.fragment,e),ga=d(e),Te=p(e,"P",{});var xt=f(Te);us=n(xt,"Extract the "),Lt=p(xt,"CODE",{});var Dl=f(Lt);cs=n(Dl,"text"),Dl.forEach(a),ds=n(xt," subfield from its nested structure with the "),lt=p(xt,"A",{href:!0,rel:!0});var zl=f(lt);Pt=p(zl,"CODE",{});var Cl=f(Pt);gs=n(Cl,"flatten"),Cl.forEach(a),zl.forEach(a),_s=n(xt," method:"),xt.forEach(a),_a=d(e),D(nt.$$.fragment,e),$a=d(e),Ae=p(e,"P",{});var Et=f(Ae);$s=n(Et,"Each subfield is now a separate column as indicated by the "),It=p(Et,"CODE",{});var Ll=f(It);ks=n(Ll,"answers"),Ll.forEach(a),ws=n(Et," prefix. Notice that "),Ot=p(Et,"CODE",{});var Pl=f(Ot);js=n(Pl,"answers.text"),Pl.forEach(a),ys=n(Et," is a list. Instead of tokenizing each sentence separately, convert the list to a string to jointly tokenize them."),Et.forEach(a),ka=d(e),yt=p(e,"P",{});var Il=f(yt);vs=n(Il,"Here is how you can create a preprocessing function to convert the list to a string and truncate sequences to be no longer than DistilGPT2\u2019s maximum input length:"),Il.forEach(a),wa=d(e),D(rt.$$.fragment,e),ja=d(e),de=p(e,"P",{});var Fe=f(de);bs=n(Fe,"Use \u{1F917} Datasets "),ot=p(Fe,"A",{href:!0,rel:!0});var Ol=f(ot);xs=n(Ol,"map"),Ol.forEach(a),Es=n(Fe," function to apply the preprocessing function over the entire dataset. You can speed up the "),Rt=p(Fe,"CODE",{});var Rl=f(Rt);Ts=n(Rl,"map"),Rl.forEach(a),As=n(Fe," function by setting "),St=p(Fe,"CODE",{});var Sl=f(St);Ms=n(Sl,"batched=True"),Sl.forEach(a),Fs=n(Fe," to process multiple elements of the dataset at once and increasing the number of processes with "),Nt=p(Fe,"CODE",{});var Nl=f(Nt);qs=n(Nl,"num_proc"),Nl.forEach(a),Ds=n(Fe,". Remove the columns you don\u2019t need:"),Fe.forEach(a),ya=d(e),D(it.$$.fragment,e),va=d(e),vt=p(e,"P",{});var Gl=f(vt);zs=n(Gl,"Now you need a second preprocessing function to capture text truncated from any lengthy examples to prevent loss of information. This preprocessing function should:"),Gl.forEach(a),ba=d(e),Re=p(e,"UL",{});var Ba=f(Re);Gt=p(Ba,"LI",{});var Bl=f(Gt);Cs=n(Bl,"Concatenate all the text."),Bl.forEach(a),Ls=d(Ba),pt=p(Ba,"LI",{});var Wa=f(pt);Ps=n(Wa,"Split the concatenated text into smaller chunks defined by "),Bt=p(Wa,"CODE",{});var Wl=f(Bt);Is=n(Wl,"block_size"),Wl.forEach(a),Os=n(Wa,"."),Wa.forEach(a),Ba.forEach(a),xa=d(e),D(ft.$$.fragment,e),Ea=d(e),Se=p(e,"P",{});var Ha=f(Se);Rs=n(Ha,"Apply the "),Wt=p(Ha,"CODE",{});var Hl=f(Wt);Ss=n(Hl,"group_texts"),Hl.forEach(a),Ns=n(Ha," function over the entire dataset:"),Ha.forEach(a),Ta=d(e),D(mt.$$.fragment,e),Aa=d(e),ge=p(e,"P",{});var qe=f(ge);Gs=n(qe,"For causal language modeling, use "),bt=p(qe,"A",{href:!0});var Yl=f(bt);Bs=n(Yl,"DataCollatorForLanguageModeling"),Yl.forEach(a),Ws=n(qe," to create a batch of examples. It will also "),Ht=p(qe,"EM",{});var Ul=f(Ht);Hs=n(Ul,"dynamically pad"),Ul.forEach(a),Ys=n(qe," your text to the length of the longest element in its batch, so they are a uniform length. While it is possible to pad your text in the "),Yt=p(qe,"CODE",{});var Jl=f(Yt);Us=n(Jl,"tokenizer"),Jl.forEach(a),Js=n(qe," function by setting "),Ut=p(qe,"CODE",{});var Kl=f(Ut);Ks=n(Kl,"padding=True"),Kl.forEach(a),Qs=n(qe,", dynamic padding is more efficient."),qe.forEach(a),Ma=d(e),D(Ne.$$.fragment,e),Fa=d(e),ze=p(e,"H2",{class:!0});var Ya=f(ze);Ge=p(Ya,"A",{id:!0,class:!0,href:!0});var Ql=f(Ge);Jt=p(Ql,"SPAN",{});var Vl=f(Jt);D(ht.$$.fragment,Vl),Vl.forEach(a),Ql.forEach(a),Vs=d(Ya),Kt=p(Ya,"SPAN",{});var Xl=f(Kt);Xs=n(Xl,"Causal language modeling"),Xl.forEach(a),Ya.forEach(a),qa=d(e),Be=p(e,"P",{});var Ua=f(Be);Zs=n(Ua,"Causal language modeling is frequently used for text generation. This section shows you how to fine-tune "),ut=p(Ua,"A",{href:!0,rel:!0});var Zl=f(ut);el=n(Zl,"DistilGPT2"),Zl.forEach(a),tl=n(Ua," to generate new text."),Ua.forEach(a),Da=d(e),Ce=p(e,"H3",{class:!0});var Ja=f(Ce);We=p(Ja,"A",{id:!0,class:!0,href:!0});var en=f(We);Qt=p(en,"SPAN",{});var tn=f(Qt);D(ct.$$.fragment,tn),tn.forEach(a),en.forEach(a),al=d(Ja),Vt=p(Ja,"SPAN",{});var an=f(Vt);sl=n(an,"Train"),an.forEach(a),Ja.forEach(a),za=d(e),D(He.$$.fragment,e),Ca=d(e),Le=p(e,"H2",{class:!0});var Ka=f(Le);Ye=p(Ka,"A",{id:!0,class:!0,href:!0});var sn=f(Ye);Xt=p(sn,"SPAN",{});var ln=f(Xt);D(dt.$$.fragment,ln),ln.forEach(a),sn.forEach(a),ll=d(Ka),Zt=p(Ka,"SPAN",{});var nn=f(Zt);nl=n(nn,"Masked language modeling"),nn.forEach(a),Ka.forEach(a),La=d(e),Ue=p(e,"P",{});var Qa=f(Ue);rl=n(Qa,"Masked language modeling is also known as a fill-mask task because it predicts a masked token in a sequence. Models for masked language modeling require a good contextual understanding of an entire sequence instead of only the left context. This section shows you how to fine-tune "),gt=p(Qa,"A",{href:!0,rel:!0});var rn=f(gt);ol=n(rn,"DistilRoBERTa"),rn.forEach(a),il=n(Qa," to predict a masked word."),Qa.forEach(a),Pa=d(e),Pe=p(e,"H3",{class:!0});var Va=f(Pe);Je=p(Va,"A",{id:!0,class:!0,href:!0});var on=f(Je);ea=p(on,"SPAN",{});var pn=f(ea);D(_t.$$.fragment,pn),pn.forEach(a),on.forEach(a),pl=d(Va),ta=p(Va,"SPAN",{});var fn=f(ta);fl=n(fn,"Train"),fn.forEach(a),Va.forEach(a),Ia=d(e),D(Ke.$$.fragment,e),Oa=d(e),D(Qe.$$.fragment,e),this.h()},h(){g(s,"name","hf:doc:metadata"),g(s,"content",JSON.stringify(Pn)),g(_,"id","language-modeling"),g(_,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),g(_,"href","#language-modeling"),g(o,"class","relative group"),g(X,"href","https://huggingface.co/distilgpt2"),g(X,"rel","nofollow"),g(U,"href","https://huggingface.co/distilroberta-base"),g(U,"rel","nofollow"),g(re,"href","https://www.reddit.com/r/askscience/"),g(re,"rel","nofollow"),g(h,"href","https://huggingface.co/datasets/eli5"),g(h,"rel","nofollow"),g(te,"id","load-eli5-dataset"),g(te,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),g(te,"href","#load-eli5-dataset"),g(pe,"class","relative group"),g(Ie,"id","preprocess"),g(Ie,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),g(Ie,"href","#preprocess"),g(De,"class","relative group"),g(lt,"href","https://huggingface.co/docs/datasets/process.html#flatten"),g(lt,"rel","nofollow"),g(ot,"href","https://huggingface.co/docs/datasets/main/en/package_reference/main_classes#datasets.Dataset.map"),g(ot,"rel","nofollow"),g(bt,"href","/docs/transformers/pr_19907/en/main_classes/data_collator#transformers.DataCollatorForLanguageModeling"),g(Ge,"id","causal-language-modeling"),g(Ge,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),g(Ge,"href","#causal-language-modeling"),g(ze,"class","relative group"),g(ut,"href","https://huggingface.co/distilgpt2"),g(ut,"rel","nofollow"),g(We,"id","train"),g(We,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),g(We,"href","#train"),g(Ce,"class","relative group"),g(Ye,"id","masked-language-modeling"),g(Ye,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),g(Ye,"href","#masked-language-modeling"),g(Le,"class","relative group"),g(gt,"href","https://huggingface.co/distilroberta-base"),g(gt,"rel","nofollow"),g(Je,"id","train"),g(Je,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),g(Je,"href","#train"),g(Pe,"class","relative group")},m(e,u){t(document.head,s),m(e,w,u),m(e,o,u),t(o,_),t(_,b),z(k,b,null),t(o,v),t(o,F),t(F,j),m(e,x,u),m(e,M,u),t(M,I),m(e,G,u),z(A,e,u),m(e,N,u),m(e,J,u),t(J,O),m(e,S,u),z(R,e,u),m(e,$,u),m(e,y,u),t(y,Y),m(e,B,u),m(e,W,u),t(W,K),t(W,X),t(X,H),t(W,fe),t(W,U),t(U,oe),t(W,he),t(W,re),t(re,Z),t(W,ee),t(W,h),t(h,T),t(W,ie),m(e,ne,u),z(V,e,u),m(e,me,u),m(e,pe,u),t(pe,te),t(te,se),z(le,se,null),t(pe,r),t(pe,E),t(E,ce),m(e,ve,u),m(e,we,u),t(we,_e),m(e,be,u),z(ue,e,u),m(e,xe,u),m(e,je,u),t(je,$e),m(e,Ee,u),z(ke,e,u),m(e,ra,u),m(e,wt,u),t(wt,Za),m(e,oa,u),z(Xe,e,u),m(e,ia,u),m(e,ye,u),t(ye,es),t(ye,Mt),t(Mt,ts),t(ye,as),t(ye,Ft),t(Ft,ss),t(ye,ls),t(ye,qt),t(qt,ns),t(ye,rs),m(e,pa,u),m(e,De,u),t(De,Ie),t(Ie,Dt),z(Ze,Dt,null),t(De,os),t(De,zt),t(zt,is),m(e,fa,u),z(et,e,u),m(e,ma,u),m(e,Oe,u),t(Oe,ps),t(Oe,Ct),t(Ct,fs),t(Oe,ms),m(e,ha,u),z(tt,e,u),m(e,ua,u),z(at,e,u),m(e,ca,u),m(e,jt,u),t(jt,hs),m(e,da,u),z(st,e,u),m(e,ga,u),m(e,Te,u),t(Te,us),t(Te,Lt),t(Lt,cs),t(Te,ds),t(Te,lt),t(lt,Pt),t(Pt,gs),t(Te,_s),m(e,_a,u),z(nt,e,u),m(e,$a,u),m(e,Ae,u),t(Ae,$s),t(Ae,It),t(It,ks),t(Ae,ws),t(Ae,Ot),t(Ot,js),t(Ae,ys),m(e,ka,u),m(e,yt,u),t(yt,vs),m(e,wa,u),z(rt,e,u),m(e,ja,u),m(e,de,u),t(de,bs),t(de,ot),t(ot,xs),t(de,Es),t(de,Rt),t(Rt,Ts),t(de,As),t(de,St),t(St,Ms),t(de,Fs),t(de,Nt),t(Nt,qs),t(de,Ds),m(e,ya,u),z(it,e,u),m(e,va,u),m(e,vt,u),t(vt,zs),m(e,ba,u),m(e,Re,u),t(Re,Gt),t(Gt,Cs),t(Re,Ls),t(Re,pt),t(pt,Ps),t(pt,Bt),t(Bt,Is),t(pt,Os),m(e,xa,u),z(ft,e,u),m(e,Ea,u),m(e,Se,u),t(Se,Rs),t(Se,Wt),t(Wt,Ss),t(Se,Ns),m(e,Ta,u),z(mt,e,u),m(e,Aa,u),m(e,ge,u),t(ge,Gs),t(ge,bt),t(bt,Bs),t(ge,Ws),t(ge,Ht),t(Ht,Hs),t(ge,Ys),t(ge,Yt),t(Yt,Us),t(ge,Js),t(ge,Ut),t(Ut,Ks),t(ge,Qs),m(e,Ma,u),z(Ne,e,u),m(e,Fa,u),m(e,ze,u),t(ze,Ge),t(Ge,Jt),z(ht,Jt,null),t(ze,Vs),t(ze,Kt),t(Kt,Xs),m(e,qa,u),m(e,Be,u),t(Be,Zs),t(Be,ut),t(ut,el),t(Be,tl),m(e,Da,u),m(e,Ce,u),t(Ce,We),t(We,Qt),z(ct,Qt,null),t(Ce,al),t(Ce,Vt),t(Vt,sl),m(e,za,u),z(He,e,u),m(e,Ca,u),m(e,Le,u),t(Le,Ye),t(Ye,Xt),z(dt,Xt,null),t(Le,ll),t(Le,Zt),t(Zt,nl),m(e,La,u),m(e,Ue,u),t(Ue,rl),t(Ue,gt),t(gt,ol),t(Ue,il),m(e,Pa,u),m(e,Pe,u),t(Pe,Je),t(Je,ea),z(_t,ea,null),t(Pe,pl),t(Pe,ta),t(ta,fl),m(e,Ia,u),z(Ke,e,u),m(e,Oa,u),z(Qe,e,u),Ra=!0},p(e,[u]){const $t={};u&2&&($t.$$scope={dirty:u,ctx:e}),V.$set($t);const aa={};u&2&&(aa.$$scope={dirty:u,ctx:e}),Ne.$set(aa);const sa={};u&2&&(sa.$$scope={dirty:u,ctx:e}),He.$set(sa);const la={};u&2&&(la.$$scope={dirty:u,ctx:e}),Ke.$set(la);const na={};u&2&&(na.$$scope={dirty:u,ctx:e}),Qe.$set(na)},i(e){Ra||(C(k.$$.fragment,e),C(A.$$.fragment,e),C(R.$$.fragment,e),C(V.$$.fragment,e),C(le.$$.fragment,e),C(ue.$$.fragment,e),C(ke.$$.fragment,e),C(Xe.$$.fragment,e),C(Ze.$$.fragment,e),C(et.$$.fragment,e),C(tt.$$.fragment,e),C(at.$$.fragment,e),C(st.$$.fragment,e),C(nt.$$.fragment,e),C(rt.$$.fragment,e),C(it.$$.fragment,e),C(ft.$$.fragment,e),C(mt.$$.fragment,e),C(Ne.$$.fragment,e),C(ht.$$.fragment,e),C(ct.$$.fragment,e),C(He.$$.fragment,e),C(dt.$$.fragment,e),C(_t.$$.fragment,e),C(Ke.$$.fragment,e),C(Qe.$$.fragment,e),Ra=!0)},o(e){L(k.$$.fragment,e),L(A.$$.fragment,e),L(R.$$.fragment,e),L(V.$$.fragment,e),L(le.$$.fragment,e),L(ue.$$.fragment,e),L(ke.$$.fragment,e),L(Xe.$$.fragment,e),L(Ze.$$.fragment,e),L(et.$$.fragment,e),L(tt.$$.fragment,e),L(at.$$.fragment,e),L(st.$$.fragment,e),L(nt.$$.fragment,e),L(rt.$$.fragment,e),L(it.$$.fragment,e),L(ft.$$.fragment,e),L(mt.$$.fragment,e),L(Ne.$$.fragment,e),L(ht.$$.fragment,e),L(ct.$$.fragment,e),L(He.$$.fragment,e),L(dt.$$.fragment,e),L(_t.$$.fragment,e),L(Ke.$$.fragment,e),L(Qe.$$.fragment,e),Ra=!1},d(e){a(s),e&&a(w),e&&a(o),P(k),e&&a(x),e&&a(M),e&&a(G),P(A,e),e&&a(N),e&&a(J),e&&a(S),P(R,e),e&&a($),e&&a(y),e&&a(B),e&&a(W),e&&a(ne),P(V,e),e&&a(me),e&&a(pe),P(le),e&&a(ve),e&&a(we),e&&a(be),P(ue,e),e&&a(xe),e&&a(je),e&&a(Ee),P(ke,e),e&&a(ra),e&&a(wt),e&&a(oa),P(Xe,e),e&&a(ia),e&&a(ye),e&&a(pa),e&&a(De),P(Ze),e&&a(fa),P(et,e),e&&a(ma),e&&a(Oe),e&&a(ha),P(tt,e),e&&a(ua),P(at,e),e&&a(ca),e&&a(jt),e&&a(da),P(st,e),e&&a(ga),e&&a(Te),e&&a(_a),P(nt,e),e&&a($a),e&&a(Ae),e&&a(ka),e&&a(yt),e&&a(wa),P(rt,e),e&&a(ja),e&&a(de),e&&a(ya),P(it,e),e&&a(va),e&&a(vt),e&&a(ba),e&&a(Re),e&&a(xa),P(ft,e),e&&a(Ea),e&&a(Se),e&&a(Ta),P(mt,e),e&&a(Aa),e&&a(ge),e&&a(Ma),P(Ne,e),e&&a(Fa),e&&a(ze),P(ht),e&&a(qa),e&&a(Be),e&&a(Da),e&&a(Ce),P(ct),e&&a(za),P(He,e),e&&a(Ca),e&&a(Le),P(dt),e&&a(La),e&&a(Ue),e&&a(Pa),e&&a(Pe),P(_t),e&&a(Ia),P(Ke,e),e&&a(Oa),P(Qe,e)}}}const Pn={local:"language-modeling",sections:[{local:"load-eli5-dataset",title:"Load ELI5 dataset"},{local:"preprocess",title:"Preprocess"},{local:"causal-language-modeling",sections:[{local:"train",title:"Train"}],title:"Causal language modeling"},{local:"masked-language-modeling",sections:[{local:"train",title:"Train"}],title:"Masked language modeling"}],title:"Language modeling"};function In(Q){return gn(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class Wn extends hn{constructor(s){super();un(this,s,In,Ln,cn,{})}}export{Wn as default,Pn as metadata};
