import{S as nf,i as rf,s as of,e as r,k as f,w as g,t as l,M as pf,c as o,d as t,m as h,a as i,x as _,h as n,b as m,F as s,g as p,y as $,q as w,o as k,B as j}from"../../chunks/vendor-4833417e.js";import{T as Es}from"../../chunks/Tip-fffd6df1.js";import{Y as En}from"../../chunks/Youtube-27813aed.js";import{I as fe}from"../../chunks/IconCopyLink-4b81c553.js";import{C as q}from"../../chunks/CodeBlock-6a3d1b46.js";import{C as lf}from"../../chunks/CodeBlockFw-27a176a0.js";import"../../chunks/CopyButton-dacfbfaf.js";function ff(P){let u,E,c,v,x,d,b,A,T,y,F,C,L,W,N,U,Ge,R,Y,he;return{c(){u=r("p"),E=l("You can fine-tune other architectures for language modeling such as "),c=r("a"),v=l("GPT-Neo"),x=l(", "),d=r("a"),b=l("GPT-J"),A=l(", and "),T=r("a"),y=l("BERT"),F=l(", following the same steps presented in this guide!"),C=f(),L=r("p"),W=l("See the text generation "),N=r("a"),U=l("task page"),Ge=l(" and fill mask "),R=r("a"),Y=l("task page"),he=l(" for more information about their associated models, datasets, and metrics."),this.h()},l(M){u=o(M,"P",{});var z=i(u);E=n(z,"You can fine-tune other architectures for language modeling such as "),c=o(z,"A",{href:!0,rel:!0});var He=i(c);v=n(He,"GPT-Neo"),He.forEach(t),x=n(z,", "),d=o(z,"A",{href:!0,rel:!0});var D=i(d);b=n(D,"GPT-J"),D.forEach(t),A=n(z,", and "),T=o(z,"A",{href:!0,rel:!0});var Jt=i(T);y=n(Jt,"BERT"),Jt.forEach(t),F=n(z,", following the same steps presented in this guide!"),z.forEach(t),C=h(M),L=o(M,"P",{});var I=i(L);W=n(I,"See the text generation "),N=o(I,"A",{href:!0,rel:!0});var Kt=i(N);U=n(Kt,"task page"),Kt.forEach(t),Ge=n(I," and fill mask "),R=o(I,"A",{href:!0,rel:!0});var Qt=i(R);Y=n(Qt,"task page"),Qt.forEach(t),he=n(I," for more information about their associated models, datasets, and metrics."),I.forEach(t),this.h()},h(){m(c,"href","https://huggingface.co/EleutherAI/gpt-neo-125M"),m(c,"rel","nofollow"),m(d,"href","https://huggingface.co/EleutherAI/gpt-j-6B"),m(d,"rel","nofollow"),m(T,"href","https://huggingface.co/bert-base-uncased"),m(T,"rel","nofollow"),m(N,"href","https://huggingface.co/tasks/text-generation"),m(N,"rel","nofollow"),m(R,"href","https://huggingface.co/tasks/fill-mask"),m(R,"rel","nofollow")},m(M,z){p(M,u,z),s(u,E),s(u,c),s(c,v),s(u,x),s(u,d),s(d,b),s(u,A),s(u,T),s(T,y),s(u,F),p(M,C,z),p(M,L,z),s(L,W),s(L,N),s(N,U),s(L,Ge),s(L,R),s(R,Y),s(L,he)},d(M){M&&t(u),M&&t(C),M&&t(L)}}}function hf(P){let u,E,c,v,x,d,b,A;return{c(){u=r("p"),E=l("If you aren\u2019t familiar with fine-tuning a model with the "),c=r("a"),v=l("Trainer"),x=l(", take a look at the basic tutorial "),d=r("a"),b=l("here"),A=l("!"),this.h()},l(T){u=o(T,"P",{});var y=i(u);E=n(y,"If you aren\u2019t familiar with fine-tuning a model with the "),c=o(y,"A",{href:!0});var F=i(c);v=n(F,"Trainer"),F.forEach(t),x=n(y,", take a look at the basic tutorial "),d=o(y,"A",{href:!0});var C=i(d);b=n(C,"here"),C.forEach(t),A=n(y,"!"),y.forEach(t),this.h()},h(){m(c,"href","/docs/transformers/pr_15839/en/main_classes/trainer#transformers.Trainer"),m(d,"href","training#finetune-with-trainer")},m(T,y){p(T,u,y),s(u,E),s(u,c),s(c,v),s(u,x),s(u,d),s(d,b),s(u,A)},d(T){T&&t(u)}}}function mf(P){let u,E,c,v,x;return{c(){u=r("p"),E=l("If you aren\u2019t familiar with fine-tuning a model with Keras, take a look at the basic tutorial "),c=r("a"),v=l("here"),x=l("!"),this.h()},l(d){u=o(d,"P",{});var b=i(u);E=n(b,"If you aren\u2019t familiar with fine-tuning a model with Keras, take a look at the basic tutorial "),c=o(b,"A",{href:!0});var A=i(c);v=n(A,"here"),A.forEach(t),x=n(b,"!"),b.forEach(t),this.h()},h(){m(c,"href","training#finetune-with-keras")},m(d,b){p(d,u,b),s(u,E),s(u,c),s(c,v),s(u,x)},d(d){d&&t(u)}}}function uf(P){let u,E,c,v,x,d,b,A;return{c(){u=r("p"),E=l("If you aren\u2019t familiar with fine-tuning a model with the "),c=r("a"),v=l("Trainer"),x=l(", take a look at the basic tutorial "),d=r("a"),b=l("here"),A=l("!"),this.h()},l(T){u=o(T,"P",{});var y=i(u);E=n(y,"If you aren\u2019t familiar with fine-tuning a model with the "),c=o(y,"A",{href:!0});var F=i(c);v=n(F,"Trainer"),F.forEach(t),x=n(y,", take a look at the basic tutorial "),d=o(y,"A",{href:!0});var C=i(d);b=n(C,"here"),C.forEach(t),A=n(y,"!"),y.forEach(t),this.h()},h(){m(c,"href","/docs/transformers/pr_15839/en/main_classes/trainer#transformers.Trainer"),m(d,"href","training#finetune-with-trainer")},m(T,y){p(T,u,y),s(u,E),s(u,c),s(c,v),s(u,x),s(u,d),s(d,b),s(u,A)},d(T){T&&t(u)}}}function cf(P){let u,E,c,v,x;return{c(){u=r("p"),E=l("If you aren\u2019t familiar with fine-tuning a model with Keras, take a look at the basic tutorial "),c=r("a"),v=l("here"),x=l("!"),this.h()},l(d){u=o(d,"P",{});var b=i(u);E=n(b,"If you aren\u2019t familiar with fine-tuning a model with Keras, take a look at the basic tutorial "),c=o(b,"A",{href:!0});var A=i(c);v=n(A,"here"),A.forEach(t),x=n(b,"!"),b.forEach(t),this.h()},h(){m(c,"href","training#finetune-with-keras")},m(d,b){p(d,u,b),s(u,E),s(u,c),s(c,v),s(u,x)},d(d){d&&t(u)}}}function df(P){let u,E,c,v,x,d,b,A;return{c(){u=r("p"),E=l(`For a more in-depth example of how to fine-tune a model for causal language modeling, take a look at the corresponding
`),c=r("a"),v=l("PyTorch notebook"),x=l(`
or `),d=r("a"),b=l("TensorFlow notebook"),A=l("."),this.h()},l(T){u=o(T,"P",{});var y=i(u);E=n(y,`For a more in-depth example of how to fine-tune a model for causal language modeling, take a look at the corresponding
`),c=o(y,"A",{href:!0,rel:!0});var F=i(c);v=n(F,"PyTorch notebook"),F.forEach(t),x=n(y,`
or `),d=o(y,"A",{href:!0,rel:!0});var C=i(d);b=n(C,"TensorFlow notebook"),C.forEach(t),A=n(y,"."),y.forEach(t),this.h()},h(){m(c,"href","https://colab.research.google.com/github/huggingface/notebooks/blob/master/examples/language_modeling.ipynb"),m(c,"rel","nofollow"),m(d,"href","https://colab.research.google.com/github/huggingface/notebooks/blob/master/examples/language_modeling-tf.ipynb"),m(d,"rel","nofollow")},m(T,y){p(T,u,y),s(u,E),s(u,c),s(c,v),s(u,x),s(u,d),s(d,b),s(u,A)},d(T){T&&t(u)}}}function gf(P){let u,E,c,v,x,d,b,A,T,y,F,C,L,W,N,U,Ge,R,Y,he,M,z,He,D,Jt,I,Kt,Qt,We,xn,Tn,Ye,An,qn,Ue,Fn,Mn,ba,me,Ea,se,ue,xs,Je,Cn,Ts,Dn,xa,Vt,zn,Ta,Ke,Aa,Xt,Pn,qa,Qe,Fa,Zt,Ln,Ma,Ve,Ca,B,In,As,Sn,On,qs,Nn,Rn,Fs,Bn,Gn,Da,ae,ce,Ms,Xe,Hn,Cs,Wn,za,Ze,Pa,de,Yn,Ds,Un,Jn,La,et,Ia,tt,Sa,es,Kn,Oa,st,Na,J,Qn,zs,Vn,Xn,at,Ps,Zn,er,Ra,lt,Ba,K,tr,Ls,sr,ar,Is,lr,nr,Ga,ts,rr,Ha,nt,Wa,S,or,rt,Ss,ir,pr,Os,fr,hr,Ns,mr,ur,Rs,cr,dr,Ya,ot,Ua,ss,gr,Ja,ge,Bs,_r,$r,it,wr,Gs,kr,jr,Ka,pt,Qa,_e,vr,Hs,yr,br,Va,ft,Xa,O,Er,as,xr,Tr,Ws,Ar,qr,Ys,Fr,Mr,Us,Cr,Dr,Za,$e,zr,Js,Pr,Lr,el,ht,tl,Q,Ir,ls,Sr,Or,Ks,Nr,Rr,sl,mt,al,le,we,Qs,ut,Br,Vs,Gr,ll,ke,Hr,ct,Wr,Yr,nl,ne,je,Xs,dt,Ur,Zs,Jr,rl,ve,Kr,ns,Qr,Vr,ol,gt,il,ye,pl,rs,Xr,fl,V,_t,Zr,os,eo,to,so,$t,ao,is,lo,no,ro,wt,oo,ps,io,po,hl,kt,ml,re,be,ea,jt,fo,ta,ho,ul,fs,mo,cl,Ee,dl,G,uo,sa,co,go,vt,aa,_o,$o,la,wo,ko,gl,yt,_l,hs,jo,$l,bt,wl,xe,vo,ms,yo,bo,kl,Et,jl,Te,Eo,xt,na,xo,To,vl,Tt,yl,Ae,Ao,At,ra,qo,Fo,bl,qt,El,oe,qe,oa,Ft,Mo,ia,Co,xl,Fe,Do,Mt,zo,Po,Tl,ie,Me,pa,Ct,Lo,fa,Io,Al,Ce,So,ha,Oo,No,ql,Dt,Fl,De,Ml,us,Ro,Cl,X,zt,Bo,cs,Go,Ho,Wo,Pt,Yo,ds,Uo,Jo,Ko,Lt,Qo,gs,Vo,Xo,Dl,It,zl,pe,ze,ma,St,Zo,ua,ei,Pl,_s,ti,Ll,Pe,Il,H,si,ca,ai,li,Ot,da,ni,ri,ga,oi,ii,Sl,Nt,Ol,$s,pi,Nl,Rt,Rl,Le,fi,ws,hi,mi,Bl,Bt,Gl,Ie,ui,Gt,_a,ci,di,Hl,Ht,Wl,Se,gi,Wt,$a,_i,$i,Yl,Yt,Ul,Oe,Jl;return d=new fe({}),W=new En({props:{id:"Vpjb1lu0MDk"}}),Y=new En({props:{id:"mqElG5QJWUg"}}),me=new Es({props:{$$slots:{default:[ff]},$$scope:{ctx:P}}}),Je=new fe({}),Ke=new q({props:{code:`from datasets import load_dataset

eli5 = load_dataset("eli5", split="train_asks[:5000]")`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> datasets <span class="hljs-keyword">import</span> load_dataset

<span class="hljs-meta">&gt;&gt;&gt; </span>eli5 = load_dataset(<span class="hljs-string">&quot;eli5&quot;</span>, split=<span class="hljs-string">&quot;train_asks[:5000]&quot;</span>)`}}),Qe=new q({props:{code:"eli5 = eli5.train_test_split(test_size=0.2)",highlighted:'eli5 = eli5.train_test_split(test_size=<span class="hljs-number">0.2</span>)'}}),Ve=new q({props:{code:'eli5["train"][0]',highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>eli5[<span class="hljs-string">&quot;train&quot;</span>][<span class="hljs-number">0</span>]
{<span class="hljs-string">&#x27;answers&#x27;</span>: {<span class="hljs-string">&#x27;a_id&#x27;</span>: [<span class="hljs-string">&#x27;c3d1aib&#x27;</span>, <span class="hljs-string">&#x27;c3d4lya&#x27;</span>],
  <span class="hljs-string">&#x27;score&#x27;</span>: [<span class="hljs-number">6</span>, <span class="hljs-number">3</span>],
  <span class="hljs-string">&#x27;text&#x27;</span>: [<span class="hljs-string">&quot;The velocity needed to remain in orbit is equal to the square root of Newton&#x27;s constant times the mass of earth divided by the distance from the center of the earth. I don&#x27;t know the altitude of that specific mission, but they&#x27;re usually around 300 km. That means he&#x27;s going 7-8 km/s.\\n\\nIn space there are no other forces acting on either the shuttle or the guy, so they stay in the same position relative to each other. If he were to become unable to return to the ship, he would presumably run out of oxygen, or slowly fall into the atmosphere and burn up.&quot;</span>,
   <span class="hljs-string">&quot;Hope you don&#x27;t mind me asking another question, but why aren&#x27;t there any stars visible in this photo?&quot;</span>]},
 <span class="hljs-string">&#x27;answers_urls&#x27;</span>: {<span class="hljs-string">&#x27;url&#x27;</span>: []},
 <span class="hljs-string">&#x27;document&#x27;</span>: <span class="hljs-string">&#x27;&#x27;</span>,
 <span class="hljs-string">&#x27;q_id&#x27;</span>: <span class="hljs-string">&#x27;nyxfp&#x27;</span>,
 <span class="hljs-string">&#x27;selftext&#x27;</span>: <span class="hljs-string">&#x27;_URL_0_\\n\\nThis was on the front page earlier and I have a few questions about it. Is it possible to calculate how fast the astronaut would be orbiting the earth? Also how does he stay close to the shuttle so that he can return safely, i.e is he orbiting at the same speed and can therefore stay next to it? And finally if his propulsion system failed, would he eventually re-enter the atmosphere and presumably die?&#x27;</span>,
 <span class="hljs-string">&#x27;selftext_urls&#x27;</span>: {<span class="hljs-string">&#x27;url&#x27;</span>: [<span class="hljs-string">&#x27;http://apod.nasa.gov/apod/image/1201/freeflyer_nasa_3000.jpg&#x27;</span>]},
 <span class="hljs-string">&#x27;subreddit&#x27;</span>: <span class="hljs-string">&#x27;askscience&#x27;</span>,
 <span class="hljs-string">&#x27;title&#x27;</span>: <span class="hljs-string">&#x27;Few questions about this space walk photograph.&#x27;</span>,
 <span class="hljs-string">&#x27;title_urls&#x27;</span>: {<span class="hljs-string">&#x27;url&#x27;</span>: []}}`}}),Xe=new fe({}),Ze=new En({props:{id:"ma1TrR7gE7I"}}),et=new q({props:{code:`from transformers import AutoTokenizer

tokenizer = AutoTokenizer.from_pretrained("distilgpt2")`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoTokenizer

<span class="hljs-meta">&gt;&gt;&gt; </span>tokenizer = AutoTokenizer.from_pretrained(<span class="hljs-string">&quot;distilgpt2&quot;</span>)`}}),tt=new En({props:{id:"8PmhEIXhBvI"}}),st=new q({props:{code:`from transformers import AutoTokenizer

tokenizer = AutoTokenizer.from_pretrained("distilroberta-base")`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoTokenizer

<span class="hljs-meta">&gt;&gt;&gt; </span>tokenizer = AutoTokenizer.from_pretrained(<span class="hljs-string">&quot;distilroberta-base&quot;</span>)`}}),lt=new q({props:{code:`eli5 = eli5.flatten()
eli5["train"][0]`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>eli5 = eli5.flatten()
<span class="hljs-meta">&gt;&gt;&gt; </span>eli5[<span class="hljs-string">&quot;train&quot;</span>][<span class="hljs-number">0</span>]
{<span class="hljs-string">&#x27;answers.a_id&#x27;</span>: [<span class="hljs-string">&#x27;c3d1aib&#x27;</span>, <span class="hljs-string">&#x27;c3d4lya&#x27;</span>],
 <span class="hljs-string">&#x27;answers.score&#x27;</span>: [<span class="hljs-number">6</span>, <span class="hljs-number">3</span>],
 <span class="hljs-string">&#x27;answers.text&#x27;</span>: [<span class="hljs-string">&quot;The velocity needed to remain in orbit is equal to the square root of Newton&#x27;s constant times the mass of earth divided by the distance from the center of the earth. I don&#x27;t know the altitude of that specific mission, but they&#x27;re usually around 300 km. That means he&#x27;s going 7-8 km/s.\\n\\nIn space there are no other forces acting on either the shuttle or the guy, so they stay in the same position relative to each other. If he were to become unable to return to the ship, he would presumably run out of oxygen, or slowly fall into the atmosphere and burn up.&quot;</span>,
  <span class="hljs-string">&quot;Hope you don&#x27;t mind me asking another question, but why aren&#x27;t there any stars visible in this photo?&quot;</span>],
 <span class="hljs-string">&#x27;answers_urls.url&#x27;</span>: [],
 <span class="hljs-string">&#x27;document&#x27;</span>: <span class="hljs-string">&#x27;&#x27;</span>,
 <span class="hljs-string">&#x27;q_id&#x27;</span>: <span class="hljs-string">&#x27;nyxfp&#x27;</span>,
 <span class="hljs-string">&#x27;selftext&#x27;</span>: <span class="hljs-string">&#x27;_URL_0_\\n\\nThis was on the front page earlier and I have a few questions about it. Is it possible to calculate how fast the astronaut would be orbiting the earth? Also how does he stay close to the shuttle so that he can return safely, i.e is he orbiting at the same speed and can therefore stay next to it? And finally if his propulsion system failed, would he eventually re-enter the atmosphere and presumably die?&#x27;</span>,
 <span class="hljs-string">&#x27;selftext_urls.url&#x27;</span>: [<span class="hljs-string">&#x27;http://apod.nasa.gov/apod/image/1201/freeflyer_nasa_3000.jpg&#x27;</span>],
 <span class="hljs-string">&#x27;subreddit&#x27;</span>: <span class="hljs-string">&#x27;askscience&#x27;</span>,
 <span class="hljs-string">&#x27;title&#x27;</span>: <span class="hljs-string">&#x27;Few questions about this space walk photograph.&#x27;</span>,
 <span class="hljs-string">&#x27;title_urls.url&#x27;</span>: []}`}}),nt=new q({props:{code:`def preprocess_function(examples):
    return tokenizer([" ".join(x) for x in examples["answers.text"]], truncation=True)`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">def</span> <span class="hljs-title function_">preprocess_function</span>(<span class="hljs-params">examples</span>):
<span class="hljs-meta">... </span>    <span class="hljs-keyword">return</span> tokenizer([<span class="hljs-string">&quot; &quot;</span>.join(x) <span class="hljs-keyword">for</span> x <span class="hljs-keyword">in</span> examples[<span class="hljs-string">&quot;answers.text&quot;</span>]], truncation=<span class="hljs-literal">True</span>)`}}),ot=new q({props:{code:`tokenized_eli5 = eli5.map(
    preprocess_function,
    batched=True,
    num_proc=4,
    remove_columns=eli5["train"].column_names,
)`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>tokenized_eli5 = eli5.<span class="hljs-built_in">map</span>(
<span class="hljs-meta">... </span>    preprocess_function,
<span class="hljs-meta">... </span>    batched=<span class="hljs-literal">True</span>,
<span class="hljs-meta">... </span>    num_proc=<span class="hljs-number">4</span>,
<span class="hljs-meta">... </span>    remove_columns=eli5[<span class="hljs-string">&quot;train&quot;</span>].column_names,
<span class="hljs-meta">... </span>)`}}),pt=new q({props:{code:`block_size = 128


def group_texts(examples):
    concatenated_examples = {k: sum(examples[k], []) for k in examples.keys()}
    total_length = len(concatenated_examples[list(examples.keys())[0]])
    result = {
        k: [t[i : i + block_size] for i in range(0, total_length, block_size)]
        for k, t in concatenated_examples.items()
    }
    result["labels"] = result["input_ids"].copy()
    return result`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>block_size = <span class="hljs-number">128</span>


<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">def</span> <span class="hljs-title function_">group_texts</span>(<span class="hljs-params">examples</span>):
<span class="hljs-meta">... </span>    concatenated_examples = {k: <span class="hljs-built_in">sum</span>(examples[k], []) <span class="hljs-keyword">for</span> k <span class="hljs-keyword">in</span> examples.keys()}
<span class="hljs-meta">... </span>    total_length = <span class="hljs-built_in">len</span>(concatenated_examples[<span class="hljs-built_in">list</span>(examples.keys())[<span class="hljs-number">0</span>]])
<span class="hljs-meta">... </span>    result = {
<span class="hljs-meta">... </span>        k: [t[i : i + block_size] <span class="hljs-keyword">for</span> i <span class="hljs-keyword">in</span> <span class="hljs-built_in">range</span>(<span class="hljs-number">0</span>, total_length, block_size)]
<span class="hljs-meta">... </span>        <span class="hljs-keyword">for</span> k, t <span class="hljs-keyword">in</span> concatenated_examples.items()
<span class="hljs-meta">... </span>    }
<span class="hljs-meta">... </span>    result[<span class="hljs-string">&quot;labels&quot;</span>] = result[<span class="hljs-string">&quot;input_ids&quot;</span>].copy()
<span class="hljs-meta">... </span>    <span class="hljs-keyword">return</span> result`}}),ft=new q({props:{code:"lm_dataset = tokenized_eli5.map(group_texts, batched=True, num_proc=4)",highlighted:'<span class="hljs-meta">&gt;&gt;&gt; </span>lm_dataset = tokenized_eli5.<span class="hljs-built_in">map</span>(group_texts, batched=<span class="hljs-literal">True</span>, num_proc=<span class="hljs-number">4</span>)'}}),ht=new lf({props:{group1:{id:"pt",code:`from transformers import DataCollatorForLanguageModeling

tokenizer.pad_token = tokenizer.eos_token
data_collator = DataCollatorForLanguageModeling(tokenizer=tokenizer, mlm=False)`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> DataCollatorForLanguageModeling

<span class="hljs-meta">&gt;&gt;&gt; </span>tokenizer.pad_token = tokenizer.eos_token
<span class="hljs-meta">&gt;&gt;&gt; </span>data_collator = DataCollatorForLanguageModeling(tokenizer=tokenizer, mlm=<span class="hljs-literal">False</span>)`},group2:{id:"tf",code:`from transformers import DataCollatorForLanguageModeling

data_collator = DataCollatorForLanguageModeling(tokenizer=tokenizer, mlm=False, return_tensors="tf")`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> DataCollatorForLanguageModeling

<span class="hljs-meta">&gt;&gt;&gt; </span>data_collator = DataCollatorForLanguageModeling(tokenizer=tokenizer, mlm=<span class="hljs-literal">False</span>, return_tensors=<span class="hljs-string">&quot;tf&quot;</span>)`}}}),mt=new lf({props:{group1:{id:"pt",code:`from transformers import DataCollatorForLanguageModeling

tokenizer.pad_token = tokenizer.eos_token
data_collator = DataCollatorForLanguageModeling(tokenizer=tokenizer, mlm_probability=0.15)`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> DataCollatorForLanguageModeling

<span class="hljs-meta">&gt;&gt;&gt; </span>tokenizer.pad_token = tokenizer.eos_token
<span class="hljs-meta">&gt;&gt;&gt; </span>data_collator = DataCollatorForLanguageModeling(tokenizer=tokenizer, mlm_probability=<span class="hljs-number">0.15</span>)`},group2:{id:"tf",code:`from transformers import DataCollatorForLanguageModeling

data_collator = DataCollatorForLanguageModeling(tokenizer=tokenizer, mlm=False, return_tensors="tf")`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> DataCollatorForLanguageModeling

<span class="hljs-meta">&gt;&gt;&gt; </span>data_collator = DataCollatorForLanguageModeling(tokenizer=tokenizer, mlm=<span class="hljs-literal">False</span>, return_tensors=<span class="hljs-string">&quot;tf&quot;</span>)`}}}),ut=new fe({}),dt=new fe({}),gt=new q({props:{code:`from transformers import AutoModelForCausalLM, TrainingArguments, Trainer

model = AutoModelForCausalLM.from_pretrained("distilgpt2")`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoModelForCausalLM, TrainingArguments, Trainer

<span class="hljs-meta">&gt;&gt;&gt; </span>model = AutoModelForCausalLM.from_pretrained(<span class="hljs-string">&quot;distilgpt2&quot;</span>)`}}),ye=new Es({props:{$$slots:{default:[hf]},$$scope:{ctx:P}}}),kt=new q({props:{code:`training_args = TrainingArguments(
    output_dir="./results",
    evaluation_strategy="epoch",
    learning_rate=2e-5,
    weight_decay=0.01,
)

trainer = Trainer(
    model=model,
    args=training_args,
    train_dataset=lm_dataset["train"],
    eval_dataset=lm_dataset["test"],
    data_collator=data_collator,
)

trainer.train()`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>training_args = TrainingArguments(
<span class="hljs-meta">... </span>    output_dir=<span class="hljs-string">&quot;./results&quot;</span>,
<span class="hljs-meta">... </span>    evaluation_strategy=<span class="hljs-string">&quot;epoch&quot;</span>,
<span class="hljs-meta">... </span>    learning_rate=<span class="hljs-number">2e-5</span>,
<span class="hljs-meta">... </span>    weight_decay=<span class="hljs-number">0.01</span>,
<span class="hljs-meta">... </span>)

<span class="hljs-meta">&gt;&gt;&gt; </span>trainer = Trainer(
<span class="hljs-meta">... </span>    model=model,
<span class="hljs-meta">... </span>    args=training_args,
<span class="hljs-meta">... </span>    train_dataset=lm_dataset[<span class="hljs-string">&quot;train&quot;</span>],
<span class="hljs-meta">... </span>    eval_dataset=lm_dataset[<span class="hljs-string">&quot;test&quot;</span>],
<span class="hljs-meta">... </span>    data_collator=data_collator,
<span class="hljs-meta">... </span>)

<span class="hljs-meta">&gt;&gt;&gt; </span>trainer.train()`}}),jt=new fe({}),Ee=new Es({props:{$$slots:{default:[mf]},$$scope:{ctx:P}}}),yt=new q({props:{code:`tf_train_set = lm_dataset["train"].to_tf_dataset(
    columns=["attention_mask", "input_ids", "labels"],
    dummy_labels=True,
    shuffle=True,
    batch_size=16,
    collate_fn=data_collator,
)

tf_test_set = lm_dataset["test"].to_tf_dataset(
    columns=["attention_mask", "input_ids", "labels"],
    dummy_labels=True,
    shuffle=False,
    batch_size=16,
    collate_fn=data_collator,
)`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>tf_train_set = lm_dataset[<span class="hljs-string">&quot;train&quot;</span>].to_tf_dataset(
<span class="hljs-meta">... </span>    columns=[<span class="hljs-string">&quot;attention_mask&quot;</span>, <span class="hljs-string">&quot;input_ids&quot;</span>, <span class="hljs-string">&quot;labels&quot;</span>],
<span class="hljs-meta">... </span>    dummy_labels=<span class="hljs-literal">True</span>,
<span class="hljs-meta">... </span>    shuffle=<span class="hljs-literal">True</span>,
<span class="hljs-meta">... </span>    batch_size=<span class="hljs-number">16</span>,
<span class="hljs-meta">... </span>    collate_fn=data_collator,
<span class="hljs-meta">... </span>)

<span class="hljs-meta">&gt;&gt;&gt; </span>tf_test_set = lm_dataset[<span class="hljs-string">&quot;test&quot;</span>].to_tf_dataset(
<span class="hljs-meta">... </span>    columns=[<span class="hljs-string">&quot;attention_mask&quot;</span>, <span class="hljs-string">&quot;input_ids&quot;</span>, <span class="hljs-string">&quot;labels&quot;</span>],
<span class="hljs-meta">... </span>    dummy_labels=<span class="hljs-literal">True</span>,
<span class="hljs-meta">... </span>    shuffle=<span class="hljs-literal">False</span>,
<span class="hljs-meta">... </span>    batch_size=<span class="hljs-number">16</span>,
<span class="hljs-meta">... </span>    collate_fn=data_collator,
<span class="hljs-meta">... </span>)`}}),bt=new q({props:{code:`from transformers import create_optimizer, AdamWeightDecay

optimizer = AdamWeightDecay(learning_rate=2e-5, weight_decay_rate=0.01)`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> create_optimizer, AdamWeightDecay

<span class="hljs-meta">&gt;&gt;&gt; </span>optimizer = AdamWeightDecay(learning_rate=<span class="hljs-number">2e-5</span>, weight_decay_rate=<span class="hljs-number">0.01</span>)`}}),Et=new q({props:{code:`from transformers import TFAutoModelForCausalLM

model = TFAutoModelForCausalLM.from_pretrained("distilgpt2")`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> TFAutoModelForCausalLM

<span class="hljs-meta">&gt;&gt;&gt; </span>model = TFAutoModelForCausalLM.from_pretrained(<span class="hljs-string">&quot;distilgpt2&quot;</span>)`}}),Tt=new q({props:{code:`import tensorflow as tf

model.compile(optimizer=optimizer)`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">import</span> tensorflow <span class="hljs-keyword">as</span> tf

<span class="hljs-meta">&gt;&gt;&gt; </span>model.<span class="hljs-built_in">compile</span>(optimizer=optimizer)`}}),qt=new q({props:{code:"model.fit(x=tf_train_set, validation_data=tf_test_set, epochs=3)",highlighted:'<span class="hljs-meta">&gt;&gt;&gt; </span>model.fit(x=tf_train_set, validation_data=tf_test_set, epochs=<span class="hljs-number">3</span>)'}}),Ft=new fe({}),Ct=new fe({}),Dt=new q({props:{code:`from transformers import AutoModelForMaskedLM

model = AutoModelForMaskedLM.from_pretrained("distilroberta-base")`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoModelForMaskedLM

<span class="hljs-meta">&gt;&gt;&gt; </span>model = AutoModelForMaskedLM.from_pretrained(<span class="hljs-string">&quot;distilroberta-base&quot;</span>)`}}),De=new Es({props:{$$slots:{default:[uf]},$$scope:{ctx:P}}}),It=new q({props:{code:`training_args = TrainingArguments(
    output_dir="./results",
    evaluation_strategy="epoch",
    learning_rate=2e-5,
    num_train_epochs=3,
    weight_decay=0.01,
)

trainer = Trainer(
    model=model,
    args=training_args,
    train_dataset=lm_dataset["train"],
    eval_dataset=lm_dataset["test"],
    data_collator=data_collator,
)

trainer.train()`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>training_args = TrainingArguments(
<span class="hljs-meta">... </span>    output_dir=<span class="hljs-string">&quot;./results&quot;</span>,
<span class="hljs-meta">... </span>    evaluation_strategy=<span class="hljs-string">&quot;epoch&quot;</span>,
<span class="hljs-meta">... </span>    learning_rate=<span class="hljs-number">2e-5</span>,
<span class="hljs-meta">... </span>    num_train_epochs=<span class="hljs-number">3</span>,
<span class="hljs-meta">... </span>    weight_decay=<span class="hljs-number">0.01</span>,
<span class="hljs-meta">... </span>)

<span class="hljs-meta">&gt;&gt;&gt; </span>trainer = Trainer(
<span class="hljs-meta">... </span>    model=model,
<span class="hljs-meta">... </span>    args=training_args,
<span class="hljs-meta">... </span>    train_dataset=lm_dataset[<span class="hljs-string">&quot;train&quot;</span>],
<span class="hljs-meta">... </span>    eval_dataset=lm_dataset[<span class="hljs-string">&quot;test&quot;</span>],
<span class="hljs-meta">... </span>    data_collator=data_collator,
<span class="hljs-meta">... </span>)

<span class="hljs-meta">&gt;&gt;&gt; </span>trainer.train()`}}),St=new fe({}),Pe=new Es({props:{$$slots:{default:[cf]},$$scope:{ctx:P}}}),Nt=new q({props:{code:`tf_train_set = lm_dataset["train"].to_tf_dataset(
    columns=["attention_mask", "input_ids", "labels"],
    dummy_labels=True,
    shuffle=True,
    batch_size=16,
    collate_fn=data_collator,
)

tf_test_set = lm_dataset["test"].to_tf_dataset(
    columns=["attention_mask", "input_ids", "labels"],
    dummy_labels=True,
    shuffle=False,
    batch_size=16,
    collate_fn=data_collator,
)`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>tf_train_set = lm_dataset[<span class="hljs-string">&quot;train&quot;</span>].to_tf_dataset(
<span class="hljs-meta">... </span>    columns=[<span class="hljs-string">&quot;attention_mask&quot;</span>, <span class="hljs-string">&quot;input_ids&quot;</span>, <span class="hljs-string">&quot;labels&quot;</span>],
<span class="hljs-meta">... </span>    dummy_labels=<span class="hljs-literal">True</span>,
<span class="hljs-meta">... </span>    shuffle=<span class="hljs-literal">True</span>,
<span class="hljs-meta">... </span>    batch_size=<span class="hljs-number">16</span>,
<span class="hljs-meta">... </span>    collate_fn=data_collator,
<span class="hljs-meta">... </span>)

<span class="hljs-meta">&gt;&gt;&gt; </span>tf_test_set = lm_dataset[<span class="hljs-string">&quot;test&quot;</span>].to_tf_dataset(
<span class="hljs-meta">... </span>    columns=[<span class="hljs-string">&quot;attention_mask&quot;</span>, <span class="hljs-string">&quot;input_ids&quot;</span>, <span class="hljs-string">&quot;labels&quot;</span>],
<span class="hljs-meta">... </span>    dummy_labels=<span class="hljs-literal">True</span>,
<span class="hljs-meta">... </span>    shuffle=<span class="hljs-literal">False</span>,
<span class="hljs-meta">... </span>    batch_size=<span class="hljs-number">16</span>,
<span class="hljs-meta">... </span>    collate_fn=data_collator,
<span class="hljs-meta">... </span>)`}}),Rt=new q({props:{code:`from transformers import create_optimizer, AdamWeightDecay

optimizer = AdamWeightDecay(learning_rate=2e-5, weight_decay_rate=0.01)`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> create_optimizer, AdamWeightDecay

<span class="hljs-meta">&gt;&gt;&gt; </span>optimizer = AdamWeightDecay(learning_rate=<span class="hljs-number">2e-5</span>, weight_decay_rate=<span class="hljs-number">0.01</span>)`}}),Bt=new q({props:{code:`from transformers import TFAutoModelForMaskedLM

model = TFAutoModelForCausalLM.from_pretrained("distilroberta-base")`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> TFAutoModelForMaskedLM

<span class="hljs-meta">&gt;&gt;&gt; </span>model = TFAutoModelForCausalLM.from_pretrained(<span class="hljs-string">&quot;distilroberta-base&quot;</span>)`}}),Ht=new q({props:{code:`import tensorflow as tf

model.compile(optimizer=optimizer)`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">import</span> tensorflow <span class="hljs-keyword">as</span> tf

<span class="hljs-meta">&gt;&gt;&gt; </span>model.<span class="hljs-built_in">compile</span>(optimizer=optimizer)`}}),Yt=new q({props:{code:"model.fit(x=tf_train_set, validation_data=tf_test_set, epochs=3)",highlighted:'<span class="hljs-meta">&gt;&gt;&gt; </span>model.fit(x=tf_train_set, validation_data=tf_test_set, epochs=<span class="hljs-number">3</span>)'}}),Oe=new Es({props:{$$slots:{default:[df]},$$scope:{ctx:P}}}),{c(){u=r("meta"),E=f(),c=r("h1"),v=r("a"),x=r("span"),g(d.$$.fragment),b=f(),A=r("span"),T=l("Language modeling"),y=f(),F=r("p"),C=l("Language modeling predicts words in a sentence. There are two forms of language modeling."),L=f(),g(W.$$.fragment),N=f(),U=r("p"),Ge=l("Causal language modeling predicts the next token in a sequence of tokens, and the model can only attend to tokens on the left."),R=f(),g(Y.$$.fragment),he=f(),M=r("p"),z=l("Masked language modeling predicts a masked token in a sequence, and the model can attend to tokens bidirectionally."),He=f(),D=r("p"),Jt=l("This guide will show you how to fine-tune "),I=r("a"),Kt=l("DistilGPT2"),Qt=l(" for causal language modeling and "),We=r("a"),xn=l("DistilRoBERTa"),Tn=l(" for masked language modeling on the "),Ye=r("a"),An=l("r/askscience"),qn=l(" subset of the "),Ue=r("a"),Fn=l("ELI5"),Mn=l(" dataset."),ba=f(),g(me.$$.fragment),Ea=f(),se=r("h2"),ue=r("a"),xs=r("span"),g(Je.$$.fragment),Cn=f(),Ts=r("span"),Dn=l("Load ELI5 dataset"),xa=f(),Vt=r("p"),zn=l("Load only the first 5000 rows of the ELI5 dataset from the \u{1F917} Datasets library since it is pretty large:"),Ta=f(),g(Ke.$$.fragment),Aa=f(),Xt=r("p"),Pn=l("Split this dataset into a train and test set:"),qa=f(),g(Qe.$$.fragment),Fa=f(),Zt=r("p"),Ln=l("Then take a look at an example:"),Ma=f(),g(Ve.$$.fragment),Ca=f(),B=r("p"),In=l("Notice "),As=r("code"),Sn=l("text"),On=l(" is a subfield nested inside the "),qs=r("code"),Nn=l("answers"),Rn=l(" dictionary. When you preprocess the dataset, you will need to extract the "),Fs=r("code"),Bn=l("text"),Gn=l(" subfield into a separate column."),Da=f(),ae=r("h2"),ce=r("a"),Ms=r("span"),g(Xe.$$.fragment),Hn=f(),Cs=r("span"),Wn=l("Preprocess"),za=f(),g(Ze.$$.fragment),Pa=f(),de=r("p"),Yn=l("For causal language modeling, load the DistilGPT2 tokenizer to process the "),Ds=r("code"),Un=l("text"),Jn=l(" subfield:"),La=f(),g(et.$$.fragment),Ia=f(),g(tt.$$.fragment),Sa=f(),es=r("p"),Kn=l("For masked language modeling, load the DistilRoBERTa tokenizer instead:"),Oa=f(),g(st.$$.fragment),Na=f(),J=r("p"),Qn=l("Extract the "),zs=r("code"),Vn=l("text"),Xn=l(" subfield from its nested structure with the "),at=r("a"),Ps=r("code"),Zn=l("flatten"),er=l(" method:"),Ra=f(),g(lt.$$.fragment),Ba=f(),K=r("p"),tr=l("Each subfield is now a separate column as indicated by the "),Ls=r("code"),sr=l("answers"),ar=l(" prefix. Notice that "),Is=r("code"),lr=l("answers.text"),nr=l(" is a list. Instead of tokenizing each sentence separately, convert the list to a string to jointly tokenize them."),Ga=f(),ts=r("p"),rr=l("Here is how you can create a preprocessing function to convert the list to a string and truncate sequences to be no longer than DistilGPT2\u2019s maximum input length:"),Ha=f(),g(nt.$$.fragment),Wa=f(),S=r("p"),or=l("Use \u{1F917} Datasets "),rt=r("a"),Ss=r("code"),ir=l("map"),pr=l(" function to apply the preprocessing function over the entire dataset. You can speed up the "),Os=r("code"),fr=l("map"),hr=l(" function by setting "),Ns=r("code"),mr=l("batched=True"),ur=l(" to process multiple elements of the dataset at once and increasing the number of processes with "),Rs=r("code"),cr=l("num_proc"),dr=l(". Remove the columns you don\u2019t need:"),Ya=f(),g(ot.$$.fragment),Ua=f(),ss=r("p"),gr=l("Now you need a second preprocessing function to capture text truncated from any lengthy examples to prevent loss of information. This preprocessing function should:"),Ja=f(),ge=r("ul"),Bs=r("li"),_r=l("Concatenate all the text."),$r=f(),it=r("li"),wr=l("Split the concatenated text into smaller chunks defined by "),Gs=r("code"),kr=l("block_size"),jr=l("."),Ka=f(),g(pt.$$.fragment),Qa=f(),_e=r("p"),vr=l("Apply the "),Hs=r("code"),yr=l("group_texts"),br=l(" function over the entire dataset:"),Va=f(),g(ft.$$.fragment),Xa=f(),O=r("p"),Er=l("For causal language modeling, use "),as=r("a"),xr=l("DataCollatorForLanguageModeling"),Tr=l(" to create a batch of examples. It will also "),Ws=r("em"),Ar=l("dynamically pad"),qr=l(" your text to the length of the longest element in its batch, so they are a uniform length. While it is possible to pad your text in the "),Ys=r("code"),Fr=l("tokenizer"),Mr=l(" function by setting "),Us=r("code"),Cr=l("padding=True"),Dr=l(", dynamic padding is more efficient."),Za=f(),$e=r("p"),zr=l("You can use the end of sequence token as the padding token, and set "),Js=r("code"),Pr=l("mlm=False"),Lr=l(". This will use the inputs as labels shifted to the right by one element:"),el=f(),g(ht.$$.fragment),tl=f(),Q=r("p"),Ir=l("For masked language modeling, use the same "),ls=r("a"),Sr=l("DataCollatorForLanguageModeling"),Or=l(" except you should specify "),Ks=r("code"),Nr=l("mlm_probability"),Rr=l(" to randomly mask tokens each time you iterate over the data."),sl=f(),g(mt.$$.fragment),al=f(),le=r("h2"),we=r("a"),Qs=r("span"),g(ut.$$.fragment),Br=f(),Vs=r("span"),Gr=l("Causal language modeling"),ll=f(),ke=r("p"),Hr=l("Causal language modeling is frequently used for text generation. This section shows you how to fine-tune "),ct=r("a"),Wr=l("DistilGPT2"),Yr=l(" to generate new text."),nl=f(),ne=r("h3"),je=r("a"),Xs=r("span"),g(dt.$$.fragment),Ur=f(),Zs=r("span"),Jr=l("Fine-tune with Trainer"),rl=f(),ve=r("p"),Kr=l("Load DistilGPT2 with "),ns=r("a"),Qr=l("AutoModelForCausalLM"),Vr=l(":"),ol=f(),g(gt.$$.fragment),il=f(),g(ye.$$.fragment),pl=f(),rs=r("p"),Xr=l("At this point, only three steps remain:"),fl=f(),V=r("ol"),_t=r("li"),Zr=l("Define your training hyperparameters in "),os=r("a"),eo=l("TrainingArguments"),to=l("."),so=f(),$t=r("li"),ao=l("Pass the training arguments to "),is=r("a"),lo=l("Trainer"),no=l(" along with the model, datasets, and data collator."),ro=f(),wt=r("li"),oo=l("Call "),ps=r("a"),io=l("train()"),po=l(" to fine-tune your model."),hl=f(),g(kt.$$.fragment),ml=f(),re=r("h3"),be=r("a"),ea=r("span"),g(jt.$$.fragment),fo=f(),ta=r("span"),ho=l("Fine-tune with TensorFlow"),ul=f(),fs=r("p"),mo=l("To fine-tune a model in TensorFlow is just as easy, with only a few differences."),cl=f(),g(Ee.$$.fragment),dl=f(),G=r("p"),uo=l("Convert your datasets to the "),sa=r("code"),co=l("tf.data.Dataset"),go=l(" format with "),vt=r("a"),aa=r("code"),_o=l("to_tf_dataset"),$o=l(". Specify inputs and labels in "),la=r("code"),wo=l("columns"),ko=l(", whether to shuffle the dataset order, batch size, and the data collator:"),gl=f(),g(yt.$$.fragment),_l=f(),hs=r("p"),jo=l("Set up an optimizer function, learning rate, and some training hyperparameters:"),$l=f(),g(bt.$$.fragment),wl=f(),xe=r("p"),vo=l("Load DistilGPT2 with "),ms=r("a"),yo=l("TFAutoModelForCausalLM"),bo=l(":"),kl=f(),g(Et.$$.fragment),jl=f(),Te=r("p"),Eo=l("Configure the model for training with "),xt=r("a"),na=r("code"),xo=l("compile"),To=l(":"),vl=f(),g(Tt.$$.fragment),yl=f(),Ae=r("p"),Ao=l("Call "),At=r("a"),ra=r("code"),qo=l("fit"),Fo=l(" to fine-tune the model:"),bl=f(),g(qt.$$.fragment),El=f(),oe=r("h2"),qe=r("a"),oa=r("span"),g(Ft.$$.fragment),Mo=f(),ia=r("span"),Co=l("Masked language modeling"),xl=f(),Fe=r("p"),Do=l("Masked language modeling is also known as a fill-mask task because it predicts a masked token in a sequence. Models for masked language modeling require a good contextual understanding of an entire sequence instead of only the left context. This section shows you how to fine-tune "),Mt=r("a"),zo=l("DistilRoBERTa"),Po=l(" to predict a masked word."),Tl=f(),ie=r("h3"),Me=r("a"),pa=r("span"),g(Ct.$$.fragment),Lo=f(),fa=r("span"),Io=l("Fine-tune with Trainer"),Al=f(),Ce=r("p"),So=l("Load DistilRoBERTa with "),ha=r("code"),Oo=l("AutoModelForMaskedlM"),No=l(":"),ql=f(),g(Dt.$$.fragment),Fl=f(),g(De.$$.fragment),Ml=f(),us=r("p"),Ro=l("At this point, only three steps remain:"),Cl=f(),X=r("ol"),zt=r("li"),Bo=l("Define your training hyperparameters in "),cs=r("a"),Go=l("TrainingArguments"),Ho=l("."),Wo=f(),Pt=r("li"),Yo=l("Pass the training arguments to "),ds=r("a"),Uo=l("Trainer"),Jo=l(" along with the model, datasets, and data collator."),Ko=f(),Lt=r("li"),Qo=l("Call "),gs=r("a"),Vo=l("train()"),Xo=l(" to fine-tune your model."),Dl=f(),g(It.$$.fragment),zl=f(),pe=r("h3"),ze=r("a"),ma=r("span"),g(St.$$.fragment),Zo=f(),ua=r("span"),ei=l("Fine-tune with TensorFlow"),Pl=f(),_s=r("p"),ti=l("To fine-tune a model in TensorFlow is just as easy, with only a few differences."),Ll=f(),g(Pe.$$.fragment),Il=f(),H=r("p"),si=l("Convert your datasets to the "),ca=r("code"),ai=l("tf.data.Dataset"),li=l(" format with "),Ot=r("a"),da=r("code"),ni=l("to_tf_dataset"),ri=l(". Specify inputs and labels in "),ga=r("code"),oi=l("columns"),ii=l(", whether to shuffle the dataset order, batch size, and the data collator:"),Sl=f(),g(Nt.$$.fragment),Ol=f(),$s=r("p"),pi=l("Set up an optimizer function, learning rate, and some training hyperparameters:"),Nl=f(),g(Rt.$$.fragment),Rl=f(),Le=r("p"),fi=l("Load DistilRoBERTa with "),ws=r("a"),hi=l("TFAutoModelForMaskedLM"),mi=l(":"),Bl=f(),g(Bt.$$.fragment),Gl=f(),Ie=r("p"),ui=l("Configure the model for training with "),Gt=r("a"),_a=r("code"),ci=l("compile"),di=l(":"),Hl=f(),g(Ht.$$.fragment),Wl=f(),Se=r("p"),gi=l("Call "),Wt=r("a"),$a=r("code"),_i=l("fit"),$i=l(" to fine-tune the model:"),Yl=f(),g(Yt.$$.fragment),Ul=f(),g(Oe.$$.fragment),this.h()},l(e){const a=pf('[data-svelte="svelte-1phssyn"]',document.head);u=o(a,"META",{name:!0,content:!0}),a.forEach(t),E=h(e),c=o(e,"H1",{class:!0});var Ut=i(c);v=o(Ut,"A",{id:!0,class:!0,href:!0});var wa=i(v);x=o(wa,"SPAN",{});var ka=i(x);_(d.$$.fragment,ka),ka.forEach(t),wa.forEach(t),b=h(Ut),A=o(Ut,"SPAN",{});var ja=i(A);T=n(ja,"Language modeling"),ja.forEach(t),Ut.forEach(t),y=h(e),F=o(e,"P",{});var va=i(F);C=n(va,"Language modeling predicts words in a sentence. There are two forms of language modeling."),va.forEach(t),L=h(e),_(W.$$.fragment,e),N=h(e),U=o(e,"P",{});var ya=i(U);Ge=n(ya,"Causal language modeling predicts the next token in a sequence of tokens, and the model can only attend to tokens on the left."),ya.forEach(t),R=h(e),_(Y.$$.fragment,e),he=h(e),M=o(e,"P",{});var wi=i(M);z=n(wi,"Masked language modeling predicts a masked token in a sequence, and the model can attend to tokens bidirectionally."),wi.forEach(t),He=h(e),D=o(e,"P",{});var Z=i(D);Jt=n(Z,"This guide will show you how to fine-tune "),I=o(Z,"A",{href:!0,rel:!0});var ki=i(I);Kt=n(ki,"DistilGPT2"),ki.forEach(t),Qt=n(Z," for causal language modeling and "),We=o(Z,"A",{href:!0,rel:!0});var ji=i(We);xn=n(ji,"DistilRoBERTa"),ji.forEach(t),Tn=n(Z," for masked language modeling on the "),Ye=o(Z,"A",{href:!0,rel:!0});var vi=i(Ye);An=n(vi,"r/askscience"),vi.forEach(t),qn=n(Z," subset of the "),Ue=o(Z,"A",{href:!0,rel:!0});var yi=i(Ue);Fn=n(yi,"ELI5"),yi.forEach(t),Mn=n(Z," dataset."),Z.forEach(t),ba=h(e),_(me.$$.fragment,e),Ea=h(e),se=o(e,"H2",{class:!0});var Kl=i(se);ue=o(Kl,"A",{id:!0,class:!0,href:!0});var bi=i(ue);xs=o(bi,"SPAN",{});var Ei=i(xs);_(Je.$$.fragment,Ei),Ei.forEach(t),bi.forEach(t),Cn=h(Kl),Ts=o(Kl,"SPAN",{});var xi=i(Ts);Dn=n(xi,"Load ELI5 dataset"),xi.forEach(t),Kl.forEach(t),xa=h(e),Vt=o(e,"P",{});var Ti=i(Vt);zn=n(Ti,"Load only the first 5000 rows of the ELI5 dataset from the \u{1F917} Datasets library since it is pretty large:"),Ti.forEach(t),Ta=h(e),_(Ke.$$.fragment,e),Aa=h(e),Xt=o(e,"P",{});var Ai=i(Xt);Pn=n(Ai,"Split this dataset into a train and test set:"),Ai.forEach(t),qa=h(e),_(Qe.$$.fragment,e),Fa=h(e),Zt=o(e,"P",{});var qi=i(Zt);Ln=n(qi,"Then take a look at an example:"),qi.forEach(t),Ma=h(e),_(Ve.$$.fragment,e),Ca=h(e),B=o(e,"P",{});var Ne=i(B);In=n(Ne,"Notice "),As=o(Ne,"CODE",{});var Fi=i(As);Sn=n(Fi,"text"),Fi.forEach(t),On=n(Ne," is a subfield nested inside the "),qs=o(Ne,"CODE",{});var Mi=i(qs);Nn=n(Mi,"answers"),Mi.forEach(t),Rn=n(Ne," dictionary. When you preprocess the dataset, you will need to extract the "),Fs=o(Ne,"CODE",{});var Ci=i(Fs);Bn=n(Ci,"text"),Ci.forEach(t),Gn=n(Ne," subfield into a separate column."),Ne.forEach(t),Da=h(e),ae=o(e,"H2",{class:!0});var Ql=i(ae);ce=o(Ql,"A",{id:!0,class:!0,href:!0});var Di=i(ce);Ms=o(Di,"SPAN",{});var zi=i(Ms);_(Xe.$$.fragment,zi),zi.forEach(t),Di.forEach(t),Hn=h(Ql),Cs=o(Ql,"SPAN",{});var Pi=i(Cs);Wn=n(Pi,"Preprocess"),Pi.forEach(t),Ql.forEach(t),za=h(e),_(Ze.$$.fragment,e),Pa=h(e),de=o(e,"P",{});var Vl=i(de);Yn=n(Vl,"For causal language modeling, load the DistilGPT2 tokenizer to process the "),Ds=o(Vl,"CODE",{});var Li=i(Ds);Un=n(Li,"text"),Li.forEach(t),Jn=n(Vl," subfield:"),Vl.forEach(t),La=h(e),_(et.$$.fragment,e),Ia=h(e),_(tt.$$.fragment,e),Sa=h(e),es=o(e,"P",{});var Ii=i(es);Kn=n(Ii,"For masked language modeling, load the DistilRoBERTa tokenizer instead:"),Ii.forEach(t),Oa=h(e),_(st.$$.fragment,e),Na=h(e),J=o(e,"P",{});var ks=i(J);Qn=n(ks,"Extract the "),zs=o(ks,"CODE",{});var Si=i(zs);Vn=n(Si,"text"),Si.forEach(t),Xn=n(ks," subfield from its nested structure with the "),at=o(ks,"A",{href:!0,rel:!0});var Oi=i(at);Ps=o(Oi,"CODE",{});var Ni=i(Ps);Zn=n(Ni,"flatten"),Ni.forEach(t),Oi.forEach(t),er=n(ks," method:"),ks.forEach(t),Ra=h(e),_(lt.$$.fragment,e),Ba=h(e),K=o(e,"P",{});var js=i(K);tr=n(js,"Each subfield is now a separate column as indicated by the "),Ls=o(js,"CODE",{});var Ri=i(Ls);sr=n(Ri,"answers"),Ri.forEach(t),ar=n(js," prefix. Notice that "),Is=o(js,"CODE",{});var Bi=i(Is);lr=n(Bi,"answers.text"),Bi.forEach(t),nr=n(js," is a list. Instead of tokenizing each sentence separately, convert the list to a string to jointly tokenize them."),js.forEach(t),Ga=h(e),ts=o(e,"P",{});var Gi=i(ts);rr=n(Gi,"Here is how you can create a preprocessing function to convert the list to a string and truncate sequences to be no longer than DistilGPT2\u2019s maximum input length:"),Gi.forEach(t),Ha=h(e),_(nt.$$.fragment,e),Wa=h(e),S=o(e,"P",{});var ee=i(S);or=n(ee,"Use \u{1F917} Datasets "),rt=o(ee,"A",{href:!0,rel:!0});var Hi=i(rt);Ss=o(Hi,"CODE",{});var Wi=i(Ss);ir=n(Wi,"map"),Wi.forEach(t),Hi.forEach(t),pr=n(ee," function to apply the preprocessing function over the entire dataset. You can speed up the "),Os=o(ee,"CODE",{});var Yi=i(Os);fr=n(Yi,"map"),Yi.forEach(t),hr=n(ee," function by setting "),Ns=o(ee,"CODE",{});var Ui=i(Ns);mr=n(Ui,"batched=True"),Ui.forEach(t),ur=n(ee," to process multiple elements of the dataset at once and increasing the number of processes with "),Rs=o(ee,"CODE",{});var Ji=i(Rs);cr=n(Ji,"num_proc"),Ji.forEach(t),dr=n(ee,". Remove the columns you don\u2019t need:"),ee.forEach(t),Ya=h(e),_(ot.$$.fragment,e),Ua=h(e),ss=o(e,"P",{});var Ki=i(ss);gr=n(Ki,"Now you need a second preprocessing function to capture text truncated from any lengthy examples to prevent loss of information. This preprocessing function should:"),Ki.forEach(t),Ja=h(e),ge=o(e,"UL",{});var Xl=i(ge);Bs=o(Xl,"LI",{});var Qi=i(Bs);_r=n(Qi,"Concatenate all the text."),Qi.forEach(t),$r=h(Xl),it=o(Xl,"LI",{});var Zl=i(it);wr=n(Zl,"Split the concatenated text into smaller chunks defined by "),Gs=o(Zl,"CODE",{});var Vi=i(Gs);kr=n(Vi,"block_size"),Vi.forEach(t),jr=n(Zl,"."),Zl.forEach(t),Xl.forEach(t),Ka=h(e),_(pt.$$.fragment,e),Qa=h(e),_e=o(e,"P",{});var en=i(_e);vr=n(en,"Apply the "),Hs=o(en,"CODE",{});var Xi=i(Hs);yr=n(Xi,"group_texts"),Xi.forEach(t),br=n(en," function over the entire dataset:"),en.forEach(t),Va=h(e),_(ft.$$.fragment,e),Xa=h(e),O=o(e,"P",{});var te=i(O);Er=n(te,"For causal language modeling, use "),as=o(te,"A",{href:!0});var Zi=i(as);xr=n(Zi,"DataCollatorForLanguageModeling"),Zi.forEach(t),Tr=n(te," to create a batch of examples. It will also "),Ws=o(te,"EM",{});var ep=i(Ws);Ar=n(ep,"dynamically pad"),ep.forEach(t),qr=n(te," your text to the length of the longest element in its batch, so they are a uniform length. While it is possible to pad your text in the "),Ys=o(te,"CODE",{});var tp=i(Ys);Fr=n(tp,"tokenizer"),tp.forEach(t),Mr=n(te," function by setting "),Us=o(te,"CODE",{});var sp=i(Us);Cr=n(sp,"padding=True"),sp.forEach(t),Dr=n(te,", dynamic padding is more efficient."),te.forEach(t),Za=h(e),$e=o(e,"P",{});var tn=i($e);zr=n(tn,"You can use the end of sequence token as the padding token, and set "),Js=o(tn,"CODE",{});var ap=i(Js);Pr=n(ap,"mlm=False"),ap.forEach(t),Lr=n(tn,". This will use the inputs as labels shifted to the right by one element:"),tn.forEach(t),el=h(e),_(ht.$$.fragment,e),tl=h(e),Q=o(e,"P",{});var vs=i(Q);Ir=n(vs,"For masked language modeling, use the same "),ls=o(vs,"A",{href:!0});var lp=i(ls);Sr=n(lp,"DataCollatorForLanguageModeling"),lp.forEach(t),Or=n(vs," except you should specify "),Ks=o(vs,"CODE",{});var np=i(Ks);Nr=n(np,"mlm_probability"),np.forEach(t),Rr=n(vs," to randomly mask tokens each time you iterate over the data."),vs.forEach(t),sl=h(e),_(mt.$$.fragment,e),al=h(e),le=o(e,"H2",{class:!0});var sn=i(le);we=o(sn,"A",{id:!0,class:!0,href:!0});var rp=i(we);Qs=o(rp,"SPAN",{});var op=i(Qs);_(ut.$$.fragment,op),op.forEach(t),rp.forEach(t),Br=h(sn),Vs=o(sn,"SPAN",{});var ip=i(Vs);Gr=n(ip,"Causal language modeling"),ip.forEach(t),sn.forEach(t),ll=h(e),ke=o(e,"P",{});var an=i(ke);Hr=n(an,"Causal language modeling is frequently used for text generation. This section shows you how to fine-tune "),ct=o(an,"A",{href:!0,rel:!0});var pp=i(ct);Wr=n(pp,"DistilGPT2"),pp.forEach(t),Yr=n(an," to generate new text."),an.forEach(t),nl=h(e),ne=o(e,"H3",{class:!0});var ln=i(ne);je=o(ln,"A",{id:!0,class:!0,href:!0});var fp=i(je);Xs=o(fp,"SPAN",{});var hp=i(Xs);_(dt.$$.fragment,hp),hp.forEach(t),fp.forEach(t),Ur=h(ln),Zs=o(ln,"SPAN",{});var mp=i(Zs);Jr=n(mp,"Fine-tune with Trainer"),mp.forEach(t),ln.forEach(t),rl=h(e),ve=o(e,"P",{});var nn=i(ve);Kr=n(nn,"Load DistilGPT2 with "),ns=o(nn,"A",{href:!0});var up=i(ns);Qr=n(up,"AutoModelForCausalLM"),up.forEach(t),Vr=n(nn,":"),nn.forEach(t),ol=h(e),_(gt.$$.fragment,e),il=h(e),_(ye.$$.fragment,e),pl=h(e),rs=o(e,"P",{});var cp=i(rs);Xr=n(cp,"At this point, only three steps remain:"),cp.forEach(t),fl=h(e),V=o(e,"OL",{});var ys=i(V);_t=o(ys,"LI",{});var rn=i(_t);Zr=n(rn,"Define your training hyperparameters in "),os=o(rn,"A",{href:!0});var dp=i(os);eo=n(dp,"TrainingArguments"),dp.forEach(t),to=n(rn,"."),rn.forEach(t),so=h(ys),$t=o(ys,"LI",{});var on=i($t);ao=n(on,"Pass the training arguments to "),is=o(on,"A",{href:!0});var gp=i(is);lo=n(gp,"Trainer"),gp.forEach(t),no=n(on," along with the model, datasets, and data collator."),on.forEach(t),ro=h(ys),wt=o(ys,"LI",{});var pn=i(wt);oo=n(pn,"Call "),ps=o(pn,"A",{href:!0});var _p=i(ps);io=n(_p,"train()"),_p.forEach(t),po=n(pn," to fine-tune your model."),pn.forEach(t),ys.forEach(t),hl=h(e),_(kt.$$.fragment,e),ml=h(e),re=o(e,"H3",{class:!0});var fn=i(re);be=o(fn,"A",{id:!0,class:!0,href:!0});var $p=i(be);ea=o($p,"SPAN",{});var wp=i(ea);_(jt.$$.fragment,wp),wp.forEach(t),$p.forEach(t),fo=h(fn),ta=o(fn,"SPAN",{});var kp=i(ta);ho=n(kp,"Fine-tune with TensorFlow"),kp.forEach(t),fn.forEach(t),ul=h(e),fs=o(e,"P",{});var jp=i(fs);mo=n(jp,"To fine-tune a model in TensorFlow is just as easy, with only a few differences."),jp.forEach(t),cl=h(e),_(Ee.$$.fragment,e),dl=h(e),G=o(e,"P",{});var Re=i(G);uo=n(Re,"Convert your datasets to the "),sa=o(Re,"CODE",{});var vp=i(sa);co=n(vp,"tf.data.Dataset"),vp.forEach(t),go=n(Re," format with "),vt=o(Re,"A",{href:!0,rel:!0});var yp=i(vt);aa=o(yp,"CODE",{});var bp=i(aa);_o=n(bp,"to_tf_dataset"),bp.forEach(t),yp.forEach(t),$o=n(Re,". Specify inputs and labels in "),la=o(Re,"CODE",{});var Ep=i(la);wo=n(Ep,"columns"),Ep.forEach(t),ko=n(Re,", whether to shuffle the dataset order, batch size, and the data collator:"),Re.forEach(t),gl=h(e),_(yt.$$.fragment,e),_l=h(e),hs=o(e,"P",{});var xp=i(hs);jo=n(xp,"Set up an optimizer function, learning rate, and some training hyperparameters:"),xp.forEach(t),$l=h(e),_(bt.$$.fragment,e),wl=h(e),xe=o(e,"P",{});var hn=i(xe);vo=n(hn,"Load DistilGPT2 with "),ms=o(hn,"A",{href:!0});var Tp=i(ms);yo=n(Tp,"TFAutoModelForCausalLM"),Tp.forEach(t),bo=n(hn,":"),hn.forEach(t),kl=h(e),_(Et.$$.fragment,e),jl=h(e),Te=o(e,"P",{});var mn=i(Te);Eo=n(mn,"Configure the model for training with "),xt=o(mn,"A",{href:!0,rel:!0});var Ap=i(xt);na=o(Ap,"CODE",{});var qp=i(na);xo=n(qp,"compile"),qp.forEach(t),Ap.forEach(t),To=n(mn,":"),mn.forEach(t),vl=h(e),_(Tt.$$.fragment,e),yl=h(e),Ae=o(e,"P",{});var un=i(Ae);Ao=n(un,"Call "),At=o(un,"A",{href:!0,rel:!0});var Fp=i(At);ra=o(Fp,"CODE",{});var Mp=i(ra);qo=n(Mp,"fit"),Mp.forEach(t),Fp.forEach(t),Fo=n(un," to fine-tune the model:"),un.forEach(t),bl=h(e),_(qt.$$.fragment,e),El=h(e),oe=o(e,"H2",{class:!0});var cn=i(oe);qe=o(cn,"A",{id:!0,class:!0,href:!0});var Cp=i(qe);oa=o(Cp,"SPAN",{});var Dp=i(oa);_(Ft.$$.fragment,Dp),Dp.forEach(t),Cp.forEach(t),Mo=h(cn),ia=o(cn,"SPAN",{});var zp=i(ia);Co=n(zp,"Masked language modeling"),zp.forEach(t),cn.forEach(t),xl=h(e),Fe=o(e,"P",{});var dn=i(Fe);Do=n(dn,"Masked language modeling is also known as a fill-mask task because it predicts a masked token in a sequence. Models for masked language modeling require a good contextual understanding of an entire sequence instead of only the left context. This section shows you how to fine-tune "),Mt=o(dn,"A",{href:!0,rel:!0});var Pp=i(Mt);zo=n(Pp,"DistilRoBERTa"),Pp.forEach(t),Po=n(dn," to predict a masked word."),dn.forEach(t),Tl=h(e),ie=o(e,"H3",{class:!0});var gn=i(ie);Me=o(gn,"A",{id:!0,class:!0,href:!0});var Lp=i(Me);pa=o(Lp,"SPAN",{});var Ip=i(pa);_(Ct.$$.fragment,Ip),Ip.forEach(t),Lp.forEach(t),Lo=h(gn),fa=o(gn,"SPAN",{});var Sp=i(fa);Io=n(Sp,"Fine-tune with Trainer"),Sp.forEach(t),gn.forEach(t),Al=h(e),Ce=o(e,"P",{});var _n=i(Ce);So=n(_n,"Load DistilRoBERTa with "),ha=o(_n,"CODE",{});var Op=i(ha);Oo=n(Op,"AutoModelForMaskedlM"),Op.forEach(t),No=n(_n,":"),_n.forEach(t),ql=h(e),_(Dt.$$.fragment,e),Fl=h(e),_(De.$$.fragment,e),Ml=h(e),us=o(e,"P",{});var Np=i(us);Ro=n(Np,"At this point, only three steps remain:"),Np.forEach(t),Cl=h(e),X=o(e,"OL",{});var bs=i(X);zt=o(bs,"LI",{});var $n=i(zt);Bo=n($n,"Define your training hyperparameters in "),cs=o($n,"A",{href:!0});var Rp=i(cs);Go=n(Rp,"TrainingArguments"),Rp.forEach(t),Ho=n($n,"."),$n.forEach(t),Wo=h(bs),Pt=o(bs,"LI",{});var wn=i(Pt);Yo=n(wn,"Pass the training arguments to "),ds=o(wn,"A",{href:!0});var Bp=i(ds);Uo=n(Bp,"Trainer"),Bp.forEach(t),Jo=n(wn," along with the model, datasets, and data collator."),wn.forEach(t),Ko=h(bs),Lt=o(bs,"LI",{});var kn=i(Lt);Qo=n(kn,"Call "),gs=o(kn,"A",{href:!0});var Gp=i(gs);Vo=n(Gp,"train()"),Gp.forEach(t),Xo=n(kn," to fine-tune your model."),kn.forEach(t),bs.forEach(t),Dl=h(e),_(It.$$.fragment,e),zl=h(e),pe=o(e,"H3",{class:!0});var jn=i(pe);ze=o(jn,"A",{id:!0,class:!0,href:!0});var Hp=i(ze);ma=o(Hp,"SPAN",{});var Wp=i(ma);_(St.$$.fragment,Wp),Wp.forEach(t),Hp.forEach(t),Zo=h(jn),ua=o(jn,"SPAN",{});var Yp=i(ua);ei=n(Yp,"Fine-tune with TensorFlow"),Yp.forEach(t),jn.forEach(t),Pl=h(e),_s=o(e,"P",{});var Up=i(_s);ti=n(Up,"To fine-tune a model in TensorFlow is just as easy, with only a few differences."),Up.forEach(t),Ll=h(e),_(Pe.$$.fragment,e),Il=h(e),H=o(e,"P",{});var Be=i(H);si=n(Be,"Convert your datasets to the "),ca=o(Be,"CODE",{});var Jp=i(ca);ai=n(Jp,"tf.data.Dataset"),Jp.forEach(t),li=n(Be," format with "),Ot=o(Be,"A",{href:!0,rel:!0});var Kp=i(Ot);da=o(Kp,"CODE",{});var Qp=i(da);ni=n(Qp,"to_tf_dataset"),Qp.forEach(t),Kp.forEach(t),ri=n(Be,". Specify inputs and labels in "),ga=o(Be,"CODE",{});var Vp=i(ga);oi=n(Vp,"columns"),Vp.forEach(t),ii=n(Be,", whether to shuffle the dataset order, batch size, and the data collator:"),Be.forEach(t),Sl=h(e),_(Nt.$$.fragment,e),Ol=h(e),$s=o(e,"P",{});var Xp=i($s);pi=n(Xp,"Set up an optimizer function, learning rate, and some training hyperparameters:"),Xp.forEach(t),Nl=h(e),_(Rt.$$.fragment,e),Rl=h(e),Le=o(e,"P",{});var vn=i(Le);fi=n(vn,"Load DistilRoBERTa with "),ws=o(vn,"A",{href:!0});var Zp=i(ws);hi=n(Zp,"TFAutoModelForMaskedLM"),Zp.forEach(t),mi=n(vn,":"),vn.forEach(t),Bl=h(e),_(Bt.$$.fragment,e),Gl=h(e),Ie=o(e,"P",{});var yn=i(Ie);ui=n(yn,"Configure the model for training with "),Gt=o(yn,"A",{href:!0,rel:!0});var ef=i(Gt);_a=o(ef,"CODE",{});var tf=i(_a);ci=n(tf,"compile"),tf.forEach(t),ef.forEach(t),di=n(yn,":"),yn.forEach(t),Hl=h(e),_(Ht.$$.fragment,e),Wl=h(e),Se=o(e,"P",{});var bn=i(Se);gi=n(bn,"Call "),Wt=o(bn,"A",{href:!0,rel:!0});var sf=i(Wt);$a=o(sf,"CODE",{});var af=i($a);_i=n(af,"fit"),af.forEach(t),sf.forEach(t),$i=n(bn," to fine-tune the model:"),bn.forEach(t),Yl=h(e),_(Yt.$$.fragment,e),Ul=h(e),_(Oe.$$.fragment,e),this.h()},h(){m(u,"name","hf:doc:metadata"),m(u,"content",JSON.stringify(_f)),m(v,"id","language-modeling"),m(v,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),m(v,"href","#language-modeling"),m(c,"class","relative group"),m(I,"href","https://huggingface.co/distilgpt2"),m(I,"rel","nofollow"),m(We,"href","https://huggingface.co/distilroberta-base"),m(We,"rel","nofollow"),m(Ye,"href","https://www.reddit.com/r/askscience/"),m(Ye,"rel","nofollow"),m(Ue,"href","https://huggingface.co/datasets/eli5"),m(Ue,"rel","nofollow"),m(ue,"id","load-eli5-dataset"),m(ue,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),m(ue,"href","#load-eli5-dataset"),m(se,"class","relative group"),m(ce,"id","preprocess"),m(ce,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),m(ce,"href","#preprocess"),m(ae,"class","relative group"),m(at,"href","https://huggingface.co/docs/datasets/process.html#flatten"),m(at,"rel","nofollow"),m(rt,"href","https://huggingface.co/docs/datasets/package_reference/main_classes.html#datasets.Dataset.map"),m(rt,"rel","nofollow"),m(as,"href","/docs/transformers/pr_15839/en/main_classes/data_collator#transformers.DataCollatorForLanguageModeling"),m(ls,"href","/docs/transformers/pr_15839/en/main_classes/data_collator#transformers.DataCollatorForLanguageModeling"),m(we,"id","causal-language-modeling"),m(we,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),m(we,"href","#causal-language-modeling"),m(le,"class","relative group"),m(ct,"href","https://huggingface.co/distilgpt2"),m(ct,"rel","nofollow"),m(je,"id","finetune-with-trainer"),m(je,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),m(je,"href","#finetune-with-trainer"),m(ne,"class","relative group"),m(ns,"href","/docs/transformers/pr_15839/en/model_doc/auto#transformers.AutoModelForCausalLM"),m(os,"href","/docs/transformers/pr_15839/en/main_classes/trainer#transformers.TrainingArguments"),m(is,"href","/docs/transformers/pr_15839/en/main_classes/trainer#transformers.Trainer"),m(ps,"href","/docs/transformers/pr_15839/en/main_classes/trainer#transformers.Trainer.train"),m(be,"id","finetune-with-tensorflow"),m(be,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),m(be,"href","#finetune-with-tensorflow"),m(re,"class","relative group"),m(vt,"href","https://huggingface.co/docs/datasets/package_reference/main_classes.html#datasets.Dataset.to_tf_dataset"),m(vt,"rel","nofollow"),m(ms,"href","/docs/transformers/pr_15839/en/model_doc/auto#transformers.TFAutoModelForCausalLM"),m(xt,"href","https://keras.io/api/models/model_training_apis/#compile-method"),m(xt,"rel","nofollow"),m(At,"href","https://keras.io/api/models/model_training_apis/#fit-method"),m(At,"rel","nofollow"),m(qe,"id","masked-language-modeling"),m(qe,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),m(qe,"href","#masked-language-modeling"),m(oe,"class","relative group"),m(Mt,"href","https://huggingface.co/distilroberta-base"),m(Mt,"rel","nofollow"),m(Me,"id","finetune-with-trainer"),m(Me,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),m(Me,"href","#finetune-with-trainer"),m(ie,"class","relative group"),m(cs,"href","/docs/transformers/pr_15839/en/main_classes/trainer#transformers.TrainingArguments"),m(ds,"href","/docs/transformers/pr_15839/en/main_classes/trainer#transformers.Trainer"),m(gs,"href","/docs/transformers/pr_15839/en/main_classes/trainer#transformers.Trainer.train"),m(ze,"id","finetune-with-tensorflow"),m(ze,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),m(ze,"href","#finetune-with-tensorflow"),m(pe,"class","relative group"),m(Ot,"href","https://huggingface.co/docs/datasets/package_reference/main_classes.html#datasets.Dataset.to_tf_dataset"),m(Ot,"rel","nofollow"),m(ws,"href","/docs/transformers/pr_15839/en/model_doc/auto#transformers.TFAutoModelForMaskedLM"),m(Gt,"href","https://keras.io/api/models/model_training_apis/#compile-method"),m(Gt,"rel","nofollow"),m(Wt,"href","https://keras.io/api/models/model_training_apis/#fit-method"),m(Wt,"rel","nofollow")},m(e,a){s(document.head,u),p(e,E,a),p(e,c,a),s(c,v),s(v,x),$(d,x,null),s(c,b),s(c,A),s(A,T),p(e,y,a),p(e,F,a),s(F,C),p(e,L,a),$(W,e,a),p(e,N,a),p(e,U,a),s(U,Ge),p(e,R,a),$(Y,e,a),p(e,he,a),p(e,M,a),s(M,z),p(e,He,a),p(e,D,a),s(D,Jt),s(D,I),s(I,Kt),s(D,Qt),s(D,We),s(We,xn),s(D,Tn),s(D,Ye),s(Ye,An),s(D,qn),s(D,Ue),s(Ue,Fn),s(D,Mn),p(e,ba,a),$(me,e,a),p(e,Ea,a),p(e,se,a),s(se,ue),s(ue,xs),$(Je,xs,null),s(se,Cn),s(se,Ts),s(Ts,Dn),p(e,xa,a),p(e,Vt,a),s(Vt,zn),p(e,Ta,a),$(Ke,e,a),p(e,Aa,a),p(e,Xt,a),s(Xt,Pn),p(e,qa,a),$(Qe,e,a),p(e,Fa,a),p(e,Zt,a),s(Zt,Ln),p(e,Ma,a),$(Ve,e,a),p(e,Ca,a),p(e,B,a),s(B,In),s(B,As),s(As,Sn),s(B,On),s(B,qs),s(qs,Nn),s(B,Rn),s(B,Fs),s(Fs,Bn),s(B,Gn),p(e,Da,a),p(e,ae,a),s(ae,ce),s(ce,Ms),$(Xe,Ms,null),s(ae,Hn),s(ae,Cs),s(Cs,Wn),p(e,za,a),$(Ze,e,a),p(e,Pa,a),p(e,de,a),s(de,Yn),s(de,Ds),s(Ds,Un),s(de,Jn),p(e,La,a),$(et,e,a),p(e,Ia,a),$(tt,e,a),p(e,Sa,a),p(e,es,a),s(es,Kn),p(e,Oa,a),$(st,e,a),p(e,Na,a),p(e,J,a),s(J,Qn),s(J,zs),s(zs,Vn),s(J,Xn),s(J,at),s(at,Ps),s(Ps,Zn),s(J,er),p(e,Ra,a),$(lt,e,a),p(e,Ba,a),p(e,K,a),s(K,tr),s(K,Ls),s(Ls,sr),s(K,ar),s(K,Is),s(Is,lr),s(K,nr),p(e,Ga,a),p(e,ts,a),s(ts,rr),p(e,Ha,a),$(nt,e,a),p(e,Wa,a),p(e,S,a),s(S,or),s(S,rt),s(rt,Ss),s(Ss,ir),s(S,pr),s(S,Os),s(Os,fr),s(S,hr),s(S,Ns),s(Ns,mr),s(S,ur),s(S,Rs),s(Rs,cr),s(S,dr),p(e,Ya,a),$(ot,e,a),p(e,Ua,a),p(e,ss,a),s(ss,gr),p(e,Ja,a),p(e,ge,a),s(ge,Bs),s(Bs,_r),s(ge,$r),s(ge,it),s(it,wr),s(it,Gs),s(Gs,kr),s(it,jr),p(e,Ka,a),$(pt,e,a),p(e,Qa,a),p(e,_e,a),s(_e,vr),s(_e,Hs),s(Hs,yr),s(_e,br),p(e,Va,a),$(ft,e,a),p(e,Xa,a),p(e,O,a),s(O,Er),s(O,as),s(as,xr),s(O,Tr),s(O,Ws),s(Ws,Ar),s(O,qr),s(O,Ys),s(Ys,Fr),s(O,Mr),s(O,Us),s(Us,Cr),s(O,Dr),p(e,Za,a),p(e,$e,a),s($e,zr),s($e,Js),s(Js,Pr),s($e,Lr),p(e,el,a),$(ht,e,a),p(e,tl,a),p(e,Q,a),s(Q,Ir),s(Q,ls),s(ls,Sr),s(Q,Or),s(Q,Ks),s(Ks,Nr),s(Q,Rr),p(e,sl,a),$(mt,e,a),p(e,al,a),p(e,le,a),s(le,we),s(we,Qs),$(ut,Qs,null),s(le,Br),s(le,Vs),s(Vs,Gr),p(e,ll,a),p(e,ke,a),s(ke,Hr),s(ke,ct),s(ct,Wr),s(ke,Yr),p(e,nl,a),p(e,ne,a),s(ne,je),s(je,Xs),$(dt,Xs,null),s(ne,Ur),s(ne,Zs),s(Zs,Jr),p(e,rl,a),p(e,ve,a),s(ve,Kr),s(ve,ns),s(ns,Qr),s(ve,Vr),p(e,ol,a),$(gt,e,a),p(e,il,a),$(ye,e,a),p(e,pl,a),p(e,rs,a),s(rs,Xr),p(e,fl,a),p(e,V,a),s(V,_t),s(_t,Zr),s(_t,os),s(os,eo),s(_t,to),s(V,so),s(V,$t),s($t,ao),s($t,is),s(is,lo),s($t,no),s(V,ro),s(V,wt),s(wt,oo),s(wt,ps),s(ps,io),s(wt,po),p(e,hl,a),$(kt,e,a),p(e,ml,a),p(e,re,a),s(re,be),s(be,ea),$(jt,ea,null),s(re,fo),s(re,ta),s(ta,ho),p(e,ul,a),p(e,fs,a),s(fs,mo),p(e,cl,a),$(Ee,e,a),p(e,dl,a),p(e,G,a),s(G,uo),s(G,sa),s(sa,co),s(G,go),s(G,vt),s(vt,aa),s(aa,_o),s(G,$o),s(G,la),s(la,wo),s(G,ko),p(e,gl,a),$(yt,e,a),p(e,_l,a),p(e,hs,a),s(hs,jo),p(e,$l,a),$(bt,e,a),p(e,wl,a),p(e,xe,a),s(xe,vo),s(xe,ms),s(ms,yo),s(xe,bo),p(e,kl,a),$(Et,e,a),p(e,jl,a),p(e,Te,a),s(Te,Eo),s(Te,xt),s(xt,na),s(na,xo),s(Te,To),p(e,vl,a),$(Tt,e,a),p(e,yl,a),p(e,Ae,a),s(Ae,Ao),s(Ae,At),s(At,ra),s(ra,qo),s(Ae,Fo),p(e,bl,a),$(qt,e,a),p(e,El,a),p(e,oe,a),s(oe,qe),s(qe,oa),$(Ft,oa,null),s(oe,Mo),s(oe,ia),s(ia,Co),p(e,xl,a),p(e,Fe,a),s(Fe,Do),s(Fe,Mt),s(Mt,zo),s(Fe,Po),p(e,Tl,a),p(e,ie,a),s(ie,Me),s(Me,pa),$(Ct,pa,null),s(ie,Lo),s(ie,fa),s(fa,Io),p(e,Al,a),p(e,Ce,a),s(Ce,So),s(Ce,ha),s(ha,Oo),s(Ce,No),p(e,ql,a),$(Dt,e,a),p(e,Fl,a),$(De,e,a),p(e,Ml,a),p(e,us,a),s(us,Ro),p(e,Cl,a),p(e,X,a),s(X,zt),s(zt,Bo),s(zt,cs),s(cs,Go),s(zt,Ho),s(X,Wo),s(X,Pt),s(Pt,Yo),s(Pt,ds),s(ds,Uo),s(Pt,Jo),s(X,Ko),s(X,Lt),s(Lt,Qo),s(Lt,gs),s(gs,Vo),s(Lt,Xo),p(e,Dl,a),$(It,e,a),p(e,zl,a),p(e,pe,a),s(pe,ze),s(ze,ma),$(St,ma,null),s(pe,Zo),s(pe,ua),s(ua,ei),p(e,Pl,a),p(e,_s,a),s(_s,ti),p(e,Ll,a),$(Pe,e,a),p(e,Il,a),p(e,H,a),s(H,si),s(H,ca),s(ca,ai),s(H,li),s(H,Ot),s(Ot,da),s(da,ni),s(H,ri),s(H,ga),s(ga,oi),s(H,ii),p(e,Sl,a),$(Nt,e,a),p(e,Ol,a),p(e,$s,a),s($s,pi),p(e,Nl,a),$(Rt,e,a),p(e,Rl,a),p(e,Le,a),s(Le,fi),s(Le,ws),s(ws,hi),s(Le,mi),p(e,Bl,a),$(Bt,e,a),p(e,Gl,a),p(e,Ie,a),s(Ie,ui),s(Ie,Gt),s(Gt,_a),s(_a,ci),s(Ie,di),p(e,Hl,a),$(Ht,e,a),p(e,Wl,a),p(e,Se,a),s(Se,gi),s(Se,Wt),s(Wt,$a),s($a,_i),s(Se,$i),p(e,Yl,a),$(Yt,e,a),p(e,Ul,a),$(Oe,e,a),Jl=!0},p(e,[a]){const Ut={};a&2&&(Ut.$$scope={dirty:a,ctx:e}),me.$set(Ut);const wa={};a&2&&(wa.$$scope={dirty:a,ctx:e}),ye.$set(wa);const ka={};a&2&&(ka.$$scope={dirty:a,ctx:e}),Ee.$set(ka);const ja={};a&2&&(ja.$$scope={dirty:a,ctx:e}),De.$set(ja);const va={};a&2&&(va.$$scope={dirty:a,ctx:e}),Pe.$set(va);const ya={};a&2&&(ya.$$scope={dirty:a,ctx:e}),Oe.$set(ya)},i(e){Jl||(w(d.$$.fragment,e),w(W.$$.fragment,e),w(Y.$$.fragment,e),w(me.$$.fragment,e),w(Je.$$.fragment,e),w(Ke.$$.fragment,e),w(Qe.$$.fragment,e),w(Ve.$$.fragment,e),w(Xe.$$.fragment,e),w(Ze.$$.fragment,e),w(et.$$.fragment,e),w(tt.$$.fragment,e),w(st.$$.fragment,e),w(lt.$$.fragment,e),w(nt.$$.fragment,e),w(ot.$$.fragment,e),w(pt.$$.fragment,e),w(ft.$$.fragment,e),w(ht.$$.fragment,e),w(mt.$$.fragment,e),w(ut.$$.fragment,e),w(dt.$$.fragment,e),w(gt.$$.fragment,e),w(ye.$$.fragment,e),w(kt.$$.fragment,e),w(jt.$$.fragment,e),w(Ee.$$.fragment,e),w(yt.$$.fragment,e),w(bt.$$.fragment,e),w(Et.$$.fragment,e),w(Tt.$$.fragment,e),w(qt.$$.fragment,e),w(Ft.$$.fragment,e),w(Ct.$$.fragment,e),w(Dt.$$.fragment,e),w(De.$$.fragment,e),w(It.$$.fragment,e),w(St.$$.fragment,e),w(Pe.$$.fragment,e),w(Nt.$$.fragment,e),w(Rt.$$.fragment,e),w(Bt.$$.fragment,e),w(Ht.$$.fragment,e),w(Yt.$$.fragment,e),w(Oe.$$.fragment,e),Jl=!0)},o(e){k(d.$$.fragment,e),k(W.$$.fragment,e),k(Y.$$.fragment,e),k(me.$$.fragment,e),k(Je.$$.fragment,e),k(Ke.$$.fragment,e),k(Qe.$$.fragment,e),k(Ve.$$.fragment,e),k(Xe.$$.fragment,e),k(Ze.$$.fragment,e),k(et.$$.fragment,e),k(tt.$$.fragment,e),k(st.$$.fragment,e),k(lt.$$.fragment,e),k(nt.$$.fragment,e),k(ot.$$.fragment,e),k(pt.$$.fragment,e),k(ft.$$.fragment,e),k(ht.$$.fragment,e),k(mt.$$.fragment,e),k(ut.$$.fragment,e),k(dt.$$.fragment,e),k(gt.$$.fragment,e),k(ye.$$.fragment,e),k(kt.$$.fragment,e),k(jt.$$.fragment,e),k(Ee.$$.fragment,e),k(yt.$$.fragment,e),k(bt.$$.fragment,e),k(Et.$$.fragment,e),k(Tt.$$.fragment,e),k(qt.$$.fragment,e),k(Ft.$$.fragment,e),k(Ct.$$.fragment,e),k(Dt.$$.fragment,e),k(De.$$.fragment,e),k(It.$$.fragment,e),k(St.$$.fragment,e),k(Pe.$$.fragment,e),k(Nt.$$.fragment,e),k(Rt.$$.fragment,e),k(Bt.$$.fragment,e),k(Ht.$$.fragment,e),k(Yt.$$.fragment,e),k(Oe.$$.fragment,e),Jl=!1},d(e){t(u),e&&t(E),e&&t(c),j(d),e&&t(y),e&&t(F),e&&t(L),j(W,e),e&&t(N),e&&t(U),e&&t(R),j(Y,e),e&&t(he),e&&t(M),e&&t(He),e&&t(D),e&&t(ba),j(me,e),e&&t(Ea),e&&t(se),j(Je),e&&t(xa),e&&t(Vt),e&&t(Ta),j(Ke,e),e&&t(Aa),e&&t(Xt),e&&t(qa),j(Qe,e),e&&t(Fa),e&&t(Zt),e&&t(Ma),j(Ve,e),e&&t(Ca),e&&t(B),e&&t(Da),e&&t(ae),j(Xe),e&&t(za),j(Ze,e),e&&t(Pa),e&&t(de),e&&t(La),j(et,e),e&&t(Ia),j(tt,e),e&&t(Sa),e&&t(es),e&&t(Oa),j(st,e),e&&t(Na),e&&t(J),e&&t(Ra),j(lt,e),e&&t(Ba),e&&t(K),e&&t(Ga),e&&t(ts),e&&t(Ha),j(nt,e),e&&t(Wa),e&&t(S),e&&t(Ya),j(ot,e),e&&t(Ua),e&&t(ss),e&&t(Ja),e&&t(ge),e&&t(Ka),j(pt,e),e&&t(Qa),e&&t(_e),e&&t(Va),j(ft,e),e&&t(Xa),e&&t(O),e&&t(Za),e&&t($e),e&&t(el),j(ht,e),e&&t(tl),e&&t(Q),e&&t(sl),j(mt,e),e&&t(al),e&&t(le),j(ut),e&&t(ll),e&&t(ke),e&&t(nl),e&&t(ne),j(dt),e&&t(rl),e&&t(ve),e&&t(ol),j(gt,e),e&&t(il),j(ye,e),e&&t(pl),e&&t(rs),e&&t(fl),e&&t(V),e&&t(hl),j(kt,e),e&&t(ml),e&&t(re),j(jt),e&&t(ul),e&&t(fs),e&&t(cl),j(Ee,e),e&&t(dl),e&&t(G),e&&t(gl),j(yt,e),e&&t(_l),e&&t(hs),e&&t($l),j(bt,e),e&&t(wl),e&&t(xe),e&&t(kl),j(Et,e),e&&t(jl),e&&t(Te),e&&t(vl),j(Tt,e),e&&t(yl),e&&t(Ae),e&&t(bl),j(qt,e),e&&t(El),e&&t(oe),j(Ft),e&&t(xl),e&&t(Fe),e&&t(Tl),e&&t(ie),j(Ct),e&&t(Al),e&&t(Ce),e&&t(ql),j(Dt,e),e&&t(Fl),j(De,e),e&&t(Ml),e&&t(us),e&&t(Cl),e&&t(X),e&&t(Dl),j(It,e),e&&t(zl),e&&t(pe),j(St),e&&t(Pl),e&&t(_s),e&&t(Ll),j(Pe,e),e&&t(Il),e&&t(H),e&&t(Sl),j(Nt,e),e&&t(Ol),e&&t($s),e&&t(Nl),j(Rt,e),e&&t(Rl),e&&t(Le),e&&t(Bl),j(Bt,e),e&&t(Gl),e&&t(Ie),e&&t(Hl),j(Ht,e),e&&t(Wl),e&&t(Se),e&&t(Yl),j(Yt,e),e&&t(Ul),j(Oe,e)}}}const _f={local:"language-modeling",sections:[{local:"load-eli5-dataset",title:"Load ELI5 dataset"},{local:"preprocess",title:"Preprocess"},{local:"causal-language-modeling",sections:[{local:"finetune-with-trainer",title:"Fine-tune with Trainer"},{local:"finetune-with-tensorflow",title:"Fine-tune with TensorFlow"}],title:"Causal language modeling"},{local:"masked-language-modeling",sections:[{local:"finetune-with-trainer",title:"Fine-tune with Trainer"},{local:"finetune-with-tensorflow",title:"Fine-tune with TensorFlow"}],title:"Masked language modeling"}],title:"Language modeling"};function $f(P,u,E){let{fw:c}=u;return P.$$set=v=>{"fw"in v&&E(0,c=v.fw)},[c]}class xf extends nf{constructor(u){super();rf(this,u,$f,gf,of,{fw:0})}}export{xf as default,_f as metadata};
