import{S as Pc,i as zc,s as qc,e as r,k as i,w as b,t as n,M as xc,c as o,d as e,m as u,x as f,a as p,h as l,b as m,F as a,g as h,y as d,q as j,o as y,B as g,v as Ic}from"../../chunks/vendor-1e8b365d.js";import{T as Al}from"../../chunks/Tip-62b14c6e.js";import{Y as Sc}from"../../chunks/Youtube-c2a8cc39.js";import{I as B}from"../../chunks/IconCopyLink-483c28ba.js";import{C as T}from"../../chunks/CodeBlock-e5764662.js";import{D as Nc}from"../../chunks/DocNotebookDropdown-37d928d3.js";import{F as Fc}from"../../chunks/FrameworkSwitchCourse-7f8f0f31.js";function Wc(S){let c,E,w,k,D,C,P,q,os,ps,re,Y,H,$s,z,N,R,hs,oe,is,ks,V,_s;return{c(){c=r("p"),E=n("\u270F\uFE0F "),w=r("strong"),k=n("Your turn!"),D=n(" As an optional challenge after we\u2019ve resolved the other issues, you can try coming back to this step and getting the model to work with the original Keras-computed loss instead of the internal loss. You\u2019ll need to add "),C=r("code"),P=n('"labels"'),q=n(" to the "),os=r("code"),ps=n("label_cols"),re=n(" argument of "),Y=r("code"),H=n("to_tf_dataset()"),$s=n(" to ensure that the labels are correctly outputted, which will get you gradients \u2014 but there\u2019s one more problem with the loss that we specified. Training will still run with this problem, but learning will be very slow and will plateau at a high training loss. Can you figure out what it is?"),z=i(),N=r("p"),R=n("A ROT13-encoded hint, if you\u2019re stuck: Vs lbh ybbx ng gur bhgchgf bs FrdhraprPynffvsvpngvba zbqryf va Genafsbezref, gurve svefg bhgchg vf "),hs=r("code"),oe=n("ybtvgf"),is=n(". Jung ner ybtvgf?"),ks=i(),V=r("p"),_s=n("And a second hint: Jura lbh fcrpvsl bcgvzvmref, npgvingvbaf be ybffrf jvgu fgevatf, Xrenf frgf nyy gur nethzrag inyhrf gb gurve qrsnhygf. Jung nethzragf qbrf FcnefrPngrtbevpnyPebffragebcl unir, naq jung ner gurve qrsnhygf?")},l(_){c=o(_,"P",{});var A=p(c);E=l(A,"\u270F\uFE0F "),w=o(A,"STRONG",{});var Es=p(w);k=l(Es,"Your turn!"),Es.forEach(e),D=l(A," As an optional challenge after we\u2019ve resolved the other issues, you can try coming back to this step and getting the model to work with the original Keras-computed loss instead of the internal loss. You\u2019ll need to add "),C=o(A,"CODE",{});var J=p(C);P=l(J,'"labels"'),J.forEach(e),q=l(A," to the "),os=o(A,"CODE",{});var na=p(os);ps=l(na,"label_cols"),na.forEach(e),re=l(A," argument of "),Y=o(A,"CODE",{});var Ts=p(Y);H=l(Ts,"to_tf_dataset()"),Ts.forEach(e),$s=l(A," to ensure that the labels are correctly outputted, which will get you gradients \u2014 but there\u2019s one more problem with the loss that we specified. Training will still run with this problem, but learning will be very slow and will plateau at a high training loss. Can you figure out what it is?"),A.forEach(e),z=u(_),N=o(_,"P",{});var As=p(N);R=l(As,"A ROT13-encoded hint, if you\u2019re stuck: Vs lbh ybbx ng gur bhgchgf bs FrdhraprPynffvsvpngvba zbqryf va Genafsbezref, gurve svefg bhgchg vf "),hs=o(As,"CODE",{});var pe=p(hs);oe=l(pe,"ybtvgf"),pe.forEach(e),is=l(As,". Jung ner ybtvgf?"),As.forEach(e),ks=u(_),V=o(_,"P",{});var X=p(V);_s=l(X,"And a second hint: Jura lbh fcrpvsl bcgvzvmref, npgvingvbaf be ybffrf jvgu fgevatf, Xrenf frgf nyy gur nethzrag inyhrf gb gurve qrsnhygf. Jung nethzragf qbrf FcnefrPngrtbevpnyPebffragebcl unir, naq jung ner gurve qrsnhygf?"),X.forEach(e)},m(_,A){h(_,c,A),a(c,E),a(c,w),a(w,k),a(c,D),a(c,C),a(C,P),a(c,q),a(c,os),a(os,ps),a(c,re),a(c,Y),a(Y,H),a(c,$s),h(_,z,A),h(_,N,A),a(N,R),a(N,hs),a(hs,oe),a(N,is),h(_,ks,A),h(_,V,A),a(V,_s)},d(_){_&&e(c),_&&e(z),_&&e(N),_&&e(ks),_&&e(V)}}}function Lc(S){let c,E,w,k,D;return{c(){c=r("p"),E=n("\u{1F4A1} You can also import the "),w=r("code"),k=n("create_optimizer()"),D=n(" function from \u{1F917} Transformers, which will give you an AdamW optimizer with correct weight decay as well as learning rate warmup and decay. This optimizer will often produce slightly better results than the ones you get with the default Adam optimizer.")},l(C){c=o(C,"P",{});var P=p(c);E=l(P,"\u{1F4A1} You can also import the "),w=o(P,"CODE",{});var q=p(w);k=l(q,"create_optimizer()"),q.forEach(e),D=l(P," function from \u{1F917} Transformers, which will give you an AdamW optimizer with correct weight decay as well as learning rate warmup and decay. This optimizer will often produce slightly better results than the ones you get with the default Adam optimizer."),P.forEach(e)},m(C,P){h(C,c,P),a(c,E),a(c,w),a(w,k),a(c,D)},d(C){C&&e(c)}}}function Mc(S){let c,E;return{c(){c=r("p"),E=n("In the next part of the course, we\u2019ll look at more advanced techniques that can help you reduce your memory footprint and let you fine-tune the biggest models.")},l(w){c=o(w,"P",{});var k=p(c);E=l(k,"In the next part of the course, we\u2019ll look at more advanced techniques that can help you reduce your memory footprint and let you fine-tune the biggest models."),k.forEach(e)},m(w,k){h(w,c,k),a(c,E)},d(w){w&&e(c)}}}function Bc(S){let c,E;return{c(){c=r("p"),E=n("\u{1F4A1} If your training data is unbalanced, make sure to build a batch of training data containing all the labels.")},l(w){c=o(w,"P",{});var k=p(c);E=l(k,"\u{1F4A1} If your training data is unbalanced, make sure to build a batch of training data containing all the labels."),k.forEach(e)},m(w,k){h(w,c,k),a(c,E)},d(w){w&&e(c)}}}function Hc(S){let c,E;return{c(){c=r("p"),E=n("\u26A0\uFE0F You will have to recreate your model and recompile after this overfitting test, as the model obtained probably won\u2019t be able to recover and learn something useful on your full dataset.")},l(w){c=o(w,"P",{});var k=p(c);E=l(k,"\u26A0\uFE0F You will have to recreate your model and recompile after this overfitting test, as the model obtained probably won\u2019t be able to recover and learn something useful on your full dataset."),k.forEach(e)},m(w,k){h(w,c,k),a(c,E)},d(w){w&&e(c)}}}function Gc(S){let c,E,w,k,D,C,P,q,os,ps,re,Y,H,$s,z,N,R,hs,oe,is,ks,V,_s,_,A,Es,J,na,Ts,As,pe,X,Cl,Cs,Mr,Wa,Br,Hr,Ol,Os,Gr,La,Kr,Ur,Dl,Ds,Yr,he,Rr,Vr,Pl,ie,zl,Ps,Jr,Ma,Xr,Zr,ql,la,Qr,xl,ue,Il,ta,so,Sl,us,zs,Ba,me,eo,Ha,ao,Nl,ra,no,Fl,O,lo,Ga,to,ro,Ka,oo,po,Ua,ho,io,Ya,uo,mo,Ra,co,bo,Va,fo,jo,Wl,ce,Ll,G,Ja,yo,go,Xa,wo,vo,Za,$o,ko,Ml,be,Bl,F,_o,Qa,Eo,To,sn,Ao,Co,en,Oo,Do,Hl,qs,Po,an,zo,qo,Gl,Z,xo,nn,Io,So,ln,No,Fo,Kl,fe,Ul,oa,Wo,Yl,xs,Rl,Is,Lo,tn,Mo,Bo,Vl,de,Jl,pa,Ho,Xl,je,rn,Go,Ko,Zl,ms,Ss,on,ye,Uo,pn,Yo,Ql,K,hn,Ro,Vo,un,Jo,Xo,mn,Zo,Qo,st,Q,sp,cn,ep,ap,bn,np,lp,et,ge,at,we,nt,v,tp,fn,rp,op,dn,pp,hp,jn,ip,up,yn,mp,cp,gn,bp,fp,wn,dp,jp,vn,yp,gp,$n,wp,vp,kn,$p,kp,_n,_p,Ep,En,Tp,Ap,Tn,Cp,Op,An,Dp,Pp,Cn,zp,qp,On,xp,Ip,lt,W,Sp,Dn,Np,Fp,Pn,Wp,Lp,zn,Mp,Bp,tt,ve,rt,ha,Hp,ot,$e,pt,U,qn,Gp,Kp,xn,Up,Yp,In,Rp,Vp,ht,ke,it,_e,ut,ia,Jp,mt,Ee,ct,Te,bt,ua,Xp,ft,Ae,dt,Ce,jt,ss,Zp,Sn,Qp,sh,Nn,eh,ah,yt,Oe,gt,De,wt,Ns,nh,Fn,lh,th,vt,Pe,$t,ze,kt,Fs,rh,Wn,oh,ph,_t,cs,Ws,Ln,qe,hh,Mn,ih,Et,es,uh,Bn,mh,ch,Hn,bh,fh,Tt,Ls,dh,xe,jh,yh,At,as,gh,Gn,wh,vh,Kn,$h,kh,Ct,Ie,Ot,Ms,Dt,ma,_h,Pt,Se,zt,Ne,qt,ca,Eh,xt,bs,Bs,Un,Fe,Th,Yn,Ah,It,ba,Ch,St,fs,Hs,Rn,We,Oh,Vn,Dh,Nt,ns,Ph,Jn,zh,qh,Xn,xh,Ih,Ft,Gs,Wt,ds,Ks,Zn,Le,Sh,Qn,Nh,Lt,ls,Fh,sl,Wh,Lh,el,Mh,Bh,Mt,fa,Hh,Bt,Us,Gh,al,Kh,Uh,Ht,js,Ys,nl,Me,Yh,ll,Rh,Gt,x,Vh,tl,Jh,Xh,rl,Zh,Qh,ol,si,ei,pl,ai,ni,Kt,Be,Ut,da,li,Yt,He,Rt,ja,ti,Vt,L,hl,ri,oi,il,pi,hi,ul,ii,ui,ml,mi,Jt,ya,ci,Xt,ga,bi,Zt,wa,fi,Qt,ys,Rs,cl,Ge,di,bl,ji,sr,va,yi,er,ts,gi,fl,wi,vi,dl,$i,ki,ar,Ke,nr,Vs,lr,Js,_i,jl,Ei,Ti,tr,$a,Ai,rr,Xs,or,gs,Zs,yl,Ue,Ci,gl,Oi,pr,Qs,Di,wl,Pi,zi,hr,ka,qi,ir,_a,xi,ur,ws,se,vl,Ye,Ii,$l,Si,mr,ee,Ni,Re,Fi,Wi,cr,Ea,Li,br,M,Ta,Ve,Mi,Bi,Hi,Aa,Je,Gi,Ki,Ui,Ca,Xe,Yi,Ri,Vi,Oa,Ze,Ji,Xi,fr,Da,Zi,dr;return w=new Fc({props:{fw:S[0]}}),q=new B({}),H=new Nc({props:{classNames:"absolute z-10 right-0 top-0",options:[{label:"Google Colab",value:"https://colab.research.google.com/github/huggingface/notebooks/blob/master/course/chapter8/section4_tf.ipynb"},{label:"Aws Studio",value:"https://studiolab.sagemaker.aws/import/github/huggingface/notebooks/blob/master/course/chapter8/section4_tf.ipynb"}]}}),J=new B({}),X=new Sc({props:{id:"N9kO52itd0Q"}}),ie=new T({props:{code:`from datasets import load_dataset, load_metric
from transformers import (
    AutoTokenizer,
    TFAutoModelForSequenceClassification,
)

raw_datasets = load_dataset("glue", "mnli")

model_checkpoint = "distilbert-base-uncased"
tokenizer = AutoTokenizer.from_pretrained(model_checkpoint)


def preprocess_function(examples):
    return tokenizer(examples["premise"], examples["hypothesis"], truncation=True)


tokenized_datasets = raw_datasets.map(preprocess_function, batched=True)

train_dataset = tokenized_datasets["train"].to_tf_dataset(
    columns=["input_ids", "labels"], batch_size=16, shuffle=True
)

validation_dataset = tokenized_datasets["validation_matched"].to_tf_dataset(
    columns=["input_ids", "labels"], batch_size=16, shuffle=True
)

model = TFAutoModelForSequenceClassification.from_pretrained(model_checkpoint)

model.compile(loss="sparse_categorical_crossentropy", optimizer="adam")

model.fit(train_dataset)`,highlighted:`<span class="hljs-keyword">from</span> datasets <span class="hljs-keyword">import</span> load_dataset, load_metric
<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> (
    AutoTokenizer,
    TFAutoModelForSequenceClassification,
)

raw_datasets = load_dataset(<span class="hljs-string">&quot;glue&quot;</span>, <span class="hljs-string">&quot;mnli&quot;</span>)

model_checkpoint = <span class="hljs-string">&quot;distilbert-base-uncased&quot;</span>
tokenizer = AutoTokenizer.from_pretrained(model_checkpoint)


<span class="hljs-keyword">def</span> <span class="hljs-title function_">preprocess_function</span>(<span class="hljs-params">examples</span>):
    <span class="hljs-keyword">return</span> tokenizer(examples[<span class="hljs-string">&quot;premise&quot;</span>], examples[<span class="hljs-string">&quot;hypothesis&quot;</span>], truncation=<span class="hljs-literal">True</span>)


tokenized_datasets = raw_datasets.<span class="hljs-built_in">map</span>(preprocess_function, batched=<span class="hljs-literal">True</span>)

train_dataset = tokenized_datasets[<span class="hljs-string">&quot;train&quot;</span>].to_tf_dataset(
    columns=[<span class="hljs-string">&quot;input_ids&quot;</span>, <span class="hljs-string">&quot;labels&quot;</span>], batch_size=<span class="hljs-number">16</span>, shuffle=<span class="hljs-literal">True</span>
)

validation_dataset = tokenized_datasets[<span class="hljs-string">&quot;validation_matched&quot;</span>].to_tf_dataset(
    columns=[<span class="hljs-string">&quot;input_ids&quot;</span>, <span class="hljs-string">&quot;labels&quot;</span>], batch_size=<span class="hljs-number">16</span>, shuffle=<span class="hljs-literal">True</span>
)

model = TFAutoModelForSequenceClassification.from_pretrained(model_checkpoint)

model.<span class="hljs-built_in">compile</span>(loss=<span class="hljs-string">&quot;sparse_categorical_crossentropy&quot;</span>, optimizer=<span class="hljs-string">&quot;adam&quot;</span>)

model.fit(train_dataset)`}}),ue=new T({props:{code:"ValueError: No gradients provided for any variable: ['tf_distil_bert_for_sequence_classification/distilbert/embeddings/word_embeddings/weight:0', '...']",highlighted:'ValueError: No gradients provided <span class="hljs-keyword">for</span> <span class="hljs-built_in">any</span> variable: [<span class="hljs-string">&#x27;tf_distil_bert_for_sequence_classification/distilbert/embeddings/word_embeddings/weight:0&#x27;</span>, <span class="hljs-string">&#x27;...&#x27;</span>]'}}),me=new B({}),ce=new T({props:{code:`for batch in train_dataset:
    break`,highlighted:`<span class="hljs-keyword">for</span> batch <span class="hljs-keyword">in</span> train_dataset:
    <span class="hljs-keyword">break</span>`}}),be=new T({props:{code:`{'attention_mask': <tf.Tensor: shape=(16, 76), dtype=int64, numpy=
 array([[1, 1, 1, ..., 0, 0, 0],
        [1, 1, 1, ..., 0, 0, 0],
        [1, 1, 1, ..., 0, 0, 0],
        ...,
        [1, 1, 1, ..., 1, 1, 1],
        [1, 1, 1, ..., 0, 0, 0],
        [1, 1, 1, ..., 0, 0, 0]])>,
 'label': <tf.Tensor: shape=(16,), dtype=int64, numpy=array([0, 2, 1, 2, 1, 1, 2, 0, 0, 0, 1, 0, 1, 2, 2, 1])>,
 'input_ids': <tf.Tensor: shape=(16, 76), dtype=int64, numpy=
 array([[ 101, 2174, 1010, ...,    0,    0,    0],
        [ 101, 3174, 2420, ...,    0,    0,    0],
        [ 101, 2044, 2048, ...,    0,    0,    0],
        ...,
        [ 101, 3398, 3398, ..., 2051, 2894,  102],
        [ 101, 1996, 4124, ...,    0,    0,    0],
        [ 101, 1999, 2070, ...,    0,    0,    0]])>}`,highlighted:`{<span class="hljs-string">&#x27;attention_mask&#x27;</span>: &lt;tf.Tensor: shape=(<span class="hljs-number">16</span>, <span class="hljs-number">76</span>), dtype=int64, numpy=
 array([[<span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, ..., <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>],
        [<span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, ..., <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>],
        [<span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, ..., <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>],
        ...,
        [<span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, ..., <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>],
        [<span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, ..., <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>],
        [<span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, ..., <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>]])&gt;,
 <span class="hljs-string">&#x27;label&#x27;</span>: &lt;tf.Tensor: shape=(<span class="hljs-number">16</span>,), dtype=int64, numpy=array([<span class="hljs-number">0</span>, <span class="hljs-number">2</span>, <span class="hljs-number">1</span>, <span class="hljs-number">2</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">2</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">1</span>, <span class="hljs-number">0</span>, <span class="hljs-number">1</span>, <span class="hljs-number">2</span>, <span class="hljs-number">2</span>, <span class="hljs-number">1</span>])&gt;,
 <span class="hljs-string">&#x27;input_ids&#x27;</span>: &lt;tf.Tensor: shape=(<span class="hljs-number">16</span>, <span class="hljs-number">76</span>), dtype=int64, numpy=
 array([[ <span class="hljs-number">101</span>, <span class="hljs-number">2174</span>, <span class="hljs-number">1010</span>, ...,    <span class="hljs-number">0</span>,    <span class="hljs-number">0</span>,    <span class="hljs-number">0</span>],
        [ <span class="hljs-number">101</span>, <span class="hljs-number">3174</span>, <span class="hljs-number">2420</span>, ...,    <span class="hljs-number">0</span>,    <span class="hljs-number">0</span>,    <span class="hljs-number">0</span>],
        [ <span class="hljs-number">101</span>, <span class="hljs-number">2044</span>, <span class="hljs-number">2048</span>, ...,    <span class="hljs-number">0</span>,    <span class="hljs-number">0</span>,    <span class="hljs-number">0</span>],
        ...,
        [ <span class="hljs-number">101</span>, <span class="hljs-number">3398</span>, <span class="hljs-number">3398</span>, ..., <span class="hljs-number">2051</span>, <span class="hljs-number">2894</span>,  <span class="hljs-number">102</span>],
        [ <span class="hljs-number">101</span>, <span class="hljs-number">1996</span>, <span class="hljs-number">4124</span>, ...,    <span class="hljs-number">0</span>,    <span class="hljs-number">0</span>,    <span class="hljs-number">0</span>],
        [ <span class="hljs-number">101</span>, <span class="hljs-number">1999</span>, <span class="hljs-number">2070</span>, ...,    <span class="hljs-number">0</span>,    <span class="hljs-number">0</span>,    <span class="hljs-number">0</span>]])&gt;}`}}),fe=new T({props:{code:'model.compile(optimizer="adam")',highlighted:'model.<span class="hljs-built_in">compile</span>(optimizer=<span class="hljs-string">&quot;adam&quot;</span>)'}}),xs=new Al({props:{$$slots:{default:[Wc]},$$scope:{ctx:S}}}),de=new T({props:{code:"  246/24543 [..............................] - ETA: 15:52 - loss: nan",highlighted:'  <span class="hljs-number">246</span>/<span class="hljs-number">24543</span> [..............................] - ETA: <span class="hljs-number">15</span>:<span class="hljs-number">52</span> - loss: nan'}}),ye=new B({}),ge=new T({props:{code:"model(batch)",highlighted:"model(batch)"}}),we=new T({props:{code:`TFSequenceClassifierOutput(loss=<tf.Tensor: shape=(16,), dtype=float32, numpy=
array([nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan,
       nan, nan, nan], dtype=float32)>, logits=<tf.Tensor: shape=(16, 2), dtype=float32, numpy=
array([[nan, nan],
       [nan, nan],
       [nan, nan],
       [nan, nan],
       [nan, nan],
       [nan, nan],
       [nan, nan],
       [nan, nan],
       [nan, nan],
       [nan, nan],
       [nan, nan],
       [nan, nan],
       [nan, nan],
       [nan, nan],
       [nan, nan],
       [nan, nan]], dtype=float32)>, hidden_states=None, attentions=None)`,highlighted:`TFSequenceClassifierOutput(loss=&lt;tf.Tensor: shape=(<span class="hljs-number">16</span>,), dtype=float32, numpy=
array([nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan,
       nan, nan, nan], dtype=float32)&gt;, logits=&lt;tf.Tensor: shape=(<span class="hljs-number">16</span>, <span class="hljs-number">2</span>), dtype=float32, numpy=
array([[nan, nan],
       [nan, nan],
       [nan, nan],
       [nan, nan],
       [nan, nan],
       [nan, nan],
       [nan, nan],
       [nan, nan],
       [nan, nan],
       [nan, nan],
       [nan, nan],
       [nan, nan],
       [nan, nan],
       [nan, nan],
       [nan, nan],
       [nan, nan]], dtype=float32)&gt;, hidden_states=<span class="hljs-literal">None</span>, attentions=<span class="hljs-literal">None</span>)`}}),ve=new T({props:{code:`model = TFAutoModelForSequenceClassification.from_pretrained(model_checkpoint)
model(batch)`,highlighted:`model = TFAutoModelForSequenceClassification.from_pretrained(model_checkpoint)
model(batch)`}}),$e=new T({props:{code:`TFSequenceClassifierOutput(loss=<tf.Tensor: shape=(16,), dtype=float32, numpy=
array([0.6844486 ,        nan,        nan, 0.67127866, 0.7068601 ,
              nan, 0.69309855,        nan, 0.65531296,        nan,
              nan,        nan, 0.675402  ,        nan,        nan,
       0.69831556], dtype=float32)>, logits=<tf.Tensor: shape=(16, 2), dtype=float32, numpy=
array([[-0.04761693, -0.06509043],
       [-0.0481936 , -0.04556257],
       [-0.0040929 , -0.05848458],
       [-0.02417453, -0.0684005 ],
       [-0.02517801, -0.05241832],
       [-0.04514256, -0.0757378 ],
       [-0.02656011, -0.02646275],
       [ 0.00766164, -0.04350497],
       [ 0.02060014, -0.05655622],
       [-0.02615328, -0.0447021 ],
       [-0.05119278, -0.06928903],
       [-0.02859691, -0.04879177],
       [-0.02210129, -0.05791225],
       [-0.02363213, -0.05962167],
       [-0.05352269, -0.0481673 ],
       [-0.08141848, -0.07110836]], dtype=float32)>, hidden_states=None, attentions=None)`,highlighted:`TFSequenceClassifierOutput(loss=&lt;tf.Tensor: shape=(<span class="hljs-number">16</span>,), dtype=float32, numpy=
array([<span class="hljs-number">0.6844486</span> ,        nan,        nan, <span class="hljs-number">0.67127866</span>, <span class="hljs-number">0.7068601</span> ,
              nan, <span class="hljs-number">0.69309855</span>,        nan, <span class="hljs-number">0.65531296</span>,        nan,
              nan,        nan, <span class="hljs-number">0.675402</span>  ,        nan,        nan,
       <span class="hljs-number">0.69831556</span>], dtype=float32)&gt;, logits=&lt;tf.Tensor: shape=(<span class="hljs-number">16</span>, <span class="hljs-number">2</span>), dtype=float32, numpy=
array([[-<span class="hljs-number">0.04761693</span>, -<span class="hljs-number">0.06509043</span>],
       [-<span class="hljs-number">0.0481936</span> , -<span class="hljs-number">0.04556257</span>],
       [-<span class="hljs-number">0.0040929</span> , -<span class="hljs-number">0.05848458</span>],
       [-<span class="hljs-number">0.02417453</span>, -<span class="hljs-number">0.0684005</span> ],
       [-<span class="hljs-number">0.02517801</span>, -<span class="hljs-number">0.05241832</span>],
       [-<span class="hljs-number">0.04514256</span>, -<span class="hljs-number">0.0757378</span> ],
       [-<span class="hljs-number">0.02656011</span>, -<span class="hljs-number">0.02646275</span>],
       [ <span class="hljs-number">0.00766164</span>, -<span class="hljs-number">0.04350497</span>],
       [ <span class="hljs-number">0.02060014</span>, -<span class="hljs-number">0.05655622</span>],
       [-<span class="hljs-number">0.02615328</span>, -<span class="hljs-number">0.0447021</span> ],
       [-<span class="hljs-number">0.05119278</span>, -<span class="hljs-number">0.06928903</span>],
       [-<span class="hljs-number">0.02859691</span>, -<span class="hljs-number">0.04879177</span>],
       [-<span class="hljs-number">0.02210129</span>, -<span class="hljs-number">0.05791225</span>],
       [-<span class="hljs-number">0.02363213</span>, -<span class="hljs-number">0.05962167</span>],
       [-<span class="hljs-number">0.05352269</span>, -<span class="hljs-number">0.0481673</span> ],
       [-<span class="hljs-number">0.08141848</span>, -<span class="hljs-number">0.07110836</span>]], dtype=float32)&gt;, hidden_states=<span class="hljs-literal">None</span>, attentions=<span class="hljs-literal">None</span>)`}}),ke=new T({props:{code:`import numpy as np

loss = model(batch).loss.numpy()
indices = np.flatnonzero(np.isnan(loss))
indices`,highlighted:`<span class="hljs-keyword">import</span> numpy <span class="hljs-keyword">as</span> np

loss = model(batch).loss.numpy()
indices = np.flatnonzero(np.isnan(loss))
indices`}}),_e=new T({props:{code:"array([ 1,  2,  5,  7,  9, 10, 11, 13, 14])",highlighted:'array([ <span class="hljs-number">1</span>,  <span class="hljs-number">2</span>,  <span class="hljs-number">5</span>,  <span class="hljs-number">7</span>,  <span class="hljs-number">9</span>, <span class="hljs-number">10</span>, <span class="hljs-number">11</span>, <span class="hljs-number">13</span>, <span class="hljs-number">14</span>])'}}),Ee=new T({props:{code:`input_ids = batch["input_ids"].numpy()
input_ids[indices]`,highlighted:`input_ids = batch[<span class="hljs-string">&quot;input_ids&quot;</span>].numpy()
input_ids[indices]`}}),Te=new T({props:{code:`array([[  101,  2007,  2032,  2001,  1037, 16480,  3917,  2594,  4135,
        23212,  3070,  2214, 10170,  1010,  2012,  4356,  1997,  3183,
         6838, 12953,  2039,  2000,  1996,  6147,  1997,  2010,  2606,
         1012,   102,  6838,  2001,  3294,  6625,  3773,  1996,  2214,
         2158,  1012,   102,     0,     0,     0,     0,     0,     0,
            0,     0,     0,     0,     0,     0,     0,     0,     0,
            0,     0,     0,     0,     0,     0,     0,     0,     0,
            0,     0,     0,     0,     0,     0,     0,     0,     0,
            0,     0,     0,     0,     0,     0,     0,     0,     0,
            0,     0,     0,     0],
       [  101,  1998,  6814,  2016,  2234,  2461,  2153,  1998, 13322,
         2009,  1012,   102,  2045,  1005,  1055,  2053,  3382,  2008,
         2016,  1005,  2222,  3046,  8103,  2075,  2009,  2153,  1012,
          102,     0,     0,     0,     0,     0,     0,     0,     0,
            0,     0,     0,     0,     0,     0,     0,     0,     0,
            0,     0,     0,     0,     0,     0,     0,     0,     0,
            0,     0,     0,     0,     0,     0,     0,     0,     0,
            0,     0,     0,     0,     0,     0,     0,     0,     0,
            0,     0,     0,     0,     0,     0,     0,     0,     0,
            0,     0,     0,     0],
       [  101,  1998,  2007,  1996,  3712,  4634,  1010,  2057,  8108,
         2025,  3404,  2028,  1012,  1996,  2616, 18449,  2125,  1999,
         1037,  9666,  1997,  4100,  8663, 11020,  6313,  2791,  1998,
         2431,  1011,  4301,  1012,   102,  2028,  1005,  1055,  5177,
         2110,  1998,  3977,  2000,  2832,  2106,  2025,  2689,  2104,
         2122,  6214,  1012,   102,     0,     0,     0,     0,     0,
            0,     0,     0,     0,     0,     0,     0,     0,     0,
            0,     0,     0,     0,     0,     0,     0,     0,     0,
            0,     0,     0,     0,     0,     0,     0,     0,     0,
            0,     0,     0,     0],
       [  101,  1045,  2001,  1999,  1037, 13090,  5948,  2007,  2048,
         2308,  2006,  2026,  5001,  2043,  2026,  2171,  2001,  2170,
         1012,   102,  1045,  2001,  3564,  1999,  2277,  1012,   102,
            0,     0,     0,     0,     0,     0,     0,     0,     0,
            0,     0,     0,     0,     0,     0,     0,     0,     0,
            0,     0,     0,     0,     0,     0,     0,     0,     0,
            0,     0,     0,     0,     0,     0,     0,     0,     0,
            0,     0,     0,     0,     0,     0,     0,     0,     0,
            0,     0,     0,     0,     0,     0,     0,     0,     0,
            0,     0,     0,     0],
       [  101,  2195,  4279,  2191,  2039,  1996,  2181,  2124,  2004,
         1996,  2225,  7363,  1012,   102,  2045,  2003,  2069,  2028,
         2451,  1999,  1996,  2225,  7363,  1012,   102,     0,     0,
            0,     0,     0,     0,     0,     0,     0,     0,     0,
            0,     0,     0,     0,     0,     0,     0,     0,     0,
            0,     0,     0,     0,     0,     0,     0,     0,     0,
            0,     0,     0,     0,     0,     0,     0,     0,     0,
            0,     0,     0,     0,     0,     0,     0,     0,     0,
            0,     0,     0,     0,     0,     0,     0,     0,     0,
            0,     0,     0,     0],
       [  101,  2061,  2008,  1045,  2123,  1005,  1056,  2113,  2065,
         2009,  2428, 10654,  7347,  2030,  2009,  7126,  2256,  2495,
         2291,   102,  2009,  2003,  5094,  2256,  2495,  2291,  2035,
         2105,  1012,   102,     0,     0,     0,     0,     0,     0,
            0,     0,     0,     0,     0,     0,     0,     0,     0,
            0,     0,     0,     0,     0,     0,     0,     0,     0,
            0,     0,     0,     0,     0,     0,     0,     0,     0,
            0,     0,     0,     0,     0,     0,     0,     0,     0,
            0,     0,     0,     0,     0,     0,     0,     0,     0,
            0,     0,     0,     0],
       [  101,  2051,  1010,  2029,  3216,  2019,  2503,  3444,  1010,
         6732,  1996,  2265,  2038, 19840,  2098,  2125,  9906,  1998,
         2003,  2770,  2041,  1997,  4784,  1012,   102,  2051,  6732,
         1996,  2265,  2003,  9525,  1998,  4569,  1012,   102,     0,
            0,     0,     0,     0,     0,     0,     0,     0,     0,
            0,     0,     0,     0,     0,     0,     0,     0,     0,
            0,     0,     0,     0,     0,     0,     0,     0,     0,
            0,     0,     0,     0,     0,     0,     0,     0,     0,
            0,     0,     0,     0,     0,     0,     0,     0,     0,
            0,     0,     0,     0],
       [  101,  1996, 10556,  2140, 11515,  2058,  1010,  2010,  2162,
         2252,  5689,  2013,  2010,  7223,  1012,   102,  2043,  1996,
        10556,  2140, 11515,  2058,  1010,  2010,  2252,  3062,  2000,
         1996,  2598,  1012,   102,     0,     0,     0,     0,     0,
            0,     0,     0,     0,     0,     0,     0,     0,     0,
            0,     0,     0,     0,     0,     0,     0,     0,     0,
            0,     0,     0,     0,     0,     0,     0,     0,     0,
            0,     0,     0,     0,     0,     0,     0,     0,     0,
            0,     0,     0,     0,     0,     0,     0,     0,     0,
            0,     0,     0,     0],
       [  101, 13543,  1999,  2049,  6143,  2933,  2443,   102,  2025,
        13543,  1999,  6143,  2933,  2003,  2443,   102,     0,     0,
            0,     0,     0,     0,     0,     0,     0,     0,     0,
            0,     0,     0,     0,     0,     0,     0,     0,     0,
            0,     0,     0,     0,     0,     0,     0,     0,     0,
            0,     0,     0,     0,     0,     0,     0,     0,     0,
            0,     0,     0,     0,     0,     0,     0,     0,     0,
            0,     0,     0,     0,     0,     0,     0,     0,     0,
            0,     0,     0,     0,     0,     0,     0,     0,     0,
            0,     0,     0,     0]])`,highlighted:`array([[  <span class="hljs-number">101</span>,  <span class="hljs-number">2007</span>,  <span class="hljs-number">2032</span>,  <span class="hljs-number">2001</span>,  <span class="hljs-number">1037</span>, <span class="hljs-number">16480</span>,  <span class="hljs-number">3917</span>,  <span class="hljs-number">2594</span>,  <span class="hljs-number">4135</span>,
        <span class="hljs-number">23212</span>,  <span class="hljs-number">3070</span>,  <span class="hljs-number">2214</span>, <span class="hljs-number">10170</span>,  <span class="hljs-number">1010</span>,  <span class="hljs-number">2012</span>,  <span class="hljs-number">4356</span>,  <span class="hljs-number">1997</span>,  <span class="hljs-number">3183</span>,
         <span class="hljs-number">6838</span>, <span class="hljs-number">12953</span>,  <span class="hljs-number">2039</span>,  <span class="hljs-number">2000</span>,  <span class="hljs-number">1996</span>,  <span class="hljs-number">6147</span>,  <span class="hljs-number">1997</span>,  <span class="hljs-number">2010</span>,  <span class="hljs-number">2606</span>,
         <span class="hljs-number">1012</span>,   <span class="hljs-number">102</span>,  <span class="hljs-number">6838</span>,  <span class="hljs-number">2001</span>,  <span class="hljs-number">3294</span>,  <span class="hljs-number">6625</span>,  <span class="hljs-number">3773</span>,  <span class="hljs-number">1996</span>,  <span class="hljs-number">2214</span>,
         <span class="hljs-number">2158</span>,  <span class="hljs-number">1012</span>,   <span class="hljs-number">102</span>,     <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,
            <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,
            <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,
            <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,
            <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,
            <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>],
       [  <span class="hljs-number">101</span>,  <span class="hljs-number">1998</span>,  <span class="hljs-number">6814</span>,  <span class="hljs-number">2016</span>,  <span class="hljs-number">2234</span>,  <span class="hljs-number">2461</span>,  <span class="hljs-number">2153</span>,  <span class="hljs-number">1998</span>, <span class="hljs-number">13322</span>,
         <span class="hljs-number">2009</span>,  <span class="hljs-number">1012</span>,   <span class="hljs-number">102</span>,  <span class="hljs-number">2045</span>,  <span class="hljs-number">1005</span>,  <span class="hljs-number">1055</span>,  <span class="hljs-number">2053</span>,  <span class="hljs-number">3382</span>,  <span class="hljs-number">2008</span>,
         <span class="hljs-number">2016</span>,  <span class="hljs-number">1005</span>,  <span class="hljs-number">2222</span>,  <span class="hljs-number">3046</span>,  <span class="hljs-number">8103</span>,  <span class="hljs-number">2075</span>,  <span class="hljs-number">2009</span>,  <span class="hljs-number">2153</span>,  <span class="hljs-number">1012</span>,
          <span class="hljs-number">102</span>,     <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,
            <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,
            <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,
            <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,
            <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,
            <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,
            <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>],
       [  <span class="hljs-number">101</span>,  <span class="hljs-number">1998</span>,  <span class="hljs-number">2007</span>,  <span class="hljs-number">1996</span>,  <span class="hljs-number">3712</span>,  <span class="hljs-number">4634</span>,  <span class="hljs-number">1010</span>,  <span class="hljs-number">2057</span>,  <span class="hljs-number">8108</span>,
         <span class="hljs-number">2025</span>,  <span class="hljs-number">3404</span>,  <span class="hljs-number">2028</span>,  <span class="hljs-number">1012</span>,  <span class="hljs-number">1996</span>,  <span class="hljs-number">2616</span>, <span class="hljs-number">18449</span>,  <span class="hljs-number">2125</span>,  <span class="hljs-number">1999</span>,
         <span class="hljs-number">1037</span>,  <span class="hljs-number">9666</span>,  <span class="hljs-number">1997</span>,  <span class="hljs-number">4100</span>,  <span class="hljs-number">8663</span>, <span class="hljs-number">11020</span>,  <span class="hljs-number">6313</span>,  <span class="hljs-number">2791</span>,  <span class="hljs-number">1998</span>,
         <span class="hljs-number">2431</span>,  <span class="hljs-number">1011</span>,  <span class="hljs-number">4301</span>,  <span class="hljs-number">1012</span>,   <span class="hljs-number">102</span>,  <span class="hljs-number">2028</span>,  <span class="hljs-number">1005</span>,  <span class="hljs-number">1055</span>,  <span class="hljs-number">5177</span>,
         <span class="hljs-number">2110</span>,  <span class="hljs-number">1998</span>,  <span class="hljs-number">3977</span>,  <span class="hljs-number">2000</span>,  <span class="hljs-number">2832</span>,  <span class="hljs-number">2106</span>,  <span class="hljs-number">2025</span>,  <span class="hljs-number">2689</span>,  <span class="hljs-number">2104</span>,
         <span class="hljs-number">2122</span>,  <span class="hljs-number">6214</span>,  <span class="hljs-number">1012</span>,   <span class="hljs-number">102</span>,     <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,
            <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,
            <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,
            <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,
            <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>],
       [  <span class="hljs-number">101</span>,  <span class="hljs-number">1045</span>,  <span class="hljs-number">2001</span>,  <span class="hljs-number">1999</span>,  <span class="hljs-number">1037</span>, <span class="hljs-number">13090</span>,  <span class="hljs-number">5948</span>,  <span class="hljs-number">2007</span>,  <span class="hljs-number">2048</span>,
         <span class="hljs-number">2308</span>,  <span class="hljs-number">2006</span>,  <span class="hljs-number">2026</span>,  <span class="hljs-number">5001</span>,  <span class="hljs-number">2043</span>,  <span class="hljs-number">2026</span>,  <span class="hljs-number">2171</span>,  <span class="hljs-number">2001</span>,  <span class="hljs-number">2170</span>,
         <span class="hljs-number">1012</span>,   <span class="hljs-number">102</span>,  <span class="hljs-number">1045</span>,  <span class="hljs-number">2001</span>,  <span class="hljs-number">3564</span>,  <span class="hljs-number">1999</span>,  <span class="hljs-number">2277</span>,  <span class="hljs-number">1012</span>,   <span class="hljs-number">102</span>,
            <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,
            <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,
            <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,
            <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,
            <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,
            <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,
            <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>],
       [  <span class="hljs-number">101</span>,  <span class="hljs-number">2195</span>,  <span class="hljs-number">4279</span>,  <span class="hljs-number">2191</span>,  <span class="hljs-number">2039</span>,  <span class="hljs-number">1996</span>,  <span class="hljs-number">2181</span>,  <span class="hljs-number">2124</span>,  <span class="hljs-number">2004</span>,
         <span class="hljs-number">1996</span>,  <span class="hljs-number">2225</span>,  <span class="hljs-number">7363</span>,  <span class="hljs-number">1012</span>,   <span class="hljs-number">102</span>,  <span class="hljs-number">2045</span>,  <span class="hljs-number">2003</span>,  <span class="hljs-number">2069</span>,  <span class="hljs-number">2028</span>,
         <span class="hljs-number">2451</span>,  <span class="hljs-number">1999</span>,  <span class="hljs-number">1996</span>,  <span class="hljs-number">2225</span>,  <span class="hljs-number">7363</span>,  <span class="hljs-number">1012</span>,   <span class="hljs-number">102</span>,     <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,
            <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,
            <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,
            <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,
            <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,
            <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,
            <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,
            <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>],
       [  <span class="hljs-number">101</span>,  <span class="hljs-number">2061</span>,  <span class="hljs-number">2008</span>,  <span class="hljs-number">1045</span>,  <span class="hljs-number">2123</span>,  <span class="hljs-number">1005</span>,  <span class="hljs-number">1056</span>,  <span class="hljs-number">2113</span>,  <span class="hljs-number">2065</span>,
         <span class="hljs-number">2009</span>,  <span class="hljs-number">2428</span>, <span class="hljs-number">10654</span>,  <span class="hljs-number">7347</span>,  <span class="hljs-number">2030</span>,  <span class="hljs-number">2009</span>,  <span class="hljs-number">7126</span>,  <span class="hljs-number">2256</span>,  <span class="hljs-number">2495</span>,
         <span class="hljs-number">2291</span>,   <span class="hljs-number">102</span>,  <span class="hljs-number">2009</span>,  <span class="hljs-number">2003</span>,  <span class="hljs-number">5094</span>,  <span class="hljs-number">2256</span>,  <span class="hljs-number">2495</span>,  <span class="hljs-number">2291</span>,  <span class="hljs-number">2035</span>,
         <span class="hljs-number">2105</span>,  <span class="hljs-number">1012</span>,   <span class="hljs-number">102</span>,     <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,
            <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,
            <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,
            <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,
            <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,
            <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,
            <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>],
       [  <span class="hljs-number">101</span>,  <span class="hljs-number">2051</span>,  <span class="hljs-number">1010</span>,  <span class="hljs-number">2029</span>,  <span class="hljs-number">3216</span>,  <span class="hljs-number">2019</span>,  <span class="hljs-number">2503</span>,  <span class="hljs-number">3444</span>,  <span class="hljs-number">1010</span>,
         <span class="hljs-number">6732</span>,  <span class="hljs-number">1996</span>,  <span class="hljs-number">2265</span>,  <span class="hljs-number">2038</span>, <span class="hljs-number">19840</span>,  <span class="hljs-number">2098</span>,  <span class="hljs-number">2125</span>,  <span class="hljs-number">9906</span>,  <span class="hljs-number">1998</span>,
         <span class="hljs-number">2003</span>,  <span class="hljs-number">2770</span>,  <span class="hljs-number">2041</span>,  <span class="hljs-number">1997</span>,  <span class="hljs-number">4784</span>,  <span class="hljs-number">1012</span>,   <span class="hljs-number">102</span>,  <span class="hljs-number">2051</span>,  <span class="hljs-number">6732</span>,
         <span class="hljs-number">1996</span>,  <span class="hljs-number">2265</span>,  <span class="hljs-number">2003</span>,  <span class="hljs-number">9525</span>,  <span class="hljs-number">1998</span>,  <span class="hljs-number">4569</span>,  <span class="hljs-number">1012</span>,   <span class="hljs-number">102</span>,     <span class="hljs-number">0</span>,
            <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,
            <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,
            <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,
            <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,
            <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,
            <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>],
       [  <span class="hljs-number">101</span>,  <span class="hljs-number">1996</span>, <span class="hljs-number">10556</span>,  <span class="hljs-number">2140</span>, <span class="hljs-number">11515</span>,  <span class="hljs-number">2058</span>,  <span class="hljs-number">1010</span>,  <span class="hljs-number">2010</span>,  <span class="hljs-number">2162</span>,
         <span class="hljs-number">2252</span>,  <span class="hljs-number">5689</span>,  <span class="hljs-number">2013</span>,  <span class="hljs-number">2010</span>,  <span class="hljs-number">7223</span>,  <span class="hljs-number">1012</span>,   <span class="hljs-number">102</span>,  <span class="hljs-number">2043</span>,  <span class="hljs-number">1996</span>,
        <span class="hljs-number">10556</span>,  <span class="hljs-number">2140</span>, <span class="hljs-number">11515</span>,  <span class="hljs-number">2058</span>,  <span class="hljs-number">1010</span>,  <span class="hljs-number">2010</span>,  <span class="hljs-number">2252</span>,  <span class="hljs-number">3062</span>,  <span class="hljs-number">2000</span>,
         <span class="hljs-number">1996</span>,  <span class="hljs-number">2598</span>,  <span class="hljs-number">1012</span>,   <span class="hljs-number">102</span>,     <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,
            <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,
            <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,
            <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,
            <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,
            <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,
            <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>],
       [  <span class="hljs-number">101</span>, <span class="hljs-number">13543</span>,  <span class="hljs-number">1999</span>,  <span class="hljs-number">2049</span>,  <span class="hljs-number">6143</span>,  <span class="hljs-number">2933</span>,  <span class="hljs-number">2443</span>,   <span class="hljs-number">102</span>,  <span class="hljs-number">2025</span>,
        <span class="hljs-number">13543</span>,  <span class="hljs-number">1999</span>,  <span class="hljs-number">6143</span>,  <span class="hljs-number">2933</span>,  <span class="hljs-number">2003</span>,  <span class="hljs-number">2443</span>,   <span class="hljs-number">102</span>,     <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,
            <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,
            <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,
            <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,
            <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,
            <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,
            <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,
            <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,
            <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>]])`}}),Ae=new T({props:{code:`labels = batch['labels'].numpy()
labels[indices]`,highlighted:`labels = batch[<span class="hljs-string">&#x27;labels&#x27;</span>].numpy()
labels[indices]`}}),Ce=new T({props:{code:"array([2, 2, 2, 2, 2, 2, 2, 2, 2])",highlighted:'array([<span class="hljs-number">2</span>, <span class="hljs-number">2</span>, <span class="hljs-number">2</span>, <span class="hljs-number">2</span>, <span class="hljs-number">2</span>, <span class="hljs-number">2</span>, <span class="hljs-number">2</span>, <span class="hljs-number">2</span>, <span class="hljs-number">2</span>])'}}),Oe=new T({props:{code:"model.config.num_labels",highlighted:"model.config.num_labels"}}),De=new T({props:{code:"2",highlighted:'<span class="hljs-number">2</span>'}}),Pe=new T({props:{code:`model = TFAutoModelForSequenceClassification.from_pretrained(model_checkpoint, num_labels=3)
model.compile(optimizer='adam')
model.fit(train_dataset)`,highlighted:`model = TFAutoModelForSequenceClassification.from_pretrained(model_checkpoint, <span class="hljs-attribute">num_labels</span>=3)
model.compile(<span class="hljs-attribute">optimizer</span>=<span class="hljs-string">&#x27;adam&#x27;</span>)
model.fit(train_dataset)`}}),ze=new T({props:{code:"  869/24543 [>.............................] - ETA: 15:29 - loss: 1.1032",highlighted:'  <span class="hljs-number">869</span>/<span class="hljs-number">24543</span> [&gt;.............................] - ETA: <span class="hljs-number">15</span>:<span class="hljs-number">29</span> - loss: <span class="hljs-number">1.1032</span>'}}),qe=new B({}),Ie=new T({props:{code:`from tensorflow.keras.optimizers import Adam

model = TFAutoModelForSequenceClassification.from_pretrained(model_checkpoint)
model.compile(optimizer=Adam(5e-5))`,highlighted:`<span class="hljs-keyword">from</span> tensorflow.keras.optimizers <span class="hljs-keyword">import</span> Adam

model = TFAutoModelForSequenceClassification.from_pretrained(model_checkpoint)
model.<span class="hljs-built_in">compile</span>(optimizer=Adam(<span class="hljs-number">5e-5</span>))`}}),Ms=new Al({props:{$$slots:{default:[Lc]},$$scope:{ctx:S}}}),Se=new T({props:{code:"model.fit(train_dataset)",highlighted:"model.fit(train_dataset)"}}),Ne=new T({props:{code:"319/24543 [..............................] - ETA: 16:07 - loss: 0.9718",highlighted:'<span class="hljs-number">319</span>/<span class="hljs-number">24543</span> [..............................] - ETA: <span class="hljs-number">16</span>:07 - loss: <span class="hljs-number">0.9718</span>'}}),Fe=new B({}),We=new B({}),Gs=new Al({props:{$$slots:{default:[Mc]},$$scope:{ctx:S}}}),Le=new B({}),Me=new B({}),Be=new T({props:{code:`input_ids = batch["input_ids"].numpy()
tokenizer.decode(input_ids[0])`,highlighted:`input_ids = batch[<span class="hljs-string">&quot;input_ids&quot;</span>].numpy()
tokenizer.decode(input_ids[<span class="hljs-number">0</span>])`}}),He=new T({props:{code:`labels = batch["labels"].numpy()
label = labels[0]`,highlighted:`labels = batch[<span class="hljs-string">&quot;labels&quot;</span>].numpy()
label = labels[<span class="hljs-number">0</span>]`}}),Ge=new B({}),Ke=new T({props:{code:`for batch in train_dataset:
    break

# Make sure you have run model.compile() and set your optimizer,
# and your loss/metrics if you're using them

model.fit(batch, epochs=20)`,highlighted:`<span class="hljs-keyword">for</span> batch <span class="hljs-keyword">in</span> train_dataset:
    <span class="hljs-keyword">break</span>

<span class="hljs-comment"># Make sure you have run model.compile() and set your optimizer,</span>
<span class="hljs-comment"># and your loss/metrics if you&#x27;re using them</span>

model.fit(batch, epochs=<span class="hljs-number">20</span>)`}}),Vs=new Al({props:{$$slots:{default:[Bc]},$$scope:{ctx:S}}}),Xs=new Al({props:{warning:!0,$$slots:{default:[Hc]},$$scope:{ctx:S}}}),Ue=new B({}),Ye=new B({}),{c(){c=r("meta"),E=i(),b(w.$$.fragment),k=i(),D=r("h1"),C=r("a"),P=r("span"),b(q.$$.fragment),os=i(),ps=r("span"),re=n("Debugging the training pipeline"),Y=i(),b(H.$$.fragment),$s=i(),z=r("p"),N=n("You\u2019ve written a beautiful script to train or fine-tune a model on a given task, dutifully following the advice from "),R=r("a"),hs=n("Chapter 7"),oe=n(". But when you launch the command "),is=r("code"),ks=n("model.fit()"),V=n(", something horrible happens: you get an error \u{1F631}! Or worse, everything seems to be fine and the training runs without error, but the resulting model is crappy. In this section, we will show you what you can do to debug these kinds of issues."),_s=i(),_=r("h2"),A=r("a"),Es=r("span"),b(J.$$.fragment),na=i(),Ts=r("span"),As=n("Debugging the training pipeline"),pe=i(),b(X.$$.fragment),Cl=i(),Cs=r("p"),Mr=n("The problem when you encounter an error in "),Wa=r("code"),Br=n("model.fit()"),Hr=n(" is that it could come from multiple sources, as training usually brings together a lot of things that you\u2019ve been working on up until that point. The problem could be something wrong in your dataset, or some issue when trying to batch elements of the datasets together. Or it could be something wrong in the model code, or your loss function or optimizer. And even if everything goes well for training, something could still go wrong during the evaluation if there is a problem with your metric."),Ol=i(),Os=r("p"),Gr=n("The best way to debug an error that arises in "),La=r("code"),Kr=n("model.fit()"),Ur=n(" is to manually go through this whole pipeline to see where things went awry. The error is then often very easy to solve."),Dl=i(),Ds=r("p"),Yr=n("To demonstrate this, we will use the following script that (tries to) fine-tune a DistilBERT model on the "),he=r("a"),Rr=n("MNLI dataset"),Vr=n(":"),Pl=i(),b(ie.$$.fragment),zl=i(),Ps=r("p"),Jr=n("If you try to execute it, you might get some "),Ma=r("code"),Xr=n("VisibleDeprecationWarning"),Zr=n("s when doing the dataset conversion \u2014 this is a known UX issue we have, so please ignore it. If you\u2019re reading the course after, say, November 2021 and it\u2019s still happening, then send rage tweets at @carrigmat until he fixes it."),ql=i(),la=r("p"),Qr=n("What\u2019s a more serious problem, though, is that we get an outright error. And it\u2019s really, terrifyingly long:"),xl=i(),b(ue.$$.fragment),Il=i(),ta=r("p"),so=n("What does that mean? We tried to train on our data, but we got no gradient? This is pretty perplexing; how do we even begin to debug something like that? When the error you get doesn\u2019t immediately suggest where the problem is, the best solution is often to walk through things in sequence, making sure at each stage that everything looks right. And of course, the place to start is always to\u2026"),Sl=i(),us=r("h3"),zs=r("a"),Ba=r("span"),b(me.$$.fragment),eo=i(),Ha=r("span"),ao=n("Check your data"),Nl=i(),ra=r("p"),no=n("This goes without saying, but if your data is corrupted, Keras is not going to be able to fix it for you. So first things first, you need to have a look at what is inside your training set."),Fl=i(),O=r("p"),lo=n("Although it\u2019s tempting to look inside "),Ga=r("code"),to=n("raw_datasets"),ro=n(" and "),Ka=r("code"),oo=n("tokenized_datasets"),po=n(", we highly recommend you go to the data right at the point where it\u2019s going to enter the model. That means reading an output from the "),Ua=r("code"),ho=n("tf.data.Dataset"),io=n(" you created with the "),Ya=r("code"),uo=n("to_tf_dataset()"),mo=n(" function! So how do we do that? "),Ra=r("code"),co=n("tf.data.Dataset"),bo=n(" objects give us whole batches at a time and don\u2019t support indexing, so we can\u2019t just ask for "),Va=r("code"),fo=n("train_dataset[0]"),jo=n(". We can, however, ask it politely for a batch:"),Wl=i(),b(ce.$$.fragment),Ll=i(),G=r("p"),Ja=r("code"),yo=n("break"),go=n(" ends the loop after one iteration, so this grabs the first batch that comes out of "),Xa=r("code"),wo=n("train_dataset"),vo=n(" and saves it as "),Za=r("code"),$o=n("batch"),ko=n(". Now, let\u2019s take a look at what\u2019s inside:"),Ml=i(),b(be.$$.fragment),Bl=i(),F=r("p"),_o=n("This looks right, doesn\u2019t it? We\u2019re passing the "),Qa=r("code"),Eo=n("labels"),To=n(", "),sn=r("code"),Ao=n("attention_mask"),Co=n(", and "),en=r("code"),Oo=n("input_ids"),Do=n(" to the model, which should be everything it needs to compute outputs and calculate the loss. So why don\u2019t we have a gradient? Look closer: we\u2019re passing a single dictionary as input, but a training batch is usually an input tensor or dictionary, plus a labels tensor. Our labels are just a key in our input dictionary."),Hl=i(),qs=r("p"),Po=n("Is this a problem? Not always, actually! But it\u2019s one of the most common issues you\u2019ll encounter when training Transformer models with TensorFlow. Our models can all compute loss internally, but to do that the labels need to be passed in the input dictionary. This is the loss that is used when we don\u2019t specify a loss value to "),an=r("code"),zo=n("compile()"),qo=n(". Keras, on the other hand, usually expects labels to be passed separately from the input dictionary, and loss computations will usually fail if you don\u2019t do that."),Gl=i(),Z=r("p"),xo=n("The problem has now become clearer: we passed a "),nn=r("code"),Io=n("loss"),So=n(" argument, which means we\u2019re asking Keras to compute losses for us, but we passed our labels as inputs to the model, not as labels in the place Keras expects them! We need to choose one or the other: either we use the model\u2019s internal loss and keep the labels where they are, or we keep using Keras losses, but we move the labels to the place Keras expects them. For simplicity, let\u2019s take the first approach. Change the call to "),ln=r("code"),No=n("compile()"),Fo=n(" to read:"),Kl=i(),b(fe.$$.fragment),Ul=i(),oa=r("p"),Wo=n("Now we\u2019ll use the model\u2019s internal loss, and this problem should be resolved!"),Yl=i(),b(xs.$$.fragment),Rl=i(),Is=r("p"),Lo=n("Now, let\u2019s try training. We should get gradients now, so hopefully (ominous music plays here) we can just call "),tn=r("code"),Mo=n("model.fit()"),Bo=n(" and everything will work fine!"),Vl=i(),b(de.$$.fragment),Jl=i(),pa=r("p"),Ho=n("Oh no."),Xl=i(),je=r("p"),rn=r("code"),Go=n("nan"),Ko=n(" is not a very encouraging loss value. Still, we\u2019ve checked our data, and it looks pretty good. If that\u2019s not the problem, where can we go next? The obvious next step is to\u2026"),Zl=i(),ms=r("h3"),Ss=r("a"),on=r("span"),b(ye.$$.fragment),Uo=i(),pn=r("span"),Yo=n("Check your model"),Ql=i(),K=r("p"),hn=r("code"),Ro=n("model.fit()"),Vo=n(" is a really great convenience function in Keras, but it does a lot of things for you, and that can make it trickier to find exactly where a problem has occurred. If you\u2019re debugging your model, one strategy that can really help is to pass just a single batch to the model, and look at the outputs for that one batch in detail. Another really helpful tip if the model is throwing errors is to "),un=r("code"),Jo=n("compile()"),Xo=n(" the model with "),mn=r("code"),Zo=n("run_eagerly=True"),Qo=n(". This will make it a lot slower, but it will make the error messages much more comprehensible, because they\u2019ll indicate exactly where in your model\u2019s code the problem occurred."),st=i(),Q=r("p"),sp=n("For now, though, we don\u2019t need "),cn=r("code"),ep=n("run_eagerly"),ap=n(" just yet. Let\u2019s run the "),bn=r("code"),np=n("batch"),lp=n(" we got before through the model and see what the outputs look like:"),et=i(),b(ge.$$.fragment),at=i(),b(we.$$.fragment),nt=i(),v=r("p"),tp=n("Well, this is tricky. Everything is "),fn=r("code"),rp=n("nan"),op=n("! But that\u2019s strange, isn\u2019t it? How would all our logits become "),dn=r("code"),pp=n("nan"),hp=n("? "),jn=r("code"),ip=n("nan"),up=n(" means \u201Cnot a number.\u201D "),yn=r("code"),mp=n("nan"),cp=n(" values often occur when you perform a forbidden operation, such as division by zero. But one thing that\u2019s very important to know about "),gn=r("code"),bp=n("nan"),fp=n(" in machine learning is that this value tends to "),wn=r("em"),dp=n("propagate"),jp=n(". If you multiply a number by "),vn=r("code"),yp=n("nan"),gp=n(", the output is also "),$n=r("code"),wp=n("nan"),vp=n(". And if you get a "),kn=r("code"),$p=n("nan"),kp=n(" anywhere in your output, your loss, or your gradient, then it will rapidly spread throughout your whole model \u2014 because when that "),_n=r("code"),_p=n("nan"),Ep=n(" value is propagated back through your network, you\u2019ll get "),En=r("code"),Tp=n("nan"),Ap=n(" gradients, and when weight updates are computed with those gradients, you\u2019ll get "),Tn=r("code"),Cp=n("nan"),Op=n(" weights, and those weights will compute even more "),An=r("code"),Dp=n("nan"),Pp=n(" outputs! Soon enough the whole network will just be one big block of "),Cn=r("code"),zp=n("nan"),qp=n("s. Once that happens, it\u2019s pretty hard to see where the problem started. How can we isolate where "),On=r("code"),xp=n("nan"),Ip=n(" first crept in?"),lt=i(),W=r("p"),Sp=n("The answer is to try "),Dn=r("em"),Np=n("reinitializing"),Fp=n(" our model. Once we started training, we got a "),Pn=r("code"),Wp=n("nan"),Lp=n(" somewhere and it quickly propagated through the whole model. So, let\u2019s load the model from a checkpoint and not do any weight updates, and see where we get a "),zn=r("code"),Mp=n("nan"),Bp=n(" value:"),tt=i(),b(ve.$$.fragment),rt=i(),ha=r("p"),Hp=n("When we run that, we get:"),ot=i(),b($e.$$.fragment),pt=i(),U=r("p"),qn=r("em"),Gp=n("Now"),Kp=n(" we\u2019re getting somewhere! There are no "),xn=r("code"),Up=n("nan"),Yp=n(" values in our logits, which is reassuring. But we do see a few "),In=r("code"),Rp=n("nan"),Vp=n(" values in our loss! Is there something about those samples in particular that\u2019s causing this problem? Let\u2019s see which ones they are (note that if you run this code yourself, you may get different indices because the dataset has been shuffled):"),ht=i(),b(ke.$$.fragment),it=i(),b(_e.$$.fragment),ut=i(),ia=r("p"),Jp=n("Let\u2019s look at the samples these indices came from:"),mt=i(),b(Ee.$$.fragment),ct=i(),b(Te.$$.fragment),bt=i(),ua=r("p"),Xp=n("Well, there\u2019s a lot in here, but nothing stands out as unusual. Let\u2019s look at the labels:"),ft=i(),b(Ae.$$.fragment),dt=i(),b(Ce.$$.fragment),jt=i(),ss=r("p"),Zp=n("Ah! The "),Sn=r("code"),Qp=n("nan"),sh=n(" samples all have the same label, and it\u2019s label 2. This is a very strong hint. The fact that we\u2019re only getting a loss of "),Nn=r("code"),eh=n("nan"),ah=n(" when our label is 2 suggests that this is a very good time to check the number of labels in our model:"),yt=i(),b(Oe.$$.fragment),gt=i(),b(De.$$.fragment),wt=i(),Ns=r("p"),nh=n("Now we see the problem: the model thinks there are only two classes, but the labels go up to 2, which means there are in fact three classes (because 0 is also a class). This is how we got a "),Fn=r("code"),lh=n("nan"),th=n(" \u2014 by trying to compute the loss for a nonexistent class! Let\u2019s try changing that and fitting the model again:"),vt=i(),b(Pe.$$.fragment),$t=i(),b(ze.$$.fragment),kt=i(),Fs=r("p"),rh=n("We\u2019re training! No more "),Wn=r("code"),oh=n("nan"),ph=n("s, and our loss is declining\u2026 sort of. If you watch it for a while, you might start to get a bit impatient, because the loss value stays stubbornly high. Let\u2019s stop training here and try to think about what could be causing this problem. At this point, we\u2019re pretty sure both the data and the model are okay, but our model isn\u2019t learning well. What else is left? It\u2019s time to\u2026"),_t=i(),cs=r("h3"),Ws=r("a"),Ln=r("span"),b(qe.$$.fragment),hh=i(),Mn=r("span"),ih=n("Check your hyperparameters"),Et=i(),es=r("p"),uh=n("If you look back at the code above, you might not be able to see any hyperparameters at all, except perhaps the "),Bn=r("code"),mh=n("batch_size"),ch=n(", and that doesn\u2019t seem like a likely culprit. Don\u2019t be fooled, though; there are always hyperparameters, and if you can\u2019t see them, it just means that you don\u2019t know what they\u2019re set to. In particular, remember a critical thing about Keras: if you set a loss, optimizer, or activation function with a string, "),Hn=r("em"),bh=n("all of its arguments will be set to their default values"),fh=n(". This means that even though using strings for this is very convenient, you should be very careful when doing so, as it can easily hide critical things from you. (Anyone trying the optional challenge above should take careful note of this fact.)"),Tt=i(),Ls=r("p"),dh=n("In this case, where have we set an argument with a string? We were setting the loss with a string initially, but we\u2019re not doing that anymore. We are, however, setting the optimizer with a string. Could that be hiding anything from us? Let\u2019s take a look at "),xe=r("a"),jh=n("its arguments"),yh=n("."),At=i(),as=r("p"),gh=n("Does anything stand out here? That\u2019s right \u2014 the learning rate! When we just use the string "),Gn=r("code"),wh=n("'adam'"),vh=n(", we\u2019re going to get the default learning rate, which is 0.001, or 1e-3. This is way too high for a Transformer model! In general, we recommend trying learning rates between 1e-5 and 1e-4 for your models; that\u2019s somewhere between 10X and 100X smaller than the value we\u2019re actually using here. That sounds like it might be a major problem, so let\u2019s try reducing it. To do that, we need to import the actual "),Kn=r("code"),$h=n("optimizer"),kh=n(" object. While we\u2019re at it, let\u2019s reinitialize the model from the checkpoint, in case training with the high learning rate damaged its weights:"),Ct=i(),b(Ie.$$.fragment),Ot=i(),b(Ms.$$.fragment),Dt=i(),ma=r("p"),_h=n("Now, we can try fitting the model with the new, improved learning rate:"),Pt=i(),b(Se.$$.fragment),zt=i(),b(Ne.$$.fragment),qt=i(),ca=r("p"),Eh=n("Now our loss is really going somewhere! Training finally looks like it\u2019s working. There\u2019s a lesson here: when your model is running but loss isn\u2019t declining, and you\u2019re sure your data is okay, it\u2019s a good idea to check hyperparameters like the learning rate and weight decay. Setting either of those too high is very likely to cause training to \u201Cstall\u201D at a high loss value."),xt=i(),bs=r("h2"),Bs=r("a"),Un=r("span"),b(Fe.$$.fragment),Th=i(),Yn=r("span"),Ah=n("Other potential issues"),It=i(),ba=r("p"),Ch=n("We\u2019ve covered the issues in the script above, but there are several other common errors you might face. Let\u2019s take a look at a (very incomplete) list."),St=i(),fs=r("h3"),Hs=r("a"),Rn=r("span"),b(We.$$.fragment),Oh=i(),Vn=r("span"),Dh=n("Dealing with out-of-memory errors"),Nt=i(),ns=r("p"),Ph=n("The telltale sign of running out of memory is an error like \u201COOM when allocating tensor\u201D \u2014 OOM is short for \u201Cout of memory.\u201D This is a very common hazard when dealing with large language models. If you encounter this, a good strategy is to halve your batch size and try again. Bear in mind, though, that some models are "),Jn=r("em"),zh=n("very"),qh=n(" large. For example, the full-size GPT-2 has 1.5B parameters, which means you\u2019ll need 6 GB of memory just to store the model, and another 6 GB for its gradients! Training the full GPT-2 model will usually require over 20 GB of VRAM no matter what batch size you use, which only a few GPUs have. More lightweight models like "),Xn=r("code"),xh=n("distilbert-base-cased"),Ih=n(" are much easier to run, and train much more quickly too."),Ft=i(),b(Gs.$$.fragment),Wt=i(),ds=r("h3"),Ks=r("a"),Zn=r("span"),b(Le.$$.fragment),Sh=i(),Qn=r("span"),Nh=n("Hungry Hungry TensorFlow \u{1F99B}"),Lt=i(),ls=r("p"),Fh=n("One particular quirk of TensorFlow that you should be aware of is that it allocates "),sl=r("em"),Wh=n("all"),Lh=n(" of your GPU memory to itself as soon as you load a model or do any training, and then it divides up that memory as required. This is different from the behavior of other frameworks, like PyTorch, which allocate memory as required with CUDA rather than doing it internally. One advantage of the TensorFlow approach is that it can often give useful errors when you run out of memory, and it can recover from that state without crashing the whole CUDA kernel. But there\u2019s also an important downside: if you run two TensorFlow processes at once, then "),el=r("strong"),Mh=n("you\u2019re going to have a bad time"),Bh=n("."),Mt=i(),fa=r("p"),Hh=n("If you\u2019re running on Colab you don\u2019t need to worry about this, but if you\u2019re running locally this is definitely something you should be careful about. In particular, be aware that closing a notebook tab does not necessarily shut that notebook down! You may need to select running notebooks (the ones with a green icon) and manually shut them down in the directory listing. Any running notebook that was using TensorFlow could still be holding on to a bunch of your GPU memory, and that means any new notebook you start may encounter some very odd issues."),Bt=i(),Us=r("p"),Gh=n("If you start getting errors about CUDA, BLAS, or cuBLAS in code that worked before, this is very often the culprit. You can use a command like "),al=r("code"),Kh=n("nvidia-smi"),Uh=n(" to check \u2014 when you shut down or restart your current notebook, is most of your memory free, or is it still in use? If it\u2019s still in use, something else is holding on to it!"),Ht=i(),js=r("h3"),Ys=r("a"),nl=r("span"),b(Me.$$.fragment),Yh=i(),ll=r("span"),Rh=n("Check your data (again!)"),Gt=i(),x=r("p"),Vh=n("Your model will only learn something if it\u2019s actually possible to learn anything from your data. If there is a bug that corrupts the data or the labels are attributed randomly, it\u2019s very likely you won\u2019t get any model training on your dataset. One helpful tool here is "),tl=r("code"),Jh=n("tokenizer.decode()"),Xh=n(". This will turn "),rl=r("code"),Zh=n("input_ids"),Qh=n(" back into strings, so you can view the data and see if your training data is teaching what you want it to teach. For example, after you get a "),ol=r("code"),si=n("batch"),ei=n(" from your "),pl=r("code"),ai=n("tf.data.Dataset"),ni=n(" like we did above, you can decode the first element like so:"),Kt=i(),b(Be.$$.fragment),Ut=i(),da=r("p"),li=n("Then you can compare it with the first label, like so:"),Yt=i(),b(He.$$.fragment),Rt=i(),ja=r("p"),ti=n("Once you can view your data like this, you can ask yourself the following questions:"),Vt=i(),L=r("ul"),hl=r("li"),ri=n("Is the decoded data understandable?"),oi=i(),il=r("li"),pi=n("Do you agree with the labels?"),hi=i(),ul=r("li"),ii=n("Is there one label that\u2019s more common than the others?"),ui=i(),ml=r("li"),mi=n("What should the loss/metric be if the model predicted a random answer/always the same answer?"),Jt=i(),ya=r("p"),ci=n("After looking at your data, go through a few of the model\u2019s predictions \u2014 if your model outputs tokens, try decoding them too! If the model is always predicting the same thing it might be because your dataset is biased toward one category (for classification problems), so techniques like oversampling rare classes might help. Alternatively, this can also be caused by training issues like bad hyperparameter settings."),Xt=i(),ga=r("p"),bi=n("If the loss/metric you get on your initial model before any training is very different from the loss/metric you would expect for random predictions, double-check the way your loss or metric is computed, as there is probably a bug there. If you are using several losses that you add at the end, make sure they are of the same scale."),Zt=i(),wa=r("p"),fi=n("When you are sure your data is perfect, you can see if the model is capable of training on it with one simple test."),Qt=i(),ys=r("h3"),Rs=r("a"),cl=r("span"),b(Ge.$$.fragment),di=i(),bl=r("span"),ji=n("Overfit your model on one batch"),sr=i(),va=r("p"),yi=n("Overfitting is usually something we try to avoid when training, as it means the model is not learning to recognize the general features we want it to but is instead just memorizing the training samples. However, trying to train your model on one batch over and over again is a good test to check if the problem as you framed it can be solved by the model you are attempting to train. It will also help you see if your initial learning rate is too high."),er=i(),ts=r("p"),gi=n("Doing this once you have defined your "),fl=r("code"),wi=n("model"),vi=n(" is really easy; just grab a batch of training data, then treat that "),dl=r("code"),$i=n("batch"),ki=n(" as your entire dataset, fitting on it for a large number of epochs:"),ar=i(),b(Ke.$$.fragment),nr=i(),b(Vs.$$.fragment),lr=i(),Js=r("p"),_i=n("The resulting model should have close-to-perfect results on the "),jl=r("code"),Ei=n("batch"),Ti=n(", with a loss declining quickly toward 0 (or the minimum value for the loss you\u2019re using)."),tr=i(),$a=r("p"),Ai=n("If you don\u2019t manage to have your model obtain perfect results like this, it means there is something wrong with the way you framed the problem or your data, so you should fix that. Only when you manage to pass the overfitting test can you be sure that your model can actually learn something."),rr=i(),b(Xs.$$.fragment),or=i(),gs=r("h3"),Zs=r("a"),yl=r("span"),b(Ue.$$.fragment),Ci=i(),gl=r("span"),Oi=n("Don't tune anything until you have a first baseline"),pr=i(),Qs=r("p"),Di=n("Intense hyperparameter tuning is always emphasized as being the hardest part of machine learning, but it\u2019s just the last step to help you gain a little bit on the metric. "),wl=r("em"),Pi=n("Very"),zi=n(" bad values for your hyperparameters, like using the default Adam learning rate of 1e-3 with a Transformer model, will make learning proceed very slowly or completely stall, of course, but most of the time \u201Creasonable\u201D hyperparameters, like a learning rate from 1e-5 to 5e-5, will work just fine to give you good results. So, don\u2019t launch into a time-consuming and costly hyperparameter search until you have something that beats the baseline you have on your dataset."),hr=i(),ka=r("p"),qi=n("Once you have a good enough model, you can start tweaking a bit. Don\u2019t try launching a thousand runs with different hyperparameters, but compare a couple of runs with different values for one hyperparameter to get an idea of which has the greatest impact."),ir=i(),_a=r("p"),xi=n("If you are tweaking the model itself, keep it simple and don\u2019t try anything you can\u2019t reasonably justify. Always make sure you go back to the overfitting test to verify that your change hasn\u2019t had any unintended consequences."),ur=i(),ws=r("h3"),se=r("a"),vl=r("span"),b(Ye.$$.fragment),Ii=i(),$l=r("span"),Si=n("Ask for help"),mr=i(),ee=r("p"),Ni=n("Hopefully you will have found some advice in this section that helped you solve your issue, but if that\u2019s not the case, remember you can always ask the community on the "),Re=r("a"),Fi=n("forums"),Wi=n("."),cr=i(),Ea=r("p"),Li=n("Here are some additional resources that may prove helpful:"),br=i(),M=r("ul"),Ta=r("li"),Ve=r("a"),Mi=n("\u201CReproducibility as a vehicle for engineering best practices\u201D"),Bi=n(" by Joel Grus"),Hi=i(),Aa=r("li"),Je=r("a"),Gi=n("\u201CChecklist for debugging neural networks\u201D"),Ki=n(" by Cecelia Shao"),Ui=i(),Ca=r("li"),Xe=r("a"),Yi=n("\u201CHow to unit test machine learning code\u201D"),Ri=n(" by Chase Roberts"),Vi=i(),Oa=r("li"),Ze=r("a"),Ji=n("\u201CA Recipe for Training Neural Networks\u201D"),Xi=n(" by Andrej Karpathy"),fr=i(),Da=r("p"),Zi=n("Of course, not every problem you encounter when training neural nets is your own fault! If you encounter something in the \u{1F917} Transformers or \u{1F917} Datasets library that does not seem right, you may have encountered a bug. You should definitely tell us all about it, and in the next section we\u2019ll explain exactly how to do that."),this.h()},l(s){const t=xc('[data-svelte="svelte-1phssyn"]',document.head);c=o(t,"META",{name:!0,content:!0}),t.forEach(e),E=u(s),f(w.$$.fragment,s),k=u(s),D=o(s,"H1",{class:!0});var Qe=p(D);C=o(Qe,"A",{id:!0,class:!0,href:!0});var kl=p(C);P=o(kl,"SPAN",{});var _l=p(P);f(q.$$.fragment,_l),_l.forEach(e),kl.forEach(e),os=u(Qe),ps=o(Qe,"SPAN",{});var El=p(ps);re=l(El,"Debugging the training pipeline"),El.forEach(e),Qe.forEach(e),Y=u(s),f(H.$$.fragment,s),$s=u(s),z=o(s,"P",{});var vs=p(z);N=l(vs,"You\u2019ve written a beautiful script to train or fine-tune a model on a given task, dutifully following the advice from "),R=o(vs,"A",{href:!0});var Tl=p(R);hs=l(Tl,"Chapter 7"),Tl.forEach(e),oe=l(vs,". But when you launch the command "),is=o(vs,"CODE",{});var lu=p(is);ks=l(lu,"model.fit()"),lu.forEach(e),V=l(vs,", something horrible happens: you get an error \u{1F631}! Or worse, everything seems to be fine and the training runs without error, but the resulting model is crappy. In this section, we will show you what you can do to debug these kinds of issues."),vs.forEach(e),_s=u(s),_=o(s,"H2",{class:!0});var jr=p(_);A=o(jr,"A",{id:!0,class:!0,href:!0});var tu=p(A);Es=o(tu,"SPAN",{});var ru=p(Es);f(J.$$.fragment,ru),ru.forEach(e),tu.forEach(e),na=u(jr),Ts=o(jr,"SPAN",{});var ou=p(Ts);As=l(ou,"Debugging the training pipeline"),ou.forEach(e),jr.forEach(e),pe=u(s),f(X.$$.fragment,s),Cl=u(s),Cs=o(s,"P",{});var yr=p(Cs);Mr=l(yr,"The problem when you encounter an error in "),Wa=o(yr,"CODE",{});var pu=p(Wa);Br=l(pu,"model.fit()"),pu.forEach(e),Hr=l(yr," is that it could come from multiple sources, as training usually brings together a lot of things that you\u2019ve been working on up until that point. The problem could be something wrong in your dataset, or some issue when trying to batch elements of the datasets together. Or it could be something wrong in the model code, or your loss function or optimizer. And even if everything goes well for training, something could still go wrong during the evaluation if there is a problem with your metric."),yr.forEach(e),Ol=u(s),Os=o(s,"P",{});var gr=p(Os);Gr=l(gr,"The best way to debug an error that arises in "),La=o(gr,"CODE",{});var hu=p(La);Kr=l(hu,"model.fit()"),hu.forEach(e),Ur=l(gr," is to manually go through this whole pipeline to see where things went awry. The error is then often very easy to solve."),gr.forEach(e),Dl=u(s),Ds=o(s,"P",{});var wr=p(Ds);Yr=l(wr,"To demonstrate this, we will use the following script that (tries to) fine-tune a DistilBERT model on the "),he=o(wr,"A",{href:!0,rel:!0});var iu=p(he);Rr=l(iu,"MNLI dataset"),iu.forEach(e),Vr=l(wr,":"),wr.forEach(e),Pl=u(s),f(ie.$$.fragment,s),zl=u(s),Ps=o(s,"P",{});var vr=p(Ps);Jr=l(vr,"If you try to execute it, you might get some "),Ma=o(vr,"CODE",{});var uu=p(Ma);Xr=l(uu,"VisibleDeprecationWarning"),uu.forEach(e),Zr=l(vr,"s when doing the dataset conversion \u2014 this is a known UX issue we have, so please ignore it. If you\u2019re reading the course after, say, November 2021 and it\u2019s still happening, then send rage tweets at @carrigmat until he fixes it."),vr.forEach(e),ql=u(s),la=o(s,"P",{});var mu=p(la);Qr=l(mu,"What\u2019s a more serious problem, though, is that we get an outright error. And it\u2019s really, terrifyingly long:"),mu.forEach(e),xl=u(s),f(ue.$$.fragment,s),Il=u(s),ta=o(s,"P",{});var cu=p(ta);so=l(cu,"What does that mean? We tried to train on our data, but we got no gradient? This is pretty perplexing; how do we even begin to debug something like that? When the error you get doesn\u2019t immediately suggest where the problem is, the best solution is often to walk through things in sequence, making sure at each stage that everything looks right. And of course, the place to start is always to\u2026"),cu.forEach(e),Sl=u(s),us=o(s,"H3",{class:!0});var $r=p(us);zs=o($r,"A",{id:!0,class:!0,href:!0});var bu=p(zs);Ba=o(bu,"SPAN",{});var fu=p(Ba);f(me.$$.fragment,fu),fu.forEach(e),bu.forEach(e),eo=u($r),Ha=o($r,"SPAN",{});var du=p(Ha);ao=l(du,"Check your data"),du.forEach(e),$r.forEach(e),Nl=u(s),ra=o(s,"P",{});var ju=p(ra);no=l(ju,"This goes without saying, but if your data is corrupted, Keras is not going to be able to fix it for you. So first things first, you need to have a look at what is inside your training set."),ju.forEach(e),Fl=u(s),O=o(s,"P",{});var I=p(O);lo=l(I,"Although it\u2019s tempting to look inside "),Ga=o(I,"CODE",{});var yu=p(Ga);to=l(yu,"raw_datasets"),yu.forEach(e),ro=l(I," and "),Ka=o(I,"CODE",{});var gu=p(Ka);oo=l(gu,"tokenized_datasets"),gu.forEach(e),po=l(I,", we highly recommend you go to the data right at the point where it\u2019s going to enter the model. That means reading an output from the "),Ua=o(I,"CODE",{});var wu=p(Ua);ho=l(wu,"tf.data.Dataset"),wu.forEach(e),io=l(I," you created with the "),Ya=o(I,"CODE",{});var vu=p(Ya);uo=l(vu,"to_tf_dataset()"),vu.forEach(e),mo=l(I," function! So how do we do that? "),Ra=o(I,"CODE",{});var $u=p(Ra);co=l($u,"tf.data.Dataset"),$u.forEach(e),bo=l(I," objects give us whole batches at a time and don\u2019t support indexing, so we can\u2019t just ask for "),Va=o(I,"CODE",{});var ku=p(Va);fo=l(ku,"train_dataset[0]"),ku.forEach(e),jo=l(I,". We can, however, ask it politely for a batch:"),I.forEach(e),Wl=u(s),f(ce.$$.fragment,s),Ll=u(s),G=o(s,"P",{});var sa=p(G);Ja=o(sa,"CODE",{});var _u=p(Ja);yo=l(_u,"break"),_u.forEach(e),go=l(sa," ends the loop after one iteration, so this grabs the first batch that comes out of "),Xa=o(sa,"CODE",{});var Eu=p(Xa);wo=l(Eu,"train_dataset"),Eu.forEach(e),vo=l(sa," and saves it as "),Za=o(sa,"CODE",{});var Tu=p(Za);$o=l(Tu,"batch"),Tu.forEach(e),ko=l(sa,". Now, let\u2019s take a look at what\u2019s inside:"),sa.forEach(e),Ml=u(s),f(be.$$.fragment,s),Bl=u(s),F=o(s,"P",{});var ae=p(F);_o=l(ae,"This looks right, doesn\u2019t it? We\u2019re passing the "),Qa=o(ae,"CODE",{});var Au=p(Qa);Eo=l(Au,"labels"),Au.forEach(e),To=l(ae,", "),sn=o(ae,"CODE",{});var Cu=p(sn);Ao=l(Cu,"attention_mask"),Cu.forEach(e),Co=l(ae,", and "),en=o(ae,"CODE",{});var Ou=p(en);Oo=l(Ou,"input_ids"),Ou.forEach(e),Do=l(ae," to the model, which should be everything it needs to compute outputs and calculate the loss. So why don\u2019t we have a gradient? Look closer: we\u2019re passing a single dictionary as input, but a training batch is usually an input tensor or dictionary, plus a labels tensor. Our labels are just a key in our input dictionary."),ae.forEach(e),Hl=u(s),qs=o(s,"P",{});var kr=p(qs);Po=l(kr,"Is this a problem? Not always, actually! But it\u2019s one of the most common issues you\u2019ll encounter when training Transformer models with TensorFlow. Our models can all compute loss internally, but to do that the labels need to be passed in the input dictionary. This is the loss that is used when we don\u2019t specify a loss value to "),an=o(kr,"CODE",{});var Du=p(an);zo=l(Du,"compile()"),Du.forEach(e),qo=l(kr,". Keras, on the other hand, usually expects labels to be passed separately from the input dictionary, and loss computations will usually fail if you don\u2019t do that."),kr.forEach(e),Gl=u(s),Z=o(s,"P",{});var Pa=p(Z);xo=l(Pa,"The problem has now become clearer: we passed a "),nn=o(Pa,"CODE",{});var Pu=p(nn);Io=l(Pu,"loss"),Pu.forEach(e),So=l(Pa," argument, which means we\u2019re asking Keras to compute losses for us, but we passed our labels as inputs to the model, not as labels in the place Keras expects them! We need to choose one or the other: either we use the model\u2019s internal loss and keep the labels where they are, or we keep using Keras losses, but we move the labels to the place Keras expects them. For simplicity, let\u2019s take the first approach. Change the call to "),ln=o(Pa,"CODE",{});var zu=p(ln);No=l(zu,"compile()"),zu.forEach(e),Fo=l(Pa," to read:"),Pa.forEach(e),Kl=u(s),f(fe.$$.fragment,s),Ul=u(s),oa=o(s,"P",{});var qu=p(oa);Wo=l(qu,"Now we\u2019ll use the model\u2019s internal loss, and this problem should be resolved!"),qu.forEach(e),Yl=u(s),f(xs.$$.fragment,s),Rl=u(s),Is=o(s,"P",{});var _r=p(Is);Lo=l(_r,"Now, let\u2019s try training. We should get gradients now, so hopefully (ominous music plays here) we can just call "),tn=o(_r,"CODE",{});var xu=p(tn);Mo=l(xu,"model.fit()"),xu.forEach(e),Bo=l(_r," and everything will work fine!"),_r.forEach(e),Vl=u(s),f(de.$$.fragment,s),Jl=u(s),pa=o(s,"P",{});var Iu=p(pa);Ho=l(Iu,"Oh no."),Iu.forEach(e),Xl=u(s),je=o(s,"P",{});var Qi=p(je);rn=o(Qi,"CODE",{});var Su=p(rn);Go=l(Su,"nan"),Su.forEach(e),Ko=l(Qi," is not a very encouraging loss value. Still, we\u2019ve checked our data, and it looks pretty good. If that\u2019s not the problem, where can we go next? The obvious next step is to\u2026"),Qi.forEach(e),Zl=u(s),ms=o(s,"H3",{class:!0});var Er=p(ms);Ss=o(Er,"A",{id:!0,class:!0,href:!0});var Nu=p(Ss);on=o(Nu,"SPAN",{});var Fu=p(on);f(ye.$$.fragment,Fu),Fu.forEach(e),Nu.forEach(e),Uo=u(Er),pn=o(Er,"SPAN",{});var Wu=p(pn);Yo=l(Wu,"Check your model"),Wu.forEach(e),Er.forEach(e),Ql=u(s),K=o(s,"P",{});var ea=p(K);hn=o(ea,"CODE",{});var Lu=p(hn);Ro=l(Lu,"model.fit()"),Lu.forEach(e),Vo=l(ea," is a really great convenience function in Keras, but it does a lot of things for you, and that can make it trickier to find exactly where a problem has occurred. If you\u2019re debugging your model, one strategy that can really help is to pass just a single batch to the model, and look at the outputs for that one batch in detail. Another really helpful tip if the model is throwing errors is to "),un=o(ea,"CODE",{});var Mu=p(un);Jo=l(Mu,"compile()"),Mu.forEach(e),Xo=l(ea," the model with "),mn=o(ea,"CODE",{});var Bu=p(mn);Zo=l(Bu,"run_eagerly=True"),Bu.forEach(e),Qo=l(ea,". This will make it a lot slower, but it will make the error messages much more comprehensible, because they\u2019ll indicate exactly where in your model\u2019s code the problem occurred."),ea.forEach(e),st=u(s),Q=o(s,"P",{});var za=p(Q);sp=l(za,"For now, though, we don\u2019t need "),cn=o(za,"CODE",{});var Hu=p(cn);ep=l(Hu,"run_eagerly"),Hu.forEach(e),ap=l(za," just yet. Let\u2019s run the "),bn=o(za,"CODE",{});var Gu=p(bn);np=l(Gu,"batch"),Gu.forEach(e),lp=l(za," we got before through the model and see what the outputs look like:"),za.forEach(e),et=u(s),f(ge.$$.fragment,s),at=u(s),f(we.$$.fragment,s),nt=u(s),v=o(s,"P",{});var $=p(v);tp=l($,"Well, this is tricky. Everything is "),fn=o($,"CODE",{});var Ku=p(fn);rp=l(Ku,"nan"),Ku.forEach(e),op=l($,"! But that\u2019s strange, isn\u2019t it? How would all our logits become "),dn=o($,"CODE",{});var Uu=p(dn);pp=l(Uu,"nan"),Uu.forEach(e),hp=l($,"? "),jn=o($,"CODE",{});var Yu=p(jn);ip=l(Yu,"nan"),Yu.forEach(e),up=l($," means \u201Cnot a number.\u201D "),yn=o($,"CODE",{});var Ru=p(yn);mp=l(Ru,"nan"),Ru.forEach(e),cp=l($," values often occur when you perform a forbidden operation, such as division by zero. But one thing that\u2019s very important to know about "),gn=o($,"CODE",{});var Vu=p(gn);bp=l(Vu,"nan"),Vu.forEach(e),fp=l($," in machine learning is that this value tends to "),wn=o($,"EM",{});var Ju=p(wn);dp=l(Ju,"propagate"),Ju.forEach(e),jp=l($,". If you multiply a number by "),vn=o($,"CODE",{});var Xu=p(vn);yp=l(Xu,"nan"),Xu.forEach(e),gp=l($,", the output is also "),$n=o($,"CODE",{});var Zu=p($n);wp=l(Zu,"nan"),Zu.forEach(e),vp=l($,". And if you get a "),kn=o($,"CODE",{});var Qu=p(kn);$p=l(Qu,"nan"),Qu.forEach(e),kp=l($," anywhere in your output, your loss, or your gradient, then it will rapidly spread throughout your whole model \u2014 because when that "),_n=o($,"CODE",{});var sm=p(_n);_p=l(sm,"nan"),sm.forEach(e),Ep=l($," value is propagated back through your network, you\u2019ll get "),En=o($,"CODE",{});var em=p(En);Tp=l(em,"nan"),em.forEach(e),Ap=l($," gradients, and when weight updates are computed with those gradients, you\u2019ll get "),Tn=o($,"CODE",{});var am=p(Tn);Cp=l(am,"nan"),am.forEach(e),Op=l($," weights, and those weights will compute even more "),An=o($,"CODE",{});var nm=p(An);Dp=l(nm,"nan"),nm.forEach(e),Pp=l($," outputs! Soon enough the whole network will just be one big block of "),Cn=o($,"CODE",{});var lm=p(Cn);zp=l(lm,"nan"),lm.forEach(e),qp=l($,"s. Once that happens, it\u2019s pretty hard to see where the problem started. How can we isolate where "),On=o($,"CODE",{});var tm=p(On);xp=l(tm,"nan"),tm.forEach(e),Ip=l($," first crept in?"),$.forEach(e),lt=u(s),W=o(s,"P",{});var ne=p(W);Sp=l(ne,"The answer is to try "),Dn=o(ne,"EM",{});var rm=p(Dn);Np=l(rm,"reinitializing"),rm.forEach(e),Fp=l(ne," our model. Once we started training, we got a "),Pn=o(ne,"CODE",{});var om=p(Pn);Wp=l(om,"nan"),om.forEach(e),Lp=l(ne," somewhere and it quickly propagated through the whole model. So, let\u2019s load the model from a checkpoint and not do any weight updates, and see where we get a "),zn=o(ne,"CODE",{});var pm=p(zn);Mp=l(pm,"nan"),pm.forEach(e),Bp=l(ne," value:"),ne.forEach(e),tt=u(s),f(ve.$$.fragment,s),rt=u(s),ha=o(s,"P",{});var hm=p(ha);Hp=l(hm,"When we run that, we get:"),hm.forEach(e),ot=u(s),f($e.$$.fragment,s),pt=u(s),U=o(s,"P",{});var aa=p(U);qn=o(aa,"EM",{});var im=p(qn);Gp=l(im,"Now"),im.forEach(e),Kp=l(aa," we\u2019re getting somewhere! There are no "),xn=o(aa,"CODE",{});var um=p(xn);Up=l(um,"nan"),um.forEach(e),Yp=l(aa," values in our logits, which is reassuring. But we do see a few "),In=o(aa,"CODE",{});var mm=p(In);Rp=l(mm,"nan"),mm.forEach(e),Vp=l(aa," values in our loss! Is there something about those samples in particular that\u2019s causing this problem? Let\u2019s see which ones they are (note that if you run this code yourself, you may get different indices because the dataset has been shuffled):"),aa.forEach(e),ht=u(s),f(ke.$$.fragment,s),it=u(s),f(_e.$$.fragment,s),ut=u(s),ia=o(s,"P",{});var cm=p(ia);Jp=l(cm,"Let\u2019s look at the samples these indices came from:"),cm.forEach(e),mt=u(s),f(Ee.$$.fragment,s),ct=u(s),f(Te.$$.fragment,s),bt=u(s),ua=o(s,"P",{});var bm=p(ua);Xp=l(bm,"Well, there\u2019s a lot in here, but nothing stands out as unusual. Let\u2019s look at the labels:"),bm.forEach(e),ft=u(s),f(Ae.$$.fragment,s),dt=u(s),f(Ce.$$.fragment,s),jt=u(s),ss=o(s,"P",{});var qa=p(ss);Zp=l(qa,"Ah! The "),Sn=o(qa,"CODE",{});var fm=p(Sn);Qp=l(fm,"nan"),fm.forEach(e),sh=l(qa," samples all have the same label, and it\u2019s label 2. This is a very strong hint. The fact that we\u2019re only getting a loss of "),Nn=o(qa,"CODE",{});var dm=p(Nn);eh=l(dm,"nan"),dm.forEach(e),ah=l(qa," when our label is 2 suggests that this is a very good time to check the number of labels in our model:"),qa.forEach(e),yt=u(s),f(Oe.$$.fragment,s),gt=u(s),f(De.$$.fragment,s),wt=u(s),Ns=o(s,"P",{});var Tr=p(Ns);nh=l(Tr,"Now we see the problem: the model thinks there are only two classes, but the labels go up to 2, which means there are in fact three classes (because 0 is also a class). This is how we got a "),Fn=o(Tr,"CODE",{});var jm=p(Fn);lh=l(jm,"nan"),jm.forEach(e),th=l(Tr," \u2014 by trying to compute the loss for a nonexistent class! Let\u2019s try changing that and fitting the model again:"),Tr.forEach(e),vt=u(s),f(Pe.$$.fragment,s),$t=u(s),f(ze.$$.fragment,s),kt=u(s),Fs=o(s,"P",{});var Ar=p(Fs);rh=l(Ar,"We\u2019re training! No more "),Wn=o(Ar,"CODE",{});var ym=p(Wn);oh=l(ym,"nan"),ym.forEach(e),ph=l(Ar,"s, and our loss is declining\u2026 sort of. If you watch it for a while, you might start to get a bit impatient, because the loss value stays stubbornly high. Let\u2019s stop training here and try to think about what could be causing this problem. At this point, we\u2019re pretty sure both the data and the model are okay, but our model isn\u2019t learning well. What else is left? It\u2019s time to\u2026"),Ar.forEach(e),_t=u(s),cs=o(s,"H3",{class:!0});var Cr=p(cs);Ws=o(Cr,"A",{id:!0,class:!0,href:!0});var gm=p(Ws);Ln=o(gm,"SPAN",{});var wm=p(Ln);f(qe.$$.fragment,wm),wm.forEach(e),gm.forEach(e),hh=u(Cr),Mn=o(Cr,"SPAN",{});var vm=p(Mn);ih=l(vm,"Check your hyperparameters"),vm.forEach(e),Cr.forEach(e),Et=u(s),es=o(s,"P",{});var xa=p(es);uh=l(xa,"If you look back at the code above, you might not be able to see any hyperparameters at all, except perhaps the "),Bn=o(xa,"CODE",{});var $m=p(Bn);mh=l($m,"batch_size"),$m.forEach(e),ch=l(xa,", and that doesn\u2019t seem like a likely culprit. Don\u2019t be fooled, though; there are always hyperparameters, and if you can\u2019t see them, it just means that you don\u2019t know what they\u2019re set to. In particular, remember a critical thing about Keras: if you set a loss, optimizer, or activation function with a string, "),Hn=o(xa,"EM",{});var km=p(Hn);bh=l(km,"all of its arguments will be set to their default values"),km.forEach(e),fh=l(xa,". This means that even though using strings for this is very convenient, you should be very careful when doing so, as it can easily hide critical things from you. (Anyone trying the optional challenge above should take careful note of this fact.)"),xa.forEach(e),Tt=u(s),Ls=o(s,"P",{});var Or=p(Ls);dh=l(Or,"In this case, where have we set an argument with a string? We were setting the loss with a string initially, but we\u2019re not doing that anymore. We are, however, setting the optimizer with a string. Could that be hiding anything from us? Let\u2019s take a look at "),xe=o(Or,"A",{href:!0,rel:!0});var _m=p(xe);jh=l(_m,"its arguments"),_m.forEach(e),yh=l(Or,"."),Or.forEach(e),At=u(s),as=o(s,"P",{});var Ia=p(as);gh=l(Ia,"Does anything stand out here? That\u2019s right \u2014 the learning rate! When we just use the string "),Gn=o(Ia,"CODE",{});var Em=p(Gn);wh=l(Em,"'adam'"),Em.forEach(e),vh=l(Ia,", we\u2019re going to get the default learning rate, which is 0.001, or 1e-3. This is way too high for a Transformer model! In general, we recommend trying learning rates between 1e-5 and 1e-4 for your models; that\u2019s somewhere between 10X and 100X smaller than the value we\u2019re actually using here. That sounds like it might be a major problem, so let\u2019s try reducing it. To do that, we need to import the actual "),Kn=o(Ia,"CODE",{});var Tm=p(Kn);$h=l(Tm,"optimizer"),Tm.forEach(e),kh=l(Ia," object. While we\u2019re at it, let\u2019s reinitialize the model from the checkpoint, in case training with the high learning rate damaged its weights:"),Ia.forEach(e),Ct=u(s),f(Ie.$$.fragment,s),Ot=u(s),f(Ms.$$.fragment,s),Dt=u(s),ma=o(s,"P",{});var Am=p(ma);_h=l(Am,"Now, we can try fitting the model with the new, improved learning rate:"),Am.forEach(e),Pt=u(s),f(Se.$$.fragment,s),zt=u(s),f(Ne.$$.fragment,s),qt=u(s),ca=o(s,"P",{});var Cm=p(ca);Eh=l(Cm,"Now our loss is really going somewhere! Training finally looks like it\u2019s working. There\u2019s a lesson here: when your model is running but loss isn\u2019t declining, and you\u2019re sure your data is okay, it\u2019s a good idea to check hyperparameters like the learning rate and weight decay. Setting either of those too high is very likely to cause training to \u201Cstall\u201D at a high loss value."),Cm.forEach(e),xt=u(s),bs=o(s,"H2",{class:!0});var Dr=p(bs);Bs=o(Dr,"A",{id:!0,class:!0,href:!0});var Om=p(Bs);Un=o(Om,"SPAN",{});var Dm=p(Un);f(Fe.$$.fragment,Dm),Dm.forEach(e),Om.forEach(e),Th=u(Dr),Yn=o(Dr,"SPAN",{});var Pm=p(Yn);Ah=l(Pm,"Other potential issues"),Pm.forEach(e),Dr.forEach(e),It=u(s),ba=o(s,"P",{});var zm=p(ba);Ch=l(zm,"We\u2019ve covered the issues in the script above, but there are several other common errors you might face. Let\u2019s take a look at a (very incomplete) list."),zm.forEach(e),St=u(s),fs=o(s,"H3",{class:!0});var Pr=p(fs);Hs=o(Pr,"A",{id:!0,class:!0,href:!0});var qm=p(Hs);Rn=o(qm,"SPAN",{});var xm=p(Rn);f(We.$$.fragment,xm),xm.forEach(e),qm.forEach(e),Oh=u(Pr),Vn=o(Pr,"SPAN",{});var Im=p(Vn);Dh=l(Im,"Dealing with out-of-memory errors"),Im.forEach(e),Pr.forEach(e),Nt=u(s),ns=o(s,"P",{});var Sa=p(ns);Ph=l(Sa,"The telltale sign of running out of memory is an error like \u201COOM when allocating tensor\u201D \u2014 OOM is short for \u201Cout of memory.\u201D This is a very common hazard when dealing with large language models. If you encounter this, a good strategy is to halve your batch size and try again. Bear in mind, though, that some models are "),Jn=o(Sa,"EM",{});var Sm=p(Jn);zh=l(Sm,"very"),Sm.forEach(e),qh=l(Sa," large. For example, the full-size GPT-2 has 1.5B parameters, which means you\u2019ll need 6 GB of memory just to store the model, and another 6 GB for its gradients! Training the full GPT-2 model will usually require over 20 GB of VRAM no matter what batch size you use, which only a few GPUs have. More lightweight models like "),Xn=o(Sa,"CODE",{});var Nm=p(Xn);xh=l(Nm,"distilbert-base-cased"),Nm.forEach(e),Ih=l(Sa," are much easier to run, and train much more quickly too."),Sa.forEach(e),Ft=u(s),f(Gs.$$.fragment,s),Wt=u(s),ds=o(s,"H3",{class:!0});var zr=p(ds);Ks=o(zr,"A",{id:!0,class:!0,href:!0});var Fm=p(Ks);Zn=o(Fm,"SPAN",{});var Wm=p(Zn);f(Le.$$.fragment,Wm),Wm.forEach(e),Fm.forEach(e),Sh=u(zr),Qn=o(zr,"SPAN",{});var Lm=p(Qn);Nh=l(Lm,"Hungry Hungry TensorFlow \u{1F99B}"),Lm.forEach(e),zr.forEach(e),Lt=u(s),ls=o(s,"P",{});var Na=p(ls);Fh=l(Na,"One particular quirk of TensorFlow that you should be aware of is that it allocates "),sl=o(Na,"EM",{});var Mm=p(sl);Wh=l(Mm,"all"),Mm.forEach(e),Lh=l(Na," of your GPU memory to itself as soon as you load a model or do any training, and then it divides up that memory as required. This is different from the behavior of other frameworks, like PyTorch, which allocate memory as required with CUDA rather than doing it internally. One advantage of the TensorFlow approach is that it can often give useful errors when you run out of memory, and it can recover from that state without crashing the whole CUDA kernel. But there\u2019s also an important downside: if you run two TensorFlow processes at once, then "),el=o(Na,"STRONG",{});var Bm=p(el);Mh=l(Bm,"you\u2019re going to have a bad time"),Bm.forEach(e),Bh=l(Na,"."),Na.forEach(e),Mt=u(s),fa=o(s,"P",{});var Hm=p(fa);Hh=l(Hm,"If you\u2019re running on Colab you don\u2019t need to worry about this, but if you\u2019re running locally this is definitely something you should be careful about. In particular, be aware that closing a notebook tab does not necessarily shut that notebook down! You may need to select running notebooks (the ones with a green icon) and manually shut them down in the directory listing. Any running notebook that was using TensorFlow could still be holding on to a bunch of your GPU memory, and that means any new notebook you start may encounter some very odd issues."),Hm.forEach(e),Bt=u(s),Us=o(s,"P",{});var qr=p(Us);Gh=l(qr,"If you start getting errors about CUDA, BLAS, or cuBLAS in code that worked before, this is very often the culprit. You can use a command like "),al=o(qr,"CODE",{});var Gm=p(al);Kh=l(Gm,"nvidia-smi"),Gm.forEach(e),Uh=l(qr," to check \u2014 when you shut down or restart your current notebook, is most of your memory free, or is it still in use? If it\u2019s still in use, something else is holding on to it!"),qr.forEach(e),Ht=u(s),js=o(s,"H3",{class:!0});var xr=p(js);Ys=o(xr,"A",{id:!0,class:!0,href:!0});var Km=p(Ys);nl=o(Km,"SPAN",{});var Um=p(nl);f(Me.$$.fragment,Um),Um.forEach(e),Km.forEach(e),Yh=u(xr),ll=o(xr,"SPAN",{});var Ym=p(ll);Rh=l(Ym,"Check your data (again!)"),Ym.forEach(e),xr.forEach(e),Gt=u(s),x=o(s,"P",{});var rs=p(x);Vh=l(rs,"Your model will only learn something if it\u2019s actually possible to learn anything from your data. If there is a bug that corrupts the data or the labels are attributed randomly, it\u2019s very likely you won\u2019t get any model training on your dataset. One helpful tool here is "),tl=o(rs,"CODE",{});var Rm=p(tl);Jh=l(Rm,"tokenizer.decode()"),Rm.forEach(e),Xh=l(rs,". This will turn "),rl=o(rs,"CODE",{});var Vm=p(rl);Zh=l(Vm,"input_ids"),Vm.forEach(e),Qh=l(rs," back into strings, so you can view the data and see if your training data is teaching what you want it to teach. For example, after you get a "),ol=o(rs,"CODE",{});var Jm=p(ol);si=l(Jm,"batch"),Jm.forEach(e),ei=l(rs," from your "),pl=o(rs,"CODE",{});var Xm=p(pl);ai=l(Xm,"tf.data.Dataset"),Xm.forEach(e),ni=l(rs," like we did above, you can decode the first element like so:"),rs.forEach(e),Kt=u(s),f(Be.$$.fragment,s),Ut=u(s),da=o(s,"P",{});var Zm=p(da);li=l(Zm,"Then you can compare it with the first label, like so:"),Zm.forEach(e),Yt=u(s),f(He.$$.fragment,s),Rt=u(s),ja=o(s,"P",{});var Qm=p(ja);ti=l(Qm,"Once you can view your data like this, you can ask yourself the following questions:"),Qm.forEach(e),Vt=u(s),L=o(s,"UL",{});var le=p(L);hl=o(le,"LI",{});var sc=p(hl);ri=l(sc,"Is the decoded data understandable?"),sc.forEach(e),oi=u(le),il=o(le,"LI",{});var ec=p(il);pi=l(ec,"Do you agree with the labels?"),ec.forEach(e),hi=u(le),ul=o(le,"LI",{});var ac=p(ul);ii=l(ac,"Is there one label that\u2019s more common than the others?"),ac.forEach(e),ui=u(le),ml=o(le,"LI",{});var nc=p(ml);mi=l(nc,"What should the loss/metric be if the model predicted a random answer/always the same answer?"),nc.forEach(e),le.forEach(e),Jt=u(s),ya=o(s,"P",{});var lc=p(ya);ci=l(lc,"After looking at your data, go through a few of the model\u2019s predictions \u2014 if your model outputs tokens, try decoding them too! If the model is always predicting the same thing it might be because your dataset is biased toward one category (for classification problems), so techniques like oversampling rare classes might help. Alternatively, this can also be caused by training issues like bad hyperparameter settings."),lc.forEach(e),Xt=u(s),ga=o(s,"P",{});var tc=p(ga);bi=l(tc,"If the loss/metric you get on your initial model before any training is very different from the loss/metric you would expect for random predictions, double-check the way your loss or metric is computed, as there is probably a bug there. If you are using several losses that you add at the end, make sure they are of the same scale."),tc.forEach(e),Zt=u(s),wa=o(s,"P",{});var rc=p(wa);fi=l(rc,"When you are sure your data is perfect, you can see if the model is capable of training on it with one simple test."),rc.forEach(e),Qt=u(s),ys=o(s,"H3",{class:!0});var Ir=p(ys);Rs=o(Ir,"A",{id:!0,class:!0,href:!0});var oc=p(Rs);cl=o(oc,"SPAN",{});var pc=p(cl);f(Ge.$$.fragment,pc),pc.forEach(e),oc.forEach(e),di=u(Ir),bl=o(Ir,"SPAN",{});var hc=p(bl);ji=l(hc,"Overfit your model on one batch"),hc.forEach(e),Ir.forEach(e),sr=u(s),va=o(s,"P",{});var ic=p(va);yi=l(ic,"Overfitting is usually something we try to avoid when training, as it means the model is not learning to recognize the general features we want it to but is instead just memorizing the training samples. However, trying to train your model on one batch over and over again is a good test to check if the problem as you framed it can be solved by the model you are attempting to train. It will also help you see if your initial learning rate is too high."),ic.forEach(e),er=u(s),ts=o(s,"P",{});var Fa=p(ts);gi=l(Fa,"Doing this once you have defined your "),fl=o(Fa,"CODE",{});var uc=p(fl);wi=l(uc,"model"),uc.forEach(e),vi=l(Fa," is really easy; just grab a batch of training data, then treat that "),dl=o(Fa,"CODE",{});var mc=p(dl);$i=l(mc,"batch"),mc.forEach(e),ki=l(Fa," as your entire dataset, fitting on it for a large number of epochs:"),Fa.forEach(e),ar=u(s),f(Ke.$$.fragment,s),nr=u(s),f(Vs.$$.fragment,s),lr=u(s),Js=o(s,"P",{});var Sr=p(Js);_i=l(Sr,"The resulting model should have close-to-perfect results on the "),jl=o(Sr,"CODE",{});var cc=p(jl);Ei=l(cc,"batch"),cc.forEach(e),Ti=l(Sr,", with a loss declining quickly toward 0 (or the minimum value for the loss you\u2019re using)."),Sr.forEach(e),tr=u(s),$a=o(s,"P",{});var bc=p($a);Ai=l(bc,"If you don\u2019t manage to have your model obtain perfect results like this, it means there is something wrong with the way you framed the problem or your data, so you should fix that. Only when you manage to pass the overfitting test can you be sure that your model can actually learn something."),bc.forEach(e),rr=u(s),f(Xs.$$.fragment,s),or=u(s),gs=o(s,"H3",{class:!0});var Nr=p(gs);Zs=o(Nr,"A",{id:!0,class:!0,href:!0});var fc=p(Zs);yl=o(fc,"SPAN",{});var dc=p(yl);f(Ue.$$.fragment,dc),dc.forEach(e),fc.forEach(e),Ci=u(Nr),gl=o(Nr,"SPAN",{});var jc=p(gl);Oi=l(jc,"Don't tune anything until you have a first baseline"),jc.forEach(e),Nr.forEach(e),pr=u(s),Qs=o(s,"P",{});var Fr=p(Qs);Di=l(Fr,"Intense hyperparameter tuning is always emphasized as being the hardest part of machine learning, but it\u2019s just the last step to help you gain a little bit on the metric. "),wl=o(Fr,"EM",{});var yc=p(wl);Pi=l(yc,"Very"),yc.forEach(e),zi=l(Fr," bad values for your hyperparameters, like using the default Adam learning rate of 1e-3 with a Transformer model, will make learning proceed very slowly or completely stall, of course, but most of the time \u201Creasonable\u201D hyperparameters, like a learning rate from 1e-5 to 5e-5, will work just fine to give you good results. So, don\u2019t launch into a time-consuming and costly hyperparameter search until you have something that beats the baseline you have on your dataset."),Fr.forEach(e),hr=u(s),ka=o(s,"P",{});var gc=p(ka);qi=l(gc,"Once you have a good enough model, you can start tweaking a bit. Don\u2019t try launching a thousand runs with different hyperparameters, but compare a couple of runs with different values for one hyperparameter to get an idea of which has the greatest impact."),gc.forEach(e),ir=u(s),_a=o(s,"P",{});var wc=p(_a);xi=l(wc,"If you are tweaking the model itself, keep it simple and don\u2019t try anything you can\u2019t reasonably justify. Always make sure you go back to the overfitting test to verify that your change hasn\u2019t had any unintended consequences."),wc.forEach(e),ur=u(s),ws=o(s,"H3",{class:!0});var Wr=p(ws);se=o(Wr,"A",{id:!0,class:!0,href:!0});var vc=p(se);vl=o(vc,"SPAN",{});var $c=p(vl);f(Ye.$$.fragment,$c),$c.forEach(e),vc.forEach(e),Ii=u(Wr),$l=o(Wr,"SPAN",{});var kc=p($l);Si=l(kc,"Ask for help"),kc.forEach(e),Wr.forEach(e),mr=u(s),ee=o(s,"P",{});var Lr=p(ee);Ni=l(Lr,"Hopefully you will have found some advice in this section that helped you solve your issue, but if that\u2019s not the case, remember you can always ask the community on the "),Re=o(Lr,"A",{href:!0,rel:!0});var _c=p(Re);Fi=l(_c,"forums"),_c.forEach(e),Wi=l(Lr,"."),Lr.forEach(e),cr=u(s),Ea=o(s,"P",{});var Ec=p(Ea);Li=l(Ec,"Here are some additional resources that may prove helpful:"),Ec.forEach(e),br=u(s),M=o(s,"UL",{});var te=p(M);Ta=o(te,"LI",{});var su=p(Ta);Ve=o(su,"A",{href:!0,rel:!0});var Tc=p(Ve);Mi=l(Tc,"\u201CReproducibility as a vehicle for engineering best practices\u201D"),Tc.forEach(e),Bi=l(su," by Joel Grus"),su.forEach(e),Hi=u(te),Aa=o(te,"LI",{});var eu=p(Aa);Je=o(eu,"A",{href:!0,rel:!0});var Ac=p(Je);Gi=l(Ac,"\u201CChecklist for debugging neural networks\u201D"),Ac.forEach(e),Ki=l(eu," by Cecelia Shao"),eu.forEach(e),Ui=u(te),Ca=o(te,"LI",{});var au=p(Ca);Xe=o(au,"A",{href:!0,rel:!0});var Cc=p(Xe);Yi=l(Cc,"\u201CHow to unit test machine learning code\u201D"),Cc.forEach(e),Ri=l(au," by Chase Roberts"),au.forEach(e),Vi=u(te),Oa=o(te,"LI",{});var nu=p(Oa);Ze=o(nu,"A",{href:!0,rel:!0});var Oc=p(Ze);Ji=l(Oc,"\u201CA Recipe for Training Neural Networks\u201D"),Oc.forEach(e),Xi=l(nu," by Andrej Karpathy"),nu.forEach(e),te.forEach(e),fr=u(s),Da=o(s,"P",{});var Dc=p(Da);Zi=l(Dc,"Of course, not every problem you encounter when training neural nets is your own fault! If you encounter something in the \u{1F917} Transformers or \u{1F917} Datasets library that does not seem right, you may have encountered a bug. You should definitely tell us all about it, and in the next section we\u2019ll explain exactly how to do that."),Dc.forEach(e),this.h()},h(){m(c,"name","hf:doc:metadata"),m(c,"content",JSON.stringify(Kc)),m(C,"id","debugging-the-training-pipeline"),m(C,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),m(C,"href","#debugging-the-training-pipeline"),m(D,"class","relative group"),m(R,"href","/course/chapter7"),m(A,"id","debugging-the-training-pipeline"),m(A,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),m(A,"href","#debugging-the-training-pipeline"),m(_,"class","relative group"),m(he,"href","https://huggingface.co/datasets/glue"),m(he,"rel","nofollow"),m(zs,"id","check-your-data"),m(zs,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),m(zs,"href","#check-your-data"),m(us,"class","relative group"),m(Ss,"id","check-your-model"),m(Ss,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),m(Ss,"href","#check-your-model"),m(ms,"class","relative group"),m(Ws,"id","check-your-hyperparameters"),m(Ws,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),m(Ws,"href","#check-your-hyperparameters"),m(cs,"class","relative group"),m(xe,"href","https://www.tensorflow.org/api_docs/python/tf/keras/optimizers/Adam"),m(xe,"rel","nofollow"),m(Bs,"id","other-potential-issues"),m(Bs,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),m(Bs,"href","#other-potential-issues"),m(bs,"class","relative group"),m(Hs,"id","dealing-with-outofmemory-errors"),m(Hs,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),m(Hs,"href","#dealing-with-outofmemory-errors"),m(fs,"class","relative group"),m(Ks,"id","hungry-hungry-tensorflow"),m(Ks,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),m(Ks,"href","#hungry-hungry-tensorflow"),m(ds,"class","relative group"),m(Ys,"id","check-your-data-again"),m(Ys,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),m(Ys,"href","#check-your-data-again"),m(js,"class","relative group"),m(Rs,"id","overfit-your-model-on-one-batch"),m(Rs,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),m(Rs,"href","#overfit-your-model-on-one-batch"),m(ys,"class","relative group"),m(Zs,"id","dont-tune-anything-until-you-have-a-first-baseline"),m(Zs,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),m(Zs,"href","#dont-tune-anything-until-you-have-a-first-baseline"),m(gs,"class","relative group"),m(se,"id","ask-for-help"),m(se,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),m(se,"href","#ask-for-help"),m(ws,"class","relative group"),m(Re,"href","https://discuss.huggingface.co/"),m(Re,"rel","nofollow"),m(Ve,"href","https://docs.google.com/presentation/d/1yHLPvPhUs2KGI5ZWo0sU-PKU3GimAk3iTsI38Z-B5Gw/edit#slide=id.p"),m(Ve,"rel","nofollow"),m(Je,"href","https://towardsdatascience.com/checklist-for-debugging-neural-networks-d8b2a9434f21"),m(Je,"rel","nofollow"),m(Xe,"href","https://medium.com/@keeper6928/how-to-unit-test-machine-learning-code-57cf6fd81765"),m(Xe,"rel","nofollow"),m(Ze,"href","http://karpathy.github.io/2019/04/25/recipe/"),m(Ze,"rel","nofollow")},m(s,t){a(document.head,c),h(s,E,t),d(w,s,t),h(s,k,t),h(s,D,t),a(D,C),a(C,P),d(q,P,null),a(D,os),a(D,ps),a(ps,re),h(s,Y,t),d(H,s,t),h(s,$s,t),h(s,z,t),a(z,N),a(z,R),a(R,hs),a(z,oe),a(z,is),a(is,ks),a(z,V),h(s,_s,t),h(s,_,t),a(_,A),a(A,Es),d(J,Es,null),a(_,na),a(_,Ts),a(Ts,As),h(s,pe,t),d(X,s,t),h(s,Cl,t),h(s,Cs,t),a(Cs,Mr),a(Cs,Wa),a(Wa,Br),a(Cs,Hr),h(s,Ol,t),h(s,Os,t),a(Os,Gr),a(Os,La),a(La,Kr),a(Os,Ur),h(s,Dl,t),h(s,Ds,t),a(Ds,Yr),a(Ds,he),a(he,Rr),a(Ds,Vr),h(s,Pl,t),d(ie,s,t),h(s,zl,t),h(s,Ps,t),a(Ps,Jr),a(Ps,Ma),a(Ma,Xr),a(Ps,Zr),h(s,ql,t),h(s,la,t),a(la,Qr),h(s,xl,t),d(ue,s,t),h(s,Il,t),h(s,ta,t),a(ta,so),h(s,Sl,t),h(s,us,t),a(us,zs),a(zs,Ba),d(me,Ba,null),a(us,eo),a(us,Ha),a(Ha,ao),h(s,Nl,t),h(s,ra,t),a(ra,no),h(s,Fl,t),h(s,O,t),a(O,lo),a(O,Ga),a(Ga,to),a(O,ro),a(O,Ka),a(Ka,oo),a(O,po),a(O,Ua),a(Ua,ho),a(O,io),a(O,Ya),a(Ya,uo),a(O,mo),a(O,Ra),a(Ra,co),a(O,bo),a(O,Va),a(Va,fo),a(O,jo),h(s,Wl,t),d(ce,s,t),h(s,Ll,t),h(s,G,t),a(G,Ja),a(Ja,yo),a(G,go),a(G,Xa),a(Xa,wo),a(G,vo),a(G,Za),a(Za,$o),a(G,ko),h(s,Ml,t),d(be,s,t),h(s,Bl,t),h(s,F,t),a(F,_o),a(F,Qa),a(Qa,Eo),a(F,To),a(F,sn),a(sn,Ao),a(F,Co),a(F,en),a(en,Oo),a(F,Do),h(s,Hl,t),h(s,qs,t),a(qs,Po),a(qs,an),a(an,zo),a(qs,qo),h(s,Gl,t),h(s,Z,t),a(Z,xo),a(Z,nn),a(nn,Io),a(Z,So),a(Z,ln),a(ln,No),a(Z,Fo),h(s,Kl,t),d(fe,s,t),h(s,Ul,t),h(s,oa,t),a(oa,Wo),h(s,Yl,t),d(xs,s,t),h(s,Rl,t),h(s,Is,t),a(Is,Lo),a(Is,tn),a(tn,Mo),a(Is,Bo),h(s,Vl,t),d(de,s,t),h(s,Jl,t),h(s,pa,t),a(pa,Ho),h(s,Xl,t),h(s,je,t),a(je,rn),a(rn,Go),a(je,Ko),h(s,Zl,t),h(s,ms,t),a(ms,Ss),a(Ss,on),d(ye,on,null),a(ms,Uo),a(ms,pn),a(pn,Yo),h(s,Ql,t),h(s,K,t),a(K,hn),a(hn,Ro),a(K,Vo),a(K,un),a(un,Jo),a(K,Xo),a(K,mn),a(mn,Zo),a(K,Qo),h(s,st,t),h(s,Q,t),a(Q,sp),a(Q,cn),a(cn,ep),a(Q,ap),a(Q,bn),a(bn,np),a(Q,lp),h(s,et,t),d(ge,s,t),h(s,at,t),d(we,s,t),h(s,nt,t),h(s,v,t),a(v,tp),a(v,fn),a(fn,rp),a(v,op),a(v,dn),a(dn,pp),a(v,hp),a(v,jn),a(jn,ip),a(v,up),a(v,yn),a(yn,mp),a(v,cp),a(v,gn),a(gn,bp),a(v,fp),a(v,wn),a(wn,dp),a(v,jp),a(v,vn),a(vn,yp),a(v,gp),a(v,$n),a($n,wp),a(v,vp),a(v,kn),a(kn,$p),a(v,kp),a(v,_n),a(_n,_p),a(v,Ep),a(v,En),a(En,Tp),a(v,Ap),a(v,Tn),a(Tn,Cp),a(v,Op),a(v,An),a(An,Dp),a(v,Pp),a(v,Cn),a(Cn,zp),a(v,qp),a(v,On),a(On,xp),a(v,Ip),h(s,lt,t),h(s,W,t),a(W,Sp),a(W,Dn),a(Dn,Np),a(W,Fp),a(W,Pn),a(Pn,Wp),a(W,Lp),a(W,zn),a(zn,Mp),a(W,Bp),h(s,tt,t),d(ve,s,t),h(s,rt,t),h(s,ha,t),a(ha,Hp),h(s,ot,t),d($e,s,t),h(s,pt,t),h(s,U,t),a(U,qn),a(qn,Gp),a(U,Kp),a(U,xn),a(xn,Up),a(U,Yp),a(U,In),a(In,Rp),a(U,Vp),h(s,ht,t),d(ke,s,t),h(s,it,t),d(_e,s,t),h(s,ut,t),h(s,ia,t),a(ia,Jp),h(s,mt,t),d(Ee,s,t),h(s,ct,t),d(Te,s,t),h(s,bt,t),h(s,ua,t),a(ua,Xp),h(s,ft,t),d(Ae,s,t),h(s,dt,t),d(Ce,s,t),h(s,jt,t),h(s,ss,t),a(ss,Zp),a(ss,Sn),a(Sn,Qp),a(ss,sh),a(ss,Nn),a(Nn,eh),a(ss,ah),h(s,yt,t),d(Oe,s,t),h(s,gt,t),d(De,s,t),h(s,wt,t),h(s,Ns,t),a(Ns,nh),a(Ns,Fn),a(Fn,lh),a(Ns,th),h(s,vt,t),d(Pe,s,t),h(s,$t,t),d(ze,s,t),h(s,kt,t),h(s,Fs,t),a(Fs,rh),a(Fs,Wn),a(Wn,oh),a(Fs,ph),h(s,_t,t),h(s,cs,t),a(cs,Ws),a(Ws,Ln),d(qe,Ln,null),a(cs,hh),a(cs,Mn),a(Mn,ih),h(s,Et,t),h(s,es,t),a(es,uh),a(es,Bn),a(Bn,mh),a(es,ch),a(es,Hn),a(Hn,bh),a(es,fh),h(s,Tt,t),h(s,Ls,t),a(Ls,dh),a(Ls,xe),a(xe,jh),a(Ls,yh),h(s,At,t),h(s,as,t),a(as,gh),a(as,Gn),a(Gn,wh),a(as,vh),a(as,Kn),a(Kn,$h),a(as,kh),h(s,Ct,t),d(Ie,s,t),h(s,Ot,t),d(Ms,s,t),h(s,Dt,t),h(s,ma,t),a(ma,_h),h(s,Pt,t),d(Se,s,t),h(s,zt,t),d(Ne,s,t),h(s,qt,t),h(s,ca,t),a(ca,Eh),h(s,xt,t),h(s,bs,t),a(bs,Bs),a(Bs,Un),d(Fe,Un,null),a(bs,Th),a(bs,Yn),a(Yn,Ah),h(s,It,t),h(s,ba,t),a(ba,Ch),h(s,St,t),h(s,fs,t),a(fs,Hs),a(Hs,Rn),d(We,Rn,null),a(fs,Oh),a(fs,Vn),a(Vn,Dh),h(s,Nt,t),h(s,ns,t),a(ns,Ph),a(ns,Jn),a(Jn,zh),a(ns,qh),a(ns,Xn),a(Xn,xh),a(ns,Ih),h(s,Ft,t),d(Gs,s,t),h(s,Wt,t),h(s,ds,t),a(ds,Ks),a(Ks,Zn),d(Le,Zn,null),a(ds,Sh),a(ds,Qn),a(Qn,Nh),h(s,Lt,t),h(s,ls,t),a(ls,Fh),a(ls,sl),a(sl,Wh),a(ls,Lh),a(ls,el),a(el,Mh),a(ls,Bh),h(s,Mt,t),h(s,fa,t),a(fa,Hh),h(s,Bt,t),h(s,Us,t),a(Us,Gh),a(Us,al),a(al,Kh),a(Us,Uh),h(s,Ht,t),h(s,js,t),a(js,Ys),a(Ys,nl),d(Me,nl,null),a(js,Yh),a(js,ll),a(ll,Rh),h(s,Gt,t),h(s,x,t),a(x,Vh),a(x,tl),a(tl,Jh),a(x,Xh),a(x,rl),a(rl,Zh),a(x,Qh),a(x,ol),a(ol,si),a(x,ei),a(x,pl),a(pl,ai),a(x,ni),h(s,Kt,t),d(Be,s,t),h(s,Ut,t),h(s,da,t),a(da,li),h(s,Yt,t),d(He,s,t),h(s,Rt,t),h(s,ja,t),a(ja,ti),h(s,Vt,t),h(s,L,t),a(L,hl),a(hl,ri),a(L,oi),a(L,il),a(il,pi),a(L,hi),a(L,ul),a(ul,ii),a(L,ui),a(L,ml),a(ml,mi),h(s,Jt,t),h(s,ya,t),a(ya,ci),h(s,Xt,t),h(s,ga,t),a(ga,bi),h(s,Zt,t),h(s,wa,t),a(wa,fi),h(s,Qt,t),h(s,ys,t),a(ys,Rs),a(Rs,cl),d(Ge,cl,null),a(ys,di),a(ys,bl),a(bl,ji),h(s,sr,t),h(s,va,t),a(va,yi),h(s,er,t),h(s,ts,t),a(ts,gi),a(ts,fl),a(fl,wi),a(ts,vi),a(ts,dl),a(dl,$i),a(ts,ki),h(s,ar,t),d(Ke,s,t),h(s,nr,t),d(Vs,s,t),h(s,lr,t),h(s,Js,t),a(Js,_i),a(Js,jl),a(jl,Ei),a(Js,Ti),h(s,tr,t),h(s,$a,t),a($a,Ai),h(s,rr,t),d(Xs,s,t),h(s,or,t),h(s,gs,t),a(gs,Zs),a(Zs,yl),d(Ue,yl,null),a(gs,Ci),a(gs,gl),a(gl,Oi),h(s,pr,t),h(s,Qs,t),a(Qs,Di),a(Qs,wl),a(wl,Pi),a(Qs,zi),h(s,hr,t),h(s,ka,t),a(ka,qi),h(s,ir,t),h(s,_a,t),a(_a,xi),h(s,ur,t),h(s,ws,t),a(ws,se),a(se,vl),d(Ye,vl,null),a(ws,Ii),a(ws,$l),a($l,Si),h(s,mr,t),h(s,ee,t),a(ee,Ni),a(ee,Re),a(Re,Fi),a(ee,Wi),h(s,cr,t),h(s,Ea,t),a(Ea,Li),h(s,br,t),h(s,M,t),a(M,Ta),a(Ta,Ve),a(Ve,Mi),a(Ta,Bi),a(M,Hi),a(M,Aa),a(Aa,Je),a(Je,Gi),a(Aa,Ki),a(M,Ui),a(M,Ca),a(Ca,Xe),a(Xe,Yi),a(Ca,Ri),a(M,Vi),a(M,Oa),a(Oa,Ze),a(Ze,Ji),a(Oa,Xi),h(s,fr,t),h(s,Da,t),a(Da,Zi),dr=!0},p(s,[t]){const Qe={};t&1&&(Qe.fw=s[0]),w.$set(Qe);const kl={};t&2&&(kl.$$scope={dirty:t,ctx:s}),xs.$set(kl);const _l={};t&2&&(_l.$$scope={dirty:t,ctx:s}),Ms.$set(_l);const El={};t&2&&(El.$$scope={dirty:t,ctx:s}),Gs.$set(El);const vs={};t&2&&(vs.$$scope={dirty:t,ctx:s}),Vs.$set(vs);const Tl={};t&2&&(Tl.$$scope={dirty:t,ctx:s}),Xs.$set(Tl)},i(s){dr||(j(w.$$.fragment,s),j(q.$$.fragment,s),j(H.$$.fragment,s),j(J.$$.fragment,s),j(X.$$.fragment,s),j(ie.$$.fragment,s),j(ue.$$.fragment,s),j(me.$$.fragment,s),j(ce.$$.fragment,s),j(be.$$.fragment,s),j(fe.$$.fragment,s),j(xs.$$.fragment,s),j(de.$$.fragment,s),j(ye.$$.fragment,s),j(ge.$$.fragment,s),j(we.$$.fragment,s),j(ve.$$.fragment,s),j($e.$$.fragment,s),j(ke.$$.fragment,s),j(_e.$$.fragment,s),j(Ee.$$.fragment,s),j(Te.$$.fragment,s),j(Ae.$$.fragment,s),j(Ce.$$.fragment,s),j(Oe.$$.fragment,s),j(De.$$.fragment,s),j(Pe.$$.fragment,s),j(ze.$$.fragment,s),j(qe.$$.fragment,s),j(Ie.$$.fragment,s),j(Ms.$$.fragment,s),j(Se.$$.fragment,s),j(Ne.$$.fragment,s),j(Fe.$$.fragment,s),j(We.$$.fragment,s),j(Gs.$$.fragment,s),j(Le.$$.fragment,s),j(Me.$$.fragment,s),j(Be.$$.fragment,s),j(He.$$.fragment,s),j(Ge.$$.fragment,s),j(Ke.$$.fragment,s),j(Vs.$$.fragment,s),j(Xs.$$.fragment,s),j(Ue.$$.fragment,s),j(Ye.$$.fragment,s),dr=!0)},o(s){y(w.$$.fragment,s),y(q.$$.fragment,s),y(H.$$.fragment,s),y(J.$$.fragment,s),y(X.$$.fragment,s),y(ie.$$.fragment,s),y(ue.$$.fragment,s),y(me.$$.fragment,s),y(ce.$$.fragment,s),y(be.$$.fragment,s),y(fe.$$.fragment,s),y(xs.$$.fragment,s),y(de.$$.fragment,s),y(ye.$$.fragment,s),y(ge.$$.fragment,s),y(we.$$.fragment,s),y(ve.$$.fragment,s),y($e.$$.fragment,s),y(ke.$$.fragment,s),y(_e.$$.fragment,s),y(Ee.$$.fragment,s),y(Te.$$.fragment,s),y(Ae.$$.fragment,s),y(Ce.$$.fragment,s),y(Oe.$$.fragment,s),y(De.$$.fragment,s),y(Pe.$$.fragment,s),y(ze.$$.fragment,s),y(qe.$$.fragment,s),y(Ie.$$.fragment,s),y(Ms.$$.fragment,s),y(Se.$$.fragment,s),y(Ne.$$.fragment,s),y(Fe.$$.fragment,s),y(We.$$.fragment,s),y(Gs.$$.fragment,s),y(Le.$$.fragment,s),y(Me.$$.fragment,s),y(Be.$$.fragment,s),y(He.$$.fragment,s),y(Ge.$$.fragment,s),y(Ke.$$.fragment,s),y(Vs.$$.fragment,s),y(Xs.$$.fragment,s),y(Ue.$$.fragment,s),y(Ye.$$.fragment,s),dr=!1},d(s){e(c),s&&e(E),g(w,s),s&&e(k),s&&e(D),g(q),s&&e(Y),g(H,s),s&&e($s),s&&e(z),s&&e(_s),s&&e(_),g(J),s&&e(pe),g(X,s),s&&e(Cl),s&&e(Cs),s&&e(Ol),s&&e(Os),s&&e(Dl),s&&e(Ds),s&&e(Pl),g(ie,s),s&&e(zl),s&&e(Ps),s&&e(ql),s&&e(la),s&&e(xl),g(ue,s),s&&e(Il),s&&e(ta),s&&e(Sl),s&&e(us),g(me),s&&e(Nl),s&&e(ra),s&&e(Fl),s&&e(O),s&&e(Wl),g(ce,s),s&&e(Ll),s&&e(G),s&&e(Ml),g(be,s),s&&e(Bl),s&&e(F),s&&e(Hl),s&&e(qs),s&&e(Gl),s&&e(Z),s&&e(Kl),g(fe,s),s&&e(Ul),s&&e(oa),s&&e(Yl),g(xs,s),s&&e(Rl),s&&e(Is),s&&e(Vl),g(de,s),s&&e(Jl),s&&e(pa),s&&e(Xl),s&&e(je),s&&e(Zl),s&&e(ms),g(ye),s&&e(Ql),s&&e(K),s&&e(st),s&&e(Q),s&&e(et),g(ge,s),s&&e(at),g(we,s),s&&e(nt),s&&e(v),s&&e(lt),s&&e(W),s&&e(tt),g(ve,s),s&&e(rt),s&&e(ha),s&&e(ot),g($e,s),s&&e(pt),s&&e(U),s&&e(ht),g(ke,s),s&&e(it),g(_e,s),s&&e(ut),s&&e(ia),s&&e(mt),g(Ee,s),s&&e(ct),g(Te,s),s&&e(bt),s&&e(ua),s&&e(ft),g(Ae,s),s&&e(dt),g(Ce,s),s&&e(jt),s&&e(ss),s&&e(yt),g(Oe,s),s&&e(gt),g(De,s),s&&e(wt),s&&e(Ns),s&&e(vt),g(Pe,s),s&&e($t),g(ze,s),s&&e(kt),s&&e(Fs),s&&e(_t),s&&e(cs),g(qe),s&&e(Et),s&&e(es),s&&e(Tt),s&&e(Ls),s&&e(At),s&&e(as),s&&e(Ct),g(Ie,s),s&&e(Ot),g(Ms,s),s&&e(Dt),s&&e(ma),s&&e(Pt),g(Se,s),s&&e(zt),g(Ne,s),s&&e(qt),s&&e(ca),s&&e(xt),s&&e(bs),g(Fe),s&&e(It),s&&e(ba),s&&e(St),s&&e(fs),g(We),s&&e(Nt),s&&e(ns),s&&e(Ft),g(Gs,s),s&&e(Wt),s&&e(ds),g(Le),s&&e(Lt),s&&e(ls),s&&e(Mt),s&&e(fa),s&&e(Bt),s&&e(Us),s&&e(Ht),s&&e(js),g(Me),s&&e(Gt),s&&e(x),s&&e(Kt),g(Be,s),s&&e(Ut),s&&e(da),s&&e(Yt),g(He,s),s&&e(Rt),s&&e(ja),s&&e(Vt),s&&e(L),s&&e(Jt),s&&e(ya),s&&e(Xt),s&&e(ga),s&&e(Zt),s&&e(wa),s&&e(Qt),s&&e(ys),g(Ge),s&&e(sr),s&&e(va),s&&e(er),s&&e(ts),s&&e(ar),g(Ke,s),s&&e(nr),g(Vs,s),s&&e(lr),s&&e(Js),s&&e(tr),s&&e($a),s&&e(rr),g(Xs,s),s&&e(or),s&&e(gs),g(Ue),s&&e(pr),s&&e(Qs),s&&e(hr),s&&e(ka),s&&e(ir),s&&e(_a),s&&e(ur),s&&e(ws),g(Ye),s&&e(mr),s&&e(ee),s&&e(cr),s&&e(Ea),s&&e(br),s&&e(M),s&&e(fr),s&&e(Da)}}}const Kc={local:"debugging-the-training-pipeline",sections:[{local:"debugging-the-training-pipeline",sections:[{local:"check-your-data",title:"Check your data"},{local:"check-your-model",title:"Check your model"},{local:"check-your-hyperparameters",title:"Check your hyperparameters"}],title:"Debugging the training pipeline"},{local:"other-potential-issues",sections:[{local:"dealing-with-outofmemory-errors",title:"Dealing with out-of-memory errors"},{local:"hungry-hungry-tensorflow",title:"Hungry Hungry TensorFlow \u{1F99B}"},{local:"check-your-data-again",title:"Check your data (again!)"},{local:"overfit-your-model-on-one-batch",title:"Overfit your model on one batch"},{local:"dont-tune-anything-until-you-have-a-first-baseline",title:"Don't tune anything until you have a first baseline"},{local:"ask-for-help",title:"Ask for help"}],title:"Other potential issues "}],title:"Debugging the training pipeline"};function Uc(S,c,E){let w="pt";return Ic(()=>{const k=new URLSearchParams(window.location.search);E(0,w=k.get("fw")||"pt")}),[w]}class s0 extends Pc{constructor(c){super();zc(this,c,Uc,Gc,qc,{})}}export{s0 as default,Kc as metadata};
