import{S as qi,i as Bi,s as Fi,e as r,k as i,w as E,t as o,M as Ki,c as s,d as t,m as p,a as n,x as D,h as l,b as m,G as e,g as b,y as j,q as k,o as P,B as x,v as Yi,L as Va}from"../../chunks/vendor-hf-doc-builder.js";import{D as A}from"../../chunks/Docstring-hf-doc-builder.js";import{C as Ma}from"../../chunks/CodeBlock-hf-doc-builder.js";import{I as gt}from"../../chunks/IconCopyLink-hf-doc-builder.js";import{E as Ga}from"../../chunks/ExampleCodeBlock-hf-doc-builder.js";function Wi(L){let g,T,w,f,$;return f=new Ma({props:{code:"debug_overflow = DebugUnderflowOverflow(model)",highlighted:"debug_overflow = DebugUnderflowOverflow(model)"}}),{c(){g=r("p"),T=o("To activate the underflow/overflow detection, initialize the object with the model :"),w=i(),E(f.$$.fragment)},l(c){g=s(c,"P",{});var _=n(g);T=l(_,"To activate the underflow/overflow detection, initialize the object with the model :"),_.forEach(t),w=p(c),D(f.$$.fragment,c)},m(c,_){b(c,g,_),e(g,T),b(c,w,_),j(f,c,_),$=!0},p:Va,i(c){$||(k(f.$$.fragment,c),$=!0)},o(c){P(f.$$.fragment,c),$=!1},d(c){c&&t(g),c&&t(w),x(f,c)}}}function Ji(L){let g,T,w,f,$;return f=new Ma({props:{code:`Detected inf/nan during batch_number=0
Last 21 forward frames:
abs min  abs max  metadata
[...]
                  encoder.block.2.layer.1.DenseReluDense.wi_0 Linear
2.17e-07 4.50e+00 weight
1.79e-06 4.65e+00 input[0]
2.68e-06 3.70e+01 output
                  encoder.block.2.layer.1.DenseReluDense.wi_1 Linear
8.08e-07 2.66e+01 weight
1.79e-06 4.65e+00 input[0]
1.27e-04 2.37e+02 output
                  encoder.block.2.layer.1.DenseReluDense.wo Linear
1.01e-06 6.44e+00 weight
0.00e+00 9.74e+03 input[0]
3.18e-04 6.27e+04 output
                  encoder.block.2.layer.1.DenseReluDense T5DenseGatedGeluDense
1.79e-06 4.65e+00 input[0]
3.18e-04 6.27e+04 output
                  encoder.block.2.layer.1.dropout Dropout
3.18e-04 6.27e+04 input[0]
0.00e+00      inf output`,highlighted:`<span class="hljs-attribute">Detected</span> inf/nan during batch_number=<span class="hljs-number">0</span>
<span class="hljs-attribute">Last</span> <span class="hljs-number">21</span> forward frames:
<span class="hljs-attribute">abs</span> min  abs max  metadata<span class="hljs-meta">
[...]</span>
                  <span class="hljs-attribute">encoder</span>.block.<span class="hljs-number">2</span>.layer.<span class="hljs-number">1</span>.DenseReluDense.wi_0 Linear
<span class="hljs-attribute">2</span>.<span class="hljs-number">17</span>e-<span class="hljs-number">07</span> <span class="hljs-number">4</span>.<span class="hljs-number">50</span>e+<span class="hljs-number">00</span> weight
<span class="hljs-attribute">1</span>.<span class="hljs-number">79</span>e-<span class="hljs-number">06</span> <span class="hljs-number">4</span>.<span class="hljs-number">65</span>e+<span class="hljs-number">00</span> input[<span class="hljs-number">0</span>]
<span class="hljs-attribute">2</span>.<span class="hljs-number">68</span>e-<span class="hljs-number">06</span> <span class="hljs-number">3</span>.<span class="hljs-number">70</span>e+<span class="hljs-number">01</span> output
                  <span class="hljs-attribute">encoder</span>.block.<span class="hljs-number">2</span>.layer.<span class="hljs-number">1</span>.DenseReluDense.wi_1 Linear
<span class="hljs-attribute">8</span>.<span class="hljs-number">08</span>e-<span class="hljs-number">07</span> <span class="hljs-number">2</span>.<span class="hljs-number">66</span>e+<span class="hljs-number">01</span> weight
<span class="hljs-attribute">1</span>.<span class="hljs-number">79</span>e-<span class="hljs-number">06</span> <span class="hljs-number">4</span>.<span class="hljs-number">65</span>e+<span class="hljs-number">00</span> input[<span class="hljs-number">0</span>]
<span class="hljs-attribute">1</span>.<span class="hljs-number">27</span>e-<span class="hljs-number">04</span> <span class="hljs-number">2</span>.<span class="hljs-number">37</span>e+<span class="hljs-number">02</span> output
                  <span class="hljs-attribute">encoder</span>.block.<span class="hljs-number">2</span>.layer.<span class="hljs-number">1</span>.DenseReluDense.wo Linear
<span class="hljs-attribute">1</span>.<span class="hljs-number">01</span>e-<span class="hljs-number">06</span> <span class="hljs-number">6</span>.<span class="hljs-number">44</span>e+<span class="hljs-number">00</span> weight
<span class="hljs-attribute">0</span>.<span class="hljs-number">00</span>e+<span class="hljs-number">00</span> <span class="hljs-number">9</span>.<span class="hljs-number">74</span>e+<span class="hljs-number">03</span> input[<span class="hljs-number">0</span>]
<span class="hljs-attribute">3</span>.<span class="hljs-number">18</span>e-<span class="hljs-number">04</span> <span class="hljs-number">6</span>.<span class="hljs-number">27</span>e+<span class="hljs-number">04</span> output
                  <span class="hljs-attribute">encoder</span>.block.<span class="hljs-number">2</span>.layer.<span class="hljs-number">1</span>.DenseReluDense T5DenseGatedGeluDense
<span class="hljs-attribute">1</span>.<span class="hljs-number">79</span>e-<span class="hljs-number">06</span> <span class="hljs-number">4</span>.<span class="hljs-number">65</span>e+<span class="hljs-number">00</span> input[<span class="hljs-number">0</span>]
<span class="hljs-attribute">3</span>.<span class="hljs-number">18</span>e-<span class="hljs-number">04</span> <span class="hljs-number">6</span>.<span class="hljs-number">27</span>e+<span class="hljs-number">04</span> output
                  <span class="hljs-attribute">encoder</span>.block.<span class="hljs-number">2</span>.layer.<span class="hljs-number">1</span>.dropout Dropout
<span class="hljs-attribute">3</span>.<span class="hljs-number">18</span>e-<span class="hljs-number">04</span> <span class="hljs-number">6</span>.<span class="hljs-number">27</span>e+<span class="hljs-number">04</span> input[<span class="hljs-number">0</span>]
<span class="hljs-attribute">0</span>.<span class="hljs-number">00</span>e+<span class="hljs-number">00</span>      inf output`}}),{c(){g=r("p"),T=o("mixed precision :"),w=i(),E(f.$$.fragment)},l(c){g=s(c,"P",{});var _=n(g);T=l(_,"mixed precision :"),_.forEach(t),w=p(c),D(f.$$.fragment,c)},m(c,_){b(c,g,_),e(g,T),b(c,w,_),j(f,c,_),$=!0},p:Va,i(c){$||(k(f.$$.fragment,c),$=!0)},o(c){P(f.$$.fragment,c),$=!1},d(c){c&&t(g),c&&t(w),x(f,c)}}}function Qi(L){let g,T,w,f,$;return f=new Ma({props:{code:"debug_overflow = DebugUnderflowOverflow(model, max_frames_to_save=100)",highlighted:'debug_overflow = DebugUnderflowOverflow(model, max_frames_to_save=<span class="hljs-number">100</span>)'}}),{c(){g=r("p"),T=o("By default the last 21 frames are printed. You can change the default to adjust for your needs. For example :"),w=i(),E(f.$$.fragment)},l(c){g=s(c,"P",{});var _=n(g);T=l(_,"By default the last 21 frames are printed. You can change the default to adjust for your needs. For example :"),_.forEach(t),w=p(c),D(f.$$.fragment,c)},m(c,_){b(c,g,_),e(g,T),b(c,w,_),j(f,c,_),$=!0},p:Va,i(c){$||(k(f.$$.fragment,c),$=!0)},o(c){P(f.$$.fragment,c),$=!1},d(c){c&&t(g),c&&t(w),x(f,c)}}}function Xi(L){let g,T,w,f,$;return f=new Ma({props:{code:"debug_overflow = DebugUnderflowOverflow(model, trace_batch_nums=[1, 3])",highlighted:'debug_overflow = DebugUnderflowOverflow(model, trace_batch_nums=[<span class="hljs-number">1</span>, <span class="hljs-number">3</span>])'}}),{c(){g=r("p"),T=o("given batch, and only do that for batches 1 and 3. Then you instantiate this class as :"),w=i(),E(f.$$.fragment)},l(c){g=s(c,"P",{});var _=n(g);T=l(_,"given batch, and only do that for batches 1 and 3. Then you instantiate this class as :"),_.forEach(t),w=p(c),D(f.$$.fragment,c)},m(c,_){b(c,g,_),e(g,T),b(c,w,_),j(f,c,_),$=!0},p:Va,i(c){$||(k(f.$$.fragment,c),$=!0)},o(c){P(f.$$.fragment,c),$=!1},d(c){c&&t(g),c&&t(w),x(f,c)}}}function Zi(L){let g,T,w,f,$;return f=new Ma({props:{code:"debug_overflow = DebugUnderflowOverflow(model, trace_batch_nums=[1, 3], abort_after_batch_num=3)",highlighted:'debug_overflow = DebugUnderflowOverflow(model, trace_batch_nums=[<span class="hljs-number">1</span>, <span class="hljs-number">3</span>], abort_after_batch_num=<span class="hljs-number">3</span>)'}}),{c(){g=r("p"),T=o("You can also specify the batch number after which to stop the training, with :"),w=i(),E(f.$$.fragment)},l(c){g=s(c,"P",{});var _=n(g);T=l(_,"You can also specify the batch number after which to stop the training, with :"),_.forEach(t),w=p(c),D(f.$$.fragment,c)},m(c,_){b(c,g,_),e(g,T),b(c,w,_),j(f,c,_),$=!0},p:Va,i(c){$||(k(f.$$.fragment,c),$=!0)},o(c){P(f.$$.fragment,c),$=!1},d(c){c&&t(g),c&&t(w),x(f,c)}}}function ep(L){let g,T,w,f,$,c,_,bt,Lr,Ra,ee,Ur,Xe,Ir,zr,qa,Ze,Sr,Ba,N,te,_t,ge,Hr,vt,Nr,Fa,G,be,Gr,wt,Vr,Ka,V,_e,Mr,$t,Rr,Ya,I,ve,qr,yt,Br,Fr,we,et,$e,Kr,Yr,Wr,tt,ye,Jr,Qr,Wa,M,Ee,Xr,C,Zr,Et,es,ts,Dt,as,rs,jt,ss,ns,kt,os,ls,Ja,R,De,is,Pt,ps,Qa,q,ae,xt,je,cs,Tt,ds,Xa,B,ke,hs,Ot,fs,Za,F,re,At,Pe,us,Ct,ms,er,v,xe,gs,Lt,bs,_s,Ut,vs,ws,It,zt,$s,ys,St,Es,Ds,K,at,js,Ht,ks,Ps,rt,xs,Nt,Ts,Os,st,As,Gt,Cs,Ls,Vt,Us,Is,Y,nt,zs,Mt,Ss,Hs,ot,Ns,Rt,Gs,Vs,lt,Ms,qt,Rs,qs,Bt,Bs,Fs,Ft,Kt,Ks,Ys,Yt,Ws,Js,Wt,Jt,Qs,Xs,Qt,Zs,en,se,Te,tn,Oe,an,Xt,rn,sn,nn,ne,Ae,on,Zt,ln,tr,W,oe,ea,Ce,pn,ta,cn,ar,O,Le,dn,Ue,hn,aa,fn,un,mn,Ie,gn,ra,bn,_n,vn,S,ze,wn,sa,$n,yn,Se,En,na,Dn,jn,kn,le,He,Pn,Ne,xn,oa,Tn,On,An,ie,Ge,Cn,Ve,Ln,la,Un,In,rr,J,pe,ia,Me,zn,pa,Sn,sr,d,Re,Hn,Q,Nn,ca,Gn,Vn,da,Mn,Rn,qn,ha,Bn,Fn,qe,fa,Kn,Yn,ua,Wn,Jn,ma,Qn,Xn,ce,Zn,z,eo,ga,to,ao,ba,ro,so,_a,no,oo,lo,Be,Fe,io,va,po,co,ho,wa,fo,uo,Ke,mo,$a,go,bo,_o,de,vo,X,wo,ya,$o,yo,Ea,Eo,Do,jo,Da,ko,Po,Ye,xo,ja,To,Oo,Ao,he,Co,ka,Lo,Uo,Pa,Io,zo,xa,So,Ho,We,No,Ta,Go,Vo,Mo,fe,Ro,Oa,qo,Bo,Aa,Fo,Ko,Ca,Yo,Wo,ue,Jo,La,Qo,Xo,it,Ua,Zo,el,tl,Z,al,Ia,rl,sl,za,nl,ol,nr;return c=new gt({}),ge=new gt({}),be=new A({props:{name:"class transformers.EvalPrediction",anchor:"transformers.EvalPrediction",parameters:[{name:"predictions",val:": typing.Union[numpy.ndarray, typing.Tuple[numpy.ndarray]]"},{name:"label_ids",val:": typing.Union[numpy.ndarray, typing.Tuple[numpy.ndarray]]"},{name:"inputs",val:": typing.Union[numpy.ndarray, typing.Tuple[numpy.ndarray], NoneType] = None"}],parametersDescription:[{anchor:"transformers.EvalPrediction.predictions",description:"<strong>predictions</strong> (<code>np.ndarray</code>) &#x2014; Predictions of the model.",name:"predictions"},{anchor:"transformers.EvalPrediction.label_ids",description:"<strong>label_ids</strong> (<code>np.ndarray</code>) &#x2014; Targets to be matched.",name:"label_ids"},{anchor:"transformers.EvalPrediction.inputs",description:"<strong>inputs</strong> (<code>np.ndarray</code>, <em>optional</em>) &#x2014;",name:"inputs"}],source:"https://github.com/huggingface/transformers/blob/vr_17548/src/transformers/trainer_utils.py#L100"}}),_e=new A({props:{name:"class transformers.IntervalStrategy",anchor:"transformers.IntervalStrategy",parameters:[{name:"value",val:""},{name:"names",val:" = None"},{name:"module",val:" = None"},{name:"qualname",val:" = None"},{name:"type",val:" = None"},{name:"start",val:" = 1"}],source:"https://github.com/huggingface/transformers/blob/vr_17548/src/transformers/trainer_utils.py#L174"}}),ve=new A({props:{name:"transformers.enable_full_determinism",anchor:"transformers.enable_full_determinism",parameters:[{name:"seed",val:": int"}],source:"https://github.com/huggingface/transformers/blob/vr_17548/src/transformers/trainer_utils.py#L58"}}),Ee=new A({props:{name:"transformers.set_seed",anchor:"transformers.set_seed",parameters:[{name:"seed",val:": int"}],parametersDescription:[{anchor:"transformers.set_seed.seed",description:"<strong>seed</strong> (<code>int</code>) &#x2014; The seed to set.",name:"seed"}],source:"https://github.com/huggingface/transformers/blob/vr_17548/src/transformers/trainer_utils.py#L83"}}),De=new A({props:{name:"transformers.torch_distributed_zero_first",anchor:"transformers.torch_distributed_zero_first",parameters:[{name:"local_rank",val:": int"}],parametersDescription:[{anchor:"transformers.torch_distributed_zero_first.local_rank",description:"<strong>local_rank</strong> (<code>int</code>) &#x2014; The rank of the local process.",name:"local_rank"}],source:"https://github.com/huggingface/transformers/blob/vr_17548/src/transformers/trainer_pt_utils.py#L219"}}),je=new gt({}),ke=new A({props:{name:"class transformers.trainer_callback.CallbackHandler",anchor:"transformers.trainer_callback.CallbackHandler",parameters:[{name:"callbacks",val:""},{name:"model",val:""},{name:"tokenizer",val:""},{name:"optimizer",val:""},{name:"lr_scheduler",val:""}],source:"https://github.com/huggingface/transformers/blob/vr_17548/src/transformers/trainer_callback.py#L284"}}),Pe=new gt({}),xe=new A({props:{name:"class transformers.trainer_pt_utils.DistributedTensorGatherer",anchor:"transformers.trainer_pt_utils.DistributedTensorGatherer",parameters:[{name:"world_size",val:""},{name:"num_samples",val:""},{name:"make_multiple_of",val:" = None"},{name:"padding_index",val:" = -100"}],parametersDescription:[{anchor:"transformers.trainer_pt_utils.DistributedTensorGatherer.world_size",description:`<strong>world_size</strong> (<code>int</code>) &#x2014;
The number of processes used in the distributed training.`,name:"world_size"},{anchor:"transformers.trainer_pt_utils.DistributedTensorGatherer.num_samples",description:`<strong>num_samples</strong> (<code>int</code>) &#x2014;
The number of samples in our dataset.`,name:"num_samples"},{anchor:"transformers.trainer_pt_utils.DistributedTensorGatherer.make_multiple_of",description:`<strong>make_multiple_of</strong> (<code>int</code>, <em>optional</em>) &#x2014;
If passed, the class assumes the datasets passed to each process are made to be a multiple of this argument
(by adding samples).`,name:"make_multiple_of"},{anchor:"transformers.trainer_pt_utils.DistributedTensorGatherer.padding_index",description:`<strong>padding_index</strong> (<code>int</code>, <em>optional</em>, defaults to -100) &#x2014;
The padding index to use if the arrays don&#x2019;t all have the same sequence length.`,name:"padding_index"}],source:"https://github.com/huggingface/transformers/blob/vr_17548/src/transformers/trainer_pt_utils.py#L345"}}),Te=new A({props:{name:"add_arrays",anchor:"transformers.trainer_pt_utils.DistributedTensorGatherer.add_arrays",parameters:[{name:"arrays",val:""}],source:"https://github.com/huggingface/transformers/blob/vr_17548/src/transformers/trainer_pt_utils.py#L406"}}),Ae=new A({props:{name:"finalize",anchor:"transformers.trainer_pt_utils.DistributedTensorGatherer.finalize",parameters:[],source:"https://github.com/huggingface/transformers/blob/vr_17548/src/transformers/trainer_pt_utils.py#L442"}}),Ce=new gt({}),Le=new A({props:{name:"class transformers.HfArgumentParser",anchor:"transformers.HfArgumentParser",parameters:[{name:"dataclass_types",val:": typing.Union[DataClassType, typing.Iterable[DataClassType]]"},{name:"**kwargs",val:""}],source:"https://github.com/huggingface/transformers/blob/vr_17548/src/transformers/hf_argparser.py#L44"}}),ze=new A({props:{name:"parse_args_into_dataclasses",anchor:"transformers.HfArgumentParser.parse_args_into_dataclasses",parameters:[{name:"args",val:" = None"},{name:"return_remaining_strings",val:" = False"},{name:"look_for_args_file",val:" = True"},{name:"args_filename",val:" = None"}],source:"https://github.com/huggingface/transformers/blob/vr_17548/src/transformers/hf_argparser.py#L168",returnDescription:`
<ul>
<li>the dataclass instances in the same order as they were passed to the initializer.abspath</li>
<li>if applicable, an additional namespace for more (non-dataclass backed) arguments added to the parser
after initialization.</li>
<li>The potential list of remaining argument strings. (same as argparse.ArgumentParser.parse_known_args)</li>
</ul>
`,returnType:`
<p>Tuple consisting of</p>
`}}),He=new A({props:{name:"parse_dict",anchor:"transformers.HfArgumentParser.parse_dict",parameters:[{name:"args",val:": dict"}],source:"https://github.com/huggingface/transformers/blob/vr_17548/src/transformers/hf_argparser.py#L241"}}),Ge=new A({props:{name:"parse_json_file",anchor:"transformers.HfArgumentParser.parse_json_file",parameters:[{name:"json_file",val:": str"}],source:"https://github.com/huggingface/transformers/blob/vr_17548/src/transformers/hf_argparser.py#L227"}}),Me=new gt({}),Re=new A({props:{name:"class transformers.debug_utils.DebugUnderflowOverflow",anchor:"transformers.debug_utils.DebugUnderflowOverflow",parameters:[{name:"model",val:""},{name:"max_frames_to_save",val:" = 21"},{name:"trace_batch_nums",val:" = []"},{name:"abort_after_batch_num",val:" = None"}],parametersDescription:[{anchor:"transformers.debug_utils.DebugUnderflowOverflow.model",description:`<strong>model</strong> (<code>nn.Module</code>) &#x2014;
The model to debug.`,name:"model"},{anchor:"transformers.debug_utils.DebugUnderflowOverflow.max_frames_to_save",description:`<strong>max_frames_to_save</strong> (<code>int</code>, <em>optional</em>, defaults to 21) &#x2014;
How many frames back to record`,name:"max_frames_to_save"},{anchor:"transformers.debug_utils.DebugUnderflowOverflow.trace_batch_nums(List[int],",description:`<strong>trace_batch_nums(<code>List[int]</code>,</strong> <em>optional</em>, defaults to <code>[]</code>) &#x2014;
Which batch numbers to trace (turns detection off)`,name:"trace_batch_nums(List[int],"},{anchor:"transformers.debug_utils.DebugUnderflowOverflow.abort_after_batch_num",description:"<strong>abort_after_batch_num</strong>  (`int&#x201C;, <em>optional</em>) &#x2014;\nWhether to abort after a certain batch number has finished",name:"abort_after_batch_num"}],source:"https://github.com/huggingface/transformers/blob/vr_17548/src/transformers/debug_utils.py#L27"}}),ce=new Ga({props:{anchor:"transformers.debug_utils.DebugUnderflowOverflow.example",$$slots:{default:[Wi]},$$scope:{ctx:L}}}),de=new Ga({props:{anchor:"transformers.debug_utils.DebugUnderflowOverflow.example-2",$$slots:{default:[Ji]},$$scope:{ctx:L}}}),he=new Ga({props:{anchor:"transformers.debug_utils.DebugUnderflowOverflow.example-3",$$slots:{default:[Qi]},$$scope:{ctx:L}}}),fe=new Ga({props:{anchor:"transformers.debug_utils.DebugUnderflowOverflow.example-4",$$slots:{default:[Xi]},$$scope:{ctx:L}}}),ue=new Ga({props:{anchor:"transformers.debug_utils.DebugUnderflowOverflow.example-5",$$slots:{default:[Zi]},$$scope:{ctx:L}}}),{c(){g=r("meta"),T=i(),w=r("h1"),f=r("a"),$=r("span"),E(c.$$.fragment),_=i(),bt=r("span"),Lr=o("Utilities for Trainer"),Ra=i(),ee=r("p"),Ur=o("This page lists all the utility functions used by "),Xe=r("a"),Ir=o("Trainer"),zr=o("."),qa=i(),Ze=r("p"),Sr=o("Most of those are only useful if you are studying the code of the Trainer in the library."),Ba=i(),N=r("h2"),te=r("a"),_t=r("span"),E(ge.$$.fragment),Hr=i(),vt=r("span"),Nr=o("Utilities"),Fa=i(),G=r("div"),E(be.$$.fragment),Gr=i(),wt=r("p"),Vr=o("Evaluation output (always contains labels), to be used to compute metrics."),Ka=i(),V=r("div"),E(_e.$$.fragment),Mr=i(),$t=r("p"),Rr=o("An enumeration."),Ya=i(),I=r("div"),E(ve.$$.fragment),qr=i(),yt=r("p"),Br=o("Helper function for reproducible behavior during distributed training. See"),Fr=i(),we=r("ul"),et=r("li"),$e=r("a"),Kr=o("https://pytorch.org/docs/stable/notes/randomness.html"),Yr=o(" for pytorch"),Wr=i(),tt=r("li"),ye=r("a"),Jr=o("https://www.tensorflow.org/api_docs/python/tf/config/experimental/enable_op_determinism"),Qr=o(" for tensorflow"),Wa=i(),M=r("div"),E(Ee.$$.fragment),Xr=i(),C=r("p"),Zr=o("Helper function for reproducible behavior to set the seed in "),Et=r("code"),es=o("random"),ts=o(", "),Dt=r("code"),as=o("numpy"),rs=o(", "),jt=r("code"),ss=o("torch"),ns=o(" and/or "),kt=r("code"),os=o("tf"),ls=o(" (if installed)."),Ja=i(),R=r("div"),E(De.$$.fragment),is=i(),Pt=r("p"),ps=o("Decorator to make all processes in distributed training wait for each local_master to do something."),Qa=i(),q=r("h2"),ae=r("a"),xt=r("span"),E(je.$$.fragment),cs=i(),Tt=r("span"),ds=o("Callbacks internals"),Xa=i(),B=r("div"),E(ke.$$.fragment),hs=i(),Ot=r("p"),fs=o("Internal class that just calls the list of callbacks in order."),Za=i(),F=r("h2"),re=r("a"),At=r("span"),E(Pe.$$.fragment),us=i(),Ct=r("span"),ms=o("Distributed Evaluation"),er=i(),v=r("div"),E(xe.$$.fragment),gs=i(),Lt=r("p"),bs=o("A class responsible for properly gathering tensors (or nested list/tuple of tensors) on the CPU by chunks."),_s=i(),Ut=r("p"),vs=o(`If our dataset has 16 samples with a batch size of 2 on 3 processes and we gather then transfer on CPU at every
step, our sampler will generate the following indices:`),ws=i(),It=r("p"),zt=r("code"),$s=o("[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 0, 1]"),ys=i(),St=r("p"),Es=o(`to get something of size a multiple of 3 (so that each process gets the same dataset length). Then process 0, 1 and
2 will be responsible of making predictions for the following samples:`),Ds=i(),K=r("ul"),at=r("li"),js=o("P0: "),Ht=r("code"),ks=o("[0, 1, 2, 3, 4, 5]"),Ps=i(),rt=r("li"),xs=o("P1: "),Nt=r("code"),Ts=o("[6, 7, 8, 9, 10, 11]"),Os=i(),st=r("li"),As=o("P2: "),Gt=r("code"),Cs=o("[12, 13, 14, 15, 0, 1]"),Ls=i(),Vt=r("p"),Us=o("The first batch treated on each process will be"),Is=i(),Y=r("ul"),nt=r("li"),zs=o("P0: "),Mt=r("code"),Ss=o("[0, 1]"),Hs=i(),ot=r("li"),Ns=o("P1: "),Rt=r("code"),Gs=o("[6, 7]"),Vs=i(),lt=r("li"),Ms=o("P2: "),qt=r("code"),Rs=o("[12, 13]"),qs=i(),Bt=r("p"),Bs=o(`So if we gather at the end of the first batch, we will get a tensor (nested list/tuple of tensor) corresponding to
the following indices:`),Fs=i(),Ft=r("p"),Kt=r("code"),Ks=o("[0, 1, 6, 7, 12, 13]"),Ys=i(),Yt=r("p"),Ws=o(`If we directly concatenate our results without taking any precautions, the user will then get the predictions for
the indices in this order at the end of the prediction loop:`),Js=i(),Wt=r("p"),Jt=r("code"),Qs=o("[0, 1, 6, 7, 12, 13, 2, 3, 8, 9, 14, 15, 4, 5, 10, 11, 0, 1]"),Xs=i(),Qt=r("p"),Zs=o("For some reason, that\u2019s not going to roll their boat. This class is there to solve that problem."),en=i(),se=r("div"),E(Te.$$.fragment),tn=i(),Oe=r("p"),an=o("Add "),Xt=r("code"),rn=o("arrays"),sn=o(` to the internal storage, Will initialize the storage to the full size at the first arrays passed
so that if we\u2019re bound to get an OOM, it happens at the beginning.`),nn=i(),ne=r("div"),E(Ae.$$.fragment),on=i(),Zt=r("p"),ln=o(`Return the properly gathered arrays and truncate to the number of samples (since the sampler added some extras
to get each process a dataset of the same length).`),tr=i(),W=r("h2"),oe=r("a"),ea=r("span"),E(Ce.$$.fragment),pn=i(),ta=r("span"),cn=o("Distributed Evaluation"),ar=i(),O=r("div"),E(Le.$$.fragment),dn=i(),Ue=r("p"),hn=o("This subclass of "),aa=r("code"),fn=o("argparse.ArgumentParser"),un=o(" uses type hints on dataclasses to generate arguments."),mn=i(),Ie=r("p"),gn=o(`The class is designed to play well with the native argparse. In particular, you can add more (non-dataclass backed)
arguments to the parser after initialization and you\u2019ll get the output back after parsing as an additional
namespace. Optional: To create sub argument groups use the `),ra=r("code"),bn=o("_argument_group_name"),_n=o(" attribute in the dataclass."),vn=i(),S=r("div"),E(ze.$$.fragment),wn=i(),sa=r("p"),$n=o("Parse command-line args into instances of the specified dataclass types."),yn=i(),Se=r("p"),En=o("This relies on argparse\u2019s "),na=r("code"),Dn=o("ArgumentParser.parse_known_args"),jn=o(`. See the doc at:
docs.python.org/3.7/library/argparse.html#argparse.ArgumentParser.parse_args`),kn=i(),le=r("div"),E(He.$$.fragment),Pn=i(),Ne=r("p"),xn=o("Alternative helper method that does not use "),oa=r("code"),Tn=o("argparse"),On=o(` at all, instead uses a dict and populating the dataclass
types.`),An=i(),ie=r("div"),E(Ge.$$.fragment),Cn=i(),Ve=r("p"),Ln=o("Alternative helper method that does not use "),la=r("code"),Un=o("argparse"),In=o(` at all, instead loading a json file and populating the
dataclass types.`),rr=i(),J=r("h2"),pe=r("a"),ia=r("span"),E(Me.$$.fragment),zn=i(),pa=r("span"),Sn=o("Debug Utilities"),sr=i(),d=r("div"),E(Re.$$.fragment),Hn=i(),Q=r("p"),Nn=o(`This debug class helps detect and understand where the model starts getting very large or very small, and more
importantly `),ca=r("code"),Gn=o("nan"),Vn=o(" or "),da=r("code"),Mn=o("inf"),Rn=o(" weight and activation elements."),qn=i(),ha=r("p"),Bn=o("There are 2 working modes:"),Fn=i(),qe=r("ol"),fa=r("li"),Kn=o("Underflow/overflow detection (default)"),Yn=i(),ua=r("li"),Wn=o("Specific batch absolute min/max tracing without detection"),Jn=i(),ma=r("p"),Qn=o("Mode 1: Underflow/overflow detection"),Xn=i(),E(ce.$$.fragment),Zn=i(),z=r("p"),eo=o("then run the training as normal and if "),ga=r("code"),to=o("nan"),ao=o(" or "),ba=r("code"),ro=o("inf"),so=o(` gets detected in at least one of the weight, input or output
elements this module will throw an exception and will print `),_a=r("code"),no=o("max_frames_to_save"),oo=o(` frames that lead to this event,
each frame reporting`),lo=i(),Be=r("ol"),Fe=r("li"),io=o("the fully qualified module name plus the class name whose "),va=r("code"),po=o("forward"),co=o(" was run"),ho=i(),wa=r("li"),fo=o("the absolute min and max value of all elements for each module weights, and the inputs and output"),uo=i(),Ke=r("p"),mo=o("For example, here is the header and the last few frames in detection report for "),$a=r("code"),go=o("google/mt5-small"),bo=o(" run in fp16"),_o=i(),E(de.$$.fragment),vo=i(),X=r("p"),wo=o("You can see here, that "),ya=r("code"),$o=o("T5DenseGatedGeluDense.forward"),yo=o(` resulted in output activations, whose absolute max value was
around 62.7K, which is very close to fp16\u2019s top limit of 64K. In the next frame we have `),Ea=r("code"),Eo=o("Dropout"),Do=o(` which
renormalizes the weights, after it zeroed some of the elements, which pushes the absolute max value to more than
64K, and we get an overlow.`),jo=i(),Da=r("p"),ko=o(`As you can see it\u2019s the previous frames that we need to look into when the numbers start going into very large for
fp16 numbers.`),Po=i(),Ye=r("p"),xo=o("The tracking is done in a forward hook, which gets invoked immediately after "),ja=r("code"),To=o("forward"),Oo=o(" has completed."),Ao=i(),E(he.$$.fragment),Co=i(),ka=r("p"),Lo=o(`To validate that you have set up this debugging feature correctly, and you intend to use it in a training that
may take hours to complete, first run it with normal tracing enabled for one of a few batches as explained in
the next section.`),Uo=i(),Pa=r("p"),Io=o("Mode 2. Specific batch absolute min/max tracing without detection"),zo=i(),xa=r("p"),So=o("The second work mode is per-batch tracing with the underflow/overflow detection feature turned off."),Ho=i(),We=r("p"),No=o("Let\u2019s say you want to watch the absolute min and max values for all the ingredients of each "),Ta=r("code"),Go=o("forward"),Vo=o(" call of a"),Mo=i(),E(fe.$$.fragment),Ro=i(),Oa=r("p"),qo=o("And now full batches 1 and 3 will be traced using the same format as explained above. Batches are 0-indexed."),Bo=i(),Aa=r("p"),Fo=o(`This is helpful if you know that the program starts misbehaving after a certain batch number, so you can
fast-forward right to that area.`),Ko=i(),Ca=r("p"),Yo=o("Early stopping:"),Wo=i(),E(ue.$$.fragment),Jo=i(),La=r("p"),Qo=o("This feature is mainly useful in the tracing mode, but you can use it for any mode."),Xo=i(),it=r("p"),Ua=r("strong"),Zo=o("Performance"),el=o(":"),tl=i(),Z=r("p"),al=o("As this module measures absolute "),Ia=r("code"),rl=o("min"),sl=o("/`"),za=r("code"),nl=o("max"),ol=o(` of each weight of the model on every forward it\u2019ll slow the training
down. Therefore remember to turn it off once the debugging needs have been met.`),this.h()},l(a){const u=Ki('[data-svelte="svelte-1phssyn"]',document.head);g=s(u,"META",{name:!0,content:!0}),u.forEach(t),T=p(a),w=s(a,"H1",{class:!0});var Je=n(w);f=s(Je,"A",{id:!0,class:!0,href:!0});var Sa=n(f);$=s(Sa,"SPAN",{});var Ha=n($);D(c.$$.fragment,Ha),Ha.forEach(t),Sa.forEach(t),_=p(Je),bt=s(Je,"SPAN",{});var Na=n(bt);Lr=l(Na,"Utilities for Trainer"),Na.forEach(t),Je.forEach(t),Ra=p(a),ee=s(a,"P",{});var Qe=n(ee);Ur=l(Qe,"This page lists all the utility functions used by "),Xe=s(Qe,"A",{href:!0});var gl=n(Xe);Ir=l(gl,"Trainer"),gl.forEach(t),zr=l(Qe,"."),Qe.forEach(t),qa=p(a),Ze=s(a,"P",{});var bl=n(Ze);Sr=l(bl,"Most of those are only useful if you are studying the code of the Trainer in the library."),bl.forEach(t),Ba=p(a),N=s(a,"H2",{class:!0});var or=n(N);te=s(or,"A",{id:!0,class:!0,href:!0});var _l=n(te);_t=s(_l,"SPAN",{});var vl=n(_t);D(ge.$$.fragment,vl),vl.forEach(t),_l.forEach(t),Hr=p(or),vt=s(or,"SPAN",{});var wl=n(vt);Nr=l(wl,"Utilities"),wl.forEach(t),or.forEach(t),Fa=p(a),G=s(a,"DIV",{class:!0});var lr=n(G);D(be.$$.fragment,lr),Gr=p(lr),wt=s(lr,"P",{});var $l=n(wt);Vr=l($l,"Evaluation output (always contains labels), to be used to compute metrics."),$l.forEach(t),lr.forEach(t),Ka=p(a),V=s(a,"DIV",{class:!0});var ir=n(V);D(_e.$$.fragment,ir),Mr=p(ir),$t=s(ir,"P",{});var yl=n($t);Rr=l(yl,"An enumeration."),yl.forEach(t),ir.forEach(t),Ya=p(a),I=s(a,"DIV",{class:!0});var pt=n(I);D(ve.$$.fragment,pt),qr=p(pt),yt=s(pt,"P",{});var El=n(yt);Br=l(El,"Helper function for reproducible behavior during distributed training. See"),El.forEach(t),Fr=p(pt),we=s(pt,"UL",{});var pr=n(we);et=s(pr,"LI",{});var ll=n(et);$e=s(ll,"A",{href:!0,rel:!0});var Dl=n($e);Kr=l(Dl,"https://pytorch.org/docs/stable/notes/randomness.html"),Dl.forEach(t),Yr=l(ll," for pytorch"),ll.forEach(t),Wr=p(pr),tt=s(pr,"LI",{});var il=n(tt);ye=s(il,"A",{href:!0,rel:!0});var jl=n(ye);Jr=l(jl,"https://www.tensorflow.org/api_docs/python/tf/config/experimental/enable_op_determinism"),jl.forEach(t),Qr=l(il," for tensorflow"),il.forEach(t),pr.forEach(t),pt.forEach(t),Wa=p(a),M=s(a,"DIV",{class:!0});var cr=n(M);D(Ee.$$.fragment,cr),Xr=p(cr),C=s(cr,"P",{});var H=n(C);Zr=l(H,"Helper function for reproducible behavior to set the seed in "),Et=s(H,"CODE",{});var kl=n(Et);es=l(kl,"random"),kl.forEach(t),ts=l(H,", "),Dt=s(H,"CODE",{});var Pl=n(Dt);as=l(Pl,"numpy"),Pl.forEach(t),rs=l(H,", "),jt=s(H,"CODE",{});var xl=n(jt);ss=l(xl,"torch"),xl.forEach(t),ns=l(H," and/or "),kt=s(H,"CODE",{});var Tl=n(kt);os=l(Tl,"tf"),Tl.forEach(t),ls=l(H," (if installed)."),H.forEach(t),cr.forEach(t),Ja=p(a),R=s(a,"DIV",{class:!0});var dr=n(R);D(De.$$.fragment,dr),is=p(dr),Pt=s(dr,"P",{});var Ol=n(Pt);ps=l(Ol,"Decorator to make all processes in distributed training wait for each local_master to do something."),Ol.forEach(t),dr.forEach(t),Qa=p(a),q=s(a,"H2",{class:!0});var hr=n(q);ae=s(hr,"A",{id:!0,class:!0,href:!0});var Al=n(ae);xt=s(Al,"SPAN",{});var Cl=n(xt);D(je.$$.fragment,Cl),Cl.forEach(t),Al.forEach(t),cs=p(hr),Tt=s(hr,"SPAN",{});var Ll=n(Tt);ds=l(Ll,"Callbacks internals"),Ll.forEach(t),hr.forEach(t),Xa=p(a),B=s(a,"DIV",{class:!0});var fr=n(B);D(ke.$$.fragment,fr),hs=p(fr),Ot=s(fr,"P",{});var Ul=n(Ot);fs=l(Ul,"Internal class that just calls the list of callbacks in order."),Ul.forEach(t),fr.forEach(t),Za=p(a),F=s(a,"H2",{class:!0});var ur=n(F);re=s(ur,"A",{id:!0,class:!0,href:!0});var Il=n(re);At=s(Il,"SPAN",{});var zl=n(At);D(Pe.$$.fragment,zl),zl.forEach(t),Il.forEach(t),us=p(ur),Ct=s(ur,"SPAN",{});var Sl=n(Ct);ms=l(Sl,"Distributed Evaluation"),Sl.forEach(t),ur.forEach(t),er=p(a),v=s(a,"DIV",{class:!0});var y=n(v);D(xe.$$.fragment,y),gs=p(y),Lt=s(y,"P",{});var Hl=n(Lt);bs=l(Hl,"A class responsible for properly gathering tensors (or nested list/tuple of tensors) on the CPU by chunks."),Hl.forEach(t),_s=p(y),Ut=s(y,"P",{});var Nl=n(Ut);vs=l(Nl,`If our dataset has 16 samples with a batch size of 2 on 3 processes and we gather then transfer on CPU at every
step, our sampler will generate the following indices:`),Nl.forEach(t),ws=p(y),It=s(y,"P",{});var Gl=n(It);zt=s(Gl,"CODE",{});var Vl=n(zt);$s=l(Vl,"[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 0, 1]"),Vl.forEach(t),Gl.forEach(t),ys=p(y),St=s(y,"P",{});var Ml=n(St);Es=l(Ml,`to get something of size a multiple of 3 (so that each process gets the same dataset length). Then process 0, 1 and
2 will be responsible of making predictions for the following samples:`),Ml.forEach(t),Ds=p(y),K=s(y,"UL",{});var ct=n(K);at=s(ct,"LI",{});var pl=n(at);js=l(pl,"P0: "),Ht=s(pl,"CODE",{});var Rl=n(Ht);ks=l(Rl,"[0, 1, 2, 3, 4, 5]"),Rl.forEach(t),pl.forEach(t),Ps=p(ct),rt=s(ct,"LI",{});var cl=n(rt);xs=l(cl,"P1: "),Nt=s(cl,"CODE",{});var ql=n(Nt);Ts=l(ql,"[6, 7, 8, 9, 10, 11]"),ql.forEach(t),cl.forEach(t),Os=p(ct),st=s(ct,"LI",{});var dl=n(st);As=l(dl,"P2: "),Gt=s(dl,"CODE",{});var Bl=n(Gt);Cs=l(Bl,"[12, 13, 14, 15, 0, 1]"),Bl.forEach(t),dl.forEach(t),ct.forEach(t),Ls=p(y),Vt=s(y,"P",{});var Fl=n(Vt);Us=l(Fl,"The first batch treated on each process will be"),Fl.forEach(t),Is=p(y),Y=s(y,"UL",{});var dt=n(Y);nt=s(dt,"LI",{});var hl=n(nt);zs=l(hl,"P0: "),Mt=s(hl,"CODE",{});var Kl=n(Mt);Ss=l(Kl,"[0, 1]"),Kl.forEach(t),hl.forEach(t),Hs=p(dt),ot=s(dt,"LI",{});var fl=n(ot);Ns=l(fl,"P1: "),Rt=s(fl,"CODE",{});var Yl=n(Rt);Gs=l(Yl,"[6, 7]"),Yl.forEach(t),fl.forEach(t),Vs=p(dt),lt=s(dt,"LI",{});var ul=n(lt);Ms=l(ul,"P2: "),qt=s(ul,"CODE",{});var Wl=n(qt);Rs=l(Wl,"[12, 13]"),Wl.forEach(t),ul.forEach(t),dt.forEach(t),qs=p(y),Bt=s(y,"P",{});var Jl=n(Bt);Bs=l(Jl,`So if we gather at the end of the first batch, we will get a tensor (nested list/tuple of tensor) corresponding to
the following indices:`),Jl.forEach(t),Fs=p(y),Ft=s(y,"P",{});var Ql=n(Ft);Kt=s(Ql,"CODE",{});var Xl=n(Kt);Ks=l(Xl,"[0, 1, 6, 7, 12, 13]"),Xl.forEach(t),Ql.forEach(t),Ys=p(y),Yt=s(y,"P",{});var Zl=n(Yt);Ws=l(Zl,`If we directly concatenate our results without taking any precautions, the user will then get the predictions for
the indices in this order at the end of the prediction loop:`),Zl.forEach(t),Js=p(y),Wt=s(y,"P",{});var ei=n(Wt);Jt=s(ei,"CODE",{});var ti=n(Jt);Qs=l(ti,"[0, 1, 6, 7, 12, 13, 2, 3, 8, 9, 14, 15, 4, 5, 10, 11, 0, 1]"),ti.forEach(t),ei.forEach(t),Xs=p(y),Qt=s(y,"P",{});var ai=n(Qt);Zs=l(ai,"For some reason, that\u2019s not going to roll their boat. This class is there to solve that problem."),ai.forEach(t),en=p(y),se=s(y,"DIV",{class:!0});var mr=n(se);D(Te.$$.fragment,mr),tn=p(mr),Oe=s(mr,"P",{});var gr=n(Oe);an=l(gr,"Add "),Xt=s(gr,"CODE",{});var ri=n(Xt);rn=l(ri,"arrays"),ri.forEach(t),sn=l(gr,` to the internal storage, Will initialize the storage to the full size at the first arrays passed
so that if we\u2019re bound to get an OOM, it happens at the beginning.`),gr.forEach(t),mr.forEach(t),nn=p(y),ne=s(y,"DIV",{class:!0});var br=n(ne);D(Ae.$$.fragment,br),on=p(br),Zt=s(br,"P",{});var si=n(Zt);ln=l(si,`Return the properly gathered arrays and truncate to the number of samples (since the sampler added some extras
to get each process a dataset of the same length).`),si.forEach(t),br.forEach(t),y.forEach(t),tr=p(a),W=s(a,"H2",{class:!0});var _r=n(W);oe=s(_r,"A",{id:!0,class:!0,href:!0});var ni=n(oe);ea=s(ni,"SPAN",{});var oi=n(ea);D(Ce.$$.fragment,oi),oi.forEach(t),ni.forEach(t),pn=p(_r),ta=s(_r,"SPAN",{});var li=n(ta);cn=l(li,"Distributed Evaluation"),li.forEach(t),_r.forEach(t),ar=p(a),O=s(a,"DIV",{class:!0});var U=n(O);D(Le.$$.fragment,U),dn=p(U),Ue=s(U,"P",{});var vr=n(Ue);hn=l(vr,"This subclass of "),aa=s(vr,"CODE",{});var ii=n(aa);fn=l(ii,"argparse.ArgumentParser"),ii.forEach(t),un=l(vr," uses type hints on dataclasses to generate arguments."),vr.forEach(t),mn=p(U),Ie=s(U,"P",{});var wr=n(Ie);gn=l(wr,`The class is designed to play well with the native argparse. In particular, you can add more (non-dataclass backed)
arguments to the parser after initialization and you\u2019ll get the output back after parsing as an additional
namespace. Optional: To create sub argument groups use the `),ra=s(wr,"CODE",{});var pi=n(ra);bn=l(pi,"_argument_group_name"),pi.forEach(t),_n=l(wr," attribute in the dataclass."),wr.forEach(t),vn=p(U),S=s(U,"DIV",{class:!0});var ht=n(S);D(ze.$$.fragment,ht),wn=p(ht),sa=s(ht,"P",{});var ci=n(sa);$n=l(ci,"Parse command-line args into instances of the specified dataclass types."),ci.forEach(t),yn=p(ht),Se=s(ht,"P",{});var $r=n(Se);En=l($r,"This relies on argparse\u2019s "),na=s($r,"CODE",{});var di=n(na);Dn=l(di,"ArgumentParser.parse_known_args"),di.forEach(t),jn=l($r,`. See the doc at:
docs.python.org/3.7/library/argparse.html#argparse.ArgumentParser.parse_args`),$r.forEach(t),ht.forEach(t),kn=p(U),le=s(U,"DIV",{class:!0});var yr=n(le);D(He.$$.fragment,yr),Pn=p(yr),Ne=s(yr,"P",{});var Er=n(Ne);xn=l(Er,"Alternative helper method that does not use "),oa=s(Er,"CODE",{});var hi=n(oa);Tn=l(hi,"argparse"),hi.forEach(t),On=l(Er,` at all, instead uses a dict and populating the dataclass
types.`),Er.forEach(t),yr.forEach(t),An=p(U),ie=s(U,"DIV",{class:!0});var Dr=n(ie);D(Ge.$$.fragment,Dr),Cn=p(Dr),Ve=s(Dr,"P",{});var jr=n(Ve);Ln=l(jr,"Alternative helper method that does not use "),la=s(jr,"CODE",{});var fi=n(la);Un=l(fi,"argparse"),fi.forEach(t),In=l(jr,` at all, instead loading a json file and populating the
dataclass types.`),jr.forEach(t),Dr.forEach(t),U.forEach(t),rr=p(a),J=s(a,"H2",{class:!0});var kr=n(J);pe=s(kr,"A",{id:!0,class:!0,href:!0});var ui=n(pe);ia=s(ui,"SPAN",{});var mi=n(ia);D(Me.$$.fragment,mi),mi.forEach(t),ui.forEach(t),zn=p(kr),pa=s(kr,"SPAN",{});var gi=n(pa);Sn=l(gi,"Debug Utilities"),gi.forEach(t),kr.forEach(t),sr=p(a),d=s(a,"DIV",{class:!0});var h=n(d);D(Re.$$.fragment,h),Hn=p(h),Q=s(h,"P",{});var ft=n(Q);Nn=l(ft,`This debug class helps detect and understand where the model starts getting very large or very small, and more
importantly `),ca=s(ft,"CODE",{});var bi=n(ca);Gn=l(bi,"nan"),bi.forEach(t),Vn=l(ft," or "),da=s(ft,"CODE",{});var _i=n(da);Mn=l(_i,"inf"),_i.forEach(t),Rn=l(ft," weight and activation elements."),ft.forEach(t),qn=p(h),ha=s(h,"P",{});var vi=n(ha);Bn=l(vi,"There are 2 working modes:"),vi.forEach(t),Fn=p(h),qe=s(h,"OL",{});var Pr=n(qe);fa=s(Pr,"LI",{});var wi=n(fa);Kn=l(wi,"Underflow/overflow detection (default)"),wi.forEach(t),Yn=p(Pr),ua=s(Pr,"LI",{});var $i=n(ua);Wn=l($i,"Specific batch absolute min/max tracing without detection"),$i.forEach(t),Pr.forEach(t),Jn=p(h),ma=s(h,"P",{});var yi=n(ma);Qn=l(yi,"Mode 1: Underflow/overflow detection"),yi.forEach(t),Xn=p(h),D(ce.$$.fragment,h),Zn=p(h),z=s(h,"P",{});var me=n(z);eo=l(me,"then run the training as normal and if "),ga=s(me,"CODE",{});var Ei=n(ga);to=l(Ei,"nan"),Ei.forEach(t),ao=l(me," or "),ba=s(me,"CODE",{});var Di=n(ba);ro=l(Di,"inf"),Di.forEach(t),so=l(me,` gets detected in at least one of the weight, input or output
elements this module will throw an exception and will print `),_a=s(me,"CODE",{});var ji=n(_a);no=l(ji,"max_frames_to_save"),ji.forEach(t),oo=l(me,` frames that lead to this event,
each frame reporting`),me.forEach(t),lo=p(h),Be=s(h,"OL",{});var xr=n(Be);Fe=s(xr,"LI",{});var Tr=n(Fe);io=l(Tr,"the fully qualified module name plus the class name whose "),va=s(Tr,"CODE",{});var ki=n(va);po=l(ki,"forward"),ki.forEach(t),co=l(Tr," was run"),Tr.forEach(t),ho=p(xr),wa=s(xr,"LI",{});var Pi=n(wa);fo=l(Pi,"the absolute min and max value of all elements for each module weights, and the inputs and output"),Pi.forEach(t),xr.forEach(t),uo=p(h),Ke=s(h,"P",{});var Or=n(Ke);mo=l(Or,"For example, here is the header and the last few frames in detection report for "),$a=s(Or,"CODE",{});var xi=n($a);go=l(xi,"google/mt5-small"),xi.forEach(t),bo=l(Or," run in fp16"),Or.forEach(t),_o=p(h),D(de.$$.fragment,h),vo=p(h),X=s(h,"P",{});var ut=n(X);wo=l(ut,"You can see here, that "),ya=s(ut,"CODE",{});var Ti=n(ya);$o=l(Ti,"T5DenseGatedGeluDense.forward"),Ti.forEach(t),yo=l(ut,` resulted in output activations, whose absolute max value was
around 62.7K, which is very close to fp16\u2019s top limit of 64K. In the next frame we have `),Ea=s(ut,"CODE",{});var Oi=n(Ea);Eo=l(Oi,"Dropout"),Oi.forEach(t),Do=l(ut,` which
renormalizes the weights, after it zeroed some of the elements, which pushes the absolute max value to more than
64K, and we get an overlow.`),ut.forEach(t),jo=p(h),Da=s(h,"P",{});var Ai=n(Da);ko=l(Ai,`As you can see it\u2019s the previous frames that we need to look into when the numbers start going into very large for
fp16 numbers.`),Ai.forEach(t),Po=p(h),Ye=s(h,"P",{});var Ar=n(Ye);xo=l(Ar,"The tracking is done in a forward hook, which gets invoked immediately after "),ja=s(Ar,"CODE",{});var Ci=n(ja);To=l(Ci,"forward"),Ci.forEach(t),Oo=l(Ar," has completed."),Ar.forEach(t),Ao=p(h),D(he.$$.fragment,h),Co=p(h),ka=s(h,"P",{});var Li=n(ka);Lo=l(Li,`To validate that you have set up this debugging feature correctly, and you intend to use it in a training that
may take hours to complete, first run it with normal tracing enabled for one of a few batches as explained in
the next section.`),Li.forEach(t),Uo=p(h),Pa=s(h,"P",{});var Ui=n(Pa);Io=l(Ui,"Mode 2. Specific batch absolute min/max tracing without detection"),Ui.forEach(t),zo=p(h),xa=s(h,"P",{});var Ii=n(xa);So=l(Ii,"The second work mode is per-batch tracing with the underflow/overflow detection feature turned off."),Ii.forEach(t),Ho=p(h),We=s(h,"P",{});var Cr=n(We);No=l(Cr,"Let\u2019s say you want to watch the absolute min and max values for all the ingredients of each "),Ta=s(Cr,"CODE",{});var zi=n(Ta);Go=l(zi,"forward"),zi.forEach(t),Vo=l(Cr," call of a"),Cr.forEach(t),Mo=p(h),D(fe.$$.fragment,h),Ro=p(h),Oa=s(h,"P",{});var Si=n(Oa);qo=l(Si,"And now full batches 1 and 3 will be traced using the same format as explained above. Batches are 0-indexed."),Si.forEach(t),Bo=p(h),Aa=s(h,"P",{});var Hi=n(Aa);Fo=l(Hi,`This is helpful if you know that the program starts misbehaving after a certain batch number, so you can
fast-forward right to that area.`),Hi.forEach(t),Ko=p(h),Ca=s(h,"P",{});var Ni=n(Ca);Yo=l(Ni,"Early stopping:"),Ni.forEach(t),Wo=p(h),D(ue.$$.fragment,h),Jo=p(h),La=s(h,"P",{});var Gi=n(La);Qo=l(Gi,"This feature is mainly useful in the tracing mode, but you can use it for any mode."),Gi.forEach(t),Xo=p(h),it=s(h,"P",{});var ml=n(it);Ua=s(ml,"STRONG",{});var Vi=n(Ua);Zo=l(Vi,"Performance"),Vi.forEach(t),el=l(ml,":"),ml.forEach(t),tl=p(h),Z=s(h,"P",{});var mt=n(Z);al=l(mt,"As this module measures absolute "),Ia=s(mt,"CODE",{});var Mi=n(Ia);rl=l(Mi,"min"),Mi.forEach(t),sl=l(mt,"/`"),za=s(mt,"CODE",{});var Ri=n(za);nl=l(Ri,"max"),Ri.forEach(t),ol=l(mt,` of each weight of the model on every forward it\u2019ll slow the training
down. Therefore remember to turn it off once the debugging needs have been met.`),mt.forEach(t),h.forEach(t),this.h()},h(){m(g,"name","hf:doc:metadata"),m(g,"content",JSON.stringify(tp)),m(f,"id","utilities-for-trainer"),m(f,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),m(f,"href","#utilities-for-trainer"),m(w,"class","relative group"),m(Xe,"href","/docs/transformers/pr_17548/en/main_classes/trainer#transformers.Trainer"),m(te,"id","transformers.EvalPrediction"),m(te,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),m(te,"href","#transformers.EvalPrediction"),m(N,"class","relative group"),m(G,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),m(V,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),m($e,"href","https://pytorch.org/docs/stable/notes/randomness.html"),m($e,"rel","nofollow"),m(ye,"href","https://www.tensorflow.org/api_docs/python/tf/config/experimental/enable_op_determinism"),m(ye,"rel","nofollow"),m(I,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),m(M,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),m(R,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),m(ae,"id","transformers.trainer_callback.CallbackHandler"),m(ae,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),m(ae,"href","#transformers.trainer_callback.CallbackHandler"),m(q,"class","relative group"),m(B,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),m(re,"id","transformers.trainer_pt_utils.DistributedTensorGatherer"),m(re,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),m(re,"href","#transformers.trainer_pt_utils.DistributedTensorGatherer"),m(F,"class","relative group"),m(se,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),m(ne,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),m(v,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),m(oe,"id","transformers.HfArgumentParser"),m(oe,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),m(oe,"href","#transformers.HfArgumentParser"),m(W,"class","relative group"),m(S,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),m(le,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),m(ie,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),m(O,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),m(pe,"id","transformers.debug_utils.DebugUnderflowOverflow"),m(pe,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),m(pe,"href","#transformers.debug_utils.DebugUnderflowOverflow"),m(J,"class","relative group"),m(d,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8")},m(a,u){e(document.head,g),b(a,T,u),b(a,w,u),e(w,f),e(f,$),j(c,$,null),e(w,_),e(w,bt),e(bt,Lr),b(a,Ra,u),b(a,ee,u),e(ee,Ur),e(ee,Xe),e(Xe,Ir),e(ee,zr),b(a,qa,u),b(a,Ze,u),e(Ze,Sr),b(a,Ba,u),b(a,N,u),e(N,te),e(te,_t),j(ge,_t,null),e(N,Hr),e(N,vt),e(vt,Nr),b(a,Fa,u),b(a,G,u),j(be,G,null),e(G,Gr),e(G,wt),e(wt,Vr),b(a,Ka,u),b(a,V,u),j(_e,V,null),e(V,Mr),e(V,$t),e($t,Rr),b(a,Ya,u),b(a,I,u),j(ve,I,null),e(I,qr),e(I,yt),e(yt,Br),e(I,Fr),e(I,we),e(we,et),e(et,$e),e($e,Kr),e(et,Yr),e(we,Wr),e(we,tt),e(tt,ye),e(ye,Jr),e(tt,Qr),b(a,Wa,u),b(a,M,u),j(Ee,M,null),e(M,Xr),e(M,C),e(C,Zr),e(C,Et),e(Et,es),e(C,ts),e(C,Dt),e(Dt,as),e(C,rs),e(C,jt),e(jt,ss),e(C,ns),e(C,kt),e(kt,os),e(C,ls),b(a,Ja,u),b(a,R,u),j(De,R,null),e(R,is),e(R,Pt),e(Pt,ps),b(a,Qa,u),b(a,q,u),e(q,ae),e(ae,xt),j(je,xt,null),e(q,cs),e(q,Tt),e(Tt,ds),b(a,Xa,u),b(a,B,u),j(ke,B,null),e(B,hs),e(B,Ot),e(Ot,fs),b(a,Za,u),b(a,F,u),e(F,re),e(re,At),j(Pe,At,null),e(F,us),e(F,Ct),e(Ct,ms),b(a,er,u),b(a,v,u),j(xe,v,null),e(v,gs),e(v,Lt),e(Lt,bs),e(v,_s),e(v,Ut),e(Ut,vs),e(v,ws),e(v,It),e(It,zt),e(zt,$s),e(v,ys),e(v,St),e(St,Es),e(v,Ds),e(v,K),e(K,at),e(at,js),e(at,Ht),e(Ht,ks),e(K,Ps),e(K,rt),e(rt,xs),e(rt,Nt),e(Nt,Ts),e(K,Os),e(K,st),e(st,As),e(st,Gt),e(Gt,Cs),e(v,Ls),e(v,Vt),e(Vt,Us),e(v,Is),e(v,Y),e(Y,nt),e(nt,zs),e(nt,Mt),e(Mt,Ss),e(Y,Hs),e(Y,ot),e(ot,Ns),e(ot,Rt),e(Rt,Gs),e(Y,Vs),e(Y,lt),e(lt,Ms),e(lt,qt),e(qt,Rs),e(v,qs),e(v,Bt),e(Bt,Bs),e(v,Fs),e(v,Ft),e(Ft,Kt),e(Kt,Ks),e(v,Ys),e(v,Yt),e(Yt,Ws),e(v,Js),e(v,Wt),e(Wt,Jt),e(Jt,Qs),e(v,Xs),e(v,Qt),e(Qt,Zs),e(v,en),e(v,se),j(Te,se,null),e(se,tn),e(se,Oe),e(Oe,an),e(Oe,Xt),e(Xt,rn),e(Oe,sn),e(v,nn),e(v,ne),j(Ae,ne,null),e(ne,on),e(ne,Zt),e(Zt,ln),b(a,tr,u),b(a,W,u),e(W,oe),e(oe,ea),j(Ce,ea,null),e(W,pn),e(W,ta),e(ta,cn),b(a,ar,u),b(a,O,u),j(Le,O,null),e(O,dn),e(O,Ue),e(Ue,hn),e(Ue,aa),e(aa,fn),e(Ue,un),e(O,mn),e(O,Ie),e(Ie,gn),e(Ie,ra),e(ra,bn),e(Ie,_n),e(O,vn),e(O,S),j(ze,S,null),e(S,wn),e(S,sa),e(sa,$n),e(S,yn),e(S,Se),e(Se,En),e(Se,na),e(na,Dn),e(Se,jn),e(O,kn),e(O,le),j(He,le,null),e(le,Pn),e(le,Ne),e(Ne,xn),e(Ne,oa),e(oa,Tn),e(Ne,On),e(O,An),e(O,ie),j(Ge,ie,null),e(ie,Cn),e(ie,Ve),e(Ve,Ln),e(Ve,la),e(la,Un),e(Ve,In),b(a,rr,u),b(a,J,u),e(J,pe),e(pe,ia),j(Me,ia,null),e(J,zn),e(J,pa),e(pa,Sn),b(a,sr,u),b(a,d,u),j(Re,d,null),e(d,Hn),e(d,Q),e(Q,Nn),e(Q,ca),e(ca,Gn),e(Q,Vn),e(Q,da),e(da,Mn),e(Q,Rn),e(d,qn),e(d,ha),e(ha,Bn),e(d,Fn),e(d,qe),e(qe,fa),e(fa,Kn),e(qe,Yn),e(qe,ua),e(ua,Wn),e(d,Jn),e(d,ma),e(ma,Qn),e(d,Xn),j(ce,d,null),e(d,Zn),e(d,z),e(z,eo),e(z,ga),e(ga,to),e(z,ao),e(z,ba),e(ba,ro),e(z,so),e(z,_a),e(_a,no),e(z,oo),e(d,lo),e(d,Be),e(Be,Fe),e(Fe,io),e(Fe,va),e(va,po),e(Fe,co),e(Be,ho),e(Be,wa),e(wa,fo),e(d,uo),e(d,Ke),e(Ke,mo),e(Ke,$a),e($a,go),e(Ke,bo),e(d,_o),j(de,d,null),e(d,vo),e(d,X),e(X,wo),e(X,ya),e(ya,$o),e(X,yo),e(X,Ea),e(Ea,Eo),e(X,Do),e(d,jo),e(d,Da),e(Da,ko),e(d,Po),e(d,Ye),e(Ye,xo),e(Ye,ja),e(ja,To),e(Ye,Oo),e(d,Ao),j(he,d,null),e(d,Co),e(d,ka),e(ka,Lo),e(d,Uo),e(d,Pa),e(Pa,Io),e(d,zo),e(d,xa),e(xa,So),e(d,Ho),e(d,We),e(We,No),e(We,Ta),e(Ta,Go),e(We,Vo),e(d,Mo),j(fe,d,null),e(d,Ro),e(d,Oa),e(Oa,qo),e(d,Bo),e(d,Aa),e(Aa,Fo),e(d,Ko),e(d,Ca),e(Ca,Yo),e(d,Wo),j(ue,d,null),e(d,Jo),e(d,La),e(La,Qo),e(d,Xo),e(d,it),e(it,Ua),e(Ua,Zo),e(it,el),e(d,tl),e(d,Z),e(Z,al),e(Z,Ia),e(Ia,rl),e(Z,sl),e(Z,za),e(za,nl),e(Z,ol),nr=!0},p(a,[u]){const Je={};u&2&&(Je.$$scope={dirty:u,ctx:a}),ce.$set(Je);const Sa={};u&2&&(Sa.$$scope={dirty:u,ctx:a}),de.$set(Sa);const Ha={};u&2&&(Ha.$$scope={dirty:u,ctx:a}),he.$set(Ha);const Na={};u&2&&(Na.$$scope={dirty:u,ctx:a}),fe.$set(Na);const Qe={};u&2&&(Qe.$$scope={dirty:u,ctx:a}),ue.$set(Qe)},i(a){nr||(k(c.$$.fragment,a),k(ge.$$.fragment,a),k(be.$$.fragment,a),k(_e.$$.fragment,a),k(ve.$$.fragment,a),k(Ee.$$.fragment,a),k(De.$$.fragment,a),k(je.$$.fragment,a),k(ke.$$.fragment,a),k(Pe.$$.fragment,a),k(xe.$$.fragment,a),k(Te.$$.fragment,a),k(Ae.$$.fragment,a),k(Ce.$$.fragment,a),k(Le.$$.fragment,a),k(ze.$$.fragment,a),k(He.$$.fragment,a),k(Ge.$$.fragment,a),k(Me.$$.fragment,a),k(Re.$$.fragment,a),k(ce.$$.fragment,a),k(de.$$.fragment,a),k(he.$$.fragment,a),k(fe.$$.fragment,a),k(ue.$$.fragment,a),nr=!0)},o(a){P(c.$$.fragment,a),P(ge.$$.fragment,a),P(be.$$.fragment,a),P(_e.$$.fragment,a),P(ve.$$.fragment,a),P(Ee.$$.fragment,a),P(De.$$.fragment,a),P(je.$$.fragment,a),P(ke.$$.fragment,a),P(Pe.$$.fragment,a),P(xe.$$.fragment,a),P(Te.$$.fragment,a),P(Ae.$$.fragment,a),P(Ce.$$.fragment,a),P(Le.$$.fragment,a),P(ze.$$.fragment,a),P(He.$$.fragment,a),P(Ge.$$.fragment,a),P(Me.$$.fragment,a),P(Re.$$.fragment,a),P(ce.$$.fragment,a),P(de.$$.fragment,a),P(he.$$.fragment,a),P(fe.$$.fragment,a),P(ue.$$.fragment,a),nr=!1},d(a){t(g),a&&t(T),a&&t(w),x(c),a&&t(Ra),a&&t(ee),a&&t(qa),a&&t(Ze),a&&t(Ba),a&&t(N),x(ge),a&&t(Fa),a&&t(G),x(be),a&&t(Ka),a&&t(V),x(_e),a&&t(Ya),a&&t(I),x(ve),a&&t(Wa),a&&t(M),x(Ee),a&&t(Ja),a&&t(R),x(De),a&&t(Qa),a&&t(q),x(je),a&&t(Xa),a&&t(B),x(ke),a&&t(Za),a&&t(F),x(Pe),a&&t(er),a&&t(v),x(xe),x(Te),x(Ae),a&&t(tr),a&&t(W),x(Ce),a&&t(ar),a&&t(O),x(Le),x(ze),x(He),x(Ge),a&&t(rr),a&&t(J),x(Me),a&&t(sr),a&&t(d),x(Re),x(ce),x(de),x(he),x(fe),x(ue)}}}const tp={local:"utilities-for-trainer",sections:[{local:"transformers.EvalPrediction",title:"Utilities"},{local:"transformers.trainer_callback.CallbackHandler",title:"Callbacks internals"},{local:"transformers.trainer_pt_utils.DistributedTensorGatherer",title:"Distributed Evaluation"},{local:"transformers.HfArgumentParser",title:"Distributed Evaluation"},{local:"transformers.debug_utils.DebugUnderflowOverflow",title:"Debug Utilities"}],title:"Utilities for Trainer"};function ap(L){return Yi(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class ip extends qi{constructor(g){super();Bi(this,g,ap,ep,Fi,{})}}export{ip as default,tp as metadata};
