import{S as Fi,i as Oi,s as Ri,e as a,k as l,w as g,t as s,M as ji,c as n,d as r,m,a as o,x as _,h as i,b as c,N as Ca,G as t,g as h,y as w,q as v,o as b,B as y,v as qi,L as Oa}from"../../chunks/vendor-hf-doc-builder.js";import{D as L}from"../../chunks/Docstring-hf-doc-builder.js";import{C as Ra}from"../../chunks/CodeBlock-hf-doc-builder.js";import{I as ne}from"../../chunks/IconCopyLink-hf-doc-builder.js";import{E as Fa}from"../../chunks/ExampleCodeBlock-hf-doc-builder.js";function Ui(C){let f,T,x,u,z;return u=new Ra({props:{code:"Adafactor(model.parameters(), scale_parameter=False, relative_step=False, warmup_init=False, lr=1e-3)",highlighted:'Adafactor(model.parameters(), scale_parameter=<span class="hljs-literal">False</span>, relative_step=<span class="hljs-literal">False</span>, warmup_init=<span class="hljs-literal">False</span>, lr=<span class="hljs-number">1e-3</span>)'}}),{c(){f=a("p"),T=s("Example:"),x=l(),g(u.$$.fragment)},l(p){f=n(p,"P",{});var $=o(f);T=i($,"Example:"),$.forEach(r),x=m(p),_(u.$$.fragment,p)},m(p,$){h(p,f,$),t(f,T),h(p,x,$),w(u,p,$),z=!0},p:Oa,i(p){z||(v(u.$$.fragment,p),z=!0)},o(p){b(u.$$.fragment,p),z=!1},d(p){p&&r(f),p&&r(x),y(u,p)}}}function Gi(C){let f,T,x,u,z;return u=new Ra({props:{code:"Adafactor(model.parameters(), scale_parameter=True, relative_step=True, warmup_init=True, lr=None)",highlighted:'Adafactor(model.parameters(), scale_parameter=<span class="hljs-literal">True</span>, relative_step=<span class="hljs-literal">True</span>, warmup_init=<span class="hljs-literal">True</span>, lr=<span class="hljs-literal">None</span>)'}}),{c(){f=a("p"),T=s("Others reported the following combination to work well:"),x=l(),g(u.$$.fragment)},l(p){f=n(p,"P",{});var $=o(f);T=i($,"Others reported the following combination to work well:"),$.forEach(r),x=m(p),_(u.$$.fragment,p)},m(p,$){h(p,f,$),t(f,T),h(p,x,$),w(u,p,$),z=!0},p:Oa,i(p){z||(v(u.$$.fragment,p),z=!0)},o(p){b(u.$$.fragment,p),z=!1},d(p){p&&r(f),p&&r(x),y(u,p)}}}function Vi(C){let f,T,x,u,z;return u=new Ra({props:{code:`from transformers.optimization import Adafactor, AdafactorSchedule

optimizer = Adafactor(model.parameters(), scale_parameter=True, relative_step=True, warmup_init=True, lr=None)
lr_scheduler = AdafactorSchedule(optimizer)
trainer = Trainer(..., optimizers=(optimizer, lr_scheduler))`,highlighted:`<span class="hljs-keyword">from</span> transformers.optimization <span class="hljs-keyword">import</span> Adafactor, AdafactorSchedule

optimizer = Adafactor(model.parameters(), scale_parameter=<span class="hljs-literal">True</span>, relative_step=<span class="hljs-literal">True</span>, warmup_init=<span class="hljs-literal">True</span>, lr=<span class="hljs-literal">None</span>)
lr_scheduler = AdafactorSchedule(optimizer)
trainer = Trainer(..., optimizers=(optimizer, lr_scheduler))`}}),{c(){f=a("p"),T=s("scheduler as following:"),x=l(),g(u.$$.fragment)},l(p){f=n(p,"P",{});var $=o(f);T=i($,"scheduler as following:"),$.forEach(r),x=m(p),_(u.$$.fragment,p)},m(p,$){h(p,f,$),t(f,T),h(p,x,$),w(u,p,$),z=!0},p:Oa,i(p){z||(v(u.$$.fragment,p),z=!0)},o(p){b(u.$$.fragment,p),z=!1},d(p){p&&r(f),p&&r(x),y(u,p)}}}function Mi(C){let f,T,x,u,z;return u=new Ra({props:{code:`# replace AdamW with Adafactor
optimizer = Adafactor(
    model.parameters(),
    lr=1e-3,
    eps=(1e-30, 1e-3),
    clip_threshold=1.0,
    decay_rate=-0.8,
    beta1=None,
    weight_decay=0.0,
    relative_step=False,
    scale_parameter=False,
    warmup_init=False,
)`,highlighted:`<span class="hljs-comment"># replace AdamW with Adafactor</span>
optimizer = Adafactor(
    model.parameters(),
    lr=<span class="hljs-number">1e-3</span>,
    eps=(<span class="hljs-number">1e-30</span>, <span class="hljs-number">1e-3</span>),
    clip_threshold=<span class="hljs-number">1.0</span>,
    decay_rate=-<span class="hljs-number">0.8</span>,
    beta1=<span class="hljs-literal">None</span>,
    weight_decay=<span class="hljs-number">0.0</span>,
    relative_step=<span class="hljs-literal">False</span>,
    scale_parameter=<span class="hljs-literal">False</span>,
    warmup_init=<span class="hljs-literal">False</span>,
)`}}),{c(){f=a("p"),T=s("Usage:"),x=l(),g(u.$$.fragment)},l(p){f=n(p,"P",{});var $=o(f);T=i($,"Usage:"),$.forEach(r),x=m(p),_(u.$$.fragment,p)},m(p,$){h(p,f,$),t(f,T),h(p,x,$),w(u,p,$),z=!0},p:Oa,i(p){z||(v(u.$$.fragment,p),z=!0)},o(p){b(u.$$.fragment,p),z=!1},d(p){p&&r(f),p&&r(x),y(u,p)}}}function Hi(C){let f,T,x,u,z,p,$,zt,ja,kr,oe,qa,Et,Ua,Ga,Sr,F,Tt,Va,Ma,ze,Ha,Dt,Ba,Ja,Ka,Lt,Qa,Nr,R,se,Pt,Ee,Xa,Wt,Ya,Ir,k,Te,Za,De,en,Le,tn,rn,an,ie,Pe,nn,kt,on,Cr,j,le,St,We,sn,Nt,ln,Fr,A,ke,mn,_t,cn,Se,pn,dn,D,hn,It,un,fn,Ne,gn,_n,Ct,wn,vn,Ft,bn,yn,Ot,$n,xn,Rt,An,zn,jt,En,Tn,Dn,qt,Ln,Pn,Ie,Wn,Ce,kn,Sn,Nn,S,Fe,Ut,In,Cn,Oe,Gt,Fn,On,Re,Rn,je,jn,qn,Un,Vt,Mt,Gn,Vn,Ht,Bt,Mn,Hn,Jt,Kt,Bn,Jn,me,Kn,ce,Qn,O,Xn,Qt,Yn,Zn,wt,eo,to,Xt,ro,ao,pe,no,de,oo,he,qe,so,Yt,io,Or,q,ue,Zt,Ue,lo,er,mo,Rr,W,Ge,co,U,po,tr,ho,uo,Ve,fo,go,_o,rr,wo,vo,fe,Me,bo,ar,yo,jr,G,He,$o,nr,xo,qr,V,ge,or,Be,Ao,sr,zo,Ur,M,_e,ir,Je,Eo,lr,To,Gr,H,Ke,Do,mr,Lo,Vr,B,Qe,Po,cr,Wo,Mr,J,Xe,ko,pr,So,Hr,K,Ye,No,dr,Io,Br,Ze,fs,Jr,Q,et,Co,hr,Fo,Kr,tt,gs,Qr,X,rt,Oo,ur,Ro,Xr,at,_s,Yr,Y,nt,jo,fr,qo,Zr,ot,ws,ea,N,st,Uo,it,Go,gr,Vo,Mo,Ho,we,Bo,_r,Jo,Ko,lt,Qo,ta,Z,ve,wr,mt,Xo,vr,Yo,ra,ee,ct,Zo,br,es,aa,te,be,yr,pt,ts,$r,rs,na,re,ye,xr,dt,as,Ar,ns,oa,I,ht,os,ae,ss,zr,is,ls,Er,ms,cs,ps,$e,ut,ds,Tr,hs,sa;return p=new ne({}),Ee=new ne({}),Te=new L({props:{name:"class transformers.AdamW",anchor:"transformers.AdamW",parameters:[{name:"params",val:": typing.Iterable[torch.nn.parameter.Parameter]"},{name:"lr",val:": float = 0.001"},{name:"betas",val:": typing.Tuple[float, float] = (0.9, 0.999)"},{name:"eps",val:": float = 1e-06"},{name:"weight_decay",val:": float = 0.0"},{name:"correct_bias",val:": bool = True"},{name:"no_deprecation_warning",val:": bool = False"}],parametersDescription:[{anchor:"transformers.AdamW.params",description:`<strong>params</strong> (<code>Iterable[nn.parameter.Parameter]</code>) &#x2014;
Iterable of parameters to optimize or dictionaries defining parameter groups.`,name:"params"},{anchor:"transformers.AdamW.lr",description:`<strong>lr</strong> (<code>float</code>, <em>optional</em>, defaults to 1e-3) &#x2014;
The learning rate to use.`,name:"lr"},{anchor:"transformers.AdamW.betas",description:`<strong>betas</strong> (<code>Tuple[float,float]</code>, <em>optional</em>, defaults to (0.9, 0.999)) &#x2014;
Adam&#x2019;s betas parameters (b1, b2).`,name:"betas"},{anchor:"transformers.AdamW.eps",description:`<strong>eps</strong> (<code>float</code>, <em>optional</em>, defaults to 1e-6) &#x2014;
Adam&#x2019;s epsilon for numerical stability.`,name:"eps"},{anchor:"transformers.AdamW.weight_decay",description:`<strong>weight_decay</strong> (<code>float</code>, <em>optional</em>, defaults to 0) &#x2014;
Decoupled weight decay to apply.`,name:"weight_decay"},{anchor:"transformers.AdamW.correct_bias",description:`<strong>correct_bias</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>True</code>) &#x2014;
Whether or not to correct bias in Adam (for instance, in Bert TF repository they use <code>False</code>).`,name:"correct_bias"},{anchor:"transformers.AdamW.no_deprecation_warning",description:`<strong>no_deprecation_warning</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>False</code>) &#x2014;
A flag used to disable the deprecation warning (set to <code>True</code> to disable the warning).`,name:"no_deprecation_warning"}],source:"https://github.com/huggingface/transformers/blob/vr_19907/src/transformers/optimization.py#L273"}}),Pe=new L({props:{name:"step",anchor:"transformers.AdamW.step",parameters:[{name:"closure",val:": typing.Callable = None"}],parametersDescription:[{anchor:"transformers.AdamW.step.closure",description:"<strong>closure</strong> (<code>Callable</code>, <em>optional</em>) &#x2014; A closure that reevaluates the model and returns the loss.",name:"closure"}],source:"https://github.com/huggingface/transformers/blob/vr_19907/src/transformers/optimization.py#L324"}}),We=new ne({}),ke=new L({props:{name:"class transformers.Adafactor",anchor:"transformers.Adafactor",parameters:[{name:"params",val:""},{name:"lr",val:" = None"},{name:"eps",val:" = (1e-30, 0.001)"},{name:"clip_threshold",val:" = 1.0"},{name:"decay_rate",val:" = -0.8"},{name:"beta1",val:" = None"},{name:"weight_decay",val:" = 0.0"},{name:"scale_parameter",val:" = True"},{name:"relative_step",val:" = True"},{name:"warmup_init",val:" = False"}],parametersDescription:[{anchor:"transformers.Adafactor.params",description:`<strong>params</strong> (<code>Iterable[nn.parameter.Parameter]</code>) &#x2014;
Iterable of parameters to optimize or dictionaries defining parameter groups.`,name:"params"},{anchor:"transformers.Adafactor.lr",description:`<strong>lr</strong> (<code>float</code>, <em>optional</em>) &#x2014;
The external learning rate.`,name:"lr"},{anchor:"transformers.Adafactor.eps",description:`<strong>eps</strong> (<code>Tuple[float, float]</code>, <em>optional</em>, defaults to (1e-30, 1e-3)) &#x2014;
Regularization constants for square gradient and parameter scale respectively`,name:"eps"},{anchor:"transformers.Adafactor.clip_threshold",description:`<strong>clip_threshold</strong> (<code>float</code>, <em>optional</em>, defaults 1.0) &#x2014;
Threshold of root mean square of final gradient update`,name:"clip_threshold"},{anchor:"transformers.Adafactor.decay_rate",description:`<strong>decay_rate</strong> (<code>float</code>, <em>optional</em>, defaults to -0.8) &#x2014;
Coefficient used to compute running averages of square`,name:"decay_rate"},{anchor:"transformers.Adafactor.beta1",description:`<strong>beta1</strong> (<code>float</code>, <em>optional</em>) &#x2014;
Coefficient used for computing running averages of gradient`,name:"beta1"},{anchor:"transformers.Adafactor.weight_decay",description:`<strong>weight_decay</strong> (<code>float</code>, <em>optional</em>, defaults to 0) &#x2014;
Weight decay (L2 penalty)`,name:"weight_decay"},{anchor:"transformers.Adafactor.scale_parameter",description:`<strong>scale_parameter</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>True</code>) &#x2014;
If True, learning rate is scaled by root mean square`,name:"scale_parameter"},{anchor:"transformers.Adafactor.relative_step",description:`<strong>relative_step</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>True</code>) &#x2014;
If True, time-dependent learning rate is computed instead of external learning rate`,name:"relative_step"},{anchor:"transformers.Adafactor.warmup_init",description:`<strong>warmup_init</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>False</code>) &#x2014;
Time-dependent learning rate computation depends on whether warm-up initialization is being used`,name:"warmup_init"}],source:"https://github.com/huggingface/transformers/blob/vr_19907/src/transformers/optimization.py#L386"}}),me=new Fa({props:{anchor:"transformers.Adafactor.example",$$slots:{default:[Ui]},$$scope:{ctx:C}}}),ce=new Fa({props:{anchor:"transformers.Adafactor.example-2",$$slots:{default:[Gi]},$$scope:{ctx:C}}}),pe=new Fa({props:{anchor:"transformers.Adafactor.example-3",$$slots:{default:[Vi]},$$scope:{ctx:C}}}),de=new Fa({props:{anchor:"transformers.Adafactor.example-4",$$slots:{default:[Mi]},$$scope:{ctx:C}}}),qe=new L({props:{name:"step",anchor:"transformers.Adafactor.step",parameters:[{name:"closure",val:" = None"}],parametersDescription:[{anchor:"transformers.Adafactor.step.closure",description:`<strong>closure</strong> (callable, optional) &#x2014; A closure that reevaluates the model
and returns the loss.`,name:"closure"}],source:"https://github.com/huggingface/transformers/blob/vr_19907/src/transformers/optimization.py#L532"}}),Ue=new ne({}),Ge=new L({props:{name:"class transformers.AdamWeightDecay",anchor:"transformers.AdamWeightDecay",parameters:[{name:"learning_rate",val:": typing.Union[float, keras.optimizers.schedules.learning_rate_schedule.LearningRateSchedule] = 0.001"},{name:"beta_1",val:": float = 0.9"},{name:"beta_2",val:": float = 0.999"},{name:"epsilon",val:": float = 1e-07"},{name:"amsgrad",val:": bool = False"},{name:"weight_decay_rate",val:": float = 0.0"},{name:"include_in_weight_decay",val:": typing.Optional[typing.List[str]] = None"},{name:"exclude_from_weight_decay",val:": typing.Optional[typing.List[str]] = None"},{name:"name",val:": str = 'AdamWeightDecay'"},{name:"**kwargs",val:""}],parametersDescription:[{anchor:"transformers.AdamWeightDecay.learning_rate",description:`<strong>learning_rate</strong> (<code>Union[float, tf.keras.optimizers.schedules.LearningRateSchedule]</code>, <em>optional</em>, defaults to 1e-3) &#x2014;
The learning rate to use or a schedule.`,name:"learning_rate"},{anchor:"transformers.AdamWeightDecay.beta_1",description:`<strong>beta_1</strong> (<code>float</code>, <em>optional</em>, defaults to 0.9) &#x2014;
The beta1 parameter in Adam, which is the exponential decay rate for the 1st momentum estimates.`,name:"beta_1"},{anchor:"transformers.AdamWeightDecay.beta_2",description:`<strong>beta_2</strong> (<code>float</code>, <em>optional</em>, defaults to 0.999) &#x2014;
The beta2 parameter in Adam, which is the exponential decay rate for the 2nd momentum estimates.`,name:"beta_2"},{anchor:"transformers.AdamWeightDecay.epsilon",description:`<strong>epsilon</strong> (<code>float</code>, <em>optional</em>, defaults to 1e-7) &#x2014;
The epsilon parameter in Adam, which is a small constant for numerical stability.`,name:"epsilon"},{anchor:"transformers.AdamWeightDecay.amsgrad",description:`<strong>amsgrad</strong> (<code>bool</code>, <em>optional</em>, default to <code>False</code>) &#x2014;
Whether to apply AMSGrad variant of this algorithm or not, see <a href="https://arxiv.org/abs/1904.09237" rel="nofollow">On the Convergence of Adam and
Beyond</a>.`,name:"amsgrad"},{anchor:"transformers.AdamWeightDecay.weight_decay_rate",description:`<strong>weight_decay_rate</strong> (<code>float</code>, <em>optional</em>, defaults to 0) &#x2014;
The weight decay to apply.`,name:"weight_decay_rate"},{anchor:"transformers.AdamWeightDecay.include_in_weight_decay",description:`<strong>include_in_weight_decay</strong> (<code>List[str]</code>, <em>optional</em>) &#x2014;
List of the parameter names (or re patterns) to apply weight decay to. If none is passed, weight decay is
applied to all parameters by default (unless they are in <code>exclude_from_weight_decay</code>).`,name:"include_in_weight_decay"},{anchor:"transformers.AdamWeightDecay.exclude_from_weight_decay",description:`<strong>exclude_from_weight_decay</strong> (<code>List[str]</code>, <em>optional</em>) &#x2014;
List of the parameter names (or re patterns) to exclude from applying weight decay to. If a
<code>include_in_weight_decay</code> is passed, the names in it will supersede this list.`,name:"exclude_from_weight_decay"},{anchor:"transformers.AdamWeightDecay.name",description:`<strong>name</strong> (<code>str</code>, <em>optional</em>, defaults to &#x2018;AdamWeightDecay&#x2019;) &#x2014;
Optional name for the operations created when applying gradients.
kwargs &#x2014;
Keyword arguments. Allowed to be {<code>clipnorm</code>, <code>clipvalue</code>, <code>lr</code>, <code>decay</code>}. <code>clipnorm</code> is clip gradients by
norm; <code>clipvalue</code> is clip gradients by value, <code>decay</code> is included for backward compatibility to allow time
inverse decay of learning rate. <code>lr</code> is included for backward compatibility, recommended to use
<code>learning_rate</code> instead.`,name:"name"}],source:"https://github.com/huggingface/transformers/blob/vr_19907/src/transformers/optimization_tf.py#L166"}}),Me=new L({props:{name:"from_config",anchor:"transformers.AdamWeightDecay.from_config",parameters:[{name:"config",val:""}],source:"https://github.com/huggingface/transformers/blob/vr_19907/src/transformers/optimization_tf.py#L223"}}),He=new L({props:{name:"transformers.create_optimizer",anchor:"transformers.create_optimizer",parameters:[{name:"init_lr",val:": float"},{name:"num_train_steps",val:": int"},{name:"num_warmup_steps",val:": int"},{name:"min_lr_ratio",val:": float = 0.0"},{name:"adam_beta1",val:": float = 0.9"},{name:"adam_beta2",val:": float = 0.999"},{name:"adam_epsilon",val:": float = 1e-08"},{name:"adam_clipnorm",val:": typing.Optional[float] = None"},{name:"adam_global_clipnorm",val:": typing.Optional[float] = None"},{name:"weight_decay_rate",val:": float = 0.0"},{name:"power",val:": float = 1.0"},{name:"include_in_weight_decay",val:": typing.Optional[typing.List[str]] = None"}],parametersDescription:[{anchor:"transformers.create_optimizer.init_lr",description:`<strong>init_lr</strong> (<code>float</code>) &#x2014;
The desired learning rate at the end of the warmup phase.`,name:"init_lr"},{anchor:"transformers.create_optimizer.num_train_steps",description:`<strong>num_train_steps</strong> (<code>int</code>) &#x2014;
The total number of training steps.`,name:"num_train_steps"},{anchor:"transformers.create_optimizer.num_warmup_steps",description:`<strong>num_warmup_steps</strong> (<code>int</code>) &#x2014;
The number of warmup steps.`,name:"num_warmup_steps"},{anchor:"transformers.create_optimizer.min_lr_ratio",description:`<strong>min_lr_ratio</strong> (<code>float</code>, <em>optional</em>, defaults to 0) &#x2014;
The final learning rate at the end of the linear decay will be <code>init_lr * min_lr_ratio</code>.`,name:"min_lr_ratio"},{anchor:"transformers.create_optimizer.adam_beta1",description:`<strong>adam_beta1</strong> (<code>float</code>, <em>optional</em>, defaults to 0.9) &#x2014;
The beta1 to use in Adam.`,name:"adam_beta1"},{anchor:"transformers.create_optimizer.adam_beta2",description:`<strong>adam_beta2</strong> (<code>float</code>, <em>optional</em>, defaults to 0.999) &#x2014;
The beta2 to use in Adam.`,name:"adam_beta2"},{anchor:"transformers.create_optimizer.adam_epsilon",description:`<strong>adam_epsilon</strong> (<code>float</code>, <em>optional</em>, defaults to 1e-8) &#x2014;
The epsilon to use in Adam.
adam_clipnorm &#x2014; (<code>float</code>, <em>optional</em>, defaults to <code>None</code>):
If not <code>None</code>, clip the gradient norm for each weight tensor to this value.
adam_global_clipnorm &#x2014; (<code>float</code>, <em>optional</em>, defaults to <code>None</code>)
If not <code>None</code>, clip gradient norm to this value. When using this argument, the norm is computed over all
weight tensors, as if they were concatenated into a single vector.`,name:"adam_epsilon"},{anchor:"transformers.create_optimizer.weight_decay_rate",description:`<strong>weight_decay_rate</strong> (<code>float</code>, <em>optional</em>, defaults to 0) &#x2014;
The weight decay to use.`,name:"weight_decay_rate"},{anchor:"transformers.create_optimizer.power",description:`<strong>power</strong> (<code>float</code>, <em>optional</em>, defaults to 1.0) &#x2014;
The power to use for PolynomialDecay.`,name:"power"},{anchor:"transformers.create_optimizer.include_in_weight_decay",description:`<strong>include_in_weight_decay</strong> (<code>List[str]</code>, <em>optional</em>) &#x2014;
List of the parameter names (or re patterns) to apply weight decay to. If none is passed, weight decay is
applied to all parameters except bias and layer norm parameters.`,name:"include_in_weight_decay"}],source:"https://github.com/huggingface/transformers/blob/vr_19907/src/transformers/optimization_tf.py#L82"}}),Be=new ne({}),Je=new ne({}),Ke=new L({props:{name:"class transformers.SchedulerType",anchor:"transformers.SchedulerType",parameters:[{name:"value",val:""},{name:"names",val:" = None"},{name:"module",val:" = None"},{name:"qualname",val:" = None"},{name:"type",val:" = None"},{name:"start",val:" = 1"}],source:"https://github.com/huggingface/transformers/blob/vr_19907/src/transformers/trainer_utils.py#L355"}}),Qe=new L({props:{name:"transformers.get_scheduler",anchor:"transformers.get_scheduler",parameters:[{name:"name",val:": typing.Union[str, transformers.trainer_utils.SchedulerType]"},{name:"optimizer",val:": Optimizer"},{name:"num_warmup_steps",val:": typing.Optional[int] = None"},{name:"num_training_steps",val:": typing.Optional[int] = None"}],parametersDescription:[{anchor:"transformers.get_scheduler.name",description:`<strong>name</strong> (<code>str</code> or <code>SchedulerType</code>) &#x2014;
The name of the scheduler to use.`,name:"name"},{anchor:"transformers.get_scheduler.optimizer",description:`<strong>optimizer</strong> (<code>torch.optim.Optimizer</code>) &#x2014;
The optimizer that will be used during training.`,name:"optimizer"},{anchor:"transformers.get_scheduler.num_warmup_steps",description:`<strong>num_warmup_steps</strong> (<code>int</code>, <em>optional</em>) &#x2014;
The number of warmup steps to do. This is not required by all schedulers (hence the argument being
optional), the function will raise an error if it&#x2019;s unset and the scheduler type requires it.`,name:"num_warmup_steps"},{anchor:"transformers.get_scheduler.num_training_steps",description:`<strong>num_training_steps</strong> (\`int&#x201C;, <em>optional</em>) &#x2014;
The number of training steps to do. This is not required by all schedulers (hence the argument being
optional), the function will raise an error if it&#x2019;s unset and the scheduler type requires it.`,name:"num_training_steps"}],source:"https://github.com/huggingface/transformers/blob/vr_19907/src/transformers/optimization.py#L233"}}),Xe=new L({props:{name:"transformers.get_constant_schedule",anchor:"transformers.get_constant_schedule",parameters:[{name:"optimizer",val:": Optimizer"},{name:"last_epoch",val:": int = -1"}],parametersDescription:[{anchor:"transformers.get_constant_schedule.optimizer",description:`<strong>optimizer</strong> (<code>~torch.optim.Optimizer</code>) &#x2014;
The optimizer for which to schedule the learning rate.`,name:"optimizer"},{anchor:"transformers.get_constant_schedule.last_epoch",description:`<strong>last_epoch</strong> (<code>int</code>, <em>optional</em>, defaults to -1) &#x2014;
The index of the last epoch when resuming training.`,name:"last_epoch"}],source:"https://github.com/huggingface/transformers/blob/vr_19907/src/transformers/optimization.py#L34",returnDescription:`
<p><code>torch.optim.lr_scheduler.LambdaLR</code> with the appropriate schedule.</p>
`}}),Ye=new L({props:{name:"transformers.get_constant_schedule_with_warmup",anchor:"transformers.get_constant_schedule_with_warmup",parameters:[{name:"optimizer",val:": Optimizer"},{name:"num_warmup_steps",val:": int"},{name:"last_epoch",val:": int = -1"}],parametersDescription:[{anchor:"transformers.get_constant_schedule_with_warmup.optimizer",description:`<strong>optimizer</strong> (<code>~torch.optim.Optimizer</code>) &#x2014;
The optimizer for which to schedule the learning rate.`,name:"optimizer"},{anchor:"transformers.get_constant_schedule_with_warmup.num_warmup_steps",description:`<strong>num_warmup_steps</strong> (<code>int</code>) &#x2014;
The number of steps for the warmup phase.`,name:"num_warmup_steps"},{anchor:"transformers.get_constant_schedule_with_warmup.last_epoch",description:`<strong>last_epoch</strong> (<code>int</code>, <em>optional</em>, defaults to -1) &#x2014;
The index of the last epoch when resuming training.`,name:"last_epoch"}],source:"https://github.com/huggingface/transformers/blob/vr_19907/src/transformers/optimization.py#L50",returnDescription:`
<p><code>torch.optim.lr_scheduler.LambdaLR</code> with the appropriate schedule.</p>
`}}),et=new L({props:{name:"transformers.get_cosine_schedule_with_warmup",anchor:"transformers.get_cosine_schedule_with_warmup",parameters:[{name:"optimizer",val:": Optimizer"},{name:"num_warmup_steps",val:": int"},{name:"num_training_steps",val:": int"},{name:"num_cycles",val:": float = 0.5"},{name:"last_epoch",val:": int = -1"}],parametersDescription:[{anchor:"transformers.get_cosine_schedule_with_warmup.optimizer",description:`<strong>optimizer</strong> (<code>~torch.optim.Optimizer</code>) &#x2014;
The optimizer for which to schedule the learning rate.`,name:"optimizer"},{anchor:"transformers.get_cosine_schedule_with_warmup.num_warmup_steps",description:`<strong>num_warmup_steps</strong> (<code>int</code>) &#x2014;
The number of steps for the warmup phase.`,name:"num_warmup_steps"},{anchor:"transformers.get_cosine_schedule_with_warmup.num_training_steps",description:`<strong>num_training_steps</strong> (<code>int</code>) &#x2014;
The total number of training steps.`,name:"num_training_steps"},{anchor:"transformers.get_cosine_schedule_with_warmup.num_cycles",description:`<strong>num_cycles</strong> (<code>float</code>, <em>optional</em>, defaults to 0.5) &#x2014;
The number of waves in the cosine schedule (the defaults is to just decrease from the max value to 0
following a half-cosine).`,name:"num_cycles"},{anchor:"transformers.get_cosine_schedule_with_warmup.last_epoch",description:`<strong>last_epoch</strong> (<code>int</code>, <em>optional</em>, defaults to -1) &#x2014;
The index of the last epoch when resuming training.`,name:"last_epoch"}],source:"https://github.com/huggingface/transformers/blob/vr_19907/src/transformers/optimization.py#L104",returnDescription:`
<p><code>torch.optim.lr_scheduler.LambdaLR</code> with the appropriate schedule.</p>
`}}),rt=new L({props:{name:"transformers.get_cosine_with_hard_restarts_schedule_with_warmup",anchor:"transformers.get_cosine_with_hard_restarts_schedule_with_warmup",parameters:[{name:"optimizer",val:": Optimizer"},{name:"num_warmup_steps",val:": int"},{name:"num_training_steps",val:": int"},{name:"num_cycles",val:": int = 1"},{name:"last_epoch",val:": int = -1"}],parametersDescription:[{anchor:"transformers.get_cosine_with_hard_restarts_schedule_with_warmup.optimizer",description:`<strong>optimizer</strong> (<code>~torch.optim.Optimizer</code>) &#x2014;
The optimizer for which to schedule the learning rate.`,name:"optimizer"},{anchor:"transformers.get_cosine_with_hard_restarts_schedule_with_warmup.num_warmup_steps",description:`<strong>num_warmup_steps</strong> (<code>int</code>) &#x2014;
The number of steps for the warmup phase.`,name:"num_warmup_steps"},{anchor:"transformers.get_cosine_with_hard_restarts_schedule_with_warmup.num_training_steps",description:`<strong>num_training_steps</strong> (<code>int</code>) &#x2014;
The total number of training steps.`,name:"num_training_steps"},{anchor:"transformers.get_cosine_with_hard_restarts_schedule_with_warmup.num_cycles",description:`<strong>num_cycles</strong> (<code>int</code>, <em>optional</em>, defaults to 1) &#x2014;
The number of hard restarts to use.`,name:"num_cycles"},{anchor:"transformers.get_cosine_with_hard_restarts_schedule_with_warmup.last_epoch",description:`<strong>last_epoch</strong> (<code>int</code>, <em>optional</em>, defaults to -1) &#x2014;
The index of the last epoch when resuming training.`,name:"last_epoch"}],source:"https://github.com/huggingface/transformers/blob/vr_19907/src/transformers/optimization.py#L138",returnDescription:`
<p><code>torch.optim.lr_scheduler.LambdaLR</code> with the appropriate schedule.</p>
`}}),nt=new L({props:{name:"transformers.get_linear_schedule_with_warmup",anchor:"transformers.get_linear_schedule_with_warmup",parameters:[{name:"optimizer",val:""},{name:"num_warmup_steps",val:""},{name:"num_training_steps",val:""},{name:"last_epoch",val:" = -1"}],parametersDescription:[{anchor:"transformers.get_linear_schedule_with_warmup.optimizer",description:`<strong>optimizer</strong> (<code>~torch.optim.Optimizer</code>) &#x2014;
The optimizer for which to schedule the learning rate.`,name:"optimizer"},{anchor:"transformers.get_linear_schedule_with_warmup.num_warmup_steps",description:`<strong>num_warmup_steps</strong> (<code>int</code>) &#x2014;
The number of steps for the warmup phase.`,name:"num_warmup_steps"},{anchor:"transformers.get_linear_schedule_with_warmup.num_training_steps",description:`<strong>num_training_steps</strong> (<code>int</code>) &#x2014;
The total number of training steps.`,name:"num_training_steps"},{anchor:"transformers.get_linear_schedule_with_warmup.last_epoch",description:`<strong>last_epoch</strong> (<code>int</code>, <em>optional</em>, defaults to -1) &#x2014;
The index of the last epoch when resuming training.`,name:"last_epoch"}],source:"https://github.com/huggingface/transformers/blob/vr_19907/src/transformers/optimization.py#L75",returnDescription:`
<p><code>torch.optim.lr_scheduler.LambdaLR</code> with the appropriate schedule.</p>
`}}),st=new L({props:{name:"transformers.get_polynomial_decay_schedule_with_warmup",anchor:"transformers.get_polynomial_decay_schedule_with_warmup",parameters:[{name:"optimizer",val:""},{name:"num_warmup_steps",val:""},{name:"num_training_steps",val:""},{name:"lr_end",val:" = 1e-07"},{name:"power",val:" = 1.0"},{name:"last_epoch",val:" = -1"}],parametersDescription:[{anchor:"transformers.get_polynomial_decay_schedule_with_warmup.optimizer",description:`<strong>optimizer</strong> (<code>~torch.optim.Optimizer</code>) &#x2014;
The optimizer for which to schedule the learning rate.`,name:"optimizer"},{anchor:"transformers.get_polynomial_decay_schedule_with_warmup.num_warmup_steps",description:`<strong>num_warmup_steps</strong> (<code>int</code>) &#x2014;
The number of steps for the warmup phase.`,name:"num_warmup_steps"},{anchor:"transformers.get_polynomial_decay_schedule_with_warmup.num_training_steps",description:`<strong>num_training_steps</strong> (<code>int</code>) &#x2014;
The total number of training steps.`,name:"num_training_steps"},{anchor:"transformers.get_polynomial_decay_schedule_with_warmup.lr_end",description:`<strong>lr_end</strong> (<code>float</code>, <em>optional</em>, defaults to 1e-7) &#x2014;
The end LR.`,name:"lr_end"},{anchor:"transformers.get_polynomial_decay_schedule_with_warmup.power",description:`<strong>power</strong> (<code>float</code>, <em>optional</em>, defaults to 1.0) &#x2014;
Power factor.`,name:"power"},{anchor:"transformers.get_polynomial_decay_schedule_with_warmup.last_epoch",description:`<strong>last_epoch</strong> (<code>int</code>, <em>optional</em>, defaults to -1) &#x2014;
The index of the last epoch when resuming training.`,name:"last_epoch"}],source:"https://github.com/huggingface/transformers/blob/vr_19907/src/transformers/optimization.py#L173",returnDescription:`
<p><code>torch.optim.lr_scheduler.LambdaLR</code> with the appropriate schedule.</p>
`}}),mt=new ne({}),ct=new L({props:{name:"class transformers.WarmUp",anchor:"transformers.WarmUp",parameters:[{name:"initial_learning_rate",val:": float"},{name:"decay_schedule_fn",val:": typing.Callable"},{name:"warmup_steps",val:": int"},{name:"power",val:": float = 1.0"},{name:"name",val:": str = None"}],parametersDescription:[{anchor:"transformers.WarmUp.initial_learning_rate",description:`<strong>initial_learning_rate</strong> (<code>float</code>) &#x2014;
The initial learning rate for the schedule after the warmup (so this will be the learning rate at the end
of the warmup).`,name:"initial_learning_rate"},{anchor:"transformers.WarmUp.decay_schedule_fn",description:`<strong>decay_schedule_fn</strong> (<code>Callable</code>) &#x2014;
The schedule function to apply after the warmup for the rest of training.`,name:"decay_schedule_fn"},{anchor:"transformers.WarmUp.warmup_steps",description:`<strong>warmup_steps</strong> (<code>int</code>) &#x2014;
The number of steps for the warmup part of training.`,name:"warmup_steps"},{anchor:"transformers.WarmUp.power",description:`<strong>power</strong> (<code>float</code>, <em>optional</em>, defaults to 1) &#x2014;
The power to use for the polynomial warmup (defaults is a linear warmup).`,name:"power"},{anchor:"transformers.WarmUp.name",description:`<strong>name</strong> (<code>str</code>, <em>optional</em>) &#x2014;
Optional name prefix for the returned tensors during the schedule.`,name:"name"}],source:"https://github.com/huggingface/transformers/blob/vr_19907/src/transformers/optimization_tf.py#L24"}}),pt=new ne({}),dt=new ne({}),ht=new L({props:{name:"class transformers.GradientAccumulator",anchor:"transformers.GradientAccumulator",parameters:[],source:"https://github.com/huggingface/transformers/blob/vr_19907/src/transformers/optimization_tf.py#L296"}}),ut=new L({props:{name:"reset",anchor:"transformers.GradientAccumulator.reset",parameters:[],source:"https://github.com/huggingface/transformers/blob/vr_19907/src/transformers/optimization_tf.py#L358"}}),{c(){f=a("meta"),T=l(),x=a("h1"),u=a("a"),z=a("span"),g(p.$$.fragment),$=l(),zt=a("span"),ja=s("Optimization"),kr=l(),oe=a("p"),qa=s("The "),Et=a("code"),Ua=s(".optimization"),Ga=s(" module provides:"),Sr=l(),F=a("ul"),Tt=a("li"),Va=s("an optimizer with weight decay fixed that can be used to fine-tuned models, and"),Ma=l(),ze=a("li"),Ha=s("several schedules in the form of schedule objects that inherit from "),Dt=a("code"),Ba=s("_LRSchedule"),Ja=s(":"),Ka=l(),Lt=a("li"),Qa=s("a gradient accumulation class to accumulate the gradients of multiple batches"),Nr=l(),R=a("h2"),se=a("a"),Pt=a("span"),g(Ee.$$.fragment),Xa=l(),Wt=a("span"),Ya=s("AdamW (PyTorch)"),Ir=l(),k=a("div"),g(Te.$$.fragment),Za=l(),De=a("p"),en=s("Implements Adam algorithm with weight decay fix as introduced in "),Le=a("a"),tn=s(`Decoupled Weight Decay
Regularization`),rn=s("."),an=l(),ie=a("div"),g(Pe.$$.fragment),nn=l(),kt=a("p"),on=s("Performs a single optimization step."),Cr=l(),j=a("h2"),le=a("a"),St=a("span"),g(We.$$.fragment),sn=l(),Nt=a("span"),ln=s("AdaFactor (PyTorch)"),Fr=l(),A=a("div"),g(ke.$$.fragment),mn=l(),_t=a("p"),cn=s(`AdaFactor pytorch implementation can be used as a drop in replacement for Adam original fairseq code:
`),Se=a("a"),pn=s("https://github.com/pytorch/fairseq/blob/master/fairseq/optim/adafactor.py"),dn=l(),D=a("p"),hn=s("Paper: "),It=a("em"),un=s("Adafactor: Adaptive Learning Rates with Sublinear Memory Cost"),fn=l(),Ne=a("a"),gn=s("https://arxiv.org/abs/1804.04235"),_n=s(` Note that
this optimizer internally adjusts the learning rate depending on the `),Ct=a("code"),wn=s("scale_parameter"),vn=s(", "),Ft=a("code"),bn=s("relative_step"),yn=s(` and
`),Ot=a("code"),$n=s("warmup_init"),xn=s(" options. To use a manual (external) learning rate schedule you should set "),Rt=a("code"),An=s("scale_parameter=False"),zn=s(` and
`),jt=a("code"),En=s("relative_step=False"),Tn=s("."),Dn=l(),qt=a("p"),Ln=s("This implementation handles low-precision (FP16, bfloat) values, but we have not thoroughly tested."),Pn=l(),Ie=a("p"),Wn=s("Recommended T5 finetuning settings ("),Ce=a("a"),kn=s("https://discuss.huggingface.co/t/t5-finetuning-tips/684/3"),Sn=s("):"),Nn=l(),S=a("ul"),Fe=a("li"),Ut=a("p"),In=s("Training without LR warmup or clip_threshold is not recommended."),Cn=l(),Oe=a("ul"),Gt=a("li"),Fn=s("use scheduled LR warm-up to fixed LR"),On=l(),Re=a("li"),Rn=s("use clip_threshold=1.0 ("),je=a("a"),jn=s("https://arxiv.org/abs/1804.04235"),qn=s(")"),Un=l(),Vt=a("li"),Mt=a("p"),Gn=s("Disable relative updates"),Vn=l(),Ht=a("li"),Bt=a("p"),Mn=s("Use scale_parameter=False"),Hn=l(),Jt=a("li"),Kt=a("p"),Bn=s("Additional optimizer operations like gradient clipping should not be used alongside Adafactor"),Jn=l(),g(me.$$.fragment),Kn=l(),g(ce.$$.fragment),Qn=l(),O=a("p"),Xn=s("When using "),Qt=a("code"),Yn=s("lr=None"),Zn=s(" with "),wt=a("a"),eo=s("Trainer"),to=s(" you will most likely need to use "),Xt=a("code"),ro=s("AdafactorSchedule"),ao=l(),g(pe.$$.fragment),no=l(),g(de.$$.fragment),oo=l(),he=a("div"),g(qe.$$.fragment),so=l(),Yt=a("p"),io=s("Performs a single optimization step"),Or=l(),q=a("h2"),ue=a("a"),Zt=a("span"),g(Ue.$$.fragment),lo=l(),er=a("span"),mo=s("AdamWeightDecay (TensorFlow)"),Rr=l(),W=a("div"),g(Ge.$$.fragment),co=l(),U=a("p"),po=s(`Adam enables L2 weight decay and clip_by_global_norm on gradients. Just adding the square of the weights to the
loss function is `),tr=a("em"),ho=s("not"),uo=s(` the correct way of using L2 regularization/weight decay with Adam, since that will interact
with the m and v parameters in strange ways as shown in `),Ve=a("a"),fo=s(`Decoupled Weight Decay
Regularization`),go=s("."),_o=l(),rr=a("p"),wo=s(`Instead we want ot decay the weights in a manner that doesn\u2019t interact with the m/v parameters. This is equivalent
to adding the square of the weights to the loss with plain (non-momentum) SGD.`),vo=l(),fe=a("div"),g(Me.$$.fragment),bo=l(),ar=a("p"),yo=s("Creates an optimizer from its config with WarmUp custom object."),jr=l(),G=a("div"),g(He.$$.fragment),$o=l(),nr=a("p"),xo=s("Creates an optimizer with a learning rate schedule using a warmup phase followed by a linear decay."),qr=l(),V=a("h2"),ge=a("a"),or=a("span"),g(Be.$$.fragment),Ao=l(),sr=a("span"),zo=s("Schedules"),Ur=l(),M=a("h3"),_e=a("a"),ir=a("span"),g(Je.$$.fragment),Eo=l(),lr=a("span"),To=s("Learning Rate Schedules (Pytorch)"),Gr=l(),H=a("div"),g(Ke.$$.fragment),Do=l(),mr=a("p"),Lo=s("An enumeration."),Vr=l(),B=a("div"),g(Qe.$$.fragment),Po=l(),cr=a("p"),Wo=s("Unified API to get any scheduler from its name."),Mr=l(),J=a("div"),g(Xe.$$.fragment),ko=l(),pr=a("p"),So=s("Create a schedule with a constant learning rate, using the learning rate set in optimizer."),Hr=l(),K=a("div"),g(Ye.$$.fragment),No=l(),dr=a("p"),Io=s(`Create a schedule with a constant learning rate preceded by a warmup period during which the learning rate
increases linearly between 0 and the initial lr set in the optimizer.`),Br=l(),Ze=a("img"),Jr=l(),Q=a("div"),g(et.$$.fragment),Co=l(),hr=a("p"),Fo=s(`Create a schedule with a learning rate that decreases following the values of the cosine function between the
initial lr set in the optimizer to 0, after a warmup period during which it increases linearly between 0 and the
initial lr set in the optimizer.`),Kr=l(),tt=a("img"),Qr=l(),X=a("div"),g(rt.$$.fragment),Oo=l(),ur=a("p"),Ro=s(`Create a schedule with a learning rate that decreases following the values of the cosine function between the
initial lr set in the optimizer to 0, with several hard restarts, after a warmup period during which it increases
linearly between 0 and the initial lr set in the optimizer.`),Xr=l(),at=a("img"),Yr=l(),Y=a("div"),g(nt.$$.fragment),jo=l(),fr=a("p"),qo=s(`Create a schedule with a learning rate that decreases linearly from the initial lr set in the optimizer to 0, after
a warmup period during which it increases linearly from 0 to the initial lr set in the optimizer.`),Zr=l(),ot=a("img"),ea=l(),N=a("div"),g(st.$$.fragment),Uo=l(),it=a("p"),Go=s(`Create a schedule with a learning rate that decreases as a polynomial decay from the initial lr set in the
optimizer to end lr defined by `),gr=a("em"),Vo=s("lr_end"),Mo=s(`, after a warmup period during which it increases linearly from 0 to the
initial lr set in the optimizer.`),Ho=l(),we=a("p"),Bo=s("Note: "),_r=a("em"),Jo=s("power"),Ko=s(` defaults to 1.0 as in the fairseq implementation, which in turn is based on the original BERT
implementation at
`),lt=a("a"),Qo=s("https://github.com/google-research/bert/blob/f39e881b169b9d53bea03d2d341b31707a6c052b/optimization.py#L37"),ta=l(),Z=a("h3"),ve=a("a"),wr=a("span"),g(mt.$$.fragment),Xo=l(),vr=a("span"),Yo=s("Warmup (TensorFlow)"),ra=l(),ee=a("div"),g(ct.$$.fragment),Zo=l(),br=a("p"),es=s("Applies a warmup schedule on a given learning rate decay schedule."),aa=l(),te=a("h2"),be=a("a"),yr=a("span"),g(pt.$$.fragment),ts=l(),$r=a("span"),rs=s("Gradient Strategies"),na=l(),re=a("h3"),ye=a("a"),xr=a("span"),g(dt.$$.fragment),as=l(),Ar=a("span"),ns=s("GradientAccumulator (TensorFlow)"),oa=l(),I=a("div"),g(ht.$$.fragment),os=l(),ae=a("p"),ss=s(`Gradient accumulation utility. When used with a distribution strategy, the accumulator should be called in a
replica context. Gradients will be accumulated locally on each replica and without synchronization. Users should
then call `),zr=a("code"),is=s(".gradients"),ls=s(", scale the gradients if required, and pass the result to "),Er=a("code"),ms=s("apply_gradients"),cs=s("."),ps=l(),$e=a("div"),g(ut.$$.fragment),ds=l(),Tr=a("p"),hs=s("Resets the accumulated gradients on the current replica."),this.h()},l(e){const d=ji('[data-svelte="svelte-1phssyn"]',document.head);f=n(d,"META",{name:!0,content:!0}),d.forEach(r),T=m(e),x=n(e,"H1",{class:!0});var ft=o(x);u=n(ft,"A",{id:!0,class:!0,href:!0});var Dr=o(u);z=n(Dr,"SPAN",{});var Lr=o(z);_(p.$$.fragment,Lr),Lr.forEach(r),Dr.forEach(r),$=m(ft),zt=n(ft,"SPAN",{});var Pr=o(zt);ja=i(Pr,"Optimization"),Pr.forEach(r),ft.forEach(r),kr=m(e),oe=n(e,"P",{});var ia=o(oe);qa=i(ia,"The "),Et=n(ia,"CODE",{});var vs=o(Et);Ua=i(vs,".optimization"),vs.forEach(r),Ga=i(ia," module provides:"),ia.forEach(r),Sr=m(e),F=n(e,"UL",{});var vt=o(F);Tt=n(vt,"LI",{});var bs=o(Tt);Va=i(bs,"an optimizer with weight decay fixed that can be used to fine-tuned models, and"),bs.forEach(r),Ma=m(vt),ze=n(vt,"LI",{});var la=o(ze);Ha=i(la,"several schedules in the form of schedule objects that inherit from "),Dt=n(la,"CODE",{});var ys=o(Dt);Ba=i(ys,"_LRSchedule"),ys.forEach(r),Ja=i(la,":"),la.forEach(r),Ka=m(vt),Lt=n(vt,"LI",{});var $s=o(Lt);Qa=i($s,"a gradient accumulation class to accumulate the gradients of multiple batches"),$s.forEach(r),vt.forEach(r),Nr=m(e),R=n(e,"H2",{class:!0});var ma=o(R);se=n(ma,"A",{id:!0,class:!0,href:!0});var xs=o(se);Pt=n(xs,"SPAN",{});var As=o(Pt);_(Ee.$$.fragment,As),As.forEach(r),xs.forEach(r),Xa=m(ma),Wt=n(ma,"SPAN",{});var zs=o(Wt);Ya=i(zs,"AdamW (PyTorch)"),zs.forEach(r),ma.forEach(r),Ir=m(e),k=n(e,"DIV",{class:!0});var bt=o(k);_(Te.$$.fragment,bt),Za=m(bt),De=n(bt,"P",{});var ca=o(De);en=i(ca,"Implements Adam algorithm with weight decay fix as introduced in "),Le=n(ca,"A",{href:!0,rel:!0});var Es=o(Le);tn=i(Es,`Decoupled Weight Decay
Regularization`),Es.forEach(r),rn=i(ca,"."),ca.forEach(r),an=m(bt),ie=n(bt,"DIV",{class:!0});var pa=o(ie);_(Pe.$$.fragment,pa),nn=m(pa),kt=n(pa,"P",{});var Ts=o(kt);on=i(Ts,"Performs a single optimization step."),Ts.forEach(r),pa.forEach(r),bt.forEach(r),Cr=m(e),j=n(e,"H2",{class:!0});var da=o(j);le=n(da,"A",{id:!0,class:!0,href:!0});var Ds=o(le);St=n(Ds,"SPAN",{});var Ls=o(St);_(We.$$.fragment,Ls),Ls.forEach(r),Ds.forEach(r),sn=m(da),Nt=n(da,"SPAN",{});var Ps=o(Nt);ln=i(Ps,"AdaFactor (PyTorch)"),Ps.forEach(r),da.forEach(r),Fr=m(e),A=n(e,"DIV",{class:!0});var E=o(A);_(ke.$$.fragment,E),mn=m(E),_t=n(E,"P",{});var us=o(_t);cn=i(us,`AdaFactor pytorch implementation can be used as a drop in replacement for Adam original fairseq code:
`),Se=n(us,"A",{href:!0,rel:!0});var Ws=o(Se);pn=i(Ws,"https://github.com/pytorch/fairseq/blob/master/fairseq/optim/adafactor.py"),Ws.forEach(r),us.forEach(r),dn=m(E),D=n(E,"P",{});var P=o(D);hn=i(P,"Paper: "),It=n(P,"EM",{});var ks=o(It);un=i(ks,"Adafactor: Adaptive Learning Rates with Sublinear Memory Cost"),ks.forEach(r),fn=m(P),Ne=n(P,"A",{href:!0,rel:!0});var Ss=o(Ne);gn=i(Ss,"https://arxiv.org/abs/1804.04235"),Ss.forEach(r),_n=i(P,` Note that
this optimizer internally adjusts the learning rate depending on the `),Ct=n(P,"CODE",{});var Ns=o(Ct);wn=i(Ns,"scale_parameter"),Ns.forEach(r),vn=i(P,", "),Ft=n(P,"CODE",{});var Is=o(Ft);bn=i(Is,"relative_step"),Is.forEach(r),yn=i(P,` and
`),Ot=n(P,"CODE",{});var Cs=o(Ot);$n=i(Cs,"warmup_init"),Cs.forEach(r),xn=i(P," options. To use a manual (external) learning rate schedule you should set "),Rt=n(P,"CODE",{});var Fs=o(Rt);An=i(Fs,"scale_parameter=False"),Fs.forEach(r),zn=i(P,` and
`),jt=n(P,"CODE",{});var Os=o(jt);En=i(Os,"relative_step=False"),Os.forEach(r),Tn=i(P,"."),P.forEach(r),Dn=m(E),qt=n(E,"P",{});var Rs=o(qt);Ln=i(Rs,"This implementation handles low-precision (FP16, bfloat) values, but we have not thoroughly tested."),Rs.forEach(r),Pn=m(E),Ie=n(E,"P",{});var ha=o(Ie);Wn=i(ha,"Recommended T5 finetuning settings ("),Ce=n(ha,"A",{href:!0,rel:!0});var js=o(Ce);kn=i(js,"https://discuss.huggingface.co/t/t5-finetuning-tips/684/3"),js.forEach(r),Sn=i(ha,"):"),ha.forEach(r),Nn=m(E),S=n(E,"UL",{});var xe=o(S);Fe=n(xe,"LI",{});var ua=o(Fe);Ut=n(ua,"P",{});var qs=o(Ut);In=i(qs,"Training without LR warmup or clip_threshold is not recommended."),qs.forEach(r),Cn=m(ua),Oe=n(ua,"UL",{});var fa=o(Oe);Gt=n(fa,"LI",{});var Us=o(Gt);Fn=i(Us,"use scheduled LR warm-up to fixed LR"),Us.forEach(r),On=m(fa),Re=n(fa,"LI",{});var ga=o(Re);Rn=i(ga,"use clip_threshold=1.0 ("),je=n(ga,"A",{href:!0,rel:!0});var Gs=o(je);jn=i(Gs,"https://arxiv.org/abs/1804.04235"),Gs.forEach(r),qn=i(ga,")"),ga.forEach(r),fa.forEach(r),ua.forEach(r),Un=m(xe),Vt=n(xe,"LI",{});var Vs=o(Vt);Mt=n(Vs,"P",{});var Ms=o(Mt);Gn=i(Ms,"Disable relative updates"),Ms.forEach(r),Vs.forEach(r),Vn=m(xe),Ht=n(xe,"LI",{});var Hs=o(Ht);Bt=n(Hs,"P",{});var Bs=o(Bt);Mn=i(Bs,"Use scale_parameter=False"),Bs.forEach(r),Hs.forEach(r),Hn=m(xe),Jt=n(xe,"LI",{});var Js=o(Jt);Kt=n(Js,"P",{});var Ks=o(Kt);Bn=i(Ks,"Additional optimizer operations like gradient clipping should not be used alongside Adafactor"),Ks.forEach(r),Js.forEach(r),xe.forEach(r),Jn=m(E),_(me.$$.fragment,E),Kn=m(E),_(ce.$$.fragment,E),Qn=m(E),O=n(E,"P",{});var gt=o(O);Xn=i(gt,"When using "),Qt=n(gt,"CODE",{});var Qs=o(Qt);Yn=i(Qs,"lr=None"),Qs.forEach(r),Zn=i(gt," with "),wt=n(gt,"A",{href:!0});var Xs=o(wt);eo=i(Xs,"Trainer"),Xs.forEach(r),to=i(gt," you will most likely need to use "),Xt=n(gt,"CODE",{});var Ys=o(Xt);ro=i(Ys,"AdafactorSchedule"),Ys.forEach(r),gt.forEach(r),ao=m(E),_(pe.$$.fragment,E),no=m(E),_(de.$$.fragment,E),oo=m(E),he=n(E,"DIV",{class:!0});var _a=o(he);_(qe.$$.fragment,_a),so=m(_a),Yt=n(_a,"P",{});var Zs=o(Yt);io=i(Zs,"Performs a single optimization step"),Zs.forEach(r),_a.forEach(r),E.forEach(r),Or=m(e),q=n(e,"H2",{class:!0});var wa=o(q);ue=n(wa,"A",{id:!0,class:!0,href:!0});var ei=o(ue);Zt=n(ei,"SPAN",{});var ti=o(Zt);_(Ue.$$.fragment,ti),ti.forEach(r),ei.forEach(r),lo=m(wa),er=n(wa,"SPAN",{});var ri=o(er);mo=i(ri,"AdamWeightDecay (TensorFlow)"),ri.forEach(r),wa.forEach(r),Rr=m(e),W=n(e,"DIV",{class:!0});var Ae=o(W);_(Ge.$$.fragment,Ae),co=m(Ae),U=n(Ae,"P",{});var yt=o(U);po=i(yt,`Adam enables L2 weight decay and clip_by_global_norm on gradients. Just adding the square of the weights to the
loss function is `),tr=n(yt,"EM",{});var ai=o(tr);ho=i(ai,"not"),ai.forEach(r),uo=i(yt,` the correct way of using L2 regularization/weight decay with Adam, since that will interact
with the m and v parameters in strange ways as shown in `),Ve=n(yt,"A",{href:!0,rel:!0});var ni=o(Ve);fo=i(ni,`Decoupled Weight Decay
Regularization`),ni.forEach(r),go=i(yt,"."),yt.forEach(r),_o=m(Ae),rr=n(Ae,"P",{});var oi=o(rr);wo=i(oi,`Instead we want ot decay the weights in a manner that doesn\u2019t interact with the m/v parameters. This is equivalent
to adding the square of the weights to the loss with plain (non-momentum) SGD.`),oi.forEach(r),vo=m(Ae),fe=n(Ae,"DIV",{class:!0});var va=o(fe);_(Me.$$.fragment,va),bo=m(va),ar=n(va,"P",{});var si=o(ar);yo=i(si,"Creates an optimizer from its config with WarmUp custom object."),si.forEach(r),va.forEach(r),Ae.forEach(r),jr=m(e),G=n(e,"DIV",{class:!0});var ba=o(G);_(He.$$.fragment,ba),$o=m(ba),nr=n(ba,"P",{});var ii=o(nr);xo=i(ii,"Creates an optimizer with a learning rate schedule using a warmup phase followed by a linear decay."),ii.forEach(r),ba.forEach(r),qr=m(e),V=n(e,"H2",{class:!0});var ya=o(V);ge=n(ya,"A",{id:!0,class:!0,href:!0});var li=o(ge);or=n(li,"SPAN",{});var mi=o(or);_(Be.$$.fragment,mi),mi.forEach(r),li.forEach(r),Ao=m(ya),sr=n(ya,"SPAN",{});var ci=o(sr);zo=i(ci,"Schedules"),ci.forEach(r),ya.forEach(r),Ur=m(e),M=n(e,"H3",{class:!0});var $a=o(M);_e=n($a,"A",{id:!0,class:!0,href:!0});var pi=o(_e);ir=n(pi,"SPAN",{});var di=o(ir);_(Je.$$.fragment,di),di.forEach(r),pi.forEach(r),Eo=m($a),lr=n($a,"SPAN",{});var hi=o(lr);To=i(hi,"Learning Rate Schedules (Pytorch)"),hi.forEach(r),$a.forEach(r),Gr=m(e),H=n(e,"DIV",{class:!0});var xa=o(H);_(Ke.$$.fragment,xa),Do=m(xa),mr=n(xa,"P",{});var ui=o(mr);Lo=i(ui,"An enumeration."),ui.forEach(r),xa.forEach(r),Vr=m(e),B=n(e,"DIV",{class:!0});var Aa=o(B);_(Qe.$$.fragment,Aa),Po=m(Aa),cr=n(Aa,"P",{});var fi=o(cr);Wo=i(fi,"Unified API to get any scheduler from its name."),fi.forEach(r),Aa.forEach(r),Mr=m(e),J=n(e,"DIV",{class:!0});var za=o(J);_(Xe.$$.fragment,za),ko=m(za),pr=n(za,"P",{});var gi=o(pr);So=i(gi,"Create a schedule with a constant learning rate, using the learning rate set in optimizer."),gi.forEach(r),za.forEach(r),Hr=m(e),K=n(e,"DIV",{class:!0});var Ea=o(K);_(Ye.$$.fragment,Ea),No=m(Ea),dr=n(Ea,"P",{});var _i=o(dr);Io=i(_i,`Create a schedule with a constant learning rate preceded by a warmup period during which the learning rate
increases linearly between 0 and the initial lr set in the optimizer.`),_i.forEach(r),Ea.forEach(r),Br=m(e),Ze=n(e,"IMG",{alt:!0,src:!0}),Jr=m(e),Q=n(e,"DIV",{class:!0});var Ta=o(Q);_(et.$$.fragment,Ta),Co=m(Ta),hr=n(Ta,"P",{});var wi=o(hr);Fo=i(wi,`Create a schedule with a learning rate that decreases following the values of the cosine function between the
initial lr set in the optimizer to 0, after a warmup period during which it increases linearly between 0 and the
initial lr set in the optimizer.`),wi.forEach(r),Ta.forEach(r),Kr=m(e),tt=n(e,"IMG",{alt:!0,src:!0}),Qr=m(e),X=n(e,"DIV",{class:!0});var Da=o(X);_(rt.$$.fragment,Da),Oo=m(Da),ur=n(Da,"P",{});var vi=o(ur);Ro=i(vi,`Create a schedule with a learning rate that decreases following the values of the cosine function between the
initial lr set in the optimizer to 0, with several hard restarts, after a warmup period during which it increases
linearly between 0 and the initial lr set in the optimizer.`),vi.forEach(r),Da.forEach(r),Xr=m(e),at=n(e,"IMG",{alt:!0,src:!0}),Yr=m(e),Y=n(e,"DIV",{class:!0});var La=o(Y);_(nt.$$.fragment,La),jo=m(La),fr=n(La,"P",{});var bi=o(fr);qo=i(bi,`Create a schedule with a learning rate that decreases linearly from the initial lr set in the optimizer to 0, after
a warmup period during which it increases linearly from 0 to the initial lr set in the optimizer.`),bi.forEach(r),La.forEach(r),Zr=m(e),ot=n(e,"IMG",{alt:!0,src:!0}),ea=m(e),N=n(e,"DIV",{class:!0});var $t=o(N);_(st.$$.fragment,$t),Uo=m($t),it=n($t,"P",{});var Pa=o(it);Go=i(Pa,`Create a schedule with a learning rate that decreases as a polynomial decay from the initial lr set in the
optimizer to end lr defined by `),gr=n(Pa,"EM",{});var yi=o(gr);Vo=i(yi,"lr_end"),yi.forEach(r),Mo=i(Pa,`, after a warmup period during which it increases linearly from 0 to the
initial lr set in the optimizer.`),Pa.forEach(r),Ho=m($t),we=n($t,"P",{});var Wr=o(we);Bo=i(Wr,"Note: "),_r=n(Wr,"EM",{});var $i=o(_r);Jo=i($i,"power"),$i.forEach(r),Ko=i(Wr,` defaults to 1.0 as in the fairseq implementation, which in turn is based on the original BERT
implementation at
`),lt=n(Wr,"A",{href:!0,rel:!0});var xi=o(lt);Qo=i(xi,"https://github.com/google-research/bert/blob/f39e881b169b9d53bea03d2d341b31707a6c052b/optimization.py#L37"),xi.forEach(r),Wr.forEach(r),$t.forEach(r),ta=m(e),Z=n(e,"H3",{class:!0});var Wa=o(Z);ve=n(Wa,"A",{id:!0,class:!0,href:!0});var Ai=o(ve);wr=n(Ai,"SPAN",{});var zi=o(wr);_(mt.$$.fragment,zi),zi.forEach(r),Ai.forEach(r),Xo=m(Wa),vr=n(Wa,"SPAN",{});var Ei=o(vr);Yo=i(Ei,"Warmup (TensorFlow)"),Ei.forEach(r),Wa.forEach(r),ra=m(e),ee=n(e,"DIV",{class:!0});var ka=o(ee);_(ct.$$.fragment,ka),Zo=m(ka),br=n(ka,"P",{});var Ti=o(br);es=i(Ti,"Applies a warmup schedule on a given learning rate decay schedule."),Ti.forEach(r),ka.forEach(r),aa=m(e),te=n(e,"H2",{class:!0});var Sa=o(te);be=n(Sa,"A",{id:!0,class:!0,href:!0});var Di=o(be);yr=n(Di,"SPAN",{});var Li=o(yr);_(pt.$$.fragment,Li),Li.forEach(r),Di.forEach(r),ts=m(Sa),$r=n(Sa,"SPAN",{});var Pi=o($r);rs=i(Pi,"Gradient Strategies"),Pi.forEach(r),Sa.forEach(r),na=m(e),re=n(e,"H3",{class:!0});var Na=o(re);ye=n(Na,"A",{id:!0,class:!0,href:!0});var Wi=o(ye);xr=n(Wi,"SPAN",{});var ki=o(xr);_(dt.$$.fragment,ki),ki.forEach(r),Wi.forEach(r),as=m(Na),Ar=n(Na,"SPAN",{});var Si=o(Ar);ns=i(Si,"GradientAccumulator (TensorFlow)"),Si.forEach(r),Na.forEach(r),oa=m(e),I=n(e,"DIV",{class:!0});var xt=o(I);_(ht.$$.fragment,xt),os=m(xt),ae=n(xt,"P",{});var At=o(ae);ss=i(At,`Gradient accumulation utility. When used with a distribution strategy, the accumulator should be called in a
replica context. Gradients will be accumulated locally on each replica and without synchronization. Users should
then call `),zr=n(At,"CODE",{});var Ni=o(zr);is=i(Ni,".gradients"),Ni.forEach(r),ls=i(At,", scale the gradients if required, and pass the result to "),Er=n(At,"CODE",{});var Ii=o(Er);ms=i(Ii,"apply_gradients"),Ii.forEach(r),cs=i(At,"."),At.forEach(r),ps=m(xt),$e=n(xt,"DIV",{class:!0});var Ia=o($e);_(ut.$$.fragment,Ia),ds=m(Ia),Tr=n(Ia,"P",{});var Ci=o(Tr);hs=i(Ci,"Resets the accumulated gradients on the current replica."),Ci.forEach(r),Ia.forEach(r),xt.forEach(r),this.h()},h(){c(f,"name","hf:doc:metadata"),c(f,"content",JSON.stringify(Bi)),c(u,"id","optimization"),c(u,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),c(u,"href","#optimization"),c(x,"class","relative group"),c(se,"id","transformers.AdamW"),c(se,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),c(se,"href","#transformers.AdamW"),c(R,"class","relative group"),c(Le,"href","https://arxiv.org/abs/1711.05101"),c(Le,"rel","nofollow"),c(ie,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),c(k,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),c(le,"id","transformers.Adafactor"),c(le,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),c(le,"href","#transformers.Adafactor"),c(j,"class","relative group"),c(Se,"href","https://github.com/pytorch/fairseq/blob/master/fairseq/optim/adafactor.py"),c(Se,"rel","nofollow"),c(Ne,"href","https://arxiv.org/abs/1804.04235"),c(Ne,"rel","nofollow"),c(Ce,"href","https://discuss.huggingface.co/t/t5-finetuning-tips/684/3"),c(Ce,"rel","nofollow"),c(je,"href","https://arxiv.org/abs/1804.04235"),c(je,"rel","nofollow"),c(wt,"href","/docs/transformers/pr_19907/en/main_classes/trainer#transformers.Trainer"),c(he,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),c(A,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),c(ue,"id","transformers.AdamWeightDecay"),c(ue,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),c(ue,"href","#transformers.AdamWeightDecay"),c(q,"class","relative group"),c(Ve,"href","https://arxiv.org/abs/1711.05101"),c(Ve,"rel","nofollow"),c(fe,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),c(W,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),c(G,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),c(ge,"id","schedules"),c(ge,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),c(ge,"href","#schedules"),c(V,"class","relative group"),c(_e,"id","transformers.SchedulerType"),c(_e,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),c(_e,"href","#transformers.SchedulerType"),c(M,"class","relative group"),c(H,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),c(B,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),c(J,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),c(K,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),c(Ze,"alt",""),Ca(Ze.src,fs="https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/warmup_constant_schedule.png")||c(Ze,"src",fs),c(Q,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),c(tt,"alt",""),Ca(tt.src,gs="https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/warmup_cosine_schedule.png")||c(tt,"src",gs),c(X,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),c(at,"alt",""),Ca(at.src,_s="https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/warmup_cosine_hard_restarts_schedule.png")||c(at,"src",_s),c(Y,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),c(ot,"alt",""),Ca(ot.src,ws="https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/warmup_linear_schedule.png")||c(ot,"src",ws),c(lt,"href","https://github.com/google-research/bert/blob/f39e881b169b9d53bea03d2d341b31707a6c052b/optimization.py#L37"),c(lt,"rel","nofollow"),c(N,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),c(ve,"id","transformers.WarmUp"),c(ve,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),c(ve,"href","#transformers.WarmUp"),c(Z,"class","relative group"),c(ee,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),c(be,"id","gradient-strategies"),c(be,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),c(be,"href","#gradient-strategies"),c(te,"class","relative group"),c(ye,"id","transformers.GradientAccumulator"),c(ye,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),c(ye,"href","#transformers.GradientAccumulator"),c(re,"class","relative group"),c($e,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),c(I,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8")},m(e,d){t(document.head,f),h(e,T,d),h(e,x,d),t(x,u),t(u,z),w(p,z,null),t(x,$),t(x,zt),t(zt,ja),h(e,kr,d),h(e,oe,d),t(oe,qa),t(oe,Et),t(Et,Ua),t(oe,Ga),h(e,Sr,d),h(e,F,d),t(F,Tt),t(Tt,Va),t(F,Ma),t(F,ze),t(ze,Ha),t(ze,Dt),t(Dt,Ba),t(ze,Ja),t(F,Ka),t(F,Lt),t(Lt,Qa),h(e,Nr,d),h(e,R,d),t(R,se),t(se,Pt),w(Ee,Pt,null),t(R,Xa),t(R,Wt),t(Wt,Ya),h(e,Ir,d),h(e,k,d),w(Te,k,null),t(k,Za),t(k,De),t(De,en),t(De,Le),t(Le,tn),t(De,rn),t(k,an),t(k,ie),w(Pe,ie,null),t(ie,nn),t(ie,kt),t(kt,on),h(e,Cr,d),h(e,j,d),t(j,le),t(le,St),w(We,St,null),t(j,sn),t(j,Nt),t(Nt,ln),h(e,Fr,d),h(e,A,d),w(ke,A,null),t(A,mn),t(A,_t),t(_t,cn),t(_t,Se),t(Se,pn),t(A,dn),t(A,D),t(D,hn),t(D,It),t(It,un),t(D,fn),t(D,Ne),t(Ne,gn),t(D,_n),t(D,Ct),t(Ct,wn),t(D,vn),t(D,Ft),t(Ft,bn),t(D,yn),t(D,Ot),t(Ot,$n),t(D,xn),t(D,Rt),t(Rt,An),t(D,zn),t(D,jt),t(jt,En),t(D,Tn),t(A,Dn),t(A,qt),t(qt,Ln),t(A,Pn),t(A,Ie),t(Ie,Wn),t(Ie,Ce),t(Ce,kn),t(Ie,Sn),t(A,Nn),t(A,S),t(S,Fe),t(Fe,Ut),t(Ut,In),t(Fe,Cn),t(Fe,Oe),t(Oe,Gt),t(Gt,Fn),t(Oe,On),t(Oe,Re),t(Re,Rn),t(Re,je),t(je,jn),t(Re,qn),t(S,Un),t(S,Vt),t(Vt,Mt),t(Mt,Gn),t(S,Vn),t(S,Ht),t(Ht,Bt),t(Bt,Mn),t(S,Hn),t(S,Jt),t(Jt,Kt),t(Kt,Bn),t(A,Jn),w(me,A,null),t(A,Kn),w(ce,A,null),t(A,Qn),t(A,O),t(O,Xn),t(O,Qt),t(Qt,Yn),t(O,Zn),t(O,wt),t(wt,eo),t(O,to),t(O,Xt),t(Xt,ro),t(A,ao),w(pe,A,null),t(A,no),w(de,A,null),t(A,oo),t(A,he),w(qe,he,null),t(he,so),t(he,Yt),t(Yt,io),h(e,Or,d),h(e,q,d),t(q,ue),t(ue,Zt),w(Ue,Zt,null),t(q,lo),t(q,er),t(er,mo),h(e,Rr,d),h(e,W,d),w(Ge,W,null),t(W,co),t(W,U),t(U,po),t(U,tr),t(tr,ho),t(U,uo),t(U,Ve),t(Ve,fo),t(U,go),t(W,_o),t(W,rr),t(rr,wo),t(W,vo),t(W,fe),w(Me,fe,null),t(fe,bo),t(fe,ar),t(ar,yo),h(e,jr,d),h(e,G,d),w(He,G,null),t(G,$o),t(G,nr),t(nr,xo),h(e,qr,d),h(e,V,d),t(V,ge),t(ge,or),w(Be,or,null),t(V,Ao),t(V,sr),t(sr,zo),h(e,Ur,d),h(e,M,d),t(M,_e),t(_e,ir),w(Je,ir,null),t(M,Eo),t(M,lr),t(lr,To),h(e,Gr,d),h(e,H,d),w(Ke,H,null),t(H,Do),t(H,mr),t(mr,Lo),h(e,Vr,d),h(e,B,d),w(Qe,B,null),t(B,Po),t(B,cr),t(cr,Wo),h(e,Mr,d),h(e,J,d),w(Xe,J,null),t(J,ko),t(J,pr),t(pr,So),h(e,Hr,d),h(e,K,d),w(Ye,K,null),t(K,No),t(K,dr),t(dr,Io),h(e,Br,d),h(e,Ze,d),h(e,Jr,d),h(e,Q,d),w(et,Q,null),t(Q,Co),t(Q,hr),t(hr,Fo),h(e,Kr,d),h(e,tt,d),h(e,Qr,d),h(e,X,d),w(rt,X,null),t(X,Oo),t(X,ur),t(ur,Ro),h(e,Xr,d),h(e,at,d),h(e,Yr,d),h(e,Y,d),w(nt,Y,null),t(Y,jo),t(Y,fr),t(fr,qo),h(e,Zr,d),h(e,ot,d),h(e,ea,d),h(e,N,d),w(st,N,null),t(N,Uo),t(N,it),t(it,Go),t(it,gr),t(gr,Vo),t(it,Mo),t(N,Ho),t(N,we),t(we,Bo),t(we,_r),t(_r,Jo),t(we,Ko),t(we,lt),t(lt,Qo),h(e,ta,d),h(e,Z,d),t(Z,ve),t(ve,wr),w(mt,wr,null),t(Z,Xo),t(Z,vr),t(vr,Yo),h(e,ra,d),h(e,ee,d),w(ct,ee,null),t(ee,Zo),t(ee,br),t(br,es),h(e,aa,d),h(e,te,d),t(te,be),t(be,yr),w(pt,yr,null),t(te,ts),t(te,$r),t($r,rs),h(e,na,d),h(e,re,d),t(re,ye),t(ye,xr),w(dt,xr,null),t(re,as),t(re,Ar),t(Ar,ns),h(e,oa,d),h(e,I,d),w(ht,I,null),t(I,os),t(I,ae),t(ae,ss),t(ae,zr),t(zr,is),t(ae,ls),t(ae,Er),t(Er,ms),t(ae,cs),t(I,ps),t(I,$e),w(ut,$e,null),t($e,ds),t($e,Tr),t(Tr,hs),sa=!0},p(e,[d]){const ft={};d&2&&(ft.$$scope={dirty:d,ctx:e}),me.$set(ft);const Dr={};d&2&&(Dr.$$scope={dirty:d,ctx:e}),ce.$set(Dr);const Lr={};d&2&&(Lr.$$scope={dirty:d,ctx:e}),pe.$set(Lr);const Pr={};d&2&&(Pr.$$scope={dirty:d,ctx:e}),de.$set(Pr)},i(e){sa||(v(p.$$.fragment,e),v(Ee.$$.fragment,e),v(Te.$$.fragment,e),v(Pe.$$.fragment,e),v(We.$$.fragment,e),v(ke.$$.fragment,e),v(me.$$.fragment,e),v(ce.$$.fragment,e),v(pe.$$.fragment,e),v(de.$$.fragment,e),v(qe.$$.fragment,e),v(Ue.$$.fragment,e),v(Ge.$$.fragment,e),v(Me.$$.fragment,e),v(He.$$.fragment,e),v(Be.$$.fragment,e),v(Je.$$.fragment,e),v(Ke.$$.fragment,e),v(Qe.$$.fragment,e),v(Xe.$$.fragment,e),v(Ye.$$.fragment,e),v(et.$$.fragment,e),v(rt.$$.fragment,e),v(nt.$$.fragment,e),v(st.$$.fragment,e),v(mt.$$.fragment,e),v(ct.$$.fragment,e),v(pt.$$.fragment,e),v(dt.$$.fragment,e),v(ht.$$.fragment,e),v(ut.$$.fragment,e),sa=!0)},o(e){b(p.$$.fragment,e),b(Ee.$$.fragment,e),b(Te.$$.fragment,e),b(Pe.$$.fragment,e),b(We.$$.fragment,e),b(ke.$$.fragment,e),b(me.$$.fragment,e),b(ce.$$.fragment,e),b(pe.$$.fragment,e),b(de.$$.fragment,e),b(qe.$$.fragment,e),b(Ue.$$.fragment,e),b(Ge.$$.fragment,e),b(Me.$$.fragment,e),b(He.$$.fragment,e),b(Be.$$.fragment,e),b(Je.$$.fragment,e),b(Ke.$$.fragment,e),b(Qe.$$.fragment,e),b(Xe.$$.fragment,e),b(Ye.$$.fragment,e),b(et.$$.fragment,e),b(rt.$$.fragment,e),b(nt.$$.fragment,e),b(st.$$.fragment,e),b(mt.$$.fragment,e),b(ct.$$.fragment,e),b(pt.$$.fragment,e),b(dt.$$.fragment,e),b(ht.$$.fragment,e),b(ut.$$.fragment,e),sa=!1},d(e){r(f),e&&r(T),e&&r(x),y(p),e&&r(kr),e&&r(oe),e&&r(Sr),e&&r(F),e&&r(Nr),e&&r(R),y(Ee),e&&r(Ir),e&&r(k),y(Te),y(Pe),e&&r(Cr),e&&r(j),y(We),e&&r(Fr),e&&r(A),y(ke),y(me),y(ce),y(pe),y(de),y(qe),e&&r(Or),e&&r(q),y(Ue),e&&r(Rr),e&&r(W),y(Ge),y(Me),e&&r(jr),e&&r(G),y(He),e&&r(qr),e&&r(V),y(Be),e&&r(Ur),e&&r(M),y(Je),e&&r(Gr),e&&r(H),y(Ke),e&&r(Vr),e&&r(B),y(Qe),e&&r(Mr),e&&r(J),y(Xe),e&&r(Hr),e&&r(K),y(Ye),e&&r(Br),e&&r(Ze),e&&r(Jr),e&&r(Q),y(et),e&&r(Kr),e&&r(tt),e&&r(Qr),e&&r(X),y(rt),e&&r(Xr),e&&r(at),e&&r(Yr),e&&r(Y),y(nt),e&&r(Zr),e&&r(ot),e&&r(ea),e&&r(N),y(st),e&&r(ta),e&&r(Z),y(mt),e&&r(ra),e&&r(ee),y(ct),e&&r(aa),e&&r(te),y(pt),e&&r(na),e&&r(re),y(dt),e&&r(oa),e&&r(I),y(ht),y(ut)}}}const Bi={local:"optimization",sections:[{local:"transformers.AdamW",title:"AdamW (PyTorch)"},{local:"transformers.Adafactor",title:"AdaFactor (PyTorch)"},{local:"transformers.AdamWeightDecay",title:"AdamWeightDecay (TensorFlow)"},{local:"schedules",sections:[{local:"transformers.SchedulerType",title:"Learning Rate Schedules (Pytorch)"},{local:"transformers.WarmUp",title:"Warmup (TensorFlow)"}],title:"Schedules"},{local:"gradient-strategies",sections:[{local:"transformers.GradientAccumulator",title:"GradientAccumulator (TensorFlow)"}],title:"Gradient Strategies"}],title:"Optimization"};function Ji(C){return qi(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class el extends Fi{constructor(f){super();Oi(this,f,Ji,Hi,Ri,{})}}export{el as default,Bi as metadata};
