import{S as Ji,i as Ki,s as Qi,e as a,k as l,w as u,t as s,M as Xi,c as n,d as r,m as c,a as o,x as f,h as i,b as m,N as ka,F as t,g as d,y as g,L as Yi,q as _,o as w,B as v}from"../../chunks/vendor-4833417e.js";import{D as $}from"../../chunks/Docstring-4f315ed9.js";import{C as Na}from"../../chunks/CodeBlock-6a3d1b46.js";import{I as Z}from"../../chunks/IconCopyLink-4b81c553.js";import"../../chunks/CopyButton-dacfbfaf.js";function Zi(Fa){let S,dt,E,x,bt,_e,Ca,$t,Oa,Lr,ee,Ra,At,ja,qa,Pr,I,zt,Ua,Ga,we,Va,Et,Ma,Ha,Ba,xt,Ja,Wr,k,te,Tt,ve,Ka,Dt,Qa,Sr,T,ye,Xa,be,Ya,$e,Za,en,tn,re,Ae,rn,Lt,an,Ir,N,ae,Pt,ze,nn,Wt,on,kr,h,Ee,sn,ht,ln,xe,cn,mn,b,pn,St,dn,hn,Te,un,fn,It,gn,_n,kt,wn,vn,Nt,yn,bn,Ft,$n,An,Ct,zn,En,xn,Ot,Tn,Dn,De,Ln,Le,Pn,Wn,Sn,D,Pe,Rt,In,kn,We,jt,Nn,Fn,Se,Cn,Ie,On,Rn,jn,qt,Ut,qn,Un,Gt,Vt,Gn,Vn,Mt,Ht,Mn,Hn,Bt,Bn,Jn,ke,Kn,Jt,Qn,Xn,Ne,Yn,L,Zn,Kt,eo,to,ut,ro,ao,Qt,no,oo,so,Fe,io,Xt,lo,co,Ce,mo,ne,Oe,po,Yt,ho,Nr,F,oe,Zt,Re,uo,er,fo,Fr,z,je,go,C,_o,tr,wo,vo,qe,yo,bo,$o,rr,Ao,zo,se,Ue,Eo,ar,xo,Cr,O,Ge,To,nr,Do,Or,R,ie,or,Ve,Lo,sr,Po,Rr,j,le,ir,Me,Wo,lr,So,jr,q,He,Io,cr,ko,qr,U,Be,No,mr,Fo,Ur,G,Je,Co,pr,Oo,Gr,V,Ke,Ro,dr,jo,Vr,Qe,ys,Mr,M,Xe,qo,hr,Uo,Hr,Ye,bs,Br,H,Ze,Go,ur,Vo,Jr,et,$s,Kr,B,tt,Mo,fr,Ho,Qr,rt,As,Xr,P,at,Bo,nt,Jo,gr,Ko,Qo,Xo,ce,Yo,_r,Zo,es,ot,ts,Yr,J,me,wr,st,rs,vr,as,Zr,K,it,ns,yr,os,ea,Q,pe,br,lt,ss,$r,is,ta,X,de,Ar,ct,ls,zr,cs,ra,W,mt,ms,Y,ps,Er,ds,hs,xr,us,fs,gs,he,pt,_s,Tr,ws,aa;return _e=new Z({}),ve=new Z({}),ye=new $({props:{name:"class transformers.AdamW",anchor:"transformers.AdamW",parameters:[{name:"params",val:": typing.Iterable[torch.nn.parameter.Parameter]"},{name:"lr",val:": float = 0.001"},{name:"betas",val:": typing.Tuple[float, float] = (0.9, 0.999)"},{name:"eps",val:": float = 1e-06"},{name:"weight_decay",val:": float = 0.0"},{name:"correct_bias",val:": bool = True"},{name:"no_deprecation_warning",val:": bool = False"}],source:"https://github.com/huggingface/transformers/blob/pr_15839/src/transformers/optimization.py#L273",parametersDescription:[{anchor:"transformers.AdamW.params",description:`<strong>params</strong> (<code>Iterable[nn.parameter.Parameter]</code>) &#x2014;
Iterable of parameters to optimize or dictionaries defining parameter groups.`,name:"params"},{anchor:"transformers.AdamW.lr",description:`<strong>lr</strong> (<code>float</code>, <em>optional</em>, defaults to 1e-3) &#x2014;
The learning rate to use.`,name:"lr"},{anchor:"transformers.AdamW.betas",description:`<strong>betas</strong> (<code>Tuple[float,float]</code>, <em>optional</em>, defaults to (0.9, 0.999)) &#x2014;
Adam&#x2019;s betas parameters (b1, b2).`,name:"betas"},{anchor:"transformers.AdamW.eps",description:`<strong>eps</strong> (<code>float</code>, <em>optional</em>, defaults to 1e-6) &#x2014;
Adam&#x2019;s epsilon for numerical stability.`,name:"eps"},{anchor:"transformers.AdamW.weight_decay",description:`<strong>weight_decay</strong> (<code>float</code>, <em>optional</em>, defaults to 0) &#x2014;
Decoupled weight decay to apply.`,name:"weight_decay"},{anchor:"transformers.AdamW.correct_bias",description:`<strong>correct_bias</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>True</code>) &#x2014;
Whether or not to correct bias in Adam (for instance, in Bert TF repository they use <code>False</code>).`,name:"correct_bias"},{anchor:"transformers.AdamW.no_deprecation_warning",description:`<strong>no_deprecation_warning</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>False</code>) &#x2014;
A flag used to disable the deprecation warning (set to <code>True</code> to disable the warning).`,name:"no_deprecation_warning"}]}}),Ae=new $({props:{name:"step",anchor:"transformers.AdamW.step",parameters:[{name:"closure",val:": typing.Callable = None"}],source:"https://github.com/huggingface/transformers/blob/pr_15839/src/transformers/optimization.py#L323",parametersDescription:[{anchor:"transformers.AdamW.step.closure",description:"<strong>closure</strong> (<code>Callable</code>, <em>optional</em>) &#x2014; A closure that reevaluates the model and returns the loss.",name:"closure"}]}}),ze=new Z({}),Ee=new $({props:{name:"class transformers.Adafactor",anchor:"transformers.Adafactor",parameters:[{name:"params",val:""},{name:"lr",val:" = None"},{name:"eps",val:" = (1e-30, 0.001)"},{name:"clip_threshold",val:" = 1.0"},{name:"decay_rate",val:" = -0.8"},{name:"beta1",val:" = None"},{name:"weight_decay",val:" = 0.0"},{name:"scale_parameter",val:" = True"},{name:"relative_step",val:" = True"},{name:"warmup_init",val:" = False"}],source:"https://github.com/huggingface/transformers/blob/pr_15839/src/transformers/optimization.py#L385",parametersDescription:[{anchor:"transformers.Adafactor.params",description:`<strong>params</strong> (<code>Iterable[nn.parameter.Parameter]</code>) &#x2014;
Iterable of parameters to optimize or dictionaries defining parameter groups.`,name:"params"},{anchor:"transformers.Adafactor.lr",description:`<strong>lr</strong> (<code>float</code>, <em>optional</em>) &#x2014;
The external learning rate.`,name:"lr"},{anchor:"transformers.Adafactor.eps",description:`<strong>eps</strong> (<code>Tuple[float, float]</code>, <em>optional</em>, defaults to (1e-30, 1e-3)) &#x2014;
Regularization constants for square gradient and parameter scale respectively`,name:"eps"},{anchor:"transformers.Adafactor.clip_threshold",description:`<strong>clip_threshold</strong> (<code>float</code>, <em>optional</em>, defaults 1.0) &#x2014;
Threshold of root mean square of final gradient update`,name:"clip_threshold"},{anchor:"transformers.Adafactor.decay_rate",description:`<strong>decay_rate</strong> (<code>float</code>, <em>optional</em>, defaults to -0.8) &#x2014;
Coefficient used to compute running averages of square`,name:"decay_rate"},{anchor:"transformers.Adafactor.beta1",description:`<strong>beta1</strong> (<code>float</code>, <em>optional</em>) &#x2014;
Coefficient used for computing running averages of gradient`,name:"beta1"},{anchor:"transformers.Adafactor.weight_decay",description:`<strong>weight_decay</strong> (<code>float</code>, <em>optional</em>, defaults to 0) &#x2014;
Weight decay (L2 penalty)`,name:"weight_decay"},{anchor:"transformers.Adafactor.scale_parameter",description:`<strong>scale_parameter</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>True</code>) &#x2014;
If True, learning rate is scaled by root mean square`,name:"scale_parameter"},{anchor:"transformers.Adafactor.relative_step",description:`<strong>relative_step</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>True</code>) &#x2014;
If True, time-dependent learning rate is computed instead of external learning rate`,name:"relative_step"},{anchor:"transformers.Adafactor.warmup_init",description:`<strong>warmup_init</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>False</code>) &#x2014;
Time-dependent learning rate computation depends on whether warm-up initialization is being used`,name:"warmup_init"}]}}),ke=new Na({props:{code:"Adafactor(model.parameters(), scale_parameter=False, relative_step=False, warmup_init=False, lr=1e-3)",highlighted:'Adafactor(model.parameters(), scale_parameter=<span class="hljs-literal">False</span>, relative_step=<span class="hljs-literal">False</span>, warmup_init=<span class="hljs-literal">False</span>, lr=<span class="hljs-number">1e-3</span>)'}}),Ne=new Na({props:{code:"Adafactor(model.parameters(), scale_parameter=True, relative_step=True, warmup_init=True, lr=None)",highlighted:'Adafactor(model.parameters(), scale_parameter=<span class="hljs-literal">True</span>, relative_step=<span class="hljs-literal">True</span>, warmup_init=<span class="hljs-literal">True</span>, lr=<span class="hljs-literal">None</span>)'}}),Fe=new Na({props:{code:`from transformers.optimization import Adafactor, AdafactorSchedule

optimizer = Adafactor(model.parameters(), scale_parameter=True, relative_step=True, warmup_init=True, lr=None)
lr_scheduler = AdafactorSchedule(optimizer)
trainer = Trainer(..., optimizers=(optimizer, lr_scheduler))`,highlighted:`<span class="hljs-keyword">from</span> transformers.optimization <span class="hljs-keyword">import</span> Adafactor, AdafactorSchedule

optimizer = Adafactor(model.parameters(), scale_parameter=<span class="hljs-literal">True</span>, relative_step=<span class="hljs-literal">True</span>, warmup_init=<span class="hljs-literal">True</span>, lr=<span class="hljs-literal">None</span>)
lr_scheduler = AdafactorSchedule(optimizer)
trainer = Trainer(..., optimizers=(optimizer, lr_scheduler))`}}),Ce=new Na({props:{code:`# replace AdamW with Adafactor
optimizer = Adafactor(
    model.parameters(),
    lr=1e-3,
    eps=(1e-30, 1e-3),
    clip_threshold=1.0,
    decay_rate=-0.8,
    beta1=None,
    weight_decay=0.0,
    relative_step=False,
    scale_parameter=False,
    warmup_init=False,
)`,highlighted:`<span class="hljs-comment"># replace AdamW with Adafactor</span>
optimizer = Adafactor(
    model.parameters(),
    lr=<span class="hljs-number">1e-3</span>,
    eps=(<span class="hljs-number">1e-30</span>, <span class="hljs-number">1e-3</span>),
    clip_threshold=<span class="hljs-number">1.0</span>,
    decay_rate=-<span class="hljs-number">0.8</span>,
    beta1=<span class="hljs-literal">None</span>,
    weight_decay=<span class="hljs-number">0.0</span>,
    relative_step=<span class="hljs-literal">False</span>,
    scale_parameter=<span class="hljs-literal">False</span>,
    warmup_init=<span class="hljs-literal">False</span>,
)`}}),Oe=new $({props:{name:"step",anchor:"transformers.Adafactor.step",parameters:[{name:"closure",val:" = None"}],source:"https://github.com/huggingface/transformers/blob/pr_15839/src/transformers/optimization.py#L531",parametersDescription:[{anchor:"transformers.Adafactor.step.closure",description:`<strong>closure</strong> (callable, optional) &#x2014; A closure that reevaluates the model
and returns the loss.`,name:"closure"}]}}),Re=new Z({}),je=new $({props:{name:"class transformers.AdamWeightDecay",anchor:"transformers.AdamWeightDecay",parameters:[{name:"learning_rate",val:": typing.Union[float, keras.optimizer_v2.learning_rate_schedule.LearningRateSchedule] = 0.001"},{name:"beta_1",val:": float = 0.9"},{name:"beta_2",val:": float = 0.999"},{name:"epsilon",val:": float = 1e-07"},{name:"amsgrad",val:": bool = False"},{name:"weight_decay_rate",val:": float = 0.0"},{name:"include_in_weight_decay",val:": typing.Optional[typing.List[str]] = None"},{name:"exclude_from_weight_decay",val:": typing.Optional[typing.List[str]] = None"},{name:"name",val:": str = 'AdamWeightDecay'"},{name:"**kwargs",val:""}],source:"https://github.com/huggingface/transformers/blob/pr_15839/src/transformers/optimization_tf.py#L152",parametersDescription:[{anchor:"transformers.AdamWeightDecay.learning_rate",description:`<strong>learning_rate</strong> (<code>Union[float, tf.keras.optimizers.schedules.LearningRateSchedule]</code>, <em>optional</em>, defaults to 1e-3) &#x2014;
The learning rate to use or a schedule.`,name:"learning_rate"},{anchor:"transformers.AdamWeightDecay.beta_1",description:`<strong>beta_1</strong> (<code>float</code>, <em>optional</em>, defaults to 0.9) &#x2014;
The beta1 parameter in Adam, which is the exponential decay rate for the 1st momentum estimates.`,name:"beta_1"},{anchor:"transformers.AdamWeightDecay.beta_2",description:`<strong>beta_2</strong> (<code>float</code>, <em>optional</em>, defaults to 0.999) &#x2014;
The beta2 parameter in Adam, which is the exponential decay rate for the 2nd momentum estimates.`,name:"beta_2"},{anchor:"transformers.AdamWeightDecay.epsilon",description:`<strong>epsilon</strong> (<code>float</code>, <em>optional</em>, defaults to 1e-7) &#x2014;
The epsilon parameter in Adam, which is a small constant for numerical stability.`,name:"epsilon"},{anchor:"transformers.AdamWeightDecay.amsgrad",description:`<strong>amsgrad</strong> (<code>bool</code>, <em>optional</em>, default to <code>False</code>) &#x2014;
Whether to apply AMSGrad variant of this algorithm or not, see <a href="https://arxiv.org/abs/1904.09237" rel="nofollow">On the Convergence of Adam and
Beyond</a>.`,name:"amsgrad"},{anchor:"transformers.AdamWeightDecay.weight_decay_rate",description:`<strong>weight_decay_rate</strong> (<code>float</code>, <em>optional</em>, defaults to 0) &#x2014;
The weight decay to apply.`,name:"weight_decay_rate"},{anchor:"transformers.AdamWeightDecay.include_in_weight_decay",description:`<strong>include_in_weight_decay</strong> (<code>List[str]</code>, <em>optional</em>) &#x2014;
List of the parameter names (or re patterns) to apply weight decay to. If none is passed, weight decay is
applied to all parameters by default (unless they are in <code>exclude_from_weight_decay</code>).`,name:"include_in_weight_decay"},{anchor:"transformers.AdamWeightDecay.exclude_from_weight_decay",description:`<strong>exclude_from_weight_decay</strong> (<code>List[str]</code>, <em>optional</em>) &#x2014;
List of the parameter names (or re patterns) to exclude from applying weight decay to. If a
<code>include_in_weight_decay</code> is passed, the names in it will supersede this list.`,name:"exclude_from_weight_decay"},{anchor:"transformers.AdamWeightDecay.name",description:`<strong>name</strong> (<code>str</code>, <em>optional</em>, defaults to &#x2018;AdamWeightDecay&#x2019;) &#x2014;
Optional name for the operations created when applying gradients.
kwargs &#x2014;
Keyword arguments. Allowed to be {<code>clipnorm</code>, <code>clipvalue</code>, <code>lr</code>, <code>decay</code>}. <code>clipnorm</code> is clip gradients by
norm; <code>clipvalue</code> is clip gradients by value, <code>decay</code> is included for backward compatibility to allow time
inverse decay of learning rate. <code>lr</code> is included for backward compatibility, recommended to use
<code>learning_rate</code> instead.`,name:"name"}]}}),Ue=new $({props:{name:"from_config",anchor:"transformers.AdamWeightDecay.from_config",parameters:[{name:"config",val:""}],source:"https://github.com/huggingface/transformers/blob/pr_15839/src/transformers/optimization_tf.py#L209"}}),Ge=new $({props:{name:"transformers.create_optimizer",anchor:"transformers.create_optimizer",parameters:[{name:"init_lr",val:": float"},{name:"num_train_steps",val:": int"},{name:"num_warmup_steps",val:": int"},{name:"min_lr_ratio",val:": float = 0.0"},{name:"adam_beta1",val:": float = 0.9"},{name:"adam_beta2",val:": float = 0.999"},{name:"adam_epsilon",val:": float = 1e-08"},{name:"weight_decay_rate",val:": float = 0.0"},{name:"power",val:": float = 1.0"},{name:"include_in_weight_decay",val:": typing.Optional[typing.List[str]] = None"}],source:"https://github.com/huggingface/transformers/blob/pr_15839/src/transformers/optimization_tf.py#L82",parametersDescription:[{anchor:"transformers.create_optimizer.init_lr",description:`<strong>init_lr</strong> (<code>float</code>) &#x2014;
The desired learning rate at the end of the warmup phase.`,name:"init_lr"},{anchor:"transformers.create_optimizer.num_train_steps",description:`<strong>num_train_steps</strong> (<code>int</code>) &#x2014;
The total number of training steps.`,name:"num_train_steps"},{anchor:"transformers.create_optimizer.num_warmup_steps",description:`<strong>num_warmup_steps</strong> (<code>int</code>) &#x2014;
The number of warmup steps.`,name:"num_warmup_steps"},{anchor:"transformers.create_optimizer.min_lr_ratio",description:`<strong>min_lr_ratio</strong> (<code>float</code>, <em>optional</em>, defaults to 0) &#x2014;
The final learning rate at the end of the linear decay will be <code>init_lr * min_lr_ratio</code>.`,name:"min_lr_ratio"},{anchor:"transformers.create_optimizer.adam_beta1",description:`<strong>adam_beta1</strong> (<code>float</code>, <em>optional</em>, defaults to 0.9) &#x2014;
The beta1 to use in Adam.`,name:"adam_beta1"},{anchor:"transformers.create_optimizer.adam_beta2",description:`<strong>adam_beta2</strong> (<code>float</code>, <em>optional</em>, defaults to 0.999) &#x2014;
The beta2 to use in Adam.`,name:"adam_beta2"},{anchor:"transformers.create_optimizer.adam_epsilon",description:`<strong>adam_epsilon</strong> (<code>float</code>, <em>optional</em>, defaults to 1e-8) &#x2014;
The epsilon to use in Adam.`,name:"adam_epsilon"},{anchor:"transformers.create_optimizer.weight_decay_rate",description:`<strong>weight_decay_rate</strong> (<code>float</code>, <em>optional</em>, defaults to 0) &#x2014;
The weight decay to use.`,name:"weight_decay_rate"},{anchor:"transformers.create_optimizer.power",description:`<strong>power</strong> (<code>float</code>, <em>optional</em>, defaults to 1.0) &#x2014;
The power to use for PolynomialDecay.`,name:"power"},{anchor:"transformers.create_optimizer.include_in_weight_decay",description:`<strong>include_in_weight_decay</strong> (<code>List[str]</code>, <em>optional</em>) &#x2014;
List of the parameter names (or re patterns) to apply weight decay to. If none is passed, weight decay is
applied to all parameters except bias and layer norm parameters.`,name:"include_in_weight_decay"}]}}),Ve=new Z({}),Me=new Z({}),He=new $({props:{name:"class transformers.SchedulerType",anchor:"transformers.SchedulerType",parameters:[{name:"value",val:""},{name:"names",val:" = None"},{name:"module",val:" = None"},{name:"qualname",val:" = None"},{name:"type",val:" = None"},{name:"start",val:" = 1"}],source:"https://github.com/huggingface/transformers/blob/pr_15839/src/transformers/trainer_utils.py#L301"}}),Be=new $({props:{name:"transformers.get_scheduler",anchor:"transformers.get_scheduler",parameters:[{name:"name",val:": typing.Union[str, transformers.trainer_utils.SchedulerType]"},{name:"optimizer",val:": Optimizer"},{name:"num_warmup_steps",val:": typing.Optional[int] = None"},{name:"num_training_steps",val:": typing.Optional[int] = None"}],source:"https://github.com/huggingface/transformers/blob/pr_15839/src/transformers/optimization.py#L233",parametersDescription:[{anchor:"transformers.get_scheduler.name",description:`<strong>name</strong> (<code>str</code> or <code>SchedulerType</code>) &#x2014;
The name of the scheduler to use.`,name:"name"},{anchor:"transformers.get_scheduler.optimizer",description:`<strong>optimizer</strong> (<code>torch.optim.Optimizer</code>) &#x2014;
The optimizer that will be used during training.`,name:"optimizer"},{anchor:"transformers.get_scheduler.num_warmup_steps",description:`<strong>num_warmup_steps</strong> (<code>int</code>, <em>optional</em>) &#x2014;
The number of warmup steps to do. This is not required by all schedulers (hence the argument being
optional), the function will raise an error if it&#x2019;s unset and the scheduler type requires it.`,name:"num_warmup_steps"},{anchor:"transformers.get_scheduler.num_training_steps",description:`<strong>num_training_steps</strong> (\`int&#x201C;, <em>optional</em>) &#x2014;
The number of training steps to do. This is not required by all schedulers (hence the argument being
optional), the function will raise an error if it&#x2019;s unset and the scheduler type requires it.`,name:"num_training_steps"}]}}),Je=new $({props:{name:"transformers.get_constant_schedule",anchor:"transformers.get_constant_schedule",parameters:[{name:"optimizer",val:": Optimizer"},{name:"last_epoch",val:": int = -1"}],source:"https://github.com/huggingface/transformers/blob/pr_15839/src/transformers/optimization.py#L34",parametersDescription:[{anchor:"transformers.get_constant_schedule.optimizer",description:`<strong>optimizer</strong> (<code>Optimizer</code>) &#x2014;
The optimizer for which to schedule the learning rate.`,name:"optimizer"},{anchor:"transformers.get_constant_schedule.last_epoch",description:`<strong>last_epoch</strong> (<code>int</code>, <em>optional</em>, defaults to -1) &#x2014;
The index of the last epoch when resuming training.`,name:"last_epoch"}],returnDescription:`
<p><code>torch.optim.lr_scheduler.LambdaLR</code> with the appropriate schedule.</p>
`}}),Ke=new $({props:{name:"transformers.get_constant_schedule_with_warmup",anchor:"transformers.get_constant_schedule_with_warmup",parameters:[{name:"optimizer",val:": Optimizer"},{name:"num_warmup_steps",val:": int"},{name:"last_epoch",val:": int = -1"}],source:"https://github.com/huggingface/transformers/blob/pr_15839/src/transformers/optimization.py#L50",parametersDescription:[{anchor:"transformers.get_constant_schedule_with_warmup.optimizer",description:`<strong>optimizer</strong> (<code>Optimizer</code>) &#x2014;
The optimizer for which to schedule the learning rate.`,name:"optimizer"},{anchor:"transformers.get_constant_schedule_with_warmup.num_warmup_steps",description:`<strong>num_warmup_steps</strong> (<code>int</code>) &#x2014;
The number of steps for the warmup phase.`,name:"num_warmup_steps"},{anchor:"transformers.get_constant_schedule_with_warmup.last_epoch",description:`<strong>last_epoch</strong> (<code>int</code>, <em>optional</em>, defaults to -1) &#x2014;
The index of the last epoch when resuming training.`,name:"last_epoch"}],returnDescription:`
<p><code>torch.optim.lr_scheduler.LambdaLR</code> with the appropriate schedule.</p>
`}}),Xe=new $({props:{name:"transformers.get_cosine_schedule_with_warmup",anchor:"transformers.get_cosine_schedule_with_warmup",parameters:[{name:"optimizer",val:": Optimizer"},{name:"num_warmup_steps",val:": int"},{name:"num_training_steps",val:": int"},{name:"num_cycles",val:": float = 0.5"},{name:"last_epoch",val:": int = -1"}],source:"https://github.com/huggingface/transformers/blob/pr_15839/src/transformers/optimization.py#L104",parametersDescription:[{anchor:"transformers.get_cosine_schedule_with_warmup.optimizer",description:`<strong>optimizer</strong> (<code>Optimizer</code>) &#x2014;
The optimizer for which to schedule the learning rate.`,name:"optimizer"},{anchor:"transformers.get_cosine_schedule_with_warmup.num_warmup_steps",description:`<strong>num_warmup_steps</strong> (<code>int</code>) &#x2014;
The number of steps for the warmup phase.`,name:"num_warmup_steps"},{anchor:"transformers.get_cosine_schedule_with_warmup.num_training_steps",description:`<strong>num_training_steps</strong> (<code>int</code>) &#x2014;
The total number of training steps.`,name:"num_training_steps"},{anchor:"transformers.get_cosine_schedule_with_warmup.num_cycles",description:`<strong>num_cycles</strong> (<code>float</code>, <em>optional</em>, defaults to 0.5) &#x2014;
The number of waves in the cosine schedule (the defaults is to just decrease from the max value to 0
following a half-cosine).`,name:"num_cycles"},{anchor:"transformers.get_cosine_schedule_with_warmup.last_epoch",description:`<strong>last_epoch</strong> (<code>int</code>, <em>optional</em>, defaults to -1) &#x2014;
The index of the last epoch when resuming training.`,name:"last_epoch"}],returnDescription:`
<p><code>torch.optim.lr_scheduler.LambdaLR</code> with the appropriate schedule.</p>
`}}),Ze=new $({props:{name:"transformers.get_cosine_with_hard_restarts_schedule_with_warmup",anchor:"transformers.get_cosine_with_hard_restarts_schedule_with_warmup",parameters:[{name:"optimizer",val:": Optimizer"},{name:"num_warmup_steps",val:": int"},{name:"num_training_steps",val:": int"},{name:"num_cycles",val:": int = 1"},{name:"last_epoch",val:": int = -1"}],source:"https://github.com/huggingface/transformers/blob/pr_15839/src/transformers/optimization.py#L138",parametersDescription:[{anchor:"transformers.get_cosine_with_hard_restarts_schedule_with_warmup.optimizer",description:`<strong>optimizer</strong> (<code>Optimizer</code>) &#x2014;
The optimizer for which to schedule the learning rate.`,name:"optimizer"},{anchor:"transformers.get_cosine_with_hard_restarts_schedule_with_warmup.num_warmup_steps",description:`<strong>num_warmup_steps</strong> (<code>int</code>) &#x2014;
The number of steps for the warmup phase.`,name:"num_warmup_steps"},{anchor:"transformers.get_cosine_with_hard_restarts_schedule_with_warmup.num_training_steps",description:`<strong>num_training_steps</strong> (<code>int</code>) &#x2014;
The total number of training steps.`,name:"num_training_steps"},{anchor:"transformers.get_cosine_with_hard_restarts_schedule_with_warmup.num_cycles",description:`<strong>num_cycles</strong> (<code>int</code>, <em>optional</em>, defaults to 1) &#x2014;
The number of hard restarts to use.`,name:"num_cycles"},{anchor:"transformers.get_cosine_with_hard_restarts_schedule_with_warmup.last_epoch",description:`<strong>last_epoch</strong> (<code>int</code>, <em>optional</em>, defaults to -1) &#x2014;
The index of the last epoch when resuming training.`,name:"last_epoch"}],returnDescription:`
<p><code>torch.optim.lr_scheduler.LambdaLR</code> with the appropriate schedule.</p>
`}}),tt=new $({props:{name:"transformers.get_linear_schedule_with_warmup",anchor:"transformers.get_linear_schedule_with_warmup",parameters:[{name:"optimizer",val:""},{name:"num_warmup_steps",val:""},{name:"num_training_steps",val:""},{name:"last_epoch",val:" = -1"}],source:"https://github.com/huggingface/transformers/blob/pr_15839/src/transformers/optimization.py#L75",parametersDescription:[{anchor:"transformers.get_linear_schedule_with_warmup.optimizer",description:`<strong>optimizer</strong> (<code>Optimizer</code>) &#x2014;
The optimizer for which to schedule the learning rate.`,name:"optimizer"},{anchor:"transformers.get_linear_schedule_with_warmup.num_warmup_steps",description:`<strong>num_warmup_steps</strong> (<code>int</code>) &#x2014;
The number of steps for the warmup phase.`,name:"num_warmup_steps"},{anchor:"transformers.get_linear_schedule_with_warmup.num_training_steps",description:`<strong>num_training_steps</strong> (<code>int</code>) &#x2014;
The total number of training steps.`,name:"num_training_steps"},{anchor:"transformers.get_linear_schedule_with_warmup.last_epoch",description:`<strong>last_epoch</strong> (<code>int</code>, <em>optional</em>, defaults to -1) &#x2014;
The index of the last epoch when resuming training.`,name:"last_epoch"}],returnDescription:`
<p><code>torch.optim.lr_scheduler.LambdaLR</code> with the appropriate schedule.</p>
`}}),at=new $({props:{name:"transformers.get_polynomial_decay_schedule_with_warmup",anchor:"transformers.get_polynomial_decay_schedule_with_warmup",parameters:[{name:"optimizer",val:""},{name:"num_warmup_steps",val:""},{name:"num_training_steps",val:""},{name:"lr_end",val:" = 1e-07"},{name:"power",val:" = 1.0"},{name:"last_epoch",val:" = -1"}],source:"https://github.com/huggingface/transformers/blob/pr_15839/src/transformers/optimization.py#L173",parametersDescription:[{anchor:"transformers.get_polynomial_decay_schedule_with_warmup.optimizer",description:`<strong>optimizer</strong> (<code>Optimizer</code>) &#x2014;
The optimizer for which to schedule the learning rate.`,name:"optimizer"},{anchor:"transformers.get_polynomial_decay_schedule_with_warmup.num_warmup_steps",description:`<strong>num_warmup_steps</strong> (<code>int</code>) &#x2014;
The number of steps for the warmup phase.`,name:"num_warmup_steps"},{anchor:"transformers.get_polynomial_decay_schedule_with_warmup.num_training_steps",description:`<strong>num_training_steps</strong> (<code>int</code>) &#x2014;
The total number of training steps.`,name:"num_training_steps"},{anchor:"transformers.get_polynomial_decay_schedule_with_warmup.lr_end",description:`<strong>lr_end</strong> (<code>float</code>, <em>optional</em>, defaults to 1e-7) &#x2014;
The end LR.`,name:"lr_end"},{anchor:"transformers.get_polynomial_decay_schedule_with_warmup.power",description:`<strong>power</strong> (<code>float</code>, <em>optional</em>, defaults to 1.0) &#x2014;
Power factor.`,name:"power"},{anchor:"transformers.get_polynomial_decay_schedule_with_warmup.last_epoch",description:`<strong>last_epoch</strong> (<code>int</code>, <em>optional</em>, defaults to -1) &#x2014;
The index of the last epoch when resuming training.`,name:"last_epoch"}],returnDescription:`
<p><code>torch.optim.lr_scheduler.LambdaLR</code> with the appropriate schedule.</p>
`}}),st=new Z({}),it=new $({props:{name:"class transformers.WarmUp",anchor:"transformers.WarmUp",parameters:[{name:"initial_learning_rate",val:": float"},{name:"decay_schedule_fn",val:": typing.Callable"},{name:"warmup_steps",val:": int"},{name:"power",val:": float = 1.0"},{name:"name",val:": str = None"}],source:"https://github.com/huggingface/transformers/blob/pr_15839/src/transformers/optimization_tf.py#L24",parametersDescription:[{anchor:"transformers.WarmUp.initial_learning_rate",description:`<strong>initial_learning_rate</strong> (<code>float</code>) &#x2014;
The initial learning rate for the schedule after the warmup (so this will be the learning rate at the end
of the warmup).`,name:"initial_learning_rate"},{anchor:"transformers.WarmUp.decay_schedule_fn",description:`<strong>decay_schedule_fn</strong> (<code>Callable</code>) &#x2014;
The schedule function to apply after the warmup for the rest of training.`,name:"decay_schedule_fn"},{anchor:"transformers.WarmUp.warmup_steps",description:`<strong>warmup_steps</strong> (<code>int</code>) &#x2014;
The number of steps for the warmup part of training.`,name:"warmup_steps"},{anchor:"transformers.WarmUp.power",description:`<strong>power</strong> (<code>float</code>, <em>optional</em>, defaults to 1) &#x2014;
The power to use for the polynomial warmup (defaults is a linear warmup).`,name:"power"},{anchor:"transformers.WarmUp.name",description:`<strong>name</strong> (<code>str</code>, <em>optional</em>) &#x2014;
Optional name prefix for the returned tensors during the schedule.`,name:"name"}]}}),lt=new Z({}),ct=new Z({}),mt=new $({props:{name:"class transformers.GradientAccumulator",anchor:"transformers.GradientAccumulator",parameters:[],source:"https://github.com/huggingface/transformers/blob/pr_15839/src/transformers/optimization_tf.py#L282"}}),pt=new $({props:{name:"reset",anchor:"transformers.GradientAccumulator.reset",parameters:[],source:"https://github.com/huggingface/transformers/blob/pr_15839/src/transformers/optimization_tf.py#L344"}}),{c(){S=a("meta"),dt=l(),E=a("h1"),x=a("a"),bt=a("span"),u(_e.$$.fragment),Ca=l(),$t=a("span"),Oa=s("Optimization"),Lr=l(),ee=a("p"),Ra=s("The "),At=a("code"),ja=s(".optimization"),qa=s(" module provides:"),Pr=l(),I=a("ul"),zt=a("li"),Ua=s("an optimizer with weight decay fixed that can be used to fine-tuned models, and"),Ga=l(),we=a("li"),Va=s("several schedules in the form of schedule objects that inherit from "),Et=a("code"),Ma=s("_LRSchedule"),Ha=s(":"),Ba=l(),xt=a("li"),Ja=s("a gradient accumulation class to accumulate the gradients of multiple batches"),Wr=l(),k=a("h2"),te=a("a"),Tt=a("span"),u(ve.$$.fragment),Ka=l(),Dt=a("span"),Qa=s("AdamW (PyTorch)"),Sr=l(),T=a("div"),u(ye.$$.fragment),Xa=l(),be=a("p"),Ya=s("Implements Adam algorithm with weight decay fix as introduced in "),$e=a("a"),Za=s(`Decoupled Weight Decay
Regularization`),en=s("."),tn=l(),re=a("div"),u(Ae.$$.fragment),rn=l(),Lt=a("p"),an=s("Performs a single optimization step."),Ir=l(),N=a("h2"),ae=a("a"),Pt=a("span"),u(ze.$$.fragment),nn=l(),Wt=a("span"),on=s("AdaFactor (PyTorch)"),kr=l(),h=a("div"),u(Ee.$$.fragment),sn=l(),ht=a("p"),ln=s(`AdaFactor pytorch implementation can be used as a drop in replacement for Adam original fairseq code:
`),xe=a("a"),cn=s("https://github.com/pytorch/fairseq/blob/master/fairseq/optim/adafactor.py"),mn=l(),b=a("p"),pn=s("Paper: "),St=a("em"),dn=s("Adafactor: Adaptive Learning Rates with Sublinear Memory Cost"),hn=l(),Te=a("a"),un=s("https://arxiv.org/abs/1804.04235"),fn=s(` Note that
this optimizer internally adjusts the learning rate depending on the `),It=a("code"),gn=s("scale_parameter"),_n=s(", "),kt=a("code"),wn=s("relative_step"),vn=s(` and
`),Nt=a("code"),yn=s("warmup_init"),bn=s(" options. To use a manual (external) learning rate schedule you should set "),Ft=a("code"),$n=s("scale_parameter=False"),An=s(` and
`),Ct=a("code"),zn=s("relative_step=False"),En=s("."),xn=l(),Ot=a("p"),Tn=s("This implementation handles low-precision (FP16, bfloat) values, but we have not thoroughly tested."),Dn=l(),De=a("p"),Ln=s("Recommended T5 finetuning settings ("),Le=a("a"),Pn=s("https://discuss.huggingface.co/t/t5-finetuning-tips/684/3"),Wn=s("):"),Sn=l(),D=a("ul"),Pe=a("li"),Rt=a("p"),In=s("Training without LR warmup or clip_threshold is not recommended."),kn=l(),We=a("ul"),jt=a("li"),Nn=s("use scheduled LR warm-up to fixed LR"),Fn=l(),Se=a("li"),Cn=s("use clip_threshold=1.0 ("),Ie=a("a"),On=s("https://arxiv.org/abs/1804.04235"),Rn=s(")"),jn=l(),qt=a("li"),Ut=a("p"),qn=s("Disable relative updates"),Un=l(),Gt=a("li"),Vt=a("p"),Gn=s("Use scale_parameter=False"),Vn=l(),Mt=a("li"),Ht=a("p"),Mn=s("Additional optimizer operations like gradient clipping should not be used alongside Adafactor"),Hn=l(),Bt=a("p"),Bn=s("Example:"),Jn=l(),u(ke.$$.fragment),Kn=l(),Jt=a("p"),Qn=s("Others reported the following combination to work well:"),Xn=l(),u(Ne.$$.fragment),Yn=l(),L=a("p"),Zn=s("When using "),Kt=a("code"),eo=s("lr=None"),to=s(" with "),ut=a("a"),ro=s("Trainer"),ao=s(" you will most likely need to use "),Qt=a("code"),no=s("AdafactorSchedule"),oo=s("scheduler as following:"),so=l(),u(Fe.$$.fragment),io=l(),Xt=a("p"),lo=s("Usage:"),co=l(),u(Ce.$$.fragment),mo=l(),ne=a("div"),u(Oe.$$.fragment),po=l(),Yt=a("p"),ho=s("Performs a single optimization step"),Nr=l(),F=a("h2"),oe=a("a"),Zt=a("span"),u(Re.$$.fragment),uo=l(),er=a("span"),fo=s("AdamWeightDecay (TensorFlow)"),Fr=l(),z=a("div"),u(je.$$.fragment),go=l(),C=a("p"),_o=s(`Adam enables L2 weight decay and clip_by_global_norm on gradients. Just adding the square of the weights to the
loss function is `),tr=a("em"),wo=s("not"),vo=s(` the correct way of using L2 regularization/weight decay with Adam, since that will interact
with the m and v parameters in strange ways as shown in `),qe=a("a"),yo=s(`Decoupled Weight Decay
Regularization`),bo=s("."),$o=l(),rr=a("p"),Ao=s(`Instead we want ot decay the weights in a manner that doesn\u2019t interact with the m/v parameters. This is equivalent
to adding the square of the weights to the loss with plain (non-momentum) SGD.`),zo=l(),se=a("div"),u(Ue.$$.fragment),Eo=l(),ar=a("p"),xo=s("Creates an optimizer from its config with WarmUp custom object."),Cr=l(),O=a("div"),u(Ge.$$.fragment),To=l(),nr=a("p"),Do=s("Creates an optimizer with a learning rate schedule using a warmup phase followed by a linear decay."),Or=l(),R=a("h2"),ie=a("a"),or=a("span"),u(Ve.$$.fragment),Lo=l(),sr=a("span"),Po=s("Schedules"),Rr=l(),j=a("h3"),le=a("a"),ir=a("span"),u(Me.$$.fragment),Wo=l(),lr=a("span"),So=s("Learning Rate Schedules (Pytorch)"),jr=l(),q=a("div"),u(He.$$.fragment),Io=l(),cr=a("p"),ko=s("An enumeration."),qr=l(),U=a("div"),u(Be.$$.fragment),No=l(),mr=a("p"),Fo=s("Unified API to get any scheduler from its name."),Ur=l(),G=a("div"),u(Je.$$.fragment),Co=l(),pr=a("p"),Oo=s("Create a schedule with a constant learning rate, using the learning rate set in optimizer."),Gr=l(),V=a("div"),u(Ke.$$.fragment),Ro=l(),dr=a("p"),jo=s(`Create a schedule with a constant learning rate preceded by a warmup period during which the learning rate
increases linearly between 0 and the initial lr set in the optimizer.`),Vr=l(),Qe=a("img"),Mr=l(),M=a("div"),u(Xe.$$.fragment),qo=l(),hr=a("p"),Uo=s(`Create a schedule with a learning rate that decreases following the values of the cosine function between the
initial lr set in the optimizer to 0, after a warmup period during which it increases linearly between 0 and the
initial lr set in the optimizer.`),Hr=l(),Ye=a("img"),Br=l(),H=a("div"),u(Ze.$$.fragment),Go=l(),ur=a("p"),Vo=s(`Create a schedule with a learning rate that decreases following the values of the cosine function between the
initial lr set in the optimizer to 0, with several hard restarts, after a warmup period during which it increases
linearly between 0 and the initial lr set in the optimizer.`),Jr=l(),et=a("img"),Kr=l(),B=a("div"),u(tt.$$.fragment),Mo=l(),fr=a("p"),Ho=s(`Create a schedule with a learning rate that decreases linearly from the initial lr set in the optimizer to 0, after
a warmup period during which it increases linearly from 0 to the initial lr set in the optimizer.`),Qr=l(),rt=a("img"),Xr=l(),P=a("div"),u(at.$$.fragment),Bo=l(),nt=a("p"),Jo=s(`Create a schedule with a learning rate that decreases as a polynomial decay from the initial lr set in the
optimizer to end lr defined by `),gr=a("em"),Ko=s("lr_end"),Qo=s(`, after a warmup period during which it increases linearly from 0 to the
initial lr set in the optimizer.`),Xo=l(),ce=a("p"),Yo=s("Note: "),_r=a("em"),Zo=s("power"),es=s(` defaults to 1.0 as in the fairseq implementation, which in turn is based on the original BERT
implementation at
`),ot=a("a"),ts=s("https://github.com/google-research/bert/blob/f39e881b169b9d53bea03d2d341b31707a6c052b/optimization.py#L37"),Yr=l(),J=a("h3"),me=a("a"),wr=a("span"),u(st.$$.fragment),rs=l(),vr=a("span"),as=s("Warmup (TensorFlow)"),Zr=l(),K=a("div"),u(it.$$.fragment),ns=l(),yr=a("p"),os=s("Applies a warmup schedule on a given learning rate decay schedule."),ea=l(),Q=a("h2"),pe=a("a"),br=a("span"),u(lt.$$.fragment),ss=l(),$r=a("span"),is=s("Gradient Strategies"),ta=l(),X=a("h3"),de=a("a"),Ar=a("span"),u(ct.$$.fragment),ls=l(),zr=a("span"),cs=s("GradientAccumulator (TensorFlow)"),ra=l(),W=a("div"),u(mt.$$.fragment),ms=l(),Y=a("p"),ps=s(`Gradient accumulation utility. When used with a distribution strategy, the accumulator should be called in a
replica context. Gradients will be accumulated locally on each replica and without synchronization. Users should
then call `),Er=a("code"),ds=s(".gradients"),hs=s(", scale the gradients if required, and pass the result to "),xr=a("code"),us=s("apply_gradients"),fs=s("."),gs=l(),he=a("div"),u(pt.$$.fragment),_s=l(),Tr=a("p"),ws=s("Resets the accumulated gradients on the current replica."),this.h()},l(e){const p=Xi('[data-svelte="svelte-1phssyn"]',document.head);S=n(p,"META",{name:!0,content:!0}),p.forEach(r),dt=c(e),E=n(e,"H1",{class:!0});var na=o(E);x=n(na,"A",{id:!0,class:!0,href:!0});var zs=o(x);bt=n(zs,"SPAN",{});var Es=o(bt);f(_e.$$.fragment,Es),Es.forEach(r),zs.forEach(r),Ca=c(na),$t=n(na,"SPAN",{});var xs=o($t);Oa=i(xs,"Optimization"),xs.forEach(r),na.forEach(r),Lr=c(e),ee=n(e,"P",{});var oa=o(ee);Ra=i(oa,"The "),At=n(oa,"CODE",{});var Ts=o(At);ja=i(Ts,".optimization"),Ts.forEach(r),qa=i(oa," module provides:"),oa.forEach(r),Pr=c(e),I=n(e,"UL",{});var ft=o(I);zt=n(ft,"LI",{});var Ds=o(zt);Ua=i(Ds,"an optimizer with weight decay fixed that can be used to fine-tuned models, and"),Ds.forEach(r),Ga=c(ft),we=n(ft,"LI",{});var sa=o(we);Va=i(sa,"several schedules in the form of schedule objects that inherit from "),Et=n(sa,"CODE",{});var Ls=o(Et);Ma=i(Ls,"_LRSchedule"),Ls.forEach(r),Ha=i(sa,":"),sa.forEach(r),Ba=c(ft),xt=n(ft,"LI",{});var Ps=o(xt);Ja=i(Ps,"a gradient accumulation class to accumulate the gradients of multiple batches"),Ps.forEach(r),ft.forEach(r),Wr=c(e),k=n(e,"H2",{class:!0});var ia=o(k);te=n(ia,"A",{id:!0,class:!0,href:!0});var Ws=o(te);Tt=n(Ws,"SPAN",{});var Ss=o(Tt);f(ve.$$.fragment,Ss),Ss.forEach(r),Ws.forEach(r),Ka=c(ia),Dt=n(ia,"SPAN",{});var Is=o(Dt);Qa=i(Is,"AdamW (PyTorch)"),Is.forEach(r),ia.forEach(r),Sr=c(e),T=n(e,"DIV",{class:!0});var gt=o(T);f(ye.$$.fragment,gt),Xa=c(gt),be=n(gt,"P",{});var la=o(be);Ya=i(la,"Implements Adam algorithm with weight decay fix as introduced in "),$e=n(la,"A",{href:!0,rel:!0});var ks=o($e);Za=i(ks,`Decoupled Weight Decay
Regularization`),ks.forEach(r),en=i(la,"."),la.forEach(r),tn=c(gt),re=n(gt,"DIV",{class:!0});var ca=o(re);f(Ae.$$.fragment,ca),rn=c(ca),Lt=n(ca,"P",{});var Ns=o(Lt);an=i(Ns,"Performs a single optimization step."),Ns.forEach(r),ca.forEach(r),gt.forEach(r),Ir=c(e),N=n(e,"H2",{class:!0});var ma=o(N);ae=n(ma,"A",{id:!0,class:!0,href:!0});var Fs=o(ae);Pt=n(Fs,"SPAN",{});var Cs=o(Pt);f(ze.$$.fragment,Cs),Cs.forEach(r),Fs.forEach(r),nn=c(ma),Wt=n(ma,"SPAN",{});var Os=o(Wt);on=i(Os,"AdaFactor (PyTorch)"),Os.forEach(r),ma.forEach(r),kr=c(e),h=n(e,"DIV",{class:!0});var y=o(h);f(Ee.$$.fragment,y),sn=c(y),ht=n(y,"P",{});var vs=o(ht);ln=i(vs,`AdaFactor pytorch implementation can be used as a drop in replacement for Adam original fairseq code:
`),xe=n(vs,"A",{href:!0,rel:!0});var Rs=o(xe);cn=i(Rs,"https://github.com/pytorch/fairseq/blob/master/fairseq/optim/adafactor.py"),Rs.forEach(r),vs.forEach(r),mn=c(y),b=n(y,"P",{});var A=o(b);pn=i(A,"Paper: "),St=n(A,"EM",{});var js=o(St);dn=i(js,"Adafactor: Adaptive Learning Rates with Sublinear Memory Cost"),js.forEach(r),hn=c(A),Te=n(A,"A",{href:!0,rel:!0});var qs=o(Te);un=i(qs,"https://arxiv.org/abs/1804.04235"),qs.forEach(r),fn=i(A,` Note that
this optimizer internally adjusts the learning rate depending on the `),It=n(A,"CODE",{});var Us=o(It);gn=i(Us,"scale_parameter"),Us.forEach(r),_n=i(A,", "),kt=n(A,"CODE",{});var Gs=o(kt);wn=i(Gs,"relative_step"),Gs.forEach(r),vn=i(A,` and
`),Nt=n(A,"CODE",{});var Vs=o(Nt);yn=i(Vs,"warmup_init"),Vs.forEach(r),bn=i(A," options. To use a manual (external) learning rate schedule you should set "),Ft=n(A,"CODE",{});var Ms=o(Ft);$n=i(Ms,"scale_parameter=False"),Ms.forEach(r),An=i(A,` and
`),Ct=n(A,"CODE",{});var Hs=o(Ct);zn=i(Hs,"relative_step=False"),Hs.forEach(r),En=i(A,"."),A.forEach(r),xn=c(y),Ot=n(y,"P",{});var Bs=o(Ot);Tn=i(Bs,"This implementation handles low-precision (FP16, bfloat) values, but we have not thoroughly tested."),Bs.forEach(r),Dn=c(y),De=n(y,"P",{});var pa=o(De);Ln=i(pa,"Recommended T5 finetuning settings ("),Le=n(pa,"A",{href:!0,rel:!0});var Js=o(Le);Pn=i(Js,"https://discuss.huggingface.co/t/t5-finetuning-tips/684/3"),Js.forEach(r),Wn=i(pa,"):"),pa.forEach(r),Sn=c(y),D=n(y,"UL",{});var ue=o(D);Pe=n(ue,"LI",{});var da=o(Pe);Rt=n(da,"P",{});var Ks=o(Rt);In=i(Ks,"Training without LR warmup or clip_threshold is not recommended."),Ks.forEach(r),kn=c(da),We=n(da,"UL",{});var ha=o(We);jt=n(ha,"LI",{});var Qs=o(jt);Nn=i(Qs,"use scheduled LR warm-up to fixed LR"),Qs.forEach(r),Fn=c(ha),Se=n(ha,"LI",{});var ua=o(Se);Cn=i(ua,"use clip_threshold=1.0 ("),Ie=n(ua,"A",{href:!0,rel:!0});var Xs=o(Ie);On=i(Xs,"https://arxiv.org/abs/1804.04235"),Xs.forEach(r),Rn=i(ua,")"),ua.forEach(r),ha.forEach(r),da.forEach(r),jn=c(ue),qt=n(ue,"LI",{});var Ys=o(qt);Ut=n(Ys,"P",{});var Zs=o(Ut);qn=i(Zs,"Disable relative updates"),Zs.forEach(r),Ys.forEach(r),Un=c(ue),Gt=n(ue,"LI",{});var ei=o(Gt);Vt=n(ei,"P",{});var ti=o(Vt);Gn=i(ti,"Use scale_parameter=False"),ti.forEach(r),ei.forEach(r),Vn=c(ue),Mt=n(ue,"LI",{});var ri=o(Mt);Ht=n(ri,"P",{});var ai=o(Ht);Mn=i(ai,"Additional optimizer operations like gradient clipping should not be used alongside Adafactor"),ai.forEach(r),ri.forEach(r),ue.forEach(r),Hn=c(y),Bt=n(y,"P",{});var ni=o(Bt);Bn=i(ni,"Example:"),ni.forEach(r),Jn=c(y),f(ke.$$.fragment,y),Kn=c(y),Jt=n(y,"P",{});var oi=o(Jt);Qn=i(oi,"Others reported the following combination to work well:"),oi.forEach(r),Xn=c(y),f(Ne.$$.fragment,y),Yn=c(y),L=n(y,"P",{});var fe=o(L);Zn=i(fe,"When using "),Kt=n(fe,"CODE",{});var si=o(Kt);eo=i(si,"lr=None"),si.forEach(r),to=i(fe," with "),ut=n(fe,"A",{href:!0});var ii=o(ut);ro=i(ii,"Trainer"),ii.forEach(r),ao=i(fe," you will most likely need to use "),Qt=n(fe,"CODE",{});var li=o(Qt);no=i(li,"AdafactorSchedule"),li.forEach(r),oo=i(fe,"scheduler as following:"),fe.forEach(r),so=c(y),f(Fe.$$.fragment,y),io=c(y),Xt=n(y,"P",{});var ci=o(Xt);lo=i(ci,"Usage:"),ci.forEach(r),co=c(y),f(Ce.$$.fragment,y),mo=c(y),ne=n(y,"DIV",{class:!0});var fa=o(ne);f(Oe.$$.fragment,fa),po=c(fa),Yt=n(fa,"P",{});var mi=o(Yt);ho=i(mi,"Performs a single optimization step"),mi.forEach(r),fa.forEach(r),y.forEach(r),Nr=c(e),F=n(e,"H2",{class:!0});var ga=o(F);oe=n(ga,"A",{id:!0,class:!0,href:!0});var pi=o(oe);Zt=n(pi,"SPAN",{});var di=o(Zt);f(Re.$$.fragment,di),di.forEach(r),pi.forEach(r),uo=c(ga),er=n(ga,"SPAN",{});var hi=o(er);fo=i(hi,"AdamWeightDecay (TensorFlow)"),hi.forEach(r),ga.forEach(r),Fr=c(e),z=n(e,"DIV",{class:!0});var ge=o(z);f(je.$$.fragment,ge),go=c(ge),C=n(ge,"P",{});var _t=o(C);_o=i(_t,`Adam enables L2 weight decay and clip_by_global_norm on gradients. Just adding the square of the weights to the
loss function is `),tr=n(_t,"EM",{});var ui=o(tr);wo=i(ui,"not"),ui.forEach(r),vo=i(_t,` the correct way of using L2 regularization/weight decay with Adam, since that will interact
with the m and v parameters in strange ways as shown in `),qe=n(_t,"A",{href:!0,rel:!0});var fi=o(qe);yo=i(fi,`Decoupled Weight Decay
Regularization`),fi.forEach(r),bo=i(_t,"."),_t.forEach(r),$o=c(ge),rr=n(ge,"P",{});var gi=o(rr);Ao=i(gi,`Instead we want ot decay the weights in a manner that doesn\u2019t interact with the m/v parameters. This is equivalent
to adding the square of the weights to the loss with plain (non-momentum) SGD.`),gi.forEach(r),zo=c(ge),se=n(ge,"DIV",{class:!0});var _a=o(se);f(Ue.$$.fragment,_a),Eo=c(_a),ar=n(_a,"P",{});var _i=o(ar);xo=i(_i,"Creates an optimizer from its config with WarmUp custom object."),_i.forEach(r),_a.forEach(r),ge.forEach(r),Cr=c(e),O=n(e,"DIV",{class:!0});var wa=o(O);f(Ge.$$.fragment,wa),To=c(wa),nr=n(wa,"P",{});var wi=o(nr);Do=i(wi,"Creates an optimizer with a learning rate schedule using a warmup phase followed by a linear decay."),wi.forEach(r),wa.forEach(r),Or=c(e),R=n(e,"H2",{class:!0});var va=o(R);ie=n(va,"A",{id:!0,class:!0,href:!0});var vi=o(ie);or=n(vi,"SPAN",{});var yi=o(or);f(Ve.$$.fragment,yi),yi.forEach(r),vi.forEach(r),Lo=c(va),sr=n(va,"SPAN",{});var bi=o(sr);Po=i(bi,"Schedules"),bi.forEach(r),va.forEach(r),Rr=c(e),j=n(e,"H3",{class:!0});var ya=o(j);le=n(ya,"A",{id:!0,class:!0,href:!0});var $i=o(le);ir=n($i,"SPAN",{});var Ai=o(ir);f(Me.$$.fragment,Ai),Ai.forEach(r),$i.forEach(r),Wo=c(ya),lr=n(ya,"SPAN",{});var zi=o(lr);So=i(zi,"Learning Rate Schedules (Pytorch)"),zi.forEach(r),ya.forEach(r),jr=c(e),q=n(e,"DIV",{class:!0});var ba=o(q);f(He.$$.fragment,ba),Io=c(ba),cr=n(ba,"P",{});var Ei=o(cr);ko=i(Ei,"An enumeration."),Ei.forEach(r),ba.forEach(r),qr=c(e),U=n(e,"DIV",{class:!0});var $a=o(U);f(Be.$$.fragment,$a),No=c($a),mr=n($a,"P",{});var xi=o(mr);Fo=i(xi,"Unified API to get any scheduler from its name."),xi.forEach(r),$a.forEach(r),Ur=c(e),G=n(e,"DIV",{class:!0});var Aa=o(G);f(Je.$$.fragment,Aa),Co=c(Aa),pr=n(Aa,"P",{});var Ti=o(pr);Oo=i(Ti,"Create a schedule with a constant learning rate, using the learning rate set in optimizer."),Ti.forEach(r),Aa.forEach(r),Gr=c(e),V=n(e,"DIV",{class:!0});var za=o(V);f(Ke.$$.fragment,za),Ro=c(za),dr=n(za,"P",{});var Di=o(dr);jo=i(Di,`Create a schedule with a constant learning rate preceded by a warmup period during which the learning rate
increases linearly between 0 and the initial lr set in the optimizer.`),Di.forEach(r),za.forEach(r),Vr=c(e),Qe=n(e,"IMG",{alt:!0,src:!0}),Mr=c(e),M=n(e,"DIV",{class:!0});var Ea=o(M);f(Xe.$$.fragment,Ea),qo=c(Ea),hr=n(Ea,"P",{});var Li=o(hr);Uo=i(Li,`Create a schedule with a learning rate that decreases following the values of the cosine function between the
initial lr set in the optimizer to 0, after a warmup period during which it increases linearly between 0 and the
initial lr set in the optimizer.`),Li.forEach(r),Ea.forEach(r),Hr=c(e),Ye=n(e,"IMG",{alt:!0,src:!0}),Br=c(e),H=n(e,"DIV",{class:!0});var xa=o(H);f(Ze.$$.fragment,xa),Go=c(xa),ur=n(xa,"P",{});var Pi=o(ur);Vo=i(Pi,`Create a schedule with a learning rate that decreases following the values of the cosine function between the
initial lr set in the optimizer to 0, with several hard restarts, after a warmup period during which it increases
linearly between 0 and the initial lr set in the optimizer.`),Pi.forEach(r),xa.forEach(r),Jr=c(e),et=n(e,"IMG",{alt:!0,src:!0}),Kr=c(e),B=n(e,"DIV",{class:!0});var Ta=o(B);f(tt.$$.fragment,Ta),Mo=c(Ta),fr=n(Ta,"P",{});var Wi=o(fr);Ho=i(Wi,`Create a schedule with a learning rate that decreases linearly from the initial lr set in the optimizer to 0, after
a warmup period during which it increases linearly from 0 to the initial lr set in the optimizer.`),Wi.forEach(r),Ta.forEach(r),Qr=c(e),rt=n(e,"IMG",{alt:!0,src:!0}),Xr=c(e),P=n(e,"DIV",{class:!0});var wt=o(P);f(at.$$.fragment,wt),Bo=c(wt),nt=n(wt,"P",{});var Da=o(nt);Jo=i(Da,`Create a schedule with a learning rate that decreases as a polynomial decay from the initial lr set in the
optimizer to end lr defined by `),gr=n(Da,"EM",{});var Si=o(gr);Ko=i(Si,"lr_end"),Si.forEach(r),Qo=i(Da,`, after a warmup period during which it increases linearly from 0 to the
initial lr set in the optimizer.`),Da.forEach(r),Xo=c(wt),ce=n(wt,"P",{});var Dr=o(ce);Yo=i(Dr,"Note: "),_r=n(Dr,"EM",{});var Ii=o(_r);Zo=i(Ii,"power"),Ii.forEach(r),es=i(Dr,` defaults to 1.0 as in the fairseq implementation, which in turn is based on the original BERT
implementation at
`),ot=n(Dr,"A",{href:!0,rel:!0});var ki=o(ot);ts=i(ki,"https://github.com/google-research/bert/blob/f39e881b169b9d53bea03d2d341b31707a6c052b/optimization.py#L37"),ki.forEach(r),Dr.forEach(r),wt.forEach(r),Yr=c(e),J=n(e,"H3",{class:!0});var La=o(J);me=n(La,"A",{id:!0,class:!0,href:!0});var Ni=o(me);wr=n(Ni,"SPAN",{});var Fi=o(wr);f(st.$$.fragment,Fi),Fi.forEach(r),Ni.forEach(r),rs=c(La),vr=n(La,"SPAN",{});var Ci=o(vr);as=i(Ci,"Warmup (TensorFlow)"),Ci.forEach(r),La.forEach(r),Zr=c(e),K=n(e,"DIV",{class:!0});var Pa=o(K);f(it.$$.fragment,Pa),ns=c(Pa),yr=n(Pa,"P",{});var Oi=o(yr);os=i(Oi,"Applies a warmup schedule on a given learning rate decay schedule."),Oi.forEach(r),Pa.forEach(r),ea=c(e),Q=n(e,"H2",{class:!0});var Wa=o(Q);pe=n(Wa,"A",{id:!0,class:!0,href:!0});var Ri=o(pe);br=n(Ri,"SPAN",{});var ji=o(br);f(lt.$$.fragment,ji),ji.forEach(r),Ri.forEach(r),ss=c(Wa),$r=n(Wa,"SPAN",{});var qi=o($r);is=i(qi,"Gradient Strategies"),qi.forEach(r),Wa.forEach(r),ta=c(e),X=n(e,"H3",{class:!0});var Sa=o(X);de=n(Sa,"A",{id:!0,class:!0,href:!0});var Ui=o(de);Ar=n(Ui,"SPAN",{});var Gi=o(Ar);f(ct.$$.fragment,Gi),Gi.forEach(r),Ui.forEach(r),ls=c(Sa),zr=n(Sa,"SPAN",{});var Vi=o(zr);cs=i(Vi,"GradientAccumulator (TensorFlow)"),Vi.forEach(r),Sa.forEach(r),ra=c(e),W=n(e,"DIV",{class:!0});var vt=o(W);f(mt.$$.fragment,vt),ms=c(vt),Y=n(vt,"P",{});var yt=o(Y);ps=i(yt,`Gradient accumulation utility. When used with a distribution strategy, the accumulator should be called in a
replica context. Gradients will be accumulated locally on each replica and without synchronization. Users should
then call `),Er=n(yt,"CODE",{});var Mi=o(Er);ds=i(Mi,".gradients"),Mi.forEach(r),hs=i(yt,", scale the gradients if required, and pass the result to "),xr=n(yt,"CODE",{});var Hi=o(xr);us=i(Hi,"apply_gradients"),Hi.forEach(r),fs=i(yt,"."),yt.forEach(r),gs=c(vt),he=n(vt,"DIV",{class:!0});var Ia=o(he);f(pt.$$.fragment,Ia),_s=c(Ia),Tr=n(Ia,"P",{});var Bi=o(Tr);ws=i(Bi,"Resets the accumulated gradients on the current replica."),Bi.forEach(r),Ia.forEach(r),vt.forEach(r),this.h()},h(){m(S,"name","hf:doc:metadata"),m(S,"content",JSON.stringify(el)),m(x,"id","optimization"),m(x,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),m(x,"href","#optimization"),m(E,"class","relative group"),m(te,"id","transformers.AdamW"),m(te,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),m(te,"href","#transformers.AdamW"),m(k,"class","relative group"),m($e,"href","https://arxiv.org/abs/1711.05101"),m($e,"rel","nofollow"),m(re,"class","docstring"),m(T,"class","docstring"),m(ae,"id","transformers.Adafactor"),m(ae,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),m(ae,"href","#transformers.Adafactor"),m(N,"class","relative group"),m(xe,"href","https://github.com/pytorch/fairseq/blob/master/fairseq/optim/adafactor.py"),m(xe,"rel","nofollow"),m(Te,"href","https://arxiv.org/abs/1804.04235"),m(Te,"rel","nofollow"),m(Le,"href","https://discuss.huggingface.co/t/t5-finetuning-tips/684/3"),m(Le,"rel","nofollow"),m(Ie,"href","https://arxiv.org/abs/1804.04235"),m(Ie,"rel","nofollow"),m(ut,"href","/docs/transformers/pr_15839/en/main_classes/trainer#transformers.Trainer"),m(ne,"class","docstring"),m(h,"class","docstring"),m(oe,"id","transformers.AdamWeightDecay"),m(oe,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),m(oe,"href","#transformers.AdamWeightDecay"),m(F,"class","relative group"),m(qe,"href","https://arxiv.org/abs/1711.05101"),m(qe,"rel","nofollow"),m(se,"class","docstring"),m(z,"class","docstring"),m(O,"class","docstring"),m(ie,"id","schedules"),m(ie,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),m(ie,"href","#schedules"),m(R,"class","relative group"),m(le,"id","transformers.SchedulerType"),m(le,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),m(le,"href","#transformers.SchedulerType"),m(j,"class","relative group"),m(q,"class","docstring"),m(U,"class","docstring"),m(G,"class","docstring"),m(V,"class","docstring"),m(Qe,"alt",""),ka(Qe.src,ys="https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/warmup_constant_schedule.png")||m(Qe,"src",ys),m(M,"class","docstring"),m(Ye,"alt",""),ka(Ye.src,bs="https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/warmup_cosine_schedule.png")||m(Ye,"src",bs),m(H,"class","docstring"),m(et,"alt",""),ka(et.src,$s="https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/warmup_cosine_hard_restarts_schedule.png")||m(et,"src",$s),m(B,"class","docstring"),m(rt,"alt",""),ka(rt.src,As="https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/warmup_linear_schedule.png")||m(rt,"src",As),m(ot,"href","https://github.com/google-research/bert/blob/f39e881b169b9d53bea03d2d341b31707a6c052b/optimization.py#L37"),m(ot,"rel","nofollow"),m(P,"class","docstring"),m(me,"id","transformers.WarmUp"),m(me,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),m(me,"href","#transformers.WarmUp"),m(J,"class","relative group"),m(K,"class","docstring"),m(pe,"id","gradient-strategies"),m(pe,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),m(pe,"href","#gradient-strategies"),m(Q,"class","relative group"),m(de,"id","transformers.GradientAccumulator"),m(de,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),m(de,"href","#transformers.GradientAccumulator"),m(X,"class","relative group"),m(he,"class","docstring"),m(W,"class","docstring")},m(e,p){t(document.head,S),d(e,dt,p),d(e,E,p),t(E,x),t(x,bt),g(_e,bt,null),t(E,Ca),t(E,$t),t($t,Oa),d(e,Lr,p),d(e,ee,p),t(ee,Ra),t(ee,At),t(At,ja),t(ee,qa),d(e,Pr,p),d(e,I,p),t(I,zt),t(zt,Ua),t(I,Ga),t(I,we),t(we,Va),t(we,Et),t(Et,Ma),t(we,Ha),t(I,Ba),t(I,xt),t(xt,Ja),d(e,Wr,p),d(e,k,p),t(k,te),t(te,Tt),g(ve,Tt,null),t(k,Ka),t(k,Dt),t(Dt,Qa),d(e,Sr,p),d(e,T,p),g(ye,T,null),t(T,Xa),t(T,be),t(be,Ya),t(be,$e),t($e,Za),t(be,en),t(T,tn),t(T,re),g(Ae,re,null),t(re,rn),t(re,Lt),t(Lt,an),d(e,Ir,p),d(e,N,p),t(N,ae),t(ae,Pt),g(ze,Pt,null),t(N,nn),t(N,Wt),t(Wt,on),d(e,kr,p),d(e,h,p),g(Ee,h,null),t(h,sn),t(h,ht),t(ht,ln),t(ht,xe),t(xe,cn),t(h,mn),t(h,b),t(b,pn),t(b,St),t(St,dn),t(b,hn),t(b,Te),t(Te,un),t(b,fn),t(b,It),t(It,gn),t(b,_n),t(b,kt),t(kt,wn),t(b,vn),t(b,Nt),t(Nt,yn),t(b,bn),t(b,Ft),t(Ft,$n),t(b,An),t(b,Ct),t(Ct,zn),t(b,En),t(h,xn),t(h,Ot),t(Ot,Tn),t(h,Dn),t(h,De),t(De,Ln),t(De,Le),t(Le,Pn),t(De,Wn),t(h,Sn),t(h,D),t(D,Pe),t(Pe,Rt),t(Rt,In),t(Pe,kn),t(Pe,We),t(We,jt),t(jt,Nn),t(We,Fn),t(We,Se),t(Se,Cn),t(Se,Ie),t(Ie,On),t(Se,Rn),t(D,jn),t(D,qt),t(qt,Ut),t(Ut,qn),t(D,Un),t(D,Gt),t(Gt,Vt),t(Vt,Gn),t(D,Vn),t(D,Mt),t(Mt,Ht),t(Ht,Mn),t(h,Hn),t(h,Bt),t(Bt,Bn),t(h,Jn),g(ke,h,null),t(h,Kn),t(h,Jt),t(Jt,Qn),t(h,Xn),g(Ne,h,null),t(h,Yn),t(h,L),t(L,Zn),t(L,Kt),t(Kt,eo),t(L,to),t(L,ut),t(ut,ro),t(L,ao),t(L,Qt),t(Qt,no),t(L,oo),t(h,so),g(Fe,h,null),t(h,io),t(h,Xt),t(Xt,lo),t(h,co),g(Ce,h,null),t(h,mo),t(h,ne),g(Oe,ne,null),t(ne,po),t(ne,Yt),t(Yt,ho),d(e,Nr,p),d(e,F,p),t(F,oe),t(oe,Zt),g(Re,Zt,null),t(F,uo),t(F,er),t(er,fo),d(e,Fr,p),d(e,z,p),g(je,z,null),t(z,go),t(z,C),t(C,_o),t(C,tr),t(tr,wo),t(C,vo),t(C,qe),t(qe,yo),t(C,bo),t(z,$o),t(z,rr),t(rr,Ao),t(z,zo),t(z,se),g(Ue,se,null),t(se,Eo),t(se,ar),t(ar,xo),d(e,Cr,p),d(e,O,p),g(Ge,O,null),t(O,To),t(O,nr),t(nr,Do),d(e,Or,p),d(e,R,p),t(R,ie),t(ie,or),g(Ve,or,null),t(R,Lo),t(R,sr),t(sr,Po),d(e,Rr,p),d(e,j,p),t(j,le),t(le,ir),g(Me,ir,null),t(j,Wo),t(j,lr),t(lr,So),d(e,jr,p),d(e,q,p),g(He,q,null),t(q,Io),t(q,cr),t(cr,ko),d(e,qr,p),d(e,U,p),g(Be,U,null),t(U,No),t(U,mr),t(mr,Fo),d(e,Ur,p),d(e,G,p),g(Je,G,null),t(G,Co),t(G,pr),t(pr,Oo),d(e,Gr,p),d(e,V,p),g(Ke,V,null),t(V,Ro),t(V,dr),t(dr,jo),d(e,Vr,p),d(e,Qe,p),d(e,Mr,p),d(e,M,p),g(Xe,M,null),t(M,qo),t(M,hr),t(hr,Uo),d(e,Hr,p),d(e,Ye,p),d(e,Br,p),d(e,H,p),g(Ze,H,null),t(H,Go),t(H,ur),t(ur,Vo),d(e,Jr,p),d(e,et,p),d(e,Kr,p),d(e,B,p),g(tt,B,null),t(B,Mo),t(B,fr),t(fr,Ho),d(e,Qr,p),d(e,rt,p),d(e,Xr,p),d(e,P,p),g(at,P,null),t(P,Bo),t(P,nt),t(nt,Jo),t(nt,gr),t(gr,Ko),t(nt,Qo),t(P,Xo),t(P,ce),t(ce,Yo),t(ce,_r),t(_r,Zo),t(ce,es),t(ce,ot),t(ot,ts),d(e,Yr,p),d(e,J,p),t(J,me),t(me,wr),g(st,wr,null),t(J,rs),t(J,vr),t(vr,as),d(e,Zr,p),d(e,K,p),g(it,K,null),t(K,ns),t(K,yr),t(yr,os),d(e,ea,p),d(e,Q,p),t(Q,pe),t(pe,br),g(lt,br,null),t(Q,ss),t(Q,$r),t($r,is),d(e,ta,p),d(e,X,p),t(X,de),t(de,Ar),g(ct,Ar,null),t(X,ls),t(X,zr),t(zr,cs),d(e,ra,p),d(e,W,p),g(mt,W,null),t(W,ms),t(W,Y),t(Y,ps),t(Y,Er),t(Er,ds),t(Y,hs),t(Y,xr),t(xr,us),t(Y,fs),t(W,gs),t(W,he),g(pt,he,null),t(he,_s),t(he,Tr),t(Tr,ws),aa=!0},p:Yi,i(e){aa||(_(_e.$$.fragment,e),_(ve.$$.fragment,e),_(ye.$$.fragment,e),_(Ae.$$.fragment,e),_(ze.$$.fragment,e),_(Ee.$$.fragment,e),_(ke.$$.fragment,e),_(Ne.$$.fragment,e),_(Fe.$$.fragment,e),_(Ce.$$.fragment,e),_(Oe.$$.fragment,e),_(Re.$$.fragment,e),_(je.$$.fragment,e),_(Ue.$$.fragment,e),_(Ge.$$.fragment,e),_(Ve.$$.fragment,e),_(Me.$$.fragment,e),_(He.$$.fragment,e),_(Be.$$.fragment,e),_(Je.$$.fragment,e),_(Ke.$$.fragment,e),_(Xe.$$.fragment,e),_(Ze.$$.fragment,e),_(tt.$$.fragment,e),_(at.$$.fragment,e),_(st.$$.fragment,e),_(it.$$.fragment,e),_(lt.$$.fragment,e),_(ct.$$.fragment,e),_(mt.$$.fragment,e),_(pt.$$.fragment,e),aa=!0)},o(e){w(_e.$$.fragment,e),w(ve.$$.fragment,e),w(ye.$$.fragment,e),w(Ae.$$.fragment,e),w(ze.$$.fragment,e),w(Ee.$$.fragment,e),w(ke.$$.fragment,e),w(Ne.$$.fragment,e),w(Fe.$$.fragment,e),w(Ce.$$.fragment,e),w(Oe.$$.fragment,e),w(Re.$$.fragment,e),w(je.$$.fragment,e),w(Ue.$$.fragment,e),w(Ge.$$.fragment,e),w(Ve.$$.fragment,e),w(Me.$$.fragment,e),w(He.$$.fragment,e),w(Be.$$.fragment,e),w(Je.$$.fragment,e),w(Ke.$$.fragment,e),w(Xe.$$.fragment,e),w(Ze.$$.fragment,e),w(tt.$$.fragment,e),w(at.$$.fragment,e),w(st.$$.fragment,e),w(it.$$.fragment,e),w(lt.$$.fragment,e),w(ct.$$.fragment,e),w(mt.$$.fragment,e),w(pt.$$.fragment,e),aa=!1},d(e){r(S),e&&r(dt),e&&r(E),v(_e),e&&r(Lr),e&&r(ee),e&&r(Pr),e&&r(I),e&&r(Wr),e&&r(k),v(ve),e&&r(Sr),e&&r(T),v(ye),v(Ae),e&&r(Ir),e&&r(N),v(ze),e&&r(kr),e&&r(h),v(Ee),v(ke),v(Ne),v(Fe),v(Ce),v(Oe),e&&r(Nr),e&&r(F),v(Re),e&&r(Fr),e&&r(z),v(je),v(Ue),e&&r(Cr),e&&r(O),v(Ge),e&&r(Or),e&&r(R),v(Ve),e&&r(Rr),e&&r(j),v(Me),e&&r(jr),e&&r(q),v(He),e&&r(qr),e&&r(U),v(Be),e&&r(Ur),e&&r(G),v(Je),e&&r(Gr),e&&r(V),v(Ke),e&&r(Vr),e&&r(Qe),e&&r(Mr),e&&r(M),v(Xe),e&&r(Hr),e&&r(Ye),e&&r(Br),e&&r(H),v(Ze),e&&r(Jr),e&&r(et),e&&r(Kr),e&&r(B),v(tt),e&&r(Qr),e&&r(rt),e&&r(Xr),e&&r(P),v(at),e&&r(Yr),e&&r(J),v(st),e&&r(Zr),e&&r(K),v(it),e&&r(ea),e&&r(Q),v(lt),e&&r(ta),e&&r(X),v(ct),e&&r(ra),e&&r(W),v(mt),v(pt)}}}const el={local:"optimization",sections:[{local:"transformers.AdamW",title:"AdamW (PyTorch)"},{local:"transformers.Adafactor",title:"AdaFactor (PyTorch)"},{local:"transformers.AdamWeightDecay",title:"AdamWeightDecay (TensorFlow)"},{local:"schedules",sections:[{local:"transformers.SchedulerType",title:"Learning Rate Schedules (Pytorch)"},{local:"transformers.WarmUp",title:"Warmup (TensorFlow)"}],title:"Schedules"},{local:"gradient-strategies",sections:[{local:"transformers.GradientAccumulator",title:"GradientAccumulator (TensorFlow)"}],title:"Gradient Strategies"}],title:"Optimization"};function tl(Fa,S,dt){let{fw:E}=S;return Fa.$$set=x=>{"fw"in x&&dt(0,E=x.fw)},[E]}class il extends Ji{constructor(S){super();Ki(this,S,tl,Zi,Qi,{fw:0})}}export{il as default,el as metadata};
