import{S as rt,i as nt,s as lt,e as l,k as h,w as d,t,M as pt,c as p,d as s,m,a as o,x as j,h as r,b as c,G as a,g as i,y as g,L as ot,q as b,o as v,B as w,v as it}from"../chunks/vendor-hf-doc-builder.js";import{I as ua}from"../chunks/IconCopyLink-hf-doc-builder.js";import{C as I}from"../chunks/CodeBlock-hf-doc-builder.js";function ct(ws){let E,He,A,z,je,M,fa,ge,_a,Ce,$,da,le,ja,ga,pe,ba,va,Se,P,D,be,L,wa,ve,ya,Fe,f,oe,$a,qa,B,ka,Ea,Y,Aa,Pa,G,Ta,xa,J,za,Da,Ne,ie,Ha,Oe,R,Ie,T,H,we,U,Ca,ye,Sa,Me,ce,Fa,Le,C,Na,K,Oa,Ia,Be,Q,Ye,S,Ma,V,La,Ba,Ge,W,Je,F,Ya,X,Ga,Ja,Re,Z,Ue,N,Ra,ee,Ua,Ka,Ke,ae,Qe,q,Qa,$e,Va,Wa,he,Xa,Za,Ve,se,We,k,es,me,as,ss,qe,ts,rs,Xe,te,Ze,u,ns,ke,ls,ps,Ee,os,is,Ae,cs,hs,Pe,ms,us,Te,fs,_s,xe,ds,js,ea,ue,gs,aa,re,sa,x,O,ze,ne,bs,De,vs,ta,ra;return M=new ua({}),L=new ua({}),R=new I({props:{code:"pip install optuna/sigopt/wandb/ray[tune] ",highlighted:"pip install optuna/sigopt/wandb/ray[tune] "}}),U=new ua({}),Q=new I({props:{code:`def sigopt_hp_space(trial):
    return [
        {"bounds": {"min": 1e-6, "max": 1e-4}, "name": "learning_rate", "type": "double"},
        {
            "categorical_values": ["16", "32", "64", "128"],
            "name": "per_device_train_batch_size",
            "type": "categorical",
        },
    ]`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">def</span> <span class="hljs-title function_">sigopt_hp_space</span>(<span class="hljs-params">trial</span>):
<span class="hljs-meta">... </span>    <span class="hljs-keyword">return</span> [
<span class="hljs-meta">... </span>        {<span class="hljs-string">&quot;bounds&quot;</span>: {<span class="hljs-string">&quot;min&quot;</span>: <span class="hljs-number">1e-6</span>, <span class="hljs-string">&quot;max&quot;</span>: <span class="hljs-number">1e-4</span>}, <span class="hljs-string">&quot;name&quot;</span>: <span class="hljs-string">&quot;learning_rate&quot;</span>, <span class="hljs-string">&quot;type&quot;</span>: <span class="hljs-string">&quot;double&quot;</span>},
<span class="hljs-meta">... </span>        {
<span class="hljs-meta">... </span>            <span class="hljs-string">&quot;categorical_values&quot;</span>: [<span class="hljs-string">&quot;16&quot;</span>, <span class="hljs-string">&quot;32&quot;</span>, <span class="hljs-string">&quot;64&quot;</span>, <span class="hljs-string">&quot;128&quot;</span>],
<span class="hljs-meta">... </span>            <span class="hljs-string">&quot;name&quot;</span>: <span class="hljs-string">&quot;per_device_train_batch_size&quot;</span>,
<span class="hljs-meta">... </span>            <span class="hljs-string">&quot;type&quot;</span>: <span class="hljs-string">&quot;categorical&quot;</span>,
<span class="hljs-meta">... </span>        },
<span class="hljs-meta">... </span>    ]`}}),W=new I({props:{code:`def optuna_hp_space(trial):
    return {
        "learning_rate": trial.suggest_float("learning_rate", 1e-6, 1e-4, log=True),
        "per_device_train_batch_size": trial.suggest_categorical("per_device_train_batch_size", [16, 32, 64, 128]),
    }`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">def</span> <span class="hljs-title function_">optuna_hp_space</span>(<span class="hljs-params">trial</span>):
<span class="hljs-meta">... </span>    <span class="hljs-keyword">return</span> {
<span class="hljs-meta">... </span>        <span class="hljs-string">&quot;learning_rate&quot;</span>: trial.suggest_float(<span class="hljs-string">&quot;learning_rate&quot;</span>, <span class="hljs-number">1e-6</span>, <span class="hljs-number">1e-4</span>, log=<span class="hljs-literal">True</span>),
<span class="hljs-meta">... </span>        <span class="hljs-string">&quot;per_device_train_batch_size&quot;</span>: trial.suggest_categorical(<span class="hljs-string">&quot;per_device_train_batch_size&quot;</span>, [<span class="hljs-number">16</span>, <span class="hljs-number">32</span>, <span class="hljs-number">64</span>, <span class="hljs-number">128</span>]),
<span class="hljs-meta">... </span>    }`}}),Z=new I({props:{code:`def ray_hp_space(trial):
    return {
        "learning_rate": tune.loguniform(1e-6, 1e-4),
        "per_device_train_batch_size": tune.choice([16, 32, 64, 128]),
    }`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">def</span> <span class="hljs-title function_">ray_hp_space</span>(<span class="hljs-params">trial</span>):
<span class="hljs-meta">... </span>    <span class="hljs-keyword">return</span> {
<span class="hljs-meta">... </span>        <span class="hljs-string">&quot;learning_rate&quot;</span>: tune.loguniform(<span class="hljs-number">1e-6</span>, <span class="hljs-number">1e-4</span>),
<span class="hljs-meta">... </span>        <span class="hljs-string">&quot;per_device_train_batch_size&quot;</span>: tune.choice([<span class="hljs-number">16</span>, <span class="hljs-number">32</span>, <span class="hljs-number">64</span>, <span class="hljs-number">128</span>]),
<span class="hljs-meta">... </span>    }`}}),ae=new I({props:{code:`def wandb_hp_space(trial):
    return {
        "method": "random",
        "metric": {"name": "objective", "goal": "minimize"},
        "parameters": {
            "learning_rate": {"distribution": "uniform", "min": 1e-6, "max": 1e-4},
            "per_device_train_batch_size": {"values": [16, 32, 64, 128]},
        },
    }`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">def</span> <span class="hljs-title function_">wandb_hp_space</span>(<span class="hljs-params">trial</span>):
<span class="hljs-meta">... </span>    <span class="hljs-keyword">return</span> {
<span class="hljs-meta">... </span>        <span class="hljs-string">&quot;method&quot;</span>: <span class="hljs-string">&quot;random&quot;</span>,
<span class="hljs-meta">... </span>        <span class="hljs-string">&quot;metric&quot;</span>: {<span class="hljs-string">&quot;name&quot;</span>: <span class="hljs-string">&quot;objective&quot;</span>, <span class="hljs-string">&quot;goal&quot;</span>: <span class="hljs-string">&quot;minimize&quot;</span>},
<span class="hljs-meta">... </span>        <span class="hljs-string">&quot;parameters&quot;</span>: {
<span class="hljs-meta">... </span>            <span class="hljs-string">&quot;learning_rate&quot;</span>: {<span class="hljs-string">&quot;distribution&quot;</span>: <span class="hljs-string">&quot;uniform&quot;</span>, <span class="hljs-string">&quot;min&quot;</span>: <span class="hljs-number">1e-6</span>, <span class="hljs-string">&quot;max&quot;</span>: <span class="hljs-number">1e-4</span>},
<span class="hljs-meta">... </span>            <span class="hljs-string">&quot;per_device_train_batch_size&quot;</span>: {<span class="hljs-string">&quot;values&quot;</span>: [<span class="hljs-number">16</span>, <span class="hljs-number">32</span>, <span class="hljs-number">64</span>, <span class="hljs-number">128</span>]},
<span class="hljs-meta">... </span>        },
<span class="hljs-meta">... </span>    }`}}),se=new I({props:{code:`def model_init(trial):
    return AutoModelForSequenceClassification.from_pretrained(
        model_args.model_name_or_path,
        from_tf=bool(".ckpt" in model_args.model_name_or_path),
        config=config,
        cache_dir=model_args.cache_dir,
        revision=model_args.model_revision,
        use_auth_token=True if model_args.use_auth_token else None,
    )`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">def</span> <span class="hljs-title function_">model_init</span>(<span class="hljs-params">trial</span>):
<span class="hljs-meta">... </span>    <span class="hljs-keyword">return</span> AutoModelForSequenceClassification.from_pretrained(
<span class="hljs-meta">... </span>        model_args.model_name_or_path,
<span class="hljs-meta">... </span>        from_tf=<span class="hljs-built_in">bool</span>(<span class="hljs-string">&quot;.ckpt&quot;</span> <span class="hljs-keyword">in</span> model_args.model_name_or_path),
<span class="hljs-meta">... </span>        config=config,
<span class="hljs-meta">... </span>        cache_dir=model_args.cache_dir,
<span class="hljs-meta">... </span>        revision=model_args.model_revision,
<span class="hljs-meta">... </span>        use_auth_token=<span class="hljs-literal">True</span> <span class="hljs-keyword">if</span> model_args.use_auth_token <span class="hljs-keyword">else</span> <span class="hljs-literal">None</span>,
<span class="hljs-meta">... </span>    )`}}),te=new I({props:{code:`trainer = Trainer(
    model=None,
    args=training_args,
    train_dataset=small_train_dataset,
    eval_dataset=small_eval_dataset,
    compute_metrics=compute_metrics,
    tokenizer=tokenizer,
    model_init=model_init,
    data_collator=data_collator,
)`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>trainer = Trainer(
<span class="hljs-meta">... </span>    model=<span class="hljs-literal">None</span>,
<span class="hljs-meta">... </span>    args=training_args,
<span class="hljs-meta">... </span>    train_dataset=small_train_dataset,
<span class="hljs-meta">... </span>    eval_dataset=small_eval_dataset,
<span class="hljs-meta">... </span>    compute_metrics=compute_metrics,
<span class="hljs-meta">... </span>    tokenizer=tokenizer,
<span class="hljs-meta">... </span>    model_init=model_init,
<span class="hljs-meta">... </span>    data_collator=data_collator,
<span class="hljs-meta">... </span>)`}}),re=new I({props:{code:`best_trial = trainer.hyperparameter_search(
    direction="maximize",
    backend="optuna",
    hp_space=optuna_hp_space,
    n_trials=20,
    compute_objective=compute_objective,
)`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>best_trial = trainer.hyperparameter_search(
<span class="hljs-meta">... </span>    direction=<span class="hljs-string">&quot;maximize&quot;</span>,
<span class="hljs-meta">... </span>    backend=<span class="hljs-string">&quot;optuna&quot;</span>,
<span class="hljs-meta">... </span>    hp_space=optuna_hp_space,
<span class="hljs-meta">... </span>    n_trials=<span class="hljs-number">20</span>,
<span class="hljs-meta">... </span>    compute_objective=compute_objective,
<span class="hljs-meta">... </span>)`}}),ne=new ua({}),{c(){E=l("meta"),He=h(),A=l("h1"),z=l("a"),je=l("span"),d(M.$$.fragment),fa=h(),ge=l("span"),_a=t("Hyperparameter Search using Trainer API"),Ce=h(),$=l("p"),da=t("\u{1F917} Transformers provides a "),le=l("a"),ja=t("Trainer"),ga=t(" class optimized for training \u{1F917} Transformers models, making it easier to start training without manually writing your own training loop. The "),pe=l("a"),ba=t("Trainer"),va=t(" provides API for hyperparameter search. This doc shows how to enable it in example."),Se=h(),P=l("h2"),D=l("a"),be=l("span"),d(L.$$.fragment),wa=h(),ve=l("span"),ya=t("Hyperparameter Search backend"),Fe=h(),f=l("p"),oe=l("a"),$a=t("Trainer"),qa=t(` supports four hyperparameter search backends currently:
`),B=l("a"),ka=t("optuna"),Ea=t(", "),Y=l("a"),Aa=t("sigopt"),Pa=t(", "),G=l("a"),Ta=t("raytune"),xa=t(" and "),J=l("a"),za=t("wandb"),Da=t("."),Ne=h(),ie=l("p"),Ha=t("you should install them before using them as the hyperparameter search backend"),Oe=h(),d(R.$$.fragment),Ie=h(),T=l("h2"),H=l("a"),we=l("span"),d(U.$$.fragment),Ca=h(),ye=l("span"),Sa=t("How to enable Hyperparameter search in example"),Me=h(),ce=l("p"),Fa=t("Define the hyperparameter search space, different backends need different format."),Le=h(),C=l("p"),Na=t("For sigopt, see sigopt "),K=l("a"),Oa=t("object_parameter"),Ia=t(", it\u2019s like following:"),Be=h(),d(Q.$$.fragment),Ye=h(),S=l("p"),Ma=t("For optuna, see optuna "),V=l("a"),La=t("object_parameter"),Ba=t(", it\u2019s like following:"),Ge=h(),d(W.$$.fragment),Je=h(),F=l("p"),Ya=t("For raytune, see raytune "),X=l("a"),Ga=t("object_parameter"),Ja=t(", it\u2019s like following:"),Re=h(),d(Z.$$.fragment),Ue=h(),N=l("p"),Ra=t("For wandb, see wandb "),ee=l("a"),Ua=t("object_parameter"),Ka=t(", it\u2019s like following:"),Ke=h(),d(ae.$$.fragment),Qe=h(),q=l("p"),Qa=t("Define a "),$e=l("code"),Va=t("model_init"),Wa=t(" function and pass it to the "),he=l("a"),Xa=t("Trainer"),Za=t(", as an example:"),Ve=h(),d(se.$$.fragment),We=h(),k=l("p"),es=t("Create a "),me=l("a"),as=t("Trainer"),ss=t(" with your "),qe=l("code"),ts=t("model_init"),rs=t(" function, training arguments, training and test datasets, and evaluation function:"),Xe=h(),d(te.$$.fragment),Ze=h(),u=l("p"),ns=t("Call hyperparameter search, get the best trial parameters, backend could be "),ke=l("code"),ls=t('"optuna"'),ps=t("/"),Ee=l("code"),os=t('"sigopt"'),is=t("/"),Ae=l("code"),cs=t('"wandb"'),hs=t("/"),Pe=l("code"),ms=t('"ray"'),us=t(". direction can be"),Te=l("code"),fs=t('"minimize"'),_s=t(" or "),xe=l("code"),ds=t('"maximize"'),js=t(", which indicates whether to optimize greater or lower objective."),ea=h(),ue=l("p"),gs=t("You could define your own compute_objective function, if not defined, the default compute_objective will be called, and the sum of eval metric like f1 is returned as objective value."),aa=h(),d(re.$$.fragment),sa=h(),x=l("h2"),O=l("a"),ze=l("span"),d(ne.$$.fragment),bs=h(),De=l("span"),vs=t("Hyperparameter search For DDP refinetune"),ta=t(`

Currently, Hyperparameter search for DDP is enabled for optuna and sigopt. Only the rank-zero process will generate the search trial and pass the argument to other ranks.`),this.h()},l(e){const n=pt('[data-svelte="svelte-1phssyn"]',document.head);E=p(n,"META",{name:!0,content:!0}),n.forEach(s),He=m(e),A=p(e,"H1",{class:!0});var na=o(A);z=p(na,"A",{id:!0,class:!0,href:!0});var ys=o(z);je=p(ys,"SPAN",{});var $s=o(je);j(M.$$.fragment,$s),$s.forEach(s),ys.forEach(s),fa=m(na),ge=p(na,"SPAN",{});var qs=o(ge);_a=r(qs,"Hyperparameter Search using Trainer API"),qs.forEach(s),na.forEach(s),Ce=m(e),$=p(e,"P",{});var fe=o($);da=r(fe,"\u{1F917} Transformers provides a "),le=p(fe,"A",{href:!0});var ks=o(le);ja=r(ks,"Trainer"),ks.forEach(s),ga=r(fe," class optimized for training \u{1F917} Transformers models, making it easier to start training without manually writing your own training loop. The "),pe=p(fe,"A",{href:!0});var Es=o(pe);ba=r(Es,"Trainer"),Es.forEach(s),va=r(fe," provides API for hyperparameter search. This doc shows how to enable it in example."),fe.forEach(s),Se=m(e),P=p(e,"H2",{class:!0});var la=o(P);D=p(la,"A",{id:!0,class:!0,href:!0});var As=o(D);be=p(As,"SPAN",{});var Ps=o(be);j(L.$$.fragment,Ps),Ps.forEach(s),As.forEach(s),wa=m(la),ve=p(la,"SPAN",{});var Ts=o(ve);ya=r(Ts,"Hyperparameter Search backend"),Ts.forEach(s),la.forEach(s),Fe=m(e),f=p(e,"P",{});var y=o(f);oe=p(y,"A",{href:!0});var xs=o(oe);$a=r(xs,"Trainer"),xs.forEach(s),qa=r(y,` supports four hyperparameter search backends currently:
`),B=p(y,"A",{href:!0,rel:!0});var zs=o(B);ka=r(zs,"optuna"),zs.forEach(s),Ea=r(y,", "),Y=p(y,"A",{href:!0,rel:!0});var Ds=o(Y);Aa=r(Ds,"sigopt"),Ds.forEach(s),Pa=r(y,", "),G=p(y,"A",{href:!0,rel:!0});var Hs=o(G);Ta=r(Hs,"raytune"),Hs.forEach(s),xa=r(y," and "),J=p(y,"A",{href:!0,rel:!0});var Cs=o(J);za=r(Cs,"wandb"),Cs.forEach(s),Da=r(y,"."),y.forEach(s),Ne=m(e),ie=p(e,"P",{});var Ss=o(ie);Ha=r(Ss,"you should install them before using them as the hyperparameter search backend"),Ss.forEach(s),Oe=m(e),j(R.$$.fragment,e),Ie=m(e),T=p(e,"H2",{class:!0});var pa=o(T);H=p(pa,"A",{id:!0,class:!0,href:!0});var Fs=o(H);we=p(Fs,"SPAN",{});var Ns=o(we);j(U.$$.fragment,Ns),Ns.forEach(s),Fs.forEach(s),Ca=m(pa),ye=p(pa,"SPAN",{});var Os=o(ye);Sa=r(Os,"How to enable Hyperparameter search in example"),Os.forEach(s),pa.forEach(s),Me=m(e),ce=p(e,"P",{});var Is=o(ce);Fa=r(Is,"Define the hyperparameter search space, different backends need different format."),Is.forEach(s),Le=m(e),C=p(e,"P",{});var oa=o(C);Na=r(oa,"For sigopt, see sigopt "),K=p(oa,"A",{href:!0,rel:!0});var Ms=o(K);Oa=r(Ms,"object_parameter"),Ms.forEach(s),Ia=r(oa,", it\u2019s like following:"),oa.forEach(s),Be=m(e),j(Q.$$.fragment,e),Ye=m(e),S=p(e,"P",{});var ia=o(S);Ma=r(ia,"For optuna, see optuna "),V=p(ia,"A",{href:!0,rel:!0});var Ls=o(V);La=r(Ls,"object_parameter"),Ls.forEach(s),Ba=r(ia,", it\u2019s like following:"),ia.forEach(s),Ge=m(e),j(W.$$.fragment,e),Je=m(e),F=p(e,"P",{});var ca=o(F);Ya=r(ca,"For raytune, see raytune "),X=p(ca,"A",{href:!0,rel:!0});var Bs=o(X);Ga=r(Bs,"object_parameter"),Bs.forEach(s),Ja=r(ca,", it\u2019s like following:"),ca.forEach(s),Re=m(e),j(Z.$$.fragment,e),Ue=m(e),N=p(e,"P",{});var ha=o(N);Ra=r(ha,"For wandb, see wandb "),ee=p(ha,"A",{href:!0,rel:!0});var Ys=o(ee);Ua=r(Ys,"object_parameter"),Ys.forEach(s),Ka=r(ha,", it\u2019s like following:"),ha.forEach(s),Ke=m(e),j(ae.$$.fragment,e),Qe=m(e),q=p(e,"P",{});var _e=o(q);Qa=r(_e,"Define a "),$e=p(_e,"CODE",{});var Gs=o($e);Va=r(Gs,"model_init"),Gs.forEach(s),Wa=r(_e," function and pass it to the "),he=p(_e,"A",{href:!0});var Js=o(he);Xa=r(Js,"Trainer"),Js.forEach(s),Za=r(_e,", as an example:"),_e.forEach(s),Ve=m(e),j(se.$$.fragment,e),We=m(e),k=p(e,"P",{});var de=o(k);es=r(de,"Create a "),me=p(de,"A",{href:!0});var Rs=o(me);as=r(Rs,"Trainer"),Rs.forEach(s),ss=r(de," with your "),qe=p(de,"CODE",{});var Us=o(qe);ts=r(Us,"model_init"),Us.forEach(s),rs=r(de," function, training arguments, training and test datasets, and evaluation function:"),de.forEach(s),Xe=m(e),j(te.$$.fragment,e),Ze=m(e),u=p(e,"P",{});var _=o(u);ns=r(_,"Call hyperparameter search, get the best trial parameters, backend could be "),ke=p(_,"CODE",{});var Ks=o(ke);ls=r(Ks,'"optuna"'),Ks.forEach(s),ps=r(_,"/"),Ee=p(_,"CODE",{});var Qs=o(Ee);os=r(Qs,'"sigopt"'),Qs.forEach(s),is=r(_,"/"),Ae=p(_,"CODE",{});var Vs=o(Ae);cs=r(Vs,'"wandb"'),Vs.forEach(s),hs=r(_,"/"),Pe=p(_,"CODE",{});var Ws=o(Pe);ms=r(Ws,'"ray"'),Ws.forEach(s),us=r(_,". direction can be"),Te=p(_,"CODE",{});var Xs=o(Te);fs=r(Xs,'"minimize"'),Xs.forEach(s),_s=r(_," or "),xe=p(_,"CODE",{});var Zs=o(xe);ds=r(Zs,'"maximize"'),Zs.forEach(s),js=r(_,", which indicates whether to optimize greater or lower objective."),_.forEach(s),ea=m(e),ue=p(e,"P",{});var et=o(ue);gs=r(et,"You could define your own compute_objective function, if not defined, the default compute_objective will be called, and the sum of eval metric like f1 is returned as objective value."),et.forEach(s),aa=m(e),j(re.$$.fragment,e),sa=m(e),x=p(e,"H2",{class:!0});var ma=o(x);O=p(ma,"A",{id:!0,class:!0,href:!0});var at=o(O);ze=p(at,"SPAN",{});var st=o(ze);j(ne.$$.fragment,st),st.forEach(s),at.forEach(s),bs=m(ma),De=p(ma,"SPAN",{});var tt=o(De);vs=r(tt,"Hyperparameter search For DDP refinetune"),tt.forEach(s),ma.forEach(s),ta=r(e,`

Currently, Hyperparameter search for DDP is enabled for optuna and sigopt. Only the rank-zero process will generate the search trial and pass the argument to other ranks.`),this.h()},h(){c(E,"name","hf:doc:metadata"),c(E,"content",JSON.stringify(ht)),c(z,"id","hyperparameter-search-using-trainer-api"),c(z,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),c(z,"href","#hyperparameter-search-using-trainer-api"),c(A,"class","relative group"),c(le,"href","/docs/transformers/pr_19043/en/main_classes/trainer#transformers.Trainer"),c(pe,"href","/docs/transformers/pr_19043/en/main_classes/trainer#transformers.Trainer"),c(D,"id","hyperparameter-search-backend"),c(D,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),c(D,"href","#hyperparameter-search-backend"),c(P,"class","relative group"),c(oe,"href","/docs/transformers/pr_19043/en/main_classes/trainer#transformers.Trainer"),c(B,"href","https://optuna.org/"),c(B,"rel","nofollow"),c(Y,"href","https://sigopt.com/"),c(Y,"rel","nofollow"),c(G,"href","https://docs.ray.io/en/latest/tune/index.html"),c(G,"rel","nofollow"),c(J,"href","https://wandb.ai/site/sweeps"),c(J,"rel","nofollow"),c(H,"id","how-to-enable-hyperparameter-search-in-example"),c(H,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),c(H,"href","#how-to-enable-hyperparameter-search-in-example"),c(T,"class","relative group"),c(K,"href","https://docs.sigopt.com/ai-module-api-references/api_reference/objects/object_parameter"),c(K,"rel","nofollow"),c(V,"href","https://optuna.readthedocs.io/en/stable/tutorial/10_key_features/002_configurations.html#sphx-glr-tutorial-10-key-features-002-configurations-py"),c(V,"rel","nofollow"),c(X,"href","https://docs.ray.io/en/latest/tune/api_docs/search_space.html"),c(X,"rel","nofollow"),c(ee,"href","https://docs.wandb.ai/guides/sweeps/configuration"),c(ee,"rel","nofollow"),c(he,"href","/docs/transformers/pr_19043/en/main_classes/trainer#transformers.Trainer"),c(me,"href","/docs/transformers/pr_19043/en/main_classes/trainer#transformers.Trainer"),c(O,"id","hyperparameter-search-for-ddp-refinetune"),c(O,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),c(O,"href","#hyperparameter-search-for-ddp-refinetune"),c(x,"class","relative group")},m(e,n){a(document.head,E),i(e,He,n),i(e,A,n),a(A,z),a(z,je),g(M,je,null),a(A,fa),a(A,ge),a(ge,_a),i(e,Ce,n),i(e,$,n),a($,da),a($,le),a(le,ja),a($,ga),a($,pe),a(pe,ba),a($,va),i(e,Se,n),i(e,P,n),a(P,D),a(D,be),g(L,be,null),a(P,wa),a(P,ve),a(ve,ya),i(e,Fe,n),i(e,f,n),a(f,oe),a(oe,$a),a(f,qa),a(f,B),a(B,ka),a(f,Ea),a(f,Y),a(Y,Aa),a(f,Pa),a(f,G),a(G,Ta),a(f,xa),a(f,J),a(J,za),a(f,Da),i(e,Ne,n),i(e,ie,n),a(ie,Ha),i(e,Oe,n),g(R,e,n),i(e,Ie,n),i(e,T,n),a(T,H),a(H,we),g(U,we,null),a(T,Ca),a(T,ye),a(ye,Sa),i(e,Me,n),i(e,ce,n),a(ce,Fa),i(e,Le,n),i(e,C,n),a(C,Na),a(C,K),a(K,Oa),a(C,Ia),i(e,Be,n),g(Q,e,n),i(e,Ye,n),i(e,S,n),a(S,Ma),a(S,V),a(V,La),a(S,Ba),i(e,Ge,n),g(W,e,n),i(e,Je,n),i(e,F,n),a(F,Ya),a(F,X),a(X,Ga),a(F,Ja),i(e,Re,n),g(Z,e,n),i(e,Ue,n),i(e,N,n),a(N,Ra),a(N,ee),a(ee,Ua),a(N,Ka),i(e,Ke,n),g(ae,e,n),i(e,Qe,n),i(e,q,n),a(q,Qa),a(q,$e),a($e,Va),a(q,Wa),a(q,he),a(he,Xa),a(q,Za),i(e,Ve,n),g(se,e,n),i(e,We,n),i(e,k,n),a(k,es),a(k,me),a(me,as),a(k,ss),a(k,qe),a(qe,ts),a(k,rs),i(e,Xe,n),g(te,e,n),i(e,Ze,n),i(e,u,n),a(u,ns),a(u,ke),a(ke,ls),a(u,ps),a(u,Ee),a(Ee,os),a(u,is),a(u,Ae),a(Ae,cs),a(u,hs),a(u,Pe),a(Pe,ms),a(u,us),a(u,Te),a(Te,fs),a(u,_s),a(u,xe),a(xe,ds),a(u,js),i(e,ea,n),i(e,ue,n),a(ue,gs),i(e,aa,n),g(re,e,n),i(e,sa,n),i(e,x,n),a(x,O),a(O,ze),g(ne,ze,null),a(x,bs),a(x,De),a(De,vs),i(e,ta,n),ra=!0},p:ot,i(e){ra||(b(M.$$.fragment,e),b(L.$$.fragment,e),b(R.$$.fragment,e),b(U.$$.fragment,e),b(Q.$$.fragment,e),b(W.$$.fragment,e),b(Z.$$.fragment,e),b(ae.$$.fragment,e),b(se.$$.fragment,e),b(te.$$.fragment,e),b(re.$$.fragment,e),b(ne.$$.fragment,e),ra=!0)},o(e){v(M.$$.fragment,e),v(L.$$.fragment,e),v(R.$$.fragment,e),v(U.$$.fragment,e),v(Q.$$.fragment,e),v(W.$$.fragment,e),v(Z.$$.fragment,e),v(ae.$$.fragment,e),v(se.$$.fragment,e),v(te.$$.fragment,e),v(re.$$.fragment,e),v(ne.$$.fragment,e),ra=!1},d(e){s(E),e&&s(He),e&&s(A),w(M),e&&s(Ce),e&&s($),e&&s(Se),e&&s(P),w(L),e&&s(Fe),e&&s(f),e&&s(Ne),e&&s(ie),e&&s(Oe),w(R,e),e&&s(Ie),e&&s(T),w(U),e&&s(Me),e&&s(ce),e&&s(Le),e&&s(C),e&&s(Be),w(Q,e),e&&s(Ye),e&&s(S),e&&s(Ge),w(W,e),e&&s(Je),e&&s(F),e&&s(Re),w(Z,e),e&&s(Ue),e&&s(N),e&&s(Ke),w(ae,e),e&&s(Qe),e&&s(q),e&&s(Ve),w(se,e),e&&s(We),e&&s(k),e&&s(Xe),w(te,e),e&&s(Ze),e&&s(u),e&&s(ea),e&&s(ue),e&&s(aa),w(re,e),e&&s(sa),e&&s(x),w(ne),e&&s(ta)}}}const ht={local:"hyperparameter-search-using-trainer-api",sections:[{local:"hyperparameter-search-backend",title:"Hyperparameter Search backend"},{local:"how-to-enable-hyperparameter-search-in-example",title:"How to enable Hyperparameter search in example"},{local:"hyperparameter-search-for-ddp-refinetune",title:"Hyperparameter search For DDP refinetune"}],title:"Hyperparameter Search using Trainer API"};function mt(ws){return it(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class dt extends rt{constructor(E){super();nt(this,E,mt,ct,lt,{})}}export{dt as default,ht as metadata};
