import{S as XE,i as FE,s as RE,e as n,k as d,w as v,t as r,M as HE,c as s,d as o,m as h,a,x as w,h as l,b as f,G as t,g as p,y as E,q as $,o as b,B as x,v as WE,L as zE}from"../chunks/vendor-hf-doc-builder.js";import{T as ds}from"../chunks/Tip-hf-doc-builder.js";import{I as L}from"../chunks/IconCopyLink-hf-doc-builder.js";import{C as I}from"../chunks/CodeBlock-hf-doc-builder.js";import{F as KE,M as LE}from"../chunks/Markdown-hf-doc-builder.js";function UE(M){let c,k,u,y,O,T,P,C,A,N,S,D,q;return c=new I({props:{code:`from transformers import AutoTokenizer, AutoModelForSequenceClassification

# Load tokenizer and PyTorch weights form the Hub
tokenizer = AutoTokenizer.from_pretrained("distilbert-base-uncased")
pt_model = AutoModelForSequenceClassification.from_pretrained("distilbert-base-uncased")
# Save to disk
tokenizer.save_pretrained("local-pt-checkpoint")
pt_model.save_pretrained("local-pt-checkpoint")`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoTokenizer, AutoModelForSequenceClassification

<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-comment"># Load tokenizer and PyTorch weights form the Hub</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>tokenizer = AutoTokenizer.from_pretrained(<span class="hljs-string">&quot;distilbert-base-uncased&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>pt_model = AutoModelForSequenceClassification.from_pretrained(<span class="hljs-string">&quot;distilbert-base-uncased&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-comment"># Save to disk</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>tokenizer.save_pretrained(<span class="hljs-string">&quot;local-pt-checkpoint&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>pt_model.save_pretrained(<span class="hljs-string">&quot;local-pt-checkpoint&quot;</span>)`}}),D=new I({props:{code:"python -m transformers.onnx --model=local-pt-checkpoint onnx/",highlighted:"python -m transformers.onnx --model=local-pt-checkpoint onnx/"}}),{c(){v(c.$$.fragment),k=d(),u=n("p"),y=r("Once the checkpoint is saved, we can export it to ONNX by pointing the "),O=n("code"),T=r("--model"),P=r(`
argument of the `),C=n("code"),A=r("transformers.onnx"),N=r(" package to the desired directory:"),S=d(),v(D.$$.fragment)},l(_){w(c.$$.fragment,_),k=h(_),u=s(_,"P",{});var j=a(u);y=l(j,"Once the checkpoint is saved, we can export it to ONNX by pointing the "),O=s(j,"CODE",{});var z=a(O);T=l(z,"--model"),z.forEach(o),P=l(j,`
argument of the `),C=s(j,"CODE",{});var V=a(C);A=l(V,"transformers.onnx"),V.forEach(o),N=l(j," package to the desired directory:"),j.forEach(o),S=h(_),w(D.$$.fragment,_)},m(_,j){E(c,_,j),p(_,k,j),p(_,u,j),t(u,y),t(u,O),t(O,T),t(u,P),t(u,C),t(C,A),t(u,N),p(_,S,j),E(D,_,j),q=!0},p:zE,i(_){q||($(c.$$.fragment,_),$(D.$$.fragment,_),q=!0)},o(_){b(c.$$.fragment,_),b(D.$$.fragment,_),q=!1},d(_){x(c,_),_&&o(k),_&&o(u),_&&o(S),x(D,_)}}}function VE(M){let c,k;return c=new LE({props:{$$slots:{default:[UE]},$$scope:{ctx:M}}}),{c(){v(c.$$.fragment)},l(u){w(c.$$.fragment,u)},m(u,y){E(c,u,y),k=!0},p(u,y){const O={};y&2&&(O.$$scope={dirty:y,ctx:u}),c.$set(O)},i(u){k||($(c.$$.fragment,u),k=!0)},o(u){b(c.$$.fragment,u),k=!1},d(u){x(c,u)}}}function GE(M){let c,k,u,y,O,T,P,C,A,N,S,D,q;return c=new I({props:{code:`from transformers import AutoTokenizer, TFAutoModelForSequenceClassification

# Load tokenizer and TensorFlow weights from the Hub
tokenizer = AutoTokenizer.from_pretrained("distilbert-base-uncased")
tf_model = TFAutoModelForSequenceClassification.from_pretrained("distilbert-base-uncased")
# Save to disk
tokenizer.save_pretrained("local-tf-checkpoint")
tf_model.save_pretrained("local-tf-checkpoint")`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoTokenizer, TFAutoModelForSequenceClassification

<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-comment"># Load tokenizer and TensorFlow weights from the Hub</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>tokenizer = AutoTokenizer.from_pretrained(<span class="hljs-string">&quot;distilbert-base-uncased&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>tf_model = TFAutoModelForSequenceClassification.from_pretrained(<span class="hljs-string">&quot;distilbert-base-uncased&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-comment"># Save to disk</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>tokenizer.save_pretrained(<span class="hljs-string">&quot;local-tf-checkpoint&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>tf_model.save_pretrained(<span class="hljs-string">&quot;local-tf-checkpoint&quot;</span>)`}}),D=new I({props:{code:"python -m transformers.onnx --model=local-tf-checkpoint onnx/",highlighted:"python -m transformers.onnx --model=local-tf-checkpoint onnx/"}}),{c(){v(c.$$.fragment),k=d(),u=n("p"),y=r("Once the checkpoint is saved, we can export it to ONNX by pointing the "),O=n("code"),T=r("--model"),P=r(`
argument of the `),C=n("code"),A=r("transformers.onnx"),N=r(" package to the desired directory:"),S=d(),v(D.$$.fragment)},l(_){w(c.$$.fragment,_),k=h(_),u=s(_,"P",{});var j=a(u);y=l(j,"Once the checkpoint is saved, we can export it to ONNX by pointing the "),O=s(j,"CODE",{});var z=a(O);T=l(z,"--model"),z.forEach(o),P=l(j,`
argument of the `),C=s(j,"CODE",{});var V=a(C);A=l(V,"transformers.onnx"),V.forEach(o),N=l(j," package to the desired directory:"),j.forEach(o),S=h(_),w(D.$$.fragment,_)},m(_,j){E(c,_,j),p(_,k,j),p(_,u,j),t(u,y),t(u,O),t(O,T),t(u,P),t(u,C),t(C,A),t(u,N),p(_,S,j),E(D,_,j),q=!0},p:zE,i(_){q||($(c.$$.fragment,_),$(D.$$.fragment,_),q=!0)},o(_){b(c.$$.fragment,_),b(D.$$.fragment,_),q=!1},d(_){x(c,_),_&&o(k),_&&o(u),_&&o(S),x(D,_)}}}function JE(M){let c,k;return c=new LE({props:{$$slots:{default:[GE]},$$scope:{ctx:M}}}),{c(){v(c.$$.fragment)},l(u){w(c.$$.fragment,u)},m(u,y){E(c,u,y),k=!0},p(u,y){const O={};y&2&&(O.$$scope={dirty:y,ctx:u}),c.$set(O)},i(u){k||($(c.$$.fragment,u),k=!0)},o(u){b(c.$$.fragment,u),k=!1},d(u){x(c,u)}}}function YE(M){let c,k,u,y,O,T,P,C;return{c(){c=n("p"),k=r("The features that have a "),u=n("code"),y=r("with-past"),O=r(" suffix (e.g. "),T=n("code"),P=r("causal-lm-with-past"),C=r(`)
correspond to model topologies with precomputed hidden states (key and values
in the attention blocks) that can be used for fast autoregressive decoding.`)},l(A){c=s(A,"P",{});var N=a(c);k=l(N,"The features that have a "),u=s(N,"CODE",{});var S=a(u);y=l(S,"with-past"),S.forEach(o),O=l(N," suffix (e.g. "),T=s(N,"CODE",{});var D=a(T);P=l(D,"causal-lm-with-past"),D.forEach(o),C=l(N,`)
correspond to model topologies with precomputed hidden states (key and values
in the attention blocks) that can be used for fast autoregressive decoding.`),N.forEach(o)},m(A,N){p(A,c,N),t(c,k),t(c,u),t(u,y),t(c,O),t(c,T),t(T,P),t(c,C)},d(A){A&&o(c)}}}function QE(M){let c,k,u,y,O;return{c(){c=n("p"),k=r(`A good way to implement a custom ONNX configuration is to look at the existing
implementation in the `),u=n("code"),y=r("configuration_<model_name>.py"),O=r(" file of a similar architecture.")},l(T){c=s(T,"P",{});var P=a(c);k=l(P,`A good way to implement a custom ONNX configuration is to look at the existing
implementation in the `),u=s(P,"CODE",{});var C=a(u);y=l(C,"configuration_<model_name>.py"),C.forEach(o),O=l(P," file of a similar architecture."),P.forEach(o)},m(T,P){p(T,c,P),t(c,k),t(c,u),t(u,y),t(c,O)},d(T){T&&o(c)}}}function ZE(M){let c,k,u,y,O,T,P,C,A,N,S,D,q,_,j,z,V,we,Re,G,H,Ee,se;return{c(){c=n("p"),k=r("Notice that "),u=n("code"),y=r("inputs"),O=r(" property for "),T=n("code"),P=r("DistilBertOnnxConfig"),C=r(` returns an
`),A=n("code"),N=r("OrderedDict"),S=r(`. This ensures that the inputs are matched with their relative
position within the `),D=n("code"),q=r("PreTrainedModel.forward()"),_=r(` method when tracing the graph.
We recommend using an `),j=n("code"),z=r("OrderedDict"),V=r(" for the "),we=n("code"),Re=r("inputs"),G=r(" and "),H=n("code"),Ee=r("outputs"),se=r(` properties
when implementing custom ONNX configurations.`)},l($e){c=s($e,"P",{});var B=a(c);k=l(B,"Notice that "),u=s(B,"CODE",{});var rn=a(u);y=l(rn,"inputs"),rn.forEach(o),O=l(B," property for "),T=s(B,"CODE",{});var qt=a(T);P=l(qt,"DistilBertOnnxConfig"),qt.forEach(o),C=l(B,` returns an
`),A=s(B,"CODE",{});var J=a(A);N=l(J,"OrderedDict"),J.forEach(o),S=l(B,`. This ensures that the inputs are matched with their relative
position within the `),D=s(B,"CODE",{});var ln=a(D);q=l(ln,"PreTrainedModel.forward()"),ln.forEach(o),_=l(B,` method when tracing the graph.
We recommend using an `),j=s(B,"CODE",{});var ae=a(j);z=l(ae,"OrderedDict"),ae.forEach(o),V=l(B," for the "),we=s(B,"CODE",{});var pn=a(we);Re=l(pn,"inputs"),pn.forEach(o),G=l(B," and "),H=s(B,"CODE",{});var dn=a(H);Ee=l(dn,"outputs"),dn.forEach(o),se=l(B,` properties
when implementing custom ONNX configurations.`),B.forEach(o)},m($e,B){p($e,c,B),t(c,k),t(c,u),t(u,y),t(c,O),t(c,T),t(T,P),t(c,C),t(c,A),t(A,N),t(c,S),t(c,D),t(D,q),t(c,_),t(c,j),t(j,z),t(c,V),t(c,we),t(we,Re),t(c,G),t(c,H),t(H,Ee),t(c,se)},d($e){$e&&o(c)}}}function e$(M){let c,k,u,y,O,T,P,C;return{c(){c=n("p"),k=r("All of the base properties and methods associated with "),u=n("a"),y=r("OnnxConfig"),O=r(` and the
other configuration classes can be overriden if needed. Check out
`),T=n("code"),P=r("BartOnnxConfig"),C=r(" for an advanced example."),this.h()},l(A){c=s(A,"P",{});var N=a(c);k=l(N,"All of the base properties and methods associated with "),u=s(N,"A",{href:!0});var S=a(u);y=l(S,"OnnxConfig"),S.forEach(o),O=l(N,` and the
other configuration classes can be overriden if needed. Check out
`),T=s(N,"CODE",{});var D=a(T);P=l(D,"BartOnnxConfig"),D.forEach(o),C=l(N," for an advanced example."),N.forEach(o),this.h()},h(){f(u,"href","/docs/transformers/pr_17548/en/main_classes/onnx#transformers.onnx.OnnxConfig")},m(A,N){p(A,c,N),t(c,k),t(c,u),t(u,y),t(c,O),t(c,T),t(T,P),t(c,C)},d(A){A&&o(c)}}}function t$(M){let c,k,u,y,O,T,P,C,A,N,S;return{c(){c=n("p"),k=r(`If your model is larger than 2GB, you will see that many additional files are
created during the export. This is `),u=n("em"),y=r("expected"),O=r(" because ONNX uses "),T=n("a"),P=r(`Protocol
Buffers`),C=r(` to store the model and
these have a size limit of 2GB. See the `),A=n("a"),N=r(`ONNX
documentation`),S=r(`
for instructions on how to load models with external data.`),this.h()},l(D){c=s(D,"P",{});var q=a(c);k=l(q,`If your model is larger than 2GB, you will see that many additional files are
created during the export. This is `),u=s(q,"EM",{});var _=a(u);y=l(_,"expected"),_.forEach(o),O=l(q," because ONNX uses "),T=s(q,"A",{href:!0,rel:!0});var j=a(T);P=l(j,`Protocol
Buffers`),j.forEach(o),C=l(q,` to store the model and
these have a size limit of 2GB. See the `),A=s(q,"A",{href:!0,rel:!0});var z=a(A);N=l(z,`ONNX
documentation`),z.forEach(o),S=l(q,`
for instructions on how to load models with external data.`),q.forEach(o),this.h()},h(){f(T,"href","https://developers.google.com/protocol-buffers/"),f(T,"rel","nofollow"),f(A,"href","https://github.com/onnx/onnx/blob/master/docs/ExternalData.md"),f(A,"rel","nofollow")},m(D,q){p(D,c,q),t(c,k),t(c,u),t(u,y),t(c,O),t(c,T),t(T,P),t(c,C),t(c,A),t(A,N),t(c,S)},d(D){D&&o(c)}}}function o$(M){let c,k;return{c(){c=n("p"),k=r(`This is the very beginning of our experiments with TorchScript and we are still exploring its capabilities with
variable-input-size models. It is a focus of interest to us and we will deepen our analysis in upcoming releases,
with more code examples, a more flexible implementation, and benchmarks comparing python-based codes with compiled
TorchScript.`)},l(u){c=s(u,"P",{});var y=a(c);k=l(y,`This is the very beginning of our experiments with TorchScript and we are still exploring its capabilities with
variable-input-size models. It is a focus of interest to us and we will deepen our analysis in upcoming releases,
with more code examples, a more flexible implementation, and benchmarks comparing python-based codes with compiled
TorchScript.`),y.forEach(o)},m(u,y){p(u,c,y),t(c,k)},d(u){u&&o(c)}}}function n$(M){let c,k,u,y,O,T,P,C,A,N,S,D,q,_,j,z,V,we,Re,G,H,Ee,se,$e,B,rn,qt,J,ln,ae,pn,dn,hs,Ed,$d,gl,hn,bd,_l,He,xd,fs,yd,kd,vl,fn,Td,wl,m,cs,Od,Nd,us,jd,Ad,ms,Dd,Cd,gs,Pd,Sd,_s,qd,Id,vs,Bd,Md,ws,zd,Ld,Es,Xd,Fd,$s,Rd,Hd,bs,Wd,Kd,xs,Ud,Vd,ys,Gd,Jd,ks,Yd,Qd,Ts,Zd,eh,Os,th,oh,Ns,nh,sh,js,ah,rh,As,lh,ih,Ds,ph,dh,Cs,hh,fh,Ps,ch,uh,Ss,mh,gh,qs,_h,vh,Is,wh,Eh,Bs,$h,bh,Ms,xh,yh,zs,kh,Th,Ls,Oh,Nh,Xs,jh,Ah,Fs,Dh,Ch,Rs,Ph,Sh,Hs,qh,Ih,Ws,Bh,Mh,Ks,zh,Lh,Us,Xh,El,cn,Fh,$l,We,It,Rh,Vs,Hh,Wh,Kh,Gs,Uh,bl,be,Ke,Js,Bt,Vh,Ys,Gh,xl,un,Jh,yl,Mt,kl,Ue,Yh,Qs,Qh,Zh,Tl,zt,Ol,mn,ef,Nl,Lt,jl,gn,tf,Al,Xt,Dl,re,of,Zs,nf,sf,ea,af,rf,Cl,Q,lf,ta,pf,df,Ft,hf,ff,Rt,cf,uf,Pl,Ht,Sl,Ve,mf,oa,gf,_f,ql,Wt,Il,Ge,vf,Kt,wf,Ef,Bl,Ut,Ml,_n,$f,zl,Je,Ll,xe,Ye,na,Vt,bf,sa,xf,Xl,Qe,yf,aa,kf,Tf,Fl,Ze,ra,Gt,la,Of,Nf,ia,jf,Af,X,Jt,Yt,pa,Df,Cf,da,Pf,Sf,ha,fa,qf,If,Qt,Zt,ca,Bf,Mf,ua,zf,Lf,ma,ga,Xf,Ff,eo,_a,va,Rf,Hf,wa,Ea,Wf,Kf,to,$a,ba,Uf,Vf,xa,ya,Gf,Jf,oo,no,ka,Yf,Qf,Ta,Zf,ec,Oa,Na,tc,oc,so,ja,Aa,nc,sc,Da,Ca,ac,rc,ao,Pa,Sa,lc,ic,qa,Ia,pc,Rl,et,dc,Ba,hc,fc,Hl,ro,Wl,le,cc,Ma,uc,mc,za,gc,_c,Kl,lo,Ul,vn,vc,Vl,io,Gl,Z,wc,La,Ec,$c,Xa,bc,xc,Fa,yc,kc,Jl,tt,Yl,ye,ot,Ra,po,Tc,Ha,Oc,Ql,wn,Nc,Zl,ie,Wa,jc,Ac,Ka,Dc,Cc,Ua,Pc,ei,En,Sc,ti,ke,nt,Va,ho,qc,Ga,Ic,oi,$n,Bc,ni,pe,bn,Mc,xn,zc,Lc,yn,Xc,kn,Fc,Rc,Tn,Hc,On,Wc,si,st,ai,at,Kc,Ja,Uc,Vc,ri,fo,li,W,Gc,Ya,Jc,Yc,Qa,Qc,Zc,Za,eu,tu,er,ou,nu,ii,rt,pi,Nn,su,di,co,hi,jn,au,fi,uo,ci,An,ru,ui,mo,mi,K,lu,tr,iu,pu,or,du,hu,nr,fu,cu,sr,uu,mu,gi,go,_i,lt,vi,Te,it,ar,_o,gu,rr,_u,wi,de,vu,lr,wu,Eu,ir,$u,bu,Ei,vo,$i,F,xu,pr,yu,ku,dr,Tu,Ou,hr,Nu,ju,fr,Au,Du,cr,Cu,Pu,bi,wo,xi,pt,yi,Oe,dt,ur,Eo,Su,mr,qu,ki,he,Iu,gr,Bu,Mu,_r,zu,Lu,Ti,$o,Oi,ht,Xu,vr,Fu,Ru,Ni,Ne,ft,wr,bo,Hu,Er,Wu,ji,Dn,Ku,Ai,fe,xo,Uu,$r,Vu,Gu,Ju,Cn,Yu,br,Qu,Zu,Pn,em,xr,tm,Di,ct,om,yo,nm,sm,Ci,je,ut,yr,ko,am,kr,rm,Pi,mt,Si,gt,lm,To,im,pm,qi,Sn,dm,Ii,qn,hm,Bi,_t,Tr,fm,cm,Oo,um,Or,mm,gm,Mi,In,_m,zi,Ae,vt,Nr,No,vm,jr,wm,Li,ce,Em,Ar,$m,bm,Dr,xm,ym,Xi,ee,km,Cr,Tm,Om,Pr,Nm,jm,Sr,Am,Dm,Fi,wt,Cm,qr,Pm,Sm,Ri,De,Et,Ir,jo,qm,Br,Im,Hi,Bn,Bm,Wi,Mn,Mm,Ki,zn,Mr,zm,Ui,Ln,Lm,Vi,Xn,Xm,Gi,Ce,$t,zr,Ao,Fm,Lr,Rm,Ji,Fn,Hm,Yi,Pe,bt,Xr,Do,Wm,Fr,Km,Qi,Y,Um,Rr,Vm,Gm,Hr,Jm,Ym,Wr,Qm,Zm,Kr,eg,Zi,Co,ep,Se,xt,Ur,Po,tg,Vr,og,tp,te,ng,Gr,sg,ag,Jr,rg,lg,Yr,ig,pg,op,So,np,qe,yt,Qr,qo,dg,Zr,hg,sp,kt,fg,el,cg,ug,ap,Io,rp,Ie,Tt,tl,Bo,mg,ol,gg,lp,ue,_g,Mo,vg,wg,zo,Eg,$g,ip,me,nl,bg,xg,Rn,yg,Lo,kg,Tg,Be,Og,Xo,Ng,jg,Fo,Ag,Dg,pp,Me,Ot,sl,Ro,Cg,al,Pg,dp,R,Sg,Ho,qg,Ig,Wo,Bg,Mg,Ko,zg,Lg,Uo,Xg,Fg,Vo,Rg,Hg,hp,ze,Nt,rl,Go,Wg,ll,Kg,fp,Hn,Ug,cp,Wn,Le,Vg,Jo,Gg,Jg,Yo,Yg,Qg,up,Xe,jt,il,Qo,Zg,pl,e_,mp,ge,t_,Zo,o_,n_,dl,s_,a_,gp,en,_p,Kn,r_,vp,Un,l_,wp,tn,Ep,Vn,i_,$p,on,bp,Gn,p_,xp,At,d_,nn,h_,f_,yp;return T=new L({}),se=new L({}),Bt=new L({}),Mt=new I({props:{code:"pip install transformers[onnx]",highlighted:"pip install transformers[onnx]"}}),zt=new I({props:{code:`python -m transformers.onnx --help

usage: Hugging Face Transformers ONNX exporter [-h] -m MODEL [--feature {causal-lm, ...}] [--opset OPSET] [--atol ATOL] output

positional arguments:
  output                Path indicating where to store generated ONNX model.

optional arguments:
  -h, --help            show this help message and exit
  -m MODEL, --model MODEL
                        Model ID on huggingface.co or path on disk to load model from.
  --feature {causal-lm, ...}
                        The type of features to export the model with.
  --opset OPSET         ONNX opset version to export the model with.
  --atol ATOL           Absolute difference tolerence when validating the model.`,highlighted:`python -m transformers.onnx --<span class="hljs-built_in">help</span>

usage: Hugging Face Transformers ONNX exporter [-h] -m MODEL [--feature {causal-lm, ...}] [--opset OPSET] [--atol ATOL] output

positional arguments:
  output                Path indicating <span class="hljs-built_in">where</span> to store generated ONNX model.

optional arguments:
  -h, --<span class="hljs-built_in">help</span>            show this <span class="hljs-built_in">help</span> message and <span class="hljs-built_in">exit</span>
  -m MODEL, --model MODEL
                        Model ID on huggingface.co or path on disk to load model from.
  --feature {causal-lm, ...}
                        The <span class="hljs-built_in">type</span> of features to <span class="hljs-built_in">export</span> the model with.
  --opset OPSET         ONNX opset version to <span class="hljs-built_in">export</span> the model with.
  --atol ATOL           Absolute difference tolerence when validating the model.`}}),Lt=new I({props:{code:"python -m transformers.onnx --model=distilbert-base-uncased onnx/",highlighted:"python -m transformers.onnx --model=distilbert-base-uncased onnx/"}}),Xt=new I({props:{code:`Validating ONNX model...
        -[\u2713] ONNX model output names match reference model ({'last_hidden_state'})
        - Validating ONNX Model output "last_hidden_state":
                -[\u2713] (2, 8, 768) matches (2, 8, 768)
                -[\u2713] all values close (atol: 1e-05)
All good, model saved at: onnx/model.onnx`,highlighted:`Validating ONNX model...
        -[\u2713] ONNX model output names match reference model ({<span class="hljs-string">&#x27;last_hidden_state&#x27;</span>})
        - Validating ONNX Model output <span class="hljs-string">&quot;last_hidden_state&quot;</span>:
                -[\u2713] (2, 8, 768) matches (2, 8, 768)
                -[\u2713] all values close (atol: 1e-05)
All good, model saved at: onnx/model.onnx`}}),Ht=new I({props:{code:`from transformers import AutoTokenizer
from onnxruntime import InferenceSession

tokenizer = AutoTokenizer.from_pretrained("distilbert-base-uncased")
session = InferenceSession("onnx/model.onnx")
# ONNX Runtime expects NumPy arrays as input
inputs = tokenizer("Using DistilBERT with ONNX Runtime!", return_tensors="np")
outputs = session.run(output_names=["last_hidden_state"], input_feed=dict(inputs))`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoTokenizer
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> onnxruntime <span class="hljs-keyword">import</span> InferenceSession

<span class="hljs-meta">&gt;&gt;&gt; </span>tokenizer = AutoTokenizer.from_pretrained(<span class="hljs-string">&quot;distilbert-base-uncased&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>session = InferenceSession(<span class="hljs-string">&quot;onnx/model.onnx&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-comment"># ONNX Runtime expects NumPy arrays as input</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>inputs = tokenizer(<span class="hljs-string">&quot;Using DistilBERT with ONNX Runtime!&quot;</span>, return_tensors=<span class="hljs-string">&quot;np&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>outputs = session.run(output_names=[<span class="hljs-string">&quot;last_hidden_state&quot;</span>], input_feed=<span class="hljs-built_in">dict</span>(inputs))`}}),Wt=new I({props:{code:`from transformers.models.distilbert import DistilBertConfig, DistilBertOnnxConfig

config = DistilBertConfig()
onnx_config = DistilBertOnnxConfig(config)
print(list(onnx_config.outputs.keys()))`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers.models.distilbert <span class="hljs-keyword">import</span> DistilBertConfig, DistilBertOnnxConfig

<span class="hljs-meta">&gt;&gt;&gt; </span>config = DistilBertConfig()
<span class="hljs-meta">&gt;&gt;&gt; </span>onnx_config = DistilBertOnnxConfig(config)
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-built_in">print</span>(<span class="hljs-built_in">list</span>(onnx_config.outputs.keys()))
[<span class="hljs-string">&quot;last_hidden_state&quot;</span>]`}}),Ut=new I({props:{code:"python -m transformers.onnx --model=keras-io/transformers-qa onnx/",highlighted:"python -m transformers.onnx --model=keras-io/transformers-qa onnx/"}}),Je=new KE({props:{pytorch:!0,tensorflow:!0,jax:!1,$$slots:{tensorflow:[JE],pytorch:[VE]},$$scope:{ctx:M}}}),Vt=new L({}),ro=new I({props:{code:`from transformers.onnx.features import FeaturesManager

distilbert_features = list(FeaturesManager.get_supported_features_for_model_type("distilbert").keys())
print(distilbert_features)`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers.onnx.features <span class="hljs-keyword">import</span> FeaturesManager

<span class="hljs-meta">&gt;&gt;&gt; </span>distilbert_features = <span class="hljs-built_in">list</span>(FeaturesManager.get_supported_features_for_model_type(<span class="hljs-string">&quot;distilbert&quot;</span>).keys())
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-built_in">print</span>(distilbert_features)
[<span class="hljs-string">&quot;default&quot;</span>, <span class="hljs-string">&quot;masked-lm&quot;</span>, <span class="hljs-string">&quot;causal-lm&quot;</span>, <span class="hljs-string">&quot;sequence-classification&quot;</span>, <span class="hljs-string">&quot;token-classification&quot;</span>, <span class="hljs-string">&quot;question-answering&quot;</span>]`}}),lo=new I({props:{code:`python -m transformers.onnx --model=distilbert-base-uncased-finetuned-sst-2-english \\
                            --feature=sequence-classification onnx/`,highlighted:`python -m transformers.onnx --model=distilbert-base-uncased-finetuned-sst-2-english \\
                            --feature=sequence-classification onnx/`}}),io=new I({props:{code:`Validating ONNX model...
        -[\u2713] ONNX model output names match reference model ({'logits'})
        - Validating ONNX Model output "logits":
                -[\u2713] (2, 2) matches (2, 2)
                -[\u2713] all values close (atol: 1e-05)
All good, model saved at: onnx/model.onnx`,highlighted:`Validating ONNX model...
        -[\u2713] ONNX model output names match reference model ({<span class="hljs-string">&#x27;logits&#x27;</span>})
        - Validating ONNX Model output <span class="hljs-string">&quot;logits&quot;</span>:
                -[\u2713] (2, 2) matches (2, 2)
                -[\u2713] all values close (atol: 1e-05)
All good, model saved at: onnx/model.onnx`}}),tt=new ds({props:{$$slots:{default:[YE]},$$scope:{ctx:M}}}),po=new L({}),ho=new L({}),st=new ds({props:{$$slots:{default:[QE]},$$scope:{ctx:M}}}),fo=new I({props:{code:`from typing import Mapping, OrderedDict
from transformers.onnx import OnnxConfig


class DistilBertOnnxConfig(OnnxConfig):
    @property
    def inputs(self) -> Mapping[str, Mapping[int, str]]:
        return OrderedDict(
            [
                ("input_ids", {0: "batch", 1: "sequence"}),
                ("attention_mask", {0: "batch", 1: "sequence"}),
            ]
        )`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> typing <span class="hljs-keyword">import</span> Mapping, OrderedDict
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers.onnx <span class="hljs-keyword">import</span> OnnxConfig


<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">class</span> <span class="hljs-title class_">DistilBertOnnxConfig</span>(<span class="hljs-title class_ inherited__">OnnxConfig</span>):
<span class="hljs-meta">... </span>    @<span class="hljs-built_in">property</span>
<span class="hljs-meta">... </span>    <span class="hljs-keyword">def</span> <span class="hljs-title function_">inputs</span>(<span class="hljs-params">self</span>) -&gt; Mapping[<span class="hljs-built_in">str</span>, Mapping[<span class="hljs-built_in">int</span>, <span class="hljs-built_in">str</span>]]:
<span class="hljs-meta">... </span>        <span class="hljs-keyword">return</span> OrderedDict(
<span class="hljs-meta">... </span>            [
<span class="hljs-meta">... </span>                (<span class="hljs-string">&quot;input_ids&quot;</span>, {<span class="hljs-number">0</span>: <span class="hljs-string">&quot;batch&quot;</span>, <span class="hljs-number">1</span>: <span class="hljs-string">&quot;sequence&quot;</span>}),
<span class="hljs-meta">... </span>                (<span class="hljs-string">&quot;attention_mask&quot;</span>, {<span class="hljs-number">0</span>: <span class="hljs-string">&quot;batch&quot;</span>, <span class="hljs-number">1</span>: <span class="hljs-string">&quot;sequence&quot;</span>}),
<span class="hljs-meta">... </span>            ]
<span class="hljs-meta">... </span>        )`}}),rt=new ds({props:{$$slots:{default:[ZE]},$$scope:{ctx:M}}}),co=new I({props:{code:`from transformers import AutoConfig

config = AutoConfig.from_pretrained("distilbert-base-uncased")
onnx_config = DistilBertOnnxConfig(config)`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoConfig

<span class="hljs-meta">&gt;&gt;&gt; </span>config = AutoConfig.from_pretrained(<span class="hljs-string">&quot;distilbert-base-uncased&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>onnx_config = DistilBertOnnxConfig(config)`}}),uo=new I({props:{code:"print(onnx_config.default_onnx_opset)",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-built_in">print</span>(onnx_config.default_onnx_opset)
<span class="hljs-number">11</span>`}}),mo=new I({props:{code:"print(onnx_config.outputs)",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-built_in">print</span>(onnx_config.outputs)
OrderedDict([(<span class="hljs-string">&quot;last_hidden_state&quot;</span>, {<span class="hljs-number">0</span>: <span class="hljs-string">&quot;batch&quot;</span>, <span class="hljs-number">1</span>: <span class="hljs-string">&quot;sequence&quot;</span>})])`}}),go=new I({props:{code:`from transformers import AutoConfig

config = AutoConfig.from_pretrained("distilbert-base-uncased")
onnx_config_for_seq_clf = DistilBertOnnxConfig(config, task="sequence-classification")
print(onnx_config_for_seq_clf.outputs)`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoConfig

<span class="hljs-meta">&gt;&gt;&gt; </span>config = AutoConfig.from_pretrained(<span class="hljs-string">&quot;distilbert-base-uncased&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>onnx_config_for_seq_clf = DistilBertOnnxConfig(config, task=<span class="hljs-string">&quot;sequence-classification&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-built_in">print</span>(onnx_config_for_seq_clf.outputs)
OrderedDict([(<span class="hljs-string">&#x27;logits&#x27;</span>, {<span class="hljs-number">0</span>: <span class="hljs-string">&#x27;batch&#x27;</span>})])`}}),lt=new ds({props:{$$slots:{default:[e$]},$$scope:{ctx:M}}}),_o=new L({}),vo=new I({props:{code:`from pathlib import Path
from transformers.onnx import export
from transformers import AutoTokenizer, AutoModel

onnx_path = Path("model.onnx")
model_ckpt = "distilbert-base-uncased"
base_model = AutoModel.from_pretrained(model_ckpt)
tokenizer = AutoTokenizer.from_pretrained(model_ckpt)

onnx_inputs, onnx_outputs = export(tokenizer, base_model, onnx_config, onnx_config.default_onnx_opset, onnx_path)`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> pathlib <span class="hljs-keyword">import</span> Path
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers.onnx <span class="hljs-keyword">import</span> export
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoTokenizer, AutoModel

<span class="hljs-meta">&gt;&gt;&gt; </span>onnx_path = Path(<span class="hljs-string">&quot;model.onnx&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>model_ckpt = <span class="hljs-string">&quot;distilbert-base-uncased&quot;</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>base_model = AutoModel.from_pretrained(model_ckpt)
<span class="hljs-meta">&gt;&gt;&gt; </span>tokenizer = AutoTokenizer.from_pretrained(model_ckpt)

<span class="hljs-meta">&gt;&gt;&gt; </span>onnx_inputs, onnx_outputs = export(tokenizer, base_model, onnx_config, onnx_config.default_onnx_opset, onnx_path)`}}),wo=new I({props:{code:`import onnx

onnx_model = onnx.load("model.onnx")
onnx.checker.check_model(onnx_model)`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">import</span> onnx

<span class="hljs-meta">&gt;&gt;&gt; </span>onnx_model = onnx.load(<span class="hljs-string">&quot;model.onnx&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>onnx.checker.check_model(onnx_model)`}}),pt=new ds({props:{$$slots:{default:[t$]},$$scope:{ctx:M}}}),Eo=new L({}),$o=new I({props:{code:`from transformers.onnx import validate_model_outputs

validate_model_outputs(
    onnx_config, tokenizer, base_model, onnx_path, onnx_outputs, onnx_config.atol_for_validation
)`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers.onnx <span class="hljs-keyword">import</span> validate_model_outputs

<span class="hljs-meta">&gt;&gt;&gt; </span>validate_model_outputs(
<span class="hljs-meta">... </span>    onnx_config, tokenizer, base_model, onnx_path, onnx_outputs, onnx_config.atol_for_validation
<span class="hljs-meta">... </span>)`}}),bo=new L({}),ko=new L({}),mt=new ds({props:{$$slots:{default:[o$]},$$scope:{ctx:M}}}),No=new L({}),jo=new L({}),Ao=new L({}),Do=new L({}),Co=new I({props:{code:`from transformers import BertModel, BertTokenizer, BertConfig
import torch

enc = BertTokenizer.from_pretrained("bert-base-uncased")

# Tokenizing input text
text = "[CLS] Who was Jim Henson ? [SEP] Jim Henson was a puppeteer [SEP]"
tokenized_text = enc.tokenize(text)

# Masking one of the input tokens
masked_index = 8
tokenized_text[masked_index] = "[MASK]"
indexed_tokens = enc.convert_tokens_to_ids(tokenized_text)
segments_ids = [0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1]

# Creating a dummy input
tokens_tensor = torch.tensor([indexed_tokens])
segments_tensors = torch.tensor([segments_ids])
dummy_input = [tokens_tensor, segments_tensors]

# Initializing the model with the torchscript flag
# Flag set to True even though it is not necessary as this model does not have an LM Head.
config = BertConfig(
    vocab_size_or_config_json_file=32000,
    hidden_size=768,
    num_hidden_layers=12,
    num_attention_heads=12,
    intermediate_size=3072,
    torchscript=True,
)

# Instantiating the model
model = BertModel(config)

# The model needs to be in evaluation mode
model.eval()

# If you are instantiating the model with *from_pretrained* you can also easily set the TorchScript flag
model = BertModel.from_pretrained("bert-base-uncased", torchscript=True)

# Creating the trace
traced_model = torch.jit.trace(model, [tokens_tensor, segments_tensors])
torch.jit.save(traced_model, "traced_bert.pt")`,highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> BertModel, BertTokenizer, BertConfig
<span class="hljs-keyword">import</span> torch

enc = BertTokenizer.from_pretrained(<span class="hljs-string">&quot;bert-base-uncased&quot;</span>)

<span class="hljs-comment"># Tokenizing input text</span>
text = <span class="hljs-string">&quot;[CLS] Who was Jim Henson ? [SEP] Jim Henson was a puppeteer [SEP]&quot;</span>
tokenized_text = enc.tokenize(text)

<span class="hljs-comment"># Masking one of the input tokens</span>
masked_index = <span class="hljs-number">8</span>
tokenized_text[masked_index] = <span class="hljs-string">&quot;[MASK]&quot;</span>
indexed_tokens = enc.convert_tokens_to_ids(tokenized_text)
segments_ids = [<span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>]

<span class="hljs-comment"># Creating a dummy input</span>
tokens_tensor = torch.tensor([indexed_tokens])
segments_tensors = torch.tensor([segments_ids])
dummy_input = [tokens_tensor, segments_tensors]

<span class="hljs-comment"># Initializing the model with the torchscript flag</span>
<span class="hljs-comment"># Flag set to True even though it is not necessary as this model does not have an LM Head.</span>
config = BertConfig(
    vocab_size_or_config_json_file=<span class="hljs-number">32000</span>,
    hidden_size=<span class="hljs-number">768</span>,
    num_hidden_layers=<span class="hljs-number">12</span>,
    num_attention_heads=<span class="hljs-number">12</span>,
    intermediate_size=<span class="hljs-number">3072</span>,
    torchscript=<span class="hljs-literal">True</span>,
)

<span class="hljs-comment"># Instantiating the model</span>
model = BertModel(config)

<span class="hljs-comment"># The model needs to be in evaluation mode</span>
model.<span class="hljs-built_in">eval</span>()

<span class="hljs-comment"># If you are instantiating the model with *from_pretrained* you can also easily set the TorchScript flag</span>
model = BertModel.from_pretrained(<span class="hljs-string">&quot;bert-base-uncased&quot;</span>, torchscript=<span class="hljs-literal">True</span>)

<span class="hljs-comment"># Creating the trace</span>
traced_model = torch.jit.trace(model, [tokens_tensor, segments_tensors])
torch.jit.save(traced_model, <span class="hljs-string">&quot;traced_bert.pt&quot;</span>)`}}),Po=new L({}),So=new I({props:{code:`loaded_model = torch.jit.load("traced_bert.pt")
loaded_model.eval()

all_encoder_layers, pooled_output = loaded_model(*dummy_input)`,highlighted:`loaded_model = torch.jit.load(<span class="hljs-string">&quot;traced_bert.pt&quot;</span>)
loaded_model.<span class="hljs-built_in">eval</span>()

all_encoder_layers, pooled_output = loaded_model(*dummy_input)`}}),qo=new L({}),Io=new I({props:{code:"traced_model(tokens_tensor, segments_tensors)",highlighted:"traced_model(tokens_tensor, segments_tensors)"}}),Bo=new L({}),Ro=new L({}),Go=new L({}),Qo=new L({}),en=new I({props:{code:`from transformers import BertModel, BertTokenizer, BertConfig
import torch
import torch.neuron`,highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> BertModel, BertTokenizer, BertConfig
<span class="hljs-keyword">import</span> torch
<span class="hljs-keyword">import</span> torch.neuron`}}),tn=new I({props:{code:"torch.jit.trace(model, [tokens_tensor, segments_tensors])",highlighted:"torch.jit.trace(model, [tokens_tensor, segments_tensors])"}}),on=new I({props:{code:"torch.neuron.trace(model, [token_tensor, segments_tensors])",highlighted:"torch.neuron.trace(model, [token_tensor, segments_tensors])"}}),{c(){c=n("meta"),k=d(),u=n("h1"),y=n("a"),O=n("span"),v(T.$$.fragment),P=d(),C=n("span"),A=r("Export \u{1F917} Transformers Models"),N=d(),S=n("p"),D=r(`If you need to deploy \u{1F917} Transformers models in production environments, we
recommend exporting them to a serialized format that can be loaded and executed
on specialized runtimes and hardware. In this guide, we\u2019ll show you how to
export \u{1F917} Transformers models in two widely used formats: ONNX and TorchScript.`),q=d(),_=n("p"),j=r(`Once exported, a model can optimized for inference via techniques such as
quantization and pruning. If you are interested in optimizing your models to run
with maximum efficiency, check out the `),z=n("a"),V=r(`\u{1F917} Optimum
library`),we=r("."),Re=d(),G=n("h2"),H=n("a"),Ee=n("span"),v(se.$$.fragment),$e=d(),B=n("span"),rn=r("ONNX"),qt=d(),J=n("p"),ln=r("The "),ae=n("a"),pn=r("ONNX (Open Neural Network eXchange)"),dn=r(` project is an open
standard that defines a common set of operators and a common file format to
represent deep learning models in a wide variety of frameworks, including
PyTorch and TensorFlow. When a model is exported to the ONNX format, these
operators are used to construct a computational graph (often called an
`),hs=n("em"),Ed=r("intermediate representation"),$d=r(`) which represents the flow of data through the
neural network.`),gl=d(),hn=n("p"),bd=r(`By exposing a graph with standardized operators and data types, ONNX makes it
easy to switch between frameworks. For example, a model trained in PyTorch can
be exported to ONNX format and then imported in TensorFlow (and vice versa).`),_l=d(),He=n("p"),xd=r("\u{1F917} Transformers provides a "),fs=n("code"),yd=r("transformers.onnx"),kd=r(` package that enables you to
convert model checkpoints to an ONNX graph by leveraging configuration objects.
These configuration objects come ready made for a number of model architectures,
and are designed to be easily extendable to other architectures.`),vl=d(),fn=n("p"),Td=r("Ready-made configurations include the following architectures:"),wl=d(),m=n("ul"),cs=n("li"),Od=r("ALBERT"),Nd=d(),us=n("li"),jd=r("BART"),Ad=d(),ms=n("li"),Dd=r("BEiT"),Cd=d(),gs=n("li"),Pd=r("BERT"),Sd=d(),_s=n("li"),qd=r("BigBird"),Id=d(),vs=n("li"),Bd=r("BigBird-Pegasus"),Md=d(),ws=n("li"),zd=r("Blenderbot"),Ld=d(),Es=n("li"),Xd=r("BlenderbotSmall"),Fd=d(),$s=n("li"),Rd=r("CamemBERT"),Hd=d(),bs=n("li"),Wd=r("ConvBERT"),Kd=d(),xs=n("li"),Ud=r("Data2VecText"),Vd=d(),ys=n("li"),Gd=r("Data2VecVision"),Jd=d(),ks=n("li"),Yd=r("DeiT"),Qd=d(),Ts=n("li"),Zd=r("DistilBERT"),eh=d(),Os=n("li"),th=r("ELECTRA"),oh=d(),Ns=n("li"),nh=r("FlauBERT"),sh=d(),js=n("li"),ah=r("GPT Neo"),rh=d(),As=n("li"),lh=r("GPT-J"),ih=d(),Ds=n("li"),ph=r("I-BERT"),dh=d(),Cs=n("li"),hh=r("LayoutLM"),fh=d(),Ps=n("li"),ch=r("M2M100"),uh=d(),Ss=n("li"),mh=r("Marian"),gh=d(),qs=n("li"),_h=r("mBART"),vh=d(),Is=n("li"),wh=r("MobileBERT"),Eh=d(),Bs=n("li"),$h=r("OpenAI GPT-2"),bh=d(),Ms=n("li"),xh=r("Perceiver"),yh=d(),zs=n("li"),kh=r("PLBart"),Th=d(),Ls=n("li"),Oh=r("RoBERTa"),Nh=d(),Xs=n("li"),jh=r("RoFormer"),Ah=d(),Fs=n("li"),Dh=r("SqueezeBERT"),Ch=d(),Rs=n("li"),Ph=r("T5"),Sh=d(),Hs=n("li"),qh=r("ViT"),Ih=d(),Ws=n("li"),Bh=r("XLM"),Mh=d(),Ks=n("li"),zh=r("XLM-RoBERTa"),Lh=d(),Us=n("li"),Xh=r("XLM-RoBERTa-XL"),El=d(),cn=n("p"),Fh=r("In the next two sections, we\u2019ll show you how to:"),$l=d(),We=n("ul"),It=n("li"),Rh=r("Export a supported model using the "),Vs=n("code"),Hh=r("transformers.onnx"),Wh=r(" package."),Kh=d(),Gs=n("li"),Uh=r("Export a custom model for an unsupported architecture."),bl=d(),be=n("h3"),Ke=n("a"),Js=n("span"),v(Bt.$$.fragment),Vh=d(),Ys=n("span"),Gh=r("Exporting a model to ONNX"),xl=d(),un=n("p"),Jh=r(`To export a \u{1F917} Transformers model to ONNX, you\u2019ll first need to install some
extra dependencies:`),yl=d(),v(Mt.$$.fragment),kl=d(),Ue=n("p"),Yh=r("The "),Qs=n("code"),Qh=r("transformers.onnx"),Zh=r(" package can then be used as a Python module:"),Tl=d(),v(zt.$$.fragment),Ol=d(),mn=n("p"),ef=r("Exporting a checkpoint using a ready-made configuration can be done as follows:"),Nl=d(),v(Lt.$$.fragment),jl=d(),gn=n("p"),tf=r("which should show the following logs:"),Al=d(),v(Xt.$$.fragment),Dl=d(),re=n("p"),of=r("This exports an ONNX graph of the checkpoint defined by the "),Zs=n("code"),nf=r("--model"),sf=r(` argument.
In this example it is `),ea=n("code"),af=r("distilbert-base-uncased"),rf=r(`, but it can be any checkpoint on
the Hugging Face Hub or one that\u2019s stored locally.`),Cl=d(),Q=n("p"),lf=r("The resulting "),ta=n("code"),pf=r("model.onnx"),df=r(" file can then be run on one of the "),Ft=n("a"),hf=r(`many
accelerators`),ff=r(` that support the
ONNX standard. For example, we can load and run the model with `),Rt=n("a"),cf=r(`ONNX
Runtime`),uf=r(" as follows:"),Pl=d(),v(Ht.$$.fragment),Sl=d(),Ve=n("p"),mf=r("The required output names (i.e. "),oa=n("code"),gf=r('["last_hidden_state"]'),_f=r(`) can be obtained by
taking a look at the ONNX configuration of each model. For example, for
DistilBERT we have:`),ql=d(),v(Wt.$$.fragment),Il=d(),Ge=n("p"),vf=r(`The process is identical for TensorFlow checkpoints on the Hub. For example, we
can export a pure TensorFlow checkpoint from the `),Kt=n("a"),wf=r(`Keras
organization`),Ef=r(" as follows:"),Bl=d(),v(Ut.$$.fragment),Ml=d(),_n=n("p"),$f=r(`To export a model that\u2019s stored locally, you\u2019ll need to have the model\u2019s weights
and tokenizer files stored in a directory. For example, we can load and save a
checkpoint as follows:`),zl=d(),v(Je.$$.fragment),Ll=d(),xe=n("h3"),Ye=n("a"),na=n("span"),v(Vt.$$.fragment),bf=d(),sa=n("span"),xf=r("Selecting features for different model topologies"),Xl=d(),Qe=n("p"),yf=r("Each ready-made configuration comes with a set of "),aa=n("em"),kf=r("features"),Tf=r(` that enable you to
export models for different types of topologies or tasks. As shown in the table
below, each feature is associated with a different auto class:`),Fl=d(),Ze=n("table"),ra=n("thead"),Gt=n("tr"),la=n("th"),Of=r("Feature"),Nf=d(),ia=n("th"),jf=r("Auto Class"),Af=d(),X=n("tbody"),Jt=n("tr"),Yt=n("td"),pa=n("code"),Df=r("causal-lm"),Cf=r(", "),da=n("code"),Pf=r("causal-lm-with-past"),Sf=d(),ha=n("td"),fa=n("code"),qf=r("AutoModelForCausalLM"),If=d(),Qt=n("tr"),Zt=n("td"),ca=n("code"),Bf=r("default"),Mf=r(", "),ua=n("code"),zf=r("default-with-past"),Lf=d(),ma=n("td"),ga=n("code"),Xf=r("AutoModel"),Ff=d(),eo=n("tr"),_a=n("td"),va=n("code"),Rf=r("masked-lm"),Hf=d(),wa=n("td"),Ea=n("code"),Wf=r("AutoModelForMaskedLM"),Kf=d(),to=n("tr"),$a=n("td"),ba=n("code"),Uf=r("question-answering"),Vf=d(),xa=n("td"),ya=n("code"),Gf=r("AutoModelForQuestionAnswering"),Jf=d(),oo=n("tr"),no=n("td"),ka=n("code"),Yf=r("seq2seq-lm"),Qf=r(", "),Ta=n("code"),Zf=r("seq2seq-lm-with-past"),ec=d(),Oa=n("td"),Na=n("code"),tc=r("AutoModelForSeq2SeqLM"),oc=d(),so=n("tr"),ja=n("td"),Aa=n("code"),nc=r("sequence-classification"),sc=d(),Da=n("td"),Ca=n("code"),ac=r("AutoModelForSequenceClassification"),rc=d(),ao=n("tr"),Pa=n("td"),Sa=n("code"),lc=r("token-classification"),ic=d(),qa=n("td"),Ia=n("code"),pc=r("AutoModelForTokenClassification"),Rl=d(),et=n("p"),dc=r(`For each configuration, you can find the list of supported features via the
`),Ba=n("code"),hc=r("FeaturesManager"),fc=r(". For example, for DistilBERT we have:"),Hl=d(),v(ro.$$.fragment),Wl=d(),le=n("p"),cc=r("You can then pass one of these features to the "),Ma=n("code"),uc=r("--feature"),mc=r(` argument in the
`),za=n("code"),gc=r("transformers.onnx"),_c=r(` package. For example, to export a text-classification model
we can pick a fine-tuned model from the Hub and run:`),Kl=d(),v(lo.$$.fragment),Ul=d(),vn=n("p"),vc=r("which will display the following logs:"),Vl=d(),v(io.$$.fragment),Gl=d(),Z=n("p"),wc=r(`Notice that in this case, the output names from the fine-tuned model are
`),La=n("code"),Ec=r("logits"),$c=r(" instead of the "),Xa=n("code"),bc=r("last_hidden_state"),xc=r(` we saw with the
`),Fa=n("code"),yc=r("distilbert-base-uncased"),kc=r(` checkpoint earlier. This is expected since the
fine-tuned model has a sequence classification head.`),Jl=d(),v(tt.$$.fragment),Yl=d(),ye=n("h3"),ot=n("a"),Ra=n("span"),v(po.$$.fragment),Tc=d(),Ha=n("span"),Oc=r("Exporting a model for an unsupported architecture"),Ql=d(),wn=n("p"),Nc=r(`If you wish to export a model whose architecture is not natively supported by
the library, there are three main steps to follow:`),Zl=d(),ie=n("ol"),Wa=n("li"),jc=r("Implement a custom ONNX configuration."),Ac=d(),Ka=n("li"),Dc=r("Export the model to ONNX."),Cc=d(),Ua=n("li"),Pc=r("Validate the outputs of the PyTorch and exported models."),ei=d(),En=n("p"),Sc=r(`In this section, we\u2019ll look at how DistilBERT was implemented to show what\u2019s
involved with each step.`),ti=d(),ke=n("h4"),nt=n("a"),Va=n("span"),v(ho.$$.fragment),qc=d(),Ga=n("span"),Ic=r("Implementing a custom ONNX configuration"),oi=d(),$n=n("p"),Bc=r(`Let\u2019s start with the ONNX configuration object. We provide three abstract
classes that you should inherit from, depending on the type of model
architecture you wish to export:`),ni=d(),pe=n("ul"),bn=n("li"),Mc=r("Encoder-based models inherit from "),xn=n("a"),zc=r("OnnxConfig"),Lc=d(),yn=n("li"),Xc=r("Decoder-based models inherit from "),kn=n("a"),Fc=r("OnnxConfigWithPast"),Rc=d(),Tn=n("li"),Hc=r("Encoder-decoder models inherit from "),On=n("a"),Wc=r("OnnxSeq2SeqConfigWithPast"),si=d(),v(st.$$.fragment),ai=d(),at=n("p"),Kc=r(`Since DistilBERT is an encoder-based model, its configuration inherits from
`),Ja=n("code"),Uc=r("OnnxConfig"),Vc=r(":"),ri=d(),v(fo.$$.fragment),li=d(),W=n("p"),Gc=r("Every configuration object must implement the "),Ya=n("code"),Jc=r("inputs"),Yc=r(` property and return a
mapping, where each key corresponds to an expected input, and each value
indicates the axis of that input. For DistilBERT, we can see that two inputs are
required: `),Qa=n("code"),Qc=r("input_ids"),Zc=r(" and "),Za=n("code"),eu=r("attention_mask"),tu=r(`. These inputs have the same shape of
`),er=n("code"),ou=r("(batch_size, sequence_length)"),nu=r(` which is why we see the same axes used in the
configuration.`),ii=d(),v(rt.$$.fragment),pi=d(),Nn=n("p"),su=r(`Once you have implemented an ONNX configuration, you can instantiate it by
providing the base model\u2019s configuration as follows:`),di=d(),v(co.$$.fragment),hi=d(),jn=n("p"),au=r(`The resulting object has several useful properties. For example you can view the
ONNX operator set that will be used during the export:`),fi=d(),v(uo.$$.fragment),ci=d(),An=n("p"),ru=r("You can also view the outputs associated with the model as follows:"),ui=d(),v(mo.$$.fragment),mi=d(),K=n("p"),lu=r(`Notice that the outputs property follows the same structure as the inputs; it
returns an `),tr=n("code"),iu=r("OrderedDict"),pu=r(` of named outputs and their shapes. The output structure
is linked to the choice of feature that the configuration is initialised with.
By default, the ONNX configuration is initialized with the `),or=n("code"),du=r("default"),hu=r(` feature
that corresponds to exporting a model loaded with the `),nr=n("code"),fu=r("AutoModel"),cu=r(` class. If you
want to export a different model topology, just provide a different feature to
the `),sr=n("code"),uu=r("task"),mu=r(` argument when you initialize the ONNX configuration. For example, if
we wished to export DistilBERT with a sequence classification head, we could
use:`),gi=d(),v(go.$$.fragment),_i=d(),v(lt.$$.fragment),vi=d(),Te=n("h4"),it=n("a"),ar=n("span"),v(_o.$$.fragment),gu=d(),rr=n("span"),_u=r("Exporting the model"),wi=d(),de=n("p"),vu=r(`Once you have implemented the ONNX configuration, the next step is to export the
model. Here we can use the `),lr=n("code"),wu=r("export()"),Eu=r(` function provided by the
`),ir=n("code"),$u=r("transformers.onnx"),bu=r(` package. This function expects the ONNX configuration, along
with the base model and tokenizer, and the path to save the exported file:`),Ei=d(),v(vo.$$.fragment),$i=d(),F=n("p"),xu=r("The "),pr=n("code"),yu=r("onnx_inputs"),ku=r(" and "),dr=n("code"),Tu=r("onnx_outputs"),Ou=r(" returned by the "),hr=n("code"),Nu=r("export()"),ju=r(` function are
lists of the keys defined in the `),fr=n("code"),Au=r("inputs"),Du=r(" and "),cr=n("code"),Cu=r("outputs"),Pu=r(` properties of the
configuration. Once the model is exported, you can test that the model is well
formed as follows:`),bi=d(),v(wo.$$.fragment),xi=d(),v(pt.$$.fragment),yi=d(),Oe=n("h4"),dt=n("a"),ur=n("span"),v(Eo.$$.fragment),Su=d(),mr=n("span"),qu=r("Validating the model outputs"),ki=d(),he=n("p"),Iu=r(`The final step is to validate that the outputs from the base and exported model
agree within some absolute tolerance. Here we can use the
`),gr=n("code"),Bu=r("validate_model_outputs()"),Mu=r(" function provided by the "),_r=n("code"),zu=r("transformers.onnx"),Lu=r(` package
as follows:`),Ti=d(),v($o.$$.fragment),Oi=d(),ht=n("p"),Xu=r("This function uses the "),vr=n("code"),Fu=r("OnnxConfig.generate_dummy_inputs()"),Ru=r(` method to generate
inputs for the base and exported model, and the absolute tolerance can be
defined in the configuration. We generally find numerical agreement in the 1e-6
to 1e-4 range, although anything smaller than 1e-3 is likely to be OK.`),Ni=d(),Ne=n("h3"),ft=n("a"),wr=n("span"),v(bo.$$.fragment),Hu=d(),Er=n("span"),Wu=r("Contributing a new configuration to \u{1F917} Transformers"),ji=d(),Dn=n("p"),Ku=r(`We are looking to expand the set of ready-made configurations and welcome
contributions from the community! If you would like to contribute your addition
to the library, you will need to:`),Ai=d(),fe=n("ul"),xo=n("li"),Uu=r("Implement the ONNX configuration in the corresponding "),$r=n("code"),Vu=r("configuration_<model_name>.py"),Gu=r(`
file`),Ju=d(),Cn=n("li"),Yu=r("Include the model architecture and corresponding features in "),br=n("code"),Qu=r("FeatureManager"),Zu=d(),Pn=n("li"),em=r("Add your model architecture to the tests in "),xr=n("code"),tm=r("test_onnx_v2.py"),Di=d(),ct=n("p"),om=r("Check out how the configuration for "),yo=n("a"),nm=r(`IBERT was
contributed`),sm=r(` to
get an idea of what\u2019s involved.`),Ci=d(),je=n("h2"),ut=n("a"),yr=n("span"),v(ko.$$.fragment),am=d(),kr=n("span"),rm=r("TorchScript"),Pi=d(),v(mt.$$.fragment),Si=d(),gt=n("p"),lm=r(`According to Pytorch\u2019s documentation: \u201CTorchScript is a way to create serializable and optimizable models from PyTorch
code\u201D. Pytorch\u2019s two modules `),To=n("a"),im=r("JIT and TRACE"),pm=r(` allow the developer to export
their model to be re-used in other programs, such as efficiency-oriented C++ programs.`),qi=d(),Sn=n("p"),dm=r(`We have provided an interface that allows the export of \u{1F917} Transformers models to TorchScript so that they can be reused
in a different environment than a Pytorch-based python program. Here we explain how to export and use our models using
TorchScript.`),Ii=d(),qn=n("p"),hm=r("Exporting a model requires two things:"),Bi=d(),_t=n("ul"),Tr=n("li"),fm=r("a forward pass with dummy inputs."),cm=d(),Oo=n("li"),um=r("model instantiation with the "),Or=n("code"),mm=r("torchscript"),gm=r(" flag."),Mi=d(),In=n("p"),_m=r("These necessities imply several things developers should be careful about. These are detailed below."),zi=d(),Ae=n("h3"),vt=n("a"),Nr=n("span"),v(No.$$.fragment),vm=d(),jr=n("span"),wm=r("TorchScript flag and tied weights"),Li=d(),ce=n("p"),Em=r(`This flag is necessary because most of the language models in this repository have tied weights between their
`),Ar=n("code"),$m=r("Embedding"),bm=r(" layer and their "),Dr=n("code"),xm=r("Decoding"),ym=r(` layer. TorchScript does not allow the export of models that have tied
weights, therefore it is necessary to untie and clone the weights beforehand.`),Xi=d(),ee=n("p"),km=r("This implies that models instantiated with the "),Cr=n("code"),Tm=r("torchscript"),Om=r(" flag have their "),Pr=n("code"),Nm=r("Embedding"),jm=r(" layer and "),Sr=n("code"),Am=r("Decoding"),Dm=r(`
layer separate, which means that they should not be trained down the line. Training would de-synchronize the two
layers, leading to unexpected results.`),Fi=d(),wt=n("p"),Cm=r(`This is not the case for models that do not have a Language Model head, as those do not have tied weights. These models
can be safely exported without the `),qr=n("code"),Pm=r("torchscript"),Sm=r(" flag."),Ri=d(),De=n("h3"),Et=n("a"),Ir=n("span"),v(jo.$$.fragment),qm=d(),Br=n("span"),Im=r("Dummy inputs and standard lengths"),Hi=d(),Bn=n("p"),Bm=r(`The dummy inputs are used to do a model forward pass. While the inputs\u2019 values are propagating through the layers,
Pytorch keeps track of the different operations executed on each tensor. These recorded operations are then used to
create the \u201Ctrace\u201D of the model.`),Wi=d(),Mn=n("p"),Mm=r(`The trace is created relatively to the inputs\u2019 dimensions. It is therefore constrained by the dimensions of the dummy
input, and will not work for any other sequence length or batch size. When trying with a different size, an error such
as:`),Ki=d(),zn=n("p"),Mr=n("code"),zm=r("The expanded size of the tensor (3) must match the existing size (7) at non-singleton dimension 2"),Ui=d(),Ln=n("p"),Lm=r(`will be raised. It is therefore recommended to trace the model with a dummy input size at least as large as the largest
input that will be fed to the model during inference. Padding can be performed to fill the missing values. As the model
will have been traced with a large input size however, the dimensions of the different matrix will be large as well,
resulting in more calculations.`),Vi=d(),Xn=n("p"),Xm=r(`It is recommended to be careful of the total number of operations done on each input and to follow performance closely
when exporting varying sequence-length models.`),Gi=d(),Ce=n("h3"),$t=n("a"),zr=n("span"),v(Ao.$$.fragment),Fm=d(),Lr=n("span"),Rm=r("Using TorchScript in Python"),Ji=d(),Fn=n("p"),Hm=r("Below is an example, showing how to save, load models as well as how to use the trace for inference."),Yi=d(),Pe=n("h4"),bt=n("a"),Xr=n("span"),v(Do.$$.fragment),Wm=d(),Fr=n("span"),Km=r("Saving a model"),Qi=d(),Y=n("p"),Um=r("This snippet shows how to use TorchScript to export a "),Rr=n("code"),Vm=r("BertModel"),Gm=r(". Here the "),Hr=n("code"),Jm=r("BertModel"),Ym=r(` is instantiated according
to a `),Wr=n("code"),Qm=r("BertConfig"),Zm=r(" class and then saved to disk under the filename "),Kr=n("code"),eg=r("traced_bert.pt"),Zi=d(),v(Co.$$.fragment),ep=d(),Se=n("h4"),xt=n("a"),Ur=n("span"),v(Po.$$.fragment),tg=d(),Vr=n("span"),og=r("Loading a model"),tp=d(),te=n("p"),ng=r("This snippet shows how to load the "),Gr=n("code"),sg=r("BertModel"),ag=r(" that was previously saved to disk under the name "),Jr=n("code"),rg=r("traced_bert.pt"),lg=r(`.
We are re-using the previously initialised `),Yr=n("code"),ig=r("dummy_input"),pg=r("."),op=d(),v(So.$$.fragment),np=d(),qe=n("h4"),yt=n("a"),Qr=n("span"),v(qo.$$.fragment),dg=d(),Zr=n("span"),hg=r("Using a traced model for inference"),sp=d(),kt=n("p"),fg=r("Using the traced model for inference is as simple as using its "),el=n("code"),cg=r("__call__"),ug=r(" dunder method:"),ap=d(),v(Io.$$.fragment),rp=d(),Ie=n("h3"),Tt=n("a"),tl=n("span"),v(Bo.$$.fragment),mg=d(),ol=n("span"),gg=r("Deploying HuggingFace TorchScript models on AWS using the Neuron SDK"),lp=d(),ue=n("p"),_g=r("AWS introduced the "),Mo=n("a"),vg=r("Amazon EC2 Inf1"),wg=r(`
instance family for low cost, high performance machine learning inference in the cloud.
The Inf1 instances are powered by the AWS Inferentia chip, a custom-built hardware accelerator,
specializing in deep learning inferencing workloads.
`),zo=n("a"),Eg=r("AWS Neuron"),$g=r(`
is the SDK for Inferentia that supports tracing and optimizing transformers models for
deployment on Inf1. The Neuron SDK provides:`),ip=d(),me=n("ol"),nl=n("li"),bg=r("Easy-to-use API with one line of code change to trace and optimize a TorchScript model for inference in the cloud."),xg=d(),Rn=n("li"),yg=r("Out of the box performance optimizations for "),Lo=n("a"),kg=r("improved cost-performance"),Tg=d(),Be=n("li"),Og=r("Support for HuggingFace transformers models built with either "),Xo=n("a"),Ng=r("PyTorch"),jg=r(`
or `),Fo=n("a"),Ag=r("TensorFlow"),Dg=r("."),pp=d(),Me=n("h4"),Ot=n("a"),sl=n("span"),v(Ro.$$.fragment),Cg=d(),al=n("span"),Pg=r("Implications"),dp=d(),R=n("p"),Sg=r("Transformers Models based on the "),Ho=n("a"),qg=r("BERT (Bidirectional Encoder Representations from Transformers)"),Ig=r(`
architecture, or its variants such as `),Wo=n("a"),Bg=r("distilBERT"),Mg=r(`
and `),Ko=n("a"),zg=r("roBERTa"),Lg=r(`
will run best on Inf1 for non-generative tasks such as Extractive Question Answering,
Sequence Classification, Token Classification. Alternatively, text generation
tasks can be adapted to run on Inf1, according to this `),Uo=n("a"),Xg=r("AWS Neuron MarianMT tutorial"),Fg=r(`.
More information about models that can be converted out of the box on Inferentia can be
found in the `),Vo=n("a"),Rg=r("Model Architecture Fit section of the Neuron documentation"),Hg=r("."),hp=d(),ze=n("h4"),Nt=n("a"),rl=n("span"),v(Go.$$.fragment),Wg=d(),ll=n("span"),Kg=r("Dependencies"),fp=d(),Hn=n("p"),Ug=r("Using AWS Neuron to convert models requires the following dependencies and environment:"),cp=d(),Wn=n("ul"),Le=n("li"),Vg=r("A "),Jo=n("a"),Gg=r("Neuron SDK environment"),Jg=r(`,
which comes pre-configured on `),Yo=n("a"),Yg=r("AWS Deep Learning AMI"),Qg=r("."),up=d(),Xe=n("h4"),jt=n("a"),il=n("span"),v(Qo.$$.fragment),Zg=d(),pl=n("span"),e_=r("Converting a Model for AWS Neuron"),mp=d(),ge=n("p"),t_=r("Using the same script as in "),Zo=n("a"),o_=r("Using TorchScript in Python"),n_=r(`
to trace a \u201CBertModel\u201D, you import `),dl=n("code"),s_=r("torch.neuron"),a_=r(` framework extension to access
the components of the Neuron SDK through a Python API.`),gp=d(),v(en.$$.fragment),_p=d(),Kn=n("p"),r_=r("And only modify the tracing line of code"),vp=d(),Un=n("p"),l_=r("from:"),wp=d(),v(tn.$$.fragment),Ep=d(),Vn=n("p"),i_=r("to:"),$p=d(),v(on.$$.fragment),bp=d(),Gn=n("p"),p_=r("This change enables Neuron SDK to trace the model and optimize it to run in Inf1 instances."),xp=d(),At=n("p"),d_=r(`To learn more about AWS Neuron SDK features, tools, example tutorials and latest updates,
please see the `),nn=n("a"),h_=r("AWS NeuronSDK documentation"),f_=r("."),this.h()},l(e){const i=HE('[data-svelte="svelte-1phssyn"]',document.head);c=s(i,"META",{name:!0,content:!0}),i.forEach(o),k=h(e),u=s(e,"H1",{class:!0});var sn=a(u);y=s(sn,"A",{id:!0,class:!0,href:!0});var hl=a(y);O=s(hl,"SPAN",{});var fl=a(O);w(T.$$.fragment,fl),fl.forEach(o),hl.forEach(o),P=h(sn),C=s(sn,"SPAN",{});var cl=a(C);A=l(cl,"Export \u{1F917} Transformers Models"),cl.forEach(o),sn.forEach(o),N=h(e),S=s(e,"P",{});var ul=a(S);D=l(ul,`If you need to deploy \u{1F917} Transformers models in production environments, we
recommend exporting them to a serialized format that can be loaded and executed
on specialized runtimes and hardware. In this guide, we\u2019ll show you how to
export \u{1F917} Transformers models in two widely used formats: ONNX and TorchScript.`),ul.forEach(o),q=h(e),_=s(e,"P",{});var an=a(_);j=l(an,`Once exported, a model can optimized for inference via techniques such as
quantization and pruning. If you are interested in optimizing your models to run
with maximum efficiency, check out the `),z=s(an,"A",{href:!0,rel:!0});var ml=a(z);V=l(ml,`\u{1F917} Optimum
library`),ml.forEach(o),we=l(an,"."),an.forEach(o),Re=h(e),G=s(e,"H2",{class:!0});var kp=a(G);H=s(kp,"A",{id:!0,class:!0,href:!0});var w_=a(H);Ee=s(w_,"SPAN",{});var E_=a(Ee);w(se.$$.fragment,E_),E_.forEach(o),w_.forEach(o),$e=h(kp),B=s(kp,"SPAN",{});var $_=a(B);rn=l($_,"ONNX"),$_.forEach(o),kp.forEach(o),qt=h(e),J=s(e,"P",{});var Jn=a(J);ln=l(Jn,"The "),ae=s(Jn,"A",{href:!0,rel:!0});var b_=a(ae);pn=l(b_,"ONNX (Open Neural Network eXchange)"),b_.forEach(o),dn=l(Jn,` project is an open
standard that defines a common set of operators and a common file format to
represent deep learning models in a wide variety of frameworks, including
PyTorch and TensorFlow. When a model is exported to the ONNX format, these
operators are used to construct a computational graph (often called an
`),hs=s(Jn,"EM",{});var x_=a(hs);Ed=l(x_,"intermediate representation"),x_.forEach(o),$d=l(Jn,`) which represents the flow of data through the
neural network.`),Jn.forEach(o),gl=h(e),hn=s(e,"P",{});var y_=a(hn);bd=l(y_,`By exposing a graph with standardized operators and data types, ONNX makes it
easy to switch between frameworks. For example, a model trained in PyTorch can
be exported to ONNX format and then imported in TensorFlow (and vice versa).`),y_.forEach(o),_l=h(e),He=s(e,"P",{});var Tp=a(He);xd=l(Tp,"\u{1F917} Transformers provides a "),fs=s(Tp,"CODE",{});var k_=a(fs);yd=l(k_,"transformers.onnx"),k_.forEach(o),kd=l(Tp,` package that enables you to
convert model checkpoints to an ONNX graph by leveraging configuration objects.
These configuration objects come ready made for a number of model architectures,
and are designed to be easily extendable to other architectures.`),Tp.forEach(o),vl=h(e),fn=s(e,"P",{});var T_=a(fn);Td=l(T_,"Ready-made configurations include the following architectures:"),T_.forEach(o),wl=h(e),m=s(e,"UL",{});var g=a(m);cs=s(g,"LI",{});var O_=a(cs);Od=l(O_,"ALBERT"),O_.forEach(o),Nd=h(g),us=s(g,"LI",{});var N_=a(us);jd=l(N_,"BART"),N_.forEach(o),Ad=h(g),ms=s(g,"LI",{});var j_=a(ms);Dd=l(j_,"BEiT"),j_.forEach(o),Cd=h(g),gs=s(g,"LI",{});var A_=a(gs);Pd=l(A_,"BERT"),A_.forEach(o),Sd=h(g),_s=s(g,"LI",{});var D_=a(_s);qd=l(D_,"BigBird"),D_.forEach(o),Id=h(g),vs=s(g,"LI",{});var C_=a(vs);Bd=l(C_,"BigBird-Pegasus"),C_.forEach(o),Md=h(g),ws=s(g,"LI",{});var P_=a(ws);zd=l(P_,"Blenderbot"),P_.forEach(o),Ld=h(g),Es=s(g,"LI",{});var S_=a(Es);Xd=l(S_,"BlenderbotSmall"),S_.forEach(o),Fd=h(g),$s=s(g,"LI",{});var q_=a($s);Rd=l(q_,"CamemBERT"),q_.forEach(o),Hd=h(g),bs=s(g,"LI",{});var I_=a(bs);Wd=l(I_,"ConvBERT"),I_.forEach(o),Kd=h(g),xs=s(g,"LI",{});var B_=a(xs);Ud=l(B_,"Data2VecText"),B_.forEach(o),Vd=h(g),ys=s(g,"LI",{});var M_=a(ys);Gd=l(M_,"Data2VecVision"),M_.forEach(o),Jd=h(g),ks=s(g,"LI",{});var z_=a(ks);Yd=l(z_,"DeiT"),z_.forEach(o),Qd=h(g),Ts=s(g,"LI",{});var L_=a(Ts);Zd=l(L_,"DistilBERT"),L_.forEach(o),eh=h(g),Os=s(g,"LI",{});var X_=a(Os);th=l(X_,"ELECTRA"),X_.forEach(o),oh=h(g),Ns=s(g,"LI",{});var F_=a(Ns);nh=l(F_,"FlauBERT"),F_.forEach(o),sh=h(g),js=s(g,"LI",{});var R_=a(js);ah=l(R_,"GPT Neo"),R_.forEach(o),rh=h(g),As=s(g,"LI",{});var H_=a(As);lh=l(H_,"GPT-J"),H_.forEach(o),ih=h(g),Ds=s(g,"LI",{});var W_=a(Ds);ph=l(W_,"I-BERT"),W_.forEach(o),dh=h(g),Cs=s(g,"LI",{});var K_=a(Cs);hh=l(K_,"LayoutLM"),K_.forEach(o),fh=h(g),Ps=s(g,"LI",{});var U_=a(Ps);ch=l(U_,"M2M100"),U_.forEach(o),uh=h(g),Ss=s(g,"LI",{});var V_=a(Ss);mh=l(V_,"Marian"),V_.forEach(o),gh=h(g),qs=s(g,"LI",{});var G_=a(qs);_h=l(G_,"mBART"),G_.forEach(o),vh=h(g),Is=s(g,"LI",{});var J_=a(Is);wh=l(J_,"MobileBERT"),J_.forEach(o),Eh=h(g),Bs=s(g,"LI",{});var Y_=a(Bs);$h=l(Y_,"OpenAI GPT-2"),Y_.forEach(o),bh=h(g),Ms=s(g,"LI",{});var Q_=a(Ms);xh=l(Q_,"Perceiver"),Q_.forEach(o),yh=h(g),zs=s(g,"LI",{});var Z_=a(zs);kh=l(Z_,"PLBart"),Z_.forEach(o),Th=h(g),Ls=s(g,"LI",{});var ev=a(Ls);Oh=l(ev,"RoBERTa"),ev.forEach(o),Nh=h(g),Xs=s(g,"LI",{});var tv=a(Xs);jh=l(tv,"RoFormer"),tv.forEach(o),Ah=h(g),Fs=s(g,"LI",{});var ov=a(Fs);Dh=l(ov,"SqueezeBERT"),ov.forEach(o),Ch=h(g),Rs=s(g,"LI",{});var nv=a(Rs);Ph=l(nv,"T5"),nv.forEach(o),Sh=h(g),Hs=s(g,"LI",{});var sv=a(Hs);qh=l(sv,"ViT"),sv.forEach(o),Ih=h(g),Ws=s(g,"LI",{});var av=a(Ws);Bh=l(av,"XLM"),av.forEach(o),Mh=h(g),Ks=s(g,"LI",{});var rv=a(Ks);zh=l(rv,"XLM-RoBERTa"),rv.forEach(o),Lh=h(g),Us=s(g,"LI",{});var lv=a(Us);Xh=l(lv,"XLM-RoBERTa-XL"),lv.forEach(o),g.forEach(o),El=h(e),cn=s(e,"P",{});var iv=a(cn);Fh=l(iv,"In the next two sections, we\u2019ll show you how to:"),iv.forEach(o),$l=h(e),We=s(e,"UL",{});var Op=a(We);It=s(Op,"LI",{});var Np=a(It);Rh=l(Np,"Export a supported model using the "),Vs=s(Np,"CODE",{});var pv=a(Vs);Hh=l(pv,"transformers.onnx"),pv.forEach(o),Wh=l(Np," package."),Np.forEach(o),Kh=h(Op),Gs=s(Op,"LI",{});var dv=a(Gs);Uh=l(dv,"Export a custom model for an unsupported architecture."),dv.forEach(o),Op.forEach(o),bl=h(e),be=s(e,"H3",{class:!0});var jp=a(be);Ke=s(jp,"A",{id:!0,class:!0,href:!0});var hv=a(Ke);Js=s(hv,"SPAN",{});var fv=a(Js);w(Bt.$$.fragment,fv),fv.forEach(o),hv.forEach(o),Vh=h(jp),Ys=s(jp,"SPAN",{});var cv=a(Ys);Gh=l(cv,"Exporting a model to ONNX"),cv.forEach(o),jp.forEach(o),xl=h(e),un=s(e,"P",{});var uv=a(un);Jh=l(uv,`To export a \u{1F917} Transformers model to ONNX, you\u2019ll first need to install some
extra dependencies:`),uv.forEach(o),yl=h(e),w(Mt.$$.fragment,e),kl=h(e),Ue=s(e,"P",{});var Ap=a(Ue);Yh=l(Ap,"The "),Qs=s(Ap,"CODE",{});var mv=a(Qs);Qh=l(mv,"transformers.onnx"),mv.forEach(o),Zh=l(Ap," package can then be used as a Python module:"),Ap.forEach(o),Tl=h(e),w(zt.$$.fragment,e),Ol=h(e),mn=s(e,"P",{});var gv=a(mn);ef=l(gv,"Exporting a checkpoint using a ready-made configuration can be done as follows:"),gv.forEach(o),Nl=h(e),w(Lt.$$.fragment,e),jl=h(e),gn=s(e,"P",{});var _v=a(gn);tf=l(_v,"which should show the following logs:"),_v.forEach(o),Al=h(e),w(Xt.$$.fragment,e),Dl=h(e),re=s(e,"P",{});var Yn=a(re);of=l(Yn,"This exports an ONNX graph of the checkpoint defined by the "),Zs=s(Yn,"CODE",{});var vv=a(Zs);nf=l(vv,"--model"),vv.forEach(o),sf=l(Yn,` argument.
In this example it is `),ea=s(Yn,"CODE",{});var wv=a(ea);af=l(wv,"distilbert-base-uncased"),wv.forEach(o),rf=l(Yn,`, but it can be any checkpoint on
the Hugging Face Hub or one that\u2019s stored locally.`),Yn.forEach(o),Cl=h(e),Q=s(e,"P",{});var Dt=a(Q);lf=l(Dt,"The resulting "),ta=s(Dt,"CODE",{});var Ev=a(ta);pf=l(Ev,"model.onnx"),Ev.forEach(o),df=l(Dt," file can then be run on one of the "),Ft=s(Dt,"A",{href:!0,rel:!0});var $v=a(Ft);hf=l($v,`many
accelerators`),$v.forEach(o),ff=l(Dt,` that support the
ONNX standard. For example, we can load and run the model with `),Rt=s(Dt,"A",{href:!0,rel:!0});var bv=a(Rt);cf=l(bv,`ONNX
Runtime`),bv.forEach(o),uf=l(Dt," as follows:"),Dt.forEach(o),Pl=h(e),w(Ht.$$.fragment,e),Sl=h(e),Ve=s(e,"P",{});var Dp=a(Ve);mf=l(Dp,"The required output names (i.e. "),oa=s(Dp,"CODE",{});var xv=a(oa);gf=l(xv,'["last_hidden_state"]'),xv.forEach(o),_f=l(Dp,`) can be obtained by
taking a look at the ONNX configuration of each model. For example, for
DistilBERT we have:`),Dp.forEach(o),ql=h(e),w(Wt.$$.fragment,e),Il=h(e),Ge=s(e,"P",{});var Cp=a(Ge);vf=l(Cp,`The process is identical for TensorFlow checkpoints on the Hub. For example, we
can export a pure TensorFlow checkpoint from the `),Kt=s(Cp,"A",{href:!0,rel:!0});var yv=a(Kt);wf=l(yv,`Keras
organization`),yv.forEach(o),Ef=l(Cp," as follows:"),Cp.forEach(o),Bl=h(e),w(Ut.$$.fragment,e),Ml=h(e),_n=s(e,"P",{});var kv=a(_n);$f=l(kv,`To export a model that\u2019s stored locally, you\u2019ll need to have the model\u2019s weights
and tokenizer files stored in a directory. For example, we can load and save a
checkpoint as follows:`),kv.forEach(o),zl=h(e),w(Je.$$.fragment,e),Ll=h(e),xe=s(e,"H3",{class:!0});var Pp=a(xe);Ye=s(Pp,"A",{id:!0,class:!0,href:!0});var Tv=a(Ye);na=s(Tv,"SPAN",{});var Ov=a(na);w(Vt.$$.fragment,Ov),Ov.forEach(o),Tv.forEach(o),bf=h(Pp),sa=s(Pp,"SPAN",{});var Nv=a(sa);xf=l(Nv,"Selecting features for different model topologies"),Nv.forEach(o),Pp.forEach(o),Xl=h(e),Qe=s(e,"P",{});var Sp=a(Qe);yf=l(Sp,"Each ready-made configuration comes with a set of "),aa=s(Sp,"EM",{});var jv=a(aa);kf=l(jv,"features"),jv.forEach(o),Tf=l(Sp,` that enable you to
export models for different types of topologies or tasks. As shown in the table
below, each feature is associated with a different auto class:`),Sp.forEach(o),Fl=h(e),Ze=s(e,"TABLE",{});var qp=a(Ze);ra=s(qp,"THEAD",{});var Av=a(ra);Gt=s(Av,"TR",{});var Ip=a(Gt);la=s(Ip,"TH",{});var Dv=a(la);Of=l(Dv,"Feature"),Dv.forEach(o),Nf=h(Ip),ia=s(Ip,"TH",{});var Cv=a(ia);jf=l(Cv,"Auto Class"),Cv.forEach(o),Ip.forEach(o),Av.forEach(o),Af=h(qp),X=s(qp,"TBODY",{});var U=a(X);Jt=s(U,"TR",{});var Bp=a(Jt);Yt=s(Bp,"TD",{});var Mp=a(Yt);pa=s(Mp,"CODE",{});var Pv=a(pa);Df=l(Pv,"causal-lm"),Pv.forEach(o),Cf=l(Mp,", "),da=s(Mp,"CODE",{});var Sv=a(da);Pf=l(Sv,"causal-lm-with-past"),Sv.forEach(o),Mp.forEach(o),Sf=h(Bp),ha=s(Bp,"TD",{});var qv=a(ha);fa=s(qv,"CODE",{});var Iv=a(fa);qf=l(Iv,"AutoModelForCausalLM"),Iv.forEach(o),qv.forEach(o),Bp.forEach(o),If=h(U),Qt=s(U,"TR",{});var zp=a(Qt);Zt=s(zp,"TD",{});var Lp=a(Zt);ca=s(Lp,"CODE",{});var Bv=a(ca);Bf=l(Bv,"default"),Bv.forEach(o),Mf=l(Lp,", "),ua=s(Lp,"CODE",{});var Mv=a(ua);zf=l(Mv,"default-with-past"),Mv.forEach(o),Lp.forEach(o),Lf=h(zp),ma=s(zp,"TD",{});var zv=a(ma);ga=s(zv,"CODE",{});var Lv=a(ga);Xf=l(Lv,"AutoModel"),Lv.forEach(o),zv.forEach(o),zp.forEach(o),Ff=h(U),eo=s(U,"TR",{});var Xp=a(eo);_a=s(Xp,"TD",{});var Xv=a(_a);va=s(Xv,"CODE",{});var Fv=a(va);Rf=l(Fv,"masked-lm"),Fv.forEach(o),Xv.forEach(o),Hf=h(Xp),wa=s(Xp,"TD",{});var Rv=a(wa);Ea=s(Rv,"CODE",{});var Hv=a(Ea);Wf=l(Hv,"AutoModelForMaskedLM"),Hv.forEach(o),Rv.forEach(o),Xp.forEach(o),Kf=h(U),to=s(U,"TR",{});var Fp=a(to);$a=s(Fp,"TD",{});var Wv=a($a);ba=s(Wv,"CODE",{});var Kv=a(ba);Uf=l(Kv,"question-answering"),Kv.forEach(o),Wv.forEach(o),Vf=h(Fp),xa=s(Fp,"TD",{});var Uv=a(xa);ya=s(Uv,"CODE",{});var Vv=a(ya);Gf=l(Vv,"AutoModelForQuestionAnswering"),Vv.forEach(o),Uv.forEach(o),Fp.forEach(o),Jf=h(U),oo=s(U,"TR",{});var Rp=a(oo);no=s(Rp,"TD",{});var Hp=a(no);ka=s(Hp,"CODE",{});var Gv=a(ka);Yf=l(Gv,"seq2seq-lm"),Gv.forEach(o),Qf=l(Hp,", "),Ta=s(Hp,"CODE",{});var Jv=a(Ta);Zf=l(Jv,"seq2seq-lm-with-past"),Jv.forEach(o),Hp.forEach(o),ec=h(Rp),Oa=s(Rp,"TD",{});var Yv=a(Oa);Na=s(Yv,"CODE",{});var Qv=a(Na);tc=l(Qv,"AutoModelForSeq2SeqLM"),Qv.forEach(o),Yv.forEach(o),Rp.forEach(o),oc=h(U),so=s(U,"TR",{});var Wp=a(so);ja=s(Wp,"TD",{});var Zv=a(ja);Aa=s(Zv,"CODE",{});var ew=a(Aa);nc=l(ew,"sequence-classification"),ew.forEach(o),Zv.forEach(o),sc=h(Wp),Da=s(Wp,"TD",{});var tw=a(Da);Ca=s(tw,"CODE",{});var ow=a(Ca);ac=l(ow,"AutoModelForSequenceClassification"),ow.forEach(o),tw.forEach(o),Wp.forEach(o),rc=h(U),ao=s(U,"TR",{});var Kp=a(ao);Pa=s(Kp,"TD",{});var nw=a(Pa);Sa=s(nw,"CODE",{});var sw=a(Sa);lc=l(sw,"token-classification"),sw.forEach(o),nw.forEach(o),ic=h(Kp),qa=s(Kp,"TD",{});var aw=a(qa);Ia=s(aw,"CODE",{});var rw=a(Ia);pc=l(rw,"AutoModelForTokenClassification"),rw.forEach(o),aw.forEach(o),Kp.forEach(o),U.forEach(o),qp.forEach(o),Rl=h(e),et=s(e,"P",{});var Up=a(et);dc=l(Up,`For each configuration, you can find the list of supported features via the
`),Ba=s(Up,"CODE",{});var lw=a(Ba);hc=l(lw,"FeaturesManager"),lw.forEach(o),fc=l(Up,". For example, for DistilBERT we have:"),Up.forEach(o),Hl=h(e),w(ro.$$.fragment,e),Wl=h(e),le=s(e,"P",{});var Qn=a(le);cc=l(Qn,"You can then pass one of these features to the "),Ma=s(Qn,"CODE",{});var iw=a(Ma);uc=l(iw,"--feature"),iw.forEach(o),mc=l(Qn,` argument in the
`),za=s(Qn,"CODE",{});var pw=a(za);gc=l(pw,"transformers.onnx"),pw.forEach(o),_c=l(Qn,` package. For example, to export a text-classification model
we can pick a fine-tuned model from the Hub and run:`),Qn.forEach(o),Kl=h(e),w(lo.$$.fragment,e),Ul=h(e),vn=s(e,"P",{});var dw=a(vn);vc=l(dw,"which will display the following logs:"),dw.forEach(o),Vl=h(e),w(io.$$.fragment,e),Gl=h(e),Z=s(e,"P",{});var Ct=a(Z);wc=l(Ct,`Notice that in this case, the output names from the fine-tuned model are
`),La=s(Ct,"CODE",{});var hw=a(La);Ec=l(hw,"logits"),hw.forEach(o),$c=l(Ct," instead of the "),Xa=s(Ct,"CODE",{});var fw=a(Xa);bc=l(fw,"last_hidden_state"),fw.forEach(o),xc=l(Ct,` we saw with the
`),Fa=s(Ct,"CODE",{});var cw=a(Fa);yc=l(cw,"distilbert-base-uncased"),cw.forEach(o),kc=l(Ct,` checkpoint earlier. This is expected since the
fine-tuned model has a sequence classification head.`),Ct.forEach(o),Jl=h(e),w(tt.$$.fragment,e),Yl=h(e),ye=s(e,"H3",{class:!0});var Vp=a(ye);ot=s(Vp,"A",{id:!0,class:!0,href:!0});var uw=a(ot);Ra=s(uw,"SPAN",{});var mw=a(Ra);w(po.$$.fragment,mw),mw.forEach(o),uw.forEach(o),Tc=h(Vp),Ha=s(Vp,"SPAN",{});var gw=a(Ha);Oc=l(gw,"Exporting a model for an unsupported architecture"),gw.forEach(o),Vp.forEach(o),Ql=h(e),wn=s(e,"P",{});var _w=a(wn);Nc=l(_w,`If you wish to export a model whose architecture is not natively supported by
the library, there are three main steps to follow:`),_w.forEach(o),Zl=h(e),ie=s(e,"OL",{});var Zn=a(ie);Wa=s(Zn,"LI",{});var vw=a(Wa);jc=l(vw,"Implement a custom ONNX configuration."),vw.forEach(o),Ac=h(Zn),Ka=s(Zn,"LI",{});var ww=a(Ka);Dc=l(ww,"Export the model to ONNX."),ww.forEach(o),Cc=h(Zn),Ua=s(Zn,"LI",{});var Ew=a(Ua);Pc=l(Ew,"Validate the outputs of the PyTorch and exported models."),Ew.forEach(o),Zn.forEach(o),ei=h(e),En=s(e,"P",{});var $w=a(En);Sc=l($w,`In this section, we\u2019ll look at how DistilBERT was implemented to show what\u2019s
involved with each step.`),$w.forEach(o),ti=h(e),ke=s(e,"H4",{class:!0});var Gp=a(ke);nt=s(Gp,"A",{id:!0,class:!0,href:!0});var bw=a(nt);Va=s(bw,"SPAN",{});var xw=a(Va);w(ho.$$.fragment,xw),xw.forEach(o),bw.forEach(o),qc=h(Gp),Ga=s(Gp,"SPAN",{});var yw=a(Ga);Ic=l(yw,"Implementing a custom ONNX configuration"),yw.forEach(o),Gp.forEach(o),oi=h(e),$n=s(e,"P",{});var kw=a($n);Bc=l(kw,`Let\u2019s start with the ONNX configuration object. We provide three abstract
classes that you should inherit from, depending on the type of model
architecture you wish to export:`),kw.forEach(o),ni=h(e),pe=s(e,"UL",{});var es=a(pe);bn=s(es,"LI",{});var c_=a(bn);Mc=l(c_,"Encoder-based models inherit from "),xn=s(c_,"A",{href:!0});var Tw=a(xn);zc=l(Tw,"OnnxConfig"),Tw.forEach(o),c_.forEach(o),Lc=h(es),yn=s(es,"LI",{});var u_=a(yn);Xc=l(u_,"Decoder-based models inherit from "),kn=s(u_,"A",{href:!0});var Ow=a(kn);Fc=l(Ow,"OnnxConfigWithPast"),Ow.forEach(o),u_.forEach(o),Rc=h(es),Tn=s(es,"LI",{});var m_=a(Tn);Hc=l(m_,"Encoder-decoder models inherit from "),On=s(m_,"A",{href:!0});var Nw=a(On);Wc=l(Nw,"OnnxSeq2SeqConfigWithPast"),Nw.forEach(o),m_.forEach(o),es.forEach(o),si=h(e),w(st.$$.fragment,e),ai=h(e),at=s(e,"P",{});var Jp=a(at);Kc=l(Jp,`Since DistilBERT is an encoder-based model, its configuration inherits from
`),Ja=s(Jp,"CODE",{});var jw=a(Ja);Uc=l(jw,"OnnxConfig"),jw.forEach(o),Vc=l(Jp,":"),Jp.forEach(o),ri=h(e),w(fo.$$.fragment,e),li=h(e),W=s(e,"P",{});var _e=a(W);Gc=l(_e,"Every configuration object must implement the "),Ya=s(_e,"CODE",{});var Aw=a(Ya);Jc=l(Aw,"inputs"),Aw.forEach(o),Yc=l(_e,` property and return a
mapping, where each key corresponds to an expected input, and each value
indicates the axis of that input. For DistilBERT, we can see that two inputs are
required: `),Qa=s(_e,"CODE",{});var Dw=a(Qa);Qc=l(Dw,"input_ids"),Dw.forEach(o),Zc=l(_e," and "),Za=s(_e,"CODE",{});var Cw=a(Za);eu=l(Cw,"attention_mask"),Cw.forEach(o),tu=l(_e,`. These inputs have the same shape of
`),er=s(_e,"CODE",{});var Pw=a(er);ou=l(Pw,"(batch_size, sequence_length)"),Pw.forEach(o),nu=l(_e,` which is why we see the same axes used in the
configuration.`),_e.forEach(o),ii=h(e),w(rt.$$.fragment,e),pi=h(e),Nn=s(e,"P",{});var Sw=a(Nn);su=l(Sw,`Once you have implemented an ONNX configuration, you can instantiate it by
providing the base model\u2019s configuration as follows:`),Sw.forEach(o),di=h(e),w(co.$$.fragment,e),hi=h(e),jn=s(e,"P",{});var qw=a(jn);au=l(qw,`The resulting object has several useful properties. For example you can view the
ONNX operator set that will be used during the export:`),qw.forEach(o),fi=h(e),w(uo.$$.fragment,e),ci=h(e),An=s(e,"P",{});var Iw=a(An);ru=l(Iw,"You can also view the outputs associated with the model as follows:"),Iw.forEach(o),ui=h(e),w(mo.$$.fragment,e),mi=h(e),K=s(e,"P",{});var ve=a(K);lu=l(ve,`Notice that the outputs property follows the same structure as the inputs; it
returns an `),tr=s(ve,"CODE",{});var Bw=a(tr);iu=l(Bw,"OrderedDict"),Bw.forEach(o),pu=l(ve,` of named outputs and their shapes. The output structure
is linked to the choice of feature that the configuration is initialised with.
By default, the ONNX configuration is initialized with the `),or=s(ve,"CODE",{});var Mw=a(or);du=l(Mw,"default"),Mw.forEach(o),hu=l(ve,` feature
that corresponds to exporting a model loaded with the `),nr=s(ve,"CODE",{});var zw=a(nr);fu=l(zw,"AutoModel"),zw.forEach(o),cu=l(ve,` class. If you
want to export a different model topology, just provide a different feature to
the `),sr=s(ve,"CODE",{});var Lw=a(sr);uu=l(Lw,"task"),Lw.forEach(o),mu=l(ve,` argument when you initialize the ONNX configuration. For example, if
we wished to export DistilBERT with a sequence classification head, we could
use:`),ve.forEach(o),gi=h(e),w(go.$$.fragment,e),_i=h(e),w(lt.$$.fragment,e),vi=h(e),Te=s(e,"H4",{class:!0});var Yp=a(Te);it=s(Yp,"A",{id:!0,class:!0,href:!0});var Xw=a(it);ar=s(Xw,"SPAN",{});var Fw=a(ar);w(_o.$$.fragment,Fw),Fw.forEach(o),Xw.forEach(o),gu=h(Yp),rr=s(Yp,"SPAN",{});var Rw=a(rr);_u=l(Rw,"Exporting the model"),Rw.forEach(o),Yp.forEach(o),wi=h(e),de=s(e,"P",{});var ts=a(de);vu=l(ts,`Once you have implemented the ONNX configuration, the next step is to export the
model. Here we can use the `),lr=s(ts,"CODE",{});var Hw=a(lr);wu=l(Hw,"export()"),Hw.forEach(o),Eu=l(ts,` function provided by the
`),ir=s(ts,"CODE",{});var Ww=a(ir);$u=l(Ww,"transformers.onnx"),Ww.forEach(o),bu=l(ts,` package. This function expects the ONNX configuration, along
with the base model and tokenizer, and the path to save the exported file:`),ts.forEach(o),Ei=h(e),w(vo.$$.fragment,e),$i=h(e),F=s(e,"P",{});var oe=a(F);xu=l(oe,"The "),pr=s(oe,"CODE",{});var Kw=a(pr);yu=l(Kw,"onnx_inputs"),Kw.forEach(o),ku=l(oe," and "),dr=s(oe,"CODE",{});var Uw=a(dr);Tu=l(Uw,"onnx_outputs"),Uw.forEach(o),Ou=l(oe," returned by the "),hr=s(oe,"CODE",{});var Vw=a(hr);Nu=l(Vw,"export()"),Vw.forEach(o),ju=l(oe,` function are
lists of the keys defined in the `),fr=s(oe,"CODE",{});var Gw=a(fr);Au=l(Gw,"inputs"),Gw.forEach(o),Du=l(oe," and "),cr=s(oe,"CODE",{});var Jw=a(cr);Cu=l(Jw,"outputs"),Jw.forEach(o),Pu=l(oe,` properties of the
configuration. Once the model is exported, you can test that the model is well
formed as follows:`),oe.forEach(o),bi=h(e),w(wo.$$.fragment,e),xi=h(e),w(pt.$$.fragment,e),yi=h(e),Oe=s(e,"H4",{class:!0});var Qp=a(Oe);dt=s(Qp,"A",{id:!0,class:!0,href:!0});var Yw=a(dt);ur=s(Yw,"SPAN",{});var Qw=a(ur);w(Eo.$$.fragment,Qw),Qw.forEach(o),Yw.forEach(o),Su=h(Qp),mr=s(Qp,"SPAN",{});var Zw=a(mr);qu=l(Zw,"Validating the model outputs"),Zw.forEach(o),Qp.forEach(o),ki=h(e),he=s(e,"P",{});var os=a(he);Iu=l(os,`The final step is to validate that the outputs from the base and exported model
agree within some absolute tolerance. Here we can use the
`),gr=s(os,"CODE",{});var e1=a(gr);Bu=l(e1,"validate_model_outputs()"),e1.forEach(o),Mu=l(os," function provided by the "),_r=s(os,"CODE",{});var t1=a(_r);zu=l(t1,"transformers.onnx"),t1.forEach(o),Lu=l(os,` package
as follows:`),os.forEach(o),Ti=h(e),w($o.$$.fragment,e),Oi=h(e),ht=s(e,"P",{});var Zp=a(ht);Xu=l(Zp,"This function uses the "),vr=s(Zp,"CODE",{});var o1=a(vr);Fu=l(o1,"OnnxConfig.generate_dummy_inputs()"),o1.forEach(o),Ru=l(Zp,` method to generate
inputs for the base and exported model, and the absolute tolerance can be
defined in the configuration. We generally find numerical agreement in the 1e-6
to 1e-4 range, although anything smaller than 1e-3 is likely to be OK.`),Zp.forEach(o),Ni=h(e),Ne=s(e,"H3",{class:!0});var ed=a(Ne);ft=s(ed,"A",{id:!0,class:!0,href:!0});var n1=a(ft);wr=s(n1,"SPAN",{});var s1=a(wr);w(bo.$$.fragment,s1),s1.forEach(o),n1.forEach(o),Hu=h(ed),Er=s(ed,"SPAN",{});var a1=a(Er);Wu=l(a1,"Contributing a new configuration to \u{1F917} Transformers"),a1.forEach(o),ed.forEach(o),ji=h(e),Dn=s(e,"P",{});var r1=a(Dn);Ku=l(r1,`We are looking to expand the set of ready-made configurations and welcome
contributions from the community! If you would like to contribute your addition
to the library, you will need to:`),r1.forEach(o),Ai=h(e),fe=s(e,"UL",{});var ns=a(fe);xo=s(ns,"LI",{});var td=a(xo);Uu=l(td,"Implement the ONNX configuration in the corresponding "),$r=s(td,"CODE",{});var l1=a($r);Vu=l(l1,"configuration_<model_name>.py"),l1.forEach(o),Gu=l(td,`
file`),td.forEach(o),Ju=h(ns),Cn=s(ns,"LI",{});var g_=a(Cn);Yu=l(g_,"Include the model architecture and corresponding features in "),br=s(g_,"CODE",{});var i1=a(br);Qu=l(i1,"FeatureManager"),i1.forEach(o),g_.forEach(o),Zu=h(ns),Pn=s(ns,"LI",{});var __=a(Pn);em=l(__,"Add your model architecture to the tests in "),xr=s(__,"CODE",{});var p1=a(xr);tm=l(p1,"test_onnx_v2.py"),p1.forEach(o),__.forEach(o),ns.forEach(o),Di=h(e),ct=s(e,"P",{});var od=a(ct);om=l(od,"Check out how the configuration for "),yo=s(od,"A",{href:!0,rel:!0});var d1=a(yo);nm=l(d1,`IBERT was
contributed`),d1.forEach(o),sm=l(od,` to
get an idea of what\u2019s involved.`),od.forEach(o),Ci=h(e),je=s(e,"H2",{class:!0});var nd=a(je);ut=s(nd,"A",{id:!0,class:!0,href:!0});var h1=a(ut);yr=s(h1,"SPAN",{});var f1=a(yr);w(ko.$$.fragment,f1),f1.forEach(o),h1.forEach(o),am=h(nd),kr=s(nd,"SPAN",{});var c1=a(kr);rm=l(c1,"TorchScript"),c1.forEach(o),nd.forEach(o),Pi=h(e),w(mt.$$.fragment,e),Si=h(e),gt=s(e,"P",{});var sd=a(gt);lm=l(sd,`According to Pytorch\u2019s documentation: \u201CTorchScript is a way to create serializable and optimizable models from PyTorch
code\u201D. Pytorch\u2019s two modules `),To=s(sd,"A",{href:!0,rel:!0});var u1=a(To);im=l(u1,"JIT and TRACE"),u1.forEach(o),pm=l(sd,` allow the developer to export
their model to be re-used in other programs, such as efficiency-oriented C++ programs.`),sd.forEach(o),qi=h(e),Sn=s(e,"P",{});var m1=a(Sn);dm=l(m1,`We have provided an interface that allows the export of \u{1F917} Transformers models to TorchScript so that they can be reused
in a different environment than a Pytorch-based python program. Here we explain how to export and use our models using
TorchScript.`),m1.forEach(o),Ii=h(e),qn=s(e,"P",{});var g1=a(qn);hm=l(g1,"Exporting a model requires two things:"),g1.forEach(o),Bi=h(e),_t=s(e,"UL",{});var ad=a(_t);Tr=s(ad,"LI",{});var _1=a(Tr);fm=l(_1,"a forward pass with dummy inputs."),_1.forEach(o),cm=h(ad),Oo=s(ad,"LI",{});var rd=a(Oo);um=l(rd,"model instantiation with the "),Or=s(rd,"CODE",{});var v1=a(Or);mm=l(v1,"torchscript"),v1.forEach(o),gm=l(rd," flag."),rd.forEach(o),ad.forEach(o),Mi=h(e),In=s(e,"P",{});var w1=a(In);_m=l(w1,"These necessities imply several things developers should be careful about. These are detailed below."),w1.forEach(o),zi=h(e),Ae=s(e,"H3",{class:!0});var ld=a(Ae);vt=s(ld,"A",{id:!0,class:!0,href:!0});var E1=a(vt);Nr=s(E1,"SPAN",{});var $1=a(Nr);w(No.$$.fragment,$1),$1.forEach(o),E1.forEach(o),vm=h(ld),jr=s(ld,"SPAN",{});var b1=a(jr);wm=l(b1,"TorchScript flag and tied weights"),b1.forEach(o),ld.forEach(o),Li=h(e),ce=s(e,"P",{});var ss=a(ce);Em=l(ss,`This flag is necessary because most of the language models in this repository have tied weights between their
`),Ar=s(ss,"CODE",{});var x1=a(Ar);$m=l(x1,"Embedding"),x1.forEach(o),bm=l(ss," layer and their "),Dr=s(ss,"CODE",{});var y1=a(Dr);xm=l(y1,"Decoding"),y1.forEach(o),ym=l(ss,` layer. TorchScript does not allow the export of models that have tied
weights, therefore it is necessary to untie and clone the weights beforehand.`),ss.forEach(o),Xi=h(e),ee=s(e,"P",{});var Pt=a(ee);km=l(Pt,"This implies that models instantiated with the "),Cr=s(Pt,"CODE",{});var k1=a(Cr);Tm=l(k1,"torchscript"),k1.forEach(o),Om=l(Pt," flag have their "),Pr=s(Pt,"CODE",{});var T1=a(Pr);Nm=l(T1,"Embedding"),T1.forEach(o),jm=l(Pt," layer and "),Sr=s(Pt,"CODE",{});var O1=a(Sr);Am=l(O1,"Decoding"),O1.forEach(o),Dm=l(Pt,`
layer separate, which means that they should not be trained down the line. Training would de-synchronize the two
layers, leading to unexpected results.`),Pt.forEach(o),Fi=h(e),wt=s(e,"P",{});var id=a(wt);Cm=l(id,`This is not the case for models that do not have a Language Model head, as those do not have tied weights. These models
can be safely exported without the `),qr=s(id,"CODE",{});var N1=a(qr);Pm=l(N1,"torchscript"),N1.forEach(o),Sm=l(id," flag."),id.forEach(o),Ri=h(e),De=s(e,"H3",{class:!0});var pd=a(De);Et=s(pd,"A",{id:!0,class:!0,href:!0});var j1=a(Et);Ir=s(j1,"SPAN",{});var A1=a(Ir);w(jo.$$.fragment,A1),A1.forEach(o),j1.forEach(o),qm=h(pd),Br=s(pd,"SPAN",{});var D1=a(Br);Im=l(D1,"Dummy inputs and standard lengths"),D1.forEach(o),pd.forEach(o),Hi=h(e),Bn=s(e,"P",{});var C1=a(Bn);Bm=l(C1,`The dummy inputs are used to do a model forward pass. While the inputs\u2019 values are propagating through the layers,
Pytorch keeps track of the different operations executed on each tensor. These recorded operations are then used to
create the \u201Ctrace\u201D of the model.`),C1.forEach(o),Wi=h(e),Mn=s(e,"P",{});var P1=a(Mn);Mm=l(P1,`The trace is created relatively to the inputs\u2019 dimensions. It is therefore constrained by the dimensions of the dummy
input, and will not work for any other sequence length or batch size. When trying with a different size, an error such
as:`),P1.forEach(o),Ki=h(e),zn=s(e,"P",{});var S1=a(zn);Mr=s(S1,"CODE",{});var q1=a(Mr);zm=l(q1,"The expanded size of the tensor (3) must match the existing size (7) at non-singleton dimension 2"),q1.forEach(o),S1.forEach(o),Ui=h(e),Ln=s(e,"P",{});var I1=a(Ln);Lm=l(I1,`will be raised. It is therefore recommended to trace the model with a dummy input size at least as large as the largest
input that will be fed to the model during inference. Padding can be performed to fill the missing values. As the model
will have been traced with a large input size however, the dimensions of the different matrix will be large as well,
resulting in more calculations.`),I1.forEach(o),Vi=h(e),Xn=s(e,"P",{});var B1=a(Xn);Xm=l(B1,`It is recommended to be careful of the total number of operations done on each input and to follow performance closely
when exporting varying sequence-length models.`),B1.forEach(o),Gi=h(e),Ce=s(e,"H3",{class:!0});var dd=a(Ce);$t=s(dd,"A",{id:!0,class:!0,href:!0});var M1=a($t);zr=s(M1,"SPAN",{});var z1=a(zr);w(Ao.$$.fragment,z1),z1.forEach(o),M1.forEach(o),Fm=h(dd),Lr=s(dd,"SPAN",{});var L1=a(Lr);Rm=l(L1,"Using TorchScript in Python"),L1.forEach(o),dd.forEach(o),Ji=h(e),Fn=s(e,"P",{});var X1=a(Fn);Hm=l(X1,"Below is an example, showing how to save, load models as well as how to use the trace for inference."),X1.forEach(o),Yi=h(e),Pe=s(e,"H4",{class:!0});var hd=a(Pe);bt=s(hd,"A",{id:!0,class:!0,href:!0});var F1=a(bt);Xr=s(F1,"SPAN",{});var R1=a(Xr);w(Do.$$.fragment,R1),R1.forEach(o),F1.forEach(o),Wm=h(hd),Fr=s(hd,"SPAN",{});var H1=a(Fr);Km=l(H1,"Saving a model"),H1.forEach(o),hd.forEach(o),Qi=h(e),Y=s(e,"P",{});var Fe=a(Y);Um=l(Fe,"This snippet shows how to use TorchScript to export a "),Rr=s(Fe,"CODE",{});var W1=a(Rr);Vm=l(W1,"BertModel"),W1.forEach(o),Gm=l(Fe,". Here the "),Hr=s(Fe,"CODE",{});var K1=a(Hr);Jm=l(K1,"BertModel"),K1.forEach(o),Ym=l(Fe,` is instantiated according
to a `),Wr=s(Fe,"CODE",{});var U1=a(Wr);Qm=l(U1,"BertConfig"),U1.forEach(o),Zm=l(Fe," class and then saved to disk under the filename "),Kr=s(Fe,"CODE",{});var V1=a(Kr);eg=l(V1,"traced_bert.pt"),V1.forEach(o),Fe.forEach(o),Zi=h(e),w(Co.$$.fragment,e),ep=h(e),Se=s(e,"H4",{class:!0});var fd=a(Se);xt=s(fd,"A",{id:!0,class:!0,href:!0});var G1=a(xt);Ur=s(G1,"SPAN",{});var J1=a(Ur);w(Po.$$.fragment,J1),J1.forEach(o),G1.forEach(o),tg=h(fd),Vr=s(fd,"SPAN",{});var Y1=a(Vr);og=l(Y1,"Loading a model"),Y1.forEach(o),fd.forEach(o),tp=h(e),te=s(e,"P",{});var St=a(te);ng=l(St,"This snippet shows how to load the "),Gr=s(St,"CODE",{});var Q1=a(Gr);sg=l(Q1,"BertModel"),Q1.forEach(o),ag=l(St," that was previously saved to disk under the name "),Jr=s(St,"CODE",{});var Z1=a(Jr);rg=l(Z1,"traced_bert.pt"),Z1.forEach(o),lg=l(St,`.
We are re-using the previously initialised `),Yr=s(St,"CODE",{});var eE=a(Yr);ig=l(eE,"dummy_input"),eE.forEach(o),pg=l(St,"."),St.forEach(o),op=h(e),w(So.$$.fragment,e),np=h(e),qe=s(e,"H4",{class:!0});var cd=a(qe);yt=s(cd,"A",{id:!0,class:!0,href:!0});var tE=a(yt);Qr=s(tE,"SPAN",{});var oE=a(Qr);w(qo.$$.fragment,oE),oE.forEach(o),tE.forEach(o),dg=h(cd),Zr=s(cd,"SPAN",{});var nE=a(Zr);hg=l(nE,"Using a traced model for inference"),nE.forEach(o),cd.forEach(o),sp=h(e),kt=s(e,"P",{});var ud=a(kt);fg=l(ud,"Using the traced model for inference is as simple as using its "),el=s(ud,"CODE",{});var sE=a(el);cg=l(sE,"__call__"),sE.forEach(o),ug=l(ud," dunder method:"),ud.forEach(o),ap=h(e),w(Io.$$.fragment,e),rp=h(e),Ie=s(e,"H3",{class:!0});var md=a(Ie);Tt=s(md,"A",{id:!0,class:!0,href:!0});var aE=a(Tt);tl=s(aE,"SPAN",{});var rE=a(tl);w(Bo.$$.fragment,rE),rE.forEach(o),aE.forEach(o),mg=h(md),ol=s(md,"SPAN",{});var lE=a(ol);gg=l(lE,"Deploying HuggingFace TorchScript models on AWS using the Neuron SDK"),lE.forEach(o),md.forEach(o),lp=h(e),ue=s(e,"P",{});var as=a(ue);_g=l(as,"AWS introduced the "),Mo=s(as,"A",{href:!0,rel:!0});var iE=a(Mo);vg=l(iE,"Amazon EC2 Inf1"),iE.forEach(o),wg=l(as,`
instance family for low cost, high performance machine learning inference in the cloud.
The Inf1 instances are powered by the AWS Inferentia chip, a custom-built hardware accelerator,
specializing in deep learning inferencing workloads.
`),zo=s(as,"A",{href:!0,rel:!0});var pE=a(zo);Eg=l(pE,"AWS Neuron"),pE.forEach(o),$g=l(as,`
is the SDK for Inferentia that supports tracing and optimizing transformers models for
deployment on Inf1. The Neuron SDK provides:`),as.forEach(o),ip=h(e),me=s(e,"OL",{});var rs=a(me);nl=s(rs,"LI",{});var dE=a(nl);bg=l(dE,"Easy-to-use API with one line of code change to trace and optimize a TorchScript model for inference in the cloud."),dE.forEach(o),xg=h(rs),Rn=s(rs,"LI",{});var v_=a(Rn);yg=l(v_,"Out of the box performance optimizations for "),Lo=s(v_,"A",{href:!0,rel:!0});var hE=a(Lo);kg=l(hE,"improved cost-performance"),hE.forEach(o),v_.forEach(o),Tg=h(rs),Be=s(rs,"LI",{});var ls=a(Be);Og=l(ls,"Support for HuggingFace transformers models built with either "),Xo=s(ls,"A",{href:!0,rel:!0});var fE=a(Xo);Ng=l(fE,"PyTorch"),fE.forEach(o),jg=l(ls,`
or `),Fo=s(ls,"A",{href:!0,rel:!0});var cE=a(Fo);Ag=l(cE,"TensorFlow"),cE.forEach(o),Dg=l(ls,"."),ls.forEach(o),rs.forEach(o),pp=h(e),Me=s(e,"H4",{class:!0});var gd=a(Me);Ot=s(gd,"A",{id:!0,class:!0,href:!0});var uE=a(Ot);sl=s(uE,"SPAN",{});var mE=a(sl);w(Ro.$$.fragment,mE),mE.forEach(o),uE.forEach(o),Cg=h(gd),al=s(gd,"SPAN",{});var gE=a(al);Pg=l(gE,"Implications"),gE.forEach(o),gd.forEach(o),dp=h(e),R=s(e,"P",{});var ne=a(R);Sg=l(ne,"Transformers Models based on the "),Ho=s(ne,"A",{href:!0,rel:!0});var _E=a(Ho);qg=l(_E,"BERT (Bidirectional Encoder Representations from Transformers)"),_E.forEach(o),Ig=l(ne,`
architecture, or its variants such as `),Wo=s(ne,"A",{href:!0,rel:!0});var vE=a(Wo);Bg=l(vE,"distilBERT"),vE.forEach(o),Mg=l(ne,`
and `),Ko=s(ne,"A",{href:!0,rel:!0});var wE=a(Ko);zg=l(wE,"roBERTa"),wE.forEach(o),Lg=l(ne,`
will run best on Inf1 for non-generative tasks such as Extractive Question Answering,
Sequence Classification, Token Classification. Alternatively, text generation
tasks can be adapted to run on Inf1, according to this `),Uo=s(ne,"A",{href:!0,rel:!0});var EE=a(Uo);Xg=l(EE,"AWS Neuron MarianMT tutorial"),EE.forEach(o),Fg=l(ne,`.
More information about models that can be converted out of the box on Inferentia can be
found in the `),Vo=s(ne,"A",{href:!0,rel:!0});var $E=a(Vo);Rg=l($E,"Model Architecture Fit section of the Neuron documentation"),$E.forEach(o),Hg=l(ne,"."),ne.forEach(o),hp=h(e),ze=s(e,"H4",{class:!0});var _d=a(ze);Nt=s(_d,"A",{id:!0,class:!0,href:!0});var bE=a(Nt);rl=s(bE,"SPAN",{});var xE=a(rl);w(Go.$$.fragment,xE),xE.forEach(o),bE.forEach(o),Wg=h(_d),ll=s(_d,"SPAN",{});var yE=a(ll);Kg=l(yE,"Dependencies"),yE.forEach(o),_d.forEach(o),fp=h(e),Hn=s(e,"P",{});var kE=a(Hn);Ug=l(kE,"Using AWS Neuron to convert models requires the following dependencies and environment:"),kE.forEach(o),cp=h(e),Wn=s(e,"UL",{});var TE=a(Wn);Le=s(TE,"LI",{});var is=a(Le);Vg=l(is,"A "),Jo=s(is,"A",{href:!0,rel:!0});var OE=a(Jo);Gg=l(OE,"Neuron SDK environment"),OE.forEach(o),Jg=l(is,`,
which comes pre-configured on `),Yo=s(is,"A",{href:!0,rel:!0});var NE=a(Yo);Yg=l(NE,"AWS Deep Learning AMI"),NE.forEach(o),Qg=l(is,"."),is.forEach(o),TE.forEach(o),up=h(e),Xe=s(e,"H4",{class:!0});var vd=a(Xe);jt=s(vd,"A",{id:!0,class:!0,href:!0});var jE=a(jt);il=s(jE,"SPAN",{});var AE=a(il);w(Qo.$$.fragment,AE),AE.forEach(o),jE.forEach(o),Zg=h(vd),pl=s(vd,"SPAN",{});var DE=a(pl);e_=l(DE,"Converting a Model for AWS Neuron"),DE.forEach(o),vd.forEach(o),mp=h(e),ge=s(e,"P",{});var ps=a(ge);t_=l(ps,"Using the same script as in "),Zo=s(ps,"A",{href:!0,rel:!0});var CE=a(Zo);o_=l(CE,"Using TorchScript in Python"),CE.forEach(o),n_=l(ps,`
to trace a \u201CBertModel\u201D, you import `),dl=s(ps,"CODE",{});var PE=a(dl);s_=l(PE,"torch.neuron"),PE.forEach(o),a_=l(ps,` framework extension to access
the components of the Neuron SDK through a Python API.`),ps.forEach(o),gp=h(e),w(en.$$.fragment,e),_p=h(e),Kn=s(e,"P",{});var SE=a(Kn);r_=l(SE,"And only modify the tracing line of code"),SE.forEach(o),vp=h(e),Un=s(e,"P",{});var qE=a(Un);l_=l(qE,"from:"),qE.forEach(o),wp=h(e),w(tn.$$.fragment,e),Ep=h(e),Vn=s(e,"P",{});var IE=a(Vn);i_=l(IE,"to:"),IE.forEach(o),$p=h(e),w(on.$$.fragment,e),bp=h(e),Gn=s(e,"P",{});var BE=a(Gn);p_=l(BE,"This change enables Neuron SDK to trace the model and optimize it to run in Inf1 instances."),BE.forEach(o),xp=h(e),At=s(e,"P",{});var wd=a(At);d_=l(wd,`To learn more about AWS Neuron SDK features, tools, example tutorials and latest updates,
please see the `),nn=s(wd,"A",{href:!0,rel:!0});var ME=a(nn);h_=l(ME,"AWS NeuronSDK documentation"),ME.forEach(o),f_=l(wd,"."),wd.forEach(o),this.h()},h(){f(c,"name","hf:doc:metadata"),f(c,"content",JSON.stringify(s$)),f(y,"id","export-transformers-models"),f(y,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),f(y,"href","#export-transformers-models"),f(u,"class","relative group"),f(z,"href","https://github.com/huggingface/optimum"),f(z,"rel","nofollow"),f(H,"id","onnx"),f(H,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),f(H,"href","#onnx"),f(G,"class","relative group"),f(ae,"href","http://onnx.ai"),f(ae,"rel","nofollow"),f(Ke,"id","exporting-a-model-to-onnx"),f(Ke,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),f(Ke,"href","#exporting-a-model-to-onnx"),f(be,"class","relative group"),f(Ft,"href","https://onnx.ai/supported-tools.html#deployModel"),f(Ft,"rel","nofollow"),f(Rt,"href","https://onnxruntime.ai/"),f(Rt,"rel","nofollow"),f(Kt,"href","https://huggingface.co/keras-io"),f(Kt,"rel","nofollow"),f(Ye,"id","selecting-features-for-different-model-topologies"),f(Ye,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),f(Ye,"href","#selecting-features-for-different-model-topologies"),f(xe,"class","relative group"),f(ot,"id","exporting-a-model-for-an-unsupported-architecture"),f(ot,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),f(ot,"href","#exporting-a-model-for-an-unsupported-architecture"),f(ye,"class","relative group"),f(nt,"id","implementing-a-custom-onnx-configuration"),f(nt,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),f(nt,"href","#implementing-a-custom-onnx-configuration"),f(ke,"class","relative group"),f(xn,"href","/docs/transformers/pr_17548/en/main_classes/onnx#transformers.onnx.OnnxConfig"),f(kn,"href","/docs/transformers/pr_17548/en/main_classes/onnx#transformers.onnx.OnnxConfigWithPast"),f(On,"href","/docs/transformers/pr_17548/en/main_classes/onnx#transformers.onnx.OnnxSeq2SeqConfigWithPast"),f(it,"id","exporting-the-model"),f(it,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),f(it,"href","#exporting-the-model"),f(Te,"class","relative group"),f(dt,"id","validating-the-model-outputs"),f(dt,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),f(dt,"href","#validating-the-model-outputs"),f(Oe,"class","relative group"),f(ft,"id","contributing-a-new-configuration-to-transformers"),f(ft,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),f(ft,"href","#contributing-a-new-configuration-to-transformers"),f(Ne,"class","relative group"),f(yo,"href","https://github.com/huggingface/transformers/pull/14868/files"),f(yo,"rel","nofollow"),f(ut,"id","torchscript"),f(ut,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),f(ut,"href","#torchscript"),f(je,"class","relative group"),f(To,"href","https://pytorch.org/docs/stable/jit.html"),f(To,"rel","nofollow"),f(vt,"id","torchscript-flag-and-tied-weights"),f(vt,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),f(vt,"href","#torchscript-flag-and-tied-weights"),f(Ae,"class","relative group"),f(Et,"id","dummy-inputs-and-standard-lengths"),f(Et,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),f(Et,"href","#dummy-inputs-and-standard-lengths"),f(De,"class","relative group"),f($t,"id","using-torchscript-in-python"),f($t,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),f($t,"href","#using-torchscript-in-python"),f(Ce,"class","relative group"),f(bt,"id","saving-a-model"),f(bt,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),f(bt,"href","#saving-a-model"),f(Pe,"class","relative group"),f(xt,"id","loading-a-model"),f(xt,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),f(xt,"href","#loading-a-model"),f(Se,"class","relative group"),f(yt,"id","using-a-traced-model-for-inference"),f(yt,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),f(yt,"href","#using-a-traced-model-for-inference"),f(qe,"class","relative group"),f(Tt,"id","deploying-huggingface-torchscript-models-on-aws-using-the-neuron-sdk"),f(Tt,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),f(Tt,"href","#deploying-huggingface-torchscript-models-on-aws-using-the-neuron-sdk"),f(Ie,"class","relative group"),f(Mo,"href","https://aws.amazon.com/ec2/instance-types/inf1/"),f(Mo,"rel","nofollow"),f(zo,"href","https://awsdocs-neuron.readthedocs-hosted.com/en/latest/#"),f(zo,"rel","nofollow"),f(Lo,"href","https://awsdocs-neuron.readthedocs-hosted.com/en/latest/neuron-guide/benchmark/%3E"),f(Lo,"rel","nofollow"),f(Xo,"href","https://awsdocs-neuron.readthedocs-hosted.com/en/latest/src/examples/pytorch/bert_tutorial/tutorial_pretrained_bert.html"),f(Xo,"rel","nofollow"),f(Fo,"href","https://awsdocs-neuron.readthedocs-hosted.com/en/latest/src/examples/tensorflow/huggingface_bert/huggingface_bert.html"),f(Fo,"rel","nofollow"),f(Ot,"id","implications"),f(Ot,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),f(Ot,"href","#implications"),f(Me,"class","relative group"),f(Ho,"href","https://huggingface.co/docs/transformers/main/model_doc/bert"),f(Ho,"rel","nofollow"),f(Wo,"href","https://huggingface.co/docs/transformers/main/model_doc/distilbert"),f(Wo,"rel","nofollow"),f(Ko,"href","https://huggingface.co/docs/transformers/main/model_doc/roberta"),f(Ko,"rel","nofollow"),f(Uo,"href","https://awsdocs-neuron.readthedocs-hosted.com/en/latest/src/examples/pytorch/transformers-marianmt.html"),f(Uo,"rel","nofollow"),f(Vo,"href","https://awsdocs-neuron.readthedocs-hosted.com/en/latest/neuron-guide/models/models-inferentia.html#models-inferentia"),f(Vo,"rel","nofollow"),f(Nt,"id","dependencies"),f(Nt,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),f(Nt,"href","#dependencies"),f(ze,"class","relative group"),f(Jo,"href","https://awsdocs-neuron.readthedocs-hosted.com/en/latest/neuron-guide/neuron-frameworks/pytorch-neuron/index.html#installation-guide"),f(Jo,"rel","nofollow"),f(Yo,"href","https://docs.aws.amazon.com/dlami/latest/devguide/tutorial-inferentia-launching.html"),f(Yo,"rel","nofollow"),f(jt,"id","converting-a-model-for-aws-neuron"),f(jt,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),f(jt,"href","#converting-a-model-for-aws-neuron"),f(Xe,"class","relative group"),f(Zo,"href","https://huggingface.co/docs/transformers/main/en/serialization#using-torchscript-in-python"),f(Zo,"rel","nofollow"),f(nn,"href","https://awsdocs-neuron.readthedocs-hosted.com/en/latest/index.html"),f(nn,"rel","nofollow")},m(e,i){t(document.head,c),p(e,k,i),p(e,u,i),t(u,y),t(y,O),E(T,O,null),t(u,P),t(u,C),t(C,A),p(e,N,i),p(e,S,i),t(S,D),p(e,q,i),p(e,_,i),t(_,j),t(_,z),t(z,V),t(_,we),p(e,Re,i),p(e,G,i),t(G,H),t(H,Ee),E(se,Ee,null),t(G,$e),t(G,B),t(B,rn),p(e,qt,i),p(e,J,i),t(J,ln),t(J,ae),t(ae,pn),t(J,dn),t(J,hs),t(hs,Ed),t(J,$d),p(e,gl,i),p(e,hn,i),t(hn,bd),p(e,_l,i),p(e,He,i),t(He,xd),t(He,fs),t(fs,yd),t(He,kd),p(e,vl,i),p(e,fn,i),t(fn,Td),p(e,wl,i),p(e,m,i),t(m,cs),t(cs,Od),t(m,Nd),t(m,us),t(us,jd),t(m,Ad),t(m,ms),t(ms,Dd),t(m,Cd),t(m,gs),t(gs,Pd),t(m,Sd),t(m,_s),t(_s,qd),t(m,Id),t(m,vs),t(vs,Bd),t(m,Md),t(m,ws),t(ws,zd),t(m,Ld),t(m,Es),t(Es,Xd),t(m,Fd),t(m,$s),t($s,Rd),t(m,Hd),t(m,bs),t(bs,Wd),t(m,Kd),t(m,xs),t(xs,Ud),t(m,Vd),t(m,ys),t(ys,Gd),t(m,Jd),t(m,ks),t(ks,Yd),t(m,Qd),t(m,Ts),t(Ts,Zd),t(m,eh),t(m,Os),t(Os,th),t(m,oh),t(m,Ns),t(Ns,nh),t(m,sh),t(m,js),t(js,ah),t(m,rh),t(m,As),t(As,lh),t(m,ih),t(m,Ds),t(Ds,ph),t(m,dh),t(m,Cs),t(Cs,hh),t(m,fh),t(m,Ps),t(Ps,ch),t(m,uh),t(m,Ss),t(Ss,mh),t(m,gh),t(m,qs),t(qs,_h),t(m,vh),t(m,Is),t(Is,wh),t(m,Eh),t(m,Bs),t(Bs,$h),t(m,bh),t(m,Ms),t(Ms,xh),t(m,yh),t(m,zs),t(zs,kh),t(m,Th),t(m,Ls),t(Ls,Oh),t(m,Nh),t(m,Xs),t(Xs,jh),t(m,Ah),t(m,Fs),t(Fs,Dh),t(m,Ch),t(m,Rs),t(Rs,Ph),t(m,Sh),t(m,Hs),t(Hs,qh),t(m,Ih),t(m,Ws),t(Ws,Bh),t(m,Mh),t(m,Ks),t(Ks,zh),t(m,Lh),t(m,Us),t(Us,Xh),p(e,El,i),p(e,cn,i),t(cn,Fh),p(e,$l,i),p(e,We,i),t(We,It),t(It,Rh),t(It,Vs),t(Vs,Hh),t(It,Wh),t(We,Kh),t(We,Gs),t(Gs,Uh),p(e,bl,i),p(e,be,i),t(be,Ke),t(Ke,Js),E(Bt,Js,null),t(be,Vh),t(be,Ys),t(Ys,Gh),p(e,xl,i),p(e,un,i),t(un,Jh),p(e,yl,i),E(Mt,e,i),p(e,kl,i),p(e,Ue,i),t(Ue,Yh),t(Ue,Qs),t(Qs,Qh),t(Ue,Zh),p(e,Tl,i),E(zt,e,i),p(e,Ol,i),p(e,mn,i),t(mn,ef),p(e,Nl,i),E(Lt,e,i),p(e,jl,i),p(e,gn,i),t(gn,tf),p(e,Al,i),E(Xt,e,i),p(e,Dl,i),p(e,re,i),t(re,of),t(re,Zs),t(Zs,nf),t(re,sf),t(re,ea),t(ea,af),t(re,rf),p(e,Cl,i),p(e,Q,i),t(Q,lf),t(Q,ta),t(ta,pf),t(Q,df),t(Q,Ft),t(Ft,hf),t(Q,ff),t(Q,Rt),t(Rt,cf),t(Q,uf),p(e,Pl,i),E(Ht,e,i),p(e,Sl,i),p(e,Ve,i),t(Ve,mf),t(Ve,oa),t(oa,gf),t(Ve,_f),p(e,ql,i),E(Wt,e,i),p(e,Il,i),p(e,Ge,i),t(Ge,vf),t(Ge,Kt),t(Kt,wf),t(Ge,Ef),p(e,Bl,i),E(Ut,e,i),p(e,Ml,i),p(e,_n,i),t(_n,$f),p(e,zl,i),E(Je,e,i),p(e,Ll,i),p(e,xe,i),t(xe,Ye),t(Ye,na),E(Vt,na,null),t(xe,bf),t(xe,sa),t(sa,xf),p(e,Xl,i),p(e,Qe,i),t(Qe,yf),t(Qe,aa),t(aa,kf),t(Qe,Tf),p(e,Fl,i),p(e,Ze,i),t(Ze,ra),t(ra,Gt),t(Gt,la),t(la,Of),t(Gt,Nf),t(Gt,ia),t(ia,jf),t(Ze,Af),t(Ze,X),t(X,Jt),t(Jt,Yt),t(Yt,pa),t(pa,Df),t(Yt,Cf),t(Yt,da),t(da,Pf),t(Jt,Sf),t(Jt,ha),t(ha,fa),t(fa,qf),t(X,If),t(X,Qt),t(Qt,Zt),t(Zt,ca),t(ca,Bf),t(Zt,Mf),t(Zt,ua),t(ua,zf),t(Qt,Lf),t(Qt,ma),t(ma,ga),t(ga,Xf),t(X,Ff),t(X,eo),t(eo,_a),t(_a,va),t(va,Rf),t(eo,Hf),t(eo,wa),t(wa,Ea),t(Ea,Wf),t(X,Kf),t(X,to),t(to,$a),t($a,ba),t(ba,Uf),t(to,Vf),t(to,xa),t(xa,ya),t(ya,Gf),t(X,Jf),t(X,oo),t(oo,no),t(no,ka),t(ka,Yf),t(no,Qf),t(no,Ta),t(Ta,Zf),t(oo,ec),t(oo,Oa),t(Oa,Na),t(Na,tc),t(X,oc),t(X,so),t(so,ja),t(ja,Aa),t(Aa,nc),t(so,sc),t(so,Da),t(Da,Ca),t(Ca,ac),t(X,rc),t(X,ao),t(ao,Pa),t(Pa,Sa),t(Sa,lc),t(ao,ic),t(ao,qa),t(qa,Ia),t(Ia,pc),p(e,Rl,i),p(e,et,i),t(et,dc),t(et,Ba),t(Ba,hc),t(et,fc),p(e,Hl,i),E(ro,e,i),p(e,Wl,i),p(e,le,i),t(le,cc),t(le,Ma),t(Ma,uc),t(le,mc),t(le,za),t(za,gc),t(le,_c),p(e,Kl,i),E(lo,e,i),p(e,Ul,i),p(e,vn,i),t(vn,vc),p(e,Vl,i),E(io,e,i),p(e,Gl,i),p(e,Z,i),t(Z,wc),t(Z,La),t(La,Ec),t(Z,$c),t(Z,Xa),t(Xa,bc),t(Z,xc),t(Z,Fa),t(Fa,yc),t(Z,kc),p(e,Jl,i),E(tt,e,i),p(e,Yl,i),p(e,ye,i),t(ye,ot),t(ot,Ra),E(po,Ra,null),t(ye,Tc),t(ye,Ha),t(Ha,Oc),p(e,Ql,i),p(e,wn,i),t(wn,Nc),p(e,Zl,i),p(e,ie,i),t(ie,Wa),t(Wa,jc),t(ie,Ac),t(ie,Ka),t(Ka,Dc),t(ie,Cc),t(ie,Ua),t(Ua,Pc),p(e,ei,i),p(e,En,i),t(En,Sc),p(e,ti,i),p(e,ke,i),t(ke,nt),t(nt,Va),E(ho,Va,null),t(ke,qc),t(ke,Ga),t(Ga,Ic),p(e,oi,i),p(e,$n,i),t($n,Bc),p(e,ni,i),p(e,pe,i),t(pe,bn),t(bn,Mc),t(bn,xn),t(xn,zc),t(pe,Lc),t(pe,yn),t(yn,Xc),t(yn,kn),t(kn,Fc),t(pe,Rc),t(pe,Tn),t(Tn,Hc),t(Tn,On),t(On,Wc),p(e,si,i),E(st,e,i),p(e,ai,i),p(e,at,i),t(at,Kc),t(at,Ja),t(Ja,Uc),t(at,Vc),p(e,ri,i),E(fo,e,i),p(e,li,i),p(e,W,i),t(W,Gc),t(W,Ya),t(Ya,Jc),t(W,Yc),t(W,Qa),t(Qa,Qc),t(W,Zc),t(W,Za),t(Za,eu),t(W,tu),t(W,er),t(er,ou),t(W,nu),p(e,ii,i),E(rt,e,i),p(e,pi,i),p(e,Nn,i),t(Nn,su),p(e,di,i),E(co,e,i),p(e,hi,i),p(e,jn,i),t(jn,au),p(e,fi,i),E(uo,e,i),p(e,ci,i),p(e,An,i),t(An,ru),p(e,ui,i),E(mo,e,i),p(e,mi,i),p(e,K,i),t(K,lu),t(K,tr),t(tr,iu),t(K,pu),t(K,or),t(or,du),t(K,hu),t(K,nr),t(nr,fu),t(K,cu),t(K,sr),t(sr,uu),t(K,mu),p(e,gi,i),E(go,e,i),p(e,_i,i),E(lt,e,i),p(e,vi,i),p(e,Te,i),t(Te,it),t(it,ar),E(_o,ar,null),t(Te,gu),t(Te,rr),t(rr,_u),p(e,wi,i),p(e,de,i),t(de,vu),t(de,lr),t(lr,wu),t(de,Eu),t(de,ir),t(ir,$u),t(de,bu),p(e,Ei,i),E(vo,e,i),p(e,$i,i),p(e,F,i),t(F,xu),t(F,pr),t(pr,yu),t(F,ku),t(F,dr),t(dr,Tu),t(F,Ou),t(F,hr),t(hr,Nu),t(F,ju),t(F,fr),t(fr,Au),t(F,Du),t(F,cr),t(cr,Cu),t(F,Pu),p(e,bi,i),E(wo,e,i),p(e,xi,i),E(pt,e,i),p(e,yi,i),p(e,Oe,i),t(Oe,dt),t(dt,ur),E(Eo,ur,null),t(Oe,Su),t(Oe,mr),t(mr,qu),p(e,ki,i),p(e,he,i),t(he,Iu),t(he,gr),t(gr,Bu),t(he,Mu),t(he,_r),t(_r,zu),t(he,Lu),p(e,Ti,i),E($o,e,i),p(e,Oi,i),p(e,ht,i),t(ht,Xu),t(ht,vr),t(vr,Fu),t(ht,Ru),p(e,Ni,i),p(e,Ne,i),t(Ne,ft),t(ft,wr),E(bo,wr,null),t(Ne,Hu),t(Ne,Er),t(Er,Wu),p(e,ji,i),p(e,Dn,i),t(Dn,Ku),p(e,Ai,i),p(e,fe,i),t(fe,xo),t(xo,Uu),t(xo,$r),t($r,Vu),t(xo,Gu),t(fe,Ju),t(fe,Cn),t(Cn,Yu),t(Cn,br),t(br,Qu),t(fe,Zu),t(fe,Pn),t(Pn,em),t(Pn,xr),t(xr,tm),p(e,Di,i),p(e,ct,i),t(ct,om),t(ct,yo),t(yo,nm),t(ct,sm),p(e,Ci,i),p(e,je,i),t(je,ut),t(ut,yr),E(ko,yr,null),t(je,am),t(je,kr),t(kr,rm),p(e,Pi,i),E(mt,e,i),p(e,Si,i),p(e,gt,i),t(gt,lm),t(gt,To),t(To,im),t(gt,pm),p(e,qi,i),p(e,Sn,i),t(Sn,dm),p(e,Ii,i),p(e,qn,i),t(qn,hm),p(e,Bi,i),p(e,_t,i),t(_t,Tr),t(Tr,fm),t(_t,cm),t(_t,Oo),t(Oo,um),t(Oo,Or),t(Or,mm),t(Oo,gm),p(e,Mi,i),p(e,In,i),t(In,_m),p(e,zi,i),p(e,Ae,i),t(Ae,vt),t(vt,Nr),E(No,Nr,null),t(Ae,vm),t(Ae,jr),t(jr,wm),p(e,Li,i),p(e,ce,i),t(ce,Em),t(ce,Ar),t(Ar,$m),t(ce,bm),t(ce,Dr),t(Dr,xm),t(ce,ym),p(e,Xi,i),p(e,ee,i),t(ee,km),t(ee,Cr),t(Cr,Tm),t(ee,Om),t(ee,Pr),t(Pr,Nm),t(ee,jm),t(ee,Sr),t(Sr,Am),t(ee,Dm),p(e,Fi,i),p(e,wt,i),t(wt,Cm),t(wt,qr),t(qr,Pm),t(wt,Sm),p(e,Ri,i),p(e,De,i),t(De,Et),t(Et,Ir),E(jo,Ir,null),t(De,qm),t(De,Br),t(Br,Im),p(e,Hi,i),p(e,Bn,i),t(Bn,Bm),p(e,Wi,i),p(e,Mn,i),t(Mn,Mm),p(e,Ki,i),p(e,zn,i),t(zn,Mr),t(Mr,zm),p(e,Ui,i),p(e,Ln,i),t(Ln,Lm),p(e,Vi,i),p(e,Xn,i),t(Xn,Xm),p(e,Gi,i),p(e,Ce,i),t(Ce,$t),t($t,zr),E(Ao,zr,null),t(Ce,Fm),t(Ce,Lr),t(Lr,Rm),p(e,Ji,i),p(e,Fn,i),t(Fn,Hm),p(e,Yi,i),p(e,Pe,i),t(Pe,bt),t(bt,Xr),E(Do,Xr,null),t(Pe,Wm),t(Pe,Fr),t(Fr,Km),p(e,Qi,i),p(e,Y,i),t(Y,Um),t(Y,Rr),t(Rr,Vm),t(Y,Gm),t(Y,Hr),t(Hr,Jm),t(Y,Ym),t(Y,Wr),t(Wr,Qm),t(Y,Zm),t(Y,Kr),t(Kr,eg),p(e,Zi,i),E(Co,e,i),p(e,ep,i),p(e,Se,i),t(Se,xt),t(xt,Ur),E(Po,Ur,null),t(Se,tg),t(Se,Vr),t(Vr,og),p(e,tp,i),p(e,te,i),t(te,ng),t(te,Gr),t(Gr,sg),t(te,ag),t(te,Jr),t(Jr,rg),t(te,lg),t(te,Yr),t(Yr,ig),t(te,pg),p(e,op,i),E(So,e,i),p(e,np,i),p(e,qe,i),t(qe,yt),t(yt,Qr),E(qo,Qr,null),t(qe,dg),t(qe,Zr),t(Zr,hg),p(e,sp,i),p(e,kt,i),t(kt,fg),t(kt,el),t(el,cg),t(kt,ug),p(e,ap,i),E(Io,e,i),p(e,rp,i),p(e,Ie,i),t(Ie,Tt),t(Tt,tl),E(Bo,tl,null),t(Ie,mg),t(Ie,ol),t(ol,gg),p(e,lp,i),p(e,ue,i),t(ue,_g),t(ue,Mo),t(Mo,vg),t(ue,wg),t(ue,zo),t(zo,Eg),t(ue,$g),p(e,ip,i),p(e,me,i),t(me,nl),t(nl,bg),t(me,xg),t(me,Rn),t(Rn,yg),t(Rn,Lo),t(Lo,kg),t(me,Tg),t(me,Be),t(Be,Og),t(Be,Xo),t(Xo,Ng),t(Be,jg),t(Be,Fo),t(Fo,Ag),t(Be,Dg),p(e,pp,i),p(e,Me,i),t(Me,Ot),t(Ot,sl),E(Ro,sl,null),t(Me,Cg),t(Me,al),t(al,Pg),p(e,dp,i),p(e,R,i),t(R,Sg),t(R,Ho),t(Ho,qg),t(R,Ig),t(R,Wo),t(Wo,Bg),t(R,Mg),t(R,Ko),t(Ko,zg),t(R,Lg),t(R,Uo),t(Uo,Xg),t(R,Fg),t(R,Vo),t(Vo,Rg),t(R,Hg),p(e,hp,i),p(e,ze,i),t(ze,Nt),t(Nt,rl),E(Go,rl,null),t(ze,Wg),t(ze,ll),t(ll,Kg),p(e,fp,i),p(e,Hn,i),t(Hn,Ug),p(e,cp,i),p(e,Wn,i),t(Wn,Le),t(Le,Vg),t(Le,Jo),t(Jo,Gg),t(Le,Jg),t(Le,Yo),t(Yo,Yg),t(Le,Qg),p(e,up,i),p(e,Xe,i),t(Xe,jt),t(jt,il),E(Qo,il,null),t(Xe,Zg),t(Xe,pl),t(pl,e_),p(e,mp,i),p(e,ge,i),t(ge,t_),t(ge,Zo),t(Zo,o_),t(ge,n_),t(ge,dl),t(dl,s_),t(ge,a_),p(e,gp,i),E(en,e,i),p(e,_p,i),p(e,Kn,i),t(Kn,r_),p(e,vp,i),p(e,Un,i),t(Un,l_),p(e,wp,i),E(tn,e,i),p(e,Ep,i),p(e,Vn,i),t(Vn,i_),p(e,$p,i),E(on,e,i),p(e,bp,i),p(e,Gn,i),t(Gn,p_),p(e,xp,i),p(e,At,i),t(At,d_),t(At,nn),t(nn,h_),t(At,f_),yp=!0},p(e,[i]){const sn={};i&2&&(sn.$$scope={dirty:i,ctx:e}),Je.$set(sn);const hl={};i&2&&(hl.$$scope={dirty:i,ctx:e}),tt.$set(hl);const fl={};i&2&&(fl.$$scope={dirty:i,ctx:e}),st.$set(fl);const cl={};i&2&&(cl.$$scope={dirty:i,ctx:e}),rt.$set(cl);const ul={};i&2&&(ul.$$scope={dirty:i,ctx:e}),lt.$set(ul);const an={};i&2&&(an.$$scope={dirty:i,ctx:e}),pt.$set(an);const ml={};i&2&&(ml.$$scope={dirty:i,ctx:e}),mt.$set(ml)},i(e){yp||($(T.$$.fragment,e),$(se.$$.fragment,e),$(Bt.$$.fragment,e),$(Mt.$$.fragment,e),$(zt.$$.fragment,e),$(Lt.$$.fragment,e),$(Xt.$$.fragment,e),$(Ht.$$.fragment,e),$(Wt.$$.fragment,e),$(Ut.$$.fragment,e),$(Je.$$.fragment,e),$(Vt.$$.fragment,e),$(ro.$$.fragment,e),$(lo.$$.fragment,e),$(io.$$.fragment,e),$(tt.$$.fragment,e),$(po.$$.fragment,e),$(ho.$$.fragment,e),$(st.$$.fragment,e),$(fo.$$.fragment,e),$(rt.$$.fragment,e),$(co.$$.fragment,e),$(uo.$$.fragment,e),$(mo.$$.fragment,e),$(go.$$.fragment,e),$(lt.$$.fragment,e),$(_o.$$.fragment,e),$(vo.$$.fragment,e),$(wo.$$.fragment,e),$(pt.$$.fragment,e),$(Eo.$$.fragment,e),$($o.$$.fragment,e),$(bo.$$.fragment,e),$(ko.$$.fragment,e),$(mt.$$.fragment,e),$(No.$$.fragment,e),$(jo.$$.fragment,e),$(Ao.$$.fragment,e),$(Do.$$.fragment,e),$(Co.$$.fragment,e),$(Po.$$.fragment,e),$(So.$$.fragment,e),$(qo.$$.fragment,e),$(Io.$$.fragment,e),$(Bo.$$.fragment,e),$(Ro.$$.fragment,e),$(Go.$$.fragment,e),$(Qo.$$.fragment,e),$(en.$$.fragment,e),$(tn.$$.fragment,e),$(on.$$.fragment,e),yp=!0)},o(e){b(T.$$.fragment,e),b(se.$$.fragment,e),b(Bt.$$.fragment,e),b(Mt.$$.fragment,e),b(zt.$$.fragment,e),b(Lt.$$.fragment,e),b(Xt.$$.fragment,e),b(Ht.$$.fragment,e),b(Wt.$$.fragment,e),b(Ut.$$.fragment,e),b(Je.$$.fragment,e),b(Vt.$$.fragment,e),b(ro.$$.fragment,e),b(lo.$$.fragment,e),b(io.$$.fragment,e),b(tt.$$.fragment,e),b(po.$$.fragment,e),b(ho.$$.fragment,e),b(st.$$.fragment,e),b(fo.$$.fragment,e),b(rt.$$.fragment,e),b(co.$$.fragment,e),b(uo.$$.fragment,e),b(mo.$$.fragment,e),b(go.$$.fragment,e),b(lt.$$.fragment,e),b(_o.$$.fragment,e),b(vo.$$.fragment,e),b(wo.$$.fragment,e),b(pt.$$.fragment,e),b(Eo.$$.fragment,e),b($o.$$.fragment,e),b(bo.$$.fragment,e),b(ko.$$.fragment,e),b(mt.$$.fragment,e),b(No.$$.fragment,e),b(jo.$$.fragment,e),b(Ao.$$.fragment,e),b(Do.$$.fragment,e),b(Co.$$.fragment,e),b(Po.$$.fragment,e),b(So.$$.fragment,e),b(qo.$$.fragment,e),b(Io.$$.fragment,e),b(Bo.$$.fragment,e),b(Ro.$$.fragment,e),b(Go.$$.fragment,e),b(Qo.$$.fragment,e),b(en.$$.fragment,e),b(tn.$$.fragment,e),b(on.$$.fragment,e),yp=!1},d(e){o(c),e&&o(k),e&&o(u),x(T),e&&o(N),e&&o(S),e&&o(q),e&&o(_),e&&o(Re),e&&o(G),x(se),e&&o(qt),e&&o(J),e&&o(gl),e&&o(hn),e&&o(_l),e&&o(He),e&&o(vl),e&&o(fn),e&&o(wl),e&&o(m),e&&o(El),e&&o(cn),e&&o($l),e&&o(We),e&&o(bl),e&&o(be),x(Bt),e&&o(xl),e&&o(un),e&&o(yl),x(Mt,e),e&&o(kl),e&&o(Ue),e&&o(Tl),x(zt,e),e&&o(Ol),e&&o(mn),e&&o(Nl),x(Lt,e),e&&o(jl),e&&o(gn),e&&o(Al),x(Xt,e),e&&o(Dl),e&&o(re),e&&o(Cl),e&&o(Q),e&&o(Pl),x(Ht,e),e&&o(Sl),e&&o(Ve),e&&o(ql),x(Wt,e),e&&o(Il),e&&o(Ge),e&&o(Bl),x(Ut,e),e&&o(Ml),e&&o(_n),e&&o(zl),x(Je,e),e&&o(Ll),e&&o(xe),x(Vt),e&&o(Xl),e&&o(Qe),e&&o(Fl),e&&o(Ze),e&&o(Rl),e&&o(et),e&&o(Hl),x(ro,e),e&&o(Wl),e&&o(le),e&&o(Kl),x(lo,e),e&&o(Ul),e&&o(vn),e&&o(Vl),x(io,e),e&&o(Gl),e&&o(Z),e&&o(Jl),x(tt,e),e&&o(Yl),e&&o(ye),x(po),e&&o(Ql),e&&o(wn),e&&o(Zl),e&&o(ie),e&&o(ei),e&&o(En),e&&o(ti),e&&o(ke),x(ho),e&&o(oi),e&&o($n),e&&o(ni),e&&o(pe),e&&o(si),x(st,e),e&&o(ai),e&&o(at),e&&o(ri),x(fo,e),e&&o(li),e&&o(W),e&&o(ii),x(rt,e),e&&o(pi),e&&o(Nn),e&&o(di),x(co,e),e&&o(hi),e&&o(jn),e&&o(fi),x(uo,e),e&&o(ci),e&&o(An),e&&o(ui),x(mo,e),e&&o(mi),e&&o(K),e&&o(gi),x(go,e),e&&o(_i),x(lt,e),e&&o(vi),e&&o(Te),x(_o),e&&o(wi),e&&o(de),e&&o(Ei),x(vo,e),e&&o($i),e&&o(F),e&&o(bi),x(wo,e),e&&o(xi),x(pt,e),e&&o(yi),e&&o(Oe),x(Eo),e&&o(ki),e&&o(he),e&&o(Ti),x($o,e),e&&o(Oi),e&&o(ht),e&&o(Ni),e&&o(Ne),x(bo),e&&o(ji),e&&o(Dn),e&&o(Ai),e&&o(fe),e&&o(Di),e&&o(ct),e&&o(Ci),e&&o(je),x(ko),e&&o(Pi),x(mt,e),e&&o(Si),e&&o(gt),e&&o(qi),e&&o(Sn),e&&o(Ii),e&&o(qn),e&&o(Bi),e&&o(_t),e&&o(Mi),e&&o(In),e&&o(zi),e&&o(Ae),x(No),e&&o(Li),e&&o(ce),e&&o(Xi),e&&o(ee),e&&o(Fi),e&&o(wt),e&&o(Ri),e&&o(De),x(jo),e&&o(Hi),e&&o(Bn),e&&o(Wi),e&&o(Mn),e&&o(Ki),e&&o(zn),e&&o(Ui),e&&o(Ln),e&&o(Vi),e&&o(Xn),e&&o(Gi),e&&o(Ce),x(Ao),e&&o(Ji),e&&o(Fn),e&&o(Yi),e&&o(Pe),x(Do),e&&o(Qi),e&&o(Y),e&&o(Zi),x(Co,e),e&&o(ep),e&&o(Se),x(Po),e&&o(tp),e&&o(te),e&&o(op),x(So,e),e&&o(np),e&&o(qe),x(qo),e&&o(sp),e&&o(kt),e&&o(ap),x(Io,e),e&&o(rp),e&&o(Ie),x(Bo),e&&o(lp),e&&o(ue),e&&o(ip),e&&o(me),e&&o(pp),e&&o(Me),x(Ro),e&&o(dp),e&&o(R),e&&o(hp),e&&o(ze),x(Go),e&&o(fp),e&&o(Hn),e&&o(cp),e&&o(Wn),e&&o(up),e&&o(Xe),x(Qo),e&&o(mp),e&&o(ge),e&&o(gp),x(en,e),e&&o(_p),e&&o(Kn),e&&o(vp),e&&o(Un),e&&o(wp),x(tn,e),e&&o(Ep),e&&o(Vn),e&&o($p),x(on,e),e&&o(bp),e&&o(Gn),e&&o(xp),e&&o(At)}}}const s$={local:"export-transformers-models",sections:[{local:"onnx",sections:[{local:"exporting-a-model-to-onnx",title:"Exporting a model to ONNX"},{local:"selecting-features-for-different-model-topologies",title:"Selecting features for different model topologies"},{local:"exporting-a-model-for-an-unsupported-architecture",sections:[{local:"implementing-a-custom-onnx-configuration",title:"Implementing a custom ONNX configuration"},{local:"exporting-the-model",title:"Exporting the model"},{local:"validating-the-model-outputs",title:"Validating the model outputs"}],title:"Exporting a model for an unsupported architecture"},{local:"contributing-a-new-configuration-to-transformers",title:"Contributing a new configuration to \u{1F917} Transformers"}],title:"ONNX"},{local:"torchscript",sections:[{local:"torchscript-flag-and-tied-weights",title:"TorchScript flag and tied weights"},{local:"dummy-inputs-and-standard-lengths",title:"Dummy inputs and standard lengths"},{local:"using-torchscript-in-python",sections:[{local:"saving-a-model",title:"Saving a model"},{local:"loading-a-model",title:"Loading a model"},{local:"using-a-traced-model-for-inference",title:"Using a traced model for inference"}],title:"Using TorchScript in Python"},{local:"deploying-huggingface-torchscript-models-on-aws-using-the-neuron-sdk",sections:[{local:"implications",title:"Implications"},{local:"dependencies",title:"Dependencies"},{local:"converting-a-model-for-aws-neuron",title:"Converting a Model for AWS Neuron"}],title:"Deploying HuggingFace TorchScript models on AWS using the Neuron SDK"}],title:"TorchScript"}],title:"Export \u{1F917} Transformers Models"};function a$(M){return WE(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class h$ extends XE{constructor(c){super();FE(this,c,a$,n$,RE,{})}}export{h$ as default,s$ as metadata};
