import{S as Gp,i as Lp,s as Fp,e as s,k as u,w as h,t as r,M as Vp,c as i,d as t,m as d,a as n,x as f,h as a,b as c,G as o,g as p,y as m,q as v,o as _,B as w,v as Xp}from"../../../chunks/vendor-hf-doc-builder.js";import{T as Hp}from"../../../chunks/Tip-hf-doc-builder.js";import{I as T}from"../../../chunks/IconCopyLink-hf-doc-builder.js";import{C as x}from"../../../chunks/CodeBlock-hf-doc-builder.js";function Qp(rs){let g,oe,E,q,j,b,Oe,I;return{c(){g=s("p"),oe=r("Due to a limitation of ONNX Runtime, it is not possible to run quantized models on "),E=s("code"),q=r("CUDAExecutionProvider"),j=r(" and only models with static quantization can be run on "),b=s("code"),Oe=r("TensorrtExecutionProvider"),I=r(".")},l(S){g=i(S,"P",{});var y=n(g);oe=a(y,"Due to a limitation of ONNX Runtime, it is not possible to run quantized models on "),E=i(y,"CODE",{});var M=n(E);q=a(M,"CUDAExecutionProvider"),M.forEach(t),j=a(y," and only models with static quantization can be run on "),b=i(y,"CODE",{});var $t=n(b);Oe=a($t,"TensorrtExecutionProvider"),$t.forEach(t),I=a(y,"."),y.forEach(t)},m(S,y){p(S,g,y),o(g,oe),o(g,E),o(E,q),o(g,j),o(g,b),o(b,Oe),o(g,I)},d(S){S&&t(g)}}}function Bp(rs){let g,oe,E,q,j,b,Oe,I,S,y,M,$t,as,Tt,Xi,ls,se,xt,Yt,Hi,Qi,Bi,ie,Jt,Wi,Yi,Re,Ji,Ki,ps,ne,us,G,re,Kt,Ue,Zi,Zt,en,ds,L,ae,eo,je,tn,to,on,cs,le,sn,Ie,nn,rn,hs,Se,fs,$,an,oo,ln,pn,so,un,dn,io,cn,hn,no,fn,mn,ms,F,pe,ro,Me,vn,ao,_n,vs,bt,wn,_s,Ge,ws,yt,gn,gs,Le,Es,qt,En,$s,V,ue,lo,Fe,$n,po,Tn,Ts,C,xn,uo,bn,yn,co,qn,An,xs,Ve,bs,de,Pn,Xe,kn,Cn,ys,He,qs,ce,zn,ho,Nn,Dn,As,Qe,Ps,At,On,ks,Be,Cs,Pt,Rn,zs,X,he,fo,We,Un,mo,jn,Ns,fe,In,vo,Sn,Mn,Ds,me,_o,k,Gn,kt,Ln,Fn,Ye,wo,Vn,Xn,Je,go,Hn,Qn,Bn,Eo,H,Wn,Ct,Yn,Jn,$o,Kn,Zn,Os,Q,ve,To,Ke,er,xo,tr,Rs,zt,or,Us,B,_e,bo,Ze,sr,yo,ir,js,W,we,qo,et,nr,Ao,rr,Is,ge,ar,Po,lr,pr,Ss,Nt,ur,Ms,z,Dt,dr,tt,cr,hr,Ot,fr,ot,mr,vr,Rt,_r,st,wr,Gs,N,gr,ko,Er,$r,it,Tr,xr,Ls,Ut,br,Fs,nt,Vs,Y,Ee,Co,rt,yr,zo,qr,Xs,jt,Ar,Hs,at,Qs,It,Pr,Bs,$e,kr,No,Cr,zr,Ws,lt,Ys,St,Nr,Js,J,Te,Do,pt,Dr,Oo,Or,Ks,D,Rr,Ro,Ur,jr,Uo,Ir,Sr,Zs,ut,ei,K,dt,Mr,jo,Gr,Lr,Io,Fr,Vr,ti,ct,oi,xe,Xr,ht,Hr,Qr,si,Z,be,So,ft,Br,Mo,Wr,ii,A,Yr,Go,Jr,Kr,Lo,Zr,ea,Mt,ta,oa,ni,mt,ri,P,sa,Fo,ia,na,Vo,ra,aa,Xo,la,pa,ai,O,ua,Ho,da,ca,Qo,ha,fa,li,ye,ma,Bo,va,_a,pi,vt,ui,qe,wa,_t,ga,Ea,di,ee,Ae,Wo,wt,$a,Yo,Ta,ci,Gt,xa,hi,R,Jo,ba,ya,Ko,qa,Aa,Zo,Pa,fi,Pe,ka,es,Ca,za,mi,te,ke,ts,gt,Na,os,Da,vi,Ce,Oa,ss,Ra,Ua,_i,Lt,ja,wi;return b=new T({}),ne=new Hp({props:{warning:!0,$$slots:{default:[Qp]},$$scope:{ctx:rs}}}),Ue=new T({}),je=new T({}),Se=new x({props:{code:"pip install optimum[onnxruntime-gpu]",highlighted:"pip install optimum[onnxruntime-gpu]"}}),Me=new T({}),Ge=new x({props:{code:`from optimum.onnxruntime import ORTModelForSequenceClassification
from transformers import AutoTokenizer

ort_model = ORTModelForSequenceClassification.from_pretrained(
    "philschmid/tiny-bert-sst2-distilled",
    from_transformers=True,
    provider="CUDAExecutionProvider",
)

tokenizer = AutoTokenizer.from_pretrained("philschmid/tiny-bert-sst2-distilled")
inputs = tokenizer("expectations were low, actual enjoyment was high", return_tensors="pt", padding=True)

outputs = ort_model(**inputs)
assert ort_model.providers == ["CUDAExecutionProvider", "CPUExecutionProvider"]`,highlighted:`<span class="hljs-keyword">from</span> optimum.onnxruntime <span class="hljs-keyword">import</span> ORTModelForSequenceClassification
<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoTokenizer

ort_model = ORTModelForSequenceClassification.from_pretrained(
    <span class="hljs-string">&quot;philschmid/tiny-bert-sst2-distilled&quot;</span>,
    from_transformers=<span class="hljs-literal">True</span>,
    provider=<span class="hljs-string">&quot;CUDAExecutionProvider&quot;</span>,
)

tokenizer = AutoTokenizer.from_pretrained(<span class="hljs-string">&quot;philschmid/tiny-bert-sst2-distilled&quot;</span>)
inputs = tokenizer(<span class="hljs-string">&quot;expectations were low, actual enjoyment was high&quot;</span>, return_tensors=<span class="hljs-string">&quot;pt&quot;</span>, padding=<span class="hljs-literal">True</span>)

outputs = ort_model(**inputs)
<span class="hljs-keyword">assert</span> ort_model.providers == [<span class="hljs-string">&quot;CUDAExecutionProvider&quot;</span>, <span class="hljs-string">&quot;CPUExecutionProvider&quot;</span>]`}}),Le=new x({props:{code:"ValueError: Asked to use CUDAExecutionProvider as an ONNX Runtime execution provider, but the available execution providers are ['CPUExecutionProvider'].",highlighted:'ValueError: Asked <span class="hljs-built_in">to</span> use CUDAExecutionProvider <span class="hljs-keyword">as</span> <span class="hljs-keyword">an</span> ONNX Runtime execution provider, but <span class="hljs-keyword">the</span> available execution providers are [<span class="hljs-string">&#x27;CPUExecutionProvider&#x27;</span>].'}}),Fe=new T({}),Ve=new x({props:{code:`from optimum.onnxruntime import ORTModelForSequenceClassification

ort_model = ORTModelForSequenceClassification.from_pretrained(
    "distilbert-base-uncased-finetuned-sst-2-english",
    from_transformers=True,
    provider="CUDAExecutionProvider",
)`,highlighted:`<span class="hljs-keyword">from</span> optimum.onnxruntime <span class="hljs-keyword">import</span> ORTModelForSequenceClassification

ort_model = ORTModelForSequenceClassification.from_pretrained(
    <span class="hljs-string">&quot;distilbert-base-uncased-finetuned-sst-2-english&quot;</span>,
    from_transformers=<span class="hljs-literal">True</span>,
    provider=<span class="hljs-string">&quot;CUDAExecutionProvider&quot;</span>,
)`}}),He=new x({props:{code:`from optimum.pipelines import pipeline
from transformers import AutoTokenizer

tokenizer = AutoTokenizer.from_pretrained(model_name)

pipe = pipeline(task="text-classification", model=ort_model, tokenizer=tokenizer)
result = pipe("Both the music and visual were astounding, not to mention the actors performance.")
print(result)
# printing: [{'label': 'POSITIVE', 'score': 0.9997727274894714}]`,highlighted:`<span class="hljs-keyword">from</span> optimum.pipelines <span class="hljs-keyword">import</span> pipeline
<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoTokenizer

tokenizer = AutoTokenizer.from_pretrained(model_name)

pipe = pipeline(task=<span class="hljs-string">&quot;text-classification&quot;</span>, model=ort_model, tokenizer=tokenizer)
result = pipe(<span class="hljs-string">&quot;Both the music and visual were astounding, not to mention the actors performance.&quot;</span>)
<span class="hljs-built_in">print</span>(result)
<span class="hljs-comment"># printing: [{&#x27;label&#x27;: &#x27;POSITIVE&#x27;, &#x27;score&#x27;: 0.9997727274894714}]</span>`}}),Qe=new x({props:{code:`import onnxruntime

session_options = onnxruntime.SessionOptions()
session_options.log_severity_level = 0

ort_model = ORTModelForSequenceClassification.from_pretrained(
    "distilbert-base-uncased-finetuned-sst-2-english",
    from_transformers=True,
    provider="CUDAExecutionProvider",
    session_options=session_options
)`,highlighted:`<span class="hljs-keyword">import</span> onnxruntime

session_options = onnxruntime.SessionOptions()
session_options.log_severity_level = <span class="hljs-number">0</span>

ort_model = ORTModelForSequenceClassification.from_pretrained(
    <span class="hljs-string">&quot;distilbert-base-uncased-finetuned-sst-2-english&quot;</span>,
    from_transformers=<span class="hljs-literal">True</span>,
    provider=<span class="hljs-string">&quot;CUDAExecutionProvider&quot;</span>,
    session_options=session_options
)`}}),Be=new x({props:{code:`2022-10-18 14:59:13.728886041 [V:onnxruntime:, session_state.cc:1193 VerifyEachN
odeIsAssignedToAnEp]  Provider: [CPUExecutionProvider]: [Gather (Gather_76), Uns
queeze (Unsqueeze_78), Gather (Gather_97), Gather (Gather_100), Concat (Concat_1
10), Unsqueeze (Unsqueeze_125), ...]
2022-10-18 14:59:13.728906431 [V:onnxruntime:, session_state.cc:1193 VerifyEachN
odeIsAssignedToAnEp]  Provider: [CUDAExecutionProvider]: [Shape (Shape_74), Slic
e (Slice_80), Gather (Gather_81), Gather (Gather_82), Add (Add_83), Shape (Shape
_95), MatMul (MatMul_101), ...]`,highlighted:`<span class="hljs-attribute">2022</span>-<span class="hljs-number">10</span>-<span class="hljs-number">18</span> <span class="hljs-number">14</span>:<span class="hljs-number">59</span>:<span class="hljs-number">13</span>.<span class="hljs-number">728886041</span><span class="hljs-meta"> [V:onnxruntime:, session_state.cc:1193 VerifyEachN
odeIsAssignedToAnEp]  Provider: [CPUExecutionProvider]: [Gather (Gather_76), Uns
queeze (Unsqueeze_78), Gather (Gather_97), Gather (Gather_100), Concat (Concat_1
10), Unsqueeze (Unsqueeze_125), ...]</span>
<span class="hljs-attribute">2022</span>-<span class="hljs-number">10</span>-<span class="hljs-number">18</span> <span class="hljs-number">14</span>:<span class="hljs-number">59</span>:<span class="hljs-number">13</span>.<span class="hljs-number">728906431</span><span class="hljs-meta"> [V:onnxruntime:, session_state.cc:1193 VerifyEachN
odeIsAssignedToAnEp]  Provider: [CUDAExecutionProvider]: [Shape (Shape_74), Slic
e (Slice_80), Gather (Gather_81), Gather (Gather_82), Add (Add_83), Shape (Shape
_95), MatMul (MatMul_101), ...]</span>`}}),We=new T({}),Ke=new T({}),Ze=new T({}),et=new T({}),nt=new x({props:{code:`export CUDA_PATH=/usr/local/cuda
export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/cuda-x.x/lib64:/path/to/TensorRT-8.x.x/lib`,highlighted:`<span class="hljs-built_in">export</span> CUDA_PATH=/usr/local/cuda
<span class="hljs-built_in">export</span> LD_LIBRARY_PATH=<span class="hljs-variable">$LD_LIBRARY_PATH</span>:/usr/local/cuda-x.x/lib64:/path/to/TensorRT-8.x.x/lib`}}),rt=new T({}),at=new x({props:{code:`from optimum.onnxruntime import ORTModelForSequenceClassification
from transformers import AutoTokenizer

ort_model = ORTModelForSequenceClassification.from_pretrained(
    "philschmid/tiny-bert-sst2-distilled",
    from_transformers=True,
    provider="TensorrtExecutionProvider",
)

tokenizer = AutoTokenizer.from_pretrained("philschmid/tiny-bert-sst2-distilled")
inp = tokenizer("expectations were low, actual enjoyment was high", return_tensors="pt", padding=True)

result = ort_model(**inp)
assert ort_model.providers == ["TensorrtExecutionProvider", "CUDAExecutionProvider", "CPUExecutionProvider"]`,highlighted:`<span class="hljs-keyword">from</span> optimum.onnxruntime <span class="hljs-keyword">import</span> ORTModelForSequenceClassification
<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoTokenizer

ort_model = ORTModelForSequenceClassification.from_pretrained(
    <span class="hljs-string">&quot;philschmid/tiny-bert-sst2-distilled&quot;</span>,
    from_transformers=<span class="hljs-literal">True</span>,
    provider=<span class="hljs-string">&quot;TensorrtExecutionProvider&quot;</span>,
)

tokenizer = AutoTokenizer.from_pretrained(<span class="hljs-string">&quot;philschmid/tiny-bert-sst2-distilled&quot;</span>)
inp = tokenizer(<span class="hljs-string">&quot;expectations were low, actual enjoyment was high&quot;</span>, return_tensors=<span class="hljs-string">&quot;pt&quot;</span>, padding=<span class="hljs-literal">True</span>)

result = ort_model(**inp)
<span class="hljs-keyword">assert</span> ort_model.providers == [<span class="hljs-string">&quot;TensorrtExecutionProvider&quot;</span>, <span class="hljs-string">&quot;CUDAExecutionProvider&quot;</span>, <span class="hljs-string">&quot;CPUExecutionProvider&quot;</span>]`}}),lt=new x({props:{code:"Failed to create TensorrtExecutionProvider. Please reference https://onnxruntime.ai/docs/execution-providers/TensorRT-ExecutionProvider.html#requirements to ensure all dependencies are met.",highlighted:'Failed to create TensorrtExecutionProvider. Please reference https:<span class="hljs-regexp">//</span>onnxruntime.ai<span class="hljs-regexp">/docs/</span>execution-providers/TensorRT-ExecutionProvider.html<span class="hljs-comment">#requirements to ensure all dependencies are met.</span>'}}),pt=new T({}),ut=new x({props:{code:`from optimum.onnxruntime import ORTModelForSequenceClassification

ort_model = ORTModelForSequenceClassification.from_pretrained(
    "distilbert-base-uncased-finetuned-sst-2-english",
    from_transformers=True,
    provider="TensorrtExecutionProvider",
)`,highlighted:`<span class="hljs-keyword">from</span> optimum.onnxruntime <span class="hljs-keyword">import</span> ORTModelForSequenceClassification

ort_model = ORTModelForSequenceClassification.from_pretrained(
    <span class="hljs-string">&quot;distilbert-base-uncased-finetuned-sst-2-english&quot;</span>,
    from_transformers=<span class="hljs-literal">True</span>,
    provider=<span class="hljs-string">&quot;TensorrtExecutionProvider&quot;</span>,
)`}}),ct=new x({props:{code:"2022-09-22 14:12:48.371513741 [V:onnxruntime:, session_state.cc:1188 VerifyEachNodeIsAssignedToAnEp] All nodes have been placed on [TensorrtExecutionProvider]",highlighted:'<span class="hljs-attribute">2022</span>-<span class="hljs-number">09</span>-<span class="hljs-number">22</span> <span class="hljs-number">14</span>:<span class="hljs-number">12</span>:<span class="hljs-number">48</span>.<span class="hljs-number">371513741</span><span class="hljs-meta"> [V:onnxruntime:, session_state.cc:1188 VerifyEachNodeIsAssignedToAnEp] All nodes have been placed on [TensorrtExecutionProvider]</span>'}}),ft=new T({}),mt=new x({props:{code:`qconfig = QuantizationConfig(
    ...,
    is_static=True,
    activations_symmetric=True,
    weights_symmetric=True,
    qdq_dedicated_pair=True,
    qdq_add_pair_to_weight=True
)`,highlighted:`qconfig = QuantizationConfig(
    ...,
    is_static=<span class="hljs-literal">True</span>,
    activations_symmetric=<span class="hljs-literal">True</span>,
    weights_symmetric=<span class="hljs-literal">True</span>,
    qdq_dedicated_pair=<span class="hljs-literal">True</span>,
    qdq_add_pair_to_weight=<span class="hljs-literal">True</span>
)`}}),vt=new x({props:{code:`session_options = onnxruntime.SessionOptions()
session_options.graph_optimization_level = onnxruntime.GraphOptimizationLevel.ORT_DISABLE_ALL

model_name = "fxmarty/distilbert-base-uncased-finetuned-sst-2-english-int8-static-symmetric-dedicated-qdq-everywhere"
ort_model = ORTModelForSequenceClassification.from_pretrained(
    model_name,
    from_transformers=False,
    provider="TensorrtExecutionProvider",
    session_options=session_options,
    provider_options={"trt_int8_enable": True},
)`,highlighted:`session_options = onnxruntime.SessionOptions()
session_options.graph_optimization_level = onnxruntime.GraphOptimizationLevel.ORT_DISABLE_ALL

model_name = <span class="hljs-string">&quot;fxmarty/distilbert-base-uncased-finetuned-sst-2-english-int8-static-symmetric-dedicated-qdq-everywhere&quot;</span>
ort_model = ORTModelForSequenceClassification.from_pretrained(
    model_name,
    from_transformers=<span class="hljs-literal">False</span>,
    provider=<span class="hljs-string">&quot;TensorrtExecutionProvider&quot;</span>,
    session_options=session_options,
    provider_options={<span class="hljs-string">&quot;trt_int8_enable&quot;</span>: <span class="hljs-literal">True</span>},
)`}}),wt=new T({}),gt=new T({}),{c(){g=s("meta"),oe=u(),E=s("h1"),q=s("a"),j=s("span"),h(b.$$.fragment),Oe=u(),I=s("span"),S=r("Accelerated inference on NVIDIA GPUs"),y=u(),M=s("p"),$t=r("By default, ONNX Runtime runs inference on CPU devices. However, it is possible to place supported operations on an NVIDIA GPU, while leaving any unsupported ones on CPU. In most cases, this allows costly operations to be placed on GPU and significantly accelerate inference."),as=u(),Tt=s("p"),Xi=r("This guide will show you how to run inference on two execution providers that ONNX Runtime supports for NVIDIA GPUs:"),ls=u(),se=s("ul"),xt=s("li"),Yt=s("code"),Hi=r("CUDAExecutionProvider"),Qi=r(": Generic acceleration on NVIDIA CUDA-enabled GPUs."),Bi=u(),ie=s("li"),Jt=s("code"),Wi=r("TensorrtExecutionProvider"),Yi=r(": Uses NVIDIA\u2019s "),Re=s("a"),Ji=r("TensorRT"),Ki=r(" inference engine and generally provides the best runtime performance."),ps=u(),h(ne.$$.fragment),us=u(),G=s("h2"),re=s("a"),Kt=s("span"),h(Ue.$$.fragment),Zi=u(),Zt=s("span"),en=r("CUDAExecutionProvider"),ds=u(),L=s("h3"),ae=s("a"),eo=s("span"),h(je.$$.fragment),tn=u(),to=s("span"),on=r("Installation"),cs=u(),le=s("p"),sn=r("Provided the CUDA and cuDNN "),Ie=s("a"),nn=r("requirements"),rn=r(" are satisfied, install the additional dependencies by running"),hs=u(),h(Se.$$.fragment),fs=u(),$=s("p"),an=r("To avoid conflicts between "),oo=s("code"),ln=r("onnxruntime"),pn=r(" and "),so=s("code"),un=r("onnxruntime-gpu"),dn=r(", make sure the package "),io=s("code"),cn=r("onnxruntime"),hn=r(" is not installed by running "),no=s("code"),fn=r("pip uninstall onnxruntime"),mn=r(" prior to installing Optimum."),ms=u(),F=s("h3"),pe=s("a"),ro=s("span"),h(Me.$$.fragment),vn=u(),ao=s("span"),_n=r("Checking the installation is successful"),vs=u(),bt=s("p"),wn=r("Before going further, run the following sample code to check whether the install was successful:"),_s=u(),h(Ge.$$.fragment),ws=u(),yt=s("p"),gn=r("In case this code runs gracefully, congratulations, the installation is successful! If you encounter the following error or similar,"),gs=u(),h(Le.$$.fragment),Es=u(),qt=s("p"),En=r("then something is wrong with the CUDA or ONNX Runtime installation."),$s=u(),V=s("h3"),ue=s("a"),lo=s("span"),h(Fe.$$.fragment),$n=u(),po=s("span"),Tn=r("Use CUDA execution provider with floating-point models"),Ts=u(),C=s("p"),xn=r("For non-quantized models, the use is straightforward. Simply specify the "),uo=s("code"),bn=r("provider"),yn=r(" argument in the "),co=s("code"),qn=r("ORTModel.from_pretrained()"),An=r(" method. Here\u2019s an example:"),xs=u(),h(Ve.$$.fragment),bs=u(),de=s("p"),Pn=r("The model can then be used with the common \u{1F917} Transformers API for inference and evaluation, such as "),Xe=s("a"),kn=r("pipelines"),Cn=r(":"),ys=u(),h(He.$$.fragment),qs=u(),ce=s("p"),zn=r("Additionally, you can pass the session option "),ho=s("code"),Nn=r("log_severity_level = 0"),Dn=r(" (verbose), to check whether all nodes are indeed placed on the CUDA execution provider or not:"),As=u(),h(Qe.$$.fragment),Ps=u(),At=s("p"),On=r("You should see the following logs:"),ks=u(),h(Be.$$.fragment),Cs=u(),Pt=s("p"),Rn=r("In this example, we can see that all the costly MatMul operations are placed on the CUDA execution provider."),zs=u(),X=s("h3"),he=s("a"),fo=s("span"),h(We.$$.fragment),Un=u(),mo=s("span"),jn=r("Use CUDA execution provider with quantized models"),Ns=u(),fe=s("p"),In=r("Due to current limitations in ONNX Runtime, it is not possible to use quantized models with "),vo=s("code"),Sn=r("CUDAExecutionProvider"),Mn=r(". The reasons are as follows:"),Ds=u(),me=s("ul"),_o=s("li"),k=s("p"),Gn=r("When using "),kt=s("a"),Ln=r("\u{1F917} Optimum dynamic quantization"),Fn=r(", nodes as "),Ye=s("a"),wo=s("code"),Vn=r("MatMulInteger"),Xn=r(", "),Je=s("a"),go=s("code"),Hn=r("DynamicQuantizeLinear"),Qn=r(" may be inserted in the ONNX graph, that cannot be consumed by the CUDA execution provider."),Bn=u(),Eo=s("li"),H=s("p"),Wn=r("When using "),Ct=s("a"),Yn=r("static quantization"),Jn=r(", the ONNX computation graph will contain matrix multiplications and convolutions in floating-point arithmetic, along with Quantize + Dequantize operations to simulate quantization. In this case, although the costly matrix multiplications and convolutions will be run on the GPU, they will use floating-point arithmetic as the "),$o=s("code"),Kn=r("CUDAExecutionProvider"),Zn=r(" can not consume the Quantize + Dequantize nodes to replace them by the operations using integer arithmetic."),Os=u(),Q=s("h3"),ve=s("a"),To=s("span"),h(Ke.$$.fragment),er=u(),xo=s("span"),tr=r("Observed time gains"),Rs=u(),zt=s("p"),or=r("Coming soon!"),Us=u(),B=s("h2"),_e=s("a"),bo=s("span"),h(Ze.$$.fragment),sr=u(),yo=s("span"),ir=r("TensorrtExecutionProvider"),js=u(),W=s("h3"),we=s("a"),qo=s("span"),h(et.$$.fragment),nr=u(),Ao=s("span"),rr=r("Installation"),Is=u(),ge=s("p"),ar=r("The easiest way to use TensorRT as the execution provider for models optimized through \u{1F917} Optimum is with the available ONNX Runtime "),Po=s("code"),lr=r("TensorrtExecutionProvider"),pr=r("."),Ss=u(),Nt=s("p"),ur=r("In order to use \u{1F917} Optimum with TensorRT in a local environment, we recommend following the NVIDIA installation guides:"),Ms=u(),z=s("ul"),Dt=s("li"),dr=r("CUDA toolkit: "),tt=s("a"),cr=r("https://docs.nvidia.com/cuda/cuda-installation-guide-linux/index.html"),hr=u(),Ot=s("li"),fr=r("cuDNN: "),ot=s("a"),mr=r("https://docs.nvidia.com/deeplearning/cudnn/install-guide/index.html"),vr=u(),Rt=s("li"),_r=r("TensorRT: "),st=s("a"),wr=r("https://docs.nvidia.com/deeplearning/tensorrt/install-guide/index.html"),Gs=u(),N=s("p"),gr=r("For TensorRT, we recommend the Tar File Installation method. Alternatively, TensorRT may be installable with "),ko=s("code"),Er=r("pip"),$r=r(" by following "),it=s("a"),Tr=r("these instructions"),xr=r("."),Ls=u(),Ut=s("p"),br=r("Once the required packages are installed, the following environment variables need to be set with the appropriate paths for ONNX Runtime to detect TensorRT installation:"),Fs=u(),h(nt.$$.fragment),Vs=u(),Y=s("h3"),Ee=s("a"),Co=s("span"),h(rt.$$.fragment),yr=u(),zo=s("span"),qr=r("Checking the installation is successful"),Xs=u(),jt=s("p"),Ar=r("Before going further, run the following sample code to check whether the install was successful:"),Hs=u(),h(at.$$.fragment),Qs=u(),It=s("p"),Pr=r("In case this code runs gracefully, congratulations, the installation is successful!"),Bs=u(),$e=s("p"),kr=r("In case the above "),No=s("code"),Cr=r("assert"),zr=r(" fails, or you encounter the following warning"),Ws=u(),h(lt.$$.fragment),Ys=u(),St=s("p"),Nr=r("something is wrong with the TensorRT or ONNX Runtime installation."),Js=u(),J=s("h3"),Te=s("a"),Do=s("span"),h(pt.$$.fragment),Dr=u(),Oo=s("span"),Or=r("Use TensorRT execution provider with floating-point models"),Ks=u(),D=s("p"),Rr=r("For non-quantized models, the use is straightforward, by simply using the "),Ro=s("code"),Ur=r("provider"),jr=r(" argument in "),Uo=s("code"),Ir=r("ORTModel.from_pretrained()"),Sr=r(". For example:"),Zs=u(),h(ut.$$.fragment),ei=u(),K=s("p"),dt=s("a"),Mr=r("As previously for "),jo=s("code"),Gr=r("CUDAExecutionProvider"),Lr=r(", by passing the session option "),Io=s("code"),Fr=r("log_severity_level = 0"),Vr=r(" (verbose), we can check in the logs whether all nodes are indeed placed on the TensorRT execution provider or not:"),ti=u(),h(ct.$$.fragment),oi=u(),xe=s("p"),Xr=r("The model can then be used with the common \u{1F917} Transformers API for inference and evaluation, such as "),ht=s("a"),Hr=r("pipelines"),Qr=r("."),si=u(),Z=s("h3"),be=s("a"),So=s("span"),h(ft.$$.fragment),Br=u(),Mo=s("span"),Wr=r("Use TensorRT execution provider with quantized models"),ii=u(),A=s("p"),Yr=r("When it comes to quantized models, TensorRT only supports models that use "),Go=s("strong"),Jr=r("static"),Kr=r(" quantization with "),Lo=s("strong"),Zr=r("symmetric quantization"),ea=r(" for weights and activations. Thus, to be able to consume with TensorRT models quantized through \u{1F917} Optimum, the following configuration needs to be passed when doing "),Mt=s("a"),ta=r("static quantization"),oa=r(":"),ni=u(),h(mt.$$.fragment),ri=u(),P=s("p"),sa=r("The "),Fo=s("code"),ia=r("qdq_dedicated_pair=True"),na=r(" argument is required by TensorRT, since it expects a single node after each "),Vo=s("code"),ra=r("QuantizeLinear"),aa=r(" + "),Xo=s("code"),la=r("DequantizeLinear"),pa=r(" (QDQ) pair."),ai=u(),O=s("p"),ua=r("The parameter "),Ho=s("code"),da=r("qdq_add_pair_to_weight=True"),ca=r(" is also required by TensorRT, that since it consumes a graph where the weights are stored in float32 with a QDQ pair. Normally, weights would be stored in fixed point 8-bits format and only a "),Qo=s("code"),ha=r("DequantizeLinear"),fa=r(" would be applied on the weights. As such, the storage savings from quantization can not be leveraged when we expect to later use the quantized ONNX model with TensorRT."),li=u(),ye=s("p"),ma=r("In the code sample below, after performing static quantization, the resulting model is loaded into the "),Bo=s("code"),va=r("ORTModel"),_a=r(" class using TensorRT as the execution provider. ONNX Runtime graph optimization need to be disabled for the model to be consumed and optimized by TensorRT, and the fact that INT8 operations are used needs to be specified to TensorRT."),pi=u(),h(vt.$$.fragment),ui=u(),qe=s("p"),wa=r("The model can then be used with the common \u{1F917} Transformers API for inference and evaluation, such as "),_t=s("a"),ga=r("pipelines"),Ea=r("."),di=u(),ee=s("h3"),Ae=s("a"),Wo=s("span"),h(wt.$$.fragment),$a=u(),Yo=s("span"),Ta=r("TensorRT limitations for quantized models"),ci=u(),Gt=s("p"),xa=r("As highlighted in the previous section, TensorRT supports only a limited range of quantized models:"),hi=u(),R=s("ul"),Jo=s("li"),ba=r("Static quantization only"),ya=u(),Ko=s("li"),qa=r("Weights and activations quantization ranges are symmetric"),Aa=u(),Zo=s("li"),Pa=r("Weights need to be stored in float32 in the ONNX model, thus there is no storage space saving from quantization."),fi=u(),Pe=s("p"),ka=r("In case "),es=s("code"),Ca=r('provider="TensorrtExecutionProvider"'),za=r(" is passed and the model has not been quantized strictly following these constraints, various errors may be raised, where error messages can be unclear."),mi=u(),te=s("h3"),ke=s("a"),ts=s("span"),h(gt.$$.fragment),Na=u(),os=s("span"),Da=r("Observed time gains"),vi=u(),Ce=s("p"),Oa=r("Nvidia Nsight Systems tool can be used to profile the execution time on GPU. Before profiling or measuring latency/throughput, it is a good practice to do a few "),ss=s("strong"),Ra=r("warmup steps"),Ua=r("."),_i=u(),Lt=s("p"),ja=r("Coming soon!"),this.h()},l(e){const l=Vp('[data-svelte="svelte-1phssyn"]',document.head);g=i(l,"META",{name:!0,content:!0}),l.forEach(t),oe=d(e),E=i(e,"H1",{class:!0});var Et=n(E);q=i(Et,"A",{id:!0,class:!0,href:!0});var Fa=n(q);j=i(Fa,"SPAN",{});var Va=n(j);f(b.$$.fragment,Va),Va.forEach(t),Fa.forEach(t),Oe=d(Et),I=i(Et,"SPAN",{});var Xa=n(I);S=a(Xa,"Accelerated inference on NVIDIA GPUs"),Xa.forEach(t),Et.forEach(t),y=d(e),M=i(e,"P",{});var Ha=n(M);$t=a(Ha,"By default, ONNX Runtime runs inference on CPU devices. However, it is possible to place supported operations on an NVIDIA GPU, while leaving any unsupported ones on CPU. In most cases, this allows costly operations to be placed on GPU and significantly accelerate inference."),Ha.forEach(t),as=d(e),Tt=i(e,"P",{});var Qa=n(Tt);Xi=a(Qa,"This guide will show you how to run inference on two execution providers that ONNX Runtime supports for NVIDIA GPUs:"),Qa.forEach(t),ls=d(e),se=i(e,"UL",{});var gi=n(se);xt=i(gi,"LI",{});var Ia=n(xt);Yt=i(Ia,"CODE",{});var Ba=n(Yt);Hi=a(Ba,"CUDAExecutionProvider"),Ba.forEach(t),Qi=a(Ia,": Generic acceleration on NVIDIA CUDA-enabled GPUs."),Ia.forEach(t),Bi=d(gi),ie=i(gi,"LI",{});var is=n(ie);Jt=i(is,"CODE",{});var Wa=n(Jt);Wi=a(Wa,"TensorrtExecutionProvider"),Wa.forEach(t),Yi=a(is,": Uses NVIDIA\u2019s "),Re=i(is,"A",{href:!0,rel:!0});var Ya=n(Re);Ji=a(Ya,"TensorRT"),Ya.forEach(t),Ki=a(is," inference engine and generally provides the best runtime performance."),is.forEach(t),gi.forEach(t),ps=d(e),f(ne.$$.fragment,e),us=d(e),G=i(e,"H2",{class:!0});var Ei=n(G);re=i(Ei,"A",{id:!0,class:!0,href:!0});var Ja=n(re);Kt=i(Ja,"SPAN",{});var Ka=n(Kt);f(Ue.$$.fragment,Ka),Ka.forEach(t),Ja.forEach(t),Zi=d(Ei),Zt=i(Ei,"SPAN",{});var Za=n(Zt);en=a(Za,"CUDAExecutionProvider"),Za.forEach(t),Ei.forEach(t),ds=d(e),L=i(e,"H3",{class:!0});var $i=n(L);ae=i($i,"A",{id:!0,class:!0,href:!0});var el=n(ae);eo=i(el,"SPAN",{});var tl=n(eo);f(je.$$.fragment,tl),tl.forEach(t),el.forEach(t),tn=d($i),to=i($i,"SPAN",{});var ol=n(to);on=a(ol,"Installation"),ol.forEach(t),$i.forEach(t),cs=d(e),le=i(e,"P",{});var Ti=n(le);sn=a(Ti,"Provided the CUDA and cuDNN "),Ie=i(Ti,"A",{href:!0,rel:!0});var sl=n(Ie);nn=a(sl,"requirements"),sl.forEach(t),rn=a(Ti," are satisfied, install the additional dependencies by running"),Ti.forEach(t),hs=d(e),f(Se.$$.fragment,e),fs=d(e),$=i(e,"P",{});var U=n($);an=a(U,"To avoid conflicts between "),oo=i(U,"CODE",{});var il=n(oo);ln=a(il,"onnxruntime"),il.forEach(t),pn=a(U," and "),so=i(U,"CODE",{});var nl=n(so);un=a(nl,"onnxruntime-gpu"),nl.forEach(t),dn=a(U,", make sure the package "),io=i(U,"CODE",{});var rl=n(io);cn=a(rl,"onnxruntime"),rl.forEach(t),hn=a(U," is not installed by running "),no=i(U,"CODE",{});var al=n(no);fn=a(al,"pip uninstall onnxruntime"),al.forEach(t),mn=a(U," prior to installing Optimum."),U.forEach(t),ms=d(e),F=i(e,"H3",{class:!0});var xi=n(F);pe=i(xi,"A",{id:!0,class:!0,href:!0});var ll=n(pe);ro=i(ll,"SPAN",{});var pl=n(ro);f(Me.$$.fragment,pl),pl.forEach(t),ll.forEach(t),vn=d(xi),ao=i(xi,"SPAN",{});var ul=n(ao);_n=a(ul,"Checking the installation is successful"),ul.forEach(t),xi.forEach(t),vs=d(e),bt=i(e,"P",{});var dl=n(bt);wn=a(dl,"Before going further, run the following sample code to check whether the install was successful:"),dl.forEach(t),_s=d(e),f(Ge.$$.fragment,e),ws=d(e),yt=i(e,"P",{});var cl=n(yt);gn=a(cl,"In case this code runs gracefully, congratulations, the installation is successful! If you encounter the following error or similar,"),cl.forEach(t),gs=d(e),f(Le.$$.fragment,e),Es=d(e),qt=i(e,"P",{});var hl=n(qt);En=a(hl,"then something is wrong with the CUDA or ONNX Runtime installation."),hl.forEach(t),$s=d(e),V=i(e,"H3",{class:!0});var bi=n(V);ue=i(bi,"A",{id:!0,class:!0,href:!0});var fl=n(ue);lo=i(fl,"SPAN",{});var ml=n(lo);f(Fe.$$.fragment,ml),ml.forEach(t),fl.forEach(t),$n=d(bi),po=i(bi,"SPAN",{});var vl=n(po);Tn=a(vl,"Use CUDA execution provider with floating-point models"),vl.forEach(t),bi.forEach(t),Ts=d(e),C=i(e,"P",{});var Ft=n(C);xn=a(Ft,"For non-quantized models, the use is straightforward. Simply specify the "),uo=i(Ft,"CODE",{});var _l=n(uo);bn=a(_l,"provider"),_l.forEach(t),yn=a(Ft," argument in the "),co=i(Ft,"CODE",{});var wl=n(co);qn=a(wl,"ORTModel.from_pretrained()"),wl.forEach(t),An=a(Ft," method. Here\u2019s an example:"),Ft.forEach(t),xs=d(e),f(Ve.$$.fragment,e),bs=d(e),de=i(e,"P",{});var yi=n(de);Pn=a(yi,"The model can then be used with the common \u{1F917} Transformers API for inference and evaluation, such as "),Xe=i(yi,"A",{href:!0,rel:!0});var gl=n(Xe);kn=a(gl,"pipelines"),gl.forEach(t),Cn=a(yi,":"),yi.forEach(t),ys=d(e),f(He.$$.fragment,e),qs=d(e),ce=i(e,"P",{});var qi=n(ce);zn=a(qi,"Additionally, you can pass the session option "),ho=i(qi,"CODE",{});var El=n(ho);Nn=a(El,"log_severity_level = 0"),El.forEach(t),Dn=a(qi," (verbose), to check whether all nodes are indeed placed on the CUDA execution provider or not:"),qi.forEach(t),As=d(e),f(Qe.$$.fragment,e),Ps=d(e),At=i(e,"P",{});var $l=n(At);On=a($l,"You should see the following logs:"),$l.forEach(t),ks=d(e),f(Be.$$.fragment,e),Cs=d(e),Pt=i(e,"P",{});var Tl=n(Pt);Rn=a(Tl,"In this example, we can see that all the costly MatMul operations are placed on the CUDA execution provider."),Tl.forEach(t),zs=d(e),X=i(e,"H3",{class:!0});var Ai=n(X);he=i(Ai,"A",{id:!0,class:!0,href:!0});var xl=n(he);fo=i(xl,"SPAN",{});var bl=n(fo);f(We.$$.fragment,bl),bl.forEach(t),xl.forEach(t),Un=d(Ai),mo=i(Ai,"SPAN",{});var yl=n(mo);jn=a(yl,"Use CUDA execution provider with quantized models"),yl.forEach(t),Ai.forEach(t),Ns=d(e),fe=i(e,"P",{});var Pi=n(fe);In=a(Pi,"Due to current limitations in ONNX Runtime, it is not possible to use quantized models with "),vo=i(Pi,"CODE",{});var ql=n(vo);Sn=a(ql,"CUDAExecutionProvider"),ql.forEach(t),Mn=a(Pi,". The reasons are as follows:"),Pi.forEach(t),Ds=d(e),me=i(e,"UL",{});var ki=n(me);_o=i(ki,"LI",{});var Al=n(_o);k=i(Al,"P",{});var ze=n(k);Gn=a(ze,"When using "),kt=i(ze,"A",{href:!0});var Pl=n(kt);Ln=a(Pl,"\u{1F917} Optimum dynamic quantization"),Pl.forEach(t),Fn=a(ze,", nodes as "),Ye=i(ze,"A",{href:!0,rel:!0});var kl=n(Ye);wo=i(kl,"CODE",{});var Cl=n(wo);Vn=a(Cl,"MatMulInteger"),Cl.forEach(t),kl.forEach(t),Xn=a(ze,", "),Je=i(ze,"A",{href:!0,rel:!0});var zl=n(Je);go=i(zl,"CODE",{});var Nl=n(go);Hn=a(Nl,"DynamicQuantizeLinear"),Nl.forEach(t),zl.forEach(t),Qn=a(ze," may be inserted in the ONNX graph, that cannot be consumed by the CUDA execution provider."),ze.forEach(t),Al.forEach(t),Bn=d(ki),Eo=i(ki,"LI",{});var Dl=n(Eo);H=i(Dl,"P",{});var Vt=n(H);Wn=a(Vt,"When using "),Ct=i(Vt,"A",{href:!0});var Ol=n(Ct);Yn=a(Ol,"static quantization"),Ol.forEach(t),Jn=a(Vt,", the ONNX computation graph will contain matrix multiplications and convolutions in floating-point arithmetic, along with Quantize + Dequantize operations to simulate quantization. In this case, although the costly matrix multiplications and convolutions will be run on the GPU, they will use floating-point arithmetic as the "),$o=i(Vt,"CODE",{});var Rl=n($o);Kn=a(Rl,"CUDAExecutionProvider"),Rl.forEach(t),Zn=a(Vt," can not consume the Quantize + Dequantize nodes to replace them by the operations using integer arithmetic."),Vt.forEach(t),Dl.forEach(t),ki.forEach(t),Os=d(e),Q=i(e,"H3",{class:!0});var Ci=n(Q);ve=i(Ci,"A",{id:!0,class:!0,href:!0});var Ul=n(ve);To=i(Ul,"SPAN",{});var jl=n(To);f(Ke.$$.fragment,jl),jl.forEach(t),Ul.forEach(t),er=d(Ci),xo=i(Ci,"SPAN",{});var Il=n(xo);tr=a(Il,"Observed time gains"),Il.forEach(t),Ci.forEach(t),Rs=d(e),zt=i(e,"P",{});var Sl=n(zt);or=a(Sl,"Coming soon!"),Sl.forEach(t),Us=d(e),B=i(e,"H2",{class:!0});var zi=n(B);_e=i(zi,"A",{id:!0,class:!0,href:!0});var Ml=n(_e);bo=i(Ml,"SPAN",{});var Gl=n(bo);f(Ze.$$.fragment,Gl),Gl.forEach(t),Ml.forEach(t),sr=d(zi),yo=i(zi,"SPAN",{});var Ll=n(yo);ir=a(Ll,"TensorrtExecutionProvider"),Ll.forEach(t),zi.forEach(t),js=d(e),W=i(e,"H3",{class:!0});var Ni=n(W);we=i(Ni,"A",{id:!0,class:!0,href:!0});var Fl=n(we);qo=i(Fl,"SPAN",{});var Vl=n(qo);f(et.$$.fragment,Vl),Vl.forEach(t),Fl.forEach(t),nr=d(Ni),Ao=i(Ni,"SPAN",{});var Xl=n(Ao);rr=a(Xl,"Installation"),Xl.forEach(t),Ni.forEach(t),Is=d(e),ge=i(e,"P",{});var Di=n(ge);ar=a(Di,"The easiest way to use TensorRT as the execution provider for models optimized through \u{1F917} Optimum is with the available ONNX Runtime "),Po=i(Di,"CODE",{});var Hl=n(Po);lr=a(Hl,"TensorrtExecutionProvider"),Hl.forEach(t),pr=a(Di,"."),Di.forEach(t),Ss=d(e),Nt=i(e,"P",{});var Ql=n(Nt);ur=a(Ql,"In order to use \u{1F917} Optimum with TensorRT in a local environment, we recommend following the NVIDIA installation guides:"),Ql.forEach(t),Ms=d(e),z=i(e,"UL",{});var Xt=n(z);Dt=i(Xt,"LI",{});var Sa=n(Dt);dr=a(Sa,"CUDA toolkit: "),tt=i(Sa,"A",{href:!0,rel:!0});var Bl=n(tt);cr=a(Bl,"https://docs.nvidia.com/cuda/cuda-installation-guide-linux/index.html"),Bl.forEach(t),Sa.forEach(t),hr=d(Xt),Ot=i(Xt,"LI",{});var Ma=n(Ot);fr=a(Ma,"cuDNN: "),ot=i(Ma,"A",{href:!0,rel:!0});var Wl=n(ot);mr=a(Wl,"https://docs.nvidia.com/deeplearning/cudnn/install-guide/index.html"),Wl.forEach(t),Ma.forEach(t),vr=d(Xt),Rt=i(Xt,"LI",{});var Ga=n(Rt);_r=a(Ga,"TensorRT: "),st=i(Ga,"A",{href:!0,rel:!0});var Yl=n(st);wr=a(Yl,"https://docs.nvidia.com/deeplearning/tensorrt/install-guide/index.html"),Yl.forEach(t),Ga.forEach(t),Xt.forEach(t),Gs=d(e),N=i(e,"P",{});var Ht=n(N);gr=a(Ht,"For TensorRT, we recommend the Tar File Installation method. Alternatively, TensorRT may be installable with "),ko=i(Ht,"CODE",{});var Jl=n(ko);Er=a(Jl,"pip"),Jl.forEach(t),$r=a(Ht," by following "),it=i(Ht,"A",{href:!0,rel:!0});var Kl=n(it);Tr=a(Kl,"these instructions"),Kl.forEach(t),xr=a(Ht,"."),Ht.forEach(t),Ls=d(e),Ut=i(e,"P",{});var Zl=n(Ut);br=a(Zl,"Once the required packages are installed, the following environment variables need to be set with the appropriate paths for ONNX Runtime to detect TensorRT installation:"),Zl.forEach(t),Fs=d(e),f(nt.$$.fragment,e),Vs=d(e),Y=i(e,"H3",{class:!0});var Oi=n(Y);Ee=i(Oi,"A",{id:!0,class:!0,href:!0});var ep=n(Ee);Co=i(ep,"SPAN",{});var tp=n(Co);f(rt.$$.fragment,tp),tp.forEach(t),ep.forEach(t),yr=d(Oi),zo=i(Oi,"SPAN",{});var op=n(zo);qr=a(op,"Checking the installation is successful"),op.forEach(t),Oi.forEach(t),Xs=d(e),jt=i(e,"P",{});var sp=n(jt);Ar=a(sp,"Before going further, run the following sample code to check whether the install was successful:"),sp.forEach(t),Hs=d(e),f(at.$$.fragment,e),Qs=d(e),It=i(e,"P",{});var ip=n(It);Pr=a(ip,"In case this code runs gracefully, congratulations, the installation is successful!"),ip.forEach(t),Bs=d(e),$e=i(e,"P",{});var Ri=n($e);kr=a(Ri,"In case the above "),No=i(Ri,"CODE",{});var np=n(No);Cr=a(np,"assert"),np.forEach(t),zr=a(Ri," fails, or you encounter the following warning"),Ri.forEach(t),Ws=d(e),f(lt.$$.fragment,e),Ys=d(e),St=i(e,"P",{});var rp=n(St);Nr=a(rp,"something is wrong with the TensorRT or ONNX Runtime installation."),rp.forEach(t),Js=d(e),J=i(e,"H3",{class:!0});var Ui=n(J);Te=i(Ui,"A",{id:!0,class:!0,href:!0});var ap=n(Te);Do=i(ap,"SPAN",{});var lp=n(Do);f(pt.$$.fragment,lp),lp.forEach(t),ap.forEach(t),Dr=d(Ui),Oo=i(Ui,"SPAN",{});var pp=n(Oo);Or=a(pp,"Use TensorRT execution provider with floating-point models"),pp.forEach(t),Ui.forEach(t),Ks=d(e),D=i(e,"P",{});var Qt=n(D);Rr=a(Qt,"For non-quantized models, the use is straightforward, by simply using the "),Ro=i(Qt,"CODE",{});var up=n(Ro);Ur=a(up,"provider"),up.forEach(t),jr=a(Qt," argument in "),Uo=i(Qt,"CODE",{});var dp=n(Uo);Ir=a(dp,"ORTModel.from_pretrained()"),dp.forEach(t),Sr=a(Qt,". For example:"),Qt.forEach(t),Zs=d(e),f(ut.$$.fragment,e),ei=d(e),K=i(e,"P",{});var ns=n(K);dt=i(ns,"A",{href:!0});var La=n(dt);Mr=a(La,"As previously for "),jo=i(La,"CODE",{});var cp=n(jo);Gr=a(cp,"CUDAExecutionProvider"),cp.forEach(t),La.forEach(t),Lr=a(ns,", by passing the session option "),Io=i(ns,"CODE",{});var hp=n(Io);Fr=a(hp,"log_severity_level = 0"),hp.forEach(t),Vr=a(ns," (verbose), we can check in the logs whether all nodes are indeed placed on the TensorRT execution provider or not:"),ns.forEach(t),ti=d(e),f(ct.$$.fragment,e),oi=d(e),xe=i(e,"P",{});var ji=n(xe);Xr=a(ji,"The model can then be used with the common \u{1F917} Transformers API for inference and evaluation, such as "),ht=i(ji,"A",{href:!0,rel:!0});var fp=n(ht);Hr=a(fp,"pipelines"),fp.forEach(t),Qr=a(ji,"."),ji.forEach(t),si=d(e),Z=i(e,"H3",{class:!0});var Ii=n(Z);be=i(Ii,"A",{id:!0,class:!0,href:!0});var mp=n(be);So=i(mp,"SPAN",{});var vp=n(So);f(ft.$$.fragment,vp),vp.forEach(t),mp.forEach(t),Br=d(Ii),Mo=i(Ii,"SPAN",{});var _p=n(Mo);Wr=a(_p,"Use TensorRT execution provider with quantized models"),_p.forEach(t),Ii.forEach(t),ii=d(e),A=i(e,"P",{});var Ne=n(A);Yr=a(Ne,"When it comes to quantized models, TensorRT only supports models that use "),Go=i(Ne,"STRONG",{});var wp=n(Go);Jr=a(wp,"static"),wp.forEach(t),Kr=a(Ne," quantization with "),Lo=i(Ne,"STRONG",{});var gp=n(Lo);Zr=a(gp,"symmetric quantization"),gp.forEach(t),ea=a(Ne," for weights and activations. Thus, to be able to consume with TensorRT models quantized through \u{1F917} Optimum, the following configuration needs to be passed when doing "),Mt=i(Ne,"A",{href:!0});var Ep=n(Mt);ta=a(Ep,"static quantization"),Ep.forEach(t),oa=a(Ne,":"),Ne.forEach(t),ni=d(e),f(mt.$$.fragment,e),ri=d(e),P=i(e,"P",{});var De=n(P);sa=a(De,"The "),Fo=i(De,"CODE",{});var $p=n(Fo);ia=a($p,"qdq_dedicated_pair=True"),$p.forEach(t),na=a(De," argument is required by TensorRT, since it expects a single node after each "),Vo=i(De,"CODE",{});var Tp=n(Vo);ra=a(Tp,"QuantizeLinear"),Tp.forEach(t),aa=a(De," + "),Xo=i(De,"CODE",{});var xp=n(Xo);la=a(xp,"DequantizeLinear"),xp.forEach(t),pa=a(De," (QDQ) pair."),De.forEach(t),ai=d(e),O=i(e,"P",{});var Bt=n(O);ua=a(Bt,"The parameter "),Ho=i(Bt,"CODE",{});var bp=n(Ho);da=a(bp,"qdq_add_pair_to_weight=True"),bp.forEach(t),ca=a(Bt," is also required by TensorRT, that since it consumes a graph where the weights are stored in float32 with a QDQ pair. Normally, weights would be stored in fixed point 8-bits format and only a "),Qo=i(Bt,"CODE",{});var yp=n(Qo);ha=a(yp,"DequantizeLinear"),yp.forEach(t),fa=a(Bt," would be applied on the weights. As such, the storage savings from quantization can not be leveraged when we expect to later use the quantized ONNX model with TensorRT."),Bt.forEach(t),li=d(e),ye=i(e,"P",{});var Si=n(ye);ma=a(Si,"In the code sample below, after performing static quantization, the resulting model is loaded into the "),Bo=i(Si,"CODE",{});var qp=n(Bo);va=a(qp,"ORTModel"),qp.forEach(t),_a=a(Si," class using TensorRT as the execution provider. ONNX Runtime graph optimization need to be disabled for the model to be consumed and optimized by TensorRT, and the fact that INT8 operations are used needs to be specified to TensorRT."),Si.forEach(t),pi=d(e),f(vt.$$.fragment,e),ui=d(e),qe=i(e,"P",{});var Mi=n(qe);wa=a(Mi,"The model can then be used with the common \u{1F917} Transformers API for inference and evaluation, such as "),_t=i(Mi,"A",{href:!0,rel:!0});var Ap=n(_t);ga=a(Ap,"pipelines"),Ap.forEach(t),Ea=a(Mi,"."),Mi.forEach(t),di=d(e),ee=i(e,"H3",{class:!0});var Gi=n(ee);Ae=i(Gi,"A",{id:!0,class:!0,href:!0});var Pp=n(Ae);Wo=i(Pp,"SPAN",{});var kp=n(Wo);f(wt.$$.fragment,kp),kp.forEach(t),Pp.forEach(t),$a=d(Gi),Yo=i(Gi,"SPAN",{});var Cp=n(Yo);Ta=a(Cp,"TensorRT limitations for quantized models"),Cp.forEach(t),Gi.forEach(t),ci=d(e),Gt=i(e,"P",{});var zp=n(Gt);xa=a(zp,"As highlighted in the previous section, TensorRT supports only a limited range of quantized models:"),zp.forEach(t),hi=d(e),R=i(e,"UL",{});var Wt=n(R);Jo=i(Wt,"LI",{});var Np=n(Jo);ba=a(Np,"Static quantization only"),Np.forEach(t),ya=d(Wt),Ko=i(Wt,"LI",{});var Dp=n(Ko);qa=a(Dp,"Weights and activations quantization ranges are symmetric"),Dp.forEach(t),Aa=d(Wt),Zo=i(Wt,"LI",{});var Op=n(Zo);Pa=a(Op,"Weights need to be stored in float32 in the ONNX model, thus there is no storage space saving from quantization."),Op.forEach(t),Wt.forEach(t),fi=d(e),Pe=i(e,"P",{});var Li=n(Pe);ka=a(Li,"In case "),es=i(Li,"CODE",{});var Rp=n(es);Ca=a(Rp,'provider="TensorrtExecutionProvider"'),Rp.forEach(t),za=a(Li," is passed and the model has not been quantized strictly following these constraints, various errors may be raised, where error messages can be unclear."),Li.forEach(t),mi=d(e),te=i(e,"H3",{class:!0});var Fi=n(te);ke=i(Fi,"A",{id:!0,class:!0,href:!0});var Up=n(ke);ts=i(Up,"SPAN",{});var jp=n(ts);f(gt.$$.fragment,jp),jp.forEach(t),Up.forEach(t),Na=d(Fi),os=i(Fi,"SPAN",{});var Ip=n(os);Da=a(Ip,"Observed time gains"),Ip.forEach(t),Fi.forEach(t),vi=d(e),Ce=i(e,"P",{});var Vi=n(Ce);Oa=a(Vi,"Nvidia Nsight Systems tool can be used to profile the execution time on GPU. Before profiling or measuring latency/throughput, it is a good practice to do a few "),ss=i(Vi,"STRONG",{});var Sp=n(ss);Ra=a(Sp,"warmup steps"),Sp.forEach(t),Ua=a(Vi,"."),Vi.forEach(t),_i=d(e),Lt=i(e,"P",{});var Mp=n(Lt);ja=a(Mp,"Coming soon!"),Mp.forEach(t),this.h()},h(){c(g,"name","hf:doc:metadata"),c(g,"content",JSON.stringify(Wp)),c(q,"id","accelerated-inference-on-nvidia-gpus"),c(q,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),c(q,"href","#accelerated-inference-on-nvidia-gpus"),c(E,"class","relative group"),c(Re,"href","https://developer.nvidia.com/tensorrt"),c(Re,"rel","nofollow"),c(re,"id","cudaexecutionprovider"),c(re,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),c(re,"href","#cudaexecutionprovider"),c(G,"class","relative group"),c(ae,"id","installation"),c(ae,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),c(ae,"href","#installation"),c(L,"class","relative group"),c(Ie,"href","https://onnxruntime.ai/docs/execution-providers/CUDA-ExecutionProvider.html#requirements"),c(Ie,"rel","nofollow"),c(pe,"id","checking-the-installation-is-successful"),c(pe,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),c(pe,"href","#checking-the-installation-is-successful"),c(F,"class","relative group"),c(ue,"id","use-cuda-execution-provider-with-floatingpoint-models"),c(ue,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),c(ue,"href","#use-cuda-execution-provider-with-floatingpoint-models"),c(V,"class","relative group"),c(Xe,"href","https://huggingface.co/docs/optimum/onnxruntime/usage_guides/pipelines"),c(Xe,"rel","nofollow"),c(he,"id","use-cuda-execution-provider-with-quantized-models"),c(he,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),c(he,"href","#use-cuda-execution-provider-with-quantized-models"),c(X,"class","relative group"),c(kt,"href","quantization#dynamic-quantization-example"),c(Ye,"href","https://github.com/onnx/onnx/blob/v1.12.0/docs/Operators.md#MatMulInteger"),c(Ye,"rel","nofollow"),c(Je,"href","https://github.com/onnx/onnx/blob/v1.12.0/docs/Operators.md#DynamicQuantizeLinear"),c(Je,"rel","nofollow"),c(Ct,"href","quantization#static-quantization-example"),c(ve,"id","observed-time-gains"),c(ve,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),c(ve,"href","#observed-time-gains"),c(Q,"class","relative group"),c(_e,"id","tensorrtexecutionprovider"),c(_e,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),c(_e,"href","#tensorrtexecutionprovider"),c(B,"class","relative group"),c(we,"id","installation"),c(we,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),c(we,"href","#installation"),c(W,"class","relative group"),c(tt,"href","https://docs.nvidia.com/cuda/cuda-installation-guide-linux/index.html"),c(tt,"rel","nofollow"),c(ot,"href","https://docs.nvidia.com/deeplearning/cudnn/install-guide/index.html"),c(ot,"rel","nofollow"),c(st,"href","https://docs.nvidia.com/deeplearning/tensorrt/install-guide/index.html"),c(st,"rel","nofollow"),c(it,"href","https://github.com/microsoft/onnxruntime/issues/9986"),c(it,"rel","nofollow"),c(Ee,"id","checking-the-installation-is-successful"),c(Ee,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),c(Ee,"href","#checking-the-installation-is-successful"),c(Y,"class","relative group"),c(Te,"id","use-tensorrt-execution-provider-with-floatingpoint-models"),c(Te,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),c(Te,"href","#use-tensorrt-execution-provider-with-floatingpoint-models"),c(J,"class","relative group"),c(dt,"href","#use-cuda-execution-provider-with-floatingpoint-models"),c(ht,"href","https://huggingface.co/docs/optimum/onnxruntime/usage_guides/pipelines"),c(ht,"rel","nofollow"),c(be,"id","use-tensorrt-execution-provider-with-quantized-models"),c(be,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),c(be,"href","#use-tensorrt-execution-provider-with-quantized-models"),c(Z,"class","relative group"),c(Mt,"href","quantization#static-quantization-example"),c(_t,"href","https://huggingface.co/docs/optimum/onnxruntime/usage_guides/pipelines"),c(_t,"rel","nofollow"),c(Ae,"id","tensorrt-limitations-for-quantized-models"),c(Ae,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),c(Ae,"href","#tensorrt-limitations-for-quantized-models"),c(ee,"class","relative group"),c(ke,"id","observed-time-gains"),c(ke,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),c(ke,"href","#observed-time-gains"),c(te,"class","relative group")},m(e,l){o(document.head,g),p(e,oe,l),p(e,E,l),o(E,q),o(q,j),m(b,j,null),o(E,Oe),o(E,I),o(I,S),p(e,y,l),p(e,M,l),o(M,$t),p(e,as,l),p(e,Tt,l),o(Tt,Xi),p(e,ls,l),p(e,se,l),o(se,xt),o(xt,Yt),o(Yt,Hi),o(xt,Qi),o(se,Bi),o(se,ie),o(ie,Jt),o(Jt,Wi),o(ie,Yi),o(ie,Re),o(Re,Ji),o(ie,Ki),p(e,ps,l),m(ne,e,l),p(e,us,l),p(e,G,l),o(G,re),o(re,Kt),m(Ue,Kt,null),o(G,Zi),o(G,Zt),o(Zt,en),p(e,ds,l),p(e,L,l),o(L,ae),o(ae,eo),m(je,eo,null),o(L,tn),o(L,to),o(to,on),p(e,cs,l),p(e,le,l),o(le,sn),o(le,Ie),o(Ie,nn),o(le,rn),p(e,hs,l),m(Se,e,l),p(e,fs,l),p(e,$,l),o($,an),o($,oo),o(oo,ln),o($,pn),o($,so),o(so,un),o($,dn),o($,io),o(io,cn),o($,hn),o($,no),o(no,fn),o($,mn),p(e,ms,l),p(e,F,l),o(F,pe),o(pe,ro),m(Me,ro,null),o(F,vn),o(F,ao),o(ao,_n),p(e,vs,l),p(e,bt,l),o(bt,wn),p(e,_s,l),m(Ge,e,l),p(e,ws,l),p(e,yt,l),o(yt,gn),p(e,gs,l),m(Le,e,l),p(e,Es,l),p(e,qt,l),o(qt,En),p(e,$s,l),p(e,V,l),o(V,ue),o(ue,lo),m(Fe,lo,null),o(V,$n),o(V,po),o(po,Tn),p(e,Ts,l),p(e,C,l),o(C,xn),o(C,uo),o(uo,bn),o(C,yn),o(C,co),o(co,qn),o(C,An),p(e,xs,l),m(Ve,e,l),p(e,bs,l),p(e,de,l),o(de,Pn),o(de,Xe),o(Xe,kn),o(de,Cn),p(e,ys,l),m(He,e,l),p(e,qs,l),p(e,ce,l),o(ce,zn),o(ce,ho),o(ho,Nn),o(ce,Dn),p(e,As,l),m(Qe,e,l),p(e,Ps,l),p(e,At,l),o(At,On),p(e,ks,l),m(Be,e,l),p(e,Cs,l),p(e,Pt,l),o(Pt,Rn),p(e,zs,l),p(e,X,l),o(X,he),o(he,fo),m(We,fo,null),o(X,Un),o(X,mo),o(mo,jn),p(e,Ns,l),p(e,fe,l),o(fe,In),o(fe,vo),o(vo,Sn),o(fe,Mn),p(e,Ds,l),p(e,me,l),o(me,_o),o(_o,k),o(k,Gn),o(k,kt),o(kt,Ln),o(k,Fn),o(k,Ye),o(Ye,wo),o(wo,Vn),o(k,Xn),o(k,Je),o(Je,go),o(go,Hn),o(k,Qn),o(me,Bn),o(me,Eo),o(Eo,H),o(H,Wn),o(H,Ct),o(Ct,Yn),o(H,Jn),o(H,$o),o($o,Kn),o(H,Zn),p(e,Os,l),p(e,Q,l),o(Q,ve),o(ve,To),m(Ke,To,null),o(Q,er),o(Q,xo),o(xo,tr),p(e,Rs,l),p(e,zt,l),o(zt,or),p(e,Us,l),p(e,B,l),o(B,_e),o(_e,bo),m(Ze,bo,null),o(B,sr),o(B,yo),o(yo,ir),p(e,js,l),p(e,W,l),o(W,we),o(we,qo),m(et,qo,null),o(W,nr),o(W,Ao),o(Ao,rr),p(e,Is,l),p(e,ge,l),o(ge,ar),o(ge,Po),o(Po,lr),o(ge,pr),p(e,Ss,l),p(e,Nt,l),o(Nt,ur),p(e,Ms,l),p(e,z,l),o(z,Dt),o(Dt,dr),o(Dt,tt),o(tt,cr),o(z,hr),o(z,Ot),o(Ot,fr),o(Ot,ot),o(ot,mr),o(z,vr),o(z,Rt),o(Rt,_r),o(Rt,st),o(st,wr),p(e,Gs,l),p(e,N,l),o(N,gr),o(N,ko),o(ko,Er),o(N,$r),o(N,it),o(it,Tr),o(N,xr),p(e,Ls,l),p(e,Ut,l),o(Ut,br),p(e,Fs,l),m(nt,e,l),p(e,Vs,l),p(e,Y,l),o(Y,Ee),o(Ee,Co),m(rt,Co,null),o(Y,yr),o(Y,zo),o(zo,qr),p(e,Xs,l),p(e,jt,l),o(jt,Ar),p(e,Hs,l),m(at,e,l),p(e,Qs,l),p(e,It,l),o(It,Pr),p(e,Bs,l),p(e,$e,l),o($e,kr),o($e,No),o(No,Cr),o($e,zr),p(e,Ws,l),m(lt,e,l),p(e,Ys,l),p(e,St,l),o(St,Nr),p(e,Js,l),p(e,J,l),o(J,Te),o(Te,Do),m(pt,Do,null),o(J,Dr),o(J,Oo),o(Oo,Or),p(e,Ks,l),p(e,D,l),o(D,Rr),o(D,Ro),o(Ro,Ur),o(D,jr),o(D,Uo),o(Uo,Ir),o(D,Sr),p(e,Zs,l),m(ut,e,l),p(e,ei,l),p(e,K,l),o(K,dt),o(dt,Mr),o(dt,jo),o(jo,Gr),o(K,Lr),o(K,Io),o(Io,Fr),o(K,Vr),p(e,ti,l),m(ct,e,l),p(e,oi,l),p(e,xe,l),o(xe,Xr),o(xe,ht),o(ht,Hr),o(xe,Qr),p(e,si,l),p(e,Z,l),o(Z,be),o(be,So),m(ft,So,null),o(Z,Br),o(Z,Mo),o(Mo,Wr),p(e,ii,l),p(e,A,l),o(A,Yr),o(A,Go),o(Go,Jr),o(A,Kr),o(A,Lo),o(Lo,Zr),o(A,ea),o(A,Mt),o(Mt,ta),o(A,oa),p(e,ni,l),m(mt,e,l),p(e,ri,l),p(e,P,l),o(P,sa),o(P,Fo),o(Fo,ia),o(P,na),o(P,Vo),o(Vo,ra),o(P,aa),o(P,Xo),o(Xo,la),o(P,pa),p(e,ai,l),p(e,O,l),o(O,ua),o(O,Ho),o(Ho,da),o(O,ca),o(O,Qo),o(Qo,ha),o(O,fa),p(e,li,l),p(e,ye,l),o(ye,ma),o(ye,Bo),o(Bo,va),o(ye,_a),p(e,pi,l),m(vt,e,l),p(e,ui,l),p(e,qe,l),o(qe,wa),o(qe,_t),o(_t,ga),o(qe,Ea),p(e,di,l),p(e,ee,l),o(ee,Ae),o(Ae,Wo),m(wt,Wo,null),o(ee,$a),o(ee,Yo),o(Yo,Ta),p(e,ci,l),p(e,Gt,l),o(Gt,xa),p(e,hi,l),p(e,R,l),o(R,Jo),o(Jo,ba),o(R,ya),o(R,Ko),o(Ko,qa),o(R,Aa),o(R,Zo),o(Zo,Pa),p(e,fi,l),p(e,Pe,l),o(Pe,ka),o(Pe,es),o(es,Ca),o(Pe,za),p(e,mi,l),p(e,te,l),o(te,ke),o(ke,ts),m(gt,ts,null),o(te,Na),o(te,os),o(os,Da),p(e,vi,l),p(e,Ce,l),o(Ce,Oa),o(Ce,ss),o(ss,Ra),o(Ce,Ua),p(e,_i,l),p(e,Lt,l),o(Lt,ja),wi=!0},p(e,[l]){const Et={};l&2&&(Et.$$scope={dirty:l,ctx:e}),ne.$set(Et)},i(e){wi||(v(b.$$.fragment,e),v(ne.$$.fragment,e),v(Ue.$$.fragment,e),v(je.$$.fragment,e),v(Se.$$.fragment,e),v(Me.$$.fragment,e),v(Ge.$$.fragment,e),v(Le.$$.fragment,e),v(Fe.$$.fragment,e),v(Ve.$$.fragment,e),v(He.$$.fragment,e),v(Qe.$$.fragment,e),v(Be.$$.fragment,e),v(We.$$.fragment,e),v(Ke.$$.fragment,e),v(Ze.$$.fragment,e),v(et.$$.fragment,e),v(nt.$$.fragment,e),v(rt.$$.fragment,e),v(at.$$.fragment,e),v(lt.$$.fragment,e),v(pt.$$.fragment,e),v(ut.$$.fragment,e),v(ct.$$.fragment,e),v(ft.$$.fragment,e),v(mt.$$.fragment,e),v(vt.$$.fragment,e),v(wt.$$.fragment,e),v(gt.$$.fragment,e),wi=!0)},o(e){_(b.$$.fragment,e),_(ne.$$.fragment,e),_(Ue.$$.fragment,e),_(je.$$.fragment,e),_(Se.$$.fragment,e),_(Me.$$.fragment,e),_(Ge.$$.fragment,e),_(Le.$$.fragment,e),_(Fe.$$.fragment,e),_(Ve.$$.fragment,e),_(He.$$.fragment,e),_(Qe.$$.fragment,e),_(Be.$$.fragment,e),_(We.$$.fragment,e),_(Ke.$$.fragment,e),_(Ze.$$.fragment,e),_(et.$$.fragment,e),_(nt.$$.fragment,e),_(rt.$$.fragment,e),_(at.$$.fragment,e),_(lt.$$.fragment,e),_(pt.$$.fragment,e),_(ut.$$.fragment,e),_(ct.$$.fragment,e),_(ft.$$.fragment,e),_(mt.$$.fragment,e),_(vt.$$.fragment,e),_(wt.$$.fragment,e),_(gt.$$.fragment,e),wi=!1},d(e){t(g),e&&t(oe),e&&t(E),w(b),e&&t(y),e&&t(M),e&&t(as),e&&t(Tt),e&&t(ls),e&&t(se),e&&t(ps),w(ne,e),e&&t(us),e&&t(G),w(Ue),e&&t(ds),e&&t(L),w(je),e&&t(cs),e&&t(le),e&&t(hs),w(Se,e),e&&t(fs),e&&t($),e&&t(ms),e&&t(F),w(Me),e&&t(vs),e&&t(bt),e&&t(_s),w(Ge,e),e&&t(ws),e&&t(yt),e&&t(gs),w(Le,e),e&&t(Es),e&&t(qt),e&&t($s),e&&t(V),w(Fe),e&&t(Ts),e&&t(C),e&&t(xs),w(Ve,e),e&&t(bs),e&&t(de),e&&t(ys),w(He,e),e&&t(qs),e&&t(ce),e&&t(As),w(Qe,e),e&&t(Ps),e&&t(At),e&&t(ks),w(Be,e),e&&t(Cs),e&&t(Pt),e&&t(zs),e&&t(X),w(We),e&&t(Ns),e&&t(fe),e&&t(Ds),e&&t(me),e&&t(Os),e&&t(Q),w(Ke),e&&t(Rs),e&&t(zt),e&&t(Us),e&&t(B),w(Ze),e&&t(js),e&&t(W),w(et),e&&t(Is),e&&t(ge),e&&t(Ss),e&&t(Nt),e&&t(Ms),e&&t(z),e&&t(Gs),e&&t(N),e&&t(Ls),e&&t(Ut),e&&t(Fs),w(nt,e),e&&t(Vs),e&&t(Y),w(rt),e&&t(Xs),e&&t(jt),e&&t(Hs),w(at,e),e&&t(Qs),e&&t(It),e&&t(Bs),e&&t($e),e&&t(Ws),w(lt,e),e&&t(Ys),e&&t(St),e&&t(Js),e&&t(J),w(pt),e&&t(Ks),e&&t(D),e&&t(Zs),w(ut,e),e&&t(ei),e&&t(K),e&&t(ti),w(ct,e),e&&t(oi),e&&t(xe),e&&t(si),e&&t(Z),w(ft),e&&t(ii),e&&t(A),e&&t(ni),w(mt,e),e&&t(ri),e&&t(P),e&&t(ai),e&&t(O),e&&t(li),e&&t(ye),e&&t(pi),w(vt,e),e&&t(ui),e&&t(qe),e&&t(di),e&&t(ee),w(wt),e&&t(ci),e&&t(Gt),e&&t(hi),e&&t(R),e&&t(fi),e&&t(Pe),e&&t(mi),e&&t(te),w(gt),e&&t(vi),e&&t(Ce),e&&t(_i),e&&t(Lt)}}}const Wp={local:"accelerated-inference-on-nvidia-gpus",sections:[{local:"cudaexecutionprovider",sections:[{local:"installation",title:"Installation"},{local:"checking-the-installation-is-successful",title:"Checking the installation is successful"},{local:"use-cuda-execution-provider-with-floatingpoint-models",title:"Use CUDA execution provider with floating-point models"},{local:"use-cuda-execution-provider-with-quantized-models",title:"Use CUDA execution provider with quantized models"},{local:"observed-time-gains",title:"Observed time gains"}],title:"CUDAExecutionProvider"},{local:"tensorrtexecutionprovider",sections:[{local:"installation",title:"Installation"},{local:"checking-the-installation-is-successful",title:"Checking the installation is successful"},{local:"use-tensorrt-execution-provider-with-floatingpoint-models",title:"Use TensorRT execution provider with floating-point models"},{local:"use-tensorrt-execution-provider-with-quantized-models",title:"Use TensorRT execution provider with quantized models"},{local:"tensorrt-limitations-for-quantized-models",title:"TensorRT limitations for quantized models"},{local:"observed-time-gains",title:"Observed time gains"}],title:"TensorrtExecutionProvider"}],title:"Accelerated inference on NVIDIA GPUs"};function Yp(rs){return Xp(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class tu extends Gp{constructor(g){super();Lp(this,g,Yp,Bp,Fp,{})}}export{tu as default,Wp as metadata};
