import{S as xt,i as Tt,s as Ct,e as s,k as f,w as j,t as r,M as Ut,c as i,d as t,m as h,a as n,x as A,h as l,b as c,G as a,g as p,y as x,q as T,o as C,B as U,v as Mt}from"../../chunks/vendor-hf-doc-builder.js";import{T as Nt}from"../../chunks/Tip-hf-doc-builder.js";import{I as xe}from"../../chunks/IconCopyLink-hf-doc-builder.js";import{C as pt}from"../../chunks/CodeBlock-hf-doc-builder.js";function Vt(pe){let u;return{c(){u=r("Attention slicing is useful even if a batch size of just 1 is used - as long as the model uses more than one attention head. If there is more than one attention head the *QK^T* attention matrix can be computed sequentially for each head which can save a significant amount of memory.")},l(d){u=l(d,"Attention slicing is useful even if a batch size of just 1 is used - as long as the model uses more than one attention head. If there is more than one attention head the *QK^T* attention matrix can be computed sequentially for each head which can save a significant amount of memory.")},m(d,m){p(d,u,m)},d(d){d&&t(u)}}}function Gt(pe){let u,d,m,$,W,M,Te,J,Ce,fe,b,Ue,X,Me,Ne,he,_,k,Y,N,Ve,z,Ge,Z,He,ce,v,Oe,ee,Ie,Fe,te,ze,Be,ue,V,me,E,Re,ae,Ke,Le,de,w,P,oe,G,Qe,se,We,ve,y,Je,ie,Xe,Ye,ne,Ze,et,ye,H,_e,g,q,re,O,tt,le,at,we,B,ot,ge,D,$e,S,st,R,it,nt,be,I,ke,K,rt,Ee;return M=new xe({}),N=new xe({}),V=new pt({props:{code:`from torch import autocast
from diffusers import StableDiffusionPipeline

pipe = StableDiffusionPipeline.from_pretrained("CompVis/stable-diffusion-v1-4", use_auth_token=True)
pipe = pipe.to("cuda")

prompt = "a photo of an astronaut riding a horse on mars"
with autocast("cuda"):
    image = pipe(prompt).images[0]  `,highlighted:`<span class="hljs-keyword">from</span> torch <span class="hljs-keyword">import</span> autocast
<span class="hljs-keyword">from</span> diffusers <span class="hljs-keyword">import</span> StableDiffusionPipeline

pipe = StableDiffusionPipeline.from_pretrained(<span class="hljs-string">&quot;CompVis/stable-diffusion-v1-4&quot;</span>, use_auth_token=<span class="hljs-literal">True</span>)
pipe = pipe.to(<span class="hljs-string">&quot;cuda&quot;</span>)

prompt = <span class="hljs-string">&quot;a photo of an astronaut riding a horse on mars&quot;</span>
<span class="hljs-keyword">with</span> autocast(<span class="hljs-string">&quot;cuda&quot;</span>):
    image = pipe(prompt).images[<span class="hljs-number">0</span>]  `}}),G=new xe({}),H=new pt({props:{code:`pipe = StableDiffusionPipeline.from_pretrained(
    "CompVis/stable-diffusion-v1-4",
    revision="fp16",
    torch_dtype=torch.float16,
    use_auth_token=True
)`,highlighted:`pipe = StableDiffusionPipeline.from_pretrained(
    <span class="hljs-string">&quot;CompVis/stable-diffusion-v1-4&quot;</span>,
    revision=<span class="hljs-string">&quot;fp16&quot;</span>,
    torch_dtype=torch.float16,
    use_auth_token=<span class="hljs-literal">True</span>
)`}}),O=new xe({}),D=new Nt({props:{$$slots:{default:[Vt]},$$scope:{ctx:pe}}}),I=new pt({props:{code:`import torch
from diffusers import StableDiffusionPipeline

pipe = StableDiffusionPipeline.from_pretrained(
    "CompVis/stable-diffusion-v1-4",
    revision="fp16",
    torch_dtype=torch.float16,
    use_auth_token=True
)
pipe = pipe.to("cuda")

prompt = "a photo of an astronaut riding a horse on mars"
pipe.enable_attention_slicing()
with torch.autocast("cuda"):
    image = pipe(prompt).images[0]  `,highlighted:`<span class="hljs-keyword">import</span> torch
<span class="hljs-keyword">from</span> diffusers <span class="hljs-keyword">import</span> StableDiffusionPipeline

pipe = StableDiffusionPipeline.from_pretrained(
    <span class="hljs-string">&quot;CompVis/stable-diffusion-v1-4&quot;</span>,
    revision=<span class="hljs-string">&quot;fp16&quot;</span>,
    torch_dtype=torch.float16,
    use_auth_token=<span class="hljs-literal">True</span>
)
pipe = pipe.to(<span class="hljs-string">&quot;cuda&quot;</span>)

prompt = <span class="hljs-string">&quot;a photo of an astronaut riding a horse on mars&quot;</span>
pipe.enable_attention_slicing()
<span class="hljs-keyword">with</span> torch.autocast(<span class="hljs-string">&quot;cuda&quot;</span>):
    image = pipe(prompt).images[<span class="hljs-number">0</span>]  `}}),{c(){u=s("meta"),d=f(),m=s("h1"),$=s("a"),W=s("span"),j(M.$$.fragment),Te=f(),J=s("span"),Ce=r("Memory and speed"),fe=f(),b=s("p"),Ue=r("We present some techniques and ideas to optimize \u{1F917} Diffusers "),X=s("em"),Me=r("inference"),Ne=r(" for memory or speed."),he=f(),_=s("h2"),k=s("a"),Y=s("span"),j(N.$$.fragment),Ve=f(),z=s("span"),Ge=r("CUDA "),Z=s("code"),He=r("autocast"),ce=f(),v=s("p"),Oe=r("If you use a CUDA GPU, you can take advantage of "),ee=s("code"),Ie=r("torch.autocast"),Fe=r(" to perform inference roughly twice as fast at the cost of slightly lower precision. All you need to do is put your inference call inside an "),te=s("code"),ze=r("autocast"),Be=r(" context manager. The following example shows how to do it using Stable Diffusion text-to-image generation as an example:"),ue=f(),j(V.$$.fragment),me=f(),E=s("p"),Re=r("Despite the precision loss, in our experience the final image results look the same as the "),ae=s("code"),Ke=r("float32"),Le=r(" versions. Feel free to experiment and report back!"),de=f(),w=s("h2"),P=s("a"),oe=s("span"),j(G.$$.fragment),Qe=f(),se=s("span"),We=r("Half precision weights"),ve=f(),y=s("p"),Je=r("To save more GPU memory, you can load the model weights directly in half precision. This involves loading the float16 version of the weights, which was saved to a branch named "),ie=s("code"),Xe=r("fp16"),Ye=r(", and telling PyTorch to use the "),ne=s("code"),Ze=r("float16"),et=r(" type when loading them:"),ye=f(),j(H.$$.fragment),_e=f(),g=s("h2"),q=s("a"),re=s("span"),j(O.$$.fragment),tt=f(),le=s("span"),at=r("Sliced attention for additional memory savings"),we=f(),B=s("p"),ot=r("For even additional memory savings, you can use a sliced version of attention that performs the computation in steps instead of all at once."),ge=f(),j(D.$$.fragment),$e=f(),S=s("p"),st=r("To perform the attention computation sequentially over each head, you only need to invoke "),R=s("a"),it=r("enable_attention_slicing()"),nt=r(" in your pipeline before inference, like here:"),be=f(),j(I.$$.fragment),ke=f(),K=s("p"),rt=r("There\u2019s a small performance penalty of about 10% slower inference times, but this method allows you to use Stable Diffusion in as little as 3.2 GB of VRAM!"),this.h()},l(e){const o=Ut('[data-svelte="svelte-1phssyn"]',document.head);u=i(o,"META",{name:!0,content:!0}),o.forEach(t),d=h(e),m=i(e,"H1",{class:!0});var F=n(m);$=i(F,"A",{id:!0,class:!0,href:!0});var ft=n($);W=i(ft,"SPAN",{});var ht=n(W);A(M.$$.fragment,ht),ht.forEach(t),ft.forEach(t),Te=h(F),J=i(F,"SPAN",{});var ct=n(J);Ce=l(ct,"Memory and speed"),ct.forEach(t),F.forEach(t),fe=h(e),b=i(e,"P",{});var Pe=n(b);Ue=l(Pe,"We present some techniques and ideas to optimize \u{1F917} Diffusers "),X=i(Pe,"EM",{});var ut=n(X);Me=l(ut,"inference"),ut.forEach(t),Ne=l(Pe," for memory or speed."),Pe.forEach(t),he=h(e),_=i(e,"H2",{class:!0});var qe=n(_);k=i(qe,"A",{id:!0,class:!0,href:!0});var mt=n(k);Y=i(mt,"SPAN",{});var dt=n(Y);A(N.$$.fragment,dt),dt.forEach(t),mt.forEach(t),Ve=h(qe),z=i(qe,"SPAN",{});var lt=n(z);Ge=l(lt,"CUDA "),Z=i(lt,"CODE",{});var vt=n(Z);He=l(vt,"autocast"),vt.forEach(t),lt.forEach(t),qe.forEach(t),ce=h(e),v=i(e,"P",{});var L=n(v);Oe=l(L,"If you use a CUDA GPU, you can take advantage of "),ee=i(L,"CODE",{});var yt=n(ee);Ie=l(yt,"torch.autocast"),yt.forEach(t),Fe=l(L," to perform inference roughly twice as fast at the cost of slightly lower precision. All you need to do is put your inference call inside an "),te=i(L,"CODE",{});var _t=n(te);ze=l(_t,"autocast"),_t.forEach(t),Be=l(L," context manager. The following example shows how to do it using Stable Diffusion text-to-image generation as an example:"),L.forEach(t),ue=h(e),A(V.$$.fragment,e),me=h(e),E=i(e,"P",{});var De=n(E);Re=l(De,"Despite the precision loss, in our experience the final image results look the same as the "),ae=i(De,"CODE",{});var wt=n(ae);Ke=l(wt,"float32"),wt.forEach(t),Le=l(De," versions. Feel free to experiment and report back!"),De.forEach(t),de=h(e),w=i(e,"H2",{class:!0});var Se=n(w);P=i(Se,"A",{id:!0,class:!0,href:!0});var gt=n(P);oe=i(gt,"SPAN",{});var $t=n(oe);A(G.$$.fragment,$t),$t.forEach(t),gt.forEach(t),Qe=h(Se),se=i(Se,"SPAN",{});var bt=n(se);We=l(bt,"Half precision weights"),bt.forEach(t),Se.forEach(t),ve=h(e),y=i(e,"P",{});var Q=n(y);Je=l(Q,"To save more GPU memory, you can load the model weights directly in half precision. This involves loading the float16 version of the weights, which was saved to a branch named "),ie=i(Q,"CODE",{});var kt=n(ie);Xe=l(kt,"fp16"),kt.forEach(t),Ye=l(Q,", and telling PyTorch to use the "),ne=i(Q,"CODE",{});var Et=n(ne);Ze=l(Et,"float16"),Et.forEach(t),et=l(Q," type when loading them:"),Q.forEach(t),ye=h(e),A(H.$$.fragment,e),_e=h(e),g=i(e,"H2",{class:!0});var je=n(g);q=i(je,"A",{id:!0,class:!0,href:!0});var Pt=n(q);re=i(Pt,"SPAN",{});var qt=n(re);A(O.$$.fragment,qt),qt.forEach(t),Pt.forEach(t),tt=h(je),le=i(je,"SPAN",{});var Dt=n(le);at=l(Dt,"Sliced attention for additional memory savings"),Dt.forEach(t),je.forEach(t),we=h(e),B=i(e,"P",{});var St=n(B);ot=l(St,"For even additional memory savings, you can use a sliced version of attention that performs the computation in steps instead of all at once."),St.forEach(t),ge=h(e),A(D.$$.fragment,e),$e=h(e),S=i(e,"P",{});var Ae=n(S);st=l(Ae,"To perform the attention computation sequentially over each head, you only need to invoke "),R=i(Ae,"A",{href:!0});var jt=n(R);it=l(jt,"enable_attention_slicing()"),jt.forEach(t),nt=l(Ae," in your pipeline before inference, like here:"),Ae.forEach(t),be=h(e),A(I.$$.fragment,e),ke=h(e),K=i(e,"P",{});var At=n(K);rt=l(At,"There\u2019s a small performance penalty of about 10% slower inference times, but this method allows you to use Stable Diffusion in as little as 3.2 GB of VRAM!"),At.forEach(t),this.h()},h(){c(u,"name","hf:doc:metadata"),c(u,"content",JSON.stringify(Ht)),c($,"id","memory-and-speed"),c($,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),c($,"href","#memory-and-speed"),c(m,"class","relative group"),c(k,"id","cuda-autocast"),c(k,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),c(k,"href","#cuda-autocast"),c(_,"class","relative group"),c(P,"id","half-precision-weights"),c(P,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),c(P,"href","#half-precision-weights"),c(w,"class","relative group"),c(q,"id","sliced-attention-for-additional-memory-savings"),c(q,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),c(q,"href","#sliced-attention-for-additional-memory-savings"),c(g,"class","relative group"),c(R,"href","/docs/diffusers/pr_581/en/api/pipelines/stable_diffusion#diffusers.StableDiffusionPipeline.enable_attention_slicing")},m(e,o){a(document.head,u),p(e,d,o),p(e,m,o),a(m,$),a($,W),x(M,W,null),a(m,Te),a(m,J),a(J,Ce),p(e,fe,o),p(e,b,o),a(b,Ue),a(b,X),a(X,Me),a(b,Ne),p(e,he,o),p(e,_,o),a(_,k),a(k,Y),x(N,Y,null),a(_,Ve),a(_,z),a(z,Ge),a(z,Z),a(Z,He),p(e,ce,o),p(e,v,o),a(v,Oe),a(v,ee),a(ee,Ie),a(v,Fe),a(v,te),a(te,ze),a(v,Be),p(e,ue,o),x(V,e,o),p(e,me,o),p(e,E,o),a(E,Re),a(E,ae),a(ae,Ke),a(E,Le),p(e,de,o),p(e,w,o),a(w,P),a(P,oe),x(G,oe,null),a(w,Qe),a(w,se),a(se,We),p(e,ve,o),p(e,y,o),a(y,Je),a(y,ie),a(ie,Xe),a(y,Ye),a(y,ne),a(ne,Ze),a(y,et),p(e,ye,o),x(H,e,o),p(e,_e,o),p(e,g,o),a(g,q),a(q,re),x(O,re,null),a(g,tt),a(g,le),a(le,at),p(e,we,o),p(e,B,o),a(B,ot),p(e,ge,o),x(D,e,o),p(e,$e,o),p(e,S,o),a(S,st),a(S,R),a(R,it),a(S,nt),p(e,be,o),x(I,e,o),p(e,ke,o),p(e,K,o),a(K,rt),Ee=!0},p(e,[o]){const F={};o&2&&(F.$$scope={dirty:o,ctx:e}),D.$set(F)},i(e){Ee||(T(M.$$.fragment,e),T(N.$$.fragment,e),T(V.$$.fragment,e),T(G.$$.fragment,e),T(H.$$.fragment,e),T(O.$$.fragment,e),T(D.$$.fragment,e),T(I.$$.fragment,e),Ee=!0)},o(e){C(M.$$.fragment,e),C(N.$$.fragment,e),C(V.$$.fragment,e),C(G.$$.fragment,e),C(H.$$.fragment,e),C(O.$$.fragment,e),C(D.$$.fragment,e),C(I.$$.fragment,e),Ee=!1},d(e){t(u),e&&t(d),e&&t(m),U(M),e&&t(fe),e&&t(b),e&&t(he),e&&t(_),U(N),e&&t(ce),e&&t(v),e&&t(ue),U(V,e),e&&t(me),e&&t(E),e&&t(de),e&&t(w),U(G),e&&t(ve),e&&t(y),e&&t(ye),U(H,e),e&&t(_e),e&&t(g),U(O),e&&t(we),e&&t(B),e&&t(ge),U(D,e),e&&t($e),e&&t(S),e&&t(be),U(I,e),e&&t(ke),e&&t(K)}}}const Ht={local:"memory-and-speed",sections:[{local:"cuda-autocast",title:"CUDA `autocast`"},{local:"half-precision-weights",title:"Half precision weights"},{local:"sliced-attention-for-additional-memory-savings",title:"Sliced attention for additional memory savings"}],title:"Memory and speed"};function Ot(pe){return Mt(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class Rt extends xt{constructor(u){super();Tt(this,u,Ot,Gt,Ct,{})}}export{Rt as default,Ht as metadata};
