import{S as zn,i as Fn,s as Qn,e as n,k as h,w,t as l,M as Zn,c as s,d as t,m as c,a as i,x as P,h as p,b as f,G as a,g as r,y as _,L as Jn,q as y,o as k,B as b,v as Kn}from"../chunks/vendor-hf-doc-builder.js";import{I as st}from"../chunks/IconCopyLink-hf-doc-builder.js";import{C as it}from"../chunks/CodeBlock-hf-doc-builder.js";function es(Bo){let E,rt,g,I,Ae,B,pa,Ve,ha,lt,$,ca,H,fa,da,pt,se,ua,ht,U,A,Se,O,ma,Le,va,ct,ie,wa,ft,C,V,xe,X,Pa,De,_a,dt,re,ya,ut,M,je,ka,ba,mt,le,Ea,vt,pe,ga,wt,he,Ua,Pt,ce,Ca,_t,fe,Na,yt,de,Ga,kt,W,Te,Ia,$a,bt,ue,Aa,Et,me,Va,gt,ve,Sa,Ut,N,S,Be,R,La,He,xa,Ct,we,Da,Nt,q,Gt,Pe,ja,It,Y,$t,_e,Ta,At,z,Vt,ye,Ba,St,F,Lt,v,Ha,Oe,Oa,Xa,Xe,Ma,Wa,xt,ke,Ra,Dt,be,qa,jt,G,L,Me,Q,Ya,We,za,Tt,Z,J,Fa,Qa,Bt,x,Za,K,Ja,Ka,Ht,Ee,Re,eo,Ot,u,to,qe,ao,oo,Ye,no,so,ze,io,ro,Xt,ge,lo,Mt,Ue,po,Wt,D,Fe,ee,Qe,ho,co,Ce,fo,uo,te,ae,Ze,mo,vo,Ne,wo,Po,oe,Je,_o,yo,Ge,ko,Rt,j,bo,Ke,Eo,go,qt,Ie,Uo,Yt,ne,zt,d,Co,et,No,Go,tt,Io,$o,at,Ao,Vo,ot,So,Lo,nt,xo,Ft;return B=new st({}),O=new st({}),X=new st({}),R=new st({}),q=new it({props:{code:"nvidia-smi topo -m",highlighted:'<span class="hljs-symbol">nvidia</span>-<span class="hljs-keyword">smi</span> topo -m'}}),Y=new it({props:{code:`        GPU0    GPU1    CPU Affinity    NUMA Affinity
GPU0     X      NV2     0-23            N/A
GPU1    NV2      X      0-23            N/A`,highlighted:`        <span class="hljs-attribute">GPU0</span>    GPU1    CPU Affinity    NUMA Affinity
<span class="hljs-attribute">GPU0</span>     X      NV2     <span class="hljs-number">0</span>-<span class="hljs-number">23</span>            N/A
<span class="hljs-attribute">GPU1</span>    NV2      X      <span class="hljs-number">0</span>-<span class="hljs-number">23</span>            N/A`}}),z=new it({props:{code:`        GPU0    GPU1    CPU Affinity    NUMA Affinity
GPU0     X      PHB     0-11            N/A
GPU1    PHB      X      0-11            N/A`,highlighted:`        <span class="hljs-attribute">GPU0</span>    GPU1    CPU Affinity    NUMA Affinity
<span class="hljs-attribute">GPU0</span>     X      PHB     <span class="hljs-number">0</span>-<span class="hljs-number">11</span>            N/A
<span class="hljs-attribute">GPU1</span>    PHB      X      <span class="hljs-number">0</span>-<span class="hljs-number">11</span>            N/A`}}),F=new it({props:{code:`  X    = Self
  SYS  = Connection traversing PCIe as well as the SMP interconnect between NUMA nodes (e.g., QPI/UPI)
  NODE = Connection traversing PCIe as well as the interconnect between PCIe Host Bridges within a NUMA node
  PHB  = Connection traversing PCIe as well as a PCIe Host Bridge (typically the CPU)
  PXB  = Connection traversing multiple PCIe bridges (without traversing the PCIe Host Bridge)
  PIX  = Connection traversing at most a single PCIe bridge
  NV#  = Connection traversing a bonded set of # NVLinks`,highlighted:`  X    = Self
  SYS  = Connection traversing PCIe <span class="hljs-keyword">as</span> well <span class="hljs-keyword">as</span> <span class="hljs-keyword">the</span> SMP interconnect between NUMA nodes (e.g., QPI/UPI)
  NODE = Connection traversing PCIe <span class="hljs-keyword">as</span> well <span class="hljs-keyword">as</span> <span class="hljs-keyword">the</span> interconnect between PCIe Host Bridges <span class="hljs-keyword">within</span> <span class="hljs-keyword">a</span> NUMA node
  PHB  = Connection traversing PCIe <span class="hljs-keyword">as</span> well <span class="hljs-keyword">as</span> <span class="hljs-keyword">a</span> PCIe Host Bridge (typically <span class="hljs-keyword">the</span> CPU)
  PXB  = Connection traversing multiple PCIe bridges (<span class="hljs-keyword">without</span> traversing <span class="hljs-keyword">the</span> PCIe Host Bridge)
  PIX  = Connection traversing <span class="hljs-keyword">at</span> most <span class="hljs-keyword">a</span> single PCIe bridge
  NV<span class="hljs-comment">#  = Connection traversing a bonded set of # NVLinks</span>`}}),Q=new st({}),ne=new it({props:{code:`# DDP w/ NVLink

rm -r /tmp/test-clm; CUDA_VISIBLE_DEVICES=0,1 python -m torch.distributed.launch \\
--nproc_per_node 2 examples/pytorch/language-modeling/run_clm.py --model_name_or_path gpt2 \\
--dataset_name wikitext --dataset_config_name wikitext-2-raw-v1 --do_train \\
--output_dir /tmp/test-clm --per_device_train_batch_size 4 --max_steps 200

{'train_runtime': 101.9003, 'train_samples_per_second': 1.963, 'epoch': 0.69}

# DDP w/o NVLink

rm -r /tmp/test-clm; CUDA_VISIBLE_DEVICES=0,1 NCCL_P2P_DISABLE=1 python -m torch.distributed.launch \\
--nproc_per_node 2 examples/pytorch/language-modeling/run_clm.py --model_name_or_path gpt2 \\
--dataset_name wikitext --dataset_config_name wikitext-2-raw-v1 --do_train
--output_dir /tmp/test-clm --per_device_train_batch_size 4 --max_steps 200

{'train_runtime': 131.4367, 'train_samples_per_second': 1.522, 'epoch': 0.69}`,highlighted:`<span class="hljs-comment"># DDP w/ NVLink</span>

<span class="hljs-built_in">rm</span> -r /tmp/test-clm; CUDA_VISIBLE_DEVICES=0,1 python -m torch.distributed.launch \\
--nproc_per_node 2 examples/pytorch/language-modeling/run_clm.py --model_name_or_path gpt2 \\
--dataset_name wikitext --dataset_config_name wikitext-2-raw-v1 --do_train \\
--output_dir /tmp/test-clm --per_device_train_batch_size 4 --max_steps 200

{<span class="hljs-string">&#x27;train_runtime&#x27;</span>: 101.9003, <span class="hljs-string">&#x27;train_samples_per_second&#x27;</span>: 1.963, <span class="hljs-string">&#x27;epoch&#x27;</span>: 0.69}

<span class="hljs-comment"># DDP w/o NVLink</span>

<span class="hljs-built_in">rm</span> -r /tmp/test-clm; CUDA_VISIBLE_DEVICES=0,1 NCCL_P2P_DISABLE=1 python -m torch.distributed.launch \\
--nproc_per_node 2 examples/pytorch/language-modeling/run_clm.py --model_name_or_path gpt2 \\
--dataset_name wikitext --dataset_config_name wikitext-2-raw-v1 --do_train
--output_dir /tmp/test-clm --per_device_train_batch_size 4 --max_steps 200

{<span class="hljs-string">&#x27;train_runtime&#x27;</span>: 131.4367, <span class="hljs-string">&#x27;train_samples_per_second&#x27;</span>: 1.522, <span class="hljs-string">&#x27;epoch&#x27;</span>: 0.69}`}}),{c(){E=n("meta"),rt=h(),g=n("h1"),I=n("a"),Ae=n("span"),w(B.$$.fragment),pa=h(),Ve=n("span"),ha=l("Custom hardware for training"),lt=h(),$=n("p"),ca=l("The hardware you use to run model training and inference can have a big effect on performance. For a deep dive into GPUs make sure to check out Tim Dettmer\u2019s excellent "),H=n("a"),fa=l("blog post"),da=l("."),pt=h(),se=n("p"),ua=l("Let\u2019s have a look at some practical advice for GPU setups."),ht=h(),U=n("h2"),A=n("a"),Se=n("span"),w(O.$$.fragment),ma=h(),Le=n("span"),va=l("GPU"),ct=l(`

When you train bigger models you have essentially three options:
- bigger GPUs
- more GPUs
- more CPU and NVMe (offloaded to by [DeepSpeed-Infinity](main_classes/deepspeed#nvme-support))
`),ie=n("p"),wa=l("Let\u2019s start at the case where you have a single GPU."),ft=h(),C=n("h3"),V=n("a"),xe=n("span"),w(X.$$.fragment),Pa=h(),De=n("span"),_a=l("Power and Cooling"),dt=h(),re=n("p"),ya=l("If you bought an expensive high end GPU make sure you give it the correct power and sufficient cooling."),ut=h(),M=n("p"),je=n("strong"),ka=l("Power"),ba=l(":"),mt=h(),le=n("p"),Ea=l("Some high end consumer GPU cards have 2 and sometimes 3 PCI-E 8-Pin power sockets. Make sure you have as many independent 12V PCI-E 8-Pin cables plugged into the card as there are sockets. Do not use the 2 splits at one end of the same cable (also known as pigtail cable). That is if you have 2 sockets on the GPU, you want 2 PCI-E 8-Pin cables going from your PSU to the card and not one that has 2 PCI-E 8-Pin connectors at the end! You won\u2019t get the full performance out of your card otherwise."),vt=h(),pe=n("p"),ga=l("Each PCI-E 8-Pin power cable needs to be plugged into a 12V rail on the PSU side and can supply up to 150W of power."),wt=h(),he=n("p"),Ua=l("Some other cards may use a PCI-E 12-Pin connectors, and these can deliver up to 500-600W of power."),Pt=h(),ce=n("p"),Ca=l("Low end cards may use 6-Pin connectors, which supply up to 75W of power."),_t=h(),fe=n("p"),Na=l("Additionally you want the high-end PSU that has stable voltage. Some lower quality ones may not give the card the stable voltage it needs to function at its peak."),yt=h(),de=n("p"),Ga=l("And of course the PSU needs to have enough unused Watts to power the card."),kt=h(),W=n("p"),Te=n("strong"),Ia=l("Cooling"),$a=l(":"),bt=h(),ue=n("p"),Aa=l("When a GPU gets overheated it will start throttling down and will not deliver full performance and it can even shutdown if it gets too hot."),Et=h(),me=n("p"),Va=l("It\u2019s hard to tell the exact best temperature to strive for when a GPU is heavily loaded, but probably anything under +80C is good, but lower is better - perhaps 70-75C is an excellent range to be in. The throttling down is likely to start at around 84-90C. But other than throttling performance a prolonged very high temperature is likely to reduce the lifespan of a GPU."),gt=h(),ve=n("p"),Sa=l("Next let\u2019s have a look at one of the most important aspects when having multiple GPUs: connectivity."),Ut=h(),N=n("h3"),S=n("a"),Be=n("span"),w(R.$$.fragment),La=h(),He=n("span"),xa=l("Multi-GPU Connectivity"),Ct=h(),we=n("p"),Da=l("If you use multiple GPUs the way cards are inter-connected can have a huge impact on the total training time. If the GPUs are on the same physical node, you can run:"),Nt=h(),w(q.$$.fragment),Gt=h(),Pe=n("p"),ja=l("and it will tell you how the GPUs are inter-connected. On a machine with dual-GPU and which are connected with NVLink, you will most likely see something like:"),It=h(),w(Y.$$.fragment),$t=h(),_e=n("p"),Ta=l("on a different machine w/o NVLink we may see:"),At=h(),w(z.$$.fragment),Vt=h(),ye=n("p"),Ba=l("The report includes this legend:"),St=h(),w(F.$$.fragment),Lt=h(),v=n("p"),Ha=l("So the first report "),Oe=n("code"),Oa=l("NV2"),Xa=l(" tells us the GPUs are interconnected with 2 NVLinks, and the second report "),Xe=n("code"),Ma=l("PHB"),Wa=l(" we have a typical consumer-level PCIe+Bridge setup."),xt=h(),ke=n("p"),Ra=l("Check what type of connectivity you have on your setup. Some of these will make the communication between cards faster (e.g. NVLink), others slower (e.g. PHB)."),Dt=h(),be=n("p"),qa=l("Depending on the type of scalability solution used, the connectivity speed could have a major or a minor impact. If the GPUs need to sync rarely, as in DDP, the impact of a slower connection will be less significant. If the GPUs need to send messages to each other often, as in ZeRO-DP, then faster connectivity becomes super important to achieve faster training."),jt=h(),G=n("h4"),L=n("a"),Me=n("span"),w(Q.$$.fragment),Ya=h(),We=n("span"),za=l("NVlink"),Tt=h(),Z=n("p"),J=n("a"),Fa=l("NVLink"),Qa=l(" is a wire-based serial multi-lane near-range communications link developed by Nvidia."),Bt=h(),x=n("p"),Za=l("Each new generation provides a faster bandwidth, e.g. here is a quote from "),K=n("a"),Ja=l("Nvidia Ampere GA102 GPU Architecture"),Ka=l(":"),Ht=h(),Ee=n("blockquote"),Re=n("p"),eo=l(`Third-Generation NVLink\xAE
GA102 GPUs utilize NVIDIA\u2019s third-generation NVLink interface, which includes four x4 links,
with each link providing 14.0625 GB/sec bandwidth in each direction between two GPUs. Four
links provide 56.25 GB/sec bandwidth in each direction, and 112.5 GB/sec total bandwidth
between two GPUs. Two RTX 3090 GPUs can be connected together for SLI using NVLink.
(Note that 3-Way and 4-Way SLI configurations are not supported.)`),Ot=h(),u=n("p"),to=l("So the higher "),qe=n("code"),ao=l("X"),oo=l(" you get in the report of "),Ye=n("code"),no=l("NVX"),so=l(" in the output of "),ze=n("code"),io=l("nvidia-smi topo -m"),ro=l(" the better. The generation will depend on your GPU architecture."),Xt=h(),ge=n("p"),lo=l("Let\u2019s compare the execution of a gpt2 language model training over a small sample of wikitext."),Mt=h(),Ue=n("p"),po=l("The results are:"),Wt=h(),D=n("table"),Fe=n("thead"),ee=n("tr"),Qe=n("th"),ho=l("NVlink"),co=h(),Ce=n("th"),fo=l("Time"),uo=h(),te=n("tbody"),ae=n("tr"),Ze=n("td"),mo=l("Y"),vo=h(),Ne=n("td"),wo=l("101s"),Po=h(),oe=n("tr"),Je=n("td"),_o=l("N"),yo=h(),Ge=n("td"),ko=l("131s"),Rt=h(),j=n("p"),bo=l("You can see that NVLink completes the training ~23% faster. In the second benchmark we use "),Ke=n("code"),Eo=l("NCCL_P2P_DISABLE=1"),go=l(" to tell the GPUs not to use NVLink."),qt=h(),Ie=n("p"),Uo=l("Here is the full benchmark code and outputs:"),Yt=h(),w(ne.$$.fragment),zt=h(),d=n("p"),Co=l("Hardware: 2x TITAN RTX 24GB each + NVlink with 2 NVLinks ("),et=n("code"),No=l("NV2"),Go=l(" in "),tt=n("code"),Io=l("nvidia-smi topo -m"),$o=l(`)
Software: `),at=n("code"),Ao=l("pytorch-1.8-to-be"),Vo=l(" + "),ot=n("code"),So=l("cuda-11.0"),Lo=l(" / "),nt=n("code"),xo=l("transformers==4.3.0.dev0"),this.h()},l(e){const o=Zn('[data-svelte="svelte-1phssyn"]',document.head);E=s(o,"META",{name:!0,content:!0}),o.forEach(t),rt=c(e),g=s(e,"H1",{class:!0});var Qt=i(g);I=s(Qt,"A",{id:!0,class:!0,href:!0});var Ho=i(I);Ae=s(Ho,"SPAN",{});var Oo=i(Ae);P(B.$$.fragment,Oo),Oo.forEach(t),Ho.forEach(t),pa=c(Qt),Ve=s(Qt,"SPAN",{});var Xo=i(Ve);ha=p(Xo,"Custom hardware for training"),Xo.forEach(t),Qt.forEach(t),lt=c(e),$=s(e,"P",{});var Zt=i($);ca=p(Zt,"The hardware you use to run model training and inference can have a big effect on performance. For a deep dive into GPUs make sure to check out Tim Dettmer\u2019s excellent "),H=s(Zt,"A",{href:!0,rel:!0});var Mo=i(H);fa=p(Mo,"blog post"),Mo.forEach(t),da=p(Zt,"."),Zt.forEach(t),pt=c(e),se=s(e,"P",{});var Wo=i(se);ua=p(Wo,"Let\u2019s have a look at some practical advice for GPU setups."),Wo.forEach(t),ht=c(e),U=s(e,"H2",{class:!0});var Jt=i(U);A=s(Jt,"A",{id:!0,class:!0,href:!0});var Ro=i(A);Se=s(Ro,"SPAN",{});var qo=i(Se);P(O.$$.fragment,qo),qo.forEach(t),Ro.forEach(t),ma=c(Jt),Le=s(Jt,"SPAN",{});var Yo=i(Le);va=p(Yo,"GPU"),Yo.forEach(t),Jt.forEach(t),ct=p(e,`

When you train bigger models you have essentially three options:
- bigger GPUs
- more GPUs
- more CPU and NVMe (offloaded to by [DeepSpeed-Infinity](main_classes/deepspeed#nvme-support))
`),ie=s(e,"P",{});var zo=i(ie);wa=p(zo,"Let\u2019s start at the case where you have a single GPU."),zo.forEach(t),ft=c(e),C=s(e,"H3",{class:!0});var Kt=i(C);V=s(Kt,"A",{id:!0,class:!0,href:!0});var Fo=i(V);xe=s(Fo,"SPAN",{});var Qo=i(xe);P(X.$$.fragment,Qo),Qo.forEach(t),Fo.forEach(t),Pa=c(Kt),De=s(Kt,"SPAN",{});var Zo=i(De);_a=p(Zo,"Power and Cooling"),Zo.forEach(t),Kt.forEach(t),dt=c(e),re=s(e,"P",{});var Jo=i(re);ya=p(Jo,"If you bought an expensive high end GPU make sure you give it the correct power and sufficient cooling."),Jo.forEach(t),ut=c(e),M=s(e,"P",{});var Do=i(M);je=s(Do,"STRONG",{});var Ko=i(je);ka=p(Ko,"Power"),Ko.forEach(t),ba=p(Do,":"),Do.forEach(t),mt=c(e),le=s(e,"P",{});var en=i(le);Ea=p(en,"Some high end consumer GPU cards have 2 and sometimes 3 PCI-E 8-Pin power sockets. Make sure you have as many independent 12V PCI-E 8-Pin cables plugged into the card as there are sockets. Do not use the 2 splits at one end of the same cable (also known as pigtail cable). That is if you have 2 sockets on the GPU, you want 2 PCI-E 8-Pin cables going from your PSU to the card and not one that has 2 PCI-E 8-Pin connectors at the end! You won\u2019t get the full performance out of your card otherwise."),en.forEach(t),vt=c(e),pe=s(e,"P",{});var tn=i(pe);ga=p(tn,"Each PCI-E 8-Pin power cable needs to be plugged into a 12V rail on the PSU side and can supply up to 150W of power."),tn.forEach(t),wt=c(e),he=s(e,"P",{});var an=i(he);Ua=p(an,"Some other cards may use a PCI-E 12-Pin connectors, and these can deliver up to 500-600W of power."),an.forEach(t),Pt=c(e),ce=s(e,"P",{});var on=i(ce);Ca=p(on,"Low end cards may use 6-Pin connectors, which supply up to 75W of power."),on.forEach(t),_t=c(e),fe=s(e,"P",{});var nn=i(fe);Na=p(nn,"Additionally you want the high-end PSU that has stable voltage. Some lower quality ones may not give the card the stable voltage it needs to function at its peak."),nn.forEach(t),yt=c(e),de=s(e,"P",{});var sn=i(de);Ga=p(sn,"And of course the PSU needs to have enough unused Watts to power the card."),sn.forEach(t),kt=c(e),W=s(e,"P",{});var jo=i(W);Te=s(jo,"STRONG",{});var rn=i(Te);Ia=p(rn,"Cooling"),rn.forEach(t),$a=p(jo,":"),jo.forEach(t),bt=c(e),ue=s(e,"P",{});var ln=i(ue);Aa=p(ln,"When a GPU gets overheated it will start throttling down and will not deliver full performance and it can even shutdown if it gets too hot."),ln.forEach(t),Et=c(e),me=s(e,"P",{});var pn=i(me);Va=p(pn,"It\u2019s hard to tell the exact best temperature to strive for when a GPU is heavily loaded, but probably anything under +80C is good, but lower is better - perhaps 70-75C is an excellent range to be in. The throttling down is likely to start at around 84-90C. But other than throttling performance a prolonged very high temperature is likely to reduce the lifespan of a GPU."),pn.forEach(t),gt=c(e),ve=s(e,"P",{});var hn=i(ve);Sa=p(hn,"Next let\u2019s have a look at one of the most important aspects when having multiple GPUs: connectivity."),hn.forEach(t),Ut=c(e),N=s(e,"H3",{class:!0});var ea=i(N);S=s(ea,"A",{id:!0,class:!0,href:!0});var cn=i(S);Be=s(cn,"SPAN",{});var fn=i(Be);P(R.$$.fragment,fn),fn.forEach(t),cn.forEach(t),La=c(ea),He=s(ea,"SPAN",{});var dn=i(He);xa=p(dn,"Multi-GPU Connectivity"),dn.forEach(t),ea.forEach(t),Ct=c(e),we=s(e,"P",{});var un=i(we);Da=p(un,"If you use multiple GPUs the way cards are inter-connected can have a huge impact on the total training time. If the GPUs are on the same physical node, you can run:"),un.forEach(t),Nt=c(e),P(q.$$.fragment,e),Gt=c(e),Pe=s(e,"P",{});var mn=i(Pe);ja=p(mn,"and it will tell you how the GPUs are inter-connected. On a machine with dual-GPU and which are connected with NVLink, you will most likely see something like:"),mn.forEach(t),It=c(e),P(Y.$$.fragment,e),$t=c(e),_e=s(e,"P",{});var vn=i(_e);Ta=p(vn,"on a different machine w/o NVLink we may see:"),vn.forEach(t),At=c(e),P(z.$$.fragment,e),Vt=c(e),ye=s(e,"P",{});var wn=i(ye);Ba=p(wn,"The report includes this legend:"),wn.forEach(t),St=c(e),P(F.$$.fragment,e),Lt=c(e),v=s(e,"P",{});var $e=i(v);Ha=p($e,"So the first report "),Oe=s($e,"CODE",{});var Pn=i(Oe);Oa=p(Pn,"NV2"),Pn.forEach(t),Xa=p($e," tells us the GPUs are interconnected with 2 NVLinks, and the second report "),Xe=s($e,"CODE",{});var _n=i(Xe);Ma=p(_n,"PHB"),_n.forEach(t),Wa=p($e," we have a typical consumer-level PCIe+Bridge setup."),$e.forEach(t),xt=c(e),ke=s(e,"P",{});var yn=i(ke);Ra=p(yn,"Check what type of connectivity you have on your setup. Some of these will make the communication between cards faster (e.g. NVLink), others slower (e.g. PHB)."),yn.forEach(t),Dt=c(e),be=s(e,"P",{});var kn=i(be);qa=p(kn,"Depending on the type of scalability solution used, the connectivity speed could have a major or a minor impact. If the GPUs need to sync rarely, as in DDP, the impact of a slower connection will be less significant. If the GPUs need to send messages to each other often, as in ZeRO-DP, then faster connectivity becomes super important to achieve faster training."),kn.forEach(t),jt=c(e),G=s(e,"H4",{class:!0});var ta=i(G);L=s(ta,"A",{id:!0,class:!0,href:!0});var bn=i(L);Me=s(bn,"SPAN",{});var En=i(Me);P(Q.$$.fragment,En),En.forEach(t),bn.forEach(t),Ya=c(ta),We=s(ta,"SPAN",{});var gn=i(We);za=p(gn,"NVlink"),gn.forEach(t),ta.forEach(t),Tt=c(e),Z=s(e,"P",{});var To=i(Z);J=s(To,"A",{href:!0,rel:!0});var Un=i(J);Fa=p(Un,"NVLink"),Un.forEach(t),Qa=p(To," is a wire-based serial multi-lane near-range communications link developed by Nvidia."),To.forEach(t),Bt=c(e),x=s(e,"P",{});var aa=i(x);Za=p(aa,"Each new generation provides a faster bandwidth, e.g. here is a quote from "),K=s(aa,"A",{href:!0,rel:!0});var Cn=i(K);Ja=p(Cn,"Nvidia Ampere GA102 GPU Architecture"),Cn.forEach(t),Ka=p(aa,":"),aa.forEach(t),Ht=c(e),Ee=s(e,"BLOCKQUOTE",{});var Nn=i(Ee);Re=s(Nn,"P",{});var Gn=i(Re);eo=p(Gn,`Third-Generation NVLink\xAE
GA102 GPUs utilize NVIDIA\u2019s third-generation NVLink interface, which includes four x4 links,
with each link providing 14.0625 GB/sec bandwidth in each direction between two GPUs. Four
links provide 56.25 GB/sec bandwidth in each direction, and 112.5 GB/sec total bandwidth
between two GPUs. Two RTX 3090 GPUs can be connected together for SLI using NVLink.
(Note that 3-Way and 4-Way SLI configurations are not supported.)`),Gn.forEach(t),Nn.forEach(t),Ot=c(e),u=s(e,"P",{});var T=i(u);to=p(T,"So the higher "),qe=s(T,"CODE",{});var In=i(qe);ao=p(In,"X"),In.forEach(t),oo=p(T," you get in the report of "),Ye=s(T,"CODE",{});var $n=i(Ye);no=p($n,"NVX"),$n.forEach(t),so=p(T," in the output of "),ze=s(T,"CODE",{});var An=i(ze);io=p(An,"nvidia-smi topo -m"),An.forEach(t),ro=p(T," the better. The generation will depend on your GPU architecture."),T.forEach(t),Xt=c(e),ge=s(e,"P",{});var Vn=i(ge);lo=p(Vn,"Let\u2019s compare the execution of a gpt2 language model training over a small sample of wikitext."),Vn.forEach(t),Mt=c(e),Ue=s(e,"P",{});var Sn=i(Ue);po=p(Sn,"The results are:"),Sn.forEach(t),Wt=c(e),D=s(e,"TABLE",{});var oa=i(D);Fe=s(oa,"THEAD",{});var Ln=i(Fe);ee=s(Ln,"TR",{});var na=i(ee);Qe=s(na,"TH",{});var xn=i(Qe);ho=p(xn,"NVlink"),xn.forEach(t),co=c(na),Ce=s(na,"TH",{align:!0});var Dn=i(Ce);fo=p(Dn,"Time"),Dn.forEach(t),na.forEach(t),Ln.forEach(t),uo=c(oa),te=s(oa,"TBODY",{});var sa=i(te);ae=s(sa,"TR",{});var ia=i(ae);Ze=s(ia,"TD",{});var jn=i(Ze);mo=p(jn,"Y"),jn.forEach(t),vo=c(ia),Ne=s(ia,"TD",{align:!0});var Tn=i(Ne);wo=p(Tn,"101s"),Tn.forEach(t),ia.forEach(t),Po=c(sa),oe=s(sa,"TR",{});var ra=i(oe);Je=s(ra,"TD",{});var Bn=i(Je);_o=p(Bn,"N"),Bn.forEach(t),yo=c(ra),Ge=s(ra,"TD",{align:!0});var Hn=i(Ge);ko=p(Hn,"131s"),Hn.forEach(t),ra.forEach(t),sa.forEach(t),oa.forEach(t),Rt=c(e),j=s(e,"P",{});var la=i(j);bo=p(la,"You can see that NVLink completes the training ~23% faster. In the second benchmark we use "),Ke=s(la,"CODE",{});var On=i(Ke);Eo=p(On,"NCCL_P2P_DISABLE=1"),On.forEach(t),go=p(la," to tell the GPUs not to use NVLink."),la.forEach(t),qt=c(e),Ie=s(e,"P",{});var Xn=i(Ie);Uo=p(Xn,"Here is the full benchmark code and outputs:"),Xn.forEach(t),Yt=c(e),P(ne.$$.fragment,e),zt=c(e),d=s(e,"P",{});var m=i(d);Co=p(m,"Hardware: 2x TITAN RTX 24GB each + NVlink with 2 NVLinks ("),et=s(m,"CODE",{});var Mn=i(et);No=p(Mn,"NV2"),Mn.forEach(t),Go=p(m," in "),tt=s(m,"CODE",{});var Wn=i(tt);Io=p(Wn,"nvidia-smi topo -m"),Wn.forEach(t),$o=p(m,`)
Software: `),at=s(m,"CODE",{});var Rn=i(at);Ao=p(Rn,"pytorch-1.8-to-be"),Rn.forEach(t),Vo=p(m," + "),ot=s(m,"CODE",{});var qn=i(ot);So=p(qn,"cuda-11.0"),qn.forEach(t),Lo=p(m," / "),nt=s(m,"CODE",{});var Yn=i(nt);xo=p(Yn,"transformers==4.3.0.dev0"),Yn.forEach(t),m.forEach(t),this.h()},h(){f(E,"name","hf:doc:metadata"),f(E,"content",JSON.stringify(ts)),f(I,"id","custom-hardware-for-training"),f(I,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),f(I,"href","#custom-hardware-for-training"),f(g,"class","relative group"),f(H,"href","https://timdettmers.com/2020/09/07/which-gpu-for-deep-learning/"),f(H,"rel","nofollow"),f(A,"id","gpu"),f(A,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),f(A,"href","#gpu"),f(U,"class","relative group"),f(V,"id","power-and-cooling"),f(V,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),f(V,"href","#power-and-cooling"),f(C,"class","relative group"),f(S,"id","multigpu-connectivity"),f(S,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),f(S,"href","#multigpu-connectivity"),f(N,"class","relative group"),f(L,"id","nvlink"),f(L,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),f(L,"href","#nvlink"),f(G,"class","relative group"),f(J,"href","https://en.wikipedia.org/wiki/NVLink"),f(J,"rel","nofollow"),f(K,"href","https://www.nvidia.com/content/dam/en-zz/Solutions/geforce/ampere/pdf/NVIDIA-ampere-GA102-GPU-Architecture-Whitepaper-V1.pdf"),f(K,"rel","nofollow"),f(Ce,"align","right"),f(Ne,"align","right"),f(Ge,"align","right")},m(e,o){a(document.head,E),r(e,rt,o),r(e,g,o),a(g,I),a(I,Ae),_(B,Ae,null),a(g,pa),a(g,Ve),a(Ve,ha),r(e,lt,o),r(e,$,o),a($,ca),a($,H),a(H,fa),a($,da),r(e,pt,o),r(e,se,o),a(se,ua),r(e,ht,o),r(e,U,o),a(U,A),a(A,Se),_(O,Se,null),a(U,ma),a(U,Le),a(Le,va),r(e,ct,o),r(e,ie,o),a(ie,wa),r(e,ft,o),r(e,C,o),a(C,V),a(V,xe),_(X,xe,null),a(C,Pa),a(C,De),a(De,_a),r(e,dt,o),r(e,re,o),a(re,ya),r(e,ut,o),r(e,M,o),a(M,je),a(je,ka),a(M,ba),r(e,mt,o),r(e,le,o),a(le,Ea),r(e,vt,o),r(e,pe,o),a(pe,ga),r(e,wt,o),r(e,he,o),a(he,Ua),r(e,Pt,o),r(e,ce,o),a(ce,Ca),r(e,_t,o),r(e,fe,o),a(fe,Na),r(e,yt,o),r(e,de,o),a(de,Ga),r(e,kt,o),r(e,W,o),a(W,Te),a(Te,Ia),a(W,$a),r(e,bt,o),r(e,ue,o),a(ue,Aa),r(e,Et,o),r(e,me,o),a(me,Va),r(e,gt,o),r(e,ve,o),a(ve,Sa),r(e,Ut,o),r(e,N,o),a(N,S),a(S,Be),_(R,Be,null),a(N,La),a(N,He),a(He,xa),r(e,Ct,o),r(e,we,o),a(we,Da),r(e,Nt,o),_(q,e,o),r(e,Gt,o),r(e,Pe,o),a(Pe,ja),r(e,It,o),_(Y,e,o),r(e,$t,o),r(e,_e,o),a(_e,Ta),r(e,At,o),_(z,e,o),r(e,Vt,o),r(e,ye,o),a(ye,Ba),r(e,St,o),_(F,e,o),r(e,Lt,o),r(e,v,o),a(v,Ha),a(v,Oe),a(Oe,Oa),a(v,Xa),a(v,Xe),a(Xe,Ma),a(v,Wa),r(e,xt,o),r(e,ke,o),a(ke,Ra),r(e,Dt,o),r(e,be,o),a(be,qa),r(e,jt,o),r(e,G,o),a(G,L),a(L,Me),_(Q,Me,null),a(G,Ya),a(G,We),a(We,za),r(e,Tt,o),r(e,Z,o),a(Z,J),a(J,Fa),a(Z,Qa),r(e,Bt,o),r(e,x,o),a(x,Za),a(x,K),a(K,Ja),a(x,Ka),r(e,Ht,o),r(e,Ee,o),a(Ee,Re),a(Re,eo),r(e,Ot,o),r(e,u,o),a(u,to),a(u,qe),a(qe,ao),a(u,oo),a(u,Ye),a(Ye,no),a(u,so),a(u,ze),a(ze,io),a(u,ro),r(e,Xt,o),r(e,ge,o),a(ge,lo),r(e,Mt,o),r(e,Ue,o),a(Ue,po),r(e,Wt,o),r(e,D,o),a(D,Fe),a(Fe,ee),a(ee,Qe),a(Qe,ho),a(ee,co),a(ee,Ce),a(Ce,fo),a(D,uo),a(D,te),a(te,ae),a(ae,Ze),a(Ze,mo),a(ae,vo),a(ae,Ne),a(Ne,wo),a(te,Po),a(te,oe),a(oe,Je),a(Je,_o),a(oe,yo),a(oe,Ge),a(Ge,ko),r(e,Rt,o),r(e,j,o),a(j,bo),a(j,Ke),a(Ke,Eo),a(j,go),r(e,qt,o),r(e,Ie,o),a(Ie,Uo),r(e,Yt,o),_(ne,e,o),r(e,zt,o),r(e,d,o),a(d,Co),a(d,et),a(et,No),a(d,Go),a(d,tt),a(tt,Io),a(d,$o),a(d,at),a(at,Ao),a(d,Vo),a(d,ot),a(ot,So),a(d,Lo),a(d,nt),a(nt,xo),Ft=!0},p:Jn,i(e){Ft||(y(B.$$.fragment,e),y(O.$$.fragment,e),y(X.$$.fragment,e),y(R.$$.fragment,e),y(q.$$.fragment,e),y(Y.$$.fragment,e),y(z.$$.fragment,e),y(F.$$.fragment,e),y(Q.$$.fragment,e),y(ne.$$.fragment,e),Ft=!0)},o(e){k(B.$$.fragment,e),k(O.$$.fragment,e),k(X.$$.fragment,e),k(R.$$.fragment,e),k(q.$$.fragment,e),k(Y.$$.fragment,e),k(z.$$.fragment,e),k(F.$$.fragment,e),k(Q.$$.fragment,e),k(ne.$$.fragment,e),Ft=!1},d(e){t(E),e&&t(rt),e&&t(g),b(B),e&&t(lt),e&&t($),e&&t(pt),e&&t(se),e&&t(ht),e&&t(U),b(O),e&&t(ct),e&&t(ie),e&&t(ft),e&&t(C),b(X),e&&t(dt),e&&t(re),e&&t(ut),e&&t(M),e&&t(mt),e&&t(le),e&&t(vt),e&&t(pe),e&&t(wt),e&&t(he),e&&t(Pt),e&&t(ce),e&&t(_t),e&&t(fe),e&&t(yt),e&&t(de),e&&t(kt),e&&t(W),e&&t(bt),e&&t(ue),e&&t(Et),e&&t(me),e&&t(gt),e&&t(ve),e&&t(Ut),e&&t(N),b(R),e&&t(Ct),e&&t(we),e&&t(Nt),b(q,e),e&&t(Gt),e&&t(Pe),e&&t(It),b(Y,e),e&&t($t),e&&t(_e),e&&t(At),b(z,e),e&&t(Vt),e&&t(ye),e&&t(St),b(F,e),e&&t(Lt),e&&t(v),e&&t(xt),e&&t(ke),e&&t(Dt),e&&t(be),e&&t(jt),e&&t(G),b(Q),e&&t(Tt),e&&t(Z),e&&t(Bt),e&&t(x),e&&t(Ht),e&&t(Ee),e&&t(Ot),e&&t(u),e&&t(Xt),e&&t(ge),e&&t(Mt),e&&t(Ue),e&&t(Wt),e&&t(D),e&&t(Rt),e&&t(j),e&&t(qt),e&&t(Ie),e&&t(Yt),b(ne,e),e&&t(zt),e&&t(d)}}}const ts={local:"custom-hardware-for-training",sections:[{local:"gpu",sections:[{local:"power-and-cooling",title:"Power and Cooling"},{local:"multigpu-connectivity",sections:[{local:"nvlink",title:"NVlink"}],title:"Multi-GPU Connectivity"}],title:"GPU"}],title:"Custom hardware for training"};function as(Bo){return Kn(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class is extends zn{constructor(E){super();Fn(this,E,as,es,Qn,{})}}export{is as default,ts as metadata};
