import{S as x6,i as M6,s as $6,e as a,k as h,w as u,t as o,M as N6,c as s,d as t,m as f,a as i,x as m,h as r,b as c,N as ie,G as l,g as p,y as v,q as P,o as w,B as _,v as z6}from"../chunks/vendor-hf-doc-builder.js";import{T as C6}from"../chunks/Tip-hf-doc-builder.js";import{I as A}from"../chunks/IconCopyLink-hf-doc-builder.js";import{C as jp}from"../chunks/CodeBlock-hf-doc-builder.js";function R6(xp){let y,Ae,E,O,oe;return{c(){y=a("p"),Ae=o("Note: Most of the strategies introduced in the "),E=a("a"),O=o("single GPU section"),oe=o(" (such as mixed precision training or gradient accumulation) are generic and apply to training models in general so make sure to have a look at it before diving into the following sections such as multi-GPU or CPU training."),this.h()},l(S){y=s(S,"P",{});var R=i(y);Ae=r(R,"Note: Most of the strategies introduced in the "),E=s(R,"A",{href:!0});var Oe=i(E);O=r(Oe,"single GPU section"),Oe.forEach(t),oe=r(R," (such as mixed precision training or gradient accumulation) are generic and apply to training models in general so make sure to have a look at it before diving into the following sections such as multi-GPU or CPU training."),R.forEach(t),this.h()},h(){c(E,"href","perf_train_gpu_one")},m(S,R){p(S,y,R),l(y,Ae),l(y,E),l(E,O),l(y,oe)},d(S){S&&t(y)}}}function Z6(xp){let y,Ae,E,O,oe,S,R,Oe,au,Mp,va,su,$p,je,Np,Pa,iu,zp,re,xe,mo,gt,ou,vo,ru,Cp,wa,nu,Rp,L,_a,Po,pu,hu,fu,ya,wo,cu,du,uu,ba,_o,mu,vu,Pu,ga,yo,wu,_u,yu,Ea,bo,bu,gu,Zp,Da,Eu,Bp,ne,Me,go,Et,Du,Eo,Lu,Vp,La,Do,Tu,qp,Z,Dt,Lo,ku,Gu,Lt,To,Uu,Iu,ko,Su,Au,Tt,Go,Ou,ju,pe,Uo,Io,xu,Mu,So,Ao,$u,Nu,he,Oo,zu,Cu,jo,Ru,Zu,xo,Bu,Vu,kt,Mo,qu,Fu,Gt,$o,Wu,Hu,No,Yu,Fp,Ta,zo,Xu,Wp,$e,Ut,Co,Ku,Ju,It,Ro,Qu,em,Zo,tm,lm,St,Bo,am,sm,Vo,qo,im,Hp,fe,Ne,Fo,At,om,Wo,rm,Yp,j,nm,Ho,pm,hm,Yo,fm,cm,Ot,dm,um,Xp,ce,ze,Xo,jt,mm,Ko,vm,Kp,de,Jo,Pm,wm,Qo,_m,ym,Jp,Ce,er,bm,gm,tr,Em,Qp,ka,Dm,eh,xt,Mt,Lm,Tm,th,Re,lr,km,Gm,Ga,Um,$t,ar,Im,Sm,Nt,Am,sr,Om,jm,lh,zt,Ct,xm,Mm,ah,Ua,$m,sh,T,ir,Nm,zm,or,Cm,Rm,Rt,Zm,rr,Bm,Vm,qm,Ia,Fm,nr,Wm,Hm,pr,Ym,ih,Sa,Xm,oh,Ze,Km,Zt,Jm,Qm,rh,Aa,ev,nh,Oa,tv,ph,ja,lv,hh,Be,av,Bt,sv,iv,fh,xa,ov,ch,Ve,hr,ue,Ma,rv,nv,fr,pv,hv,$a,fv,cv,me,ve,Na,dv,uv,cr,mv,vv,za,Pv,wv,Pe,Ca,_v,yv,dr,bv,gv,Ra,Ev,Dv,we,Za,Lv,Tv,ur,kv,Gv,Ba,Uv,dh,Va,Iv,uh,qa,Sv,mh,Fa,Av,vh,Wa,Ov,Ph,Vt,mr,jv,xv,wh,qt,_h,D,Mv,vr,$v,Nv,Pr,zv,Cv,wr,Rv,Zv,_r,Bv,Vv,yr,qv,yh,_e,qe,br,Ft,Fv,gr,Wv,bh,ye,Hv,Wt,Yv,Xv,Ha,eg,gh,Fe,Kv,Er,Jv,Qv,Eh,Ya,eP,Dh,Ht,Lh,Xa,tP,Th,Ka,lP,kh,Yt,Gh,Ja,aP,Uh,Qa,sP,Ih,Xt,Sh,es,iP,Ah,ts,oP,Oh,ls,rP,jh,as,nP,xh,ss,pP,Mh,is,hP,$h,os,fP,Nh,rs,cP,zh,ns,dP,Ch,B,Dr,uP,mP,Lr,vP,PP,Tr,wP,Rh,ps,_P,Zh,hs,yP,Bh,fs,bP,Vh,cs,gP,qh,ds,EP,Fh,V,us,Kt,DP,LP,TP,ms,Jt,kP,GP,UP,kr,Qt,Gr,IP,SP,Wh,be,We,Ur,el,AP,Ir,OP,Hh,He,jP,Sr,xP,MP,Yh,vs,$P,Xh,tl,Kh,Ps,NP,Jh,ws,zP,Qh,_s,CP,ef,ys,RP,tf,Ye,Ar,ZP,BP,Or,VP,lf,bs,qP,af,Xe,FP,ll,WP,HP,sf,gs,Es,tg,of,Ds,YP,rf,Ls,XP,nf,q,KP,jr,JP,QP,xr,e1,t1,pf,Ke,l1,Mr,a1,s1,hf,Je,i1,$r,o1,r1,ff,F,n1,Nr,p1,h1,zr,f1,c1,cf,Ts,d1,df,W,u1,Cr,m1,v1,Rr,P1,w1,uf,x,_1,Zr,y1,b1,Br,g1,E1,Vr,D1,L1,mf,ks,T1,vf,Gs,k1,Pf,M,qr,G1,U1,Fr,I1,S1,Wr,A1,O1,Hr,j1,wf,Us,x1,_f,Qe,Yr,M1,$1,Xr,N1,yf,Is,z1,bf,$,al,C1,Kr,R1,Z1,B1,Ss,V1,sl,q1,F1,Jr,W1,H1,Qr,Y1,gf,As,X1,Ef,Os,K1,Df,b,il,ol,J1,Q1,rl,ew,tw,en,nl,lw,aw,tn,pl,sw,iw,js,hl,ow,rw,nw,ln,fl,pw,hw,xs,cl,fw,cw,dw,Ms,dl,uw,mw,Lf,et,vw,an,Pw,ww,Tf,$s,_w,kf,ge,yw,ul,bw,gw,Ns,lg,Gf,zs,Ew,Uf,Cs,Dw,If,tt,Lw,sn,Tw,kw,Sf,Ee,lt,on,ml,Gw,rn,Uw,Af,Rs,Iw,Of,H,Sw,vl,Aw,Ow,Pl,jw,xw,jf,Y,Mw,nn,$w,Nw,pn,zw,Cw,xf,k,Rw,hn,Zw,Bw,fn,Vw,qw,cn,Fw,Ww,dn,Hw,Yw,Mf,wl,Xw,Zs,ag,$f,d,Kw,un,Jw,Qw,mn,e2,t2,vn,l2,a2,Pn,s2,i2,wn,o2,r2,_n,n2,p2,yn,h2,f2,Bs,sg,Nf,_l,c2,Vs,ig,zf,yl,d2,qs,og,Cf,Fs,u2,Rf,X,m2,bl,v2,P2,gl,w2,_2,Zf,Ws,y2,Bf,Hs,b2,Vf,Ys,Xs,g2,El,E2,qf,Ks,D2,Ff,N,Js,Dl,L2,T2,k2,Qs,Ll,G2,U2,I2,ei,Tl,S2,A2,O2,ti,kl,j2,x2,Wf,li,M2,Hf,K,bn,$2,N2,Gl,z2,Ul,C2,R2,Z2,ai,B2,Il,V2,Yf,De,at,gn,Sl,q2,En,F2,Xf,st,W2,Al,H2,Y2,Kf,si,ii,rg,Jf,oi,X2,Qf,ri,K2,ec,ni,J2,tc,G,Dn,Ol,Q2,e_,Ln,jl,t_,l_,Tn,xl,a_,s_,kn,Ml,i_,o_,Gn,$l,r_,lc,pi,n_,ac,Le,it,Un,Nl,p_,In,h_,sc,hi,f_,ic,fi,ci,ng,oc,ot,c_,zl,d_,u_,rc,di,m_,nc,ui,v_,pc,U,mi,Cl,P_,w_,__,Sn,Rl,y_,b_,An,Zl,g_,E_,On,Bl,D_,L_,jn,Vl,T_,hc,vi,k_,fc,Te,rt,xn,ql,G_,Mn,U_,cc,nt,I_,Pi,S_,A_,dc,wi,O_,uc,_i,j_,mc,pt,x_,$n,M_,$_,vc,yi,N_,Pc,bi,z_,wc,gi,C_,_c,ht,ft,Fl,R_,Z_,Wl,B_,V_,q_,Nn,Hl,F_,yc,Ei,W_,bc,Di,zn,Yl,H_,gc,Li,Y_,Ec,ke,ct,Cn,Xl,X_,Rn,K_,Dc,Kl,Jl,J_,Q_,Lc,Ql,ey,ea,ty,Tc,Ti,ly,kc,z,Zn,ay,sy,Bn,iy,oy,Vn,ry,ny,qn,py,Gc,ki,hy,Uc,Gi,Fn,fy,Ic,Ui,cy,Sc,Ii,Wn,dy,Ac,Si,uy,Oc,Ai,Hn,my,jc,Oi,vy,xc,ji,Yn,Py,Mc,xi,wy,$c,Mi,$i,pg,Nc,Ni,_y,zc,zi,yy,Cc,Ci,by,Rc,dt,gy,ta,Ey,Dy,Zc,Ge,ut,Xn,la,Ly,Kn,Ty,Bc,Ri,ky,Vc,Zi,Jn,Gy,qc,J,aa,Qn,Uy,Iy,ep,tp,Sy,Ay,sa,lp,Oy,jy,ia,ap,xy,My,sp,$y,Ny,ip,op,zy,Fc,Bi,Ue,Cy,oa,Ry,Zy,rp,By,Vy,Wc,Vi,np,qy,Hc,Q,ra,pp,Fy,Wy,na,hp,Hy,Yy,fp,Xy,Ky,pa,cp,Jy,Qy,Ie,dp,up,eb,tb,mp,vp,lb,ab,Se,Pp,sb,ib,wp,ob,rb,_p,nb,pb,ha,yp,hb,fb,fa,bp,cb,db,gp,ub,Yc,qi,Ep,mb,Xc,mt,ca,Dp,vb,Pb,da,Lp,wb,_b,Tp,yb,bb,ua,kp,gb,Eb,Gp,Up,Db,Kc;return S=new A({}),je=new C6({props:{$$slots:{default:[R6]},$$scope:{ctx:xp}}}),gt=new A({}),Et=new A({}),At=new A({}),jt=new A({}),qt=new jp({props:{code:`
# DP
rm -r /tmp/test-clm; CUDA_VISIBLE_DEVICES=0,1 \\
python examples/pytorch/language-modeling/run_clm.py \\
--model_name_or_path gpt2 --dataset_name wikitext --dataset_config_name wikitext-2-raw-v1 \\
--do_train --output_dir /tmp/test-clm --per_device_train_batch_size 4 --max_steps 200

{'train_runtime': 110.5948, 'train_samples_per_second': 1.808, 'epoch': 0.69}

# DDP w/ NVlink
rm -r /tmp/test-clm; CUDA_VISIBLE_DEVICES=0,1 \\
python -m torch.distributed.launch --nproc_per_node 2 examples/pytorch/language-modeling/run_clm.py \\
--model_name_or_path gpt2 --dataset_name wikitext --dataset_config_name wikitext-2-raw-v1 \\
--do_train --output_dir /tmp/test-clm --per_device_train_batch_size 4 --max_steps 200

{'train_runtime': 101.9003, 'train_samples_per_second': 1.963, 'epoch': 0.69}

# DDP w/o NVlink
rm -r /tmp/test-clm; NCCL_P2P_DISABLE=1 CUDA_VISIBLE_DEVICES=0,1 \\
python -m torch.distributed.launch --nproc_per_node 2 examples/pytorch/language-modeling/run_clm.py \\
--model_name_or_path gpt2 --dataset_name wikitext --dataset_config_name wikitext-2-raw-v1 \\
--do_train --output_dir /tmp/test-clm --per_device_train_batch_size 4 --max_steps 200

{'train_runtime': 131.4367, 'train_samples_per_second': 1.522, 'epoch': 0.69}`,highlighted:`
<span class="hljs-comment"># DP</span>
<span class="hljs-string">rm</span> -<span class="hljs-string">r</span> /<span class="hljs-string">tmp</span>/<span class="hljs-string">test-clm</span>; <span class="hljs-string">CUDA_VISIBLE_DEVICES</span>=<span class="hljs-string">0</span>,<span class="hljs-string">1</span> \\
<span class="hljs-string">python</span> <span class="hljs-string">examples</span>/<span class="hljs-string">pytorch</span>/<span class="hljs-string">language-modeling</span>/<span class="hljs-string">run_clm</span>.<span class="hljs-string">py</span> \\
<span class="hljs-built_in">--model_name_or_path</span> <span class="hljs-string">gpt2</span> <span class="hljs-built_in">--dataset_name</span> <span class="hljs-string">wikitext</span> <span class="hljs-built_in">--dataset_config_name</span> <span class="hljs-string">wikitext-2-raw-v1</span> \\
<span class="hljs-built_in">--do_train</span> <span class="hljs-built_in">--output_dir</span> /<span class="hljs-string">tmp</span>/<span class="hljs-string">test-clm</span> <span class="hljs-built_in">--per_device_train_batch_size</span> <span class="hljs-string">4</span> <span class="hljs-built_in">--max_steps</span> <span class="hljs-string">200</span>

{<span class="hljs-string">&#x27;train_runtime&#x27;</span>: <span class="hljs-string">110</span>.<span class="hljs-string">5948</span>, <span class="hljs-string">&#x27;train_samples_per_second&#x27;</span>: <span class="hljs-string">1</span>.<span class="hljs-string">808</span>, <span class="hljs-string">&#x27;epoch&#x27;</span>: <span class="hljs-string">0</span>.<span class="hljs-string">69</span>}

<span class="hljs-comment"># DDP w/ NVlink</span>
<span class="hljs-string">rm</span> -<span class="hljs-string">r</span> /<span class="hljs-string">tmp</span>/<span class="hljs-string">test-clm</span>; <span class="hljs-string">CUDA_VISIBLE_DEVICES</span>=<span class="hljs-string">0</span>,<span class="hljs-string">1</span> \\
<span class="hljs-string">python</span> -<span class="hljs-string">m</span> <span class="hljs-string">torch</span>.<span class="hljs-string">distributed</span>.<span class="hljs-string">launch</span> <span class="hljs-built_in">--nproc_per_node</span> <span class="hljs-string">2</span> <span class="hljs-string">examples</span>/<span class="hljs-string">pytorch</span>/<span class="hljs-string">language-modeling</span>/<span class="hljs-string">run_clm</span>.<span class="hljs-string">py</span> \\
<span class="hljs-built_in">--model_name_or_path</span> <span class="hljs-string">gpt2</span> <span class="hljs-built_in">--dataset_name</span> <span class="hljs-string">wikitext</span> <span class="hljs-built_in">--dataset_config_name</span> <span class="hljs-string">wikitext-2-raw-v1</span> \\
<span class="hljs-built_in">--do_train</span> <span class="hljs-built_in">--output_dir</span> /<span class="hljs-string">tmp</span>/<span class="hljs-string">test-clm</span> <span class="hljs-built_in">--per_device_train_batch_size</span> <span class="hljs-string">4</span> <span class="hljs-built_in">--max_steps</span> <span class="hljs-string">200</span>

{<span class="hljs-string">&#x27;train_runtime&#x27;</span>: <span class="hljs-string">101</span>.<span class="hljs-string">9003</span>, <span class="hljs-string">&#x27;train_samples_per_second&#x27;</span>: <span class="hljs-string">1</span>.<span class="hljs-string">963</span>, <span class="hljs-string">&#x27;epoch&#x27;</span>: <span class="hljs-string">0</span>.<span class="hljs-string">69</span>}

<span class="hljs-comment"># DDP w/o NVlink</span>
<span class="hljs-string">rm</span> -<span class="hljs-string">r</span> /<span class="hljs-string">tmp</span>/<span class="hljs-string">test-clm</span>; <span class="hljs-string">NCCL_P2P_DISABLE</span>=<span class="hljs-string">1</span> <span class="hljs-string">CUDA_VISIBLE_DEVICES</span>=<span class="hljs-string">0</span>,<span class="hljs-string">1</span> \\
<span class="hljs-string">python</span> -<span class="hljs-string">m</span> <span class="hljs-string">torch</span>.<span class="hljs-string">distributed</span>.<span class="hljs-string">launch</span> <span class="hljs-built_in">--nproc_per_node</span> <span class="hljs-string">2</span> <span class="hljs-string">examples</span>/<span class="hljs-string">pytorch</span>/<span class="hljs-string">language-modeling</span>/<span class="hljs-string">run_clm</span>.<span class="hljs-string">py</span> \\
<span class="hljs-built_in">--model_name_or_path</span> <span class="hljs-string">gpt2</span> <span class="hljs-built_in">--dataset_name</span> <span class="hljs-string">wikitext</span> <span class="hljs-built_in">--dataset_config_name</span> <span class="hljs-string">wikitext-2-raw-v1</span> \\
<span class="hljs-built_in">--do_train</span> <span class="hljs-built_in">--output_dir</span> /<span class="hljs-string">tmp</span>/<span class="hljs-string">test-clm</span> <span class="hljs-built_in">--per_device_train_batch_size</span> <span class="hljs-string">4</span> <span class="hljs-built_in">--max_steps</span> <span class="hljs-string">200</span>

{<span class="hljs-string">&#x27;train_runtime&#x27;</span>: <span class="hljs-string">131</span>.<span class="hljs-string">4367</span>, <span class="hljs-string">&#x27;train_samples_per_second&#x27;</span>: <span class="hljs-string">1</span>.<span class="hljs-string">522</span>, <span class="hljs-string">&#x27;epoch&#x27;</span>: <span class="hljs-string">0</span>.<span class="hljs-string">69</span>}`}}),Ft=new A({}),Ht=new jp({props:{code:`La | Lb | Lc
---|----|---
a0 | b0 | c0
a1 | b1 | c1
a2 | b2 | c2`,highlighted:`La |<span class="hljs-string"> Lb </span>|<span class="hljs-string"> Lc
---</span>|<span class="hljs-string">----</span>|<span class="hljs-string">---
a0 </span>|<span class="hljs-string"> b0 </span>|<span class="hljs-string"> c0
a1 </span>|<span class="hljs-string"> b1 </span>|<span class="hljs-string"> c1
a2 </span>|<span class="hljs-string"> b2 </span>|<span class="hljs-string"> c2</span>`}}),Yt=new jp({props:{code:`GPU0:
La | Lb | Lc
---|----|---
a0 | b0 | c0

GPU1:
La | Lb | Lc
---|----|---
a1 | b1 | c1

GPU2:
La | Lb | Lc
---|----|---
a2 | b2 | c2`,highlighted:`GPU0:
La |<span class="hljs-string"> Lb </span>|<span class="hljs-string"> Lc
---</span>|<span class="hljs-string">----</span>|<span class="hljs-string">---
a0 </span>|<span class="hljs-string"> b0 </span>|<span class="hljs-string"> c0

GPU1:
La </span>|<span class="hljs-string"> Lb </span>|<span class="hljs-string"> Lc
---</span>|<span class="hljs-string">----</span>|<span class="hljs-string">---
a1 </span>|<span class="hljs-string"> b1 </span>|<span class="hljs-string"> c1

GPU2:
La </span>|<span class="hljs-string"> Lb </span>|<span class="hljs-string"> Lc
---</span>|<span class="hljs-string">----</span>|<span class="hljs-string">---
a2 </span>|<span class="hljs-string"> b2 </span>|<span class="hljs-string"> c2</span>`}}),Xt=new jp({props:{code:`x0 => GPU0
x1 => GPU1
x2 => GPU2`,highlighted:`<span class="hljs-attribute">x0</span> <span class="hljs-operator">=</span>&gt; GPU0
<span class="hljs-attribute">x1</span> <span class="hljs-operator">=</span>&gt; GPU1
<span class="hljs-attribute">x2</span> <span class="hljs-operator">=</span>&gt; GPU2`}}),el=new A({}),tl=new jp({props:{code:`===================  ===================
|  0 | 1 | 2 | 3  |  |  4 | 5 | 6 | 7  |
===================  ===================
        gpu0                 gpu1`,highlighted:`===================  ===================
|<span class="hljs-string">  0 </span>|<span class="hljs-string"> 1 </span>|<span class="hljs-string"> 2 </span>|<span class="hljs-string"> 3  </span>|<span class="hljs-string">  </span>|<span class="hljs-string">  4 </span>|<span class="hljs-string"> 5 </span>|<span class="hljs-string"> 6 </span>|<span class="hljs-string"> 7  </span>|
===================  ===================
        gpu0                 gpu1`}}),ml=new A({}),Sl=new A({}),Nl=new A({}),ql=new A({}),Xl=new A({}),la=new A({}),{c(){y=a("meta"),Ae=h(),E=a("h1"),O=a("a"),oe=a("span"),u(S.$$.fragment),R=h(),Oe=a("span"),au=o("Efficient Training on Multiple GPUs"),Mp=h(),va=a("p"),su=o("When training on a single GPU is too slow or the model weights don\u2019t fit in a single GPUs memory we use a mutli-GPU setup. Switching from a single GPU to multiple requires some form of parallelism as the work needs to be distributed. There are several techniques to achieve parallism such as data, tensor, or pipeline parallism. However, there is no one solution to fit them all and which settings works best depends on the hardware you are running on. While the main concepts most likely will apply to any other framework, this article is focused on PyTorch-based implementations."),$p=h(),u(je.$$.fragment),Np=h(),Pa=a("p"),iu=o("We will first discuss in depth various 1D parallelism techniques and their pros and cons and then look at how they can be combined into 2D and 3D parallelism to enable an even faster training and to support even bigger models. Various other powerful alternative approaches will be presented."),zp=h(),re=a("h2"),xe=a("a"),mo=a("span"),u(gt.$$.fragment),ou=h(),vo=a("span"),ru=o("Concepts"),Cp=h(),wa=a("p"),nu=o("The following is the brief description of the main concepts that will be described later in depth in this document."),Rp=h(),L=a("ol"),_a=a("li"),Po=a("strong"),pu=o("DataParallel (DP)"),hu=o(" - the same setup is replicated multiple times, and each being fed a slice of the data. The processing is done in parallel and all setups are synchronized at the end of each training step."),fu=h(),ya=a("li"),wo=a("strong"),cu=o("TensorParallel (TP)"),du=o(" - each tensor is split up into multiple chunks, so instead of having the whole tensor reside on a single gpu, each shard of the tensor resides on its designated gpu. During processing each shard gets processed separately and in parallel on different GPUs and the results are synced at the end of the step. This is what one may call horizontal parallelism, as the splitting happens on horizontal level."),uu=h(),ba=a("li"),_o=a("strong"),mu=o("PipelineParallel (PP)"),vu=o(" - the model is split up vertically (layer-level) across multiple GPUs, so that only one or several layers of the model are places on a single gpu. Each gpu processes in parallel different stages of the pipeline and working on a small chunk of the batch."),Pu=h(),ga=a("li"),yo=a("strong"),wu=o("Zero Redundancy Optimizer (ZeRO)"),_u=o(" - Also performs sharding of the tensors somewhat similar to TP, except the whole tensor gets reconstructed in time for a forward or backward computation, therefore the model doesn\u2019t need to be modified. It also supports various offloading techniques to compensate for limited GPU memory."),yu=h(),Ea=a("li"),bo=a("strong"),bu=o("Sharded DDP"),gu=o(" - is another name for the foundational ZeRO concept as used by various other implementations of ZeRO."),Zp=h(),Da=a("p"),Eu=o("Before diving deeper into the specifics of each concept we first have a look at the rough decision process when training large models on a large infrastructure."),Bp=h(),ne=a("h2"),Me=a("a"),go=a("span"),u(Et.$$.fragment),Du=h(),Eo=a("span"),Lu=o("Scalability Strategy"),Vp=h(),La=a("p"),Do=a("strong"),Tu=o("\u21E8 Single Node / Multi-GPU"),qp=h(),Z=a("ul"),Dt=a("li"),Lo=a("p"),ku=o("Model fits onto a single GPU:"),Gu=h(),Lt=a("ol"),To=a("li"),Uu=o("DDP - Distributed DP"),Iu=h(),ko=a("li"),Su=o("ZeRO - may or may not be faster depending on the situation and configuration used"),Au=h(),Tt=a("li"),Go=a("p"),Ou=o("Model doesn\u2019t fit onto a single GPU:"),ju=h(),pe=a("ol"),Uo=a("li"),Io=a("p"),xu=o("PP"),Mu=h(),So=a("li"),Ao=a("p"),$u=o("ZeRO"),Nu=h(),he=a("li"),Oo=a("p"),zu=o("TP"),Cu=h(),jo=a("p"),Ru=o("With very fast intra-node connectivity of NVLINK or NVSwitch all three should be mostly on par, without these PP will be faster than TP or ZeRO. The degree of TP may also make a difference. Best to experiment to find the winner on your particular setup."),Zu=h(),xo=a("p"),Bu=o("TP is almost always used within a single node. That is TP size <= gpus per node."),Vu=h(),kt=a("li"),Mo=a("p"),qu=o("Largest Layer not fitting into a single GPU:"),Fu=h(),Gt=a("ol"),$o=a("li"),Wu=o("If not using ZeRO - must use TP, as PP alone won\u2019t be able to fit."),Hu=h(),No=a("li"),Yu=o("With ZeRO see the same entry for \u201CSingle GPU\u201D above"),Fp=h(),Ta=a("p"),zo=a("strong"),Xu=o("\u21E8 Multi-Node / Multi-GPU"),Wp=h(),$e=a("ul"),Ut=a("li"),Co=a("p"),Ku=o("When you have fast inter-node connectivity:"),Ju=h(),It=a("ol"),Ro=a("li"),Qu=o("ZeRO - as it requires close to no modifications to the model"),em=h(),Zo=a("li"),tm=o("PP+TP+DP - less communications, but requires massive changes to the model"),lm=h(),St=a("li"),Bo=a("p"),am=o("when you have slow inter-node connectivity and still low on GPU memory:"),sm=h(),Vo=a("ol"),qo=a("li"),im=o("DP+PP+TP+ZeRO-1"),Hp=h(),fe=a("h2"),Ne=a("a"),Fo=a("span"),u(At.$$.fragment),om=h(),Wo=a("span"),rm=o("Data Parallelism"),Yp=h(),j=a("p"),nm=o("Most users with just 2 GPUs already enjoy the increased training speed up thanks to "),Ho=a("code"),pm=o("DataParallel"),hm=o(" (DP) and "),Yo=a("code"),fm=o("DistributedDataParallel"),cm=o(" (DDP) that are almost trivial to use. This is a built-in feature of Pytorch. Note that in general it is advised to use DDP as it is better maintained and works for all models while DP might fail for some models. "),Ot=a("a"),dm=o("PyTorch documentation"),um=o(" itself recommends the use of DDP."),Xp=h(),ce=a("h3"),ze=a("a"),Xo=a("span"),u(jt.$$.fragment),mm=h(),Ko=a("span"),vm=o("DP vs DDP"),Kp=h(),de=a("p"),Jo=a("code"),Pm=o("DistributedDataParallel"),wm=o(" (DDP) is typically faster than "),Qo=a("code"),_m=o("DataParallel"),ym=o(" (DP), but it is not always the case:"),Jp=h(),Ce=a("ul"),er=a("li"),bm=o("while DP is python threads-based, DDP is multiprocess-based - and as such it has no python threads limitations, such as GIL"),gm=h(),tr=a("li"),Em=o("on the other hand a slow inter-connectivity between the GPU cards could lead to an actual slower outcome with DDP"),Qp=h(),ka=a("p"),Dm=o("Here are the main differences in the inter-GPU communication overhead between the two modes:"),eh=h(),xt=a("p"),Mt=a("a"),Lm=o("DDP"),Tm=o(":"),th=h(),Re=a("ul"),lr=a("li"),km=o("At the start time the main process replicates the model once from gpu 0 to the rest of gpus"),Gm=h(),Ga=a("li"),Um=o("Then for each batch:"),$t=a("ol"),ar=a("li"),Im=o("each gpu consumes each own mini-batch of data directly"),Sm=h(),Nt=a("li"),Am=o("during "),sr=a("code"),Om=o("backward"),jm=o(", once the local gradients are ready, they are then averaged across all processes"),lh=h(),zt=a("p"),Ct=a("a"),xm=o("DP"),Mm=o(":"),ah=h(),Ua=a("p"),$m=o("For each batch:"),sh=h(),T=a("ol"),ir=a("li"),Nm=o("gpu 0 reads the batch of data and then sends a mini-batch to each gpu"),zm=h(),or=a("li"),Cm=o("replicates the up-to-date model from gpu 0 to each gpu"),Rm=h(),Rt=a("li"),Zm=o("runs "),rr=a("code"),Bm=o("forward"),Vm=o(" and sends output from each gpu to gpu 0, computes loss"),qm=h(),Ia=a("li"),Fm=o("scatters loss from gpu 0 to all gpus, runs "),nr=a("code"),Wm=o("backward"),Hm=h(),pr=a("li"),Ym=o("sends gradients from each gpu to gpu 0 and averages those"),ih=h(),Sa=a("p"),Xm=o("The only communication DDP performs per batch is sending gradients, whereas DP does 5 different data exchanges per batch."),oh=h(),Ze=a("p"),Km=o("DP copies data within the process via python threads, whereas DDP copies data via "),Zt=a("a"),Jm=o("torch.distributed"),Qm=o("."),rh=h(),Aa=a("p"),ev=o("Under DP gpu 0 performs a lot more work than the rest of the gpus, thus resulting in under-utilization of gpus."),nh=h(),Oa=a("p"),tv=o("You can use DDP across multiple machines, but this is not the case with DP."),ph=h(),ja=a("p"),lv=o("There are other differences between DP and DDP but they aren\u2019t relevant to this discussion."),hh=h(),Be=a("p"),av=o("If you want to go really deep into understanding these 2 modes, this "),Bt=a("a"),sv=o("article"),iv=o(" is highly recommended, as it has great diagrams, includes multiple benchmarks and profiler outputs on various hardware, explains all the nuances that you may need to know."),fh=h(),xa=a("p"),ov=o("Let\u2019s look at an actual benchmark:"),ch=h(),Ve=a("table"),hr=a("thead"),ue=a("tr"),Ma=a("th"),rv=o("Type"),nv=h(),fr=a("th"),pv=o("NVlink"),hv=h(),$a=a("th"),fv=o("Time"),cv=h(),me=a("tbody"),ve=a("tr"),Na=a("td"),dv=o("2:DP"),uv=h(),cr=a("td"),mv=o("Y"),vv=h(),za=a("td"),Pv=o("110s"),wv=h(),Pe=a("tr"),Ca=a("td"),_v=o("2:DDP"),yv=h(),dr=a("td"),bv=o("Y"),gv=h(),Ra=a("td"),Ev=o("101s"),Dv=h(),we=a("tr"),Za=a("td"),Lv=o("2:DDP"),Tv=h(),ur=a("td"),kv=o("N"),Gv=h(),Ba=a("td"),Uv=o("131s"),dh=h(),Va=a("p"),Iv=o("Analysis:"),uh=h(),qa=a("p"),Sv=o("Here DP is ~10% slower than DDP w/ NVlink, but ~15% faster than DDP w/o NVlink"),mh=h(),Fa=a("p"),Av=o("The real difference will depend on how much data each GPU needs to sync with the others - the more there is to sync, the more a slow link will slow down the total runtime."),vh=h(),Wa=a("p"),Ov=o("Here is the full benchmark code and outputs:"),Ph=h(),Vt=a("p"),mr=a("code"),jv=o("NCCL_P2P_DISABLE=1"),xv=o(" was used to disable the NVLink feature on the corresponding benchmark."),wh=h(),u(qt.$$.fragment),_h=h(),D=a("p"),Mv=o("Hardware: 2x TITAN RTX 24GB each + NVlink with 2 NVLinks ("),vr=a("code"),$v=o("NV2"),Nv=o(" in "),Pr=a("code"),zv=o("nvidia-smi topo -m"),Cv=o(`)
Software: `),wr=a("code"),Rv=o("pytorch-1.8-to-be"),Zv=o(" + "),_r=a("code"),Bv=o("cuda-11.0"),Vv=o(" / "),yr=a("code"),qv=o("transformers==4.3.0.dev0"),yh=h(),_e=a("h2"),qe=a("a"),br=a("span"),u(Ft.$$.fragment),Fv=h(),gr=a("span"),Wv=o("ZeRO Data Parallelism"),bh=h(),ye=a("p"),Hv=o("ZeRO-powered data parallelism (ZeRO-DP) is described on the following diagram from this "),Wt=a("a"),Yv=o("blog post"),Xv=h(),Ha=a("img"),gh=h(),Fe=a("p"),Kv=o("It can be difficult to wrap one\u2019s head around it, but in reality the concept is quite simple. This is just the usual "),Er=a("code"),Jv=o("DataParallel"),Qv=o(" (DP), except, instead of replicating the full model params, gradients and optimizer states, each GPU stores only a slice of it.  And then at run-time when the full layer params are needed just for the given layer, all GPUs synchronize to give each other parts that they miss - this is it."),Eh=h(),Ya=a("p"),eP=o("Consider this simple model with 3 layers, where each layer has 3 params:"),Dh=h(),u(Ht.$$.fragment),Lh=h(),Xa=a("p"),tP=o("Layer La has weights a0, a1 and a2."),Th=h(),Ka=a("p"),lP=o("If we have 3 GPUs, the Sharded DDP (= Zero-DP) splits the model onto 3 GPUs like so:"),kh=h(),u(Yt.$$.fragment),Gh=h(),Ja=a("p"),aP=o("In a way this is the same horizontal slicing, as tensor parallelism, if you imagine the typical DNN diagram. Vertical slicing is where one puts whole layer-groups on different GPUs. But it\u2019s just the starting point."),Uh=h(),Qa=a("p"),sP=o("Now each of these GPUs will get the usual mini-batch as it works in DP:"),Ih=h(),u(Xt.$$.fragment),Sh=h(),es=a("p"),iP=o("The inputs are unmodified - they think they are going to be processed by the normal model."),Ah=h(),ts=a("p"),oP=o("First, the inputs hit the layer La."),Oh=h(),ls=a("p"),rP=o("Let\u2019s focus just on GPU0: x0 needs a0, a1, a2 params to do its forward path, but GPU0 has only a0 - it gets sent a1 from GPU1 and a2 from GPU2, bringing all pieces of the model together."),jh=h(),as=a("p"),nP=o("In parallel, GPU1 gets mini-batch x1 and it only has a1, but needs a0 and a2 params, so it gets those from GPU0 and GPU2."),xh=h(),ss=a("p"),pP=o("Same happens to GPU2 that gets input x2. It gets a0 and a1 from GPU0 and GPU1, and with its a2 it reconstructs the full tensor."),Mh=h(),is=a("p"),hP=o("All 3 GPUs get the full tensors reconstructed and a forward happens."),$h=h(),os=a("p"),fP=o("As soon as the calculation is done, the data that is no longer needed gets dropped - it\u2019s only used during the calculation. The reconstruction is done efficiently via a pre-fetch."),Nh=h(),rs=a("p"),cP=o("And the whole process is repeated for layer Lb, then Lc forward-wise, and then backward Lc -> Lb -> La."),zh=h(),ns=a("p"),dP=o("To me this sounds like an efficient group backpacking weight distribution strategy:"),Ch=h(),B=a("ol"),Dr=a("li"),uP=o("person A carries the tent"),mP=h(),Lr=a("li"),vP=o("person B carries the stove"),PP=h(),Tr=a("li"),wP=o("person C carries the axe"),Rh=h(),ps=a("p"),_P=o("Now each night they all share what they have with others and get from others what they don\u2019t have, and in the morning they pack up their allocated type of gear and continue on their way. This is Sharded DDP / Zero DP."),Zh=h(),hs=a("p"),yP=o("Compare this strategy to the simple one where each person has to carry their own tent, stove and axe, which would be far more inefficient. This is DataParallel (DP and DDP) in Pytorch."),Bh=h(),fs=a("p"),bP=o("While reading the literature on this topic you may encounter the following synonyms: Sharded, Partitioned."),Vh=h(),cs=a("p"),gP=o("If you pay close attention the way ZeRO partitions the model\u2019s weights - it looks very similar to tensor parallelism which will be discussed later. This is because it partitions/shards each layer\u2019s weights, unlike vertical model parallelism which is discussed next."),qh=h(),ds=a("p"),EP=o("Implementations:"),Fh=h(),V=a("ul"),us=a("li"),Kt=a("a"),DP=o("DeepSpeed"),LP=o(" ZeRO-DP stages 1+2+3"),TP=h(),ms=a("li"),Jt=a("a"),kP=o("Fairscale"),GP=o(" ZeRO-DP stages 1+2+3"),UP=h(),kr=a("li"),Qt=a("a"),Gr=a("code"),IP=o("transformers"),SP=o(" integration"),Wh=h(),be=a("h2"),We=a("a"),Ur=a("span"),u(el.$$.fragment),AP=h(),Ir=a("span"),OP=o("Naive Model Parallelism (Vertical) and Pipeline Parallelism"),Hh=h(),He=a("p"),jP=o("Naive Model Parallelism (MP) is where one spreads groups of model layers across multiple GPUs. The mechanism is relatively simple - switch the desired layers "),Sr=a("code"),xP=o(".to()"),MP=o(" the desired devices and now whenever the data goes in and out those layers switch the data to the same device as the layer and leave the rest unmodified."),Yh=h(),vs=a("p"),$P=o("We refer to it as Vertical MP, because if you remember how most models are drawn, we slice the layers vertically. For example, if the following diagram shows an 8-layer model:"),Xh=h(),u(tl.$$.fragment),Kh=h(),Ps=a("p"),NP=o("we just sliced it in 2 vertically, placing layers 0-3 onto GPU0 and 4-7 to GPU1."),Jh=h(),ws=a("p"),zP=o("Now while data travels from layer 0 to 1, 1 to 2 and 2 to 3 this is just the normal model. But when data needs to pass from layer 3 to layer 4 it needs to travel from GPU0 to GPU1 which introduces a communication overhead. If the participating GPUs are on the same compute node (e.g. same physical machine) this copying is pretty fast, but if the GPUs are located on different compute nodes (e.g. multiple machines) the communication overhead could be significantly larger."),Qh=h(),_s=a("p"),CP=o("Then layers 4 to 5 to 6 to 7 are as a normal model would have and when the 7th layer completes we often need to send the data back to layer 0 where the labels are (or alternatively send the labels to the last layer). Now the loss can be computed and the optimizer can do its work."),ef=h(),ys=a("p"),RP=o("Problems:"),tf=h(),Ye=a("ul"),Ar=a("li"),ZP=o("the main deficiency and why this one is called \u201Cnaive\u201D MP, is that all but one GPU is idle at any given moment. So if 4 GPUs are used, it\u2019s almost identical to quadrupling the amount of memory of a single GPU, and ignoring the rest of the hardware. Plus there is the overhead of copying the data between devices. So 4x 6GB cards will be able to accommodate the same size as 1x 24GB card using naive MP, except the latter will complete the training faster, since it doesn\u2019t have the data copying overhead. But, say, if you have 40GB cards and need to fit a 45GB model you can with 4x 40GB cards (but barely because of the gradient and optimizer states)"),BP=h(),Or=a("li"),VP=o("shared embeddings may need to get copied back and forth between GPUs."),lf=h(),bs=a("p"),qP=o("Pipeline Parallelism (PP) is almost identical to a naive MP, but it solves the GPU idling problem, by chunking the incoming batch into micro-batches and artificially creating a pipeline, which allows different GPUs to concurrently participate in the computation process."),af=h(),Xe=a("p"),FP=o("The following illustration from the "),ll=a("a"),WP=o("GPipe paper"),HP=o(" shows the naive MP on the top, and PP on the bottom:"),sf=h(),gs=a("p"),Es=a("img"),of=h(),Ds=a("p"),YP=o("It\u2019s easy to see from the bottom diagram how PP has less dead zones, where GPUs are idle. The idle parts are referred to as the \u201Cbubble\u201D."),rf=h(),Ls=a("p"),XP=o("Both parts of the diagram show a parallelism that is of degree 4. That is 4 GPUs are participating in the pipeline. So there is the forward path of 4 pipe stages F0, F1, F2 and F3 and then the return reverse order backward path of B3, B2, B1 and B0."),nf=h(),q=a("p"),KP=o("PP introduces a new hyper-parameter to tune and it\u2019s "),jr=a("code"),JP=o("chunks"),QP=o(" which defines how many chunks of data are sent in a sequence through the same pipe stage. For example, in the bottomw diagram you can see that "),xr=a("code"),e1=o("chunks=4"),t1=o(". GPU0 performs the same forward path on chunk 0, 1, 2 and 3 (F0,0, F0,1, F0,2, F0,3) and then it waits for other GPUs to do their work and only when their work is starting to be complete, GPU0 starts to work again doing the backward path for chunks 3, 2, 1 and 0 (B0,3, B0,2, B0,1, B0,0)."),pf=h(),Ke=a("p"),l1=o("Note that conceptually this is the same concept as gradient accumulation steps (GAS). Pytorch uses "),Mr=a("code"),a1=o("chunks"),s1=o(", whereas DeepSpeed refers to the same hyper-parameter as GAS."),hf=h(),Je=a("p"),i1=o("Because of the chunks, PP introduces the concept of micro-batches (MBS). DP splits the global data batch size into mini-batches, so if you have a DP degree of 4, a global batch size of 1024 gets split up into 4 mini-batches of 256 each (1024/4). And if the number of "),$r=a("code"),o1=o("chunks"),r1=o(" (or GAS) is 32 we end up with a micro-batch size of 8 (256/32). Each Pipeline stage works with a single micro-batch at a time."),ff=h(),F=a("p"),n1=o("To calculate the global batch size of the DP + PP setup we then do: "),Nr=a("code"),p1=o("mbs*chunks*dp_degree"),h1=o(" ("),zr=a("code"),f1=o("8*32*4=1024"),c1=o(")."),cf=h(),Ts=a("p"),d1=o("Let\u2019s go back to the diagram."),df=h(),W=a("p"),u1=o("With "),Cr=a("code"),m1=o("chunks=1"),v1=o(" you end up with the naive MP, which is very inefficient. With a very large "),Rr=a("code"),P1=o("chunks"),w1=o(" value you end up with tiny micro-batch sizes which could be not every efficient either. So one has to experiment to find the value that leads to the highest efficient utilization of the gpus."),uf=h(),x=a("p"),_1=o("While the diagram shows that there is a bubble of \u201Cdead\u201D time that can\u2019t be parallelized because the last "),Zr=a("code"),y1=o("forward"),b1=o(" stage has to wait for "),Br=a("code"),g1=o("backward"),E1=o(" to complete the pipeline, the purpose of finding the best value for "),Vr=a("code"),D1=o("chunks"),L1=o(" is to enable a high concurrent GPU utilization across all participating GPUs which translates to minimizing the size of the bubble."),mf=h(),ks=a("p"),T1=o("There are 2 groups of solutions - the traditional Pipeline API and the more modern solutions that make things much easier for the end user."),vf=h(),Gs=a("p"),k1=o("Traditional Pipeline API solutions:"),Pf=h(),M=a("ul"),qr=a("li"),G1=o("PyTorch"),U1=h(),Fr=a("li"),I1=o("FairScale"),S1=h(),Wr=a("li"),A1=o("DeepSpeed"),O1=h(),Hr=a("li"),j1=o("Megatron-LM"),wf=h(),Us=a("p"),x1=o("Modern solutions:"),_f=h(),Qe=a("ul"),Yr=a("li"),M1=o("Varuna"),$1=h(),Xr=a("li"),N1=o("Sagemaker"),yf=h(),Is=a("p"),z1=o("Problems with traditional Pipeline API solutions:"),bf=h(),$=a("ul"),al=a("li"),C1=o("have to modify the model quite heavily, because Pipeline requires one to rewrite the normal flow of modules into a "),Kr=a("code"),R1=o("nn.Sequential"),Z1=o(" sequence of the same, which may require changes to the design of the model."),B1=h(),Ss=a("li"),V1=o("currently the Pipeline API is very restricted. If you had a bunch of python variables being passed in the very first stage of the Pipeline, you will have to find a way around it. Currently, the pipeline interface requires either a single Tensor or a tuple of Tensors as the only input and output. These tensors must have a batch size as the very first dimension, since pipeline is going to chunk the mini batch into micro-batches. Possible improvements are being discussed here "),sl=a("a"),q1=o("https://github.com/pytorch/pytorch/pull/50693"),F1=h(),Jr=a("li"),W1=o("conditional control flow at the level of pipe stages is not possible - e.g., Encoder-Decoder models like T5 require special workarounds to handle a conditional encoder stage."),H1=h(),Qr=a("li"),Y1=o("have to arrange each layer so that the output of one model becomes an input to the other model."),gf=h(),As=a("p"),X1=o("We are yet to experiment with Varuna and SageMaker but their papers report that they have overcome the list of problems mentioned above and that they require much smaller changes to the user\u2019s model."),Ef=h(),Os=a("p"),K1=o("Implementations:"),Df=h(),b=a("ul"),il=a("li"),ol=a("a"),J1=o("Pytorch"),Q1=o(" (initial support in pytorch-1.8, and progressively getting improved in 1.9 and more so in 1.10). Some "),rl=a("a"),ew=o("examples"),tw=h(),en=a("li"),nl=a("a"),lw=o("FairScale"),aw=h(),tn=a("li"),pl=a("a"),sw=o("DeepSpeed"),iw=h(),js=a("li"),hl=a("a"),ow=o("Megatron-LM"),rw=o(" has an internal implementation - no API."),nw=h(),ln=a("li"),fl=a("a"),pw=o("Varuna"),hw=h(),xs=a("li"),cl=a("a"),fw=o("SageMaker"),cw=o(" - this is a proprietary solution that can only be used on AWS."),dw=h(),Ms=a("li"),dl=a("a"),uw=o("OSLO"),mw=o(" - this is implemented based on the Hugging Face Transformers."),Lf=h(),et=a("p"),vw=o("\u{1F917} Transformers status: as of this writing none of the models supports full-PP. GPT2 and T5 models have naive MP support. The main obstacle is being unable to convert the models to "),an=a("code"),Pw=o("nn.Sequential"),ww=o(" and have all the inputs to be Tensors. This is because currently the models include many features that make the conversion very complicated, and will need to be removed to accomplish that."),Tf=h(),$s=a("p"),_w=o("Other approaches:"),kf=h(),ge=a("p"),yw=o("DeepSpeed, Varuna and SageMaker use the concept of an "),ul=a("a"),bw=o("Interleaved Pipeline"),gw=h(),Ns=a("img"),Gf=h(),zs=a("p"),Ew=o("Here the bubble (idle time) is further minimized by prioritizing backward passes."),Uf=h(),Cs=a("p"),Dw=o("Varuna further tries to improve the schedule by using simulations to discover the most efficient scheduling."),If=h(),tt=a("p"),Lw=o("OSLO has pipeline parallelism implementation based on the Transformers without "),sn=a("code"),Tw=o("nn.Sequential"),kw=o(" converting."),Sf=h(),Ee=a("h2"),lt=a("a"),on=a("span"),u(ml.$$.fragment),Gw=h(),rn=a("span"),Uw=o("Tensor Parallelism"),Af=h(),Rs=a("p"),Iw=o("In Tensor Parallelism each GPU processes only a slice of a tensor and only aggregates the full tensor for operations that require the whole thing."),Of=h(),H=a("p"),Sw=o("In this section we use concepts and diagrams from the "),vl=a("a"),Aw=o("Megatron-LM"),Ow=o(" paper: "),Pl=a("a"),jw=o("Efficient Large-Scale Language Model Training on GPU Clusters"),xw=o("."),jf=h(),Y=a("p"),Mw=o("The main building block of any transformer is a fully connected "),nn=a("code"),$w=o("nn.Linear"),Nw=o(" followed by a nonlinear activation "),pn=a("code"),zw=o("GeLU"),Cw=o("."),xf=h(),k=a("p"),Rw=o("Following the Megatron\u2019s paper notation, we can write the dot-product part of it as "),hn=a("code"),Zw=o("Y = GeLU(XA)"),Bw=o(", where "),fn=a("code"),Vw=o("X"),qw=o(" and "),cn=a("code"),Fw=o("Y"),Ww=o(" are the input and output vectors, and "),dn=a("code"),Hw=o("A"),Yw=o(" is the weight matrix."),Mf=h(),wl=a("p"),Xw=o(`If we look at the computation in matrix form, it\u2019s easy to see how the matrix multiplication can be split between multiple GPUs:
`),Zs=a("img"),$f=h(),d=a("p"),Kw=o("If we split the weight matrix "),un=a("code"),Jw=o("A"),Qw=o(" column-wise across "),mn=a("code"),e2=o("N"),t2=o(" GPUs and perform matrix multiplications "),vn=a("code"),l2=o("XA_1"),a2=o(" through "),Pn=a("code"),s2=o("XA_n"),i2=o(" in parallel, then we will end up with "),wn=a("code"),o2=o("N"),r2=o(" output vectors "),_n=a("code"),n2=o("Y_1, Y_2, ..., Y_n"),p2=o(" which can be fed into "),yn=a("code"),h2=o("GeLU"),f2=o(` independently:
`),Bs=a("img"),Nf=h(),_l=a("p"),c2=o(`Using this principle, we can update an MLP of arbitrary depth, without the need for any synchronization between GPUs until the very end, where we need to reconstruct the output vector from shards. The Megatron-LM paper authors provide a helpful illustration for that:
`),Vs=a("img"),zf=h(),yl=a("p"),d2=o(`Parallelizing the multi-headed attention layers is even simpler, since they are already inherently parallel, due to having multiple independent heads!
`),qs=a("img"),Cf=h(),Fs=a("p"),u2=o("Special considerations: TP requires very fast network, and therefore it\u2019s not advisable to do TP across more than one node. Practically, if a node has 4 GPUs, the highest TP degree is therefore 4. If you need a TP degree of 8, you need to use nodes that have at least 8 GPUs."),Rf=h(),X=a("p"),m2=o("This section is based on the original much more "),bl=a("a"),v2=o("detailed TP overview"),P2=o(`.
by `),gl=a("a"),w2=o("@anton-l"),_2=o("."),Zf=h(),Ws=a("p"),y2=o("SageMaker combines TP with DP for a more efficient processing."),Bf=h(),Hs=a("p"),b2=o("Alternative names:"),Vf=h(),Ys=a("ul"),Xs=a("li"),g2=o("DeepSpeed calls it "),El=a("a"),E2=o("tensor slicing"),qf=h(),Ks=a("p"),D2=o("Implementations:"),Ff=h(),N=a("ul"),Js=a("li"),Dl=a("a"),L2=o("Megatron-LM"),T2=o(" has an internal implementation, as it\u2019s very model-specific"),k2=h(),Qs=a("li"),Ll=a("a"),G2=o("parallelformers"),U2=o(" (only inference at the moment)"),I2=h(),ei=a("li"),Tl=a("a"),S2=o("SageMaker"),A2=o(" - this is a proprietary solution that can only be used on AWS."),O2=h(),ti=a("li"),kl=a("a"),j2=o("OSLO"),x2=o(" has the tensor parallelism implementation based on the Transformers."),Wf=h(),li=a("p"),M2=o("\u{1F917} Transformers status:"),Hf=h(),K=a("ul"),bn=a("li"),$2=o("core: not yet implemented in the core"),N2=h(),Gl=a("li"),z2=o("but if you want inference "),Ul=a("a"),C2=o("parallelformers"),R2=o(" provides this support for most of our models. So until this is implemented in the core you can use theirs. And hopefully training mode will be supported too."),Z2=h(),ai=a("li"),B2=o("Deepspeed-Inference also supports our BERT, GPT-2, and GPT-Neo models in their super-fast CUDA-kernel-based inference mode, see more "),Il=a("a"),V2=o("here"),Yf=h(),De=a("h2"),at=a("a"),gn=a("span"),u(Sl.$$.fragment),q2=h(),En=a("span"),F2=o("DP+PP"),Xf=h(),st=a("p"),W2=o("The following diagram from the DeepSpeed "),Al=a("a"),H2=o("pipeline tutorial"),Y2=o(" demonstrates how one combines DP with PP."),Kf=h(),si=a("p"),ii=a("img"),Jf=h(),oi=a("p"),X2=o("Here it\u2019s important to see how DP rank 0 doesn\u2019t see GPU2 and DP rank 1 doesn\u2019t see GPU3. To DP there is just GPUs 0 and 1 where it feeds data as if there were just 2 GPUs. GPU0 \u201Csecretly\u201D offloads some of its load to GPU2 using PP. And GPU1 does the same by enlisting GPU3 to its aid."),Qf=h(),ri=a("p"),K2=o("Since each dimension requires at least 2 GPUs, here you\u2019d need at least 4 GPUs."),ec=h(),ni=a("p"),J2=o("Implementations:"),tc=h(),G=a("ul"),Dn=a("li"),Ol=a("a"),Q2=o("DeepSpeed"),e_=h(),Ln=a("li"),jl=a("a"),t_=o("Megatron-LM"),l_=h(),Tn=a("li"),xl=a("a"),a_=o("Varuna"),s_=h(),kn=a("li"),Ml=a("a"),i_=o("SageMaker"),o_=h(),Gn=a("li"),$l=a("a"),r_=o("OSLO"),lc=h(),pi=a("p"),n_=o("\u{1F917} Transformers status: not yet implemented"),ac=h(),Le=a("h2"),it=a("a"),Un=a("span"),u(Nl.$$.fragment),p_=h(),In=a("span"),h_=o("DP+PP+TP"),sc=h(),hi=a("p"),f_=o("To get an even more efficient training a 3D parallelism is used where PP is combined with TP and DP. This can be seen in the following diagram."),ic=h(),fi=a("p"),ci=a("img"),oc=h(),ot=a("p"),c_=o("This diagram is from a blog post "),zl=a("a"),d_=o("3D parallelism: Scaling to trillion-parameter models"),u_=o(", which is a good read as well."),rc=h(),di=a("p"),m_=o("Since each dimension requires at least 2 GPUs, here you\u2019d need at least 8 GPUs."),nc=h(),ui=a("p"),v_=o("Implementations:"),pc=h(),U=a("ul"),mi=a("li"),Cl=a("a"),P_=o("DeepSpeed"),w_=o(" - DeepSpeed also includes an even more efficient DP, which they call ZeRO-DP."),__=h(),Sn=a("li"),Rl=a("a"),y_=o("Megatron-LM"),b_=h(),An=a("li"),Zl=a("a"),g_=o("Varuna"),E_=h(),On=a("li"),Bl=a("a"),D_=o("SageMaker"),L_=h(),jn=a("li"),Vl=a("a"),T_=o("OSLO"),hc=h(),vi=a("p"),k_=o("\u{1F917} Transformers status: not yet implemented, since we have no PP and TP."),fc=h(),Te=a("h2"),rt=a("a"),xn=a("span"),u(ql.$$.fragment),G_=h(),Mn=a("span"),U_=o("ZeRO DP+PP+TP"),cc=h(),nt=a("p"),I_=o("One of the main features of DeepSpeed is ZeRO, which is a super-scalable extension of DP. It has already been discussed in "),Pi=a("a"),S_=o("ZeRO Data Parallelism"),A_=o(". Normally it\u2019s a standalone feature that doesn\u2019t require PP or TP. But it can be combined with PP and TP."),dc=h(),wi=a("p"),O_=o("When ZeRO-DP is combined with PP (and optionally TP) it typically enables only ZeRO stage 1 (optimizer sharding)."),uc=h(),_i=a("p"),j_=o("While it\u2019s theoretically possible to use ZeRO stage 2 (gradient sharding) with Pipeline Parallelism, it will have bad performance impacts. There would need to be an additional reduce-scatter collective for every micro-batch to aggregate the gradients before sharding, which adds a potentially significant communication overhead. By nature of Pipeline Parallelism, small micro-batches are used and instead the focus is on trying to balance arithmetic intensity (micro-batch size) with minimizing the Pipeline bubble (number of micro-batches). Therefore those communication costs are going to hurt."),mc=h(),pt=a("p"),x_=o("In addition, There are already fewer layers than normal due to PP and so the memory savings won\u2019t be huge. PP already reduces gradient size by "),$n=a("code"),M_=o("1/PP"),$_=o(", and so gradient sharding savings on top of that are less significant than pure DP."),vc=h(),yi=a("p"),N_=o("ZeRO stage 3 is not a good choice either for the same reason - more inter-node communications required."),Pc=h(),bi=a("p"),z_=o("And since we have ZeRO, the other benefit is ZeRO-Offload. Since this is stage 1 optimizer states can be offloaded to CPU."),wc=h(),gi=a("p"),C_=o("Implementations:"),_c=h(),ht=a("ul"),ft=a("li"),Fl=a("a"),R_=o("Megatron-DeepSpeed"),Z_=o(" and "),Wl=a("a"),B_=o("Megatron-Deepspeed from BigScience"),V_=o(", which is the fork of the former repo."),q_=h(),Nn=a("li"),Hl=a("a"),F_=o("OSLO"),yc=h(),Ei=a("p"),W_=o("Important papers:"),bc=h(),Di=a("ul"),zn=a("li"),Yl=a("a"),H_=o("Using DeepSpeed and Megatron to Train Megatron-Turing NLG 530B, A Large-Scale Generative Language Model"),gc=h(),Li=a("p"),Y_=o("\u{1F917} Transformers status: not yet implemented, since we have no PP and TP."),Ec=h(),ke=a("h2"),ct=a("a"),Cn=a("span"),u(Xl.$$.fragment),X_=h(),Rn=a("span"),K_=o("FlexFlow"),Dc=h(),Kl=a("p"),Jl=a("a"),J_=o("FlexFlow"),Q_=o(" also solves the parallelization problem in a slightly different approach."),Lc=h(),Ql=a("p"),ey=o("Paper: "),ea=a("a"),ty=o("\u201CBeyond Data and Model Parallelism for Deep Neural Networks\u201D by Zhihao Jia, Matei Zaharia, Alex Aiken"),Tc=h(),Ti=a("p"),ly=o("It performs a sort of 4D Parallelism over Sample-Operator-Attribute-Parameter."),kc=h(),z=a("ol"),Zn=a("li"),ay=o("Sample = Data Parallelism (sample-wise parallel)"),sy=h(),Bn=a("li"),iy=o("Operator = Parallelize a single operation into several sub-operations"),oy=h(),Vn=a("li"),ry=o("Attribute = Data Parallelism (length-wise parallel)"),ny=h(),qn=a("li"),py=o("Parameter = Model Parallelism (regardless of dimension - horizontal or vertical)"),Gc=h(),ki=a("p"),hy=o("Examples:"),Uc=h(),Gi=a("ul"),Fn=a("li"),fy=o("Sample"),Ic=h(),Ui=a("p"),cy=o("Let\u2019s take 10 batches of sequence length 512. If we parallelize them by sample dimension into 2 devices, we get 10 x 512 which becomes be 5 x 2 x 512."),Sc=h(),Ii=a("ul"),Wn=a("li"),dy=o("Operator"),Ac=h(),Si=a("p"),uy=o("If we perform layer normalization, we compute std first and mean second, and then we can normalize data. Operator parallelism allows computing std and mean in parallel. So if we parallelize them by operator dimension into 2 devices (cuda:0, cuda:1), first we copy input data into both devices, and cuda:0 computes std, cuda:1 computes mean at the same time."),Oc=h(),Ai=a("ul"),Hn=a("li"),my=o("Attribute"),jc=h(),Oi=a("p"),vy=o("We have 10 batches of 512 length. If we parallelize them by attribute dimension into 2 devices, 10 x 512 will be 10 x 2 x 256."),xc=h(),ji=a("ul"),Yn=a("li"),Py=o("Parameter"),Mc=h(),xi=a("p"),wy=o("It is similar with tensor model parallelism or naive layer-wise model parallelism."),$c=h(),Mi=a("p"),$i=a("img"),Nc=h(),Ni=a("p"),_y=o("The significance of this framework is that it takes resources like (1) GPU/TPU/CPU vs. (2) RAM/DRAM vs. (3) fast-intra-connect/slow-inter-connect and it automatically optimizes all these  algorithmically deciding which parallelisation to use where."),zc=h(),zi=a("p"),yy=o("One very important aspect is that FlexFlow is designed for optimizing DNN parallelizations for models with static and fixed workloads, since models with dynamic behavior may prefer different parallelization strategies across iterations."),Cc=h(),Ci=a("p"),by=o("So the promise is very attractive - it runs a 30min simulation on the cluster of choice and it comes up with the best strategy to utilise this specific environment. If you add/remove/replace any parts it\u2019ll run and re-optimize the plan for that. And then you can train. A different setup will have its own custom optimization."),Rc=h(),dt=a("p"),gy=o("\u{1F917} Transformers status: not yet integrated. We already have our models FX-trace-able via "),ta=a("a"),Ey=o("transformers.utils.fx"),Dy=o(", which is a prerequisite for FlexFlow, so someone needs to figure out what needs to be done to make FlexFlow work with our models."),Zc=h(),Ge=a("h2"),ut=a("a"),Xn=a("span"),u(la.$$.fragment),Ly=h(),Kn=a("span"),Ty=o("Which Strategy To Use When"),Bc=h(),Ri=a("p"),ky=o("Here is a very rough outline at which parallelism strategy to use when. The first on each list is typically faster."),Vc=h(),Zi=a("p"),Jn=a("strong"),Gy=o("\u21E8 Single GPU"),qc=h(),J=a("ul"),aa=a("li"),Qn=a("p"),Uy=o("Model fits onto a single GPU:"),Iy=h(),ep=a("ol"),tp=a("li"),Sy=o("Normal use"),Ay=h(),sa=a("li"),lp=a("p"),Oy=o("Model doesn\u2019t fit onto a single GPU:"),jy=h(),ia=a("ol"),ap=a("li"),xy=o("ZeRO + Offload CPU and optionally NVMe"),My=h(),sp=a("li"),$y=o("as above plus Memory Centric Tiling (see below for details) if the largest layer can\u2019t fit into a single GPU"),Ny=h(),ip=a("li"),op=a("p"),zy=o("Largest Layer not fitting into a single GPU:"),Fc=h(),Bi=a("ol"),Ue=a("li"),Cy=o("ZeRO - Enable "),oa=a("a"),Ry=o("Memory Centric Tiling"),Zy=o(" (MCT). It allows you to run arbitrarily large layers by automatically splitting them and executing them sequentially. MCT reduces the number of parameters that are live on a GPU, but it does not affect the activation memory. As this need is very rare as of this writing a manual override of "),rp=a("code"),By=o("torch.nn.Linear"),Vy=o(" needs to be done by the user."),Wc=h(),Vi=a("p"),np=a("strong"),qy=o("\u21E8 Single Node / Multi-GPU"),Hc=h(),Q=a("ul"),ra=a("li"),pp=a("p"),Fy=o("Model fits onto a single GPU:"),Wy=h(),na=a("ol"),hp=a("li"),Hy=o("DDP - Distributed DP"),Yy=h(),fp=a("li"),Xy=o("ZeRO - may or may not be faster depending on the situation and configuration used"),Ky=h(),pa=a("li"),cp=a("p"),Jy=o("Model doesn\u2019t fit onto a single GPU:"),Qy=h(),Ie=a("ol"),dp=a("li"),up=a("p"),eb=o("PP"),tb=h(),mp=a("li"),vp=a("p"),lb=o("ZeRO"),ab=h(),Se=a("li"),Pp=a("p"),sb=o("TP"),ib=h(),wp=a("p"),ob=o("With very fast intra-node connectivity of NVLINK or NVSwitch all three should be mostly on par, without these PP will be faster than TP or ZeRO. The degree of TP may also make a difference. Best to experiment to find the winner on your particular setup."),rb=h(),_p=a("p"),nb=o("TP is almost always used within a single node. That is TP size <= gpus per node."),pb=h(),ha=a("li"),yp=a("p"),hb=o("Largest Layer not fitting into a single GPU:"),fb=h(),fa=a("ol"),bp=a("li"),cb=o("If not using ZeRO - must use TP, as PP alone won\u2019t be able to fit."),db=h(),gp=a("li"),ub=o("With ZeRO see the same entry for \u201CSingle GPU\u201D above"),Yc=h(),qi=a("p"),Ep=a("strong"),mb=o("\u21E8 Multi-Node / Multi-GPU"),Xc=h(),mt=a("ul"),ca=a("li"),Dp=a("p"),vb=o("When you have fast inter-node connectivity:"),Pb=h(),da=a("ol"),Lp=a("li"),wb=o("ZeRO - as it requires close to no modifications to the model"),_b=h(),Tp=a("li"),yb=o("PP+TP+DP - less communications, but requires massive changes to the model"),bb=h(),ua=a("li"),kp=a("p"),gb=o("when you have slow inter-node connectivity and still low on GPU memory:"),Eb=h(),Gp=a("ol"),Up=a("li"),Db=o("DP+PP+TP+ZeRO-1"),this.h()},l(e){const n=N6('[data-svelte="svelte-1phssyn"]',document.head);y=s(n,"META",{name:!0,content:!0}),n.forEach(t),Ae=f(e),E=s(e,"H1",{class:!0});var ma=i(E);O=s(ma,"A",{id:!0,class:!0,href:!0});var hg=i(O);oe=s(hg,"SPAN",{});var fg=i(oe);m(S.$$.fragment,fg),fg.forEach(t),hg.forEach(t),R=f(ma),Oe=s(ma,"SPAN",{});var cg=i(Oe);au=r(cg,"Efficient Training on Multiple GPUs"),cg.forEach(t),ma.forEach(t),Mp=f(e),va=s(e,"P",{});var dg=i(va);su=r(dg,"When training on a single GPU is too slow or the model weights don\u2019t fit in a single GPUs memory we use a mutli-GPU setup. Switching from a single GPU to multiple requires some form of parallelism as the work needs to be distributed. There are several techniques to achieve parallism such as data, tensor, or pipeline parallism. However, there is no one solution to fit them all and which settings works best depends on the hardware you are running on. While the main concepts most likely will apply to any other framework, this article is focused on PyTorch-based implementations."),dg.forEach(t),$p=f(e),m(je.$$.fragment,e),Np=f(e),Pa=s(e,"P",{});var ug=i(Pa);iu=r(ug,"We will first discuss in depth various 1D parallelism techniques and their pros and cons and then look at how they can be combined into 2D and 3D parallelism to enable an even faster training and to support even bigger models. Various other powerful alternative approaches will be presented."),ug.forEach(t),zp=f(e),re=s(e,"H2",{class:!0});var Jc=i(re);xe=s(Jc,"A",{id:!0,class:!0,href:!0});var mg=i(xe);mo=s(mg,"SPAN",{});var vg=i(mo);m(gt.$$.fragment,vg),vg.forEach(t),mg.forEach(t),ou=f(Jc),vo=s(Jc,"SPAN",{});var Pg=i(vo);ru=r(Pg,"Concepts"),Pg.forEach(t),Jc.forEach(t),Cp=f(e),wa=s(e,"P",{});var wg=i(wa);nu=r(wg,"The following is the brief description of the main concepts that will be described later in depth in this document."),wg.forEach(t),Rp=f(e),L=s(e,"OL",{});var ee=i(L);_a=s(ee,"LI",{});var Lb=i(_a);Po=s(Lb,"STRONG",{});var _g=i(Po);pu=r(_g,"DataParallel (DP)"),_g.forEach(t),hu=r(Lb," - the same setup is replicated multiple times, and each being fed a slice of the data. The processing is done in parallel and all setups are synchronized at the end of each training step."),Lb.forEach(t),fu=f(ee),ya=s(ee,"LI",{});var Tb=i(ya);wo=s(Tb,"STRONG",{});var yg=i(wo);cu=r(yg,"TensorParallel (TP)"),yg.forEach(t),du=r(Tb," - each tensor is split up into multiple chunks, so instead of having the whole tensor reside on a single gpu, each shard of the tensor resides on its designated gpu. During processing each shard gets processed separately and in parallel on different GPUs and the results are synced at the end of the step. This is what one may call horizontal parallelism, as the splitting happens on horizontal level."),Tb.forEach(t),uu=f(ee),ba=s(ee,"LI",{});var kb=i(ba);_o=s(kb,"STRONG",{});var bg=i(_o);mu=r(bg,"PipelineParallel (PP)"),bg.forEach(t),vu=r(kb," - the model is split up vertically (layer-level) across multiple GPUs, so that only one or several layers of the model are places on a single gpu. Each gpu processes in parallel different stages of the pipeline and working on a small chunk of the batch."),kb.forEach(t),Pu=f(ee),ga=s(ee,"LI",{});var Gb=i(ga);yo=s(Gb,"STRONG",{});var gg=i(yo);wu=r(gg,"Zero Redundancy Optimizer (ZeRO)"),gg.forEach(t),_u=r(Gb," - Also performs sharding of the tensors somewhat similar to TP, except the whole tensor gets reconstructed in time for a forward or backward computation, therefore the model doesn\u2019t need to be modified. It also supports various offloading techniques to compensate for limited GPU memory."),Gb.forEach(t),yu=f(ee),Ea=s(ee,"LI",{});var Ub=i(Ea);bo=s(Ub,"STRONG",{});var Eg=i(bo);bu=r(Eg,"Sharded DDP"),Eg.forEach(t),gu=r(Ub," - is another name for the foundational ZeRO concept as used by various other implementations of ZeRO."),Ub.forEach(t),ee.forEach(t),Zp=f(e),Da=s(e,"P",{});var Dg=i(Da);Eu=r(Dg,"Before diving deeper into the specifics of each concept we first have a look at the rough decision process when training large models on a large infrastructure."),Dg.forEach(t),Bp=f(e),ne=s(e,"H2",{class:!0});var Qc=i(ne);Me=s(Qc,"A",{id:!0,class:!0,href:!0});var Lg=i(Me);go=s(Lg,"SPAN",{});var Tg=i(go);m(Et.$$.fragment,Tg),Tg.forEach(t),Lg.forEach(t),Du=f(Qc),Eo=s(Qc,"SPAN",{});var kg=i(Eo);Lu=r(kg,"Scalability Strategy"),kg.forEach(t),Qc.forEach(t),Vp=f(e),La=s(e,"P",{});var Gg=i(La);Do=s(Gg,"STRONG",{});var Ug=i(Do);Tu=r(Ug,"\u21E8 Single Node / Multi-GPU"),Ug.forEach(t),Gg.forEach(t),qp=f(e),Z=s(e,"UL",{});var Fi=i(Z);Dt=s(Fi,"LI",{});var ed=i(Dt);Lo=s(ed,"P",{});var Ig=i(Lo);ku=r(Ig,"Model fits onto a single GPU:"),Ig.forEach(t),Gu=f(ed),Lt=s(ed,"OL",{});var td=i(Lt);To=s(td,"LI",{});var Sg=i(To);Uu=r(Sg,"DDP - Distributed DP"),Sg.forEach(t),Iu=f(td),ko=s(td,"LI",{});var Ag=i(ko);Su=r(Ag,"ZeRO - may or may not be faster depending on the situation and configuration used"),Ag.forEach(t),td.forEach(t),ed.forEach(t),Au=f(Fi),Tt=s(Fi,"LI",{});var ld=i(Tt);Go=s(ld,"P",{});var Og=i(Go);Ou=r(Og,"Model doesn\u2019t fit onto a single GPU:"),Og.forEach(t),ju=f(ld),pe=s(ld,"OL",{});var Wi=i(pe);Uo=s(Wi,"LI",{});var jg=i(Uo);Io=s(jg,"P",{});var xg=i(Io);xu=r(xg,"PP"),xg.forEach(t),jg.forEach(t),Mu=f(Wi),So=s(Wi,"LI",{});var Mg=i(So);Ao=s(Mg,"P",{});var $g=i(Ao);$u=r($g,"ZeRO"),$g.forEach(t),Mg.forEach(t),Nu=f(Wi),he=s(Wi,"LI",{});var Hi=i(he);Oo=s(Hi,"P",{});var Ng=i(Oo);zu=r(Ng,"TP"),Ng.forEach(t),Cu=f(Hi),jo=s(Hi,"P",{});var zg=i(jo);Ru=r(zg,"With very fast intra-node connectivity of NVLINK or NVSwitch all three should be mostly on par, without these PP will be faster than TP or ZeRO. The degree of TP may also make a difference. Best to experiment to find the winner on your particular setup."),zg.forEach(t),Zu=f(Hi),xo=s(Hi,"P",{});var Cg=i(xo);Bu=r(Cg,"TP is almost always used within a single node. That is TP size <= gpus per node."),Cg.forEach(t),Hi.forEach(t),Wi.forEach(t),ld.forEach(t),Vu=f(Fi),kt=s(Fi,"LI",{});var ad=i(kt);Mo=s(ad,"P",{});var Rg=i(Mo);qu=r(Rg,"Largest Layer not fitting into a single GPU:"),Rg.forEach(t),Fu=f(ad),Gt=s(ad,"OL",{});var sd=i(Gt);$o=s(sd,"LI",{});var Zg=i($o);Wu=r(Zg,"If not using ZeRO - must use TP, as PP alone won\u2019t be able to fit."),Zg.forEach(t),Hu=f(sd),No=s(sd,"LI",{});var Bg=i(No);Yu=r(Bg,"With ZeRO see the same entry for \u201CSingle GPU\u201D above"),Bg.forEach(t),sd.forEach(t),ad.forEach(t),Fi.forEach(t),Fp=f(e),Ta=s(e,"P",{});var Vg=i(Ta);zo=s(Vg,"STRONG",{});var qg=i(zo);Xu=r(qg,"\u21E8 Multi-Node / Multi-GPU"),qg.forEach(t),Vg.forEach(t),Wp=f(e),$e=s(e,"UL",{});var id=i($e);Ut=s(id,"LI",{});var od=i(Ut);Co=s(od,"P",{});var Fg=i(Co);Ku=r(Fg,"When you have fast inter-node connectivity:"),Fg.forEach(t),Ju=f(od),It=s(od,"OL",{});var rd=i(It);Ro=s(rd,"LI",{});var Wg=i(Ro);Qu=r(Wg,"ZeRO - as it requires close to no modifications to the model"),Wg.forEach(t),em=f(rd),Zo=s(rd,"LI",{});var Hg=i(Zo);tm=r(Hg,"PP+TP+DP - less communications, but requires massive changes to the model"),Hg.forEach(t),rd.forEach(t),od.forEach(t),lm=f(id),St=s(id,"LI",{});var nd=i(St);Bo=s(nd,"P",{});var Yg=i(Bo);am=r(Yg,"when you have slow inter-node connectivity and still low on GPU memory:"),Yg.forEach(t),sm=f(nd),Vo=s(nd,"OL",{});var Xg=i(Vo);qo=s(Xg,"LI",{});var Kg=i(qo);im=r(Kg,"DP+PP+TP+ZeRO-1"),Kg.forEach(t),Xg.forEach(t),nd.forEach(t),id.forEach(t),Hp=f(e),fe=s(e,"H2",{class:!0});var pd=i(fe);Ne=s(pd,"A",{id:!0,class:!0,href:!0});var Jg=i(Ne);Fo=s(Jg,"SPAN",{});var Qg=i(Fo);m(At.$$.fragment,Qg),Qg.forEach(t),Jg.forEach(t),om=f(pd),Wo=s(pd,"SPAN",{});var eE=i(Wo);rm=r(eE,"Data Parallelism"),eE.forEach(t),pd.forEach(t),Yp=f(e),j=s(e,"P",{});var vt=i(j);nm=r(vt,"Most users with just 2 GPUs already enjoy the increased training speed up thanks to "),Ho=s(vt,"CODE",{});var tE=i(Ho);pm=r(tE,"DataParallel"),tE.forEach(t),hm=r(vt," (DP) and "),Yo=s(vt,"CODE",{});var lE=i(Yo);fm=r(lE,"DistributedDataParallel"),lE.forEach(t),cm=r(vt," (DDP) that are almost trivial to use. This is a built-in feature of Pytorch. Note that in general it is advised to use DDP as it is better maintained and works for all models while DP might fail for some models. "),Ot=s(vt,"A",{href:!0,rel:!0});var aE=i(Ot);dm=r(aE,"PyTorch documentation"),aE.forEach(t),um=r(vt," itself recommends the use of DDP."),vt.forEach(t),Xp=f(e),ce=s(e,"H3",{class:!0});var hd=i(ce);ze=s(hd,"A",{id:!0,class:!0,href:!0});var sE=i(ze);Xo=s(sE,"SPAN",{});var iE=i(Xo);m(jt.$$.fragment,iE),iE.forEach(t),sE.forEach(t),mm=f(hd),Ko=s(hd,"SPAN",{});var oE=i(Ko);vm=r(oE,"DP vs DDP"),oE.forEach(t),hd.forEach(t),Kp=f(e),de=s(e,"P",{});var Ip=i(de);Jo=s(Ip,"CODE",{});var rE=i(Jo);Pm=r(rE,"DistributedDataParallel"),rE.forEach(t),wm=r(Ip," (DDP) is typically faster than "),Qo=s(Ip,"CODE",{});var nE=i(Qo);_m=r(nE,"DataParallel"),nE.forEach(t),ym=r(Ip," (DP), but it is not always the case:"),Ip.forEach(t),Jp=f(e),Ce=s(e,"UL",{});var fd=i(Ce);er=s(fd,"LI",{});var pE=i(er);bm=r(pE,"while DP is python threads-based, DDP is multiprocess-based - and as such it has no python threads limitations, such as GIL"),pE.forEach(t),gm=f(fd),tr=s(fd,"LI",{});var hE=i(tr);Em=r(hE,"on the other hand a slow inter-connectivity between the GPU cards could lead to an actual slower outcome with DDP"),hE.forEach(t),fd.forEach(t),Qp=f(e),ka=s(e,"P",{});var fE=i(ka);Dm=r(fE,"Here are the main differences in the inter-GPU communication overhead between the two modes:"),fE.forEach(t),eh=f(e),xt=s(e,"P",{});var Ib=i(xt);Mt=s(Ib,"A",{href:!0,rel:!0});var cE=i(Mt);Lm=r(cE,"DDP"),cE.forEach(t),Tm=r(Ib,":"),Ib.forEach(t),th=f(e),Re=s(e,"UL",{});var cd=i(Re);lr=s(cd,"LI",{});var dE=i(lr);km=r(dE,"At the start time the main process replicates the model once from gpu 0 to the rest of gpus"),dE.forEach(t),Gm=f(cd),Ga=s(cd,"LI",{});var Sb=i(Ga);Um=r(Sb,"Then for each batch:"),$t=s(Sb,"OL",{});var dd=i($t);ar=s(dd,"LI",{});var uE=i(ar);Im=r(uE,"each gpu consumes each own mini-batch of data directly"),uE.forEach(t),Sm=f(dd),Nt=s(dd,"LI",{});var ud=i(Nt);Am=r(ud,"during "),sr=s(ud,"CODE",{});var mE=i(sr);Om=r(mE,"backward"),mE.forEach(t),jm=r(ud,", once the local gradients are ready, they are then averaged across all processes"),ud.forEach(t),dd.forEach(t),Sb.forEach(t),cd.forEach(t),lh=f(e),zt=s(e,"P",{});var Ab=i(zt);Ct=s(Ab,"A",{href:!0,rel:!0});var vE=i(Ct);xm=r(vE,"DP"),vE.forEach(t),Mm=r(Ab,":"),Ab.forEach(t),ah=f(e),Ua=s(e,"P",{});var PE=i(Ua);$m=r(PE,"For each batch:"),PE.forEach(t),sh=f(e),T=s(e,"OL",{});var te=i(T);ir=s(te,"LI",{});var wE=i(ir);Nm=r(wE,"gpu 0 reads the batch of data and then sends a mini-batch to each gpu"),wE.forEach(t),zm=f(te),or=s(te,"LI",{});var _E=i(or);Cm=r(_E,"replicates the up-to-date model from gpu 0 to each gpu"),_E.forEach(t),Rm=f(te),Rt=s(te,"LI",{});var md=i(Rt);Zm=r(md,"runs "),rr=s(md,"CODE",{});var yE=i(rr);Bm=r(yE,"forward"),yE.forEach(t),Vm=r(md," and sends output from each gpu to gpu 0, computes loss"),md.forEach(t),qm=f(te),Ia=s(te,"LI",{});var Ob=i(Ia);Fm=r(Ob,"scatters loss from gpu 0 to all gpus, runs "),nr=s(Ob,"CODE",{});var bE=i(nr);Wm=r(bE,"backward"),bE.forEach(t),Ob.forEach(t),Hm=f(te),pr=s(te,"LI",{});var gE=i(pr);Ym=r(gE,"sends gradients from each gpu to gpu 0 and averages those"),gE.forEach(t),te.forEach(t),ih=f(e),Sa=s(e,"P",{});var EE=i(Sa);Xm=r(EE,"The only communication DDP performs per batch is sending gradients, whereas DP does 5 different data exchanges per batch."),EE.forEach(t),oh=f(e),Ze=s(e,"P",{});var vd=i(Ze);Km=r(vd,"DP copies data within the process via python threads, whereas DDP copies data via "),Zt=s(vd,"A",{href:!0,rel:!0});var DE=i(Zt);Jm=r(DE,"torch.distributed"),DE.forEach(t),Qm=r(vd,"."),vd.forEach(t),rh=f(e),Aa=s(e,"P",{});var LE=i(Aa);ev=r(LE,"Under DP gpu 0 performs a lot more work than the rest of the gpus, thus resulting in under-utilization of gpus."),LE.forEach(t),nh=f(e),Oa=s(e,"P",{});var TE=i(Oa);tv=r(TE,"You can use DDP across multiple machines, but this is not the case with DP."),TE.forEach(t),ph=f(e),ja=s(e,"P",{});var kE=i(ja);lv=r(kE,"There are other differences between DP and DDP but they aren\u2019t relevant to this discussion."),kE.forEach(t),hh=f(e),Be=s(e,"P",{});var Pd=i(Be);av=r(Pd,"If you want to go really deep into understanding these 2 modes, this "),Bt=s(Pd,"A",{href:!0,rel:!0});var GE=i(Bt);sv=r(GE,"article"),GE.forEach(t),iv=r(Pd," is highly recommended, as it has great diagrams, includes multiple benchmarks and profiler outputs on various hardware, explains all the nuances that you may need to know."),Pd.forEach(t),fh=f(e),xa=s(e,"P",{});var UE=i(xa);ov=r(UE,"Let\u2019s look at an actual benchmark:"),UE.forEach(t),ch=f(e),Ve=s(e,"TABLE",{});var wd=i(Ve);hr=s(wd,"THEAD",{});var IE=i(hr);ue=s(IE,"TR",{});var Yi=i(ue);Ma=s(Yi,"TH",{align:!0});var SE=i(Ma);rv=r(SE,"Type"),SE.forEach(t),nv=f(Yi),fr=s(Yi,"TH",{});var AE=i(fr);pv=r(AE,"NVlink"),AE.forEach(t),hv=f(Yi),$a=s(Yi,"TH",{align:!0});var OE=i($a);fv=r(OE,"Time"),OE.forEach(t),Yi.forEach(t),IE.forEach(t),cv=f(wd),me=s(wd,"TBODY",{});var Xi=i(me);ve=s(Xi,"TR",{});var Ki=i(ve);Na=s(Ki,"TD",{align:!0});var jE=i(Na);dv=r(jE,"2:DP"),jE.forEach(t),uv=f(Ki),cr=s(Ki,"TD",{});var xE=i(cr);mv=r(xE,"Y"),xE.forEach(t),vv=f(Ki),za=s(Ki,"TD",{align:!0});var ME=i(za);Pv=r(ME,"110s"),ME.forEach(t),Ki.forEach(t),wv=f(Xi),Pe=s(Xi,"TR",{});var Ji=i(Pe);Ca=s(Ji,"TD",{align:!0});var $E=i(Ca);_v=r($E,"2:DDP"),$E.forEach(t),yv=f(Ji),dr=s(Ji,"TD",{});var NE=i(dr);bv=r(NE,"Y"),NE.forEach(t),gv=f(Ji),Ra=s(Ji,"TD",{align:!0});var zE=i(Ra);Ev=r(zE,"101s"),zE.forEach(t),Ji.forEach(t),Dv=f(Xi),we=s(Xi,"TR",{});var Qi=i(we);Za=s(Qi,"TD",{align:!0});var CE=i(Za);Lv=r(CE,"2:DDP"),CE.forEach(t),Tv=f(Qi),ur=s(Qi,"TD",{});var RE=i(ur);kv=r(RE,"N"),RE.forEach(t),Gv=f(Qi),Ba=s(Qi,"TD",{align:!0});var ZE=i(Ba);Uv=r(ZE,"131s"),ZE.forEach(t),Qi.forEach(t),Xi.forEach(t),wd.forEach(t),dh=f(e),Va=s(e,"P",{});var BE=i(Va);Iv=r(BE,"Analysis:"),BE.forEach(t),uh=f(e),qa=s(e,"P",{});var VE=i(qa);Sv=r(VE,"Here DP is ~10% slower than DDP w/ NVlink, but ~15% faster than DDP w/o NVlink"),VE.forEach(t),mh=f(e),Fa=s(e,"P",{});var qE=i(Fa);Av=r(qE,"The real difference will depend on how much data each GPU needs to sync with the others - the more there is to sync, the more a slow link will slow down the total runtime."),qE.forEach(t),vh=f(e),Wa=s(e,"P",{});var FE=i(Wa);Ov=r(FE,"Here is the full benchmark code and outputs:"),FE.forEach(t),Ph=f(e),Vt=s(e,"P",{});var jb=i(Vt);mr=s(jb,"CODE",{});var WE=i(mr);jv=r(WE,"NCCL_P2P_DISABLE=1"),WE.forEach(t),xv=r(jb," was used to disable the NVLink feature on the corresponding benchmark."),jb.forEach(t),wh=f(e),m(qt.$$.fragment,e),_h=f(e),D=s(e,"P",{});var C=i(D);Mv=r(C,"Hardware: 2x TITAN RTX 24GB each + NVlink with 2 NVLinks ("),vr=s(C,"CODE",{});var HE=i(vr);$v=r(HE,"NV2"),HE.forEach(t),Nv=r(C," in "),Pr=s(C,"CODE",{});var YE=i(Pr);zv=r(YE,"nvidia-smi topo -m"),YE.forEach(t),Cv=r(C,`)
Software: `),wr=s(C,"CODE",{});var XE=i(wr);Rv=r(XE,"pytorch-1.8-to-be"),XE.forEach(t),Zv=r(C," + "),_r=s(C,"CODE",{});var KE=i(_r);Bv=r(KE,"cuda-11.0"),KE.forEach(t),Vv=r(C," / "),yr=s(C,"CODE",{});var JE=i(yr);qv=r(JE,"transformers==4.3.0.dev0"),JE.forEach(t),C.forEach(t),yh=f(e),_e=s(e,"H2",{class:!0});var _d=i(_e);qe=s(_d,"A",{id:!0,class:!0,href:!0});var QE=i(qe);br=s(QE,"SPAN",{});var e3=i(br);m(Ft.$$.fragment,e3),e3.forEach(t),QE.forEach(t),Fv=f(_d),gr=s(_d,"SPAN",{});var t3=i(gr);Wv=r(t3,"ZeRO Data Parallelism"),t3.forEach(t),_d.forEach(t),bh=f(e),ye=s(e,"P",{});var Sp=i(ye);Hv=r(Sp,"ZeRO-powered data parallelism (ZeRO-DP) is described on the following diagram from this "),Wt=s(Sp,"A",{href:!0,rel:!0});var l3=i(Wt);Yv=r(l3,"blog post"),l3.forEach(t),Xv=f(Sp),Ha=s(Sp,"IMG",{src:!0,alt:!0}),Sp.forEach(t),gh=f(e),Fe=s(e,"P",{});var yd=i(Fe);Kv=r(yd,"It can be difficult to wrap one\u2019s head around it, but in reality the concept is quite simple. This is just the usual "),Er=s(yd,"CODE",{});var a3=i(Er);Jv=r(a3,"DataParallel"),a3.forEach(t),Qv=r(yd," (DP), except, instead of replicating the full model params, gradients and optimizer states, each GPU stores only a slice of it.  And then at run-time when the full layer params are needed just for the given layer, all GPUs synchronize to give each other parts that they miss - this is it."),yd.forEach(t),Eh=f(e),Ya=s(e,"P",{});var s3=i(Ya);eP=r(s3,"Consider this simple model with 3 layers, where each layer has 3 params:"),s3.forEach(t),Dh=f(e),m(Ht.$$.fragment,e),Lh=f(e),Xa=s(e,"P",{});var i3=i(Xa);tP=r(i3,"Layer La has weights a0, a1 and a2."),i3.forEach(t),Th=f(e),Ka=s(e,"P",{});var o3=i(Ka);lP=r(o3,"If we have 3 GPUs, the Sharded DDP (= Zero-DP) splits the model onto 3 GPUs like so:"),o3.forEach(t),kh=f(e),m(Yt.$$.fragment,e),Gh=f(e),Ja=s(e,"P",{});var r3=i(Ja);aP=r(r3,"In a way this is the same horizontal slicing, as tensor parallelism, if you imagine the typical DNN diagram. Vertical slicing is where one puts whole layer-groups on different GPUs. But it\u2019s just the starting point."),r3.forEach(t),Uh=f(e),Qa=s(e,"P",{});var n3=i(Qa);sP=r(n3,"Now each of these GPUs will get the usual mini-batch as it works in DP:"),n3.forEach(t),Ih=f(e),m(Xt.$$.fragment,e),Sh=f(e),es=s(e,"P",{});var p3=i(es);iP=r(p3,"The inputs are unmodified - they think they are going to be processed by the normal model."),p3.forEach(t),Ah=f(e),ts=s(e,"P",{});var h3=i(ts);oP=r(h3,"First, the inputs hit the layer La."),h3.forEach(t),Oh=f(e),ls=s(e,"P",{});var f3=i(ls);rP=r(f3,"Let\u2019s focus just on GPU0: x0 needs a0, a1, a2 params to do its forward path, but GPU0 has only a0 - it gets sent a1 from GPU1 and a2 from GPU2, bringing all pieces of the model together."),f3.forEach(t),jh=f(e),as=s(e,"P",{});var c3=i(as);nP=r(c3,"In parallel, GPU1 gets mini-batch x1 and it only has a1, but needs a0 and a2 params, so it gets those from GPU0 and GPU2."),c3.forEach(t),xh=f(e),ss=s(e,"P",{});var d3=i(ss);pP=r(d3,"Same happens to GPU2 that gets input x2. It gets a0 and a1 from GPU0 and GPU1, and with its a2 it reconstructs the full tensor."),d3.forEach(t),Mh=f(e),is=s(e,"P",{});var u3=i(is);hP=r(u3,"All 3 GPUs get the full tensors reconstructed and a forward happens."),u3.forEach(t),$h=f(e),os=s(e,"P",{});var m3=i(os);fP=r(m3,"As soon as the calculation is done, the data that is no longer needed gets dropped - it\u2019s only used during the calculation. The reconstruction is done efficiently via a pre-fetch."),m3.forEach(t),Nh=f(e),rs=s(e,"P",{});var v3=i(rs);cP=r(v3,"And the whole process is repeated for layer Lb, then Lc forward-wise, and then backward Lc -> Lb -> La."),v3.forEach(t),zh=f(e),ns=s(e,"P",{});var P3=i(ns);dP=r(P3,"To me this sounds like an efficient group backpacking weight distribution strategy:"),P3.forEach(t),Ch=f(e),B=s(e,"OL",{});var eo=i(B);Dr=s(eo,"LI",{});var w3=i(Dr);uP=r(w3,"person A carries the tent"),w3.forEach(t),mP=f(eo),Lr=s(eo,"LI",{});var _3=i(Lr);vP=r(_3,"person B carries the stove"),_3.forEach(t),PP=f(eo),Tr=s(eo,"LI",{});var y3=i(Tr);wP=r(y3,"person C carries the axe"),y3.forEach(t),eo.forEach(t),Rh=f(e),ps=s(e,"P",{});var b3=i(ps);_P=r(b3,"Now each night they all share what they have with others and get from others what they don\u2019t have, and in the morning they pack up their allocated type of gear and continue on their way. This is Sharded DDP / Zero DP."),b3.forEach(t),Zh=f(e),hs=s(e,"P",{});var g3=i(hs);yP=r(g3,"Compare this strategy to the simple one where each person has to carry their own tent, stove and axe, which would be far more inefficient. This is DataParallel (DP and DDP) in Pytorch."),g3.forEach(t),Bh=f(e),fs=s(e,"P",{});var E3=i(fs);bP=r(E3,"While reading the literature on this topic you may encounter the following synonyms: Sharded, Partitioned."),E3.forEach(t),Vh=f(e),cs=s(e,"P",{});var D3=i(cs);gP=r(D3,"If you pay close attention the way ZeRO partitions the model\u2019s weights - it looks very similar to tensor parallelism which will be discussed later. This is because it partitions/shards each layer\u2019s weights, unlike vertical model parallelism which is discussed next."),D3.forEach(t),qh=f(e),ds=s(e,"P",{});var L3=i(ds);EP=r(L3,"Implementations:"),L3.forEach(t),Fh=f(e),V=s(e,"UL",{});var to=i(V);us=s(to,"LI",{});var xb=i(us);Kt=s(xb,"A",{href:!0,rel:!0});var T3=i(Kt);DP=r(T3,"DeepSpeed"),T3.forEach(t),LP=r(xb," ZeRO-DP stages 1+2+3"),xb.forEach(t),TP=f(to),ms=s(to,"LI",{});var Mb=i(ms);Jt=s(Mb,"A",{href:!0,rel:!0});var k3=i(Jt);kP=r(k3,"Fairscale"),k3.forEach(t),GP=r(Mb," ZeRO-DP stages 1+2+3"),Mb.forEach(t),UP=f(to),kr=s(to,"LI",{});var G3=i(kr);Qt=s(G3,"A",{href:!0});var $b=i(Qt);Gr=s($b,"CODE",{});var U3=i(Gr);IP=r(U3,"transformers"),U3.forEach(t),SP=r($b," integration"),$b.forEach(t),G3.forEach(t),to.forEach(t),Wh=f(e),be=s(e,"H2",{class:!0});var bd=i(be);We=s(bd,"A",{id:!0,class:!0,href:!0});var I3=i(We);Ur=s(I3,"SPAN",{});var S3=i(Ur);m(el.$$.fragment,S3),S3.forEach(t),I3.forEach(t),AP=f(bd),Ir=s(bd,"SPAN",{});var A3=i(Ir);OP=r(A3,"Naive Model Parallelism (Vertical) and Pipeline Parallelism"),A3.forEach(t),bd.forEach(t),Hh=f(e),He=s(e,"P",{});var gd=i(He);jP=r(gd,"Naive Model Parallelism (MP) is where one spreads groups of model layers across multiple GPUs. The mechanism is relatively simple - switch the desired layers "),Sr=s(gd,"CODE",{});var O3=i(Sr);xP=r(O3,".to()"),O3.forEach(t),MP=r(gd," the desired devices and now whenever the data goes in and out those layers switch the data to the same device as the layer and leave the rest unmodified."),gd.forEach(t),Yh=f(e),vs=s(e,"P",{});var j3=i(vs);$P=r(j3,"We refer to it as Vertical MP, because if you remember how most models are drawn, we slice the layers vertically. For example, if the following diagram shows an 8-layer model:"),j3.forEach(t),Xh=f(e),m(tl.$$.fragment,e),Kh=f(e),Ps=s(e,"P",{});var x3=i(Ps);NP=r(x3,"we just sliced it in 2 vertically, placing layers 0-3 onto GPU0 and 4-7 to GPU1."),x3.forEach(t),Jh=f(e),ws=s(e,"P",{});var M3=i(ws);zP=r(M3,"Now while data travels from layer 0 to 1, 1 to 2 and 2 to 3 this is just the normal model. But when data needs to pass from layer 3 to layer 4 it needs to travel from GPU0 to GPU1 which introduces a communication overhead. If the participating GPUs are on the same compute node (e.g. same physical machine) this copying is pretty fast, but if the GPUs are located on different compute nodes (e.g. multiple machines) the communication overhead could be significantly larger."),M3.forEach(t),Qh=f(e),_s=s(e,"P",{});var $3=i(_s);CP=r($3,"Then layers 4 to 5 to 6 to 7 are as a normal model would have and when the 7th layer completes we often need to send the data back to layer 0 where the labels are (or alternatively send the labels to the last layer). Now the loss can be computed and the optimizer can do its work."),$3.forEach(t),ef=f(e),ys=s(e,"P",{});var N3=i(ys);RP=r(N3,"Problems:"),N3.forEach(t),tf=f(e),Ye=s(e,"UL",{});var Ed=i(Ye);Ar=s(Ed,"LI",{});var z3=i(Ar);ZP=r(z3,"the main deficiency and why this one is called \u201Cnaive\u201D MP, is that all but one GPU is idle at any given moment. So if 4 GPUs are used, it\u2019s almost identical to quadrupling the amount of memory of a single GPU, and ignoring the rest of the hardware. Plus there is the overhead of copying the data between devices. So 4x 6GB cards will be able to accommodate the same size as 1x 24GB card using naive MP, except the latter will complete the training faster, since it doesn\u2019t have the data copying overhead. But, say, if you have 40GB cards and need to fit a 45GB model you can with 4x 40GB cards (but barely because of the gradient and optimizer states)"),z3.forEach(t),BP=f(Ed),Or=s(Ed,"LI",{});var C3=i(Or);VP=r(C3,"shared embeddings may need to get copied back and forth between GPUs."),C3.forEach(t),Ed.forEach(t),lf=f(e),bs=s(e,"P",{});var R3=i(bs);qP=r(R3,"Pipeline Parallelism (PP) is almost identical to a naive MP, but it solves the GPU idling problem, by chunking the incoming batch into micro-batches and artificially creating a pipeline, which allows different GPUs to concurrently participate in the computation process."),R3.forEach(t),af=f(e),Xe=s(e,"P",{});var Dd=i(Xe);FP=r(Dd,"The following illustration from the "),ll=s(Dd,"A",{href:!0,rel:!0});var Z3=i(ll);WP=r(Z3,"GPipe paper"),Z3.forEach(t),HP=r(Dd," shows the naive MP on the top, and PP on the bottom:"),Dd.forEach(t),sf=f(e),gs=s(e,"P",{});var B3=i(gs);Es=s(B3,"IMG",{src:!0,alt:!0}),B3.forEach(t),of=f(e),Ds=s(e,"P",{});var V3=i(Ds);YP=r(V3,"It\u2019s easy to see from the bottom diagram how PP has less dead zones, where GPUs are idle. The idle parts are referred to as the \u201Cbubble\u201D."),V3.forEach(t),rf=f(e),Ls=s(e,"P",{});var q3=i(Ls);XP=r(q3,"Both parts of the diagram show a parallelism that is of degree 4. That is 4 GPUs are participating in the pipeline. So there is the forward path of 4 pipe stages F0, F1, F2 and F3 and then the return reverse order backward path of B3, B2, B1 and B0."),q3.forEach(t),nf=f(e),q=s(e,"P",{});var lo=i(q);KP=r(lo,"PP introduces a new hyper-parameter to tune and it\u2019s "),jr=s(lo,"CODE",{});var F3=i(jr);JP=r(F3,"chunks"),F3.forEach(t),QP=r(lo," which defines how many chunks of data are sent in a sequence through the same pipe stage. For example, in the bottomw diagram you can see that "),xr=s(lo,"CODE",{});var W3=i(xr);e1=r(W3,"chunks=4"),W3.forEach(t),t1=r(lo,". GPU0 performs the same forward path on chunk 0, 1, 2 and 3 (F0,0, F0,1, F0,2, F0,3) and then it waits for other GPUs to do their work and only when their work is starting to be complete, GPU0 starts to work again doing the backward path for chunks 3, 2, 1 and 0 (B0,3, B0,2, B0,1, B0,0)."),lo.forEach(t),pf=f(e),Ke=s(e,"P",{});var Ld=i(Ke);l1=r(Ld,"Note that conceptually this is the same concept as gradient accumulation steps (GAS). Pytorch uses "),Mr=s(Ld,"CODE",{});var H3=i(Mr);a1=r(H3,"chunks"),H3.forEach(t),s1=r(Ld,", whereas DeepSpeed refers to the same hyper-parameter as GAS."),Ld.forEach(t),hf=f(e),Je=s(e,"P",{});var Td=i(Je);i1=r(Td,"Because of the chunks, PP introduces the concept of micro-batches (MBS). DP splits the global data batch size into mini-batches, so if you have a DP degree of 4, a global batch size of 1024 gets split up into 4 mini-batches of 256 each (1024/4). And if the number of "),$r=s(Td,"CODE",{});var Y3=i($r);o1=r(Y3,"chunks"),Y3.forEach(t),r1=r(Td," (or GAS) is 32 we end up with a micro-batch size of 8 (256/32). Each Pipeline stage works with a single micro-batch at a time."),Td.forEach(t),ff=f(e),F=s(e,"P",{});var ao=i(F);n1=r(ao,"To calculate the global batch size of the DP + PP setup we then do: "),Nr=s(ao,"CODE",{});var X3=i(Nr);p1=r(X3,"mbs*chunks*dp_degree"),X3.forEach(t),h1=r(ao," ("),zr=s(ao,"CODE",{});var K3=i(zr);f1=r(K3,"8*32*4=1024"),K3.forEach(t),c1=r(ao,")."),ao.forEach(t),cf=f(e),Ts=s(e,"P",{});var J3=i(Ts);d1=r(J3,"Let\u2019s go back to the diagram."),J3.forEach(t),df=f(e),W=s(e,"P",{});var so=i(W);u1=r(so,"With "),Cr=s(so,"CODE",{});var Q3=i(Cr);m1=r(Q3,"chunks=1"),Q3.forEach(t),v1=r(so," you end up with the naive MP, which is very inefficient. With a very large "),Rr=s(so,"CODE",{});var e0=i(Rr);P1=r(e0,"chunks"),e0.forEach(t),w1=r(so," value you end up with tiny micro-batch sizes which could be not every efficient either. So one has to experiment to find the value that leads to the highest efficient utilization of the gpus."),so.forEach(t),uf=f(e),x=s(e,"P",{});var Pt=i(x);_1=r(Pt,"While the diagram shows that there is a bubble of \u201Cdead\u201D time that can\u2019t be parallelized because the last "),Zr=s(Pt,"CODE",{});var t0=i(Zr);y1=r(t0,"forward"),t0.forEach(t),b1=r(Pt," stage has to wait for "),Br=s(Pt,"CODE",{});var l0=i(Br);g1=r(l0,"backward"),l0.forEach(t),E1=r(Pt," to complete the pipeline, the purpose of finding the best value for "),Vr=s(Pt,"CODE",{});var a0=i(Vr);D1=r(a0,"chunks"),a0.forEach(t),L1=r(Pt," is to enable a high concurrent GPU utilization across all participating GPUs which translates to minimizing the size of the bubble."),Pt.forEach(t),mf=f(e),ks=s(e,"P",{});var s0=i(ks);T1=r(s0,"There are 2 groups of solutions - the traditional Pipeline API and the more modern solutions that make things much easier for the end user."),s0.forEach(t),vf=f(e),Gs=s(e,"P",{});var i0=i(Gs);k1=r(i0,"Traditional Pipeline API solutions:"),i0.forEach(t),Pf=f(e),M=s(e,"UL",{});var wt=i(M);qr=s(wt,"LI",{});var o0=i(qr);G1=r(o0,"PyTorch"),o0.forEach(t),U1=f(wt),Fr=s(wt,"LI",{});var r0=i(Fr);I1=r(r0,"FairScale"),r0.forEach(t),S1=f(wt),Wr=s(wt,"LI",{});var n0=i(Wr);A1=r(n0,"DeepSpeed"),n0.forEach(t),O1=f(wt),Hr=s(wt,"LI",{});var p0=i(Hr);j1=r(p0,"Megatron-LM"),p0.forEach(t),wt.forEach(t),wf=f(e),Us=s(e,"P",{});var h0=i(Us);x1=r(h0,"Modern solutions:"),h0.forEach(t),_f=f(e),Qe=s(e,"UL",{});var kd=i(Qe);Yr=s(kd,"LI",{});var f0=i(Yr);M1=r(f0,"Varuna"),f0.forEach(t),$1=f(kd),Xr=s(kd,"LI",{});var c0=i(Xr);N1=r(c0,"Sagemaker"),c0.forEach(t),kd.forEach(t),yf=f(e),Is=s(e,"P",{});var d0=i(Is);z1=r(d0,"Problems with traditional Pipeline API solutions:"),d0.forEach(t),bf=f(e),$=s(e,"UL",{});var _t=i($);al=s(_t,"LI",{});var Gd=i(al);C1=r(Gd,"have to modify the model quite heavily, because Pipeline requires one to rewrite the normal flow of modules into a "),Kr=s(Gd,"CODE",{});var u0=i(Kr);R1=r(u0,"nn.Sequential"),u0.forEach(t),Z1=r(Gd," sequence of the same, which may require changes to the design of the model."),Gd.forEach(t),B1=f(_t),Ss=s(_t,"LI",{});var Nb=i(Ss);V1=r(Nb,"currently the Pipeline API is very restricted. If you had a bunch of python variables being passed in the very first stage of the Pipeline, you will have to find a way around it. Currently, the pipeline interface requires either a single Tensor or a tuple of Tensors as the only input and output. These tensors must have a batch size as the very first dimension, since pipeline is going to chunk the mini batch into micro-batches. Possible improvements are being discussed here "),sl=s(Nb,"A",{href:!0,rel:!0});var m0=i(sl);q1=r(m0,"https://github.com/pytorch/pytorch/pull/50693"),m0.forEach(t),Nb.forEach(t),F1=f(_t),Jr=s(_t,"LI",{});var v0=i(Jr);W1=r(v0,"conditional control flow at the level of pipe stages is not possible - e.g., Encoder-Decoder models like T5 require special workarounds to handle a conditional encoder stage."),v0.forEach(t),H1=f(_t),Qr=s(_t,"LI",{});var P0=i(Qr);Y1=r(P0,"have to arrange each layer so that the output of one model becomes an input to the other model."),P0.forEach(t),_t.forEach(t),gf=f(e),As=s(e,"P",{});var w0=i(As);X1=r(w0,"We are yet to experiment with Varuna and SageMaker but their papers report that they have overcome the list of problems mentioned above and that they require much smaller changes to the user\u2019s model."),w0.forEach(t),Ef=f(e),Os=s(e,"P",{});var _0=i(Os);K1=r(_0,"Implementations:"),_0.forEach(t),Df=f(e),b=s(e,"UL",{});var I=i(b);il=s(I,"LI",{});var Ud=i(il);ol=s(Ud,"A",{href:!0,rel:!0});var y0=i(ol);J1=r(y0,"Pytorch"),y0.forEach(t),Q1=r(Ud," (initial support in pytorch-1.8, and progressively getting improved in 1.9 and more so in 1.10). Some "),rl=s(Ud,"A",{href:!0,rel:!0});var b0=i(rl);ew=r(b0,"examples"),b0.forEach(t),Ud.forEach(t),tw=f(I),en=s(I,"LI",{});var g0=i(en);nl=s(g0,"A",{href:!0,rel:!0});var E0=i(nl);lw=r(E0,"FairScale"),E0.forEach(t),g0.forEach(t),aw=f(I),tn=s(I,"LI",{});var D0=i(tn);pl=s(D0,"A",{href:!0,rel:!0});var L0=i(pl);sw=r(L0,"DeepSpeed"),L0.forEach(t),D0.forEach(t),iw=f(I),js=s(I,"LI",{});var zb=i(js);hl=s(zb,"A",{href:!0,rel:!0});var T0=i(hl);ow=r(T0,"Megatron-LM"),T0.forEach(t),rw=r(zb," has an internal implementation - no API."),zb.forEach(t),nw=f(I),ln=s(I,"LI",{});var k0=i(ln);fl=s(k0,"A",{href:!0,rel:!0});var G0=i(fl);pw=r(G0,"Varuna"),G0.forEach(t),k0.forEach(t),hw=f(I),xs=s(I,"LI",{});var Cb=i(xs);cl=s(Cb,"A",{href:!0,rel:!0});var U0=i(cl);fw=r(U0,"SageMaker"),U0.forEach(t),cw=r(Cb," - this is a proprietary solution that can only be used on AWS."),Cb.forEach(t),dw=f(I),Ms=s(I,"LI",{});var Rb=i(Ms);dl=s(Rb,"A",{href:!0,rel:!0});var I0=i(dl);uw=r(I0,"OSLO"),I0.forEach(t),mw=r(Rb," - this is implemented based on the Hugging Face Transformers."),Rb.forEach(t),I.forEach(t),Lf=f(e),et=s(e,"P",{});var Id=i(et);vw=r(Id,"\u{1F917} Transformers status: as of this writing none of the models supports full-PP. GPT2 and T5 models have naive MP support. The main obstacle is being unable to convert the models to "),an=s(Id,"CODE",{});var S0=i(an);Pw=r(S0,"nn.Sequential"),S0.forEach(t),ww=r(Id," and have all the inputs to be Tensors. This is because currently the models include many features that make the conversion very complicated, and will need to be removed to accomplish that."),Id.forEach(t),Tf=f(e),$s=s(e,"P",{});var A0=i($s);_w=r(A0,"Other approaches:"),A0.forEach(t),kf=f(e),ge=s(e,"P",{});var Ap=i(ge);yw=r(Ap,"DeepSpeed, Varuna and SageMaker use the concept of an "),ul=s(Ap,"A",{href:!0,rel:!0});var O0=i(ul);bw=r(O0,"Interleaved Pipeline"),O0.forEach(t),gw=f(Ap),Ns=s(Ap,"IMG",{src:!0,alt:!0}),Ap.forEach(t),Gf=f(e),zs=s(e,"P",{});var j0=i(zs);Ew=r(j0,"Here the bubble (idle time) is further minimized by prioritizing backward passes."),j0.forEach(t),Uf=f(e),Cs=s(e,"P",{});var x0=i(Cs);Dw=r(x0,"Varuna further tries to improve the schedule by using simulations to discover the most efficient scheduling."),x0.forEach(t),If=f(e),tt=s(e,"P",{});var Sd=i(tt);Lw=r(Sd,"OSLO has pipeline parallelism implementation based on the Transformers without "),sn=s(Sd,"CODE",{});var M0=i(sn);Tw=r(M0,"nn.Sequential"),M0.forEach(t),kw=r(Sd," converting."),Sd.forEach(t),Sf=f(e),Ee=s(e,"H2",{class:!0});var Ad=i(Ee);lt=s(Ad,"A",{id:!0,class:!0,href:!0});var $0=i(lt);on=s($0,"SPAN",{});var N0=i(on);m(ml.$$.fragment,N0),N0.forEach(t),$0.forEach(t),Gw=f(Ad),rn=s(Ad,"SPAN",{});var z0=i(rn);Uw=r(z0,"Tensor Parallelism"),z0.forEach(t),Ad.forEach(t),Af=f(e),Rs=s(e,"P",{});var C0=i(Rs);Iw=r(C0,"In Tensor Parallelism each GPU processes only a slice of a tensor and only aggregates the full tensor for operations that require the whole thing."),C0.forEach(t),Of=f(e),H=s(e,"P",{});var io=i(H);Sw=r(io,"In this section we use concepts and diagrams from the "),vl=s(io,"A",{href:!0,rel:!0});var R0=i(vl);Aw=r(R0,"Megatron-LM"),R0.forEach(t),Ow=r(io," paper: "),Pl=s(io,"A",{href:!0,rel:!0});var Z0=i(Pl);jw=r(Z0,"Efficient Large-Scale Language Model Training on GPU Clusters"),Z0.forEach(t),xw=r(io,"."),io.forEach(t),jf=f(e),Y=s(e,"P",{});var oo=i(Y);Mw=r(oo,"The main building block of any transformer is a fully connected "),nn=s(oo,"CODE",{});var B0=i(nn);$w=r(B0,"nn.Linear"),B0.forEach(t),Nw=r(oo," followed by a nonlinear activation "),pn=s(oo,"CODE",{});var V0=i(pn);zw=r(V0,"GeLU"),V0.forEach(t),Cw=r(oo,"."),oo.forEach(t),xf=f(e),k=s(e,"P",{});var le=i(k);Rw=r(le,"Following the Megatron\u2019s paper notation, we can write the dot-product part of it as "),hn=s(le,"CODE",{});var q0=i(hn);Zw=r(q0,"Y = GeLU(XA)"),q0.forEach(t),Bw=r(le,", where "),fn=s(le,"CODE",{});var F0=i(fn);Vw=r(F0,"X"),F0.forEach(t),qw=r(le," and "),cn=s(le,"CODE",{});var W0=i(cn);Fw=r(W0,"Y"),W0.forEach(t),Ww=r(le," are the input and output vectors, and "),dn=s(le,"CODE",{});var H0=i(dn);Hw=r(H0,"A"),H0.forEach(t),Yw=r(le," is the weight matrix."),le.forEach(t),Mf=f(e),wl=s(e,"P",{});var Zb=i(wl);Xw=r(Zb,`If we look at the computation in matrix form, it\u2019s easy to see how the matrix multiplication can be split between multiple GPUs:
`),Zs=s(Zb,"IMG",{src:!0,alt:!0}),Zb.forEach(t),$f=f(e),d=s(e,"P",{});var g=i(d);Kw=r(g,"If we split the weight matrix "),un=s(g,"CODE",{});var Y0=i(un);Jw=r(Y0,"A"),Y0.forEach(t),Qw=r(g," column-wise across "),mn=s(g,"CODE",{});var X0=i(mn);e2=r(X0,"N"),X0.forEach(t),t2=r(g," GPUs and perform matrix multiplications "),vn=s(g,"CODE",{});var K0=i(vn);l2=r(K0,"XA_1"),K0.forEach(t),a2=r(g," through "),Pn=s(g,"CODE",{});var J0=i(Pn);s2=r(J0,"XA_n"),J0.forEach(t),i2=r(g," in parallel, then we will end up with "),wn=s(g,"CODE",{});var Q0=i(wn);o2=r(Q0,"N"),Q0.forEach(t),r2=r(g," output vectors "),_n=s(g,"CODE",{});var e4=i(_n);n2=r(e4,"Y_1, Y_2, ..., Y_n"),e4.forEach(t),p2=r(g," which can be fed into "),yn=s(g,"CODE",{});var t4=i(yn);h2=r(t4,"GeLU"),t4.forEach(t),f2=r(g,` independently:
`),Bs=s(g,"IMG",{src:!0,alt:!0}),g.forEach(t),Nf=f(e),_l=s(e,"P",{});var Bb=i(_l);c2=r(Bb,`Using this principle, we can update an MLP of arbitrary depth, without the need for any synchronization between GPUs until the very end, where we need to reconstruct the output vector from shards. The Megatron-LM paper authors provide a helpful illustration for that:
`),Vs=s(Bb,"IMG",{src:!0,alt:!0}),Bb.forEach(t),zf=f(e),yl=s(e,"P",{});var Vb=i(yl);d2=r(Vb,`Parallelizing the multi-headed attention layers is even simpler, since they are already inherently parallel, due to having multiple independent heads!
`),qs=s(Vb,"IMG",{src:!0,alt:!0}),Vb.forEach(t),Cf=f(e),Fs=s(e,"P",{});var l4=i(Fs);u2=r(l4,"Special considerations: TP requires very fast network, and therefore it\u2019s not advisable to do TP across more than one node. Practically, if a node has 4 GPUs, the highest TP degree is therefore 4. If you need a TP degree of 8, you need to use nodes that have at least 8 GPUs."),l4.forEach(t),Rf=f(e),X=s(e,"P",{});var ro=i(X);m2=r(ro,"This section is based on the original much more "),bl=s(ro,"A",{href:!0,rel:!0});var a4=i(bl);v2=r(a4,"detailed TP overview"),a4.forEach(t),P2=r(ro,`.
by `),gl=s(ro,"A",{href:!0,rel:!0});var s4=i(gl);w2=r(s4,"@anton-l"),s4.forEach(t),_2=r(ro,"."),ro.forEach(t),Zf=f(e),Ws=s(e,"P",{});var i4=i(Ws);y2=r(i4,"SageMaker combines TP with DP for a more efficient processing."),i4.forEach(t),Bf=f(e),Hs=s(e,"P",{});var o4=i(Hs);b2=r(o4,"Alternative names:"),o4.forEach(t),Vf=f(e),Ys=s(e,"UL",{});var r4=i(Ys);Xs=s(r4,"LI",{});var qb=i(Xs);g2=r(qb,"DeepSpeed calls it "),El=s(qb,"A",{href:!0,rel:!0});var n4=i(El);E2=r(n4,"tensor slicing"),n4.forEach(t),qb.forEach(t),r4.forEach(t),qf=f(e),Ks=s(e,"P",{});var p4=i(Ks);D2=r(p4,"Implementations:"),p4.forEach(t),Ff=f(e),N=s(e,"UL",{});var yt=i(N);Js=s(yt,"LI",{});var Fb=i(Js);Dl=s(Fb,"A",{href:!0,rel:!0});var h4=i(Dl);L2=r(h4,"Megatron-LM"),h4.forEach(t),T2=r(Fb," has an internal implementation, as it\u2019s very model-specific"),Fb.forEach(t),k2=f(yt),Qs=s(yt,"LI",{});var Wb=i(Qs);Ll=s(Wb,"A",{href:!0,rel:!0});var f4=i(Ll);G2=r(f4,"parallelformers"),f4.forEach(t),U2=r(Wb," (only inference at the moment)"),Wb.forEach(t),I2=f(yt),ei=s(yt,"LI",{});var Hb=i(ei);Tl=s(Hb,"A",{href:!0,rel:!0});var c4=i(Tl);S2=r(c4,"SageMaker"),c4.forEach(t),A2=r(Hb," - this is a proprietary solution that can only be used on AWS."),Hb.forEach(t),O2=f(yt),ti=s(yt,"LI",{});var Yb=i(ti);kl=s(Yb,"A",{href:!0,rel:!0});var d4=i(kl);j2=r(d4,"OSLO"),d4.forEach(t),x2=r(Yb," has the tensor parallelism implementation based on the Transformers."),Yb.forEach(t),yt.forEach(t),Wf=f(e),li=s(e,"P",{});var u4=i(li);M2=r(u4,"\u{1F917} Transformers status:"),u4.forEach(t),Hf=f(e),K=s(e,"UL",{});var no=i(K);bn=s(no,"LI",{});var m4=i(bn);$2=r(m4,"core: not yet implemented in the core"),m4.forEach(t),N2=f(no),Gl=s(no,"LI",{});var Od=i(Gl);z2=r(Od,"but if you want inference "),Ul=s(Od,"A",{href:!0,rel:!0});var v4=i(Ul);C2=r(v4,"parallelformers"),v4.forEach(t),R2=r(Od," provides this support for most of our models. So until this is implemented in the core you can use theirs. And hopefully training mode will be supported too."),Od.forEach(t),Z2=f(no),ai=s(no,"LI",{});var Xb=i(ai);B2=r(Xb,"Deepspeed-Inference also supports our BERT, GPT-2, and GPT-Neo models in their super-fast CUDA-kernel-based inference mode, see more "),Il=s(Xb,"A",{href:!0,rel:!0});var P4=i(Il);V2=r(P4,"here"),P4.forEach(t),Xb.forEach(t),no.forEach(t),Yf=f(e),De=s(e,"H2",{class:!0});var jd=i(De);at=s(jd,"A",{id:!0,class:!0,href:!0});var w4=i(at);gn=s(w4,"SPAN",{});var _4=i(gn);m(Sl.$$.fragment,_4),_4.forEach(t),w4.forEach(t),q2=f(jd),En=s(jd,"SPAN",{});var y4=i(En);F2=r(y4,"DP+PP"),y4.forEach(t),jd.forEach(t),Xf=f(e),st=s(e,"P",{});var xd=i(st);W2=r(xd,"The following diagram from the DeepSpeed "),Al=s(xd,"A",{href:!0,rel:!0});var b4=i(Al);H2=r(b4,"pipeline tutorial"),b4.forEach(t),Y2=r(xd," demonstrates how one combines DP with PP."),xd.forEach(t),Kf=f(e),si=s(e,"P",{});var g4=i(si);ii=s(g4,"IMG",{src:!0,alt:!0}),g4.forEach(t),Jf=f(e),oi=s(e,"P",{});var E4=i(oi);X2=r(E4,"Here it\u2019s important to see how DP rank 0 doesn\u2019t see GPU2 and DP rank 1 doesn\u2019t see GPU3. To DP there is just GPUs 0 and 1 where it feeds data as if there were just 2 GPUs. GPU0 \u201Csecretly\u201D offloads some of its load to GPU2 using PP. And GPU1 does the same by enlisting GPU3 to its aid."),E4.forEach(t),Qf=f(e),ri=s(e,"P",{});var D4=i(ri);K2=r(D4,"Since each dimension requires at least 2 GPUs, here you\u2019d need at least 4 GPUs."),D4.forEach(t),ec=f(e),ni=s(e,"P",{});var L4=i(ni);J2=r(L4,"Implementations:"),L4.forEach(t),tc=f(e),G=s(e,"UL",{});var ae=i(G);Dn=s(ae,"LI",{});var T4=i(Dn);Ol=s(T4,"A",{href:!0,rel:!0});var k4=i(Ol);Q2=r(k4,"DeepSpeed"),k4.forEach(t),T4.forEach(t),e_=f(ae),Ln=s(ae,"LI",{});var G4=i(Ln);jl=s(G4,"A",{href:!0,rel:!0});var U4=i(jl);t_=r(U4,"Megatron-LM"),U4.forEach(t),G4.forEach(t),l_=f(ae),Tn=s(ae,"LI",{});var I4=i(Tn);xl=s(I4,"A",{href:!0,rel:!0});var S4=i(xl);a_=r(S4,"Varuna"),S4.forEach(t),I4.forEach(t),s_=f(ae),kn=s(ae,"LI",{});var A4=i(kn);Ml=s(A4,"A",{href:!0,rel:!0});var O4=i(Ml);i_=r(O4,"SageMaker"),O4.forEach(t),A4.forEach(t),o_=f(ae),Gn=s(ae,"LI",{});var j4=i(Gn);$l=s(j4,"A",{href:!0,rel:!0});var x4=i($l);r_=r(x4,"OSLO"),x4.forEach(t),j4.forEach(t),ae.forEach(t),lc=f(e),pi=s(e,"P",{});var M4=i(pi);n_=r(M4,"\u{1F917} Transformers status: not yet implemented"),M4.forEach(t),ac=f(e),Le=s(e,"H2",{class:!0});var Md=i(Le);it=s(Md,"A",{id:!0,class:!0,href:!0});var $4=i(it);Un=s($4,"SPAN",{});var N4=i(Un);m(Nl.$$.fragment,N4),N4.forEach(t),$4.forEach(t),p_=f(Md),In=s(Md,"SPAN",{});var z4=i(In);h_=r(z4,"DP+PP+TP"),z4.forEach(t),Md.forEach(t),sc=f(e),hi=s(e,"P",{});var C4=i(hi);f_=r(C4,"To get an even more efficient training a 3D parallelism is used where PP is combined with TP and DP. This can be seen in the following diagram."),C4.forEach(t),ic=f(e),fi=s(e,"P",{});var R4=i(fi);ci=s(R4,"IMG",{src:!0,alt:!0}),R4.forEach(t),oc=f(e),ot=s(e,"P",{});var $d=i(ot);c_=r($d,"This diagram is from a blog post "),zl=s($d,"A",{href:!0,rel:!0});var Z4=i(zl);d_=r(Z4,"3D parallelism: Scaling to trillion-parameter models"),Z4.forEach(t),u_=r($d,", which is a good read as well."),$d.forEach(t),rc=f(e),di=s(e,"P",{});var B4=i(di);m_=r(B4,"Since each dimension requires at least 2 GPUs, here you\u2019d need at least 8 GPUs."),B4.forEach(t),nc=f(e),ui=s(e,"P",{});var V4=i(ui);v_=r(V4,"Implementations:"),V4.forEach(t),pc=f(e),U=s(e,"UL",{});var se=i(U);mi=s(se,"LI",{});var Kb=i(mi);Cl=s(Kb,"A",{href:!0,rel:!0});var q4=i(Cl);P_=r(q4,"DeepSpeed"),q4.forEach(t),w_=r(Kb," - DeepSpeed also includes an even more efficient DP, which they call ZeRO-DP."),Kb.forEach(t),__=f(se),Sn=s(se,"LI",{});var F4=i(Sn);Rl=s(F4,"A",{href:!0,rel:!0});var W4=i(Rl);y_=r(W4,"Megatron-LM"),W4.forEach(t),F4.forEach(t),b_=f(se),An=s(se,"LI",{});var H4=i(An);Zl=s(H4,"A",{href:!0,rel:!0});var Y4=i(Zl);g_=r(Y4,"Varuna"),Y4.forEach(t),H4.forEach(t),E_=f(se),On=s(se,"LI",{});var X4=i(On);Bl=s(X4,"A",{href:!0,rel:!0});var K4=i(Bl);D_=r(K4,"SageMaker"),K4.forEach(t),X4.forEach(t),L_=f(se),jn=s(se,"LI",{});var J4=i(jn);Vl=s(J4,"A",{href:!0,rel:!0});var Q4=i(Vl);T_=r(Q4,"OSLO"),Q4.forEach(t),J4.forEach(t),se.forEach(t),hc=f(e),vi=s(e,"P",{});var e5=i(vi);k_=r(e5,"\u{1F917} Transformers status: not yet implemented, since we have no PP and TP."),e5.forEach(t),fc=f(e),Te=s(e,"H2",{class:!0});var Nd=i(Te);rt=s(Nd,"A",{id:!0,class:!0,href:!0});var t5=i(rt);xn=s(t5,"SPAN",{});var l5=i(xn);m(ql.$$.fragment,l5),l5.forEach(t),t5.forEach(t),G_=f(Nd),Mn=s(Nd,"SPAN",{});var a5=i(Mn);U_=r(a5,"ZeRO DP+PP+TP"),a5.forEach(t),Nd.forEach(t),cc=f(e),nt=s(e,"P",{});var zd=i(nt);I_=r(zd,"One of the main features of DeepSpeed is ZeRO, which is a super-scalable extension of DP. It has already been discussed in "),Pi=s(zd,"A",{href:!0});var s5=i(Pi);S_=r(s5,"ZeRO Data Parallelism"),s5.forEach(t),A_=r(zd,". Normally it\u2019s a standalone feature that doesn\u2019t require PP or TP. But it can be combined with PP and TP."),zd.forEach(t),dc=f(e),wi=s(e,"P",{});var i5=i(wi);O_=r(i5,"When ZeRO-DP is combined with PP (and optionally TP) it typically enables only ZeRO stage 1 (optimizer sharding)."),i5.forEach(t),uc=f(e),_i=s(e,"P",{});var o5=i(_i);j_=r(o5,"While it\u2019s theoretically possible to use ZeRO stage 2 (gradient sharding) with Pipeline Parallelism, it will have bad performance impacts. There would need to be an additional reduce-scatter collective for every micro-batch to aggregate the gradients before sharding, which adds a potentially significant communication overhead. By nature of Pipeline Parallelism, small micro-batches are used and instead the focus is on trying to balance arithmetic intensity (micro-batch size) with minimizing the Pipeline bubble (number of micro-batches). Therefore those communication costs are going to hurt."),o5.forEach(t),mc=f(e),pt=s(e,"P",{});var Cd=i(pt);x_=r(Cd,"In addition, There are already fewer layers than normal due to PP and so the memory savings won\u2019t be huge. PP already reduces gradient size by "),$n=s(Cd,"CODE",{});var r5=i($n);M_=r(r5,"1/PP"),r5.forEach(t),$_=r(Cd,", and so gradient sharding savings on top of that are less significant than pure DP."),Cd.forEach(t),vc=f(e),yi=s(e,"P",{});var n5=i(yi);N_=r(n5,"ZeRO stage 3 is not a good choice either for the same reason - more inter-node communications required."),n5.forEach(t),Pc=f(e),bi=s(e,"P",{});var p5=i(bi);z_=r(p5,"And since we have ZeRO, the other benefit is ZeRO-Offload. Since this is stage 1 optimizer states can be offloaded to CPU."),p5.forEach(t),wc=f(e),gi=s(e,"P",{});var h5=i(gi);C_=r(h5,"Implementations:"),h5.forEach(t),_c=f(e),ht=s(e,"UL",{});var Rd=i(ht);ft=s(Rd,"LI",{});var Op=i(ft);Fl=s(Op,"A",{href:!0,rel:!0});var f5=i(Fl);R_=r(f5,"Megatron-DeepSpeed"),f5.forEach(t),Z_=r(Op," and "),Wl=s(Op,"A",{href:!0,rel:!0});var c5=i(Wl);B_=r(c5,"Megatron-Deepspeed from BigScience"),c5.forEach(t),V_=r(Op,", which is the fork of the former repo."),Op.forEach(t),q_=f(Rd),Nn=s(Rd,"LI",{});var d5=i(Nn);Hl=s(d5,"A",{href:!0,rel:!0});var u5=i(Hl);F_=r(u5,"OSLO"),u5.forEach(t),d5.forEach(t),Rd.forEach(t),yc=f(e),Ei=s(e,"P",{});var m5=i(Ei);W_=r(m5,"Important papers:"),m5.forEach(t),bc=f(e),Di=s(e,"UL",{});var v5=i(Di);zn=s(v5,"LI",{});var P5=i(zn);Yl=s(P5,"A",{href:!0,rel:!0});var w5=i(Yl);H_=r(w5,"Using DeepSpeed and Megatron to Train Megatron-Turing NLG 530B, A Large-Scale Generative Language Model"),w5.forEach(t),P5.forEach(t),v5.forEach(t),gc=f(e),Li=s(e,"P",{});var _5=i(Li);Y_=r(_5,"\u{1F917} Transformers status: not yet implemented, since we have no PP and TP."),_5.forEach(t),Ec=f(e),ke=s(e,"H2",{class:!0});var Zd=i(ke);ct=s(Zd,"A",{id:!0,class:!0,href:!0});var y5=i(ct);Cn=s(y5,"SPAN",{});var b5=i(Cn);m(Xl.$$.fragment,b5),b5.forEach(t),y5.forEach(t),X_=f(Zd),Rn=s(Zd,"SPAN",{});var g5=i(Rn);K_=r(g5,"FlexFlow"),g5.forEach(t),Zd.forEach(t),Dc=f(e),Kl=s(e,"P",{});var Jb=i(Kl);Jl=s(Jb,"A",{href:!0,rel:!0});var E5=i(Jl);J_=r(E5,"FlexFlow"),E5.forEach(t),Q_=r(Jb," also solves the parallelization problem in a slightly different approach."),Jb.forEach(t),Lc=f(e),Ql=s(e,"P",{});var Qb=i(Ql);ey=r(Qb,"Paper: "),ea=s(Qb,"A",{href:!0,rel:!0});var D5=i(ea);ty=r(D5,"\u201CBeyond Data and Model Parallelism for Deep Neural Networks\u201D by Zhihao Jia, Matei Zaharia, Alex Aiken"),D5.forEach(t),Qb.forEach(t),Tc=f(e),Ti=s(e,"P",{});var L5=i(Ti);ly=r(L5,"It performs a sort of 4D Parallelism over Sample-Operator-Attribute-Parameter."),L5.forEach(t),kc=f(e),z=s(e,"OL",{});var bt=i(z);Zn=s(bt,"LI",{});var T5=i(Zn);ay=r(T5,"Sample = Data Parallelism (sample-wise parallel)"),T5.forEach(t),sy=f(bt),Bn=s(bt,"LI",{});var k5=i(Bn);iy=r(k5,"Operator = Parallelize a single operation into several sub-operations"),k5.forEach(t),oy=f(bt),Vn=s(bt,"LI",{});var G5=i(Vn);ry=r(G5,"Attribute = Data Parallelism (length-wise parallel)"),G5.forEach(t),ny=f(bt),qn=s(bt,"LI",{});var U5=i(qn);py=r(U5,"Parameter = Model Parallelism (regardless of dimension - horizontal or vertical)"),U5.forEach(t),bt.forEach(t),Gc=f(e),ki=s(e,"P",{});var I5=i(ki);hy=r(I5,"Examples:"),I5.forEach(t),Uc=f(e),Gi=s(e,"UL",{});var S5=i(Gi);Fn=s(S5,"LI",{});var A5=i(Fn);fy=r(A5,"Sample"),A5.forEach(t),S5.forEach(t),Ic=f(e),Ui=s(e,"P",{});var O5=i(Ui);cy=r(O5,"Let\u2019s take 10 batches of sequence length 512. If we parallelize them by sample dimension into 2 devices, we get 10 x 512 which becomes be 5 x 2 x 512."),O5.forEach(t),Sc=f(e),Ii=s(e,"UL",{});var j5=i(Ii);Wn=s(j5,"LI",{});var x5=i(Wn);dy=r(x5,"Operator"),x5.forEach(t),j5.forEach(t),Ac=f(e),Si=s(e,"P",{});var M5=i(Si);uy=r(M5,"If we perform layer normalization, we compute std first and mean second, and then we can normalize data. Operator parallelism allows computing std and mean in parallel. So if we parallelize them by operator dimension into 2 devices (cuda:0, cuda:1), first we copy input data into both devices, and cuda:0 computes std, cuda:1 computes mean at the same time."),M5.forEach(t),Oc=f(e),Ai=s(e,"UL",{});var $5=i(Ai);Hn=s($5,"LI",{});var N5=i(Hn);my=r(N5,"Attribute"),N5.forEach(t),$5.forEach(t),jc=f(e),Oi=s(e,"P",{});var z5=i(Oi);vy=r(z5,"We have 10 batches of 512 length. If we parallelize them by attribute dimension into 2 devices, 10 x 512 will be 10 x 2 x 256."),z5.forEach(t),xc=f(e),ji=s(e,"UL",{});var C5=i(ji);Yn=s(C5,"LI",{});var R5=i(Yn);Py=r(R5,"Parameter"),R5.forEach(t),C5.forEach(t),Mc=f(e),xi=s(e,"P",{});var Z5=i(xi);wy=r(Z5,"It is similar with tensor model parallelism or naive layer-wise model parallelism."),Z5.forEach(t),$c=f(e),Mi=s(e,"P",{});var B5=i(Mi);$i=s(B5,"IMG",{src:!0,alt:!0}),B5.forEach(t),Nc=f(e),Ni=s(e,"P",{});var V5=i(Ni);_y=r(V5,"The significance of this framework is that it takes resources like (1) GPU/TPU/CPU vs. (2) RAM/DRAM vs. (3) fast-intra-connect/slow-inter-connect and it automatically optimizes all these  algorithmically deciding which parallelisation to use where."),V5.forEach(t),zc=f(e),zi=s(e,"P",{});var q5=i(zi);yy=r(q5,"One very important aspect is that FlexFlow is designed for optimizing DNN parallelizations for models with static and fixed workloads, since models with dynamic behavior may prefer different parallelization strategies across iterations."),q5.forEach(t),Cc=f(e),Ci=s(e,"P",{});var F5=i(Ci);by=r(F5,"So the promise is very attractive - it runs a 30min simulation on the cluster of choice and it comes up with the best strategy to utilise this specific environment. If you add/remove/replace any parts it\u2019ll run and re-optimize the plan for that. And then you can train. A different setup will have its own custom optimization."),F5.forEach(t),Rc=f(e),dt=s(e,"P",{});var Bd=i(dt);gy=r(Bd,"\u{1F917} Transformers status: not yet integrated. We already have our models FX-trace-able via "),ta=s(Bd,"A",{href:!0,rel:!0});var W5=i(ta);Ey=r(W5,"transformers.utils.fx"),W5.forEach(t),Dy=r(Bd,", which is a prerequisite for FlexFlow, so someone needs to figure out what needs to be done to make FlexFlow work with our models."),Bd.forEach(t),Zc=f(e),Ge=s(e,"H2",{class:!0});var Vd=i(Ge);ut=s(Vd,"A",{id:!0,class:!0,href:!0});var H5=i(ut);Xn=s(H5,"SPAN",{});var Y5=i(Xn);m(la.$$.fragment,Y5),Y5.forEach(t),H5.forEach(t),Ly=f(Vd),Kn=s(Vd,"SPAN",{});var X5=i(Kn);Ty=r(X5,"Which Strategy To Use When"),X5.forEach(t),Vd.forEach(t),Bc=f(e),Ri=s(e,"P",{});var K5=i(Ri);ky=r(K5,"Here is a very rough outline at which parallelism strategy to use when. The first on each list is typically faster."),K5.forEach(t),Vc=f(e),Zi=s(e,"P",{});var J5=i(Zi);Jn=s(J5,"STRONG",{});var Q5=i(Jn);Gy=r(Q5,"\u21E8 Single GPU"),Q5.forEach(t),J5.forEach(t),qc=f(e),J=s(e,"UL",{});var po=i(J);aa=s(po,"LI",{});var qd=i(aa);Qn=s(qd,"P",{});var e6=i(Qn);Uy=r(e6,"Model fits onto a single GPU:"),e6.forEach(t),Iy=f(qd),ep=s(qd,"OL",{});var t6=i(ep);tp=s(t6,"LI",{});var l6=i(tp);Sy=r(l6,"Normal use"),l6.forEach(t),t6.forEach(t),qd.forEach(t),Ay=f(po),sa=s(po,"LI",{});var Fd=i(sa);lp=s(Fd,"P",{});var a6=i(lp);Oy=r(a6,"Model doesn\u2019t fit onto a single GPU:"),a6.forEach(t),jy=f(Fd),ia=s(Fd,"OL",{});var Wd=i(ia);ap=s(Wd,"LI",{});var s6=i(ap);xy=r(s6,"ZeRO + Offload CPU and optionally NVMe"),s6.forEach(t),My=f(Wd),sp=s(Wd,"LI",{});var i6=i(sp);$y=r(i6,"as above plus Memory Centric Tiling (see below for details) if the largest layer can\u2019t fit into a single GPU"),i6.forEach(t),Wd.forEach(t),Fd.forEach(t),Ny=f(po),ip=s(po,"LI",{});var o6=i(ip);op=s(o6,"P",{});var r6=i(op);zy=r(r6,"Largest Layer not fitting into a single GPU:"),r6.forEach(t),o6.forEach(t),po.forEach(t),Fc=f(e),Bi=s(e,"OL",{});var n6=i(Bi);Ue=s(n6,"LI",{});var ho=i(Ue);Cy=r(ho,"ZeRO - Enable "),oa=s(ho,"A",{href:!0,rel:!0});var p6=i(oa);Ry=r(p6,"Memory Centric Tiling"),p6.forEach(t),Zy=r(ho," (MCT). It allows you to run arbitrarily large layers by automatically splitting them and executing them sequentially. MCT reduces the number of parameters that are live on a GPU, but it does not affect the activation memory. As this need is very rare as of this writing a manual override of "),rp=s(ho,"CODE",{});var h6=i(rp);By=r(h6,"torch.nn.Linear"),h6.forEach(t),Vy=r(ho," needs to be done by the user."),ho.forEach(t),n6.forEach(t),Wc=f(e),Vi=s(e,"P",{});var f6=i(Vi);np=s(f6,"STRONG",{});var c6=i(np);qy=r(c6,"\u21E8 Single Node / Multi-GPU"),c6.forEach(t),f6.forEach(t),Hc=f(e),Q=s(e,"UL",{});var fo=i(Q);ra=s(fo,"LI",{});var Hd=i(ra);pp=s(Hd,"P",{});var d6=i(pp);Fy=r(d6,"Model fits onto a single GPU:"),d6.forEach(t),Wy=f(Hd),na=s(Hd,"OL",{});var Yd=i(na);hp=s(Yd,"LI",{});var u6=i(hp);Hy=r(u6,"DDP - Distributed DP"),u6.forEach(t),Yy=f(Yd),fp=s(Yd,"LI",{});var m6=i(fp);Xy=r(m6,"ZeRO - may or may not be faster depending on the situation and configuration used"),m6.forEach(t),Yd.forEach(t),Hd.forEach(t),Ky=f(fo),pa=s(fo,"LI",{});var Xd=i(pa);cp=s(Xd,"P",{});var v6=i(cp);Jy=r(v6,"Model doesn\u2019t fit onto a single GPU:"),v6.forEach(t),Qy=f(Xd),Ie=s(Xd,"OL",{});var co=i(Ie);dp=s(co,"LI",{});var P6=i(dp);up=s(P6,"P",{});var w6=i(up);eb=r(w6,"PP"),w6.forEach(t),P6.forEach(t),tb=f(co),mp=s(co,"LI",{});var _6=i(mp);vp=s(_6,"P",{});var y6=i(vp);lb=r(y6,"ZeRO"),y6.forEach(t),_6.forEach(t),ab=f(co),Se=s(co,"LI",{});var uo=i(Se);Pp=s(uo,"P",{});var b6=i(Pp);sb=r(b6,"TP"),b6.forEach(t),ib=f(uo),wp=s(uo,"P",{});var g6=i(wp);ob=r(g6,"With very fast intra-node connectivity of NVLINK or NVSwitch all three should be mostly on par, without these PP will be faster than TP or ZeRO. The degree of TP may also make a difference. Best to experiment to find the winner on your particular setup."),g6.forEach(t),rb=f(uo),_p=s(uo,"P",{});var E6=i(_p);nb=r(E6,"TP is almost always used within a single node. That is TP size <= gpus per node."),E6.forEach(t),uo.forEach(t),co.forEach(t),Xd.forEach(t),pb=f(fo),ha=s(fo,"LI",{});var Kd=i(ha);yp=s(Kd,"P",{});var D6=i(yp);hb=r(D6,"Largest Layer not fitting into a single GPU:"),D6.forEach(t),fb=f(Kd),fa=s(Kd,"OL",{});var Jd=i(fa);bp=s(Jd,"LI",{});var L6=i(bp);cb=r(L6,"If not using ZeRO - must use TP, as PP alone won\u2019t be able to fit."),L6.forEach(t),db=f(Jd),gp=s(Jd,"LI",{});var T6=i(gp);ub=r(T6,"With ZeRO see the same entry for \u201CSingle GPU\u201D above"),T6.forEach(t),Jd.forEach(t),Kd.forEach(t),fo.forEach(t),Yc=f(e),qi=s(e,"P",{});var k6=i(qi);Ep=s(k6,"STRONG",{});var G6=i(Ep);mb=r(G6,"\u21E8 Multi-Node / Multi-GPU"),G6.forEach(t),k6.forEach(t),Xc=f(e),mt=s(e,"UL",{});var Qd=i(mt);ca=s(Qd,"LI",{});var eu=i(ca);Dp=s(eu,"P",{});var U6=i(Dp);vb=r(U6,"When you have fast inter-node connectivity:"),U6.forEach(t),Pb=f(eu),da=s(eu,"OL",{});var tu=i(da);Lp=s(tu,"LI",{});var I6=i(Lp);wb=r(I6,"ZeRO - as it requires close to no modifications to the model"),I6.forEach(t),_b=f(tu),Tp=s(tu,"LI",{});var S6=i(Tp);yb=r(S6,"PP+TP+DP - less communications, but requires massive changes to the model"),S6.forEach(t),tu.forEach(t),eu.forEach(t),bb=f(Qd),ua=s(Qd,"LI",{});var lu=i(ua);kp=s(lu,"P",{});var A6=i(kp);gb=r(A6,"when you have slow inter-node connectivity and still low on GPU memory:"),A6.forEach(t),Eb=f(lu),Gp=s(lu,"OL",{});var O6=i(Gp);Up=s(O6,"LI",{});var j6=i(Up);Db=r(j6,"DP+PP+TP+ZeRO-1"),j6.forEach(t),O6.forEach(t),lu.forEach(t),Qd.forEach(t),this.h()},h(){c(y,"name","hf:doc:metadata"),c(y,"content",JSON.stringify(B6)),c(O,"id","efficient-training-on-multiple-gpus"),c(O,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),c(O,"href","#efficient-training-on-multiple-gpus"),c(E,"class","relative group"),c(xe,"id","concepts"),c(xe,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),c(xe,"href","#concepts"),c(re,"class","relative group"),c(Me,"id","scalability-strategy"),c(Me,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),c(Me,"href","#scalability-strategy"),c(ne,"class","relative group"),c(Ne,"id","data-parallelism"),c(Ne,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),c(Ne,"href","#data-parallelism"),c(fe,"class","relative group"),c(Ot,"href","https://pytorch.org/docs/master/generated/torch.nn.DataParallel.html"),c(Ot,"rel","nofollow"),c(ze,"id","dp-vs-ddp"),c(ze,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),c(ze,"href","#dp-vs-ddp"),c(ce,"class","relative group"),c(Mt,"href","https://pytorch.org/docs/master/notes/ddp.html"),c(Mt,"rel","nofollow"),c(Ct,"href","https://pytorch.org/docs/master/generated/torch.nn.DataParallel.html"),c(Ct,"rel","nofollow"),c(Zt,"href","https://pytorch.org/docs/master/distributed.html"),c(Zt,"rel","nofollow"),c(Bt,"href","https://www.telesens.co/2019/04/04/distributed-data-parallel-training-using-pytorch-on-aws/"),c(Bt,"rel","nofollow"),c(Ma,"align","left"),c($a,"align","right"),c(Na,"align","left"),c(za,"align","right"),c(Ca,"align","left"),c(Ra,"align","right"),c(Za,"align","left"),c(Ba,"align","right"),c(qe,"id","zero-data-parallelism"),c(qe,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),c(qe,"href","#zero-data-parallelism"),c(_e,"class","relative group"),c(Wt,"href","https://www.microsoft.com/en-us/research/blog/zero-deepspeed-new-system-optimizations-enable-training-models-with-over-100-billion-parameters/"),c(Wt,"rel","nofollow"),ie(Ha.src,eg="https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/parallelism-zero.png")||c(Ha,"src",eg),c(Ha,"alt","DeepSpeed-Image-1"),c(Kt,"href","https://www.deepspeed.ai/features/#the-zero-redundancy-optimizer"),c(Kt,"rel","nofollow"),c(Jt,"href","https://github.com/facebookresearch/fairscale/#optimizer-state-sharding-zero"),c(Jt,"rel","nofollow"),c(Qt,"href","main_classes/trainer#trainer-integrations"),c(We,"id","naive-model-parallelism-vertical-and-pipeline-parallelism"),c(We,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),c(We,"href","#naive-model-parallelism-vertical-and-pipeline-parallelism"),c(be,"class","relative group"),c(ll,"href","https://ai.googleblog.com/2019/03/introducing-gpipe-open-source-library.html"),c(ll,"rel","nofollow"),ie(Es.src,tg="https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/parallelism-gpipe-bubble.png")||c(Es,"src",tg),c(Es,"alt","mp-pp"),c(sl,"href","https://github.com/pytorch/pytorch/pull/50693"),c(sl,"rel","nofollow"),c(ol,"href","https://pytorch.org/docs/stable/pipeline.html"),c(ol,"rel","nofollow"),c(rl,"href","https://github.com/pytorch/pytorch/blob/master/benchmarks/distributed/pipeline/pipe.py"),c(rl,"rel","nofollow"),c(nl,"href","https://fairscale.readthedocs.io/en/latest/tutorials/pipe.html"),c(nl,"rel","nofollow"),c(pl,"href","https://www.deepspeed.ai/tutorials/pipeline/"),c(pl,"rel","nofollow"),c(hl,"href","https://github.com/NVIDIA/Megatron-LM"),c(hl,"rel","nofollow"),c(fl,"href","https://github.com/microsoft/varuna"),c(fl,"rel","nofollow"),c(cl,"href","https://arxiv.org/abs/2111.05972"),c(cl,"rel","nofollow"),c(dl,"href","https://github.com/tunib-ai/oslo"),c(dl,"rel","nofollow"),c(ul,"href","https://docs.aws.amazon.com/sagemaker/latest/dg/model-parallel-core-features.html"),c(ul,"rel","nofollow"),ie(Ns.src,lg="https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/parallelism-sagemaker-interleaved-pipeline.png")||c(Ns,"src",lg),c(Ns,"alt","interleaved-pipeline-execution"),c(lt,"id","tensor-parallelism"),c(lt,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),c(lt,"href","#tensor-parallelism"),c(Ee,"class","relative group"),c(vl,"href","https://github.com/NVIDIA/Megatron-LM"),c(vl,"rel","nofollow"),c(Pl,"href","https://arxiv.org/abs/2104.04473"),c(Pl,"rel","nofollow"),ie(Zs.src,ag="https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/parallelism-tp-parallel_gemm.png")||c(Zs,"src",ag),c(Zs,"alt","Parallel GEMM"),ie(Bs.src,sg="https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/parallelism-tp-independent-gelu.png")||c(Bs,"src",sg),c(Bs,"alt","independent GeLU"),ie(Vs.src,ig="https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/parallelism-tp-parallel_shard_processing.png")||c(Vs,"src",ig),c(Vs,"alt","parallel shard processing"),ie(qs.src,og="https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/parallelism-tp-parallel_self_attention.png")||c(qs,"src",og),c(qs,"alt","parallel self-attention"),c(bl,"href","https://github.com/huggingface/transformers/issues/10321#issuecomment-783543530"),c(bl,"rel","nofollow"),c(gl,"href","https://github.com/anton-l"),c(gl,"rel","nofollow"),c(El,"href","https://www.deepspeed.ai/features/#model-parallelism"),c(El,"rel","nofollow"),c(Dl,"href","https://github.com/NVIDIA/Megatron-LM"),c(Dl,"rel","nofollow"),c(Ll,"href","https://github.com/tunib-ai/parallelformers"),c(Ll,"rel","nofollow"),c(Tl,"href","https://arxiv.org/abs/2111.05972"),c(Tl,"rel","nofollow"),c(kl,"href","https://github.com/tunib-ai/oslo"),c(kl,"rel","nofollow"),c(Ul,"href","https://github.com/tunib-ai/parallelformers"),c(Ul,"rel","nofollow"),c(Il,"href","https://www.deepspeed.ai/tutorials/inference-tutorial/"),c(Il,"rel","nofollow"),c(at,"id","dppp"),c(at,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),c(at,"href","#dppp"),c(De,"class","relative group"),c(Al,"href","https://www.deepspeed.ai/tutorials/pipeline/"),c(Al,"rel","nofollow"),ie(ii.src,rg="https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/parallelism-zero-dp-pp.png")||c(ii,"src",rg),c(ii,"alt","dp-pp-2d"),c(Ol,"href","https://github.com/microsoft/DeepSpeed"),c(Ol,"rel","nofollow"),c(jl,"href","https://github.com/NVIDIA/Megatron-LM"),c(jl,"rel","nofollow"),c(xl,"href","https://github.com/microsoft/varuna"),c(xl,"rel","nofollow"),c(Ml,"href","https://arxiv.org/abs/2111.05972"),c(Ml,"rel","nofollow"),c($l,"href","https://github.com/tunib-ai/oslo"),c($l,"rel","nofollow"),c(it,"id","dppptp"),c(it,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),c(it,"href","#dppptp"),c(Le,"class","relative group"),ie(ci.src,ng="https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/parallelism-deepspeed-3d.png")||c(ci,"src",ng),c(ci,"alt","dp-pp-tp-3d"),c(zl,"href","https://www.microsoft.com/en-us/research/blog/deepspeed-extreme-scale-model-training-for-everyone/"),c(zl,"rel","nofollow"),c(Cl,"href","https://github.com/microsoft/DeepSpeed"),c(Cl,"rel","nofollow"),c(Rl,"href","https://github.com/NVIDIA/Megatron-LM"),c(Rl,"rel","nofollow"),c(Zl,"href","https://github.com/microsoft/varuna"),c(Zl,"rel","nofollow"),c(Bl,"href","https://arxiv.org/abs/2111.05972"),c(Bl,"rel","nofollow"),c(Vl,"href","https://github.com/tunib-ai/oslo"),c(Vl,"rel","nofollow"),c(rt,"id","zero-dppptp"),c(rt,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),c(rt,"href","#zero-dppptp"),c(Te,"class","relative group"),c(Pi,"href","#zero-data-parallelism"),c(Fl,"href","https://github.com/microsoft/Megatron-DeepSpeed"),c(Fl,"rel","nofollow"),c(Wl,"href","https://github.com/bigscience-workshop/Megatron-DeepSpeed"),c(Wl,"rel","nofollow"),c(Hl,"href","https://github.com/tunib-ai/oslo"),c(Hl,"rel","nofollow"),c(Yl,"href","https://arxiv.org/abs/2201.11990"),c(Yl,"rel","nofollow"),c(ct,"id","flexflow"),c(ct,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),c(ct,"href","#flexflow"),c(ke,"class","relative group"),c(Jl,"href","https://github.com/flexflow/FlexFlow"),c(Jl,"rel","nofollow"),c(ea,"href","https://arxiv.org/abs/1807.05358"),c(ea,"rel","nofollow"),ie($i.src,pg="https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/parallelism-flexflow.jpeg")||c($i,"src",pg),c($i,"alt","flex-flow-soap"),c(ta,"href","https://github.com/huggingface/transformers/blob/master/src/transformers/utils/fx.py"),c(ta,"rel","nofollow"),c(ut,"id","which-strategy-to-use-when"),c(ut,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),c(ut,"href","#which-strategy-to-use-when"),c(Ge,"class","relative group"),c(oa,"href","https://deepspeed.readthedocs.io/en/latest/zero3.html#memory-centric-tiling"),c(oa,"rel","nofollow")},m(e,n){l(document.head,y),p(e,Ae,n),p(e,E,n),l(E,O),l(O,oe),v(S,oe,null),l(E,R),l(E,Oe),l(Oe,au),p(e,Mp,n),p(e,va,n),l(va,su),p(e,$p,n),v(je,e,n),p(e,Np,n),p(e,Pa,n),l(Pa,iu),p(e,zp,n),p(e,re,n),l(re,xe),l(xe,mo),v(gt,mo,null),l(re,ou),l(re,vo),l(vo,ru),p(e,Cp,n),p(e,wa,n),l(wa,nu),p(e,Rp,n),p(e,L,n),l(L,_a),l(_a,Po),l(Po,pu),l(_a,hu),l(L,fu),l(L,ya),l(ya,wo),l(wo,cu),l(ya,du),l(L,uu),l(L,ba),l(ba,_o),l(_o,mu),l(ba,vu),l(L,Pu),l(L,ga),l(ga,yo),l(yo,wu),l(ga,_u),l(L,yu),l(L,Ea),l(Ea,bo),l(bo,bu),l(Ea,gu),p(e,Zp,n),p(e,Da,n),l(Da,Eu),p(e,Bp,n),p(e,ne,n),l(ne,Me),l(Me,go),v(Et,go,null),l(ne,Du),l(ne,Eo),l(Eo,Lu),p(e,Vp,n),p(e,La,n),l(La,Do),l(Do,Tu),p(e,qp,n),p(e,Z,n),l(Z,Dt),l(Dt,Lo),l(Lo,ku),l(Dt,Gu),l(Dt,Lt),l(Lt,To),l(To,Uu),l(Lt,Iu),l(Lt,ko),l(ko,Su),l(Z,Au),l(Z,Tt),l(Tt,Go),l(Go,Ou),l(Tt,ju),l(Tt,pe),l(pe,Uo),l(Uo,Io),l(Io,xu),l(pe,Mu),l(pe,So),l(So,Ao),l(Ao,$u),l(pe,Nu),l(pe,he),l(he,Oo),l(Oo,zu),l(he,Cu),l(he,jo),l(jo,Ru),l(he,Zu),l(he,xo),l(xo,Bu),l(Z,Vu),l(Z,kt),l(kt,Mo),l(Mo,qu),l(kt,Fu),l(kt,Gt),l(Gt,$o),l($o,Wu),l(Gt,Hu),l(Gt,No),l(No,Yu),p(e,Fp,n),p(e,Ta,n),l(Ta,zo),l(zo,Xu),p(e,Wp,n),p(e,$e,n),l($e,Ut),l(Ut,Co),l(Co,Ku),l(Ut,Ju),l(Ut,It),l(It,Ro),l(Ro,Qu),l(It,em),l(It,Zo),l(Zo,tm),l($e,lm),l($e,St),l(St,Bo),l(Bo,am),l(St,sm),l(St,Vo),l(Vo,qo),l(qo,im),p(e,Hp,n),p(e,fe,n),l(fe,Ne),l(Ne,Fo),v(At,Fo,null),l(fe,om),l(fe,Wo),l(Wo,rm),p(e,Yp,n),p(e,j,n),l(j,nm),l(j,Ho),l(Ho,pm),l(j,hm),l(j,Yo),l(Yo,fm),l(j,cm),l(j,Ot),l(Ot,dm),l(j,um),p(e,Xp,n),p(e,ce,n),l(ce,ze),l(ze,Xo),v(jt,Xo,null),l(ce,mm),l(ce,Ko),l(Ko,vm),p(e,Kp,n),p(e,de,n),l(de,Jo),l(Jo,Pm),l(de,wm),l(de,Qo),l(Qo,_m),l(de,ym),p(e,Jp,n),p(e,Ce,n),l(Ce,er),l(er,bm),l(Ce,gm),l(Ce,tr),l(tr,Em),p(e,Qp,n),p(e,ka,n),l(ka,Dm),p(e,eh,n),p(e,xt,n),l(xt,Mt),l(Mt,Lm),l(xt,Tm),p(e,th,n),p(e,Re,n),l(Re,lr),l(lr,km),l(Re,Gm),l(Re,Ga),l(Ga,Um),l(Ga,$t),l($t,ar),l(ar,Im),l($t,Sm),l($t,Nt),l(Nt,Am),l(Nt,sr),l(sr,Om),l(Nt,jm),p(e,lh,n),p(e,zt,n),l(zt,Ct),l(Ct,xm),l(zt,Mm),p(e,ah,n),p(e,Ua,n),l(Ua,$m),p(e,sh,n),p(e,T,n),l(T,ir),l(ir,Nm),l(T,zm),l(T,or),l(or,Cm),l(T,Rm),l(T,Rt),l(Rt,Zm),l(Rt,rr),l(rr,Bm),l(Rt,Vm),l(T,qm),l(T,Ia),l(Ia,Fm),l(Ia,nr),l(nr,Wm),l(T,Hm),l(T,pr),l(pr,Ym),p(e,ih,n),p(e,Sa,n),l(Sa,Xm),p(e,oh,n),p(e,Ze,n),l(Ze,Km),l(Ze,Zt),l(Zt,Jm),l(Ze,Qm),p(e,rh,n),p(e,Aa,n),l(Aa,ev),p(e,nh,n),p(e,Oa,n),l(Oa,tv),p(e,ph,n),p(e,ja,n),l(ja,lv),p(e,hh,n),p(e,Be,n),l(Be,av),l(Be,Bt),l(Bt,sv),l(Be,iv),p(e,fh,n),p(e,xa,n),l(xa,ov),p(e,ch,n),p(e,Ve,n),l(Ve,hr),l(hr,ue),l(ue,Ma),l(Ma,rv),l(ue,nv),l(ue,fr),l(fr,pv),l(ue,hv),l(ue,$a),l($a,fv),l(Ve,cv),l(Ve,me),l(me,ve),l(ve,Na),l(Na,dv),l(ve,uv),l(ve,cr),l(cr,mv),l(ve,vv),l(ve,za),l(za,Pv),l(me,wv),l(me,Pe),l(Pe,Ca),l(Ca,_v),l(Pe,yv),l(Pe,dr),l(dr,bv),l(Pe,gv),l(Pe,Ra),l(Ra,Ev),l(me,Dv),l(me,we),l(we,Za),l(Za,Lv),l(we,Tv),l(we,ur),l(ur,kv),l(we,Gv),l(we,Ba),l(Ba,Uv),p(e,dh,n),p(e,Va,n),l(Va,Iv),p(e,uh,n),p(e,qa,n),l(qa,Sv),p(e,mh,n),p(e,Fa,n),l(Fa,Av),p(e,vh,n),p(e,Wa,n),l(Wa,Ov),p(e,Ph,n),p(e,Vt,n),l(Vt,mr),l(mr,jv),l(Vt,xv),p(e,wh,n),v(qt,e,n),p(e,_h,n),p(e,D,n),l(D,Mv),l(D,vr),l(vr,$v),l(D,Nv),l(D,Pr),l(Pr,zv),l(D,Cv),l(D,wr),l(wr,Rv),l(D,Zv),l(D,_r),l(_r,Bv),l(D,Vv),l(D,yr),l(yr,qv),p(e,yh,n),p(e,_e,n),l(_e,qe),l(qe,br),v(Ft,br,null),l(_e,Fv),l(_e,gr),l(gr,Wv),p(e,bh,n),p(e,ye,n),l(ye,Hv),l(ye,Wt),l(Wt,Yv),l(ye,Xv),l(ye,Ha),p(e,gh,n),p(e,Fe,n),l(Fe,Kv),l(Fe,Er),l(Er,Jv),l(Fe,Qv),p(e,Eh,n),p(e,Ya,n),l(Ya,eP),p(e,Dh,n),v(Ht,e,n),p(e,Lh,n),p(e,Xa,n),l(Xa,tP),p(e,Th,n),p(e,Ka,n),l(Ka,lP),p(e,kh,n),v(Yt,e,n),p(e,Gh,n),p(e,Ja,n),l(Ja,aP),p(e,Uh,n),p(e,Qa,n),l(Qa,sP),p(e,Ih,n),v(Xt,e,n),p(e,Sh,n),p(e,es,n),l(es,iP),p(e,Ah,n),p(e,ts,n),l(ts,oP),p(e,Oh,n),p(e,ls,n),l(ls,rP),p(e,jh,n),p(e,as,n),l(as,nP),p(e,xh,n),p(e,ss,n),l(ss,pP),p(e,Mh,n),p(e,is,n),l(is,hP),p(e,$h,n),p(e,os,n),l(os,fP),p(e,Nh,n),p(e,rs,n),l(rs,cP),p(e,zh,n),p(e,ns,n),l(ns,dP),p(e,Ch,n),p(e,B,n),l(B,Dr),l(Dr,uP),l(B,mP),l(B,Lr),l(Lr,vP),l(B,PP),l(B,Tr),l(Tr,wP),p(e,Rh,n),p(e,ps,n),l(ps,_P),p(e,Zh,n),p(e,hs,n),l(hs,yP),p(e,Bh,n),p(e,fs,n),l(fs,bP),p(e,Vh,n),p(e,cs,n),l(cs,gP),p(e,qh,n),p(e,ds,n),l(ds,EP),p(e,Fh,n),p(e,V,n),l(V,us),l(us,Kt),l(Kt,DP),l(us,LP),l(V,TP),l(V,ms),l(ms,Jt),l(Jt,kP),l(ms,GP),l(V,UP),l(V,kr),l(kr,Qt),l(Qt,Gr),l(Gr,IP),l(Qt,SP),p(e,Wh,n),p(e,be,n),l(be,We),l(We,Ur),v(el,Ur,null),l(be,AP),l(be,Ir),l(Ir,OP),p(e,Hh,n),p(e,He,n),l(He,jP),l(He,Sr),l(Sr,xP),l(He,MP),p(e,Yh,n),p(e,vs,n),l(vs,$P),p(e,Xh,n),v(tl,e,n),p(e,Kh,n),p(e,Ps,n),l(Ps,NP),p(e,Jh,n),p(e,ws,n),l(ws,zP),p(e,Qh,n),p(e,_s,n),l(_s,CP),p(e,ef,n),p(e,ys,n),l(ys,RP),p(e,tf,n),p(e,Ye,n),l(Ye,Ar),l(Ar,ZP),l(Ye,BP),l(Ye,Or),l(Or,VP),p(e,lf,n),p(e,bs,n),l(bs,qP),p(e,af,n),p(e,Xe,n),l(Xe,FP),l(Xe,ll),l(ll,WP),l(Xe,HP),p(e,sf,n),p(e,gs,n),l(gs,Es),p(e,of,n),p(e,Ds,n),l(Ds,YP),p(e,rf,n),p(e,Ls,n),l(Ls,XP),p(e,nf,n),p(e,q,n),l(q,KP),l(q,jr),l(jr,JP),l(q,QP),l(q,xr),l(xr,e1),l(q,t1),p(e,pf,n),p(e,Ke,n),l(Ke,l1),l(Ke,Mr),l(Mr,a1),l(Ke,s1),p(e,hf,n),p(e,Je,n),l(Je,i1),l(Je,$r),l($r,o1),l(Je,r1),p(e,ff,n),p(e,F,n),l(F,n1),l(F,Nr),l(Nr,p1),l(F,h1),l(F,zr),l(zr,f1),l(F,c1),p(e,cf,n),p(e,Ts,n),l(Ts,d1),p(e,df,n),p(e,W,n),l(W,u1),l(W,Cr),l(Cr,m1),l(W,v1),l(W,Rr),l(Rr,P1),l(W,w1),p(e,uf,n),p(e,x,n),l(x,_1),l(x,Zr),l(Zr,y1),l(x,b1),l(x,Br),l(Br,g1),l(x,E1),l(x,Vr),l(Vr,D1),l(x,L1),p(e,mf,n),p(e,ks,n),l(ks,T1),p(e,vf,n),p(e,Gs,n),l(Gs,k1),p(e,Pf,n),p(e,M,n),l(M,qr),l(qr,G1),l(M,U1),l(M,Fr),l(Fr,I1),l(M,S1),l(M,Wr),l(Wr,A1),l(M,O1),l(M,Hr),l(Hr,j1),p(e,wf,n),p(e,Us,n),l(Us,x1),p(e,_f,n),p(e,Qe,n),l(Qe,Yr),l(Yr,M1),l(Qe,$1),l(Qe,Xr),l(Xr,N1),p(e,yf,n),p(e,Is,n),l(Is,z1),p(e,bf,n),p(e,$,n),l($,al),l(al,C1),l(al,Kr),l(Kr,R1),l(al,Z1),l($,B1),l($,Ss),l(Ss,V1),l(Ss,sl),l(sl,q1),l($,F1),l($,Jr),l(Jr,W1),l($,H1),l($,Qr),l(Qr,Y1),p(e,gf,n),p(e,As,n),l(As,X1),p(e,Ef,n),p(e,Os,n),l(Os,K1),p(e,Df,n),p(e,b,n),l(b,il),l(il,ol),l(ol,J1),l(il,Q1),l(il,rl),l(rl,ew),l(b,tw),l(b,en),l(en,nl),l(nl,lw),l(b,aw),l(b,tn),l(tn,pl),l(pl,sw),l(b,iw),l(b,js),l(js,hl),l(hl,ow),l(js,rw),l(b,nw),l(b,ln),l(ln,fl),l(fl,pw),l(b,hw),l(b,xs),l(xs,cl),l(cl,fw),l(xs,cw),l(b,dw),l(b,Ms),l(Ms,dl),l(dl,uw),l(Ms,mw),p(e,Lf,n),p(e,et,n),l(et,vw),l(et,an),l(an,Pw),l(et,ww),p(e,Tf,n),p(e,$s,n),l($s,_w),p(e,kf,n),p(e,ge,n),l(ge,yw),l(ge,ul),l(ul,bw),l(ge,gw),l(ge,Ns),p(e,Gf,n),p(e,zs,n),l(zs,Ew),p(e,Uf,n),p(e,Cs,n),l(Cs,Dw),p(e,If,n),p(e,tt,n),l(tt,Lw),l(tt,sn),l(sn,Tw),l(tt,kw),p(e,Sf,n),p(e,Ee,n),l(Ee,lt),l(lt,on),v(ml,on,null),l(Ee,Gw),l(Ee,rn),l(rn,Uw),p(e,Af,n),p(e,Rs,n),l(Rs,Iw),p(e,Of,n),p(e,H,n),l(H,Sw),l(H,vl),l(vl,Aw),l(H,Ow),l(H,Pl),l(Pl,jw),l(H,xw),p(e,jf,n),p(e,Y,n),l(Y,Mw),l(Y,nn),l(nn,$w),l(Y,Nw),l(Y,pn),l(pn,zw),l(Y,Cw),p(e,xf,n),p(e,k,n),l(k,Rw),l(k,hn),l(hn,Zw),l(k,Bw),l(k,fn),l(fn,Vw),l(k,qw),l(k,cn),l(cn,Fw),l(k,Ww),l(k,dn),l(dn,Hw),l(k,Yw),p(e,Mf,n),p(e,wl,n),l(wl,Xw),l(wl,Zs),p(e,$f,n),p(e,d,n),l(d,Kw),l(d,un),l(un,Jw),l(d,Qw),l(d,mn),l(mn,e2),l(d,t2),l(d,vn),l(vn,l2),l(d,a2),l(d,Pn),l(Pn,s2),l(d,i2),l(d,wn),l(wn,o2),l(d,r2),l(d,_n),l(_n,n2),l(d,p2),l(d,yn),l(yn,h2),l(d,f2),l(d,Bs),p(e,Nf,n),p(e,_l,n),l(_l,c2),l(_l,Vs),p(e,zf,n),p(e,yl,n),l(yl,d2),l(yl,qs),p(e,Cf,n),p(e,Fs,n),l(Fs,u2),p(e,Rf,n),p(e,X,n),l(X,m2),l(X,bl),l(bl,v2),l(X,P2),l(X,gl),l(gl,w2),l(X,_2),p(e,Zf,n),p(e,Ws,n),l(Ws,y2),p(e,Bf,n),p(e,Hs,n),l(Hs,b2),p(e,Vf,n),p(e,Ys,n),l(Ys,Xs),l(Xs,g2),l(Xs,El),l(El,E2),p(e,qf,n),p(e,Ks,n),l(Ks,D2),p(e,Ff,n),p(e,N,n),l(N,Js),l(Js,Dl),l(Dl,L2),l(Js,T2),l(N,k2),l(N,Qs),l(Qs,Ll),l(Ll,G2),l(Qs,U2),l(N,I2),l(N,ei),l(ei,Tl),l(Tl,S2),l(ei,A2),l(N,O2),l(N,ti),l(ti,kl),l(kl,j2),l(ti,x2),p(e,Wf,n),p(e,li,n),l(li,M2),p(e,Hf,n),p(e,K,n),l(K,bn),l(bn,$2),l(K,N2),l(K,Gl),l(Gl,z2),l(Gl,Ul),l(Ul,C2),l(Gl,R2),l(K,Z2),l(K,ai),l(ai,B2),l(ai,Il),l(Il,V2),p(e,Yf,n),p(e,De,n),l(De,at),l(at,gn),v(Sl,gn,null),l(De,q2),l(De,En),l(En,F2),p(e,Xf,n),p(e,st,n),l(st,W2),l(st,Al),l(Al,H2),l(st,Y2),p(e,Kf,n),p(e,si,n),l(si,ii),p(e,Jf,n),p(e,oi,n),l(oi,X2),p(e,Qf,n),p(e,ri,n),l(ri,K2),p(e,ec,n),p(e,ni,n),l(ni,J2),p(e,tc,n),p(e,G,n),l(G,Dn),l(Dn,Ol),l(Ol,Q2),l(G,e_),l(G,Ln),l(Ln,jl),l(jl,t_),l(G,l_),l(G,Tn),l(Tn,xl),l(xl,a_),l(G,s_),l(G,kn),l(kn,Ml),l(Ml,i_),l(G,o_),l(G,Gn),l(Gn,$l),l($l,r_),p(e,lc,n),p(e,pi,n),l(pi,n_),p(e,ac,n),p(e,Le,n),l(Le,it),l(it,Un),v(Nl,Un,null),l(Le,p_),l(Le,In),l(In,h_),p(e,sc,n),p(e,hi,n),l(hi,f_),p(e,ic,n),p(e,fi,n),l(fi,ci),p(e,oc,n),p(e,ot,n),l(ot,c_),l(ot,zl),l(zl,d_),l(ot,u_),p(e,rc,n),p(e,di,n),l(di,m_),p(e,nc,n),p(e,ui,n),l(ui,v_),p(e,pc,n),p(e,U,n),l(U,mi),l(mi,Cl),l(Cl,P_),l(mi,w_),l(U,__),l(U,Sn),l(Sn,Rl),l(Rl,y_),l(U,b_),l(U,An),l(An,Zl),l(Zl,g_),l(U,E_),l(U,On),l(On,Bl),l(Bl,D_),l(U,L_),l(U,jn),l(jn,Vl),l(Vl,T_),p(e,hc,n),p(e,vi,n),l(vi,k_),p(e,fc,n),p(e,Te,n),l(Te,rt),l(rt,xn),v(ql,xn,null),l(Te,G_),l(Te,Mn),l(Mn,U_),p(e,cc,n),p(e,nt,n),l(nt,I_),l(nt,Pi),l(Pi,S_),l(nt,A_),p(e,dc,n),p(e,wi,n),l(wi,O_),p(e,uc,n),p(e,_i,n),l(_i,j_),p(e,mc,n),p(e,pt,n),l(pt,x_),l(pt,$n),l($n,M_),l(pt,$_),p(e,vc,n),p(e,yi,n),l(yi,N_),p(e,Pc,n),p(e,bi,n),l(bi,z_),p(e,wc,n),p(e,gi,n),l(gi,C_),p(e,_c,n),p(e,ht,n),l(ht,ft),l(ft,Fl),l(Fl,R_),l(ft,Z_),l(ft,Wl),l(Wl,B_),l(ft,V_),l(ht,q_),l(ht,Nn),l(Nn,Hl),l(Hl,F_),p(e,yc,n),p(e,Ei,n),l(Ei,W_),p(e,bc,n),p(e,Di,n),l(Di,zn),l(zn,Yl),l(Yl,H_),p(e,gc,n),p(e,Li,n),l(Li,Y_),p(e,Ec,n),p(e,ke,n),l(ke,ct),l(ct,Cn),v(Xl,Cn,null),l(ke,X_),l(ke,Rn),l(Rn,K_),p(e,Dc,n),p(e,Kl,n),l(Kl,Jl),l(Jl,J_),l(Kl,Q_),p(e,Lc,n),p(e,Ql,n),l(Ql,ey),l(Ql,ea),l(ea,ty),p(e,Tc,n),p(e,Ti,n),l(Ti,ly),p(e,kc,n),p(e,z,n),l(z,Zn),l(Zn,ay),l(z,sy),l(z,Bn),l(Bn,iy),l(z,oy),l(z,Vn),l(Vn,ry),l(z,ny),l(z,qn),l(qn,py),p(e,Gc,n),p(e,ki,n),l(ki,hy),p(e,Uc,n),p(e,Gi,n),l(Gi,Fn),l(Fn,fy),p(e,Ic,n),p(e,Ui,n),l(Ui,cy),p(e,Sc,n),p(e,Ii,n),l(Ii,Wn),l(Wn,dy),p(e,Ac,n),p(e,Si,n),l(Si,uy),p(e,Oc,n),p(e,Ai,n),l(Ai,Hn),l(Hn,my),p(e,jc,n),p(e,Oi,n),l(Oi,vy),p(e,xc,n),p(e,ji,n),l(ji,Yn),l(Yn,Py),p(e,Mc,n),p(e,xi,n),l(xi,wy),p(e,$c,n),p(e,Mi,n),l(Mi,$i),p(e,Nc,n),p(e,Ni,n),l(Ni,_y),p(e,zc,n),p(e,zi,n),l(zi,yy),p(e,Cc,n),p(e,Ci,n),l(Ci,by),p(e,Rc,n),p(e,dt,n),l(dt,gy),l(dt,ta),l(ta,Ey),l(dt,Dy),p(e,Zc,n),p(e,Ge,n),l(Ge,ut),l(ut,Xn),v(la,Xn,null),l(Ge,Ly),l(Ge,Kn),l(Kn,Ty),p(e,Bc,n),p(e,Ri,n),l(Ri,ky),p(e,Vc,n),p(e,Zi,n),l(Zi,Jn),l(Jn,Gy),p(e,qc,n),p(e,J,n),l(J,aa),l(aa,Qn),l(Qn,Uy),l(aa,Iy),l(aa,ep),l(ep,tp),l(tp,Sy),l(J,Ay),l(J,sa),l(sa,lp),l(lp,Oy),l(sa,jy),l(sa,ia),l(ia,ap),l(ap,xy),l(ia,My),l(ia,sp),l(sp,$y),l(J,Ny),l(J,ip),l(ip,op),l(op,zy),p(e,Fc,n),p(e,Bi,n),l(Bi,Ue),l(Ue,Cy),l(Ue,oa),l(oa,Ry),l(Ue,Zy),l(Ue,rp),l(rp,By),l(Ue,Vy),p(e,Wc,n),p(e,Vi,n),l(Vi,np),l(np,qy),p(e,Hc,n),p(e,Q,n),l(Q,ra),l(ra,pp),l(pp,Fy),l(ra,Wy),l(ra,na),l(na,hp),l(hp,Hy),l(na,Yy),l(na,fp),l(fp,Xy),l(Q,Ky),l(Q,pa),l(pa,cp),l(cp,Jy),l(pa,Qy),l(pa,Ie),l(Ie,dp),l(dp,up),l(up,eb),l(Ie,tb),l(Ie,mp),l(mp,vp),l(vp,lb),l(Ie,ab),l(Ie,Se),l(Se,Pp),l(Pp,sb),l(Se,ib),l(Se,wp),l(wp,ob),l(Se,rb),l(Se,_p),l(_p,nb),l(Q,pb),l(Q,ha),l(ha,yp),l(yp,hb),l(ha,fb),l(ha,fa),l(fa,bp),l(bp,cb),l(fa,db),l(fa,gp),l(gp,ub),p(e,Yc,n),p(e,qi,n),l(qi,Ep),l(Ep,mb),p(e,Xc,n),p(e,mt,n),l(mt,ca),l(ca,Dp),l(Dp,vb),l(ca,Pb),l(ca,da),l(da,Lp),l(Lp,wb),l(da,_b),l(da,Tp),l(Tp,yb),l(mt,bb),l(mt,ua),l(ua,kp),l(kp,gb),l(ua,Eb),l(ua,Gp),l(Gp,Up),l(Up,Db),Kc=!0},p(e,[n]){const ma={};n&2&&(ma.$$scope={dirty:n,ctx:e}),je.$set(ma)},i(e){Kc||(P(S.$$.fragment,e),P(je.$$.fragment,e),P(gt.$$.fragment,e),P(Et.$$.fragment,e),P(At.$$.fragment,e),P(jt.$$.fragment,e),P(qt.$$.fragment,e),P(Ft.$$.fragment,e),P(Ht.$$.fragment,e),P(Yt.$$.fragment,e),P(Xt.$$.fragment,e),P(el.$$.fragment,e),P(tl.$$.fragment,e),P(ml.$$.fragment,e),P(Sl.$$.fragment,e),P(Nl.$$.fragment,e),P(ql.$$.fragment,e),P(Xl.$$.fragment,e),P(la.$$.fragment,e),Kc=!0)},o(e){w(S.$$.fragment,e),w(je.$$.fragment,e),w(gt.$$.fragment,e),w(Et.$$.fragment,e),w(At.$$.fragment,e),w(jt.$$.fragment,e),w(qt.$$.fragment,e),w(Ft.$$.fragment,e),w(Ht.$$.fragment,e),w(Yt.$$.fragment,e),w(Xt.$$.fragment,e),w(el.$$.fragment,e),w(tl.$$.fragment,e),w(ml.$$.fragment,e),w(Sl.$$.fragment,e),w(Nl.$$.fragment,e),w(ql.$$.fragment,e),w(Xl.$$.fragment,e),w(la.$$.fragment,e),Kc=!1},d(e){t(y),e&&t(Ae),e&&t(E),_(S),e&&t(Mp),e&&t(va),e&&t($p),_(je,e),e&&t(Np),e&&t(Pa),e&&t(zp),e&&t(re),_(gt),e&&t(Cp),e&&t(wa),e&&t(Rp),e&&t(L),e&&t(Zp),e&&t(Da),e&&t(Bp),e&&t(ne),_(Et),e&&t(Vp),e&&t(La),e&&t(qp),e&&t(Z),e&&t(Fp),e&&t(Ta),e&&t(Wp),e&&t($e),e&&t(Hp),e&&t(fe),_(At),e&&t(Yp),e&&t(j),e&&t(Xp),e&&t(ce),_(jt),e&&t(Kp),e&&t(de),e&&t(Jp),e&&t(Ce),e&&t(Qp),e&&t(ka),e&&t(eh),e&&t(xt),e&&t(th),e&&t(Re),e&&t(lh),e&&t(zt),e&&t(ah),e&&t(Ua),e&&t(sh),e&&t(T),e&&t(ih),e&&t(Sa),e&&t(oh),e&&t(Ze),e&&t(rh),e&&t(Aa),e&&t(nh),e&&t(Oa),e&&t(ph),e&&t(ja),e&&t(hh),e&&t(Be),e&&t(fh),e&&t(xa),e&&t(ch),e&&t(Ve),e&&t(dh),e&&t(Va),e&&t(uh),e&&t(qa),e&&t(mh),e&&t(Fa),e&&t(vh),e&&t(Wa),e&&t(Ph),e&&t(Vt),e&&t(wh),_(qt,e),e&&t(_h),e&&t(D),e&&t(yh),e&&t(_e),_(Ft),e&&t(bh),e&&t(ye),e&&t(gh),e&&t(Fe),e&&t(Eh),e&&t(Ya),e&&t(Dh),_(Ht,e),e&&t(Lh),e&&t(Xa),e&&t(Th),e&&t(Ka),e&&t(kh),_(Yt,e),e&&t(Gh),e&&t(Ja),e&&t(Uh),e&&t(Qa),e&&t(Ih),_(Xt,e),e&&t(Sh),e&&t(es),e&&t(Ah),e&&t(ts),e&&t(Oh),e&&t(ls),e&&t(jh),e&&t(as),e&&t(xh),e&&t(ss),e&&t(Mh),e&&t(is),e&&t($h),e&&t(os),e&&t(Nh),e&&t(rs),e&&t(zh),e&&t(ns),e&&t(Ch),e&&t(B),e&&t(Rh),e&&t(ps),e&&t(Zh),e&&t(hs),e&&t(Bh),e&&t(fs),e&&t(Vh),e&&t(cs),e&&t(qh),e&&t(ds),e&&t(Fh),e&&t(V),e&&t(Wh),e&&t(be),_(el),e&&t(Hh),e&&t(He),e&&t(Yh),e&&t(vs),e&&t(Xh),_(tl,e),e&&t(Kh),e&&t(Ps),e&&t(Jh),e&&t(ws),e&&t(Qh),e&&t(_s),e&&t(ef),e&&t(ys),e&&t(tf),e&&t(Ye),e&&t(lf),e&&t(bs),e&&t(af),e&&t(Xe),e&&t(sf),e&&t(gs),e&&t(of),e&&t(Ds),e&&t(rf),e&&t(Ls),e&&t(nf),e&&t(q),e&&t(pf),e&&t(Ke),e&&t(hf),e&&t(Je),e&&t(ff),e&&t(F),e&&t(cf),e&&t(Ts),e&&t(df),e&&t(W),e&&t(uf),e&&t(x),e&&t(mf),e&&t(ks),e&&t(vf),e&&t(Gs),e&&t(Pf),e&&t(M),e&&t(wf),e&&t(Us),e&&t(_f),e&&t(Qe),e&&t(yf),e&&t(Is),e&&t(bf),e&&t($),e&&t(gf),e&&t(As),e&&t(Ef),e&&t(Os),e&&t(Df),e&&t(b),e&&t(Lf),e&&t(et),e&&t(Tf),e&&t($s),e&&t(kf),e&&t(ge),e&&t(Gf),e&&t(zs),e&&t(Uf),e&&t(Cs),e&&t(If),e&&t(tt),e&&t(Sf),e&&t(Ee),_(ml),e&&t(Af),e&&t(Rs),e&&t(Of),e&&t(H),e&&t(jf),e&&t(Y),e&&t(xf),e&&t(k),e&&t(Mf),e&&t(wl),e&&t($f),e&&t(d),e&&t(Nf),e&&t(_l),e&&t(zf),e&&t(yl),e&&t(Cf),e&&t(Fs),e&&t(Rf),e&&t(X),e&&t(Zf),e&&t(Ws),e&&t(Bf),e&&t(Hs),e&&t(Vf),e&&t(Ys),e&&t(qf),e&&t(Ks),e&&t(Ff),e&&t(N),e&&t(Wf),e&&t(li),e&&t(Hf),e&&t(K),e&&t(Yf),e&&t(De),_(Sl),e&&t(Xf),e&&t(st),e&&t(Kf),e&&t(si),e&&t(Jf),e&&t(oi),e&&t(Qf),e&&t(ri),e&&t(ec),e&&t(ni),e&&t(tc),e&&t(G),e&&t(lc),e&&t(pi),e&&t(ac),e&&t(Le),_(Nl),e&&t(sc),e&&t(hi),e&&t(ic),e&&t(fi),e&&t(oc),e&&t(ot),e&&t(rc),e&&t(di),e&&t(nc),e&&t(ui),e&&t(pc),e&&t(U),e&&t(hc),e&&t(vi),e&&t(fc),e&&t(Te),_(ql),e&&t(cc),e&&t(nt),e&&t(dc),e&&t(wi),e&&t(uc),e&&t(_i),e&&t(mc),e&&t(pt),e&&t(vc),e&&t(yi),e&&t(Pc),e&&t(bi),e&&t(wc),e&&t(gi),e&&t(_c),e&&t(ht),e&&t(yc),e&&t(Ei),e&&t(bc),e&&t(Di),e&&t(gc),e&&t(Li),e&&t(Ec),e&&t(ke),_(Xl),e&&t(Dc),e&&t(Kl),e&&t(Lc),e&&t(Ql),e&&t(Tc),e&&t(Ti),e&&t(kc),e&&t(z),e&&t(Gc),e&&t(ki),e&&t(Uc),e&&t(Gi),e&&t(Ic),e&&t(Ui),e&&t(Sc),e&&t(Ii),e&&t(Ac),e&&t(Si),e&&t(Oc),e&&t(Ai),e&&t(jc),e&&t(Oi),e&&t(xc),e&&t(ji),e&&t(Mc),e&&t(xi),e&&t($c),e&&t(Mi),e&&t(Nc),e&&t(Ni),e&&t(zc),e&&t(zi),e&&t(Cc),e&&t(Ci),e&&t(Rc),e&&t(dt),e&&t(Zc),e&&t(Ge),_(la),e&&t(Bc),e&&t(Ri),e&&t(Vc),e&&t(Zi),e&&t(qc),e&&t(J),e&&t(Fc),e&&t(Bi),e&&t(Wc),e&&t(Vi),e&&t(Hc),e&&t(Q),e&&t(Yc),e&&t(qi),e&&t(Xc),e&&t(mt)}}}const B6={local:"efficient-training-on-multiple-gpus",sections:[{local:"concepts",title:"Concepts"},{local:"scalability-strategy",title:"Scalability Strategy"},{local:"data-parallelism",sections:[{local:"dp-vs-ddp",title:"DP vs DDP"}],title:"Data Parallelism"},{local:"zero-data-parallelism",title:"ZeRO Data Parallelism"},{local:"naive-model-parallelism-vertical-and-pipeline-parallelism",title:"Naive Model Parallelism (Vertical) and Pipeline Parallelism"},{local:"tensor-parallelism",title:"Tensor Parallelism"},{local:"dppp",title:"DP+PP"},{local:"dppptp",title:"DP+PP+TP"},{local:"zero-dppptp",title:"ZeRO DP+PP+TP"},{local:"flexflow",title:"FlexFlow"},{local:"which-strategy-to-use-when",title:"Which Strategy To Use When"}],title:"Efficient Training on Multiple GPUs"};function V6(xp){return z6(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class Y6 extends x6{constructor(y){super();M6(this,y,V6,Z6,$6,{})}}export{Y6 as default,B6 as metadata};
