import{S as dL,i as cL,s as mL,e as l,k as f,w as d,t as o,M as _L,c as a,d as s,m as h,a as i,x as c,h as r,b as u,G as t,g as p,y as m,q as _,o as v,B as y,v as vL}from"../chunks/vendor-hf-doc-builder.js";import{T as fg}from"../chunks/Tip-hf-doc-builder.js";import{I as E}from"../chunks/IconCopyLink-hf-doc-builder.js";import{C as w}from"../chunks/CodeBlock-hf-doc-builder.js";function yL(ye){let $,O,g,x,A,C,R,D;return{c(){$=l("p"),O=o("This plugin doesn\u2019t work with "),g=l("code"),x=o("-n"),A=o(" flag from "),C=l("code"),R=o("pytest-xdist"),D=o(".")},l(P){$=a(P,"P",{});var b=i($);O=r(b,"This plugin doesn\u2019t work with "),g=a(b,"CODE",{});var S=i(g);x=r(S,"-n"),S.forEach(s),A=r(b," flag from "),C=a(b,"CODE",{});var z=i(C);R=r(z,"pytest-xdist"),z.forEach(s),D=r(b,"."),b.forEach(s)},m(P,b){p(P,$,b),t($,O),t($,g),t(g,x),t($,A),t($,C),t(C,R),t($,D)},d(P){P&&s($)}}}function wL(ye){let $,O,g,x,A,C,R,D;return{c(){$=l("p"),O=o("There is another plugin "),g=l("code"),x=o("pytest-repeat"),A=o(", but it doesn\u2019t work with "),C=l("code"),R=o("unittest"),D=o(".")},l(P){$=a(P,"P",{});var b=i($);O=r(b,"There is another plugin "),g=a(b,"CODE",{});var S=i(g);x=r(S,"pytest-repeat"),S.forEach(s),A=r(b,", but it doesn\u2019t work with "),C=a(b,"CODE",{});var z=i(C);R=r(z,"unittest"),z.forEach(s),D=r(b,"."),b.forEach(s)},m(P,b){p(P,$,b),t($,O),t($,g),t(g,x),t($,A),t($,C),t(C,R),t($,D)},d(P){P&&s($)}}}function $L(ye){let $,O,g,x,A,C,R,D,P,b,S,z,Rt,we;return{c(){$=l("p"),O=o("In order to run the equivalent of "),g=l("code"),x=o("rm -r"),A=o(` safely, only subdirs of the project repository checkout are allowed if
an explicit `),C=l("code"),R=o("tmp_dir"),D=o(" is used, so that by mistake no "),P=l("code"),b=o("/tmp"),S=o(` or similar important part of the filesystem will
get nuked. i.e. please always pass paths that start with `),z=l("code"),Rt=o("./"),we=o(".")},l(Ye){$=a(Ye,"P",{});var q=i($);O=r(q,"In order to run the equivalent of "),g=a(q,"CODE",{});var me=i(g);x=r(me,"rm -r"),me.forEach(s),A=r(q,` safely, only subdirs of the project repository checkout are allowed if
an explicit `),C=a(q,"CODE",{});var Ve=i(C);R=r(Ve,"tmp_dir"),Ve.forEach(s),D=r(q," is used, so that by mistake no "),P=a(q,"CODE",{});var zt=i(P);b=r(zt,"/tmp"),zt.forEach(s),S=r(q,` or similar important part of the filesystem will
get nuked. i.e. please always pass paths that start with `),z=a(q,"CODE",{});var ci=i(z);Rt=r(ci,"./"),ci.forEach(s),we=r(q,"."),q.forEach(s)},m(Ye,q){p(Ye,$,q),t($,O),t($,g),t(g,x),t($,A),t($,C),t(C,R),t($,D),t($,P),t(P,b),t($,S),t($,z),t(z,Rt),t($,we)},d(Ye){Ye&&s($)}}}function EL(ye){let $,O;return{c(){$=l("p"),O=o(`Each test can register multiple temporary directories and they all will get auto-removed, unless requested
otherwise.`)},l(g){$=a(g,"P",{});var x=i($);O=r(x,`Each test can register multiple temporary directories and they all will get auto-removed, unless requested
otherwise.`),x.forEach(s)},m(g,x){p(g,$,x),t($,O)},d(g){g&&s($)}}}function gL(ye){let $,O,g,x,A,C,R,D,P,b,S,z,Rt,we,Ye,q,me,Ve,zt,ci,hg,ug,mi,Cp,dg,cg,Dm,Xe,Gt,jp,Ro,mg,Pp,_g,qm,Ft,zo,Go,vg,Fo,yg,wg,$g,Wo,Eg,Op,gg,bg,kg,Mo,Bo,xg,Yo,Cg,jg,Pg,Je,Ip,_i,Vo,Og,Ig,Tg,Tp,L,Xo,Ag,Dg,Ap,qg,Sg,Dp,Lg,Ug,qp,Ng,Hg,Sp,Rg,zg,Lp,Gg,Fg,Wg,Up,$e,Jo,Mg,Bg,Np,Yg,Vg,Hp,Xg,Jg,Sm,Zo,Lm,Wt,Zg,Ko,Kg,Qg,Um,Ze,Mt,Rp,Qo,eb,zp,tb,Nm,Ke,Bt,Gp,er,sb,Fp,ob,Hm,Yt,rb,tr,lb,ab,Rm,vi,ib,zm,yi,nb,Gm,sr,Fm,wi,pb,Wm,or,Mm,$i,fb,Bm,rr,Ym,Ei,hb,Vm,Q,Wp,ub,db,Mp,cb,mb,Bp,_b,vb,Yp,yb,Xm,Qe,Vt,Vp,lr,wb,Xp,$b,Jm,gi,Eb,Zm,ar,Km,bi,gb,Qm,ir,e_,et,Xt,Jp,nr,bb,Zp,kb,t_,ki,xb,s_,pr,o_,tt,Jt,Kp,fr,Cb,Qp,jb,r_,xi,Pb,l_,hr,a_,Ci,Ob,i_,Ee,ji,ef,Ib,Tb,Ab,Pi,tf,Db,qb,Sb,Oi,sf,Lb,Ub,n_,Ii,Nb,p_,ur,f_,Ti,Hb,h_,Zt,Rb,of,zb,Gb,u_,dr,d_,Ai,Fb,c_,Kt,Wb,rf,Mb,Bb,m_,cr,__,ee,Yb,lf,Vb,Xb,af,Jb,Zb,nf,Kb,Qb,v_,Qt,e3,pf,t3,s3,y_,mr,w_,Di,o3,$_,_r,E_,ge,r3,ff,l3,a3,hf,i3,n3,g_,vr,b_,es,p3,uf,f3,h3,k_,ts,u3,df,d3,c3,x_,yr,C_,st,ss,cf,wr,m3,mf,_3,j_,be,v3,_f,y3,w3,os,vf,$3,E3,g3,P_,$r,O_,qi,b3,I_,Si,Er,k3,yf,x3,C3,T_,gr,A_,br,wf,j3,D_,kr,q_,xr,Cr,P3,$f,O3,I3,S_,jr,L_,ot,rs,Ef,Pr,T3,gf,A3,U_,ls,D3,Or,q3,S3,N_,Ir,H_,Tr,R_,Li,L3,z_,rt,as,bf,Ar,U3,kf,N3,G_,Dr,qr,H3,R3,F_,Sr,W_,lt,z3,xf,G3,F3,Cf,W3,M_,ke,M3,jf,B3,Y3,Pf,V3,X3,B_,Lr,Y_,xe,J3,Of,Z3,K3,If,Q3,e2,V_,Ur,X_,Ui,t2,J_,Nr,Hr,s2,o2,Z_,at,is,Tf,Rr,r2,Af,l2,K_,ns,a2,Df,i2,n2,Q_,zr,ev,it,ps,qf,Gr,p2,Sf,f2,tv,Ni,h2,sv,Fr,ov,nt,fs,Lf,Wr,u2,Uf,d2,rv,W,c2,Nf,m2,_2,Hf,v2,y2,Rf,w2,$2,zf,E2,g2,lv,_e,Gf,b2,k2,Ff,x2,C2,Wf,j2,P2,av,Ce,O2,Mf,I2,T2,Mr,A2,D2,iv,pt,hs,Bf,Br,q2,Yf,S2,nv,Hi,L2,pv,ft,us,Vf,Yr,U2,Xf,N2,fv,Ri,zi,Vr,H2,R2,hv,Xr,uv,Gi,z2,dv,Jr,cv,ds,mv,cs,_v,ht,ms,Jf,Zr,G2,Zf,F2,vv,Kr,yv,_s,W2,Kf,M2,B2,wv,vs,Y2,Qf,V2,X2,$v,Qr,Ev,Fi,J2,gv,el,bv,ys,Z2,eh,K2,Q2,kv,tl,xv,Wi,e5,Cv,sl,jv,I,t5,th,s5,o5,sh,r5,l5,oh,a5,i5,rh,n5,p5,lh,f5,h5,ol,u5,d5,Pv,je,c5,rl,ah,m5,_5,ih,v5,y5,Ov,ut,ws,nh,ll,w5,ph,$5,Iv,dt,$s,fh,al,E5,hh,g5,Tv,il,nl,b5,k5,Av,pl,Dv,Mi,x5,qv,fl,Sv,Bi,C5,Lv,ct,Es,uh,hl,j5,dh,P5,Uv,Pe,O5,ch,I5,T5,mh,A5,D5,Nv,ul,Hv,mt,gs,_h,dl,q5,vh,S5,Rv,cl,ml,L5,U5,zv,_l,Gv,vl,Fv,_t,bs,yh,yl,N5,wh,H5,Wv,ks,R5,$h,z5,G5,Mv,wl,Bv,te,F5,Eh,W5,M5,gh,B5,Y5,bh,V5,X5,Yv,$l,Vv,Yi,J5,Xv,Vi,Z5,Jv,U,Xi,kh,K5,Q5,ek,xs,xh,tk,sk,Ch,ok,rk,lk,Cs,jh,ak,ik,Ph,nk,pk,fk,js,Oh,hk,uk,Ih,dk,ck,mk,Ps,Th,_k,vk,Ah,yk,wk,$k,Os,Dh,Ek,gk,qh,bk,kk,Zv,Ji,xk,Kv,k,Ck,Sh,jk,Pk,Lh,Ok,Ik,Uh,Tk,Ak,Nh,Dk,qk,Hh,Sk,Lk,Rh,Uk,Nk,zh,Hk,Rk,Gh,zk,Gk,Fh,Fk,Wk,Wh,Mk,Bk,Qv,Zi,Yk,e1,El,t1,Oe,Vk,Mh,Xk,Jk,Bh,Zk,Kk,s1,gl,o1,Ki,Qk,r1,bl,l1,Ie,e0,Yh,t0,s0,Vh,o0,r0,a1,kl,i1,Is,l0,Xh,a0,i0,n1,Qi,n0,p1,en,Jh,p0,f1,xl,h1,vt,Ts,Zh,Cl,f0,Kh,h0,u1,yt,Qh,u0,d0,eu,c0,m0,d1,tn,_0,c1,As,tu,jl,v0,y0,su,Pl,w0,m1,Ds,$0,ou,E0,g0,_1,sn,b0,v1,Ol,y1,wt,qs,ru,Il,k0,lu,x0,w1,Te,C0,au,j0,P0,iu,O0,I0,$1,M,T0,nu,A0,D0,pu,q0,S0,fu,L0,U0,hu,N0,H0,E1,Tl,g1,on,R0,b1,Al,k1,$t,Ss,uu,Dl,z0,du,G0,x1,rn,F0,C1,ql,j1,Et,Ls,cu,Sl,W0,mu,M0,P1,ln,B0,O1,Ll,I1,an,Y0,T1,nn,V0,A1,Ul,D1,gt,Us,_u,Nl,X0,vu,J0,q1,Ae,Z0,yu,K0,Q0,wu,e4,t4,S1,se,s4,Hl,o4,r4,$u,l4,a4,Eu,i4,n4,L1,bt,Ns,gu,Rl,p4,bu,f4,U1,pn,h4,N1,zl,H1,Hs,u4,ku,d4,c4,R1,De,m4,xu,_4,v4,Cu,y4,w4,z1,Gl,G1,Rs,$4,ju,E4,g4,F1,Fl,W1,zs,b4,Pu,k4,x4,M1,Wl,B1,fn,C4,Y1,Ml,V1,hn,j4,X1,Bl,J1,B,P4,Yl,O4,I4,Ou,T4,A4,Iu,D4,q4,Tu,S4,L4,Z1,oe,U4,Au,N4,H4,Du,R4,z4,qu,G4,F4,K1,qe,W4,Su,M4,B4,Lu,Y4,V4,Q1,Vl,ey,re,X4,Uu,J4,Z4,Nu,K4,Q4,Hu,e6,t6,ty,Xl,sy,un,s6,oy,Jl,ry,dn,o6,ly,Zl,ay,cn,r6,iy,kt,Gs,Ru,Kl,l6,zu,a6,ny,Fs,i6,Gu,n6,p6,py,Ws,Ql,mn,Fu,f6,h6,u6,G,ea,Wu,d6,c6,Mu,m6,_6,_n,Bu,v6,y6,w6,Ms,Yu,$6,E6,Vu,g6,b6,k6,Bs,Xu,x6,C6,Ju,j6,P6,O6,vn,Zu,I6,T6,A6,Se,Ku,D6,q6,Qu,S6,L6,ed,U6,N6,H6,ta,sa,R6,td,z6,G6,F6,F,sd,od,W6,M6,rd,ld,B6,Y6,ad,id,V6,X6,nd,pd,J6,Z6,fd,hd,K6,Q6,ud,dd,e7,fy,Ys,t7,cd,s7,o7,hy,oa,uy,Y,r7,md,l7,a7,_d,i7,n7,vd,p7,f7,yd,h7,u7,dy,ra,cy,xt,Vs,wd,la,d7,$d,c7,my,Xs,m7,Ed,_7,v7,_y,yn,y7,vy,Le,w7,gd,$7,E7,bd,g7,b7,yy,wn,k7,wy,aa,$y,Js,x7,kd,C7,j7,Ey,$n,xd,P7,gy,ia,by,na,Cd,O7,I7,ky,En,jd,T7,xy,pa,Cy,gn,A7,jy,bn,fa,Ct,D7,Pd,q7,S7,Od,L7,U7,N7,ve,kn,Id,H7,R7,z7,xn,Td,G7,F7,W7,Cn,Ad,M7,B7,Y7,jn,Dd,V7,X7,Py,Zs,Oy,Ks,Iy,jt,Qs,qd,ha,J7,Sd,Z7,Ty,Ue,K7,Ld,Q7,e8,Ud,t8,s8,Ay,ua,Dy,Pt,eo,Nd,da,o8,Hd,r8,qy,to,l8,Rd,a8,i8,Sy,Pn,n8,Ly,so,zd,ca,p8,Gd,f8,h8,u8,Fd,ma,d8,Wd,c8,m8,Uy,le,_8,Md,v8,y8,Bd,w8,$8,Yd,E8,g8,Ny,Ot,oo,Vd,_a,b8,Xd,k8,Hy,On,Jd,x8,Ry,va,zy,In,C8,Gy,ya,Fy,ro,j8,Zd,P8,O8,Wy,wa,My,Tn,Kd,I8,By,$a,Yy,An,T8,Vy,Ea,Xy,lo,A8,Qd,D8,q8,Jy,ga,Zy,Dn,ec,S8,Ky,ba,Qy,qn,tc,L8,ew,ka,tw,Sn,U8,sw,xa,ow,Ln,N8,rw,Ca,lw,ao,H8,ja,R8,z8,aw,It,io,sc,Pa,G8,oc,F8,iw,Un,W8,nw,Oa,pw,Ne,M8,rc,B8,Y8,lc,V8,X8,fw,Ia,hw,ae,J8,ac,Z8,K8,ic,Q8,e9,nc,t9,s9,uw,Ta,dw,Nn,o9,cw,Hn,r9,mw,Rn,l9,_w,ie,pc,a9,i9,fc,n9,p9,Tt,f9,hc,h9,u9,uc,d9,c9,m9,dc,_9,vw,He,v9,cc,y9,w9,mc,$9,E9,yw,Aa,ww,Re,g9,Da,b9,k9,qa,x9,C9,$w,no,j9,_c,P9,O9,Ew,zn,I9,gw,At,po,vc,Sa,T9,yc,A9,bw,V,D9,wc,q9,S9,$c,L9,U9,Ec,N9,H9,La,R9,z9,kw,Ua,xw,fo,G9,gc,F9,W9,Cw,Na,jw,ho,M9,bc,B9,Y9,Pw,Ha,Ow,T,V9,kc,X9,J9,xc,Z9,K9,Cc,Q9,ex,jc,tx,sx,Pc,ox,rx,Oc,lx,ax,Iw,uo,ix,Ic,nx,px,Tw,Ra,Aw,Gn,fx,Dw,za,qw,ze,hx,Tc,ux,dx,Ac,cx,mx,Sw,Ga,Lw,co,_x,Dc,vx,yx,Uw,Fa,Nw,Fn,wx,Hw,Dt,mo,qc,Wa,$x,Sc,Ex,Rw,_o,gx,Lc,bx,kx,zw,Ma,Gw,qt,vo,Uc,Ba,xx,Nc,Cx,Fw,Ya,jx,Hc,Px,Ww,Va,Mw,ne,Ox,Rc,Ix,Tx,zc,Ax,Dx,Gc,qx,Sx,Bw,Xa,Yw,N,Lx,Fc,Ux,Nx,Wc,Hx,Rx,Mc,zx,Gx,Bc,Fx,Wx,Yc,Mx,Bx,Vw,yo,Yx,Vc,Vx,Xx,Xw,St,wo,Xc,Ja,Jx,Jc,Zx,Jw,Wn,Kx,Zw,Za,Kw,Lt,$o,Zc,Ka,Qx,Kc,eC,Qw,Mn,tC,e$,Qa,t$,Ut,Eo,Qc,ei,sC,em,oC,s$,Bn,rC,o$,pe,ti,lC,tm,aC,iC,nC,H,pC,sm,fC,hC,om,uC,dC,rm,cC,mC,lm,_C,vC,si,yC,wC,am,$C,EC,im,gC,bC,oi,kC,ri,xC,CC,r$,Nt,go,nm,li,jC,pm,PC,l$,Yn,OC,a$,fe,fm,IC,TC,hm,AC,DC,um,qC,SC,dm,LC,i$,Vn,UC,n$,Xn,NC,p$,Jn,HC,f$,Zn,RC,h$,bo,Kn,cm,zC,GC,FC,Ht,WC,mm,MC,BC,_m,YC,VC,u$,Qn,XC,d$,ai,c$,ep,JC,m$,ii,_$,ko,ZC,vm,KC,QC,v$,xo,ej,ym,tj,sj,y$,tp,oj,w$,Co,wm,ni,rj,lj,$m,pi,aj,$$;return C=new E({}),Ro=new E({}),Zo=new w({props:{code:`RUN_SLOW=1 pytest tests/
RUN_SLOW=1 pytest examples/`,highlighted:`RUN_SLOW=1 pytest tests/
RUN_SLOW=1 pytest examples/`}}),Qo=new E({}),er=new E({}),sr=new w({props:{code:"pytest",highlighted:"pytest"}}),or=new w({props:{code:"make test",highlighted:'make <span class="hljs-built_in">test</span>'}}),rr=new w({props:{code:"python -m pytest -n auto --dist=loadfile -s -v ./tests/",highlighted:"python -m pytest -n auto --dist=loadfile -s -v ./tests/"}}),lr=new E({}),ar=new w({props:{code:"pytest --collect-only -q",highlighted:"pytest --collect-only -q"}}),ir=new w({props:{code:"pytest tests/test_optimization.py --collect-only -q",highlighted:"pytest tests/test_optimization.py --collect-only -q"}}),nr=new E({}),pr=new w({props:{code:"pytest tests/test_logging.py",highlighted:"pytest tests/test_logging.py"}}),fr=new E({}),hr=new w({props:{code:"pytest tests/test_optimization.py::OptimizationTest::test_adam_w",highlighted:"pytest tests/test_optimization.py::OptimizationTest::test_adam_w"}}),ur=new w({props:{code:"pytest tests/test_optimization.py::OptimizationTest",highlighted:"pytest tests/test_optimization.py::OptimizationTest"}}),dr=new w({props:{code:"pytest tests/test_optimization.py::OptimizationTest --collect-only -q",highlighted:"pytest tests/test_optimization.py::OptimizationTest --collect-only -q"}}),cr=new w({props:{code:"pytest -k adam tests/test_optimization.py",highlighted:"pytest -k adam tests/test_optimization.py"}}),mr=new w({props:{code:'pytest -k "not adam" tests/test_optimization.py',highlighted:'pytest -k <span class="hljs-string">&quot;not adam&quot;</span> tests/test_optimization.py'}}),_r=new w({props:{code:'pytest -k "ada and not adam" tests/test_optimization.py',highlighted:'pytest -k <span class="hljs-string">&quot;ada and not adam&quot;</span> tests/test_optimization.py'}}),vr=new w({props:{code:'pytest -k "test_adam_w or test_adam_w" tests/test_optimization.py',highlighted:'pytest -k <span class="hljs-string">&quot;test_adam_w or test_adam_w&quot;</span> tests/test_optimization.py'}}),yr=new w({props:{code:'pytest -k "test and ada" tests/test_optimization.py',highlighted:'pytest -k <span class="hljs-string">&quot;test and ada&quot;</span> tests/test_optimization.py'}}),wr=new E({}),$r=new w({props:{code:`r"""
Returns:

Example:
    \`\`\`python
    >>> import torch
    >>> from transformers import WhisperModel, WhisperFeatureExtractor
    >>> from datasets import load_dataset

    >>> model = WhisperModel.from_pretrained("openai/whisper-base")
    >>> feature_extractor = WhisperFeatureExtractor.from_pretrained("openai/whisper-base")
    >>> ds = load_dataset("hf-internal-testing/librispeech_asr_dummy", "clean", split="validation")
    >>> inputs = feature_extractor(ds[0]["audio"]["array"], return_tensors="pt")
    >>> input_features = inputs.input_features
    >>> decoder_input_ids = torch.tensor([[1, 1]]) * model.config.decoder_start_token_id
    >>> last_hidden_state = model(input_features, decoder_input_ids=decoder_input_ids).last_hidden_state
    >>> list(last_hidden_state.shape)
    [1, 2, 512]
    \`\`\`"""
`,highlighted:`<span class="hljs-string">r&quot;&quot;&quot;
Returns:

Example:
    \`\`\`python
    &gt;&gt;&gt; import torch
    &gt;&gt;&gt; from transformers import WhisperModel, WhisperFeatureExtractor
    &gt;&gt;&gt; from datasets import load_dataset

    &gt;&gt;&gt; model = WhisperModel.from_pretrained(&quot;openai/whisper-base&quot;)
    &gt;&gt;&gt; feature_extractor = WhisperFeatureExtractor.from_pretrained(&quot;openai/whisper-base&quot;)
    &gt;&gt;&gt; ds = load_dataset(&quot;hf-internal-testing/librispeech_asr_dummy&quot;, &quot;clean&quot;, split=&quot;validation&quot;)
    &gt;&gt;&gt; inputs = feature_extractor(ds[0][&quot;audio&quot;][&quot;array&quot;], return_tensors=&quot;pt&quot;)
    &gt;&gt;&gt; input_features = inputs.input_features
    &gt;&gt;&gt; decoder_input_ids = torch.tensor([[1, 1]]) * model.config.decoder_start_token_id
    &gt;&gt;&gt; last_hidden_state = model(input_features, decoder_input_ids=decoder_input_ids).last_hidden_state
    &gt;&gt;&gt; list(last_hidden_state.shape)
    [1, 2, 512]
    \`\`\`&quot;&quot;&quot;</span>
`}}),gr=new w({props:{code:"python utils/prepare_for_doc_test.py <path_to_file_or_dir>",highlighted:"python utils/prepare_for_doc_test.py &lt;path_to_file_or_dir&gt;"}}),kr=new w({props:{code:"pytest --doctest-modules <path_to_file_or_dir>",highlighted:"pytest --doctest-modules &lt;path_to_file_or_dir&gt;"}}),jr=new w({props:{code:"python utils/prepare_for_doc_test.py <path_to_file_or_dir> --remove_new_line",highlighted:"python utils/prepare_for_doc_test.py &lt;path_to_file_or_dir&gt; --remove_new_line"}}),Pr=new E({}),Ir=new w({props:{code:"pip install pytest-picked",highlighted:"pip install pytest-picked"}}),Tr=new w({props:{code:"pytest --picked",highlighted:"pytest --picked"}}),Ar=new E({}),Sr=new w({props:{code:"pip install pytest-xdist",highlighted:"pip install pytest-xdist"}}),Lr=new w({props:{code:`[tool:pytest]
looponfailroots = transformers tests`,highlighted:`<span class="hljs-section">[tool:pytest]</span>
<span class="hljs-attr">looponfailroots</span> = transformers tests`}}),Ur=new w({props:{code:`[pytest]
looponfailroots = transformers tests`,highlighted:`<span class="hljs-section">[pytest]</span>
<span class="hljs-attr">looponfailroots</span> = transformers tests`}}),Rr=new E({}),zr=new w({props:{code:"pytest *ls -1 tests/*py | grep -v test_modeling*",highlighted:'pytest *<span class="hljs-built_in">ls</span> -1 tests/*py | grep -v test_modeling*'}}),Gr=new E({}),Fr=new w({props:{code:"pytest --cache-clear tests",highlighted:"pytest --cache-clear tests"}}),Wr=new E({}),Br=new E({}),Yr=new E({}),Xr=new w({props:{code:"pip install pytest-flakefinder",highlighted:"pip install pytest-flakefinder"}}),Jr=new w({props:{code:"pytest --flake-finder --flake-runs=5 tests/test_failing_test.py",highlighted:"pytest --flake-finder --flake-runs=5 tests/test_failing_test.py"}}),ds=new fg({props:{$$slots:{default:[yL]},$$scope:{ctx:ye}}}),cs=new fg({props:{$$slots:{default:[wL]},$$scope:{ctx:ye}}}),Zr=new E({}),Kr=new w({props:{code:"pip install pytest-random-order",highlighted:"pip install pytest-random-order"}}),Qr=new w({props:{code:`pytest tests
[...]
Using --random-order-bucket=module
Using --random-order-seed=573663`,highlighted:`pytest tests
[...]
Using --random-order-bucket=module
Using --random-order-seed=573663`}}),el=new w({props:{code:`pytest --random-order-seed=573663
[...]
Using --random-order-bucket=module
Using --random-order-seed=573663`,highlighted:`pytest --random-order-seed=573663
[...]
Using --random-order-bucket=module
Using --random-order-seed=573663`}}),tl=new w({props:{code:"pytest --random-order-bucket=none tests/test_a.py tests/test_c.py tests/test_b.py",highlighted:"pytest --random-order-bucket=none tests/test_a.py tests/test_c.py tests/test_b.py"}}),sl=new w({props:{code:"pytest --random-order-bucket=none",highlighted:"pytest --random-order-bucket=none"}}),ll=new E({}),al=new E({}),pl=new w({props:{code:"pip install pytest-sugar",highlighted:"pip install pytest-sugar"}}),fl=new w({props:{code:"pytest -p no:sugar",highlighted:"pytest -p no:sugar"}}),hl=new E({}),ul=new w({props:{code:"pytest --pspec tests/test_optimization.py",highlighted:"pytest --pspec tests/test_optimization.py"}}),dl=new E({}),_l=new w({props:{code:"pip install pytest-instafail",highlighted:"pip install pytest-instafail"}}),vl=new w({props:{code:"pytest --instafail",highlighted:"pytest --instafail"}}),yl=new E({}),wl=new w({props:{code:'CUDA_VISIBLE_DEVICES="" pytest tests/test_logging.py',highlighted:'CUDA_VISIBLE_DEVICES=<span class="hljs-string">&quot;&quot;</span> pytest tests/test_logging.py'}}),$l=new w({props:{code:'CUDA_VISIBLE_DEVICES="1" pytest tests/test_logging.py',highlighted:'CUDA_VISIBLE_DEVICES=<span class="hljs-string">&quot;1&quot;</span> pytest tests/test_logging.py'}}),El=new w({props:{code:`@require_torch_multi_gpu
def test_example_with_multi_gpu():`,highlighted:`<span class="hljs-meta">@require_torch_multi_gpu</span>
<span class="hljs-keyword">def</span> <span class="hljs-title function_">test_example_with_multi_gpu</span>():`}}),gl=new w({props:{code:`@require_tf
def test_tf_thing_with_tensorflow():`,highlighted:`<span class="hljs-meta">@require_tf</span>
<span class="hljs-keyword">def</span> <span class="hljs-title function_">test_tf_thing_with_tensorflow</span>():`}}),bl=new w({props:{code:`@require_torch_gpu
@slow
def test_example_slow_on_gpu():`,highlighted:`<span class="hljs-meta">@require_torch_gpu</span>
<span class="hljs-meta">@slow</span>
<span class="hljs-keyword">def</span> <span class="hljs-title function_">test_example_slow_on_gpu</span>():`}}),kl=new w({props:{code:`@parameterized.expand(...)
@require_torch_multi_gpu
def test_integration_foo():`,highlighted:`<span class="hljs-meta">@parameterized.expand(<span class="hljs-params">...</span>)</span>
<span class="hljs-meta">@require_torch_multi_gpu</span>
<span class="hljs-keyword">def</span> <span class="hljs-title function_">test_integration_foo</span>():`}}),xl=new w({props:{code:`from transformers.testing_utils import get_gpu_count

n_gpu = get_gpu_count()  # works with torch and tf`,highlighted:`<span class="hljs-keyword">from</span> transformers.testing_utils <span class="hljs-keyword">import</span> get_gpu_count

n_gpu = get_gpu_count()  <span class="hljs-comment"># works with torch and tf</span>`}}),Cl=new E({}),Ol=new w({props:{code:"CUDA_VISIBLE_DEVICES=0,1 RUN_SLOW=1 pytest -sv tests/test_trainer_distributed.py",highlighted:"CUDA_VISIBLE_DEVICES=0,1 RUN_SLOW=1 pytest -sv tests/test_trainer_distributed.py"}}),Il=new E({}),Tl=new w({props:{code:"pytest -s tests/test_logging.py",highlighted:"pytest -s tests/test_logging.py"}}),Al=new w({props:{code:"py.test tests --junitxml=result.xml",highlighted:"py.test tests --junitxml=result.xml"}}),Dl=new E({}),ql=new w({props:{code:"pytest --color=no tests/test_logging.py",highlighted:"pytest --color=no tests/test_logging.py"}}),Sl=new E({}),Ll=new w({props:{code:"pytest --pastebin=failed tests/test_logging.py",highlighted:"pytest --pastebin=failed tests/test_logging.py"}}),Ul=new w({props:{code:"pytest --pastebin=all tests/test_logging.py",highlighted:"pytest --pastebin=all tests/test_logging.py"}}),Nl=new E({}),Rl=new E({}),zl=new w({props:{code:`# test_this1.py
import unittest
from parameterized import parameterized


class TestMathUnitTest(unittest.TestCase):
    @parameterized.expand(
        [
            ("negative", -1.5, -2.0),
            ("integer", 1, 1.0),
            ("large fraction", 1.6, 1),
        ]
    )
    def test_floor(self, name, input, expected):
        assert_equal(math.floor(input), expected)`,highlighted:`<span class="hljs-comment"># test_this1.py</span>
<span class="hljs-keyword">import</span> unittest
<span class="hljs-keyword">from</span> parameterized <span class="hljs-keyword">import</span> parameterized


<span class="hljs-keyword">class</span> <span class="hljs-title class_">TestMathUnitTest</span>(unittest.TestCase):
<span class="hljs-meta">    @parameterized.expand(<span class="hljs-params">
        [
            (<span class="hljs-params"><span class="hljs-string">&quot;negative&quot;</span>, -<span class="hljs-number">1.5</span>, -<span class="hljs-number">2.0</span></span>),
            (<span class="hljs-params"><span class="hljs-string">&quot;integer&quot;</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1.0</span></span>),
            (<span class="hljs-params"><span class="hljs-string">&quot;large fraction&quot;</span>, <span class="hljs-number">1.6</span>, <span class="hljs-number">1</span></span>),
        ]
    </span>)</span>
    <span class="hljs-keyword">def</span> <span class="hljs-title function_">test_floor</span>(<span class="hljs-params">self, name, <span class="hljs-built_in">input</span>, expected</span>):
        assert_equal(math.floor(<span class="hljs-built_in">input</span>), expected)`}}),Gl=new w({props:{code:'pytest -k "negative and integer" tests/test_mytest.py',highlighted:'pytest -k <span class="hljs-string">&quot;negative and integer&quot;</span> tests/test_mytest.py'}}),Fl=new w({props:{code:'pytest -k "not negative" tests/test_mytest.py',highlighted:'pytest -k <span class="hljs-string">&quot;not negative&quot;</span> tests/test_mytest.py'}}),Wl=new w({props:{code:"pytest test_this1.py --collect-only -q",highlighted:"pytest test_this1.py --collect-only -q"}}),Ml=new w({props:{code:`test_this1.py::TestMathUnitTest::test_floor_0_negative
test_this1.py::TestMathUnitTest::test_floor_1_integer
test_this1.py::TestMathUnitTest::test_floor_2_large_fraction`,highlighted:`test_this1.py::TestMathUnitTest::test_floor_0_negative
test_this1.py::TestMathUnitTest::test_floor_1_integer
test_this1.py::TestMathUnitTest::test_floor_2_large_fraction`}}),Bl=new w({props:{code:"pytest test_this1.py::TestMathUnitTest::test_floor_0_negative  test_this1.py::TestMathUnitTest::test_floor_1_integer",highlighted:"pytest test_this1.py::TestMathUnitTest::test_floor_0_negative  test_this1.py::TestMathUnitTest::test_floor_1_integer"}}),Vl=new w({props:{code:`# test_this2.py
import pytest


@pytest.mark.parametrize(
    "name, input, expected",
    [
        ("negative", -1.5, -2.0),
        ("integer", 1, 1.0),
        ("large fraction", 1.6, 1),
    ],
)
def test_floor(name, input, expected):
    assert_equal(math.floor(input), expected)`,highlighted:`<span class="hljs-comment"># test_this2.py</span>
<span class="hljs-keyword">import</span> pytest


<span class="hljs-meta">@pytest.mark.parametrize(<span class="hljs-params">
    <span class="hljs-string">&quot;name, input, expected&quot;</span>,
    [
        (<span class="hljs-params"><span class="hljs-string">&quot;negative&quot;</span>, -<span class="hljs-number">1.5</span>, -<span class="hljs-number">2.0</span></span>),
        (<span class="hljs-params"><span class="hljs-string">&quot;integer&quot;</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1.0</span></span>),
        (<span class="hljs-params"><span class="hljs-string">&quot;large fraction&quot;</span>, <span class="hljs-number">1.6</span>, <span class="hljs-number">1</span></span>),
    ],
</span>)</span>
<span class="hljs-keyword">def</span> <span class="hljs-title function_">test_floor</span>(<span class="hljs-params">name, <span class="hljs-built_in">input</span>, expected</span>):
    assert_equal(math.floor(<span class="hljs-built_in">input</span>), expected)`}}),Xl=new w({props:{code:"pytest test_this2.py --collect-only -q",highlighted:"pytest test_this2.py --collect-only -q"}}),Jl=new w({props:{code:`test_this2.py::test_floor[integer-1-1.0]
test_this2.py::test_floor[negative--1.5--2.0]
test_this2.py::test_floor[large fraction-1.6-1]`,highlighted:`test_this2.py::test_floor[integer-1-1.0]
test_this2.py::test_floor[negative--1.5--2.0]
test_this2.py::test_floor[large fraction-1.6-1]`}}),Zl=new w({props:{code:"pytest test_this2.py::test_floor[negative--1.5--2.0] test_this2.py::test_floor[integer-1-1.0]",highlighted:"pytest test_this2.py::test_floor[negative--1.5--2.0] test_this2.py::test_floor[integer-1-1.0]"}}),Kl=new E({}),oa=new w({props:{code:`from transformers.testing_utils import TestCasePlus


class PathExampleTest(TestCasePlus):
    def test_something_involving_local_locations(self):
        data_dir = self.tests_dir / "fixtures/tests_samples/wmt_en_ro"`,highlighted:`<span class="hljs-keyword">from</span> transformers.testing_utils <span class="hljs-keyword">import</span> TestCasePlus


<span class="hljs-keyword">class</span> <span class="hljs-title class_">PathExampleTest</span>(<span class="hljs-title class_ inherited__">TestCasePlus</span>):
    <span class="hljs-keyword">def</span> <span class="hljs-title function_">test_something_involving_local_locations</span>(<span class="hljs-params">self</span>):
        data_dir = self.tests_dir / <span class="hljs-string">&quot;fixtures/tests_samples/wmt_en_ro&quot;</span>`}}),ra=new w({props:{code:`from transformers.testing_utils import TestCasePlus


class PathExampleTest(TestCasePlus):
    def test_something_involving_stringified_locations(self):
        examples_dir = self.examples_dir_str`,highlighted:`<span class="hljs-keyword">from</span> transformers.testing_utils <span class="hljs-keyword">import</span> TestCasePlus


<span class="hljs-keyword">class</span> <span class="hljs-title class_">PathExampleTest</span>(<span class="hljs-title class_ inherited__">TestCasePlus</span>):
    <span class="hljs-keyword">def</span> <span class="hljs-title function_">test_something_involving_stringified_locations</span>(<span class="hljs-params">self</span>):
        examples_dir = self.examples_dir_str`}}),la=new E({}),aa=new w({props:{code:`from transformers.testing_utils import TestCasePlus


class ExamplesTests(TestCasePlus):
    def test_whatever(self):
        tmp_dir = self.get_auto_remove_tmp_dir()`,highlighted:`<span class="hljs-keyword">from</span> transformers.testing_utils <span class="hljs-keyword">import</span> TestCasePlus


<span class="hljs-keyword">class</span> <span class="hljs-title class_">ExamplesTests</span>(<span class="hljs-title class_ inherited__">TestCasePlus</span>):
    <span class="hljs-keyword">def</span> <span class="hljs-title function_">test_whatever</span>(<span class="hljs-params">self</span>):
        tmp_dir = self.get_auto_remove_tmp_dir()`}}),ia=new w({props:{code:`def test_whatever(self):
    tmp_dir = self.get_auto_remove_tmp_dir()`,highlighted:`<span class="hljs-keyword">def</span> <span class="hljs-title function_">test_whatever</span>(<span class="hljs-params">self</span>):
    tmp_dir = self.get_auto_remove_tmp_dir()`}}),pa=new w({props:{code:`def test_whatever(self):
    tmp_dir = self.get_auto_remove_tmp_dir("./xxx")`,highlighted:`<span class="hljs-keyword">def</span> <span class="hljs-title function_">test_whatever</span>(<span class="hljs-params">self</span>):
    tmp_dir = self.get_auto_remove_tmp_dir(<span class="hljs-string">&quot;./xxx&quot;</span>)`}}),Zs=new fg({props:{$$slots:{default:[$L]},$$scope:{ctx:ye}}}),Ks=new fg({props:{$$slots:{default:[EL]},$$scope:{ctx:ye}}}),ha=new E({}),ua=new w({props:{code:`import os
from transformers.testing_utils import ExtendSysPath

bindir = os.path.abspath(os.path.dirname(__file__))
with ExtendSysPath(f"{bindir}/.."):
    from test_trainer import TrainerIntegrationCommon  # noqa`,highlighted:`<span class="hljs-keyword">import</span> os
<span class="hljs-keyword">from</span> transformers.testing_utils <span class="hljs-keyword">import</span> ExtendSysPath

bindir = os.path.abspath(os.path.dirname(__file__))
<span class="hljs-keyword">with</span> ExtendSysPath(<span class="hljs-string">f&quot;<span class="hljs-subst">{bindir}</span>/..&quot;</span>):
    <span class="hljs-keyword">from</span> test_trainer <span class="hljs-keyword">import</span> TrainerIntegrationCommon  <span class="hljs-comment"># noqa</span>`}}),da=new E({}),_a=new E({}),va=new w({props:{code:`@unittest.skip("this bug needs to be fixed")
def test_feature_x():`,highlighted:`<span class="hljs-meta">@unittest.skip(<span class="hljs-params"><span class="hljs-string">&quot;this bug needs to be fixed&quot;</span></span>)</span>
<span class="hljs-keyword">def</span> <span class="hljs-title function_">test_feature_x</span>():`}}),ya=new w({props:{code:'@pytest.mark.skip(reason="this bug needs to be fixed")',highlighted:'<span class="hljs-meta">@pytest.mark.skip(<span class="hljs-params">reason=<span class="hljs-string">&quot;this bug needs to be fixed&quot;</span></span>)</span>'}}),wa=new w({props:{code:`@pytest.mark.xfail
def test_feature_x():`,highlighted:`<span class="hljs-meta">@pytest.mark.xfail</span>
<span class="hljs-keyword">def</span> <span class="hljs-title function_">test_feature_x</span>():`}}),$a=new w({props:{code:`def test_feature_x():
    if not has_something():
        pytest.skip("unsupported configuration")`,highlighted:`<span class="hljs-keyword">def</span> <span class="hljs-title function_">test_feature_x</span>():
    <span class="hljs-keyword">if</span> <span class="hljs-keyword">not</span> has_something():
        pytest.skip(<span class="hljs-string">&quot;unsupported configuration&quot;</span>)`}}),Ea=new w({props:{code:`import pytest

if not pytest.config.getoption("--custom-flag"):
    pytest.skip("--custom-flag is missing, skipping tests", allow_module_level=True)`,highlighted:`<span class="hljs-keyword">import</span> pytest

<span class="hljs-keyword">if</span> <span class="hljs-keyword">not</span> pytest.config.getoption(<span class="hljs-string">&quot;--custom-flag&quot;</span>):
    pytest.skip(<span class="hljs-string">&quot;--custom-flag is missing, skipping tests&quot;</span>, allow_module_level=<span class="hljs-literal">True</span>)`}}),ga=new w({props:{code:`def test_feature_x():
    pytest.xfail("expected to fail until bug XYZ is fixed")`,highlighted:`<span class="hljs-keyword">def</span> <span class="hljs-title function_">test_feature_x</span>():
    pytest.xfail(<span class="hljs-string">&quot;expected to fail until bug XYZ is fixed&quot;</span>)`}}),ba=new w({props:{code:'docutils = pytest.importorskip("docutils", minversion="0.3")',highlighted:'docutils = pytest.importorskip(<span class="hljs-string">&quot;docutils&quot;</span>, minversion=<span class="hljs-string">&quot;0.3&quot;</span>)'}}),ka=new w({props:{code:`@pytest.mark.skipif(sys.version_info < (3,6), reason="requires python3.6 or higher")
def test_feature_x():`,highlighted:`<span class="hljs-meta">@pytest.mark.skipif(<span class="hljs-params">sys.version_info &lt; (<span class="hljs-params"><span class="hljs-number">3</span>,<span class="hljs-number">6</span></span>), reason=<span class="hljs-string">&quot;requires python3.6 or higher&quot;</span></span>)</span>
<span class="hljs-keyword">def</span> <span class="hljs-title function_">test_feature_x</span>():`}}),xa=new w({props:{code:`@unittest.skipIf(torch_device == "cpu", "Can't do half precision")
def test_feature_x():`,highlighted:`<span class="hljs-meta">@unittest.skipIf(<span class="hljs-params">torch_device == <span class="hljs-string">&quot;cpu&quot;</span>, <span class="hljs-string">&quot;Can&#x27;t do half precision&quot;</span></span>)</span>
<span class="hljs-keyword">def</span> <span class="hljs-title function_">test_feature_x</span>():`}}),Ca=new w({props:{code:`@pytest.mark.skipif(sys.platform == 'win32', reason="does not run on windows")
class TestClass():
    def test_feature_x(self):`,highlighted:`<span class="hljs-meta">@pytest.mark.skipif(<span class="hljs-params">sys.platform == <span class="hljs-string">&#x27;win32&#x27;</span>, reason=<span class="hljs-string">&quot;does not run on windows&quot;</span></span>)</span>
<span class="hljs-keyword">class</span> <span class="hljs-title class_">TestClass</span>():
    <span class="hljs-keyword">def</span> <span class="hljs-title function_">test_feature_x</span>(<span class="hljs-params">self</span>):`}}),Pa=new E({}),Oa=new w({props:{code:`from transformers.testing_utils import slow
@slow
def test_integration_foo():`,highlighted:`<span class="hljs-keyword">from</span> transformers.testing_utils <span class="hljs-keyword">import</span> slow
<span class="hljs-meta">@slow</span>
<span class="hljs-keyword">def</span> <span class="hljs-title function_">test_integration_foo</span>():`}}),Ia=new w({props:{code:"RUN_SLOW=1 pytest tests",highlighted:"RUN_SLOW=1 pytest tests"}}),Ta=new w({props:{code:`@parameteriz ed.expand(...)
@slow
def test_integration_foo():`,highlighted:`<span class="hljs-meta">@parameteriz ed.expand(<span class="hljs-params">...</span>)</span>
<span class="hljs-meta">@slow</span>
<span class="hljs-keyword">def</span> <span class="hljs-title function_">test_integration_foo</span>():`}}),Aa=new w({props:{code:"grep tiny tests examples",highlighted:"grep tiny tests examples"}}),Sa=new E({}),Ua=new w({props:{code:`import sys


def print_to_stdout(s):
    print(s)


def print_to_stderr(s):
    sys.stderr.write(s)


def test_result_and_stdout(capsys):
    msg = "Hello"
    print_to_stdout(msg)
    print_to_stderr(msg)
    out, err = capsys.readouterr()  # consume the captured output streams
    # optional: if you want to replay the consumed streams:
    sys.stdout.write(out)
    sys.stderr.write(err)
    # test:
    assert msg in out
    assert msg in err`,highlighted:`<span class="hljs-keyword">import</span> sys


<span class="hljs-keyword">def</span> <span class="hljs-title function_">print_to_stdout</span>(<span class="hljs-params">s</span>):
    <span class="hljs-built_in">print</span>(s)


<span class="hljs-keyword">def</span> <span class="hljs-title function_">print_to_stderr</span>(<span class="hljs-params">s</span>):
    sys.stderr.write(s)


<span class="hljs-keyword">def</span> <span class="hljs-title function_">test_result_and_stdout</span>(<span class="hljs-params">capsys</span>):
    msg = <span class="hljs-string">&quot;Hello&quot;</span>
    print_to_stdout(msg)
    print_to_stderr(msg)
    out, err = capsys.readouterr()  <span class="hljs-comment"># consume the captured output streams</span>
    <span class="hljs-comment"># optional: if you want to replay the consumed streams:</span>
    sys.stdout.write(out)
    sys.stderr.write(err)
    <span class="hljs-comment"># test:</span>
    <span class="hljs-keyword">assert</span> msg <span class="hljs-keyword">in</span> out
    <span class="hljs-keyword">assert</span> msg <span class="hljs-keyword">in</span> err`}}),Na=new w({props:{code:`def raise_exception(msg):
    raise ValueError(msg)


def test_something_exception():
    msg = "Not a good value"
    error = ""
    try:
        raise_exception(msg)
    except Exception as e:
        error = str(e)
        assert msg in error, f"{msg} is in the exception:\\n{error}"`,highlighted:`<span class="hljs-keyword">def</span> <span class="hljs-title function_">raise_exception</span>(<span class="hljs-params">msg</span>):
    <span class="hljs-keyword">raise</span> ValueError(msg)


<span class="hljs-keyword">def</span> <span class="hljs-title function_">test_something_exception</span>():
    msg = <span class="hljs-string">&quot;Not a good value&quot;</span>
    error = <span class="hljs-string">&quot;&quot;</span>
    <span class="hljs-keyword">try</span>:
        raise_exception(msg)
    <span class="hljs-keyword">except</span> Exception <span class="hljs-keyword">as</span> e:
        error = <span class="hljs-built_in">str</span>(e)
        <span class="hljs-keyword">assert</span> msg <span class="hljs-keyword">in</span> error, <span class="hljs-string">f&quot;<span class="hljs-subst">{msg}</span> is in the exception:\\n<span class="hljs-subst">{error}</span>&quot;</span>`}}),Ha=new w({props:{code:`from io import StringIO
from contextlib import redirect_stdout


def print_to_stdout(s):
    print(s)


def test_result_and_stdout():
    msg = "Hello"
    buffer = StringIO()
    with redirect_stdout(buffer):
        print_to_stdout(msg)
    out = buffer.getvalue()
    # optional: if you want to replay the consumed streams:
    sys.stdout.write(out)
    # test:
    assert msg in out`,highlighted:`<span class="hljs-keyword">from</span> io <span class="hljs-keyword">import</span> StringIO
<span class="hljs-keyword">from</span> contextlib <span class="hljs-keyword">import</span> redirect_stdout


<span class="hljs-keyword">def</span> <span class="hljs-title function_">print_to_stdout</span>(<span class="hljs-params">s</span>):
    <span class="hljs-built_in">print</span>(s)


<span class="hljs-keyword">def</span> <span class="hljs-title function_">test_result_and_stdout</span>():
    msg = <span class="hljs-string">&quot;Hello&quot;</span>
    buffer = StringIO()
    <span class="hljs-keyword">with</span> redirect_stdout(buffer):
        print_to_stdout(msg)
    out = buffer.getvalue()
    <span class="hljs-comment"># optional: if you want to replay the consumed streams:</span>
    sys.stdout.write(out)
    <span class="hljs-comment"># test:</span>
    <span class="hljs-keyword">assert</span> msg <span class="hljs-keyword">in</span> out`}}),Ra=new w({props:{code:`from transformers.testing_utils import CaptureStdout

with CaptureStdout() as cs:
    function_that_writes_to_stdout()
print(cs.out)`,highlighted:`<span class="hljs-keyword">from</span> transformers.testing_utils <span class="hljs-keyword">import</span> CaptureStdout

<span class="hljs-keyword">with</span> CaptureStdout() <span class="hljs-keyword">as</span> cs:
    function_that_writes_to_stdout()
<span class="hljs-built_in">print</span>(cs.out)`}}),za=new w({props:{code:`from transformers.testing_utils import CaptureStdout

msg = "Secret message\\r"
final = "Hello World"
with CaptureStdout() as cs:
    print(msg + final)
assert cs.out == final + "\\n", f"captured: {cs.out}, expecting {final}"`,highlighted:`<span class="hljs-keyword">from</span> transformers.testing_utils <span class="hljs-keyword">import</span> CaptureStdout

msg = <span class="hljs-string">&quot;Secret message\\r&quot;</span>
final = <span class="hljs-string">&quot;Hello World&quot;</span>
<span class="hljs-keyword">with</span> CaptureStdout() <span class="hljs-keyword">as</span> cs:
    <span class="hljs-built_in">print</span>(msg + final)
<span class="hljs-keyword">assert</span> cs.out == final + <span class="hljs-string">&quot;\\n&quot;</span>, <span class="hljs-string">f&quot;captured: <span class="hljs-subst">{cs.out}</span>, expecting <span class="hljs-subst">{final}</span>&quot;</span>`}}),Ga=new w({props:{code:`from transformers.testing_utils import CaptureStderr

with CaptureStderr() as cs:
    function_that_writes_to_stderr()
print(cs.err)`,highlighted:`<span class="hljs-keyword">from</span> transformers.testing_utils <span class="hljs-keyword">import</span> CaptureStderr

<span class="hljs-keyword">with</span> CaptureStderr() <span class="hljs-keyword">as</span> cs:
    function_that_writes_to_stderr()
<span class="hljs-built_in">print</span>(cs.err)`}}),Fa=new w({props:{code:`from transformers.testing_utils import CaptureStd

with CaptureStd() as cs:
    function_that_writes_to_stdout_and_stderr()
print(cs.err, cs.out)`,highlighted:`<span class="hljs-keyword">from</span> transformers.testing_utils <span class="hljs-keyword">import</span> CaptureStd

<span class="hljs-keyword">with</span> CaptureStd() <span class="hljs-keyword">as</span> cs:
    function_that_writes_to_stdout_and_stderr()
<span class="hljs-built_in">print</span>(cs.err, cs.out)`}}),Wa=new E({}),Ma=new w({props:{code:`from transformers import logging
from transformers.testing_utils import CaptureLogger

msg = "Testing 1, 2, 3"
logging.set_verbosity_info()
logger = logging.get_logger("transformers.models.bart.tokenization_bart")
with CaptureLogger(logger) as cl:
    logger.info(msg)
assert cl.out, msg + "\\n"`,highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> logging
<span class="hljs-keyword">from</span> transformers.testing_utils <span class="hljs-keyword">import</span> CaptureLogger

msg = <span class="hljs-string">&quot;Testing 1, 2, 3&quot;</span>
logging.set_verbosity_info()
logger = logging.get_logger(<span class="hljs-string">&quot;transformers.models.bart.tokenization_bart&quot;</span>)
<span class="hljs-keyword">with</span> CaptureLogger(logger) <span class="hljs-keyword">as</span> cl:
    logger.info(msg)
<span class="hljs-keyword">assert</span> cl.out, msg + <span class="hljs-string">&quot;\\n&quot;</span>`}}),Ba=new E({}),Va=new w({props:{code:`from transformers.testing_utils import mockenv


class HfArgumentParserTest(unittest.TestCase):
    @mockenv(TRANSFORMERS_VERBOSITY="error")
    def test_env_override(self):
        env_level_str = os.getenv("TRANSFORMERS_VERBOSITY", None)`,highlighted:`<span class="hljs-keyword">from</span> transformers.testing_utils <span class="hljs-keyword">import</span> mockenv


<span class="hljs-keyword">class</span> <span class="hljs-title class_">HfArgumentParserTest</span>(unittest.TestCase):
<span class="hljs-meta">    @mockenv(<span class="hljs-params">TRANSFORMERS_VERBOSITY=<span class="hljs-string">&quot;error&quot;</span></span>)</span>
    <span class="hljs-keyword">def</span> <span class="hljs-title function_">test_env_override</span>(<span class="hljs-params">self</span>):
        env_level_str = os.getenv(<span class="hljs-string">&quot;TRANSFORMERS_VERBOSITY&quot;</span>, <span class="hljs-literal">None</span>)`}}),Xa=new w({props:{code:`from transformers.testing_utils import TestCasePlus


class EnvExampleTest(TestCasePlus):
    def test_external_prog(self):
        env = self.get_env()
        # now call the external program, passing \`env\` to it`,highlighted:`<span class="hljs-keyword">from</span> transformers.testing_utils <span class="hljs-keyword">import</span> TestCasePlus


<span class="hljs-keyword">class</span> <span class="hljs-title class_">EnvExampleTest</span>(<span class="hljs-title class_ inherited__">TestCasePlus</span>):
    <span class="hljs-keyword">def</span> <span class="hljs-title function_">test_external_prog</span>(<span class="hljs-params">self</span>):
        env = self.get_env()
        <span class="hljs-comment"># now call the external program, passing \`env\` to it</span>`}}),Ja=new E({}),Za=new w({props:{code:`seed = 42

# python RNG
import random

random.seed(seed)

# pytorch RNGs
import torch

torch.manual_seed(seed)
torch.backends.cudnn.deterministic = True
if torch.cuda.is_available():
    torch.cuda.manual_seed_all(seed)

# numpy RNG
import numpy as np

np.random.seed(seed)

# tf RNG
tf.random.set_seed(seed)`,highlighted:`seed = <span class="hljs-number">42</span>

<span class="hljs-comment"># python RNG</span>
<span class="hljs-keyword">import</span> random

random.seed(seed)

<span class="hljs-comment"># pytorch RNGs</span>
<span class="hljs-keyword">import</span> torch

torch.manual_seed(seed)
torch.backends.cudnn.deterministic = <span class="hljs-literal">True</span>
<span class="hljs-keyword">if</span> torch.cuda.is_available():
    torch.cuda.manual_seed_all(seed)

<span class="hljs-comment"># numpy RNG</span>
<span class="hljs-keyword">import</span> numpy <span class="hljs-keyword">as</span> np

np.random.seed(seed)

<span class="hljs-comment"># tf RNG</span>
tf.random.set_seed(seed)`}}),Ka=new E({}),Qa=new w({props:{code:"pytest tests/test_logging.py -W error::UserWarning --pdb",highlighted:"pytest tests/test_logging.py -W error::UserWarning --pdb"}}),ei=new E({}),li=new E({}),ai=new w({props:{code:`- run:
    name: run CI experiment
    command: |
        set +euo pipefail
        echo "setting run-all-despite-any-errors-mode"
        this_command_will_fail
        echo "but bash continues to run"
        # emulate another failure
        false
        # but the last command must be a success
        echo "during experiment do not remove: reporting success to CI, even if there were failures"`,highlighted:`<span class="hljs-bullet">-</span> <span class="hljs-attr">run:</span>
    <span class="hljs-attr">name:</span> <span class="hljs-string">run</span> <span class="hljs-string">CI</span> <span class="hljs-string">experiment</span>
    <span class="hljs-attr">command:</span> <span class="hljs-string">|
        set +euo pipefail
        echo &quot;setting run-all-despite-any-errors-mode&quot;
        this_command_will_fail
        echo &quot;but bash continues to run&quot;
        # emulate another failure
        false
        # but the last command must be a success
        echo &quot;during experiment do not remove: reporting success to CI, even if there were failures&quot;</span>`}}),ii=new w({props:{code:"cmd_that_may_fail || true",highlighted:'cmd_that_may_fail || <span class="hljs-literal">true</span>'}}),{c(){$=l("meta"),O=f(),g=l("h1"),x=l("a"),A=l("span"),d(C.$$.fragment),R=f(),D=l("span"),P=o("Testing"),b=f(),S=l("p"),z=o("Let\u2019s take a look at how \u{1F917} Transformers models are tested and how you can write new tests and improve the existing ones."),Rt=f(),we=l("p"),Ye=o("There are 2 test suites in the repository:"),q=f(),me=l("ol"),Ve=l("li"),zt=l("code"),ci=o("tests"),hg=o(" \u2014 tests for the general API"),ug=f(),mi=l("li"),Cp=l("code"),dg=o("examples"),cg=o(" \u2014 tests primarily for various applications that aren\u2019t part of the API"),Dm=f(),Xe=l("h2"),Gt=l("a"),jp=l("span"),d(Ro.$$.fragment),mg=f(),Pp=l("span"),_g=o("How transformers are tested"),qm=f(),Ft=l("ol"),zo=l("li"),Go=l("p"),vg=o(`Once a PR is submitted it gets tested with 9 CircleCi jobs. Every new commit to that PR gets retested. These jobs
are defined in this `),Fo=l("a"),yg=o("config file"),wg=o(`, so that if needed you can reproduce the same
environment on your machine.`),$g=f(),Wo=l("p"),Eg=o("These CI jobs don\u2019t run "),Op=l("code"),gg=o("@slow"),bg=o(" tests."),kg=f(),Mo=l("li"),Bo=l("p"),xg=o("There are 3 jobs run by "),Yo=l("a"),Cg=o("github actions"),jg=o(":"),Pg=f(),Je=l("ul"),Ip=l("li"),_i=l("p"),Vo=l("a"),Og=o("torch hub integration"),Ig=o(`: checks whether torch hub
integration works.`),Tg=f(),Tp=l("li"),L=l("p"),Xo=l("a"),Ag=o("self-hosted (push)"),Dg=o(`: runs fast tests on GPU only on commits on
`),Ap=l("code"),qg=o("main"),Sg=o(". It only runs if a commit on "),Dp=l("code"),Lg=o("main"),Ug=o(" has updated the code in one of the following folders: "),qp=l("code"),Ng=o("src"),Hg=o(`,
`),Sp=l("code"),Rg=o("tests"),zg=o(", "),Lp=l("code"),Gg=o(".github"),Fg=o(" (to prevent running on added model cards, notebooks, etc.)"),Wg=f(),Up=l("li"),$e=l("p"),Jo=l("a"),Mg=o("self-hosted runner"),Bg=o(`: runs normal and slow tests on GPU in
`),Np=l("code"),Yg=o("tests"),Vg=o(" and "),Hp=l("code"),Xg=o("examples"),Jg=o(":"),Sm=f(),d(Zo.$$.fragment),Lm=f(),Wt=l("p"),Zg=o("The results can be observed "),Ko=l("a"),Kg=o("here"),Qg=o("."),Um=f(),Ze=l("h2"),Mt=l("a"),Rp=l("span"),d(Qo.$$.fragment),eb=f(),zp=l("span"),tb=o("Running tests"),Nm=f(),Ke=l("h3"),Bt=l("a"),Gp=l("span"),d(er.$$.fragment),sb=f(),Fp=l("span"),ob=o("Choosing which tests to run"),Hm=f(),Yt=l("p"),rb=o(`This document goes into many details of how tests can be run. If after reading everything, you need even more details
you will find them `),tr=l("a"),lb=o("here"),ab=o("."),Rm=f(),vi=l("p"),ib=o("Here are some most useful ways of running tests."),zm=f(),yi=l("p"),nb=o("Run all:"),Gm=f(),d(sr.$$.fragment),Fm=f(),wi=l("p"),pb=o("or:"),Wm=f(),d(or.$$.fragment),Mm=f(),$i=l("p"),fb=o("Note that the latter is defined as:"),Bm=f(),d(rr.$$.fragment),Ym=f(),Ei=l("p"),hb=o("which tells pytest to:"),Vm=f(),Q=l("ul"),Wp=l("li"),ub=o("run as many test processes as they are CPU cores (which could be too many if you don\u2019t have a ton of RAM!)"),db=f(),Mp=l("li"),cb=o("ensure that all tests from the same file will be run by the same test process"),mb=f(),Bp=l("li"),_b=o("do not capture output"),vb=f(),Yp=l("li"),yb=o("run in verbose mode"),Xm=f(),Qe=l("h3"),Vt=l("a"),Vp=l("span"),d(lr.$$.fragment),wb=f(),Xp=l("span"),$b=o("Getting the list of all tests"),Jm=f(),gi=l("p"),Eb=o("All tests of the test suite:"),Zm=f(),d(ar.$$.fragment),Km=f(),bi=l("p"),gb=o("All tests of a given test file:"),Qm=f(),d(ir.$$.fragment),e_=f(),et=l("h3"),Xt=l("a"),Jp=l("span"),d(nr.$$.fragment),bb=f(),Zp=l("span"),kb=o("Run a specific test module"),t_=f(),ki=l("p"),xb=o("To run an individual test module:"),s_=f(),d(pr.$$.fragment),o_=f(),tt=l("h3"),Jt=l("a"),Kp=l("span"),d(fr.$$.fragment),Cb=f(),Qp=l("span"),jb=o("Run specific tests"),r_=f(),xi=l("p"),Pb=o(`Since unittest is used inside most of the tests, to run specific subtests you need to know the name of the unittest
class containing those tests. For example, it could be:`),l_=f(),d(hr.$$.fragment),a_=f(),Ci=l("p"),Ob=o("Here:"),i_=f(),Ee=l("ul"),ji=l("li"),ef=l("code"),Ib=o("tests/test_optimization.py"),Tb=o(" - the file with tests"),Ab=f(),Pi=l("li"),tf=l("code"),Db=o("OptimizationTest"),qb=o(" - the name of the class"),Sb=f(),Oi=l("li"),sf=l("code"),Lb=o("test_adam_w"),Ub=o(" - the name of the specific test function"),n_=f(),Ii=l("p"),Nb=o("If the file contains multiple classes, you can choose to run only tests of a given class. For example:"),p_=f(),d(ur.$$.fragment),f_=f(),Ti=l("p"),Hb=o("will run all the tests inside that class."),h_=f(),Zt=l("p"),Rb=o("As mentioned earlier you can see what tests are contained inside the "),of=l("code"),zb=o("OptimizationTest"),Gb=o(" class by running:"),u_=f(),d(dr.$$.fragment),d_=f(),Ai=l("p"),Fb=o("You can run tests by keyword expressions."),c_=f(),Kt=l("p"),Wb=o("To run only tests whose name contains "),rf=l("code"),Mb=o("adam"),Bb=o(":"),m_=f(),d(cr.$$.fragment),__=f(),ee=l("p"),Yb=o("Logical "),lf=l("code"),Vb=o("and"),Xb=o(" and "),af=l("code"),Jb=o("or"),Zb=o(" can be used to indicate whether all keywords should match or either. "),nf=l("code"),Kb=o("not"),Qb=o(` can be used to
negate.`),v_=f(),Qt=l("p"),e3=o("To run all tests except those whose name contains "),pf=l("code"),t3=o("adam"),s3=o(":"),y_=f(),d(mr.$$.fragment),w_=f(),Di=l("p"),o3=o("And you can combine the two patterns in one:"),$_=f(),d(_r.$$.fragment),E_=f(),ge=l("p"),r3=o("For example to run both "),ff=l("code"),l3=o("test_adafactor"),a3=o(" and "),hf=l("code"),i3=o("test_adam_w"),n3=o(" you can use:"),g_=f(),d(vr.$$.fragment),b_=f(),es=l("p"),p3=o("Note that we use "),uf=l("code"),f3=o("or"),h3=o(" here, since we want either of the keywords to match to include both."),k_=f(),ts=l("p"),u3=o("If you want to include only tests that include both patterns, "),df=l("code"),d3=o("and"),c3=o(" is to be used:"),x_=f(),d(yr.$$.fragment),C_=f(),st=l("h3"),ss=l("a"),cf=l("span"),d(wr.$$.fragment),m3=f(),mf=l("span"),_3=o("Run documentation tests"),j_=f(),be=l("p"),v3=o("In order to test whether the documentation examples are correct, you should checkt that the "),_f=l("code"),y3=o("doctests"),w3=o(` are passing.
As an example, let\u2019s use `),os=l("a"),vf=l("code"),$3=o("WhisperModel.forward"),E3=o("\u2019s docstring"),g3=o(":"),P_=f(),d($r.$$.fragment),O_=f(),qi=l("p"),b3=o("3 steps are required to debug the docstring examples :"),I_=f(),Si=l("ol"),Er=l("li"),k3=o("In order to properly run the test, "),yf=l("strong"),x3=o("an extra line has to be added"),C3=o(" at the end of the docstring. This can be automatically done on any file using :"),T_=f(),d(gr.$$.fragment),A_=f(),br=l("ol"),wf=l("li"),j3=o("Then, you can use the following line to automatically test every docstring example in the desired file :"),D_=f(),d(kr.$$.fragment),q_=f(),xr=l("ol"),Cr=l("li"),P3=o("Once you are done debugging, you need to remove the extra line added in step "),$f=l("strong"),O3=o("1."),I3=o(" by running the follwing :"),S_=f(),d(jr.$$.fragment),L_=f(),ot=l("h3"),rs=l("a"),Ef=l("span"),d(Pr.$$.fragment),T3=f(),gf=l("span"),A3=o("Run only modified tests"),U_=f(),ls=l("p"),D3=o("You can run the tests related to the unstaged files or the current branch (according to Git) by using "),Or=l("a"),q3=o("pytest-picked"),S3=o(`. This is a great way of quickly testing your changes didn\u2019t break
anything, since it won\u2019t run the tests related to files you didn\u2019t touch.`),N_=f(),d(Ir.$$.fragment),H_=f(),d(Tr.$$.fragment),R_=f(),Li=l("p"),L3=o("All tests will be run from files and folders which are modified, but not yet committed."),z_=f(),rt=l("h3"),as=l("a"),bf=l("span"),d(Ar.$$.fragment),U3=f(),kf=l("span"),N3=o("Automatically rerun failed tests on source modification"),G_=f(),Dr=l("p"),qr=l("a"),H3=o("pytest-xdist"),R3=o(` provides a very useful feature of detecting all failed
tests, and then waiting for you to modify files and continuously re-rerun those failing tests until they pass while you
fix them. So that you don\u2019t need to re start pytest after you made the fix. This is repeated until all tests pass after
which again a full run is performed.`),F_=f(),d(Sr.$$.fragment),W_=f(),lt=l("p"),z3=o("To enter the mode: "),xf=l("code"),G3=o("pytest -f"),F3=o(" or "),Cf=l("code"),W3=o("pytest --looponfail"),M_=f(),ke=l("p"),M3=o("File changes are detected by looking at "),jf=l("code"),B3=o("looponfailroots"),Y3=o(` root directories and all of their contents (recursively).
If the default for this value does not work for you, you can change it in your project by setting a configuration
option in `),Pf=l("code"),V3=o("setup.cfg"),X3=o(":"),B_=f(),d(Lr.$$.fragment),Y_=f(),xe=l("p"),J3=o("or "),Of=l("code"),Z3=o("pytest.ini"),K3=o("/"),If=l("code"),Q3=o("tox.ini"),e2=o(" files:"),V_=f(),d(Ur.$$.fragment),X_=f(),Ui=l("p"),t2=o(`This would lead to only looking for file changes in the respective directories, specified relatively to the ini-file\u2019s
directory.`),J_=f(),Nr=l("p"),Hr=l("a"),s2=o("pytest-watch"),o2=o(" is an alternative implementation of this functionality."),Z_=f(),at=l("h3"),is=l("a"),Tf=l("span"),d(Rr.$$.fragment),r2=f(),Af=l("span"),l2=o("Skip a test module"),K_=f(),ns=l("p"),a2=o(`If you want to run all test modules, except a few you can exclude them by giving an explicit list of tests to run. For
example, to run all except `),Df=l("code"),i2=o("test_modeling_*.py"),n2=o(" tests:"),Q_=f(),d(zr.$$.fragment),ev=f(),it=l("h3"),ps=l("a"),qf=l("span"),d(Gr.$$.fragment),p2=f(),Sf=l("span"),f2=o("Clearing state"),tv=f(),Ni=l("p"),h2=o("CI builds and when isolation is important (against speed), cache should be cleared:"),sv=f(),d(Fr.$$.fragment),ov=f(),nt=l("h3"),fs=l("a"),Lf=l("span"),d(Wr.$$.fragment),u2=f(),Uf=l("span"),d2=o("Running tests in parallel"),rv=f(),W=l("p"),c2=o("As mentioned earlier "),Nf=l("code"),m2=o("make test"),_2=o(" runs tests in parallel via "),Hf=l("code"),v2=o("pytest-xdist"),y2=o(" plugin ("),Rf=l("code"),w2=o("-n X"),$2=o(" argument, e.g. "),zf=l("code"),E2=o("-n 2"),g2=o(`
to run 2 parallel jobs).`),lv=f(),_e=l("p"),Gf=l("code"),b2=o("pytest-xdist"),k2=o("\u2019s "),Ff=l("code"),x2=o("--dist="),C2=o(" option allows one to control how the tests are grouped. "),Wf=l("code"),j2=o("--dist=loadfile"),P2=o(` puts the
tests located in one file onto the same process.`),av=f(),Ce=l("p"),O2=o("Since the order of executed tests is different and unpredictable, if running the test suite with "),Mf=l("code"),I2=o("pytest-xdist"),T2=o(`
produces failures (meaning we have some undetected coupled tests), use `),Mr=l("a"),A2=o("pytest-replay"),D2=o(` to replay the tests in the same order, which should help with then somehow
reducing that failing sequence to a minimum.`),iv=f(),pt=l("h3"),hs=l("a"),Bf=l("span"),d(Br.$$.fragment),q2=f(),Yf=l("span"),S2=o("Test order and repetition"),nv=f(),Hi=l("p"),L2=o(`It\u2019s good to repeat the tests several times, in sequence, randomly, or in sets, to detect any potential
inter-dependency and state-related bugs (tear down). And the straightforward multiple repetition is just good to detect
some problems that get uncovered by randomness of DL.`),pv=f(),ft=l("h4"),us=l("a"),Vf=l("span"),d(Yr.$$.fragment),U2=f(),Xf=l("span"),N2=o("Repeat tests"),fv=f(),Ri=l("ul"),zi=l("li"),Vr=l("a"),H2=o("pytest-flakefinder"),R2=o(":"),hv=f(),d(Xr.$$.fragment),uv=f(),Gi=l("p"),z2=o("And then run every test multiple times (50 by default):"),dv=f(),d(Jr.$$.fragment),cv=f(),d(ds.$$.fragment),mv=f(),d(cs.$$.fragment),_v=f(),ht=l("h4"),ms=l("a"),Jf=l("span"),d(Zr.$$.fragment),G2=f(),Zf=l("span"),F2=o("Run tests in a random order"),vv=f(),d(Kr.$$.fragment),yv=f(),_s=l("p"),W2=o("Important: the presence of "),Kf=l("code"),M2=o("pytest-random-order"),B2=o(` will automatically randomize tests, no configuration change or
command line options is required.`),wv=f(),vs=l("p"),Y2=o(`As explained earlier this allows detection of coupled tests - where one test\u2019s state affects the state of another. When
`),Qf=l("code"),V2=o("pytest-random-order"),X2=o(" is installed it will print the random seed it used for that session, e.g:"),$v=f(),d(Qr.$$.fragment),Ev=f(),Fi=l("p"),J2=o("So that if the given particular sequence fails, you can reproduce it by adding that exact seed, e.g.:"),gv=f(),d(el.$$.fragment),bv=f(),ys=l("p"),Z2=o(`It will only reproduce the exact order if you use the exact same list of tests (or no list at all). Once you start to
manually narrowing down the list you can no longer rely on the seed, but have to list them manually in the exact order
they failed and tell pytest to not randomize them instead using `),eh=l("code"),K2=o("--random-order-bucket=none"),Q2=o(", e.g.:"),kv=f(),d(tl.$$.fragment),xv=f(),Wi=l("p"),e5=o("To disable the shuffling for all tests:"),Cv=f(),d(sl.$$.fragment),jv=f(),I=l("p"),t5=o("By default "),th=l("code"),s5=o("--random-order-bucket=module"),o5=o(` is implied, which will shuffle the files on the module levels. It can also
shuffle on `),sh=l("code"),r5=o("class"),l5=o(", "),oh=l("code"),a5=o("package"),i5=o(", "),rh=l("code"),n5=o("global"),p5=o(" and "),lh=l("code"),f5=o("none"),h5=o(` levels. For the complete details please see its
`),ol=l("a"),u5=o("documentation"),d5=o("."),Pv=f(),je=l("p"),c5=o("Another randomization alternative is: "),rl=l("a"),ah=l("code"),m5=o("pytest-randomly"),_5=o(`. This
module has a very similar functionality/interface, but it doesn\u2019t have the bucket modes available in
`),ih=l("code"),v5=o("pytest-random-order"),y5=o(". It has the same problem of imposing itself once installed."),Ov=f(),ut=l("h3"),ws=l("a"),nh=l("span"),d(ll.$$.fragment),w5=f(),ph=l("span"),$5=o("Look and feel variations"),Iv=f(),dt=l("h4"),$s=l("a"),fh=l("span"),d(al.$$.fragment),E5=f(),hh=l("span"),g5=o("pytest-sugar"),Tv=f(),il=l("p"),nl=l("a"),b5=o("pytest-sugar"),k5=o(` is a plugin that improves the look-n-feel, adds a
progressbar, and show tests that fail and the assert instantly. It gets activated automatically upon installation.`),Av=f(),d(pl.$$.fragment),Dv=f(),Mi=l("p"),x5=o("To run tests without it, run:"),qv=f(),d(fl.$$.fragment),Sv=f(),Bi=l("p"),C5=o("or uninstall it."),Lv=f(),ct=l("h4"),Es=l("a"),uh=l("span"),d(hl.$$.fragment),j5=f(),dh=l("span"),P5=o("Report each sub-test name and its progress"),Uv=f(),Pe=l("p"),O5=o("For a single or a group of tests via "),ch=l("code"),I5=o("pytest"),T5=o(" (after "),mh=l("code"),A5=o("pip install pytest-pspec"),D5=o("):"),Nv=f(),d(ul.$$.fragment),Hv=f(),mt=l("h4"),gs=l("a"),_h=l("span"),d(dl.$$.fragment),q5=f(),vh=l("span"),S5=o("Instantly shows failed tests"),Rv=f(),cl=l("p"),ml=l("a"),L5=o("pytest-instafail"),U5=o(` shows failures and errors instantly instead of
waiting until the end of test session.`),zv=f(),d(_l.$$.fragment),Gv=f(),d(vl.$$.fragment),Fv=f(),_t=l("h3"),bs=l("a"),yh=l("span"),d(yl.$$.fragment),N5=f(),wh=l("span"),H5=o("To GPU or not to GPU"),Wv=f(),ks=l("p"),R5=o("On a GPU-enabled setup, to test in CPU-only mode add "),$h=l("code"),z5=o('CUDA_VISIBLE_DEVICES=""'),G5=o(":"),Mv=f(),d(wl.$$.fragment),Bv=f(),te=l("p"),F5=o("or if you have multiple gpus, you can specify which one is to be used by "),Eh=l("code"),W5=o("pytest"),M5=o(`. For example, to use only the
second gpu if you have gpus `),gh=l("code"),B5=o("0"),Y5=o(" and "),bh=l("code"),V5=o("1"),X5=o(", you can run:"),Yv=f(),d($l.$$.fragment),Vv=f(),Yi=l("p"),J5=o("This is handy when you want to run different tasks on different GPUs."),Xv=f(),Vi=l("p"),Z5=o(`Some tests must be run on CPU-only, others on either CPU or GPU or TPU, yet others on multiple-GPUs. The following skip
decorators are used to set the requirements of tests CPU/GPU/TPU-wise:`),Jv=f(),U=l("ul"),Xi=l("li"),kh=l("code"),K5=o("require_torch"),Q5=o(" - this test will run only under torch"),ek=f(),xs=l("li"),xh=l("code"),tk=o("require_torch_gpu"),sk=o(" - as "),Ch=l("code"),ok=o("require_torch"),rk=o(" plus requires at least 1 GPU"),lk=f(),Cs=l("li"),jh=l("code"),ak=o("require_torch_multi_gpu"),ik=o(" - as "),Ph=l("code"),nk=o("require_torch"),pk=o(" plus requires at least 2 GPUs"),fk=f(),js=l("li"),Oh=l("code"),hk=o("require_torch_non_multi_gpu"),uk=o(" - as "),Ih=l("code"),dk=o("require_torch"),ck=o(" plus requires 0 or 1 GPUs"),mk=f(),Ps=l("li"),Th=l("code"),_k=o("require_torch_up_to_2_gpus"),vk=o(" - as "),Ah=l("code"),yk=o("require_torch"),wk=o(" plus requires 0 or 1 or 2 GPUs"),$k=f(),Os=l("li"),Dh=l("code"),Ek=o("require_torch_tpu"),gk=o(" - as "),qh=l("code"),bk=o("require_torch"),kk=o(" plus requires at least 1 TPU"),Zv=f(),Ji=l("p"),xk=o("Let\u2019s depict the GPU requirements in the following table:"),Kv=f(),k=l("p"),Ck=o(`| n gpus | decorator                      |
|--------+--------------------------------|
| `),Sh=l("code"),jk=o(">= 0"),Pk=o(" | "),Lh=l("code"),Ok=o("@require_torch"),Ik=o(`               |
| `),Uh=l("code"),Tk=o(">= 1"),Ak=o(" | "),Nh=l("code"),Dk=o("@require_torch_gpu"),qk=o(`           |
| `),Hh=l("code"),Sk=o(">= 2"),Lk=o(" | "),Rh=l("code"),Uk=o("@require_torch_multi_gpu"),Nk=o(`     |
| `),zh=l("code"),Hk=o("< 2"),Rk=o("  | "),Gh=l("code"),zk=o("@require_torch_non_multi_gpu"),Gk=o(` |
| `),Fh=l("code"),Fk=o("< 3"),Wk=o("  | "),Wh=l("code"),Mk=o("@require_torch_up_to_2_gpus"),Bk=o("  |"),Qv=f(),Zi=l("p"),Yk=o("For example, here is a test that must be run only when there are 2 or more GPUs available and pytorch is installed:"),e1=f(),d(El.$$.fragment),t1=f(),Oe=l("p"),Vk=o("If a test requires "),Mh=l("code"),Xk=o("tensorflow"),Jk=o(" use the "),Bh=l("code"),Zk=o("require_tf"),Kk=o(" decorator. For example:"),s1=f(),d(gl.$$.fragment),o1=f(),Ki=l("p"),Qk=o(`These decorators can be stacked. For example, if a test is slow and requires at least one GPU under pytorch, here is
how to set it up:`),r1=f(),d(bl.$$.fragment),l1=f(),Ie=l("p"),e0=o("Some decorators like "),Yh=l("code"),t0=o("@parametrized"),s0=o(" rewrite test names, therefore "),Vh=l("code"),o0=o("@require_*"),r0=o(` skip decorators have to be listed
last for them to work correctly. Here is an example of the correct usage:`),a1=f(),d(kl.$$.fragment),i1=f(),Is=l("p"),l0=o("This order problem doesn\u2019t exist with "),Xh=l("code"),a0=o("@pytest.mark.parametrize"),i0=o(`, you can put it first or last and it will still
work. But it only works with non-unittests.`),n1=f(),Qi=l("p"),n0=o("Inside tests:"),p1=f(),en=l("ul"),Jh=l("li"),p0=o("How many GPUs are available:"),f1=f(),d(xl.$$.fragment),h1=f(),vt=l("h3"),Ts=l("a"),Zh=l("span"),d(Cl.$$.fragment),f0=f(),Kh=l("span"),h0=o("Distributed training"),u1=f(),yt=l("p"),Qh=l("code"),u0=o("pytest"),d0=o(` can\u2019t deal with distributed training directly. If this is attempted - the sub-processes don\u2019t do the right
thing and end up thinking they are `),eu=l("code"),c0=o("pytest"),m0=o(` and start running the test suite in loops. It works, however, if one
spawns a normal process that then spawns off multiple workers and manages the IO pipes.`),d1=f(),tn=l("p"),_0=o("Here are some tests that use it:"),c1=f(),As=l("ul"),tu=l("li"),jl=l("a"),v0=o("test_trainer_distributed.py"),y0=f(),su=l("li"),Pl=l("a"),w0=o("test_deepspeed.py"),m1=f(),Ds=l("p"),$0=o("To jump right into the execution point, search for the "),ou=l("code"),E0=o("execute_subprocess_async"),g0=o(" call in those tests."),_1=f(),sn=l("p"),b0=o("You will need at least 2 GPUs to see these tests in action:"),v1=f(),d(Ol.$$.fragment),y1=f(),wt=l("h3"),qs=l("a"),ru=l("span"),d(Il.$$.fragment),k0=f(),lu=l("span"),x0=o("Output capture"),w1=f(),Te=l("p"),C0=o("During test execution any output sent to "),au=l("code"),j0=o("stdout"),P0=o(" and "),iu=l("code"),O0=o("stderr"),I0=o(` is captured. If a test or a setup method fails, its
according captured output will usually be shown along with the failure traceback.`),$1=f(),M=l("p"),T0=o("To disable output capturing and to get the "),nu=l("code"),A0=o("stdout"),D0=o(" and "),pu=l("code"),q0=o("stderr"),S0=o(" normally, use "),fu=l("code"),L0=o("-s"),U0=o(" or "),hu=l("code"),N0=o("--capture=no"),H0=o(":"),E1=f(),d(Tl.$$.fragment),g1=f(),on=l("p"),R0=o("To send test results to JUnit format output:"),b1=f(),d(Al.$$.fragment),k1=f(),$t=l("h3"),Ss=l("a"),uu=l("span"),d(Dl.$$.fragment),z0=f(),du=l("span"),G0=o("Color control"),x1=f(),rn=l("p"),F0=o("To have no color (e.g., yellow on white background is not readable):"),C1=f(),d(ql.$$.fragment),j1=f(),Et=l("h3"),Ls=l("a"),cu=l("span"),d(Sl.$$.fragment),W0=f(),mu=l("span"),M0=o("Sending test report to online pastebin service"),P1=f(),ln=l("p"),B0=o("Creating a URL for each test failure:"),O1=f(),d(Ll.$$.fragment),I1=f(),an=l("p"),Y0=o(`This will submit test run information to a remote Paste service and provide a URL for each failure. You may select
tests as usual or add for example -x if you only want to send one particular failure.`),T1=f(),nn=l("p"),V0=o("Creating a URL for a whole test session log:"),A1=f(),d(Ul.$$.fragment),D1=f(),gt=l("h2"),Us=l("a"),_u=l("span"),d(Nl.$$.fragment),X0=f(),vu=l("span"),J0=o("Writing tests"),q1=f(),Ae=l("p"),Z0=o("\u{1F917} transformers tests are based on "),yu=l("code"),K0=o("unittest"),Q0=o(", but run by "),wu=l("code"),e4=o("pytest"),t4=o(`, so most of the time features from both systems
can be used.`),S1=f(),se=l("p"),s4=o("You can read "),Hl=l("a"),o4=o("here"),r4=o(` which features are supported, but the important
thing to remember is that most `),$u=l("code"),l4=o("pytest"),a4=o(` fixtures don\u2019t work. Neither parametrization, but we use the module
`),Eu=l("code"),i4=o("parameterized"),n4=o(" that works in a similar way."),L1=f(),bt=l("h3"),Ns=l("a"),gu=l("span"),d(Rl.$$.fragment),p4=f(),bu=l("span"),f4=o("Parametrization"),U1=f(),pn=l("p"),h4=o(`Often, there is a need to run the same test multiple times, but with different arguments. It could be done from within
the test, but then there is no way of running that test for just one set of arguments.`),N1=f(),d(zl.$$.fragment),H1=f(),Hs=l("p"),u4=o("Now, by default this test will be run 3 times, each time with the last 3 arguments of "),ku=l("code"),d4=o("test_floor"),c4=o(` being assigned the
corresponding arguments in the parameter list.`),R1=f(),De=l("p"),m4=o("and you could run just the "),xu=l("code"),_4=o("negative"),v4=o(" and "),Cu=l("code"),y4=o("integer"),w4=o(" sets of params with:"),z1=f(),d(Gl.$$.fragment),G1=f(),Rs=l("p"),$4=o("or all but "),ju=l("code"),E4=o("negative"),g4=o(" sub-tests, with:"),F1=f(),d(Fl.$$.fragment),W1=f(),zs=l("p"),b4=o("Besides using the "),Pu=l("code"),k4=o("-k"),x4=o(` filter that was just mentioned, you can find out the exact name of each sub-test and run any
or all of them using their exact names.`),M1=f(),d(Wl.$$.fragment),B1=f(),fn=l("p"),C4=o("and it will list:"),Y1=f(),d(Ml.$$.fragment),V1=f(),hn=l("p"),j4=o("So now you can run just 2 specific sub-tests:"),X1=f(),d(Bl.$$.fragment),J1=f(),B=l("p"),P4=o("The module "),Yl=l("a"),O4=o("parameterized"),I4=o(` which is already in the developer dependencies
of `),Ou=l("code"),T4=o("transformers"),A4=o(" works for both: "),Iu=l("code"),D4=o("unittests"),q4=o(" and "),Tu=l("code"),S4=o("pytest"),L4=o(" tests."),Z1=f(),oe=l("p"),U4=o("If, however, the test is not a "),Au=l("code"),N4=o("unittest"),H4=o(", you may use "),Du=l("code"),R4=o("pytest.mark.parametrize"),z4=o(` (or you may see it being used in
some existing tests, mostly under `),qu=l("code"),G4=o("examples"),F4=o(")."),K1=f(),qe=l("p"),W4=o("Here is the same example, this time using "),Su=l("code"),M4=o("pytest"),B4=o("\u2019s "),Lu=l("code"),Y4=o("parametrize"),V4=o(" marker:"),Q1=f(),d(Vl.$$.fragment),ey=f(),re=l("p"),X4=o("Same as with "),Uu=l("code"),J4=o("parameterized"),Z4=o(", with "),Nu=l("code"),K4=o("pytest.mark.parametrize"),Q4=o(` you can have a fine control over which sub-tests are
run, if the `),Hu=l("code"),e6=o("-k"),t6=o(` filter doesn\u2019t do the job. Except, this parametrization function creates a slightly different set of
names for the sub-tests. Here is what they look like:`),ty=f(),d(Xl.$$.fragment),sy=f(),un=l("p"),s6=o("and it will list:"),oy=f(),d(Jl.$$.fragment),ry=f(),dn=l("p"),o6=o("So now you can run just the specific test:"),ly=f(),d(Zl.$$.fragment),ay=f(),cn=l("p"),r6=o("as in the previous example."),iy=f(),kt=l("h3"),Gs=l("a"),Ru=l("span"),d(Kl.$$.fragment),l6=f(),zu=l("span"),a6=o("Files and directories"),ny=f(),Fs=l("p"),i6=o(`In tests often we need to know where things are relative to the current test file, and it\u2019s not trivial since the test
could be invoked from more than one directory or could reside in sub-directories with different depths. A helper class
`),Gu=l("code"),n6=o("transformers.test_utils.TestCasePlus"),p6=o(` solves this problem by sorting out all the basic paths and provides easy
accessors to them:`),py=f(),Ws=l("ul"),Ql=l("li"),mn=l("p"),Fu=l("code"),f6=o("pathlib"),h6=o(" objects (all fully resolved):"),u6=f(),G=l("ul"),ea=l("li"),Wu=l("code"),d6=o("test_file_path"),c6=o(" - the current test file path, i.e. "),Mu=l("code"),m6=o("__file__"),_6=f(),_n=l("li"),Bu=l("code"),v6=o("test_file_dir"),y6=o(" - the directory containing the current test file"),w6=f(),Ms=l("li"),Yu=l("code"),$6=o("tests_dir"),E6=o(" - the directory of the "),Vu=l("code"),g6=o("tests"),b6=o(" test suite"),k6=f(),Bs=l("li"),Xu=l("code"),x6=o("examples_dir"),C6=o(" - the directory of the "),Ju=l("code"),j6=o("examples"),P6=o(" test suite"),O6=f(),vn=l("li"),Zu=l("code"),I6=o("repo_root_dir"),T6=o(" - the directory of the repository"),A6=f(),Se=l("li"),Ku=l("code"),D6=o("src_dir"),q6=o(" - the directory of "),Qu=l("code"),S6=o("src"),L6=o(" (i.e. where the "),ed=l("code"),U6=o("transformers"),N6=o(" sub-dir resides)"),H6=f(),ta=l("li"),sa=l("p"),R6=o("stringified paths---same as above but these return paths as strings, rather than "),td=l("code"),z6=o("pathlib"),G6=o(" objects:"),F6=f(),F=l("ul"),sd=l("li"),od=l("code"),W6=o("test_file_path_str"),M6=f(),rd=l("li"),ld=l("code"),B6=o("test_file_dir_str"),Y6=f(),ad=l("li"),id=l("code"),V6=o("tests_dir_str"),X6=f(),nd=l("li"),pd=l("code"),J6=o("examples_dir_str"),Z6=f(),fd=l("li"),hd=l("code"),K6=o("repo_root_dir_str"),Q6=f(),ud=l("li"),dd=l("code"),e7=o("src_dir_str"),fy=f(),Ys=l("p"),t7=o(`To start using those all you need is to make sure that the test resides in a subclass of
`),cd=l("code"),s7=o("transformers.test_utils.TestCasePlus"),o7=o(". For example:"),hy=f(),d(oa.$$.fragment),uy=f(),Y=l("p"),r7=o("If you don\u2019t need to manipulate paths via "),md=l("code"),l7=o("pathlib"),a7=o(` or you just need a path as a string, you can always invoked
`),_d=l("code"),i7=o("str()"),n7=o(" on the "),vd=l("code"),p7=o("pathlib"),f7=o(" object or use the accessors ending with "),yd=l("code"),h7=o("_str"),u7=o(". For example:"),dy=f(),d(ra.$$.fragment),cy=f(),xt=l("h3"),Vs=l("a"),wd=l("span"),d(la.$$.fragment),d7=f(),$d=l("span"),c7=o("Temporary files and directories"),my=f(),Xs=l("p"),m7=o(`Using unique temporary files and directories are essential for parallel test running, so that the tests won\u2019t overwrite
each other\u2019s data. Also we want to get the temporary files and directories removed at the end of each test that created
them. Therefore, using packages like `),Ed=l("code"),_7=o("tempfile"),v7=o(", which address these needs is essential."),_y=f(),yn=l("p"),y7=o(`However, when debugging tests, you need to be able to see what goes into the temporary file or directory and you want
to know it\u2019s exact path and not having it randomized on every test re-run.`),vy=f(),Le=l("p"),w7=o("A helper class "),gd=l("code"),$7=o("transformers.test_utils.TestCasePlus"),E7=o(` is best used for such purposes. It\u2019s a sub-class of
`),bd=l("code"),g7=o("unittest.TestCase"),b7=o(", so we can easily inherit from it in the test modules."),yy=f(),wn=l("p"),k7=o("Here is an example of its usage:"),wy=f(),d(aa.$$.fragment),$y=f(),Js=l("p"),x7=o("This code creates a unique temporary directory, and sets "),kd=l("code"),C7=o("tmp_dir"),j7=o(" to its location."),Ey=f(),$n=l("ul"),xd=l("li"),P7=o("Create a unique temporary dir:"),gy=f(),d(ia.$$.fragment),by=f(),na=l("p"),Cd=l("code"),O7=o("tmp_dir"),I7=o(` will contain the path to the created temporary dir. It will be automatically removed at the end of the
test.`),ky=f(),En=l("ul"),jd=l("li"),T7=o("Create a temporary dir of my choice, ensure it\u2019s empty before the test starts and don\u2019t empty it after the test."),xy=f(),d(pa.$$.fragment),Cy=f(),gn=l("p"),A7=o(`This is useful for debug when you want to monitor a specific directory and want to make sure the previous tests didn\u2019t
leave any data in there.`),jy=f(),bn=l("ul"),fa=l("li"),Ct=l("p"),D7=o("You can override the default behavior by directly overriding the "),Pd=l("code"),q7=o("before"),S7=o(" and "),Od=l("code"),L7=o("after"),U7=o(` args, leading to one of the
following behaviors:`),N7=f(),ve=l("ul"),kn=l("li"),Id=l("code"),H7=o("before=True"),R7=o(": the temporary dir will always be cleared at the beginning of the test."),z7=f(),xn=l("li"),Td=l("code"),G7=o("before=False"),F7=o(": if the temporary dir already existed, any existing files will remain there."),W7=f(),Cn=l("li"),Ad=l("code"),M7=o("after=True"),B7=o(": the temporary dir will always be deleted at the end of the test."),Y7=f(),jn=l("li"),Dd=l("code"),V7=o("after=False"),X7=o(": the temporary dir will always be left intact at the end of the test."),Py=f(),d(Zs.$$.fragment),Oy=f(),d(Ks.$$.fragment),Iy=f(),jt=l("h3"),Qs=l("a"),qd=l("span"),d(ha.$$.fragment),J7=f(),Sd=l("span"),Z7=o("Temporary sys.path override"),Ty=f(),Ue=l("p"),K7=o("If you need to temporary override "),Ld=l("code"),Q7=o("sys.path"),e8=o(` to import from another test for example, you can use the
`),Ud=l("code"),t8=o("ExtendSysPath"),s8=o(" context manager. Example:"),Ay=f(),d(ua.$$.fragment),Dy=f(),Pt=l("h3"),eo=l("a"),Nd=l("span"),d(da.$$.fragment),o8=f(),Hd=l("span"),r8=o("Skipping tests"),qy=f(),to=l("p"),l8=o(`This is useful when a bug is found and a new test is written, yet the bug is not fixed yet. In order to be able to
commit it to the main repository we need make sure it\u2019s skipped during `),Rd=l("code"),a8=o("make test"),i8=o("."),Sy=f(),Pn=l("p"),n8=o("Methods:"),Ly=f(),so=l("ul"),zd=l("li"),ca=l("p"),p8=o("A "),Gd=l("strong"),f8=o("skip"),h8=o(` means that you expect your test to pass only if some conditions are met, otherwise pytest should skip
running the test altogether. Common examples are skipping windows-only tests on non-windows platforms, or skipping
tests that depend on an external resource which is not available at the moment (for example a database).`),u8=f(),Fd=l("li"),ma=l("p"),d8=o("A "),Wd=l("strong"),c8=o("xfail"),m8=o(` means that you expect a test to fail for some reason. A common example is a test for a feature not yet
implemented, or a bug not yet fixed. When a test passes despite being expected to fail (marked with
pytest.mark.xfail), it\u2019s an xpass and will be reported in the test summary.`),Uy=f(),le=l("p"),_8=o("One of the important differences between the two is that "),Md=l("code"),v8=o("skip"),y8=o(" doesn\u2019t run the test, and "),Bd=l("code"),w8=o("xfail"),$8=o(` does. So if the
code that\u2019s buggy causes some bad state that will affect other tests, do not use `),Yd=l("code"),E8=o("xfail"),g8=o("."),Ny=f(),Ot=l("h4"),oo=l("a"),Vd=l("span"),d(_a.$$.fragment),b8=f(),Xd=l("span"),k8=o("Implementation"),Hy=f(),On=l("ul"),Jd=l("li"),x8=o("Here is how to skip whole test unconditionally:"),Ry=f(),d(va.$$.fragment),zy=f(),In=l("p"),C8=o("or via pytest:"),Gy=f(),d(ya.$$.fragment),Fy=f(),ro=l("p"),j8=o("or the "),Zd=l("code"),P8=o("xfail"),O8=o(" way:"),Wy=f(),d(wa.$$.fragment),My=f(),Tn=l("ul"),Kd=l("li"),I8=o("Here is how to skip a test based on some internal check inside the test:"),By=f(),d($a.$$.fragment),Yy=f(),An=l("p"),T8=o("or the whole module:"),Vy=f(),d(Ea.$$.fragment),Xy=f(),lo=l("p"),A8=o("or the "),Qd=l("code"),D8=o("xfail"),q8=o(" way:"),Jy=f(),d(ga.$$.fragment),Zy=f(),Dn=l("ul"),ec=l("li"),S8=o("Here is how to skip all tests in a module if some import is missing:"),Ky=f(),d(ba.$$.fragment),Qy=f(),qn=l("ul"),tc=l("li"),L8=o("Skip a test based on a condition:"),ew=f(),d(ka.$$.fragment),tw=f(),Sn=l("p"),U8=o("or:"),sw=f(),d(xa.$$.fragment),ow=f(),Ln=l("p"),N8=o("or skip the whole module:"),rw=f(),d(Ca.$$.fragment),lw=f(),ao=l("p"),H8=o("More details, example and ways are "),ja=l("a"),R8=o("here"),z8=o("."),aw=f(),It=l("h3"),io=l("a"),sc=l("span"),d(Pa.$$.fragment),G8=f(),oc=l("span"),F8=o("Slow tests"),iw=f(),Un=l("p"),W8=o(`The library of tests is ever-growing, and some of the tests take minutes to run, therefore we can\u2019t afford waiting for
an hour for the test suite to complete on CI. Therefore, with some exceptions for essential tests, slow tests should be
marked as in the example below:`),nw=f(),d(Oa.$$.fragment),pw=f(),Ne=l("p"),M8=o("Once a test is marked as "),rc=l("code"),B8=o("@slow"),Y8=o(", to run such tests set "),lc=l("code"),V8=o("RUN_SLOW=1"),X8=o(" env var, e.g.:"),fw=f(),d(Ia.$$.fragment),hw=f(),ae=l("p"),J8=o("Some decorators like "),ac=l("code"),Z8=o("@parameterized"),K8=o(" rewrite test names, therefore "),ic=l("code"),Q8=o("@slow"),e9=o(` and the rest of the skip decorators
`),nc=l("code"),t9=o("@require_*"),s9=o(" have to be listed last for them to work correctly. Here is an example of the correct usage:"),uw=f(),d(Ta.$$.fragment),dw=f(),Nn=l("p"),o9=o(`As explained at the beginning of this document, slow tests get to run on a scheduled basis, rather than in PRs CI
checks. So it\u2019s possible that some problems will be missed during a PR submission and get merged. Such problems will
get caught during the next scheduled CI job. But it also means that it\u2019s important to run the slow tests on your
machine before submitting the PR.`),cw=f(),Hn=l("p"),r9=o("Here is a rough decision making mechanism for choosing which tests should be marked as slow:"),mw=f(),Rn=l("p"),l9=o(`If the test is focused on one of the library\u2019s internal components (e.g., modeling files, tokenization files,
pipelines), then we should run that test in the non-slow test suite. If it\u2019s focused on an other aspect of the library,
such as the documentation or the examples, then we should run these tests in the slow test suite. And then, to refine
this approach we should have exceptions:`),_w=f(),ie=l("ul"),pc=l("li"),a9=o(`All tests that need to download a heavy set of weights or a dataset that is larger than ~50MB (e.g., model or
tokenizer integration tests, pipeline integration tests) should be set to slow. If you\u2019re adding a new model, you
should create and upload to the hub a tiny version of it (with random weights) for integration tests. This is
discussed in the following paragraphs.`),i9=f(),fc=l("li"),n9=o("All tests that need to do a training not specifically optimized to be fast should be set to slow."),p9=f(),Tt=l("li"),f9=o(`We can introduce exceptions if some of these should-be-non-slow tests are excruciatingly slow, and set them to
`),hc=l("code"),h9=o("@slow"),u9=o(`. Auto-modeling tests, which save and load large files to disk, are a good example of tests that are marked
as `),uc=l("code"),d9=o("@slow"),c9=o("."),m9=f(),dc=l("li"),_9=o("If a test completes under 1 second on CI (including downloads if any) then it should be a normal test regardless."),vw=f(),He=l("p"),v9=o(`Collectively, all the non-slow tests need to cover entirely the different internals, while remaining fast. For example,
a significant coverage can be achieved by testing with specially created tiny models with random weights. Such models
have the very minimal number of layers (e.g., 2), vocab size (e.g., 1000), etc. Then the `),cc=l("code"),y9=o("@slow"),w9=o(` tests can use large
slow models to do qualitative testing. To see the use of these simply look for `),mc=l("em"),$9=o("tiny"),E9=o(" models with:"),yw=f(),d(Aa.$$.fragment),ww=f(),Re=l("p"),g9=o("Here is a an example of a "),Da=l("a"),b9=o("script"),k9=o(` that created the tiny model
`),qa=l("a"),x9=o("stas/tiny-wmt19-en-de"),C9=o(`. You can easily adjust it to your specific
model\u2019s architecture.`),$w=f(),no=l("p"),j9=o(`It\u2019s easy to measure the run-time incorrectly if for example there is an overheard of downloading a huge model, but if
you test it locally the downloaded files would be cached and thus the download time not measured. Hence check the
execution speed report in CI logs instead (the output of `),_c=l("code"),P9=o("pytest --durations=0 tests"),O9=o(")."),Ew=f(),zn=l("p"),I9=o(`That report is also useful to find slow outliers that aren\u2019t marked as such, or which need to be re-written to be fast.
If you notice that the test suite starts getting slow on CI, the top listing of this report will show the slowest
tests.`),gw=f(),At=l("h3"),po=l("a"),vc=l("span"),d(Sa.$$.fragment),T9=f(),yc=l("span"),A9=o("Testing the stdout/stderr output"),bw=f(),V=l("p"),D9=o("In order to test functions that write to "),wc=l("code"),q9=o("stdout"),S9=o(" and/or "),$c=l("code"),L9=o("stderr"),U9=o(`, the test can access those streams using the
`),Ec=l("code"),N9=o("pytest"),H9=o("\u2019s "),La=l("a"),R9=o("capsys system"),z9=o(". Here is how this is accomplished:"),kw=f(),d(Ua.$$.fragment),xw=f(),fo=l("p"),G9=o("And, of course, most of the time, "),gc=l("code"),F9=o("stderr"),W9=o(` will come as a part of an exception, so try/except has to be used in such
a case:`),Cw=f(),d(Na.$$.fragment),jw=f(),ho=l("p"),M9=o("Another approach to capturing stdout is via "),bc=l("code"),B9=o("contextlib.redirect_stdout"),Y9=o(":"),Pw=f(),d(Ha.$$.fragment),Ow=f(),T=l("p"),V9=o("An important potential issue with capturing stdout is that it may contain "),kc=l("code"),X9=o("\\r"),J9=o(" characters that in normal "),xc=l("code"),Z9=o("print"),K9=o(`
reset everything that has been printed so far. There is no problem with `),Cc=l("code"),Q9=o("pytest"),ex=o(", but with "),jc=l("code"),tx=o("pytest -s"),sx=o(` these
characters get included in the buffer, so to be able to have the test run with and without `),Pc=l("code"),ox=o("-s"),rx=o(`, you have to make an
extra cleanup to the captured output, using `),Oc=l("code"),lx=o("re.sub(r'~.*\\r', '', buf, 0, re.M)"),ax=o("."),Iw=f(),uo=l("p"),ix=o(`But, then we have a helper context manager wrapper to automatically take care of it all, regardless of whether it has
some `),Ic=l("code"),nx=o("\\r"),px=o("\u2019s in it or not, so it\u2019s a simple:"),Tw=f(),d(Ra.$$.fragment),Aw=f(),Gn=l("p"),fx=o("Here is a full test example:"),Dw=f(),d(za.$$.fragment),qw=f(),ze=l("p"),hx=o("If you\u2019d like to capture "),Tc=l("code"),ux=o("stderr"),dx=o(" use the "),Ac=l("code"),cx=o("CaptureStderr"),mx=o(" class instead:"),Sw=f(),d(Ga.$$.fragment),Lw=f(),co=l("p"),_x=o("If you need to capture both streams at once, use the parent "),Dc=l("code"),vx=o("CaptureStd"),yx=o(" class:"),Uw=f(),d(Fa.$$.fragment),Nw=f(),Fn=l("p"),wx=o(`Also, to aid debugging test issues, by default these context managers automatically replay the captured streams on exit
from the context.`),Hw=f(),Dt=l("h3"),mo=l("a"),qc=l("span"),d(Wa.$$.fragment),$x=f(),Sc=l("span"),Ex=o("Capturing logger stream"),Rw=f(),_o=l("p"),gx=o("If you need to validate the output of a logger, you can use "),Lc=l("code"),bx=o("CaptureLogger"),kx=o(":"),zw=f(),d(Ma.$$.fragment),Gw=f(),qt=l("h3"),vo=l("a"),Uc=l("span"),d(Ba.$$.fragment),xx=f(),Nc=l("span"),Cx=o("Testing with environment variables"),Fw=f(),Ya=l("p"),jx=o(`If you want to test the impact of environment variables for a specific test you can use a helper decorator
`),Hc=l("code"),Px=o("transformers.testing_utils.mockenv"),Ww=f(),d(Va.$$.fragment),Mw=f(),ne=l("p"),Ox=o("At times an external program needs to be called, which requires setting "),Rc=l("code"),Ix=o("PYTHONPATH"),Tx=o(" in "),zc=l("code"),Ax=o("os.environ"),Dx=o(` to include
multiple local paths. A helper class `),Gc=l("code"),qx=o("transformers.test_utils.TestCasePlus"),Sx=o(" comes to help:"),Bw=f(),d(Xa.$$.fragment),Yw=f(),N=l("p"),Lx=o("Depending on whether the test file was under the "),Fc=l("code"),Ux=o("tests"),Nx=o(" test suite or "),Wc=l("code"),Hx=o("examples"),Rx=o(` it\u2019ll correctly set up
`),Mc=l("code"),zx=o("env[PYTHONPATH]"),Gx=o(" to include one of these two directories, and also the "),Bc=l("code"),Fx=o("src"),Wx=o(` directory to ensure the testing is
done against the current repo, and finally with whatever `),Yc=l("code"),Mx=o("env[PYTHONPATH]"),Bx=o(` was already set to before the test was
called if anything.`),Vw=f(),yo=l("p"),Yx=o("This helper method creates a copy of the "),Vc=l("code"),Vx=o("os.environ"),Xx=o(" object, so the original remains intact."),Xw=f(),St=l("h3"),wo=l("a"),Xc=l("span"),d(Ja.$$.fragment),Jx=f(),Jc=l("span"),Zx=o("Getting reproducible results"),Jw=f(),Wn=l("p"),Kx=o(`In some situations you may want to remove randomness for your tests. To get identical reproducable results set, you
will need to fix the seed:`),Zw=f(),d(Za.$$.fragment),Kw=f(),Lt=l("h3"),$o=l("a"),Zc=l("span"),d(Ka.$$.fragment),Qx=f(),Kc=l("span"),eC=o("Debugging tests"),Qw=f(),Mn=l("p"),tC=o("To start a debugger at the point of the warning, do this:"),e$=f(),d(Qa.$$.fragment),t$=f(),Ut=l("h2"),Eo=l("a"),Qc=l("span"),d(ei.$$.fragment),sC=f(),em=l("span"),oC=o("Working with github actions workflows"),s$=f(),Bn=l("p"),rC=o("To trigger a self-push workflow CI job, you must:"),o$=f(),pe=l("ol"),ti=l("li"),lC=o("Create a new branch on "),tm=l("code"),aC=o("transformers"),iC=o(" origin (not a fork!)."),nC=f(),H=l("li"),pC=o("The branch name has to start with either "),sm=l("code"),fC=o("ci_"),hC=o(" or "),om=l("code"),uC=o("ci-"),dC=o(" ("),rm=l("code"),cC=o("main"),mC=o(` triggers it too, but we can\u2019t do PRs on
`),lm=l("code"),_C=o("main"),vC=o(`). It also gets triggered only for specific paths - you can find the up-to-date definition in case it
changed since this document has been written `),si=l("a"),yC=o("here"),wC=o(" under "),am=l("em"),$C=o("push:"),EC=f(),im=l("li"),gC=o("Create a PR from this branch."),bC=f(),oi=l("li"),kC=o("Then you can see the job appear "),ri=l("a"),xC=o("here"),CC=o(`. It may not run right away if there
is a backlog.`),r$=f(),Nt=l("h2"),go=l("a"),nm=l("span"),d(li.$$.fragment),jC=f(),pm=l("span"),PC=o("Testing Experimental CI Features"),l$=f(),Yn=l("p"),OC=o(`Testing CI features can be potentially problematic as it can interfere with the normal CI functioning. Therefore if a
new CI feature is to be added, it should be done as following.`),a$=f(),fe=l("ol"),fm=l("li"),IC=o("Create a new dedicated job that tests what needs to be tested"),TC=f(),hm=l("li"),AC=o("The new job must always succeed so that it gives us a green \u2713 (details below)."),DC=f(),um=l("li"),qC=o(`Let it run for some days to see that a variety of different PR types get to run on it (user fork branches,
non-forked branches, branches originating from github.com UI direct file edit, various forced pushes, etc. - there
are so many) while monitoring the experimental job\u2019s logs (not the overall job green as it\u2019s purposefully always
green)`),SC=f(),dm=l("li"),LC=o("When it\u2019s clear that everything is solid, then merge the new changes into existing jobs."),i$=f(),Vn=l("p"),UC=o("That way experiments on CI functionality itself won\u2019t interfere with the normal workflow."),n$=f(),Xn=l("p"),NC=o("Now how can we make the job always succeed while the new CI feature is being developed?"),p$=f(),Jn=l("p"),HC=o(`Some CIs, like TravisCI support ignore-step-failure and will report the overall job as successful, but CircleCI and
Github Actions as of this writing don\u2019t support that.`),f$=f(),Zn=l("p"),RC=o("So the following workaround can be used:"),h$=f(),bo=l("ol"),Kn=l("li"),cm=l("code"),zC=o("set +euo pipefail"),GC=o(" at the beginning of the run command to suppress most potential failures in the bash script."),FC=f(),Ht=l("li"),WC=o("the last command must be a success: "),mm=l("code"),MC=o('echo "done"'),BC=o(" or just "),_m=l("code"),YC=o("true"),VC=o(" will do"),u$=f(),Qn=l("p"),XC=o("Here is an example:"),d$=f(),d(ai.$$.fragment),c$=f(),ep=l("p"),JC=o("For simple commands you could also do:"),m$=f(),d(ii.$$.fragment),_$=f(),ko=l("p"),ZC=o(`Of course, once satisfied with the results, integrate the experimental step or job with the rest of the normal jobs,
while removing `),vm=l("code"),KC=o("set +euo pipefail"),QC=o(` or any other things you may have added to ensure that the experimental job doesn\u2019t
interfere with the normal CI functioning.`),v$=f(),xo=l("p"),ej=o("This whole process would have been much easier if we only could set something like "),ym=l("code"),tj=o("allow-failure"),sj=o(` for the
experimental step, and let it fail without impacting the overall status of PRs. But as mentioned earlier CircleCI and
Github Actions don\u2019t support it at the moment.`),y$=f(),tp=l("p"),oj=o("You can vote for this feature and see where it is at these CI-specific threads:"),w$=f(),Co=l("ul"),wm=l("li"),ni=l("a"),rj=o("Github Actions:"),lj=f(),$m=l("li"),pi=l("a"),aj=o("CircleCI:"),this.h()},l(e){const n=_L('[data-svelte="svelte-1phssyn"]',document.head);$=a(n,"META",{name:!0,content:!0}),n.forEach(s),O=h(e),g=a(e,"H1",{class:!0});var fi=i(g);x=a(fi,"A",{id:!0,class:!0,href:!0});var Em=i(x);A=a(Em,"SPAN",{});var gm=i(A);c(C.$$.fragment,gm),gm.forEach(s),Em.forEach(s),R=h(fi),D=a(fi,"SPAN",{});var bm=i(D);P=r(bm,"Testing"),bm.forEach(s),fi.forEach(s),b=h(e),S=a(e,"P",{});var Ij=i(S);z=r(Ij,"Let\u2019s take a look at how \u{1F917} Transformers models are tested and how you can write new tests and improve the existing ones."),Ij.forEach(s),Rt=h(e),we=a(e,"P",{});var Tj=i(we);Ye=r(Tj,"There are 2 test suites in the repository:"),Tj.forEach(s),q=h(e),me=a(e,"OL",{});var E$=i(me);Ve=a(E$,"LI",{});var ij=i(Ve);zt=a(ij,"CODE",{});var Aj=i(zt);ci=r(Aj,"tests"),Aj.forEach(s),hg=r(ij," \u2014 tests for the general API"),ij.forEach(s),ug=h(E$),mi=a(E$,"LI",{});var nj=i(mi);Cp=a(nj,"CODE",{});var Dj=i(Cp);dg=r(Dj,"examples"),Dj.forEach(s),cg=r(nj," \u2014 tests primarily for various applications that aren\u2019t part of the API"),nj.forEach(s),E$.forEach(s),Dm=h(e),Xe=a(e,"H2",{class:!0});var g$=i(Xe);Gt=a(g$,"A",{id:!0,class:!0,href:!0});var qj=i(Gt);jp=a(qj,"SPAN",{});var Sj=i(jp);c(Ro.$$.fragment,Sj),Sj.forEach(s),qj.forEach(s),mg=h(g$),Pp=a(g$,"SPAN",{});var Lj=i(Pp);_g=r(Lj,"How transformers are tested"),Lj.forEach(s),g$.forEach(s),qm=h(e),Ft=a(e,"OL",{});var b$=i(Ft);zo=a(b$,"LI",{});var k$=i(zo);Go=a(k$,"P",{});var x$=i(Go);vg=r(x$,`Once a PR is submitted it gets tested with 9 CircleCi jobs. Every new commit to that PR gets retested. These jobs
are defined in this `),Fo=a(x$,"A",{href:!0,rel:!0});var Uj=i(Fo);yg=r(Uj,"config file"),Uj.forEach(s),wg=r(x$,`, so that if needed you can reproduce the same
environment on your machine.`),x$.forEach(s),$g=h(k$),Wo=a(k$,"P",{});var C$=i(Wo);Eg=r(C$,"These CI jobs don\u2019t run "),Op=a(C$,"CODE",{});var Nj=i(Op);gg=r(Nj,"@slow"),Nj.forEach(s),bg=r(C$," tests."),C$.forEach(s),k$.forEach(s),kg=h(b$),Mo=a(b$,"LI",{});var j$=i(Mo);Bo=a(j$,"P",{});var P$=i(Bo);xg=r(P$,"There are 3 jobs run by "),Yo=a(P$,"A",{href:!0,rel:!0});var Hj=i(Yo);Cg=r(Hj,"github actions"),Hj.forEach(s),jg=r(P$,":"),P$.forEach(s),Pg=h(j$),Je=a(j$,"UL",{});var sp=i(Je);Ip=a(sp,"LI",{});var Rj=i(Ip);_i=a(Rj,"P",{});var pj=i(_i);Vo=a(pj,"A",{href:!0,rel:!0});var zj=i(Vo);Og=r(zj,"torch hub integration"),zj.forEach(s),Ig=r(pj,`: checks whether torch hub
integration works.`),pj.forEach(s),Rj.forEach(s),Tg=h(sp),Tp=a(sp,"LI",{});var Gj=i(Tp);L=a(Gj,"P",{});var Z=i(L);Xo=a(Z,"A",{href:!0,rel:!0});var Fj=i(Xo);Ag=r(Fj,"self-hosted (push)"),Fj.forEach(s),Dg=r(Z,`: runs fast tests on GPU only on commits on
`),Ap=a(Z,"CODE",{});var Wj=i(Ap);qg=r(Wj,"main"),Wj.forEach(s),Sg=r(Z,". It only runs if a commit on "),Dp=a(Z,"CODE",{});var Mj=i(Dp);Lg=r(Mj,"main"),Mj.forEach(s),Ug=r(Z," has updated the code in one of the following folders: "),qp=a(Z,"CODE",{});var Bj=i(qp);Ng=r(Bj,"src"),Bj.forEach(s),Hg=r(Z,`,
`),Sp=a(Z,"CODE",{});var Yj=i(Sp);Rg=r(Yj,"tests"),Yj.forEach(s),zg=r(Z,", "),Lp=a(Z,"CODE",{});var Vj=i(Lp);Gg=r(Vj,".github"),Vj.forEach(s),Fg=r(Z," (to prevent running on added model cards, notebooks, etc.)"),Z.forEach(s),Gj.forEach(s),Wg=h(sp),Up=a(sp,"LI",{});var Xj=i(Up);$e=a(Xj,"P",{});var hi=i($e);Jo=a(hi,"A",{href:!0,rel:!0});var Jj=i(Jo);Mg=r(Jj,"self-hosted runner"),Jj.forEach(s),Bg=r(hi,`: runs normal and slow tests on GPU in
`),Np=a(hi,"CODE",{});var Zj=i(Np);Yg=r(Zj,"tests"),Zj.forEach(s),Vg=r(hi," and "),Hp=a(hi,"CODE",{});var Kj=i(Hp);Xg=r(Kj,"examples"),Kj.forEach(s),Jg=r(hi,":"),hi.forEach(s),Xj.forEach(s),sp.forEach(s),j$.forEach(s),b$.forEach(s),Sm=h(e),c(Zo.$$.fragment,e),Lm=h(e),Wt=a(e,"P",{});var O$=i(Wt);Zg=r(O$,"The results can be observed "),Ko=a(O$,"A",{href:!0,rel:!0});var Qj=i(Ko);Kg=r(Qj,"here"),Qj.forEach(s),Qg=r(O$,"."),O$.forEach(s),Um=h(e),Ze=a(e,"H2",{class:!0});var I$=i(Ze);Mt=a(I$,"A",{id:!0,class:!0,href:!0});var eP=i(Mt);Rp=a(eP,"SPAN",{});var tP=i(Rp);c(Qo.$$.fragment,tP),tP.forEach(s),eP.forEach(s),eb=h(I$),zp=a(I$,"SPAN",{});var sP=i(zp);tb=r(sP,"Running tests"),sP.forEach(s),I$.forEach(s),Nm=h(e),Ke=a(e,"H3",{class:!0});var T$=i(Ke);Bt=a(T$,"A",{id:!0,class:!0,href:!0});var oP=i(Bt);Gp=a(oP,"SPAN",{});var rP=i(Gp);c(er.$$.fragment,rP),rP.forEach(s),oP.forEach(s),sb=h(T$),Fp=a(T$,"SPAN",{});var lP=i(Fp);ob=r(lP,"Choosing which tests to run"),lP.forEach(s),T$.forEach(s),Hm=h(e),Yt=a(e,"P",{});var A$=i(Yt);rb=r(A$,`This document goes into many details of how tests can be run. If after reading everything, you need even more details
you will find them `),tr=a(A$,"A",{href:!0,rel:!0});var aP=i(tr);lb=r(aP,"here"),aP.forEach(s),ab=r(A$,"."),A$.forEach(s),Rm=h(e),vi=a(e,"P",{});var iP=i(vi);ib=r(iP,"Here are some most useful ways of running tests."),iP.forEach(s),zm=h(e),yi=a(e,"P",{});var nP=i(yi);nb=r(nP,"Run all:"),nP.forEach(s),Gm=h(e),c(sr.$$.fragment,e),Fm=h(e),wi=a(e,"P",{});var pP=i(wi);pb=r(pP,"or:"),pP.forEach(s),Wm=h(e),c(or.$$.fragment,e),Mm=h(e),$i=a(e,"P",{});var fP=i($i);fb=r(fP,"Note that the latter is defined as:"),fP.forEach(s),Bm=h(e),c(rr.$$.fragment,e),Ym=h(e),Ei=a(e,"P",{});var hP=i(Ei);hb=r(hP,"which tells pytest to:"),hP.forEach(s),Vm=h(e),Q=a(e,"UL",{});var jo=i(Q);Wp=a(jo,"LI",{});var uP=i(Wp);ub=r(uP,"run as many test processes as they are CPU cores (which could be too many if you don\u2019t have a ton of RAM!)"),uP.forEach(s),db=h(jo),Mp=a(jo,"LI",{});var dP=i(Mp);cb=r(dP,"ensure that all tests from the same file will be run by the same test process"),dP.forEach(s),mb=h(jo),Bp=a(jo,"LI",{});var cP=i(Bp);_b=r(cP,"do not capture output"),cP.forEach(s),vb=h(jo),Yp=a(jo,"LI",{});var mP=i(Yp);yb=r(mP,"run in verbose mode"),mP.forEach(s),jo.forEach(s),Xm=h(e),Qe=a(e,"H3",{class:!0});var D$=i(Qe);Vt=a(D$,"A",{id:!0,class:!0,href:!0});var _P=i(Vt);Vp=a(_P,"SPAN",{});var vP=i(Vp);c(lr.$$.fragment,vP),vP.forEach(s),_P.forEach(s),wb=h(D$),Xp=a(D$,"SPAN",{});var yP=i(Xp);$b=r(yP,"Getting the list of all tests"),yP.forEach(s),D$.forEach(s),Jm=h(e),gi=a(e,"P",{});var wP=i(gi);Eb=r(wP,"All tests of the test suite:"),wP.forEach(s),Zm=h(e),c(ar.$$.fragment,e),Km=h(e),bi=a(e,"P",{});var $P=i(bi);gb=r($P,"All tests of a given test file:"),$P.forEach(s),Qm=h(e),c(ir.$$.fragment,e),e_=h(e),et=a(e,"H3",{class:!0});var q$=i(et);Xt=a(q$,"A",{id:!0,class:!0,href:!0});var EP=i(Xt);Jp=a(EP,"SPAN",{});var gP=i(Jp);c(nr.$$.fragment,gP),gP.forEach(s),EP.forEach(s),bb=h(q$),Zp=a(q$,"SPAN",{});var bP=i(Zp);kb=r(bP,"Run a specific test module"),bP.forEach(s),q$.forEach(s),t_=h(e),ki=a(e,"P",{});var kP=i(ki);xb=r(kP,"To run an individual test module:"),kP.forEach(s),s_=h(e),c(pr.$$.fragment,e),o_=h(e),tt=a(e,"H3",{class:!0});var S$=i(tt);Jt=a(S$,"A",{id:!0,class:!0,href:!0});var xP=i(Jt);Kp=a(xP,"SPAN",{});var CP=i(Kp);c(fr.$$.fragment,CP),CP.forEach(s),xP.forEach(s),Cb=h(S$),Qp=a(S$,"SPAN",{});var jP=i(Qp);jb=r(jP,"Run specific tests"),jP.forEach(s),S$.forEach(s),r_=h(e),xi=a(e,"P",{});var PP=i(xi);Pb=r(PP,`Since unittest is used inside most of the tests, to run specific subtests you need to know the name of the unittest
class containing those tests. For example, it could be:`),PP.forEach(s),l_=h(e),c(hr.$$.fragment,e),a_=h(e),Ci=a(e,"P",{});var OP=i(Ci);Ob=r(OP,"Here:"),OP.forEach(s),i_=h(e),Ee=a(e,"UL",{});var op=i(Ee);ji=a(op,"LI",{});var fj=i(ji);ef=a(fj,"CODE",{});var IP=i(ef);Ib=r(IP,"tests/test_optimization.py"),IP.forEach(s),Tb=r(fj," - the file with tests"),fj.forEach(s),Ab=h(op),Pi=a(op,"LI",{});var hj=i(Pi);tf=a(hj,"CODE",{});var TP=i(tf);Db=r(TP,"OptimizationTest"),TP.forEach(s),qb=r(hj," - the name of the class"),hj.forEach(s),Sb=h(op),Oi=a(op,"LI",{});var uj=i(Oi);sf=a(uj,"CODE",{});var AP=i(sf);Lb=r(AP,"test_adam_w"),AP.forEach(s),Ub=r(uj," - the name of the specific test function"),uj.forEach(s),op.forEach(s),n_=h(e),Ii=a(e,"P",{});var DP=i(Ii);Nb=r(DP,"If the file contains multiple classes, you can choose to run only tests of a given class. For example:"),DP.forEach(s),p_=h(e),c(ur.$$.fragment,e),f_=h(e),Ti=a(e,"P",{});var qP=i(Ti);Hb=r(qP,"will run all the tests inside that class."),qP.forEach(s),h_=h(e),Zt=a(e,"P",{});var L$=i(Zt);Rb=r(L$,"As mentioned earlier you can see what tests are contained inside the "),of=a(L$,"CODE",{});var SP=i(of);zb=r(SP,"OptimizationTest"),SP.forEach(s),Gb=r(L$," class by running:"),L$.forEach(s),u_=h(e),c(dr.$$.fragment,e),d_=h(e),Ai=a(e,"P",{});var LP=i(Ai);Fb=r(LP,"You can run tests by keyword expressions."),LP.forEach(s),c_=h(e),Kt=a(e,"P",{});var U$=i(Kt);Wb=r(U$,"To run only tests whose name contains "),rf=a(U$,"CODE",{});var UP=i(rf);Mb=r(UP,"adam"),UP.forEach(s),Bb=r(U$,":"),U$.forEach(s),m_=h(e),c(cr.$$.fragment,e),__=h(e),ee=a(e,"P",{});var Po=i(ee);Yb=r(Po,"Logical "),lf=a(Po,"CODE",{});var NP=i(lf);Vb=r(NP,"and"),NP.forEach(s),Xb=r(Po," and "),af=a(Po,"CODE",{});var HP=i(af);Jb=r(HP,"or"),HP.forEach(s),Zb=r(Po," can be used to indicate whether all keywords should match or either. "),nf=a(Po,"CODE",{});var RP=i(nf);Kb=r(RP,"not"),RP.forEach(s),Qb=r(Po,` can be used to
negate.`),Po.forEach(s),v_=h(e),Qt=a(e,"P",{});var N$=i(Qt);e3=r(N$,"To run all tests except those whose name contains "),pf=a(N$,"CODE",{});var zP=i(pf);t3=r(zP,"adam"),zP.forEach(s),s3=r(N$,":"),N$.forEach(s),y_=h(e),c(mr.$$.fragment,e),w_=h(e),Di=a(e,"P",{});var GP=i(Di);o3=r(GP,"And you can combine the two patterns in one:"),GP.forEach(s),$_=h(e),c(_r.$$.fragment,e),E_=h(e),ge=a(e,"P",{});var rp=i(ge);r3=r(rp,"For example to run both "),ff=a(rp,"CODE",{});var FP=i(ff);l3=r(FP,"test_adafactor"),FP.forEach(s),a3=r(rp," and "),hf=a(rp,"CODE",{});var WP=i(hf);i3=r(WP,"test_adam_w"),WP.forEach(s),n3=r(rp," you can use:"),rp.forEach(s),g_=h(e),c(vr.$$.fragment,e),b_=h(e),es=a(e,"P",{});var H$=i(es);p3=r(H$,"Note that we use "),uf=a(H$,"CODE",{});var MP=i(uf);f3=r(MP,"or"),MP.forEach(s),h3=r(H$," here, since we want either of the keywords to match to include both."),H$.forEach(s),k_=h(e),ts=a(e,"P",{});var R$=i(ts);u3=r(R$,"If you want to include only tests that include both patterns, "),df=a(R$,"CODE",{});var BP=i(df);d3=r(BP,"and"),BP.forEach(s),c3=r(R$," is to be used:"),R$.forEach(s),x_=h(e),c(yr.$$.fragment,e),C_=h(e),st=a(e,"H3",{class:!0});var z$=i(st);ss=a(z$,"A",{id:!0,class:!0,href:!0});var YP=i(ss);cf=a(YP,"SPAN",{});var VP=i(cf);c(wr.$$.fragment,VP),VP.forEach(s),YP.forEach(s),m3=h(z$),mf=a(z$,"SPAN",{});var XP=i(mf);_3=r(XP,"Run documentation tests"),XP.forEach(s),z$.forEach(s),j_=h(e),be=a(e,"P",{});var lp=i(be);v3=r(lp,"In order to test whether the documentation examples are correct, you should checkt that the "),_f=a(lp,"CODE",{});var JP=i(_f);y3=r(JP,"doctests"),JP.forEach(s),w3=r(lp,` are passing.
As an example, let\u2019s use `),os=a(lp,"A",{href:!0,rel:!0});var dj=i(os);vf=a(dj,"CODE",{});var ZP=i(vf);$3=r(ZP,"WhisperModel.forward"),ZP.forEach(s),E3=r(dj,"\u2019s docstring"),dj.forEach(s),g3=r(lp,":"),lp.forEach(s),P_=h(e),c($r.$$.fragment,e),O_=h(e),qi=a(e,"P",{});var KP=i(qi);b3=r(KP,"3 steps are required to debug the docstring examples :"),KP.forEach(s),I_=h(e),Si=a(e,"OL",{});var QP=i(Si);Er=a(QP,"LI",{});var G$=i(Er);k3=r(G$,"In order to properly run the test, "),yf=a(G$,"STRONG",{});var eO=i(yf);x3=r(eO,"an extra line has to be added"),eO.forEach(s),C3=r(G$," at the end of the docstring. This can be automatically done on any file using :"),G$.forEach(s),QP.forEach(s),T_=h(e),c(gr.$$.fragment,e),A_=h(e),br=a(e,"OL",{start:!0});var tO=i(br);wf=a(tO,"LI",{});var sO=i(wf);j3=r(sO,"Then, you can use the following line to automatically test every docstring example in the desired file :"),sO.forEach(s),tO.forEach(s),D_=h(e),c(kr.$$.fragment,e),q_=h(e),xr=a(e,"OL",{start:!0});var oO=i(xr);Cr=a(oO,"LI",{});var F$=i(Cr);P3=r(F$,"Once you are done debugging, you need to remove the extra line added in step "),$f=a(F$,"STRONG",{});var rO=i($f);O3=r(rO,"1."),rO.forEach(s),I3=r(F$," by running the follwing :"),F$.forEach(s),oO.forEach(s),S_=h(e),c(jr.$$.fragment,e),L_=h(e),ot=a(e,"H3",{class:!0});var W$=i(ot);rs=a(W$,"A",{id:!0,class:!0,href:!0});var lO=i(rs);Ef=a(lO,"SPAN",{});var aO=i(Ef);c(Pr.$$.fragment,aO),aO.forEach(s),lO.forEach(s),T3=h(W$),gf=a(W$,"SPAN",{});var iO=i(gf);A3=r(iO,"Run only modified tests"),iO.forEach(s),W$.forEach(s),U_=h(e),ls=a(e,"P",{});var M$=i(ls);D3=r(M$,"You can run the tests related to the unstaged files or the current branch (according to Git) by using "),Or=a(M$,"A",{href:!0,rel:!0});var nO=i(Or);q3=r(nO,"pytest-picked"),nO.forEach(s),S3=r(M$,`. This is a great way of quickly testing your changes didn\u2019t break
anything, since it won\u2019t run the tests related to files you didn\u2019t touch.`),M$.forEach(s),N_=h(e),c(Ir.$$.fragment,e),H_=h(e),c(Tr.$$.fragment,e),R_=h(e),Li=a(e,"P",{});var pO=i(Li);L3=r(pO,"All tests will be run from files and folders which are modified, but not yet committed."),pO.forEach(s),z_=h(e),rt=a(e,"H3",{class:!0});var B$=i(rt);as=a(B$,"A",{id:!0,class:!0,href:!0});var fO=i(as);bf=a(fO,"SPAN",{});var hO=i(bf);c(Ar.$$.fragment,hO),hO.forEach(s),fO.forEach(s),U3=h(B$),kf=a(B$,"SPAN",{});var uO=i(kf);N3=r(uO,"Automatically rerun failed tests on source modification"),uO.forEach(s),B$.forEach(s),G_=h(e),Dr=a(e,"P",{});var cj=i(Dr);qr=a(cj,"A",{href:!0,rel:!0});var dO=i(qr);H3=r(dO,"pytest-xdist"),dO.forEach(s),R3=r(cj,` provides a very useful feature of detecting all failed
tests, and then waiting for you to modify files and continuously re-rerun those failing tests until they pass while you
fix them. So that you don\u2019t need to re start pytest after you made the fix. This is repeated until all tests pass after
which again a full run is performed.`),cj.forEach(s),F_=h(e),c(Sr.$$.fragment,e),W_=h(e),lt=a(e,"P",{});var km=i(lt);z3=r(km,"To enter the mode: "),xf=a(km,"CODE",{});var cO=i(xf);G3=r(cO,"pytest -f"),cO.forEach(s),F3=r(km," or "),Cf=a(km,"CODE",{});var mO=i(Cf);W3=r(mO,"pytest --looponfail"),mO.forEach(s),km.forEach(s),M_=h(e),ke=a(e,"P",{});var ap=i(ke);M3=r(ap,"File changes are detected by looking at "),jf=a(ap,"CODE",{});var _O=i(jf);B3=r(_O,"looponfailroots"),_O.forEach(s),Y3=r(ap,` root directories and all of their contents (recursively).
If the default for this value does not work for you, you can change it in your project by setting a configuration
option in `),Pf=a(ap,"CODE",{});var vO=i(Pf);V3=r(vO,"setup.cfg"),vO.forEach(s),X3=r(ap,":"),ap.forEach(s),B_=h(e),c(Lr.$$.fragment,e),Y_=h(e),xe=a(e,"P",{});var ip=i(xe);J3=r(ip,"or "),Of=a(ip,"CODE",{});var yO=i(Of);Z3=r(yO,"pytest.ini"),yO.forEach(s),K3=r(ip,"/"),If=a(ip,"CODE",{});var wO=i(If);Q3=r(wO,"tox.ini"),wO.forEach(s),e2=r(ip," files:"),ip.forEach(s),V_=h(e),c(Ur.$$.fragment,e),X_=h(e),Ui=a(e,"P",{});var $O=i(Ui);t2=r($O,`This would lead to only looking for file changes in the respective directories, specified relatively to the ini-file\u2019s
directory.`),$O.forEach(s),J_=h(e),Nr=a(e,"P",{});var mj=i(Nr);Hr=a(mj,"A",{href:!0,rel:!0});var EO=i(Hr);s2=r(EO,"pytest-watch"),EO.forEach(s),o2=r(mj," is an alternative implementation of this functionality."),mj.forEach(s),Z_=h(e),at=a(e,"H3",{class:!0});var Y$=i(at);is=a(Y$,"A",{id:!0,class:!0,href:!0});var gO=i(is);Tf=a(gO,"SPAN",{});var bO=i(Tf);c(Rr.$$.fragment,bO),bO.forEach(s),gO.forEach(s),r2=h(Y$),Af=a(Y$,"SPAN",{});var kO=i(Af);l2=r(kO,"Skip a test module"),kO.forEach(s),Y$.forEach(s),K_=h(e),ns=a(e,"P",{});var V$=i(ns);a2=r(V$,`If you want to run all test modules, except a few you can exclude them by giving an explicit list of tests to run. For
example, to run all except `),Df=a(V$,"CODE",{});var xO=i(Df);i2=r(xO,"test_modeling_*.py"),xO.forEach(s),n2=r(V$," tests:"),V$.forEach(s),Q_=h(e),c(zr.$$.fragment,e),ev=h(e),it=a(e,"H3",{class:!0});var X$=i(it);ps=a(X$,"A",{id:!0,class:!0,href:!0});var CO=i(ps);qf=a(CO,"SPAN",{});var jO=i(qf);c(Gr.$$.fragment,jO),jO.forEach(s),CO.forEach(s),p2=h(X$),Sf=a(X$,"SPAN",{});var PO=i(Sf);f2=r(PO,"Clearing state"),PO.forEach(s),X$.forEach(s),tv=h(e),Ni=a(e,"P",{});var OO=i(Ni);h2=r(OO,"CI builds and when isolation is important (against speed), cache should be cleared:"),OO.forEach(s),sv=h(e),c(Fr.$$.fragment,e),ov=h(e),nt=a(e,"H3",{class:!0});var J$=i(nt);fs=a(J$,"A",{id:!0,class:!0,href:!0});var IO=i(fs);Lf=a(IO,"SPAN",{});var TO=i(Lf);c(Wr.$$.fragment,TO),TO.forEach(s),IO.forEach(s),u2=h(J$),Uf=a(J$,"SPAN",{});var AO=i(Uf);d2=r(AO,"Running tests in parallel"),AO.forEach(s),J$.forEach(s),rv=h(e),W=a(e,"P",{});var Ge=i(W);c2=r(Ge,"As mentioned earlier "),Nf=a(Ge,"CODE",{});var DO=i(Nf);m2=r(DO,"make test"),DO.forEach(s),_2=r(Ge," runs tests in parallel via "),Hf=a(Ge,"CODE",{});var qO=i(Hf);v2=r(qO,"pytest-xdist"),qO.forEach(s),y2=r(Ge," plugin ("),Rf=a(Ge,"CODE",{});var SO=i(Rf);w2=r(SO,"-n X"),SO.forEach(s),$2=r(Ge," argument, e.g. "),zf=a(Ge,"CODE",{});var LO=i(zf);E2=r(LO,"-n 2"),LO.forEach(s),g2=r(Ge,`
to run 2 parallel jobs).`),Ge.forEach(s),lv=h(e),_e=a(e,"P",{});var ui=i(_e);Gf=a(ui,"CODE",{});var UO=i(Gf);b2=r(UO,"pytest-xdist"),UO.forEach(s),k2=r(ui,"\u2019s "),Ff=a(ui,"CODE",{});var NO=i(Ff);x2=r(NO,"--dist="),NO.forEach(s),C2=r(ui," option allows one to control how the tests are grouped. "),Wf=a(ui,"CODE",{});var HO=i(Wf);j2=r(HO,"--dist=loadfile"),HO.forEach(s),P2=r(ui,` puts the
tests located in one file onto the same process.`),ui.forEach(s),av=h(e),Ce=a(e,"P",{});var np=i(Ce);O2=r(np,"Since the order of executed tests is different and unpredictable, if running the test suite with "),Mf=a(np,"CODE",{});var RO=i(Mf);I2=r(RO,"pytest-xdist"),RO.forEach(s),T2=r(np,`
produces failures (meaning we have some undetected coupled tests), use `),Mr=a(np,"A",{href:!0,rel:!0});var zO=i(Mr);A2=r(zO,"pytest-replay"),zO.forEach(s),D2=r(np,` to replay the tests in the same order, which should help with then somehow
reducing that failing sequence to a minimum.`),np.forEach(s),iv=h(e),pt=a(e,"H3",{class:!0});var Z$=i(pt);hs=a(Z$,"A",{id:!0,class:!0,href:!0});var GO=i(hs);Bf=a(GO,"SPAN",{});var FO=i(Bf);c(Br.$$.fragment,FO),FO.forEach(s),GO.forEach(s),q2=h(Z$),Yf=a(Z$,"SPAN",{});var WO=i(Yf);S2=r(WO,"Test order and repetition"),WO.forEach(s),Z$.forEach(s),nv=h(e),Hi=a(e,"P",{});var MO=i(Hi);L2=r(MO,`It\u2019s good to repeat the tests several times, in sequence, randomly, or in sets, to detect any potential
inter-dependency and state-related bugs (tear down). And the straightforward multiple repetition is just good to detect
some problems that get uncovered by randomness of DL.`),MO.forEach(s),pv=h(e),ft=a(e,"H4",{class:!0});var K$=i(ft);us=a(K$,"A",{id:!0,class:!0,href:!0});var BO=i(us);Vf=a(BO,"SPAN",{});var YO=i(Vf);c(Yr.$$.fragment,YO),YO.forEach(s),BO.forEach(s),U2=h(K$),Xf=a(K$,"SPAN",{});var VO=i(Xf);N2=r(VO,"Repeat tests"),VO.forEach(s),K$.forEach(s),fv=h(e),Ri=a(e,"UL",{});var XO=i(Ri);zi=a(XO,"LI",{});var _j=i(zi);Vr=a(_j,"A",{href:!0,rel:!0});var JO=i(Vr);H2=r(JO,"pytest-flakefinder"),JO.forEach(s),R2=r(_j,":"),_j.forEach(s),XO.forEach(s),hv=h(e),c(Xr.$$.fragment,e),uv=h(e),Gi=a(e,"P",{});var ZO=i(Gi);z2=r(ZO,"And then run every test multiple times (50 by default):"),ZO.forEach(s),dv=h(e),c(Jr.$$.fragment,e),cv=h(e),c(ds.$$.fragment,e),mv=h(e),c(cs.$$.fragment,e),_v=h(e),ht=a(e,"H4",{class:!0});var Q$=i(ht);ms=a(Q$,"A",{id:!0,class:!0,href:!0});var KO=i(ms);Jf=a(KO,"SPAN",{});var QO=i(Jf);c(Zr.$$.fragment,QO),QO.forEach(s),KO.forEach(s),G2=h(Q$),Zf=a(Q$,"SPAN",{});var eI=i(Zf);F2=r(eI,"Run tests in a random order"),eI.forEach(s),Q$.forEach(s),vv=h(e),c(Kr.$$.fragment,e),yv=h(e),_s=a(e,"P",{});var eE=i(_s);W2=r(eE,"Important: the presence of "),Kf=a(eE,"CODE",{});var tI=i(Kf);M2=r(tI,"pytest-random-order"),tI.forEach(s),B2=r(eE,` will automatically randomize tests, no configuration change or
command line options is required.`),eE.forEach(s),wv=h(e),vs=a(e,"P",{});var tE=i(vs);Y2=r(tE,`As explained earlier this allows detection of coupled tests - where one test\u2019s state affects the state of another. When
`),Qf=a(tE,"CODE",{});var sI=i(Qf);V2=r(sI,"pytest-random-order"),sI.forEach(s),X2=r(tE," is installed it will print the random seed it used for that session, e.g:"),tE.forEach(s),$v=h(e),c(Qr.$$.fragment,e),Ev=h(e),Fi=a(e,"P",{});var oI=i(Fi);J2=r(oI,"So that if the given particular sequence fails, you can reproduce it by adding that exact seed, e.g.:"),oI.forEach(s),gv=h(e),c(el.$$.fragment,e),bv=h(e),ys=a(e,"P",{});var sE=i(ys);Z2=r(sE,`It will only reproduce the exact order if you use the exact same list of tests (or no list at all). Once you start to
manually narrowing down the list you can no longer rely on the seed, but have to list them manually in the exact order
they failed and tell pytest to not randomize them instead using `),eh=a(sE,"CODE",{});var rI=i(eh);K2=r(rI,"--random-order-bucket=none"),rI.forEach(s),Q2=r(sE,", e.g.:"),sE.forEach(s),kv=h(e),c(tl.$$.fragment,e),xv=h(e),Wi=a(e,"P",{});var lI=i(Wi);e5=r(lI,"To disable the shuffling for all tests:"),lI.forEach(s),Cv=h(e),c(sl.$$.fragment,e),jv=h(e),I=a(e,"P",{});var X=i(I);t5=r(X,"By default "),th=a(X,"CODE",{});var aI=i(th);s5=r(aI,"--random-order-bucket=module"),aI.forEach(s),o5=r(X,` is implied, which will shuffle the files on the module levels. It can also
shuffle on `),sh=a(X,"CODE",{});var iI=i(sh);r5=r(iI,"class"),iI.forEach(s),l5=r(X,", "),oh=a(X,"CODE",{});var nI=i(oh);a5=r(nI,"package"),nI.forEach(s),i5=r(X,", "),rh=a(X,"CODE",{});var pI=i(rh);n5=r(pI,"global"),pI.forEach(s),p5=r(X," and "),lh=a(X,"CODE",{});var fI=i(lh);f5=r(fI,"none"),fI.forEach(s),h5=r(X,` levels. For the complete details please see its
`),ol=a(X,"A",{href:!0,rel:!0});var hI=i(ol);u5=r(hI,"documentation"),hI.forEach(s),d5=r(X,"."),X.forEach(s),Pv=h(e),je=a(e,"P",{});var pp=i(je);c5=r(pp,"Another randomization alternative is: "),rl=a(pp,"A",{href:!0,rel:!0});var uI=i(rl);ah=a(uI,"CODE",{});var dI=i(ah);m5=r(dI,"pytest-randomly"),dI.forEach(s),uI.forEach(s),_5=r(pp,`. This
module has a very similar functionality/interface, but it doesn\u2019t have the bucket modes available in
`),ih=a(pp,"CODE",{});var cI=i(ih);v5=r(cI,"pytest-random-order"),cI.forEach(s),y5=r(pp,". It has the same problem of imposing itself once installed."),pp.forEach(s),Ov=h(e),ut=a(e,"H3",{class:!0});var oE=i(ut);ws=a(oE,"A",{id:!0,class:!0,href:!0});var mI=i(ws);nh=a(mI,"SPAN",{});var _I=i(nh);c(ll.$$.fragment,_I),_I.forEach(s),mI.forEach(s),w5=h(oE),ph=a(oE,"SPAN",{});var vI=i(ph);$5=r(vI,"Look and feel variations"),vI.forEach(s),oE.forEach(s),Iv=h(e),dt=a(e,"H4",{class:!0});var rE=i(dt);$s=a(rE,"A",{id:!0,class:!0,href:!0});var yI=i($s);fh=a(yI,"SPAN",{});var wI=i(fh);c(al.$$.fragment,wI),wI.forEach(s),yI.forEach(s),E5=h(rE),hh=a(rE,"SPAN",{});var $I=i(hh);g5=r($I,"pytest-sugar"),$I.forEach(s),rE.forEach(s),Tv=h(e),il=a(e,"P",{});var vj=i(il);nl=a(vj,"A",{href:!0,rel:!0});var EI=i(nl);b5=r(EI,"pytest-sugar"),EI.forEach(s),k5=r(vj,` is a plugin that improves the look-n-feel, adds a
progressbar, and show tests that fail and the assert instantly. It gets activated automatically upon installation.`),vj.forEach(s),Av=h(e),c(pl.$$.fragment,e),Dv=h(e),Mi=a(e,"P",{});var gI=i(Mi);x5=r(gI,"To run tests without it, run:"),gI.forEach(s),qv=h(e),c(fl.$$.fragment,e),Sv=h(e),Bi=a(e,"P",{});var bI=i(Bi);C5=r(bI,"or uninstall it."),bI.forEach(s),Lv=h(e),ct=a(e,"H4",{class:!0});var lE=i(ct);Es=a(lE,"A",{id:!0,class:!0,href:!0});var kI=i(Es);uh=a(kI,"SPAN",{});var xI=i(uh);c(hl.$$.fragment,xI),xI.forEach(s),kI.forEach(s),j5=h(lE),dh=a(lE,"SPAN",{});var CI=i(dh);P5=r(CI,"Report each sub-test name and its progress"),CI.forEach(s),lE.forEach(s),Uv=h(e),Pe=a(e,"P",{});var fp=i(Pe);O5=r(fp,"For a single or a group of tests via "),ch=a(fp,"CODE",{});var jI=i(ch);I5=r(jI,"pytest"),jI.forEach(s),T5=r(fp," (after "),mh=a(fp,"CODE",{});var PI=i(mh);A5=r(PI,"pip install pytest-pspec"),PI.forEach(s),D5=r(fp,"):"),fp.forEach(s),Nv=h(e),c(ul.$$.fragment,e),Hv=h(e),mt=a(e,"H4",{class:!0});var aE=i(mt);gs=a(aE,"A",{id:!0,class:!0,href:!0});var OI=i(gs);_h=a(OI,"SPAN",{});var II=i(_h);c(dl.$$.fragment,II),II.forEach(s),OI.forEach(s),q5=h(aE),vh=a(aE,"SPAN",{});var TI=i(vh);S5=r(TI,"Instantly shows failed tests"),TI.forEach(s),aE.forEach(s),Rv=h(e),cl=a(e,"P",{});var yj=i(cl);ml=a(yj,"A",{href:!0,rel:!0});var AI=i(ml);L5=r(AI,"pytest-instafail"),AI.forEach(s),U5=r(yj,` shows failures and errors instantly instead of
waiting until the end of test session.`),yj.forEach(s),zv=h(e),c(_l.$$.fragment,e),Gv=h(e),c(vl.$$.fragment,e),Fv=h(e),_t=a(e,"H3",{class:!0});var iE=i(_t);bs=a(iE,"A",{id:!0,class:!0,href:!0});var DI=i(bs);yh=a(DI,"SPAN",{});var qI=i(yh);c(yl.$$.fragment,qI),qI.forEach(s),DI.forEach(s),N5=h(iE),wh=a(iE,"SPAN",{});var SI=i(wh);H5=r(SI,"To GPU or not to GPU"),SI.forEach(s),iE.forEach(s),Wv=h(e),ks=a(e,"P",{});var nE=i(ks);R5=r(nE,"On a GPU-enabled setup, to test in CPU-only mode add "),$h=a(nE,"CODE",{});var LI=i($h);z5=r(LI,'CUDA_VISIBLE_DEVICES=""'),LI.forEach(s),G5=r(nE,":"),nE.forEach(s),Mv=h(e),c(wl.$$.fragment,e),Bv=h(e),te=a(e,"P",{});var Oo=i(te);F5=r(Oo,"or if you have multiple gpus, you can specify which one is to be used by "),Eh=a(Oo,"CODE",{});var UI=i(Eh);W5=r(UI,"pytest"),UI.forEach(s),M5=r(Oo,`. For example, to use only the
second gpu if you have gpus `),gh=a(Oo,"CODE",{});var NI=i(gh);B5=r(NI,"0"),NI.forEach(s),Y5=r(Oo," and "),bh=a(Oo,"CODE",{});var HI=i(bh);V5=r(HI,"1"),HI.forEach(s),X5=r(Oo,", you can run:"),Oo.forEach(s),Yv=h(e),c($l.$$.fragment,e),Vv=h(e),Yi=a(e,"P",{});var RI=i(Yi);J5=r(RI,"This is handy when you want to run different tasks on different GPUs."),RI.forEach(s),Xv=h(e),Vi=a(e,"P",{});var zI=i(Vi);Z5=r(zI,`Some tests must be run on CPU-only, others on either CPU or GPU or TPU, yet others on multiple-GPUs. The following skip
decorators are used to set the requirements of tests CPU/GPU/TPU-wise:`),zI.forEach(s),Jv=h(e),U=a(e,"UL",{});var he=i(U);Xi=a(he,"LI",{});var wj=i(Xi);kh=a(wj,"CODE",{});var GI=i(kh);K5=r(GI,"require_torch"),GI.forEach(s),Q5=r(wj," - this test will run only under torch"),wj.forEach(s),ek=h(he),xs=a(he,"LI",{});var xm=i(xs);xh=a(xm,"CODE",{});var FI=i(xh);tk=r(FI,"require_torch_gpu"),FI.forEach(s),sk=r(xm," - as "),Ch=a(xm,"CODE",{});var WI=i(Ch);ok=r(WI,"require_torch"),WI.forEach(s),rk=r(xm," plus requires at least 1 GPU"),xm.forEach(s),lk=h(he),Cs=a(he,"LI",{});var Cm=i(Cs);jh=a(Cm,"CODE",{});var MI=i(jh);ak=r(MI,"require_torch_multi_gpu"),MI.forEach(s),ik=r(Cm," - as "),Ph=a(Cm,"CODE",{});var BI=i(Ph);nk=r(BI,"require_torch"),BI.forEach(s),pk=r(Cm," plus requires at least 2 GPUs"),Cm.forEach(s),fk=h(he),js=a(he,"LI",{});var jm=i(js);Oh=a(jm,"CODE",{});var YI=i(Oh);hk=r(YI,"require_torch_non_multi_gpu"),YI.forEach(s),uk=r(jm," - as "),Ih=a(jm,"CODE",{});var VI=i(Ih);dk=r(VI,"require_torch"),VI.forEach(s),ck=r(jm," plus requires 0 or 1 GPUs"),jm.forEach(s),mk=h(he),Ps=a(he,"LI",{});var Pm=i(Ps);Th=a(Pm,"CODE",{});var XI=i(Th);_k=r(XI,"require_torch_up_to_2_gpus"),XI.forEach(s),vk=r(Pm," - as "),Ah=a(Pm,"CODE",{});var JI=i(Ah);yk=r(JI,"require_torch"),JI.forEach(s),wk=r(Pm," plus requires 0 or 1 or 2 GPUs"),Pm.forEach(s),$k=h(he),Os=a(he,"LI",{});var Om=i(Os);Dh=a(Om,"CODE",{});var ZI=i(Dh);Ek=r(ZI,"require_torch_tpu"),ZI.forEach(s),gk=r(Om," - as "),qh=a(Om,"CODE",{});var KI=i(qh);bk=r(KI,"require_torch"),KI.forEach(s),kk=r(Om," plus requires at least 1 TPU"),Om.forEach(s),he.forEach(s),Zv=h(e),Ji=a(e,"P",{});var QI=i(Ji);xk=r(QI,"Let\u2019s depict the GPU requirements in the following table:"),QI.forEach(s),Kv=h(e),k=a(e,"P",{});var j=i(k);Ck=r(j,`| n gpus | decorator                      |
|--------+--------------------------------|
| `),Sh=a(j,"CODE",{});var eT=i(Sh);jk=r(eT,">= 0"),eT.forEach(s),Pk=r(j," | "),Lh=a(j,"CODE",{});var tT=i(Lh);Ok=r(tT,"@require_torch"),tT.forEach(s),Ik=r(j,`               |
| `),Uh=a(j,"CODE",{});var sT=i(Uh);Tk=r(sT,">= 1"),sT.forEach(s),Ak=r(j," | "),Nh=a(j,"CODE",{});var oT=i(Nh);Dk=r(oT,"@require_torch_gpu"),oT.forEach(s),qk=r(j,`           |
| `),Hh=a(j,"CODE",{});var rT=i(Hh);Sk=r(rT,">= 2"),rT.forEach(s),Lk=r(j," | "),Rh=a(j,"CODE",{});var lT=i(Rh);Uk=r(lT,"@require_torch_multi_gpu"),lT.forEach(s),Nk=r(j,`     |
| `),zh=a(j,"CODE",{});var aT=i(zh);Hk=r(aT,"< 2"),aT.forEach(s),Rk=r(j,"  | "),Gh=a(j,"CODE",{});var iT=i(Gh);zk=r(iT,"@require_torch_non_multi_gpu"),iT.forEach(s),Gk=r(j,` |
| `),Fh=a(j,"CODE",{});var nT=i(Fh);Fk=r(nT,"< 3"),nT.forEach(s),Wk=r(j,"  | "),Wh=a(j,"CODE",{});var pT=i(Wh);Mk=r(pT,"@require_torch_up_to_2_gpus"),pT.forEach(s),Bk=r(j,"  |"),j.forEach(s),Qv=h(e),Zi=a(e,"P",{});var fT=i(Zi);Yk=r(fT,"For example, here is a test that must be run only when there are 2 or more GPUs available and pytorch is installed:"),fT.forEach(s),e1=h(e),c(El.$$.fragment,e),t1=h(e),Oe=a(e,"P",{});var hp=i(Oe);Vk=r(hp,"If a test requires "),Mh=a(hp,"CODE",{});var hT=i(Mh);Xk=r(hT,"tensorflow"),hT.forEach(s),Jk=r(hp," use the "),Bh=a(hp,"CODE",{});var uT=i(Bh);Zk=r(uT,"require_tf"),uT.forEach(s),Kk=r(hp," decorator. For example:"),hp.forEach(s),s1=h(e),c(gl.$$.fragment,e),o1=h(e),Ki=a(e,"P",{});var dT=i(Ki);Qk=r(dT,`These decorators can be stacked. For example, if a test is slow and requires at least one GPU under pytorch, here is
how to set it up:`),dT.forEach(s),r1=h(e),c(bl.$$.fragment,e),l1=h(e),Ie=a(e,"P",{});var up=i(Ie);e0=r(up,"Some decorators like "),Yh=a(up,"CODE",{});var cT=i(Yh);t0=r(cT,"@parametrized"),cT.forEach(s),s0=r(up," rewrite test names, therefore "),Vh=a(up,"CODE",{});var mT=i(Vh);o0=r(mT,"@require_*"),mT.forEach(s),r0=r(up,` skip decorators have to be listed
last for them to work correctly. Here is an example of the correct usage:`),up.forEach(s),a1=h(e),c(kl.$$.fragment,e),i1=h(e),Is=a(e,"P",{});var pE=i(Is);l0=r(pE,"This order problem doesn\u2019t exist with "),Xh=a(pE,"CODE",{});var _T=i(Xh);a0=r(_T,"@pytest.mark.parametrize"),_T.forEach(s),i0=r(pE,`, you can put it first or last and it will still
work. But it only works with non-unittests.`),pE.forEach(s),n1=h(e),Qi=a(e,"P",{});var vT=i(Qi);n0=r(vT,"Inside tests:"),vT.forEach(s),p1=h(e),en=a(e,"UL",{});var yT=i(en);Jh=a(yT,"LI",{});var wT=i(Jh);p0=r(wT,"How many GPUs are available:"),wT.forEach(s),yT.forEach(s),f1=h(e),c(xl.$$.fragment,e),h1=h(e),vt=a(e,"H3",{class:!0});var fE=i(vt);Ts=a(fE,"A",{id:!0,class:!0,href:!0});var $T=i(Ts);Zh=a($T,"SPAN",{});var ET=i(Zh);c(Cl.$$.fragment,ET),ET.forEach(s),$T.forEach(s),f0=h(fE),Kh=a(fE,"SPAN",{});var gT=i(Kh);h0=r(gT,"Distributed training"),gT.forEach(s),fE.forEach(s),u1=h(e),yt=a(e,"P",{});var Im=i(yt);Qh=a(Im,"CODE",{});var bT=i(Qh);u0=r(bT,"pytest"),bT.forEach(s),d0=r(Im,` can\u2019t deal with distributed training directly. If this is attempted - the sub-processes don\u2019t do the right
thing and end up thinking they are `),eu=a(Im,"CODE",{});var kT=i(eu);c0=r(kT,"pytest"),kT.forEach(s),m0=r(Im,` and start running the test suite in loops. It works, however, if one
spawns a normal process that then spawns off multiple workers and manages the IO pipes.`),Im.forEach(s),d1=h(e),tn=a(e,"P",{});var xT=i(tn);_0=r(xT,"Here are some tests that use it:"),xT.forEach(s),c1=h(e),As=a(e,"UL",{});var hE=i(As);tu=a(hE,"LI",{});var CT=i(tu);jl=a(CT,"A",{href:!0,rel:!0});var jT=i(jl);v0=r(jT,"test_trainer_distributed.py"),jT.forEach(s),CT.forEach(s),y0=h(hE),su=a(hE,"LI",{});var PT=i(su);Pl=a(PT,"A",{href:!0,rel:!0});var OT=i(Pl);w0=r(OT,"test_deepspeed.py"),OT.forEach(s),PT.forEach(s),hE.forEach(s),m1=h(e),Ds=a(e,"P",{});var uE=i(Ds);$0=r(uE,"To jump right into the execution point, search for the "),ou=a(uE,"CODE",{});var IT=i(ou);E0=r(IT,"execute_subprocess_async"),IT.forEach(s),g0=r(uE," call in those tests."),uE.forEach(s),_1=h(e),sn=a(e,"P",{});var TT=i(sn);b0=r(TT,"You will need at least 2 GPUs to see these tests in action:"),TT.forEach(s),v1=h(e),c(Ol.$$.fragment,e),y1=h(e),wt=a(e,"H3",{class:!0});var dE=i(wt);qs=a(dE,"A",{id:!0,class:!0,href:!0});var AT=i(qs);ru=a(AT,"SPAN",{});var DT=i(ru);c(Il.$$.fragment,DT),DT.forEach(s),AT.forEach(s),k0=h(dE),lu=a(dE,"SPAN",{});var qT=i(lu);x0=r(qT,"Output capture"),qT.forEach(s),dE.forEach(s),w1=h(e),Te=a(e,"P",{});var dp=i(Te);C0=r(dp,"During test execution any output sent to "),au=a(dp,"CODE",{});var ST=i(au);j0=r(ST,"stdout"),ST.forEach(s),P0=r(dp," and "),iu=a(dp,"CODE",{});var LT=i(iu);O0=r(LT,"stderr"),LT.forEach(s),I0=r(dp,` is captured. If a test or a setup method fails, its
according captured output will usually be shown along with the failure traceback.`),dp.forEach(s),$1=h(e),M=a(e,"P",{});var Fe=i(M);T0=r(Fe,"To disable output capturing and to get the "),nu=a(Fe,"CODE",{});var UT=i(nu);A0=r(UT,"stdout"),UT.forEach(s),D0=r(Fe," and "),pu=a(Fe,"CODE",{});var NT=i(pu);q0=r(NT,"stderr"),NT.forEach(s),S0=r(Fe," normally, use "),fu=a(Fe,"CODE",{});var HT=i(fu);L0=r(HT,"-s"),HT.forEach(s),U0=r(Fe," or "),hu=a(Fe,"CODE",{});var RT=i(hu);N0=r(RT,"--capture=no"),RT.forEach(s),H0=r(Fe,":"),Fe.forEach(s),E1=h(e),c(Tl.$$.fragment,e),g1=h(e),on=a(e,"P",{});var zT=i(on);R0=r(zT,"To send test results to JUnit format output:"),zT.forEach(s),b1=h(e),c(Al.$$.fragment,e),k1=h(e),$t=a(e,"H3",{class:!0});var cE=i($t);Ss=a(cE,"A",{id:!0,class:!0,href:!0});var GT=i(Ss);uu=a(GT,"SPAN",{});var FT=i(uu);c(Dl.$$.fragment,FT),FT.forEach(s),GT.forEach(s),z0=h(cE),du=a(cE,"SPAN",{});var WT=i(du);G0=r(WT,"Color control"),WT.forEach(s),cE.forEach(s),x1=h(e),rn=a(e,"P",{});var MT=i(rn);F0=r(MT,"To have no color (e.g., yellow on white background is not readable):"),MT.forEach(s),C1=h(e),c(ql.$$.fragment,e),j1=h(e),Et=a(e,"H3",{class:!0});var mE=i(Et);Ls=a(mE,"A",{id:!0,class:!0,href:!0});var BT=i(Ls);cu=a(BT,"SPAN",{});var YT=i(cu);c(Sl.$$.fragment,YT),YT.forEach(s),BT.forEach(s),W0=h(mE),mu=a(mE,"SPAN",{});var VT=i(mu);M0=r(VT,"Sending test report to online pastebin service"),VT.forEach(s),mE.forEach(s),P1=h(e),ln=a(e,"P",{});var XT=i(ln);B0=r(XT,"Creating a URL for each test failure:"),XT.forEach(s),O1=h(e),c(Ll.$$.fragment,e),I1=h(e),an=a(e,"P",{});var JT=i(an);Y0=r(JT,`This will submit test run information to a remote Paste service and provide a URL for each failure. You may select
tests as usual or add for example -x if you only want to send one particular failure.`),JT.forEach(s),T1=h(e),nn=a(e,"P",{});var ZT=i(nn);V0=r(ZT,"Creating a URL for a whole test session log:"),ZT.forEach(s),A1=h(e),c(Ul.$$.fragment,e),D1=h(e),gt=a(e,"H2",{class:!0});var _E=i(gt);Us=a(_E,"A",{id:!0,class:!0,href:!0});var KT=i(Us);_u=a(KT,"SPAN",{});var QT=i(_u);c(Nl.$$.fragment,QT),QT.forEach(s),KT.forEach(s),X0=h(_E),vu=a(_E,"SPAN",{});var eA=i(vu);J0=r(eA,"Writing tests"),eA.forEach(s),_E.forEach(s),q1=h(e),Ae=a(e,"P",{});var cp=i(Ae);Z0=r(cp,"\u{1F917} transformers tests are based on "),yu=a(cp,"CODE",{});var tA=i(yu);K0=r(tA,"unittest"),tA.forEach(s),Q0=r(cp,", but run by "),wu=a(cp,"CODE",{});var sA=i(wu);e4=r(sA,"pytest"),sA.forEach(s),t4=r(cp,`, so most of the time features from both systems
can be used.`),cp.forEach(s),S1=h(e),se=a(e,"P",{});var Io=i(se);s4=r(Io,"You can read "),Hl=a(Io,"A",{href:!0,rel:!0});var oA=i(Hl);o4=r(oA,"here"),oA.forEach(s),r4=r(Io,` which features are supported, but the important
thing to remember is that most `),$u=a(Io,"CODE",{});var rA=i($u);l4=r(rA,"pytest"),rA.forEach(s),a4=r(Io,` fixtures don\u2019t work. Neither parametrization, but we use the module
`),Eu=a(Io,"CODE",{});var lA=i(Eu);i4=r(lA,"parameterized"),lA.forEach(s),n4=r(Io," that works in a similar way."),Io.forEach(s),L1=h(e),bt=a(e,"H3",{class:!0});var vE=i(bt);Ns=a(vE,"A",{id:!0,class:!0,href:!0});var aA=i(Ns);gu=a(aA,"SPAN",{});var iA=i(gu);c(Rl.$$.fragment,iA),iA.forEach(s),aA.forEach(s),p4=h(vE),bu=a(vE,"SPAN",{});var nA=i(bu);f4=r(nA,"Parametrization"),nA.forEach(s),vE.forEach(s),U1=h(e),pn=a(e,"P",{});var pA=i(pn);h4=r(pA,`Often, there is a need to run the same test multiple times, but with different arguments. It could be done from within
the test, but then there is no way of running that test for just one set of arguments.`),pA.forEach(s),N1=h(e),c(zl.$$.fragment,e),H1=h(e),Hs=a(e,"P",{});var yE=i(Hs);u4=r(yE,"Now, by default this test will be run 3 times, each time with the last 3 arguments of "),ku=a(yE,"CODE",{});var fA=i(ku);d4=r(fA,"test_floor"),fA.forEach(s),c4=r(yE,` being assigned the
corresponding arguments in the parameter list.`),yE.forEach(s),R1=h(e),De=a(e,"P",{});var mp=i(De);m4=r(mp,"and you could run just the "),xu=a(mp,"CODE",{});var hA=i(xu);_4=r(hA,"negative"),hA.forEach(s),v4=r(mp," and "),Cu=a(mp,"CODE",{});var uA=i(Cu);y4=r(uA,"integer"),uA.forEach(s),w4=r(mp," sets of params with:"),mp.forEach(s),z1=h(e),c(Gl.$$.fragment,e),G1=h(e),Rs=a(e,"P",{});var wE=i(Rs);$4=r(wE,"or all but "),ju=a(wE,"CODE",{});var dA=i(ju);E4=r(dA,"negative"),dA.forEach(s),g4=r(wE," sub-tests, with:"),wE.forEach(s),F1=h(e),c(Fl.$$.fragment,e),W1=h(e),zs=a(e,"P",{});var $E=i(zs);b4=r($E,"Besides using the "),Pu=a($E,"CODE",{});var cA=i(Pu);k4=r(cA,"-k"),cA.forEach(s),x4=r($E,` filter that was just mentioned, you can find out the exact name of each sub-test and run any
or all of them using their exact names.`),$E.forEach(s),M1=h(e),c(Wl.$$.fragment,e),B1=h(e),fn=a(e,"P",{});var mA=i(fn);C4=r(mA,"and it will list:"),mA.forEach(s),Y1=h(e),c(Ml.$$.fragment,e),V1=h(e),hn=a(e,"P",{});var _A=i(hn);j4=r(_A,"So now you can run just 2 specific sub-tests:"),_A.forEach(s),X1=h(e),c(Bl.$$.fragment,e),J1=h(e),B=a(e,"P",{});var We=i(B);P4=r(We,"The module "),Yl=a(We,"A",{href:!0,rel:!0});var vA=i(Yl);O4=r(vA,"parameterized"),vA.forEach(s),I4=r(We,` which is already in the developer dependencies
of `),Ou=a(We,"CODE",{});var yA=i(Ou);T4=r(yA,"transformers"),yA.forEach(s),A4=r(We," works for both: "),Iu=a(We,"CODE",{});var wA=i(Iu);D4=r(wA,"unittests"),wA.forEach(s),q4=r(We," and "),Tu=a(We,"CODE",{});var $A=i(Tu);S4=r($A,"pytest"),$A.forEach(s),L4=r(We," tests."),We.forEach(s),Z1=h(e),oe=a(e,"P",{});var To=i(oe);U4=r(To,"If, however, the test is not a "),Au=a(To,"CODE",{});var EA=i(Au);N4=r(EA,"unittest"),EA.forEach(s),H4=r(To,", you may use "),Du=a(To,"CODE",{});var gA=i(Du);R4=r(gA,"pytest.mark.parametrize"),gA.forEach(s),z4=r(To,` (or you may see it being used in
some existing tests, mostly under `),qu=a(To,"CODE",{});var bA=i(qu);G4=r(bA,"examples"),bA.forEach(s),F4=r(To,")."),To.forEach(s),K1=h(e),qe=a(e,"P",{});var _p=i(qe);W4=r(_p,"Here is the same example, this time using "),Su=a(_p,"CODE",{});var kA=i(Su);M4=r(kA,"pytest"),kA.forEach(s),B4=r(_p,"\u2019s "),Lu=a(_p,"CODE",{});var xA=i(Lu);Y4=r(xA,"parametrize"),xA.forEach(s),V4=r(_p," marker:"),_p.forEach(s),Q1=h(e),c(Vl.$$.fragment,e),ey=h(e),re=a(e,"P",{});var Ao=i(re);X4=r(Ao,"Same as with "),Uu=a(Ao,"CODE",{});var CA=i(Uu);J4=r(CA,"parameterized"),CA.forEach(s),Z4=r(Ao,", with "),Nu=a(Ao,"CODE",{});var jA=i(Nu);K4=r(jA,"pytest.mark.parametrize"),jA.forEach(s),Q4=r(Ao,` you can have a fine control over which sub-tests are
run, if the `),Hu=a(Ao,"CODE",{});var PA=i(Hu);e6=r(PA,"-k"),PA.forEach(s),t6=r(Ao,` filter doesn\u2019t do the job. Except, this parametrization function creates a slightly different set of
names for the sub-tests. Here is what they look like:`),Ao.forEach(s),ty=h(e),c(Xl.$$.fragment,e),sy=h(e),un=a(e,"P",{});var OA=i(un);s6=r(OA,"and it will list:"),OA.forEach(s),oy=h(e),c(Jl.$$.fragment,e),ry=h(e),dn=a(e,"P",{});var IA=i(dn);o6=r(IA,"So now you can run just the specific test:"),IA.forEach(s),ly=h(e),c(Zl.$$.fragment,e),ay=h(e),cn=a(e,"P",{});var TA=i(cn);r6=r(TA,"as in the previous example."),TA.forEach(s),iy=h(e),kt=a(e,"H3",{class:!0});var EE=i(kt);Gs=a(EE,"A",{id:!0,class:!0,href:!0});var AA=i(Gs);Ru=a(AA,"SPAN",{});var DA=i(Ru);c(Kl.$$.fragment,DA),DA.forEach(s),AA.forEach(s),l6=h(EE),zu=a(EE,"SPAN",{});var qA=i(zu);a6=r(qA,"Files and directories"),qA.forEach(s),EE.forEach(s),ny=h(e),Fs=a(e,"P",{});var gE=i(Fs);i6=r(gE,`In tests often we need to know where things are relative to the current test file, and it\u2019s not trivial since the test
could be invoked from more than one directory or could reside in sub-directories with different depths. A helper class
`),Gu=a(gE,"CODE",{});var SA=i(Gu);n6=r(SA,"transformers.test_utils.TestCasePlus"),SA.forEach(s),p6=r(gE,` solves this problem by sorting out all the basic paths and provides easy
accessors to them:`),gE.forEach(s),py=h(e),Ws=a(e,"UL",{});var bE=i(Ws);Ql=a(bE,"LI",{});var kE=i(Ql);mn=a(kE,"P",{});var $j=i(mn);Fu=a($j,"CODE",{});var LA=i(Fu);f6=r(LA,"pathlib"),LA.forEach(s),h6=r($j," objects (all fully resolved):"),$j.forEach(s),u6=h(kE),G=a(kE,"UL",{});var ue=i(G);ea=a(ue,"LI",{});var xE=i(ea);Wu=a(xE,"CODE",{});var UA=i(Wu);d6=r(UA,"test_file_path"),UA.forEach(s),c6=r(xE," - the current test file path, i.e. "),Mu=a(xE,"CODE",{});var NA=i(Mu);m6=r(NA,"__file__"),NA.forEach(s),xE.forEach(s),_6=h(ue),_n=a(ue,"LI",{});var Ej=i(_n);Bu=a(Ej,"CODE",{});var HA=i(Bu);v6=r(HA,"test_file_dir"),HA.forEach(s),y6=r(Ej," - the directory containing the current test file"),Ej.forEach(s),w6=h(ue),Ms=a(ue,"LI",{});var Tm=i(Ms);Yu=a(Tm,"CODE",{});var RA=i(Yu);$6=r(RA,"tests_dir"),RA.forEach(s),E6=r(Tm," - the directory of the "),Vu=a(Tm,"CODE",{});var zA=i(Vu);g6=r(zA,"tests"),zA.forEach(s),b6=r(Tm," test suite"),Tm.forEach(s),k6=h(ue),Bs=a(ue,"LI",{});var Am=i(Bs);Xu=a(Am,"CODE",{});var GA=i(Xu);x6=r(GA,"examples_dir"),GA.forEach(s),C6=r(Am," - the directory of the "),Ju=a(Am,"CODE",{});var FA=i(Ju);j6=r(FA,"examples"),FA.forEach(s),P6=r(Am," test suite"),Am.forEach(s),O6=h(ue),vn=a(ue,"LI",{});var gj=i(vn);Zu=a(gj,"CODE",{});var WA=i(Zu);I6=r(WA,"repo_root_dir"),WA.forEach(s),T6=r(gj," - the directory of the repository"),gj.forEach(s),A6=h(ue),Se=a(ue,"LI",{});var di=i(Se);Ku=a(di,"CODE",{});var MA=i(Ku);D6=r(MA,"src_dir"),MA.forEach(s),q6=r(di," - the directory of "),Qu=a(di,"CODE",{});var BA=i(Qu);S6=r(BA,"src"),BA.forEach(s),L6=r(di," (i.e. where the "),ed=a(di,"CODE",{});var YA=i(ed);U6=r(YA,"transformers"),YA.forEach(s),N6=r(di," sub-dir resides)"),di.forEach(s),ue.forEach(s),kE.forEach(s),H6=h(bE),ta=a(bE,"LI",{});var CE=i(ta);sa=a(CE,"P",{});var jE=i(sa);R6=r(jE,"stringified paths---same as above but these return paths as strings, rather than "),td=a(jE,"CODE",{});var VA=i(td);z6=r(VA,"pathlib"),VA.forEach(s),G6=r(jE," objects:"),jE.forEach(s),F6=h(CE),F=a(CE,"UL",{});var de=i(F);sd=a(de,"LI",{});var XA=i(sd);od=a(XA,"CODE",{});var JA=i(od);W6=r(JA,"test_file_path_str"),JA.forEach(s),XA.forEach(s),M6=h(de),rd=a(de,"LI",{});var ZA=i(rd);ld=a(ZA,"CODE",{});var KA=i(ld);B6=r(KA,"test_file_dir_str"),KA.forEach(s),ZA.forEach(s),Y6=h(de),ad=a(de,"LI",{});var QA=i(ad);id=a(QA,"CODE",{});var eD=i(id);V6=r(eD,"tests_dir_str"),eD.forEach(s),QA.forEach(s),X6=h(de),nd=a(de,"LI",{});var tD=i(nd);pd=a(tD,"CODE",{});var sD=i(pd);J6=r(sD,"examples_dir_str"),sD.forEach(s),tD.forEach(s),Z6=h(de),fd=a(de,"LI",{});var oD=i(fd);hd=a(oD,"CODE",{});var rD=i(hd);K6=r(rD,"repo_root_dir_str"),rD.forEach(s),oD.forEach(s),Q6=h(de),ud=a(de,"LI",{});var lD=i(ud);dd=a(lD,"CODE",{});var aD=i(dd);e7=r(aD,"src_dir_str"),aD.forEach(s),lD.forEach(s),de.forEach(s),CE.forEach(s),bE.forEach(s),fy=h(e),Ys=a(e,"P",{});var PE=i(Ys);t7=r(PE,`To start using those all you need is to make sure that the test resides in a subclass of
`),cd=a(PE,"CODE",{});var iD=i(cd);s7=r(iD,"transformers.test_utils.TestCasePlus"),iD.forEach(s),o7=r(PE,". For example:"),PE.forEach(s),hy=h(e),c(oa.$$.fragment,e),uy=h(e),Y=a(e,"P",{});var Me=i(Y);r7=r(Me,"If you don\u2019t need to manipulate paths via "),md=a(Me,"CODE",{});var nD=i(md);l7=r(nD,"pathlib"),nD.forEach(s),a7=r(Me,` or you just need a path as a string, you can always invoked
`),_d=a(Me,"CODE",{});var pD=i(_d);i7=r(pD,"str()"),pD.forEach(s),n7=r(Me," on the "),vd=a(Me,"CODE",{});var fD=i(vd);p7=r(fD,"pathlib"),fD.forEach(s),f7=r(Me," object or use the accessors ending with "),yd=a(Me,"CODE",{});var hD=i(yd);h7=r(hD,"_str"),hD.forEach(s),u7=r(Me,". For example:"),Me.forEach(s),dy=h(e),c(ra.$$.fragment,e),cy=h(e),xt=a(e,"H3",{class:!0});var OE=i(xt);Vs=a(OE,"A",{id:!0,class:!0,href:!0});var uD=i(Vs);wd=a(uD,"SPAN",{});var dD=i(wd);c(la.$$.fragment,dD),dD.forEach(s),uD.forEach(s),d7=h(OE),$d=a(OE,"SPAN",{});var cD=i($d);c7=r(cD,"Temporary files and directories"),cD.forEach(s),OE.forEach(s),my=h(e),Xs=a(e,"P",{});var IE=i(Xs);m7=r(IE,`Using unique temporary files and directories are essential for parallel test running, so that the tests won\u2019t overwrite
each other\u2019s data. Also we want to get the temporary files and directories removed at the end of each test that created
them. Therefore, using packages like `),Ed=a(IE,"CODE",{});var mD=i(Ed);_7=r(mD,"tempfile"),mD.forEach(s),v7=r(IE,", which address these needs is essential."),IE.forEach(s),_y=h(e),yn=a(e,"P",{});var _D=i(yn);y7=r(_D,`However, when debugging tests, you need to be able to see what goes into the temporary file or directory and you want
to know it\u2019s exact path and not having it randomized on every test re-run.`),_D.forEach(s),vy=h(e),Le=a(e,"P",{});var vp=i(Le);w7=r(vp,"A helper class "),gd=a(vp,"CODE",{});var vD=i(gd);$7=r(vD,"transformers.test_utils.TestCasePlus"),vD.forEach(s),E7=r(vp,` is best used for such purposes. It\u2019s a sub-class of
`),bd=a(vp,"CODE",{});var yD=i(bd);g7=r(yD,"unittest.TestCase"),yD.forEach(s),b7=r(vp,", so we can easily inherit from it in the test modules."),vp.forEach(s),yy=h(e),wn=a(e,"P",{});var wD=i(wn);k7=r(wD,"Here is an example of its usage:"),wD.forEach(s),wy=h(e),c(aa.$$.fragment,e),$y=h(e),Js=a(e,"P",{});var TE=i(Js);x7=r(TE,"This code creates a unique temporary directory, and sets "),kd=a(TE,"CODE",{});var $D=i(kd);C7=r($D,"tmp_dir"),$D.forEach(s),j7=r(TE," to its location."),TE.forEach(s),Ey=h(e),$n=a(e,"UL",{});var ED=i($n);xd=a(ED,"LI",{});var gD=i(xd);P7=r(gD,"Create a unique temporary dir:"),gD.forEach(s),ED.forEach(s),gy=h(e),c(ia.$$.fragment,e),by=h(e),na=a(e,"P",{});var bj=i(na);Cd=a(bj,"CODE",{});var bD=i(Cd);O7=r(bD,"tmp_dir"),bD.forEach(s),I7=r(bj,` will contain the path to the created temporary dir. It will be automatically removed at the end of the
test.`),bj.forEach(s),ky=h(e),En=a(e,"UL",{});var kD=i(En);jd=a(kD,"LI",{});var xD=i(jd);T7=r(xD,"Create a temporary dir of my choice, ensure it\u2019s empty before the test starts and don\u2019t empty it after the test."),xD.forEach(s),kD.forEach(s),xy=h(e),c(pa.$$.fragment,e),Cy=h(e),gn=a(e,"P",{});var CD=i(gn);A7=r(CD,`This is useful for debug when you want to monitor a specific directory and want to make sure the previous tests didn\u2019t
leave any data in there.`),CD.forEach(s),jy=h(e),bn=a(e,"UL",{});var jD=i(bn);fa=a(jD,"LI",{});var AE=i(fa);Ct=a(AE,"P",{});var yp=i(Ct);D7=r(yp,"You can override the default behavior by directly overriding the "),Pd=a(yp,"CODE",{});var PD=i(Pd);q7=r(PD,"before"),PD.forEach(s),S7=r(yp," and "),Od=a(yp,"CODE",{});var OD=i(Od);L7=r(OD,"after"),OD.forEach(s),U7=r(yp,` args, leading to one of the
following behaviors:`),yp.forEach(s),N7=h(AE),ve=a(AE,"UL",{});var Do=i(ve);kn=a(Do,"LI",{});var kj=i(kn);Id=a(kj,"CODE",{});var ID=i(Id);H7=r(ID,"before=True"),ID.forEach(s),R7=r(kj,": the temporary dir will always be cleared at the beginning of the test."),kj.forEach(s),z7=h(Do),xn=a(Do,"LI",{});var xj=i(xn);Td=a(xj,"CODE",{});var TD=i(Td);G7=r(TD,"before=False"),TD.forEach(s),F7=r(xj,": if the temporary dir already existed, any existing files will remain there."),xj.forEach(s),W7=h(Do),Cn=a(Do,"LI",{});var Cj=i(Cn);Ad=a(Cj,"CODE",{});var AD=i(Ad);M7=r(AD,"after=True"),AD.forEach(s),B7=r(Cj,": the temporary dir will always be deleted at the end of the test."),Cj.forEach(s),Y7=h(Do),jn=a(Do,"LI",{});var jj=i(jn);Dd=a(jj,"CODE",{});var DD=i(Dd);V7=r(DD,"after=False"),DD.forEach(s),X7=r(jj,": the temporary dir will always be left intact at the end of the test."),jj.forEach(s),Do.forEach(s),AE.forEach(s),jD.forEach(s),Py=h(e),c(Zs.$$.fragment,e),Oy=h(e),c(Ks.$$.fragment,e),Iy=h(e),jt=a(e,"H3",{class:!0});var DE=i(jt);Qs=a(DE,"A",{id:!0,class:!0,href:!0});var qD=i(Qs);qd=a(qD,"SPAN",{});var SD=i(qd);c(ha.$$.fragment,SD),SD.forEach(s),qD.forEach(s),J7=h(DE),Sd=a(DE,"SPAN",{});var LD=i(Sd);Z7=r(LD,"Temporary sys.path override"),LD.forEach(s),DE.forEach(s),Ty=h(e),Ue=a(e,"P",{});var wp=i(Ue);K7=r(wp,"If you need to temporary override "),Ld=a(wp,"CODE",{});var UD=i(Ld);Q7=r(UD,"sys.path"),UD.forEach(s),e8=r(wp,` to import from another test for example, you can use the
`),Ud=a(wp,"CODE",{});var ND=i(Ud);t8=r(ND,"ExtendSysPath"),ND.forEach(s),s8=r(wp," context manager. Example:"),wp.forEach(s),Ay=h(e),c(ua.$$.fragment,e),Dy=h(e),Pt=a(e,"H3",{class:!0});var qE=i(Pt);eo=a(qE,"A",{id:!0,class:!0,href:!0});var HD=i(eo);Nd=a(HD,"SPAN",{});var RD=i(Nd);c(da.$$.fragment,RD),RD.forEach(s),HD.forEach(s),o8=h(qE),Hd=a(qE,"SPAN",{});var zD=i(Hd);r8=r(zD,"Skipping tests"),zD.forEach(s),qE.forEach(s),qy=h(e),to=a(e,"P",{});var SE=i(to);l8=r(SE,`This is useful when a bug is found and a new test is written, yet the bug is not fixed yet. In order to be able to
commit it to the main repository we need make sure it\u2019s skipped during `),Rd=a(SE,"CODE",{});var GD=i(Rd);a8=r(GD,"make test"),GD.forEach(s),i8=r(SE,"."),SE.forEach(s),Sy=h(e),Pn=a(e,"P",{});var FD=i(Pn);n8=r(FD,"Methods:"),FD.forEach(s),Ly=h(e),so=a(e,"UL",{});var LE=i(so);zd=a(LE,"LI",{});var WD=i(zd);ca=a(WD,"P",{});var UE=i(ca);p8=r(UE,"A "),Gd=a(UE,"STRONG",{});var MD=i(Gd);f8=r(MD,"skip"),MD.forEach(s),h8=r(UE,` means that you expect your test to pass only if some conditions are met, otherwise pytest should skip
running the test altogether. Common examples are skipping windows-only tests on non-windows platforms, or skipping
tests that depend on an external resource which is not available at the moment (for example a database).`),UE.forEach(s),WD.forEach(s),u8=h(LE),Fd=a(LE,"LI",{});var BD=i(Fd);ma=a(BD,"P",{});var NE=i(ma);d8=r(NE,"A "),Wd=a(NE,"STRONG",{});var YD=i(Wd);c8=r(YD,"xfail"),YD.forEach(s),m8=r(NE,` means that you expect a test to fail for some reason. A common example is a test for a feature not yet
implemented, or a bug not yet fixed. When a test passes despite being expected to fail (marked with
pytest.mark.xfail), it\u2019s an xpass and will be reported in the test summary.`),NE.forEach(s),BD.forEach(s),LE.forEach(s),Uy=h(e),le=a(e,"P",{});var qo=i(le);_8=r(qo,"One of the important differences between the two is that "),Md=a(qo,"CODE",{});var VD=i(Md);v8=r(VD,"skip"),VD.forEach(s),y8=r(qo," doesn\u2019t run the test, and "),Bd=a(qo,"CODE",{});var XD=i(Bd);w8=r(XD,"xfail"),XD.forEach(s),$8=r(qo,` does. So if the
code that\u2019s buggy causes some bad state that will affect other tests, do not use `),Yd=a(qo,"CODE",{});var JD=i(Yd);E8=r(JD,"xfail"),JD.forEach(s),g8=r(qo,"."),qo.forEach(s),Ny=h(e),Ot=a(e,"H4",{class:!0});var HE=i(Ot);oo=a(HE,"A",{id:!0,class:!0,href:!0});var ZD=i(oo);Vd=a(ZD,"SPAN",{});var KD=i(Vd);c(_a.$$.fragment,KD),KD.forEach(s),ZD.forEach(s),b8=h(HE),Xd=a(HE,"SPAN",{});var QD=i(Xd);k8=r(QD,"Implementation"),QD.forEach(s),HE.forEach(s),Hy=h(e),On=a(e,"UL",{});var eq=i(On);Jd=a(eq,"LI",{});var tq=i(Jd);x8=r(tq,"Here is how to skip whole test unconditionally:"),tq.forEach(s),eq.forEach(s),Ry=h(e),c(va.$$.fragment,e),zy=h(e),In=a(e,"P",{});var sq=i(In);C8=r(sq,"or via pytest:"),sq.forEach(s),Gy=h(e),c(ya.$$.fragment,e),Fy=h(e),ro=a(e,"P",{});var RE=i(ro);j8=r(RE,"or the "),Zd=a(RE,"CODE",{});var oq=i(Zd);P8=r(oq,"xfail"),oq.forEach(s),O8=r(RE," way:"),RE.forEach(s),Wy=h(e),c(wa.$$.fragment,e),My=h(e),Tn=a(e,"UL",{});var rq=i(Tn);Kd=a(rq,"LI",{});var lq=i(Kd);I8=r(lq,"Here is how to skip a test based on some internal check inside the test:"),lq.forEach(s),rq.forEach(s),By=h(e),c($a.$$.fragment,e),Yy=h(e),An=a(e,"P",{});var aq=i(An);T8=r(aq,"or the whole module:"),aq.forEach(s),Vy=h(e),c(Ea.$$.fragment,e),Xy=h(e),lo=a(e,"P",{});var zE=i(lo);A8=r(zE,"or the "),Qd=a(zE,"CODE",{});var iq=i(Qd);D8=r(iq,"xfail"),iq.forEach(s),q8=r(zE," way:"),zE.forEach(s),Jy=h(e),c(ga.$$.fragment,e),Zy=h(e),Dn=a(e,"UL",{});var nq=i(Dn);ec=a(nq,"LI",{});var pq=i(ec);S8=r(pq,"Here is how to skip all tests in a module if some import is missing:"),pq.forEach(s),nq.forEach(s),Ky=h(e),c(ba.$$.fragment,e),Qy=h(e),qn=a(e,"UL",{});var fq=i(qn);tc=a(fq,"LI",{});var hq=i(tc);L8=r(hq,"Skip a test based on a condition:"),hq.forEach(s),fq.forEach(s),ew=h(e),c(ka.$$.fragment,e),tw=h(e),Sn=a(e,"P",{});var uq=i(Sn);U8=r(uq,"or:"),uq.forEach(s),sw=h(e),c(xa.$$.fragment,e),ow=h(e),Ln=a(e,"P",{});var dq=i(Ln);N8=r(dq,"or skip the whole module:"),dq.forEach(s),rw=h(e),c(Ca.$$.fragment,e),lw=h(e),ao=a(e,"P",{});var GE=i(ao);H8=r(GE,"More details, example and ways are "),ja=a(GE,"A",{href:!0,rel:!0});var cq=i(ja);R8=r(cq,"here"),cq.forEach(s),z8=r(GE,"."),GE.forEach(s),aw=h(e),It=a(e,"H3",{class:!0});var FE=i(It);io=a(FE,"A",{id:!0,class:!0,href:!0});var mq=i(io);sc=a(mq,"SPAN",{});var _q=i(sc);c(Pa.$$.fragment,_q),_q.forEach(s),mq.forEach(s),G8=h(FE),oc=a(FE,"SPAN",{});var vq=i(oc);F8=r(vq,"Slow tests"),vq.forEach(s),FE.forEach(s),iw=h(e),Un=a(e,"P",{});var yq=i(Un);W8=r(yq,`The library of tests is ever-growing, and some of the tests take minutes to run, therefore we can\u2019t afford waiting for
an hour for the test suite to complete on CI. Therefore, with some exceptions for essential tests, slow tests should be
marked as in the example below:`),yq.forEach(s),nw=h(e),c(Oa.$$.fragment,e),pw=h(e),Ne=a(e,"P",{});var $p=i(Ne);M8=r($p,"Once a test is marked as "),rc=a($p,"CODE",{});var wq=i(rc);B8=r(wq,"@slow"),wq.forEach(s),Y8=r($p,", to run such tests set "),lc=a($p,"CODE",{});var $q=i(lc);V8=r($q,"RUN_SLOW=1"),$q.forEach(s),X8=r($p," env var, e.g.:"),$p.forEach(s),fw=h(e),c(Ia.$$.fragment,e),hw=h(e),ae=a(e,"P",{});var So=i(ae);J8=r(So,"Some decorators like "),ac=a(So,"CODE",{});var Eq=i(ac);Z8=r(Eq,"@parameterized"),Eq.forEach(s),K8=r(So," rewrite test names, therefore "),ic=a(So,"CODE",{});var gq=i(ic);Q8=r(gq,"@slow"),gq.forEach(s),e9=r(So,` and the rest of the skip decorators
`),nc=a(So,"CODE",{});var bq=i(nc);t9=r(bq,"@require_*"),bq.forEach(s),s9=r(So," have to be listed last for them to work correctly. Here is an example of the correct usage:"),So.forEach(s),uw=h(e),c(Ta.$$.fragment,e),dw=h(e),Nn=a(e,"P",{});var kq=i(Nn);o9=r(kq,`As explained at the beginning of this document, slow tests get to run on a scheduled basis, rather than in PRs CI
checks. So it\u2019s possible that some problems will be missed during a PR submission and get merged. Such problems will
get caught during the next scheduled CI job. But it also means that it\u2019s important to run the slow tests on your
machine before submitting the PR.`),kq.forEach(s),cw=h(e),Hn=a(e,"P",{});var xq=i(Hn);r9=r(xq,"Here is a rough decision making mechanism for choosing which tests should be marked as slow:"),xq.forEach(s),mw=h(e),Rn=a(e,"P",{});var Cq=i(Rn);l9=r(Cq,`If the test is focused on one of the library\u2019s internal components (e.g., modeling files, tokenization files,
pipelines), then we should run that test in the non-slow test suite. If it\u2019s focused on an other aspect of the library,
such as the documentation or the examples, then we should run these tests in the slow test suite. And then, to refine
this approach we should have exceptions:`),Cq.forEach(s),_w=h(e),ie=a(e,"UL",{});var Lo=i(ie);pc=a(Lo,"LI",{});var jq=i(pc);a9=r(jq,`All tests that need to download a heavy set of weights or a dataset that is larger than ~50MB (e.g., model or
tokenizer integration tests, pipeline integration tests) should be set to slow. If you\u2019re adding a new model, you
should create and upload to the hub a tiny version of it (with random weights) for integration tests. This is
discussed in the following paragraphs.`),jq.forEach(s),i9=h(Lo),fc=a(Lo,"LI",{});var Pq=i(fc);n9=r(Pq,"All tests that need to do a training not specifically optimized to be fast should be set to slow."),Pq.forEach(s),p9=h(Lo),Tt=a(Lo,"LI",{});var Ep=i(Tt);f9=r(Ep,`We can introduce exceptions if some of these should-be-non-slow tests are excruciatingly slow, and set them to
`),hc=a(Ep,"CODE",{});var Oq=i(hc);h9=r(Oq,"@slow"),Oq.forEach(s),u9=r(Ep,`. Auto-modeling tests, which save and load large files to disk, are a good example of tests that are marked
as `),uc=a(Ep,"CODE",{});var Iq=i(uc);d9=r(Iq,"@slow"),Iq.forEach(s),c9=r(Ep,"."),Ep.forEach(s),m9=h(Lo),dc=a(Lo,"LI",{});var Tq=i(dc);_9=r(Tq,"If a test completes under 1 second on CI (including downloads if any) then it should be a normal test regardless."),Tq.forEach(s),Lo.forEach(s),vw=h(e),He=a(e,"P",{});var gp=i(He);v9=r(gp,`Collectively, all the non-slow tests need to cover entirely the different internals, while remaining fast. For example,
a significant coverage can be achieved by testing with specially created tiny models with random weights. Such models
have the very minimal number of layers (e.g., 2), vocab size (e.g., 1000), etc. Then the `),cc=a(gp,"CODE",{});var Aq=i(cc);y9=r(Aq,"@slow"),Aq.forEach(s),w9=r(gp,` tests can use large
slow models to do qualitative testing. To see the use of these simply look for `),mc=a(gp,"EM",{});var Dq=i(mc);$9=r(Dq,"tiny"),Dq.forEach(s),E9=r(gp," models with:"),gp.forEach(s),yw=h(e),c(Aa.$$.fragment,e),ww=h(e),Re=a(e,"P",{});var bp=i(Re);g9=r(bp,"Here is a an example of a "),Da=a(bp,"A",{href:!0,rel:!0});var qq=i(Da);b9=r(qq,"script"),qq.forEach(s),k9=r(bp,` that created the tiny model
`),qa=a(bp,"A",{href:!0,rel:!0});var Sq=i(qa);x9=r(Sq,"stas/tiny-wmt19-en-de"),Sq.forEach(s),C9=r(bp,`. You can easily adjust it to your specific
model\u2019s architecture.`),bp.forEach(s),$w=h(e),no=a(e,"P",{});var WE=i(no);j9=r(WE,`It\u2019s easy to measure the run-time incorrectly if for example there is an overheard of downloading a huge model, but if
you test it locally the downloaded files would be cached and thus the download time not measured. Hence check the
execution speed report in CI logs instead (the output of `),_c=a(WE,"CODE",{});var Lq=i(_c);P9=r(Lq,"pytest --durations=0 tests"),Lq.forEach(s),O9=r(WE,")."),WE.forEach(s),Ew=h(e),zn=a(e,"P",{});var Uq=i(zn);I9=r(Uq,`That report is also useful to find slow outliers that aren\u2019t marked as such, or which need to be re-written to be fast.
If you notice that the test suite starts getting slow on CI, the top listing of this report will show the slowest
tests.`),Uq.forEach(s),gw=h(e),At=a(e,"H3",{class:!0});var ME=i(At);po=a(ME,"A",{id:!0,class:!0,href:!0});var Nq=i(po);vc=a(Nq,"SPAN",{});var Hq=i(vc);c(Sa.$$.fragment,Hq),Hq.forEach(s),Nq.forEach(s),T9=h(ME),yc=a(ME,"SPAN",{});var Rq=i(yc);A9=r(Rq,"Testing the stdout/stderr output"),Rq.forEach(s),ME.forEach(s),bw=h(e),V=a(e,"P",{});var Be=i(V);D9=r(Be,"In order to test functions that write to "),wc=a(Be,"CODE",{});var zq=i(wc);q9=r(zq,"stdout"),zq.forEach(s),S9=r(Be," and/or "),$c=a(Be,"CODE",{});var Gq=i($c);L9=r(Gq,"stderr"),Gq.forEach(s),U9=r(Be,`, the test can access those streams using the
`),Ec=a(Be,"CODE",{});var Fq=i(Ec);N9=r(Fq,"pytest"),Fq.forEach(s),H9=r(Be,"\u2019s "),La=a(Be,"A",{href:!0,rel:!0});var Wq=i(La);R9=r(Wq,"capsys system"),Wq.forEach(s),z9=r(Be,". Here is how this is accomplished:"),Be.forEach(s),kw=h(e),c(Ua.$$.fragment,e),xw=h(e),fo=a(e,"P",{});var BE=i(fo);G9=r(BE,"And, of course, most of the time, "),gc=a(BE,"CODE",{});var Mq=i(gc);F9=r(Mq,"stderr"),Mq.forEach(s),W9=r(BE,` will come as a part of an exception, so try/except has to be used in such
a case:`),BE.forEach(s),Cw=h(e),c(Na.$$.fragment,e),jw=h(e),ho=a(e,"P",{});var YE=i(ho);M9=r(YE,"Another approach to capturing stdout is via "),bc=a(YE,"CODE",{});var Bq=i(bc);B9=r(Bq,"contextlib.redirect_stdout"),Bq.forEach(s),Y9=r(YE,":"),YE.forEach(s),Pw=h(e),c(Ha.$$.fragment,e),Ow=h(e),T=a(e,"P",{});var J=i(T);V9=r(J,"An important potential issue with capturing stdout is that it may contain "),kc=a(J,"CODE",{});var Yq=i(kc);X9=r(Yq,"\\r"),Yq.forEach(s),J9=r(J," characters that in normal "),xc=a(J,"CODE",{});var Vq=i(xc);Z9=r(Vq,"print"),Vq.forEach(s),K9=r(J,`
reset everything that has been printed so far. There is no problem with `),Cc=a(J,"CODE",{});var Xq=i(Cc);Q9=r(Xq,"pytest"),Xq.forEach(s),ex=r(J,", but with "),jc=a(J,"CODE",{});var Jq=i(jc);tx=r(Jq,"pytest -s"),Jq.forEach(s),sx=r(J,` these
characters get included in the buffer, so to be able to have the test run with and without `),Pc=a(J,"CODE",{});var Zq=i(Pc);ox=r(Zq,"-s"),Zq.forEach(s),rx=r(J,`, you have to make an
extra cleanup to the captured output, using `),Oc=a(J,"CODE",{});var Kq=i(Oc);lx=r(Kq,"re.sub(r'~.*\\r', '', buf, 0, re.M)"),Kq.forEach(s),ax=r(J,"."),J.forEach(s),Iw=h(e),uo=a(e,"P",{});var VE=i(uo);ix=r(VE,`But, then we have a helper context manager wrapper to automatically take care of it all, regardless of whether it has
some `),Ic=a(VE,"CODE",{});var Qq=i(Ic);nx=r(Qq,"\\r"),Qq.forEach(s),px=r(VE,"\u2019s in it or not, so it\u2019s a simple:"),VE.forEach(s),Tw=h(e),c(Ra.$$.fragment,e),Aw=h(e),Gn=a(e,"P",{});var eS=i(Gn);fx=r(eS,"Here is a full test example:"),eS.forEach(s),Dw=h(e),c(za.$$.fragment,e),qw=h(e),ze=a(e,"P",{});var kp=i(ze);hx=r(kp,"If you\u2019d like to capture "),Tc=a(kp,"CODE",{});var tS=i(Tc);ux=r(tS,"stderr"),tS.forEach(s),dx=r(kp," use the "),Ac=a(kp,"CODE",{});var sS=i(Ac);cx=r(sS,"CaptureStderr"),sS.forEach(s),mx=r(kp," class instead:"),kp.forEach(s),Sw=h(e),c(Ga.$$.fragment,e),Lw=h(e),co=a(e,"P",{});var XE=i(co);_x=r(XE,"If you need to capture both streams at once, use the parent "),Dc=a(XE,"CODE",{});var oS=i(Dc);vx=r(oS,"CaptureStd"),oS.forEach(s),yx=r(XE," class:"),XE.forEach(s),Uw=h(e),c(Fa.$$.fragment,e),Nw=h(e),Fn=a(e,"P",{});var rS=i(Fn);wx=r(rS,`Also, to aid debugging test issues, by default these context managers automatically replay the captured streams on exit
from the context.`),rS.forEach(s),Hw=h(e),Dt=a(e,"H3",{class:!0});var JE=i(Dt);mo=a(JE,"A",{id:!0,class:!0,href:!0});var lS=i(mo);qc=a(lS,"SPAN",{});var aS=i(qc);c(Wa.$$.fragment,aS),aS.forEach(s),lS.forEach(s),$x=h(JE),Sc=a(JE,"SPAN",{});var iS=i(Sc);Ex=r(iS,"Capturing logger stream"),iS.forEach(s),JE.forEach(s),Rw=h(e),_o=a(e,"P",{});var ZE=i(_o);gx=r(ZE,"If you need to validate the output of a logger, you can use "),Lc=a(ZE,"CODE",{});var nS=i(Lc);bx=r(nS,"CaptureLogger"),nS.forEach(s),kx=r(ZE,":"),ZE.forEach(s),zw=h(e),c(Ma.$$.fragment,e),Gw=h(e),qt=a(e,"H3",{class:!0});var KE=i(qt);vo=a(KE,"A",{id:!0,class:!0,href:!0});var pS=i(vo);Uc=a(pS,"SPAN",{});var fS=i(Uc);c(Ba.$$.fragment,fS),fS.forEach(s),pS.forEach(s),xx=h(KE),Nc=a(KE,"SPAN",{});var hS=i(Nc);Cx=r(hS,"Testing with environment variables"),hS.forEach(s),KE.forEach(s),Fw=h(e),Ya=a(e,"P",{});var Pj=i(Ya);jx=r(Pj,`If you want to test the impact of environment variables for a specific test you can use a helper decorator
`),Hc=a(Pj,"CODE",{});var uS=i(Hc);Px=r(uS,"transformers.testing_utils.mockenv"),uS.forEach(s),Pj.forEach(s),Ww=h(e),c(Va.$$.fragment,e),Mw=h(e),ne=a(e,"P",{});var Uo=i(ne);Ox=r(Uo,"At times an external program needs to be called, which requires setting "),Rc=a(Uo,"CODE",{});var dS=i(Rc);Ix=r(dS,"PYTHONPATH"),dS.forEach(s),Tx=r(Uo," in "),zc=a(Uo,"CODE",{});var cS=i(zc);Ax=r(cS,"os.environ"),cS.forEach(s),Dx=r(Uo,` to include
multiple local paths. A helper class `),Gc=a(Uo,"CODE",{});var mS=i(Gc);qx=r(mS,"transformers.test_utils.TestCasePlus"),mS.forEach(s),Sx=r(Uo," comes to help:"),Uo.forEach(s),Bw=h(e),c(Xa.$$.fragment,e),Yw=h(e),N=a(e,"P",{});var ce=i(N);Lx=r(ce,"Depending on whether the test file was under the "),Fc=a(ce,"CODE",{});var _S=i(Fc);Ux=r(_S,"tests"),_S.forEach(s),Nx=r(ce," test suite or "),Wc=a(ce,"CODE",{});var vS=i(Wc);Hx=r(vS,"examples"),vS.forEach(s),Rx=r(ce,` it\u2019ll correctly set up
`),Mc=a(ce,"CODE",{});var yS=i(Mc);zx=r(yS,"env[PYTHONPATH]"),yS.forEach(s),Gx=r(ce," to include one of these two directories, and also the "),Bc=a(ce,"CODE",{});var wS=i(Bc);Fx=r(wS,"src"),wS.forEach(s),Wx=r(ce,` directory to ensure the testing is
done against the current repo, and finally with whatever `),Yc=a(ce,"CODE",{});var $S=i(Yc);Mx=r($S,"env[PYTHONPATH]"),$S.forEach(s),Bx=r(ce,` was already set to before the test was
called if anything.`),ce.forEach(s),Vw=h(e),yo=a(e,"P",{});var QE=i(yo);Yx=r(QE,"This helper method creates a copy of the "),Vc=a(QE,"CODE",{});var ES=i(Vc);Vx=r(ES,"os.environ"),ES.forEach(s),Xx=r(QE," object, so the original remains intact."),QE.forEach(s),Xw=h(e),St=a(e,"H3",{class:!0});var eg=i(St);wo=a(eg,"A",{id:!0,class:!0,href:!0});var gS=i(wo);Xc=a(gS,"SPAN",{});var bS=i(Xc);c(Ja.$$.fragment,bS),bS.forEach(s),gS.forEach(s),Jx=h(eg),Jc=a(eg,"SPAN",{});var kS=i(Jc);Zx=r(kS,"Getting reproducible results"),kS.forEach(s),eg.forEach(s),Jw=h(e),Wn=a(e,"P",{});var xS=i(Wn);Kx=r(xS,`In some situations you may want to remove randomness for your tests. To get identical reproducable results set, you
will need to fix the seed:`),xS.forEach(s),Zw=h(e),c(Za.$$.fragment,e),Kw=h(e),Lt=a(e,"H3",{class:!0});var tg=i(Lt);$o=a(tg,"A",{id:!0,class:!0,href:!0});var CS=i($o);Zc=a(CS,"SPAN",{});var jS=i(Zc);c(Ka.$$.fragment,jS),jS.forEach(s),CS.forEach(s),Qx=h(tg),Kc=a(tg,"SPAN",{});var PS=i(Kc);eC=r(PS,"Debugging tests"),PS.forEach(s),tg.forEach(s),Qw=h(e),Mn=a(e,"P",{});var OS=i(Mn);tC=r(OS,"To start a debugger at the point of the warning, do this:"),OS.forEach(s),e$=h(e),c(Qa.$$.fragment,e),t$=h(e),Ut=a(e,"H2",{class:!0});var sg=i(Ut);Eo=a(sg,"A",{id:!0,class:!0,href:!0});var IS=i(Eo);Qc=a(IS,"SPAN",{});var TS=i(Qc);c(ei.$$.fragment,TS),TS.forEach(s),IS.forEach(s),sC=h(sg),em=a(sg,"SPAN",{});var AS=i(em);oC=r(AS,"Working with github actions workflows"),AS.forEach(s),sg.forEach(s),s$=h(e),Bn=a(e,"P",{});var DS=i(Bn);rC=r(DS,"To trigger a self-push workflow CI job, you must:"),DS.forEach(s),o$=h(e),pe=a(e,"OL",{});var No=i(pe);ti=a(No,"LI",{});var og=i(ti);lC=r(og,"Create a new branch on "),tm=a(og,"CODE",{});var qS=i(tm);aC=r(qS,"transformers"),qS.forEach(s),iC=r(og," origin (not a fork!)."),og.forEach(s),nC=h(No),H=a(No,"LI",{});var K=i(H);pC=r(K,"The branch name has to start with either "),sm=a(K,"CODE",{});var SS=i(sm);fC=r(SS,"ci_"),SS.forEach(s),hC=r(K," or "),om=a(K,"CODE",{});var LS=i(om);uC=r(LS,"ci-"),LS.forEach(s),dC=r(K," ("),rm=a(K,"CODE",{});var US=i(rm);cC=r(US,"main"),US.forEach(s),mC=r(K,` triggers it too, but we can\u2019t do PRs on
`),lm=a(K,"CODE",{});var NS=i(lm);_C=r(NS,"main"),NS.forEach(s),vC=r(K,`). It also gets triggered only for specific paths - you can find the up-to-date definition in case it
changed since this document has been written `),si=a(K,"A",{href:!0,rel:!0});var HS=i(si);yC=r(HS,"here"),HS.forEach(s),wC=r(K," under "),am=a(K,"EM",{});var RS=i(am);$C=r(RS,"push:"),RS.forEach(s),K.forEach(s),EC=h(No),im=a(No,"LI",{});var zS=i(im);gC=r(zS,"Create a PR from this branch."),zS.forEach(s),bC=h(No),oi=a(No,"LI",{});var rg=i(oi);kC=r(rg,"Then you can see the job appear "),ri=a(rg,"A",{href:!0,rel:!0});var GS=i(ri);xC=r(GS,"here"),GS.forEach(s),CC=r(rg,`. It may not run right away if there
is a backlog.`),rg.forEach(s),No.forEach(s),r$=h(e),Nt=a(e,"H2",{class:!0});var lg=i(Nt);go=a(lg,"A",{id:!0,class:!0,href:!0});var FS=i(go);nm=a(FS,"SPAN",{});var WS=i(nm);c(li.$$.fragment,WS),WS.forEach(s),FS.forEach(s),jC=h(lg),pm=a(lg,"SPAN",{});var MS=i(pm);PC=r(MS,"Testing Experimental CI Features"),MS.forEach(s),lg.forEach(s),l$=h(e),Yn=a(e,"P",{});var BS=i(Yn);OC=r(BS,`Testing CI features can be potentially problematic as it can interfere with the normal CI functioning. Therefore if a
new CI feature is to be added, it should be done as following.`),BS.forEach(s),a$=h(e),fe=a(e,"OL",{});var Ho=i(fe);fm=a(Ho,"LI",{});var YS=i(fm);IC=r(YS,"Create a new dedicated job that tests what needs to be tested"),YS.forEach(s),TC=h(Ho),hm=a(Ho,"LI",{});var VS=i(hm);AC=r(VS,"The new job must always succeed so that it gives us a green \u2713 (details below)."),VS.forEach(s),DC=h(Ho),um=a(Ho,"LI",{});var XS=i(um);qC=r(XS,`Let it run for some days to see that a variety of different PR types get to run on it (user fork branches,
non-forked branches, branches originating from github.com UI direct file edit, various forced pushes, etc. - there
are so many) while monitoring the experimental job\u2019s logs (not the overall job green as it\u2019s purposefully always
green)`),XS.forEach(s),SC=h(Ho),dm=a(Ho,"LI",{});var JS=i(dm);LC=r(JS,"When it\u2019s clear that everything is solid, then merge the new changes into existing jobs."),JS.forEach(s),Ho.forEach(s),i$=h(e),Vn=a(e,"P",{});var ZS=i(Vn);UC=r(ZS,"That way experiments on CI functionality itself won\u2019t interfere with the normal workflow."),ZS.forEach(s),n$=h(e),Xn=a(e,"P",{});var KS=i(Xn);NC=r(KS,"Now how can we make the job always succeed while the new CI feature is being developed?"),KS.forEach(s),p$=h(e),Jn=a(e,"P",{});var QS=i(Jn);HC=r(QS,`Some CIs, like TravisCI support ignore-step-failure and will report the overall job as successful, but CircleCI and
Github Actions as of this writing don\u2019t support that.`),QS.forEach(s),f$=h(e),Zn=a(e,"P",{});var eL=i(Zn);RC=r(eL,"So the following workaround can be used:"),eL.forEach(s),h$=h(e),bo=a(e,"OL",{});var ag=i(bo);Kn=a(ag,"LI",{});var Oj=i(Kn);cm=a(Oj,"CODE",{});var tL=i(cm);zC=r(tL,"set +euo pipefail"),tL.forEach(s),GC=r(Oj," at the beginning of the run command to suppress most potential failures in the bash script."),Oj.forEach(s),FC=h(ag),Ht=a(ag,"LI",{});var xp=i(Ht);WC=r(xp,"the last command must be a success: "),mm=a(xp,"CODE",{});var sL=i(mm);MC=r(sL,'echo "done"'),sL.forEach(s),BC=r(xp," or just "),_m=a(xp,"CODE",{});var oL=i(_m);YC=r(oL,"true"),oL.forEach(s),VC=r(xp," will do"),xp.forEach(s),ag.forEach(s),u$=h(e),Qn=a(e,"P",{});var rL=i(Qn);XC=r(rL,"Here is an example:"),rL.forEach(s),d$=h(e),c(ai.$$.fragment,e),c$=h(e),ep=a(e,"P",{});var lL=i(ep);JC=r(lL,"For simple commands you could also do:"),lL.forEach(s),m$=h(e),c(ii.$$.fragment,e),_$=h(e),ko=a(e,"P",{});var ig=i(ko);ZC=r(ig,`Of course, once satisfied with the results, integrate the experimental step or job with the rest of the normal jobs,
while removing `),vm=a(ig,"CODE",{});var aL=i(vm);KC=r(aL,"set +euo pipefail"),aL.forEach(s),QC=r(ig,` or any other things you may have added to ensure that the experimental job doesn\u2019t
interfere with the normal CI functioning.`),ig.forEach(s),v$=h(e),xo=a(e,"P",{});var ng=i(xo);ej=r(ng,"This whole process would have been much easier if we only could set something like "),ym=a(ng,"CODE",{});var iL=i(ym);tj=r(iL,"allow-failure"),iL.forEach(s),sj=r(ng,` for the
experimental step, and let it fail without impacting the overall status of PRs. But as mentioned earlier CircleCI and
Github Actions don\u2019t support it at the moment.`),ng.forEach(s),y$=h(e),tp=a(e,"P",{});var nL=i(tp);oj=r(nL,"You can vote for this feature and see where it is at these CI-specific threads:"),nL.forEach(s),w$=h(e),Co=a(e,"UL",{});var pg=i(Co);wm=a(pg,"LI",{});var pL=i(wm);ni=a(pL,"A",{href:!0,rel:!0});var fL=i(ni);rj=r(fL,"Github Actions:"),fL.forEach(s),pL.forEach(s),lj=h(pg),$m=a(pg,"LI",{});var hL=i($m);pi=a(hL,"A",{href:!0,rel:!0});var uL=i(pi);aj=r(uL,"CircleCI:"),uL.forEach(s),hL.forEach(s),pg.forEach(s),this.h()},h(){u($,"name","hf:doc:metadata"),u($,"content",JSON.stringify(bL)),u(x,"id","testing"),u(x,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),u(x,"href","#testing"),u(g,"class","relative group"),u(Gt,"id","how-transformers-are-tested"),u(Gt,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),u(Gt,"href","#how-transformers-are-tested"),u(Xe,"class","relative group"),u(Fo,"href","https://github.com/huggingface/transformers/tree/main/.circleci/config.yml"),u(Fo,"rel","nofollow"),u(Yo,"href","https://github.com/huggingface/transformers/actions"),u(Yo,"rel","nofollow"),u(Vo,"href","https://github.com/huggingface/transformers/tree/main/.github/workflows/github-torch-hub.yml"),u(Vo,"rel","nofollow"),u(Xo,"href","https://github.com/huggingface/transformers/tree/main/.github/workflows/self-push.yml"),u(Xo,"rel","nofollow"),u(Jo,"href","https://github.com/huggingface/transformers/tree/main/.github/workflows/self-scheduled.yml"),u(Jo,"rel","nofollow"),u(Ko,"href","https://github.com/huggingface/transformers/actions"),u(Ko,"rel","nofollow"),u(Mt,"id","running-tests"),u(Mt,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),u(Mt,"href","#running-tests"),u(Ze,"class","relative group"),u(Bt,"id","choosing-which-tests-to-run"),u(Bt,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),u(Bt,"href","#choosing-which-tests-to-run"),u(Ke,"class","relative group"),u(tr,"href","https://docs.pytest.org/en/latest/usage.html"),u(tr,"rel","nofollow"),u(Vt,"id","getting-the-list-of-all-tests"),u(Vt,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),u(Vt,"href","#getting-the-list-of-all-tests"),u(Qe,"class","relative group"),u(Xt,"id","run-a-specific-test-module"),u(Xt,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),u(Xt,"href","#run-a-specific-test-module"),u(et,"class","relative group"),u(Jt,"id","run-specific-tests"),u(Jt,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),u(Jt,"href","#run-specific-tests"),u(tt,"class","relative group"),u(ss,"id","run-documentation-tests"),u(ss,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),u(ss,"href","#run-documentation-tests"),u(st,"class","relative group"),u(os,"href","https://github.com/huggingface/transformers/blob/main/src/transformers/models/whisper/modeling_whisper.py#L1017-L1035"),u(os,"rel","nofollow"),u(br,"start","2"),u(xr,"start","3"),u(rs,"id","run-only-modified-tests"),u(rs,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),u(rs,"href","#run-only-modified-tests"),u(ot,"class","relative group"),u(Or,"href","https://github.com/anapaulagomes/pytest-picked"),u(Or,"rel","nofollow"),u(as,"id","automatically-rerun-failed-tests-on-source-modification"),u(as,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),u(as,"href","#automatically-rerun-failed-tests-on-source-modification"),u(rt,"class","relative group"),u(qr,"href","https://github.com/pytest-dev/pytest-xdist"),u(qr,"rel","nofollow"),u(Hr,"href","https://github.com/joeyespo/pytest-watch"),u(Hr,"rel","nofollow"),u(is,"id","skip-a-test-module"),u(is,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),u(is,"href","#skip-a-test-module"),u(at,"class","relative group"),u(ps,"id","clearing-state"),u(ps,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),u(ps,"href","#clearing-state"),u(it,"class","relative group"),u(fs,"id","running-tests-in-parallel"),u(fs,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),u(fs,"href","#running-tests-in-parallel"),u(nt,"class","relative group"),u(Mr,"href","https://github.com/ESSS/pytest-replay"),u(Mr,"rel","nofollow"),u(hs,"id","test-order-and-repetition"),u(hs,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),u(hs,"href","#test-order-and-repetition"),u(pt,"class","relative group"),u(us,"id","repeat-tests"),u(us,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),u(us,"href","#repeat-tests"),u(ft,"class","relative group"),u(Vr,"href","https://github.com/dropbox/pytest-flakefinder"),u(Vr,"rel","nofollow"),u(ms,"id","run-tests-in-a-random-order"),u(ms,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),u(ms,"href","#run-tests-in-a-random-order"),u(ht,"class","relative group"),u(ol,"href","https://github.com/jbasko/pytest-random-order"),u(ol,"rel","nofollow"),u(rl,"href","https://github.com/pytest-dev/pytest-randomly"),u(rl,"rel","nofollow"),u(ws,"id","look-and-feel-variations"),u(ws,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),u(ws,"href","#look-and-feel-variations"),u(ut,"class","relative group"),u($s,"id","pytestsugar"),u($s,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),u($s,"href","#pytestsugar"),u(dt,"class","relative group"),u(nl,"href","https://github.com/Frozenball/pytest-sugar"),u(nl,"rel","nofollow"),u(Es,"id","report-each-subtest-name-and-its-progress"),u(Es,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),u(Es,"href","#report-each-subtest-name-and-its-progress"),u(ct,"class","relative group"),u(gs,"id","instantly-shows-failed-tests"),u(gs,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),u(gs,"href","#instantly-shows-failed-tests"),u(mt,"class","relative group"),u(ml,"href","https://github.com/pytest-dev/pytest-instafail"),u(ml,"rel","nofollow"),u(bs,"id","to-gpu-or-not-to-gpu"),u(bs,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),u(bs,"href","#to-gpu-or-not-to-gpu"),u(_t,"class","relative group"),u(Ts,"id","distributed-training"),u(Ts,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),u(Ts,"href","#distributed-training"),u(vt,"class","relative group"),u(jl,"href","https://github.com/huggingface/transformers/tree/main/tests/trainer/test_trainer_distributed.py"),u(jl,"rel","nofollow"),u(Pl,"href","https://github.com/huggingface/transformers/tree/main/tests/deepspeed/test_deepspeed.py"),u(Pl,"rel","nofollow"),u(qs,"id","output-capture"),u(qs,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),u(qs,"href","#output-capture"),u(wt,"class","relative group"),u(Ss,"id","color-control"),u(Ss,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),u(Ss,"href","#color-control"),u($t,"class","relative group"),u(Ls,"id","sending-test-report-to-online-pastebin-service"),u(Ls,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),u(Ls,"href","#sending-test-report-to-online-pastebin-service"),u(Et,"class","relative group"),u(Us,"id","writing-tests"),u(Us,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),u(Us,"href","#writing-tests"),u(gt,"class","relative group"),u(Hl,"href","https://docs.pytest.org/en/stable/unittest.html"),u(Hl,"rel","nofollow"),u(Ns,"id","parametrization"),u(Ns,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),u(Ns,"href","#parametrization"),u(bt,"class","relative group"),u(Yl,"href","https://pypi.org/project/parameterized/"),u(Yl,"rel","nofollow"),u(Gs,"id","files-and-directories"),u(Gs,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),u(Gs,"href","#files-and-directories"),u(kt,"class","relative group"),u(Vs,"id","temporary-files-and-directories"),u(Vs,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),u(Vs,"href","#temporary-files-and-directories"),u(xt,"class","relative group"),u(Qs,"id","temporary-syspath-override"),u(Qs,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),u(Qs,"href","#temporary-syspath-override"),u(jt,"class","relative group"),u(eo,"id","skipping-tests"),u(eo,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),u(eo,"href","#skipping-tests"),u(Pt,"class","relative group"),u(oo,"id","implementation"),u(oo,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),u(oo,"href","#implementation"),u(Ot,"class","relative group"),u(ja,"href","https://docs.pytest.org/en/latest/skipping.html"),u(ja,"rel","nofollow"),u(io,"id","slow-tests"),u(io,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),u(io,"href","#slow-tests"),u(It,"class","relative group"),u(Da,"href","https://github.com/huggingface/transformers/tree/main/scripts/fsmt/fsmt-make-tiny-model.py"),u(Da,"rel","nofollow"),u(qa,"href","https://huggingface.co/stas/tiny-wmt19-en-de"),u(qa,"rel","nofollow"),u(po,"id","testing-the-stdoutstderr-output"),u(po,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),u(po,"href","#testing-the-stdoutstderr-output"),u(At,"class","relative group"),u(La,"href","https://docs.pytest.org/en/latest/capture.html"),u(La,"rel","nofollow"),u(mo,"id","capturing-logger-stream"),u(mo,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),u(mo,"href","#capturing-logger-stream"),u(Dt,"class","relative group"),u(vo,"id","testing-with-environment-variables"),u(vo,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),u(vo,"href","#testing-with-environment-variables"),u(qt,"class","relative group"),u(wo,"id","getting-reproducible-results"),u(wo,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),u(wo,"href","#getting-reproducible-results"),u(St,"class","relative group"),u($o,"id","debugging-tests"),u($o,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),u($o,"href","#debugging-tests"),u(Lt,"class","relative group"),u(Eo,"id","working-with-github-actions-workflows"),u(Eo,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),u(Eo,"href","#working-with-github-actions-workflows"),u(Ut,"class","relative group"),u(si,"href","https://github.com/huggingface/transformers/blob/main/.github/workflows/self-push.yml"),u(si,"rel","nofollow"),u(ri,"href","https://github.com/huggingface/transformers/actions/workflows/self-push.yml"),u(ri,"rel","nofollow"),u(go,"id","testing-experimental-ci-features"),u(go,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),u(go,"href","#testing-experimental-ci-features"),u(Nt,"class","relative group"),u(ni,"href","https://github.com/actions/toolkit/issues/399"),u(ni,"rel","nofollow"),u(pi,"href","https://ideas.circleci.com/ideas/CCI-I-344"),u(pi,"rel","nofollow")},m(e,n){t(document.head,$),p(e,O,n),p(e,g,n),t(g,x),t(x,A),m(C,A,null),t(g,R),t(g,D),t(D,P),p(e,b,n),p(e,S,n),t(S,z),p(e,Rt,n),p(e,we,n),t(we,Ye),p(e,q,n),p(e,me,n),t(me,Ve),t(Ve,zt),t(zt,ci),t(Ve,hg),t(me,ug),t(me,mi),t(mi,Cp),t(Cp,dg),t(mi,cg),p(e,Dm,n),p(e,Xe,n),t(Xe,Gt),t(Gt,jp),m(Ro,jp,null),t(Xe,mg),t(Xe,Pp),t(Pp,_g),p(e,qm,n),p(e,Ft,n),t(Ft,zo),t(zo,Go),t(Go,vg),t(Go,Fo),t(Fo,yg),t(Go,wg),t(zo,$g),t(zo,Wo),t(Wo,Eg),t(Wo,Op),t(Op,gg),t(Wo,bg),t(Ft,kg),t(Ft,Mo),t(Mo,Bo),t(Bo,xg),t(Bo,Yo),t(Yo,Cg),t(Bo,jg),t(Mo,Pg),t(Mo,Je),t(Je,Ip),t(Ip,_i),t(_i,Vo),t(Vo,Og),t(_i,Ig),t(Je,Tg),t(Je,Tp),t(Tp,L),t(L,Xo),t(Xo,Ag),t(L,Dg),t(L,Ap),t(Ap,qg),t(L,Sg),t(L,Dp),t(Dp,Lg),t(L,Ug),t(L,qp),t(qp,Ng),t(L,Hg),t(L,Sp),t(Sp,Rg),t(L,zg),t(L,Lp),t(Lp,Gg),t(L,Fg),t(Je,Wg),t(Je,Up),t(Up,$e),t($e,Jo),t(Jo,Mg),t($e,Bg),t($e,Np),t(Np,Yg),t($e,Vg),t($e,Hp),t(Hp,Xg),t($e,Jg),p(e,Sm,n),m(Zo,e,n),p(e,Lm,n),p(e,Wt,n),t(Wt,Zg),t(Wt,Ko),t(Ko,Kg),t(Wt,Qg),p(e,Um,n),p(e,Ze,n),t(Ze,Mt),t(Mt,Rp),m(Qo,Rp,null),t(Ze,eb),t(Ze,zp),t(zp,tb),p(e,Nm,n),p(e,Ke,n),t(Ke,Bt),t(Bt,Gp),m(er,Gp,null),t(Ke,sb),t(Ke,Fp),t(Fp,ob),p(e,Hm,n),p(e,Yt,n),t(Yt,rb),t(Yt,tr),t(tr,lb),t(Yt,ab),p(e,Rm,n),p(e,vi,n),t(vi,ib),p(e,zm,n),p(e,yi,n),t(yi,nb),p(e,Gm,n),m(sr,e,n),p(e,Fm,n),p(e,wi,n),t(wi,pb),p(e,Wm,n),m(or,e,n),p(e,Mm,n),p(e,$i,n),t($i,fb),p(e,Bm,n),m(rr,e,n),p(e,Ym,n),p(e,Ei,n),t(Ei,hb),p(e,Vm,n),p(e,Q,n),t(Q,Wp),t(Wp,ub),t(Q,db),t(Q,Mp),t(Mp,cb),t(Q,mb),t(Q,Bp),t(Bp,_b),t(Q,vb),t(Q,Yp),t(Yp,yb),p(e,Xm,n),p(e,Qe,n),t(Qe,Vt),t(Vt,Vp),m(lr,Vp,null),t(Qe,wb),t(Qe,Xp),t(Xp,$b),p(e,Jm,n),p(e,gi,n),t(gi,Eb),p(e,Zm,n),m(ar,e,n),p(e,Km,n),p(e,bi,n),t(bi,gb),p(e,Qm,n),m(ir,e,n),p(e,e_,n),p(e,et,n),t(et,Xt),t(Xt,Jp),m(nr,Jp,null),t(et,bb),t(et,Zp),t(Zp,kb),p(e,t_,n),p(e,ki,n),t(ki,xb),p(e,s_,n),m(pr,e,n),p(e,o_,n),p(e,tt,n),t(tt,Jt),t(Jt,Kp),m(fr,Kp,null),t(tt,Cb),t(tt,Qp),t(Qp,jb),p(e,r_,n),p(e,xi,n),t(xi,Pb),p(e,l_,n),m(hr,e,n),p(e,a_,n),p(e,Ci,n),t(Ci,Ob),p(e,i_,n),p(e,Ee,n),t(Ee,ji),t(ji,ef),t(ef,Ib),t(ji,Tb),t(Ee,Ab),t(Ee,Pi),t(Pi,tf),t(tf,Db),t(Pi,qb),t(Ee,Sb),t(Ee,Oi),t(Oi,sf),t(sf,Lb),t(Oi,Ub),p(e,n_,n),p(e,Ii,n),t(Ii,Nb),p(e,p_,n),m(ur,e,n),p(e,f_,n),p(e,Ti,n),t(Ti,Hb),p(e,h_,n),p(e,Zt,n),t(Zt,Rb),t(Zt,of),t(of,zb),t(Zt,Gb),p(e,u_,n),m(dr,e,n),p(e,d_,n),p(e,Ai,n),t(Ai,Fb),p(e,c_,n),p(e,Kt,n),t(Kt,Wb),t(Kt,rf),t(rf,Mb),t(Kt,Bb),p(e,m_,n),m(cr,e,n),p(e,__,n),p(e,ee,n),t(ee,Yb),t(ee,lf),t(lf,Vb),t(ee,Xb),t(ee,af),t(af,Jb),t(ee,Zb),t(ee,nf),t(nf,Kb),t(ee,Qb),p(e,v_,n),p(e,Qt,n),t(Qt,e3),t(Qt,pf),t(pf,t3),t(Qt,s3),p(e,y_,n),m(mr,e,n),p(e,w_,n),p(e,Di,n),t(Di,o3),p(e,$_,n),m(_r,e,n),p(e,E_,n),p(e,ge,n),t(ge,r3),t(ge,ff),t(ff,l3),t(ge,a3),t(ge,hf),t(hf,i3),t(ge,n3),p(e,g_,n),m(vr,e,n),p(e,b_,n),p(e,es,n),t(es,p3),t(es,uf),t(uf,f3),t(es,h3),p(e,k_,n),p(e,ts,n),t(ts,u3),t(ts,df),t(df,d3),t(ts,c3),p(e,x_,n),m(yr,e,n),p(e,C_,n),p(e,st,n),t(st,ss),t(ss,cf),m(wr,cf,null),t(st,m3),t(st,mf),t(mf,_3),p(e,j_,n),p(e,be,n),t(be,v3),t(be,_f),t(_f,y3),t(be,w3),t(be,os),t(os,vf),t(vf,$3),t(os,E3),t(be,g3),p(e,P_,n),m($r,e,n),p(e,O_,n),p(e,qi,n),t(qi,b3),p(e,I_,n),p(e,Si,n),t(Si,Er),t(Er,k3),t(Er,yf),t(yf,x3),t(Er,C3),p(e,T_,n),m(gr,e,n),p(e,A_,n),p(e,br,n),t(br,wf),t(wf,j3),p(e,D_,n),m(kr,e,n),p(e,q_,n),p(e,xr,n),t(xr,Cr),t(Cr,P3),t(Cr,$f),t($f,O3),t(Cr,I3),p(e,S_,n),m(jr,e,n),p(e,L_,n),p(e,ot,n),t(ot,rs),t(rs,Ef),m(Pr,Ef,null),t(ot,T3),t(ot,gf),t(gf,A3),p(e,U_,n),p(e,ls,n),t(ls,D3),t(ls,Or),t(Or,q3),t(ls,S3),p(e,N_,n),m(Ir,e,n),p(e,H_,n),m(Tr,e,n),p(e,R_,n),p(e,Li,n),t(Li,L3),p(e,z_,n),p(e,rt,n),t(rt,as),t(as,bf),m(Ar,bf,null),t(rt,U3),t(rt,kf),t(kf,N3),p(e,G_,n),p(e,Dr,n),t(Dr,qr),t(qr,H3),t(Dr,R3),p(e,F_,n),m(Sr,e,n),p(e,W_,n),p(e,lt,n),t(lt,z3),t(lt,xf),t(xf,G3),t(lt,F3),t(lt,Cf),t(Cf,W3),p(e,M_,n),p(e,ke,n),t(ke,M3),t(ke,jf),t(jf,B3),t(ke,Y3),t(ke,Pf),t(Pf,V3),t(ke,X3),p(e,B_,n),m(Lr,e,n),p(e,Y_,n),p(e,xe,n),t(xe,J3),t(xe,Of),t(Of,Z3),t(xe,K3),t(xe,If),t(If,Q3),t(xe,e2),p(e,V_,n),m(Ur,e,n),p(e,X_,n),p(e,Ui,n),t(Ui,t2),p(e,J_,n),p(e,Nr,n),t(Nr,Hr),t(Hr,s2),t(Nr,o2),p(e,Z_,n),p(e,at,n),t(at,is),t(is,Tf),m(Rr,Tf,null),t(at,r2),t(at,Af),t(Af,l2),p(e,K_,n),p(e,ns,n),t(ns,a2),t(ns,Df),t(Df,i2),t(ns,n2),p(e,Q_,n),m(zr,e,n),p(e,ev,n),p(e,it,n),t(it,ps),t(ps,qf),m(Gr,qf,null),t(it,p2),t(it,Sf),t(Sf,f2),p(e,tv,n),p(e,Ni,n),t(Ni,h2),p(e,sv,n),m(Fr,e,n),p(e,ov,n),p(e,nt,n),t(nt,fs),t(fs,Lf),m(Wr,Lf,null),t(nt,u2),t(nt,Uf),t(Uf,d2),p(e,rv,n),p(e,W,n),t(W,c2),t(W,Nf),t(Nf,m2),t(W,_2),t(W,Hf),t(Hf,v2),t(W,y2),t(W,Rf),t(Rf,w2),t(W,$2),t(W,zf),t(zf,E2),t(W,g2),p(e,lv,n),p(e,_e,n),t(_e,Gf),t(Gf,b2),t(_e,k2),t(_e,Ff),t(Ff,x2),t(_e,C2),t(_e,Wf),t(Wf,j2),t(_e,P2),p(e,av,n),p(e,Ce,n),t(Ce,O2),t(Ce,Mf),t(Mf,I2),t(Ce,T2),t(Ce,Mr),t(Mr,A2),t(Ce,D2),p(e,iv,n),p(e,pt,n),t(pt,hs),t(hs,Bf),m(Br,Bf,null),t(pt,q2),t(pt,Yf),t(Yf,S2),p(e,nv,n),p(e,Hi,n),t(Hi,L2),p(e,pv,n),p(e,ft,n),t(ft,us),t(us,Vf),m(Yr,Vf,null),t(ft,U2),t(ft,Xf),t(Xf,N2),p(e,fv,n),p(e,Ri,n),t(Ri,zi),t(zi,Vr),t(Vr,H2),t(zi,R2),p(e,hv,n),m(Xr,e,n),p(e,uv,n),p(e,Gi,n),t(Gi,z2),p(e,dv,n),m(Jr,e,n),p(e,cv,n),m(ds,e,n),p(e,mv,n),m(cs,e,n),p(e,_v,n),p(e,ht,n),t(ht,ms),t(ms,Jf),m(Zr,Jf,null),t(ht,G2),t(ht,Zf),t(Zf,F2),p(e,vv,n),m(Kr,e,n),p(e,yv,n),p(e,_s,n),t(_s,W2),t(_s,Kf),t(Kf,M2),t(_s,B2),p(e,wv,n),p(e,vs,n),t(vs,Y2),t(vs,Qf),t(Qf,V2),t(vs,X2),p(e,$v,n),m(Qr,e,n),p(e,Ev,n),p(e,Fi,n),t(Fi,J2),p(e,gv,n),m(el,e,n),p(e,bv,n),p(e,ys,n),t(ys,Z2),t(ys,eh),t(eh,K2),t(ys,Q2),p(e,kv,n),m(tl,e,n),p(e,xv,n),p(e,Wi,n),t(Wi,e5),p(e,Cv,n),m(sl,e,n),p(e,jv,n),p(e,I,n),t(I,t5),t(I,th),t(th,s5),t(I,o5),t(I,sh),t(sh,r5),t(I,l5),t(I,oh),t(oh,a5),t(I,i5),t(I,rh),t(rh,n5),t(I,p5),t(I,lh),t(lh,f5),t(I,h5),t(I,ol),t(ol,u5),t(I,d5),p(e,Pv,n),p(e,je,n),t(je,c5),t(je,rl),t(rl,ah),t(ah,m5),t(je,_5),t(je,ih),t(ih,v5),t(je,y5),p(e,Ov,n),p(e,ut,n),t(ut,ws),t(ws,nh),m(ll,nh,null),t(ut,w5),t(ut,ph),t(ph,$5),p(e,Iv,n),p(e,dt,n),t(dt,$s),t($s,fh),m(al,fh,null),t(dt,E5),t(dt,hh),t(hh,g5),p(e,Tv,n),p(e,il,n),t(il,nl),t(nl,b5),t(il,k5),p(e,Av,n),m(pl,e,n),p(e,Dv,n),p(e,Mi,n),t(Mi,x5),p(e,qv,n),m(fl,e,n),p(e,Sv,n),p(e,Bi,n),t(Bi,C5),p(e,Lv,n),p(e,ct,n),t(ct,Es),t(Es,uh),m(hl,uh,null),t(ct,j5),t(ct,dh),t(dh,P5),p(e,Uv,n),p(e,Pe,n),t(Pe,O5),t(Pe,ch),t(ch,I5),t(Pe,T5),t(Pe,mh),t(mh,A5),t(Pe,D5),p(e,Nv,n),m(ul,e,n),p(e,Hv,n),p(e,mt,n),t(mt,gs),t(gs,_h),m(dl,_h,null),t(mt,q5),t(mt,vh),t(vh,S5),p(e,Rv,n),p(e,cl,n),t(cl,ml),t(ml,L5),t(cl,U5),p(e,zv,n),m(_l,e,n),p(e,Gv,n),m(vl,e,n),p(e,Fv,n),p(e,_t,n),t(_t,bs),t(bs,yh),m(yl,yh,null),t(_t,N5),t(_t,wh),t(wh,H5),p(e,Wv,n),p(e,ks,n),t(ks,R5),t(ks,$h),t($h,z5),t(ks,G5),p(e,Mv,n),m(wl,e,n),p(e,Bv,n),p(e,te,n),t(te,F5),t(te,Eh),t(Eh,W5),t(te,M5),t(te,gh),t(gh,B5),t(te,Y5),t(te,bh),t(bh,V5),t(te,X5),p(e,Yv,n),m($l,e,n),p(e,Vv,n),p(e,Yi,n),t(Yi,J5),p(e,Xv,n),p(e,Vi,n),t(Vi,Z5),p(e,Jv,n),p(e,U,n),t(U,Xi),t(Xi,kh),t(kh,K5),t(Xi,Q5),t(U,ek),t(U,xs),t(xs,xh),t(xh,tk),t(xs,sk),t(xs,Ch),t(Ch,ok),t(xs,rk),t(U,lk),t(U,Cs),t(Cs,jh),t(jh,ak),t(Cs,ik),t(Cs,Ph),t(Ph,nk),t(Cs,pk),t(U,fk),t(U,js),t(js,Oh),t(Oh,hk),t(js,uk),t(js,Ih),t(Ih,dk),t(js,ck),t(U,mk),t(U,Ps),t(Ps,Th),t(Th,_k),t(Ps,vk),t(Ps,Ah),t(Ah,yk),t(Ps,wk),t(U,$k),t(U,Os),t(Os,Dh),t(Dh,Ek),t(Os,gk),t(Os,qh),t(qh,bk),t(Os,kk),p(e,Zv,n),p(e,Ji,n),t(Ji,xk),p(e,Kv,n),p(e,k,n),t(k,Ck),t(k,Sh),t(Sh,jk),t(k,Pk),t(k,Lh),t(Lh,Ok),t(k,Ik),t(k,Uh),t(Uh,Tk),t(k,Ak),t(k,Nh),t(Nh,Dk),t(k,qk),t(k,Hh),t(Hh,Sk),t(k,Lk),t(k,Rh),t(Rh,Uk),t(k,Nk),t(k,zh),t(zh,Hk),t(k,Rk),t(k,Gh),t(Gh,zk),t(k,Gk),t(k,Fh),t(Fh,Fk),t(k,Wk),t(k,Wh),t(Wh,Mk),t(k,Bk),p(e,Qv,n),p(e,Zi,n),t(Zi,Yk),p(e,e1,n),m(El,e,n),p(e,t1,n),p(e,Oe,n),t(Oe,Vk),t(Oe,Mh),t(Mh,Xk),t(Oe,Jk),t(Oe,Bh),t(Bh,Zk),t(Oe,Kk),p(e,s1,n),m(gl,e,n),p(e,o1,n),p(e,Ki,n),t(Ki,Qk),p(e,r1,n),m(bl,e,n),p(e,l1,n),p(e,Ie,n),t(Ie,e0),t(Ie,Yh),t(Yh,t0),t(Ie,s0),t(Ie,Vh),t(Vh,o0),t(Ie,r0),p(e,a1,n),m(kl,e,n),p(e,i1,n),p(e,Is,n),t(Is,l0),t(Is,Xh),t(Xh,a0),t(Is,i0),p(e,n1,n),p(e,Qi,n),t(Qi,n0),p(e,p1,n),p(e,en,n),t(en,Jh),t(Jh,p0),p(e,f1,n),m(xl,e,n),p(e,h1,n),p(e,vt,n),t(vt,Ts),t(Ts,Zh),m(Cl,Zh,null),t(vt,f0),t(vt,Kh),t(Kh,h0),p(e,u1,n),p(e,yt,n),t(yt,Qh),t(Qh,u0),t(yt,d0),t(yt,eu),t(eu,c0),t(yt,m0),p(e,d1,n),p(e,tn,n),t(tn,_0),p(e,c1,n),p(e,As,n),t(As,tu),t(tu,jl),t(jl,v0),t(As,y0),t(As,su),t(su,Pl),t(Pl,w0),p(e,m1,n),p(e,Ds,n),t(Ds,$0),t(Ds,ou),t(ou,E0),t(Ds,g0),p(e,_1,n),p(e,sn,n),t(sn,b0),p(e,v1,n),m(Ol,e,n),p(e,y1,n),p(e,wt,n),t(wt,qs),t(qs,ru),m(Il,ru,null),t(wt,k0),t(wt,lu),t(lu,x0),p(e,w1,n),p(e,Te,n),t(Te,C0),t(Te,au),t(au,j0),t(Te,P0),t(Te,iu),t(iu,O0),t(Te,I0),p(e,$1,n),p(e,M,n),t(M,T0),t(M,nu),t(nu,A0),t(M,D0),t(M,pu),t(pu,q0),t(M,S0),t(M,fu),t(fu,L0),t(M,U0),t(M,hu),t(hu,N0),t(M,H0),p(e,E1,n),m(Tl,e,n),p(e,g1,n),p(e,on,n),t(on,R0),p(e,b1,n),m(Al,e,n),p(e,k1,n),p(e,$t,n),t($t,Ss),t(Ss,uu),m(Dl,uu,null),t($t,z0),t($t,du),t(du,G0),p(e,x1,n),p(e,rn,n),t(rn,F0),p(e,C1,n),m(ql,e,n),p(e,j1,n),p(e,Et,n),t(Et,Ls),t(Ls,cu),m(Sl,cu,null),t(Et,W0),t(Et,mu),t(mu,M0),p(e,P1,n),p(e,ln,n),t(ln,B0),p(e,O1,n),m(Ll,e,n),p(e,I1,n),p(e,an,n),t(an,Y0),p(e,T1,n),p(e,nn,n),t(nn,V0),p(e,A1,n),m(Ul,e,n),p(e,D1,n),p(e,gt,n),t(gt,Us),t(Us,_u),m(Nl,_u,null),t(gt,X0),t(gt,vu),t(vu,J0),p(e,q1,n),p(e,Ae,n),t(Ae,Z0),t(Ae,yu),t(yu,K0),t(Ae,Q0),t(Ae,wu),t(wu,e4),t(Ae,t4),p(e,S1,n),p(e,se,n),t(se,s4),t(se,Hl),t(Hl,o4),t(se,r4),t(se,$u),t($u,l4),t(se,a4),t(se,Eu),t(Eu,i4),t(se,n4),p(e,L1,n),p(e,bt,n),t(bt,Ns),t(Ns,gu),m(Rl,gu,null),t(bt,p4),t(bt,bu),t(bu,f4),p(e,U1,n),p(e,pn,n),t(pn,h4),p(e,N1,n),m(zl,e,n),p(e,H1,n),p(e,Hs,n),t(Hs,u4),t(Hs,ku),t(ku,d4),t(Hs,c4),p(e,R1,n),p(e,De,n),t(De,m4),t(De,xu),t(xu,_4),t(De,v4),t(De,Cu),t(Cu,y4),t(De,w4),p(e,z1,n),m(Gl,e,n),p(e,G1,n),p(e,Rs,n),t(Rs,$4),t(Rs,ju),t(ju,E4),t(Rs,g4),p(e,F1,n),m(Fl,e,n),p(e,W1,n),p(e,zs,n),t(zs,b4),t(zs,Pu),t(Pu,k4),t(zs,x4),p(e,M1,n),m(Wl,e,n),p(e,B1,n),p(e,fn,n),t(fn,C4),p(e,Y1,n),m(Ml,e,n),p(e,V1,n),p(e,hn,n),t(hn,j4),p(e,X1,n),m(Bl,e,n),p(e,J1,n),p(e,B,n),t(B,P4),t(B,Yl),t(Yl,O4),t(B,I4),t(B,Ou),t(Ou,T4),t(B,A4),t(B,Iu),t(Iu,D4),t(B,q4),t(B,Tu),t(Tu,S4),t(B,L4),p(e,Z1,n),p(e,oe,n),t(oe,U4),t(oe,Au),t(Au,N4),t(oe,H4),t(oe,Du),t(Du,R4),t(oe,z4),t(oe,qu),t(qu,G4),t(oe,F4),p(e,K1,n),p(e,qe,n),t(qe,W4),t(qe,Su),t(Su,M4),t(qe,B4),t(qe,Lu),t(Lu,Y4),t(qe,V4),p(e,Q1,n),m(Vl,e,n),p(e,ey,n),p(e,re,n),t(re,X4),t(re,Uu),t(Uu,J4),t(re,Z4),t(re,Nu),t(Nu,K4),t(re,Q4),t(re,Hu),t(Hu,e6),t(re,t6),p(e,ty,n),m(Xl,e,n),p(e,sy,n),p(e,un,n),t(un,s6),p(e,oy,n),m(Jl,e,n),p(e,ry,n),p(e,dn,n),t(dn,o6),p(e,ly,n),m(Zl,e,n),p(e,ay,n),p(e,cn,n),t(cn,r6),p(e,iy,n),p(e,kt,n),t(kt,Gs),t(Gs,Ru),m(Kl,Ru,null),t(kt,l6),t(kt,zu),t(zu,a6),p(e,ny,n),p(e,Fs,n),t(Fs,i6),t(Fs,Gu),t(Gu,n6),t(Fs,p6),p(e,py,n),p(e,Ws,n),t(Ws,Ql),t(Ql,mn),t(mn,Fu),t(Fu,f6),t(mn,h6),t(Ql,u6),t(Ql,G),t(G,ea),t(ea,Wu),t(Wu,d6),t(ea,c6),t(ea,Mu),t(Mu,m6),t(G,_6),t(G,_n),t(_n,Bu),t(Bu,v6),t(_n,y6),t(G,w6),t(G,Ms),t(Ms,Yu),t(Yu,$6),t(Ms,E6),t(Ms,Vu),t(Vu,g6),t(Ms,b6),t(G,k6),t(G,Bs),t(Bs,Xu),t(Xu,x6),t(Bs,C6),t(Bs,Ju),t(Ju,j6),t(Bs,P6),t(G,O6),t(G,vn),t(vn,Zu),t(Zu,I6),t(vn,T6),t(G,A6),t(G,Se),t(Se,Ku),t(Ku,D6),t(Se,q6),t(Se,Qu),t(Qu,S6),t(Se,L6),t(Se,ed),t(ed,U6),t(Se,N6),t(Ws,H6),t(Ws,ta),t(ta,sa),t(sa,R6),t(sa,td),t(td,z6),t(sa,G6),t(ta,F6),t(ta,F),t(F,sd),t(sd,od),t(od,W6),t(F,M6),t(F,rd),t(rd,ld),t(ld,B6),t(F,Y6),t(F,ad),t(ad,id),t(id,V6),t(F,X6),t(F,nd),t(nd,pd),t(pd,J6),t(F,Z6),t(F,fd),t(fd,hd),t(hd,K6),t(F,Q6),t(F,ud),t(ud,dd),t(dd,e7),p(e,fy,n),p(e,Ys,n),t(Ys,t7),t(Ys,cd),t(cd,s7),t(Ys,o7),p(e,hy,n),m(oa,e,n),p(e,uy,n),p(e,Y,n),t(Y,r7),t(Y,md),t(md,l7),t(Y,a7),t(Y,_d),t(_d,i7),t(Y,n7),t(Y,vd),t(vd,p7),t(Y,f7),t(Y,yd),t(yd,h7),t(Y,u7),p(e,dy,n),m(ra,e,n),p(e,cy,n),p(e,xt,n),t(xt,Vs),t(Vs,wd),m(la,wd,null),t(xt,d7),t(xt,$d),t($d,c7),p(e,my,n),p(e,Xs,n),t(Xs,m7),t(Xs,Ed),t(Ed,_7),t(Xs,v7),p(e,_y,n),p(e,yn,n),t(yn,y7),p(e,vy,n),p(e,Le,n),t(Le,w7),t(Le,gd),t(gd,$7),t(Le,E7),t(Le,bd),t(bd,g7),t(Le,b7),p(e,yy,n),p(e,wn,n),t(wn,k7),p(e,wy,n),m(aa,e,n),p(e,$y,n),p(e,Js,n),t(Js,x7),t(Js,kd),t(kd,C7),t(Js,j7),p(e,Ey,n),p(e,$n,n),t($n,xd),t(xd,P7),p(e,gy,n),m(ia,e,n),p(e,by,n),p(e,na,n),t(na,Cd),t(Cd,O7),t(na,I7),p(e,ky,n),p(e,En,n),t(En,jd),t(jd,T7),p(e,xy,n),m(pa,e,n),p(e,Cy,n),p(e,gn,n),t(gn,A7),p(e,jy,n),p(e,bn,n),t(bn,fa),t(fa,Ct),t(Ct,D7),t(Ct,Pd),t(Pd,q7),t(Ct,S7),t(Ct,Od),t(Od,L7),t(Ct,U7),t(fa,N7),t(fa,ve),t(ve,kn),t(kn,Id),t(Id,H7),t(kn,R7),t(ve,z7),t(ve,xn),t(xn,Td),t(Td,G7),t(xn,F7),t(ve,W7),t(ve,Cn),t(Cn,Ad),t(Ad,M7),t(Cn,B7),t(ve,Y7),t(ve,jn),t(jn,Dd),t(Dd,V7),t(jn,X7),p(e,Py,n),m(Zs,e,n),p(e,Oy,n),m(Ks,e,n),p(e,Iy,n),p(e,jt,n),t(jt,Qs),t(Qs,qd),m(ha,qd,null),t(jt,J7),t(jt,Sd),t(Sd,Z7),p(e,Ty,n),p(e,Ue,n),t(Ue,K7),t(Ue,Ld),t(Ld,Q7),t(Ue,e8),t(Ue,Ud),t(Ud,t8),t(Ue,s8),p(e,Ay,n),m(ua,e,n),p(e,Dy,n),p(e,Pt,n),t(Pt,eo),t(eo,Nd),m(da,Nd,null),t(Pt,o8),t(Pt,Hd),t(Hd,r8),p(e,qy,n),p(e,to,n),t(to,l8),t(to,Rd),t(Rd,a8),t(to,i8),p(e,Sy,n),p(e,Pn,n),t(Pn,n8),p(e,Ly,n),p(e,so,n),t(so,zd),t(zd,ca),t(ca,p8),t(ca,Gd),t(Gd,f8),t(ca,h8),t(so,u8),t(so,Fd),t(Fd,ma),t(ma,d8),t(ma,Wd),t(Wd,c8),t(ma,m8),p(e,Uy,n),p(e,le,n),t(le,_8),t(le,Md),t(Md,v8),t(le,y8),t(le,Bd),t(Bd,w8),t(le,$8),t(le,Yd),t(Yd,E8),t(le,g8),p(e,Ny,n),p(e,Ot,n),t(Ot,oo),t(oo,Vd),m(_a,Vd,null),t(Ot,b8),t(Ot,Xd),t(Xd,k8),p(e,Hy,n),p(e,On,n),t(On,Jd),t(Jd,x8),p(e,Ry,n),m(va,e,n),p(e,zy,n),p(e,In,n),t(In,C8),p(e,Gy,n),m(ya,e,n),p(e,Fy,n),p(e,ro,n),t(ro,j8),t(ro,Zd),t(Zd,P8),t(ro,O8),p(e,Wy,n),m(wa,e,n),p(e,My,n),p(e,Tn,n),t(Tn,Kd),t(Kd,I8),p(e,By,n),m($a,e,n),p(e,Yy,n),p(e,An,n),t(An,T8),p(e,Vy,n),m(Ea,e,n),p(e,Xy,n),p(e,lo,n),t(lo,A8),t(lo,Qd),t(Qd,D8),t(lo,q8),p(e,Jy,n),m(ga,e,n),p(e,Zy,n),p(e,Dn,n),t(Dn,ec),t(ec,S8),p(e,Ky,n),m(ba,e,n),p(e,Qy,n),p(e,qn,n),t(qn,tc),t(tc,L8),p(e,ew,n),m(ka,e,n),p(e,tw,n),p(e,Sn,n),t(Sn,U8),p(e,sw,n),m(xa,e,n),p(e,ow,n),p(e,Ln,n),t(Ln,N8),p(e,rw,n),m(Ca,e,n),p(e,lw,n),p(e,ao,n),t(ao,H8),t(ao,ja),t(ja,R8),t(ao,z8),p(e,aw,n),p(e,It,n),t(It,io),t(io,sc),m(Pa,sc,null),t(It,G8),t(It,oc),t(oc,F8),p(e,iw,n),p(e,Un,n),t(Un,W8),p(e,nw,n),m(Oa,e,n),p(e,pw,n),p(e,Ne,n),t(Ne,M8),t(Ne,rc),t(rc,B8),t(Ne,Y8),t(Ne,lc),t(lc,V8),t(Ne,X8),p(e,fw,n),m(Ia,e,n),p(e,hw,n),p(e,ae,n),t(ae,J8),t(ae,ac),t(ac,Z8),t(ae,K8),t(ae,ic),t(ic,Q8),t(ae,e9),t(ae,nc),t(nc,t9),t(ae,s9),p(e,uw,n),m(Ta,e,n),p(e,dw,n),p(e,Nn,n),t(Nn,o9),p(e,cw,n),p(e,Hn,n),t(Hn,r9),p(e,mw,n),p(e,Rn,n),t(Rn,l9),p(e,_w,n),p(e,ie,n),t(ie,pc),t(pc,a9),t(ie,i9),t(ie,fc),t(fc,n9),t(ie,p9),t(ie,Tt),t(Tt,f9),t(Tt,hc),t(hc,h9),t(Tt,u9),t(Tt,uc),t(uc,d9),t(Tt,c9),t(ie,m9),t(ie,dc),t(dc,_9),p(e,vw,n),p(e,He,n),t(He,v9),t(He,cc),t(cc,y9),t(He,w9),t(He,mc),t(mc,$9),t(He,E9),p(e,yw,n),m(Aa,e,n),p(e,ww,n),p(e,Re,n),t(Re,g9),t(Re,Da),t(Da,b9),t(Re,k9),t(Re,qa),t(qa,x9),t(Re,C9),p(e,$w,n),p(e,no,n),t(no,j9),t(no,_c),t(_c,P9),t(no,O9),p(e,Ew,n),p(e,zn,n),t(zn,I9),p(e,gw,n),p(e,At,n),t(At,po),t(po,vc),m(Sa,vc,null),t(At,T9),t(At,yc),t(yc,A9),p(e,bw,n),p(e,V,n),t(V,D9),t(V,wc),t(wc,q9),t(V,S9),t(V,$c),t($c,L9),t(V,U9),t(V,Ec),t(Ec,N9),t(V,H9),t(V,La),t(La,R9),t(V,z9),p(e,kw,n),m(Ua,e,n),p(e,xw,n),p(e,fo,n),t(fo,G9),t(fo,gc),t(gc,F9),t(fo,W9),p(e,Cw,n),m(Na,e,n),p(e,jw,n),p(e,ho,n),t(ho,M9),t(ho,bc),t(bc,B9),t(ho,Y9),p(e,Pw,n),m(Ha,e,n),p(e,Ow,n),p(e,T,n),t(T,V9),t(T,kc),t(kc,X9),t(T,J9),t(T,xc),t(xc,Z9),t(T,K9),t(T,Cc),t(Cc,Q9),t(T,ex),t(T,jc),t(jc,tx),t(T,sx),t(T,Pc),t(Pc,ox),t(T,rx),t(T,Oc),t(Oc,lx),t(T,ax),p(e,Iw,n),p(e,uo,n),t(uo,ix),t(uo,Ic),t(Ic,nx),t(uo,px),p(e,Tw,n),m(Ra,e,n),p(e,Aw,n),p(e,Gn,n),t(Gn,fx),p(e,Dw,n),m(za,e,n),p(e,qw,n),p(e,ze,n),t(ze,hx),t(ze,Tc),t(Tc,ux),t(ze,dx),t(ze,Ac),t(Ac,cx),t(ze,mx),p(e,Sw,n),m(Ga,e,n),p(e,Lw,n),p(e,co,n),t(co,_x),t(co,Dc),t(Dc,vx),t(co,yx),p(e,Uw,n),m(Fa,e,n),p(e,Nw,n),p(e,Fn,n),t(Fn,wx),p(e,Hw,n),p(e,Dt,n),t(Dt,mo),t(mo,qc),m(Wa,qc,null),t(Dt,$x),t(Dt,Sc),t(Sc,Ex),p(e,Rw,n),p(e,_o,n),t(_o,gx),t(_o,Lc),t(Lc,bx),t(_o,kx),p(e,zw,n),m(Ma,e,n),p(e,Gw,n),p(e,qt,n),t(qt,vo),t(vo,Uc),m(Ba,Uc,null),t(qt,xx),t(qt,Nc),t(Nc,Cx),p(e,Fw,n),p(e,Ya,n),t(Ya,jx),t(Ya,Hc),t(Hc,Px),p(e,Ww,n),m(Va,e,n),p(e,Mw,n),p(e,ne,n),t(ne,Ox),t(ne,Rc),t(Rc,Ix),t(ne,Tx),t(ne,zc),t(zc,Ax),t(ne,Dx),t(ne,Gc),t(Gc,qx),t(ne,Sx),p(e,Bw,n),m(Xa,e,n),p(e,Yw,n),p(e,N,n),t(N,Lx),t(N,Fc),t(Fc,Ux),t(N,Nx),t(N,Wc),t(Wc,Hx),t(N,Rx),t(N,Mc),t(Mc,zx),t(N,Gx),t(N,Bc),t(Bc,Fx),t(N,Wx),t(N,Yc),t(Yc,Mx),t(N,Bx),p(e,Vw,n),p(e,yo,n),t(yo,Yx),t(yo,Vc),t(Vc,Vx),t(yo,Xx),p(e,Xw,n),p(e,St,n),t(St,wo),t(wo,Xc),m(Ja,Xc,null),t(St,Jx),t(St,Jc),t(Jc,Zx),p(e,Jw,n),p(e,Wn,n),t(Wn,Kx),p(e,Zw,n),m(Za,e,n),p(e,Kw,n),p(e,Lt,n),t(Lt,$o),t($o,Zc),m(Ka,Zc,null),t(Lt,Qx),t(Lt,Kc),t(Kc,eC),p(e,Qw,n),p(e,Mn,n),t(Mn,tC),p(e,e$,n),m(Qa,e,n),p(e,t$,n),p(e,Ut,n),t(Ut,Eo),t(Eo,Qc),m(ei,Qc,null),t(Ut,sC),t(Ut,em),t(em,oC),p(e,s$,n),p(e,Bn,n),t(Bn,rC),p(e,o$,n),p(e,pe,n),t(pe,ti),t(ti,lC),t(ti,tm),t(tm,aC),t(ti,iC),t(pe,nC),t(pe,H),t(H,pC),t(H,sm),t(sm,fC),t(H,hC),t(H,om),t(om,uC),t(H,dC),t(H,rm),t(rm,cC),t(H,mC),t(H,lm),t(lm,_C),t(H,vC),t(H,si),t(si,yC),t(H,wC),t(H,am),t(am,$C),t(pe,EC),t(pe,im),t(im,gC),t(pe,bC),t(pe,oi),t(oi,kC),t(oi,ri),t(ri,xC),t(oi,CC),p(e,r$,n),p(e,Nt,n),t(Nt,go),t(go,nm),m(li,nm,null),t(Nt,jC),t(Nt,pm),t(pm,PC),p(e,l$,n),p(e,Yn,n),t(Yn,OC),p(e,a$,n),p(e,fe,n),t(fe,fm),t(fm,IC),t(fe,TC),t(fe,hm),t(hm,AC),t(fe,DC),t(fe,um),t(um,qC),t(fe,SC),t(fe,dm),t(dm,LC),p(e,i$,n),p(e,Vn,n),t(Vn,UC),p(e,n$,n),p(e,Xn,n),t(Xn,NC),p(e,p$,n),p(e,Jn,n),t(Jn,HC),p(e,f$,n),p(e,Zn,n),t(Zn,RC),p(e,h$,n),p(e,bo,n),t(bo,Kn),t(Kn,cm),t(cm,zC),t(Kn,GC),t(bo,FC),t(bo,Ht),t(Ht,WC),t(Ht,mm),t(mm,MC),t(Ht,BC),t(Ht,_m),t(_m,YC),t(Ht,VC),p(e,u$,n),p(e,Qn,n),t(Qn,XC),p(e,d$,n),m(ai,e,n),p(e,c$,n),p(e,ep,n),t(ep,JC),p(e,m$,n),m(ii,e,n),p(e,_$,n),p(e,ko,n),t(ko,ZC),t(ko,vm),t(vm,KC),t(ko,QC),p(e,v$,n),p(e,xo,n),t(xo,ej),t(xo,ym),t(ym,tj),t(xo,sj),p(e,y$,n),p(e,tp,n),t(tp,oj),p(e,w$,n),p(e,Co,n),t(Co,wm),t(wm,ni),t(ni,rj),t(Co,lj),t(Co,$m),t($m,pi),t(pi,aj),$$=!0},p(e,[n]){const fi={};n&2&&(fi.$$scope={dirty:n,ctx:e}),ds.$set(fi);const Em={};n&2&&(Em.$$scope={dirty:n,ctx:e}),cs.$set(Em);const gm={};n&2&&(gm.$$scope={dirty:n,ctx:e}),Zs.$set(gm);const bm={};n&2&&(bm.$$scope={dirty:n,ctx:e}),Ks.$set(bm)},i(e){$$||(_(C.$$.fragment,e),_(Ro.$$.fragment,e),_(Zo.$$.fragment,e),_(Qo.$$.fragment,e),_(er.$$.fragment,e),_(sr.$$.fragment,e),_(or.$$.fragment,e),_(rr.$$.fragment,e),_(lr.$$.fragment,e),_(ar.$$.fragment,e),_(ir.$$.fragment,e),_(nr.$$.fragment,e),_(pr.$$.fragment,e),_(fr.$$.fragment,e),_(hr.$$.fragment,e),_(ur.$$.fragment,e),_(dr.$$.fragment,e),_(cr.$$.fragment,e),_(mr.$$.fragment,e),_(_r.$$.fragment,e),_(vr.$$.fragment,e),_(yr.$$.fragment,e),_(wr.$$.fragment,e),_($r.$$.fragment,e),_(gr.$$.fragment,e),_(kr.$$.fragment,e),_(jr.$$.fragment,e),_(Pr.$$.fragment,e),_(Ir.$$.fragment,e),_(Tr.$$.fragment,e),_(Ar.$$.fragment,e),_(Sr.$$.fragment,e),_(Lr.$$.fragment,e),_(Ur.$$.fragment,e),_(Rr.$$.fragment,e),_(zr.$$.fragment,e),_(Gr.$$.fragment,e),_(Fr.$$.fragment,e),_(Wr.$$.fragment,e),_(Br.$$.fragment,e),_(Yr.$$.fragment,e),_(Xr.$$.fragment,e),_(Jr.$$.fragment,e),_(ds.$$.fragment,e),_(cs.$$.fragment,e),_(Zr.$$.fragment,e),_(Kr.$$.fragment,e),_(Qr.$$.fragment,e),_(el.$$.fragment,e),_(tl.$$.fragment,e),_(sl.$$.fragment,e),_(ll.$$.fragment,e),_(al.$$.fragment,e),_(pl.$$.fragment,e),_(fl.$$.fragment,e),_(hl.$$.fragment,e),_(ul.$$.fragment,e),_(dl.$$.fragment,e),_(_l.$$.fragment,e),_(vl.$$.fragment,e),_(yl.$$.fragment,e),_(wl.$$.fragment,e),_($l.$$.fragment,e),_(El.$$.fragment,e),_(gl.$$.fragment,e),_(bl.$$.fragment,e),_(kl.$$.fragment,e),_(xl.$$.fragment,e),_(Cl.$$.fragment,e),_(Ol.$$.fragment,e),_(Il.$$.fragment,e),_(Tl.$$.fragment,e),_(Al.$$.fragment,e),_(Dl.$$.fragment,e),_(ql.$$.fragment,e),_(Sl.$$.fragment,e),_(Ll.$$.fragment,e),_(Ul.$$.fragment,e),_(Nl.$$.fragment,e),_(Rl.$$.fragment,e),_(zl.$$.fragment,e),_(Gl.$$.fragment,e),_(Fl.$$.fragment,e),_(Wl.$$.fragment,e),_(Ml.$$.fragment,e),_(Bl.$$.fragment,e),_(Vl.$$.fragment,e),_(Xl.$$.fragment,e),_(Jl.$$.fragment,e),_(Zl.$$.fragment,e),_(Kl.$$.fragment,e),_(oa.$$.fragment,e),_(ra.$$.fragment,e),_(la.$$.fragment,e),_(aa.$$.fragment,e),_(ia.$$.fragment,e),_(pa.$$.fragment,e),_(Zs.$$.fragment,e),_(Ks.$$.fragment,e),_(ha.$$.fragment,e),_(ua.$$.fragment,e),_(da.$$.fragment,e),_(_a.$$.fragment,e),_(va.$$.fragment,e),_(ya.$$.fragment,e),_(wa.$$.fragment,e),_($a.$$.fragment,e),_(Ea.$$.fragment,e),_(ga.$$.fragment,e),_(ba.$$.fragment,e),_(ka.$$.fragment,e),_(xa.$$.fragment,e),_(Ca.$$.fragment,e),_(Pa.$$.fragment,e),_(Oa.$$.fragment,e),_(Ia.$$.fragment,e),_(Ta.$$.fragment,e),_(Aa.$$.fragment,e),_(Sa.$$.fragment,e),_(Ua.$$.fragment,e),_(Na.$$.fragment,e),_(Ha.$$.fragment,e),_(Ra.$$.fragment,e),_(za.$$.fragment,e),_(Ga.$$.fragment,e),_(Fa.$$.fragment,e),_(Wa.$$.fragment,e),_(Ma.$$.fragment,e),_(Ba.$$.fragment,e),_(Va.$$.fragment,e),_(Xa.$$.fragment,e),_(Ja.$$.fragment,e),_(Za.$$.fragment,e),_(Ka.$$.fragment,e),_(Qa.$$.fragment,e),_(ei.$$.fragment,e),_(li.$$.fragment,e),_(ai.$$.fragment,e),_(ii.$$.fragment,e),$$=!0)},o(e){v(C.$$.fragment,e),v(Ro.$$.fragment,e),v(Zo.$$.fragment,e),v(Qo.$$.fragment,e),v(er.$$.fragment,e),v(sr.$$.fragment,e),v(or.$$.fragment,e),v(rr.$$.fragment,e),v(lr.$$.fragment,e),v(ar.$$.fragment,e),v(ir.$$.fragment,e),v(nr.$$.fragment,e),v(pr.$$.fragment,e),v(fr.$$.fragment,e),v(hr.$$.fragment,e),v(ur.$$.fragment,e),v(dr.$$.fragment,e),v(cr.$$.fragment,e),v(mr.$$.fragment,e),v(_r.$$.fragment,e),v(vr.$$.fragment,e),v(yr.$$.fragment,e),v(wr.$$.fragment,e),v($r.$$.fragment,e),v(gr.$$.fragment,e),v(kr.$$.fragment,e),v(jr.$$.fragment,e),v(Pr.$$.fragment,e),v(Ir.$$.fragment,e),v(Tr.$$.fragment,e),v(Ar.$$.fragment,e),v(Sr.$$.fragment,e),v(Lr.$$.fragment,e),v(Ur.$$.fragment,e),v(Rr.$$.fragment,e),v(zr.$$.fragment,e),v(Gr.$$.fragment,e),v(Fr.$$.fragment,e),v(Wr.$$.fragment,e),v(Br.$$.fragment,e),v(Yr.$$.fragment,e),v(Xr.$$.fragment,e),v(Jr.$$.fragment,e),v(ds.$$.fragment,e),v(cs.$$.fragment,e),v(Zr.$$.fragment,e),v(Kr.$$.fragment,e),v(Qr.$$.fragment,e),v(el.$$.fragment,e),v(tl.$$.fragment,e),v(sl.$$.fragment,e),v(ll.$$.fragment,e),v(al.$$.fragment,e),v(pl.$$.fragment,e),v(fl.$$.fragment,e),v(hl.$$.fragment,e),v(ul.$$.fragment,e),v(dl.$$.fragment,e),v(_l.$$.fragment,e),v(vl.$$.fragment,e),v(yl.$$.fragment,e),v(wl.$$.fragment,e),v($l.$$.fragment,e),v(El.$$.fragment,e),v(gl.$$.fragment,e),v(bl.$$.fragment,e),v(kl.$$.fragment,e),v(xl.$$.fragment,e),v(Cl.$$.fragment,e),v(Ol.$$.fragment,e),v(Il.$$.fragment,e),v(Tl.$$.fragment,e),v(Al.$$.fragment,e),v(Dl.$$.fragment,e),v(ql.$$.fragment,e),v(Sl.$$.fragment,e),v(Ll.$$.fragment,e),v(Ul.$$.fragment,e),v(Nl.$$.fragment,e),v(Rl.$$.fragment,e),v(zl.$$.fragment,e),v(Gl.$$.fragment,e),v(Fl.$$.fragment,e),v(Wl.$$.fragment,e),v(Ml.$$.fragment,e),v(Bl.$$.fragment,e),v(Vl.$$.fragment,e),v(Xl.$$.fragment,e),v(Jl.$$.fragment,e),v(Zl.$$.fragment,e),v(Kl.$$.fragment,e),v(oa.$$.fragment,e),v(ra.$$.fragment,e),v(la.$$.fragment,e),v(aa.$$.fragment,e),v(ia.$$.fragment,e),v(pa.$$.fragment,e),v(Zs.$$.fragment,e),v(Ks.$$.fragment,e),v(ha.$$.fragment,e),v(ua.$$.fragment,e),v(da.$$.fragment,e),v(_a.$$.fragment,e),v(va.$$.fragment,e),v(ya.$$.fragment,e),v(wa.$$.fragment,e),v($a.$$.fragment,e),v(Ea.$$.fragment,e),v(ga.$$.fragment,e),v(ba.$$.fragment,e),v(ka.$$.fragment,e),v(xa.$$.fragment,e),v(Ca.$$.fragment,e),v(Pa.$$.fragment,e),v(Oa.$$.fragment,e),v(Ia.$$.fragment,e),v(Ta.$$.fragment,e),v(Aa.$$.fragment,e),v(Sa.$$.fragment,e),v(Ua.$$.fragment,e),v(Na.$$.fragment,e),v(Ha.$$.fragment,e),v(Ra.$$.fragment,e),v(za.$$.fragment,e),v(Ga.$$.fragment,e),v(Fa.$$.fragment,e),v(Wa.$$.fragment,e),v(Ma.$$.fragment,e),v(Ba.$$.fragment,e),v(Va.$$.fragment,e),v(Xa.$$.fragment,e),v(Ja.$$.fragment,e),v(Za.$$.fragment,e),v(Ka.$$.fragment,e),v(Qa.$$.fragment,e),v(ei.$$.fragment,e),v(li.$$.fragment,e),v(ai.$$.fragment,e),v(ii.$$.fragment,e),$$=!1},d(e){s($),e&&s(O),e&&s(g),y(C),e&&s(b),e&&s(S),e&&s(Rt),e&&s(we),e&&s(q),e&&s(me),e&&s(Dm),e&&s(Xe),y(Ro),e&&s(qm),e&&s(Ft),e&&s(Sm),y(Zo,e),e&&s(Lm),e&&s(Wt),e&&s(Um),e&&s(Ze),y(Qo),e&&s(Nm),e&&s(Ke),y(er),e&&s(Hm),e&&s(Yt),e&&s(Rm),e&&s(vi),e&&s(zm),e&&s(yi),e&&s(Gm),y(sr,e),e&&s(Fm),e&&s(wi),e&&s(Wm),y(or,e),e&&s(Mm),e&&s($i),e&&s(Bm),y(rr,e),e&&s(Ym),e&&s(Ei),e&&s(Vm),e&&s(Q),e&&s(Xm),e&&s(Qe),y(lr),e&&s(Jm),e&&s(gi),e&&s(Zm),y(ar,e),e&&s(Km),e&&s(bi),e&&s(Qm),y(ir,e),e&&s(e_),e&&s(et),y(nr),e&&s(t_),e&&s(ki),e&&s(s_),y(pr,e),e&&s(o_),e&&s(tt),y(fr),e&&s(r_),e&&s(xi),e&&s(l_),y(hr,e),e&&s(a_),e&&s(Ci),e&&s(i_),e&&s(Ee),e&&s(n_),e&&s(Ii),e&&s(p_),y(ur,e),e&&s(f_),e&&s(Ti),e&&s(h_),e&&s(Zt),e&&s(u_),y(dr,e),e&&s(d_),e&&s(Ai),e&&s(c_),e&&s(Kt),e&&s(m_),y(cr,e),e&&s(__),e&&s(ee),e&&s(v_),e&&s(Qt),e&&s(y_),y(mr,e),e&&s(w_),e&&s(Di),e&&s($_),y(_r,e),e&&s(E_),e&&s(ge),e&&s(g_),y(vr,e),e&&s(b_),e&&s(es),e&&s(k_),e&&s(ts),e&&s(x_),y(yr,e),e&&s(C_),e&&s(st),y(wr),e&&s(j_),e&&s(be),e&&s(P_),y($r,e),e&&s(O_),e&&s(qi),e&&s(I_),e&&s(Si),e&&s(T_),y(gr,e),e&&s(A_),e&&s(br),e&&s(D_),y(kr,e),e&&s(q_),e&&s(xr),e&&s(S_),y(jr,e),e&&s(L_),e&&s(ot),y(Pr),e&&s(U_),e&&s(ls),e&&s(N_),y(Ir,e),e&&s(H_),y(Tr,e),e&&s(R_),e&&s(Li),e&&s(z_),e&&s(rt),y(Ar),e&&s(G_),e&&s(Dr),e&&s(F_),y(Sr,e),e&&s(W_),e&&s(lt),e&&s(M_),e&&s(ke),e&&s(B_),y(Lr,e),e&&s(Y_),e&&s(xe),e&&s(V_),y(Ur,e),e&&s(X_),e&&s(Ui),e&&s(J_),e&&s(Nr),e&&s(Z_),e&&s(at),y(Rr),e&&s(K_),e&&s(ns),e&&s(Q_),y(zr,e),e&&s(ev),e&&s(it),y(Gr),e&&s(tv),e&&s(Ni),e&&s(sv),y(Fr,e),e&&s(ov),e&&s(nt),y(Wr),e&&s(rv),e&&s(W),e&&s(lv),e&&s(_e),e&&s(av),e&&s(Ce),e&&s(iv),e&&s(pt),y(Br),e&&s(nv),e&&s(Hi),e&&s(pv),e&&s(ft),y(Yr),e&&s(fv),e&&s(Ri),e&&s(hv),y(Xr,e),e&&s(uv),e&&s(Gi),e&&s(dv),y(Jr,e),e&&s(cv),y(ds,e),e&&s(mv),y(cs,e),e&&s(_v),e&&s(ht),y(Zr),e&&s(vv),y(Kr,e),e&&s(yv),e&&s(_s),e&&s(wv),e&&s(vs),e&&s($v),y(Qr,e),e&&s(Ev),e&&s(Fi),e&&s(gv),y(el,e),e&&s(bv),e&&s(ys),e&&s(kv),y(tl,e),e&&s(xv),e&&s(Wi),e&&s(Cv),y(sl,e),e&&s(jv),e&&s(I),e&&s(Pv),e&&s(je),e&&s(Ov),e&&s(ut),y(ll),e&&s(Iv),e&&s(dt),y(al),e&&s(Tv),e&&s(il),e&&s(Av),y(pl,e),e&&s(Dv),e&&s(Mi),e&&s(qv),y(fl,e),e&&s(Sv),e&&s(Bi),e&&s(Lv),e&&s(ct),y(hl),e&&s(Uv),e&&s(Pe),e&&s(Nv),y(ul,e),e&&s(Hv),e&&s(mt),y(dl),e&&s(Rv),e&&s(cl),e&&s(zv),y(_l,e),e&&s(Gv),y(vl,e),e&&s(Fv),e&&s(_t),y(yl),e&&s(Wv),e&&s(ks),e&&s(Mv),y(wl,e),e&&s(Bv),e&&s(te),e&&s(Yv),y($l,e),e&&s(Vv),e&&s(Yi),e&&s(Xv),e&&s(Vi),e&&s(Jv),e&&s(U),e&&s(Zv),e&&s(Ji),e&&s(Kv),e&&s(k),e&&s(Qv),e&&s(Zi),e&&s(e1),y(El,e),e&&s(t1),e&&s(Oe),e&&s(s1),y(gl,e),e&&s(o1),e&&s(Ki),e&&s(r1),y(bl,e),e&&s(l1),e&&s(Ie),e&&s(a1),y(kl,e),e&&s(i1),e&&s(Is),e&&s(n1),e&&s(Qi),e&&s(p1),e&&s(en),e&&s(f1),y(xl,e),e&&s(h1),e&&s(vt),y(Cl),e&&s(u1),e&&s(yt),e&&s(d1),e&&s(tn),e&&s(c1),e&&s(As),e&&s(m1),e&&s(Ds),e&&s(_1),e&&s(sn),e&&s(v1),y(Ol,e),e&&s(y1),e&&s(wt),y(Il),e&&s(w1),e&&s(Te),e&&s($1),e&&s(M),e&&s(E1),y(Tl,e),e&&s(g1),e&&s(on),e&&s(b1),y(Al,e),e&&s(k1),e&&s($t),y(Dl),e&&s(x1),e&&s(rn),e&&s(C1),y(ql,e),e&&s(j1),e&&s(Et),y(Sl),e&&s(P1),e&&s(ln),e&&s(O1),y(Ll,e),e&&s(I1),e&&s(an),e&&s(T1),e&&s(nn),e&&s(A1),y(Ul,e),e&&s(D1),e&&s(gt),y(Nl),e&&s(q1),e&&s(Ae),e&&s(S1),e&&s(se),e&&s(L1),e&&s(bt),y(Rl),e&&s(U1),e&&s(pn),e&&s(N1),y(zl,e),e&&s(H1),e&&s(Hs),e&&s(R1),e&&s(De),e&&s(z1),y(Gl,e),e&&s(G1),e&&s(Rs),e&&s(F1),y(Fl,e),e&&s(W1),e&&s(zs),e&&s(M1),y(Wl,e),e&&s(B1),e&&s(fn),e&&s(Y1),y(Ml,e),e&&s(V1),e&&s(hn),e&&s(X1),y(Bl,e),e&&s(J1),e&&s(B),e&&s(Z1),e&&s(oe),e&&s(K1),e&&s(qe),e&&s(Q1),y(Vl,e),e&&s(ey),e&&s(re),e&&s(ty),y(Xl,e),e&&s(sy),e&&s(un),e&&s(oy),y(Jl,e),e&&s(ry),e&&s(dn),e&&s(ly),y(Zl,e),e&&s(ay),e&&s(cn),e&&s(iy),e&&s(kt),y(Kl),e&&s(ny),e&&s(Fs),e&&s(py),e&&s(Ws),e&&s(fy),e&&s(Ys),e&&s(hy),y(oa,e),e&&s(uy),e&&s(Y),e&&s(dy),y(ra,e),e&&s(cy),e&&s(xt),y(la),e&&s(my),e&&s(Xs),e&&s(_y),e&&s(yn),e&&s(vy),e&&s(Le),e&&s(yy),e&&s(wn),e&&s(wy),y(aa,e),e&&s($y),e&&s(Js),e&&s(Ey),e&&s($n),e&&s(gy),y(ia,e),e&&s(by),e&&s(na),e&&s(ky),e&&s(En),e&&s(xy),y(pa,e),e&&s(Cy),e&&s(gn),e&&s(jy),e&&s(bn),e&&s(Py),y(Zs,e),e&&s(Oy),y(Ks,e),e&&s(Iy),e&&s(jt),y(ha),e&&s(Ty),e&&s(Ue),e&&s(Ay),y(ua,e),e&&s(Dy),e&&s(Pt),y(da),e&&s(qy),e&&s(to),e&&s(Sy),e&&s(Pn),e&&s(Ly),e&&s(so),e&&s(Uy),e&&s(le),e&&s(Ny),e&&s(Ot),y(_a),e&&s(Hy),e&&s(On),e&&s(Ry),y(va,e),e&&s(zy),e&&s(In),e&&s(Gy),y(ya,e),e&&s(Fy),e&&s(ro),e&&s(Wy),y(wa,e),e&&s(My),e&&s(Tn),e&&s(By),y($a,e),e&&s(Yy),e&&s(An),e&&s(Vy),y(Ea,e),e&&s(Xy),e&&s(lo),e&&s(Jy),y(ga,e),e&&s(Zy),e&&s(Dn),e&&s(Ky),y(ba,e),e&&s(Qy),e&&s(qn),e&&s(ew),y(ka,e),e&&s(tw),e&&s(Sn),e&&s(sw),y(xa,e),e&&s(ow),e&&s(Ln),e&&s(rw),y(Ca,e),e&&s(lw),e&&s(ao),e&&s(aw),e&&s(It),y(Pa),e&&s(iw),e&&s(Un),e&&s(nw),y(Oa,e),e&&s(pw),e&&s(Ne),e&&s(fw),y(Ia,e),e&&s(hw),e&&s(ae),e&&s(uw),y(Ta,e),e&&s(dw),e&&s(Nn),e&&s(cw),e&&s(Hn),e&&s(mw),e&&s(Rn),e&&s(_w),e&&s(ie),e&&s(vw),e&&s(He),e&&s(yw),y(Aa,e),e&&s(ww),e&&s(Re),e&&s($w),e&&s(no),e&&s(Ew),e&&s(zn),e&&s(gw),e&&s(At),y(Sa),e&&s(bw),e&&s(V),e&&s(kw),y(Ua,e),e&&s(xw),e&&s(fo),e&&s(Cw),y(Na,e),e&&s(jw),e&&s(ho),e&&s(Pw),y(Ha,e),e&&s(Ow),e&&s(T),e&&s(Iw),e&&s(uo),e&&s(Tw),y(Ra,e),e&&s(Aw),e&&s(Gn),e&&s(Dw),y(za,e),e&&s(qw),e&&s(ze),e&&s(Sw),y(Ga,e),e&&s(Lw),e&&s(co),e&&s(Uw),y(Fa,e),e&&s(Nw),e&&s(Fn),e&&s(Hw),e&&s(Dt),y(Wa),e&&s(Rw),e&&s(_o),e&&s(zw),y(Ma,e),e&&s(Gw),e&&s(qt),y(Ba),e&&s(Fw),e&&s(Ya),e&&s(Ww),y(Va,e),e&&s(Mw),e&&s(ne),e&&s(Bw),y(Xa,e),e&&s(Yw),e&&s(N),e&&s(Vw),e&&s(yo),e&&s(Xw),e&&s(St),y(Ja),e&&s(Jw),e&&s(Wn),e&&s(Zw),y(Za,e),e&&s(Kw),e&&s(Lt),y(Ka),e&&s(Qw),e&&s(Mn),e&&s(e$),y(Qa,e),e&&s(t$),e&&s(Ut),y(ei),e&&s(s$),e&&s(Bn),e&&s(o$),e&&s(pe),e&&s(r$),e&&s(Nt),y(li),e&&s(l$),e&&s(Yn),e&&s(a$),e&&s(fe),e&&s(i$),e&&s(Vn),e&&s(n$),e&&s(Xn),e&&s(p$),e&&s(Jn),e&&s(f$),e&&s(Zn),e&&s(h$),e&&s(bo),e&&s(u$),e&&s(Qn),e&&s(d$),y(ai,e),e&&s(c$),e&&s(ep),e&&s(m$),y(ii,e),e&&s(_$),e&&s(ko),e&&s(v$),e&&s(xo),e&&s(y$),e&&s(tp),e&&s(w$),e&&s(Co)}}}const bL={local:"testing",sections:[{local:"how-transformers-are-tested",title:"How transformers are tested"},{local:"running-tests",sections:[{local:"choosing-which-tests-to-run",title:"Choosing which tests to run"},{local:"getting-the-list-of-all-tests",title:"Getting the list of all tests"},{local:"run-a-specific-test-module",title:"Run a specific test module"},{local:"run-specific-tests",title:"Run specific tests"},{local:"run-documentation-tests",title:"Run documentation tests "},{local:"run-only-modified-tests",title:"Run only modified tests"},{local:"automatically-rerun-failed-tests-on-source-modification",title:"Automatically rerun failed tests on source modification"},{local:"skip-a-test-module",title:"Skip a test module"},{local:"clearing-state",title:"Clearing state"},{local:"running-tests-in-parallel",title:"Running tests in parallel"},{local:"test-order-and-repetition",sections:[{local:"repeat-tests",title:"Repeat tests"},{local:"run-tests-in-a-random-order",title:"Run tests in a random order"}],title:"Test order and repetition"},{local:"look-and-feel-variations",sections:[{local:"pytestsugar",title:"pytest-sugar"},{local:"report-each-subtest-name-and-its-progress",title:"Report each sub-test name and its progress"},{local:"instantly-shows-failed-tests",title:"Instantly shows failed tests"}],title:"Look and feel variations"},{local:"to-gpu-or-not-to-gpu",title:"To GPU or not to GPU"},{local:"distributed-training",title:"Distributed training"},{local:"output-capture",title:"Output capture"},{local:"color-control",title:"Color control"},{local:"sending-test-report-to-online-pastebin-service",title:"Sending test report to online pastebin service"}],title:"Running tests"},{local:"writing-tests",sections:[{local:"parametrization",title:"Parametrization"},{local:"files-and-directories",title:"Files and directories"},{local:"temporary-files-and-directories",title:"Temporary files and directories"},{local:"temporary-syspath-override",title:"Temporary sys.path override"},{local:"skipping-tests",sections:[{local:"implementation",title:"Implementation"}],title:"Skipping tests"},{local:"slow-tests",title:"Slow tests"},{local:"testing-the-stdoutstderr-output",title:"Testing the stdout/stderr output"},{local:"capturing-logger-stream",title:"Capturing logger stream"},{local:"testing-with-environment-variables",title:"Testing with environment variables"},{local:"getting-reproducible-results",title:"Getting reproducible results"},{local:"debugging-tests",title:"Debugging tests"}],title:"Writing tests"},{local:"working-with-github-actions-workflows",title:"Working with github actions workflows"},{local:"testing-experimental-ci-features",title:"Testing Experimental CI Features"}],title:"Testing"};function kL(ye){return vL(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class OL extends dL{constructor($){super();cL(this,$,kL,gL,mL,{})}}export{OL as default,bL as metadata};
