import{S as w8,i as v8,s as b8,e as a,k as h,w as p,t as r,M as _8,c as s,d as o,m as f,a as i,x as u,h as n,b as m,N as E8,G as t,g as d,y as c,q as y,o as g,B as w,v as k8}from"../chunks/vendor-hf-doc-builder.js";import{T as T8}from"../chunks/Tip-hf-doc-builder.js";import{I as ue}from"../chunks/IconCopyLink-hf-doc-builder.js";import{C as E}from"../chunks/CodeBlock-hf-doc-builder.js";function $8(Rh){let P,nt,M,H,Oe,D,po;return{c(){P=a("p"),nt=r("In case you are using Windows, you should replace "),M=a("code"),H=r("RUN_SLOW=1"),Oe=r(" with "),D=a("code"),po=r("SET RUN_SLOW=1")},l(le){P=s(le,"P",{});var de=i(P);nt=n(de,"In case you are using Windows, you should replace "),M=s(de,"CODE",{});var uo=i(M);H=n(uo,"RUN_SLOW=1"),uo.forEach(o),Oe=n(de," with "),D=s(de,"CODE",{});var R=i(D);po=n(R,"SET RUN_SLOW=1"),R.forEach(o),de.forEach(o)},m(le,de){d(le,P,de),t(P,nt),t(P,M),t(M,H),t(P,Oe),t(P,D),t(D,po)},d(le){le&&o(P)}}}function P8(Rh){let P,nt,M,H,Oe,D,po,le,de,uo,R,dc,Ka,hc,fc,Qa,mc,pc,Fh,at,uc,co,cc,yc,zh,Wr,gc,Hh,W,Va,wc,vc,Za,bc,_c,es,Ec,kc,he,Tc,ts,$c,Pc,os,Ic,Lc,rs,Ac,Bc,Wh,st,Mc,yo,Nc,xc,Gh,Gr,Oc,Uh,je,it,ns,go,jc,as,Cc,Yh,Ur,qc,Jh,lt,Sc,Yr,Dc,Rc,Xh,ce,ss,Fc,zc,is,Hc,Wc,wo,Gc,ls,Uc,Yc,Kh,dt,Jc,ds,Xc,Kc,Qh,Jr,Qc,Vh,Ce,ht,hs,vo,Vc,fs,Zc,Zh,G,ey,Xr,ty,oy,Kr,ry,ny,ms,ay,sy,ef,Qr,iy,tf,Vr,hk,of,v,ly,ps,dy,hy,us,fy,my,Zr,py,uy,en,cy,yy,tn,gy,wy,on,vy,by,cs,_y,Ey,ys,ky,Ty,gs,$y,Py,ws,Iy,Ly,vs,Ay,By,bs,My,Ny,rn,xy,Oy,_s,jy,Cy,Es,qy,Sy,rf,bo,nf,I,Dy,nn,Ry,Fy,ks,zy,Hy,Ts,Wy,Gy,an,Uy,Yy,sn,Jy,Xy,af,qe,ft,$s,_o,Ky,Ps,Qy,sf,ln,Vy,lf,x,Se,Zy,Is,eg,tg,Eo,og,rg,ng,De,ag,Ls,sg,ig,As,lg,dg,hg,Bs,fg,mg,Re,pg,Ms,ug,cg,Ns,yg,gg,wg,xs,vg,df,Fe,mt,Os,ko,bg,js,_g,hf,dn,Eg,ff,ze,pt,Cs,To,kg,qs,Tg,mf,hn,$g,pf,ut,$o,Po,Pg,Ig,Io,Lg,Ag,Lo,Ao,Bg,Mg,Bo,Ng,uf,fn,xg,cf,ye,fe,Og,Mo,jg,Cg,No,qg,Sg,Ss,Dg,Rg,Fg,Ds,zg,Hg,Rs,Wg,yf,mn,Gg,gf,pn,Ug,wf,b,Fs,zs,Hs,Yg,Jg,Ws,un,Gs,Xg,Kg,Us,cn,Ys,Qg,Vg,Js,yn,Xs,Zg,ew,Ks,gn,Qs,tw,ow,Vs,wn,Zs,rw,nw,ei,vn,ti,aw,sw,oi,bn,ri,iw,lw,ni,_n,ai,dw,hw,si,En,ii,fw,mw,li,kn,di,pw,uw,hi,Tn,fi,cw,yw,mi,$n,pi,gw,ww,ui,Pn,ci,vw,vf,O,bw,yi,_w,Ew,gi,kw,Tw,wi,$w,Pw,vi,Iw,Lw,bf,He,ct,bi,xo,Aw,_i,Bw,_f,yt,Mw,Ei,Nw,xw,Ef,j,We,Ow,ki,jw,Cw,In,qw,Sw,Dw,Ge,Rw,Ti,Fw,zw,$i,Hw,Ww,Gw,Pi,Uw,Yw,Ue,Jw,Oo,Xw,Kw,Ii,Qw,Vw,Zw,Li,ev,kf,Ln,tv,Tf,Ye,gt,Ai,jo,ov,Bi,rv,$f,wt,Mi,Co,nv,qo,av,sv,iv,Ni,So,lv,xi,dv,hv,Pf,Do,If,Ro,Oi,fv,Lf,Fo,Af,An,mv,Bf,zo,Mf,Ho,Je,pv,ji,uv,cv,Wo,yv,gv,Nf,Go,Ci,wv,vv,xf,Uo,Yo,bv,qi,_v,Ev,Of,Jo,jf,vt,kv,Si,Tv,$v,Cf,Xe,bt,Di,Xo,Pv,Ri,Iv,qf,L,Lv,Fi,Av,Bv,zi,Mv,Nv,Hi,xv,Ov,Wi,jv,Cv,Gi,qv,Sv,Sf,Bn,Dv,Df,_t,Rv,Ui,Fv,zv,Rf,A,Yi,Hv,Wv,Ji,Gv,Uv,Xi,Yv,Jv,Ki,Xv,Kv,F,Qv,Qi,Vv,Zv,Vi,eb,tb,Zi,ob,rb,el,nb,ab,sb,Ke,ib,tl,lb,db,ol,hb,fb,Ff,Et,mb,rl,pb,ub,zf,Mn,cb,Hf,Nn,yb,Wf,kt,Ko,Qo,gb,wb,Vo,vb,bb,nl,_b,Gf,xn,Eb,Uf,Tt,kb,al,Tb,$b,Yf,$t,Pb,sl,Ib,Lb,Jf,Zo,Xf,On,Ab,Kf,Pt,il,Bb,Mb,Qe,Nb,ll,xb,Ob,dl,jb,Cb,Qf,jn,qb,Vf,It,Sb,hl,Db,Rb,Zf,U,fl,Fb,zb,ml,Hb,Wb,pl,Gb,Ub,ul,Yb,em,er,tr,Jb,Xb,tm,Lt,Kb,or,Qb,Vb,om,Cn,Zb,rm,qn,e_,nm,B,cl,t_,o_,yl,r_,n_,gl,a_,s_,wl,i_,l_,vl,d_,h_,bl,f_,am,Ve,m_,_l,p_,u_,El,c_,sm,Sn,y_,im,rr,lm,ge,g_,kl,w_,v_,Tl,b_,__,dm,C,z,E_,nr,k_,T_,$l,$_,P_,Pl,I_,L_,ar,A_,B_,M_,Il,N_,x_,T,O_,Ll,j_,C_,Al,q_,S_,Bl,D_,R_,Ml,F_,z_,Nl,H_,W_,xl,G_,U_,Ol,Y_,J_,jl,X_,K_,Cl,Q_,V_,ql,Z_,e1,t1,sr,o1,Sl,r1,n1,a1,me,s1,Dl,i1,l1,Rl,d1,h1,Fl,f1,m1,hm,At,p1,zl,u1,c1,fm,Ze,Bt,Hl,ir,y1,Wl,g1,mm,Dn,w1,pm,lr,um,Mt,v1,Rn,b1,_1,cm,Fn,E1,ym,Nt,zn,Gl,k1,T1,$1,Hn,Ul,P1,I1,gm,we,L1,Yl,A1,B1,dr,M1,N1,wm,Wn,Jl,x1,vm,ve,O1,Xl,j1,C1,Kl,q1,S1,bm,Gn,D1,_m,Un,Ql,R1,Em,hr,km,fr,Vl,F1,Tm,mr,$m,pr,Zl,z1,Pm,ur,Im,cr,ed,H1,Lm,yr,Am,et,td,od,W1,G1,rd,nd,U1,Bm,Yn,Y1,Mm,gr,Nm,Jn,J1,xm,Xn,X1,Om,Kn,K1,jm,Qn,ad,Q1,Cm,be,V1,sd,Z1,e0,id,t0,o0,qm,Y,r0,ld,n0,a0,dd,s0,i0,hd,l0,d0,Sm,N,fd,h0,f0,md,m0,p0,pd,u0,c0,ud,y0,g0,cd,w0,v0,Dm,wr,Rm,_e,b0,yd,_0,E0,gd,k0,T0,Fm,Vn,wd,$0,zm,J,P0,vd,I0,L0,bd,A0,B0,_d,M0,N0,Hm,xt,Zn,x0,vr,O0,j0,ea,C0,br,q0,Wm,Ot,S0,Ed,D0,R0,Gm,_r,Um,X,F0,kd,z0,H0,Td,W0,G0,$d,U0,Y0,Ym,Er,Jm,ta,J0,Xm,kr,Km,oa,X0,Qm,Tr,Vm,ra,K0,Zm,$r,ep,Pr,Q0,Pd,V0,tp,Ir,op,Ee,Z0,Id,e2,t2,Ld,o2,r2,rp,Lr,np,Ar,n2,Ad,a2,ap,Br,sp,na,s2,ip,jt,i2,Bd,l2,d2,lp,K,h2,Md,f2,m2,Nd,p2,u2,xd,c2,y2,dp,Q,g2,Od,w2,v2,jd,b2,_2,Cd,E2,k2,hp,Mr,fp,aa,qd,T2,mp,Ct,$2,sa,P2,I2,pp,Nr,up,q,L2,Sd,A2,B2,Dd,M2,N2,Rd,x2,O2,Fd,j2,C2,cp,V,q2,zd,S2,D2,Hd,R2,F2,Wd,z2,H2,yp,Z,tt,W2,Gd,G2,U2,Ud,Y2,J2,X2,Yd,K2,Q2,Jd,V2,Z2,ee,eE,Xd,tE,oE,Kd,rE,nE,Qd,aE,sE,xr,iE,gp,ke,lE,Vd,dE,hE,Zd,fE,mE,wp,qt,pE,eh,uE,cE,vp,ia,th,yE,bp,St,gE,oh,wE,vE,_p,Or,Ep,la,bE,kp,Dt,da,_E,rh,EE,kE,nh,TE,Tp,Rt,$E,ah,PE,IE,$p,jr,Pp,Ft,Ip,te,LE,sh,AE,BE,ih,ME,NE,lh,xE,OE,Lp,zt,Cr,jE,dh,CE,qE,SE,hh,DE,Ap,ha,fh,RE,Bp,Ht,FE,mh,zE,HE,Mp,fa,WE,Np,ma,GE,xp,qr,Op,Wt,UE,ph,YE,JE,jp,Sr,Cp,Gt,XE,uh,KE,QE,qp,Te,VE,ch,ZE,e3,yh,t3,o3,Sp,pa,gh,r3,Dp,oe,n3,wh,a3,s3,vh,i3,l3,bh,d3,h3,Rp,ua,_h,f3,Fp,re,m3,Eh,p3,u3,kh,c3,y3,Th,g3,w3,zp,$e,v3,$h,b3,_3,ca,E3,k3,Hp,ya,Ph,T3,Wp,Ut,$3,Ih,P3,I3,Gp,Dr,Up,ga,L3,Yp,Rr,Jp,wa,A3,Xp,va,B3,Kp,ba,M3,Qp,_a,Lh,N3,Vp,S,x3,Ea,O3,j3,Ah,C3,q3,Bh,S3,D3,Mh,R3,F3,Zp,Fr,eu,Yt,z3,Nh,H3,W3,tu,ka,xh,G3,ou,Jt,U3,Oh,Y3,J3,ru,Ta,jh,X3,nu,$a,K3,au,ot,Xt,Ch,zr,Q3,qh,V3,su,Pa,Z3,iu,Ia,Sh,ek,lu;return D=new ue({}),go=new ue({}),vo=new ue({}),bo=new E({props:{code:`model = BrandNewBertModel.from_pretrained("brandy/brand_new_bert")
model.config  # model has access to its config`,highlighted:`model = BrandNewBertModel.from_pretrained(<span class="hljs-string">&quot;brandy/brand_new_bert&quot;</span>)
model.config  <span class="hljs-comment"># model has access to its config</span>`}}),_o=new ue({}),ko=new ue({}),To=new ue({}),xo=new ue({}),jo=new ue({}),Do=new E({props:{code:`git clone https://github.com/[your Github handle]/transformers.git
cd transformers
git remote add upstream https://github.com/huggingface/transformers.git`,highlighted:`git <span class="hljs-built_in">clone</span> https://github.com/[your Github handle]/transformers.git
<span class="hljs-built_in">cd</span> transformers
git remote add upstream https://github.com/huggingface/transformers.git`}}),Fo=new E({props:{code:`python -m venv .env
source .env/bin/activate
pip install -e ".[dev]"`,highlighted:`python -m venv .<span class="hljs-built_in">env</span>
<span class="hljs-built_in">source</span> .<span class="hljs-built_in">env</span>/bin/activate
pip install -e <span class="hljs-string">&quot;.[dev]&quot;</span>`}}),zo=new E({props:{code:"cd ..",highlighted:'<span class="hljs-built_in">cd</span> ..'}}),Jo=new E({props:{code:`git clone https://github.com/org_that_created_brand_new_bert_org/brand_new_bert.git
cd brand_new_bert
pip install -e .`,highlighted:`git <span class="hljs-built_in">clone</span> https://github.com/org_that_created_brand_new_bert_org/brand_new_bert.git
<span class="hljs-built_in">cd</span> brand_new_bert
pip install -e .`}}),Xo=new ue({}),Zo=new E({props:{code:`model = BrandNewBertModel.load_pretrained_checkpoint("/path/to/checkpoint/")
input_ids = [0, 4, 5, 2, 3, 7, 9]  # vector of input ids
original_output = model.predict(input_ids)`,highlighted:`model = BrandNewBertModel.load_pretrained_checkpoint(<span class="hljs-string">&quot;/path/to/checkpoint/&quot;</span>)
input_ids = [<span class="hljs-number">0</span>, <span class="hljs-number">4</span>, <span class="hljs-number">5</span>, <span class="hljs-number">2</span>, <span class="hljs-number">3</span>, <span class="hljs-number">7</span>, <span class="hljs-number">9</span>]  <span class="hljs-comment"># vector of input ids</span>
original_output = model.predict(input_ids)`}}),rr=new E({props:{code:`[[
 [-0.1465, -0.6501,  0.1993,  ...,  0.1451,  0.3430,  0.6024],
 [-0.4417, -0.5920,  0.3450,  ..., -0.3062,  0.6182,  0.7132],
 [-0.5009, -0.7122,  0.4548,  ..., -0.3662,  0.6091,  0.7648],
 ...,
 [-0.5613, -0.6332,  0.4324,  ..., -0.3792,  0.7372,  0.9288],
 [-0.5416, -0.6345,  0.4180,  ..., -0.3564,  0.6992,  0.9191],
 [-0.5334, -0.6403,  0.4271,  ..., -0.3339,  0.6533,  0.8694]]],`,highlighted:`<span class="hljs-comment">[<span class="hljs-comment">[
 <span class="hljs-comment">[-0.1465, -0.6501,  0.1993,  ...,  0.1451,  0.3430,  0.6024]</span>,
 <span class="hljs-comment">[-0.4417, -0.5920,  0.3450,  ..., -0.3062,  0.6182,  0.7132]</span>,
 <span class="hljs-comment">[-0.5009, -0.7122,  0.4548,  ..., -0.3662,  0.6091,  0.7648]</span>,
 ...,
 <span class="hljs-comment">[-0.5613, -0.6332,  0.4324,  ..., -0.3792,  0.7372,  0.9288]</span>,
 <span class="hljs-comment">[-0.5416, -0.6345,  0.4180,  ..., -0.3564,  0.6992,  0.9191]</span>,
 <span class="hljs-comment">[-0.5334, -0.6403,  0.4271,  ..., -0.3339,  0.6533,  0.8694]</span>]</span>]</span>,`}}),ir=new ue({}),lr=new E({props:{code:"cd transformers",highlighted:'<span class="hljs-built_in">cd</span> transformers'}}),hr=new E({props:{code:"git checkout -b add_brand_new_bert",highlighted:"git checkout -b add_brand_new_bert"}}),mr=new E({props:{code:`git add .
git commit`,highlighted:`git add .
git commit`}}),ur=new E({props:{code:`git fetch upstream
git rebase upstream/main`,highlighted:`git fetch upstream
git rebase upstream/main`}}),yr=new E({props:{code:"git push -u origin a-descriptive-name-for-my-changes",highlighted:"git push -u origin a-descriptive-name-for-my-changes"}}),gr=new E({props:{code:`git fetch upstream
git merge upstream/main`,highlighted:`git fetch upstream
git merge upstream/main`}}),wr=new E({props:{code:`from transformers import BrandNewBertModel, BrandNewBertConfig

model = BrandNewBertModel(BrandNewBertConfig())`,highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> BrandNewBertModel, BrandNewBertConfig

model = BrandNewBertModel(BrandNewBertConfig())`}}),_r=new E({props:{code:`from torch import nn


class SimpleModel(nn.Module):
    def __init__(self):
        super().__init__()
        self.dense = nn.Linear(10, 10)
        self.intermediate = nn.Linear(10, 10)
        self.layer_norm = nn.LayerNorm(10)`,highlighted:`<span class="hljs-keyword">from</span> torch <span class="hljs-keyword">import</span> nn


<span class="hljs-keyword">class</span> <span class="hljs-title class_">SimpleModel</span>(nn.Module):
    <span class="hljs-keyword">def</span> <span class="hljs-title function_">__init__</span>(<span class="hljs-params">self</span>):
        <span class="hljs-built_in">super</span>().__init__()
        self.dense = nn.Linear(<span class="hljs-number">10</span>, <span class="hljs-number">10</span>)
        self.intermediate = nn.Linear(<span class="hljs-number">10</span>, <span class="hljs-number">10</span>)
        self.layer_norm = nn.LayerNorm(<span class="hljs-number">10</span>)`}}),Er=new E({props:{code:`model = SimpleModel()

print(model)`,highlighted:`model = SimpleModel()

<span class="hljs-built_in">print</span>(model)`}}),kr=new E({props:{code:`SimpleModel(
  (dense): Linear(in_features=10, out_features=10, bias=True)
  (intermediate): Linear(in_features=10, out_features=10, bias=True)
  (layer_norm): LayerNorm((10,), eps=1e-05, elementwise_affine=True)
)`,highlighted:`SimpleModel(
  (dense): Linear(<span class="hljs-attribute">in_features</span>=10, <span class="hljs-attribute">out_features</span>=10, <span class="hljs-attribute">bias</span>=<span class="hljs-literal">True</span>)
  (intermediate): Linear(<span class="hljs-attribute">in_features</span>=10, <span class="hljs-attribute">out_features</span>=10, <span class="hljs-attribute">bias</span>=<span class="hljs-literal">True</span>)
  (layer_norm): LayerNorm((10,), <span class="hljs-attribute">eps</span>=1e-05, <span class="hljs-attribute">elementwise_affine</span>=<span class="hljs-literal">True</span>)
)`}}),Tr=new E({props:{code:"print(model.dense.weight.data)",highlighted:'<span class="hljs-built_in">print</span>(model.dense.weight.data)'}}),$r=new E({props:{code:`tensor([[-0.0818,  0.2207, -0.0749, -0.0030,  0.0045, -0.1569, -0.1598,  0.0212,
         -0.2077,  0.2157],
        [ 0.1044,  0.0201,  0.0990,  0.2482,  0.3116,  0.2509,  0.2866, -0.2190,
          0.2166, -0.0212],
        [-0.2000,  0.1107, -0.1999, -0.3119,  0.1559,  0.0993,  0.1776, -0.1950,
         -0.1023, -0.0447],
        [-0.0888, -0.1092,  0.2281,  0.0336,  0.1817, -0.0115,  0.2096,  0.1415,
         -0.1876, -0.2467],
        [ 0.2208, -0.2352, -0.1426, -0.2636, -0.2889, -0.2061, -0.2849, -0.0465,
          0.2577,  0.0402],
        [ 0.1502,  0.2465,  0.2566,  0.0693,  0.2352, -0.0530,  0.1859, -0.0604,
          0.2132,  0.1680],
        [ 0.1733, -0.2407, -0.1721,  0.1484,  0.0358, -0.0633, -0.0721, -0.0090,
          0.2707, -0.2509],
        [-0.1173,  0.1561,  0.2945,  0.0595, -0.1996,  0.2988, -0.0802,  0.0407,
          0.1829, -0.1568],
        [-0.1164, -0.2228, -0.0403,  0.0428,  0.1339,  0.0047,  0.1967,  0.2923,
          0.0333, -0.0536],
        [-0.1492, -0.1616,  0.1057,  0.1950, -0.2807, -0.2710, -0.1586,  0.0739,
          0.2220,  0.2358]]).`,highlighted:`tensor([[<span class="hljs-string">-0</span>.0818,  0.2207, <span class="hljs-string">-0</span>.0749, <span class="hljs-string">-0</span>.0030,  0.0045, <span class="hljs-string">-0</span>.1569, <span class="hljs-string">-0</span>.1598,  0.0212,
         <span class="hljs-string">-0</span>.2077,  0.2157],
        [ 0.1044,  0.0201,  0.0990,  0.2482,  0.3116,  0.2509,  0.2866, <span class="hljs-string">-0</span>.2190,
          0.2166, <span class="hljs-string">-0</span>.0212],
        [<span class="hljs-string">-0</span>.2000,  0.1107, <span class="hljs-string">-0</span>.1999, <span class="hljs-string">-0</span>.3119,  0.1559,  0.0993,  0.1776, <span class="hljs-string">-0</span>.1950,
         <span class="hljs-string">-0</span>.1023, <span class="hljs-string">-0</span>.0447],
        [<span class="hljs-string">-0</span>.0888, <span class="hljs-string">-0</span>.1092,  0.2281,  0.0336,  0.1817, <span class="hljs-string">-0</span>.0115,  0.2096,  0.1415,
         <span class="hljs-string">-0</span>.1876, <span class="hljs-string">-0</span>.2467],
        [ 0.2208, <span class="hljs-string">-0</span>.2352, <span class="hljs-string">-0</span>.1426, <span class="hljs-string">-0</span>.2636, <span class="hljs-string">-0</span>.2889, <span class="hljs-string">-0</span>.2061, <span class="hljs-string">-0</span>.2849, <span class="hljs-string">-0</span>.0465,
          0.2577,  0.0402],
        [ 0.1502,  0.2465,  0.2566,  0.0693,  0.2352, <span class="hljs-string">-0</span>.0530,  0.1859, <span class="hljs-string">-0</span>.0604,
          0.2132,  0.1680],
        [ 0.1733, <span class="hljs-string">-0</span>.2407, <span class="hljs-string">-0</span>.1721,  0.1484,  0.0358, <span class="hljs-string">-0</span>.0633, <span class="hljs-string">-0</span>.0721, <span class="hljs-string">-0</span>.0090,
          0.2707, <span class="hljs-string">-0</span>.2509],
        [<span class="hljs-string">-0</span>.1173,  0.1561,  0.2945,  0.0595, <span class="hljs-string">-0</span>.1996,  0.2988, <span class="hljs-string">-0</span>.0802,  0.0407,
          0.1829, <span class="hljs-string">-0</span>.1568],
        [<span class="hljs-string">-0</span>.1164, <span class="hljs-string">-0</span>.2228, <span class="hljs-string">-0</span>.0403,  0.0428,  0.1339,  0.0047,  0.1967,  0.2923,
          0.0333, <span class="hljs-string">-0</span>.0536],
        [<span class="hljs-string">-0</span>.1492, <span class="hljs-string">-0</span>.1616,  0.1057,  0.1950, <span class="hljs-string">-0</span>.2807, <span class="hljs-string">-0</span>.2710, <span class="hljs-string">-0</span>.1586,  0.0739,
          0.2220,  0.2358]]).`}}),Ir=new E({props:{code:`# retrieve matching layer weights, e.g. by
# recursive algorithm
layer_name = "dense"
pretrained_weight = array_of_dense_layer

model_pointer = getattr(model, "dense")

model_pointer.weight.data = torch.from_numpy(pretrained_weight)`,highlighted:`<span class="hljs-comment"># retrieve matching layer weights, e.g. by</span>
<span class="hljs-comment"># recursive algorithm</span>
layer_name = <span class="hljs-string">&quot;dense&quot;</span>
pretrained_weight = array_of_dense_layer

model_pointer = <span class="hljs-built_in">getattr</span>(model, <span class="hljs-string">&quot;dense&quot;</span>)

model_pointer.weight.data = torch.from_numpy(pretrained_weight)`}}),Lr=new E({props:{code:`assert (
    model_pointer.weight.shape == pretrained_weight.shape
), f"Pointer shape of random weight {model_pointer.shape} and array shape of checkpoint weight {pretrained_weight.shape} mismatched"`,highlighted:`<span class="hljs-keyword">assert</span> (
    model_pointer.weight.shape == pretrained_weight.shape
), <span class="hljs-string">f&quot;Pointer shape of random weight <span class="hljs-subst">{model_pointer.shape}</span> and array shape of checkpoint weight <span class="hljs-subst">{pretrained_weight.shape}</span> mismatched&quot;</span>`}}),Br=new E({props:{code:'logger.info(f"Initialize PyTorch weight {layer_name} from {pretrained_weight.name}")',highlighted:'logger.info(<span class="hljs-string">f&quot;Initialize PyTorch weight <span class="hljs-subst">{layer_name}</span> from <span class="hljs-subst">{pretrained_weight.name}</span>&quot;</span>)'}}),Mr=new E({props:{code:'model.save_pretrained("/path/to/converted/checkpoint/folder")',highlighted:'model.save_pretrained(<span class="hljs-string">&quot;/path/to/converted/checkpoint/folder&quot;</span>)'}}),Nr=new E({props:{code:`model = BrandNewBertModel.from_pretrained("/path/to/converted/checkpoint/folder")
input_ids = [0, 4, 4, 3, 2, 4, 1, 7, 19]
output = model(input_ids).last_hidden_states`,highlighted:`model = BrandNewBertModel.from_pretrained(<span class="hljs-string">&quot;/path/to/converted/checkpoint/folder&quot;</span>)
input_ids = [<span class="hljs-number">0</span>, <span class="hljs-number">4</span>, <span class="hljs-number">4</span>, <span class="hljs-number">3</span>, <span class="hljs-number">2</span>, <span class="hljs-number">4</span>, <span class="hljs-number">1</span>, <span class="hljs-number">7</span>, <span class="hljs-number">19</span>]
output = model(input_ids).last_hidden_states`}}),Or=new E({props:{code:"pytest tests/models/brand_new_bert/test_modeling_brand_new_bert.py",highlighted:"pytest tests/models/brand_new_bert/test_modeling_brand_new_bert.py"}}),jr=new E({props:{code:"RUN_SLOW=1 pytest -sv tests/models/brand_new_bert/test_modeling_brand_new_bert.py::BrandNewBertModelIntegrationTests",highlighted:"RUN_SLOW=1 pytest -sv tests/models/brand_new_bert/test_modeling_brand_new_bert.py::BrandNewBertModelIntegrationTests"}}),Ft=new T8({props:{$$slots:{default:[$8]},$$scope:{ctx:Rh}}}),qr=new E({props:{code:`input_str = "This is a long example input string containing special characters .$?-, numbers 2872 234 12 and words."
model = BrandNewBertModel.load_pretrained_checkpoint("/path/to/checkpoint/")
input_ids = model.tokenize(input_str)`,highlighted:`input_str = <span class="hljs-string">&quot;This is a long example input string containing special characters .$?-, numbers 2872 234 12 and words.&quot;</span>
model = BrandNewBertModel.load_pretrained_checkpoint(<span class="hljs-string">&quot;/path/to/checkpoint/&quot;</span>)
input_ids = model.tokenize(input_str)`}}),Sr=new E({props:{code:`from transformers import BrandNewBertTokenizer

input_str = "This is a long example input string containing special characters .$?-, numbers 2872 234 12 and words."

tokenizer = BrandNewBertTokenizer.from_pretrained("/path/to/tokenizer/folder/")

input_ids = tokenizer(input_str).input_ids`,highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> BrandNewBertTokenizer

input_str = <span class="hljs-string">&quot;This is a long example input string containing special characters .$?-, numbers 2872 234 12 and words.&quot;</span>

tokenizer = BrandNewBertTokenizer.from_pretrained(<span class="hljs-string">&quot;/path/to/tokenizer/folder/&quot;</span>)

input_ids = tokenizer(input_str).input_ids`}}),Dr=new E({props:{code:"make style",highlighted:"make style"}}),Rr=new E({props:{code:"make quality",highlighted:"make quality"}}),Fr=new E({props:{code:`brand_new_bert.push_to_hub("brand_new_bert")
# Uncomment the following line to push to an organization.
# brand_new_bert.push_to_hub("<organization>/brand_new_bert")`,highlighted:`brand_new_bert.push_to_hub(<span class="hljs-string">&quot;brand_new_bert&quot;</span>)
<span class="hljs-comment"># Uncomment the following line to push to an organization.</span>
<span class="hljs-comment"># brand_new_bert.push_to_hub(&quot;&lt;organization&gt;/brand_new_bert&quot;)</span>`}}),zr=new ue({}),{c(){P=a("meta"),nt=h(),M=a("h1"),H=a("a"),Oe=a("span"),p(D.$$.fragment),po=h(),le=a("span"),de=r("How to add a model to \u{1F917} Transformers?"),uo=h(),R=a("p"),dc=r(`Adding a new model is often difficult and requires an in-depth knowledge of the \u{1F917} Transformers library and ideally also
of the model\u2019s original repository. At Hugging Face, we are trying to empower the community more and more to add models
independently. Thus, for some new models that the community wants to be added to \u{1F917} Transformers, we create a customized
`),Ka=a("em"),hc=r("call-for-model-addition"),fc=r(` that explains step-by-step how to add the requested model. With this
`),Qa=a("em"),mc=r("call-for-model-addition"),pc=r(`, we want to teach a motivated and experienced contributor of the community how to port a
model to \u{1F917} Transformers.`),Fh=h(),at=a("p"),uc=r(`If this sounds like something you would be interested in, feel free to check out the currently open
\u201Ccalls-for-model-addition\u201D `),co=a("a"),cc=r("here"),yc=r(`
and to contact us.`),zh=h(),Wr=a("p"),gc=r(`If selected, you will then work closely with one member of the Hugging Face team to integrate the model into \u{1F917}
Transformers. By doing so, you will both gain a theoretical and deep practical understanding of the proposed model. But
more importantly, you will have made a major open-source contribution to \u{1F917} Transformers. Along the way, you will:`),Hh=h(),W=a("ul"),Va=a("li"),wc=r("get insights into open-source best practices"),vc=h(),Za=a("li"),bc=r("understand the design principles of one of the most popular NLP libraries"),_c=h(),es=a("li"),Ec=r("learn how to do efficiently test large NLP models"),kc=h(),he=a("li"),Tc=r("learn how to integrate Python utilities like "),ts=a("code"),$c=r("black"),Pc=r(", "),os=a("code"),Ic=r("isort"),Lc=r(", "),rs=a("code"),Ac=r("make fix-copies"),Bc=r(` into a library to always
ensure clean and readable code`),Wh=h(),st=a("p"),Mc=r(`We are also more than happy if you want to add a model that cannot be found in the \u201Ccalls-for-model-addition\u201D folder.
The following sections explain in detail how to add a new model. It might also be very helpful to check out already
added models to see if those resemble the model you would like to add `),yo=a("a"),Nc=r("here"),xc=r("."),Gh=h(),Gr=a("p"),Oc=r("To start, let\u2019s try to get a general overview of the Transformers library."),Uh=h(),je=a("h2"),it=a("a"),ns=a("span"),p(go.$$.fragment),jc=h(),as=a("span"),Cc=r("General overview of \u{1F917} Transformers"),Yh=h(),Ur=a("p"),qc=r(`First, you should get a general overview of \u{1F917} Transformers. \u{1F917} Transformers is a very opinionated library, so there is a
chance that you don\u2019t agree with some of the library\u2019s philosophies or design choices. From our experience, however, we
found that the fundamental design choices and philosophies of the library are crucial to efficiently scale \u{1F917}
Transformers while keeping maintenance costs at a reasonable level.`),Jh=h(),lt=a("p"),Sc=r("A good first starting point to better understand the library is to read the "),Yr=a("a"),Dc=r("documentation of our philosophy"),Rc=r(". As a result of our way of working, there are some choices that we try to apply to all models:"),Xh=h(),ce=a("ul"),ss=a("li"),Fc=r("Composition is generally favored over-abstraction"),zc=h(),is=a("li"),Hc=r("Duplicating code is not always bad if it strongly improves the readability or accessibility of a model"),Wc=h(),wo=a("li"),Gc=r(`Model files are as self-contained as possible so that when you read the code of a specific model, you ideally only
have to look into the respective `),ls=a("code"),Uc=r("modeling_....py"),Yc=r(" file."),Kh=h(),dt=a("p"),Jc=r("In our opinion, the library\u2019s code is not just a means to provide a product, "),ds=a("em"),Xc=r("e.g."),Kc=r(` the ability to use BERT for
inference, but also as the very product that we want to improve. Hence, when adding a model, the user is not only the
person that will use your model, but also everybody that will read, try to understand, and possibly tweak your code.`),Qh=h(),Jr=a("p"),Qc=r("With this in mind, let\u2019s go a bit deeper into the general library design."),Vh=h(),Ce=a("h3"),ht=a("a"),hs=a("span"),p(vo.$$.fragment),Vc=h(),fs=a("span"),Zc=r("Overview of models"),Zh=h(),G=a("p"),ey=r(`To successfully add a model, it is important to understand the interaction between your model and its config,
`),Xr=a("a"),ty=r("PreTrainedModel"),oy=r(", and "),Kr=a("a"),ry=r("PretrainedConfig"),ny=r(`. For exemplary purposes, we will
call the model to be added to \u{1F917} Transformers `),ms=a("code"),ay=r("BrandNewBert"),sy=r("."),ef=h(),Qr=a("p"),iy=r("Let\u2019s take a look:"),tf=h(),Vr=a("img"),of=h(),v=a("p"),ly=r(`As you can see, we do make use of inheritance in \u{1F917} Transformers, but we keep the level of abstraction to an absolute
minimum. There are never more than two levels of abstraction for any model in the library. `),ps=a("code"),dy=r("BrandNewBertModel"),hy=r(`
inherits from `),us=a("code"),fy=r("BrandNewBertPreTrainedModel"),my=r(" which in turn inherits from "),Zr=a("a"),py=r("PreTrainedModel"),uy=r(` and
that\u2019s it. As a general rule, we want to make sure that a new model only depends on
`),en=a("a"),cy=r("PreTrainedModel"),yy=r(`. The important functionalities that are automatically provided to every new
model are `),tn=a("a"),gy=r("from_pretrained()"),wy=r(` and
`),on=a("a"),vy=r("save_pretrained()"),by=r(`, which are used for serialization and deserialization. All of the
other important functionalities, such as `),cs=a("code"),_y=r("BrandNewBertModel.forward"),Ey=r(` should be completely defined in the new
`),ys=a("code"),ky=r("modeling_brand_new_bert.py"),Ty=r(` script. Next, we want to make sure that a model with a specific head layer, such as
`),gs=a("code"),$y=r("BrandNewBertForMaskedLM"),Py=r(" does not inherit from "),ws=a("code"),Iy=r("BrandNewBertModel"),Ly=r(", but rather uses "),vs=a("code"),Ay=r("BrandNewBertModel"),By=r(`
as a component that can be called in its forward pass to keep the level of abstraction low. Every new model requires a
configuration class, called `),bs=a("code"),My=r("BrandNewBertConfig"),Ny=r(`. This configuration is always stored as an attribute in
`),rn=a("a"),xy=r("PreTrainedModel"),Oy=r(", and thus can be accessed via the "),_s=a("code"),jy=r("config"),Cy=r(` attribute for all classes
inheriting from `),Es=a("code"),qy=r("BrandNewBertPreTrainedModel"),Sy=r(":"),rf=h(),p(bo.$$.fragment),nf=h(),I=a("p"),Dy=r(`Similar to the model, the configuration inherits basic serialization and deserialization functionalities from
`),nn=a("a"),Ry=r("PretrainedConfig"),Fy=r(`. Note that the configuration and the model are always serialized into two
different formats - the model to a `),ks=a("em"),zy=r("pytorch_model.bin"),Hy=r(" file and the configuration to a "),Ts=a("em"),Wy=r("config.json"),Gy=r(` file. Calling
`),an=a("a"),Uy=r("save_pretrained()"),Yy=r(` will automatically call
`),sn=a("a"),Jy=r("save_pretrained()"),Xy=r(", so that both model and configuration are saved."),af=h(),qe=a("h3"),ft=a("a"),$s=a("span"),p(_o.$$.fragment),Ky=h(),Ps=a("span"),Qy=r("Code style"),sf=h(),ln=a("p"),Vy=r(`When coding your new model, keep in mind that Transformers is an opinionated library and we have a few quirks of our
own regarding how code should be written :-)`),lf=h(),x=a("ol"),Se=a("li"),Zy=r(`The forward pass of your model should be fully written in the modeling file while being fully independent of other
models in the library. If you want to reuse a block from another model, copy the code and paste it with a
`),Is=a("code"),eg=r("# Copied from"),tg=r(" comment on top (see "),Eo=a("a"),og=r("here"),rg=r(`
for a good example).`),ng=h(),De=a("li"),ag=r(`The code should be fully understandable, even by a non-native English speaker. This means you should pick
descriptive variable names and avoid abbreviations. As an example, `),Ls=a("code"),sg=r("activation"),ig=r(" is preferred to "),As=a("code"),lg=r("act"),dg=r(`.
One-letter variable names are strongly discouraged unless it\u2019s an index in a for loop.`),hg=h(),Bs=a("li"),fg=r("More generally we prefer longer explicit code to short magical one."),mg=h(),Re=a("li"),pg=r("Avoid subclassing "),Ms=a("code"),ug=r("nn.Sequential"),cg=r(" in PyTorch but subclass "),Ns=a("code"),yg=r("nn.Module"),gg=r(` and write the forward pass, so that anyone
using your code can quickly debug it by adding print statements or breaking points.`),wg=h(),xs=a("li"),vg=r(`Your function signature should be type-annotated. For the rest, good variable names are way more readable and
understandable than type annotations.`),df=h(),Fe=a("h3"),mt=a("a"),Os=a("span"),p(ko.$$.fragment),bg=h(),js=a("span"),_g=r("Overview of tokenizers"),hf=h(),dn=a("p"),Eg=r("Not quite ready yet :-( This section will be added soon!"),ff=h(),ze=a("h2"),pt=a("a"),Cs=a("span"),p(To.$$.fragment),kg=h(),qs=a("span"),Tg=r("Step-by-step recipe to add a model to \u{1F917} Transformers"),mf=h(),hn=a("p"),$g=r(`Everyone has different preferences of how to port a model so it can be very helpful for you to take a look at summaries
of how other contributors ported models to Hugging Face. Here is a list of community blog posts on how to port a model:`),pf=h(),ut=a("ol"),$o=a("li"),Po=a("a"),Pg=r("Porting GPT2 Model"),Ig=r(" by "),Io=a("a"),Lg=r("Thomas"),Ag=h(),Lo=a("li"),Ao=a("a"),Bg=r("Porting WMT19 MT Model"),Mg=r(" by "),Bo=a("a"),Ng=r("Stas"),uf=h(),fn=a("p"),xg=r("From experience, we can tell you that the most important things to keep in mind when adding a model are:"),cf=h(),ye=a("ul"),fe=a("li"),Og=r(`Don\u2019t reinvent the wheel! Most parts of the code you will add for the new \u{1F917} Transformers model already exist
somewhere in \u{1F917} Transformers. Take some time to find similar, already existing models and tokenizers you can copy
from. `),Mo=a("a"),jg=r("grep"),Cg=r(" and "),No=a("a"),qg=r("rg"),Sg=r(` are your
friends. Note that it might very well happen that your model\u2019s tokenizer is based on one model implementation, and
your model\u2019s modeling code on another one. `),Ss=a("em"),Dg=r("E.g."),Rg=r(` FSMT\u2019s modeling code is based on BART, while FSMT\u2019s tokenizer code
is based on XLM.`),Fg=h(),Ds=a("li"),zg=r(`It\u2019s more of an engineering challenge than a scientific challenge. You should spend more time on creating an
efficient debugging environment than trying to understand all theoretical aspects of the model in the paper.`),Hg=h(),Rs=a("li"),Wg=r(`Ask for help, when you\u2019re stuck! Models are the core component of \u{1F917} Transformers so that we at Hugging Face are more
than happy to help you at every step to add your model. Don\u2019t hesitate to ask if you notice you are not making
progress.`),yf=h(),mn=a("p"),Gg=r("In the following, we try to give you a general recipe that we found most useful when porting a model to \u{1F917} Transformers."),gf=h(),pn=a("p"),Ug=r(`The following list is a summary of everything that has to be done to add a model and can be used by you as a To-Do
List:`),wf=h(),b=a("ul"),Fs=a("li"),zs=a("ol"),Hs=a("li"),Yg=r("\u2610 (Optional) Understood theoretical aspects"),Jg=h(),Ws=a("li"),un=a("ol"),Gs=a("li"),Xg=r("\u2610 Prepared transformers dev environment"),Kg=h(),Us=a("li"),cn=a("ol"),Ys=a("li"),Qg=r("\u2610 Set up debugging environment of the original repository"),Vg=h(),Js=a("li"),yn=a("ol"),Xs=a("li"),Zg=r("\u2610 Created script that successfully runs forward pass using original repository and checkpoint"),ew=h(),Ks=a("li"),gn=a("ol"),Qs=a("li"),tw=r("\u2610 Successfully added the model skeleton to Transformers"),ow=h(),Vs=a("li"),wn=a("ol"),Zs=a("li"),rw=r("\u2610 Successfully converted original checkpoint to Transformers checkpoint"),nw=h(),ei=a("li"),vn=a("ol"),ti=a("li"),aw=r("\u2610 Successfully ran forward pass in Transformers that gives identical output to original checkpoint"),sw=h(),oi=a("li"),bn=a("ol"),ri=a("li"),iw=r("\u2610 Finished model tests in Transformers"),lw=h(),ni=a("li"),_n=a("ol"),ai=a("li"),dw=r("\u2610 Successfully added Tokenizer in Transformers"),hw=h(),si=a("li"),En=a("ol"),ii=a("li"),fw=r("\u2610 Run end-to-end integration tests"),mw=h(),li=a("li"),kn=a("ol"),di=a("li"),pw=r("\u2610 Finished docs"),uw=h(),hi=a("li"),Tn=a("ol"),fi=a("li"),cw=r("\u2610 Uploaded model weights to the hub"),yw=h(),mi=a("li"),$n=a("ol"),pi=a("li"),gw=r("\u2610 Submitted the pull request"),ww=h(),ui=a("li"),Pn=a("ol"),ci=a("li"),vw=r("\u2610 (Optional) Added a demo notebook"),vf=h(),O=a("p"),bw=r("To begin with, we usually recommend to start by getting a good theoretical understanding of "),yi=a("code"),_w=r("BrandNewBert"),Ew=r(`. However,
if you prefer to understand the theoretical aspects of the model `),gi=a("em"),kw=r("on-the-job"),Tw=r(`, then it is totally fine to directly dive
into the `),wi=a("code"),$w=r("BrandNewBert"),Pw=r(`\u2019s code-base. This option might suit you better, if your engineering skills are better than
your theoretical skill, if you have trouble understanding `),vi=a("code"),Iw=r("BrandNewBert"),Lw=r(`\u2019s paper, or if you just enjoy programming
much more than reading scientific papers.`),bf=h(),He=a("h3"),ct=a("a"),bi=a("span"),p(xo.$$.fragment),Aw=h(),_i=a("span"),Bw=r("1. (Optional) Theoretical aspects of BrandNewBert"),_f=h(),yt=a("p"),Mw=r("You should take some time to read "),Ei=a("em"),Nw=r("BrandNewBert\u2019s"),xw=r(` paper, if such descriptive work exists. There might be large
sections of the paper that are difficult to understand. If this is the case, this is fine - don\u2019t worry! The goal is
not to get a deep theoretical understanding of the paper, but to extract the necessary information required to
effectively re-implement the model in \u{1F917} Transformers. That being said, you don\u2019t have to spend too much time on the
theoretical aspects, but rather focus on the practical ones, namely:`),Ef=h(),j=a("ul"),We=a("li"),Ow=r("What type of model is "),ki=a("em"),jw=r("brand_new_bert"),Cw=r(`? BERT-like encoder-only model? GPT2-like decoder-only model? BART-like
encoder-decoder model? Look at the `),In=a("a"),qw=r("model_summary"),Sw=r(" if you\u2019re not familiar with the differences between those."),Dw=h(),Ge=a("li"),Rw=r("What are the applications of "),Ti=a("em"),Fw=r("brand_new_bert"),zw=r("? Text classification? Text generation? Seq2Seq tasks, "),$i=a("em"),Hw=r("e.g.,"),Ww=r(`
summarization?`),Gw=h(),Pi=a("li"),Uw=r("What is the novel feature of the model making it different from BERT/GPT-2/BART?"),Yw=h(),Ue=a("li"),Jw=r("Which of the already existing "),Oo=a("a"),Xw=r("\u{1F917} Transformers models"),Kw=r(` is most
similar to `),Ii=a("em"),Qw=r("brand_new_bert"),Vw=r("?"),Zw=h(),Li=a("li"),ev=r(`What type of tokenizer is used? A sentencepiece tokenizer? Word piece tokenizer? Is it the same tokenizer as used
for BERT or BART?`),kf=h(),Ln=a("p"),tv=r(`After you feel like you have gotten a good overview of the architecture of the model, you might want to write to the
Hugging Face team with any questions you might have. This might include questions regarding the model\u2019s architecture,
its attention layer, etc. We will be more than happy to help you.`),Tf=h(),Ye=a("h3"),gt=a("a"),Ai=a("span"),p(jo.$$.fragment),ov=h(),Bi=a("span"),rv=r("2. Next prepare your environment"),$f=h(),wt=a("ol"),Mi=a("li"),Co=a("p"),nv=r("Fork the "),qo=a("a"),av=r("repository"),sv=r(` by clicking on the \u2018Fork\u2019 button on the
repository\u2019s page. This creates a copy of the code under your GitHub user account.`),iv=h(),Ni=a("li"),So=a("p"),lv=r("Clone your "),xi=a("code"),dv=r("transformers"),hv=r(" fork to your local disk, and add the base repository as a remote:"),Pf=h(),p(Do.$$.fragment),If=h(),Ro=a("ol"),Oi=a("li"),fv=r("Set up a development environment, for instance by running the following command:"),Lf=h(),p(Fo.$$.fragment),Af=h(),An=a("p"),mv=r("and return to the parent directory"),Bf=h(),p(zo.$$.fragment),Mf=h(),Ho=a("ol"),Je=a("li"),pv=r("We recommend adding the PyTorch version of "),ji=a("em"),uv=r("brand_new_bert"),cv=r(` to Transformers. To install PyTorch, please follow the
instructions on `),Wo=a("a"),yv=r("https://pytorch.org/get-started/locally/"),gv=r("."),Nf=h(),Go=a("p"),Ci=a("strong"),wv=r("Note:"),vv=r(" You don\u2019t need to have CUDA installed. Making the new model work on CPU is sufficient."),xf=h(),Uo=a("ol"),Yo=a("li"),bv=r("To port "),qi=a("em"),_v=r("brand_new_bert"),Ev=r(", you will also need access to its original repository:"),Of=h(),p(Jo.$$.fragment),jf=h(),vt=a("p"),kv=r("Now you have set up a development environment to port "),Si=a("em"),Tv=r("brand_new_bert"),$v=r(" to \u{1F917} Transformers."),Cf=h(),Xe=a("h3"),bt=a("a"),Di=a("span"),p(Xo.$$.fragment),Pv=h(),Ri=a("span"),Iv=r("3.-4. Run a pretrained checkpoint using the original repository"),qf=h(),L=a("p"),Lv=r("At first, you will work on the original "),Fi=a("em"),Av=r("brand_new_bert"),Bv=r(` repository. Often, the original implementation is very
\u201Cresearchy\u201D. Meaning that documentation might be lacking and the code can be difficult to understand. But this should
be exactly your motivation to reimplement `),zi=a("em"),Mv=r("brand_new_bert"),Nv=r(". At Hugging Face, one of our main goals is to "),Hi=a("em"),xv=r(`make people
stand on the shoulders of giants`),Ov=r(` which translates here very well into taking a working model and rewriting it to make
it as `),Wi=a("strong"),jv=r("accessible, user-friendly, and beautiful"),Cv=r(` as possible. This is the number-one motivation to re-implement
models into \u{1F917} Transformers - trying to make complex new NLP technology accessible to `),Gi=a("strong"),qv=r("everybody"),Sv=r("."),Sf=h(),Bn=a("p"),Dv=r("You should start thereby by diving into the original repository."),Df=h(),_t=a("p"),Rv=r("Successfully running the official pretrained model in the original repository is often "),Ui=a("strong"),Fv=r("the most difficult"),zv=r(` step.
From our experience, it is very important to spend some time getting familiar with the original code-base. You need to
figure out the following:`),Rf=h(),A=a("ul"),Yi=a("li"),Hv=r("Where to find the pretrained weights?"),Wv=h(),Ji=a("li"),Gv=r("How to load the pretrained weights into the corresponding model?"),Uv=h(),Xi=a("li"),Yv=r("How to run the tokenizer independently from the model?"),Jv=h(),Ki=a("li"),Xv=r(`Trace one forward pass so that you know which classes and functions are required for a simple forward pass. Usually,
you only have to reimplement those functions.`),Kv=h(),F=a("li"),Qv=r(`Be able to locate the important components of the model: Where is the model\u2019s class? Are there model sub-classes,
`),Qi=a("em"),Vv=r("e.g."),Zv=r(` EncoderModel, DecoderModel? Where is the self-attention layer? Are there multiple different attention layers,
`),Vi=a("em"),eb=r("e.g."),tb=h(),Zi=a("em"),ob=r("self-attention"),rb=r(", "),el=a("em"),nb=r("cross-attention"),ab=r("\u2026?"),sb=h(),Ke=a("li"),ib=r("How can you debug the model in the original environment of the repo? Do you have to add "),tl=a("em"),lb=r("print"),db=r(` statements, can you
work with an interactive debugger like `),ol=a("em"),hb=r("ipdb"),fb=r(", or should you use an efficient IDE to debug the model, like PyCharm?"),Ff=h(),Et=a("p"),mb=r("It is very important that before you start the porting process, that you can "),rl=a("strong"),pb=r("efficiently"),ub=r(` debug code in the original
repository! Also, remember that you are working with an open-source library, so do not hesitate to open an issue, or
even a pull request in the original repository. The maintainers of this repository are most likely very happy about
someone looking into their code!`),zf=h(),Mn=a("p"),cb=r(`At this point, it is really up to you which debugging environment and strategy you prefer to use to debug the original
model. We strongly advise against setting up a costly GPU environment, but simply work on a CPU both when starting to
dive into the original repository and also when starting to write the \u{1F917} Transformers implementation of the model. Only
at the very end, when the model has already been successfully ported to \u{1F917} Transformers, one should verify that the
model also works as expected on GPU.`),Hf=h(),Nn=a("p"),yb=r("In general, there are two possible debugging environments for running the original model"),Wf=h(),kt=a("ul"),Ko=a("li"),Qo=a("a"),gb=r("Jupyter notebooks"),wb=r(" / "),Vo=a("a"),vb=r("google colab"),bb=h(),nl=a("li"),_b=r("Local python scripts."),Gf=h(),xn=a("p"),Eb=r(`Jupyter notebooks have the advantage that they allow for cell-by-cell execution which can be helpful to better split
logical components from one another and to have faster debugging cycles as intermediate results can be stored. Also,
notebooks are often easier to share with other contributors, which might be very helpful if you want to ask the Hugging
Face team for help. If you are familiar with Jupiter notebooks, we strongly recommend you to work with them.`),Uf=h(),Tt=a("p"),kb=r(`The obvious disadvantage of Jupyter notebooks is that if you are not used to working with them you will have to spend
some time adjusting to the new programming environment and that you might not be able to use your known debugging tools
anymore, like `),al=a("code"),Tb=r("ipdb"),$b=r("."),Yf=h(),$t=a("p"),Pb=r("For each code-base, a good first step is always to load a "),sl=a("strong"),Ib=r("small"),Lb=r(` pretrained checkpoint and to be able to reproduce a
single forward pass using a dummy integer vector of input IDs as an input. Such a script could look like this (in
pseudocode):`),Jf=h(),p(Zo.$$.fragment),Xf=h(),On=a("p"),Ab=r("Next, regarding the debugging strategy, there are generally a few from which to choose from:"),Kf=h(),Pt=a("ul"),il=a("li"),Bb=r(`Decompose the original model into many small testable components and run a forward pass on each of those for
verification`),Mb=h(),Qe=a("li"),Nb=r("Decompose the original model only into the original "),ll=a("em"),xb=r("tokenizer"),Ob=r(" and the original "),dl=a("em"),jb=r("model"),Cb=r(`, run a forward pass on
those, and use intermediate print statements or breakpoints for verification`),Qf=h(),jn=a("p"),qb=r(`Again, it is up to you which strategy to choose. Often, one or the other is advantageous depending on the original code
base.`),Vf=h(),It=a("p"),Sb=r("If the original code-base allows you to decompose the model into smaller sub-components, "),hl=a("em"),Db=r("e.g."),Rb=r(` if the original
code-base can easily be run in eager mode, it is usually worth the effort to do so. There are some important advantages
to taking the more difficult road in the beginning:`),Zf=h(),U=a("ul"),fl=a("li"),Fb=r(`at a later stage when comparing the original model to the Hugging Face implementation, you can verify automatically
for each component individually that the corresponding component of the \u{1F917} Transformers implementation matches instead
of relying on visual comparison via print statements`),zb=h(),ml=a("li"),Hb=r(`it can give you some rope to decompose the big problem of porting a model into smaller problems of just porting
individual components and thus structure your work better`),Wb=h(),pl=a("li"),Gb=r(`separating the model into logical meaningful components will help you to get a better overview of the model\u2019s design
and thus to better understand the model`),Ub=h(),ul=a("li"),Yb=r(`at a later stage those component-by-component tests help you to ensure that no regression occurs as you continue
changing your code`),em=h(),er=a("p"),tr=a("a"),Jb=r("Lysandre\u2019s"),Xb=r(` integration checks for ELECTRA
gives a nice example of how this can be done.`),tm=h(),Lt=a("p"),Kb=r(`However, if the original code-base is very complex or only allows intermediate components to be run in a compiled mode,
it might be too time-consuming or even impossible to separate the model into smaller testable sub-components. A good
example is `),or=a("a"),Qb=r("T5\u2019s MeshTensorFlow"),Vb=r(` library which is
very complex and does not offer a simple way to decompose the model into its sub-components. For such libraries, one
often relies on verifying print statements.`),om=h(),Cn=a("p"),Zb=r(`No matter which strategy you choose, the recommended procedure is often the same in that you should start to debug the
starting layers first and the ending layers last.`),rm=h(),qn=a("p"),e_=r(`It is recommended that you retrieve the output, either by print statements or sub-component functions, of the following
layers in the following order:`),nm=h(),B=a("ol"),cl=a("li"),t_=r("Retrieve the input IDs passed to the model"),o_=h(),yl=a("li"),r_=r("Retrieve the word embeddings"),n_=h(),gl=a("li"),a_=r("Retrieve the input of the first Transformer layer"),s_=h(),wl=a("li"),i_=r("Retrieve the output of the first Transformer layer"),l_=h(),vl=a("li"),d_=r("Retrieve the output of the following n - 1 Transformer layers"),h_=h(),bl=a("li"),f_=r("Retrieve the output of the whole BrandNewBert Model"),am=h(),Ve=a("p"),m_=r("Input IDs should thereby consists of an array of integers, "),_l=a("em"),p_=r("e.g."),u_=h(),El=a("code"),c_=r("input_ids = [0, 4, 4, 3, 2, 4, 1, 7, 19]"),sm=h(),Sn=a("p"),y_=r("The outputs of the following layers often consist of multi-dimensional float arrays and can look like this:"),im=h(),p(rr.$$.fragment),lm=h(),ge=a("p"),g_=r(`We expect that every model added to \u{1F917} Transformers passes a couple of integration tests, meaning that the original
model and the reimplemented version in \u{1F917} Transformers have to give the exact same output up to a precision of 0.001!
Since it is normal that the exact same model written in different libraries can give a slightly different output
depending on the library framework, we accept an error tolerance of 1e-3 (0.001). It is not enough if the model gives
nearly the same output, they have to be the almost identical. Therefore, you will certainly compare the intermediate
outputs of the \u{1F917} Transformers version multiple times against the intermediate outputs of the original implementation of
`),kl=a("em"),w_=r("brand_new_bert"),v_=r(" in which case an "),Tl=a("strong"),b_=r("efficient"),__=r(` debugging environment of the original repository is absolutely
important. Here is some advice is to make your debugging environment as efficient as possible.`),dm=h(),C=a("ul"),z=a("li"),E_=r(`Find the best way of debugging intermediate results. Is the original repository written in PyTorch? Then you should
probably take the time to write a longer script that decomposes the original model into smaller sub-components to
retrieve intermediate values. Is the original repository written in Tensorflow 1? Then you might have to rely on
TensorFlow print operations like `),nr=a("a"),k_=r("tf.print"),T_=r(` to output
intermediate values. Is the original repository written in Jax? Then make sure that the model is `),$l=a("strong"),$_=r("not jitted"),P_=r(` when
running the forward pass, `),Pl=a("em"),I_=r("e.g."),L_=r(" check-out "),ar=a("a"),A_=r("this link"),B_=r("."),M_=h(),Il=a("li"),N_=r(`Use the smallest pretrained checkpoint you can find. The smaller the checkpoint, the faster your debug cycle
becomes. It is not efficient if your pretrained model is so big that your forward pass takes more than 10 seconds.
In case only very large checkpoints are available, it might make more sense to create a dummy model in the new
environment with randomly initialized weights and save those weights for comparison with the \u{1F917} Transformers version
of your model`),x_=h(),T=a("li"),O_=r(`Make sure you are using the easiest way of calling a forward pass in the original repository. Ideally, you want to
find the function in the original repository that `),Ll=a("strong"),j_=r("only"),C_=r(" calls a single forward pass, "),Al=a("em"),q_=r("i.e."),S_=r(` that is often called
`),Bl=a("code"),D_=r("predict"),R_=r(", "),Ml=a("code"),F_=r("evaluate"),z_=r(", "),Nl=a("code"),H_=r("forward"),W_=r(" or "),xl=a("code"),G_=r("__call__"),U_=r(". You don\u2019t want to debug a function that calls "),Ol=a("code"),Y_=r("forward"),J_=r(`
multiple times, `),jl=a("em"),X_=r("e.g."),K_=r(" to generate text, like "),Cl=a("code"),Q_=r("autoregressive_sample"),V_=r(", "),ql=a("code"),Z_=r("generate"),e1=r("."),t1=h(),sr=a("li"),o1=r("Try to separate the tokenization from the model\u2019s "),Sl=a("em"),r1=r("forward"),n1=r(` pass. If the original repository shows examples where
you have to input a string, then try to find out where in the forward call the string input is changed to input ids
and start from this point. This might mean that you have to possibly write a small script yourself or change the
original code so that you can directly input the ids instead of an input string.`),a1=h(),me=a("li"),s1=r("Make sure that the model in your debugging setup is "),Dl=a("strong"),i1=r("not"),l1=r(` in training mode, which often causes the model to yield
random outputs due to multiple dropout layers in the model. Make sure that the forward pass in your debugging
environment is `),Rl=a("strong"),d1=r("deterministic"),h1=r(" so that the dropout layers are not used. Or use "),Fl=a("em"),f1=r("transformers.utils.set_seed"),m1=r(`
if the old and new implementations are in the same framework.`),hm=h(),At=a("p"),p1=r("The following section gives you more specific details/tips on how you can do this for "),zl=a("em"),u1=r("brand_new_bert"),c1=r("."),fm=h(),Ze=a("h3"),Bt=a("a"),Hl=a("span"),p(ir.$$.fragment),y1=h(),Wl=a("span"),g1=r("5.-14. Port BrandNewBert to \u{1F917} Transformers"),mm=h(),Dn=a("p"),w1=r("Next, you can finally start adding new code to \u{1F917} Transformers. Go into the clone of your \u{1F917} Transformers\u2019 fork:"),pm=h(),p(lr.$$.fragment),um=h(),Mt=a("p"),v1=r(`In the special case that you are adding a model whose architecture exactly matches the model architecture of an
existing model you only have to add a conversion script as described in `),Rn=a("a"),b1=r("this section"),_1=r(`.
In this case, you can just re-use the whole model architecture of the already existing model.`),cm=h(),Fn=a("p"),E1=r("Otherwise, let\u2019s start generating a new model. You have two choices here:"),ym=h(),Nt=a("ul"),zn=a("li"),Gl=a("code"),k1=r("transformers-cli add-new-model-like"),T1=r(" to add a new model like an existing one"),$1=h(),Hn=a("li"),Ul=a("code"),P1=r("transformers-cli add-new-model"),I1=r(" to add a new model from our template (will look like BERT or Bart depending on the type of model you select)"),gm=h(),we=a("p"),L1=r("In both cases, you will be prompted with a questionnaire to fill the basic information of your model. The second command requires to install "),Yl=a("code"),A1=r("cookiecutter"),B1=r(", you can find more information on it "),dr=a("a"),M1=r("here"),N1=r("."),wm=h(),Wn=a("p"),Jl=a("strong"),x1=r("Open a Pull Request on the main huggingface/transformers repo"),vm=h(),ve=a("p"),O1=r(`Before starting to adapt the automatically generated code, now is the time to open a \u201CWork in progress (WIP)\u201D pull
request, `),Xl=a("em"),j1=r("e.g."),C1=r(" \u201C[WIP] Add "),Kl=a("em"),q1=r("brand_new_bert"),S1=r(`\u201D, in \u{1F917} Transformers so that you and the Hugging Face team can work
side-by-side on integrating the model into \u{1F917} Transformers.`),bm=h(),Gn=a("p"),D1=r("You should do the following:"),_m=h(),Un=a("ol"),Ql=a("li"),R1=r("Create a branch with a descriptive name from your main branch"),Em=h(),p(hr.$$.fragment),km=h(),fr=a("ol"),Vl=a("li"),F1=r("Commit the automatically generated code:"),Tm=h(),p(mr.$$.fragment),$m=h(),pr=a("ol"),Zl=a("li"),z1=r("Fetch and rebase to current main"),Pm=h(),p(ur.$$.fragment),Im=h(),cr=a("ol"),ed=a("li"),H1=r("Push the changes to your account using:"),Lm=h(),p(yr.$$.fragment),Am=h(),et=a("ol"),td=a("li"),od=a("p"),W1=r(`Once you are satisfied, go to the webpage of your fork on GitHub. Click on \u201CPull request\u201D. Make sure to add the
GitHub handle of some members of the Hugging Face team as reviewers, so that the Hugging Face team gets notified for
future changes.`),G1=h(),rd=a("li"),nd=a("p"),U1=r("Change the PR into a draft by clicking on \u201CConvert to draft\u201D on the right of the GitHub pull request web page."),Bm=h(),Yn=a("p"),Y1=r(`In the following, whenever you have done some progress, don\u2019t forget to commit your work and push it to your account so
that it shows in the pull request. Additionally, you should make sure to update your work with the current main from
time to time by doing:`),Mm=h(),p(gr.$$.fragment),Nm=h(),Jn=a("p"),J1=r(`In general, all questions you might have regarding the model or your implementation should be asked in your PR and
discussed/solved in the PR. This way, the Hugging Face team will always be notified when you are committing new code or
if you have a question. It is often very helpful to point the Hugging Face team to your added code so that the Hugging
Face team can efficiently understand your problem or question.`),xm=h(),Xn=a("p"),X1=r(`To do so, you can go to the \u201CFiles changed\u201D tab where you see all of your changes, go to a line regarding which you
want to ask a question, and click on the \u201C+\u201D symbol to add a comment. Whenever a question or problem has been solved,
you can click on the \u201CResolve\u201D button of the created comment.`),Om=h(),Kn=a("p"),K1=r(`In the same way, the Hugging Face team will open comments when reviewing your code. We recommend asking most questions
on GitHub on your PR. For some very general questions that are not very useful for the public, feel free to ping the
Hugging Face team by Slack or email.`),jm=h(),Qn=a("p"),ad=a("strong"),Q1=r("5. Adapt the generated models code for brand_new_bert"),Cm=h(),be=a("p"),V1=r(`At first, we will focus only on the model itself and not care about the tokenizer. All the relevant code should be
found in the generated files `),sd=a("code"),Z1=r("src/transformers/models/brand_new_bert/modeling_brand_new_bert.py"),e0=r(` and
`),id=a("code"),t0=r("src/transformers/models/brand_new_bert/configuration_brand_new_bert.py"),o0=r("."),qm=h(),Y=a("p"),r0=r(`Now you can finally start coding :). The generated code in
`),ld=a("code"),n0=r("src/transformers/models/brand_new_bert/modeling_brand_new_bert.py"),a0=r(` will either have the same architecture as BERT if
it\u2019s an encoder-only model or BART if it\u2019s an encoder-decoder model. At this point, you should remind yourself what
you\u2019ve learned in the beginning about the theoretical aspects of the model: `),dd=a("em"),s0=r(`How is the model different from BERT or
BART?`),i0=r("\u201D. Implement those changes which often means to change the "),hd=a("em"),l0=r("self-attention"),d0=r(` layer, the order of the normalization
layer, etc\u2026 Again, it is often useful to look at the similar architecture of already existing models in Transformers to
get a better feeling of how your model should be implemented.`),Sm=h(),N=a("p"),fd=a("strong"),h0=r("Note"),f0=r(` that at this point, you don\u2019t have to be very sure that your code is fully correct or clean. Rather, it is
advised to add a first `),md=a("em"),m0=r("unclean"),p0=r(`, copy-pasted version of the original code to
`),pd=a("code"),u0=r("src/transformers/models/brand_new_bert/modeling_brand_new_bert.py"),c0=r(` until you feel like all the necessary code is
added. From our experience, it is much more efficient to quickly add a first version of the required code and
improve/correct the code iteratively with the conversion script as described in the next section. The only thing that
has to work at this point is that you can instantiate the \u{1F917} Transformers implementation of `),ud=a("em"),y0=r("brand_new_bert"),g0=r(", "),cd=a("em"),w0=r("i.e."),v0=r(` the
following command should work:`),Dm=h(),p(wr.$$.fragment),Rm=h(),_e=a("p"),b0=r("The above command will create a model according to the default parameters as defined in "),yd=a("code"),_0=r("BrandNewBertConfig()"),E0=r(` with
random weights, thus making sure that the `),gd=a("code"),k0=r("init()"),T0=r(" methods of all components works."),Fm=h(),Vn=a("p"),wd=a("strong"),$0=r("6. Write a conversion script"),zm=h(),J=a("p"),P0=r("Next, you should write a conversion script that lets you convert the checkpoint you used to debug "),vd=a("em"),I0=r("brand_new_bert"),L0=r(` in
the original repository to a checkpoint compatible with your just created \u{1F917} Transformers implementation of
`),bd=a("em"),A0=r("brand_new_bert"),B0=r(`. It is not advised to write the conversion script from scratch, but rather to look through already
existing conversion scripts in \u{1F917} Transformers for one that has been used to convert a similar model that was written in
the same framework as `),_d=a("em"),M0=r("brand_new_bert"),N0=r(`. Usually, it is enough to copy an already existing conversion script and
slightly adapt it for your use case. Don\u2019t hesitate to ask the Hugging Face team to point you to a similar already
existing conversion script for your model.`),Hm=h(),xt=a("ul"),Zn=a("li"),x0=r("If you are porting a model from TensorFlow to PyTorch, a good starting point might be BERT\u2019s conversion script "),vr=a("a"),O0=r("here"),j0=h(),ea=a("li"),C0=r("If you are porting a model from PyTorch to PyTorch, a good starting point might be BART\u2019s conversion script "),br=a("a"),q0=r("here"),Wm=h(),Ot=a("p"),S0=r(`In the following, we\u2019ll quickly explain how PyTorch models store layer weights and define layer names. In PyTorch, the
name of a layer is defined by the name of the class attribute you give the layer. Let\u2019s define a dummy model in
PyTorch, called `),Ed=a("code"),D0=r("SimpleModel"),R0=r(" as follows:"),Gm=h(),p(_r.$$.fragment),Um=h(),X=a("p"),F0=r("Now we can create an instance of this model definition which will fill all weights: "),kd=a("code"),z0=r("dense"),H0=r(", "),Td=a("code"),W0=r("intermediate"),G0=r(`,
`),$d=a("code"),U0=r("layer_norm"),Y0=r(" with random weights. We can print the model to see its architecture"),Ym=h(),p(Er.$$.fragment),Jm=h(),ta=a("p"),J0=r("This will print out the following:"),Xm=h(),p(kr.$$.fragment),Km=h(),oa=a("p"),X0=r(`We can see that the layer names are defined by the name of the class attribute in PyTorch. You can print out the weight
values of a specific layer:`),Qm=h(),p(Tr.$$.fragment),Vm=h(),ra=a("p"),K0=r("to see that the weights were randomly initialized"),Zm=h(),p($r.$$.fragment),ep=h(),Pr=a("p"),Q0=r(`In the conversion script, you should fill those randomly initialized weights with the exact weights of the
corresponding layer in the checkpoint. `),Pd=a("em"),V0=r("E.g."),tp=h(),p(Ir.$$.fragment),op=h(),Ee=a("p"),Z0=r(`While doing so, you must verify that each randomly initialized weight of your PyTorch model and its corresponding
pretrained checkpoint weight exactly match in both `),Id=a("strong"),e2=r("shape and name"),t2=r(". To do so, it is "),Ld=a("strong"),o2=r("necessary"),r2=r(` to add assert
statements for the shape and print out the names of the checkpoints weights. E.g. you should add statements like:`),rp=h(),p(Lr.$$.fragment),np=h(),Ar=a("p"),n2=r("Besides, you should also print out the names of both weights to make sure they match, "),Ad=a("em"),a2=r("e.g."),ap=h(),p(Br.$$.fragment),sp=h(),na=a("p"),s2=r(`If either the shape or the name doesn\u2019t match, you probably assigned the wrong checkpoint weight to a randomly
initialized layer of the \u{1F917} Transformers implementation.`),ip=h(),jt=a("p"),i2=r("An incorrect shape is most likely due to an incorrect setting of the config parameters in "),Bd=a("code"),l2=r("BrandNewBertConfig()"),d2=r(` that
do not exactly match those that were used for the checkpoint you want to convert. However, it could also be that
PyTorch\u2019s implementation of a layer requires the weight to be transposed beforehand.`),lp=h(),K=a("p"),h2=r("Finally, you should also check that "),Md=a("strong"),f2=r("all"),m2=r(` required weights are initialized and print out all checkpoint weights that
were not used for initialization to make sure the model is correctly converted. It is completely normal, that the
conversion trials fail with either a wrong shape statement or wrong name assignment. This is most likely because either
you used incorrect parameters in `),Nd=a("code"),p2=r("BrandNewBertConfig()"),u2=r(`, have a wrong architecture in the \u{1F917} Transformers
implementation, you have a bug in the `),xd=a("code"),c2=r("init()"),y2=r(` functions of one of the components of the \u{1F917} Transformers
implementation or you need to transpose one of the checkpoint weights.`),dp=h(),Q=a("p"),g2=r(`This step should be iterated with the previous step until all weights of the checkpoint are correctly loaded in the
Transformers model. Having correctly loaded the checkpoint into the \u{1F917} Transformers implementation, you can then save
the model under a folder of your choice `),Od=a("code"),w2=r("/path/to/converted/checkpoint/folder"),v2=r(` that should then contain both a
`),jd=a("code"),b2=r("pytorch_model.bin"),_2=r(" file and a "),Cd=a("code"),E2=r("config.json"),k2=r(" file:"),hp=h(),p(Mr.$$.fragment),fp=h(),aa=a("p"),qd=a("strong"),T2=r("7. Implement the forward pass"),mp=h(),Ct=a("p"),$2=r(`Having managed to correctly load the pretrained weights into the \u{1F917} Transformers implementation, you should now make
sure that the forward pass is correctly implemented. In `),sa=a("a"),P2=r("Get familiar with the original repository"),I2=r(`, you have already created a script that runs a forward
pass of the model using the original repository. Now you should write an analogous script using the \u{1F917} Transformers
implementation instead of the original one. It should look as follows:`),pp=h(),p(Nr.$$.fragment),up=h(),q=a("p"),L2=r(`It is very likely that the \u{1F917} Transformers implementation and the original model implementation don\u2019t give the exact
same output the very first time or that the forward pass throws an error. Don\u2019t be disappointed - it\u2019s expected! First,
you should make sure that the forward pass doesn\u2019t throw any errors. It often happens that the wrong dimensions are
used leading to a `),Sd=a("em"),A2=r("Dimensionality mismatch"),B2=r(" error or that the wrong data type object is used, "),Dd=a("em"),M2=r("e.g."),N2=h(),Rd=a("code"),x2=r("torch.long"),O2=r(`
instead of `),Fd=a("code"),j2=r("torch.float32"),C2=r(`. Don\u2019t hesitate to ask the Hugging Face team for help, if you don\u2019t manage to solve
certain errors.`),cp=h(),V=a("p"),q2=r(`The final part to make sure the \u{1F917} Transformers implementation works correctly is to ensure that the outputs are
equivalent to a precision of `),zd=a("code"),S2=r("1e-3"),D2=r(". First, you should ensure that the output shapes are identical, "),Hd=a("em"),R2=r("i.e."),F2=h(),Wd=a("code"),z2=r("outputs.shape"),H2=r(` should yield the same value for the script of the \u{1F917} Transformers implementation and the original
implementation. Next, you should make sure that the output values are identical as well. This one of the most difficult
parts of adding a new model. Common mistakes why the outputs are not identical are:`),yp=h(),Z=a("ul"),tt=a("li"),W2=r("Some layers were not added, "),Gd=a("em"),G2=r("i.e."),U2=r(" an "),Ud=a("em"),Y2=r("activation"),J2=r(" layer was not added, or the residual connection was forgotten"),X2=h(),Yd=a("li"),K2=r("The word embedding matrix was not tied"),Q2=h(),Jd=a("li"),V2=r("The wrong positional embeddings are used because the original implementation uses on offset"),Z2=h(),ee=a("li"),eE=r("Dropout is applied during the forward pass. To fix this make sure "),Xd=a("em"),tE=r("model.training is False"),oE=r(` and that no dropout
layer is falsely activated during the forward pass, `),Kd=a("em"),rE=r("i.e."),nE=r(" pass "),Qd=a("em"),aE=r("self.training"),sE=r(" to "),xr=a("a"),iE=r("PyTorch\u2019s functional dropout"),gp=h(),ke=a("p"),lE=r(`The best way to fix the problem is usually to look at the forward pass of the original implementation and the \u{1F917}
Transformers implementation side-by-side and check if there are any differences. Ideally, you should debug/print out
intermediate outputs of both implementations of the forward pass to find the exact position in the network where the \u{1F917}
Transformers implementation shows a different output than the original implementation. First, make sure that the
hard-coded `),Vd=a("code"),dE=r("input_ids"),hE=r(` in both scripts are identical. Next, verify that the outputs of the first transformation of
the `),Zd=a("code"),fE=r("input_ids"),mE=r(` (usually the word embeddings) are identical. And then work your way up to the very last layer of the
network. At some point, you will notice a difference between the two implementations, which should point you to the bug
in the \u{1F917} Transformers implementation. From our experience, a simple and efficient way is to add many print statements
in both the original implementation and \u{1F917} Transformers implementation, at the same positions in the network
respectively, and to successively remove print statements showing the same values for intermediate presentations.`),wp=h(),qt=a("p"),pE=r(`When you\u2019re confident that both implementations yield the same output, verifying the outputs with
`),eh=a("code"),uE=r("torch.allclose(original_output, output, atol=1e-3)"),cE=r(`, you\u2019re done with the most difficult part! Congratulations - the
work left to be done should be a cakewalk \u{1F60A}.`),vp=h(),ia=a("p"),th=a("strong"),yE=r("8. Adding all necessary model tests"),bp=h(),St=a("p"),gE=r(`At this point, you have successfully added a new model. However, it is very much possible that the model does not yet
fully comply with the required design. To make sure, the implementation is fully compatible with \u{1F917} Transformers, all
common tests should pass. The Cookiecutter should have automatically added a test file for your model, probably under
the same `),oh=a("code"),wE=r("tests/models/brand_new_bert/test_modeling_brand_new_bert.py"),vE=r(`. Run this test file to verify that all common
tests pass:`),_p=h(),p(Or.$$.fragment),Ep=h(),la=a("p"),bE=r("Having fixed all common tests, it is now crucial to ensure that all the nice work you have done is well tested, so that"),kp=h(),Dt=a("ul"),da=a("li"),_E=r("a) The community can easily understand your work by looking at specific tests of "),rh=a("em"),EE=r("brand_new_bert"),kE=h(),nh=a("li"),TE=r("b) Future changes to your model will not break any important feature of the model."),Tp=h(),Rt=a("p"),$E=r(`At first, integration tests should be added. Those integration tests essentially do the same as the debugging scripts
you used earlier to implement the model to \u{1F917} Transformers. A template of those model tests is already added by the
Cookiecutter, called `),ah=a("code"),PE=r("BrandNewBertModelIntegrationTests"),IE=r(` and only has to be filled out by you. To ensure that those
tests are passing, run`),$p=h(),p(jr.$$.fragment),Pp=h(),p(Ft.$$.fragment),Ip=h(),te=a("p"),LE=r("Second, all features that are special to "),sh=a("em"),AE=r("brand_new_bert"),BE=r(` should be tested additionally in a separate test under
`),ih=a("code"),ME=r("BrandNewBertModelTester"),NE=r("/`"),lh=a("code"),xE=r("BrandNewBertModelTest"),OE=r(`. This part is often forgotten but is extremely useful in two
ways:`),Lp=h(),zt=a("ul"),Cr=a("li"),jE=r(`It helps to transfer the knowledge you have acquired during the model addition to the community by showing how the
special features of `),dh=a("em"),CE=r("brand_new_bert"),qE=r(" should work."),SE=h(),hh=a("li"),DE=r("Future contributors can quickly test changes to the model by running those special tests."),Ap=h(),ha=a("p"),fh=a("strong"),RE=r("9. Implement the tokenizer"),Bp=h(),Ht=a("p"),FE=r("Next, we should add the tokenizer of "),mh=a("em"),zE=r("brand_new_bert"),HE=r(`. Usually, the tokenizer is equivalent or very similar to an
already existing tokenizer of \u{1F917} Transformers.`),Mp=h(),fa=a("p"),WE=r(`It is very important to find/extract the original tokenizer file and to manage to load this file into the \u{1F917}
Transformers\u2019 implementation of the tokenizer.`),Np=h(),ma=a("p"),GE=r("To ensure that the tokenizer works correctly, it is recommended to first create a script in the original repository\nthat inputs a string and returns the `input_ids\u201C. It could look similar to this (in pseudo-code):"),xp=h(),p(qr.$$.fragment),Op=h(),Wt=a("p"),UE=r(`You might have to take a deeper look again into the original repository to find the correct tokenizer function or you
might even have to do changes to your clone of the original repository to only output the `),ph=a("code"),YE=r("input_ids"),JE=r(`. Having written
a functional tokenization script that uses the original repository, an analogous script for \u{1F917} Transformers should be
created. It should look similar to this:`),jp=h(),p(Sr.$$.fragment),Cp=h(),Gt=a("p"),XE=r("When both "),uh=a("code"),KE=r("input_ids"),QE=r(" yield the same values, as a final step a tokenizer test file should also be added."),qp=h(),Te=a("p"),VE=r("Analogous to the modeling test files of "),ch=a("em"),ZE=r("brand_new_bert"),e3=r(", the tokenization test files of "),yh=a("em"),t3=r("brand_new_bert"),o3=r(` should
contain a couple of hard-coded integration tests.`),Sp=h(),pa=a("p"),gh=a("strong"),r3=r("10. Run End-to-end integration tests"),Dp=h(),oe=a("p"),n3=r(`Having added the tokenizer, you should also add a couple of end-to-end integration tests using both the model and the
tokenizer to `),wh=a("code"),a3=r("tests/models/brand_new_bert/test_modeling_brand_new_bert.py"),s3=r(` in \u{1F917} Transformers.
Such a test should show on a meaningful
text-to-text sample that the \u{1F917} Transformers implementation works as expected. A meaningful text-to-text sample can
include `),vh=a("em"),i3=r("e.g."),l3=r(` a source-to-target-translation pair, an article-to-summary pair, a question-to-answer pair, etc\u2026 If none
of the ported checkpoints has been fine-tuned on a downstream task it is enough to simply rely on the model tests. In a
final step to ensure that the model is fully functional, it is advised that you also run all tests on GPU. It can
happen that you forgot to add some `),bh=a("code"),d3=r(".to(self.device)"),h3=r(` statements to internal tensors of the model, which in such a
test would show in an error. In case you have no access to a GPU, the Hugging Face team can take care of running those
tests for you.`),Rp=h(),ua=a("p"),_h=a("strong"),f3=r("11. Add Docstring"),Fp=h(),re=a("p"),m3=r("Now, all the necessary functionality for "),Eh=a("em"),p3=r("brand_new_bert"),u3=r(` is added - you\u2019re almost done! The only thing left to add is
a nice docstring and a doc page. The Cookiecutter should have added a template file called
`),kh=a("code"),c3=r("docs/source/model_doc/brand_new_bert.rst"),y3=r(` that you should fill out. Users of your model will usually first look at
this page before using your model. Hence, the documentation must be understandable and concise. It is very useful for
the community to add some `),Th=a("em"),g3=r("Tips"),w3=r(` to show how the model should be used. Don\u2019t hesitate to ping the Hugging Face team
regarding the docstrings.`),zp=h(),$e=a("p"),v3=r("Next, make sure that the docstring added to "),$h=a("code"),b3=r("src/transformers/models/brand_new_bert/modeling_brand_new_bert.py"),_3=r(` is
correct and included all necessary inputs and outputs. We have a detailed guide about writing documentation and our docstring format `),ca=a("a"),E3=r("here"),k3=r(`. It is always to good to remind oneself that documentation should
be treated at least as carefully as the code in \u{1F917} Transformers since the documentation is usually the first contact
point of the community with the model.`),Hp=h(),ya=a("p"),Ph=a("strong"),T3=r("Code refactor"),Wp=h(),Ut=a("p"),$3=r("Great, now you have added all the necessary code for "),Ih=a("em"),P3=r("brand_new_bert"),I3=r(`. At this point, you should correct some potential
incorrect code style by running:`),Gp=h(),p(Dr.$$.fragment),Up=h(),ga=a("p"),L3=r("and verify that your coding style passes the quality check:"),Yp=h(),p(Rr.$$.fragment),Jp=h(),wa=a("p"),A3=r(`There are a couple of other very strict design tests in \u{1F917} Transformers that might still be failing, which shows up in
the tests of your pull request. This is often because of some missing information in the docstring or some incorrect
naming. The Hugging Face team will surely help you if you\u2019re stuck here.`),Xp=h(),va=a("p"),B3=r(`Lastly, it is always a good idea to refactor one\u2019s code after having ensured that the code works correctly. With all
tests passing, now it\u2019s a good time to go over the added code again and do some refactoring.`),Kp=h(),ba=a("p"),M3=r("You have now finished the coding part, congratulation! \u{1F389} You are Awesome! \u{1F60E}"),Qp=h(),_a=a("p"),Lh=a("strong"),N3=r("12. Upload the models to the model hub"),Vp=h(),S=a("p"),x3=r(`In this final part, you should convert and upload all checkpoints to the model hub and add a model card for each
uploaded model checkpoint. You can get familiar with the hub functionalities by reading our `),Ea=a("a"),O3=r("Model sharing and uploading Page"),j3=r(`. You should work alongside the Hugging Face team here to decide on a fitting name for each
checkpoint and to get the required access rights to be able to upload the model under the author\u2019s organization of
`),Ah=a("em"),C3=r("brand_new_bert"),q3=r(". The "),Bh=a("code"),S3=r("push_to_hub"),D3=r(" method, present in all models in "),Mh=a("code"),R3=r("transformers"),F3=r(", is a quick and efficient way to push your checkpoint to the hub. A little snippet is pasted below:"),Zp=h(),p(Fr.$$.fragment),eu=h(),Yt=a("p"),z3=r(`It is worth spending some time to create fitting model cards for each checkpoint. The model cards should highlight the
specific characteristics of this particular checkpoint, `),Nh=a("em"),H3=r("e.g."),W3=r(` On which dataset was the checkpoint
pretrained/fine-tuned on? On what down-stream task should the model be used? And also include some code on how to
correctly use the model.`),tu=h(),ka=a("p"),xh=a("strong"),G3=r("13. (Optional) Add notebook"),ou=h(),Jt=a("p"),U3=r("It is very helpful to add a notebook that showcases in-detail how "),Oh=a("em"),Y3=r("brand_new_bert"),J3=r(` can be used for inference and/or
fine-tuned on a downstream task. This is not mandatory to merge your PR, but very useful for the community.`),ru=h(),Ta=a("p"),jh=a("strong"),X3=r("14. Submit your finished PR"),nu=h(),$a=a("p"),K3=r(`You\u2019re done programming now and can move to the last step, which is getting your PR merged into main. Usually, the
Hugging Face team should have helped you already at this point, but it is worth taking some time to give your finished
PR a nice description and eventually add comments to your code, if you want to point out certain design choices to your
reviewer.`),au=h(),ot=a("h3"),Xt=a("a"),Ch=a("span"),p(zr.$$.fragment),Q3=h(),qh=a("span"),V3=r("Share your work!!"),su=h(),Pa=a("p"),Z3=r(`Now, it\u2019s time to get some credit from the community for your work! Having completed a model addition is a major
contribution to Transformers and the whole NLP community. Your code and the ported pre-trained models will certainly be
used by hundreds and possibly even thousands of developers and researchers. You should be proud of your work and share
your achievement with the community.`),iu=h(),Ia=a("p"),Sh=a("strong"),ek=r("You have made another model that is super easy to access for everyone in the community! \u{1F92F}"),this.h()},l(e){const l=_8('[data-svelte="svelte-1phssyn"]',document.head);P=s(l,"META",{name:!0,content:!0}),l.forEach(o),nt=f(e),M=s(e,"H1",{class:!0});var Hr=i(M);H=s(Hr,"A",{id:!0,class:!0,href:!0});var fk=i(H);Oe=s(fk,"SPAN",{});var mk=i(Oe);u(D.$$.fragment,mk),mk.forEach(o),fk.forEach(o),po=f(Hr),le=s(Hr,"SPAN",{});var pk=i(le);de=n(pk,"How to add a model to \u{1F917} Transformers?"),pk.forEach(o),Hr.forEach(o),uo=f(e),R=s(e,"P",{});var La=i(R);dc=n(La,`Adding a new model is often difficult and requires an in-depth knowledge of the \u{1F917} Transformers library and ideally also
of the model\u2019s original repository. At Hugging Face, we are trying to empower the community more and more to add models
independently. Thus, for some new models that the community wants to be added to \u{1F917} Transformers, we create a customized
`),Ka=s(La,"EM",{});var uk=i(Ka);hc=n(uk,"call-for-model-addition"),uk.forEach(o),fc=n(La,` that explains step-by-step how to add the requested model. With this
`),Qa=s(La,"EM",{});var ck=i(Qa);mc=n(ck,"call-for-model-addition"),ck.forEach(o),pc=n(La,`, we want to teach a motivated and experienced contributor of the community how to port a
model to \u{1F917} Transformers.`),La.forEach(o),Fh=f(e),at=s(e,"P",{});var du=i(at);uc=n(du,`If this sounds like something you would be interested in, feel free to check out the currently open
\u201Ccalls-for-model-addition\u201D `),co=s(du,"A",{href:!0,rel:!0});var yk=i(co);cc=n(yk,"here"),yk.forEach(o),yc=n(du,`
and to contact us.`),du.forEach(o),zh=f(e),Wr=s(e,"P",{});var gk=i(Wr);gc=n(gk,`If selected, you will then work closely with one member of the Hugging Face team to integrate the model into \u{1F917}
Transformers. By doing so, you will both gain a theoretical and deep practical understanding of the proposed model. But
more importantly, you will have made a major open-source contribution to \u{1F917} Transformers. Along the way, you will:`),gk.forEach(o),Hh=f(e),W=s(e,"UL",{});var Kt=i(W);Va=s(Kt,"LI",{});var wk=i(Va);wc=n(wk,"get insights into open-source best practices"),wk.forEach(o),vc=f(Kt),Za=s(Kt,"LI",{});var vk=i(Za);bc=n(vk,"understand the design principles of one of the most popular NLP libraries"),vk.forEach(o),_c=f(Kt),es=s(Kt,"LI",{});var bk=i(es);Ec=n(bk,"learn how to do efficiently test large NLP models"),bk.forEach(o),kc=f(Kt),he=s(Kt,"LI",{});var Qt=i(he);Tc=n(Qt,"learn how to integrate Python utilities like "),ts=s(Qt,"CODE",{});var _k=i(ts);$c=n(_k,"black"),_k.forEach(o),Pc=n(Qt,", "),os=s(Qt,"CODE",{});var Ek=i(os);Ic=n(Ek,"isort"),Ek.forEach(o),Lc=n(Qt,", "),rs=s(Qt,"CODE",{});var kk=i(rs);Ac=n(kk,"make fix-copies"),kk.forEach(o),Bc=n(Qt,` into a library to always
ensure clean and readable code`),Qt.forEach(o),Kt.forEach(o),Wh=f(e),st=s(e,"P",{});var hu=i(st);Mc=n(hu,`We are also more than happy if you want to add a model that cannot be found in the \u201Ccalls-for-model-addition\u201D folder.
The following sections explain in detail how to add a new model. It might also be very helpful to check out already
added models to see if those resemble the model you would like to add `),yo=s(hu,"A",{href:!0,rel:!0});var Tk=i(yo);Nc=n(Tk,"here"),Tk.forEach(o),xc=n(hu,"."),hu.forEach(o),Gh=f(e),Gr=s(e,"P",{});var $k=i(Gr);Oc=n($k,"To start, let\u2019s try to get a general overview of the Transformers library."),$k.forEach(o),Uh=f(e),je=s(e,"H2",{class:!0});var fu=i(je);it=s(fu,"A",{id:!0,class:!0,href:!0});var Pk=i(it);ns=s(Pk,"SPAN",{});var Ik=i(ns);u(go.$$.fragment,Ik),Ik.forEach(o),Pk.forEach(o),jc=f(fu),as=s(fu,"SPAN",{});var Lk=i(as);Cc=n(Lk,"General overview of \u{1F917} Transformers"),Lk.forEach(o),fu.forEach(o),Yh=f(e),Ur=s(e,"P",{});var Ak=i(Ur);qc=n(Ak,`First, you should get a general overview of \u{1F917} Transformers. \u{1F917} Transformers is a very opinionated library, so there is a
chance that you don\u2019t agree with some of the library\u2019s philosophies or design choices. From our experience, however, we
found that the fundamental design choices and philosophies of the library are crucial to efficiently scale \u{1F917}
Transformers while keeping maintenance costs at a reasonable level.`),Ak.forEach(o),Jh=f(e),lt=s(e,"P",{});var mu=i(lt);Sc=n(mu,"A good first starting point to better understand the library is to read the "),Yr=s(mu,"A",{href:!0});var Bk=i(Yr);Dc=n(Bk,"documentation of our philosophy"),Bk.forEach(o),Rc=n(mu,". As a result of our way of working, there are some choices that we try to apply to all models:"),mu.forEach(o),Xh=f(e),ce=s(e,"UL",{});var Aa=i(ce);ss=s(Aa,"LI",{});var Mk=i(ss);Fc=n(Mk,"Composition is generally favored over-abstraction"),Mk.forEach(o),zc=f(Aa),is=s(Aa,"LI",{});var Nk=i(is);Hc=n(Nk,"Duplicating code is not always bad if it strongly improves the readability or accessibility of a model"),Nk.forEach(o),Wc=f(Aa),wo=s(Aa,"LI",{});var pu=i(wo);Gc=n(pu,`Model files are as self-contained as possible so that when you read the code of a specific model, you ideally only
have to look into the respective `),ls=s(pu,"CODE",{});var xk=i(ls);Uc=n(xk,"modeling_....py"),xk.forEach(o),Yc=n(pu," file."),pu.forEach(o),Aa.forEach(o),Kh=f(e),dt=s(e,"P",{});var uu=i(dt);Jc=n(uu,"In our opinion, the library\u2019s code is not just a means to provide a product, "),ds=s(uu,"EM",{});var Ok=i(ds);Xc=n(Ok,"e.g."),Ok.forEach(o),Kc=n(uu,` the ability to use BERT for
inference, but also as the very product that we want to improve. Hence, when adding a model, the user is not only the
person that will use your model, but also everybody that will read, try to understand, and possibly tweak your code.`),uu.forEach(o),Qh=f(e),Jr=s(e,"P",{});var jk=i(Jr);Qc=n(jk,"With this in mind, let\u2019s go a bit deeper into the general library design."),jk.forEach(o),Vh=f(e),Ce=s(e,"H3",{class:!0});var cu=i(Ce);ht=s(cu,"A",{id:!0,class:!0,href:!0});var Ck=i(ht);hs=s(Ck,"SPAN",{});var qk=i(hs);u(vo.$$.fragment,qk),qk.forEach(o),Ck.forEach(o),Vc=f(cu),fs=s(cu,"SPAN",{});var Sk=i(fs);Zc=n(Sk,"Overview of models"),Sk.forEach(o),cu.forEach(o),Zh=f(e),G=s(e,"P",{});var Vt=i(G);ey=n(Vt,`To successfully add a model, it is important to understand the interaction between your model and its config,
`),Xr=s(Vt,"A",{href:!0});var Dk=i(Xr);ty=n(Dk,"PreTrainedModel"),Dk.forEach(o),oy=n(Vt,", and "),Kr=s(Vt,"A",{href:!0});var Rk=i(Kr);ry=n(Rk,"PretrainedConfig"),Rk.forEach(o),ny=n(Vt,`. For exemplary purposes, we will
call the model to be added to \u{1F917} Transformers `),ms=s(Vt,"CODE",{});var Fk=i(ms);ay=n(Fk,"BrandNewBert"),Fk.forEach(o),sy=n(Vt,"."),Vt.forEach(o),ef=f(e),Qr=s(e,"P",{});var zk=i(Qr);iy=n(zk,"Let\u2019s take a look:"),zk.forEach(o),tf=f(e),Vr=s(e,"IMG",{src:!0}),of=f(e),v=s(e,"P",{});var _=i(v);ly=n(_,`As you can see, we do make use of inheritance in \u{1F917} Transformers, but we keep the level of abstraction to an absolute
minimum. There are never more than two levels of abstraction for any model in the library. `),ps=s(_,"CODE",{});var Hk=i(ps);dy=n(Hk,"BrandNewBertModel"),Hk.forEach(o),hy=n(_,`
inherits from `),us=s(_,"CODE",{});var Wk=i(us);fy=n(Wk,"BrandNewBertPreTrainedModel"),Wk.forEach(o),my=n(_," which in turn inherits from "),Zr=s(_,"A",{href:!0});var Gk=i(Zr);py=n(Gk,"PreTrainedModel"),Gk.forEach(o),uy=n(_,` and
that\u2019s it. As a general rule, we want to make sure that a new model only depends on
`),en=s(_,"A",{href:!0});var Uk=i(en);cy=n(Uk,"PreTrainedModel"),Uk.forEach(o),yy=n(_,`. The important functionalities that are automatically provided to every new
model are `),tn=s(_,"A",{href:!0});var Yk=i(tn);gy=n(Yk,"from_pretrained()"),Yk.forEach(o),wy=n(_,` and
`),on=s(_,"A",{href:!0});var Jk=i(on);vy=n(Jk,"save_pretrained()"),Jk.forEach(o),by=n(_,`, which are used for serialization and deserialization. All of the
other important functionalities, such as `),cs=s(_,"CODE",{});var Xk=i(cs);_y=n(Xk,"BrandNewBertModel.forward"),Xk.forEach(o),Ey=n(_,` should be completely defined in the new
`),ys=s(_,"CODE",{});var Kk=i(ys);ky=n(Kk,"modeling_brand_new_bert.py"),Kk.forEach(o),Ty=n(_,` script. Next, we want to make sure that a model with a specific head layer, such as
`),gs=s(_,"CODE",{});var Qk=i(gs);$y=n(Qk,"BrandNewBertForMaskedLM"),Qk.forEach(o),Py=n(_," does not inherit from "),ws=s(_,"CODE",{});var Vk=i(ws);Iy=n(Vk,"BrandNewBertModel"),Vk.forEach(o),Ly=n(_,", but rather uses "),vs=s(_,"CODE",{});var Zk=i(vs);Ay=n(Zk,"BrandNewBertModel"),Zk.forEach(o),By=n(_,`
as a component that can be called in its forward pass to keep the level of abstraction low. Every new model requires a
configuration class, called `),bs=s(_,"CODE",{});var e4=i(bs);My=n(e4,"BrandNewBertConfig"),e4.forEach(o),Ny=n(_,`. This configuration is always stored as an attribute in
`),rn=s(_,"A",{href:!0});var t4=i(rn);xy=n(t4,"PreTrainedModel"),t4.forEach(o),Oy=n(_,", and thus can be accessed via the "),_s=s(_,"CODE",{});var o4=i(_s);jy=n(o4,"config"),o4.forEach(o),Cy=n(_,` attribute for all classes
inheriting from `),Es=s(_,"CODE",{});var r4=i(Es);qy=n(r4,"BrandNewBertPreTrainedModel"),r4.forEach(o),Sy=n(_,":"),_.forEach(o),rf=f(e),u(bo.$$.fragment,e),nf=f(e),I=s(e,"P",{});var ne=i(I);Dy=n(ne,`Similar to the model, the configuration inherits basic serialization and deserialization functionalities from
`),nn=s(ne,"A",{href:!0});var n4=i(nn);Ry=n(n4,"PretrainedConfig"),n4.forEach(o),Fy=n(ne,`. Note that the configuration and the model are always serialized into two
different formats - the model to a `),ks=s(ne,"EM",{});var a4=i(ks);zy=n(a4,"pytorch_model.bin"),a4.forEach(o),Hy=n(ne," file and the configuration to a "),Ts=s(ne,"EM",{});var s4=i(Ts);Wy=n(s4,"config.json"),s4.forEach(o),Gy=n(ne,` file. Calling
`),an=s(ne,"A",{href:!0});var i4=i(an);Uy=n(i4,"save_pretrained()"),i4.forEach(o),Yy=n(ne,` will automatically call
`),sn=s(ne,"A",{href:!0});var l4=i(sn);Jy=n(l4,"save_pretrained()"),l4.forEach(o),Xy=n(ne,", so that both model and configuration are saved."),ne.forEach(o),af=f(e),qe=s(e,"H3",{class:!0});var yu=i(qe);ft=s(yu,"A",{id:!0,class:!0,href:!0});var d4=i(ft);$s=s(d4,"SPAN",{});var h4=i($s);u(_o.$$.fragment,h4),h4.forEach(o),d4.forEach(o),Ky=f(yu),Ps=s(yu,"SPAN",{});var f4=i(Ps);Qy=n(f4,"Code style"),f4.forEach(o),yu.forEach(o),sf=f(e),ln=s(e,"P",{});var m4=i(ln);Vy=n(m4,`When coding your new model, keep in mind that Transformers is an opinionated library and we have a few quirks of our
own regarding how code should be written :-)`),m4.forEach(o),lf=f(e),x=s(e,"OL",{});var Pe=i(x);Se=s(Pe,"LI",{});var Ba=i(Se);Zy=n(Ba,`The forward pass of your model should be fully written in the modeling file while being fully independent of other
models in the library. If you want to reuse a block from another model, copy the code and paste it with a
`),Is=s(Ba,"CODE",{});var p4=i(Is);eg=n(p4,"# Copied from"),p4.forEach(o),tg=n(Ba," comment on top (see "),Eo=s(Ba,"A",{href:!0,rel:!0});var u4=i(Eo);og=n(u4,"here"),u4.forEach(o),rg=n(Ba,`
for a good example).`),Ba.forEach(o),ng=f(Pe),De=s(Pe,"LI",{});var Ma=i(De);ag=n(Ma,`The code should be fully understandable, even by a non-native English speaker. This means you should pick
descriptive variable names and avoid abbreviations. As an example, `),Ls=s(Ma,"CODE",{});var c4=i(Ls);sg=n(c4,"activation"),c4.forEach(o),ig=n(Ma," is preferred to "),As=s(Ma,"CODE",{});var y4=i(As);lg=n(y4,"act"),y4.forEach(o),dg=n(Ma,`.
One-letter variable names are strongly discouraged unless it\u2019s an index in a for loop.`),Ma.forEach(o),hg=f(Pe),Bs=s(Pe,"LI",{});var g4=i(Bs);fg=n(g4,"More generally we prefer longer explicit code to short magical one."),g4.forEach(o),mg=f(Pe),Re=s(Pe,"LI",{});var Na=i(Re);pg=n(Na,"Avoid subclassing "),Ms=s(Na,"CODE",{});var w4=i(Ms);ug=n(w4,"nn.Sequential"),w4.forEach(o),cg=n(Na," in PyTorch but subclass "),Ns=s(Na,"CODE",{});var v4=i(Ns);yg=n(v4,"nn.Module"),v4.forEach(o),gg=n(Na,` and write the forward pass, so that anyone
using your code can quickly debug it by adding print statements or breaking points.`),Na.forEach(o),wg=f(Pe),xs=s(Pe,"LI",{});var b4=i(xs);vg=n(b4,`Your function signature should be type-annotated. For the rest, good variable names are way more readable and
understandable than type annotations.`),b4.forEach(o),Pe.forEach(o),df=f(e),Fe=s(e,"H3",{class:!0});var gu=i(Fe);mt=s(gu,"A",{id:!0,class:!0,href:!0});var _4=i(mt);Os=s(_4,"SPAN",{});var E4=i(Os);u(ko.$$.fragment,E4),E4.forEach(o),_4.forEach(o),bg=f(gu),js=s(gu,"SPAN",{});var k4=i(js);_g=n(k4,"Overview of tokenizers"),k4.forEach(o),gu.forEach(o),hf=f(e),dn=s(e,"P",{});var T4=i(dn);Eg=n(T4,"Not quite ready yet :-( This section will be added soon!"),T4.forEach(o),ff=f(e),ze=s(e,"H2",{class:!0});var wu=i(ze);pt=s(wu,"A",{id:!0,class:!0,href:!0});var $4=i(pt);Cs=s($4,"SPAN",{});var P4=i(Cs);u(To.$$.fragment,P4),P4.forEach(o),$4.forEach(o),kg=f(wu),qs=s(wu,"SPAN",{});var I4=i(qs);Tg=n(I4,"Step-by-step recipe to add a model to \u{1F917} Transformers"),I4.forEach(o),wu.forEach(o),mf=f(e),hn=s(e,"P",{});var L4=i(hn);$g=n(L4,`Everyone has different preferences of how to port a model so it can be very helpful for you to take a look at summaries
of how other contributors ported models to Hugging Face. Here is a list of community blog posts on how to port a model:`),L4.forEach(o),pf=f(e),ut=s(e,"OL",{});var vu=i(ut);$o=s(vu,"LI",{});var bu=i($o);Po=s(bu,"A",{href:!0,rel:!0});var A4=i(Po);Pg=n(A4,"Porting GPT2 Model"),A4.forEach(o),Ig=n(bu," by "),Io=s(bu,"A",{href:!0,rel:!0});var B4=i(Io);Lg=n(B4,"Thomas"),B4.forEach(o),bu.forEach(o),Ag=f(vu),Lo=s(vu,"LI",{});var _u=i(Lo);Ao=s(_u,"A",{href:!0,rel:!0});var M4=i(Ao);Bg=n(M4,"Porting WMT19 MT Model"),M4.forEach(o),Mg=n(_u," by "),Bo=s(_u,"A",{href:!0,rel:!0});var N4=i(Bo);Ng=n(N4,"Stas"),N4.forEach(o),_u.forEach(o),vu.forEach(o),uf=f(e),fn=s(e,"P",{});var x4=i(fn);xg=n(x4,"From experience, we can tell you that the most important things to keep in mind when adding a model are:"),x4.forEach(o),cf=f(e),ye=s(e,"UL",{});var xa=i(ye);fe=s(xa,"LI",{});var Zt=i(fe);Og=n(Zt,`Don\u2019t reinvent the wheel! Most parts of the code you will add for the new \u{1F917} Transformers model already exist
somewhere in \u{1F917} Transformers. Take some time to find similar, already existing models and tokenizers you can copy
from. `),Mo=s(Zt,"A",{href:!0,rel:!0});var O4=i(Mo);jg=n(O4,"grep"),O4.forEach(o),Cg=n(Zt," and "),No=s(Zt,"A",{href:!0,rel:!0});var j4=i(No);qg=n(j4,"rg"),j4.forEach(o),Sg=n(Zt,` are your
friends. Note that it might very well happen that your model\u2019s tokenizer is based on one model implementation, and
your model\u2019s modeling code on another one. `),Ss=s(Zt,"EM",{});var C4=i(Ss);Dg=n(C4,"E.g."),C4.forEach(o),Rg=n(Zt,` FSMT\u2019s modeling code is based on BART, while FSMT\u2019s tokenizer code
is based on XLM.`),Zt.forEach(o),Fg=f(xa),Ds=s(xa,"LI",{});var q4=i(Ds);zg=n(q4,`It\u2019s more of an engineering challenge than a scientific challenge. You should spend more time on creating an
efficient debugging environment than trying to understand all theoretical aspects of the model in the paper.`),q4.forEach(o),Hg=f(xa),Rs=s(xa,"LI",{});var S4=i(Rs);Wg=n(S4,`Ask for help, when you\u2019re stuck! Models are the core component of \u{1F917} Transformers so that we at Hugging Face are more
than happy to help you at every step to add your model. Don\u2019t hesitate to ask if you notice you are not making
progress.`),S4.forEach(o),xa.forEach(o),yf=f(e),mn=s(e,"P",{});var D4=i(mn);Gg=n(D4,"In the following, we try to give you a general recipe that we found most useful when porting a model to \u{1F917} Transformers."),D4.forEach(o),gf=f(e),pn=s(e,"P",{});var R4=i(pn);Ug=n(R4,`The following list is a summary of everything that has to be done to add a model and can be used by you as a To-Do
List:`),R4.forEach(o),wf=f(e),b=s(e,"UL",{});var k=i(b);Fs=s(k,"LI",{});var F4=i(Fs);zs=s(F4,"OL",{});var z4=i(zs);Hs=s(z4,"LI",{});var H4=i(Hs);Yg=n(H4,"\u2610 (Optional) Understood theoretical aspects"),H4.forEach(o),z4.forEach(o),F4.forEach(o),Jg=f(k),Ws=s(k,"LI",{});var W4=i(Ws);un=s(W4,"OL",{start:!0});var G4=i(un);Gs=s(G4,"LI",{});var U4=i(Gs);Xg=n(U4,"\u2610 Prepared transformers dev environment"),U4.forEach(o),G4.forEach(o),W4.forEach(o),Kg=f(k),Us=s(k,"LI",{});var Y4=i(Us);cn=s(Y4,"OL",{start:!0});var J4=i(cn);Ys=s(J4,"LI",{});var X4=i(Ys);Qg=n(X4,"\u2610 Set up debugging environment of the original repository"),X4.forEach(o),J4.forEach(o),Y4.forEach(o),Vg=f(k),Js=s(k,"LI",{});var K4=i(Js);yn=s(K4,"OL",{start:!0});var Q4=i(yn);Xs=s(Q4,"LI",{});var V4=i(Xs);Zg=n(V4,"\u2610 Created script that successfully runs forward pass using original repository and checkpoint"),V4.forEach(o),Q4.forEach(o),K4.forEach(o),ew=f(k),Ks=s(k,"LI",{});var Z4=i(Ks);gn=s(Z4,"OL",{start:!0});var e5=i(gn);Qs=s(e5,"LI",{});var t5=i(Qs);tw=n(t5,"\u2610 Successfully added the model skeleton to Transformers"),t5.forEach(o),e5.forEach(o),Z4.forEach(o),ow=f(k),Vs=s(k,"LI",{});var o5=i(Vs);wn=s(o5,"OL",{start:!0});var r5=i(wn);Zs=s(r5,"LI",{});var n5=i(Zs);rw=n(n5,"\u2610 Successfully converted original checkpoint to Transformers checkpoint"),n5.forEach(o),r5.forEach(o),o5.forEach(o),nw=f(k),ei=s(k,"LI",{});var a5=i(ei);vn=s(a5,"OL",{start:!0});var s5=i(vn);ti=s(s5,"LI",{});var i5=i(ti);aw=n(i5,"\u2610 Successfully ran forward pass in Transformers that gives identical output to original checkpoint"),i5.forEach(o),s5.forEach(o),a5.forEach(o),sw=f(k),oi=s(k,"LI",{});var l5=i(oi);bn=s(l5,"OL",{start:!0});var d5=i(bn);ri=s(d5,"LI",{});var h5=i(ri);iw=n(h5,"\u2610 Finished model tests in Transformers"),h5.forEach(o),d5.forEach(o),l5.forEach(o),lw=f(k),ni=s(k,"LI",{});var f5=i(ni);_n=s(f5,"OL",{start:!0});var m5=i(_n);ai=s(m5,"LI",{});var p5=i(ai);dw=n(p5,"\u2610 Successfully added Tokenizer in Transformers"),p5.forEach(o),m5.forEach(o),f5.forEach(o),hw=f(k),si=s(k,"LI",{});var u5=i(si);En=s(u5,"OL",{start:!0});var c5=i(En);ii=s(c5,"LI",{});var y5=i(ii);fw=n(y5,"\u2610 Run end-to-end integration tests"),y5.forEach(o),c5.forEach(o),u5.forEach(o),mw=f(k),li=s(k,"LI",{});var g5=i(li);kn=s(g5,"OL",{start:!0});var w5=i(kn);di=s(w5,"LI",{});var v5=i(di);pw=n(v5,"\u2610 Finished docs"),v5.forEach(o),w5.forEach(o),g5.forEach(o),uw=f(k),hi=s(k,"LI",{});var b5=i(hi);Tn=s(b5,"OL",{start:!0});var _5=i(Tn);fi=s(_5,"LI",{});var E5=i(fi);cw=n(E5,"\u2610 Uploaded model weights to the hub"),E5.forEach(o),_5.forEach(o),b5.forEach(o),yw=f(k),mi=s(k,"LI",{});var k5=i(mi);$n=s(k5,"OL",{start:!0});var T5=i($n);pi=s(T5,"LI",{});var $5=i(pi);gw=n($5,"\u2610 Submitted the pull request"),$5.forEach(o),T5.forEach(o),k5.forEach(o),ww=f(k),ui=s(k,"LI",{});var P5=i(ui);Pn=s(P5,"OL",{start:!0});var I5=i(Pn);ci=s(I5,"LI",{});var L5=i(ci);vw=n(L5,"\u2610 (Optional) Added a demo notebook"),L5.forEach(o),I5.forEach(o),P5.forEach(o),k.forEach(o),vf=f(e),O=s(e,"P",{});var Ie=i(O);bw=n(Ie,"To begin with, we usually recommend to start by getting a good theoretical understanding of "),yi=s(Ie,"CODE",{});var A5=i(yi);_w=n(A5,"BrandNewBert"),A5.forEach(o),Ew=n(Ie,`. However,
if you prefer to understand the theoretical aspects of the model `),gi=s(Ie,"EM",{});var B5=i(gi);kw=n(B5,"on-the-job"),B5.forEach(o),Tw=n(Ie,`, then it is totally fine to directly dive
into the `),wi=s(Ie,"CODE",{});var M5=i(wi);$w=n(M5,"BrandNewBert"),M5.forEach(o),Pw=n(Ie,`\u2019s code-base. This option might suit you better, if your engineering skills are better than
your theoretical skill, if you have trouble understanding `),vi=s(Ie,"CODE",{});var N5=i(vi);Iw=n(N5,"BrandNewBert"),N5.forEach(o),Lw=n(Ie,`\u2019s paper, or if you just enjoy programming
much more than reading scientific papers.`),Ie.forEach(o),bf=f(e),He=s(e,"H3",{class:!0});var Eu=i(He);ct=s(Eu,"A",{id:!0,class:!0,href:!0});var x5=i(ct);bi=s(x5,"SPAN",{});var O5=i(bi);u(xo.$$.fragment,O5),O5.forEach(o),x5.forEach(o),Aw=f(Eu),_i=s(Eu,"SPAN",{});var j5=i(_i);Bw=n(j5,"1. (Optional) Theoretical aspects of BrandNewBert"),j5.forEach(o),Eu.forEach(o),_f=f(e),yt=s(e,"P",{});var ku=i(yt);Mw=n(ku,"You should take some time to read "),Ei=s(ku,"EM",{});var C5=i(Ei);Nw=n(C5,"BrandNewBert\u2019s"),C5.forEach(o),xw=n(ku,` paper, if such descriptive work exists. There might be large
sections of the paper that are difficult to understand. If this is the case, this is fine - don\u2019t worry! The goal is
not to get a deep theoretical understanding of the paper, but to extract the necessary information required to
effectively re-implement the model in \u{1F917} Transformers. That being said, you don\u2019t have to spend too much time on the
theoretical aspects, but rather focus on the practical ones, namely:`),ku.forEach(o),Ef=f(e),j=s(e,"UL",{});var Le=i(j);We=s(Le,"LI",{});var Oa=i(We);Ow=n(Oa,"What type of model is "),ki=s(Oa,"EM",{});var q5=i(ki);jw=n(q5,"brand_new_bert"),q5.forEach(o),Cw=n(Oa,`? BERT-like encoder-only model? GPT2-like decoder-only model? BART-like
encoder-decoder model? Look at the `),In=s(Oa,"A",{href:!0});var S5=i(In);qw=n(S5,"model_summary"),S5.forEach(o),Sw=n(Oa," if you\u2019re not familiar with the differences between those."),Oa.forEach(o),Dw=f(Le),Ge=s(Le,"LI",{});var ja=i(Ge);Rw=n(ja,"What are the applications of "),Ti=s(ja,"EM",{});var D5=i(Ti);Fw=n(D5,"brand_new_bert"),D5.forEach(o),zw=n(ja,"? Text classification? Text generation? Seq2Seq tasks, "),$i=s(ja,"EM",{});var R5=i($i);Hw=n(R5,"e.g.,"),R5.forEach(o),Ww=n(ja,`
summarization?`),ja.forEach(o),Gw=f(Le),Pi=s(Le,"LI",{});var F5=i(Pi);Uw=n(F5,"What is the novel feature of the model making it different from BERT/GPT-2/BART?"),F5.forEach(o),Yw=f(Le),Ue=s(Le,"LI",{});var Ca=i(Ue);Jw=n(Ca,"Which of the already existing "),Oo=s(Ca,"A",{href:!0,rel:!0});var z5=i(Oo);Xw=n(z5,"\u{1F917} Transformers models"),z5.forEach(o),Kw=n(Ca,` is most
similar to `),Ii=s(Ca,"EM",{});var H5=i(Ii);Qw=n(H5,"brand_new_bert"),H5.forEach(o),Vw=n(Ca,"?"),Ca.forEach(o),Zw=f(Le),Li=s(Le,"LI",{});var W5=i(Li);ev=n(W5,`What type of tokenizer is used? A sentencepiece tokenizer? Word piece tokenizer? Is it the same tokenizer as used
for BERT or BART?`),W5.forEach(o),Le.forEach(o),kf=f(e),Ln=s(e,"P",{});var G5=i(Ln);tv=n(G5,`After you feel like you have gotten a good overview of the architecture of the model, you might want to write to the
Hugging Face team with any questions you might have. This might include questions regarding the model\u2019s architecture,
its attention layer, etc. We will be more than happy to help you.`),G5.forEach(o),Tf=f(e),Ye=s(e,"H3",{class:!0});var Tu=i(Ye);gt=s(Tu,"A",{id:!0,class:!0,href:!0});var U5=i(gt);Ai=s(U5,"SPAN",{});var Y5=i(Ai);u(jo.$$.fragment,Y5),Y5.forEach(o),U5.forEach(o),ov=f(Tu),Bi=s(Tu,"SPAN",{});var J5=i(Bi);rv=n(J5,"2. Next prepare your environment"),J5.forEach(o),Tu.forEach(o),$f=f(e),wt=s(e,"OL",{});var $u=i(wt);Mi=s($u,"LI",{});var X5=i(Mi);Co=s(X5,"P",{});var Pu=i(Co);nv=n(Pu,"Fork the "),qo=s(Pu,"A",{href:!0,rel:!0});var K5=i(qo);av=n(K5,"repository"),K5.forEach(o),sv=n(Pu,` by clicking on the \u2018Fork\u2019 button on the
repository\u2019s page. This creates a copy of the code under your GitHub user account.`),Pu.forEach(o),X5.forEach(o),iv=f($u),Ni=s($u,"LI",{});var Q5=i(Ni);So=s(Q5,"P",{});var Iu=i(So);lv=n(Iu,"Clone your "),xi=s(Iu,"CODE",{});var V5=i(xi);dv=n(V5,"transformers"),V5.forEach(o),hv=n(Iu," fork to your local disk, and add the base repository as a remote:"),Iu.forEach(o),Q5.forEach(o),$u.forEach(o),Pf=f(e),u(Do.$$.fragment,e),If=f(e),Ro=s(e,"OL",{start:!0});var Z5=i(Ro);Oi=s(Z5,"LI",{});var e6=i(Oi);fv=n(e6,"Set up a development environment, for instance by running the following command:"),e6.forEach(o),Z5.forEach(o),Lf=f(e),u(Fo.$$.fragment,e),Af=f(e),An=s(e,"P",{});var t6=i(An);mv=n(t6,"and return to the parent directory"),t6.forEach(o),Bf=f(e),u(zo.$$.fragment,e),Mf=f(e),Ho=s(e,"OL",{start:!0});var o6=i(Ho);Je=s(o6,"LI",{});var qa=i(Je);pv=n(qa,"We recommend adding the PyTorch version of "),ji=s(qa,"EM",{});var r6=i(ji);uv=n(r6,"brand_new_bert"),r6.forEach(o),cv=n(qa,` to Transformers. To install PyTorch, please follow the
instructions on `),Wo=s(qa,"A",{href:!0,rel:!0});var n6=i(Wo);yv=n(n6,"https://pytorch.org/get-started/locally/"),n6.forEach(o),gv=n(qa,"."),qa.forEach(o),o6.forEach(o),Nf=f(e),Go=s(e,"P",{});var tk=i(Go);Ci=s(tk,"STRONG",{});var a6=i(Ci);wv=n(a6,"Note:"),a6.forEach(o),vv=n(tk," You don\u2019t need to have CUDA installed. Making the new model work on CPU is sufficient."),tk.forEach(o),xf=f(e),Uo=s(e,"OL",{start:!0});var s6=i(Uo);Yo=s(s6,"LI",{});var Lu=i(Yo);bv=n(Lu,"To port "),qi=s(Lu,"EM",{});var i6=i(qi);_v=n(i6,"brand_new_bert"),i6.forEach(o),Ev=n(Lu,", you will also need access to its original repository:"),Lu.forEach(o),s6.forEach(o),Of=f(e),u(Jo.$$.fragment,e),jf=f(e),vt=s(e,"P",{});var Au=i(vt);kv=n(Au,"Now you have set up a development environment to port "),Si=s(Au,"EM",{});var l6=i(Si);Tv=n(l6,"brand_new_bert"),l6.forEach(o),$v=n(Au," to \u{1F917} Transformers."),Au.forEach(o),Cf=f(e),Xe=s(e,"H3",{class:!0});var Bu=i(Xe);bt=s(Bu,"A",{id:!0,class:!0,href:!0});var d6=i(bt);Di=s(d6,"SPAN",{});var h6=i(Di);u(Xo.$$.fragment,h6),h6.forEach(o),d6.forEach(o),Pv=f(Bu),Ri=s(Bu,"SPAN",{});var f6=i(Ri);Iv=n(f6,"3.-4. Run a pretrained checkpoint using the original repository"),f6.forEach(o),Bu.forEach(o),qf=f(e),L=s(e,"P",{});var ae=i(L);Lv=n(ae,"At first, you will work on the original "),Fi=s(ae,"EM",{});var m6=i(Fi);Av=n(m6,"brand_new_bert"),m6.forEach(o),Bv=n(ae,` repository. Often, the original implementation is very
\u201Cresearchy\u201D. Meaning that documentation might be lacking and the code can be difficult to understand. But this should
be exactly your motivation to reimplement `),zi=s(ae,"EM",{});var p6=i(zi);Mv=n(p6,"brand_new_bert"),p6.forEach(o),Nv=n(ae,". At Hugging Face, one of our main goals is to "),Hi=s(ae,"EM",{});var u6=i(Hi);xv=n(u6,`make people
stand on the shoulders of giants`),u6.forEach(o),Ov=n(ae,` which translates here very well into taking a working model and rewriting it to make
it as `),Wi=s(ae,"STRONG",{});var c6=i(Wi);jv=n(c6,"accessible, user-friendly, and beautiful"),c6.forEach(o),Cv=n(ae,` as possible. This is the number-one motivation to re-implement
models into \u{1F917} Transformers - trying to make complex new NLP technology accessible to `),Gi=s(ae,"STRONG",{});var y6=i(Gi);qv=n(y6,"everybody"),y6.forEach(o),Sv=n(ae,"."),ae.forEach(o),Sf=f(e),Bn=s(e,"P",{});var g6=i(Bn);Dv=n(g6,"You should start thereby by diving into the original repository."),g6.forEach(o),Df=f(e),_t=s(e,"P",{});var Mu=i(_t);Rv=n(Mu,"Successfully running the official pretrained model in the original repository is often "),Ui=s(Mu,"STRONG",{});var w6=i(Ui);Fv=n(w6,"the most difficult"),w6.forEach(o),zv=n(Mu,` step.
From our experience, it is very important to spend some time getting familiar with the original code-base. You need to
figure out the following:`),Mu.forEach(o),Rf=f(e),A=s(e,"UL",{});var se=i(A);Yi=s(se,"LI",{});var v6=i(Yi);Hv=n(v6,"Where to find the pretrained weights?"),v6.forEach(o),Wv=f(se),Ji=s(se,"LI",{});var b6=i(Ji);Gv=n(b6,"How to load the pretrained weights into the corresponding model?"),b6.forEach(o),Uv=f(se),Xi=s(se,"LI",{});var _6=i(Xi);Yv=n(_6,"How to run the tokenizer independently from the model?"),_6.forEach(o),Jv=f(se),Ki=s(se,"LI",{});var E6=i(Ki);Xv=n(E6,`Trace one forward pass so that you know which classes and functions are required for a simple forward pass. Usually,
you only have to reimplement those functions.`),E6.forEach(o),Kv=f(se),F=s(se,"LI",{});var Ae=i(F);Qv=n(Ae,`Be able to locate the important components of the model: Where is the model\u2019s class? Are there model sub-classes,
`),Qi=s(Ae,"EM",{});var k6=i(Qi);Vv=n(k6,"e.g."),k6.forEach(o),Zv=n(Ae,` EncoderModel, DecoderModel? Where is the self-attention layer? Are there multiple different attention layers,
`),Vi=s(Ae,"EM",{});var T6=i(Vi);eb=n(T6,"e.g."),T6.forEach(o),tb=f(Ae),Zi=s(Ae,"EM",{});var $6=i(Zi);ob=n($6,"self-attention"),$6.forEach(o),rb=n(Ae,", "),el=s(Ae,"EM",{});var P6=i(el);nb=n(P6,"cross-attention"),P6.forEach(o),ab=n(Ae,"\u2026?"),Ae.forEach(o),sb=f(se),Ke=s(se,"LI",{});var Sa=i(Ke);ib=n(Sa,"How can you debug the model in the original environment of the repo? Do you have to add "),tl=s(Sa,"EM",{});var I6=i(tl);lb=n(I6,"print"),I6.forEach(o),db=n(Sa,` statements, can you
work with an interactive debugger like `),ol=s(Sa,"EM",{});var L6=i(ol);hb=n(L6,"ipdb"),L6.forEach(o),fb=n(Sa,", or should you use an efficient IDE to debug the model, like PyCharm?"),Sa.forEach(o),se.forEach(o),Ff=f(e),Et=s(e,"P",{});var Nu=i(Et);mb=n(Nu,"It is very important that before you start the porting process, that you can "),rl=s(Nu,"STRONG",{});var A6=i(rl);pb=n(A6,"efficiently"),A6.forEach(o),ub=n(Nu,` debug code in the original
repository! Also, remember that you are working with an open-source library, so do not hesitate to open an issue, or
even a pull request in the original repository. The maintainers of this repository are most likely very happy about
someone looking into their code!`),Nu.forEach(o),zf=f(e),Mn=s(e,"P",{});var B6=i(Mn);cb=n(B6,`At this point, it is really up to you which debugging environment and strategy you prefer to use to debug the original
model. We strongly advise against setting up a costly GPU environment, but simply work on a CPU both when starting to
dive into the original repository and also when starting to write the \u{1F917} Transformers implementation of the model. Only
at the very end, when the model has already been successfully ported to \u{1F917} Transformers, one should verify that the
model also works as expected on GPU.`),B6.forEach(o),Hf=f(e),Nn=s(e,"P",{});var M6=i(Nn);yb=n(M6,"In general, there are two possible debugging environments for running the original model"),M6.forEach(o),Wf=f(e),kt=s(e,"UL",{});var xu=i(kt);Ko=s(xu,"LI",{});var Ou=i(Ko);Qo=s(Ou,"A",{href:!0,rel:!0});var N6=i(Qo);gb=n(N6,"Jupyter notebooks"),N6.forEach(o),wb=n(Ou," / "),Vo=s(Ou,"A",{href:!0,rel:!0});var x6=i(Vo);vb=n(x6,"google colab"),x6.forEach(o),Ou.forEach(o),bb=f(xu),nl=s(xu,"LI",{});var O6=i(nl);_b=n(O6,"Local python scripts."),O6.forEach(o),xu.forEach(o),Gf=f(e),xn=s(e,"P",{});var j6=i(xn);Eb=n(j6,`Jupyter notebooks have the advantage that they allow for cell-by-cell execution which can be helpful to better split
logical components from one another and to have faster debugging cycles as intermediate results can be stored. Also,
notebooks are often easier to share with other contributors, which might be very helpful if you want to ask the Hugging
Face team for help. If you are familiar with Jupiter notebooks, we strongly recommend you to work with them.`),j6.forEach(o),Uf=f(e),Tt=s(e,"P",{});var ju=i(Tt);kb=n(ju,`The obvious disadvantage of Jupyter notebooks is that if you are not used to working with them you will have to spend
some time adjusting to the new programming environment and that you might not be able to use your known debugging tools
anymore, like `),al=s(ju,"CODE",{});var C6=i(al);Tb=n(C6,"ipdb"),C6.forEach(o),$b=n(ju,"."),ju.forEach(o),Yf=f(e),$t=s(e,"P",{});var Cu=i($t);Pb=n(Cu,"For each code-base, a good first step is always to load a "),sl=s(Cu,"STRONG",{});var q6=i(sl);Ib=n(q6,"small"),q6.forEach(o),Lb=n(Cu,` pretrained checkpoint and to be able to reproduce a
single forward pass using a dummy integer vector of input IDs as an input. Such a script could look like this (in
pseudocode):`),Cu.forEach(o),Jf=f(e),u(Zo.$$.fragment,e),Xf=f(e),On=s(e,"P",{});var S6=i(On);Ab=n(S6,"Next, regarding the debugging strategy, there are generally a few from which to choose from:"),S6.forEach(o),Kf=f(e),Pt=s(e,"UL",{});var qu=i(Pt);il=s(qu,"LI",{});var D6=i(il);Bb=n(D6,`Decompose the original model into many small testable components and run a forward pass on each of those for
verification`),D6.forEach(o),Mb=f(qu),Qe=s(qu,"LI",{});var Da=i(Qe);Nb=n(Da,"Decompose the original model only into the original "),ll=s(Da,"EM",{});var R6=i(ll);xb=n(R6,"tokenizer"),R6.forEach(o),Ob=n(Da," and the original "),dl=s(Da,"EM",{});var F6=i(dl);jb=n(F6,"model"),F6.forEach(o),Cb=n(Da,`, run a forward pass on
those, and use intermediate print statements or breakpoints for verification`),Da.forEach(o),qu.forEach(o),Qf=f(e),jn=s(e,"P",{});var z6=i(jn);qb=n(z6,`Again, it is up to you which strategy to choose. Often, one or the other is advantageous depending on the original code
base.`),z6.forEach(o),Vf=f(e),It=s(e,"P",{});var Su=i(It);Sb=n(Su,"If the original code-base allows you to decompose the model into smaller sub-components, "),hl=s(Su,"EM",{});var H6=i(hl);Db=n(H6,"e.g."),H6.forEach(o),Rb=n(Su,` if the original
code-base can easily be run in eager mode, it is usually worth the effort to do so. There are some important advantages
to taking the more difficult road in the beginning:`),Su.forEach(o),Zf=f(e),U=s(e,"UL",{});var eo=i(U);fl=s(eo,"LI",{});var W6=i(fl);Fb=n(W6,`at a later stage when comparing the original model to the Hugging Face implementation, you can verify automatically
for each component individually that the corresponding component of the \u{1F917} Transformers implementation matches instead
of relying on visual comparison via print statements`),W6.forEach(o),zb=f(eo),ml=s(eo,"LI",{});var G6=i(ml);Hb=n(G6,`it can give you some rope to decompose the big problem of porting a model into smaller problems of just porting
individual components and thus structure your work better`),G6.forEach(o),Wb=f(eo),pl=s(eo,"LI",{});var U6=i(pl);Gb=n(U6,`separating the model into logical meaningful components will help you to get a better overview of the model\u2019s design
and thus to better understand the model`),U6.forEach(o),Ub=f(eo),ul=s(eo,"LI",{});var Y6=i(ul);Yb=n(Y6,`at a later stage those component-by-component tests help you to ensure that no regression occurs as you continue
changing your code`),Y6.forEach(o),eo.forEach(o),em=f(e),er=s(e,"P",{});var ok=i(er);tr=s(ok,"A",{href:!0,rel:!0});var J6=i(tr);Jb=n(J6,"Lysandre\u2019s"),J6.forEach(o),Xb=n(ok,` integration checks for ELECTRA
gives a nice example of how this can be done.`),ok.forEach(o),tm=f(e),Lt=s(e,"P",{});var Du=i(Lt);Kb=n(Du,`However, if the original code-base is very complex or only allows intermediate components to be run in a compiled mode,
it might be too time-consuming or even impossible to separate the model into smaller testable sub-components. A good
example is `),or=s(Du,"A",{href:!0,rel:!0});var X6=i(or);Qb=n(X6,"T5\u2019s MeshTensorFlow"),X6.forEach(o),Vb=n(Du,` library which is
very complex and does not offer a simple way to decompose the model into its sub-components. For such libraries, one
often relies on verifying print statements.`),Du.forEach(o),om=f(e),Cn=s(e,"P",{});var K6=i(Cn);Zb=n(K6,`No matter which strategy you choose, the recommended procedure is often the same in that you should start to debug the
starting layers first and the ending layers last.`),K6.forEach(o),rm=f(e),qn=s(e,"P",{});var Q6=i(qn);e_=n(Q6,`It is recommended that you retrieve the output, either by print statements or sub-component functions, of the following
layers in the following order:`),Q6.forEach(o),nm=f(e),B=s(e,"OL",{});var ie=i(B);cl=s(ie,"LI",{});var V6=i(cl);t_=n(V6,"Retrieve the input IDs passed to the model"),V6.forEach(o),o_=f(ie),yl=s(ie,"LI",{});var Z6=i(yl);r_=n(Z6,"Retrieve the word embeddings"),Z6.forEach(o),n_=f(ie),gl=s(ie,"LI",{});var e7=i(gl);a_=n(e7,"Retrieve the input of the first Transformer layer"),e7.forEach(o),s_=f(ie),wl=s(ie,"LI",{});var t7=i(wl);i_=n(t7,"Retrieve the output of the first Transformer layer"),t7.forEach(o),l_=f(ie),vl=s(ie,"LI",{});var o7=i(vl);d_=n(o7,"Retrieve the output of the following n - 1 Transformer layers"),o7.forEach(o),h_=f(ie),bl=s(ie,"LI",{});var r7=i(bl);f_=n(r7,"Retrieve the output of the whole BrandNewBert Model"),r7.forEach(o),ie.forEach(o),am=f(e),Ve=s(e,"P",{});var Dh=i(Ve);m_=n(Dh,"Input IDs should thereby consists of an array of integers, "),_l=s(Dh,"EM",{});var n7=i(_l);p_=n(n7,"e.g."),n7.forEach(o),u_=f(Dh),El=s(Dh,"CODE",{});var a7=i(El);c_=n(a7,"input_ids = [0, 4, 4, 3, 2, 4, 1, 7, 19]"),a7.forEach(o),Dh.forEach(o),sm=f(e),Sn=s(e,"P",{});var s7=i(Sn);y_=n(s7,"The outputs of the following layers often consist of multi-dimensional float arrays and can look like this:"),s7.forEach(o),im=f(e),u(rr.$$.fragment,e),lm=f(e),ge=s(e,"P",{});var Ra=i(ge);g_=n(Ra,`We expect that every model added to \u{1F917} Transformers passes a couple of integration tests, meaning that the original
model and the reimplemented version in \u{1F917} Transformers have to give the exact same output up to a precision of 0.001!
Since it is normal that the exact same model written in different libraries can give a slightly different output
depending on the library framework, we accept an error tolerance of 1e-3 (0.001). It is not enough if the model gives
nearly the same output, they have to be the almost identical. Therefore, you will certainly compare the intermediate
outputs of the \u{1F917} Transformers version multiple times against the intermediate outputs of the original implementation of
`),kl=s(Ra,"EM",{});var i7=i(kl);w_=n(i7,"brand_new_bert"),i7.forEach(o),v_=n(Ra," in which case an "),Tl=s(Ra,"STRONG",{});var l7=i(Tl);b_=n(l7,"efficient"),l7.forEach(o),__=n(Ra,` debugging environment of the original repository is absolutely
important. Here is some advice is to make your debugging environment as efficient as possible.`),Ra.forEach(o),dm=f(e),C=s(e,"UL",{});var Be=i(C);z=s(Be,"LI",{});var Me=i(z);E_=n(Me,`Find the best way of debugging intermediate results. Is the original repository written in PyTorch? Then you should
probably take the time to write a longer script that decomposes the original model into smaller sub-components to
retrieve intermediate values. Is the original repository written in Tensorflow 1? Then you might have to rely on
TensorFlow print operations like `),nr=s(Me,"A",{href:!0,rel:!0});var d7=i(nr);k_=n(d7,"tf.print"),d7.forEach(o),T_=n(Me,` to output
intermediate values. Is the original repository written in Jax? Then make sure that the model is `),$l=s(Me,"STRONG",{});var h7=i($l);$_=n(h7,"not jitted"),h7.forEach(o),P_=n(Me,` when
running the forward pass, `),Pl=s(Me,"EM",{});var f7=i(Pl);I_=n(f7,"e.g."),f7.forEach(o),L_=n(Me," check-out "),ar=s(Me,"A",{href:!0,rel:!0});var m7=i(ar);A_=n(m7,"this link"),m7.forEach(o),B_=n(Me,"."),Me.forEach(o),M_=f(Be),Il=s(Be,"LI",{});var p7=i(Il);N_=n(p7,`Use the smallest pretrained checkpoint you can find. The smaller the checkpoint, the faster your debug cycle
becomes. It is not efficient if your pretrained model is so big that your forward pass takes more than 10 seconds.
In case only very large checkpoints are available, it might make more sense to create a dummy model in the new
environment with randomly initialized weights and save those weights for comparison with the \u{1F917} Transformers version
of your model`),p7.forEach(o),x_=f(Be),T=s(Be,"LI",{});var $=i(T);O_=n($,`Make sure you are using the easiest way of calling a forward pass in the original repository. Ideally, you want to
find the function in the original repository that `),Ll=s($,"STRONG",{});var u7=i(Ll);j_=n(u7,"only"),u7.forEach(o),C_=n($," calls a single forward pass, "),Al=s($,"EM",{});var c7=i(Al);q_=n(c7,"i.e."),c7.forEach(o),S_=n($,` that is often called
`),Bl=s($,"CODE",{});var y7=i(Bl);D_=n(y7,"predict"),y7.forEach(o),R_=n($,", "),Ml=s($,"CODE",{});var g7=i(Ml);F_=n(g7,"evaluate"),g7.forEach(o),z_=n($,", "),Nl=s($,"CODE",{});var w7=i(Nl);H_=n(w7,"forward"),w7.forEach(o),W_=n($," or "),xl=s($,"CODE",{});var v7=i(xl);G_=n(v7,"__call__"),v7.forEach(o),U_=n($,". You don\u2019t want to debug a function that calls "),Ol=s($,"CODE",{});var b7=i(Ol);Y_=n(b7,"forward"),b7.forEach(o),J_=n($,`
multiple times, `),jl=s($,"EM",{});var _7=i(jl);X_=n(_7,"e.g."),_7.forEach(o),K_=n($," to generate text, like "),Cl=s($,"CODE",{});var E7=i(Cl);Q_=n(E7,"autoregressive_sample"),E7.forEach(o),V_=n($,", "),ql=s($,"CODE",{});var k7=i(ql);Z_=n(k7,"generate"),k7.forEach(o),e1=n($,"."),$.forEach(o),t1=f(Be),sr=s(Be,"LI",{});var Ru=i(sr);o1=n(Ru,"Try to separate the tokenization from the model\u2019s "),Sl=s(Ru,"EM",{});var T7=i(Sl);r1=n(T7,"forward"),T7.forEach(o),n1=n(Ru,` pass. If the original repository shows examples where
you have to input a string, then try to find out where in the forward call the string input is changed to input ids
and start from this point. This might mean that you have to possibly write a small script yourself or change the
original code so that you can directly input the ids instead of an input string.`),Ru.forEach(o),a1=f(Be),me=s(Be,"LI",{});var to=i(me);s1=n(to,"Make sure that the model in your debugging setup is "),Dl=s(to,"STRONG",{});var $7=i(Dl);i1=n($7,"not"),$7.forEach(o),l1=n(to,` in training mode, which often causes the model to yield
random outputs due to multiple dropout layers in the model. Make sure that the forward pass in your debugging
environment is `),Rl=s(to,"STRONG",{});var P7=i(Rl);d1=n(P7,"deterministic"),P7.forEach(o),h1=n(to," so that the dropout layers are not used. Or use "),Fl=s(to,"EM",{});var I7=i(Fl);f1=n(I7,"transformers.utils.set_seed"),I7.forEach(o),m1=n(to,`
if the old and new implementations are in the same framework.`),to.forEach(o),Be.forEach(o),hm=f(e),At=s(e,"P",{});var Fu=i(At);p1=n(Fu,"The following section gives you more specific details/tips on how you can do this for "),zl=s(Fu,"EM",{});var L7=i(zl);u1=n(L7,"brand_new_bert"),L7.forEach(o),c1=n(Fu,"."),Fu.forEach(o),fm=f(e),Ze=s(e,"H3",{class:!0});var zu=i(Ze);Bt=s(zu,"A",{id:!0,class:!0,href:!0});var A7=i(Bt);Hl=s(A7,"SPAN",{});var B7=i(Hl);u(ir.$$.fragment,B7),B7.forEach(o),A7.forEach(o),y1=f(zu),Wl=s(zu,"SPAN",{});var M7=i(Wl);g1=n(M7,"5.-14. Port BrandNewBert to \u{1F917} Transformers"),M7.forEach(o),zu.forEach(o),mm=f(e),Dn=s(e,"P",{});var N7=i(Dn);w1=n(N7,"Next, you can finally start adding new code to \u{1F917} Transformers. Go into the clone of your \u{1F917} Transformers\u2019 fork:"),N7.forEach(o),pm=f(e),u(lr.$$.fragment,e),um=f(e),Mt=s(e,"P",{});var Hu=i(Mt);v1=n(Hu,`In the special case that you are adding a model whose architecture exactly matches the model architecture of an
existing model you only have to add a conversion script as described in `),Rn=s(Hu,"A",{href:!0});var x7=i(Rn);b1=n(x7,"this section"),x7.forEach(o),_1=n(Hu,`.
In this case, you can just re-use the whole model architecture of the already existing model.`),Hu.forEach(o),cm=f(e),Fn=s(e,"P",{});var O7=i(Fn);E1=n(O7,"Otherwise, let\u2019s start generating a new model. You have two choices here:"),O7.forEach(o),ym=f(e),Nt=s(e,"UL",{});var Wu=i(Nt);zn=s(Wu,"LI",{});var rk=i(zn);Gl=s(rk,"CODE",{});var j7=i(Gl);k1=n(j7,"transformers-cli add-new-model-like"),j7.forEach(o),T1=n(rk," to add a new model like an existing one"),rk.forEach(o),$1=f(Wu),Hn=s(Wu,"LI",{});var nk=i(Hn);Ul=s(nk,"CODE",{});var C7=i(Ul);P1=n(C7,"transformers-cli add-new-model"),C7.forEach(o),I1=n(nk," to add a new model from our template (will look like BERT or Bart depending on the type of model you select)"),nk.forEach(o),Wu.forEach(o),gm=f(e),we=s(e,"P",{});var Fa=i(we);L1=n(Fa,"In both cases, you will be prompted with a questionnaire to fill the basic information of your model. The second command requires to install "),Yl=s(Fa,"CODE",{});var q7=i(Yl);A1=n(q7,"cookiecutter"),q7.forEach(o),B1=n(Fa,", you can find more information on it "),dr=s(Fa,"A",{href:!0,rel:!0});var S7=i(dr);M1=n(S7,"here"),S7.forEach(o),N1=n(Fa,"."),Fa.forEach(o),wm=f(e),Wn=s(e,"P",{});var D7=i(Wn);Jl=s(D7,"STRONG",{});var R7=i(Jl);x1=n(R7,"Open a Pull Request on the main huggingface/transformers repo"),R7.forEach(o),D7.forEach(o),vm=f(e),ve=s(e,"P",{});var za=i(ve);O1=n(za,`Before starting to adapt the automatically generated code, now is the time to open a \u201CWork in progress (WIP)\u201D pull
request, `),Xl=s(za,"EM",{});var F7=i(Xl);j1=n(F7,"e.g."),F7.forEach(o),C1=n(za," \u201C[WIP] Add "),Kl=s(za,"EM",{});var z7=i(Kl);q1=n(z7,"brand_new_bert"),z7.forEach(o),S1=n(za,`\u201D, in \u{1F917} Transformers so that you and the Hugging Face team can work
side-by-side on integrating the model into \u{1F917} Transformers.`),za.forEach(o),bm=f(e),Gn=s(e,"P",{});var H7=i(Gn);D1=n(H7,"You should do the following:"),H7.forEach(o),_m=f(e),Un=s(e,"OL",{});var W7=i(Un);Ql=s(W7,"LI",{});var G7=i(Ql);R1=n(G7,"Create a branch with a descriptive name from your main branch"),G7.forEach(o),W7.forEach(o),Em=f(e),u(hr.$$.fragment,e),km=f(e),fr=s(e,"OL",{start:!0});var U7=i(fr);Vl=s(U7,"LI",{});var Y7=i(Vl);F1=n(Y7,"Commit the automatically generated code:"),Y7.forEach(o),U7.forEach(o),Tm=f(e),u(mr.$$.fragment,e),$m=f(e),pr=s(e,"OL",{start:!0});var J7=i(pr);Zl=s(J7,"LI",{});var X7=i(Zl);z1=n(X7,"Fetch and rebase to current main"),X7.forEach(o),J7.forEach(o),Pm=f(e),u(ur.$$.fragment,e),Im=f(e),cr=s(e,"OL",{start:!0});var K7=i(cr);ed=s(K7,"LI",{});var Q7=i(ed);H1=n(Q7,"Push the changes to your account using:"),Q7.forEach(o),K7.forEach(o),Lm=f(e),u(yr.$$.fragment,e),Am=f(e),et=s(e,"OL",{start:!0});var Gu=i(et);td=s(Gu,"LI",{});var V7=i(td);od=s(V7,"P",{});var Z7=i(od);W1=n(Z7,`Once you are satisfied, go to the webpage of your fork on GitHub. Click on \u201CPull request\u201D. Make sure to add the
GitHub handle of some members of the Hugging Face team as reviewers, so that the Hugging Face team gets notified for
future changes.`),Z7.forEach(o),V7.forEach(o),G1=f(Gu),rd=s(Gu,"LI",{});var eT=i(rd);nd=s(eT,"P",{});var tT=i(nd);U1=n(tT,"Change the PR into a draft by clicking on \u201CConvert to draft\u201D on the right of the GitHub pull request web page."),tT.forEach(o),eT.forEach(o),Gu.forEach(o),Bm=f(e),Yn=s(e,"P",{});var oT=i(Yn);Y1=n(oT,`In the following, whenever you have done some progress, don\u2019t forget to commit your work and push it to your account so
that it shows in the pull request. Additionally, you should make sure to update your work with the current main from
time to time by doing:`),oT.forEach(o),Mm=f(e),u(gr.$$.fragment,e),Nm=f(e),Jn=s(e,"P",{});var rT=i(Jn);J1=n(rT,`In general, all questions you might have regarding the model or your implementation should be asked in your PR and
discussed/solved in the PR. This way, the Hugging Face team will always be notified when you are committing new code or
if you have a question. It is often very helpful to point the Hugging Face team to your added code so that the Hugging
Face team can efficiently understand your problem or question.`),rT.forEach(o),xm=f(e),Xn=s(e,"P",{});var nT=i(Xn);X1=n(nT,`To do so, you can go to the \u201CFiles changed\u201D tab where you see all of your changes, go to a line regarding which you
want to ask a question, and click on the \u201C+\u201D symbol to add a comment. Whenever a question or problem has been solved,
you can click on the \u201CResolve\u201D button of the created comment.`),nT.forEach(o),Om=f(e),Kn=s(e,"P",{});var aT=i(Kn);K1=n(aT,`In the same way, the Hugging Face team will open comments when reviewing your code. We recommend asking most questions
on GitHub on your PR. For some very general questions that are not very useful for the public, feel free to ping the
Hugging Face team by Slack or email.`),aT.forEach(o),jm=f(e),Qn=s(e,"P",{});var sT=i(Qn);ad=s(sT,"STRONG",{});var iT=i(ad);Q1=n(iT,"5. Adapt the generated models code for brand_new_bert"),iT.forEach(o),sT.forEach(o),Cm=f(e),be=s(e,"P",{});var Ha=i(be);V1=n(Ha,`At first, we will focus only on the model itself and not care about the tokenizer. All the relevant code should be
found in the generated files `),sd=s(Ha,"CODE",{});var lT=i(sd);Z1=n(lT,"src/transformers/models/brand_new_bert/modeling_brand_new_bert.py"),lT.forEach(o),e0=n(Ha,` and
`),id=s(Ha,"CODE",{});var dT=i(id);t0=n(dT,"src/transformers/models/brand_new_bert/configuration_brand_new_bert.py"),dT.forEach(o),o0=n(Ha,"."),Ha.forEach(o),qm=f(e),Y=s(e,"P",{});var oo=i(Y);r0=n(oo,`Now you can finally start coding :). The generated code in
`),ld=s(oo,"CODE",{});var hT=i(ld);n0=n(hT,"src/transformers/models/brand_new_bert/modeling_brand_new_bert.py"),hT.forEach(o),a0=n(oo,` will either have the same architecture as BERT if
it\u2019s an encoder-only model or BART if it\u2019s an encoder-decoder model. At this point, you should remind yourself what
you\u2019ve learned in the beginning about the theoretical aspects of the model: `),dd=s(oo,"EM",{});var fT=i(dd);s0=n(fT,`How is the model different from BERT or
BART?`),fT.forEach(o),i0=n(oo,"\u201D. Implement those changes which often means to change the "),hd=s(oo,"EM",{});var mT=i(hd);l0=n(mT,"self-attention"),mT.forEach(o),d0=n(oo,` layer, the order of the normalization
layer, etc\u2026 Again, it is often useful to look at the similar architecture of already existing models in Transformers to
get a better feeling of how your model should be implemented.`),oo.forEach(o),Sm=f(e),N=s(e,"P",{});var pe=i(N);fd=s(pe,"STRONG",{});var pT=i(fd);h0=n(pT,"Note"),pT.forEach(o),f0=n(pe,` that at this point, you don\u2019t have to be very sure that your code is fully correct or clean. Rather, it is
advised to add a first `),md=s(pe,"EM",{});var uT=i(md);m0=n(uT,"unclean"),uT.forEach(o),p0=n(pe,`, copy-pasted version of the original code to
`),pd=s(pe,"CODE",{});var cT=i(pd);u0=n(cT,"src/transformers/models/brand_new_bert/modeling_brand_new_bert.py"),cT.forEach(o),c0=n(pe,` until you feel like all the necessary code is
added. From our experience, it is much more efficient to quickly add a first version of the required code and
improve/correct the code iteratively with the conversion script as described in the next section. The only thing that
has to work at this point is that you can instantiate the \u{1F917} Transformers implementation of `),ud=s(pe,"EM",{});var yT=i(ud);y0=n(yT,"brand_new_bert"),yT.forEach(o),g0=n(pe,", "),cd=s(pe,"EM",{});var gT=i(cd);w0=n(gT,"i.e."),gT.forEach(o),v0=n(pe,` the
following command should work:`),pe.forEach(o),Dm=f(e),u(wr.$$.fragment,e),Rm=f(e),_e=s(e,"P",{});var Wa=i(_e);b0=n(Wa,"The above command will create a model according to the default parameters as defined in "),yd=s(Wa,"CODE",{});var wT=i(yd);_0=n(wT,"BrandNewBertConfig()"),wT.forEach(o),E0=n(Wa,` with
random weights, thus making sure that the `),gd=s(Wa,"CODE",{});var vT=i(gd);k0=n(vT,"init()"),vT.forEach(o),T0=n(Wa," methods of all components works."),Wa.forEach(o),Fm=f(e),Vn=s(e,"P",{});var bT=i(Vn);wd=s(bT,"STRONG",{});var _T=i(wd);$0=n(_T,"6. Write a conversion script"),_T.forEach(o),bT.forEach(o),zm=f(e),J=s(e,"P",{});var ro=i(J);P0=n(ro,"Next, you should write a conversion script that lets you convert the checkpoint you used to debug "),vd=s(ro,"EM",{});var ET=i(vd);I0=n(ET,"brand_new_bert"),ET.forEach(o),L0=n(ro,` in
the original repository to a checkpoint compatible with your just created \u{1F917} Transformers implementation of
`),bd=s(ro,"EM",{});var kT=i(bd);A0=n(kT,"brand_new_bert"),kT.forEach(o),B0=n(ro,`. It is not advised to write the conversion script from scratch, but rather to look through already
existing conversion scripts in \u{1F917} Transformers for one that has been used to convert a similar model that was written in
the same framework as `),_d=s(ro,"EM",{});var TT=i(_d);M0=n(TT,"brand_new_bert"),TT.forEach(o),N0=n(ro,`. Usually, it is enough to copy an already existing conversion script and
slightly adapt it for your use case. Don\u2019t hesitate to ask the Hugging Face team to point you to a similar already
existing conversion script for your model.`),ro.forEach(o),Hm=f(e),xt=s(e,"UL",{});var Uu=i(xt);Zn=s(Uu,"LI",{});var ak=i(Zn);x0=n(ak,"If you are porting a model from TensorFlow to PyTorch, a good starting point might be BERT\u2019s conversion script "),vr=s(ak,"A",{href:!0,rel:!0});var $T=i(vr);O0=n($T,"here"),$T.forEach(o),ak.forEach(o),j0=f(Uu),ea=s(Uu,"LI",{});var sk=i(ea);C0=n(sk,"If you are porting a model from PyTorch to PyTorch, a good starting point might be BART\u2019s conversion script "),br=s(sk,"A",{href:!0,rel:!0});var PT=i(br);q0=n(PT,"here"),PT.forEach(o),sk.forEach(o),Uu.forEach(o),Wm=f(e),Ot=s(e,"P",{});var Yu=i(Ot);S0=n(Yu,`In the following, we\u2019ll quickly explain how PyTorch models store layer weights and define layer names. In PyTorch, the
name of a layer is defined by the name of the class attribute you give the layer. Let\u2019s define a dummy model in
PyTorch, called `),Ed=s(Yu,"CODE",{});var IT=i(Ed);D0=n(IT,"SimpleModel"),IT.forEach(o),R0=n(Yu," as follows:"),Yu.forEach(o),Gm=f(e),u(_r.$$.fragment,e),Um=f(e),X=s(e,"P",{});var no=i(X);F0=n(no,"Now we can create an instance of this model definition which will fill all weights: "),kd=s(no,"CODE",{});var LT=i(kd);z0=n(LT,"dense"),LT.forEach(o),H0=n(no,", "),Td=s(no,"CODE",{});var AT=i(Td);W0=n(AT,"intermediate"),AT.forEach(o),G0=n(no,`,
`),$d=s(no,"CODE",{});var BT=i($d);U0=n(BT,"layer_norm"),BT.forEach(o),Y0=n(no," with random weights. We can print the model to see its architecture"),no.forEach(o),Ym=f(e),u(Er.$$.fragment,e),Jm=f(e),ta=s(e,"P",{});var MT=i(ta);J0=n(MT,"This will print out the following:"),MT.forEach(o),Xm=f(e),u(kr.$$.fragment,e),Km=f(e),oa=s(e,"P",{});var NT=i(oa);X0=n(NT,`We can see that the layer names are defined by the name of the class attribute in PyTorch. You can print out the weight
values of a specific layer:`),NT.forEach(o),Qm=f(e),u(Tr.$$.fragment,e),Vm=f(e),ra=s(e,"P",{});var xT=i(ra);K0=n(xT,"to see that the weights were randomly initialized"),xT.forEach(o),Zm=f(e),u($r.$$.fragment,e),ep=f(e),Pr=s(e,"P",{});var ik=i(Pr);Q0=n(ik,`In the conversion script, you should fill those randomly initialized weights with the exact weights of the
corresponding layer in the checkpoint. `),Pd=s(ik,"EM",{});var OT=i(Pd);V0=n(OT,"E.g."),OT.forEach(o),ik.forEach(o),tp=f(e),u(Ir.$$.fragment,e),op=f(e),Ee=s(e,"P",{});var Ga=i(Ee);Z0=n(Ga,`While doing so, you must verify that each randomly initialized weight of your PyTorch model and its corresponding
pretrained checkpoint weight exactly match in both `),Id=s(Ga,"STRONG",{});var jT=i(Id);e2=n(jT,"shape and name"),jT.forEach(o),t2=n(Ga,". To do so, it is "),Ld=s(Ga,"STRONG",{});var CT=i(Ld);o2=n(CT,"necessary"),CT.forEach(o),r2=n(Ga,` to add assert
statements for the shape and print out the names of the checkpoints weights. E.g. you should add statements like:`),Ga.forEach(o),rp=f(e),u(Lr.$$.fragment,e),np=f(e),Ar=s(e,"P",{});var lk=i(Ar);n2=n(lk,"Besides, you should also print out the names of both weights to make sure they match, "),Ad=s(lk,"EM",{});var qT=i(Ad);a2=n(qT,"e.g."),qT.forEach(o),lk.forEach(o),ap=f(e),u(Br.$$.fragment,e),sp=f(e),na=s(e,"P",{});var ST=i(na);s2=n(ST,`If either the shape or the name doesn\u2019t match, you probably assigned the wrong checkpoint weight to a randomly
initialized layer of the \u{1F917} Transformers implementation.`),ST.forEach(o),ip=f(e),jt=s(e,"P",{});var Ju=i(jt);i2=n(Ju,"An incorrect shape is most likely due to an incorrect setting of the config parameters in "),Bd=s(Ju,"CODE",{});var DT=i(Bd);l2=n(DT,"BrandNewBertConfig()"),DT.forEach(o),d2=n(Ju,` that
do not exactly match those that were used for the checkpoint you want to convert. However, it could also be that
PyTorch\u2019s implementation of a layer requires the weight to be transposed beforehand.`),Ju.forEach(o),lp=f(e),K=s(e,"P",{});var ao=i(K);h2=n(ao,"Finally, you should also check that "),Md=s(ao,"STRONG",{});var RT=i(Md);f2=n(RT,"all"),RT.forEach(o),m2=n(ao,` required weights are initialized and print out all checkpoint weights that
were not used for initialization to make sure the model is correctly converted. It is completely normal, that the
conversion trials fail with either a wrong shape statement or wrong name assignment. This is most likely because either
you used incorrect parameters in `),Nd=s(ao,"CODE",{});var FT=i(Nd);p2=n(FT,"BrandNewBertConfig()"),FT.forEach(o),u2=n(ao,`, have a wrong architecture in the \u{1F917} Transformers
implementation, you have a bug in the `),xd=s(ao,"CODE",{});var zT=i(xd);c2=n(zT,"init()"),zT.forEach(o),y2=n(ao,` functions of one of the components of the \u{1F917} Transformers
implementation or you need to transpose one of the checkpoint weights.`),ao.forEach(o),dp=f(e),Q=s(e,"P",{});var so=i(Q);g2=n(so,`This step should be iterated with the previous step until all weights of the checkpoint are correctly loaded in the
Transformers model. Having correctly loaded the checkpoint into the \u{1F917} Transformers implementation, you can then save
the model under a folder of your choice `),Od=s(so,"CODE",{});var HT=i(Od);w2=n(HT,"/path/to/converted/checkpoint/folder"),HT.forEach(o),v2=n(so,` that should then contain both a
`),jd=s(so,"CODE",{});var WT=i(jd);b2=n(WT,"pytorch_model.bin"),WT.forEach(o),_2=n(so," file and a "),Cd=s(so,"CODE",{});var GT=i(Cd);E2=n(GT,"config.json"),GT.forEach(o),k2=n(so," file:"),so.forEach(o),hp=f(e),u(Mr.$$.fragment,e),fp=f(e),aa=s(e,"P",{});var UT=i(aa);qd=s(UT,"STRONG",{});var YT=i(qd);T2=n(YT,"7. Implement the forward pass"),YT.forEach(o),UT.forEach(o),mp=f(e),Ct=s(e,"P",{});var Xu=i(Ct);$2=n(Xu,`Having managed to correctly load the pretrained weights into the \u{1F917} Transformers implementation, you should now make
sure that the forward pass is correctly implemented. In `),sa=s(Xu,"A",{href:!0});var JT=i(sa);P2=n(JT,"Get familiar with the original repository"),JT.forEach(o),I2=n(Xu,`, you have already created a script that runs a forward
pass of the model using the original repository. Now you should write an analogous script using the \u{1F917} Transformers
implementation instead of the original one. It should look as follows:`),Xu.forEach(o),pp=f(e),u(Nr.$$.fragment,e),up=f(e),q=s(e,"P",{});var Ne=i(q);L2=n(Ne,`It is very likely that the \u{1F917} Transformers implementation and the original model implementation don\u2019t give the exact
same output the very first time or that the forward pass throws an error. Don\u2019t be disappointed - it\u2019s expected! First,
you should make sure that the forward pass doesn\u2019t throw any errors. It often happens that the wrong dimensions are
used leading to a `),Sd=s(Ne,"EM",{});var XT=i(Sd);A2=n(XT,"Dimensionality mismatch"),XT.forEach(o),B2=n(Ne," error or that the wrong data type object is used, "),Dd=s(Ne,"EM",{});var KT=i(Dd);M2=n(KT,"e.g."),KT.forEach(o),N2=f(Ne),Rd=s(Ne,"CODE",{});var QT=i(Rd);x2=n(QT,"torch.long"),QT.forEach(o),O2=n(Ne,`
instead of `),Fd=s(Ne,"CODE",{});var VT=i(Fd);j2=n(VT,"torch.float32"),VT.forEach(o),C2=n(Ne,`. Don\u2019t hesitate to ask the Hugging Face team for help, if you don\u2019t manage to solve
certain errors.`),Ne.forEach(o),cp=f(e),V=s(e,"P",{});var io=i(V);q2=n(io,`The final part to make sure the \u{1F917} Transformers implementation works correctly is to ensure that the outputs are
equivalent to a precision of `),zd=s(io,"CODE",{});var ZT=i(zd);S2=n(ZT,"1e-3"),ZT.forEach(o),D2=n(io,". First, you should ensure that the output shapes are identical, "),Hd=s(io,"EM",{});var e9=i(Hd);R2=n(e9,"i.e."),e9.forEach(o),F2=f(io),Wd=s(io,"CODE",{});var t9=i(Wd);z2=n(t9,"outputs.shape"),t9.forEach(o),H2=n(io,` should yield the same value for the script of the \u{1F917} Transformers implementation and the original
implementation. Next, you should make sure that the output values are identical as well. This one of the most difficult
parts of adding a new model. Common mistakes why the outputs are not identical are:`),io.forEach(o),yp=f(e),Z=s(e,"UL",{});var lo=i(Z);tt=s(lo,"LI",{});var Ua=i(tt);W2=n(Ua,"Some layers were not added, "),Gd=s(Ua,"EM",{});var o9=i(Gd);G2=n(o9,"i.e."),o9.forEach(o),U2=n(Ua," an "),Ud=s(Ua,"EM",{});var r9=i(Ud);Y2=n(r9,"activation"),r9.forEach(o),J2=n(Ua," layer was not added, or the residual connection was forgotten"),Ua.forEach(o),X2=f(lo),Yd=s(lo,"LI",{});var n9=i(Yd);K2=n(n9,"The word embedding matrix was not tied"),n9.forEach(o),Q2=f(lo),Jd=s(lo,"LI",{});var a9=i(Jd);V2=n(a9,"The wrong positional embeddings are used because the original implementation uses on offset"),a9.forEach(o),Z2=f(lo),ee=s(lo,"LI",{});var rt=i(ee);eE=n(rt,"Dropout is applied during the forward pass. To fix this make sure "),Xd=s(rt,"EM",{});var s9=i(Xd);tE=n(s9,"model.training is False"),s9.forEach(o),oE=n(rt,` and that no dropout
layer is falsely activated during the forward pass, `),Kd=s(rt,"EM",{});var i9=i(Kd);rE=n(i9,"i.e."),i9.forEach(o),nE=n(rt," pass "),Qd=s(rt,"EM",{});var l9=i(Qd);aE=n(l9,"self.training"),l9.forEach(o),sE=n(rt," to "),xr=s(rt,"A",{href:!0,rel:!0});var d9=i(xr);iE=n(d9,"PyTorch\u2019s functional dropout"),d9.forEach(o),rt.forEach(o),lo.forEach(o),gp=f(e),ke=s(e,"P",{});var Ya=i(ke);lE=n(Ya,`The best way to fix the problem is usually to look at the forward pass of the original implementation and the \u{1F917}
Transformers implementation side-by-side and check if there are any differences. Ideally, you should debug/print out
intermediate outputs of both implementations of the forward pass to find the exact position in the network where the \u{1F917}
Transformers implementation shows a different output than the original implementation. First, make sure that the
hard-coded `),Vd=s(Ya,"CODE",{});var h9=i(Vd);dE=n(h9,"input_ids"),h9.forEach(o),hE=n(Ya,` in both scripts are identical. Next, verify that the outputs of the first transformation of
the `),Zd=s(Ya,"CODE",{});var f9=i(Zd);fE=n(f9,"input_ids"),f9.forEach(o),mE=n(Ya,` (usually the word embeddings) are identical. And then work your way up to the very last layer of the
network. At some point, you will notice a difference between the two implementations, which should point you to the bug
in the \u{1F917} Transformers implementation. From our experience, a simple and efficient way is to add many print statements
in both the original implementation and \u{1F917} Transformers implementation, at the same positions in the network
respectively, and to successively remove print statements showing the same values for intermediate presentations.`),Ya.forEach(o),wp=f(e),qt=s(e,"P",{});var Ku=i(qt);pE=n(Ku,`When you\u2019re confident that both implementations yield the same output, verifying the outputs with
`),eh=s(Ku,"CODE",{});var m9=i(eh);uE=n(m9,"torch.allclose(original_output, output, atol=1e-3)"),m9.forEach(o),cE=n(Ku,`, you\u2019re done with the most difficult part! Congratulations - the
work left to be done should be a cakewalk \u{1F60A}.`),Ku.forEach(o),vp=f(e),ia=s(e,"P",{});var p9=i(ia);th=s(p9,"STRONG",{});var u9=i(th);yE=n(u9,"8. Adding all necessary model tests"),u9.forEach(o),p9.forEach(o),bp=f(e),St=s(e,"P",{});var Qu=i(St);gE=n(Qu,`At this point, you have successfully added a new model. However, it is very much possible that the model does not yet
fully comply with the required design. To make sure, the implementation is fully compatible with \u{1F917} Transformers, all
common tests should pass. The Cookiecutter should have automatically added a test file for your model, probably under
the same `),oh=s(Qu,"CODE",{});var c9=i(oh);wE=n(c9,"tests/models/brand_new_bert/test_modeling_brand_new_bert.py"),c9.forEach(o),vE=n(Qu,`. Run this test file to verify that all common
tests pass:`),Qu.forEach(o),_p=f(e),u(Or.$$.fragment,e),Ep=f(e),la=s(e,"P",{});var y9=i(la);bE=n(y9,"Having fixed all common tests, it is now crucial to ensure that all the nice work you have done is well tested, so that"),y9.forEach(o),kp=f(e),Dt=s(e,"UL",{});var Vu=i(Dt);da=s(Vu,"LI",{});var dk=i(da);_E=n(dk,"a) The community can easily understand your work by looking at specific tests of "),rh=s(dk,"EM",{});var g9=i(rh);EE=n(g9,"brand_new_bert"),g9.forEach(o),dk.forEach(o),kE=f(Vu),nh=s(Vu,"LI",{});var w9=i(nh);TE=n(w9,"b) Future changes to your model will not break any important feature of the model."),w9.forEach(o),Vu.forEach(o),Tp=f(e),Rt=s(e,"P",{});var Zu=i(Rt);$E=n(Zu,`At first, integration tests should be added. Those integration tests essentially do the same as the debugging scripts
you used earlier to implement the model to \u{1F917} Transformers. A template of those model tests is already added by the
Cookiecutter, called `),ah=s(Zu,"CODE",{});var v9=i(ah);PE=n(v9,"BrandNewBertModelIntegrationTests"),v9.forEach(o),IE=n(Zu,` and only has to be filled out by you. To ensure that those
tests are passing, run`),Zu.forEach(o),$p=f(e),u(jr.$$.fragment,e),Pp=f(e),u(Ft.$$.fragment,e),Ip=f(e),te=s(e,"P",{});var ho=i(te);LE=n(ho,"Second, all features that are special to "),sh=s(ho,"EM",{});var b9=i(sh);AE=n(b9,"brand_new_bert"),b9.forEach(o),BE=n(ho,` should be tested additionally in a separate test under
`),ih=s(ho,"CODE",{});var _9=i(ih);ME=n(_9,"BrandNewBertModelTester"),_9.forEach(o),NE=n(ho,"/`"),lh=s(ho,"CODE",{});var E9=i(lh);xE=n(E9,"BrandNewBertModelTest"),E9.forEach(o),OE=n(ho,`. This part is often forgotten but is extremely useful in two
ways:`),ho.forEach(o),Lp=f(e),zt=s(e,"UL",{});var ec=i(zt);Cr=s(ec,"LI",{});var tc=i(Cr);jE=n(tc,`It helps to transfer the knowledge you have acquired during the model addition to the community by showing how the
special features of `),dh=s(tc,"EM",{});var k9=i(dh);CE=n(k9,"brand_new_bert"),k9.forEach(o),qE=n(tc," should work."),tc.forEach(o),SE=f(ec),hh=s(ec,"LI",{});var T9=i(hh);DE=n(T9,"Future contributors can quickly test changes to the model by running those special tests."),T9.forEach(o),ec.forEach(o),Ap=f(e),ha=s(e,"P",{});var $9=i(ha);fh=s($9,"STRONG",{});var P9=i(fh);RE=n(P9,"9. Implement the tokenizer"),P9.forEach(o),$9.forEach(o),Bp=f(e),Ht=s(e,"P",{});var oc=i(Ht);FE=n(oc,"Next, we should add the tokenizer of "),mh=s(oc,"EM",{});var I9=i(mh);zE=n(I9,"brand_new_bert"),I9.forEach(o),HE=n(oc,`. Usually, the tokenizer is equivalent or very similar to an
already existing tokenizer of \u{1F917} Transformers.`),oc.forEach(o),Mp=f(e),fa=s(e,"P",{});var L9=i(fa);WE=n(L9,`It is very important to find/extract the original tokenizer file and to manage to load this file into the \u{1F917}
Transformers\u2019 implementation of the tokenizer.`),L9.forEach(o),Np=f(e),ma=s(e,"P",{});var A9=i(ma);GE=n(A9,"To ensure that the tokenizer works correctly, it is recommended to first create a script in the original repository\nthat inputs a string and returns the `input_ids\u201C. It could look similar to this (in pseudo-code):"),A9.forEach(o),xp=f(e),u(qr.$$.fragment,e),Op=f(e),Wt=s(e,"P",{});var rc=i(Wt);UE=n(rc,`You might have to take a deeper look again into the original repository to find the correct tokenizer function or you
might even have to do changes to your clone of the original repository to only output the `),ph=s(rc,"CODE",{});var B9=i(ph);YE=n(B9,"input_ids"),B9.forEach(o),JE=n(rc,`. Having written
a functional tokenization script that uses the original repository, an analogous script for \u{1F917} Transformers should be
created. It should look similar to this:`),rc.forEach(o),jp=f(e),u(Sr.$$.fragment,e),Cp=f(e),Gt=s(e,"P",{});var nc=i(Gt);XE=n(nc,"When both "),uh=s(nc,"CODE",{});var M9=i(uh);KE=n(M9,"input_ids"),M9.forEach(o),QE=n(nc," yield the same values, as a final step a tokenizer test file should also be added."),nc.forEach(o),qp=f(e),Te=s(e,"P",{});var Ja=i(Te);VE=n(Ja,"Analogous to the modeling test files of "),ch=s(Ja,"EM",{});var N9=i(ch);ZE=n(N9,"brand_new_bert"),N9.forEach(o),e3=n(Ja,", the tokenization test files of "),yh=s(Ja,"EM",{});var x9=i(yh);t3=n(x9,"brand_new_bert"),x9.forEach(o),o3=n(Ja,` should
contain a couple of hard-coded integration tests.`),Ja.forEach(o),Sp=f(e),pa=s(e,"P",{});var O9=i(pa);gh=s(O9,"STRONG",{});var j9=i(gh);r3=n(j9,"10. Run End-to-end integration tests"),j9.forEach(o),O9.forEach(o),Dp=f(e),oe=s(e,"P",{});var fo=i(oe);n3=n(fo,`Having added the tokenizer, you should also add a couple of end-to-end integration tests using both the model and the
tokenizer to `),wh=s(fo,"CODE",{});var C9=i(wh);a3=n(C9,"tests/models/brand_new_bert/test_modeling_brand_new_bert.py"),C9.forEach(o),s3=n(fo,` in \u{1F917} Transformers.
Such a test should show on a meaningful
text-to-text sample that the \u{1F917} Transformers implementation works as expected. A meaningful text-to-text sample can
include `),vh=s(fo,"EM",{});var q9=i(vh);i3=n(q9,"e.g."),q9.forEach(o),l3=n(fo,` a source-to-target-translation pair, an article-to-summary pair, a question-to-answer pair, etc\u2026 If none
of the ported checkpoints has been fine-tuned on a downstream task it is enough to simply rely on the model tests. In a
final step to ensure that the model is fully functional, it is advised that you also run all tests on GPU. It can
happen that you forgot to add some `),bh=s(fo,"CODE",{});var S9=i(bh);d3=n(S9,".to(self.device)"),S9.forEach(o),h3=n(fo,` statements to internal tensors of the model, which in such a
test would show in an error. In case you have no access to a GPU, the Hugging Face team can take care of running those
tests for you.`),fo.forEach(o),Rp=f(e),ua=s(e,"P",{});var D9=i(ua);_h=s(D9,"STRONG",{});var R9=i(_h);f3=n(R9,"11. Add Docstring"),R9.forEach(o),D9.forEach(o),Fp=f(e),re=s(e,"P",{});var mo=i(re);m3=n(mo,"Now, all the necessary functionality for "),Eh=s(mo,"EM",{});var F9=i(Eh);p3=n(F9,"brand_new_bert"),F9.forEach(o),u3=n(mo,` is added - you\u2019re almost done! The only thing left to add is
a nice docstring and a doc page. The Cookiecutter should have added a template file called
`),kh=s(mo,"CODE",{});var z9=i(kh);c3=n(z9,"docs/source/model_doc/brand_new_bert.rst"),z9.forEach(o),y3=n(mo,` that you should fill out. Users of your model will usually first look at
this page before using your model. Hence, the documentation must be understandable and concise. It is very useful for
the community to add some `),Th=s(mo,"EM",{});var H9=i(Th);g3=n(H9,"Tips"),H9.forEach(o),w3=n(mo,` to show how the model should be used. Don\u2019t hesitate to ping the Hugging Face team
regarding the docstrings.`),mo.forEach(o),zp=f(e),$e=s(e,"P",{});var Xa=i($e);v3=n(Xa,"Next, make sure that the docstring added to "),$h=s(Xa,"CODE",{});var W9=i($h);b3=n(W9,"src/transformers/models/brand_new_bert/modeling_brand_new_bert.py"),W9.forEach(o),_3=n(Xa,` is
correct and included all necessary inputs and outputs. We have a detailed guide about writing documentation and our docstring format `),ca=s(Xa,"A",{href:!0});var G9=i(ca);E3=n(G9,"here"),G9.forEach(o),k3=n(Xa,`. It is always to good to remind oneself that documentation should
be treated at least as carefully as the code in \u{1F917} Transformers since the documentation is usually the first contact
point of the community with the model.`),Xa.forEach(o),Hp=f(e),ya=s(e,"P",{});var U9=i(ya);Ph=s(U9,"STRONG",{});var Y9=i(Ph);T3=n(Y9,"Code refactor"),Y9.forEach(o),U9.forEach(o),Wp=f(e),Ut=s(e,"P",{});var ac=i(Ut);$3=n(ac,"Great, now you have added all the necessary code for "),Ih=s(ac,"EM",{});var J9=i(Ih);P3=n(J9,"brand_new_bert"),J9.forEach(o),I3=n(ac,`. At this point, you should correct some potential
incorrect code style by running:`),ac.forEach(o),Gp=f(e),u(Dr.$$.fragment,e),Up=f(e),ga=s(e,"P",{});var X9=i(ga);L3=n(X9,"and verify that your coding style passes the quality check:"),X9.forEach(o),Yp=f(e),u(Rr.$$.fragment,e),Jp=f(e),wa=s(e,"P",{});var K9=i(wa);A3=n(K9,`There are a couple of other very strict design tests in \u{1F917} Transformers that might still be failing, which shows up in
the tests of your pull request. This is often because of some missing information in the docstring or some incorrect
naming. The Hugging Face team will surely help you if you\u2019re stuck here.`),K9.forEach(o),Xp=f(e),va=s(e,"P",{});var Q9=i(va);B3=n(Q9,`Lastly, it is always a good idea to refactor one\u2019s code after having ensured that the code works correctly. With all
tests passing, now it\u2019s a good time to go over the added code again and do some refactoring.`),Q9.forEach(o),Kp=f(e),ba=s(e,"P",{});var V9=i(ba);M3=n(V9,"You have now finished the coding part, congratulation! \u{1F389} You are Awesome! \u{1F60E}"),V9.forEach(o),Qp=f(e),_a=s(e,"P",{});var Z9=i(_a);Lh=s(Z9,"STRONG",{});var e8=i(Lh);N3=n(e8,"12. Upload the models to the model hub"),e8.forEach(o),Z9.forEach(o),Vp=f(e),S=s(e,"P",{});var xe=i(S);x3=n(xe,`In this final part, you should convert and upload all checkpoints to the model hub and add a model card for each
uploaded model checkpoint. You can get familiar with the hub functionalities by reading our `),Ea=s(xe,"A",{href:!0});var t8=i(Ea);O3=n(t8,"Model sharing and uploading Page"),t8.forEach(o),j3=n(xe,`. You should work alongside the Hugging Face team here to decide on a fitting name for each
checkpoint and to get the required access rights to be able to upload the model under the author\u2019s organization of
`),Ah=s(xe,"EM",{});var o8=i(Ah);C3=n(o8,"brand_new_bert"),o8.forEach(o),q3=n(xe,". The "),Bh=s(xe,"CODE",{});var r8=i(Bh);S3=n(r8,"push_to_hub"),r8.forEach(o),D3=n(xe," method, present in all models in "),Mh=s(xe,"CODE",{});var n8=i(Mh);R3=n(n8,"transformers"),n8.forEach(o),F3=n(xe,", is a quick and efficient way to push your checkpoint to the hub. A little snippet is pasted below:"),xe.forEach(o),Zp=f(e),u(Fr.$$.fragment,e),eu=f(e),Yt=s(e,"P",{});var sc=i(Yt);z3=n(sc,`It is worth spending some time to create fitting model cards for each checkpoint. The model cards should highlight the
specific characteristics of this particular checkpoint, `),Nh=s(sc,"EM",{});var a8=i(Nh);H3=n(a8,"e.g."),a8.forEach(o),W3=n(sc,` On which dataset was the checkpoint
pretrained/fine-tuned on? On what down-stream task should the model be used? And also include some code on how to
correctly use the model.`),sc.forEach(o),tu=f(e),ka=s(e,"P",{});var s8=i(ka);xh=s(s8,"STRONG",{});var i8=i(xh);G3=n(i8,"13. (Optional) Add notebook"),i8.forEach(o),s8.forEach(o),ou=f(e),Jt=s(e,"P",{});var ic=i(Jt);U3=n(ic,"It is very helpful to add a notebook that showcases in-detail how "),Oh=s(ic,"EM",{});var l8=i(Oh);Y3=n(l8,"brand_new_bert"),l8.forEach(o),J3=n(ic,` can be used for inference and/or
fine-tuned on a downstream task. This is not mandatory to merge your PR, but very useful for the community.`),ic.forEach(o),ru=f(e),Ta=s(e,"P",{});var d8=i(Ta);jh=s(d8,"STRONG",{});var h8=i(jh);X3=n(h8,"14. Submit your finished PR"),h8.forEach(o),d8.forEach(o),nu=f(e),$a=s(e,"P",{});var f8=i($a);K3=n(f8,`You\u2019re done programming now and can move to the last step, which is getting your PR merged into main. Usually, the
Hugging Face team should have helped you already at this point, but it is worth taking some time to give your finished
PR a nice description and eventually add comments to your code, if you want to point out certain design choices to your
reviewer.`),f8.forEach(o),au=f(e),ot=s(e,"H3",{class:!0});var lc=i(ot);Xt=s(lc,"A",{id:!0,class:!0,href:!0});var m8=i(Xt);Ch=s(m8,"SPAN",{});var p8=i(Ch);u(zr.$$.fragment,p8),p8.forEach(o),m8.forEach(o),Q3=f(lc),qh=s(lc,"SPAN",{});var u8=i(qh);V3=n(u8,"Share your work!!"),u8.forEach(o),lc.forEach(o),su=f(e),Pa=s(e,"P",{});var c8=i(Pa);Z3=n(c8,`Now, it\u2019s time to get some credit from the community for your work! Having completed a model addition is a major
contribution to Transformers and the whole NLP community. Your code and the ported pre-trained models will certainly be
used by hundreds and possibly even thousands of developers and researchers. You should be proud of your work and share
your achievement with the community.`),c8.forEach(o),iu=f(e),Ia=s(e,"P",{});var y8=i(Ia);Sh=s(y8,"STRONG",{});var g8=i(Sh);ek=n(g8,"You have made another model that is super easy to access for everyone in the community! \u{1F92F}"),g8.forEach(o),y8.forEach(o),this.h()},h(){m(P,"name","hf:doc:metadata"),m(P,"content",JSON.stringify(I8)),m(H,"id","how-to-add-a-model-to-transformers"),m(H,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),m(H,"href","#how-to-add-a-model-to-transformers"),m(M,"class","relative group"),m(co,"href","https://github.com/huggingface/transformers/tree/main/templates/adding_a_new_model/open_model_proposals/README.md"),m(co,"rel","nofollow"),m(yo,"href","https://github.com/huggingface/transformers/pulls?q=is%3Apr+label%3A%22PR+for+Model+Addition%22+is%3Aclosed"),m(yo,"rel","nofollow"),m(it,"id","general-overview-of-transformers"),m(it,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),m(it,"href","#general-overview-of-transformers"),m(je,"class","relative group"),m(Yr,"href","philosophy"),m(ht,"id","overview-of-models"),m(ht,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),m(ht,"href","#overview-of-models"),m(Ce,"class","relative group"),m(Xr,"href","/docs/transformers/pr_19043/en/main_classes/model#transformers.PreTrainedModel"),m(Kr,"href","/docs/transformers/pr_19043/en/main_classes/configuration#transformers.PretrainedConfig"),E8(Vr.src,hk="https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/transformers_overview.png")||m(Vr,"src",hk),m(Zr,"href","/docs/transformers/pr_19043/en/main_classes/model#transformers.PreTrainedModel"),m(en,"href","/docs/transformers/pr_19043/en/main_classes/model#transformers.PreTrainedModel"),m(tn,"href","/docs/transformers/pr_19043/en/main_classes/model#transformers.PreTrainedModel.from_pretrained"),m(on,"href","/docs/transformers/pr_19043/en/main_classes/model#transformers.PreTrainedModel.save_pretrained"),m(rn,"href","/docs/transformers/pr_19043/en/main_classes/model#transformers.PreTrainedModel"),m(nn,"href","/docs/transformers/pr_19043/en/main_classes/configuration#transformers.PretrainedConfig"),m(an,"href","/docs/transformers/pr_19043/en/main_classes/model#transformers.PreTrainedModel.save_pretrained"),m(sn,"href","/docs/transformers/pr_19043/en/main_classes/configuration#transformers.PretrainedConfig.save_pretrained"),m(ft,"id","code-style"),m(ft,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),m(ft,"href","#code-style"),m(qe,"class","relative group"),m(Eo,"href","https://github.com/huggingface/transformers/blob/v4.17.0/src/transformers/models/roberta/modeling_roberta.py#L160"),m(Eo,"rel","nofollow"),m(mt,"id","overview-of-tokenizers"),m(mt,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),m(mt,"href","#overview-of-tokenizers"),m(Fe,"class","relative group"),m(pt,"id","stepbystep-recipe-to-add-a-model-to-transformers"),m(pt,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),m(pt,"href","#stepbystep-recipe-to-add-a-model-to-transformers"),m(ze,"class","relative group"),m(Po,"href","https://medium.com/huggingface/from-tensorflow-to-pytorch-265f40ef2a28"),m(Po,"rel","nofollow"),m(Io,"href","https://huggingface.co/thomwolf"),m(Io,"rel","nofollow"),m(Ao,"href","https://huggingface.co/blog/porting-fsmt"),m(Ao,"rel","nofollow"),m(Bo,"href","https://huggingface.co/stas"),m(Bo,"rel","nofollow"),m(Mo,"href","https://www.gnu.org/software/grep/"),m(Mo,"rel","nofollow"),m(No,"href","https://github.com/BurntSushi/ripgrep"),m(No,"rel","nofollow"),m(un,"start","2"),m(cn,"start","3"),m(yn,"start","4"),m(gn,"start","5"),m(wn,"start","6"),m(vn,"start","7"),m(bn,"start","8"),m(_n,"start","9"),m(En,"start","10"),m(kn,"start","11"),m(Tn,"start","12"),m($n,"start","13"),m(Pn,"start","14"),m(ct,"id","1-optional-theoretical-aspects-of-brandnewbert"),m(ct,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),m(ct,"href","#1-optional-theoretical-aspects-of-brandnewbert"),m(He,"class","relative group"),m(In,"href","model_summary"),m(Oo,"href","https://huggingface.co/transformers/#contents"),m(Oo,"rel","nofollow"),m(gt,"id","2-next-prepare-your-environment"),m(gt,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),m(gt,"href","#2-next-prepare-your-environment"),m(Ye,"class","relative group"),m(qo,"href","https://github.com/huggingface/transformers"),m(qo,"rel","nofollow"),m(Ro,"start","3"),m(Wo,"href","https://pytorch.org/get-started/locally/"),m(Wo,"rel","nofollow"),m(Ho,"start","4"),m(Uo,"start","5"),m(bt,"id","34-run-a-pretrained-checkpoint-using-the-original-repository"),m(bt,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),m(bt,"href","#34-run-a-pretrained-checkpoint-using-the-original-repository"),m(Xe,"class","relative group"),m(Qo,"href","https://jupyter.org/"),m(Qo,"rel","nofollow"),m(Vo,"href","https://colab.research.google.com/notebooks/intro.ipynb"),m(Vo,"rel","nofollow"),m(tr,"href","https://gist.github.com/LysandreJik/db4c948f6b4483960de5cbac598ad4ed"),m(tr,"rel","nofollow"),m(or,"href","https://github.com/tensorflow/mesh/tree/master/mesh_tensorflow"),m(or,"rel","nofollow"),m(nr,"href","https://www.tensorflow.org/api_docs/python/tf/print"),m(nr,"rel","nofollow"),m(ar,"href","https://github.com/google/jax/issues/196"),m(ar,"rel","nofollow"),m(Bt,"id","514-port-brandnewbert-to-transformers"),m(Bt,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),m(Bt,"href","#514-port-brandnewbert-to-transformers"),m(Ze,"class","relative group"),m(Rn,"href","#write-a-conversion-script"),m(dr,"href","https://github.com/huggingface/transformers/tree/main/templates/adding_a_new_model"),m(dr,"rel","nofollow"),m(fr,"start","2"),m(pr,"start","3"),m(cr,"start","4"),m(et,"start","5"),m(vr,"href","https://github.com/huggingface/transformers/blob/7acfa95afb8194f8f9c1f4d2c6028224dbed35a2/src/transformers/models/bert/modeling_bert.py#L91"),m(vr,"rel","nofollow"),m(br,"href","https://github.com/huggingface/transformers/blob/main/src/transformers/models/bart/convert_bart_original_pytorch_checkpoint_to_pytorch.py"),m(br,"rel","nofollow"),m(sa,"href","#run-a-pretrained-checkpoint-using-the-original-repository"),m(xr,"href","https://pytorch.org/docs/stable/nn.functional.html?highlight=dropout#torch.nn.functional.dropout"),m(xr,"rel","nofollow"),m(ca,"href","writing-documentation"),m(Ea,"href","model_sharing"),m(Xt,"id","share-your-work"),m(Xt,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),m(Xt,"href","#share-your-work"),m(ot,"class","relative group")},m(e,l){t(document.head,P),d(e,nt,l),d(e,M,l),t(M,H),t(H,Oe),c(D,Oe,null),t(M,po),t(M,le),t(le,de),d(e,uo,l),d(e,R,l),t(R,dc),t(R,Ka),t(Ka,hc),t(R,fc),t(R,Qa),t(Qa,mc),t(R,pc),d(e,Fh,l),d(e,at,l),t(at,uc),t(at,co),t(co,cc),t(at,yc),d(e,zh,l),d(e,Wr,l),t(Wr,gc),d(e,Hh,l),d(e,W,l),t(W,Va),t(Va,wc),t(W,vc),t(W,Za),t(Za,bc),t(W,_c),t(W,es),t(es,Ec),t(W,kc),t(W,he),t(he,Tc),t(he,ts),t(ts,$c),t(he,Pc),t(he,os),t(os,Ic),t(he,Lc),t(he,rs),t(rs,Ac),t(he,Bc),d(e,Wh,l),d(e,st,l),t(st,Mc),t(st,yo),t(yo,Nc),t(st,xc),d(e,Gh,l),d(e,Gr,l),t(Gr,Oc),d(e,Uh,l),d(e,je,l),t(je,it),t(it,ns),c(go,ns,null),t(je,jc),t(je,as),t(as,Cc),d(e,Yh,l),d(e,Ur,l),t(Ur,qc),d(e,Jh,l),d(e,lt,l),t(lt,Sc),t(lt,Yr),t(Yr,Dc),t(lt,Rc),d(e,Xh,l),d(e,ce,l),t(ce,ss),t(ss,Fc),t(ce,zc),t(ce,is),t(is,Hc),t(ce,Wc),t(ce,wo),t(wo,Gc),t(wo,ls),t(ls,Uc),t(wo,Yc),d(e,Kh,l),d(e,dt,l),t(dt,Jc),t(dt,ds),t(ds,Xc),t(dt,Kc),d(e,Qh,l),d(e,Jr,l),t(Jr,Qc),d(e,Vh,l),d(e,Ce,l),t(Ce,ht),t(ht,hs),c(vo,hs,null),t(Ce,Vc),t(Ce,fs),t(fs,Zc),d(e,Zh,l),d(e,G,l),t(G,ey),t(G,Xr),t(Xr,ty),t(G,oy),t(G,Kr),t(Kr,ry),t(G,ny),t(G,ms),t(ms,ay),t(G,sy),d(e,ef,l),d(e,Qr,l),t(Qr,iy),d(e,tf,l),d(e,Vr,l),d(e,of,l),d(e,v,l),t(v,ly),t(v,ps),t(ps,dy),t(v,hy),t(v,us),t(us,fy),t(v,my),t(v,Zr),t(Zr,py),t(v,uy),t(v,en),t(en,cy),t(v,yy),t(v,tn),t(tn,gy),t(v,wy),t(v,on),t(on,vy),t(v,by),t(v,cs),t(cs,_y),t(v,Ey),t(v,ys),t(ys,ky),t(v,Ty),t(v,gs),t(gs,$y),t(v,Py),t(v,ws),t(ws,Iy),t(v,Ly),t(v,vs),t(vs,Ay),t(v,By),t(v,bs),t(bs,My),t(v,Ny),t(v,rn),t(rn,xy),t(v,Oy),t(v,_s),t(_s,jy),t(v,Cy),t(v,Es),t(Es,qy),t(v,Sy),d(e,rf,l),c(bo,e,l),d(e,nf,l),d(e,I,l),t(I,Dy),t(I,nn),t(nn,Ry),t(I,Fy),t(I,ks),t(ks,zy),t(I,Hy),t(I,Ts),t(Ts,Wy),t(I,Gy),t(I,an),t(an,Uy),t(I,Yy),t(I,sn),t(sn,Jy),t(I,Xy),d(e,af,l),d(e,qe,l),t(qe,ft),t(ft,$s),c(_o,$s,null),t(qe,Ky),t(qe,Ps),t(Ps,Qy),d(e,sf,l),d(e,ln,l),t(ln,Vy),d(e,lf,l),d(e,x,l),t(x,Se),t(Se,Zy),t(Se,Is),t(Is,eg),t(Se,tg),t(Se,Eo),t(Eo,og),t(Se,rg),t(x,ng),t(x,De),t(De,ag),t(De,Ls),t(Ls,sg),t(De,ig),t(De,As),t(As,lg),t(De,dg),t(x,hg),t(x,Bs),t(Bs,fg),t(x,mg),t(x,Re),t(Re,pg),t(Re,Ms),t(Ms,ug),t(Re,cg),t(Re,Ns),t(Ns,yg),t(Re,gg),t(x,wg),t(x,xs),t(xs,vg),d(e,df,l),d(e,Fe,l),t(Fe,mt),t(mt,Os),c(ko,Os,null),t(Fe,bg),t(Fe,js),t(js,_g),d(e,hf,l),d(e,dn,l),t(dn,Eg),d(e,ff,l),d(e,ze,l),t(ze,pt),t(pt,Cs),c(To,Cs,null),t(ze,kg),t(ze,qs),t(qs,Tg),d(e,mf,l),d(e,hn,l),t(hn,$g),d(e,pf,l),d(e,ut,l),t(ut,$o),t($o,Po),t(Po,Pg),t($o,Ig),t($o,Io),t(Io,Lg),t(ut,Ag),t(ut,Lo),t(Lo,Ao),t(Ao,Bg),t(Lo,Mg),t(Lo,Bo),t(Bo,Ng),d(e,uf,l),d(e,fn,l),t(fn,xg),d(e,cf,l),d(e,ye,l),t(ye,fe),t(fe,Og),t(fe,Mo),t(Mo,jg),t(fe,Cg),t(fe,No),t(No,qg),t(fe,Sg),t(fe,Ss),t(Ss,Dg),t(fe,Rg),t(ye,Fg),t(ye,Ds),t(Ds,zg),t(ye,Hg),t(ye,Rs),t(Rs,Wg),d(e,yf,l),d(e,mn,l),t(mn,Gg),d(e,gf,l),d(e,pn,l),t(pn,Ug),d(e,wf,l),d(e,b,l),t(b,Fs),t(Fs,zs),t(zs,Hs),t(Hs,Yg),t(b,Jg),t(b,Ws),t(Ws,un),t(un,Gs),t(Gs,Xg),t(b,Kg),t(b,Us),t(Us,cn),t(cn,Ys),t(Ys,Qg),t(b,Vg),t(b,Js),t(Js,yn),t(yn,Xs),t(Xs,Zg),t(b,ew),t(b,Ks),t(Ks,gn),t(gn,Qs),t(Qs,tw),t(b,ow),t(b,Vs),t(Vs,wn),t(wn,Zs),t(Zs,rw),t(b,nw),t(b,ei),t(ei,vn),t(vn,ti),t(ti,aw),t(b,sw),t(b,oi),t(oi,bn),t(bn,ri),t(ri,iw),t(b,lw),t(b,ni),t(ni,_n),t(_n,ai),t(ai,dw),t(b,hw),t(b,si),t(si,En),t(En,ii),t(ii,fw),t(b,mw),t(b,li),t(li,kn),t(kn,di),t(di,pw),t(b,uw),t(b,hi),t(hi,Tn),t(Tn,fi),t(fi,cw),t(b,yw),t(b,mi),t(mi,$n),t($n,pi),t(pi,gw),t(b,ww),t(b,ui),t(ui,Pn),t(Pn,ci),t(ci,vw),d(e,vf,l),d(e,O,l),t(O,bw),t(O,yi),t(yi,_w),t(O,Ew),t(O,gi),t(gi,kw),t(O,Tw),t(O,wi),t(wi,$w),t(O,Pw),t(O,vi),t(vi,Iw),t(O,Lw),d(e,bf,l),d(e,He,l),t(He,ct),t(ct,bi),c(xo,bi,null),t(He,Aw),t(He,_i),t(_i,Bw),d(e,_f,l),d(e,yt,l),t(yt,Mw),t(yt,Ei),t(Ei,Nw),t(yt,xw),d(e,Ef,l),d(e,j,l),t(j,We),t(We,Ow),t(We,ki),t(ki,jw),t(We,Cw),t(We,In),t(In,qw),t(We,Sw),t(j,Dw),t(j,Ge),t(Ge,Rw),t(Ge,Ti),t(Ti,Fw),t(Ge,zw),t(Ge,$i),t($i,Hw),t(Ge,Ww),t(j,Gw),t(j,Pi),t(Pi,Uw),t(j,Yw),t(j,Ue),t(Ue,Jw),t(Ue,Oo),t(Oo,Xw),t(Ue,Kw),t(Ue,Ii),t(Ii,Qw),t(Ue,Vw),t(j,Zw),t(j,Li),t(Li,ev),d(e,kf,l),d(e,Ln,l),t(Ln,tv),d(e,Tf,l),d(e,Ye,l),t(Ye,gt),t(gt,Ai),c(jo,Ai,null),t(Ye,ov),t(Ye,Bi),t(Bi,rv),d(e,$f,l),d(e,wt,l),t(wt,Mi),t(Mi,Co),t(Co,nv),t(Co,qo),t(qo,av),t(Co,sv),t(wt,iv),t(wt,Ni),t(Ni,So),t(So,lv),t(So,xi),t(xi,dv),t(So,hv),d(e,Pf,l),c(Do,e,l),d(e,If,l),d(e,Ro,l),t(Ro,Oi),t(Oi,fv),d(e,Lf,l),c(Fo,e,l),d(e,Af,l),d(e,An,l),t(An,mv),d(e,Bf,l),c(zo,e,l),d(e,Mf,l),d(e,Ho,l),t(Ho,Je),t(Je,pv),t(Je,ji),t(ji,uv),t(Je,cv),t(Je,Wo),t(Wo,yv),t(Je,gv),d(e,Nf,l),d(e,Go,l),t(Go,Ci),t(Ci,wv),t(Go,vv),d(e,xf,l),d(e,Uo,l),t(Uo,Yo),t(Yo,bv),t(Yo,qi),t(qi,_v),t(Yo,Ev),d(e,Of,l),c(Jo,e,l),d(e,jf,l),d(e,vt,l),t(vt,kv),t(vt,Si),t(Si,Tv),t(vt,$v),d(e,Cf,l),d(e,Xe,l),t(Xe,bt),t(bt,Di),c(Xo,Di,null),t(Xe,Pv),t(Xe,Ri),t(Ri,Iv),d(e,qf,l),d(e,L,l),t(L,Lv),t(L,Fi),t(Fi,Av),t(L,Bv),t(L,zi),t(zi,Mv),t(L,Nv),t(L,Hi),t(Hi,xv),t(L,Ov),t(L,Wi),t(Wi,jv),t(L,Cv),t(L,Gi),t(Gi,qv),t(L,Sv),d(e,Sf,l),d(e,Bn,l),t(Bn,Dv),d(e,Df,l),d(e,_t,l),t(_t,Rv),t(_t,Ui),t(Ui,Fv),t(_t,zv),d(e,Rf,l),d(e,A,l),t(A,Yi),t(Yi,Hv),t(A,Wv),t(A,Ji),t(Ji,Gv),t(A,Uv),t(A,Xi),t(Xi,Yv),t(A,Jv),t(A,Ki),t(Ki,Xv),t(A,Kv),t(A,F),t(F,Qv),t(F,Qi),t(Qi,Vv),t(F,Zv),t(F,Vi),t(Vi,eb),t(F,tb),t(F,Zi),t(Zi,ob),t(F,rb),t(F,el),t(el,nb),t(F,ab),t(A,sb),t(A,Ke),t(Ke,ib),t(Ke,tl),t(tl,lb),t(Ke,db),t(Ke,ol),t(ol,hb),t(Ke,fb),d(e,Ff,l),d(e,Et,l),t(Et,mb),t(Et,rl),t(rl,pb),t(Et,ub),d(e,zf,l),d(e,Mn,l),t(Mn,cb),d(e,Hf,l),d(e,Nn,l),t(Nn,yb),d(e,Wf,l),d(e,kt,l),t(kt,Ko),t(Ko,Qo),t(Qo,gb),t(Ko,wb),t(Ko,Vo),t(Vo,vb),t(kt,bb),t(kt,nl),t(nl,_b),d(e,Gf,l),d(e,xn,l),t(xn,Eb),d(e,Uf,l),d(e,Tt,l),t(Tt,kb),t(Tt,al),t(al,Tb),t(Tt,$b),d(e,Yf,l),d(e,$t,l),t($t,Pb),t($t,sl),t(sl,Ib),t($t,Lb),d(e,Jf,l),c(Zo,e,l),d(e,Xf,l),d(e,On,l),t(On,Ab),d(e,Kf,l),d(e,Pt,l),t(Pt,il),t(il,Bb),t(Pt,Mb),t(Pt,Qe),t(Qe,Nb),t(Qe,ll),t(ll,xb),t(Qe,Ob),t(Qe,dl),t(dl,jb),t(Qe,Cb),d(e,Qf,l),d(e,jn,l),t(jn,qb),d(e,Vf,l),d(e,It,l),t(It,Sb),t(It,hl),t(hl,Db),t(It,Rb),d(e,Zf,l),d(e,U,l),t(U,fl),t(fl,Fb),t(U,zb),t(U,ml),t(ml,Hb),t(U,Wb),t(U,pl),t(pl,Gb),t(U,Ub),t(U,ul),t(ul,Yb),d(e,em,l),d(e,er,l),t(er,tr),t(tr,Jb),t(er,Xb),d(e,tm,l),d(e,Lt,l),t(Lt,Kb),t(Lt,or),t(or,Qb),t(Lt,Vb),d(e,om,l),d(e,Cn,l),t(Cn,Zb),d(e,rm,l),d(e,qn,l),t(qn,e_),d(e,nm,l),d(e,B,l),t(B,cl),t(cl,t_),t(B,o_),t(B,yl),t(yl,r_),t(B,n_),t(B,gl),t(gl,a_),t(B,s_),t(B,wl),t(wl,i_),t(B,l_),t(B,vl),t(vl,d_),t(B,h_),t(B,bl),t(bl,f_),d(e,am,l),d(e,Ve,l),t(Ve,m_),t(Ve,_l),t(_l,p_),t(Ve,u_),t(Ve,El),t(El,c_),d(e,sm,l),d(e,Sn,l),t(Sn,y_),d(e,im,l),c(rr,e,l),d(e,lm,l),d(e,ge,l),t(ge,g_),t(ge,kl),t(kl,w_),t(ge,v_),t(ge,Tl),t(Tl,b_),t(ge,__),d(e,dm,l),d(e,C,l),t(C,z),t(z,E_),t(z,nr),t(nr,k_),t(z,T_),t(z,$l),t($l,$_),t(z,P_),t(z,Pl),t(Pl,I_),t(z,L_),t(z,ar),t(ar,A_),t(z,B_),t(C,M_),t(C,Il),t(Il,N_),t(C,x_),t(C,T),t(T,O_),t(T,Ll),t(Ll,j_),t(T,C_),t(T,Al),t(Al,q_),t(T,S_),t(T,Bl),t(Bl,D_),t(T,R_),t(T,Ml),t(Ml,F_),t(T,z_),t(T,Nl),t(Nl,H_),t(T,W_),t(T,xl),t(xl,G_),t(T,U_),t(T,Ol),t(Ol,Y_),t(T,J_),t(T,jl),t(jl,X_),t(T,K_),t(T,Cl),t(Cl,Q_),t(T,V_),t(T,ql),t(ql,Z_),t(T,e1),t(C,t1),t(C,sr),t(sr,o1),t(sr,Sl),t(Sl,r1),t(sr,n1),t(C,a1),t(C,me),t(me,s1),t(me,Dl),t(Dl,i1),t(me,l1),t(me,Rl),t(Rl,d1),t(me,h1),t(me,Fl),t(Fl,f1),t(me,m1),d(e,hm,l),d(e,At,l),t(At,p1),t(At,zl),t(zl,u1),t(At,c1),d(e,fm,l),d(e,Ze,l),t(Ze,Bt),t(Bt,Hl),c(ir,Hl,null),t(Ze,y1),t(Ze,Wl),t(Wl,g1),d(e,mm,l),d(e,Dn,l),t(Dn,w1),d(e,pm,l),c(lr,e,l),d(e,um,l),d(e,Mt,l),t(Mt,v1),t(Mt,Rn),t(Rn,b1),t(Mt,_1),d(e,cm,l),d(e,Fn,l),t(Fn,E1),d(e,ym,l),d(e,Nt,l),t(Nt,zn),t(zn,Gl),t(Gl,k1),t(zn,T1),t(Nt,$1),t(Nt,Hn),t(Hn,Ul),t(Ul,P1),t(Hn,I1),d(e,gm,l),d(e,we,l),t(we,L1),t(we,Yl),t(Yl,A1),t(we,B1),t(we,dr),t(dr,M1),t(we,N1),d(e,wm,l),d(e,Wn,l),t(Wn,Jl),t(Jl,x1),d(e,vm,l),d(e,ve,l),t(ve,O1),t(ve,Xl),t(Xl,j1),t(ve,C1),t(ve,Kl),t(Kl,q1),t(ve,S1),d(e,bm,l),d(e,Gn,l),t(Gn,D1),d(e,_m,l),d(e,Un,l),t(Un,Ql),t(Ql,R1),d(e,Em,l),c(hr,e,l),d(e,km,l),d(e,fr,l),t(fr,Vl),t(Vl,F1),d(e,Tm,l),c(mr,e,l),d(e,$m,l),d(e,pr,l),t(pr,Zl),t(Zl,z1),d(e,Pm,l),c(ur,e,l),d(e,Im,l),d(e,cr,l),t(cr,ed),t(ed,H1),d(e,Lm,l),c(yr,e,l),d(e,Am,l),d(e,et,l),t(et,td),t(td,od),t(od,W1),t(et,G1),t(et,rd),t(rd,nd),t(nd,U1),d(e,Bm,l),d(e,Yn,l),t(Yn,Y1),d(e,Mm,l),c(gr,e,l),d(e,Nm,l),d(e,Jn,l),t(Jn,J1),d(e,xm,l),d(e,Xn,l),t(Xn,X1),d(e,Om,l),d(e,Kn,l),t(Kn,K1),d(e,jm,l),d(e,Qn,l),t(Qn,ad),t(ad,Q1),d(e,Cm,l),d(e,be,l),t(be,V1),t(be,sd),t(sd,Z1),t(be,e0),t(be,id),t(id,t0),t(be,o0),d(e,qm,l),d(e,Y,l),t(Y,r0),t(Y,ld),t(ld,n0),t(Y,a0),t(Y,dd),t(dd,s0),t(Y,i0),t(Y,hd),t(hd,l0),t(Y,d0),d(e,Sm,l),d(e,N,l),t(N,fd),t(fd,h0),t(N,f0),t(N,md),t(md,m0),t(N,p0),t(N,pd),t(pd,u0),t(N,c0),t(N,ud),t(ud,y0),t(N,g0),t(N,cd),t(cd,w0),t(N,v0),d(e,Dm,l),c(wr,e,l),d(e,Rm,l),d(e,_e,l),t(_e,b0),t(_e,yd),t(yd,_0),t(_e,E0),t(_e,gd),t(gd,k0),t(_e,T0),d(e,Fm,l),d(e,Vn,l),t(Vn,wd),t(wd,$0),d(e,zm,l),d(e,J,l),t(J,P0),t(J,vd),t(vd,I0),t(J,L0),t(J,bd),t(bd,A0),t(J,B0),t(J,_d),t(_d,M0),t(J,N0),d(e,Hm,l),d(e,xt,l),t(xt,Zn),t(Zn,x0),t(Zn,vr),t(vr,O0),t(xt,j0),t(xt,ea),t(ea,C0),t(ea,br),t(br,q0),d(e,Wm,l),d(e,Ot,l),t(Ot,S0),t(Ot,Ed),t(Ed,D0),t(Ot,R0),d(e,Gm,l),c(_r,e,l),d(e,Um,l),d(e,X,l),t(X,F0),t(X,kd),t(kd,z0),t(X,H0),t(X,Td),t(Td,W0),t(X,G0),t(X,$d),t($d,U0),t(X,Y0),d(e,Ym,l),c(Er,e,l),d(e,Jm,l),d(e,ta,l),t(ta,J0),d(e,Xm,l),c(kr,e,l),d(e,Km,l),d(e,oa,l),t(oa,X0),d(e,Qm,l),c(Tr,e,l),d(e,Vm,l),d(e,ra,l),t(ra,K0),d(e,Zm,l),c($r,e,l),d(e,ep,l),d(e,Pr,l),t(Pr,Q0),t(Pr,Pd),t(Pd,V0),d(e,tp,l),c(Ir,e,l),d(e,op,l),d(e,Ee,l),t(Ee,Z0),t(Ee,Id),t(Id,e2),t(Ee,t2),t(Ee,Ld),t(Ld,o2),t(Ee,r2),d(e,rp,l),c(Lr,e,l),d(e,np,l),d(e,Ar,l),t(Ar,n2),t(Ar,Ad),t(Ad,a2),d(e,ap,l),c(Br,e,l),d(e,sp,l),d(e,na,l),t(na,s2),d(e,ip,l),d(e,jt,l),t(jt,i2),t(jt,Bd),t(Bd,l2),t(jt,d2),d(e,lp,l),d(e,K,l),t(K,h2),t(K,Md),t(Md,f2),t(K,m2),t(K,Nd),t(Nd,p2),t(K,u2),t(K,xd),t(xd,c2),t(K,y2),d(e,dp,l),d(e,Q,l),t(Q,g2),t(Q,Od),t(Od,w2),t(Q,v2),t(Q,jd),t(jd,b2),t(Q,_2),t(Q,Cd),t(Cd,E2),t(Q,k2),d(e,hp,l),c(Mr,e,l),d(e,fp,l),d(e,aa,l),t(aa,qd),t(qd,T2),d(e,mp,l),d(e,Ct,l),t(Ct,$2),t(Ct,sa),t(sa,P2),t(Ct,I2),d(e,pp,l),c(Nr,e,l),d(e,up,l),d(e,q,l),t(q,L2),t(q,Sd),t(Sd,A2),t(q,B2),t(q,Dd),t(Dd,M2),t(q,N2),t(q,Rd),t(Rd,x2),t(q,O2),t(q,Fd),t(Fd,j2),t(q,C2),d(e,cp,l),d(e,V,l),t(V,q2),t(V,zd),t(zd,S2),t(V,D2),t(V,Hd),t(Hd,R2),t(V,F2),t(V,Wd),t(Wd,z2),t(V,H2),d(e,yp,l),d(e,Z,l),t(Z,tt),t(tt,W2),t(tt,Gd),t(Gd,G2),t(tt,U2),t(tt,Ud),t(Ud,Y2),t(tt,J2),t(Z,X2),t(Z,Yd),t(Yd,K2),t(Z,Q2),t(Z,Jd),t(Jd,V2),t(Z,Z2),t(Z,ee),t(ee,eE),t(ee,Xd),t(Xd,tE),t(ee,oE),t(ee,Kd),t(Kd,rE),t(ee,nE),t(ee,Qd),t(Qd,aE),t(ee,sE),t(ee,xr),t(xr,iE),d(e,gp,l),d(e,ke,l),t(ke,lE),t(ke,Vd),t(Vd,dE),t(ke,hE),t(ke,Zd),t(Zd,fE),t(ke,mE),d(e,wp,l),d(e,qt,l),t(qt,pE),t(qt,eh),t(eh,uE),t(qt,cE),d(e,vp,l),d(e,ia,l),t(ia,th),t(th,yE),d(e,bp,l),d(e,St,l),t(St,gE),t(St,oh),t(oh,wE),t(St,vE),d(e,_p,l),c(Or,e,l),d(e,Ep,l),d(e,la,l),t(la,bE),d(e,kp,l),d(e,Dt,l),t(Dt,da),t(da,_E),t(da,rh),t(rh,EE),t(Dt,kE),t(Dt,nh),t(nh,TE),d(e,Tp,l),d(e,Rt,l),t(Rt,$E),t(Rt,ah),t(ah,PE),t(Rt,IE),d(e,$p,l),c(jr,e,l),d(e,Pp,l),c(Ft,e,l),d(e,Ip,l),d(e,te,l),t(te,LE),t(te,sh),t(sh,AE),t(te,BE),t(te,ih),t(ih,ME),t(te,NE),t(te,lh),t(lh,xE),t(te,OE),d(e,Lp,l),d(e,zt,l),t(zt,Cr),t(Cr,jE),t(Cr,dh),t(dh,CE),t(Cr,qE),t(zt,SE),t(zt,hh),t(hh,DE),d(e,Ap,l),d(e,ha,l),t(ha,fh),t(fh,RE),d(e,Bp,l),d(e,Ht,l),t(Ht,FE),t(Ht,mh),t(mh,zE),t(Ht,HE),d(e,Mp,l),d(e,fa,l),t(fa,WE),d(e,Np,l),d(e,ma,l),t(ma,GE),d(e,xp,l),c(qr,e,l),d(e,Op,l),d(e,Wt,l),t(Wt,UE),t(Wt,ph),t(ph,YE),t(Wt,JE),d(e,jp,l),c(Sr,e,l),d(e,Cp,l),d(e,Gt,l),t(Gt,XE),t(Gt,uh),t(uh,KE),t(Gt,QE),d(e,qp,l),d(e,Te,l),t(Te,VE),t(Te,ch),t(ch,ZE),t(Te,e3),t(Te,yh),t(yh,t3),t(Te,o3),d(e,Sp,l),d(e,pa,l),t(pa,gh),t(gh,r3),d(e,Dp,l),d(e,oe,l),t(oe,n3),t(oe,wh),t(wh,a3),t(oe,s3),t(oe,vh),t(vh,i3),t(oe,l3),t(oe,bh),t(bh,d3),t(oe,h3),d(e,Rp,l),d(e,ua,l),t(ua,_h),t(_h,f3),d(e,Fp,l),d(e,re,l),t(re,m3),t(re,Eh),t(Eh,p3),t(re,u3),t(re,kh),t(kh,c3),t(re,y3),t(re,Th),t(Th,g3),t(re,w3),d(e,zp,l),d(e,$e,l),t($e,v3),t($e,$h),t($h,b3),t($e,_3),t($e,ca),t(ca,E3),t($e,k3),d(e,Hp,l),d(e,ya,l),t(ya,Ph),t(Ph,T3),d(e,Wp,l),d(e,Ut,l),t(Ut,$3),t(Ut,Ih),t(Ih,P3),t(Ut,I3),d(e,Gp,l),c(Dr,e,l),d(e,Up,l),d(e,ga,l),t(ga,L3),d(e,Yp,l),c(Rr,e,l),d(e,Jp,l),d(e,wa,l),t(wa,A3),d(e,Xp,l),d(e,va,l),t(va,B3),d(e,Kp,l),d(e,ba,l),t(ba,M3),d(e,Qp,l),d(e,_a,l),t(_a,Lh),t(Lh,N3),d(e,Vp,l),d(e,S,l),t(S,x3),t(S,Ea),t(Ea,O3),t(S,j3),t(S,Ah),t(Ah,C3),t(S,q3),t(S,Bh),t(Bh,S3),t(S,D3),t(S,Mh),t(Mh,R3),t(S,F3),d(e,Zp,l),c(Fr,e,l),d(e,eu,l),d(e,Yt,l),t(Yt,z3),t(Yt,Nh),t(Nh,H3),t(Yt,W3),d(e,tu,l),d(e,ka,l),t(ka,xh),t(xh,G3),d(e,ou,l),d(e,Jt,l),t(Jt,U3),t(Jt,Oh),t(Oh,Y3),t(Jt,J3),d(e,ru,l),d(e,Ta,l),t(Ta,jh),t(jh,X3),d(e,nu,l),d(e,$a,l),t($a,K3),d(e,au,l),d(e,ot,l),t(ot,Xt),t(Xt,Ch),c(zr,Ch,null),t(ot,Q3),t(ot,qh),t(qh,V3),d(e,su,l),d(e,Pa,l),t(Pa,Z3),d(e,iu,l),d(e,Ia,l),t(Ia,Sh),t(Sh,ek),lu=!0},p(e,[l]){const Hr={};l&2&&(Hr.$$scope={dirty:l,ctx:e}),Ft.$set(Hr)},i(e){lu||(y(D.$$.fragment,e),y(go.$$.fragment,e),y(vo.$$.fragment,e),y(bo.$$.fragment,e),y(_o.$$.fragment,e),y(ko.$$.fragment,e),y(To.$$.fragment,e),y(xo.$$.fragment,e),y(jo.$$.fragment,e),y(Do.$$.fragment,e),y(Fo.$$.fragment,e),y(zo.$$.fragment,e),y(Jo.$$.fragment,e),y(Xo.$$.fragment,e),y(Zo.$$.fragment,e),y(rr.$$.fragment,e),y(ir.$$.fragment,e),y(lr.$$.fragment,e),y(hr.$$.fragment,e),y(mr.$$.fragment,e),y(ur.$$.fragment,e),y(yr.$$.fragment,e),y(gr.$$.fragment,e),y(wr.$$.fragment,e),y(_r.$$.fragment,e),y(Er.$$.fragment,e),y(kr.$$.fragment,e),y(Tr.$$.fragment,e),y($r.$$.fragment,e),y(Ir.$$.fragment,e),y(Lr.$$.fragment,e),y(Br.$$.fragment,e),y(Mr.$$.fragment,e),y(Nr.$$.fragment,e),y(Or.$$.fragment,e),y(jr.$$.fragment,e),y(Ft.$$.fragment,e),y(qr.$$.fragment,e),y(Sr.$$.fragment,e),y(Dr.$$.fragment,e),y(Rr.$$.fragment,e),y(Fr.$$.fragment,e),y(zr.$$.fragment,e),lu=!0)},o(e){g(D.$$.fragment,e),g(go.$$.fragment,e),g(vo.$$.fragment,e),g(bo.$$.fragment,e),g(_o.$$.fragment,e),g(ko.$$.fragment,e),g(To.$$.fragment,e),g(xo.$$.fragment,e),g(jo.$$.fragment,e),g(Do.$$.fragment,e),g(Fo.$$.fragment,e),g(zo.$$.fragment,e),g(Jo.$$.fragment,e),g(Xo.$$.fragment,e),g(Zo.$$.fragment,e),g(rr.$$.fragment,e),g(ir.$$.fragment,e),g(lr.$$.fragment,e),g(hr.$$.fragment,e),g(mr.$$.fragment,e),g(ur.$$.fragment,e),g(yr.$$.fragment,e),g(gr.$$.fragment,e),g(wr.$$.fragment,e),g(_r.$$.fragment,e),g(Er.$$.fragment,e),g(kr.$$.fragment,e),g(Tr.$$.fragment,e),g($r.$$.fragment,e),g(Ir.$$.fragment,e),g(Lr.$$.fragment,e),g(Br.$$.fragment,e),g(Mr.$$.fragment,e),g(Nr.$$.fragment,e),g(Or.$$.fragment,e),g(jr.$$.fragment,e),g(Ft.$$.fragment,e),g(qr.$$.fragment,e),g(Sr.$$.fragment,e),g(Dr.$$.fragment,e),g(Rr.$$.fragment,e),g(Fr.$$.fragment,e),g(zr.$$.fragment,e),lu=!1},d(e){o(P),e&&o(nt),e&&o(M),w(D),e&&o(uo),e&&o(R),e&&o(Fh),e&&o(at),e&&o(zh),e&&o(Wr),e&&o(Hh),e&&o(W),e&&o(Wh),e&&o(st),e&&o(Gh),e&&o(Gr),e&&o(Uh),e&&o(je),w(go),e&&o(Yh),e&&o(Ur),e&&o(Jh),e&&o(lt),e&&o(Xh),e&&o(ce),e&&o(Kh),e&&o(dt),e&&o(Qh),e&&o(Jr),e&&o(Vh),e&&o(Ce),w(vo),e&&o(Zh),e&&o(G),e&&o(ef),e&&o(Qr),e&&o(tf),e&&o(Vr),e&&o(of),e&&o(v),e&&o(rf),w(bo,e),e&&o(nf),e&&o(I),e&&o(af),e&&o(qe),w(_o),e&&o(sf),e&&o(ln),e&&o(lf),e&&o(x),e&&o(df),e&&o(Fe),w(ko),e&&o(hf),e&&o(dn),e&&o(ff),e&&o(ze),w(To),e&&o(mf),e&&o(hn),e&&o(pf),e&&o(ut),e&&o(uf),e&&o(fn),e&&o(cf),e&&o(ye),e&&o(yf),e&&o(mn),e&&o(gf),e&&o(pn),e&&o(wf),e&&o(b),e&&o(vf),e&&o(O),e&&o(bf),e&&o(He),w(xo),e&&o(_f),e&&o(yt),e&&o(Ef),e&&o(j),e&&o(kf),e&&o(Ln),e&&o(Tf),e&&o(Ye),w(jo),e&&o($f),e&&o(wt),e&&o(Pf),w(Do,e),e&&o(If),e&&o(Ro),e&&o(Lf),w(Fo,e),e&&o(Af),e&&o(An),e&&o(Bf),w(zo,e),e&&o(Mf),e&&o(Ho),e&&o(Nf),e&&o(Go),e&&o(xf),e&&o(Uo),e&&o(Of),w(Jo,e),e&&o(jf),e&&o(vt),e&&o(Cf),e&&o(Xe),w(Xo),e&&o(qf),e&&o(L),e&&o(Sf),e&&o(Bn),e&&o(Df),e&&o(_t),e&&o(Rf),e&&o(A),e&&o(Ff),e&&o(Et),e&&o(zf),e&&o(Mn),e&&o(Hf),e&&o(Nn),e&&o(Wf),e&&o(kt),e&&o(Gf),e&&o(xn),e&&o(Uf),e&&o(Tt),e&&o(Yf),e&&o($t),e&&o(Jf),w(Zo,e),e&&o(Xf),e&&o(On),e&&o(Kf),e&&o(Pt),e&&o(Qf),e&&o(jn),e&&o(Vf),e&&o(It),e&&o(Zf),e&&o(U),e&&o(em),e&&o(er),e&&o(tm),e&&o(Lt),e&&o(om),e&&o(Cn),e&&o(rm),e&&o(qn),e&&o(nm),e&&o(B),e&&o(am),e&&o(Ve),e&&o(sm),e&&o(Sn),e&&o(im),w(rr,e),e&&o(lm),e&&o(ge),e&&o(dm),e&&o(C),e&&o(hm),e&&o(At),e&&o(fm),e&&o(Ze),w(ir),e&&o(mm),e&&o(Dn),e&&o(pm),w(lr,e),e&&o(um),e&&o(Mt),e&&o(cm),e&&o(Fn),e&&o(ym),e&&o(Nt),e&&o(gm),e&&o(we),e&&o(wm),e&&o(Wn),e&&o(vm),e&&o(ve),e&&o(bm),e&&o(Gn),e&&o(_m),e&&o(Un),e&&o(Em),w(hr,e),e&&o(km),e&&o(fr),e&&o(Tm),w(mr,e),e&&o($m),e&&o(pr),e&&o(Pm),w(ur,e),e&&o(Im),e&&o(cr),e&&o(Lm),w(yr,e),e&&o(Am),e&&o(et),e&&o(Bm),e&&o(Yn),e&&o(Mm),w(gr,e),e&&o(Nm),e&&o(Jn),e&&o(xm),e&&o(Xn),e&&o(Om),e&&o(Kn),e&&o(jm),e&&o(Qn),e&&o(Cm),e&&o(be),e&&o(qm),e&&o(Y),e&&o(Sm),e&&o(N),e&&o(Dm),w(wr,e),e&&o(Rm),e&&o(_e),e&&o(Fm),e&&o(Vn),e&&o(zm),e&&o(J),e&&o(Hm),e&&o(xt),e&&o(Wm),e&&o(Ot),e&&o(Gm),w(_r,e),e&&o(Um),e&&o(X),e&&o(Ym),w(Er,e),e&&o(Jm),e&&o(ta),e&&o(Xm),w(kr,e),e&&o(Km),e&&o(oa),e&&o(Qm),w(Tr,e),e&&o(Vm),e&&o(ra),e&&o(Zm),w($r,e),e&&o(ep),e&&o(Pr),e&&o(tp),w(Ir,e),e&&o(op),e&&o(Ee),e&&o(rp),w(Lr,e),e&&o(np),e&&o(Ar),e&&o(ap),w(Br,e),e&&o(sp),e&&o(na),e&&o(ip),e&&o(jt),e&&o(lp),e&&o(K),e&&o(dp),e&&o(Q),e&&o(hp),w(Mr,e),e&&o(fp),e&&o(aa),e&&o(mp),e&&o(Ct),e&&o(pp),w(Nr,e),e&&o(up),e&&o(q),e&&o(cp),e&&o(V),e&&o(yp),e&&o(Z),e&&o(gp),e&&o(ke),e&&o(wp),e&&o(qt),e&&o(vp),e&&o(ia),e&&o(bp),e&&o(St),e&&o(_p),w(Or,e),e&&o(Ep),e&&o(la),e&&o(kp),e&&o(Dt),e&&o(Tp),e&&o(Rt),e&&o($p),w(jr,e),e&&o(Pp),w(Ft,e),e&&o(Ip),e&&o(te),e&&o(Lp),e&&o(zt),e&&o(Ap),e&&o(ha),e&&o(Bp),e&&o(Ht),e&&o(Mp),e&&o(fa),e&&o(Np),e&&o(ma),e&&o(xp),w(qr,e),e&&o(Op),e&&o(Wt),e&&o(jp),w(Sr,e),e&&o(Cp),e&&o(Gt),e&&o(qp),e&&o(Te),e&&o(Sp),e&&o(pa),e&&o(Dp),e&&o(oe),e&&o(Rp),e&&o(ua),e&&o(Fp),e&&o(re),e&&o(zp),e&&o($e),e&&o(Hp),e&&o(ya),e&&o(Wp),e&&o(Ut),e&&o(Gp),w(Dr,e),e&&o(Up),e&&o(ga),e&&o(Yp),w(Rr,e),e&&o(Jp),e&&o(wa),e&&o(Xp),e&&o(va),e&&o(Kp),e&&o(ba),e&&o(Qp),e&&o(_a),e&&o(Vp),e&&o(S),e&&o(Zp),w(Fr,e),e&&o(eu),e&&o(Yt),e&&o(tu),e&&o(ka),e&&o(ou),e&&o(Jt),e&&o(ru),e&&o(Ta),e&&o(nu),e&&o($a),e&&o(au),e&&o(ot),w(zr),e&&o(su),e&&o(Pa),e&&o(iu),e&&o(Ia)}}}const I8={local:"how-to-add-a-model-to-transformers",sections:[{local:"general-overview-of-transformers",sections:[{local:"overview-of-models",title:"Overview of models"},{local:"code-style",title:"Code style"},{local:"overview-of-tokenizers",title:"Overview of tokenizers"}],title:"General overview of \u{1F917} Transformers"},{local:"stepbystep-recipe-to-add-a-model-to-transformers",sections:[{local:"1-optional-theoretical-aspects-of-brandnewbert",title:"1. (Optional) Theoretical aspects of BrandNewBert"},{local:"2-next-prepare-your-environment",title:"2. Next prepare your environment"},{local:"34-run-a-pretrained-checkpoint-using-the-original-repository",title:"3.-4. Run a pretrained checkpoint using the original repository"},{local:"514-port-brandnewbert-to-transformers",title:"5.-14. Port BrandNewBert to \u{1F917} Transformers"},{local:"share-your-work",title:"Share your work!!"}],title:"Step-by-step recipe to add a model to \u{1F917} Transformers"}],title:"How to add a model to \u{1F917} Transformers?"};function L8(Rh){return k8(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class x8 extends w8{constructor(P){super();v8(this,P,L8,P8,b8,{})}}export{x8 as default,I8 as metadata};
