import{S as I9,i as q9,s as G9,e as s,k as h,w as u,t as p,Y as le,M as N9,c as i,d as t,m as f,a as l,x as d,h as m,Z as ne,b as r,N as c,G as a,g as o,y as v,q as g,o as b,B as _,v as B9}from"../chunks/vendor-hf-doc-builder.js";import{T as E9}from"../chunks/Tip-hf-doc-builder.js";import{Y as hE}from"../chunks/Youtube-hf-doc-builder.js";import{I as w}from"../chunks/IconCopyLink-hf-doc-builder.js";function F9(Vs){let k,P;return{c(){k=s("p"),P=p(`This model could be very well be used in an autoencoding setting, there is no checkpoint for such a
pretraining yet, though.`)},l(x){k=i(x,"P",{});var A=l(k);P=m(A,`This model could be very well be used in an autoencoding setting, there is no checkpoint for such a
pretraining yet, though.`),A.forEach(t)},m(x,A){o(x,k,A),a(k,P)},d(x){x&&t(k)}}}function D9(Vs){let k,P;return{c(){k=s("p"),P=p(`This model could be very well be used in an autoregressive setting, there is no checkpoint for such a
pretraining yet, though.`)},l(x){k=i(x,"P",{});var A=l(k);P=m(A,`This model could be very well be used in an autoregressive setting, there is no checkpoint for such a
pretraining yet, though.`),A.forEach(t)},m(x,A){o(x,k,A),a(k,P)},d(x){x&&t(k)}}}function H9(Vs){let k,P,x,A,kh,Yt,Cv,xh,zv,Ip,L,jv,Zt,Uv,Xv,Kt,Vv,Yv,Ot,Zv,Kv,qp,Ys,Ov,Gp,T,Ah,Zs,Wv,Jv,Th,Ks,Qv,eg,$h,Os,tg,ag,Mh,Ws,sg,ig,Ph,Js,rg,Np,M,fE,Bp,Qs,lg,Fp,ei,ng,Dp,ti,og,Hp,ai,hg,Cp,si,fg,zp,ii,jp,ge,et,Lh,Wt,pg,Sh,mg,Up,ri,cg,Xp,Jt,Vp,be,tt,Rh,Qt,ug,Ih,dg,Yp,q,li,ni,pE,vg,oi,hi,mE,gg,fi,pi,cE,Zp,ea,ta,bg,_g,Kp,mi,wg,Op,ci,Eg,Wp,_e,at,qh,aa,yg,Gh,kg,Jp,G,ui,di,uE,xg,vi,gi,dE,Ag,bi,_i,vE,Qp,sa,ia,Tg,$g,em,wi,Mg,tm,Ei,Pg,am,we,st,Nh,ra,Lg,Bh,Sg,sm,N,yi,ki,gE,Rg,xi,Ai,bE,Ig,Ti,$i,_E,im,la,na,qg,Gg,rm,Mi,Ng,lm,Pi,Bg,nm,Ee,it,Fh,oa,Fg,Dh,Dg,om,B,Li,Si,wE,Hg,Ri,Ii,EE,Cg,qi,Gi,yE,hm,ha,fa,zg,jg,fm,Ni,Ug,pm,Bi,Xg,mm,Fi,Vg,cm,Di,Yg,um,Hi,dm,ye,rt,Hh,pa,Zg,Ch,Kg,vm,F,Ci,zi,kE,Og,ji,Ui,xE,Wg,Xi,Vi,AE,gm,ma,ca,Jg,Qg,bm,Yi,e1,_m,S,ua,t1,Zi,a1,s1,i1,da,r1,Ki,l1,n1,o1,zh,h1,f1,jh,p1,wm,Oi,m1,Em,lt,ym,Wi,c1,km,ke,nt,Uh,va,u1,Xh,d1,xm,D,Ji,Qi,TE,v1,er,tr,$E,g1,ar,sr,ME,Am,ga,ba,b1,_1,Tm,ir,w1,$m,rr,E1,Mm,lr,y1,Pm,nr,Lm,xe,ot,Vh,_a,k1,Yh,x1,Sm,or,A1,Rm,wa,Im,Ae,ht,Zh,Ea,T1,Kh,$1,qm,H,hr,fr,PE,M1,pr,mr,LE,P1,cr,ur,SE,Gm,ya,ka,L1,S1,Nm,dr,R1,Bm,oe,Oh,I1,q1,Wh,G1,N1,Jh,B1,Fm,vr,F1,Dm,gr,D1,Hm,Te,ft,Qh,xa,H1,ef,C1,Cm,C,br,_r,RE,z1,wr,Er,IE,j1,yr,kr,qE,zm,Aa,Ta,U1,X1,jm,xr,V1,Um,he,tf,Y1,Z1,af,K1,O1,sf,W1,Xm,Ar,J1,Vm,$e,pt,rf,$a,Q1,lf,eb,Ym,z,Tr,$r,GE,tb,Mr,Pr,NE,ab,Lr,Sr,BE,Zm,Ma,Pa,sb,ib,Km,Rr,rb,Om,R,nf,lb,nb,of,ob,hb,hf,fb,pb,ff,mb,Wm,Ir,cb,Jm,Me,mt,pf,La,ub,mf,db,Qm,j,qr,Gr,FE,vb,Nr,Br,DE,gb,Fr,Dr,HE,ec,Sa,Ra,bb,_b,tc,Hr,wb,ac,fe,cf,Eb,yb,uf,kb,xb,df,Ab,sc,Cr,Tb,ic,Pe,ct,vf,Ia,$b,gf,Mb,rc,U,zr,jr,CE,Pb,Ur,Xr,zE,Lb,Vr,Yr,jE,lc,qa,Ga,Sb,Rb,nc,Zr,Ib,oc,Kr,qb,hc,Le,ut,bf,Na,Gb,_f,Nb,fc,X,Or,Wr,UE,Bb,Jr,Qr,XE,Fb,el,tl,VE,pc,Ba,Fa,Db,Hb,mc,al,Cb,cc,pe,wf,zb,jb,Ef,Ub,Xb,yf,Vb,uc,I,Yb,kf,Zb,Kb,xf,Ob,Wb,Af,Jb,Qb,dc,sl,e_,vc,Se,dt,Tf,Da,t_,$f,a_,gc,V,il,rl,YE,s_,ll,nl,ZE,i_,ol,hl,KE,bc,Ha,Ca,r_,l_,_c,fl,n_,wc,pl,o_,Ec,Re,vt,Mf,za,h_,Pf,f_,yc,Y,ml,cl,OE,p_,ul,dl,WE,m_,vl,gl,JE,kc,ja,Ua,c_,u_,xc,bl,d_,Ac,_l,v_,Tc,Ie,gt,Lf,Xa,g_,Sf,b_,$c,Z,wl,El,QE,__,yl,kl,ey,w_,xl,Al,ty,Mc,Va,Ya,E_,y_,Pc,Tl,k_,Lc,$l,x_,Sc,qe,bt,Rf,Za,A_,If,T_,Rc,K,Ml,Pl,ay,$_,Ll,Sl,sy,M_,Rl,Il,iy,Ic,Ka,Oa,P_,L_,qc,ql,S_,Gc,Gl,R_,Nc,Nl,I_,Bc,Bl,q_,Fc,Fl,Dc,Ge,_t,qf,Wa,G_,Gf,N_,Hc,O,Dl,Hl,ry,B_,Cl,zl,ly,F_,jl,Ul,ny,Cc,Ja,Qa,D_,H_,zc,wt,C_,Xl,z_,j_,jc,Vl,U_,Uc,Et,Xc,Yl,X_,Vc,Zl,Yc,Ne,yt,Nf,es,V_,Bf,Y_,Zc,Kl,Z_,Kc,ts,Oc,Be,kt,Ff,as,K_,Df,O_,Wc,W,Ol,Wl,oy,W_,Jl,Ql,hy,J_,en,tn,fy,Jc,ss,is,Q_,ew,Qc,an,tw,eu,$,Hf,aw,sw,Cf,iw,rw,zf,lw,nw,jf,ow,hw,Uf,fw,tu,sn,pw,au,Fe,xt,Xf,rs,mw,Vf,cw,su,J,rn,ln,py,uw,nn,on,my,dw,hn,fn,cy,iu,ls,ns,vw,gw,ru,pn,bw,lu,At,Yf,_w,ww,Zf,Ew,nu,mn,yw,ou,cn,kw,hu,De,Tt,Kf,os,xw,Of,Aw,fu,Q,un,dn,uy,Tw,vn,gn,dy,$w,bn,_n,vy,pu,hs,fs,Mw,Pw,mu,wn,Lw,cu,En,Sw,uu,He,$t,Wf,ps,Rw,Jf,Iw,du,ee,yn,kn,gy,qw,xn,An,by,Gw,Tn,$n,_y,vu,ms,cs,Nw,Bw,gu,Mn,Fw,bu,Pn,Dw,_u,Ln,Hw,wu,Sn,Cw,Eu,Rn,zw,yu,Ce,Mt,Qf,us,jw,ep,Uw,ku,te,In,qn,wy,Xw,Gn,Nn,Ey,Vw,Bn,Fn,yy,xu,ds,vs,Yw,Zw,Au,Dn,Kw,Tu,Hn,Ow,$u,ze,Pt,tp,gs,Ww,ap,Jw,Mu,ae,Cn,zn,ky,Qw,jn,Un,xy,e2,Xn,Vn,Ay,Pu,bs,_s,t2,a2,Lu,Yn,s2,Su,Zn,i2,Ru,Lt,r2,ws,l2,n2,Iu,me,o2,Es,h2,f2,sp,p2,m2,qu,je,St,ip,ys,c2,rp,u2,Gu,se,Kn,On,Ty,d2,Wn,Jn,$y,v2,Qn,eo,My,Nu,ks,xs,g2,b2,Bu,ce,_2,lp,w2,E2,np,y2,k2,Fu,to,x2,Du,Ue,Rt,op,As,A2,hp,T2,Hu,ie,ao,so,Py,$2,io,ro,Ly,M2,lo,no,Sy,Cu,Ts,$s,P2,L2,zu,It,S2,Ms,R2,I2,ju,oo,q2,Uu,ho,Xu,Xe,qt,fp,Ps,G2,pp,N2,Vu,fo,B2,Yu,Ve,Gt,mp,Ls,F2,cp,D2,Zu,Ss,Rs,H2,C2,Ku,po,z2,Ou,mo,j2,Wu,co,U2,Ju,uo,Qu,Ye,Nt,up,Is,X2,dp,V2,ed,vo,Y2,td,Ze,Bt,vp,qs,Z2,gp,K2,ad,re,go,bo,Ry,O2,_o,wo,Iy,W2,Eo,yo,qy,sd,Gs,Ns,J2,Q2,id,ko,e0,rd,xo,t0,ld,ue,bp,a0,s0,_p,i0,r0,wp,l0,nd,Ao,n0,od,Ke,Ft,Ep,Bs,o0,yp,h0,hd,Oe,To,$o,Gy,f0,Mo,Po,Ny,fd,Fs,Ds,p0,m0,pd,Lo,c0,md,So,u0,cd,We,Dt,kp,Hs,d0,xp,v0,ud,Je,Ht,Ap,Cs,g0,Tp,b0,dd,Ro,_0,vd,Io,gd,qo,$p,w0,bd,zs,Go,E0,y0,_d,No,wd,Bo,Mp,k0,Ed,js,Fo,x0,A0,yd,Do,T0,kd,Ct,By,xd,Ho,$0,Ad,Qe,zt,Pp,Us,M0,Lp,P0,Td,Co,$d,zo,Sp,L0,Md,E,jo,S0,R0,Pd,y9='<span class="katex"><span class="katex-mathml"><math xmlns="http://www.w3.org/1998/Math/MathML"><semantics><mrow><mi>l</mi></mrow><annotation encoding="application/x-tex">l</annotation></semantics></math></span><span class="katex-html" aria-hidden="true"><span class="base"><span class="strut" style="height:0.6944em;"></span><span class="mord mathnormal" style="margin-right:0.01968em;">l</span></span></span></span>',Ld,Sd,k9='<span class="katex"><span class="katex-mathml"><math xmlns="http://www.w3.org/1998/Math/MathML"><semantics><mrow><mi>d</mi></mrow><annotation encoding="application/x-tex">d</annotation></semantics></math></span><span class="katex-html" aria-hidden="true"><span class="base"><span class="strut" style="height:0.6944em;"></span><span class="mord mathnormal">d</span></span></span></span>',Rd,Id,x9='<span class="katex"><span class="katex-mathml"><math xmlns="http://www.w3.org/1998/Math/MathML"><semantics><mrow><mi>l</mi></mrow><annotation encoding="application/x-tex">l</annotation></semantics></math></span><span class="katex-html" aria-hidden="true"><span class="base"><span class="strut" style="height:0.6944em;"></span><span class="mord mathnormal" style="margin-right:0.01968em;">l</span></span></span></span>',qd,Gd,A9='<span class="katex"><span class="katex-mathml"><math xmlns="http://www.w3.org/1998/Math/MathML"><semantics><mrow><mi>d</mi></mrow><annotation encoding="application/x-tex">d</annotation></semantics></math></span><span class="katex-html" aria-hidden="true"><span class="base"><span class="strut" style="height:0.6944em;"></span><span class="mord mathnormal">d</span></span></span></span>',Nd,Bd,T9='<span class="katex"><span class="katex-mathml"><math xmlns="http://www.w3.org/1998/Math/MathML"><semantics><mrow><msub><mi>l</mi><mn>1</mn></msub><mo>\xD7</mo><msub><mi>d</mi><mn>1</mn></msub></mrow><annotation encoding="application/x-tex">l_{1} \\times d_{1}</annotation></semantics></math></span><span class="katex-html" aria-hidden="true"><span class="base"><span class="strut" style="height:0.8444em;vertical-align:-0.15em;"></span><span class="mord"><span class="mord mathnormal" style="margin-right:0.01968em;">l</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:0.3011em;"><span style="top:-2.55em;margin-left:-0.0197em;margin-right:0.05em;"><span class="pstrut" style="height:2.7em;"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mtight"><span class="mord mtight">1</span></span></span></span></span><span class="vlist-s">\u200B</span></span><span class="vlist-r"><span class="vlist" style="height:0.15em;"><span></span></span></span></span></span></span><span class="mspace" style="margin-right:0.2222em;"></span><span class="mbin">\xD7</span><span class="mspace" style="margin-right:0.2222em;"></span></span><span class="base"><span class="strut" style="height:0.8444em;vertical-align:-0.15em;"></span><span class="mord"><span class="mord mathnormal">d</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:0.3011em;"><span style="top:-2.55em;margin-left:0em;margin-right:0.05em;"><span class="pstrut" style="height:2.7em;"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mtight"><span class="mord mtight">1</span></span></span></span></span><span class="vlist-s">\u200B</span></span><span class="vlist-r"><span class="vlist" style="height:0.15em;"><span></span></span></span></span></span></span></span></span></span>',Fd,Dd,$9='<span class="katex"><span class="katex-mathml"><math xmlns="http://www.w3.org/1998/Math/MathML"><semantics><mrow><msub><mi>l</mi><mn>2</mn></msub><mo>\xD7</mo><msub><mi>d</mi><mn>2</mn></msub></mrow><annotation encoding="application/x-tex">l_{2} \\times d_{2}</annotation></semantics></math></span><span class="katex-html" aria-hidden="true"><span class="base"><span class="strut" style="height:0.8444em;vertical-align:-0.15em;"></span><span class="mord"><span class="mord mathnormal" style="margin-right:0.01968em;">l</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:0.3011em;"><span style="top:-2.55em;margin-left:-0.0197em;margin-right:0.05em;"><span class="pstrut" style="height:2.7em;"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mtight"><span class="mord mtight">2</span></span></span></span></span><span class="vlist-s">\u200B</span></span><span class="vlist-r"><span class="vlist" style="height:0.15em;"><span></span></span></span></span></span></span><span class="mspace" style="margin-right:0.2222em;"></span><span class="mbin">\xD7</span><span class="mspace" style="margin-right:0.2222em;"></span></span><span class="base"><span class="strut" style="height:0.8444em;vertical-align:-0.15em;"></span><span class="mord"><span class="mord mathnormal">d</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:0.3011em;"><span style="top:-2.55em;margin-left:0em;margin-right:0.05em;"><span class="pstrut" style="height:2.7em;"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mtight"><span class="mord mtight">2</span></span></span></span></span><span class="vlist-s">\u200B</span></span><span class="vlist-r"><span class="vlist" style="height:0.15em;"><span></span></span></span></span></span></span></span></span></span>',Hd,Cd,M9='<span class="katex"><span class="katex-mathml"><math xmlns="http://www.w3.org/1998/Math/MathML"><semantics><mrow><msub><mi>l</mi><mn>1</mn></msub><mo>\xD7</mo><msub><mi>l</mi><mn>2</mn></msub><mo>=</mo><mi>l</mi></mrow><annotation encoding="application/x-tex">l_{1} \\times l_{2} = l</annotation></semantics></math></span><span class="katex-html" aria-hidden="true"><span class="base"><span class="strut" style="height:0.8444em;vertical-align:-0.15em;"></span><span class="mord"><span class="mord mathnormal" style="margin-right:0.01968em;">l</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:0.3011em;"><span style="top:-2.55em;margin-left:-0.0197em;margin-right:0.05em;"><span class="pstrut" style="height:2.7em;"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mtight"><span class="mord mtight">1</span></span></span></span></span><span class="vlist-s">\u200B</span></span><span class="vlist-r"><span class="vlist" style="height:0.15em;"><span></span></span></span></span></span></span><span class="mspace" style="margin-right:0.2222em;"></span><span class="mbin">\xD7</span><span class="mspace" style="margin-right:0.2222em;"></span></span><span class="base"><span class="strut" style="height:0.8444em;vertical-align:-0.15em;"></span><span class="mord"><span class="mord mathnormal" style="margin-right:0.01968em;">l</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:0.3011em;"><span style="top:-2.55em;margin-left:-0.0197em;margin-right:0.05em;"><span class="pstrut" style="height:2.7em;"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mtight"><span class="mord mtight">2</span></span></span></span></span><span class="vlist-s">\u200B</span></span><span class="vlist-r"><span class="vlist" style="height:0.15em;"><span></span></span></span></span></span></span><span class="mspace" style="margin-right:0.2778em;"></span><span class="mrel">=</span><span class="mspace" style="margin-right:0.2778em;"></span></span><span class="base"><span class="strut" style="height:0.6944em;"></span><span class="mord mathnormal" style="margin-right:0.01968em;">l</span></span></span></span>',zd,jd,P9='<span class="katex"><span class="katex-mathml"><math xmlns="http://www.w3.org/1998/Math/MathML"><semantics><mrow><msub><mi>d</mi><mn>1</mn></msub><mo>+</mo><msub><mi>d</mi><mn>2</mn></msub><mo>=</mo><mi>d</mi></mrow><annotation encoding="application/x-tex">d_{1} + d_{2} = d</annotation></semantics></math></span><span class="katex-html" aria-hidden="true"><span class="base"><span class="strut" style="height:0.8444em;vertical-align:-0.15em;"></span><span class="mord"><span class="mord mathnormal">d</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:0.3011em;"><span style="top:-2.55em;margin-left:0em;margin-right:0.05em;"><span class="pstrut" style="height:2.7em;"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mtight"><span class="mord mtight">1</span></span></span></span></span><span class="vlist-s">\u200B</span></span><span class="vlist-r"><span class="vlist" style="height:0.15em;"><span></span></span></span></span></span></span><span class="mspace" style="margin-right:0.2222em;"></span><span class="mbin">+</span><span class="mspace" style="margin-right:0.2222em;"></span></span><span class="base"><span class="strut" style="height:0.8444em;vertical-align:-0.15em;"></span><span class="mord"><span class="mord mathnormal">d</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:0.3011em;"><span style="top:-2.55em;margin-left:0em;margin-right:0.05em;"><span class="pstrut" style="height:2.7em;"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mtight"><span class="mord mtight">2</span></span></span></span></span><span class="vlist-s">\u200B</span></span><span class="vlist-r"><span class="vlist" style="height:0.15em;"><span></span></span></span></span></span></span><span class="mspace" style="margin-right:0.2778em;"></span><span class="mrel">=</span><span class="mspace" style="margin-right:0.2778em;"></span></span><span class="base"><span class="strut" style="height:0.6944em;"></span><span class="mord mathnormal">d</span></span></span></span>',Ud,Xd,L9='<span class="katex"><span class="katex-mathml"><math xmlns="http://www.w3.org/1998/Math/MathML"><semantics><mrow><mi>j</mi></mrow><annotation encoding="application/x-tex">j</annotation></semantics></math></span><span class="katex-html" aria-hidden="true"><span class="base"><span class="strut" style="height:0.854em;vertical-align:-0.1944em;"></span><span class="mord mathnormal" style="margin-right:0.05724em;">j</span></span></span></span>',Vd,Yd,S9='<span class="katex"><span class="katex-mathml"><math xmlns="http://www.w3.org/1998/Math/MathML"><semantics><mrow><mi>j</mi><mi mathvariant="normal">%</mi><mi>l</mi><mn>1</mn></mrow><annotation encoding="application/x-tex">j \\% l1</annotation></semantics></math></span><span class="katex-html" aria-hidden="true"><span class="base"><span class="strut" style="height:0.9444em;vertical-align:-0.1944em;"></span><span class="mord mathnormal" style="margin-right:0.05724em;">j</span><span class="mord">%</span><span class="mord mathnormal" style="margin-right:0.01968em;">l</span><span class="mord">1</span></span></span></span>',Zd,Kd,R9='<span class="katex"><span class="katex-mathml"><math xmlns="http://www.w3.org/1998/Math/MathML"><semantics><mrow><mi>j</mi><mi mathvariant="normal">/</mi><mi mathvariant="normal">/</mi><mi>l</mi><mn>1</mn></mrow><annotation encoding="application/x-tex">j // l1</annotation></semantics></math></span><span class="katex-html" aria-hidden="true"><span class="base"><span class="strut" style="height:1em;vertical-align:-0.25em;"></span><span class="mord mathnormal" style="margin-right:0.05724em;">j</span><span class="mord">//</span><span class="mord mathnormal" style="margin-right:0.01968em;">l</span><span class="mord">1</span></span></span></span>',Od,Wd;return Yt=new w({}),Wt=new w({}),Jt=new hE({props:{id:"d_ixlCubqQw"}}),Qt=new w({}),aa=new w({}),ra=new w({}),oa=new w({}),pa=new w({}),lt=new E9({props:{$$slots:{default:[F9]},$$scope:{ctx:Vs}}}),va=new w({}),_a=new w({}),wa=new hE({props:{id:"MUqNwgPjJvQ"}}),Ea=new w({}),xa=new w({}),$a=new w({}),La=new w({}),Ia=new w({}),Na=new w({}),Da=new w({}),za=new w({}),Xa=new w({}),Za=new w({}),Wa=new w({}),Et=new E9({props:{$$slots:{default:[D9]},$$scope:{ctx:Vs}}}),es=new w({}),ts=new hE({props:{id:"0_4KEb08xrE"}}),as=new w({}),rs=new w({}),os=new w({}),ps=new w({}),us=new w({}),gs=new w({}),ys=new w({}),As=new w({}),Ps=new w({}),Ls=new w({}),Is=new w({}),qs=new w({}),Bs=new w({}),Hs=new w({}),Cs=new w({}),Us=new w({}),{c(){k=s("meta"),P=h(),x=s("h1"),A=s("a"),kh=s("span"),u(Yt.$$.fragment),Cv=h(),xh=s("span"),zv=p("Summary of the models"),Ip=h(),L=s("p"),jv=p("This is a summary of the models available in \u{1F917} Transformers. It assumes you\u2019re familiar with the original "),Zt=s("a"),Uv=p(`transformer
model`),Xv=p(". For a gentle introduction check the "),Kt=s("a"),Vv=p("annotated transformer"),Yv=p(`. Here we focus on the high-level differences between the
models. You can check them more in detail in their respective documentation. Also check out `),Ot=s("a"),Zv=p("the Model Hub"),Kv=p(" where you can filter the checkpoints by model architecture."),qp=h(),Ys=s("p"),Ov=p("Each one of the models in the library falls into one of the following categories:"),Gp=h(),T=s("ul"),Ah=s("li"),Zs=s("a"),Wv=p("autoregressive-models"),Jv=h(),Th=s("li"),Ks=s("a"),Qv=p("autoencoding-models"),eg=h(),$h=s("li"),Os=s("a"),tg=p("seq-to-seq-models"),ag=h(),Mh=s("li"),Ws=s("a"),sg=p("multimodal-models"),ig=h(),Ph=s("li"),Js=s("a"),rg=p("retrieval-based-models"),Np=h(),M=s("iframe"),Bp=h(),Qs=s("p"),lg=p(`Autoregressive models are pretrained on the classic language modeling task: guess the next token having read all the
previous ones. They correspond to the decoder of the original transformer model, and a mask is used on top of the full
sentence so that the attention heads can only see what was before in the text, and not what\u2019s after. Although those
models can be fine-tuned and achieve great results on many tasks, the most natural application is text generation. A
typical example of such models is GPT.`),Fp=h(),ei=s("p"),ng=p(`Autoencoding models are pretrained by corrupting the input tokens in some way and trying to reconstruct the original
sentence. They correspond to the encoder of the original transformer model in the sense that they get access to the
full inputs without any mask. Those models usually build a bidirectional representation of the whole sentence. They can
be fine-tuned and achieve great results on many tasks such as text generation, but their most natural application is
sentence classification or token classification. A typical example of such models is BERT.`),Dp=h(),ti=s("p"),og=p(`Note that the only difference between autoregressive models and autoencoding models is in the way the model is
pretrained. Therefore, the same architecture can be used for both autoregressive and autoencoding models. When a given
model has been used for both types of pretraining, we have put it in the category corresponding to the article where it
was first introduced.`),Hp=h(),ai=s("p"),hg=p(`Sequence-to-sequence models use both the encoder and the decoder of the original transformer, either for translation
tasks or by transforming other tasks to sequence-to-sequence problems. They can be fine-tuned to many tasks but their
most natural applications are translation, summarization and question answering. The original transformer model is an
example of such a model (only for translation), T5 is an example that can be fine-tuned on other tasks.`),Cp=h(),si=s("p"),fg=p("Multimodal models mix text inputs with other kinds (e.g. images) and are more specific to a given task."),zp=h(),ii=s("a"),jp=h(),ge=s("h2"),et=s("a"),Lh=s("span"),u(Wt.$$.fragment),pg=h(),Sh=s("span"),mg=p("Decoders or autoregressive models"),Up=h(),ri=s("p"),cg=p(`As mentioned before, these models rely on the decoder part of the original transformer and use an attention mask so
that at each position, the model can only look at the tokens before the attention heads.`),Xp=h(),u(Jt.$$.fragment),Vp=h(),be=s("h3"),tt=s("a"),Rh=s("span"),u(Qt.$$.fragment),ug=h(),Ih=s("span"),dg=p("Original GPT"),Yp=h(),q=s("div"),li=s("a"),ni=s("img"),vg=h(),oi=s("a"),hi=s("img"),gg=h(),fi=s("a"),pi=s("img"),Zp=h(),ea=s("p"),ta=s("a"),bg=p("Improving Language Understanding by Generative Pre-Training"),_g=p(", Alec Radford et al."),Kp=h(),mi=s("p"),wg=p("The first autoregressive model based on the transformer architecture, pretrained on the Book Corpus dataset."),Op=h(),ci=s("p"),Eg=p(`The library provides versions of the model for language modeling and multitask language modeling/multiple choice
classification.`),Wp=h(),_e=s("h3"),at=s("a"),qh=s("span"),u(aa.$$.fragment),yg=h(),Gh=s("span"),kg=p("GPT-2"),Jp=h(),G=s("div"),ui=s("a"),di=s("img"),xg=h(),vi=s("a"),gi=s("img"),Ag=h(),bi=s("a"),_i=s("img"),Qp=h(),sa=s("p"),ia=s("a"),Tg=p("Language Models are Unsupervised Multitask Learners"),$g=p(`,
Alec Radford et al.`),em=h(),wi=s("p"),Mg=p(`A bigger and better version of GPT, pretrained on WebText (web pages from outgoing links in Reddit with 3 karmas or
more).`),tm=h(),Ei=s("p"),Pg=p(`The library provides versions of the model for language modeling and multitask language modeling/multiple choice
classification.`),am=h(),we=s("h3"),st=s("a"),Nh=s("span"),u(ra.$$.fragment),Lg=h(),Bh=s("span"),Sg=p("CTRL"),sm=h(),N=s("div"),yi=s("a"),ki=s("img"),Rg=h(),xi=s("a"),Ai=s("img"),Ig=h(),Ti=s("a"),$i=s("img"),im=h(),la=s("p"),na=s("a"),qg=p("CTRL: A Conditional Transformer Language Model for Controllable Generation"),Gg=p(`,
Nitish Shirish Keskar et al.`),rm=h(),Mi=s("p"),Ng=p(`Same as the GPT model but adds the idea of control codes. Text is generated from a prompt (can be empty) and one (or
several) of those control codes which are then used to influence the text generation: generate with the style of
wikipedia article, a book or a movie review.`),lm=h(),Pi=s("p"),Bg=p("The library provides a version of the model for language modeling only."),nm=h(),Ee=s("h3"),it=s("a"),Fh=s("span"),u(oa.$$.fragment),Fg=h(),Dh=s("span"),Dg=p("Transformer-XL"),om=h(),B=s("div"),Li=s("a"),Si=s("img"),Hg=h(),Ri=s("a"),Ii=s("img"),Cg=h(),qi=s("a"),Gi=s("img"),hm=h(),ha=s("p"),fa=s("a"),zg=p("Transformer-XL: Attentive Language Models Beyond a Fixed-Length Context"),jg=p(`, Zihang
Dai et al.`),fm=h(),Ni=s("p"),Ug=p(`Same as a regular GPT model, but introduces a recurrence mechanism for two consecutive segments (similar to a regular
RNNs with two consecutive inputs). In this context, a segment is a number of consecutive tokens (for instance 512) that
may span across multiple documents, and segments are fed in order to the model.`),pm=h(),Bi=s("p"),Xg=p(`Basically, the hidden states of the previous segment are concatenated to the current input to compute the attention
scores. This allows the model to pay attention to information that was in the previous segment as well as the current
one. By stacking multiple attention layers, the receptive field can be increased to multiple previous segments.`),mm=h(),Fi=s("p"),Vg=p(`This changes the positional embeddings to positional relative embeddings (as the regular positional embeddings would
give the same results in the current input and the current hidden state at a given position) and needs to make some
adjustments in the way attention scores are computed.`),cm=h(),Di=s("p"),Yg=p("The library provides a version of the model for language modeling only."),um=h(),Hi=s("a"),dm=h(),ye=s("h3"),rt=s("a"),Hh=s("span"),u(pa.$$.fragment),Zg=h(),Ch=s("span"),Kg=p("Reformer"),vm=h(),F=s("div"),Ci=s("a"),zi=s("img"),Og=h(),ji=s("a"),Ui=s("img"),Wg=h(),Xi=s("a"),Vi=s("img"),gm=h(),ma=s("p"),ca=s("a"),Jg=p("Reformer: The Efficient Transformer"),Qg=p(", Nikita Kitaev et al ."),bm=h(),Yi=s("p"),e1=p(`An autoregressive transformer model with lots of tricks to reduce memory footprint and compute time. Those tricks
include:`),_m=h(),S=s("ul"),ua=s("li"),t1=p("Use "),Zi=s("a"),a1=p("Axial position encoding"),s1=p(` (see below for more details). It\u2019s a mechanism to avoid
having a huge positional encoding matrix (when the sequence length is very big) by factorizing it into smaller
matrices.`),i1=h(),da=s("li"),r1=p("Replace traditional attention by "),Ki=s("a"),l1=p("LSH (local-sensitive hashing) attention"),n1=p(` (see below for more
details). It\u2019s a technique to avoid computing the full product query-key in the attention layers.`),o1=h(),zh=s("li"),h1=p(`Avoid storing the intermediate results of each layer by using reversible transformer layers to obtain them during
the backward pass (subtracting the residuals from the input of the next layer gives them back) or recomputing them
for results inside a given layer (less efficient than storing them but saves memory).`),f1=h(),jh=s("li"),p1=p("Compute the feedforward operations by chunks and not on the whole batch."),wm=h(),Oi=s("p"),m1=p("With those tricks, the model can be fed much larger sentences than traditional transformer autoregressive models."),Em=h(),u(lt.$$.fragment),ym=h(),Wi=s("p"),c1=p("The library provides a version of the model for language modeling only."),km=h(),ke=s("h3"),nt=s("a"),Uh=s("span"),u(va.$$.fragment),u1=h(),Xh=s("span"),d1=p("XLNet"),xm=h(),D=s("div"),Ji=s("a"),Qi=s("img"),v1=h(),er=s("a"),tr=s("img"),g1=h(),ar=s("a"),sr=s("img"),Am=h(),ga=s("p"),ba=s("a"),b1=p("XLNet: Generalized Autoregressive Pretraining for Language Understanding"),_1=p(`, Zhilin
Yang et al.`),Tm=h(),ir=s("p"),w1=p(`XLNet is not a traditional autoregressive model but uses a training strategy that builds on that. It permutes the
tokens in the sentence, then allows the model to use the last n tokens to predict the token n+1. Since this is all done
with a mask, the sentence is actually fed in the model in the right order, but instead of masking the first n tokens
for n+1, XLNet uses a mask that hides the previous tokens in some given permutation of 1,\u2026,sequence length.`),$m=h(),rr=s("p"),E1=p("XLNet also uses the same recurrence mechanism as Transformer-XL to build long-term dependencies."),Mm=h(),lr=s("p"),y1=p(`The library provides a version of the model for language modeling, token classification, sentence classification,
multiple choice classification and question answering.`),Pm=h(),nr=s("a"),Lm=h(),xe=s("h2"),ot=s("a"),Vh=s("span"),u(_a.$$.fragment),k1=h(),Yh=s("span"),x1=p("Encoders or autoencoding models"),Sm=h(),or=s("p"),A1=p(`As mentioned before, these models rely on the encoder part of the original transformer and use no mask so the model can
look at all the tokens in the attention heads. For pretraining, targets are the original sentences and inputs are their
corrupted versions.`),Rm=h(),u(wa.$$.fragment),Im=h(),Ae=s("h3"),ht=s("a"),Zh=s("span"),u(Ea.$$.fragment),T1=h(),Kh=s("span"),$1=p("BERT"),qm=h(),H=s("div"),hr=s("a"),fr=s("img"),M1=h(),pr=s("a"),mr=s("img"),P1=h(),cr=s("a"),ur=s("img"),Gm=h(),ya=s("p"),ka=s("a"),L1=p("BERT: Pre-training of Deep Bidirectional Transformers for Language Understanding"),S1=p(`,
Jacob Devlin et al.`),Nm=h(),dr=s("p"),R1=p(`Corrupts the inputs by using random masking, more precisely, during pretraining, a given percentage of tokens (usually
15%) is masked by:`),Bm=h(),oe=s("ul"),Oh=s("li"),I1=p("a special mask token with probability 0.8"),q1=h(),Wh=s("li"),G1=p("a random token different from the one masked with probability 0.1"),N1=h(),Jh=s("li"),B1=p("the same token with probability 0.1"),Fm=h(),vr=s("p"),F1=p(`The model must predict the original sentence, but has a second objective: inputs are two sentences A and B (with a
separation token in between). With probability 50%, the sentences are consecutive in the corpus, in the remaining 50%
they are not related. The model has to predict if the sentences are consecutive or not.`),Dm=h(),gr=s("p"),D1=p(`The library provides a version of the model for language modeling (traditional or masked), next sentence prediction,
token classification, sentence classification, multiple choice classification and question answering.`),Hm=h(),Te=s("h3"),ft=s("a"),Qh=s("span"),u(xa.$$.fragment),H1=h(),ef=s("span"),C1=p("ALBERT"),Cm=h(),C=s("div"),br=s("a"),_r=s("img"),z1=h(),wr=s("a"),Er=s("img"),j1=h(),yr=s("a"),kr=s("img"),zm=h(),Aa=s("p"),Ta=s("a"),U1=p("ALBERT: A Lite BERT for Self-supervised Learning of Language Representations"),X1=p(`,
Zhenzhong Lan et al.`),jm=h(),xr=s("p"),V1=p("Same as BERT but with a few tweaks:"),Um=h(),he=s("ul"),tf=s("li"),Y1=p(`Embedding size E is different from hidden size H justified because the embeddings are context independent (one
embedding vector represents one token), whereas hidden states are context dependent (one hidden state represents a
sequence of tokens) so it\u2019s more logical to have H >> E. Also, the embedding matrix is large since it\u2019s V x E (V
being the vocab size). If E < H, it has less parameters.`),Z1=h(),af=s("li"),K1=p("Layers are split in groups that share parameters (to save memory)."),O1=h(),sf=s("li"),W1=p(`Next sentence prediction is replaced by a sentence ordering prediction: in the inputs, we have two sentences A and
B (that are consecutive) and we either feed A followed by B or B followed by A. The model must predict if they have
been swapped or not.`),Xm=h(),Ar=s("p"),J1=p(`The library provides a version of the model for masked language modeling, token classification, sentence
classification, multiple choice classification and question answering.`),Vm=h(),$e=s("h3"),pt=s("a"),rf=s("span"),u($a.$$.fragment),Q1=h(),lf=s("span"),eb=p("RoBERTa"),Ym=h(),z=s("div"),Tr=s("a"),$r=s("img"),tb=h(),Mr=s("a"),Pr=s("img"),ab=h(),Lr=s("a"),Sr=s("img"),Zm=h(),Ma=s("p"),Pa=s("a"),sb=p("RoBERTa: A Robustly Optimized BERT Pretraining Approach"),ib=p(", Yinhan Liu et al."),Km=h(),Rr=s("p"),rb=p("Same as BERT with better pretraining tricks:"),Om=h(),R=s("ul"),nf=s("li"),lb=p("dynamic masking: tokens are masked differently at each epoch, whereas BERT does it once and for all"),nb=h(),of=s("li"),ob=p(`no NSP (next sentence prediction) loss and instead of putting just two sentences together, put a chunk of
contiguous texts together to reach 512 tokens (so the sentences are in an order than may span several documents)`),hb=h(),hf=s("li"),fb=p("train with larger batches"),pb=h(),ff=s("li"),mb=p("use BPE with bytes as a subunit and not characters (because of unicode characters)"),Wm=h(),Ir=s("p"),cb=p(`The library provides a version of the model for masked language modeling, token classification, sentence
classification, multiple choice classification and question answering.`),Jm=h(),Me=s("h3"),mt=s("a"),pf=s("span"),u(La.$$.fragment),ub=h(),mf=s("span"),db=p("DistilBERT"),Qm=h(),j=s("div"),qr=s("a"),Gr=s("img"),vb=h(),Nr=s("a"),Br=s("img"),gb=h(),Fr=s("a"),Dr=s("img"),ec=h(),Sa=s("p"),Ra=s("a"),bb=p("DistilBERT, a distilled version of BERT: smaller, faster, cheaper and lighter"),_b=p(`,
Victor Sanh et al.`),tc=h(),Hr=s("p"),wb=p(`Same as BERT but smaller. Trained by distillation of the pretrained BERT model, meaning it\u2019s been trained to predict
the same probabilities as the larger model. The actual objective is a combination of:`),ac=h(),fe=s("ul"),cf=s("li"),Eb=p("finding the same probabilities as the teacher model"),yb=h(),uf=s("li"),kb=p("predicting the masked tokens correctly (but no next-sentence objective)"),xb=h(),df=s("li"),Ab=p("a cosine similarity between the hidden states of the student and the teacher model"),sc=h(),Cr=s("p"),Tb=p(`The library provides a version of the model for masked language modeling, token classification, sentence classification
and question answering.`),ic=h(),Pe=s("h3"),ct=s("a"),vf=s("span"),u(Ia.$$.fragment),$b=h(),gf=s("span"),Mb=p("ConvBERT"),rc=h(),U=s("div"),zr=s("a"),jr=s("img"),Pb=h(),Ur=s("a"),Xr=s("img"),Lb=h(),Vr=s("a"),Yr=s("img"),lc=h(),qa=s("p"),Ga=s("a"),Sb=p("ConvBERT: Improving BERT with Span-based Dynamic Convolution"),Rb=p(`, Zihang Jiang,
Weihao Yu, Daquan Zhou, Yunpeng Chen, Jiashi Feng, Shuicheng Yan.`),nc=h(),Zr=s("p"),Ib=p(`Pre-trained language models like BERT and its variants have recently achieved impressive performance in various natural
language understanding tasks. However, BERT heavily relies on the global self-attention block and thus suffers large
memory footprint and computation cost. Although all its attention heads query on the whole input sequence for
generating the attention map from a global perspective, we observe some heads only need to learn local dependencies,
which means the existence of computation redundancy. We therefore propose a novel span-based dynamic convolution to
replace these self-attention heads to directly model local dependencies. The novel convolution heads, together with the
rest self-attention heads, form a new mixed attention block that is more efficient at both global and local context
learning. We equip BERT with this mixed attention design and build a ConvBERT model. Experiments have shown that
ConvBERT significantly outperforms BERT and its variants in various downstream tasks, with lower training cost and
fewer model parameters. Remarkably, ConvBERTbase model achieves 86.4 GLUE score, 0.7 higher than ELECTRAbase, while
using less than 1/4 training cost.`),oc=h(),Kr=s("p"),qb=p(`The library provides a version of the model for masked language modeling, token classification, sentence classification
and question answering.`),hc=h(),Le=s("h3"),ut=s("a"),bf=s("span"),u(Na.$$.fragment),Gb=h(),_f=s("span"),Nb=p("XLM"),fc=h(),X=s("div"),Or=s("a"),Wr=s("img"),Bb=h(),Jr=s("a"),Qr=s("img"),Fb=h(),el=s("a"),tl=s("img"),pc=h(),Ba=s("p"),Fa=s("a"),Db=p("Cross-lingual Language Model Pretraining"),Hb=p(", Guillaume Lample and Alexis Conneau"),mc=h(),al=s("p"),Cb=p(`A transformer model trained on several languages. There are three different type of training for this model and the
library provides checkpoints for all of them:`),cc=h(),pe=s("ul"),wf=s("li"),zb=p(`Causal language modeling (CLM) which is the traditional autoregressive training (so this model could be in the
previous section as well). One of the languages is selected for each training sample, and the model input is a
sentence of 256 tokens, that may span over several documents in one of those languages.`),jb=h(),Ef=s("li"),Ub=p(`Masked language modeling (MLM) which is like RoBERTa. One of the languages is selected for each training sample,
and the model input is a sentence of 256 tokens, that may span over several documents in one of those languages,
with dynamic masking of the tokens.`),Xb=h(),yf=s("li"),Vb=p(`A combination of MLM and translation language modeling (TLM). This consists of concatenating a sentence in two
different languages, with random masking. To predict one of the masked tokens, the model can use both, the
surrounding context in language 1 and the context given by language 2.`),uc=h(),I=s("p"),Yb=p("Checkpoints refer to which method was used for pretraining by having "),kf=s("em"),Zb=p("clm"),Kb=p(", "),xf=s("em"),Ob=p("mlm"),Wb=p(" or "),Af=s("em"),Jb=p("mlm-tlm"),Qb=p(` in their names. On top
of positional embeddings, the model has language embeddings. When training using MLM/CLM, this gives the model an
indication of the language used, and when training using MLM+TLM, an indication of the language used for each part.`),dc=h(),sl=s("p"),e_=p(`The library provides a version of the model for language modeling, token classification, sentence classification and
question answering.`),vc=h(),Se=s("h3"),dt=s("a"),Tf=s("span"),u(Da.$$.fragment),t_=h(),$f=s("span"),a_=p("XLM-RoBERTa"),gc=h(),V=s("div"),il=s("a"),rl=s("img"),s_=h(),ll=s("a"),nl=s("img"),i_=h(),ol=s("a"),hl=s("img"),bc=h(),Ha=s("p"),Ca=s("a"),r_=p("Unsupervised Cross-lingual Representation Learning at Scale"),l_=p(`, Alexis Conneau et
al.`),_c=h(),fl=s("p"),n_=p(`Uses RoBERTa tricks on the XLM approach, but does not use the translation language modeling objective. It only uses
masked language modeling on sentences coming from one language. However, the model is trained on many more languages
(100) and doesn\u2019t use the language embeddings, so it\u2019s capable of detecting the input language by itself.`),wc=h(),pl=s("p"),o_=p(`The library provides a version of the model for masked language modeling, token classification, sentence
classification, multiple choice classification and question answering.`),Ec=h(),Re=s("h3"),vt=s("a"),Mf=s("span"),u(za.$$.fragment),h_=h(),Pf=s("span"),f_=p("FlauBERT"),yc=h(),Y=s("div"),ml=s("a"),cl=s("img"),p_=h(),ul=s("a"),dl=s("img"),m_=h(),vl=s("a"),gl=s("img"),kc=h(),ja=s("p"),Ua=s("a"),c_=p("FlauBERT: Unsupervised Language Model Pre-training for French"),u_=p(", Hang Le et al."),xc=h(),bl=s("p"),d_=p("Like RoBERTa, without the sentence ordering prediction (so just trained on the MLM objective)."),Ac=h(),_l=s("p"),v_=p("The library provides a version of the model for language modeling and sentence classification."),Tc=h(),Ie=s("h3"),gt=s("a"),Lf=s("span"),u(Xa.$$.fragment),g_=h(),Sf=s("span"),b_=p("ELECTRA"),$c=h(),Z=s("div"),wl=s("a"),El=s("img"),__=h(),yl=s("a"),kl=s("img"),w_=h(),xl=s("a"),Al=s("img"),Mc=h(),Va=s("p"),Ya=s("a"),E_=p("ELECTRA: Pre-training Text Encoders as Discriminators Rather Than Generators"),y_=p(`,
Kevin Clark et al.`),Pc=h(),Tl=s("p"),k_=p(`ELECTRA is a transformer model pretrained with the use of another (small) masked language model. The inputs are
corrupted by that language model, which takes an input text that is randomly masked and outputs a text in which ELECTRA
has to predict which token is an original and which one has been replaced. Like for GAN training, the small language
model is trained for a few steps (but with the original texts as objective, not to fool the ELECTRA model like in a
traditional GAN setting) then the ELECTRA model is trained for a few steps.`),Lc=h(),$l=s("p"),x_=p(`The library provides a version of the model for masked language modeling, token classification and sentence
classification.`),Sc=h(),qe=s("h3"),bt=s("a"),Rf=s("span"),u(Za.$$.fragment),A_=h(),If=s("span"),T_=p("Funnel Transformer"),Rc=h(),K=s("div"),Ml=s("a"),Pl=s("img"),$_=h(),Ll=s("a"),Sl=s("img"),M_=h(),Rl=s("a"),Il=s("img"),Ic=h(),Ka=s("p"),Oa=s("a"),P_=p("Funnel-Transformer: Filtering out Sequential Redundancy for Efficient Language Processing"),L_=p(", Zihang Dai et al."),qc=h(),ql=s("p"),S_=p(`Funnel Transformer is a transformer model using pooling, a bit like a ResNet model: layers are grouped in blocks, and
at the beginning of each block (except the first one), the hidden states are pooled among the sequence dimension. This
way, their length is divided by 2, which speeds up the computation of the next hidden states. All pretrained models
have three blocks, which means the final hidden state has a sequence length that is one fourth of the original sequence
length.`),Gc=h(),Gl=s("p"),R_=p(`For tasks such as classification, this is not a problem, but for tasks like masked language modeling or token
classification, we need a hidden state with the same sequence length as the original input. In those cases, the final
hidden states are upsampled to the input sequence length and go through two additional layers. That\u2019s why there are two
versions of each checkpoint. The version suffixed with \u201C-base\u201D contains only the three blocks, while the version
without that suffix contains the three blocks and the upsampling head with its additional layers.`),Nc=h(),Nl=s("p"),I_=p("The pretrained models available use the same pretraining objective as ELECTRA."),Bc=h(),Bl=s("p"),q_=p(`The library provides a version of the model for masked language modeling, token classification, sentence
classification, multiple choice classification and question answering.`),Fc=h(),Fl=s("a"),Dc=h(),Ge=s("h3"),_t=s("a"),qf=s("span"),u(Wa.$$.fragment),G_=h(),Gf=s("span"),N_=p("Longformer"),Hc=h(),O=s("div"),Dl=s("a"),Hl=s("img"),B_=h(),Cl=s("a"),zl=s("img"),F_=h(),jl=s("a"),Ul=s("img"),Cc=h(),Ja=s("p"),Qa=s("a"),D_=p("Longformer: The Long-Document Transformer"),H_=p(", Iz Beltagy et al."),zc=h(),wt=s("p"),C_=p(`A transformer model replacing the attention matrices by sparse matrices to go faster. Often, the local context (e.g.,
what are the two tokens left and right?) is enough to take action for a given token. Some preselected input tokens are
still given global attention, but the attention matrix has way less parameters, resulting in a speed-up. See the
`),Xl=s("a"),z_=p("local attention section"),j_=p(" for more information."),jc=h(),Vl=s("p"),U_=p("It is pretrained the same way a RoBERTa otherwise."),Uc=h(),u(Et.$$.fragment),Xc=h(),Yl=s("p"),X_=p(`The library provides a version of the model for masked language modeling, token classification, sentence
classification, multiple choice classification and question answering.`),Vc=h(),Zl=s("a"),Yc=h(),Ne=s("h2"),yt=s("a"),Nf=s("span"),u(es.$$.fragment),V_=h(),Bf=s("span"),Y_=p("Sequence-to-sequence models"),Zc=h(),Kl=s("p"),Z_=p("As mentioned before, these models keep both the encoder and the decoder of the original transformer."),Kc=h(),u(ts.$$.fragment),Oc=h(),Be=s("h3"),kt=s("a"),Ff=s("span"),u(as.$$.fragment),K_=h(),Df=s("span"),O_=p("BART"),Wc=h(),W=s("div"),Ol=s("a"),Wl=s("img"),W_=h(),Jl=s("a"),Ql=s("img"),J_=h(),en=s("a"),tn=s("img"),Jc=h(),ss=s("p"),is=s("a"),Q_=p("BART: Denoising Sequence-to-Sequence Pre-training for Natural Language Generation, Translation, and Comprehension"),ew=p(", Mike Lewis et al."),Qc=h(),an=s("p"),tw=p(`Sequence-to-sequence model with an encoder and a decoder. Encoder is fed a corrupted version of the tokens, decoder is
fed the original tokens (but has a mask to hide the future words like a regular transformers decoder). A composition of
the following transformations are applied on the pretraining tasks for the encoder:`),eu=h(),$=s("ul"),Hf=s("li"),aw=p("mask random tokens (like in BERT)"),sw=h(),Cf=s("li"),iw=p("delete random tokens"),rw=h(),zf=s("li"),lw=p("mask a span of k tokens with a single mask token (a span of 0 tokens is an insertion of a mask token)"),nw=h(),jf=s("li"),ow=p("permute sentences"),hw=h(),Uf=s("li"),fw=p("rotate the document to make it start at a specific token"),tu=h(),sn=s("p"),pw=p("The library provides a version of this model for conditional generation and sequence classification."),au=h(),Fe=s("h3"),xt=s("a"),Xf=s("span"),u(rs.$$.fragment),mw=h(),Vf=s("span"),cw=p("Pegasus"),su=h(),J=s("div"),rn=s("a"),ln=s("img"),uw=h(),nn=s("a"),on=s("img"),dw=h(),hn=s("a"),fn=s("img"),iu=h(),ls=s("p"),ns=s("a"),vw=p("PEGASUS: Pre-training with Extracted Gap-sentences forAbstractive Summarization"),gw=p(", Jingqing Zhang, Yao Zhao, Mohammad Saleh and Peter J. Liu on Dec 18, 2019."),ru=h(),pn=s("p"),bw=p(`Sequence-to-sequence model with the same encoder-decoder model architecture as BART. Pegasus is pre-trained jointly on
two self-supervised objective functions: Masked Language Modeling (MLM) and a novel summarization specific pretraining
objective, called Gap Sentence Generation (GSG).`),lu=h(),At=s("ul"),Yf=s("li"),_w=p(`MLM: encoder input tokens are randomly replaced by a mask tokens and have to be predicted by the encoder (like in
BERT)`),ww=h(),Zf=s("li"),Ew=p(`GSG: whole encoder input sentences are replaced by a second mask token and fed to the decoder, but which has a
causal mask to hide the future words like a regular auto-regressive transformer decoder.`),nu=h(),mn=s("p"),yw=p(`In contrast to BART, Pegasus\u2019 pretraining task is intentionally similar to summarization: important sentences are
masked and are generated together as one output sequence from the remaining sentences, similar to an extractive
summary.`),ou=h(),cn=s("p"),kw=p("The library provides a version of this model for conditional generation, which should be used for summarization."),hu=h(),De=s("h3"),Tt=s("a"),Kf=s("span"),u(os.$$.fragment),xw=h(),Of=s("span"),Aw=p("MarianMT"),fu=h(),Q=s("div"),un=s("a"),dn=s("img"),Tw=h(),vn=s("a"),gn=s("img"),$w=h(),bn=s("a"),_n=s("img"),pu=h(),hs=s("p"),fs=s("a"),Mw=p("Marian: Fast Neural Machine Translation in C++"),Pw=p(", Marcin Junczys-Dowmunt et al."),mu=h(),wn=s("p"),Lw=p("A framework for translation models, using the same models as BART"),cu=h(),En=s("p"),Sw=p("The library provides a version of this model for conditional generation."),uu=h(),He=s("h3"),$t=s("a"),Wf=s("span"),u(ps.$$.fragment),Rw=h(),Jf=s("span"),Iw=p("T5"),du=h(),ee=s("div"),yn=s("a"),kn=s("img"),qw=h(),xn=s("a"),An=s("img"),Gw=h(),Tn=s("a"),$n=s("img"),vu=h(),ms=s("p"),cs=s("a"),Nw=p("Exploring the Limits of Transfer Learning with a Unified Text-to-Text Transformer"),Bw=p(", Colin Raffel et al."),gu=h(),Mn=s("p"),Fw=p(`Uses the traditional transformer model (with a slight change in the positional embeddings, which are learned at each
layer). To be able to operate on all NLP tasks, it transforms them into text-to-text problems by using specific
prefixes: \u201Csummarize: \u201D, \u201Cquestion: \u201D, \u201Ctranslate English to German: \u201D and so forth.`),bu=h(),Pn=s("p"),Dw=p(`The pretraining includes both supervised and self-supervised training. Supervised training is conducted on downstream
tasks provided by the GLUE and SuperGLUE benchmarks (converting them into text-to-text tasks as explained above).`),_u=h(),Ln=s("p"),Hw=p(`Self-supervised training uses corrupted tokens, by randomly removing 15% of the tokens and replacing them with
individual sentinel tokens (if several consecutive tokens are marked for removal, the whole group is replaced with a
single sentinel token). The input of the encoder is the corrupted sentence, the input of the decoder is the original
sentence and the target is then the dropped out tokens delimited by their sentinel tokens.`),wu=h(),Sn=s("p"),Cw=p(`For instance, if we have the sentence \u201CMy dog is very cute .\u201D, and we decide to remove the tokens: \u201Cdog\u201D, \u201Cis\u201D and
\u201Ccute\u201D, the encoder input becomes \u201CMy <x> very <y> .\u201D and the target input becomes \u201C<x> dog is <y> cute .<z>\u201D`),Eu=h(),Rn=s("p"),zw=p("The library provides a version of this model for conditional generation."),yu=h(),Ce=s("h3"),Mt=s("a"),Qf=s("span"),u(us.$$.fragment),jw=h(),ep=s("span"),Uw=p("MT5"),ku=h(),te=s("div"),In=s("a"),qn=s("img"),Xw=h(),Gn=s("a"),Nn=s("img"),Vw=h(),Bn=s("a"),Fn=s("img"),xu=h(),ds=s("p"),vs=s("a"),Yw=p("mT5: A massively multilingual pre-trained text-to-text transformer"),Zw=p(`, Linting Xue
et al.`),Au=h(),Dn=s("p"),Kw=p(`The model architecture is same as T5. mT5\u2019s pretraining objective includes T5\u2019s self-supervised training, but not T5\u2019s
supervised training. mT5 is trained on 101 languages.`),Tu=h(),Hn=s("p"),Ow=p("The library provides a version of this model for conditional generation."),$u=h(),ze=s("h3"),Pt=s("a"),tp=s("span"),u(gs.$$.fragment),Ww=h(),ap=s("span"),Jw=p("MBart"),Mu=h(),ae=s("div"),Cn=s("a"),zn=s("img"),Qw=h(),jn=s("a"),Un=s("img"),e2=h(),Xn=s("a"),Vn=s("img"),Pu=h(),bs=s("p"),_s=s("a"),t2=p("Multilingual Denoising Pre-training for Neural Machine Translation"),a2=p(` by Yinhan Liu,
Jiatao Gu, Naman Goyal, Xian Li, Sergey Edunov Marjan Ghazvininejad, Mike Lewis, Luke Zettlemoyer.`),Lu=h(),Yn=s("p"),s2=p(`The model architecture and pretraining objective is same as BART, but MBart is trained on 25 languages and is intended
for supervised and unsupervised machine translation. MBart is one of the first methods for pretraining a complete
sequence-to-sequence model by denoising full texts in multiple languages,`),Su=h(),Zn=s("p"),i2=p("The library provides a version of this model for conditional generation."),Ru=h(),Lt=s("p"),r2=p("The "),ws=s("a"),l2=p("mbart-large-en-ro checkpoint"),n2=p(` can be used for english ->
romanian translation.`),Iu=h(),me=s("p"),o2=p("The "),Es=s("a"),h2=p("mbart-large-cc25"),f2=p(` checkpoint can be finetuned for other
translation and summarization tasks, using code in `),sp=s("code"),p2=p("examples/pytorch/translation/"),m2=p(` , but is not very useful without
finetuning.`),qu=h(),je=s("h3"),St=s("a"),ip=s("span"),u(ys.$$.fragment),c2=h(),rp=s("span"),u2=p("ProphetNet"),Gu=h(),se=s("div"),Kn=s("a"),On=s("img"),d2=h(),Wn=s("a"),Jn=s("img"),v2=h(),Qn=s("a"),eo=s("img"),Nu=h(),ks=s("p"),xs=s("a"),g2=p("ProphetNet: Predicting Future N-gram for Sequence-to-Sequence Pre-training,"),b2=p(` by
Yu Yan, Weizhen Qi, Yeyun Gong, Dayiheng Liu, Nan Duan, Jiusheng Chen, Ruofei Zhang, Ming Zhou.`),Bu=h(),ce=s("p"),_2=p("ProphetNet introduces a novel "),lp=s("em"),w2=p("sequence-to-sequence"),E2=p(" pretraining objective, called "),np=s("em"),y2=p("future n-gram prediction"),k2=p(`. In
future n-gram prediction, the model predicts the next n tokens simultaneously based on previous context tokens at each
time step instead instead of just the single next token. The future n-gram prediction explicitly encourages the model
to plan for the future tokens and prevent overfitting on strong local correlations. The model architecture is based on
the original Transformer, but replaces the \u201Cstandard\u201D self-attention mechanism in the decoder by a a main
self-attention mechanism and a self and n-stream (predict) self-attention mechanism.`),Fu=h(),to=s("p"),x2=p(`The library provides a pre-trained version of this model for conditional generation and a fine-tuned version for
summarization.`),Du=h(),Ue=s("h3"),Rt=s("a"),op=s("span"),u(As.$$.fragment),A2=h(),hp=s("span"),T2=p("XLM-ProphetNet"),Hu=h(),ie=s("div"),ao=s("a"),so=s("img"),$2=h(),io=s("a"),ro=s("img"),M2=h(),lo=s("a"),no=s("img"),Cu=h(),Ts=s("p"),$s=s("a"),P2=p("ProphetNet: Predicting Future N-gram for Sequence-to-Sequence Pre-training,"),L2=p(` by
Yu Yan, Weizhen Qi, Yeyun Gong, Dayiheng Liu, Nan Duan, Jiusheng Chen, Ruofei Zhang, Ming Zhou.`),zu=h(),It=s("p"),S2=p(`XLM-ProphetNet\u2019s model architecture and pretraining objective is same as ProphetNet, but XLM-ProphetNet was pre-trained
on the cross-lingual dataset `),Ms=s("a"),R2=p("XGLUE"),I2=p("."),ju=h(),oo=s("p"),q2=p(`The library provides a pre-trained version of this model for multi-lingual conditional generation and fine-tuned
versions for headline generation and question generation, respectively.`),Uu=h(),ho=s("a"),Xu=h(),Xe=s("h2"),qt=s("a"),fp=s("span"),u(Ps.$$.fragment),G2=h(),pp=s("span"),N2=p("Multimodal models"),Vu=h(),fo=s("p"),B2=p(`There is one multimodal model in the library which has not been pretrained in the self-supervised fashion like the
others.`),Yu=h(),Ve=s("h3"),Gt=s("a"),mp=s("span"),u(Ls.$$.fragment),F2=h(),cp=s("span"),D2=p("MMBT"),Zu=h(),Ss=s("p"),Rs=s("a"),H2=p("Supervised Multimodal Bitransformers for Classifying Images and Text"),C2=p(`, Douwe Kiela
et al.`),Ku=h(),po=s("p"),z2=p(`A transformers model used in multimodal settings, combining a text and an image to make predictions. The transformer
model takes as inputs the embeddings of the tokenized text and the final activations of a pretrained on images resnet
(after the pooling layer) that goes through a linear layer (to go from number of features at the end of the resnet to
the hidden state dimension of the transformer).`),Ou=h(),mo=s("p"),j2=p(`The different inputs are concatenated, and on top of the positional embeddings, a segment embedding is added to let the
model know which part of the input vector corresponds to the text and which to the image.`),Wu=h(),co=s("p"),U2=p("The pretrained model only works for classification."),Ju=h(),uo=s("a"),Qu=h(),Ye=s("h2"),Nt=s("a"),up=s("span"),u(Is.$$.fragment),X2=h(),dp=s("span"),V2=p("Retrieval-based models"),ed=h(),vo=s("p"),Y2=p("Some models use documents retrieval during (pre)training and inference for open-domain question answering, for example."),td=h(),Ze=s("h3"),Bt=s("a"),vp=s("span"),u(qs.$$.fragment),Z2=h(),gp=s("span"),K2=p("DPR"),ad=h(),re=s("div"),go=s("a"),bo=s("img"),O2=h(),_o=s("a"),wo=s("img"),W2=h(),Eo=s("a"),yo=s("img"),sd=h(),Gs=s("p"),Ns=s("a"),J2=p("Dense Passage Retrieval for Open-Domain Question Answering"),Q2=p(`, Vladimir Karpukhin et
al.`),id=h(),ko=s("p"),e0=p(`Dense Passage Retrieval (DPR) - is a set of tools and models for state-of-the-art open-domain question-answering
research.`),rd=h(),xo=s("p"),t0=p("DPR consists in three models:"),ld=h(),ue=s("ul"),bp=s("li"),a0=p("Question encoder: encode questions as vectors"),s0=h(),_p=s("li"),i0=p("Context encoder: encode contexts as vectors"),r0=h(),wp=s("li"),l0=p(`Reader: extract the answer of the questions inside retrieved contexts, along with a relevance score (high if the
inferred span actually answers the question).`),nd=h(),Ao=s("p"),n0=p(`DPR\u2019s pipeline (not implemented yet) uses a retrieval step to find the top k contexts given a certain question, and
then it calls the reader with the question and the retrieved documents to get the answer.`),od=h(),Ke=s("h3"),Ft=s("a"),Ep=s("span"),u(Bs.$$.fragment),o0=h(),yp=s("span"),h0=p("RAG"),hd=h(),Oe=s("div"),To=s("a"),$o=s("img"),f0=h(),Mo=s("a"),Po=s("img"),fd=h(),Fs=s("p"),Ds=s("a"),p0=p("Retrieval-Augmented Generation for Knowledge-Intensive NLP Tasks"),m0=p(`, Patrick Lewis,
Ethan Perez, Aleksandara Piktus, Fabio Petroni, Vladimir Karpukhin, Naman Goyal, Heinrich K\xFCttler, Mike Lewis, Wen-tau
Yih, Tim Rockt\xE4schel, Sebastian Riedel, Douwe Kiela`),pd=h(),Lo=s("p"),c0=p(`Retrieval-augmented generation (\u201CRAG\u201D) models combine the powers of pretrained dense retrieval (DPR) and Seq2Seq
models. RAG models retrieve docs, pass them to a seq2seq model, then marginalize to generate outputs. The retriever and
seq2seq modules are initialized from pretrained models, and fine-tuned jointly, allowing both retrieval and generation
to adapt to downstream tasks.`),md=h(),So=s("p"),u0=p("The two models RAG-Token and RAG-Sequence are available for generation."),cd=h(),We=s("h2"),Dt=s("a"),kp=s("span"),u(Hs.$$.fragment),d0=h(),xp=s("span"),v0=p("More technical aspects"),ud=h(),Je=s("h3"),Ht=s("a"),Ap=s("span"),u(Cs.$$.fragment),g0=h(),Tp=s("span"),b0=p("Full vs sparse attention"),dd=h(),Ro=s("p"),_0=p(`Most transformer models use full attention in the sense that the attention matrix is square. It can be a big
computational bottleneck when you have long texts. Longformer and reformer are models that try to be more efficient and
use a sparse version of the attention matrix to speed up training.`),vd=h(),Io=s("a"),gd=h(),qo=s("p"),$p=s("strong"),w0=p("LSH attention"),bd=h(),zs=s("p"),Go=s("a"),E0=p("Reformer"),y0=p(` uses LSH attention. In the softmax(QK^t), only the biggest elements (in the softmax
dimension) of the matrix QK^t are going to give useful contributions. So for each query q in Q, we can consider only
the keys k in K that are close to q. A hash function is used to determine if q and k are close. The attention mask is
modified to mask the current token (except at the first position), because it will give a query and a key equal (so
very similar to each other). Since the hash can be a bit random, several hash functions are used in practice
(determined by a n_rounds parameter) and then are averaged together.`),_d=h(),No=s("a"),wd=h(),Bo=s("p"),Mp=s("strong"),k0=p("Local attention"),Ed=h(),js=s("p"),Fo=s("a"),x0=p("Longformer"),A0=p(` uses local attention: often, the local context (e.g., what are the two tokens to the
left and right?) is enough to take action for a given token. Also, by stacking attention layers that have a small
window, the last layer will have a receptive field of more than just the tokens in the window, allowing them to build a
representation of the whole sentence.`),yd=h(),Do=s("p"),T0=p(`Some preselected input tokens are also given global attention: for those few tokens, the attention matrix can access
all tokens and this process is symmetric: all other tokens have access to those specific tokens (on top of the ones in
their local window). This is shown in Figure 2d of the paper, see below for a sample attention mask:`),kd=h(),Ct=s("img"),xd=h(),Ho=s("p"),$0=p(`Using those attention matrices with less parameters then allows the model to have inputs having a bigger sequence
length.`),Ad=h(),Qe=s("h3"),zt=s("a"),Pp=s("span"),u(Us.$$.fragment),M0=h(),Lp=s("span"),P0=p("Other tricks"),Td=h(),Co=s("a"),$d=h(),zo=s("p"),Sp=s("strong"),L0=p("Axial positional encodings"),Md=h(),E=s("p"),jo=s("a"),S0=p("Reformer"),R0=p(` uses axial positional encodings: in traditional transformer models, the positional encoding
E is a matrix of size `),Pd=new le,Ld=p(" by "),Sd=new le,Rd=p(", "),Id=new le,qd=p(" being the sequence length and "),Gd=new le,Nd=p(` the dimension of the
hidden state. If you have very long texts, this matrix can be huge and take way too much space on the GPU. To alleviate
that, axial positional encodings consist of factorizing that big matrix E in two smaller matrices E1 and E2, with
dimensions `),Bd=new le,Fd=p(" and "),Dd=new le,Hd=p(", such that "),Cd=new le,zd=p(` and
`),jd=new le,Ud=p(` (with the product for the lengths, this ends up being way smaller). The embedding for time
step `),Xd=new le,Vd=p(" in E is obtained by concatenating the embeddings for timestep "),Yd=new le,Zd=p(" in E1 and "),Kd=new le,Od=p(`
in E2.`),this.h()},l(e){const n=N9('[data-svelte="svelte-1phssyn"]',document.head);k=i(n,"META",{name:!0,content:!0}),n.forEach(t),P=f(e),x=i(e,"H1",{class:!0});var Xs=l(x);A=i(Xs,"A",{id:!0,class:!0,href:!0});var Rp=l(A);kh=i(Rp,"SPAN",{});var Fy=l(kh);d(Yt.$$.fragment,Fy),Fy.forEach(t),Rp.forEach(t),Cv=f(Xs),xh=i(Xs,"SPAN",{});var Dy=l(xh);zv=m(Dy,"Summary of the models"),Dy.forEach(t),Xs.forEach(t),Ip=f(e),L=i(e,"P",{});var jt=l(L);jv=m(jt,"This is a summary of the models available in \u{1F917} Transformers. It assumes you\u2019re familiar with the original "),Zt=i(jt,"A",{href:!0,rel:!0});var Hy=l(Zt);Uv=m(Hy,`transformer
model`),Hy.forEach(t),Xv=m(jt,". For a gentle introduction check the "),Kt=i(jt,"A",{href:!0,rel:!0});var Cy=l(Kt);Vv=m(Cy,"annotated transformer"),Cy.forEach(t),Yv=m(jt,`. Here we focus on the high-level differences between the
models. You can check them more in detail in their respective documentation. Also check out `),Ot=i(jt,"A",{href:!0,rel:!0});var zy=l(Ot);Zv=m(zy,"the Model Hub"),zy.forEach(t),Kv=m(jt," where you can filter the checkpoints by model architecture."),jt.forEach(t),qp=f(e),Ys=i(e,"P",{});var jy=l(Ys);Ov=m(jy,"Each one of the models in the library falls into one of the following categories:"),jy.forEach(t),Gp=f(e),T=i(e,"UL",{});var de=l(T);Ah=i(de,"LI",{});var Uy=l(Ah);Zs=i(Uy,"A",{href:!0});var Xy=l(Zs);Wv=m(Xy,"autoregressive-models"),Xy.forEach(t),Uy.forEach(t),Jv=f(de),Th=i(de,"LI",{});var Vy=l(Th);Ks=i(Vy,"A",{href:!0});var Yy=l(Ks);Qv=m(Yy,"autoencoding-models"),Yy.forEach(t),Vy.forEach(t),eg=f(de),$h=i(de,"LI",{});var Zy=l($h);Os=i(Zy,"A",{href:!0});var Ky=l(Os);tg=m(Ky,"seq-to-seq-models"),Ky.forEach(t),Zy.forEach(t),ag=f(de),Mh=i(de,"LI",{});var Oy=l(Mh);Ws=i(Oy,"A",{href:!0});var Wy=l(Ws);sg=m(Wy,"multimodal-models"),Wy.forEach(t),Oy.forEach(t),ig=f(de),Ph=i(de,"LI",{});var Jy=l(Ph);Js=i(Jy,"A",{href:!0});var Qy=l(Js);rg=m(Qy,"retrieval-based-models"),Qy.forEach(t),Jy.forEach(t),de.forEach(t),Np=f(e),M=i(e,"IFRAME",{width:!0,height:!0,src:!0,title:!0,frameborder:!0,allow:!0}),l(M).forEach(t),Bp=f(e),Qs=i(e,"P",{});var e3=l(Qs);lg=m(e3,`Autoregressive models are pretrained on the classic language modeling task: guess the next token having read all the
previous ones. They correspond to the decoder of the original transformer model, and a mask is used on top of the full
sentence so that the attention heads can only see what was before in the text, and not what\u2019s after. Although those
models can be fine-tuned and achieve great results on many tasks, the most natural application is text generation. A
typical example of such models is GPT.`),e3.forEach(t),Fp=f(e),ei=i(e,"P",{});var t3=l(ei);ng=m(t3,`Autoencoding models are pretrained by corrupting the input tokens in some way and trying to reconstruct the original
sentence. They correspond to the encoder of the original transformer model in the sense that they get access to the
full inputs without any mask. Those models usually build a bidirectional representation of the whole sentence. They can
be fine-tuned and achieve great results on many tasks such as text generation, but their most natural application is
sentence classification or token classification. A typical example of such models is BERT.`),t3.forEach(t),Dp=f(e),ti=i(e,"P",{});var a3=l(ti);og=m(a3,`Note that the only difference between autoregressive models and autoencoding models is in the way the model is
pretrained. Therefore, the same architecture can be used for both autoregressive and autoencoding models. When a given
model has been used for both types of pretraining, we have put it in the category corresponding to the article where it
was first introduced.`),a3.forEach(t),Hp=f(e),ai=i(e,"P",{});var s3=l(ai);hg=m(s3,`Sequence-to-sequence models use both the encoder and the decoder of the original transformer, either for translation
tasks or by transforming other tasks to sequence-to-sequence problems. They can be fine-tuned to many tasks but their
most natural applications are translation, summarization and question answering. The original transformer model is an
example of such a model (only for translation), T5 is an example that can be fine-tuned on other tasks.`),s3.forEach(t),Cp=f(e),si=i(e,"P",{});var i3=l(si);fg=m(i3,"Multimodal models mix text inputs with other kinds (e.g. images) and are more specific to a given task."),i3.forEach(t),zp=f(e),ii=i(e,"A",{id:!0}),l(ii).forEach(t),jp=f(e),ge=i(e,"H2",{class:!0});var Jd=l(ge);et=i(Jd,"A",{id:!0,class:!0,href:!0});var r3=l(et);Lh=i(r3,"SPAN",{});var l3=l(Lh);d(Wt.$$.fragment,l3),l3.forEach(t),r3.forEach(t),pg=f(Jd),Sh=i(Jd,"SPAN",{});var n3=l(Sh);mg=m(n3,"Decoders or autoregressive models"),n3.forEach(t),Jd.forEach(t),Up=f(e),ri=i(e,"P",{});var o3=l(ri);cg=m(o3,`As mentioned before, these models rely on the decoder part of the original transformer and use an attention mask so
that at each position, the model can only look at the tokens before the attention heads.`),o3.forEach(t),Xp=f(e),d(Jt.$$.fragment,e),Vp=f(e),be=i(e,"H3",{class:!0});var Qd=l(be);tt=i(Qd,"A",{id:!0,class:!0,href:!0});var h3=l(tt);Rh=i(h3,"SPAN",{});var f3=l(Rh);d(Qt.$$.fragment,f3),f3.forEach(t),h3.forEach(t),ug=f(Qd),Ih=i(Qd,"SPAN",{});var p3=l(Ih);dg=m(p3,"Original GPT"),p3.forEach(t),Qd.forEach(t),Yp=f(e),q=i(e,"DIV",{class:!0});var Uo=l(q);li=i(Uo,"A",{href:!0});var m3=l(li);ni=i(m3,"IMG",{alt:!0,src:!0}),m3.forEach(t),vg=f(Uo),oi=i(Uo,"A",{href:!0});var c3=l(oi);hi=i(c3,"IMG",{alt:!0,src:!0}),c3.forEach(t),gg=f(Uo),fi=i(Uo,"A",{href:!0});var u3=l(fi);pi=i(u3,"IMG",{alt:!0,src:!0}),u3.forEach(t),Uo.forEach(t),Zp=f(e),ea=i(e,"P",{});var I0=l(ea);ta=i(I0,"A",{href:!0,rel:!0});var d3=l(ta);bg=m(d3,"Improving Language Understanding by Generative Pre-Training"),d3.forEach(t),_g=m(I0,", Alec Radford et al."),I0.forEach(t),Kp=f(e),mi=i(e,"P",{});var v3=l(mi);wg=m(v3,"The first autoregressive model based on the transformer architecture, pretrained on the Book Corpus dataset."),v3.forEach(t),Op=f(e),ci=i(e,"P",{});var g3=l(ci);Eg=m(g3,`The library provides versions of the model for language modeling and multitask language modeling/multiple choice
classification.`),g3.forEach(t),Wp=f(e),_e=i(e,"H3",{class:!0});var ev=l(_e);at=i(ev,"A",{id:!0,class:!0,href:!0});var b3=l(at);qh=i(b3,"SPAN",{});var _3=l(qh);d(aa.$$.fragment,_3),_3.forEach(t),b3.forEach(t),yg=f(ev),Gh=i(ev,"SPAN",{});var w3=l(Gh);kg=m(w3,"GPT-2"),w3.forEach(t),ev.forEach(t),Jp=f(e),G=i(e,"DIV",{class:!0});var Xo=l(G);ui=i(Xo,"A",{href:!0});var E3=l(ui);di=i(E3,"IMG",{alt:!0,src:!0}),E3.forEach(t),xg=f(Xo),vi=i(Xo,"A",{href:!0});var y3=l(vi);gi=i(y3,"IMG",{alt:!0,src:!0}),y3.forEach(t),Ag=f(Xo),bi=i(Xo,"A",{href:!0});var k3=l(bi);_i=i(k3,"IMG",{alt:!0,src:!0}),k3.forEach(t),Xo.forEach(t),Qp=f(e),sa=i(e,"P",{});var q0=l(sa);ia=i(q0,"A",{href:!0,rel:!0});var x3=l(ia);Tg=m(x3,"Language Models are Unsupervised Multitask Learners"),x3.forEach(t),$g=m(q0,`,
Alec Radford et al.`),q0.forEach(t),em=f(e),wi=i(e,"P",{});var A3=l(wi);Mg=m(A3,`A bigger and better version of GPT, pretrained on WebText (web pages from outgoing links in Reddit with 3 karmas or
more).`),A3.forEach(t),tm=f(e),Ei=i(e,"P",{});var T3=l(Ei);Pg=m(T3,`The library provides versions of the model for language modeling and multitask language modeling/multiple choice
classification.`),T3.forEach(t),am=f(e),we=i(e,"H3",{class:!0});var tv=l(we);st=i(tv,"A",{id:!0,class:!0,href:!0});var $3=l(st);Nh=i($3,"SPAN",{});var M3=l(Nh);d(ra.$$.fragment,M3),M3.forEach(t),$3.forEach(t),Lg=f(tv),Bh=i(tv,"SPAN",{});var P3=l(Bh);Sg=m(P3,"CTRL"),P3.forEach(t),tv.forEach(t),sm=f(e),N=i(e,"DIV",{class:!0});var Vo=l(N);yi=i(Vo,"A",{href:!0});var L3=l(yi);ki=i(L3,"IMG",{alt:!0,src:!0}),L3.forEach(t),Rg=f(Vo),xi=i(Vo,"A",{href:!0});var S3=l(xi);Ai=i(S3,"IMG",{alt:!0,src:!0}),S3.forEach(t),Ig=f(Vo),Ti=i(Vo,"A",{href:!0});var R3=l(Ti);$i=i(R3,"IMG",{alt:!0,src:!0}),R3.forEach(t),Vo.forEach(t),im=f(e),la=i(e,"P",{});var G0=l(la);na=i(G0,"A",{href:!0,rel:!0});var I3=l(na);qg=m(I3,"CTRL: A Conditional Transformer Language Model for Controllable Generation"),I3.forEach(t),Gg=m(G0,`,
Nitish Shirish Keskar et al.`),G0.forEach(t),rm=f(e),Mi=i(e,"P",{});var q3=l(Mi);Ng=m(q3,`Same as the GPT model but adds the idea of control codes. Text is generated from a prompt (can be empty) and one (or
several) of those control codes which are then used to influence the text generation: generate with the style of
wikipedia article, a book or a movie review.`),q3.forEach(t),lm=f(e),Pi=i(e,"P",{});var G3=l(Pi);Bg=m(G3,"The library provides a version of the model for language modeling only."),G3.forEach(t),nm=f(e),Ee=i(e,"H3",{class:!0});var av=l(Ee);it=i(av,"A",{id:!0,class:!0,href:!0});var N3=l(it);Fh=i(N3,"SPAN",{});var B3=l(Fh);d(oa.$$.fragment,B3),B3.forEach(t),N3.forEach(t),Fg=f(av),Dh=i(av,"SPAN",{});var F3=l(Dh);Dg=m(F3,"Transformer-XL"),F3.forEach(t),av.forEach(t),om=f(e),B=i(e,"DIV",{class:!0});var Yo=l(B);Li=i(Yo,"A",{href:!0});var D3=l(Li);Si=i(D3,"IMG",{alt:!0,src:!0}),D3.forEach(t),Hg=f(Yo),Ri=i(Yo,"A",{href:!0});var H3=l(Ri);Ii=i(H3,"IMG",{alt:!0,src:!0}),H3.forEach(t),Cg=f(Yo),qi=i(Yo,"A",{href:!0});var C3=l(qi);Gi=i(C3,"IMG",{alt:!0,src:!0}),C3.forEach(t),Yo.forEach(t),hm=f(e),ha=i(e,"P",{});var N0=l(ha);fa=i(N0,"A",{href:!0,rel:!0});var z3=l(fa);zg=m(z3,"Transformer-XL: Attentive Language Models Beyond a Fixed-Length Context"),z3.forEach(t),jg=m(N0,`, Zihang
Dai et al.`),N0.forEach(t),fm=f(e),Ni=i(e,"P",{});var j3=l(Ni);Ug=m(j3,`Same as a regular GPT model, but introduces a recurrence mechanism for two consecutive segments (similar to a regular
RNNs with two consecutive inputs). In this context, a segment is a number of consecutive tokens (for instance 512) that
may span across multiple documents, and segments are fed in order to the model.`),j3.forEach(t),pm=f(e),Bi=i(e,"P",{});var U3=l(Bi);Xg=m(U3,`Basically, the hidden states of the previous segment are concatenated to the current input to compute the attention
scores. This allows the model to pay attention to information that was in the previous segment as well as the current
one. By stacking multiple attention layers, the receptive field can be increased to multiple previous segments.`),U3.forEach(t),mm=f(e),Fi=i(e,"P",{});var X3=l(Fi);Vg=m(X3,`This changes the positional embeddings to positional relative embeddings (as the regular positional embeddings would
give the same results in the current input and the current hidden state at a given position) and needs to make some
adjustments in the way attention scores are computed.`),X3.forEach(t),cm=f(e),Di=i(e,"P",{});var V3=l(Di);Yg=m(V3,"The library provides a version of the model for language modeling only."),V3.forEach(t),um=f(e),Hi=i(e,"A",{id:!0}),l(Hi).forEach(t),dm=f(e),ye=i(e,"H3",{class:!0});var sv=l(ye);rt=i(sv,"A",{id:!0,class:!0,href:!0});var Y3=l(rt);Hh=i(Y3,"SPAN",{});var Z3=l(Hh);d(pa.$$.fragment,Z3),Z3.forEach(t),Y3.forEach(t),Zg=f(sv),Ch=i(sv,"SPAN",{});var K3=l(Ch);Kg=m(K3,"Reformer"),K3.forEach(t),sv.forEach(t),vm=f(e),F=i(e,"DIV",{class:!0});var Zo=l(F);Ci=i(Zo,"A",{href:!0});var O3=l(Ci);zi=i(O3,"IMG",{alt:!0,src:!0}),O3.forEach(t),Og=f(Zo),ji=i(Zo,"A",{href:!0});var W3=l(ji);Ui=i(W3,"IMG",{alt:!0,src:!0}),W3.forEach(t),Wg=f(Zo),Xi=i(Zo,"A",{href:!0});var J3=l(Xi);Vi=i(J3,"IMG",{alt:!0,src:!0}),J3.forEach(t),Zo.forEach(t),gm=f(e),ma=i(e,"P",{});var B0=l(ma);ca=i(B0,"A",{href:!0,rel:!0});var Q3=l(ca);Jg=m(Q3,"Reformer: The Efficient Transformer"),Q3.forEach(t),Qg=m(B0,", Nikita Kitaev et al ."),B0.forEach(t),bm=f(e),Yi=i(e,"P",{});var e5=l(Yi);e1=m(e5,`An autoregressive transformer model with lots of tricks to reduce memory footprint and compute time. Those tricks
include:`),e5.forEach(t),_m=f(e),S=i(e,"UL",{});var Ut=l(S);ua=i(Ut,"LI",{});var iv=l(ua);t1=m(iv,"Use "),Zi=i(iv,"A",{href:!0});var t5=l(Zi);a1=m(t5,"Axial position encoding"),t5.forEach(t),s1=m(iv,` (see below for more details). It\u2019s a mechanism to avoid
having a huge positional encoding matrix (when the sequence length is very big) by factorizing it into smaller
matrices.`),iv.forEach(t),i1=f(Ut),da=i(Ut,"LI",{});var rv=l(da);r1=m(rv,"Replace traditional attention by "),Ki=i(rv,"A",{href:!0});var a5=l(Ki);l1=m(a5,"LSH (local-sensitive hashing) attention"),a5.forEach(t),n1=m(rv,` (see below for more
details). It\u2019s a technique to avoid computing the full product query-key in the attention layers.`),rv.forEach(t),o1=f(Ut),zh=i(Ut,"LI",{});var s5=l(zh);h1=m(s5,`Avoid storing the intermediate results of each layer by using reversible transformer layers to obtain them during
the backward pass (subtracting the residuals from the input of the next layer gives them back) or recomputing them
for results inside a given layer (less efficient than storing them but saves memory).`),s5.forEach(t),f1=f(Ut),jh=i(Ut,"LI",{});var i5=l(jh);p1=m(i5,"Compute the feedforward operations by chunks and not on the whole batch."),i5.forEach(t),Ut.forEach(t),wm=f(e),Oi=i(e,"P",{});var r5=l(Oi);m1=m(r5,"With those tricks, the model can be fed much larger sentences than traditional transformer autoregressive models."),r5.forEach(t),Em=f(e),d(lt.$$.fragment,e),ym=f(e),Wi=i(e,"P",{});var l5=l(Wi);c1=m(l5,"The library provides a version of the model for language modeling only."),l5.forEach(t),km=f(e),ke=i(e,"H3",{class:!0});var lv=l(ke);nt=i(lv,"A",{id:!0,class:!0,href:!0});var n5=l(nt);Uh=i(n5,"SPAN",{});var o5=l(Uh);d(va.$$.fragment,o5),o5.forEach(t),n5.forEach(t),u1=f(lv),Xh=i(lv,"SPAN",{});var h5=l(Xh);d1=m(h5,"XLNet"),h5.forEach(t),lv.forEach(t),xm=f(e),D=i(e,"DIV",{class:!0});var Ko=l(D);Ji=i(Ko,"A",{href:!0});var f5=l(Ji);Qi=i(f5,"IMG",{alt:!0,src:!0}),f5.forEach(t),v1=f(Ko),er=i(Ko,"A",{href:!0});var p5=l(er);tr=i(p5,"IMG",{alt:!0,src:!0}),p5.forEach(t),g1=f(Ko),ar=i(Ko,"A",{href:!0});var m5=l(ar);sr=i(m5,"IMG",{alt:!0,src:!0}),m5.forEach(t),Ko.forEach(t),Am=f(e),ga=i(e,"P",{});var F0=l(ga);ba=i(F0,"A",{href:!0,rel:!0});var c5=l(ba);b1=m(c5,"XLNet: Generalized Autoregressive Pretraining for Language Understanding"),c5.forEach(t),_1=m(F0,`, Zhilin
Yang et al.`),F0.forEach(t),Tm=f(e),ir=i(e,"P",{});var u5=l(ir);w1=m(u5,`XLNet is not a traditional autoregressive model but uses a training strategy that builds on that. It permutes the
tokens in the sentence, then allows the model to use the last n tokens to predict the token n+1. Since this is all done
with a mask, the sentence is actually fed in the model in the right order, but instead of masking the first n tokens
for n+1, XLNet uses a mask that hides the previous tokens in some given permutation of 1,\u2026,sequence length.`),u5.forEach(t),$m=f(e),rr=i(e,"P",{});var d5=l(rr);E1=m(d5,"XLNet also uses the same recurrence mechanism as Transformer-XL to build long-term dependencies."),d5.forEach(t),Mm=f(e),lr=i(e,"P",{});var v5=l(lr);y1=m(v5,`The library provides a version of the model for language modeling, token classification, sentence classification,
multiple choice classification and question answering.`),v5.forEach(t),Pm=f(e),nr=i(e,"A",{id:!0}),l(nr).forEach(t),Lm=f(e),xe=i(e,"H2",{class:!0});var nv=l(xe);ot=i(nv,"A",{id:!0,class:!0,href:!0});var g5=l(ot);Vh=i(g5,"SPAN",{});var b5=l(Vh);d(_a.$$.fragment,b5),b5.forEach(t),g5.forEach(t),k1=f(nv),Yh=i(nv,"SPAN",{});var _5=l(Yh);x1=m(_5,"Encoders or autoencoding models"),_5.forEach(t),nv.forEach(t),Sm=f(e),or=i(e,"P",{});var w5=l(or);A1=m(w5,`As mentioned before, these models rely on the encoder part of the original transformer and use no mask so the model can
look at all the tokens in the attention heads. For pretraining, targets are the original sentences and inputs are their
corrupted versions.`),w5.forEach(t),Rm=f(e),d(wa.$$.fragment,e),Im=f(e),Ae=i(e,"H3",{class:!0});var ov=l(Ae);ht=i(ov,"A",{id:!0,class:!0,href:!0});var E5=l(ht);Zh=i(E5,"SPAN",{});var y5=l(Zh);d(Ea.$$.fragment,y5),y5.forEach(t),E5.forEach(t),T1=f(ov),Kh=i(ov,"SPAN",{});var k5=l(Kh);$1=m(k5,"BERT"),k5.forEach(t),ov.forEach(t),qm=f(e),H=i(e,"DIV",{class:!0});var Oo=l(H);hr=i(Oo,"A",{href:!0});var x5=l(hr);fr=i(x5,"IMG",{alt:!0,src:!0}),x5.forEach(t),M1=f(Oo),pr=i(Oo,"A",{href:!0});var A5=l(pr);mr=i(A5,"IMG",{alt:!0,src:!0}),A5.forEach(t),P1=f(Oo),cr=i(Oo,"A",{href:!0});var T5=l(cr);ur=i(T5,"IMG",{alt:!0,src:!0}),T5.forEach(t),Oo.forEach(t),Gm=f(e),ya=i(e,"P",{});var D0=l(ya);ka=i(D0,"A",{href:!0,rel:!0});var $5=l(ka);L1=m($5,"BERT: Pre-training of Deep Bidirectional Transformers for Language Understanding"),$5.forEach(t),S1=m(D0,`,
Jacob Devlin et al.`),D0.forEach(t),Nm=f(e),dr=i(e,"P",{});var M5=l(dr);R1=m(M5,`Corrupts the inputs by using random masking, more precisely, during pretraining, a given percentage of tokens (usually
15%) is masked by:`),M5.forEach(t),Bm=f(e),oe=i(e,"UL",{});var Wo=l(oe);Oh=i(Wo,"LI",{});var P5=l(Oh);I1=m(P5,"a special mask token with probability 0.8"),P5.forEach(t),q1=f(Wo),Wh=i(Wo,"LI",{});var L5=l(Wh);G1=m(L5,"a random token different from the one masked with probability 0.1"),L5.forEach(t),N1=f(Wo),Jh=i(Wo,"LI",{});var S5=l(Jh);B1=m(S5,"the same token with probability 0.1"),S5.forEach(t),Wo.forEach(t),Fm=f(e),vr=i(e,"P",{});var R5=l(vr);F1=m(R5,`The model must predict the original sentence, but has a second objective: inputs are two sentences A and B (with a
separation token in between). With probability 50%, the sentences are consecutive in the corpus, in the remaining 50%
they are not related. The model has to predict if the sentences are consecutive or not.`),R5.forEach(t),Dm=f(e),gr=i(e,"P",{});var I5=l(gr);D1=m(I5,`The library provides a version of the model for language modeling (traditional or masked), next sentence prediction,
token classification, sentence classification, multiple choice classification and question answering.`),I5.forEach(t),Hm=f(e),Te=i(e,"H3",{class:!0});var hv=l(Te);ft=i(hv,"A",{id:!0,class:!0,href:!0});var q5=l(ft);Qh=i(q5,"SPAN",{});var G5=l(Qh);d(xa.$$.fragment,G5),G5.forEach(t),q5.forEach(t),H1=f(hv),ef=i(hv,"SPAN",{});var N5=l(ef);C1=m(N5,"ALBERT"),N5.forEach(t),hv.forEach(t),Cm=f(e),C=i(e,"DIV",{class:!0});var Jo=l(C);br=i(Jo,"A",{href:!0});var B5=l(br);_r=i(B5,"IMG",{alt:!0,src:!0}),B5.forEach(t),z1=f(Jo),wr=i(Jo,"A",{href:!0});var F5=l(wr);Er=i(F5,"IMG",{alt:!0,src:!0}),F5.forEach(t),j1=f(Jo),yr=i(Jo,"A",{href:!0});var D5=l(yr);kr=i(D5,"IMG",{alt:!0,src:!0}),D5.forEach(t),Jo.forEach(t),zm=f(e),Aa=i(e,"P",{});var H0=l(Aa);Ta=i(H0,"A",{href:!0,rel:!0});var H5=l(Ta);U1=m(H5,"ALBERT: A Lite BERT for Self-supervised Learning of Language Representations"),H5.forEach(t),X1=m(H0,`,
Zhenzhong Lan et al.`),H0.forEach(t),jm=f(e),xr=i(e,"P",{});var C5=l(xr);V1=m(C5,"Same as BERT but with a few tweaks:"),C5.forEach(t),Um=f(e),he=i(e,"UL",{});var Qo=l(he);tf=i(Qo,"LI",{});var z5=l(tf);Y1=m(z5,`Embedding size E is different from hidden size H justified because the embeddings are context independent (one
embedding vector represents one token), whereas hidden states are context dependent (one hidden state represents a
sequence of tokens) so it\u2019s more logical to have H >> E. Also, the embedding matrix is large since it\u2019s V x E (V
being the vocab size). If E < H, it has less parameters.`),z5.forEach(t),Z1=f(Qo),af=i(Qo,"LI",{});var j5=l(af);K1=m(j5,"Layers are split in groups that share parameters (to save memory)."),j5.forEach(t),O1=f(Qo),sf=i(Qo,"LI",{});var U5=l(sf);W1=m(U5,`Next sentence prediction is replaced by a sentence ordering prediction: in the inputs, we have two sentences A and
B (that are consecutive) and we either feed A followed by B or B followed by A. The model must predict if they have
been swapped or not.`),U5.forEach(t),Qo.forEach(t),Xm=f(e),Ar=i(e,"P",{});var X5=l(Ar);J1=m(X5,`The library provides a version of the model for masked language modeling, token classification, sentence
classification, multiple choice classification and question answering.`),X5.forEach(t),Vm=f(e),$e=i(e,"H3",{class:!0});var fv=l($e);pt=i(fv,"A",{id:!0,class:!0,href:!0});var V5=l(pt);rf=i(V5,"SPAN",{});var Y5=l(rf);d($a.$$.fragment,Y5),Y5.forEach(t),V5.forEach(t),Q1=f(fv),lf=i(fv,"SPAN",{});var Z5=l(lf);eb=m(Z5,"RoBERTa"),Z5.forEach(t),fv.forEach(t),Ym=f(e),z=i(e,"DIV",{class:!0});var eh=l(z);Tr=i(eh,"A",{href:!0});var K5=l(Tr);$r=i(K5,"IMG",{alt:!0,src:!0}),K5.forEach(t),tb=f(eh),Mr=i(eh,"A",{href:!0});var O5=l(Mr);Pr=i(O5,"IMG",{alt:!0,src:!0}),O5.forEach(t),ab=f(eh),Lr=i(eh,"A",{href:!0});var W5=l(Lr);Sr=i(W5,"IMG",{alt:!0,src:!0}),W5.forEach(t),eh.forEach(t),Zm=f(e),Ma=i(e,"P",{});var C0=l(Ma);Pa=i(C0,"A",{href:!0,rel:!0});var J5=l(Pa);sb=m(J5,"RoBERTa: A Robustly Optimized BERT Pretraining Approach"),J5.forEach(t),ib=m(C0,", Yinhan Liu et al."),C0.forEach(t),Km=f(e),Rr=i(e,"P",{});var Q5=l(Rr);rb=m(Q5,"Same as BERT with better pretraining tricks:"),Q5.forEach(t),Om=f(e),R=i(e,"UL",{});var Xt=l(R);nf=i(Xt,"LI",{});var ek=l(nf);lb=m(ek,"dynamic masking: tokens are masked differently at each epoch, whereas BERT does it once and for all"),ek.forEach(t),nb=f(Xt),of=i(Xt,"LI",{});var tk=l(of);ob=m(tk,`no NSP (next sentence prediction) loss and instead of putting just two sentences together, put a chunk of
contiguous texts together to reach 512 tokens (so the sentences are in an order than may span several documents)`),tk.forEach(t),hb=f(Xt),hf=i(Xt,"LI",{});var ak=l(hf);fb=m(ak,"train with larger batches"),ak.forEach(t),pb=f(Xt),ff=i(Xt,"LI",{});var sk=l(ff);mb=m(sk,"use BPE with bytes as a subunit and not characters (because of unicode characters)"),sk.forEach(t),Xt.forEach(t),Wm=f(e),Ir=i(e,"P",{});var ik=l(Ir);cb=m(ik,`The library provides a version of the model for masked language modeling, token classification, sentence
classification, multiple choice classification and question answering.`),ik.forEach(t),Jm=f(e),Me=i(e,"H3",{class:!0});var pv=l(Me);mt=i(pv,"A",{id:!0,class:!0,href:!0});var rk=l(mt);pf=i(rk,"SPAN",{});var lk=l(pf);d(La.$$.fragment,lk),lk.forEach(t),rk.forEach(t),ub=f(pv),mf=i(pv,"SPAN",{});var nk=l(mf);db=m(nk,"DistilBERT"),nk.forEach(t),pv.forEach(t),Qm=f(e),j=i(e,"DIV",{class:!0});var th=l(j);qr=i(th,"A",{href:!0});var ok=l(qr);Gr=i(ok,"IMG",{alt:!0,src:!0}),ok.forEach(t),vb=f(th),Nr=i(th,"A",{href:!0});var hk=l(Nr);Br=i(hk,"IMG",{alt:!0,src:!0}),hk.forEach(t),gb=f(th),Fr=i(th,"A",{href:!0});var fk=l(Fr);Dr=i(fk,"IMG",{alt:!0,src:!0}),fk.forEach(t),th.forEach(t),ec=f(e),Sa=i(e,"P",{});var z0=l(Sa);Ra=i(z0,"A",{href:!0,rel:!0});var pk=l(Ra);bb=m(pk,"DistilBERT, a distilled version of BERT: smaller, faster, cheaper and lighter"),pk.forEach(t),_b=m(z0,`,
Victor Sanh et al.`),z0.forEach(t),tc=f(e),Hr=i(e,"P",{});var mk=l(Hr);wb=m(mk,`Same as BERT but smaller. Trained by distillation of the pretrained BERT model, meaning it\u2019s been trained to predict
the same probabilities as the larger model. The actual objective is a combination of:`),mk.forEach(t),ac=f(e),fe=i(e,"UL",{});var ah=l(fe);cf=i(ah,"LI",{});var ck=l(cf);Eb=m(ck,"finding the same probabilities as the teacher model"),ck.forEach(t),yb=f(ah),uf=i(ah,"LI",{});var uk=l(uf);kb=m(uk,"predicting the masked tokens correctly (but no next-sentence objective)"),uk.forEach(t),xb=f(ah),df=i(ah,"LI",{});var dk=l(df);Ab=m(dk,"a cosine similarity between the hidden states of the student and the teacher model"),dk.forEach(t),ah.forEach(t),sc=f(e),Cr=i(e,"P",{});var vk=l(Cr);Tb=m(vk,`The library provides a version of the model for masked language modeling, token classification, sentence classification
and question answering.`),vk.forEach(t),ic=f(e),Pe=i(e,"H3",{class:!0});var mv=l(Pe);ct=i(mv,"A",{id:!0,class:!0,href:!0});var gk=l(ct);vf=i(gk,"SPAN",{});var bk=l(vf);d(Ia.$$.fragment,bk),bk.forEach(t),gk.forEach(t),$b=f(mv),gf=i(mv,"SPAN",{});var _k=l(gf);Mb=m(_k,"ConvBERT"),_k.forEach(t),mv.forEach(t),rc=f(e),U=i(e,"DIV",{class:!0});var sh=l(U);zr=i(sh,"A",{href:!0});var wk=l(zr);jr=i(wk,"IMG",{alt:!0,src:!0}),wk.forEach(t),Pb=f(sh),Ur=i(sh,"A",{href:!0});var Ek=l(Ur);Xr=i(Ek,"IMG",{alt:!0,src:!0}),Ek.forEach(t),Lb=f(sh),Vr=i(sh,"A",{href:!0});var yk=l(Vr);Yr=i(yk,"IMG",{alt:!0,src:!0}),yk.forEach(t),sh.forEach(t),lc=f(e),qa=i(e,"P",{});var j0=l(qa);Ga=i(j0,"A",{href:!0,rel:!0});var kk=l(Ga);Sb=m(kk,"ConvBERT: Improving BERT with Span-based Dynamic Convolution"),kk.forEach(t),Rb=m(j0,`, Zihang Jiang,
Weihao Yu, Daquan Zhou, Yunpeng Chen, Jiashi Feng, Shuicheng Yan.`),j0.forEach(t),nc=f(e),Zr=i(e,"P",{});var xk=l(Zr);Ib=m(xk,`Pre-trained language models like BERT and its variants have recently achieved impressive performance in various natural
language understanding tasks. However, BERT heavily relies on the global self-attention block and thus suffers large
memory footprint and computation cost. Although all its attention heads query on the whole input sequence for
generating the attention map from a global perspective, we observe some heads only need to learn local dependencies,
which means the existence of computation redundancy. We therefore propose a novel span-based dynamic convolution to
replace these self-attention heads to directly model local dependencies. The novel convolution heads, together with the
rest self-attention heads, form a new mixed attention block that is more efficient at both global and local context
learning. We equip BERT with this mixed attention design and build a ConvBERT model. Experiments have shown that
ConvBERT significantly outperforms BERT and its variants in various downstream tasks, with lower training cost and
fewer model parameters. Remarkably, ConvBERTbase model achieves 86.4 GLUE score, 0.7 higher than ELECTRAbase, while
using less than 1/4 training cost.`),xk.forEach(t),oc=f(e),Kr=i(e,"P",{});var Ak=l(Kr);qb=m(Ak,`The library provides a version of the model for masked language modeling, token classification, sentence classification
and question answering.`),Ak.forEach(t),hc=f(e),Le=i(e,"H3",{class:!0});var cv=l(Le);ut=i(cv,"A",{id:!0,class:!0,href:!0});var Tk=l(ut);bf=i(Tk,"SPAN",{});var $k=l(bf);d(Na.$$.fragment,$k),$k.forEach(t),Tk.forEach(t),Gb=f(cv),_f=i(cv,"SPAN",{});var Mk=l(_f);Nb=m(Mk,"XLM"),Mk.forEach(t),cv.forEach(t),fc=f(e),X=i(e,"DIV",{class:!0});var ih=l(X);Or=i(ih,"A",{href:!0});var Pk=l(Or);Wr=i(Pk,"IMG",{alt:!0,src:!0}),Pk.forEach(t),Bb=f(ih),Jr=i(ih,"A",{href:!0});var Lk=l(Jr);Qr=i(Lk,"IMG",{alt:!0,src:!0}),Lk.forEach(t),Fb=f(ih),el=i(ih,"A",{href:!0});var Sk=l(el);tl=i(Sk,"IMG",{alt:!0,src:!0}),Sk.forEach(t),ih.forEach(t),pc=f(e),Ba=i(e,"P",{});var U0=l(Ba);Fa=i(U0,"A",{href:!0,rel:!0});var Rk=l(Fa);Db=m(Rk,"Cross-lingual Language Model Pretraining"),Rk.forEach(t),Hb=m(U0,", Guillaume Lample and Alexis Conneau"),U0.forEach(t),mc=f(e),al=i(e,"P",{});var Ik=l(al);Cb=m(Ik,`A transformer model trained on several languages. There are three different type of training for this model and the
library provides checkpoints for all of them:`),Ik.forEach(t),cc=f(e),pe=i(e,"UL",{});var rh=l(pe);wf=i(rh,"LI",{});var qk=l(wf);zb=m(qk,`Causal language modeling (CLM) which is the traditional autoregressive training (so this model could be in the
previous section as well). One of the languages is selected for each training sample, and the model input is a
sentence of 256 tokens, that may span over several documents in one of those languages.`),qk.forEach(t),jb=f(rh),Ef=i(rh,"LI",{});var Gk=l(Ef);Ub=m(Gk,`Masked language modeling (MLM) which is like RoBERTa. One of the languages is selected for each training sample,
and the model input is a sentence of 256 tokens, that may span over several documents in one of those languages,
with dynamic masking of the tokens.`),Gk.forEach(t),Xb=f(rh),yf=i(rh,"LI",{});var Nk=l(yf);Vb=m(Nk,`A combination of MLM and translation language modeling (TLM). This consists of concatenating a sentence in two
different languages, with random masking. To predict one of the masked tokens, the model can use both, the
surrounding context in language 1 and the context given by language 2.`),Nk.forEach(t),rh.forEach(t),uc=f(e),I=i(e,"P",{});var Vt=l(I);Yb=m(Vt,"Checkpoints refer to which method was used for pretraining by having "),kf=i(Vt,"EM",{});var Bk=l(kf);Zb=m(Bk,"clm"),Bk.forEach(t),Kb=m(Vt,", "),xf=i(Vt,"EM",{});var Fk=l(xf);Ob=m(Fk,"mlm"),Fk.forEach(t),Wb=m(Vt," or "),Af=i(Vt,"EM",{});var Dk=l(Af);Jb=m(Dk,"mlm-tlm"),Dk.forEach(t),Qb=m(Vt,` in their names. On top
of positional embeddings, the model has language embeddings. When training using MLM/CLM, this gives the model an
indication of the language used, and when training using MLM+TLM, an indication of the language used for each part.`),Vt.forEach(t),dc=f(e),sl=i(e,"P",{});var Hk=l(sl);e_=m(Hk,`The library provides a version of the model for language modeling, token classification, sentence classification and
question answering.`),Hk.forEach(t),vc=f(e),Se=i(e,"H3",{class:!0});var uv=l(Se);dt=i(uv,"A",{id:!0,class:!0,href:!0});var Ck=l(dt);Tf=i(Ck,"SPAN",{});var zk=l(Tf);d(Da.$$.fragment,zk),zk.forEach(t),Ck.forEach(t),t_=f(uv),$f=i(uv,"SPAN",{});var jk=l($f);a_=m(jk,"XLM-RoBERTa"),jk.forEach(t),uv.forEach(t),gc=f(e),V=i(e,"DIV",{class:!0});var lh=l(V);il=i(lh,"A",{href:!0});var Uk=l(il);rl=i(Uk,"IMG",{alt:!0,src:!0}),Uk.forEach(t),s_=f(lh),ll=i(lh,"A",{href:!0});var Xk=l(ll);nl=i(Xk,"IMG",{alt:!0,src:!0}),Xk.forEach(t),i_=f(lh),ol=i(lh,"A",{href:!0});var Vk=l(ol);hl=i(Vk,"IMG",{alt:!0,src:!0}),Vk.forEach(t),lh.forEach(t),bc=f(e),Ha=i(e,"P",{});var X0=l(Ha);Ca=i(X0,"A",{href:!0,rel:!0});var Yk=l(Ca);r_=m(Yk,"Unsupervised Cross-lingual Representation Learning at Scale"),Yk.forEach(t),l_=m(X0,`, Alexis Conneau et
al.`),X0.forEach(t),_c=f(e),fl=i(e,"P",{});var Zk=l(fl);n_=m(Zk,`Uses RoBERTa tricks on the XLM approach, but does not use the translation language modeling objective. It only uses
masked language modeling on sentences coming from one language. However, the model is trained on many more languages
(100) and doesn\u2019t use the language embeddings, so it\u2019s capable of detecting the input language by itself.`),Zk.forEach(t),wc=f(e),pl=i(e,"P",{});var Kk=l(pl);o_=m(Kk,`The library provides a version of the model for masked language modeling, token classification, sentence
classification, multiple choice classification and question answering.`),Kk.forEach(t),Ec=f(e),Re=i(e,"H3",{class:!0});var dv=l(Re);vt=i(dv,"A",{id:!0,class:!0,href:!0});var Ok=l(vt);Mf=i(Ok,"SPAN",{});var Wk=l(Mf);d(za.$$.fragment,Wk),Wk.forEach(t),Ok.forEach(t),h_=f(dv),Pf=i(dv,"SPAN",{});var Jk=l(Pf);f_=m(Jk,"FlauBERT"),Jk.forEach(t),dv.forEach(t),yc=f(e),Y=i(e,"DIV",{class:!0});var nh=l(Y);ml=i(nh,"A",{href:!0});var Qk=l(ml);cl=i(Qk,"IMG",{alt:!0,src:!0}),Qk.forEach(t),p_=f(nh),ul=i(nh,"A",{href:!0});var e4=l(ul);dl=i(e4,"IMG",{alt:!0,src:!0}),e4.forEach(t),m_=f(nh),vl=i(nh,"A",{href:!0});var t4=l(vl);gl=i(t4,"IMG",{alt:!0,src:!0}),t4.forEach(t),nh.forEach(t),kc=f(e),ja=i(e,"P",{});var V0=l(ja);Ua=i(V0,"A",{href:!0,rel:!0});var a4=l(Ua);c_=m(a4,"FlauBERT: Unsupervised Language Model Pre-training for French"),a4.forEach(t),u_=m(V0,", Hang Le et al."),V0.forEach(t),xc=f(e),bl=i(e,"P",{});var s4=l(bl);d_=m(s4,"Like RoBERTa, without the sentence ordering prediction (so just trained on the MLM objective)."),s4.forEach(t),Ac=f(e),_l=i(e,"P",{});var i4=l(_l);v_=m(i4,"The library provides a version of the model for language modeling and sentence classification."),i4.forEach(t),Tc=f(e),Ie=i(e,"H3",{class:!0});var vv=l(Ie);gt=i(vv,"A",{id:!0,class:!0,href:!0});var r4=l(gt);Lf=i(r4,"SPAN",{});var l4=l(Lf);d(Xa.$$.fragment,l4),l4.forEach(t),r4.forEach(t),g_=f(vv),Sf=i(vv,"SPAN",{});var n4=l(Sf);b_=m(n4,"ELECTRA"),n4.forEach(t),vv.forEach(t),$c=f(e),Z=i(e,"DIV",{class:!0});var oh=l(Z);wl=i(oh,"A",{href:!0});var o4=l(wl);El=i(o4,"IMG",{alt:!0,src:!0}),o4.forEach(t),__=f(oh),yl=i(oh,"A",{href:!0});var h4=l(yl);kl=i(h4,"IMG",{alt:!0,src:!0}),h4.forEach(t),w_=f(oh),xl=i(oh,"A",{href:!0});var f4=l(xl);Al=i(f4,"IMG",{alt:!0,src:!0}),f4.forEach(t),oh.forEach(t),Mc=f(e),Va=i(e,"P",{});var Y0=l(Va);Ya=i(Y0,"A",{href:!0,rel:!0});var p4=l(Ya);E_=m(p4,"ELECTRA: Pre-training Text Encoders as Discriminators Rather Than Generators"),p4.forEach(t),y_=m(Y0,`,
Kevin Clark et al.`),Y0.forEach(t),Pc=f(e),Tl=i(e,"P",{});var m4=l(Tl);k_=m(m4,`ELECTRA is a transformer model pretrained with the use of another (small) masked language model. The inputs are
corrupted by that language model, which takes an input text that is randomly masked and outputs a text in which ELECTRA
has to predict which token is an original and which one has been replaced. Like for GAN training, the small language
model is trained for a few steps (but with the original texts as objective, not to fool the ELECTRA model like in a
traditional GAN setting) then the ELECTRA model is trained for a few steps.`),m4.forEach(t),Lc=f(e),$l=i(e,"P",{});var c4=l($l);x_=m(c4,`The library provides a version of the model for masked language modeling, token classification and sentence
classification.`),c4.forEach(t),Sc=f(e),qe=i(e,"H3",{class:!0});var gv=l(qe);bt=i(gv,"A",{id:!0,class:!0,href:!0});var u4=l(bt);Rf=i(u4,"SPAN",{});var d4=l(Rf);d(Za.$$.fragment,d4),d4.forEach(t),u4.forEach(t),A_=f(gv),If=i(gv,"SPAN",{});var v4=l(If);T_=m(v4,"Funnel Transformer"),v4.forEach(t),gv.forEach(t),Rc=f(e),K=i(e,"DIV",{class:!0});var hh=l(K);Ml=i(hh,"A",{href:!0});var g4=l(Ml);Pl=i(g4,"IMG",{alt:!0,src:!0}),g4.forEach(t),$_=f(hh),Ll=i(hh,"A",{href:!0});var b4=l(Ll);Sl=i(b4,"IMG",{alt:!0,src:!0}),b4.forEach(t),M_=f(hh),Rl=i(hh,"A",{href:!0});var _4=l(Rl);Il=i(_4,"IMG",{alt:!0,src:!0}),_4.forEach(t),hh.forEach(t),Ic=f(e),Ka=i(e,"P",{});var Z0=l(Ka);Oa=i(Z0,"A",{href:!0,rel:!0});var w4=l(Oa);P_=m(w4,"Funnel-Transformer: Filtering out Sequential Redundancy for Efficient Language Processing"),w4.forEach(t),L_=m(Z0,", Zihang Dai et al."),Z0.forEach(t),qc=f(e),ql=i(e,"P",{});var E4=l(ql);S_=m(E4,`Funnel Transformer is a transformer model using pooling, a bit like a ResNet model: layers are grouped in blocks, and
at the beginning of each block (except the first one), the hidden states are pooled among the sequence dimension. This
way, their length is divided by 2, which speeds up the computation of the next hidden states. All pretrained models
have three blocks, which means the final hidden state has a sequence length that is one fourth of the original sequence
length.`),E4.forEach(t),Gc=f(e),Gl=i(e,"P",{});var y4=l(Gl);R_=m(y4,`For tasks such as classification, this is not a problem, but for tasks like masked language modeling or token
classification, we need a hidden state with the same sequence length as the original input. In those cases, the final
hidden states are upsampled to the input sequence length and go through two additional layers. That\u2019s why there are two
versions of each checkpoint. The version suffixed with \u201C-base\u201D contains only the three blocks, while the version
without that suffix contains the three blocks and the upsampling head with its additional layers.`),y4.forEach(t),Nc=f(e),Nl=i(e,"P",{});var k4=l(Nl);I_=m(k4,"The pretrained models available use the same pretraining objective as ELECTRA."),k4.forEach(t),Bc=f(e),Bl=i(e,"P",{});var x4=l(Bl);q_=m(x4,`The library provides a version of the model for masked language modeling, token classification, sentence
classification, multiple choice classification and question answering.`),x4.forEach(t),Fc=f(e),Fl=i(e,"A",{id:!0}),l(Fl).forEach(t),Dc=f(e),Ge=i(e,"H3",{class:!0});var bv=l(Ge);_t=i(bv,"A",{id:!0,class:!0,href:!0});var A4=l(_t);qf=i(A4,"SPAN",{});var T4=l(qf);d(Wa.$$.fragment,T4),T4.forEach(t),A4.forEach(t),G_=f(bv),Gf=i(bv,"SPAN",{});var $4=l(Gf);N_=m($4,"Longformer"),$4.forEach(t),bv.forEach(t),Hc=f(e),O=i(e,"DIV",{class:!0});var fh=l(O);Dl=i(fh,"A",{href:!0});var M4=l(Dl);Hl=i(M4,"IMG",{alt:!0,src:!0}),M4.forEach(t),B_=f(fh),Cl=i(fh,"A",{href:!0});var P4=l(Cl);zl=i(P4,"IMG",{alt:!0,src:!0}),P4.forEach(t),F_=f(fh),jl=i(fh,"A",{href:!0});var L4=l(jl);Ul=i(L4,"IMG",{alt:!0,src:!0}),L4.forEach(t),fh.forEach(t),Cc=f(e),Ja=i(e,"P",{});var K0=l(Ja);Qa=i(K0,"A",{href:!0,rel:!0});var S4=l(Qa);D_=m(S4,"Longformer: The Long-Document Transformer"),S4.forEach(t),H_=m(K0,", Iz Beltagy et al."),K0.forEach(t),zc=f(e),wt=i(e,"P",{});var _v=l(wt);C_=m(_v,`A transformer model replacing the attention matrices by sparse matrices to go faster. Often, the local context (e.g.,
what are the two tokens left and right?) is enough to take action for a given token. Some preselected input tokens are
still given global attention, but the attention matrix has way less parameters, resulting in a speed-up. See the
`),Xl=i(_v,"A",{href:!0});var R4=l(Xl);z_=m(R4,"local attention section"),R4.forEach(t),j_=m(_v," for more information."),_v.forEach(t),jc=f(e),Vl=i(e,"P",{});var I4=l(Vl);U_=m(I4,"It is pretrained the same way a RoBERTa otherwise."),I4.forEach(t),Uc=f(e),d(Et.$$.fragment,e),Xc=f(e),Yl=i(e,"P",{});var q4=l(Yl);X_=m(q4,`The library provides a version of the model for masked language modeling, token classification, sentence
classification, multiple choice classification and question answering.`),q4.forEach(t),Vc=f(e),Zl=i(e,"A",{id:!0}),l(Zl).forEach(t),Yc=f(e),Ne=i(e,"H2",{class:!0});var wv=l(Ne);yt=i(wv,"A",{id:!0,class:!0,href:!0});var G4=l(yt);Nf=i(G4,"SPAN",{});var N4=l(Nf);d(es.$$.fragment,N4),N4.forEach(t),G4.forEach(t),V_=f(wv),Bf=i(wv,"SPAN",{});var B4=l(Bf);Y_=m(B4,"Sequence-to-sequence models"),B4.forEach(t),wv.forEach(t),Zc=f(e),Kl=i(e,"P",{});var F4=l(Kl);Z_=m(F4,"As mentioned before, these models keep both the encoder and the decoder of the original transformer."),F4.forEach(t),Kc=f(e),d(ts.$$.fragment,e),Oc=f(e),Be=i(e,"H3",{class:!0});var Ev=l(Be);kt=i(Ev,"A",{id:!0,class:!0,href:!0});var D4=l(kt);Ff=i(D4,"SPAN",{});var H4=l(Ff);d(as.$$.fragment,H4),H4.forEach(t),D4.forEach(t),K_=f(Ev),Df=i(Ev,"SPAN",{});var C4=l(Df);O_=m(C4,"BART"),C4.forEach(t),Ev.forEach(t),Wc=f(e),W=i(e,"DIV",{class:!0});var ph=l(W);Ol=i(ph,"A",{href:!0});var z4=l(Ol);Wl=i(z4,"IMG",{alt:!0,src:!0}),z4.forEach(t),W_=f(ph),Jl=i(ph,"A",{href:!0});var j4=l(Jl);Ql=i(j4,"IMG",{alt:!0,src:!0}),j4.forEach(t),J_=f(ph),en=i(ph,"A",{href:!0});var U4=l(en);tn=i(U4,"IMG",{alt:!0,src:!0}),U4.forEach(t),ph.forEach(t),Jc=f(e),ss=i(e,"P",{});var O0=l(ss);is=i(O0,"A",{href:!0,rel:!0});var X4=l(is);Q_=m(X4,"BART: Denoising Sequence-to-Sequence Pre-training for Natural Language Generation, Translation, and Comprehension"),X4.forEach(t),ew=m(O0,", Mike Lewis et al."),O0.forEach(t),Qc=f(e),an=i(e,"P",{});var V4=l(an);tw=m(V4,`Sequence-to-sequence model with an encoder and a decoder. Encoder is fed a corrupted version of the tokens, decoder is
fed the original tokens (but has a mask to hide the future words like a regular transformers decoder). A composition of
the following transformations are applied on the pretraining tasks for the encoder:`),V4.forEach(t),eu=f(e),$=i(e,"UL",{});var ve=l($);Hf=i(ve,"LI",{});var Y4=l(Hf);aw=m(Y4,"mask random tokens (like in BERT)"),Y4.forEach(t),sw=f(ve),Cf=i(ve,"LI",{});var Z4=l(Cf);iw=m(Z4,"delete random tokens"),Z4.forEach(t),rw=f(ve),zf=i(ve,"LI",{});var K4=l(zf);lw=m(K4,"mask a span of k tokens with a single mask token (a span of 0 tokens is an insertion of a mask token)"),K4.forEach(t),nw=f(ve),jf=i(ve,"LI",{});var O4=l(jf);ow=m(O4,"permute sentences"),O4.forEach(t),hw=f(ve),Uf=i(ve,"LI",{});var W4=l(Uf);fw=m(W4,"rotate the document to make it start at a specific token"),W4.forEach(t),ve.forEach(t),tu=f(e),sn=i(e,"P",{});var J4=l(sn);pw=m(J4,"The library provides a version of this model for conditional generation and sequence classification."),J4.forEach(t),au=f(e),Fe=i(e,"H3",{class:!0});var yv=l(Fe);xt=i(yv,"A",{id:!0,class:!0,href:!0});var Q4=l(xt);Xf=i(Q4,"SPAN",{});var ex=l(Xf);d(rs.$$.fragment,ex),ex.forEach(t),Q4.forEach(t),mw=f(yv),Vf=i(yv,"SPAN",{});var tx=l(Vf);cw=m(tx,"Pegasus"),tx.forEach(t),yv.forEach(t),su=f(e),J=i(e,"DIV",{class:!0});var mh=l(J);rn=i(mh,"A",{href:!0});var ax=l(rn);ln=i(ax,"IMG",{alt:!0,src:!0}),ax.forEach(t),uw=f(mh),nn=i(mh,"A",{href:!0});var sx=l(nn);on=i(sx,"IMG",{alt:!0,src:!0}),sx.forEach(t),dw=f(mh),hn=i(mh,"A",{href:!0});var ix=l(hn);fn=i(ix,"IMG",{alt:!0,src:!0}),ix.forEach(t),mh.forEach(t),iu=f(e),ls=i(e,"P",{});var W0=l(ls);ns=i(W0,"A",{href:!0,rel:!0});var rx=l(ns);vw=m(rx,"PEGASUS: Pre-training with Extracted Gap-sentences forAbstractive Summarization"),rx.forEach(t),gw=m(W0,", Jingqing Zhang, Yao Zhao, Mohammad Saleh and Peter J. Liu on Dec 18, 2019."),W0.forEach(t),ru=f(e),pn=i(e,"P",{});var lx=l(pn);bw=m(lx,`Sequence-to-sequence model with the same encoder-decoder model architecture as BART. Pegasus is pre-trained jointly on
two self-supervised objective functions: Masked Language Modeling (MLM) and a novel summarization specific pretraining
objective, called Gap Sentence Generation (GSG).`),lx.forEach(t),lu=f(e),At=i(e,"UL",{});var kv=l(At);Yf=i(kv,"LI",{});var nx=l(Yf);_w=m(nx,`MLM: encoder input tokens are randomly replaced by a mask tokens and have to be predicted by the encoder (like in
BERT)`),nx.forEach(t),ww=f(kv),Zf=i(kv,"LI",{});var ox=l(Zf);Ew=m(ox,`GSG: whole encoder input sentences are replaced by a second mask token and fed to the decoder, but which has a
causal mask to hide the future words like a regular auto-regressive transformer decoder.`),ox.forEach(t),kv.forEach(t),nu=f(e),mn=i(e,"P",{});var hx=l(mn);yw=m(hx,`In contrast to BART, Pegasus\u2019 pretraining task is intentionally similar to summarization: important sentences are
masked and are generated together as one output sequence from the remaining sentences, similar to an extractive
summary.`),hx.forEach(t),ou=f(e),cn=i(e,"P",{});var fx=l(cn);kw=m(fx,"The library provides a version of this model for conditional generation, which should be used for summarization."),fx.forEach(t),hu=f(e),De=i(e,"H3",{class:!0});var xv=l(De);Tt=i(xv,"A",{id:!0,class:!0,href:!0});var px=l(Tt);Kf=i(px,"SPAN",{});var mx=l(Kf);d(os.$$.fragment,mx),mx.forEach(t),px.forEach(t),xw=f(xv),Of=i(xv,"SPAN",{});var cx=l(Of);Aw=m(cx,"MarianMT"),cx.forEach(t),xv.forEach(t),fu=f(e),Q=i(e,"DIV",{class:!0});var ch=l(Q);un=i(ch,"A",{href:!0});var ux=l(un);dn=i(ux,"IMG",{alt:!0,src:!0}),ux.forEach(t),Tw=f(ch),vn=i(ch,"A",{href:!0});var dx=l(vn);gn=i(dx,"IMG",{alt:!0,src:!0}),dx.forEach(t),$w=f(ch),bn=i(ch,"A",{href:!0});var vx=l(bn);_n=i(vx,"IMG",{alt:!0,src:!0}),vx.forEach(t),ch.forEach(t),pu=f(e),hs=i(e,"P",{});var J0=l(hs);fs=i(J0,"A",{href:!0,rel:!0});var gx=l(fs);Mw=m(gx,"Marian: Fast Neural Machine Translation in C++"),gx.forEach(t),Pw=m(J0,", Marcin Junczys-Dowmunt et al."),J0.forEach(t),mu=f(e),wn=i(e,"P",{});var bx=l(wn);Lw=m(bx,"A framework for translation models, using the same models as BART"),bx.forEach(t),cu=f(e),En=i(e,"P",{});var _x=l(En);Sw=m(_x,"The library provides a version of this model for conditional generation."),_x.forEach(t),uu=f(e),He=i(e,"H3",{class:!0});var Av=l(He);$t=i(Av,"A",{id:!0,class:!0,href:!0});var wx=l($t);Wf=i(wx,"SPAN",{});var Ex=l(Wf);d(ps.$$.fragment,Ex),Ex.forEach(t),wx.forEach(t),Rw=f(Av),Jf=i(Av,"SPAN",{});var yx=l(Jf);Iw=m(yx,"T5"),yx.forEach(t),Av.forEach(t),du=f(e),ee=i(e,"DIV",{class:!0});var uh=l(ee);yn=i(uh,"A",{href:!0});var kx=l(yn);kn=i(kx,"IMG",{alt:!0,src:!0}),kx.forEach(t),qw=f(uh),xn=i(uh,"A",{href:!0});var xx=l(xn);An=i(xx,"IMG",{alt:!0,src:!0}),xx.forEach(t),Gw=f(uh),Tn=i(uh,"A",{href:!0});var Ax=l(Tn);$n=i(Ax,"IMG",{alt:!0,src:!0}),Ax.forEach(t),uh.forEach(t),vu=f(e),ms=i(e,"P",{});var Q0=l(ms);cs=i(Q0,"A",{href:!0,rel:!0});var Tx=l(cs);Nw=m(Tx,"Exploring the Limits of Transfer Learning with a Unified Text-to-Text Transformer"),Tx.forEach(t),Bw=m(Q0,", Colin Raffel et al."),Q0.forEach(t),gu=f(e),Mn=i(e,"P",{});var $x=l(Mn);Fw=m($x,`Uses the traditional transformer model (with a slight change in the positional embeddings, which are learned at each
layer). To be able to operate on all NLP tasks, it transforms them into text-to-text problems by using specific
prefixes: \u201Csummarize: \u201D, \u201Cquestion: \u201D, \u201Ctranslate English to German: \u201D and so forth.`),$x.forEach(t),bu=f(e),Pn=i(e,"P",{});var Mx=l(Pn);Dw=m(Mx,`The pretraining includes both supervised and self-supervised training. Supervised training is conducted on downstream
tasks provided by the GLUE and SuperGLUE benchmarks (converting them into text-to-text tasks as explained above).`),Mx.forEach(t),_u=f(e),Ln=i(e,"P",{});var Px=l(Ln);Hw=m(Px,`Self-supervised training uses corrupted tokens, by randomly removing 15% of the tokens and replacing them with
individual sentinel tokens (if several consecutive tokens are marked for removal, the whole group is replaced with a
single sentinel token). The input of the encoder is the corrupted sentence, the input of the decoder is the original
sentence and the target is then the dropped out tokens delimited by their sentinel tokens.`),Px.forEach(t),wu=f(e),Sn=i(e,"P",{});var Lx=l(Sn);Cw=m(Lx,`For instance, if we have the sentence \u201CMy dog is very cute .\u201D, and we decide to remove the tokens: \u201Cdog\u201D, \u201Cis\u201D and
\u201Ccute\u201D, the encoder input becomes \u201CMy <x> very <y> .\u201D and the target input becomes \u201C<x> dog is <y> cute .<z>\u201D`),Lx.forEach(t),Eu=f(e),Rn=i(e,"P",{});var Sx=l(Rn);zw=m(Sx,"The library provides a version of this model for conditional generation."),Sx.forEach(t),yu=f(e),Ce=i(e,"H3",{class:!0});var Tv=l(Ce);Mt=i(Tv,"A",{id:!0,class:!0,href:!0});var Rx=l(Mt);Qf=i(Rx,"SPAN",{});var Ix=l(Qf);d(us.$$.fragment,Ix),Ix.forEach(t),Rx.forEach(t),jw=f(Tv),ep=i(Tv,"SPAN",{});var qx=l(ep);Uw=m(qx,"MT5"),qx.forEach(t),Tv.forEach(t),ku=f(e),te=i(e,"DIV",{class:!0});var dh=l(te);In=i(dh,"A",{href:!0});var Gx=l(In);qn=i(Gx,"IMG",{alt:!0,src:!0}),Gx.forEach(t),Xw=f(dh),Gn=i(dh,"A",{href:!0});var Nx=l(Gn);Nn=i(Nx,"IMG",{alt:!0,src:!0}),Nx.forEach(t),Vw=f(dh),Bn=i(dh,"A",{href:!0});var Bx=l(Bn);Fn=i(Bx,"IMG",{alt:!0,src:!0}),Bx.forEach(t),dh.forEach(t),xu=f(e),ds=i(e,"P",{});var eE=l(ds);vs=i(eE,"A",{href:!0,rel:!0});var Fx=l(vs);Yw=m(Fx,"mT5: A massively multilingual pre-trained text-to-text transformer"),Fx.forEach(t),Zw=m(eE,`, Linting Xue
et al.`),eE.forEach(t),Au=f(e),Dn=i(e,"P",{});var Dx=l(Dn);Kw=m(Dx,`The model architecture is same as T5. mT5\u2019s pretraining objective includes T5\u2019s self-supervised training, but not T5\u2019s
supervised training. mT5 is trained on 101 languages.`),Dx.forEach(t),Tu=f(e),Hn=i(e,"P",{});var Hx=l(Hn);Ow=m(Hx,"The library provides a version of this model for conditional generation."),Hx.forEach(t),$u=f(e),ze=i(e,"H3",{class:!0});var $v=l(ze);Pt=i($v,"A",{id:!0,class:!0,href:!0});var Cx=l(Pt);tp=i(Cx,"SPAN",{});var zx=l(tp);d(gs.$$.fragment,zx),zx.forEach(t),Cx.forEach(t),Ww=f($v),ap=i($v,"SPAN",{});var jx=l(ap);Jw=m(jx,"MBart"),jx.forEach(t),$v.forEach(t),Mu=f(e),ae=i(e,"DIV",{class:!0});var vh=l(ae);Cn=i(vh,"A",{href:!0});var Ux=l(Cn);zn=i(Ux,"IMG",{alt:!0,src:!0}),Ux.forEach(t),Qw=f(vh),jn=i(vh,"A",{href:!0});var Xx=l(jn);Un=i(Xx,"IMG",{alt:!0,src:!0}),Xx.forEach(t),e2=f(vh),Xn=i(vh,"A",{href:!0});var Vx=l(Xn);Vn=i(Vx,"IMG",{alt:!0,src:!0}),Vx.forEach(t),vh.forEach(t),Pu=f(e),bs=i(e,"P",{});var tE=l(bs);_s=i(tE,"A",{href:!0,rel:!0});var Yx=l(_s);t2=m(Yx,"Multilingual Denoising Pre-training for Neural Machine Translation"),Yx.forEach(t),a2=m(tE,` by Yinhan Liu,
Jiatao Gu, Naman Goyal, Xian Li, Sergey Edunov Marjan Ghazvininejad, Mike Lewis, Luke Zettlemoyer.`),tE.forEach(t),Lu=f(e),Yn=i(e,"P",{});var Zx=l(Yn);s2=m(Zx,`The model architecture and pretraining objective is same as BART, but MBart is trained on 25 languages and is intended
for supervised and unsupervised machine translation. MBart is one of the first methods for pretraining a complete
sequence-to-sequence model by denoising full texts in multiple languages,`),Zx.forEach(t),Su=f(e),Zn=i(e,"P",{});var Kx=l(Zn);i2=m(Kx,"The library provides a version of this model for conditional generation."),Kx.forEach(t),Ru=f(e),Lt=i(e,"P",{});var Mv=l(Lt);r2=m(Mv,"The "),ws=i(Mv,"A",{href:!0,rel:!0});var Ox=l(ws);l2=m(Ox,"mbart-large-en-ro checkpoint"),Ox.forEach(t),n2=m(Mv,` can be used for english ->
romanian translation.`),Mv.forEach(t),Iu=f(e),me=i(e,"P",{});var gh=l(me);o2=m(gh,"The "),Es=i(gh,"A",{href:!0,rel:!0});var Wx=l(Es);h2=m(Wx,"mbart-large-cc25"),Wx.forEach(t),f2=m(gh,` checkpoint can be finetuned for other
translation and summarization tasks, using code in `),sp=i(gh,"CODE",{});var Jx=l(sp);p2=m(Jx,"examples/pytorch/translation/"),Jx.forEach(t),m2=m(gh,` , but is not very useful without
finetuning.`),gh.forEach(t),qu=f(e),je=i(e,"H3",{class:!0});var Pv=l(je);St=i(Pv,"A",{id:!0,class:!0,href:!0});var Qx=l(St);ip=i(Qx,"SPAN",{});var eA=l(ip);d(ys.$$.fragment,eA),eA.forEach(t),Qx.forEach(t),c2=f(Pv),rp=i(Pv,"SPAN",{});var tA=l(rp);u2=m(tA,"ProphetNet"),tA.forEach(t),Pv.forEach(t),Gu=f(e),se=i(e,"DIV",{class:!0});var bh=l(se);Kn=i(bh,"A",{href:!0});var aA=l(Kn);On=i(aA,"IMG",{alt:!0,src:!0}),aA.forEach(t),d2=f(bh),Wn=i(bh,"A",{href:!0});var sA=l(Wn);Jn=i(sA,"IMG",{alt:!0,src:!0}),sA.forEach(t),v2=f(bh),Qn=i(bh,"A",{href:!0});var iA=l(Qn);eo=i(iA,"IMG",{alt:!0,src:!0}),iA.forEach(t),bh.forEach(t),Nu=f(e),ks=i(e,"P",{});var aE=l(ks);xs=i(aE,"A",{href:!0,rel:!0});var rA=l(xs);g2=m(rA,"ProphetNet: Predicting Future N-gram for Sequence-to-Sequence Pre-training,"),rA.forEach(t),b2=m(aE,` by
Yu Yan, Weizhen Qi, Yeyun Gong, Dayiheng Liu, Nan Duan, Jiusheng Chen, Ruofei Zhang, Ming Zhou.`),aE.forEach(t),Bu=f(e),ce=i(e,"P",{});var _h=l(ce);_2=m(_h,"ProphetNet introduces a novel "),lp=i(_h,"EM",{});var lA=l(lp);w2=m(lA,"sequence-to-sequence"),lA.forEach(t),E2=m(_h," pretraining objective, called "),np=i(_h,"EM",{});var nA=l(np);y2=m(nA,"future n-gram prediction"),nA.forEach(t),k2=m(_h,`. In
future n-gram prediction, the model predicts the next n tokens simultaneously based on previous context tokens at each
time step instead instead of just the single next token. The future n-gram prediction explicitly encourages the model
to plan for the future tokens and prevent overfitting on strong local correlations. The model architecture is based on
the original Transformer, but replaces the \u201Cstandard\u201D self-attention mechanism in the decoder by a a main
self-attention mechanism and a self and n-stream (predict) self-attention mechanism.`),_h.forEach(t),Fu=f(e),to=i(e,"P",{});var oA=l(to);x2=m(oA,`The library provides a pre-trained version of this model for conditional generation and a fine-tuned version for
summarization.`),oA.forEach(t),Du=f(e),Ue=i(e,"H3",{class:!0});var Lv=l(Ue);Rt=i(Lv,"A",{id:!0,class:!0,href:!0});var hA=l(Rt);op=i(hA,"SPAN",{});var fA=l(op);d(As.$$.fragment,fA),fA.forEach(t),hA.forEach(t),A2=f(Lv),hp=i(Lv,"SPAN",{});var pA=l(hp);T2=m(pA,"XLM-ProphetNet"),pA.forEach(t),Lv.forEach(t),Hu=f(e),ie=i(e,"DIV",{class:!0});var wh=l(ie);ao=i(wh,"A",{href:!0});var mA=l(ao);so=i(mA,"IMG",{alt:!0,src:!0}),mA.forEach(t),$2=f(wh),io=i(wh,"A",{href:!0});var cA=l(io);ro=i(cA,"IMG",{alt:!0,src:!0}),cA.forEach(t),M2=f(wh),lo=i(wh,"A",{href:!0});var uA=l(lo);no=i(uA,"IMG",{alt:!0,src:!0}),uA.forEach(t),wh.forEach(t),Cu=f(e),Ts=i(e,"P",{});var sE=l(Ts);$s=i(sE,"A",{href:!0,rel:!0});var dA=l($s);P2=m(dA,"ProphetNet: Predicting Future N-gram for Sequence-to-Sequence Pre-training,"),dA.forEach(t),L2=m(sE,` by
Yu Yan, Weizhen Qi, Yeyun Gong, Dayiheng Liu, Nan Duan, Jiusheng Chen, Ruofei Zhang, Ming Zhou.`),sE.forEach(t),zu=f(e),It=i(e,"P",{});var Sv=l(It);S2=m(Sv,`XLM-ProphetNet\u2019s model architecture and pretraining objective is same as ProphetNet, but XLM-ProphetNet was pre-trained
on the cross-lingual dataset `),Ms=i(Sv,"A",{href:!0,rel:!0});var vA=l(Ms);R2=m(vA,"XGLUE"),vA.forEach(t),I2=m(Sv,"."),Sv.forEach(t),ju=f(e),oo=i(e,"P",{});var gA=l(oo);q2=m(gA,`The library provides a pre-trained version of this model for multi-lingual conditional generation and fine-tuned
versions for headline generation and question generation, respectively.`),gA.forEach(t),Uu=f(e),ho=i(e,"A",{id:!0}),l(ho).forEach(t),Xu=f(e),Xe=i(e,"H2",{class:!0});var Rv=l(Xe);qt=i(Rv,"A",{id:!0,class:!0,href:!0});var bA=l(qt);fp=i(bA,"SPAN",{});var _A=l(fp);d(Ps.$$.fragment,_A),_A.forEach(t),bA.forEach(t),G2=f(Rv),pp=i(Rv,"SPAN",{});var wA=l(pp);N2=m(wA,"Multimodal models"),wA.forEach(t),Rv.forEach(t),Vu=f(e),fo=i(e,"P",{});var EA=l(fo);B2=m(EA,`There is one multimodal model in the library which has not been pretrained in the self-supervised fashion like the
others.`),EA.forEach(t),Yu=f(e),Ve=i(e,"H3",{class:!0});var Iv=l(Ve);Gt=i(Iv,"A",{id:!0,class:!0,href:!0});var yA=l(Gt);mp=i(yA,"SPAN",{});var kA=l(mp);d(Ls.$$.fragment,kA),kA.forEach(t),yA.forEach(t),F2=f(Iv),cp=i(Iv,"SPAN",{});var xA=l(cp);D2=m(xA,"MMBT"),xA.forEach(t),Iv.forEach(t),Zu=f(e),Ss=i(e,"P",{});var iE=l(Ss);Rs=i(iE,"A",{href:!0,rel:!0});var AA=l(Rs);H2=m(AA,"Supervised Multimodal Bitransformers for Classifying Images and Text"),AA.forEach(t),C2=m(iE,`, Douwe Kiela
et al.`),iE.forEach(t),Ku=f(e),po=i(e,"P",{});var TA=l(po);z2=m(TA,`A transformers model used in multimodal settings, combining a text and an image to make predictions. The transformer
model takes as inputs the embeddings of the tokenized text and the final activations of a pretrained on images resnet
(after the pooling layer) that goes through a linear layer (to go from number of features at the end of the resnet to
the hidden state dimension of the transformer).`),TA.forEach(t),Ou=f(e),mo=i(e,"P",{});var $A=l(mo);j2=m($A,`The different inputs are concatenated, and on top of the positional embeddings, a segment embedding is added to let the
model know which part of the input vector corresponds to the text and which to the image.`),$A.forEach(t),Wu=f(e),co=i(e,"P",{});var MA=l(co);U2=m(MA,"The pretrained model only works for classification."),MA.forEach(t),Ju=f(e),uo=i(e,"A",{id:!0}),l(uo).forEach(t),Qu=f(e),Ye=i(e,"H2",{class:!0});var qv=l(Ye);Nt=i(qv,"A",{id:!0,class:!0,href:!0});var PA=l(Nt);up=i(PA,"SPAN",{});var LA=l(up);d(Is.$$.fragment,LA),LA.forEach(t),PA.forEach(t),X2=f(qv),dp=i(qv,"SPAN",{});var SA=l(dp);V2=m(SA,"Retrieval-based models"),SA.forEach(t),qv.forEach(t),ed=f(e),vo=i(e,"P",{});var RA=l(vo);Y2=m(RA,"Some models use documents retrieval during (pre)training and inference for open-domain question answering, for example."),RA.forEach(t),td=f(e),Ze=i(e,"H3",{class:!0});var Gv=l(Ze);Bt=i(Gv,"A",{id:!0,class:!0,href:!0});var IA=l(Bt);vp=i(IA,"SPAN",{});var qA=l(vp);d(qs.$$.fragment,qA),qA.forEach(t),IA.forEach(t),Z2=f(Gv),gp=i(Gv,"SPAN",{});var GA=l(gp);K2=m(GA,"DPR"),GA.forEach(t),Gv.forEach(t),ad=f(e),re=i(e,"DIV",{class:!0});var Eh=l(re);go=i(Eh,"A",{href:!0});var NA=l(go);bo=i(NA,"IMG",{alt:!0,src:!0}),NA.forEach(t),O2=f(Eh),_o=i(Eh,"A",{href:!0});var BA=l(_o);wo=i(BA,"IMG",{alt:!0,src:!0}),BA.forEach(t),W2=f(Eh),Eo=i(Eh,"A",{href:!0});var FA=l(Eo);yo=i(FA,"IMG",{alt:!0,src:!0}),FA.forEach(t),Eh.forEach(t),sd=f(e),Gs=i(e,"P",{});var rE=l(Gs);Ns=i(rE,"A",{href:!0,rel:!0});var DA=l(Ns);J2=m(DA,"Dense Passage Retrieval for Open-Domain Question Answering"),DA.forEach(t),Q2=m(rE,`, Vladimir Karpukhin et
al.`),rE.forEach(t),id=f(e),ko=i(e,"P",{});var HA=l(ko);e0=m(HA,`Dense Passage Retrieval (DPR) - is a set of tools and models for state-of-the-art open-domain question-answering
research.`),HA.forEach(t),rd=f(e),xo=i(e,"P",{});var CA=l(xo);t0=m(CA,"DPR consists in three models:"),CA.forEach(t),ld=f(e),ue=i(e,"UL",{});var yh=l(ue);bp=i(yh,"LI",{});var zA=l(bp);a0=m(zA,"Question encoder: encode questions as vectors"),zA.forEach(t),s0=f(yh),_p=i(yh,"LI",{});var jA=l(_p);i0=m(jA,"Context encoder: encode contexts as vectors"),jA.forEach(t),r0=f(yh),wp=i(yh,"LI",{});var UA=l(wp);l0=m(UA,`Reader: extract the answer of the questions inside retrieved contexts, along with a relevance score (high if the
inferred span actually answers the question).`),UA.forEach(t),yh.forEach(t),nd=f(e),Ao=i(e,"P",{});var XA=l(Ao);n0=m(XA,`DPR\u2019s pipeline (not implemented yet) uses a retrieval step to find the top k contexts given a certain question, and
then it calls the reader with the question and the retrieved documents to get the answer.`),XA.forEach(t),od=f(e),Ke=i(e,"H3",{class:!0});var Nv=l(Ke);Ft=i(Nv,"A",{id:!0,class:!0,href:!0});var VA=l(Ft);Ep=i(VA,"SPAN",{});var YA=l(Ep);d(Bs.$$.fragment,YA),YA.forEach(t),VA.forEach(t),o0=f(Nv),yp=i(Nv,"SPAN",{});var ZA=l(yp);h0=m(ZA,"RAG"),ZA.forEach(t),Nv.forEach(t),hd=f(e),Oe=i(e,"DIV",{class:!0});var Bv=l(Oe);To=i(Bv,"A",{href:!0});var KA=l(To);$o=i(KA,"IMG",{alt:!0,src:!0}),KA.forEach(t),f0=f(Bv),Mo=i(Bv,"A",{href:!0});var OA=l(Mo);Po=i(OA,"IMG",{alt:!0,src:!0}),OA.forEach(t),Bv.forEach(t),fd=f(e),Fs=i(e,"P",{});var lE=l(Fs);Ds=i(lE,"A",{href:!0,rel:!0});var WA=l(Ds);p0=m(WA,"Retrieval-Augmented Generation for Knowledge-Intensive NLP Tasks"),WA.forEach(t),m0=m(lE,`, Patrick Lewis,
Ethan Perez, Aleksandara Piktus, Fabio Petroni, Vladimir Karpukhin, Naman Goyal, Heinrich K\xFCttler, Mike Lewis, Wen-tau
Yih, Tim Rockt\xE4schel, Sebastian Riedel, Douwe Kiela`),lE.forEach(t),pd=f(e),Lo=i(e,"P",{});var JA=l(Lo);c0=m(JA,`Retrieval-augmented generation (\u201CRAG\u201D) models combine the powers of pretrained dense retrieval (DPR) and Seq2Seq
models. RAG models retrieve docs, pass them to a seq2seq model, then marginalize to generate outputs. The retriever and
seq2seq modules are initialized from pretrained models, and fine-tuned jointly, allowing both retrieval and generation
to adapt to downstream tasks.`),JA.forEach(t),md=f(e),So=i(e,"P",{});var QA=l(So);u0=m(QA,"The two models RAG-Token and RAG-Sequence are available for generation."),QA.forEach(t),cd=f(e),We=i(e,"H2",{class:!0});var Fv=l(We);Dt=i(Fv,"A",{id:!0,class:!0,href:!0});var e9=l(Dt);kp=i(e9,"SPAN",{});var t9=l(kp);d(Hs.$$.fragment,t9),t9.forEach(t),e9.forEach(t),d0=f(Fv),xp=i(Fv,"SPAN",{});var a9=l(xp);v0=m(a9,"More technical aspects"),a9.forEach(t),Fv.forEach(t),ud=f(e),Je=i(e,"H3",{class:!0});var Dv=l(Je);Ht=i(Dv,"A",{id:!0,class:!0,href:!0});var s9=l(Ht);Ap=i(s9,"SPAN",{});var i9=l(Ap);d(Cs.$$.fragment,i9),i9.forEach(t),s9.forEach(t),g0=f(Dv),Tp=i(Dv,"SPAN",{});var r9=l(Tp);b0=m(r9,"Full vs sparse attention"),r9.forEach(t),Dv.forEach(t),dd=f(e),Ro=i(e,"P",{});var l9=l(Ro);_0=m(l9,`Most transformer models use full attention in the sense that the attention matrix is square. It can be a big
computational bottleneck when you have long texts. Longformer and reformer are models that try to be more efficient and
use a sparse version of the attention matrix to speed up training.`),l9.forEach(t),vd=f(e),Io=i(e,"A",{id:!0}),l(Io).forEach(t),gd=f(e),qo=i(e,"P",{});var n9=l(qo);$p=i(n9,"STRONG",{});var o9=l($p);w0=m(o9,"LSH attention"),o9.forEach(t),n9.forEach(t),bd=f(e),zs=i(e,"P",{});var nE=l(zs);Go=i(nE,"A",{href:!0});var h9=l(Go);E0=m(h9,"Reformer"),h9.forEach(t),y0=m(nE,` uses LSH attention. In the softmax(QK^t), only the biggest elements (in the softmax
dimension) of the matrix QK^t are going to give useful contributions. So for each query q in Q, we can consider only
the keys k in K that are close to q. A hash function is used to determine if q and k are close. The attention mask is
modified to mask the current token (except at the first position), because it will give a query and a key equal (so
very similar to each other). Since the hash can be a bit random, several hash functions are used in practice
(determined by a n_rounds parameter) and then are averaged together.`),nE.forEach(t),_d=f(e),No=i(e,"A",{id:!0}),l(No).forEach(t),wd=f(e),Bo=i(e,"P",{});var f9=l(Bo);Mp=i(f9,"STRONG",{});var p9=l(Mp);k0=m(p9,"Local attention"),p9.forEach(t),f9.forEach(t),Ed=f(e),js=i(e,"P",{});var oE=l(js);Fo=i(oE,"A",{href:!0});var m9=l(Fo);x0=m(m9,"Longformer"),m9.forEach(t),A0=m(oE,` uses local attention: often, the local context (e.g., what are the two tokens to the
left and right?) is enough to take action for a given token. Also, by stacking attention layers that have a small
window, the last layer will have a receptive field of more than just the tokens in the window, allowing them to build a
representation of the whole sentence.`),oE.forEach(t),yd=f(e),Do=i(e,"P",{});var c9=l(Do);T0=m(c9,`Some preselected input tokens are also given global attention: for those few tokens, the attention matrix can access
all tokens and this process is symmetric: all other tokens have access to those specific tokens (on top of the ones in
their local window). This is shown in Figure 2d of the paper, see below for a sample attention mask:`),c9.forEach(t),kd=f(e),Ct=i(e,"IMG",{scale:!0,align:!0,src:!0}),xd=f(e),Ho=i(e,"P",{});var u9=l(Ho);$0=m(u9,`Using those attention matrices with less parameters then allows the model to have inputs having a bigger sequence
length.`),u9.forEach(t),Ad=f(e),Qe=i(e,"H3",{class:!0});var Hv=l(Qe);zt=i(Hv,"A",{id:!0,class:!0,href:!0});var d9=l(zt);Pp=i(d9,"SPAN",{});var v9=l(Pp);d(Us.$$.fragment,v9),v9.forEach(t),d9.forEach(t),M0=f(Hv),Lp=i(Hv,"SPAN",{});var g9=l(Lp);P0=m(g9,"Other tricks"),g9.forEach(t),Hv.forEach(t),Td=f(e),Co=i(e,"A",{id:!0}),l(Co).forEach(t),$d=f(e),zo=i(e,"P",{});var b9=l(zo);Sp=i(b9,"STRONG",{});var _9=l(Sp);L0=m(_9,"Axial positional encodings"),_9.forEach(t),b9.forEach(t),Md=f(e),E=i(e,"P",{});var y=l(E);jo=i(y,"A",{href:!0});var w9=l(jo);S0=m(w9,"Reformer"),w9.forEach(t),R0=m(y,` uses axial positional encodings: in traditional transformer models, the positional encoding
E is a matrix of size `),Pd=ne(y),Ld=m(y," by "),Sd=ne(y),Rd=m(y,", "),Id=ne(y),qd=m(y," being the sequence length and "),Gd=ne(y),Nd=m(y,` the dimension of the
hidden state. If you have very long texts, this matrix can be huge and take way too much space on the GPU. To alleviate
that, axial positional encodings consist of factorizing that big matrix E in two smaller matrices E1 and E2, with
dimensions `),Bd=ne(y),Fd=m(y," and "),Dd=ne(y),Hd=m(y,", such that "),Cd=ne(y),zd=m(y,` and
`),jd=ne(y),Ud=m(y,` (with the product for the lengths, this ends up being way smaller). The embedding for time
step `),Xd=ne(y),Vd=m(y," in E is obtained by concatenating the embeddings for timestep "),Yd=ne(y),Zd=m(y," in E1 and "),Kd=ne(y),Od=m(y,`
in E2.`),y.forEach(t),this.h()},h(){r(k,"name","hf:doc:metadata"),r(k,"content",JSON.stringify(C9)),r(A,"id","summary-of-the-models"),r(A,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),r(A,"href","#summary-of-the-models"),r(x,"class","relative group"),r(Zt,"href","https://arxiv.org/abs/1706.03762"),r(Zt,"rel","nofollow"),r(Kt,"href","http://nlp.seas.harvard.edu/2018/04/03/attention.html"),r(Kt,"rel","nofollow"),r(Ot,"href","https://huggingface.co/models"),r(Ot,"rel","nofollow"),r(Zs,"href","#autoregressive-models"),r(Ks,"href","#autoencoding-models"),r(Os,"href","#seq-to-seq-models"),r(Ws,"href","#multimodal-models"),r(Js,"href","#retrieval-based-models"),r(M,"width","560"),r(M,"height","315"),c(M.src,fE="https://www.youtube.com/embed/H39Z_720T5s")||r(M,"src",fE),r(M,"title","YouTube video player"),r(M,"frameborder","0"),r(M,"allow",`accelerometer; autoplay; clipboard-write; encrypted-media; gyroscope;
picture-in-picture`),M.allowFullscreen=!0,r(ii,"id","autoregressive-models"),r(et,"id","decoders-or-autoregressive-models"),r(et,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),r(et,"href","#decoders-or-autoregressive-models"),r(ge,"class","relative group"),r(tt,"id","original-gpt"),r(tt,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),r(tt,"href","#original-gpt"),r(be,"class","relative group"),r(ni,"alt","Models"),c(ni.src,pE="https://img.shields.io/badge/All_model_pages-openai--gpt-blueviolet")||r(ni,"src",pE),r(li,"href","https://huggingface.co/models?filter=openai-gpt"),r(hi,"alt","Doc"),c(hi.src,mE="https://img.shields.io/badge/Model_documentation-openai--gpt-blueviolet")||r(hi,"src",mE),r(oi,"href","model_doc/openai-gpt"),r(pi,"alt","Spaces"),c(pi.src,cE="https://img.shields.io/badge/%F0%9F%A4%97%20Hugging%20Face-Spaces-blue")||r(pi,"src",cE),r(fi,"href","https://huggingface.co/spaces/docs-demos/openai-gpt"),r(q,"class","flex flex-wrap space-x-1"),r(ta,"href","https://cdn.openai.com/research-covers/language-unsupervised/language_understanding_paper.pdf"),r(ta,"rel","nofollow"),r(at,"id","gpt2"),r(at,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),r(at,"href","#gpt2"),r(_e,"class","relative group"),r(di,"alt","Models"),c(di.src,uE="https://img.shields.io/badge/All_model_pages-gpt2-blueviolet")||r(di,"src",uE),r(ui,"href","https://huggingface.co/models?filter=gpt2"),r(gi,"alt","Doc"),c(gi.src,dE="https://img.shields.io/badge/Model_documentation-gpt2-blueviolet")||r(gi,"src",dE),r(vi,"href","model_doc/gpt2"),r(_i,"alt","Spaces"),c(_i.src,vE="https://img.shields.io/badge/%F0%9F%A4%97%20Hugging%20Face-Spaces-blue")||r(_i,"src",vE),r(bi,"href","https://huggingface.co/spaces/docs-demos/gpt2"),r(G,"class","flex flex-wrap space-x-1"),r(ia,"href","https://d4mucfpksywv.cloudfront.net/better-language-models/language_models_are_unsupervised_multitask_learners.pdf"),r(ia,"rel","nofollow"),r(st,"id","ctrl"),r(st,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),r(st,"href","#ctrl"),r(we,"class","relative group"),r(ki,"alt","Models"),c(ki.src,gE="https://img.shields.io/badge/All_model_pages-ctrl-blueviolet")||r(ki,"src",gE),r(yi,"href","https://huggingface.co/models?filter=ctrl"),r(Ai,"alt","Doc"),c(Ai.src,bE="https://img.shields.io/badge/Model_documentation-ctrl-blueviolet")||r(Ai,"src",bE),r(xi,"href","model_doc/ctrl"),r($i,"alt","Spaces"),c($i.src,_E="https://img.shields.io/badge/%F0%9F%A4%97%20Hugging%20Face-Spaces-blue")||r($i,"src",_E),r(Ti,"href","https://huggingface.co/spaces/docs-demos/tiny-ctrl"),r(N,"class","flex flex-wrap space-x-1"),r(na,"href","https://arxiv.org/abs/1909.05858"),r(na,"rel","nofollow"),r(it,"id","transformerxl"),r(it,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),r(it,"href","#transformerxl"),r(Ee,"class","relative group"),r(Si,"alt","Models"),c(Si.src,wE="https://img.shields.io/badge/All_model_pages-transfo--xl-blueviolet")||r(Si,"src",wE),r(Li,"href","https://huggingface.co/models?filter=transfo-xl"),r(Ii,"alt","Doc"),c(Ii.src,EE="https://img.shields.io/badge/Model_documentation-transfo--xl-blueviolet")||r(Ii,"src",EE),r(Ri,"href","model_doc/transfo-xl"),r(Gi,"alt","Spaces"),c(Gi.src,yE="https://img.shields.io/badge/%F0%9F%A4%97%20Hugging%20Face-Spaces-blue")||r(Gi,"src",yE),r(qi,"href","https://huggingface.co/spaces/docs-demos/transfo-xl-wt103"),r(B,"class","flex flex-wrap space-x-1"),r(fa,"href","https://arxiv.org/abs/1901.02860"),r(fa,"rel","nofollow"),r(Hi,"id","reformer"),r(rt,"id","reformer"),r(rt,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),r(rt,"href","#reformer"),r(ye,"class","relative group"),r(zi,"alt","Models"),c(zi.src,kE="https://img.shields.io/badge/All_model_pages-reformer-blueviolet")||r(zi,"src",kE),r(Ci,"href","https://huggingface.co/models?filter=reformer"),r(Ui,"alt","Doc"),c(Ui.src,xE="https://img.shields.io/badge/Model_documentation-reformer-blueviolet")||r(Ui,"src",xE),r(ji,"href","model_doc/reformer"),r(Vi,"alt","Spaces"),c(Vi.src,AE="https://img.shields.io/badge/%F0%9F%A4%97%20Hugging%20Face-Spaces-blue")||r(Vi,"src",AE),r(Xi,"href","https://huggingface.co/spaces/docs-demos/reformer-crime-and-punishment"),r(F,"class","flex flex-wrap space-x-1"),r(ca,"href","https://arxiv.org/abs/2001.04451"),r(ca,"rel","nofollow"),r(Zi,"href","#axial-pos-encoding"),r(Ki,"href","#lsh-attention"),r(nt,"id","xlnet"),r(nt,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),r(nt,"href","#xlnet"),r(ke,"class","relative group"),r(Qi,"alt","Models"),c(Qi.src,TE="https://img.shields.io/badge/All_model_pages-xlnet-blueviolet")||r(Qi,"src",TE),r(Ji,"href","https://huggingface.co/models?filter=xlnet"),r(tr,"alt","Doc"),c(tr.src,$E="https://img.shields.io/badge/Model_documentation-xlnet-blueviolet")||r(tr,"src",$E),r(er,"href","model_doc/xlnet"),r(sr,"alt","Spaces"),c(sr.src,ME="https://img.shields.io/badge/%F0%9F%A4%97%20Hugging%20Face-Spaces-blue")||r(sr,"src",ME),r(ar,"href","https://huggingface.co/spaces/docs-demos/xlnet-base-cased"),r(D,"class","flex flex-wrap space-x-1"),r(ba,"href","https://arxiv.org/abs/1906.08237"),r(ba,"rel","nofollow"),r(nr,"id","autoencoding-models"),r(ot,"id","encoders-or-autoencoding-models"),r(ot,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),r(ot,"href","#encoders-or-autoencoding-models"),r(xe,"class","relative group"),r(ht,"id","bert"),r(ht,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),r(ht,"href","#bert"),r(Ae,"class","relative group"),r(fr,"alt","Models"),c(fr.src,PE="https://img.shields.io/badge/All_model_pages-bert-blueviolet")||r(fr,"src",PE),r(hr,"href","https://huggingface.co/models?filter=bert"),r(mr,"alt","Doc"),c(mr.src,LE="https://img.shields.io/badge/Model_documentation-bert-blueviolet")||r(mr,"src",LE),r(pr,"href","model_doc/bert"),r(ur,"alt","Spaces"),c(ur.src,SE="https://img.shields.io/badge/%F0%9F%A4%97%20Hugging%20Face-Spaces-blue")||r(ur,"src",SE),r(cr,"href","https://huggingface.co/spaces/docs-demos/bert-base-uncased"),r(H,"class","flex flex-wrap space-x-1"),r(ka,"href","https://arxiv.org/abs/1810.04805"),r(ka,"rel","nofollow"),r(ft,"id","albert"),r(ft,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),r(ft,"href","#albert"),r(Te,"class","relative group"),r(_r,"alt","Models"),c(_r.src,RE="https://img.shields.io/badge/All_model_pages-albert-blueviolet")||r(_r,"src",RE),r(br,"href","https://huggingface.co/models?filter=albert"),r(Er,"alt","Doc"),c(Er.src,IE="https://img.shields.io/badge/Model_documentation-albert-blueviolet")||r(Er,"src",IE),r(wr,"href","model_doc/albert"),r(kr,"alt","Spaces"),c(kr.src,qE="https://img.shields.io/badge/%F0%9F%A4%97%20Hugging%20Face-Spaces-blue")||r(kr,"src",qE),r(yr,"href","https://huggingface.co/spaces/docs-demos/albert-base-v2"),r(C,"class","flex flex-wrap space-x-1"),r(Ta,"href","https://arxiv.org/abs/1909.11942"),r(Ta,"rel","nofollow"),r(pt,"id","roberta"),r(pt,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),r(pt,"href","#roberta"),r($e,"class","relative group"),r($r,"alt","Models"),c($r.src,GE="https://img.shields.io/badge/All_model_pages-roberta-blueviolet")||r($r,"src",GE),r(Tr,"href","https://huggingface.co/models?filter=roberta"),r(Pr,"alt","Doc"),c(Pr.src,NE="https://img.shields.io/badge/Model_documentation-roberta-blueviolet")||r(Pr,"src",NE),r(Mr,"href","model_doc/roberta"),r(Sr,"alt","Spaces"),c(Sr.src,BE="https://img.shields.io/badge/%F0%9F%A4%97%20Hugging%20Face-Spaces-blue")||r(Sr,"src",BE),r(Lr,"href","https://huggingface.co/spaces/docs-demos/roberta-base"),r(z,"class","flex flex-wrap space-x-1"),r(Pa,"href","https://arxiv.org/abs/1907.11692"),r(Pa,"rel","nofollow"),r(mt,"id","distilbert"),r(mt,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),r(mt,"href","#distilbert"),r(Me,"class","relative group"),r(Gr,"alt","Models"),c(Gr.src,FE="https://img.shields.io/badge/All_model_pages-distilbert-blueviolet")||r(Gr,"src",FE),r(qr,"href","https://huggingface.co/models?filter=distilbert"),r(Br,"alt","Doc"),c(Br.src,DE="https://img.shields.io/badge/Model_documentation-distilbert-blueviolet")||r(Br,"src",DE),r(Nr,"href","model_doc/distilbert"),r(Dr,"alt","Spaces"),c(Dr.src,HE="https://img.shields.io/badge/%F0%9F%A4%97%20Hugging%20Face-Spaces-blue")||r(Dr,"src",HE),r(Fr,"href","https://huggingface.co/spaces/docs-demos/distilbert-base-uncased"),r(j,"class","flex flex-wrap space-x-1"),r(Ra,"href","https://arxiv.org/abs/1910.01108"),r(Ra,"rel","nofollow"),r(ct,"id","convbert"),r(ct,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),r(ct,"href","#convbert"),r(Pe,"class","relative group"),r(jr,"alt","Models"),c(jr.src,CE="https://img.shields.io/badge/All_model_pages-convbert-blueviolet")||r(jr,"src",CE),r(zr,"href","https://huggingface.co/models?filter=convbert"),r(Xr,"alt","Doc"),c(Xr.src,zE="https://img.shields.io/badge/Model_documentation-convbert-blueviolet")||r(Xr,"src",zE),r(Ur,"href","model_doc/convbert"),r(Yr,"alt","Spaces"),c(Yr.src,jE="https://img.shields.io/badge/%F0%9F%A4%97%20Hugging%20Face-Spaces-blue")||r(Yr,"src",jE),r(Vr,"href","https://huggingface.co/spaces/docs-demos/conv-bert-base"),r(U,"class","flex flex-wrap space-x-1"),r(Ga,"href","https://arxiv.org/abs/2008.02496"),r(Ga,"rel","nofollow"),r(ut,"id","xlm"),r(ut,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),r(ut,"href","#xlm"),r(Le,"class","relative group"),r(Wr,"alt","Models"),c(Wr.src,UE="https://img.shields.io/badge/All_model_pages-xlm-blueviolet")||r(Wr,"src",UE),r(Or,"href","https://huggingface.co/models?filter=xlm"),r(Qr,"alt","Doc"),c(Qr.src,XE="https://img.shields.io/badge/Model_documentation-xlm-blueviolet")||r(Qr,"src",XE),r(Jr,"href","model_doc/xlm"),r(tl,"alt","Spaces"),c(tl.src,VE="https://img.shields.io/badge/%F0%9F%A4%97%20Hugging%20Face-Spaces-blue")||r(tl,"src",VE),r(el,"href","https://huggingface.co/spaces/docs-demos/xlm-mlm-en-2048"),r(X,"class","flex flex-wrap space-x-1"),r(Fa,"href","https://arxiv.org/abs/1901.07291"),r(Fa,"rel","nofollow"),r(dt,"id","xlmroberta"),r(dt,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),r(dt,"href","#xlmroberta"),r(Se,"class","relative group"),r(rl,"alt","Models"),c(rl.src,YE="https://img.shields.io/badge/All_model_pages-xlm--roberta-blueviolet")||r(rl,"src",YE),r(il,"href","https://huggingface.co/models?filter=xlm-roberta"),r(nl,"alt","Doc"),c(nl.src,ZE="https://img.shields.io/badge/Model_documentation-xlm--roberta-blueviolet")||r(nl,"src",ZE),r(ll,"href","model_doc/xlm-roberta"),r(hl,"alt","Spaces"),c(hl.src,KE="https://img.shields.io/badge/%F0%9F%A4%97%20Hugging%20Face-Spaces-blue")||r(hl,"src",KE),r(ol,"href","https://huggingface.co/spaces/docs-demos/xlm-roberta-base"),r(V,"class","flex flex-wrap space-x-1"),r(Ca,"href","https://arxiv.org/abs/1911.02116"),r(Ca,"rel","nofollow"),r(vt,"id","flaubert"),r(vt,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),r(vt,"href","#flaubert"),r(Re,"class","relative group"),r(cl,"alt","Models"),c(cl.src,OE="https://img.shields.io/badge/All_model_pages-flaubert-blueviolet")||r(cl,"src",OE),r(ml,"href","https://huggingface.co/models?filter=flaubert"),r(dl,"alt","Doc"),c(dl.src,WE="https://img.shields.io/badge/Model_documentation-flaubert-blueviolet")||r(dl,"src",WE),r(ul,"href","model_doc/flaubert"),r(gl,"alt","Spaces"),c(gl.src,JE="https://img.shields.io/badge/%F0%9F%A4%97%20Hugging%20Face-Spaces-blue")||r(gl,"src",JE),r(vl,"href","https://huggingface.co/spaces/docs-demos/flaubert_small_cased"),r(Y,"class","flex flex-wrap space-x-1"),r(Ua,"href","https://arxiv.org/abs/1912.05372"),r(Ua,"rel","nofollow"),r(gt,"id","electra"),r(gt,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),r(gt,"href","#electra"),r(Ie,"class","relative group"),r(El,"alt","Models"),c(El.src,QE="https://img.shields.io/badge/All_model_pages-electra-blueviolet")||r(El,"src",QE),r(wl,"href","https://huggingface.co/models?filter=electra"),r(kl,"alt","Doc"),c(kl.src,ey="https://img.shields.io/badge/Model_documentation-electra-blueviolet")||r(kl,"src",ey),r(yl,"href","model_doc/electra"),r(Al,"alt","Spaces"),c(Al.src,ty="https://img.shields.io/badge/%F0%9F%A4%97%20Hugging%20Face-Spaces-blue")||r(Al,"src",ty),r(xl,"href","https://huggingface.co/spaces/docs-demos/electra_large_discriminator_squad2_512"),r(Z,"class","flex flex-wrap space-x-1"),r(Ya,"href","https://arxiv.org/abs/2003.10555"),r(Ya,"rel","nofollow"),r(bt,"id","funnel-transformer"),r(bt,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),r(bt,"href","#funnel-transformer"),r(qe,"class","relative group"),r(Pl,"alt","Models"),c(Pl.src,ay="https://img.shields.io/badge/All_model_pages-funnel-blueviolet")||r(Pl,"src",ay),r(Ml,"href","https://huggingface.co/models?filter=funnel"),r(Sl,"alt","Doc"),c(Sl.src,sy="https://img.shields.io/badge/Model_documentation-funnel-blueviolet")||r(Sl,"src",sy),r(Ll,"href","model_doc/funnel"),r(Il,"alt","Spaces"),c(Il.src,iy="https://img.shields.io/badge/%F0%9F%A4%97%20Hugging%20Face-Spaces-blue")||r(Il,"src",iy),r(Rl,"href","https://huggingface.co/spaces/docs-demos/funnel-transformer-small"),r(K,"class","flex flex-wrap space-x-1"),r(Oa,"href","https://arxiv.org/abs/2006.03236"),r(Oa,"rel","nofollow"),r(Fl,"id","longformer"),r(_t,"id","longformer"),r(_t,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),r(_t,"href","#longformer"),r(Ge,"class","relative group"),r(Hl,"alt","Models"),c(Hl.src,ry="https://img.shields.io/badge/All_model_pages-longformer-blueviolet")||r(Hl,"src",ry),r(Dl,"href","https://huggingface.co/models?filter=longformer"),r(zl,"alt","Doc"),c(zl.src,ly="https://img.shields.io/badge/Model_documentation-longformer-blueviolet")||r(zl,"src",ly),r(Cl,"href","model_doc/longformer"),r(Ul,"alt","Spaces"),c(Ul.src,ny="https://img.shields.io/badge/%F0%9F%A4%97%20Hugging%20Face-Spaces-blue")||r(Ul,"src",ny),r(jl,"href","https://huggingface.co/spaces/docs-demos/longformer-base-4096-finetuned-squadv1"),r(O,"class","flex flex-wrap space-x-1"),r(Qa,"href","https://arxiv.org/abs/2004.05150"),r(Qa,"rel","nofollow"),r(Xl,"href","#local-attention"),r(Zl,"id","seq-to-seq-models"),r(yt,"id","sequencetosequence-models"),r(yt,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),r(yt,"href","#sequencetosequence-models"),r(Ne,"class","relative group"),r(kt,"id","bart"),r(kt,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),r(kt,"href","#bart"),r(Be,"class","relative group"),r(Wl,"alt","Models"),c(Wl.src,oy="https://img.shields.io/badge/All_model_pages-bart-blueviolet")||r(Wl,"src",oy),r(Ol,"href","https://huggingface.co/models?filter=bart"),r(Ql,"alt","Doc"),c(Ql.src,hy="https://img.shields.io/badge/Model_documentation-bart-blueviolet")||r(Ql,"src",hy),r(Jl,"href","model_doc/bart"),r(tn,"alt","Spaces"),c(tn.src,fy="https://img.shields.io/badge/%F0%9F%A4%97%20Hugging%20Face-Spaces-blue")||r(tn,"src",fy),r(en,"href","https://huggingface.co/spaces/docs-demos/bart-large-mnli"),r(W,"class","flex flex-wrap space-x-1"),r(is,"href","https://arxiv.org/abs/1910.13461"),r(is,"rel","nofollow"),r(xt,"id","pegasus"),r(xt,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),r(xt,"href","#pegasus"),r(Fe,"class","relative group"),r(ln,"alt","Models"),c(ln.src,py="https://img.shields.io/badge/All_model_pages-pegasus-blueviolet")||r(ln,"src",py),r(rn,"href","https://huggingface.co/models?filter=pegasus"),r(on,"alt","Doc"),c(on.src,my="https://img.shields.io/badge/Model_documentation-pegasus-blueviolet")||r(on,"src",my),r(nn,"href","model_doc/pegasus"),r(fn,"alt","Spaces"),c(fn.src,cy="https://img.shields.io/badge/%F0%9F%A4%97%20Hugging%20Face-Spaces-blue")||r(fn,"src",cy),r(hn,"href","https://huggingface.co/spaces/docs-demos/pegasus_paraphrase"),r(J,"class","flex flex-wrap space-x-1"),r(ns,"href","https://arxiv.org/pdf/1912.08777.pdf"),r(ns,"rel","nofollow"),r(Tt,"id","marianmt"),r(Tt,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),r(Tt,"href","#marianmt"),r(De,"class","relative group"),r(dn,"alt","Models"),c(dn.src,uy="https://img.shields.io/badge/All_model_pages-marian-blueviolet")||r(dn,"src",uy),r(un,"href","https://huggingface.co/models?filter=marian"),r(gn,"alt","Doc"),c(gn.src,dy="https://img.shields.io/badge/Model_documentation-marian-blueviolet")||r(gn,"src",dy),r(vn,"href","model_doc/marian"),r(_n,"alt","Spaces"),c(_n.src,vy="https://img.shields.io/badge/%F0%9F%A4%97%20Hugging%20Face-Spaces-blue")||r(_n,"src",vy),r(bn,"href","https://huggingface.co/spaces/docs-demos/opus-mt-zh-en"),r(Q,"class","flex flex-wrap space-x-1"),r(fs,"href","https://arxiv.org/abs/1804.00344"),r(fs,"rel","nofollow"),r($t,"id","t5"),r($t,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),r($t,"href","#t5"),r(He,"class","relative group"),r(kn,"alt","Models"),c(kn.src,gy="https://img.shields.io/badge/All_model_pages-t5-blueviolet")||r(kn,"src",gy),r(yn,"href","https://huggingface.co/models?filter=t5"),r(An,"alt","Doc"),c(An.src,by="https://img.shields.io/badge/Model_documentation-t5-blueviolet")||r(An,"src",by),r(xn,"href","model_doc/t5"),r($n,"alt","Spaces"),c($n.src,_y="https://img.shields.io/badge/%F0%9F%A4%97%20Hugging%20Face-Spaces-blue")||r($n,"src",_y),r(Tn,"href","https://huggingface.co/spaces/docs-demos/t5-base"),r(ee,"class","flex flex-wrap space-x-1"),r(cs,"href","https://arxiv.org/abs/1910.10683"),r(cs,"rel","nofollow"),r(Mt,"id","mt5"),r(Mt,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),r(Mt,"href","#mt5"),r(Ce,"class","relative group"),r(qn,"alt","Models"),c(qn.src,wy="https://img.shields.io/badge/All_model_pages-mt5-blueviolet")||r(qn,"src",wy),r(In,"href","https://huggingface.co/models?filter=mt5"),r(Nn,"alt","Doc"),c(Nn.src,Ey="https://img.shields.io/badge/Model_documentation-mt5-blueviolet")||r(Nn,"src",Ey),r(Gn,"href","model_doc/mt5"),r(Fn,"alt","Spaces"),c(Fn.src,yy="https://img.shields.io/badge/%F0%9F%A4%97%20Hugging%20Face-Spaces-blue")||r(Fn,"src",yy),r(Bn,"href","https://huggingface.co/spaces/docs-demos/mt5-small-finetuned-arxiv-cs-finetuned-arxiv-cs-full"),r(te,"class","flex flex-wrap space-x-1"),r(vs,"href","https://arxiv.org/abs/2010.11934"),r(vs,"rel","nofollow"),r(Pt,"id","mbart"),r(Pt,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),r(Pt,"href","#mbart"),r(ze,"class","relative group"),r(zn,"alt","Models"),c(zn.src,ky="https://img.shields.io/badge/All_model_pages-mbart-blueviolet")||r(zn,"src",ky),r(Cn,"href","https://huggingface.co/models?filter=mbart"),r(Un,"alt","Doc"),c(Un.src,xy="https://img.shields.io/badge/Model_documentation-mbart-blueviolet")||r(Un,"src",xy),r(jn,"href","model_doc/mbart"),r(Vn,"alt","Spaces"),c(Vn.src,Ay="https://img.shields.io/badge/%F0%9F%A4%97%20Hugging%20Face-Spaces-blue")||r(Vn,"src",Ay),r(Xn,"href","https://huggingface.co/spaces/docs-demos/mbart-large-50-one-to-many-mmt"),r(ae,"class","flex flex-wrap space-x-1"),r(_s,"href","https://arxiv.org/abs/2001.08210"),r(_s,"rel","nofollow"),r(ws,"href","https://huggingface.co/facebook/mbart-large-en-ro"),r(ws,"rel","nofollow"),r(Es,"href","https://huggingface.co/facebook/mbart-large-cc25"),r(Es,"rel","nofollow"),r(St,"id","prophetnet"),r(St,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),r(St,"href","#prophetnet"),r(je,"class","relative group"),r(On,"alt","Models"),c(On.src,Ty="https://img.shields.io/badge/All_model_pages-prophetnet-blueviolet")||r(On,"src",Ty),r(Kn,"href","https://huggingface.co/models?filter=prophetnet"),r(Jn,"alt","Doc"),c(Jn.src,$y="https://img.shields.io/badge/Model_documentation-prophetnet-blueviolet")||r(Jn,"src",$y),r(Wn,"href","model_doc/prophetnet"),r(eo,"alt","Spaces"),c(eo.src,My="https://img.shields.io/badge/%F0%9F%A4%97%20Hugging%20Face-Spaces-blue")||r(eo,"src",My),r(Qn,"href","https://huggingface.co/spaces/docs-demos/prophetnet-large-uncased"),r(se,"class","flex flex-wrap space-x-1"),r(xs,"href","https://arxiv.org/abs/2001.04063"),r(xs,"rel","nofollow"),r(Rt,"id","xlmprophetnet"),r(Rt,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),r(Rt,"href","#xlmprophetnet"),r(Ue,"class","relative group"),r(so,"alt","Models"),c(so.src,Py="https://img.shields.io/badge/All_model_pages-xprophetnet-blueviolet")||r(so,"src",Py),r(ao,"href","https://huggingface.co/models?filter=xprophetnet"),r(ro,"alt","Doc"),c(ro.src,Ly="https://img.shields.io/badge/Model_documentation-xprophetnet-blueviolet")||r(ro,"src",Ly),r(io,"href","model_doc/xlm-prophetnet"),r(no,"alt","Spaces"),c(no.src,Sy="https://img.shields.io/badge/%F0%9F%A4%97%20Hugging%20Face-Spaces-blue")||r(no,"src",Sy),r(lo,"href","https://huggingface.co/spaces/docs-demos/xprophetnet-large-wiki100-cased-xglue-ntg"),r(ie,"class","flex flex-wrap space-x-1"),r($s,"href","https://arxiv.org/abs/2001.04063"),r($s,"rel","nofollow"),r(Ms,"href","https://arxiv.org/abs/2004.01401"),r(Ms,"rel","nofollow"),r(ho,"id","multimodal-models"),r(qt,"id","multimodal-models"),r(qt,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),r(qt,"href","#multimodal-models"),r(Xe,"class","relative group"),r(Gt,"id","mmbt"),r(Gt,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),r(Gt,"href","#mmbt"),r(Ve,"class","relative group"),r(Rs,"href","https://arxiv.org/abs/1909.02950"),r(Rs,"rel","nofollow"),r(uo,"id","retrieval-based-models"),r(Nt,"id","retrievalbased-models"),r(Nt,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),r(Nt,"href","#retrievalbased-models"),r(Ye,"class","relative group"),r(Bt,"id","dpr"),r(Bt,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),r(Bt,"href","#dpr"),r(Ze,"class","relative group"),r(bo,"alt","Models"),c(bo.src,Ry="https://img.shields.io/badge/All_model_pages-dpr-blueviolet")||r(bo,"src",Ry),r(go,"href","https://huggingface.co/models?filter=dpr"),r(wo,"alt","Doc"),c(wo.src,Iy="https://img.shields.io/badge/Model_documentation-dpr-blueviolet")||r(wo,"src",Iy),r(_o,"href","model_doc/dpr"),r(yo,"alt","Spaces"),c(yo.src,qy="https://img.shields.io/badge/%F0%9F%A4%97%20Hugging%20Face-Spaces-blue")||r(yo,"src",qy),r(Eo,"href","https://huggingface.co/spaces/docs-demos/dpr-question_encoder-bert-base-multilingual"),r(re,"class","flex flex-wrap space-x-1"),r(Ns,"href","https://arxiv.org/abs/2004.04906"),r(Ns,"rel","nofollow"),r(Ft,"id","rag"),r(Ft,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),r(Ft,"href","#rag"),r(Ke,"class","relative group"),r($o,"alt","Models"),c($o.src,Gy="https://img.shields.io/badge/All_model_pages-rag-blueviolet")||r($o,"src",Gy),r(To,"href","https://huggingface.co/models?filter=rag"),r(Po,"alt","Doc"),c(Po.src,Ny="https://img.shields.io/badge/Model_documentation-rag-blueviolet")||r(Po,"src",Ny),r(Mo,"href","model_doc/rag"),r(Oe,"class","flex flex-wrap space-x-1"),r(Ds,"href","https://arxiv.org/abs/2005.11401"),r(Ds,"rel","nofollow"),r(Dt,"id","more-technical-aspects"),r(Dt,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),r(Dt,"href","#more-technical-aspects"),r(We,"class","relative group"),r(Ht,"id","full-vs-sparse-attention"),r(Ht,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),r(Ht,"href","#full-vs-sparse-attention"),r(Je,"class","relative group"),r(Io,"id","lsh-attention"),r(Go,"href","#reformer"),r(No,"id","local-attention"),r(Fo,"href","#longformer"),r(Ct,"scale","50 %"),r(Ct,"align","center"),c(Ct.src,By="https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/local_attention_mask.png")||r(Ct,"src",By),r(zt,"id","other-tricks"),r(zt,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),r(zt,"href","#other-tricks"),r(Qe,"class","relative group"),r(Co,"id","axial-pos-encoding"),r(jo,"href","#reformer"),Pd.a=Ld,Sd.a=Rd,Id.a=qd,Gd.a=Nd,Bd.a=Fd,Dd.a=Hd,Cd.a=zd,jd.a=Ud,Xd.a=Vd,Yd.a=Zd,Kd.a=Od},m(e,n){a(document.head,k),o(e,P,n),o(e,x,n),a(x,A),a(A,kh),v(Yt,kh,null),a(x,Cv),a(x,xh),a(xh,zv),o(e,Ip,n),o(e,L,n),a(L,jv),a(L,Zt),a(Zt,Uv),a(L,Xv),a(L,Kt),a(Kt,Vv),a(L,Yv),a(L,Ot),a(Ot,Zv),a(L,Kv),o(e,qp,n),o(e,Ys,n),a(Ys,Ov),o(e,Gp,n),o(e,T,n),a(T,Ah),a(Ah,Zs),a(Zs,Wv),a(T,Jv),a(T,Th),a(Th,Ks),a(Ks,Qv),a(T,eg),a(T,$h),a($h,Os),a(Os,tg),a(T,ag),a(T,Mh),a(Mh,Ws),a(Ws,sg),a(T,ig),a(T,Ph),a(Ph,Js),a(Js,rg),o(e,Np,n),o(e,M,n),o(e,Bp,n),o(e,Qs,n),a(Qs,lg),o(e,Fp,n),o(e,ei,n),a(ei,ng),o(e,Dp,n),o(e,ti,n),a(ti,og),o(e,Hp,n),o(e,ai,n),a(ai,hg),o(e,Cp,n),o(e,si,n),a(si,fg),o(e,zp,n),o(e,ii,n),o(e,jp,n),o(e,ge,n),a(ge,et),a(et,Lh),v(Wt,Lh,null),a(ge,pg),a(ge,Sh),a(Sh,mg),o(e,Up,n),o(e,ri,n),a(ri,cg),o(e,Xp,n),v(Jt,e,n),o(e,Vp,n),o(e,be,n),a(be,tt),a(tt,Rh),v(Qt,Rh,null),a(be,ug),a(be,Ih),a(Ih,dg),o(e,Yp,n),o(e,q,n),a(q,li),a(li,ni),a(q,vg),a(q,oi),a(oi,hi),a(q,gg),a(q,fi),a(fi,pi),o(e,Zp,n),o(e,ea,n),a(ea,ta),a(ta,bg),a(ea,_g),o(e,Kp,n),o(e,mi,n),a(mi,wg),o(e,Op,n),o(e,ci,n),a(ci,Eg),o(e,Wp,n),o(e,_e,n),a(_e,at),a(at,qh),v(aa,qh,null),a(_e,yg),a(_e,Gh),a(Gh,kg),o(e,Jp,n),o(e,G,n),a(G,ui),a(ui,di),a(G,xg),a(G,vi),a(vi,gi),a(G,Ag),a(G,bi),a(bi,_i),o(e,Qp,n),o(e,sa,n),a(sa,ia),a(ia,Tg),a(sa,$g),o(e,em,n),o(e,wi,n),a(wi,Mg),o(e,tm,n),o(e,Ei,n),a(Ei,Pg),o(e,am,n),o(e,we,n),a(we,st),a(st,Nh),v(ra,Nh,null),a(we,Lg),a(we,Bh),a(Bh,Sg),o(e,sm,n),o(e,N,n),a(N,yi),a(yi,ki),a(N,Rg),a(N,xi),a(xi,Ai),a(N,Ig),a(N,Ti),a(Ti,$i),o(e,im,n),o(e,la,n),a(la,na),a(na,qg),a(la,Gg),o(e,rm,n),o(e,Mi,n),a(Mi,Ng),o(e,lm,n),o(e,Pi,n),a(Pi,Bg),o(e,nm,n),o(e,Ee,n),a(Ee,it),a(it,Fh),v(oa,Fh,null),a(Ee,Fg),a(Ee,Dh),a(Dh,Dg),o(e,om,n),o(e,B,n),a(B,Li),a(Li,Si),a(B,Hg),a(B,Ri),a(Ri,Ii),a(B,Cg),a(B,qi),a(qi,Gi),o(e,hm,n),o(e,ha,n),a(ha,fa),a(fa,zg),a(ha,jg),o(e,fm,n),o(e,Ni,n),a(Ni,Ug),o(e,pm,n),o(e,Bi,n),a(Bi,Xg),o(e,mm,n),o(e,Fi,n),a(Fi,Vg),o(e,cm,n),o(e,Di,n),a(Di,Yg),o(e,um,n),o(e,Hi,n),o(e,dm,n),o(e,ye,n),a(ye,rt),a(rt,Hh),v(pa,Hh,null),a(ye,Zg),a(ye,Ch),a(Ch,Kg),o(e,vm,n),o(e,F,n),a(F,Ci),a(Ci,zi),a(F,Og),a(F,ji),a(ji,Ui),a(F,Wg),a(F,Xi),a(Xi,Vi),o(e,gm,n),o(e,ma,n),a(ma,ca),a(ca,Jg),a(ma,Qg),o(e,bm,n),o(e,Yi,n),a(Yi,e1),o(e,_m,n),o(e,S,n),a(S,ua),a(ua,t1),a(ua,Zi),a(Zi,a1),a(ua,s1),a(S,i1),a(S,da),a(da,r1),a(da,Ki),a(Ki,l1),a(da,n1),a(S,o1),a(S,zh),a(zh,h1),a(S,f1),a(S,jh),a(jh,p1),o(e,wm,n),o(e,Oi,n),a(Oi,m1),o(e,Em,n),v(lt,e,n),o(e,ym,n),o(e,Wi,n),a(Wi,c1),o(e,km,n),o(e,ke,n),a(ke,nt),a(nt,Uh),v(va,Uh,null),a(ke,u1),a(ke,Xh),a(Xh,d1),o(e,xm,n),o(e,D,n),a(D,Ji),a(Ji,Qi),a(D,v1),a(D,er),a(er,tr),a(D,g1),a(D,ar),a(ar,sr),o(e,Am,n),o(e,ga,n),a(ga,ba),a(ba,b1),a(ga,_1),o(e,Tm,n),o(e,ir,n),a(ir,w1),o(e,$m,n),o(e,rr,n),a(rr,E1),o(e,Mm,n),o(e,lr,n),a(lr,y1),o(e,Pm,n),o(e,nr,n),o(e,Lm,n),o(e,xe,n),a(xe,ot),a(ot,Vh),v(_a,Vh,null),a(xe,k1),a(xe,Yh),a(Yh,x1),o(e,Sm,n),o(e,or,n),a(or,A1),o(e,Rm,n),v(wa,e,n),o(e,Im,n),o(e,Ae,n),a(Ae,ht),a(ht,Zh),v(Ea,Zh,null),a(Ae,T1),a(Ae,Kh),a(Kh,$1),o(e,qm,n),o(e,H,n),a(H,hr),a(hr,fr),a(H,M1),a(H,pr),a(pr,mr),a(H,P1),a(H,cr),a(cr,ur),o(e,Gm,n),o(e,ya,n),a(ya,ka),a(ka,L1),a(ya,S1),o(e,Nm,n),o(e,dr,n),a(dr,R1),o(e,Bm,n),o(e,oe,n),a(oe,Oh),a(Oh,I1),a(oe,q1),a(oe,Wh),a(Wh,G1),a(oe,N1),a(oe,Jh),a(Jh,B1),o(e,Fm,n),o(e,vr,n),a(vr,F1),o(e,Dm,n),o(e,gr,n),a(gr,D1),o(e,Hm,n),o(e,Te,n),a(Te,ft),a(ft,Qh),v(xa,Qh,null),a(Te,H1),a(Te,ef),a(ef,C1),o(e,Cm,n),o(e,C,n),a(C,br),a(br,_r),a(C,z1),a(C,wr),a(wr,Er),a(C,j1),a(C,yr),a(yr,kr),o(e,zm,n),o(e,Aa,n),a(Aa,Ta),a(Ta,U1),a(Aa,X1),o(e,jm,n),o(e,xr,n),a(xr,V1),o(e,Um,n),o(e,he,n),a(he,tf),a(tf,Y1),a(he,Z1),a(he,af),a(af,K1),a(he,O1),a(he,sf),a(sf,W1),o(e,Xm,n),o(e,Ar,n),a(Ar,J1),o(e,Vm,n),o(e,$e,n),a($e,pt),a(pt,rf),v($a,rf,null),a($e,Q1),a($e,lf),a(lf,eb),o(e,Ym,n),o(e,z,n),a(z,Tr),a(Tr,$r),a(z,tb),a(z,Mr),a(Mr,Pr),a(z,ab),a(z,Lr),a(Lr,Sr),o(e,Zm,n),o(e,Ma,n),a(Ma,Pa),a(Pa,sb),a(Ma,ib),o(e,Km,n),o(e,Rr,n),a(Rr,rb),o(e,Om,n),o(e,R,n),a(R,nf),a(nf,lb),a(R,nb),a(R,of),a(of,ob),a(R,hb),a(R,hf),a(hf,fb),a(R,pb),a(R,ff),a(ff,mb),o(e,Wm,n),o(e,Ir,n),a(Ir,cb),o(e,Jm,n),o(e,Me,n),a(Me,mt),a(mt,pf),v(La,pf,null),a(Me,ub),a(Me,mf),a(mf,db),o(e,Qm,n),o(e,j,n),a(j,qr),a(qr,Gr),a(j,vb),a(j,Nr),a(Nr,Br),a(j,gb),a(j,Fr),a(Fr,Dr),o(e,ec,n),o(e,Sa,n),a(Sa,Ra),a(Ra,bb),a(Sa,_b),o(e,tc,n),o(e,Hr,n),a(Hr,wb),o(e,ac,n),o(e,fe,n),a(fe,cf),a(cf,Eb),a(fe,yb),a(fe,uf),a(uf,kb),a(fe,xb),a(fe,df),a(df,Ab),o(e,sc,n),o(e,Cr,n),a(Cr,Tb),o(e,ic,n),o(e,Pe,n),a(Pe,ct),a(ct,vf),v(Ia,vf,null),a(Pe,$b),a(Pe,gf),a(gf,Mb),o(e,rc,n),o(e,U,n),a(U,zr),a(zr,jr),a(U,Pb),a(U,Ur),a(Ur,Xr),a(U,Lb),a(U,Vr),a(Vr,Yr),o(e,lc,n),o(e,qa,n),a(qa,Ga),a(Ga,Sb),a(qa,Rb),o(e,nc,n),o(e,Zr,n),a(Zr,Ib),o(e,oc,n),o(e,Kr,n),a(Kr,qb),o(e,hc,n),o(e,Le,n),a(Le,ut),a(ut,bf),v(Na,bf,null),a(Le,Gb),a(Le,_f),a(_f,Nb),o(e,fc,n),o(e,X,n),a(X,Or),a(Or,Wr),a(X,Bb),a(X,Jr),a(Jr,Qr),a(X,Fb),a(X,el),a(el,tl),o(e,pc,n),o(e,Ba,n),a(Ba,Fa),a(Fa,Db),a(Ba,Hb),o(e,mc,n),o(e,al,n),a(al,Cb),o(e,cc,n),o(e,pe,n),a(pe,wf),a(wf,zb),a(pe,jb),a(pe,Ef),a(Ef,Ub),a(pe,Xb),a(pe,yf),a(yf,Vb),o(e,uc,n),o(e,I,n),a(I,Yb),a(I,kf),a(kf,Zb),a(I,Kb),a(I,xf),a(xf,Ob),a(I,Wb),a(I,Af),a(Af,Jb),a(I,Qb),o(e,dc,n),o(e,sl,n),a(sl,e_),o(e,vc,n),o(e,Se,n),a(Se,dt),a(dt,Tf),v(Da,Tf,null),a(Se,t_),a(Se,$f),a($f,a_),o(e,gc,n),o(e,V,n),a(V,il),a(il,rl),a(V,s_),a(V,ll),a(ll,nl),a(V,i_),a(V,ol),a(ol,hl),o(e,bc,n),o(e,Ha,n),a(Ha,Ca),a(Ca,r_),a(Ha,l_),o(e,_c,n),o(e,fl,n),a(fl,n_),o(e,wc,n),o(e,pl,n),a(pl,o_),o(e,Ec,n),o(e,Re,n),a(Re,vt),a(vt,Mf),v(za,Mf,null),a(Re,h_),a(Re,Pf),a(Pf,f_),o(e,yc,n),o(e,Y,n),a(Y,ml),a(ml,cl),a(Y,p_),a(Y,ul),a(ul,dl),a(Y,m_),a(Y,vl),a(vl,gl),o(e,kc,n),o(e,ja,n),a(ja,Ua),a(Ua,c_),a(ja,u_),o(e,xc,n),o(e,bl,n),a(bl,d_),o(e,Ac,n),o(e,_l,n),a(_l,v_),o(e,Tc,n),o(e,Ie,n),a(Ie,gt),a(gt,Lf),v(Xa,Lf,null),a(Ie,g_),a(Ie,Sf),a(Sf,b_),o(e,$c,n),o(e,Z,n),a(Z,wl),a(wl,El),a(Z,__),a(Z,yl),a(yl,kl),a(Z,w_),a(Z,xl),a(xl,Al),o(e,Mc,n),o(e,Va,n),a(Va,Ya),a(Ya,E_),a(Va,y_),o(e,Pc,n),o(e,Tl,n),a(Tl,k_),o(e,Lc,n),o(e,$l,n),a($l,x_),o(e,Sc,n),o(e,qe,n),a(qe,bt),a(bt,Rf),v(Za,Rf,null),a(qe,A_),a(qe,If),a(If,T_),o(e,Rc,n),o(e,K,n),a(K,Ml),a(Ml,Pl),a(K,$_),a(K,Ll),a(Ll,Sl),a(K,M_),a(K,Rl),a(Rl,Il),o(e,Ic,n),o(e,Ka,n),a(Ka,Oa),a(Oa,P_),a(Ka,L_),o(e,qc,n),o(e,ql,n),a(ql,S_),o(e,Gc,n),o(e,Gl,n),a(Gl,R_),o(e,Nc,n),o(e,Nl,n),a(Nl,I_),o(e,Bc,n),o(e,Bl,n),a(Bl,q_),o(e,Fc,n),o(e,Fl,n),o(e,Dc,n),o(e,Ge,n),a(Ge,_t),a(_t,qf),v(Wa,qf,null),a(Ge,G_),a(Ge,Gf),a(Gf,N_),o(e,Hc,n),o(e,O,n),a(O,Dl),a(Dl,Hl),a(O,B_),a(O,Cl),a(Cl,zl),a(O,F_),a(O,jl),a(jl,Ul),o(e,Cc,n),o(e,Ja,n),a(Ja,Qa),a(Qa,D_),a(Ja,H_),o(e,zc,n),o(e,wt,n),a(wt,C_),a(wt,Xl),a(Xl,z_),a(wt,j_),o(e,jc,n),o(e,Vl,n),a(Vl,U_),o(e,Uc,n),v(Et,e,n),o(e,Xc,n),o(e,Yl,n),a(Yl,X_),o(e,Vc,n),o(e,Zl,n),o(e,Yc,n),o(e,Ne,n),a(Ne,yt),a(yt,Nf),v(es,Nf,null),a(Ne,V_),a(Ne,Bf),a(Bf,Y_),o(e,Zc,n),o(e,Kl,n),a(Kl,Z_),o(e,Kc,n),v(ts,e,n),o(e,Oc,n),o(e,Be,n),a(Be,kt),a(kt,Ff),v(as,Ff,null),a(Be,K_),a(Be,Df),a(Df,O_),o(e,Wc,n),o(e,W,n),a(W,Ol),a(Ol,Wl),a(W,W_),a(W,Jl),a(Jl,Ql),a(W,J_),a(W,en),a(en,tn),o(e,Jc,n),o(e,ss,n),a(ss,is),a(is,Q_),a(ss,ew),o(e,Qc,n),o(e,an,n),a(an,tw),o(e,eu,n),o(e,$,n),a($,Hf),a(Hf,aw),a($,sw),a($,Cf),a(Cf,iw),a($,rw),a($,zf),a(zf,lw),a($,nw),a($,jf),a(jf,ow),a($,hw),a($,Uf),a(Uf,fw),o(e,tu,n),o(e,sn,n),a(sn,pw),o(e,au,n),o(e,Fe,n),a(Fe,xt),a(xt,Xf),v(rs,Xf,null),a(Fe,mw),a(Fe,Vf),a(Vf,cw),o(e,su,n),o(e,J,n),a(J,rn),a(rn,ln),a(J,uw),a(J,nn),a(nn,on),a(J,dw),a(J,hn),a(hn,fn),o(e,iu,n),o(e,ls,n),a(ls,ns),a(ns,vw),a(ls,gw),o(e,ru,n),o(e,pn,n),a(pn,bw),o(e,lu,n),o(e,At,n),a(At,Yf),a(Yf,_w),a(At,ww),a(At,Zf),a(Zf,Ew),o(e,nu,n),o(e,mn,n),a(mn,yw),o(e,ou,n),o(e,cn,n),a(cn,kw),o(e,hu,n),o(e,De,n),a(De,Tt),a(Tt,Kf),v(os,Kf,null),a(De,xw),a(De,Of),a(Of,Aw),o(e,fu,n),o(e,Q,n),a(Q,un),a(un,dn),a(Q,Tw),a(Q,vn),a(vn,gn),a(Q,$w),a(Q,bn),a(bn,_n),o(e,pu,n),o(e,hs,n),a(hs,fs),a(fs,Mw),a(hs,Pw),o(e,mu,n),o(e,wn,n),a(wn,Lw),o(e,cu,n),o(e,En,n),a(En,Sw),o(e,uu,n),o(e,He,n),a(He,$t),a($t,Wf),v(ps,Wf,null),a(He,Rw),a(He,Jf),a(Jf,Iw),o(e,du,n),o(e,ee,n),a(ee,yn),a(yn,kn),a(ee,qw),a(ee,xn),a(xn,An),a(ee,Gw),a(ee,Tn),a(Tn,$n),o(e,vu,n),o(e,ms,n),a(ms,cs),a(cs,Nw),a(ms,Bw),o(e,gu,n),o(e,Mn,n),a(Mn,Fw),o(e,bu,n),o(e,Pn,n),a(Pn,Dw),o(e,_u,n),o(e,Ln,n),a(Ln,Hw),o(e,wu,n),o(e,Sn,n),a(Sn,Cw),o(e,Eu,n),o(e,Rn,n),a(Rn,zw),o(e,yu,n),o(e,Ce,n),a(Ce,Mt),a(Mt,Qf),v(us,Qf,null),a(Ce,jw),a(Ce,ep),a(ep,Uw),o(e,ku,n),o(e,te,n),a(te,In),a(In,qn),a(te,Xw),a(te,Gn),a(Gn,Nn),a(te,Vw),a(te,Bn),a(Bn,Fn),o(e,xu,n),o(e,ds,n),a(ds,vs),a(vs,Yw),a(ds,Zw),o(e,Au,n),o(e,Dn,n),a(Dn,Kw),o(e,Tu,n),o(e,Hn,n),a(Hn,Ow),o(e,$u,n),o(e,ze,n),a(ze,Pt),a(Pt,tp),v(gs,tp,null),a(ze,Ww),a(ze,ap),a(ap,Jw),o(e,Mu,n),o(e,ae,n),a(ae,Cn),a(Cn,zn),a(ae,Qw),a(ae,jn),a(jn,Un),a(ae,e2),a(ae,Xn),a(Xn,Vn),o(e,Pu,n),o(e,bs,n),a(bs,_s),a(_s,t2),a(bs,a2),o(e,Lu,n),o(e,Yn,n),a(Yn,s2),o(e,Su,n),o(e,Zn,n),a(Zn,i2),o(e,Ru,n),o(e,Lt,n),a(Lt,r2),a(Lt,ws),a(ws,l2),a(Lt,n2),o(e,Iu,n),o(e,me,n),a(me,o2),a(me,Es),a(Es,h2),a(me,f2),a(me,sp),a(sp,p2),a(me,m2),o(e,qu,n),o(e,je,n),a(je,St),a(St,ip),v(ys,ip,null),a(je,c2),a(je,rp),a(rp,u2),o(e,Gu,n),o(e,se,n),a(se,Kn),a(Kn,On),a(se,d2),a(se,Wn),a(Wn,Jn),a(se,v2),a(se,Qn),a(Qn,eo),o(e,Nu,n),o(e,ks,n),a(ks,xs),a(xs,g2),a(ks,b2),o(e,Bu,n),o(e,ce,n),a(ce,_2),a(ce,lp),a(lp,w2),a(ce,E2),a(ce,np),a(np,y2),a(ce,k2),o(e,Fu,n),o(e,to,n),a(to,x2),o(e,Du,n),o(e,Ue,n),a(Ue,Rt),a(Rt,op),v(As,op,null),a(Ue,A2),a(Ue,hp),a(hp,T2),o(e,Hu,n),o(e,ie,n),a(ie,ao),a(ao,so),a(ie,$2),a(ie,io),a(io,ro),a(ie,M2),a(ie,lo),a(lo,no),o(e,Cu,n),o(e,Ts,n),a(Ts,$s),a($s,P2),a(Ts,L2),o(e,zu,n),o(e,It,n),a(It,S2),a(It,Ms),a(Ms,R2),a(It,I2),o(e,ju,n),o(e,oo,n),a(oo,q2),o(e,Uu,n),o(e,ho,n),o(e,Xu,n),o(e,Xe,n),a(Xe,qt),a(qt,fp),v(Ps,fp,null),a(Xe,G2),a(Xe,pp),a(pp,N2),o(e,Vu,n),o(e,fo,n),a(fo,B2),o(e,Yu,n),o(e,Ve,n),a(Ve,Gt),a(Gt,mp),v(Ls,mp,null),a(Ve,F2),a(Ve,cp),a(cp,D2),o(e,Zu,n),o(e,Ss,n),a(Ss,Rs),a(Rs,H2),a(Ss,C2),o(e,Ku,n),o(e,po,n),a(po,z2),o(e,Ou,n),o(e,mo,n),a(mo,j2),o(e,Wu,n),o(e,co,n),a(co,U2),o(e,Ju,n),o(e,uo,n),o(e,Qu,n),o(e,Ye,n),a(Ye,Nt),a(Nt,up),v(Is,up,null),a(Ye,X2),a(Ye,dp),a(dp,V2),o(e,ed,n),o(e,vo,n),a(vo,Y2),o(e,td,n),o(e,Ze,n),a(Ze,Bt),a(Bt,vp),v(qs,vp,null),a(Ze,Z2),a(Ze,gp),a(gp,K2),o(e,ad,n),o(e,re,n),a(re,go),a(go,bo),a(re,O2),a(re,_o),a(_o,wo),a(re,W2),a(re,Eo),a(Eo,yo),o(e,sd,n),o(e,Gs,n),a(Gs,Ns),a(Ns,J2),a(Gs,Q2),o(e,id,n),o(e,ko,n),a(ko,e0),o(e,rd,n),o(e,xo,n),a(xo,t0),o(e,ld,n),o(e,ue,n),a(ue,bp),a(bp,a0),a(ue,s0),a(ue,_p),a(_p,i0),a(ue,r0),a(ue,wp),a(wp,l0),o(e,nd,n),o(e,Ao,n),a(Ao,n0),o(e,od,n),o(e,Ke,n),a(Ke,Ft),a(Ft,Ep),v(Bs,Ep,null),a(Ke,o0),a(Ke,yp),a(yp,h0),o(e,hd,n),o(e,Oe,n),a(Oe,To),a(To,$o),a(Oe,f0),a(Oe,Mo),a(Mo,Po),o(e,fd,n),o(e,Fs,n),a(Fs,Ds),a(Ds,p0),a(Fs,m0),o(e,pd,n),o(e,Lo,n),a(Lo,c0),o(e,md,n),o(e,So,n),a(So,u0),o(e,cd,n),o(e,We,n),a(We,Dt),a(Dt,kp),v(Hs,kp,null),a(We,d0),a(We,xp),a(xp,v0),o(e,ud,n),o(e,Je,n),a(Je,Ht),a(Ht,Ap),v(Cs,Ap,null),a(Je,g0),a(Je,Tp),a(Tp,b0),o(e,dd,n),o(e,Ro,n),a(Ro,_0),o(e,vd,n),o(e,Io,n),o(e,gd,n),o(e,qo,n),a(qo,$p),a($p,w0),o(e,bd,n),o(e,zs,n),a(zs,Go),a(Go,E0),a(zs,y0),o(e,_d,n),o(e,No,n),o(e,wd,n),o(e,Bo,n),a(Bo,Mp),a(Mp,k0),o(e,Ed,n),o(e,js,n),a(js,Fo),a(Fo,x0),a(js,A0),o(e,yd,n),o(e,Do,n),a(Do,T0),o(e,kd,n),o(e,Ct,n),o(e,xd,n),o(e,Ho,n),a(Ho,$0),o(e,Ad,n),o(e,Qe,n),a(Qe,zt),a(zt,Pp),v(Us,Pp,null),a(Qe,M0),a(Qe,Lp),a(Lp,P0),o(e,Td,n),o(e,Co,n),o(e,$d,n),o(e,zo,n),a(zo,Sp),a(Sp,L0),o(e,Md,n),o(e,E,n),a(E,jo),a(jo,S0),a(E,R0),Pd.m(y9,E),a(E,Ld),Sd.m(k9,E),a(E,Rd),Id.m(x9,E),a(E,qd),Gd.m(A9,E),a(E,Nd),Bd.m(T9,E),a(E,Fd),Dd.m($9,E),a(E,Hd),Cd.m(M9,E),a(E,zd),jd.m(P9,E),a(E,Ud),Xd.m(L9,E),a(E,Vd),Yd.m(S9,E),a(E,Zd),Kd.m(R9,E),a(E,Od),Wd=!0},p(e,[n]){const Xs={};n&2&&(Xs.$$scope={dirty:n,ctx:e}),lt.$set(Xs);const Rp={};n&2&&(Rp.$$scope={dirty:n,ctx:e}),Et.$set(Rp)},i(e){Wd||(g(Yt.$$.fragment,e),g(Wt.$$.fragment,e),g(Jt.$$.fragment,e),g(Qt.$$.fragment,e),g(aa.$$.fragment,e),g(ra.$$.fragment,e),g(oa.$$.fragment,e),g(pa.$$.fragment,e),g(lt.$$.fragment,e),g(va.$$.fragment,e),g(_a.$$.fragment,e),g(wa.$$.fragment,e),g(Ea.$$.fragment,e),g(xa.$$.fragment,e),g($a.$$.fragment,e),g(La.$$.fragment,e),g(Ia.$$.fragment,e),g(Na.$$.fragment,e),g(Da.$$.fragment,e),g(za.$$.fragment,e),g(Xa.$$.fragment,e),g(Za.$$.fragment,e),g(Wa.$$.fragment,e),g(Et.$$.fragment,e),g(es.$$.fragment,e),g(ts.$$.fragment,e),g(as.$$.fragment,e),g(rs.$$.fragment,e),g(os.$$.fragment,e),g(ps.$$.fragment,e),g(us.$$.fragment,e),g(gs.$$.fragment,e),g(ys.$$.fragment,e),g(As.$$.fragment,e),g(Ps.$$.fragment,e),g(Ls.$$.fragment,e),g(Is.$$.fragment,e),g(qs.$$.fragment,e),g(Bs.$$.fragment,e),g(Hs.$$.fragment,e),g(Cs.$$.fragment,e),g(Us.$$.fragment,e),Wd=!0)},o(e){b(Yt.$$.fragment,e),b(Wt.$$.fragment,e),b(Jt.$$.fragment,e),b(Qt.$$.fragment,e),b(aa.$$.fragment,e),b(ra.$$.fragment,e),b(oa.$$.fragment,e),b(pa.$$.fragment,e),b(lt.$$.fragment,e),b(va.$$.fragment,e),b(_a.$$.fragment,e),b(wa.$$.fragment,e),b(Ea.$$.fragment,e),b(xa.$$.fragment,e),b($a.$$.fragment,e),b(La.$$.fragment,e),b(Ia.$$.fragment,e),b(Na.$$.fragment,e),b(Da.$$.fragment,e),b(za.$$.fragment,e),b(Xa.$$.fragment,e),b(Za.$$.fragment,e),b(Wa.$$.fragment,e),b(Et.$$.fragment,e),b(es.$$.fragment,e),b(ts.$$.fragment,e),b(as.$$.fragment,e),b(rs.$$.fragment,e),b(os.$$.fragment,e),b(ps.$$.fragment,e),b(us.$$.fragment,e),b(gs.$$.fragment,e),b(ys.$$.fragment,e),b(As.$$.fragment,e),b(Ps.$$.fragment,e),b(Ls.$$.fragment,e),b(Is.$$.fragment,e),b(qs.$$.fragment,e),b(Bs.$$.fragment,e),b(Hs.$$.fragment,e),b(Cs.$$.fragment,e),b(Us.$$.fragment,e),Wd=!1},d(e){t(k),e&&t(P),e&&t(x),_(Yt),e&&t(Ip),e&&t(L),e&&t(qp),e&&t(Ys),e&&t(Gp),e&&t(T),e&&t(Np),e&&t(M),e&&t(Bp),e&&t(Qs),e&&t(Fp),e&&t(ei),e&&t(Dp),e&&t(ti),e&&t(Hp),e&&t(ai),e&&t(Cp),e&&t(si),e&&t(zp),e&&t(ii),e&&t(jp),e&&t(ge),_(Wt),e&&t(Up),e&&t(ri),e&&t(Xp),_(Jt,e),e&&t(Vp),e&&t(be),_(Qt),e&&t(Yp),e&&t(q),e&&t(Zp),e&&t(ea),e&&t(Kp),e&&t(mi),e&&t(Op),e&&t(ci),e&&t(Wp),e&&t(_e),_(aa),e&&t(Jp),e&&t(G),e&&t(Qp),e&&t(sa),e&&t(em),e&&t(wi),e&&t(tm),e&&t(Ei),e&&t(am),e&&t(we),_(ra),e&&t(sm),e&&t(N),e&&t(im),e&&t(la),e&&t(rm),e&&t(Mi),e&&t(lm),e&&t(Pi),e&&t(nm),e&&t(Ee),_(oa),e&&t(om),e&&t(B),e&&t(hm),e&&t(ha),e&&t(fm),e&&t(Ni),e&&t(pm),e&&t(Bi),e&&t(mm),e&&t(Fi),e&&t(cm),e&&t(Di),e&&t(um),e&&t(Hi),e&&t(dm),e&&t(ye),_(pa),e&&t(vm),e&&t(F),e&&t(gm),e&&t(ma),e&&t(bm),e&&t(Yi),e&&t(_m),e&&t(S),e&&t(wm),e&&t(Oi),e&&t(Em),_(lt,e),e&&t(ym),e&&t(Wi),e&&t(km),e&&t(ke),_(va),e&&t(xm),e&&t(D),e&&t(Am),e&&t(ga),e&&t(Tm),e&&t(ir),e&&t($m),e&&t(rr),e&&t(Mm),e&&t(lr),e&&t(Pm),e&&t(nr),e&&t(Lm),e&&t(xe),_(_a),e&&t(Sm),e&&t(or),e&&t(Rm),_(wa,e),e&&t(Im),e&&t(Ae),_(Ea),e&&t(qm),e&&t(H),e&&t(Gm),e&&t(ya),e&&t(Nm),e&&t(dr),e&&t(Bm),e&&t(oe),e&&t(Fm),e&&t(vr),e&&t(Dm),e&&t(gr),e&&t(Hm),e&&t(Te),_(xa),e&&t(Cm),e&&t(C),e&&t(zm),e&&t(Aa),e&&t(jm),e&&t(xr),e&&t(Um),e&&t(he),e&&t(Xm),e&&t(Ar),e&&t(Vm),e&&t($e),_($a),e&&t(Ym),e&&t(z),e&&t(Zm),e&&t(Ma),e&&t(Km),e&&t(Rr),e&&t(Om),e&&t(R),e&&t(Wm),e&&t(Ir),e&&t(Jm),e&&t(Me),_(La),e&&t(Qm),e&&t(j),e&&t(ec),e&&t(Sa),e&&t(tc),e&&t(Hr),e&&t(ac),e&&t(fe),e&&t(sc),e&&t(Cr),e&&t(ic),e&&t(Pe),_(Ia),e&&t(rc),e&&t(U),e&&t(lc),e&&t(qa),e&&t(nc),e&&t(Zr),e&&t(oc),e&&t(Kr),e&&t(hc),e&&t(Le),_(Na),e&&t(fc),e&&t(X),e&&t(pc),e&&t(Ba),e&&t(mc),e&&t(al),e&&t(cc),e&&t(pe),e&&t(uc),e&&t(I),e&&t(dc),e&&t(sl),e&&t(vc),e&&t(Se),_(Da),e&&t(gc),e&&t(V),e&&t(bc),e&&t(Ha),e&&t(_c),e&&t(fl),e&&t(wc),e&&t(pl),e&&t(Ec),e&&t(Re),_(za),e&&t(yc),e&&t(Y),e&&t(kc),e&&t(ja),e&&t(xc),e&&t(bl),e&&t(Ac),e&&t(_l),e&&t(Tc),e&&t(Ie),_(Xa),e&&t($c),e&&t(Z),e&&t(Mc),e&&t(Va),e&&t(Pc),e&&t(Tl),e&&t(Lc),e&&t($l),e&&t(Sc),e&&t(qe),_(Za),e&&t(Rc),e&&t(K),e&&t(Ic),e&&t(Ka),e&&t(qc),e&&t(ql),e&&t(Gc),e&&t(Gl),e&&t(Nc),e&&t(Nl),e&&t(Bc),e&&t(Bl),e&&t(Fc),e&&t(Fl),e&&t(Dc),e&&t(Ge),_(Wa),e&&t(Hc),e&&t(O),e&&t(Cc),e&&t(Ja),e&&t(zc),e&&t(wt),e&&t(jc),e&&t(Vl),e&&t(Uc),_(Et,e),e&&t(Xc),e&&t(Yl),e&&t(Vc),e&&t(Zl),e&&t(Yc),e&&t(Ne),_(es),e&&t(Zc),e&&t(Kl),e&&t(Kc),_(ts,e),e&&t(Oc),e&&t(Be),_(as),e&&t(Wc),e&&t(W),e&&t(Jc),e&&t(ss),e&&t(Qc),e&&t(an),e&&t(eu),e&&t($),e&&t(tu),e&&t(sn),e&&t(au),e&&t(Fe),_(rs),e&&t(su),e&&t(J),e&&t(iu),e&&t(ls),e&&t(ru),e&&t(pn),e&&t(lu),e&&t(At),e&&t(nu),e&&t(mn),e&&t(ou),e&&t(cn),e&&t(hu),e&&t(De),_(os),e&&t(fu),e&&t(Q),e&&t(pu),e&&t(hs),e&&t(mu),e&&t(wn),e&&t(cu),e&&t(En),e&&t(uu),e&&t(He),_(ps),e&&t(du),e&&t(ee),e&&t(vu),e&&t(ms),e&&t(gu),e&&t(Mn),e&&t(bu),e&&t(Pn),e&&t(_u),e&&t(Ln),e&&t(wu),e&&t(Sn),e&&t(Eu),e&&t(Rn),e&&t(yu),e&&t(Ce),_(us),e&&t(ku),e&&t(te),e&&t(xu),e&&t(ds),e&&t(Au),e&&t(Dn),e&&t(Tu),e&&t(Hn),e&&t($u),e&&t(ze),_(gs),e&&t(Mu),e&&t(ae),e&&t(Pu),e&&t(bs),e&&t(Lu),e&&t(Yn),e&&t(Su),e&&t(Zn),e&&t(Ru),e&&t(Lt),e&&t(Iu),e&&t(me),e&&t(qu),e&&t(je),_(ys),e&&t(Gu),e&&t(se),e&&t(Nu),e&&t(ks),e&&t(Bu),e&&t(ce),e&&t(Fu),e&&t(to),e&&t(Du),e&&t(Ue),_(As),e&&t(Hu),e&&t(ie),e&&t(Cu),e&&t(Ts),e&&t(zu),e&&t(It),e&&t(ju),e&&t(oo),e&&t(Uu),e&&t(ho),e&&t(Xu),e&&t(Xe),_(Ps),e&&t(Vu),e&&t(fo),e&&t(Yu),e&&t(Ve),_(Ls),e&&t(Zu),e&&t(Ss),e&&t(Ku),e&&t(po),e&&t(Ou),e&&t(mo),e&&t(Wu),e&&t(co),e&&t(Ju),e&&t(uo),e&&t(Qu),e&&t(Ye),_(Is),e&&t(ed),e&&t(vo),e&&t(td),e&&t(Ze),_(qs),e&&t(ad),e&&t(re),e&&t(sd),e&&t(Gs),e&&t(id),e&&t(ko),e&&t(rd),e&&t(xo),e&&t(ld),e&&t(ue),e&&t(nd),e&&t(Ao),e&&t(od),e&&t(Ke),_(Bs),e&&t(hd),e&&t(Oe),e&&t(fd),e&&t(Fs),e&&t(pd),e&&t(Lo),e&&t(md),e&&t(So),e&&t(cd),e&&t(We),_(Hs),e&&t(ud),e&&t(Je),_(Cs),e&&t(dd),e&&t(Ro),e&&t(vd),e&&t(Io),e&&t(gd),e&&t(qo),e&&t(bd),e&&t(zs),e&&t(_d),e&&t(No),e&&t(wd),e&&t(Bo),e&&t(Ed),e&&t(js),e&&t(yd),e&&t(Do),e&&t(kd),e&&t(Ct),e&&t(xd),e&&t(Ho),e&&t(Ad),e&&t(Qe),_(Us),e&&t(Td),e&&t(Co),e&&t($d),e&&t(zo),e&&t(Md),e&&t(E)}}}const C9={local:"summary-of-the-models",sections:[{local:"decoders-or-autoregressive-models",sections:[{local:"original-gpt",title:"Original GPT"},{local:"gpt2",title:"GPT-2"},{local:"ctrl",title:"CTRL"},{local:"transformerxl",title:"Transformer-XL"},{local:"reformer",title:"Reformer"},{local:"xlnet",title:"XLNet"}],title:"Decoders or autoregressive models"},{local:"encoders-or-autoencoding-models",sections:[{local:"bert",title:"BERT"},{local:"albert",title:"ALBERT"},{local:"roberta",title:"RoBERTa"},{local:"distilbert",title:"DistilBERT"},{local:"convbert",title:"ConvBERT"},{local:"xlm",title:"XLM"},{local:"xlmroberta",title:"XLM-RoBERTa"},{local:"flaubert",title:"FlauBERT"},{local:"electra",title:"ELECTRA"},{local:"funnel-transformer",title:"Funnel Transformer"},{local:"longformer",title:"Longformer"}],title:"Encoders or autoencoding models"},{local:"sequencetosequence-models",sections:[{local:"bart",title:"BART"},{local:"pegasus",title:"Pegasus"},{local:"marianmt",title:"MarianMT"},{local:"t5",title:"T5"},{local:"mt5",title:"MT5"},{local:"mbart",title:"MBart"},{local:"prophetnet",title:"ProphetNet"},{local:"xlmprophetnet",title:"XLM-ProphetNet"}],title:"Sequence-to-sequence models"},{local:"multimodal-models",sections:[{local:"mmbt",title:"MMBT"}],title:"Multimodal models"},{local:"retrievalbased-models",sections:[{local:"dpr",title:"DPR"},{local:"rag",title:"RAG"}],title:"Retrieval-based models"},{local:"more-technical-aspects",sections:[{local:"full-vs-sparse-attention",title:"Full vs sparse attention"},{local:"other-tricks",title:"Other tricks"}],title:"More technical aspects"}],title:"Summary of the models"};function z9(Vs){return B9(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class Y9 extends I9{constructor(k){super();q9(this,k,z9,H9,G9,{})}}export{Y9 as default,C9 as metadata};
