import{S as Dc,i as xc,s as yc,e as o,k as l,w as Oc,t as d,M as Cc,c as n,d as t,m as c,a,x as kc,h as r,b as Ne,G as e,g as u,y as wc,L as zc,q as qc,o as Rc,B as Ic,v as Pc}from"../chunks/vendor-hf-doc-builder.js";import{I as Ac}from"../chunks/IconCopyLink-hf-doc-builder.js";function Sc(Hr){let O,To,C,V,Ge,le,dn,Ye,rn,bo,W,ln,Le,cn,sn,Do,m,hn,$e,fn,un,He,_n,pn,Be,mn,gn,xo,X,En,Fe,vn,Tn,yo,y,Z,Me,bn,Dn,Ue,xn,yn,On,g,Je,Cn,kn,je,wn,zn,Ke,qn,Rn,Qe,In,Pn,An,ee,Ve,Sn,Nn,We,Gn,Yn,Oo,te,Ln,Xe,$n,Hn,Co,E,p,Ze,Bn,Fn,et,Mn,Un,tt,Jn,jn,ot,Kn,Qn,nt,Vn,Wn,Xn,v,at,Zn,ea,dt,ta,oa,rt,na,aa,lt,da,ra,la,T,ct,ca,ia,it,sa,ha,st,fa,ua,ht,_a,pa,ma,oe,ft,ga,Ea,ut,va,Ta,ko,b,ba,_t,Da,xa,pt,ya,Oa,mt,Ca,ka,wo,_,wa,gt,za,qa,Et,Ra,Ia,vt,Pa,Aa,Tt,Sa,Na,bt,Ga,Ya,zo,ne,Dt,k,xt,La,$a,yt,Ha,Ba,Ot,Fa,Ma,s,w,Ct,Ua,Ja,kt,ja,Ka,wt,zt,Qa,Va,z,qo,Wa,qt,Xa,Za,ce,Rt,ed,td,od,q,Ro,nd,Io,ad,It,Pt,dd,rd,R,Po,ld,At,cd,id,St,Nt,sd,hd,I,Ao,fd,Gt,ud,_d,Yt,Lt,pd,md,P,$t,gd,Ed,Ht,vd,Td,ie,Bt,bd,Dd,xd,A,So,yd,No,Od,Ft,Mt,Cd,kd,S,Go,wd,Ut,zd,qd,se,Jt,Rd,Id,Pd,N,Yo,Ad,Lo,Sd,jt,Kt,Nd,Gd,G,$o,Yd,Qt,Ld,$d,he,Vt,Hd,Bd,Fd,Y,Ho,Md,Bo,Ud,Wt,Xt,Jd,jd,L,Fo,Kd,Zt,Qd,Vd,eo,Wd,Xd,$,to,Zd,er,oo,tr,or,fe,no,nr,ar,dr,H,Mo,rr,Uo,lr,ao,ro,cr,ir,B,Jo,sr,lo,hr,fr,ue,co,ur,_r,pr,F,jo,mr,Ko,gr,io,so,Er,vr,M,Qo,Tr,ho,br,Dr,fo,xr,yr,U,Vo,Or,uo,Cr,kr,_e,_o,wr,zr,qr,J,Wo,Rr,Xo,Ir,po,mo,Pr,Zo;return le=new Ac({}),{c(){O=o("meta"),To=l(),C=o("h1"),V=o("a"),Ge=o("span"),Oc(le.$$.fragment),dn=l(),Ye=o("span"),rn=d("Padding and truncation"),bo=l(),W=o("p"),ln=d("Batched inputs are often different lengths, so they can\u2019t be converted to fixed-size tensors. Padding and truncation are strategies for dealing with this problem, to create rectangular tensors from batches of varying lengths. Padding adds a special "),Le=o("strong"),cn=d("padding token"),sn=d(" to ensure shorter sequences will have the same length as either the longest sequence in a batch or the maximum length accepted by the model. Truncation works in the other direction by truncating long sequences."),Do=l(),m=o("p"),hn=d("In most cases, padding your batch to the length of the longest sequence and truncating to the maximum length a model can accept works pretty well. However, the API supports more strategies if you need them. The three arguments you need to are: "),$e=o("code"),fn=d("padding"),un=d(", "),He=o("code"),_n=d("truncation"),pn=d(" and "),Be=o("code"),mn=d("max_length"),gn=d("."),xo=l(),X=o("p"),En=d("The "),Fe=o("code"),vn=d("padding"),Tn=d(" argument controls padding. It can be a boolean or a string:"),yo=l(),y=o("ul"),Z=o("li"),Me=o("code"),bn=d("True"),Dn=d(" or "),Ue=o("code"),xn=d("'longest'"),yn=d(`: pad to the longest sequence in the batch (no padding is applied if you only provide
a single sequence).`),On=l(),g=o("li"),Je=o("code"),Cn=d("'max_length'"),kn=d(": pad to a length specified by the "),je=o("code"),wn=d("max_length"),zn=d(` argument or the maximum length accepted
by the model if no `),Ke=o("code"),qn=d("max_length"),Rn=d(" is provided ("),Qe=o("code"),In=d("max_length=None"),Pn=d("). Padding will still be applied if you only provide a single sequence."),An=l(),ee=o("li"),Ve=o("code"),Sn=d("False"),Nn=d(" or "),We=o("code"),Gn=d("'do_not_pad'"),Yn=d(": no padding is applied. This is the default behavior."),Oo=l(),te=o("p"),Ln=d("The "),Xe=o("code"),$n=d("truncation"),Hn=d(" argument controls truncation. It can be a boolean or a string:"),Co=l(),E=o("ul"),p=o("li"),Ze=o("code"),Bn=d("True"),Fn=d(" or "),et=o("code"),Mn=d("'longest_first'"),Un=d(": truncate to a maximum length specified by the "),tt=o("code"),Jn=d("max_length"),jn=d(` argument or
the maximum length accepted by the model if no `),ot=o("code"),Kn=d("max_length"),Qn=d(" is provided ("),nt=o("code"),Vn=d("max_length=None"),Wn=d(`). This will
truncate token by token, removing a token from the longest sequence in the pair until the proper length is
reached.`),Xn=l(),v=o("li"),at=o("code"),Zn=d("'only_second'"),ea=d(": truncate to a maximum length specified by the "),dt=o("code"),ta=d("max_length"),oa=d(` argument or the maximum
length accepted by the model if no `),rt=o("code"),na=d("max_length"),aa=d(" is provided ("),lt=o("code"),da=d("max_length=None"),ra=d(`). This will only truncate
the second sentence of a pair if a pair of sequences (or a batch of pairs of sequences) is provided.`),la=l(),T=o("li"),ct=o("code"),ca=d("'only_first'"),ia=d(": truncate to a maximum length specified by the "),it=o("code"),sa=d("max_length"),ha=d(` argument or the maximum
length accepted by the model if no `),st=o("code"),fa=d("max_length"),ua=d(" is provided ("),ht=o("code"),_a=d("max_length=None"),pa=d(`). This will only truncate
the first sentence of a pair if a pair of sequences (or a batch of pairs of sequences) is provided.`),ma=l(),oe=o("li"),ft=o("code"),ga=d("False"),Ea=d(" or "),ut=o("code"),va=d("'do_not_truncate'"),Ta=d(": no truncation is applied. This is the default behavior."),ko=l(),b=o("p"),ba=d("The "),_t=o("code"),Da=d("max_length"),xa=d(" argument controls the length of the padding and truncation. It can be an integer or "),pt=o("code"),ya=d("None"),Oa=d(", in which case it will default to the maximum length the model can accept. If the model has no specific maximum input length, truncation or padding to "),mt=o("code"),Ca=d("max_length"),ka=d(" is deactivated."),wo=l(),_=o("p"),wa=d("The following table summarizes the recommended way to setup padding and truncation. If you use pairs of input sequences in any of the following examples, you can replace "),gt=o("code"),za=d("truncation=True"),qa=d(" by a "),Et=o("code"),Ra=d("STRATEGY"),Ia=d(` selected in
`),vt=o("code"),Pa=d("['only_first', 'only_second', 'longest_first']"),Aa=d(", i.e. "),Tt=o("code"),Sa=d("truncation='only_second'"),Na=d(" or "),bt=o("code"),Ga=d("truncation='longest_first'"),Ya=d(" to control how both sequences in the pair are truncated as detailed before."),zo=l(),ne=o("table"),Dt=o("thead"),k=o("tr"),xt=o("th"),La=d("Truncation"),$a=l(),yt=o("th"),Ha=d("Padding"),Ba=l(),Ot=o("th"),Fa=d("Instruction"),Ma=l(),s=o("tbody"),w=o("tr"),Ct=o("td"),Ua=d("no truncation"),Ja=l(),kt=o("td"),ja=d("no padding"),Ka=l(),wt=o("td"),zt=o("code"),Qa=d("tokenizer(batch_sentences)"),Va=l(),z=o("tr"),qo=o("td"),Wa=l(),qt=o("td"),Xa=d("padding to max sequence in batch"),Za=l(),ce=o("td"),Rt=o("code"),ed=d("tokenizer(batch_sentences, padding=True)"),td=d(" or"),od=l(),q=o("tr"),Ro=o("td"),nd=l(),Io=o("td"),ad=l(),It=o("td"),Pt=o("code"),dd=d("tokenizer(batch_sentences, padding='longest')"),rd=l(),R=o("tr"),Po=o("td"),ld=l(),At=o("td"),cd=d("padding to max model input length"),id=l(),St=o("td"),Nt=o("code"),sd=d("tokenizer(batch_sentences, padding='max_length')"),hd=l(),I=o("tr"),Ao=o("td"),fd=l(),Gt=o("td"),ud=d("padding to specific length"),_d=l(),Yt=o("td"),Lt=o("code"),pd=d("tokenizer(batch_sentences, padding='max_length', max_length=42)"),md=l(),P=o("tr"),$t=o("td"),gd=d("truncation to max model input length"),Ed=l(),Ht=o("td"),vd=d("no padding"),Td=l(),ie=o("td"),Bt=o("code"),bd=d("tokenizer(batch_sentences, truncation=True)"),Dd=d(" or"),xd=l(),A=o("tr"),So=o("td"),yd=l(),No=o("td"),Od=l(),Ft=o("td"),Mt=o("code"),Cd=d("tokenizer(batch_sentences, truncation=STRATEGY)"),kd=l(),S=o("tr"),Go=o("td"),wd=l(),Ut=o("td"),zd=d("padding to max sequence in batch"),qd=l(),se=o("td"),Jt=o("code"),Rd=d("tokenizer(batch_sentences, padding=True, truncation=True)"),Id=d(" or"),Pd=l(),N=o("tr"),Yo=o("td"),Ad=l(),Lo=o("td"),Sd=l(),jt=o("td"),Kt=o("code"),Nd=d("tokenizer(batch_sentences, padding=True, truncation=STRATEGY)"),Gd=l(),G=o("tr"),$o=o("td"),Yd=l(),Qt=o("td"),Ld=d("padding to max model input length"),$d=l(),he=o("td"),Vt=o("code"),Hd=d("tokenizer(batch_sentences, padding='max_length', truncation=True)"),Bd=d(" or"),Fd=l(),Y=o("tr"),Ho=o("td"),Md=l(),Bo=o("td"),Ud=l(),Wt=o("td"),Xt=o("code"),Jd=d("tokenizer(batch_sentences, padding='max_length', truncation=STRATEGY)"),jd=l(),L=o("tr"),Fo=o("td"),Kd=l(),Zt=o("td"),Qd=d("padding to specific length"),Vd=l(),eo=o("td"),Wd=d("Not possible"),Xd=l(),$=o("tr"),to=o("td"),Zd=d("truncation to specific length"),er=l(),oo=o("td"),tr=d("no padding"),or=l(),fe=o("td"),no=o("code"),nr=d("tokenizer(batch_sentences, truncation=True, max_length=42)"),ar=d(" or"),dr=l(),H=o("tr"),Mo=o("td"),rr=l(),Uo=o("td"),lr=l(),ao=o("td"),ro=o("code"),cr=d("tokenizer(batch_sentences, truncation=STRATEGY, max_length=42)"),ir=l(),B=o("tr"),Jo=o("td"),sr=l(),lo=o("td"),hr=d("padding to max sequence in batch"),fr=l(),ue=o("td"),co=o("code"),ur=d("tokenizer(batch_sentences, padding=True, truncation=True, max_length=42)"),_r=d(" or"),pr=l(),F=o("tr"),jo=o("td"),mr=l(),Ko=o("td"),gr=l(),io=o("td"),so=o("code"),Er=d("tokenizer(batch_sentences, padding=True, truncation=STRATEGY, max_length=42)"),vr=l(),M=o("tr"),Qo=o("td"),Tr=l(),ho=o("td"),br=d("padding to max model input length"),Dr=l(),fo=o("td"),xr=d("Not possible"),yr=l(),U=o("tr"),Vo=o("td"),Or=l(),uo=o("td"),Cr=d("padding to specific length"),kr=l(),_e=o("td"),_o=o("code"),wr=d("tokenizer(batch_sentences, padding='max_length', truncation=True, max_length=42)"),zr=d(" or"),qr=l(),J=o("tr"),Wo=o("td"),Rr=l(),Xo=o("td"),Ir=l(),po=o("td"),mo=o("code"),Pr=d("tokenizer(batch_sentences, padding='max_length', truncation=STRATEGY, max_length=42)"),this.h()},l(i){const f=Cc('[data-svelte="svelte-1phssyn"]',document.head);O=n(f,"META",{name:!0,content:!0}),f.forEach(t),To=c(i),C=n(i,"H1",{class:!0});var en=a(C);V=n(en,"A",{id:!0,class:!0,href:!0});var Br=a(V);Ge=n(Br,"SPAN",{});var Fr=a(Ge);kc(le.$$.fragment,Fr),Fr.forEach(t),Br.forEach(t),dn=c(en),Ye=n(en,"SPAN",{});var Mr=a(Ye);rn=r(Mr,"Padding and truncation"),Mr.forEach(t),en.forEach(t),bo=c(i),W=n(i,"P",{});var tn=a(W);ln=r(tn,"Batched inputs are often different lengths, so they can\u2019t be converted to fixed-size tensors. Padding and truncation are strategies for dealing with this problem, to create rectangular tensors from batches of varying lengths. Padding adds a special "),Le=n(tn,"STRONG",{});var Ur=a(Le);cn=r(Ur,"padding token"),Ur.forEach(t),sn=r(tn," to ensure shorter sequences will have the same length as either the longest sequence in a batch or the maximum length accepted by the model. Truncation works in the other direction by truncating long sequences."),tn.forEach(t),Do=c(i),m=n(i,"P",{});var ae=a(m);hn=r(ae,"In most cases, padding your batch to the length of the longest sequence and truncating to the maximum length a model can accept works pretty well. However, the API supports more strategies if you need them. The three arguments you need to are: "),$e=n(ae,"CODE",{});var Jr=a($e);fn=r(Jr,"padding"),Jr.forEach(t),un=r(ae,", "),He=n(ae,"CODE",{});var jr=a(He);_n=r(jr,"truncation"),jr.forEach(t),pn=r(ae," and "),Be=n(ae,"CODE",{});var Kr=a(Be);mn=r(Kr,"max_length"),Kr.forEach(t),gn=r(ae,"."),ae.forEach(t),xo=c(i),X=n(i,"P",{});var on=a(X);En=r(on,"The "),Fe=n(on,"CODE",{});var Qr=a(Fe);vn=r(Qr,"padding"),Qr.forEach(t),Tn=r(on," argument controls padding. It can be a boolean or a string:"),on.forEach(t),yo=c(i),y=n(i,"UL",{});var pe=a(y);Z=n(pe,"LI",{});var go=a(Z);Me=n(go,"CODE",{});var Vr=a(Me);bn=r(Vr,"True"),Vr.forEach(t),Dn=r(go," or "),Ue=n(go,"CODE",{});var Wr=a(Ue);xn=r(Wr,"'longest'"),Wr.forEach(t),yn=r(go,`: pad to the longest sequence in the batch (no padding is applied if you only provide
a single sequence).`),go.forEach(t),On=c(pe),g=n(pe,"LI",{});var j=a(g);Je=n(j,"CODE",{});var Xr=a(Je);Cn=r(Xr,"'max_length'"),Xr.forEach(t),kn=r(j,": pad to a length specified by the "),je=n(j,"CODE",{});var Zr=a(je);wn=r(Zr,"max_length"),Zr.forEach(t),zn=r(j,` argument or the maximum length accepted
by the model if no `),Ke=n(j,"CODE",{});var el=a(Ke);qn=r(el,"max_length"),el.forEach(t),Rn=r(j," is provided ("),Qe=n(j,"CODE",{});var tl=a(Qe);In=r(tl,"max_length=None"),tl.forEach(t),Pn=r(j,"). Padding will still be applied if you only provide a single sequence."),j.forEach(t),An=c(pe),ee=n(pe,"LI",{});var Eo=a(ee);Ve=n(Eo,"CODE",{});var ol=a(Ve);Sn=r(ol,"False"),ol.forEach(t),Nn=r(Eo," or "),We=n(Eo,"CODE",{});var nl=a(We);Gn=r(nl,"'do_not_pad'"),nl.forEach(t),Yn=r(Eo,": no padding is applied. This is the default behavior."),Eo.forEach(t),pe.forEach(t),Oo=c(i),te=n(i,"P",{});var nn=a(te);Ln=r(nn,"The "),Xe=n(nn,"CODE",{});var al=a(Xe);$n=r(al,"truncation"),al.forEach(t),Hn=r(nn," argument controls truncation. It can be a boolean or a string:"),nn.forEach(t),Co=c(i),E=n(i,"UL",{});var de=a(E);p=n(de,"LI",{});var x=a(p);Ze=n(x,"CODE",{});var dl=a(Ze);Bn=r(dl,"True"),dl.forEach(t),Fn=r(x," or "),et=n(x,"CODE",{});var rl=a(et);Mn=r(rl,"'longest_first'"),rl.forEach(t),Un=r(x,": truncate to a maximum length specified by the "),tt=n(x,"CODE",{});var ll=a(tt);Jn=r(ll,"max_length"),ll.forEach(t),jn=r(x,` argument or
the maximum length accepted by the model if no `),ot=n(x,"CODE",{});var cl=a(ot);Kn=r(cl,"max_length"),cl.forEach(t),Qn=r(x," is provided ("),nt=n(x,"CODE",{});var il=a(nt);Vn=r(il,"max_length=None"),il.forEach(t),Wn=r(x,`). This will
truncate token by token, removing a token from the longest sequence in the pair until the proper length is
reached.`),x.forEach(t),Xn=c(de),v=n(de,"LI",{});var K=a(v);at=n(K,"CODE",{});var sl=a(at);Zn=r(sl,"'only_second'"),sl.forEach(t),ea=r(K,": truncate to a maximum length specified by the "),dt=n(K,"CODE",{});var hl=a(dt);ta=r(hl,"max_length"),hl.forEach(t),oa=r(K,` argument or the maximum
length accepted by the model if no `),rt=n(K,"CODE",{});var fl=a(rt);na=r(fl,"max_length"),fl.forEach(t),aa=r(K," is provided ("),lt=n(K,"CODE",{});var ul=a(lt);da=r(ul,"max_length=None"),ul.forEach(t),ra=r(K,`). This will only truncate
the second sentence of a pair if a pair of sequences (or a batch of pairs of sequences) is provided.`),K.forEach(t),la=c(de),T=n(de,"LI",{});var Q=a(T);ct=n(Q,"CODE",{});var _l=a(ct);ca=r(_l,"'only_first'"),_l.forEach(t),ia=r(Q,": truncate to a maximum length specified by the "),it=n(Q,"CODE",{});var pl=a(it);sa=r(pl,"max_length"),pl.forEach(t),ha=r(Q,` argument or the maximum
length accepted by the model if no `),st=n(Q,"CODE",{});var ml=a(st);fa=r(ml,"max_length"),ml.forEach(t),ua=r(Q," is provided ("),ht=n(Q,"CODE",{});var gl=a(ht);_a=r(gl,"max_length=None"),gl.forEach(t),pa=r(Q,`). This will only truncate
the first sentence of a pair if a pair of sequences (or a batch of pairs of sequences) is provided.`),Q.forEach(t),ma=c(de),oe=n(de,"LI",{});var vo=a(oe);ft=n(vo,"CODE",{});var El=a(ft);ga=r(El,"False"),El.forEach(t),Ea=r(vo," or "),ut=n(vo,"CODE",{});var vl=a(ut);va=r(vl,"'do_not_truncate'"),vl.forEach(t),Ta=r(vo,": no truncation is applied. This is the default behavior."),vo.forEach(t),de.forEach(t),ko=c(i),b=n(i,"P",{});var re=a(b);ba=r(re,"The "),_t=n(re,"CODE",{});var Tl=a(_t);Da=r(Tl,"max_length"),Tl.forEach(t),xa=r(re," argument controls the length of the padding and truncation. It can be an integer or "),pt=n(re,"CODE",{});var bl=a(pt);ya=r(bl,"None"),bl.forEach(t),Oa=r(re,", in which case it will default to the maximum length the model can accept. If the model has no specific maximum input length, truncation or padding to "),mt=n(re,"CODE",{});var Dl=a(mt);Ca=r(Dl,"max_length"),Dl.forEach(t),ka=r(re," is deactivated."),re.forEach(t),wo=c(i),_=n(i,"P",{});var D=a(_);wa=r(D,"The following table summarizes the recommended way to setup padding and truncation. If you use pairs of input sequences in any of the following examples, you can replace "),gt=n(D,"CODE",{});var xl=a(gt);za=r(xl,"truncation=True"),xl.forEach(t),qa=r(D," by a "),Et=n(D,"CODE",{});var yl=a(Et);Ra=r(yl,"STRATEGY"),yl.forEach(t),Ia=r(D,` selected in
`),vt=n(D,"CODE",{});var Ol=a(vt);Pa=r(Ol,"['only_first', 'only_second', 'longest_first']"),Ol.forEach(t),Aa=r(D,", i.e. "),Tt=n(D,"CODE",{});var Cl=a(Tt);Sa=r(Cl,"truncation='only_second'"),Cl.forEach(t),Na=r(D," or "),bt=n(D,"CODE",{});var kl=a(bt);Ga=r(kl,"truncation='longest_first'"),kl.forEach(t),Ya=r(D," to control how both sequences in the pair are truncated as detailed before."),D.forEach(t),zo=c(i),ne=n(i,"TABLE",{});var an=a(ne);Dt=n(an,"THEAD",{});var wl=a(Dt);k=n(wl,"TR",{});var me=a(k);xt=n(me,"TH",{});var zl=a(xt);La=r(zl,"Truncation"),zl.forEach(t),$a=c(me),yt=n(me,"TH",{});var ql=a(yt);Ha=r(ql,"Padding"),ql.forEach(t),Ba=c(me),Ot=n(me,"TH",{});var Rl=a(Ot);Fa=r(Rl,"Instruction"),Rl.forEach(t),me.forEach(t),wl.forEach(t),Ma=c(an),s=n(an,"TBODY",{});var h=a(s);w=n(h,"TR",{});var ge=a(w);Ct=n(ge,"TD",{});var Il=a(Ct);Ua=r(Il,"no truncation"),Il.forEach(t),Ja=c(ge),kt=n(ge,"TD",{});var Pl=a(kt);ja=r(Pl,"no padding"),Pl.forEach(t),Ka=c(ge),wt=n(ge,"TD",{});var Al=a(wt);zt=n(Al,"CODE",{});var Sl=a(zt);Qa=r(Sl,"tokenizer(batch_sentences)"),Sl.forEach(t),Al.forEach(t),ge.forEach(t),Va=c(h),z=n(h,"TR",{});var Ee=a(z);qo=n(Ee,"TD",{}),a(qo).forEach(t),Wa=c(Ee),qt=n(Ee,"TD",{});var Nl=a(qt);Xa=r(Nl,"padding to max sequence in batch"),Nl.forEach(t),Za=c(Ee),ce=n(Ee,"TD",{});var Ar=a(ce);Rt=n(Ar,"CODE",{});var Gl=a(Rt);ed=r(Gl,"tokenizer(batch_sentences, padding=True)"),Gl.forEach(t),td=r(Ar," or"),Ar.forEach(t),Ee.forEach(t),od=c(h),q=n(h,"TR",{});var ve=a(q);Ro=n(ve,"TD",{}),a(Ro).forEach(t),nd=c(ve),Io=n(ve,"TD",{}),a(Io).forEach(t),ad=c(ve),It=n(ve,"TD",{});var Yl=a(It);Pt=n(Yl,"CODE",{});var Ll=a(Pt);dd=r(Ll,"tokenizer(batch_sentences, padding='longest')"),Ll.forEach(t),Yl.forEach(t),ve.forEach(t),rd=c(h),R=n(h,"TR",{});var Te=a(R);Po=n(Te,"TD",{}),a(Po).forEach(t),ld=c(Te),At=n(Te,"TD",{});var $l=a(At);cd=r($l,"padding to max model input length"),$l.forEach(t),id=c(Te),St=n(Te,"TD",{});var Hl=a(St);Nt=n(Hl,"CODE",{});var Bl=a(Nt);sd=r(Bl,"tokenizer(batch_sentences, padding='max_length')"),Bl.forEach(t),Hl.forEach(t),Te.forEach(t),hd=c(h),I=n(h,"TR",{});var be=a(I);Ao=n(be,"TD",{}),a(Ao).forEach(t),fd=c(be),Gt=n(be,"TD",{});var Fl=a(Gt);ud=r(Fl,"padding to specific length"),Fl.forEach(t),_d=c(be),Yt=n(be,"TD",{});var Ml=a(Yt);Lt=n(Ml,"CODE",{});var Ul=a(Lt);pd=r(Ul,"tokenizer(batch_sentences, padding='max_length', max_length=42)"),Ul.forEach(t),Ml.forEach(t),be.forEach(t),md=c(h),P=n(h,"TR",{});var De=a(P);$t=n(De,"TD",{});var Jl=a($t);gd=r(Jl,"truncation to max model input length"),Jl.forEach(t),Ed=c(De),Ht=n(De,"TD",{});var jl=a(Ht);vd=r(jl,"no padding"),jl.forEach(t),Td=c(De),ie=n(De,"TD",{});var Sr=a(ie);Bt=n(Sr,"CODE",{});var Kl=a(Bt);bd=r(Kl,"tokenizer(batch_sentences, truncation=True)"),Kl.forEach(t),Dd=r(Sr," or"),Sr.forEach(t),De.forEach(t),xd=c(h),A=n(h,"TR",{});var xe=a(A);So=n(xe,"TD",{}),a(So).forEach(t),yd=c(xe),No=n(xe,"TD",{}),a(No).forEach(t),Od=c(xe),Ft=n(xe,"TD",{});var Ql=a(Ft);Mt=n(Ql,"CODE",{});var Vl=a(Mt);Cd=r(Vl,"tokenizer(batch_sentences, truncation=STRATEGY)"),Vl.forEach(t),Ql.forEach(t),xe.forEach(t),kd=c(h),S=n(h,"TR",{});var ye=a(S);Go=n(ye,"TD",{}),a(Go).forEach(t),wd=c(ye),Ut=n(ye,"TD",{});var Wl=a(Ut);zd=r(Wl,"padding to max sequence in batch"),Wl.forEach(t),qd=c(ye),se=n(ye,"TD",{});var Nr=a(se);Jt=n(Nr,"CODE",{});var Xl=a(Jt);Rd=r(Xl,"tokenizer(batch_sentences, padding=True, truncation=True)"),Xl.forEach(t),Id=r(Nr," or"),Nr.forEach(t),ye.forEach(t),Pd=c(h),N=n(h,"TR",{});var Oe=a(N);Yo=n(Oe,"TD",{}),a(Yo).forEach(t),Ad=c(Oe),Lo=n(Oe,"TD",{}),a(Lo).forEach(t),Sd=c(Oe),jt=n(Oe,"TD",{});var Zl=a(jt);Kt=n(Zl,"CODE",{});var ec=a(Kt);Nd=r(ec,"tokenizer(batch_sentences, padding=True, truncation=STRATEGY)"),ec.forEach(t),Zl.forEach(t),Oe.forEach(t),Gd=c(h),G=n(h,"TR",{});var Ce=a(G);$o=n(Ce,"TD",{}),a($o).forEach(t),Yd=c(Ce),Qt=n(Ce,"TD",{});var tc=a(Qt);Ld=r(tc,"padding to max model input length"),tc.forEach(t),$d=c(Ce),he=n(Ce,"TD",{});var Gr=a(he);Vt=n(Gr,"CODE",{});var oc=a(Vt);Hd=r(oc,"tokenizer(batch_sentences, padding='max_length', truncation=True)"),oc.forEach(t),Bd=r(Gr," or"),Gr.forEach(t),Ce.forEach(t),Fd=c(h),Y=n(h,"TR",{});var ke=a(Y);Ho=n(ke,"TD",{}),a(Ho).forEach(t),Md=c(ke),Bo=n(ke,"TD",{}),a(Bo).forEach(t),Ud=c(ke),Wt=n(ke,"TD",{});var nc=a(Wt);Xt=n(nc,"CODE",{});var ac=a(Xt);Jd=r(ac,"tokenizer(batch_sentences, padding='max_length', truncation=STRATEGY)"),ac.forEach(t),nc.forEach(t),ke.forEach(t),jd=c(h),L=n(h,"TR",{});var we=a(L);Fo=n(we,"TD",{}),a(Fo).forEach(t),Kd=c(we),Zt=n(we,"TD",{});var dc=a(Zt);Qd=r(dc,"padding to specific length"),dc.forEach(t),Vd=c(we),eo=n(we,"TD",{});var rc=a(eo);Wd=r(rc,"Not possible"),rc.forEach(t),we.forEach(t),Xd=c(h),$=n(h,"TR",{});var ze=a($);to=n(ze,"TD",{});var lc=a(to);Zd=r(lc,"truncation to specific length"),lc.forEach(t),er=c(ze),oo=n(ze,"TD",{});var cc=a(oo);tr=r(cc,"no padding"),cc.forEach(t),or=c(ze),fe=n(ze,"TD",{});var Yr=a(fe);no=n(Yr,"CODE",{});var ic=a(no);nr=r(ic,"tokenizer(batch_sentences, truncation=True, max_length=42)"),ic.forEach(t),ar=r(Yr," or"),Yr.forEach(t),ze.forEach(t),dr=c(h),H=n(h,"TR",{});var qe=a(H);Mo=n(qe,"TD",{}),a(Mo).forEach(t),rr=c(qe),Uo=n(qe,"TD",{}),a(Uo).forEach(t),lr=c(qe),ao=n(qe,"TD",{});var sc=a(ao);ro=n(sc,"CODE",{});var hc=a(ro);cr=r(hc,"tokenizer(batch_sentences, truncation=STRATEGY, max_length=42)"),hc.forEach(t),sc.forEach(t),qe.forEach(t),ir=c(h),B=n(h,"TR",{});var Re=a(B);Jo=n(Re,"TD",{}),a(Jo).forEach(t),sr=c(Re),lo=n(Re,"TD",{});var fc=a(lo);hr=r(fc,"padding to max sequence in batch"),fc.forEach(t),fr=c(Re),ue=n(Re,"TD",{});var Lr=a(ue);co=n(Lr,"CODE",{});var uc=a(co);ur=r(uc,"tokenizer(batch_sentences, padding=True, truncation=True, max_length=42)"),uc.forEach(t),_r=r(Lr," or"),Lr.forEach(t),Re.forEach(t),pr=c(h),F=n(h,"TR",{});var Ie=a(F);jo=n(Ie,"TD",{}),a(jo).forEach(t),mr=c(Ie),Ko=n(Ie,"TD",{}),a(Ko).forEach(t),gr=c(Ie),io=n(Ie,"TD",{});var _c=a(io);so=n(_c,"CODE",{});var pc=a(so);Er=r(pc,"tokenizer(batch_sentences, padding=True, truncation=STRATEGY, max_length=42)"),pc.forEach(t),_c.forEach(t),Ie.forEach(t),vr=c(h),M=n(h,"TR",{});var Pe=a(M);Qo=n(Pe,"TD",{}),a(Qo).forEach(t),Tr=c(Pe),ho=n(Pe,"TD",{});var mc=a(ho);br=r(mc,"padding to max model input length"),mc.forEach(t),Dr=c(Pe),fo=n(Pe,"TD",{});var gc=a(fo);xr=r(gc,"Not possible"),gc.forEach(t),Pe.forEach(t),yr=c(h),U=n(h,"TR",{});var Ae=a(U);Vo=n(Ae,"TD",{}),a(Vo).forEach(t),Or=c(Ae),uo=n(Ae,"TD",{});var Ec=a(uo);Cr=r(Ec,"padding to specific length"),Ec.forEach(t),kr=c(Ae),_e=n(Ae,"TD",{});var $r=a(_e);_o=n($r,"CODE",{});var vc=a(_o);wr=r(vc,"tokenizer(batch_sentences, padding='max_length', truncation=True, max_length=42)"),vc.forEach(t),zr=r($r," or"),$r.forEach(t),Ae.forEach(t),qr=c(h),J=n(h,"TR",{});var Se=a(J);Wo=n(Se,"TD",{}),a(Wo).forEach(t),Rr=c(Se),Xo=n(Se,"TD",{}),a(Xo).forEach(t),Ir=c(Se),po=n(Se,"TD",{});var Tc=a(po);mo=n(Tc,"CODE",{});var bc=a(mo);Pr=r(bc,"tokenizer(batch_sentences, padding='max_length', truncation=STRATEGY, max_length=42)"),bc.forEach(t),Tc.forEach(t),Se.forEach(t),h.forEach(t),an.forEach(t),this.h()},h(){Ne(O,"name","hf:doc:metadata"),Ne(O,"content",JSON.stringify(Nc)),Ne(V,"id","padding-and-truncation"),Ne(V,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),Ne(V,"href","#padding-and-truncation"),Ne(C,"class","relative group")},m(i,f){e(document.head,O),u(i,To,f),u(i,C,f),e(C,V),e(V,Ge),wc(le,Ge,null),e(C,dn),e(C,Ye),e(Ye,rn),u(i,bo,f),u(i,W,f),e(W,ln),e(W,Le),e(Le,cn),e(W,sn),u(i,Do,f),u(i,m,f),e(m,hn),e(m,$e),e($e,fn),e(m,un),e(m,He),e(He,_n),e(m,pn),e(m,Be),e(Be,mn),e(m,gn),u(i,xo,f),u(i,X,f),e(X,En),e(X,Fe),e(Fe,vn),e(X,Tn),u(i,yo,f),u(i,y,f),e(y,Z),e(Z,Me),e(Me,bn),e(Z,Dn),e(Z,Ue),e(Ue,xn),e(Z,yn),e(y,On),e(y,g),e(g,Je),e(Je,Cn),e(g,kn),e(g,je),e(je,wn),e(g,zn),e(g,Ke),e(Ke,qn),e(g,Rn),e(g,Qe),e(Qe,In),e(g,Pn),e(y,An),e(y,ee),e(ee,Ve),e(Ve,Sn),e(ee,Nn),e(ee,We),e(We,Gn),e(ee,Yn),u(i,Oo,f),u(i,te,f),e(te,Ln),e(te,Xe),e(Xe,$n),e(te,Hn),u(i,Co,f),u(i,E,f),e(E,p),e(p,Ze),e(Ze,Bn),e(p,Fn),e(p,et),e(et,Mn),e(p,Un),e(p,tt),e(tt,Jn),e(p,jn),e(p,ot),e(ot,Kn),e(p,Qn),e(p,nt),e(nt,Vn),e(p,Wn),e(E,Xn),e(E,v),e(v,at),e(at,Zn),e(v,ea),e(v,dt),e(dt,ta),e(v,oa),e(v,rt),e(rt,na),e(v,aa),e(v,lt),e(lt,da),e(v,ra),e(E,la),e(E,T),e(T,ct),e(ct,ca),e(T,ia),e(T,it),e(it,sa),e(T,ha),e(T,st),e(st,fa),e(T,ua),e(T,ht),e(ht,_a),e(T,pa),e(E,ma),e(E,oe),e(oe,ft),e(ft,ga),e(oe,Ea),e(oe,ut),e(ut,va),e(oe,Ta),u(i,ko,f),u(i,b,f),e(b,ba),e(b,_t),e(_t,Da),e(b,xa),e(b,pt),e(pt,ya),e(b,Oa),e(b,mt),e(mt,Ca),e(b,ka),u(i,wo,f),u(i,_,f),e(_,wa),e(_,gt),e(gt,za),e(_,qa),e(_,Et),e(Et,Ra),e(_,Ia),e(_,vt),e(vt,Pa),e(_,Aa),e(_,Tt),e(Tt,Sa),e(_,Na),e(_,bt),e(bt,Ga),e(_,Ya),u(i,zo,f),u(i,ne,f),e(ne,Dt),e(Dt,k),e(k,xt),e(xt,La),e(k,$a),e(k,yt),e(yt,Ha),e(k,Ba),e(k,Ot),e(Ot,Fa),e(ne,Ma),e(ne,s),e(s,w),e(w,Ct),e(Ct,Ua),e(w,Ja),e(w,kt),e(kt,ja),e(w,Ka),e(w,wt),e(wt,zt),e(zt,Qa),e(s,Va),e(s,z),e(z,qo),e(z,Wa),e(z,qt),e(qt,Xa),e(z,Za),e(z,ce),e(ce,Rt),e(Rt,ed),e(ce,td),e(s,od),e(s,q),e(q,Ro),e(q,nd),e(q,Io),e(q,ad),e(q,It),e(It,Pt),e(Pt,dd),e(s,rd),e(s,R),e(R,Po),e(R,ld),e(R,At),e(At,cd),e(R,id),e(R,St),e(St,Nt),e(Nt,sd),e(s,hd),e(s,I),e(I,Ao),e(I,fd),e(I,Gt),e(Gt,ud),e(I,_d),e(I,Yt),e(Yt,Lt),e(Lt,pd),e(s,md),e(s,P),e(P,$t),e($t,gd),e(P,Ed),e(P,Ht),e(Ht,vd),e(P,Td),e(P,ie),e(ie,Bt),e(Bt,bd),e(ie,Dd),e(s,xd),e(s,A),e(A,So),e(A,yd),e(A,No),e(A,Od),e(A,Ft),e(Ft,Mt),e(Mt,Cd),e(s,kd),e(s,S),e(S,Go),e(S,wd),e(S,Ut),e(Ut,zd),e(S,qd),e(S,se),e(se,Jt),e(Jt,Rd),e(se,Id),e(s,Pd),e(s,N),e(N,Yo),e(N,Ad),e(N,Lo),e(N,Sd),e(N,jt),e(jt,Kt),e(Kt,Nd),e(s,Gd),e(s,G),e(G,$o),e(G,Yd),e(G,Qt),e(Qt,Ld),e(G,$d),e(G,he),e(he,Vt),e(Vt,Hd),e(he,Bd),e(s,Fd),e(s,Y),e(Y,Ho),e(Y,Md),e(Y,Bo),e(Y,Ud),e(Y,Wt),e(Wt,Xt),e(Xt,Jd),e(s,jd),e(s,L),e(L,Fo),e(L,Kd),e(L,Zt),e(Zt,Qd),e(L,Vd),e(L,eo),e(eo,Wd),e(s,Xd),e(s,$),e($,to),e(to,Zd),e($,er),e($,oo),e(oo,tr),e($,or),e($,fe),e(fe,no),e(no,nr),e(fe,ar),e(s,dr),e(s,H),e(H,Mo),e(H,rr),e(H,Uo),e(H,lr),e(H,ao),e(ao,ro),e(ro,cr),e(s,ir),e(s,B),e(B,Jo),e(B,sr),e(B,lo),e(lo,hr),e(B,fr),e(B,ue),e(ue,co),e(co,ur),e(ue,_r),e(s,pr),e(s,F),e(F,jo),e(F,mr),e(F,Ko),e(F,gr),e(F,io),e(io,so),e(so,Er),e(s,vr),e(s,M),e(M,Qo),e(M,Tr),e(M,ho),e(ho,br),e(M,Dr),e(M,fo),e(fo,xr),e(s,yr),e(s,U),e(U,Vo),e(U,Or),e(U,uo),e(uo,Cr),e(U,kr),e(U,_e),e(_e,_o),e(_o,wr),e(_e,zr),e(s,qr),e(s,J),e(J,Wo),e(J,Rr),e(J,Xo),e(J,Ir),e(J,po),e(po,mo),e(mo,Pr),Zo=!0},p:zc,i(i){Zo||(qc(le.$$.fragment,i),Zo=!0)},o(i){Rc(le.$$.fragment,i),Zo=!1},d(i){t(O),i&&t(To),i&&t(C),Ic(le),i&&t(bo),i&&t(W),i&&t(Do),i&&t(m),i&&t(xo),i&&t(X),i&&t(yo),i&&t(y),i&&t(Oo),i&&t(te),i&&t(Co),i&&t(E),i&&t(ko),i&&t(b),i&&t(wo),i&&t(_),i&&t(zo),i&&t(ne)}}}const Nc={local:"padding-and-truncation",title:"Padding and truncation"};function Gc(Hr){return Pc(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class $c extends Dc{constructor(O){super();xc(this,O,Gc,Sc,yc,{})}}export{$c as default,Nc as metadata};
