import{S as jv,i as _v,s as gv,e as t,k as c,w as m,t as r,M as vv,c as n,d as a,m as h,a as l,x as d,h as o,b as u,N as bv,F as e,g as i,y as f,q as b,o as j,B as _}from"../chunks/vendor-4833417e.js";import{T as Ev}from"../chunks/Tip-fffd6df1.js";import{Y as wv}from"../chunks/Youtube-27813aed.js";import{I as k}from"../chunks/IconCopyLink-4b81c553.js";import{C as E}from"../chunks/CodeBlock-6a3d1b46.js";import{C as $v}from"../chunks/CodeBlockFw-27a176a0.js";import{D as yv}from"../chunks/DocNotebookDropdown-ecff2a90.js";import"../chunks/CopyButton-dacfbfaf.js";function kv(In){let w,R,$,x,ns;return{c(){w=t("p"),R=r("If you plan on using a pretrained model, it\u2019s important to use the associated pretrained tokenizer. This ensures the text is split the same way as the pretraining corpus, and uses the same corresponding tokens-to-index (usually referrred to as the "),$=t("em"),x=r("vocab"),ns=r(") during pretraining.")},l(q){w=n(q,"P",{});var F=l(w);R=o(F,"If you plan on using a pretrained model, it\u2019s important to use the associated pretrained tokenizer. This ensures the text is split the same way as the pretraining corpus, and uses the same corresponding tokens-to-index (usually referrred to as the "),$=n(F,"EM",{});var Gs=l($);x=o(Gs,"vocab"),Gs.forEach(a),ns=o(F,") during pretraining."),F.forEach(a)},m(q,F){i(q,w,F),e(w,R),e(w,$),e($,x),e(w,ns)},d(q){q&&a(w)}}}function xv(In){let w,R,$,x,ns,q,F,Gs,oi,Uo,Ce,Mo,at,pi,Vo,H,Rn,ci,hi,Fn,ii,ui,Hn,mi,Ko,ls,Us,Bn,Oe,di,Jn,fi,Qo,Le,Xo,B,bi,tt,ji,_i,Wn,gi,vi,Zo,Ms,sp,J,Ei,nt,wi,$i,Yn,yi,ki,ep,rs,Vs,Gn,Se,xi,Un,Ti,ap,Ks,qi,lt,Di,Ai,tp,Ne,np,rt,zi,lp,Ie,rp,ot,Pi,op,W,pt,ct,Ci,Oi,Li,ht,it,Si,Ni,Ii,ut,mt,Ri,Fi,pp,Qs,Hi,Mn,Bi,Ji,cp,Re,hp,Y,Wi,Vn,Yi,Gi,Kn,Ui,Mi,ip,dt,Vi,up,Fe,mp,os,Xs,Qn,He,Ki,Xn,Qi,dp,Zs,Xi,Zn,Zi,su,fp,G,eu,sl,au,tu,el,nu,lu,bp,Be,jp,se,ru,al,ou,pu,_p,ps,ee,tl,Je,cu,nl,hu,gp,ft,iu,vp,U,uu,ll,mu,du,rl,fu,bu,Ep,We,wp,cs,ae,ol,Ye,ju,pl,_u,$p,bt,gu,yp,D,vu,cl,Eu,wu,hl,$u,yu,il,ku,xu,kp,Ge,xp,hs,te,ul,Ue,Tu,ml,qu,Tp,ne,Du,jt,Au,zu,qp,Me,Dp,M,Pu,Ve,Cu,Ou,Ke,Lu,Su,Ap,Qe,zp,V,Nu,dl,Iu,Ru,fl,Fu,Hu,Pp,Xe,Cp,_t,Bu,Op,K,gt,bl,Ju,Wu,Yu,vt,jl,Gu,Uu,Mu,Et,_l,Vu,Ku,Lp,is,le,gl,Ze,Qu,vl,Xu,Sp,re,Zu,sa,sm,em,Np,oe,am,ea,tm,nm,Ip,aa,Rp,wt,ta,lm,na,El,rm,om,Fp,la,Hp,ra,wl,pm,Bp,oa,Jp,pe,cm,$l,hm,im,Wp,us,ce,yl,pa,um,kl,mm,Yp,A,dm,xl,fm,bm,Tl,jm,_m,ql,gm,vm,Gp,he,Em,$t,wm,$m,Up,ca,Mp,Q,ym,Dl,km,xm,Al,Tm,qm,Vp,ha,Kp,ms,ie,zl,ia,Dm,Pl,Am,Qp,yt,zm,Xp,ua,Zp,kt,Pm,sc,ma,ec,xt,Cm,ac,da,tc,Tt,Om,nc,fa,lc,qt,Lm,rc,ds,ue,Cl,ba,Sm,Ol,Nm,oc,Dt,Im,pc,X,Rm,ja,Fm,Hm,Ll,Bm,Jm,cc,_a,hc,me,Wm,ga,Sl,Ym,Gm,ic,va,uc,At,zt,V1,mc,fs,de,Nl,Ea,Um,Il,Mm,dc,fe,Vm,Pt,Km,Qm,fc,wa,bc,bs,be,Rl,$a,Xm,Fl,Zm,jc,je,sd,ya,Hl,ed,ad,_c,Ct,S,td,ka,Bl,nd,ld,xa,Jl,rd,od,Ta,Wl,pd,cd,gc,qa,vc,Da,js,hd,Ot,Yl,id,ud,Gl,md,dd,Ec,Aa,wc,za,Pa,fd,Ca,Ul,bd,jd,$c,Oa,yc,La,Sa,_d,Ml,gd,vd,kc,Na,xc,Lt,Ed,Tc,Ia,qc,St,Nt,K1,Dc,_s,_e,Vl,Ra,wd,Kl,$d,Ac,It,yd,zc,ge,Ql,kd,xd,Xl,Td,Pc,ve,qd,Fa,Dd,Ad,Cc,Ha,Oc,Z,zd,Zl,Pd,Cd,sr,Od,Ld,Lc,Ba,Sc,ss,Sd,er,Nd,Id,ar,Rd,Fd,Nc,Ja,Ic,Ee,Hd,Rt,Bd,Jd,Rc,Wa,Fc,gs,we,tr,Ya,Wd,nr,Yd,Hc,Ft,Gd,Bc,Ga,Jc,Ht,vs,Ud,lr,Md,Vd,rr,Kd,Qd,Wc,Ua,Yc,Ma,Va,Xd,or,Zd,sf,Gc,Ka,Uc,es,ef,pr,af,tf,cr,nf,lf,Mc,Bt,rf,Vc,Es,$e,hr,Qa,of,ir,pf,Kc,z,cf,ur,hf,uf,mr,mf,df,dr,ff,bf,Qc,as,Xa,Jt,fr,jf,_f,gf,ws,ye,br,vf,Ef,jr,wf,$f,yf,P,_r,kf,xf,gr,Tf,qf,vr,Df,Af,Er,zf,Pf,Cf,ke,wr,Of,Lf,$r,Sf,Nf,If,Za,Wt,yr,Rf,Ff,Hf,N,T,kr,Bf,Jf,xr,Wf,Yf,Tr,Gf,Uf,qr,Mf,Vf,Dr,Kf,Qf,Xf,C,Ar,Zf,sb,zr,eb,ab,Pr,tb,nb,Cr,lb,rb,ob,O,Or,pb,cb,Lr,hb,ib,Sr,ub,mb,Nr,db,fb,bb,xe,Ir,jb,_b,Rr,gb,vb,Eb,Fr,ts,Hr,wb,$b,Br,yb,kb,Jr,xb,Tb,Xc,y,qb,Wr,Db,Ab,Yr,zb,Pb,Gr,Cb,Ob,Ur,Lb,Sb,Mr,Nb,Ib,Zc,Te,Vr,$s,Kr,Rb,Fb,Qr,Hb,Bb,Xr,Jb,Wb,g,ys,Zr,Yb,Gb,so,Ub,Mb,eo,ao,Vb,Kb,ks,sh,Qb,to,Xb,Zb,Yt,no,sj,ej,aj,xs,eh,tj,ah,nj,lo,ro,lj,rj,Ts,th,oj,oo,pj,cj,po,co,hj,ij,qs,nh,uj,ho,mj,dj,io,uo,fj,bj,Ds,mo,jj,_j,fo,gj,vj,Gt,bo,Ej,wj,$j,As,lh,yj,rh,kj,jo,_o,xj,Tj,zs,oh,qj,go,Dj,Aj,Ut,vo,zj,Pj,Cj,Ps,ph,Oj,ch,Lj,Eo,wo,Sj,Nj,Cs,hh,Ij,$o,Rj,Fj,Mt,yo,Hj,Bj,Jj,Os,ih,Wj,uh,Yj,ko,xo,Gj,Uj,Ls,mh,Mj,To,Vj,Kj,qo,Qj,Xj,Ss,Do,Zj,s1,Ao,e1,a1,Vt,zo,t1,n1,l1,Ns,dh,r1,fh,o1,Po,Co,p1,c1,Is,bh,h1,Oo,i1,u1,Kt,Lo,m1,d1,f1,Rs,jh,b1,_h,j1,So,No,_1,g1,Fs,gh,v1,Io,E1,w1,Ro,$1,y1,Hs,vh,k1,Fo,x1,T1,Qt,Ho,q1,D1,A1,Bs,Eh,z1,wh,P1,Bo,Jo,C1,$h;return q=new k({}),Ce=new yv({props:{classNames:"absolute z-10 right-0 top-0",options:[{label:"Mixed",value:"https://colab.research.google.com/github/huggingface/notebooks/blob/master/transformers_doc/preprocessing.ipynb"},{label:"PyTorch",value:"https://colab.research.google.com/github/huggingface/notebooks/blob/master/transformers_doc/pytorch/preprocessing.ipynb"},{label:"TensorFlow",value:"https://colab.research.google.com/github/huggingface/notebooks/blob/master/transformers_doc/tensorflow/preprocessing.ipynb"},{label:"Mixed",value:"https://studiolab.sagemaker.aws/import/github/huggingface/notebooks/blob/master/transformers_doc/preprocessing.ipynb"},{label:"PyTorch",value:"https://studiolab.sagemaker.aws/import/github/huggingface/notebooks/blob/master/transformers_doc/pytorch/preprocessing.ipynb"},{label:"TensorFlow",value:"https://studiolab.sagemaker.aws/import/github/huggingface/notebooks/blob/master/transformers_doc/tensorflow/preprocessing.ipynb"}]}}),Oe=new k({}),Le=new wv({props:{id:"Yffk5aydLzg"}}),Ms=new Ev({props:{$$slots:{default:[kv]},$$scope:{ctx:In}}}),Se=new k({}),Ne=new E({props:{code:`from transformers import AutoTokenizer

tokenizer = AutoTokenizer.from_pretrained("bert-base-cased")`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoTokenizer

<span class="hljs-meta">&gt;&gt;&gt; </span>tokenizer = AutoTokenizer.from_pretrained(<span class="hljs-string">&quot;bert-base-cased&quot;</span>)`}}),Ie=new E({props:{code:`encoded_input = tokenizer("Do not meddle in the affairs of wizards, for they are subtle and quick to anger.")
print(encoded_input)`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>encoded_input = tokenizer(<span class="hljs-string">&quot;Do not meddle in the affairs of wizards, for they are subtle and quick to anger.&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-built_in">print</span>(encoded_input)
{<span class="hljs-string">&#x27;input_ids&#x27;</span>: [<span class="hljs-number">101</span>, <span class="hljs-number">2079</span>, <span class="hljs-number">2025</span>, <span class="hljs-number">19960</span>, <span class="hljs-number">10362</span>, <span class="hljs-number">1999</span>, <span class="hljs-number">1996</span>, <span class="hljs-number">3821</span>, <span class="hljs-number">1997</span>, <span class="hljs-number">16657</span>, <span class="hljs-number">1010</span>, <span class="hljs-number">2005</span>, <span class="hljs-number">2027</span>, <span class="hljs-number">2024</span>, <span class="hljs-number">11259</span>, <span class="hljs-number">1998</span>, <span class="hljs-number">4248</span>, <span class="hljs-number">2000</span>, <span class="hljs-number">4963</span>, <span class="hljs-number">1012</span>, <span class="hljs-number">102</span>], 
 <span class="hljs-string">&#x27;token_type_ids&#x27;</span>: [<span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>], 
 <span class="hljs-string">&#x27;attention_mask&#x27;</span>: [<span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>]}`}}),Re=new E({props:{code:'tokenizer.decode(encoded_input["input_ids"])',highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>tokenizer.decode(encoded_input[<span class="hljs-string">&quot;input_ids&quot;</span>])
<span class="hljs-string">&#x27;[CLS] Do not meddle in the affairs of wizards, for they are subtle and quick to anger. [SEP]&#x27;</span>`}}),Fe=new E({props:{code:`batch_sentences = [
    "But what about second breakfast?",
    "Don't think he knows about second breakfast, Pip.",
    "What about elevensies?",
]
encoded_inputs = tokenizer(batch_sentences)
print(encoded_inputs)`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>batch_sentences = [
<span class="hljs-meta">... </span>    <span class="hljs-string">&quot;But what about second breakfast?&quot;</span>,
<span class="hljs-meta">... </span>    <span class="hljs-string">&quot;Don&#x27;t think he knows about second breakfast, Pip.&quot;</span>,
<span class="hljs-meta">... </span>    <span class="hljs-string">&quot;What about elevensies?&quot;</span>,
<span class="hljs-meta">... </span>]
<span class="hljs-meta">&gt;&gt;&gt; </span>encoded_inputs = tokenizer(batch_sentences)
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-built_in">print</span>(encoded_inputs)
{<span class="hljs-string">&#x27;input_ids&#x27;</span>: [[<span class="hljs-number">101</span>, <span class="hljs-number">1252</span>, <span class="hljs-number">1184</span>, <span class="hljs-number">1164</span>, <span class="hljs-number">1248</span>, <span class="hljs-number">6462</span>, <span class="hljs-number">136</span>, <span class="hljs-number">102</span>], 
               [<span class="hljs-number">101</span>, <span class="hljs-number">1790</span>, <span class="hljs-number">112</span>, <span class="hljs-number">189</span>, <span class="hljs-number">1341</span>, <span class="hljs-number">1119</span>, <span class="hljs-number">3520</span>, <span class="hljs-number">1164</span>, <span class="hljs-number">1248</span>, <span class="hljs-number">6462</span>, <span class="hljs-number">117</span>, <span class="hljs-number">21902</span>, <span class="hljs-number">1643</span>, <span class="hljs-number">119</span>, <span class="hljs-number">102</span>], 
               [<span class="hljs-number">101</span>, <span class="hljs-number">1327</span>, <span class="hljs-number">1164</span>, <span class="hljs-number">5450</span>, <span class="hljs-number">23434</span>, <span class="hljs-number">136</span>, <span class="hljs-number">102</span>]], 
 <span class="hljs-string">&#x27;token_type_ids&#x27;</span>: [[<span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>], 
                    [<span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>], 
                    [<span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>]], 
 <span class="hljs-string">&#x27;attention_mask&#x27;</span>: [[<span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>], 
                    [<span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>], 
                    [<span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>]]}`}}),He=new k({}),Be=new E({props:{code:`batch_sentences = [
    "But what about second breakfast?",
    "Don't think he knows about second breakfast, Pip.",
    "What about elevensies?",
]
encoded_input = tokenizer(batch_sentences, padding=True)
print(encoded_input)`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>batch_sentences = [
<span class="hljs-meta">... </span>    <span class="hljs-string">&quot;But what about second breakfast?&quot;</span>,
<span class="hljs-meta">... </span>    <span class="hljs-string">&quot;Don&#x27;t think he knows about second breakfast, Pip.&quot;</span>,
<span class="hljs-meta">... </span>    <span class="hljs-string">&quot;What about elevensies?&quot;</span>,
<span class="hljs-meta">... </span>]
<span class="hljs-meta">&gt;&gt;&gt; </span>encoded_input = tokenizer(batch_sentences, padding=<span class="hljs-literal">True</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-built_in">print</span>(encoded_input)
{<span class="hljs-string">&#x27;input_ids&#x27;</span>: [[<span class="hljs-number">101</span>, <span class="hljs-number">1252</span>, <span class="hljs-number">1184</span>, <span class="hljs-number">1164</span>, <span class="hljs-number">1248</span>, <span class="hljs-number">6462</span>, <span class="hljs-number">136</span>, <span class="hljs-number">102</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>], 
               [<span class="hljs-number">101</span>, <span class="hljs-number">1790</span>, <span class="hljs-number">112</span>, <span class="hljs-number">189</span>, <span class="hljs-number">1341</span>, <span class="hljs-number">1119</span>, <span class="hljs-number">3520</span>, <span class="hljs-number">1164</span>, <span class="hljs-number">1248</span>, <span class="hljs-number">6462</span>, <span class="hljs-number">117</span>, <span class="hljs-number">21902</span>, <span class="hljs-number">1643</span>, <span class="hljs-number">119</span>, <span class="hljs-number">102</span>], 
               [<span class="hljs-number">101</span>, <span class="hljs-number">1327</span>, <span class="hljs-number">1164</span>, <span class="hljs-number">5450</span>, <span class="hljs-number">23434</span>, <span class="hljs-number">136</span>, <span class="hljs-number">102</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>]], 
 <span class="hljs-string">&#x27;token_type_ids&#x27;</span>: [[<span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>], 
                    [<span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>], 
                    [<span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>]], 
 <span class="hljs-string">&#x27;attention_mask&#x27;</span>: [[<span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>], 
                    [<span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>], 
                    [<span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>]]}`}}),Je=new k({}),We=new E({props:{code:`batch_sentences = [
    "But what about second breakfast?",
    "Don't think he knows about second breakfast, Pip.",
    "What about elevensies?",
]
encoded_input = tokenizer(batch_sentences, padding=True, truncation=True)
print(encoded_input)`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>batch_sentences = [
<span class="hljs-meta">... </span>    <span class="hljs-string">&quot;But what about second breakfast?&quot;</span>,
<span class="hljs-meta">... </span>    <span class="hljs-string">&quot;Don&#x27;t think he knows about second breakfast, Pip.&quot;</span>,
<span class="hljs-meta">... </span>    <span class="hljs-string">&quot;What about elevensies?&quot;</span>,
<span class="hljs-meta">... </span>]
<span class="hljs-meta">&gt;&gt;&gt; </span>encoded_input = tokenizer(batch_sentences, padding=<span class="hljs-literal">True</span>, truncation=<span class="hljs-literal">True</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-built_in">print</span>(encoded_input)
{<span class="hljs-string">&#x27;input_ids&#x27;</span>: [[<span class="hljs-number">101</span>, <span class="hljs-number">1252</span>, <span class="hljs-number">1184</span>, <span class="hljs-number">1164</span>, <span class="hljs-number">1248</span>, <span class="hljs-number">6462</span>, <span class="hljs-number">136</span>, <span class="hljs-number">102</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>], 
               [<span class="hljs-number">101</span>, <span class="hljs-number">1790</span>, <span class="hljs-number">112</span>, <span class="hljs-number">189</span>, <span class="hljs-number">1341</span>, <span class="hljs-number">1119</span>, <span class="hljs-number">3520</span>, <span class="hljs-number">1164</span>, <span class="hljs-number">1248</span>, <span class="hljs-number">6462</span>, <span class="hljs-number">117</span>, <span class="hljs-number">21902</span>, <span class="hljs-number">1643</span>, <span class="hljs-number">119</span>, <span class="hljs-number">102</span>], 
               [<span class="hljs-number">101</span>, <span class="hljs-number">1327</span>, <span class="hljs-number">1164</span>, <span class="hljs-number">5450</span>, <span class="hljs-number">23434</span>, <span class="hljs-number">136</span>, <span class="hljs-number">102</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>]], 
 <span class="hljs-string">&#x27;token_type_ids&#x27;</span>: [[<span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>], 
                    [<span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>], 
                    [<span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>]], 
 <span class="hljs-string">&#x27;attention_mask&#x27;</span>: [[<span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>], 
                    [<span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>], 
                    [<span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>]]}`}}),Ye=new k({}),Ge=new $v({props:{group1:{id:"pt",code:`batch_sentences = [
    "But what about second breakfast?",
    "Don't think he knows about second breakfast, Pip.",
    "What about elevensies?",
]
encoded_input = tokenizer(batch, padding=True, truncation=True, return_tensors="pt")
print(encoded_input)`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>batch_sentences = [
<span class="hljs-meta">... </span>    <span class="hljs-string">&quot;But what about second breakfast?&quot;</span>,
<span class="hljs-meta">... </span>    <span class="hljs-string">&quot;Don&#x27;t think he knows about second breakfast, Pip.&quot;</span>,
<span class="hljs-meta">... </span>    <span class="hljs-string">&quot;What about elevensies?&quot;</span>,
<span class="hljs-meta">... </span>]
<span class="hljs-meta">&gt;&gt;&gt; </span>encoded_input = tokenizer(batch, padding=<span class="hljs-literal">True</span>, truncation=<span class="hljs-literal">True</span>, return_tensors=<span class="hljs-string">&quot;pt&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-built_in">print</span>(encoded_input)
{<span class="hljs-string">&#x27;input_ids&#x27;</span>: tensor([[  <span class="hljs-number">101</span>,   <span class="hljs-number">153</span>,  <span class="hljs-number">7719</span>, <span class="hljs-number">21490</span>,  <span class="hljs-number">1122</span>,  <span class="hljs-number">1114</span>,  <span class="hljs-number">9582</span>,  <span class="hljs-number">1623</span>,   <span class="hljs-number">102</span>],
                      [  <span class="hljs-number">101</span>,  <span class="hljs-number">5226</span>,  <span class="hljs-number">1122</span>,  <span class="hljs-number">9649</span>,  <span class="hljs-number">1199</span>,  <span class="hljs-number">2610</span>,  <span class="hljs-number">1236</span>,   <span class="hljs-number">102</span>,     <span class="hljs-number">0</span>]]), 
 <span class="hljs-string">&#x27;token_type_ids&#x27;</span>: tensor([[<span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>],
                           [<span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>]]), 
 <span class="hljs-string">&#x27;attention_mask&#x27;</span>: tensor([[<span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>],
                           [<span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">0</span>]])}`},group2:{id:"tf",code:`batch_sentences = [
    "But what about second breakfast?",
    "Don't think he knows about second breakfast, Pip.",
    "What about elevensies?",
]
encoded_input = tokenizer(batch, padding=True, truncation=True, return_tensors="tf")
print(encoded_input)`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>batch_sentences = [
<span class="hljs-meta">... </span>    <span class="hljs-string">&quot;But what about second breakfast?&quot;</span>,
<span class="hljs-meta">... </span>    <span class="hljs-string">&quot;Don&#x27;t think he knows about second breakfast, Pip.&quot;</span>,
<span class="hljs-meta">... </span>    <span class="hljs-string">&quot;What about elevensies?&quot;</span>,
<span class="hljs-meta">... </span>]
<span class="hljs-meta">&gt;&gt;&gt; </span>encoded_input = tokenizer(batch, padding=<span class="hljs-literal">True</span>, truncation=<span class="hljs-literal">True</span>, return_tensors=<span class="hljs-string">&quot;tf&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-built_in">print</span>(encoded_input)
{<span class="hljs-string">&#x27;input_ids&#x27;</span>: &lt;tf.Tensor: shape=(<span class="hljs-number">2</span>, <span class="hljs-number">9</span>), dtype=int32, numpy=
array([[  <span class="hljs-number">101</span>,   <span class="hljs-number">153</span>,  <span class="hljs-number">7719</span>, <span class="hljs-number">21490</span>,  <span class="hljs-number">1122</span>,  <span class="hljs-number">1114</span>,  <span class="hljs-number">9582</span>,  <span class="hljs-number">1623</span>,   <span class="hljs-number">102</span>],
       [  <span class="hljs-number">101</span>,  <span class="hljs-number">5226</span>,  <span class="hljs-number">1122</span>,  <span class="hljs-number">9649</span>,  <span class="hljs-number">1199</span>,  <span class="hljs-number">2610</span>,  <span class="hljs-number">1236</span>,   <span class="hljs-number">102</span>,     <span class="hljs-number">0</span>]],
      dtype=int32)&gt;, 
 <span class="hljs-string">&#x27;token_type_ids&#x27;</span>: &lt;tf.Tensor: shape=(<span class="hljs-number">2</span>, <span class="hljs-number">9</span>), dtype=int32, numpy=
array([[<span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>],
       [<span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>]], dtype=int32)&gt;, 
 <span class="hljs-string">&#x27;attention_mask&#x27;</span>: &lt;tf.Tensor: shape=(<span class="hljs-number">2</span>, <span class="hljs-number">9</span>), dtype=int32, numpy=
array([[<span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>],
       [<span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">0</span>]], dtype=int32)&gt;}`}}}),Ue=new k({}),Me=new E({props:{code:"pip install datasets",highlighted:"pip install datasets"}}),Qe=new E({props:{code:`from datasets import load_dataset, Audio

dataset = load_dataset("superb", "ks")`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> datasets <span class="hljs-keyword">import</span> load_dataset, Audio

<span class="hljs-meta">&gt;&gt;&gt; </span>dataset = load_dataset(<span class="hljs-string">&quot;superb&quot;</span>, <span class="hljs-string">&quot;ks&quot;</span>)`}}),Xe=new E({props:{code:'dataset["train"][0]["audio"]',highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>dataset[<span class="hljs-string">&quot;train&quot;</span>][<span class="hljs-number">0</span>][<span class="hljs-string">&quot;audio&quot;</span>]
{<span class="hljs-string">&#x27;array&#x27;</span>: array([ <span class="hljs-number">0.</span>        ,  <span class="hljs-number">0.</span>        ,  <span class="hljs-number">0.</span>        , ..., -<span class="hljs-number">0.00592041</span>,
        -<span class="hljs-number">0.00405884</span>, -<span class="hljs-number">0.00253296</span>], dtype=float32),
 <span class="hljs-string">&#x27;path&#x27;</span>: <span class="hljs-string">&#x27;/root/.cache/huggingface/datasets/downloads/extracted/05734a36d88019a09725c20cc024e1c4e7982e37d7d55c0c1ca1742ea1cdd47f/_background_noise_/doing_the_dishes.wav&#x27;</span>,
 <span class="hljs-string">&#x27;sampling_rate&#x27;</span>: <span class="hljs-number">16000</span>}`}}),Ze=new k({}),aa=new E({props:{code:`lj_speech = load_dataset("lj_speech", split="train")
lj_speech[0]["audio"]`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>lj_speech = load_dataset(<span class="hljs-string">&quot;lj_speech&quot;</span>, split=<span class="hljs-string">&quot;train&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>lj_speech[<span class="hljs-number">0</span>][<span class="hljs-string">&quot;audio&quot;</span>]
{<span class="hljs-string">&#x27;array&#x27;</span>: array([-<span class="hljs-number">7.3242188e-04</span>, -<span class="hljs-number">7.6293945e-04</span>, -<span class="hljs-number">6.4086914e-04</span>, ...,
         <span class="hljs-number">7.3242188e-04</span>,  <span class="hljs-number">2.1362305e-04</span>,  <span class="hljs-number">6.1035156e-05</span>], dtype=float32),
 <span class="hljs-string">&#x27;path&#x27;</span>: <span class="hljs-string">&#x27;/root/.cache/huggingface/datasets/downloads/extracted/917ece08c95cf0c4115e45294e3cd0dee724a1165b7fc11798369308a465bd26/LJSpeech-1.1/wavs/LJ001-0001.wav&#x27;</span>,
 <span class="hljs-string">&#x27;sampling_rate&#x27;</span>: <span class="hljs-number">22050</span>}`}}),la=new E({props:{code:'lj_speech = lj_speech.cast_column("audio", Audio(sampling_rate=16_000))',highlighted:'<span class="hljs-meta">&gt;&gt;&gt; </span>lj_speech = lj_speech.cast_column(<span class="hljs-string">&quot;audio&quot;</span>, Audio(sampling_rate=<span class="hljs-number">16_000</span>))'}}),oa=new E({props:{code:'lj_speech[0]["audio"]',highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>lj_speech[<span class="hljs-number">0</span>][<span class="hljs-string">&quot;audio&quot;</span>]
{<span class="hljs-string">&#x27;array&#x27;</span>: array([-<span class="hljs-number">0.00064146</span>, -<span class="hljs-number">0.00074657</span>, -<span class="hljs-number">0.00068768</span>, ...,  <span class="hljs-number">0.00068341</span>,
         <span class="hljs-number">0.00014045</span>,  <span class="hljs-number">0.</span>        ], dtype=float32),
 <span class="hljs-string">&#x27;path&#x27;</span>: <span class="hljs-string">&#x27;/root/.cache/huggingface/datasets/downloads/extracted/917ece08c95cf0c4115e45294e3cd0dee724a1165b7fc11798369308a465bd26/LJSpeech-1.1/wavs/LJ001-0001.wav&#x27;</span>,
 <span class="hljs-string">&#x27;sampling_rate&#x27;</span>: <span class="hljs-number">16000</span>}`}}),pa=new k({}),ca=new E({props:{code:`from transformers import AutoFeatureExtractor

feature_extractor = AutoFeatureExtractor.from_pretrained("facebook/wav2vec2-base")`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoFeatureExtractor

<span class="hljs-meta">&gt;&gt;&gt; </span>feature_extractor = AutoFeatureExtractor.from_pretrained(<span class="hljs-string">&quot;facebook/wav2vec2-base&quot;</span>)`}}),ha=new E({props:{code:`audio_input = [dataset["train"][0]["audio"]["array"]]
feature_extractor(audio_input, sampling_rate=16000)`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>audio_input = [dataset[<span class="hljs-string">&quot;train&quot;</span>][<span class="hljs-number">0</span>][<span class="hljs-string">&quot;audio&quot;</span>][<span class="hljs-string">&quot;array&quot;</span>]]
<span class="hljs-meta">&gt;&gt;&gt; </span>feature_extractor(audio_input, sampling_rate=<span class="hljs-number">16000</span>)
{<span class="hljs-string">&#x27;input_values&#x27;</span>: [array([ <span class="hljs-number">0.00045439</span>,  <span class="hljs-number">0.00045439</span>,  <span class="hljs-number">0.00045439</span>, ..., -<span class="hljs-number">0.1578519</span> , -<span class="hljs-number">0.10807519</span>, -<span class="hljs-number">0.06727459</span>], dtype=float32)]}`}}),ia=new k({}),ua=new E({props:{code:`dataset["train"][0]["audio"]["array"].shape

dataset["train"][1]["audio"]["array"].shape`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>dataset[<span class="hljs-string">&quot;train&quot;</span>][<span class="hljs-number">0</span>][<span class="hljs-string">&quot;audio&quot;</span>][<span class="hljs-string">&quot;array&quot;</span>].shape
(<span class="hljs-number">1522930</span>,)

<span class="hljs-meta">&gt;&gt;&gt; </span>dataset[<span class="hljs-string">&quot;train&quot;</span>][<span class="hljs-number">1</span>][<span class="hljs-string">&quot;audio&quot;</span>][<span class="hljs-string">&quot;array&quot;</span>].shape
(<span class="hljs-number">988891</span>,)`}}),ma=new E({props:{code:`def preprocess_function(examples):
    audio_arrays = [x["array"] for x in examples["audio"]]
    inputs = feature_extractor(
        audio_arrays,
        sampling_rate=16000,
        padding=True,
        max_length=1000000,
        truncation=True,
    )
    return inputs`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">def</span> <span class="hljs-title function_">preprocess_function</span>(<span class="hljs-params">examples</span>):
<span class="hljs-meta">... </span>    audio_arrays = [x[<span class="hljs-string">&quot;array&quot;</span>] <span class="hljs-keyword">for</span> x <span class="hljs-keyword">in</span> examples[<span class="hljs-string">&quot;audio&quot;</span>]]
<span class="hljs-meta">... </span>    inputs = feature_extractor(
<span class="hljs-meta">... </span>        audio_arrays,
<span class="hljs-meta">... </span>        sampling_rate=<span class="hljs-number">16000</span>,
<span class="hljs-meta">... </span>        padding=<span class="hljs-literal">True</span>,
<span class="hljs-meta">... </span>        max_length=<span class="hljs-number">1000000</span>,
<span class="hljs-meta">... </span>        truncation=<span class="hljs-literal">True</span>,
<span class="hljs-meta">... </span>    )
<span class="hljs-meta">... </span>    <span class="hljs-keyword">return</span> inputs`}}),da=new E({props:{code:'processed_dataset = preprocess_function(dataset["train"][:5])',highlighted:'<span class="hljs-meta">&gt;&gt;&gt; </span>processed_dataset = preprocess_function(dataset[<span class="hljs-string">&quot;train&quot;</span>][:<span class="hljs-number">5</span>])'}}),fa=new E({props:{code:`processed_dataset["input_values"][0].shape

processed_dataset["input_values"][1].shape`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>processed_dataset[<span class="hljs-string">&quot;input_values&quot;</span>][<span class="hljs-number">0</span>].shape
(<span class="hljs-number">1000000</span>,)

<span class="hljs-meta">&gt;&gt;&gt; </span>processed_dataset[<span class="hljs-string">&quot;input_values&quot;</span>][<span class="hljs-number">1</span>].shape
(<span class="hljs-number">1000000</span>,)`}}),ba=new k({}),_a=new E({props:{code:`from datasets import load_dataset

dataset = load_dataset("food101", split="train[:100]")`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> datasets <span class="hljs-keyword">import</span> load_dataset

<span class="hljs-meta">&gt;&gt;&gt; </span>dataset = load_dataset(<span class="hljs-string">&quot;food101&quot;</span>, split=<span class="hljs-string">&quot;train[:100]&quot;</span>)`}}),va=new E({props:{code:'dataset[0]["image"]',highlighted:'<span class="hljs-meta">&gt;&gt;&gt; </span>dataset[<span class="hljs-number">0</span>][<span class="hljs-string">&quot;image&quot;</span>]'}}),Ea=new k({}),wa=new E({props:{code:`from transformers import AutoFeatureExtractor

feature_extractor = AutoFeatureExtractor.from_pretrained("google/vit-base-patch16-224")`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoFeatureExtractor

<span class="hljs-meta">&gt;&gt;&gt; </span>feature_extractor = AutoFeatureExtractor.from_pretrained(<span class="hljs-string">&quot;google/vit-base-patch16-224&quot;</span>)`}}),$a=new k({}),qa=new E({props:{code:`from torchvision.transforms import Compose, Normalize, RandomResizedCrop, ColorJitter, ToTensor

normalize = Normalize(mean=feature_extractor.image_mean, std=feature_extractor.image_std)
_transforms = Compose(
    [RandomResizedCrop(feature_extractor.size), ColorJitter(brightness=0.5, hue=0.5), ToTensor(), normalize]
)`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> torchvision.transforms <span class="hljs-keyword">import</span> Compose, Normalize, RandomResizedCrop, ColorJitter, ToTensor

<span class="hljs-meta">&gt;&gt;&gt; </span>normalize = Normalize(mean=feature_extractor.image_mean, std=feature_extractor.image_std)
<span class="hljs-meta">&gt;&gt;&gt; </span>_transforms = Compose(
<span class="hljs-meta">... </span>    [RandomResizedCrop(feature_extractor.size), ColorJitter(brightness=<span class="hljs-number">0.5</span>, hue=<span class="hljs-number">0.5</span>), ToTensor(), normalize]
<span class="hljs-meta">... </span>)`}}),Aa=new E({props:{code:`def transforms(examples):
    examples["pixel_values"] = [_transforms(image.convert("RGB")) for image in examples["image"]]
    return examples`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">def</span> <span class="hljs-title function_">transforms</span>(<span class="hljs-params">examples</span>):
<span class="hljs-meta">... </span>    examples[<span class="hljs-string">&quot;pixel_values&quot;</span>] = [_transforms(image.convert(<span class="hljs-string">&quot;RGB&quot;</span>)) <span class="hljs-keyword">for</span> image <span class="hljs-keyword">in</span> examples[<span class="hljs-string">&quot;image&quot;</span>]]
<span class="hljs-meta">... </span>    <span class="hljs-keyword">return</span> examples`}}),Oa=new E({props:{code:"dataset.set_transform(transforms)",highlighted:'<span class="hljs-meta">&gt;&gt;&gt; </span>dataset.set_transform(transforms)'}}),Na=new E({props:{code:'dataset[0]["image"]',highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>dataset[<span class="hljs-number">0</span>][<span class="hljs-string">&quot;image&quot;</span>]
{<span class="hljs-string">&#x27;image&#x27;</span>: &lt;PIL.JpegImagePlugin.JpegImageFile image mode=RGB size=384x512 at <span class="hljs-number">0x7F1A7B0630D0</span>&gt;,
 <span class="hljs-string">&#x27;label&#x27;</span>: <span class="hljs-number">6</span>,
 <span class="hljs-string">&#x27;pixel_values&#x27;</span>: tensor([[[ <span class="hljs-number">0.0353</span>,  <span class="hljs-number">0.0745</span>,  <span class="hljs-number">0.1216</span>,  ..., -<span class="hljs-number">0.9922</span>, -<span class="hljs-number">0.9922</span>, -<span class="hljs-number">0.9922</span>],
          [-<span class="hljs-number">0.0196</span>,  <span class="hljs-number">0.0667</span>,  <span class="hljs-number">0.1294</span>,  ..., -<span class="hljs-number">0.9765</span>, -<span class="hljs-number">0.9843</span>, -<span class="hljs-number">0.9922</span>],
          [ <span class="hljs-number">0.0196</span>,  <span class="hljs-number">0.0824</span>,  <span class="hljs-number">0.1137</span>,  ..., -<span class="hljs-number">0.9765</span>, -<span class="hljs-number">0.9686</span>, -<span class="hljs-number">0.8667</span>],
          ...,
          [ <span class="hljs-number">0.0275</span>,  <span class="hljs-number">0.0745</span>,  <span class="hljs-number">0.0510</span>,  ..., -<span class="hljs-number">0.1137</span>, -<span class="hljs-number">0.1216</span>, -<span class="hljs-number">0.0824</span>],
          [ <span class="hljs-number">0.0667</span>,  <span class="hljs-number">0.0824</span>,  <span class="hljs-number">0.0667</span>,  ..., -<span class="hljs-number">0.0588</span>, -<span class="hljs-number">0.0745</span>, -<span class="hljs-number">0.0980</span>],
          [ <span class="hljs-number">0.0353</span>,  <span class="hljs-number">0.0353</span>,  <span class="hljs-number">0.0431</span>,  ..., -<span class="hljs-number">0.0039</span>, -<span class="hljs-number">0.0039</span>, -<span class="hljs-number">0.0588</span>]],
 
         [[ <span class="hljs-number">0.2078</span>,  <span class="hljs-number">0.2471</span>,  <span class="hljs-number">0.2863</span>,  ..., -<span class="hljs-number">0.9451</span>, -<span class="hljs-number">0.9373</span>, -<span class="hljs-number">0.9451</span>],
          [ <span class="hljs-number">0.1608</span>,  <span class="hljs-number">0.2471</span>,  <span class="hljs-number">0.3098</span>,  ..., -<span class="hljs-number">0.9373</span>, -<span class="hljs-number">0.9451</span>, -<span class="hljs-number">0.9373</span>],
          [ <span class="hljs-number">0.2078</span>,  <span class="hljs-number">0.2706</span>,  <span class="hljs-number">0.3020</span>,  ..., -<span class="hljs-number">0.9608</span>, -<span class="hljs-number">0.9373</span>, -<span class="hljs-number">0.8275</span>],
          ...,
          [-<span class="hljs-number">0.0353</span>,  <span class="hljs-number">0.0118</span>, -<span class="hljs-number">0.0039</span>,  ..., -<span class="hljs-number">0.2392</span>, -<span class="hljs-number">0.2471</span>, -<span class="hljs-number">0.2078</span>],
          [ <span class="hljs-number">0.0196</span>,  <span class="hljs-number">0.0353</span>,  <span class="hljs-number">0.0196</span>,  ..., -<span class="hljs-number">0.1843</span>, -<span class="hljs-number">0.2000</span>, -<span class="hljs-number">0.2235</span>],
          [-<span class="hljs-number">0.0118</span>, -<span class="hljs-number">0.0039</span>, -<span class="hljs-number">0.0039</span>,  ..., -<span class="hljs-number">0.0980</span>, -<span class="hljs-number">0.0980</span>, -<span class="hljs-number">0.1529</span>]],
 
         [[ <span class="hljs-number">0.3961</span>,  <span class="hljs-number">0.4431</span>,  <span class="hljs-number">0.4980</span>,  ..., -<span class="hljs-number">0.9216</span>, -<span class="hljs-number">0.9137</span>, -<span class="hljs-number">0.9216</span>],
          [ <span class="hljs-number">0.3569</span>,  <span class="hljs-number">0.4510</span>,  <span class="hljs-number">0.5216</span>,  ..., -<span class="hljs-number">0.9059</span>, -<span class="hljs-number">0.9137</span>, -<span class="hljs-number">0.9137</span>],
          [ <span class="hljs-number">0.4118</span>,  <span class="hljs-number">0.4745</span>,  <span class="hljs-number">0.5216</span>,  ..., -<span class="hljs-number">0.9137</span>, -<span class="hljs-number">0.8902</span>, -<span class="hljs-number">0.7804</span>],
          ...,
          [-<span class="hljs-number">0.2314</span>, -<span class="hljs-number">0.1922</span>, -<span class="hljs-number">0.2078</span>,  ..., -<span class="hljs-number">0.4196</span>, -<span class="hljs-number">0.4275</span>, -<span class="hljs-number">0.3882</span>],
          [-<span class="hljs-number">0.1843</span>, -<span class="hljs-number">0.1686</span>, -<span class="hljs-number">0.2000</span>,  ..., -<span class="hljs-number">0.3647</span>, -<span class="hljs-number">0.3804</span>, -<span class="hljs-number">0.4039</span>],
          [-<span class="hljs-number">0.1922</span>, -<span class="hljs-number">0.1922</span>, -<span class="hljs-number">0.1922</span>,  ..., -<span class="hljs-number">0.2941</span>, -<span class="hljs-number">0.2863</span>, -<span class="hljs-number">0.3412</span>]]])}`}}),Ia=new E({props:{code:`import numpy as np
import matplotlib.pyplot as plt

img = dataset[0]["pixel_values"]
plt.imshow(img.permute(1, 2, 0))`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">import</span> numpy <span class="hljs-keyword">as</span> np
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">import</span> matplotlib.pyplot <span class="hljs-keyword">as</span> plt

<span class="hljs-meta">&gt;&gt;&gt; </span>img = dataset[<span class="hljs-number">0</span>][<span class="hljs-string">&quot;pixel_values&quot;</span>]
<span class="hljs-meta">&gt;&gt;&gt; </span>plt.imshow(img.permute(<span class="hljs-number">1</span>, <span class="hljs-number">2</span>, <span class="hljs-number">0</span>))`}}),Ra=new k({}),Ha=new E({props:{code:`from datasets import load_dataset

lj_speech = load_dataset("lj_speech", split="train")`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> datasets <span class="hljs-keyword">import</span> load_dataset

<span class="hljs-meta">&gt;&gt;&gt; </span>lj_speech = load_dataset(<span class="hljs-string">&quot;lj_speech&quot;</span>, split=<span class="hljs-string">&quot;train&quot;</span>)`}}),Ba=new E({props:{code:'lj_speech = lj_speech.map(remove_columns=["file", "id", "normalized_text"])',highlighted:'<span class="hljs-meta">&gt;&gt;&gt; </span>lj_speech = lj_speech.<span class="hljs-built_in">map</span>(remove_columns=[<span class="hljs-string">&quot;file&quot;</span>, <span class="hljs-string">&quot;id&quot;</span>, <span class="hljs-string">&quot;normalized_text&quot;</span>])'}}),Ja=new E({props:{code:`lj_speech[0]["audio"]

lj_speech[0]["text"]`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>lj_speech[<span class="hljs-number">0</span>][<span class="hljs-string">&quot;audio&quot;</span>]
{<span class="hljs-string">&#x27;array&#x27;</span>: array([-<span class="hljs-number">7.3242188e-04</span>, -<span class="hljs-number">7.6293945e-04</span>, -<span class="hljs-number">6.4086914e-04</span>, ...,
         <span class="hljs-number">7.3242188e-04</span>,  <span class="hljs-number">2.1362305e-04</span>,  <span class="hljs-number">6.1035156e-05</span>], dtype=float32),
 <span class="hljs-string">&#x27;path&#x27;</span>: <span class="hljs-string">&#x27;/root/.cache/huggingface/datasets/downloads/extracted/917ece08c95cf0c4115e45294e3cd0dee724a1165b7fc11798369308a465bd26/LJSpeech-1.1/wavs/LJ001-0001.wav&#x27;</span>,
 <span class="hljs-string">&#x27;sampling_rate&#x27;</span>: <span class="hljs-number">22050</span>}

<span class="hljs-meta">&gt;&gt;&gt; </span>lj_speech[<span class="hljs-number">0</span>][<span class="hljs-string">&quot;text&quot;</span>]
<span class="hljs-string">&#x27;Printing, in the only sense with which we are at present concerned, differs from most if not from all the arts and crafts represented in the Exhibition&#x27;</span>`}}),Wa=new E({props:{code:'lj_speech = lj_speech.cast_column("audio", Audio(sampling_rate=16_000))',highlighted:'<span class="hljs-meta">&gt;&gt;&gt; </span>lj_speech = lj_speech.cast_column(<span class="hljs-string">&quot;audio&quot;</span>, Audio(sampling_rate=<span class="hljs-number">16_000</span>))'}}),Ya=new k({}),Ga=new E({props:{code:`from transformers import AutoProcessor

processor = AutoProcessor.from_pretrained("facebook/wav2vec2-base-960h")`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoProcessor

<span class="hljs-meta">&gt;&gt;&gt; </span>processor = AutoProcessor.from_pretrained(<span class="hljs-string">&quot;facebook/wav2vec2-base-960h&quot;</span>)`}}),Ua=new E({props:{code:`def prepare_dataset(example):
    audio = example["audio"]

    example["input_values"] = processor(audio["array"], sampling_rate=16000)

    with processor.as_target_processor():
        example["labels"] = processor(example["text"]).input_ids
    return example`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">def</span> <span class="hljs-title function_">prepare_dataset</span>(<span class="hljs-params">example</span>):
<span class="hljs-meta">... </span>    audio = example[<span class="hljs-string">&quot;audio&quot;</span>]

<span class="hljs-meta">... </span>    example[<span class="hljs-string">&quot;input_values&quot;</span>] = processor(audio[<span class="hljs-string">&quot;array&quot;</span>], sampling_rate=<span class="hljs-number">16000</span>)

<span class="hljs-meta">... </span>    <span class="hljs-keyword">with</span> processor.as_target_processor():
<span class="hljs-meta">... </span>        example[<span class="hljs-string">&quot;labels&quot;</span>] = processor(example[<span class="hljs-string">&quot;text&quot;</span>]).input_ids
<span class="hljs-meta">... </span>    <span class="hljs-keyword">return</span> example`}}),Ka=new E({props:{code:"prepare_dataset(lj_speech[0])",highlighted:'<span class="hljs-meta">&gt;&gt;&gt; </span>prepare_dataset(lj_speech[<span class="hljs-number">0</span>])'}}),Qa=new k({}),{c(){w=t("meta"),R=c(),$=t("h1"),x=t("a"),ns=t("span"),m(q.$$.fragment),F=c(),Gs=t("span"),oi=r("Preprocess"),Uo=c(),m(Ce.$$.fragment),Mo=c(),at=t("p"),pi=r("Before you can use your data in a model, the data needs to be processed into an acceptable format for the model. A model does not understand raw text, images or audio. These inputs need to be converted into numbers and assembled into tensors. In this tutorial, you will:"),Vo=c(),H=t("ul"),Rn=t("li"),ci=r("Preprocess textual data with a tokenizer."),hi=c(),Fn=t("li"),ii=r("Preprocess image or audio data with a feature extractor."),ui=c(),Hn=t("li"),mi=r("Preprocess data for a multimodal task with a processor."),Ko=c(),ls=t("h2"),Us=t("a"),Bn=t("span"),m(Oe.$$.fragment),di=c(),Jn=t("span"),fi=r("NLP"),Qo=c(),m(Le.$$.fragment),Xo=c(),B=t("p"),bi=r("The main tool for processing textual data is a "),tt=t("a"),ji=r("tokenizer"),_i=r(". A tokenizer starts by splitting text into "),Wn=t("em"),gi=r("tokens"),vi=r(" according to a set of rules. The tokens are converted into numbers, which are used to build tensors as input to a model. Any additional inputs required by a model are also added by the tokenizer."),Zo=c(),m(Ms.$$.fragment),sp=c(),J=t("p"),Ei=r("Get started quickly by loading a pretrained tokenizer with the "),nt=t("a"),wi=r("AutoTokenizer"),$i=r(" class. This downloads the "),Yn=t("em"),yi=r("vocab"),ki=r(" used when a model is pretrained."),ep=c(),rs=t("h3"),Vs=t("a"),Gn=t("span"),m(Se.$$.fragment),xi=c(),Un=t("span"),Ti=r("Tokenize"),ap=c(),Ks=t("p"),qi=r("Load a pretrained tokenizer with "),lt=t("a"),Di=r("AutoTokenizer.from_pretrained()"),Ai=r(":"),tp=c(),m(Ne.$$.fragment),np=c(),rt=t("p"),zi=r("Then pass your sentence to the tokenizer:"),lp=c(),m(Ie.$$.fragment),rp=c(),ot=t("p"),Pi=r("The tokenizer returns a dictionary with three important itmes:"),op=c(),W=t("ul"),pt=t("li"),ct=t("a"),Ci=r("input_ids"),Oi=r(" are the indices corresponding to each token in the sentence."),Li=c(),ht=t("li"),it=t("a"),Si=r("attention_mask"),Ni=r(" indicates whether a token should be attended to or not."),Ii=c(),ut=t("li"),mt=t("a"),Ri=r("token_type_ids"),Fi=r(" identifies which sequence a token belongs to when there is more than one sequence."),pp=c(),Qs=t("p"),Hi=r("You can decode the "),Mn=t("code"),Bi=r("input_ids"),Ji=r(" to return the original input:"),cp=c(),m(Re.$$.fragment),hp=c(),Y=t("p"),Wi=r("As you can see, the tokenizer added two special tokens - "),Vn=t("code"),Yi=r("CLS"),Gi=r(" and "),Kn=t("code"),Ui=r("SEP"),Mi=r(` (classifier and separator) - to the sentence. Not all models need
special tokens, but if they do, the tokenizer will automatically add them for you.`),ip=c(),dt=t("p"),Vi=r("If there are several sentences you want to process, pass the sentences as a list to the tokenizer:"),up=c(),m(Fe.$$.fragment),mp=c(),os=t("h3"),Xs=t("a"),Qn=t("span"),m(He.$$.fragment),Ki=c(),Xn=t("span"),Qi=r("Pad"),dp=c(),Zs=t("p"),Xi=r("This brings us to an important topic. When you process a batch of sentences, they aren\u2019t always the same length. This is a problem because tensors, the input to the model, need to have a uniform shape. Padding is a strategy for ensuring tensors are rectangular by adding a special "),Zn=t("em"),Zi=r("padding token"),su=r(" to sentences with fewer tokens."),fp=c(),G=t("p"),eu=r("Set the "),sl=t("code"),au=r("padding"),tu=r(" parameter to "),el=t("code"),nu=r("True"),lu=r(" to pad the shorter sequences in the batch to match the longest sequence:"),bp=c(),m(Be.$$.fragment),jp=c(),se=t("p"),ru=r("Notice the tokenizer padded the first and third sentences with a "),al=t("code"),ou=r("0"),pu=r(" because they are shorter!"),_p=c(),ps=t("h3"),ee=t("a"),tl=t("span"),m(Je.$$.fragment),cu=c(),nl=t("span"),hu=r("Truncation"),gp=c(),ft=t("p"),iu=r("On the other end of the spectrum, sometimes a sequence may be too long for a model to handle. In this case, you will need to truncate the sequence to a shorter length."),vp=c(),U=t("p"),uu=r("Set the "),ll=t("code"),mu=r("truncation"),du=r(" parameter to "),rl=t("code"),fu=r("True"),bu=r(" to truncate a sequence to the maximum length accepted by the model:"),Ep=c(),m(We.$$.fragment),wp=c(),cs=t("h3"),ae=t("a"),ol=t("span"),m(Ye.$$.fragment),ju=c(),pl=t("span"),_u=r("Build tensors"),$p=c(),bt=t("p"),gu=r("Finally, you want the tokenizer to return the actual tensors that are fed to the model."),yp=c(),D=t("p"),vu=r("Set the "),cl=t("code"),Eu=r("return_tensors"),wu=r(" parameter to either "),hl=t("code"),$u=r("pt"),yu=r(" for PyTorch, or "),il=t("code"),ku=r("tf"),xu=r(" for TensorFlow:"),kp=c(),m(Ge.$$.fragment),xp=c(),hs=t("h2"),te=t("a"),ul=t("span"),m(Ue.$$.fragment),Tu=c(),ml=t("span"),qu=r("Audio"),Tp=c(),ne=t("p"),Du=r("Audio inputs are preprocessed differently than textual inputs, but the end goal remains the same: create numerical sequences the model can understand. A "),jt=t("a"),Au=r("feature extractor"),zu=r(" is designed for the express purpose of extracting features from raw image or audio data and converting them into tensors. Before you begin, install \u{1F917} Datasets to load an audio dataset to experiment with:"),qp=c(),m(Me.$$.fragment),Dp=c(),M=t("p"),Pu=r("Load the keyword spotting task from the "),Ve=t("a"),Cu=r("SUPERB"),Ou=r(" benchmark (see the \u{1F917} "),Ke=t("a"),Lu=r("Datasets tutorial"),Su=r(" for more details on how to load a dataset):"),Ap=c(),m(Qe.$$.fragment),zp=c(),V=t("p"),Nu=r("Access the first element of the "),dl=t("code"),Iu=r("audio"),Ru=r(" column to take a look at the input. Calling the "),fl=t("code"),Fu=r("audio"),Hu=r(" column will automatically load and resample the audio file:"),Pp=c(),m(Xe.$$.fragment),Cp=c(),_t=t("p"),Bu=r("This returns three items:"),Op=c(),K=t("ul"),gt=t("li"),bl=t("code"),Ju=r("array"),Wu=r(" is the speech signal loaded - and potentially resampled - as a 1D array."),Yu=c(),vt=t("li"),jl=t("code"),Gu=r("path"),Uu=r(" points to the location of the audio file."),Mu=c(),Et=t("li"),_l=t("code"),Vu=r("sampling_rate"),Ku=r(" refers to how many data points in the speech signal are measured per second."),Lp=c(),is=t("h3"),le=t("a"),gl=t("span"),m(Ze.$$.fragment),Qu=c(),vl=t("span"),Xu=r("Resample"),Sp=c(),re=t("p"),Zu=r("For this tutorial, you will use the "),sa=t("a"),sm=r("Wav2Vec2"),em=r(" model. As you can see from the model card, the Wav2Vec2 model is pretrained on 16kHz sampled speech audio. It is important your audio data\u2019s sampling rate matches the sampling rate of the dataset used to pretrain the model. If your data\u2019s sampling rate isn\u2019t the same, then you need to resample your audio data."),Np=c(),oe=t("p"),am=r("For example, load the "),ea=t("a"),tm=r("LJ Speech"),nm=r(" dataset which has a sampling rate of 22050kHz. In order to use the Wav2Vec2 model with this dataset, downsample the sampling rate to 16kHz:"),Ip=c(),m(aa.$$.fragment),Rp=c(),wt=t("ol"),ta=t("li"),lm=r("Use \u{1F917} Datasets\u2019 "),na=t("a"),El=t("code"),rm=r("cast_column"),om=r(" method to downsample the sampling rate to 16kHz:"),Fp=c(),m(la.$$.fragment),Hp=c(),ra=t("ol"),wl=t("li"),pm=r("Load the audio file:"),Bp=c(),m(oa.$$.fragment),Jp=c(),pe=t("p"),cm=r("As you can see, the "),$l=t("code"),hm=r("sampling_rate"),im=r(" was downsampled to 16kHz. Now that you know how resampling works, let\u2019s return to our previous example with the SUPERB dataset!"),Wp=c(),us=t("h3"),ce=t("a"),yl=t("span"),m(pa.$$.fragment),um=c(),kl=t("span"),mm=r("Feature extractor"),Yp=c(),A=t("p"),dm=r("The next step is to load a feature extractor to normalize and pad the input. When padding textual data, a "),xl=t("code"),fm=r("0"),bm=r(" is added for shorter sequences. The same idea applies to audio data, and the audio feature extractor will add a "),Tl=t("code"),jm=r("0"),_m=r(" - interpreted as silence - to "),ql=t("code"),gm=r("array"),vm=r("."),Gp=c(),he=t("p"),Em=r("Load the feature extractor with "),$t=t("a"),wm=r("AutoFeatureExtractor.from_pretrained()"),$m=r(":"),Up=c(),m(ca.$$.fragment),Mp=c(),Q=t("p"),ym=r("Pass the audio "),Dl=t("code"),km=r("array"),xm=r(" to the feature extractor. We also recommend adding the "),Al=t("code"),Tm=r("sampling_rate"),qm=r(" argument in the feature extractor in order to better debug any silent errors that may occur."),Vp=c(),m(ha.$$.fragment),Kp=c(),ms=t("h3"),ie=t("a"),zl=t("span"),m(ia.$$.fragment),Dm=c(),Pl=t("span"),Am=r("Pad and truncate"),Qp=c(),yt=t("p"),zm=r("Just like the tokenizer, you can apply padding or truncation to handle variable sequences in a batch. Take a look at the sequence length of these two audio samples:"),Xp=c(),m(ua.$$.fragment),Zp=c(),kt=t("p"),Pm=r("As you can see, the first sample has a longer sequence than the second sample. Let\u2019s create a function that will preprocess the dataset. Specify a maximum sample length, and the feature extractor will either pad or truncate the sequences to match it:"),sc=c(),m(ma.$$.fragment),ec=c(),xt=t("p"),Cm=r("Apply the function to the the first few examples in the dataset:"),ac=c(),m(da.$$.fragment),tc=c(),Tt=t("p"),Om=r("Now take another look at the processed sample lengths:"),nc=c(),m(fa.$$.fragment),lc=c(),qt=t("p"),Lm=r("The lengths of the first two samples now match the maximum length you specified."),rc=c(),ds=t("h2"),ue=t("a"),Cl=t("span"),m(ba.$$.fragment),Sm=c(),Ol=t("span"),Nm=r("Vision"),oc=c(),Dt=t("p"),Im=r("A feature extractor is also used to process images for vision tasks. Once again, the goal is to convert the raw image into a batch of tensors as input."),pc=c(),X=t("p"),Rm=r("Let\u2019s load the "),ja=t("a"),Fm=r("food101"),Hm=r(" dataset for this tutorial. Use \u{1F917} Datasets "),Ll=t("code"),Bm=r("split"),Jm=r(" parameter to only load a small sample from the training split since the dataset is quite large:"),cc=c(),m(_a.$$.fragment),hc=c(),me=t("p"),Wm=r("Next, take a look at the image with \u{1F917} Datasets "),ga=t("a"),Sl=t("code"),Ym=r("Image"),Gm=r(" feature:"),ic=c(),m(va.$$.fragment),uc=c(),At=t("p"),zt=t("img"),mc=c(),fs=t("h3"),de=t("a"),Nl=t("span"),m(Ea.$$.fragment),Um=c(),Il=t("span"),Mm=r("Feature extractor"),dc=c(),fe=t("p"),Vm=r("Load the feature extractor with "),Pt=t("a"),Km=r("AutoFeatureExtractor.from_pretrained()"),Qm=r(":"),fc=c(),m(wa.$$.fragment),bc=c(),bs=t("h3"),be=t("a"),Rl=t("span"),m($a.$$.fragment),Xm=c(),Fl=t("span"),Zm=r("Data augmentation"),jc=c(),je=t("p"),sd=r("For vision tasks, it is common to add some type of data augmentation to the images as a part of preprocessing. You can add augmentations with any library you\u2019d like, but in this tutorial, you will use torchvision\u2019s "),ya=t("a"),Hl=t("code"),ed=r("transforms"),ad=r(" module."),_c=c(),Ct=t("ol"),S=t("li"),td=r("Normalize the image and use "),ka=t("a"),Bl=t("code"),nd=r("Compose"),ld=r(" to chain some transforms - "),xa=t("a"),Jl=t("code"),rd=r("RandomResizedCrop"),od=r(" and "),Ta=t("a"),Wl=t("code"),pd=r("ColorJitter"),cd=r(" - together:"),gc=c(),m(qa.$$.fragment),vc=c(),Da=t("ol"),js=t("li"),hd=r("The model accepts "),Ot=t("a"),Yl=t("code"),id=r("pixel_values"),ud=r(" as it\u2019s input. This value is generated by the feature extractor. Create a function that generates "),Gl=t("code"),md=r("pixel_values"),dd=r(" from the transforms:"),Ec=c(),m(Aa.$$.fragment),wc=c(),za=t("ol"),Pa=t("li"),fd=r("Then use \u{1F917} Datasets "),Ca=t("a"),Ul=t("code"),bd=r("set_transform"),jd=r(" to apply the transforms on-the-fly:"),$c=c(),m(Oa.$$.fragment),yc=c(),La=t("ol"),Sa=t("li"),_d=r("Now when you access the image, you will notice the feature extractor has added the model input "),Ml=t("code"),gd=r("pixel_values"),vd=r(":"),kc=c(),m(Na.$$.fragment),xc=c(),Lt=t("p"),Ed=r("Here is what the image looks like after you preprocess it. Just as you\u2019d expect from the applied transforms, the image has been randomly cropped and it\u2019s color properties are different."),Tc=c(),m(Ia.$$.fragment),qc=c(),St=t("p"),Nt=t("img"),Dc=c(),_s=t("h2"),_e=t("a"),Vl=t("span"),m(Ra.$$.fragment),wd=c(),Kl=t("span"),$d=r("Multimodal"),Ac=c(),It=t("p"),yd=r("For multimodal tasks. you will use a combination of everything you\u2019ve learned so far and apply your skills to a automatic speech recognition (ASR) task. This means you will need a:"),zc=c(),ge=t("ul"),Ql=t("li"),kd=r("Feature extractor to preprocess the audio data."),xd=c(),Xl=t("li"),Td=r("Tokenizer to process the text."),Pc=c(),ve=t("p"),qd=r("Let\u2019s return to the "),Fa=t("a"),Dd=r("LJ Speech"),Ad=r(" dataset:"),Cc=c(),m(Ha.$$.fragment),Oc=c(),Z=t("p"),zd=r("Since you are mainly interested in the "),Zl=t("code"),Pd=r("audio"),Cd=r(" and "),sr=t("code"),Od=r("text"),Ld=r(" column, remove the other columns:"),Lc=c(),m(Ba.$$.fragment),Sc=c(),ss=t("p"),Sd=r("Now take a look at the "),er=t("code"),Nd=r("audio"),Id=r(" and "),ar=t("code"),Rd=r("text"),Fd=r(" columns:"),Nc=c(),m(Ja.$$.fragment),Ic=c(),Ee=t("p"),Hd=r("Remember from the earlier section on processing audio data, you should always "),Rt=t("a"),Bd=r("resample"),Jd=r(" your audio data\u2019s sampling rate to match the sampling rate of the dataset used to pretrain a model:"),Rc=c(),m(Wa.$$.fragment),Fc=c(),gs=t("h3"),we=t("a"),tr=t("span"),m(Ya.$$.fragment),Wd=c(),nr=t("span"),Yd=r("Processor"),Hc=c(),Ft=t("p"),Gd=r("A processor combines a feature extractor and tokenizer. Load a processor with [`AutoProcessor.from_pretrained]:"),Bc=c(),m(Ga.$$.fragment),Jc=c(),Ht=t("ol"),vs=t("li"),Ud=r("Create a function to process the audio data to "),lr=t("code"),Md=r("input_values"),Vd=r(", and tokenizes the text to "),rr=t("code"),Kd=r("labels"),Qd=r(". These are your inputs to the model:"),Wc=c(),m(Ua.$$.fragment),Yc=c(),Ma=t("ol"),Va=t("li"),Xd=r("Apply the "),or=t("code"),Zd=r("prepare_dataset"),sf=r(" function to a sample:"),Gc=c(),m(Ka.$$.fragment),Uc=c(),es=t("p"),ef=r("Notice the processor has added "),pr=t("code"),af=r("input_values"),tf=r(" and "),cr=t("code"),nf=r("labels"),lf=r(". The sampling rate has also been correctly downsampled to 16kHz."),Mc=c(),Bt=t("p"),rf=r("Awesome, you should now be able to preprocess data for any modality and even combine different modalities! In the next tutorial, learn how to fine-tune a model on your newly preprocessed data."),Vc=c(),Es=t("h2"),$e=t("a"),hr=t("span"),m(Qa.$$.fragment),of=c(),ir=t("span"),pf=r("Everything you always wanted to know about padding and truncation"),Kc=c(),z=t("p"),cf=r(`We have seen the commands that will work for most cases (pad your batch to the length of the maximum sentence and
truncate to the maximum length the model can accept). However, the API supports more strategies if you need them. The
three arguments you need to know for this are `),ur=t("code"),hf=r("padding"),uf=r(", "),mr=t("code"),mf=r("truncation"),df=r(" and "),dr=t("code"),ff=r("max_length"),bf=r("."),Qc=c(),as=t("ul"),Xa=t("li"),Jt=t("p"),fr=t("code"),jf=r("padding"),_f=r(" controls the padding. It can be a boolean or a string which should be:"),gf=c(),ws=t("ul"),ye=t("li"),br=t("code"),vf=r("True"),Ef=r(" or "),jr=t("code"),wf=r("'longest'"),$f=r(` to pad to the longest sequence in the batch (doing no padding if you only provide
a single sequence).`),yf=c(),P=t("li"),_r=t("code"),kf=r("'max_length'"),xf=r(" to pad to a length specified by the "),gr=t("code"),Tf=r("max_length"),qf=r(` argument or the maximum length accepted
by the model if no `),vr=t("code"),Df=r("max_length"),Af=r(" is provided ("),Er=t("code"),zf=r("max_length=None"),Pf=r(`). If you only provide a single sequence,
padding will still be applied to it.`),Cf=c(),ke=t("li"),wr=t("code"),Of=r("False"),Lf=r(" or "),$r=t("code"),Sf=r("'do_not_pad'"),Nf=r(` to not pad the sequences. As we have seen before, this is the default
behavior.`),If=c(),Za=t("li"),Wt=t("p"),yr=t("code"),Rf=r("truncation"),Ff=r(" controls the truncation. It can be a boolean or a string which should be:"),Hf=c(),N=t("ul"),T=t("li"),kr=t("code"),Bf=r("True"),Jf=r(" or "),xr=t("code"),Wf=r("'longest_first'"),Yf=r(" truncate to a maximum length specified by the "),Tr=t("code"),Gf=r("max_length"),Uf=r(` argument or
the maximum length accepted by the model if no `),qr=t("code"),Mf=r("max_length"),Vf=r(" is provided ("),Dr=t("code"),Kf=r("max_length=None"),Qf=r(`). This will
truncate token by token, removing a token from the longest sequence in the pair until the proper length is
reached.`),Xf=c(),C=t("li"),Ar=t("code"),Zf=r("'only_second'"),sb=r(" truncate to a maximum length specified by the "),zr=t("code"),eb=r("max_length"),ab=r(` argument or the maximum
length accepted by the model if no `),Pr=t("code"),tb=r("max_length"),nb=r(" is provided ("),Cr=t("code"),lb=r("max_length=None"),rb=r(`). This will only truncate
the second sentence of a pair if a pair of sequence (or a batch of pairs of sequences) is provided.`),ob=c(),O=t("li"),Or=t("code"),pb=r("'only_first'"),cb=r(" truncate to a maximum length specified by the "),Lr=t("code"),hb=r("max_length"),ib=r(` argument or the maximum
length accepted by the model if no `),Sr=t("code"),ub=r("max_length"),mb=r(" is provided ("),Nr=t("code"),db=r("max_length=None"),fb=r(`). This will only truncate
the first sentence of a pair if a pair of sequence (or a batch of pairs of sequences) is provided.`),bb=c(),xe=t("li"),Ir=t("code"),jb=r("False"),_b=r(" or "),Rr=t("code"),gb=r("'do_not_truncate'"),vb=r(` to not truncate the sequences. As we have seen before, this is the
default behavior.`),Eb=c(),Fr=t("li"),ts=t("p"),Hr=t("code"),wb=r("max_length"),$b=r(" to control the length of the padding/truncation. It can be an integer or "),Br=t("code"),yb=r("None"),kb=r(`, in which case
it will default to the maximum length the model can accept. If the model has no specific maximum input length,
truncation/padding to `),Jr=t("code"),xb=r("max_length"),Tb=r(" is deactivated."),Xc=c(),y=t("p"),qb=r(`Here is a table summarizing the recommend way to setup padding and truncation. If you use pair of inputs sequence in
any of the following examples, you can replace `),Wr=t("code"),Db=r("truncation=True"),Ab=r(" by a "),Yr=t("code"),zb=r("STRATEGY"),Pb=r(` selected in
`),Gr=t("code"),Cb=r("['only_first', 'only_second', 'longest_first']"),Ob=r(", i.e. "),Ur=t("code"),Lb=r("truncation='only_second'"),Sb=r(" or "),Mr=t("code"),Nb=r("truncation= 'longest_first'"),Ib=r(" to control how both sequence in the pair are truncated as detailed before."),Zc=c(),Te=t("table"),Vr=t("thead"),$s=t("tr"),Kr=t("th"),Rb=r("Truncation"),Fb=c(),Qr=t("th"),Hb=r("Padding"),Bb=c(),Xr=t("th"),Jb=r("Instruction"),Wb=c(),g=t("tbody"),ys=t("tr"),Zr=t("td"),Yb=r("no truncation"),Gb=c(),so=t("td"),Ub=r("no padding"),Mb=c(),eo=t("td"),ao=t("code"),Vb=r("tokenizer(batch_sentences)"),Kb=c(),ks=t("tr"),sh=t("td"),Qb=c(),to=t("td"),Xb=r("padding to max sequence in batch"),Zb=c(),Yt=t("td"),no=t("code"),sj=r("tokenizer(batch_sentences, padding=True)"),ej=r(" or"),aj=c(),xs=t("tr"),eh=t("td"),tj=c(),ah=t("td"),nj=c(),lo=t("td"),ro=t("code"),lj=r("tokenizer(batch_sentences, padding='longest')"),rj=c(),Ts=t("tr"),th=t("td"),oj=c(),oo=t("td"),pj=r("padding to max model input length"),cj=c(),po=t("td"),co=t("code"),hj=r("tokenizer(batch_sentences, padding='max_length')"),ij=c(),qs=t("tr"),nh=t("td"),uj=c(),ho=t("td"),mj=r("padding to specific length"),dj=c(),io=t("td"),uo=t("code"),fj=r("tokenizer(batch_sentences, padding='max_length', max_length=42)"),bj=c(),Ds=t("tr"),mo=t("td"),jj=r("truncation to max model input length"),_j=c(),fo=t("td"),gj=r("no padding"),vj=c(),Gt=t("td"),bo=t("code"),Ej=r("tokenizer(batch_sentences, truncation=True)"),wj=r(" or"),$j=c(),As=t("tr"),lh=t("td"),yj=c(),rh=t("td"),kj=c(),jo=t("td"),_o=t("code"),xj=r("tokenizer(batch_sentences, truncation=STRATEGY)"),Tj=c(),zs=t("tr"),oh=t("td"),qj=c(),go=t("td"),Dj=r("padding to max sequence in batch"),Aj=c(),Ut=t("td"),vo=t("code"),zj=r("tokenizer(batch_sentences, padding=True, truncation=True)"),Pj=r(" or"),Cj=c(),Ps=t("tr"),ph=t("td"),Oj=c(),ch=t("td"),Lj=c(),Eo=t("td"),wo=t("code"),Sj=r("tokenizer(batch_sentences, padding=True, truncation=STRATEGY)"),Nj=c(),Cs=t("tr"),hh=t("td"),Ij=c(),$o=t("td"),Rj=r("padding to max model input length"),Fj=c(),Mt=t("td"),yo=t("code"),Hj=r("tokenizer(batch_sentences, padding='max_length', truncation=True)"),Bj=r(" or"),Jj=c(),Os=t("tr"),ih=t("td"),Wj=c(),uh=t("td"),Yj=c(),ko=t("td"),xo=t("code"),Gj=r("tokenizer(batch_sentences, padding='max_length', truncation=STRATEGY)"),Uj=c(),Ls=t("tr"),mh=t("td"),Mj=c(),To=t("td"),Vj=r("padding to specific length"),Kj=c(),qo=t("td"),Qj=r("Not possible"),Xj=c(),Ss=t("tr"),Do=t("td"),Zj=r("truncation to specific length"),s1=c(),Ao=t("td"),e1=r("no padding"),a1=c(),Vt=t("td"),zo=t("code"),t1=r("tokenizer(batch_sentences, truncation=True, max_length=42)"),n1=r(" or"),l1=c(),Ns=t("tr"),dh=t("td"),r1=c(),fh=t("td"),o1=c(),Po=t("td"),Co=t("code"),p1=r("tokenizer(batch_sentences, truncation=STRATEGY, max_length=42)"),c1=c(),Is=t("tr"),bh=t("td"),h1=c(),Oo=t("td"),i1=r("padding to max sequence in batch"),u1=c(),Kt=t("td"),Lo=t("code"),m1=r("tokenizer(batch_sentences, padding=True, truncation=True, max_length=42)"),d1=r(" or"),f1=c(),Rs=t("tr"),jh=t("td"),b1=c(),_h=t("td"),j1=c(),So=t("td"),No=t("code"),_1=r("tokenizer(batch_sentences, padding=True, truncation=STRATEGY, max_length=42)"),g1=c(),Fs=t("tr"),gh=t("td"),v1=c(),Io=t("td"),E1=r("padding to max model input length"),w1=c(),Ro=t("td"),$1=r("Not possible"),y1=c(),Hs=t("tr"),vh=t("td"),k1=c(),Fo=t("td"),x1=r("padding to specific length"),T1=c(),Qt=t("td"),Ho=t("code"),q1=r("tokenizer(batch_sentences, padding='max_length', truncation=True, max_length=42)"),D1=r(" or"),A1=c(),Bs=t("tr"),Eh=t("td"),z1=c(),wh=t("td"),P1=c(),Bo=t("td"),Jo=t("code"),C1=r("tokenizer(batch_sentences, padding='max_length', truncation=STRATEGY, max_length=42)"),this.h()},l(s){const p=vv('[data-svelte="svelte-1phssyn"]',document.head);w=n(p,"META",{name:!0,content:!0}),p.forEach(a),R=h(s),$=n(s,"H1",{class:!0});var st=l($);x=n(st,"A",{id:!0,class:!0,href:!0});var Q1=l(x);ns=n(Q1,"SPAN",{});var X1=l(ns);d(q.$$.fragment,X1),X1.forEach(a),Q1.forEach(a),F=h(st),Gs=n(st,"SPAN",{});var Z1=l(Gs);oi=o(Z1,"Preprocess"),Z1.forEach(a),st.forEach(a),Uo=h(s),d(Ce.$$.fragment,s),Mo=h(s),at=n(s,"P",{});var s0=l(at);pi=o(s0,"Before you can use your data in a model, the data needs to be processed into an acceptable format for the model. A model does not understand raw text, images or audio. These inputs need to be converted into numbers and assembled into tensors. In this tutorial, you will:"),s0.forEach(a),Vo=h(s),H=n(s,"UL",{});var Xt=l(H);Rn=n(Xt,"LI",{});var e0=l(Rn);ci=o(e0,"Preprocess textual data with a tokenizer."),e0.forEach(a),hi=h(Xt),Fn=n(Xt,"LI",{});var a0=l(Fn);ii=o(a0,"Preprocess image or audio data with a feature extractor."),a0.forEach(a),ui=h(Xt),Hn=n(Xt,"LI",{});var t0=l(Hn);mi=o(t0,"Preprocess data for a multimodal task with a processor."),t0.forEach(a),Xt.forEach(a),Ko=h(s),ls=n(s,"H2",{class:!0});var yh=l(ls);Us=n(yh,"A",{id:!0,class:!0,href:!0});var n0=l(Us);Bn=n(n0,"SPAN",{});var l0=l(Bn);d(Oe.$$.fragment,l0),l0.forEach(a),n0.forEach(a),di=h(yh),Jn=n(yh,"SPAN",{});var r0=l(Jn);fi=o(r0,"NLP"),r0.forEach(a),yh.forEach(a),Qo=h(s),d(Le.$$.fragment,s),Xo=h(s),B=n(s,"P",{});var Zt=l(B);bi=o(Zt,"The main tool for processing textual data is a "),tt=n(Zt,"A",{href:!0});var o0=l(tt);ji=o(o0,"tokenizer"),o0.forEach(a),_i=o(Zt,". A tokenizer starts by splitting text into "),Wn=n(Zt,"EM",{});var p0=l(Wn);gi=o(p0,"tokens"),p0.forEach(a),vi=o(Zt," according to a set of rules. The tokens are converted into numbers, which are used to build tensors as input to a model. Any additional inputs required by a model are also added by the tokenizer."),Zt.forEach(a),Zo=h(s),d(Ms.$$.fragment,s),sp=h(s),J=n(s,"P",{});var sn=l(J);Ei=o(sn,"Get started quickly by loading a pretrained tokenizer with the "),nt=n(sn,"A",{href:!0});var c0=l(nt);wi=o(c0,"AutoTokenizer"),c0.forEach(a),$i=o(sn," class. This downloads the "),Yn=n(sn,"EM",{});var h0=l(Yn);yi=o(h0,"vocab"),h0.forEach(a),ki=o(sn," used when a model is pretrained."),sn.forEach(a),ep=h(s),rs=n(s,"H3",{class:!0});var kh=l(rs);Vs=n(kh,"A",{id:!0,class:!0,href:!0});var i0=l(Vs);Gn=n(i0,"SPAN",{});var u0=l(Gn);d(Se.$$.fragment,u0),u0.forEach(a),i0.forEach(a),xi=h(kh),Un=n(kh,"SPAN",{});var m0=l(Un);Ti=o(m0,"Tokenize"),m0.forEach(a),kh.forEach(a),ap=h(s),Ks=n(s,"P",{});var xh=l(Ks);qi=o(xh,"Load a pretrained tokenizer with "),lt=n(xh,"A",{href:!0});var d0=l(lt);Di=o(d0,"AutoTokenizer.from_pretrained()"),d0.forEach(a),Ai=o(xh,":"),xh.forEach(a),tp=h(s),d(Ne.$$.fragment,s),np=h(s),rt=n(s,"P",{});var f0=l(rt);zi=o(f0,"Then pass your sentence to the tokenizer:"),f0.forEach(a),lp=h(s),d(Ie.$$.fragment,s),rp=h(s),ot=n(s,"P",{});var b0=l(ot);Pi=o(b0,"The tokenizer returns a dictionary with three important itmes:"),b0.forEach(a),op=h(s),W=n(s,"UL",{});var en=l(W);pt=n(en,"LI",{});var O1=l(pt);ct=n(O1,"A",{href:!0});var j0=l(ct);Ci=o(j0,"input_ids"),j0.forEach(a),Oi=o(O1," are the indices corresponding to each token in the sentence."),O1.forEach(a),Li=h(en),ht=n(en,"LI",{});var L1=l(ht);it=n(L1,"A",{href:!0});var _0=l(it);Si=o(_0,"attention_mask"),_0.forEach(a),Ni=o(L1," indicates whether a token should be attended to or not."),L1.forEach(a),Ii=h(en),ut=n(en,"LI",{});var S1=l(ut);mt=n(S1,"A",{href:!0});var g0=l(mt);Ri=o(g0,"token_type_ids"),g0.forEach(a),Fi=o(S1," identifies which sequence a token belongs to when there is more than one sequence."),S1.forEach(a),en.forEach(a),pp=h(s),Qs=n(s,"P",{});var Th=l(Qs);Hi=o(Th,"You can decode the "),Mn=n(Th,"CODE",{});var v0=l(Mn);Bi=o(v0,"input_ids"),v0.forEach(a),Ji=o(Th," to return the original input:"),Th.forEach(a),cp=h(s),d(Re.$$.fragment,s),hp=h(s),Y=n(s,"P",{});var an=l(Y);Wi=o(an,"As you can see, the tokenizer added two special tokens - "),Vn=n(an,"CODE",{});var E0=l(Vn);Yi=o(E0,"CLS"),E0.forEach(a),Gi=o(an," and "),Kn=n(an,"CODE",{});var w0=l(Kn);Ui=o(w0,"SEP"),w0.forEach(a),Mi=o(an,` (classifier and separator) - to the sentence. Not all models need
special tokens, but if they do, the tokenizer will automatically add them for you.`),an.forEach(a),ip=h(s),dt=n(s,"P",{});var $0=l(dt);Vi=o($0,"If there are several sentences you want to process, pass the sentences as a list to the tokenizer:"),$0.forEach(a),up=h(s),d(Fe.$$.fragment,s),mp=h(s),os=n(s,"H3",{class:!0});var qh=l(os);Xs=n(qh,"A",{id:!0,class:!0,href:!0});var y0=l(Xs);Qn=n(y0,"SPAN",{});var k0=l(Qn);d(He.$$.fragment,k0),k0.forEach(a),y0.forEach(a),Ki=h(qh),Xn=n(qh,"SPAN",{});var x0=l(Xn);Qi=o(x0,"Pad"),x0.forEach(a),qh.forEach(a),dp=h(s),Zs=n(s,"P",{});var Dh=l(Zs);Xi=o(Dh,"This brings us to an important topic. When you process a batch of sentences, they aren\u2019t always the same length. This is a problem because tensors, the input to the model, need to have a uniform shape. Padding is a strategy for ensuring tensors are rectangular by adding a special "),Zn=n(Dh,"EM",{});var T0=l(Zn);Zi=o(T0,"padding token"),T0.forEach(a),su=o(Dh," to sentences with fewer tokens."),Dh.forEach(a),fp=h(s),G=n(s,"P",{});var tn=l(G);eu=o(tn,"Set the "),sl=n(tn,"CODE",{});var q0=l(sl);au=o(q0,"padding"),q0.forEach(a),tu=o(tn," parameter to "),el=n(tn,"CODE",{});var D0=l(el);nu=o(D0,"True"),D0.forEach(a),lu=o(tn," to pad the shorter sequences in the batch to match the longest sequence:"),tn.forEach(a),bp=h(s),d(Be.$$.fragment,s),jp=h(s),se=n(s,"P",{});var Ah=l(se);ru=o(Ah,"Notice the tokenizer padded the first and third sentences with a "),al=n(Ah,"CODE",{});var A0=l(al);ou=o(A0,"0"),A0.forEach(a),pu=o(Ah," because they are shorter!"),Ah.forEach(a),_p=h(s),ps=n(s,"H3",{class:!0});var zh=l(ps);ee=n(zh,"A",{id:!0,class:!0,href:!0});var z0=l(ee);tl=n(z0,"SPAN",{});var P0=l(tl);d(Je.$$.fragment,P0),P0.forEach(a),z0.forEach(a),cu=h(zh),nl=n(zh,"SPAN",{});var C0=l(nl);hu=o(C0,"Truncation"),C0.forEach(a),zh.forEach(a),gp=h(s),ft=n(s,"P",{});var O0=l(ft);iu=o(O0,"On the other end of the spectrum, sometimes a sequence may be too long for a model to handle. In this case, you will need to truncate the sequence to a shorter length."),O0.forEach(a),vp=h(s),U=n(s,"P",{});var nn=l(U);uu=o(nn,"Set the "),ll=n(nn,"CODE",{});var L0=l(ll);mu=o(L0,"truncation"),L0.forEach(a),du=o(nn," parameter to "),rl=n(nn,"CODE",{});var S0=l(rl);fu=o(S0,"True"),S0.forEach(a),bu=o(nn," to truncate a sequence to the maximum length accepted by the model:"),nn.forEach(a),Ep=h(s),d(We.$$.fragment,s),wp=h(s),cs=n(s,"H3",{class:!0});var Ph=l(cs);ae=n(Ph,"A",{id:!0,class:!0,href:!0});var N0=l(ae);ol=n(N0,"SPAN",{});var I0=l(ol);d(Ye.$$.fragment,I0),I0.forEach(a),N0.forEach(a),ju=h(Ph),pl=n(Ph,"SPAN",{});var R0=l(pl);_u=o(R0,"Build tensors"),R0.forEach(a),Ph.forEach(a),$p=h(s),bt=n(s,"P",{});var F0=l(bt);gu=o(F0,"Finally, you want the tokenizer to return the actual tensors that are fed to the model."),F0.forEach(a),yp=h(s),D=n(s,"P",{});var qe=l(D);vu=o(qe,"Set the "),cl=n(qe,"CODE",{});var H0=l(cl);Eu=o(H0,"return_tensors"),H0.forEach(a),wu=o(qe," parameter to either "),hl=n(qe,"CODE",{});var B0=l(hl);$u=o(B0,"pt"),B0.forEach(a),yu=o(qe," for PyTorch, or "),il=n(qe,"CODE",{});var J0=l(il);ku=o(J0,"tf"),J0.forEach(a),xu=o(qe," for TensorFlow:"),qe.forEach(a),kp=h(s),d(Ge.$$.fragment,s),xp=h(s),hs=n(s,"H2",{class:!0});var Ch=l(hs);te=n(Ch,"A",{id:!0,class:!0,href:!0});var W0=l(te);ul=n(W0,"SPAN",{});var Y0=l(ul);d(Ue.$$.fragment,Y0),Y0.forEach(a),W0.forEach(a),Tu=h(Ch),ml=n(Ch,"SPAN",{});var G0=l(ml);qu=o(G0,"Audio"),G0.forEach(a),Ch.forEach(a),Tp=h(s),ne=n(s,"P",{});var Oh=l(ne);Du=o(Oh,"Audio inputs are preprocessed differently than textual inputs, but the end goal remains the same: create numerical sequences the model can understand. A "),jt=n(Oh,"A",{href:!0});var U0=l(jt);Au=o(U0,"feature extractor"),U0.forEach(a),zu=o(Oh," is designed for the express purpose of extracting features from raw image or audio data and converting them into tensors. Before you begin, install \u{1F917} Datasets to load an audio dataset to experiment with:"),Oh.forEach(a),qp=h(s),d(Me.$$.fragment,s),Dp=h(s),M=n(s,"P",{});var ln=l(M);Pu=o(ln,"Load the keyword spotting task from the "),Ve=n(ln,"A",{href:!0,rel:!0});var M0=l(Ve);Cu=o(M0,"SUPERB"),M0.forEach(a),Ou=o(ln," benchmark (see the \u{1F917} "),Ke=n(ln,"A",{href:!0,rel:!0});var V0=l(Ke);Lu=o(V0,"Datasets tutorial"),V0.forEach(a),Su=o(ln," for more details on how to load a dataset):"),ln.forEach(a),Ap=h(s),d(Qe.$$.fragment,s),zp=h(s),V=n(s,"P",{});var rn=l(V);Nu=o(rn,"Access the first element of the "),dl=n(rn,"CODE",{});var K0=l(dl);Iu=o(K0,"audio"),K0.forEach(a),Ru=o(rn," column to take a look at the input. Calling the "),fl=n(rn,"CODE",{});var Q0=l(fl);Fu=o(Q0,"audio"),Q0.forEach(a),Hu=o(rn," column will automatically load and resample the audio file:"),rn.forEach(a),Pp=h(s),d(Xe.$$.fragment,s),Cp=h(s),_t=n(s,"P",{});var X0=l(_t);Bu=o(X0,"This returns three items:"),X0.forEach(a),Op=h(s),K=n(s,"UL",{});var on=l(K);gt=n(on,"LI",{});var N1=l(gt);bl=n(N1,"CODE",{});var Z0=l(bl);Ju=o(Z0,"array"),Z0.forEach(a),Wu=o(N1," is the speech signal loaded - and potentially resampled - as a 1D array."),N1.forEach(a),Yu=h(on),vt=n(on,"LI",{});var I1=l(vt);jl=n(I1,"CODE",{});var s_=l(jl);Gu=o(s_,"path"),s_.forEach(a),Uu=o(I1," points to the location of the audio file."),I1.forEach(a),Mu=h(on),Et=n(on,"LI",{});var R1=l(Et);_l=n(R1,"CODE",{});var e_=l(_l);Vu=o(e_,"sampling_rate"),e_.forEach(a),Ku=o(R1," refers to how many data points in the speech signal are measured per second."),R1.forEach(a),on.forEach(a),Lp=h(s),is=n(s,"H3",{class:!0});var Lh=l(is);le=n(Lh,"A",{id:!0,class:!0,href:!0});var a_=l(le);gl=n(a_,"SPAN",{});var t_=l(gl);d(Ze.$$.fragment,t_),t_.forEach(a),a_.forEach(a),Qu=h(Lh),vl=n(Lh,"SPAN",{});var n_=l(vl);Xu=o(n_,"Resample"),n_.forEach(a),Lh.forEach(a),Sp=h(s),re=n(s,"P",{});var Sh=l(re);Zu=o(Sh,"For this tutorial, you will use the "),sa=n(Sh,"A",{href:!0,rel:!0});var l_=l(sa);sm=o(l_,"Wav2Vec2"),l_.forEach(a),em=o(Sh," model. As you can see from the model card, the Wav2Vec2 model is pretrained on 16kHz sampled speech audio. It is important your audio data\u2019s sampling rate matches the sampling rate of the dataset used to pretrain the model. If your data\u2019s sampling rate isn\u2019t the same, then you need to resample your audio data."),Sh.forEach(a),Np=h(s),oe=n(s,"P",{});var Nh=l(oe);am=o(Nh,"For example, load the "),ea=n(Nh,"A",{href:!0,rel:!0});var r_=l(ea);tm=o(r_,"LJ Speech"),r_.forEach(a),nm=o(Nh," dataset which has a sampling rate of 22050kHz. In order to use the Wav2Vec2 model with this dataset, downsample the sampling rate to 16kHz:"),Nh.forEach(a),Ip=h(s),d(aa.$$.fragment,s),Rp=h(s),wt=n(s,"OL",{});var o_=l(wt);ta=n(o_,"LI",{});var Ih=l(ta);lm=o(Ih,"Use \u{1F917} Datasets\u2019 "),na=n(Ih,"A",{href:!0,rel:!0});var p_=l(na);El=n(p_,"CODE",{});var c_=l(El);rm=o(c_,"cast_column"),c_.forEach(a),p_.forEach(a),om=o(Ih," method to downsample the sampling rate to 16kHz:"),Ih.forEach(a),o_.forEach(a),Fp=h(s),d(la.$$.fragment,s),Hp=h(s),ra=n(s,"OL",{start:!0});var h_=l(ra);wl=n(h_,"LI",{});var i_=l(wl);pm=o(i_,"Load the audio file:"),i_.forEach(a),h_.forEach(a),Bp=h(s),d(oa.$$.fragment,s),Jp=h(s),pe=n(s,"P",{});var Rh=l(pe);cm=o(Rh,"As you can see, the "),$l=n(Rh,"CODE",{});var u_=l($l);hm=o(u_,"sampling_rate"),u_.forEach(a),im=o(Rh," was downsampled to 16kHz. Now that you know how resampling works, let\u2019s return to our previous example with the SUPERB dataset!"),Rh.forEach(a),Wp=h(s),us=n(s,"H3",{class:!0});var Fh=l(us);ce=n(Fh,"A",{id:!0,class:!0,href:!0});var m_=l(ce);yl=n(m_,"SPAN",{});var d_=l(yl);d(pa.$$.fragment,d_),d_.forEach(a),m_.forEach(a),um=h(Fh),kl=n(Fh,"SPAN",{});var f_=l(kl);mm=o(f_,"Feature extractor"),f_.forEach(a),Fh.forEach(a),Yp=h(s),A=n(s,"P",{});var De=l(A);dm=o(De,"The next step is to load a feature extractor to normalize and pad the input. When padding textual data, a "),xl=n(De,"CODE",{});var b_=l(xl);fm=o(b_,"0"),b_.forEach(a),bm=o(De," is added for shorter sequences. The same idea applies to audio data, and the audio feature extractor will add a "),Tl=n(De,"CODE",{});var j_=l(Tl);jm=o(j_,"0"),j_.forEach(a),_m=o(De," - interpreted as silence - to "),ql=n(De,"CODE",{});var __=l(ql);gm=o(__,"array"),__.forEach(a),vm=o(De,"."),De.forEach(a),Gp=h(s),he=n(s,"P",{});var Hh=l(he);Em=o(Hh,"Load the feature extractor with "),$t=n(Hh,"A",{href:!0});var g_=l($t);wm=o(g_,"AutoFeatureExtractor.from_pretrained()"),g_.forEach(a),$m=o(Hh,":"),Hh.forEach(a),Up=h(s),d(ca.$$.fragment,s),Mp=h(s),Q=n(s,"P",{});var pn=l(Q);ym=o(pn,"Pass the audio "),Dl=n(pn,"CODE",{});var v_=l(Dl);km=o(v_,"array"),v_.forEach(a),xm=o(pn," to the feature extractor. We also recommend adding the "),Al=n(pn,"CODE",{});var E_=l(Al);Tm=o(E_,"sampling_rate"),E_.forEach(a),qm=o(pn," argument in the feature extractor in order to better debug any silent errors that may occur."),pn.forEach(a),Vp=h(s),d(ha.$$.fragment,s),Kp=h(s),ms=n(s,"H3",{class:!0});var Bh=l(ms);ie=n(Bh,"A",{id:!0,class:!0,href:!0});var w_=l(ie);zl=n(w_,"SPAN",{});var $_=l(zl);d(ia.$$.fragment,$_),$_.forEach(a),w_.forEach(a),Dm=h(Bh),Pl=n(Bh,"SPAN",{});var y_=l(Pl);Am=o(y_,"Pad and truncate"),y_.forEach(a),Bh.forEach(a),Qp=h(s),yt=n(s,"P",{});var k_=l(yt);zm=o(k_,"Just like the tokenizer, you can apply padding or truncation to handle variable sequences in a batch. Take a look at the sequence length of these two audio samples:"),k_.forEach(a),Xp=h(s),d(ua.$$.fragment,s),Zp=h(s),kt=n(s,"P",{});var x_=l(kt);Pm=o(x_,"As you can see, the first sample has a longer sequence than the second sample. Let\u2019s create a function that will preprocess the dataset. Specify a maximum sample length, and the feature extractor will either pad or truncate the sequences to match it:"),x_.forEach(a),sc=h(s),d(ma.$$.fragment,s),ec=h(s),xt=n(s,"P",{});var T_=l(xt);Cm=o(T_,"Apply the function to the the first few examples in the dataset:"),T_.forEach(a),ac=h(s),d(da.$$.fragment,s),tc=h(s),Tt=n(s,"P",{});var q_=l(Tt);Om=o(q_,"Now take another look at the processed sample lengths:"),q_.forEach(a),nc=h(s),d(fa.$$.fragment,s),lc=h(s),qt=n(s,"P",{});var D_=l(qt);Lm=o(D_,"The lengths of the first two samples now match the maximum length you specified."),D_.forEach(a),rc=h(s),ds=n(s,"H2",{class:!0});var Jh=l(ds);ue=n(Jh,"A",{id:!0,class:!0,href:!0});var A_=l(ue);Cl=n(A_,"SPAN",{});var z_=l(Cl);d(ba.$$.fragment,z_),z_.forEach(a),A_.forEach(a),Sm=h(Jh),Ol=n(Jh,"SPAN",{});var P_=l(Ol);Nm=o(P_,"Vision"),P_.forEach(a),Jh.forEach(a),oc=h(s),Dt=n(s,"P",{});var C_=l(Dt);Im=o(C_,"A feature extractor is also used to process images for vision tasks. Once again, the goal is to convert the raw image into a batch of tensors as input."),C_.forEach(a),pc=h(s),X=n(s,"P",{});var cn=l(X);Rm=o(cn,"Let\u2019s load the "),ja=n(cn,"A",{href:!0,rel:!0});var O_=l(ja);Fm=o(O_,"food101"),O_.forEach(a),Hm=o(cn," dataset for this tutorial. Use \u{1F917} Datasets "),Ll=n(cn,"CODE",{});var L_=l(Ll);Bm=o(L_,"split"),L_.forEach(a),Jm=o(cn," parameter to only load a small sample from the training split since the dataset is quite large:"),cn.forEach(a),cc=h(s),d(_a.$$.fragment,s),hc=h(s),me=n(s,"P",{});var Wh=l(me);Wm=o(Wh,"Next, take a look at the image with \u{1F917} Datasets "),ga=n(Wh,"A",{href:!0,rel:!0});var S_=l(ga);Sl=n(S_,"CODE",{});var N_=l(Sl);Ym=o(N_,"Image"),N_.forEach(a),S_.forEach(a),Gm=o(Wh," feature:"),Wh.forEach(a),ic=h(s),d(va.$$.fragment,s),uc=h(s),At=n(s,"P",{});var I_=l(At);zt=n(I_,"IMG",{src:!0,alt:!0}),I_.forEach(a),mc=h(s),fs=n(s,"H3",{class:!0});var Yh=l(fs);de=n(Yh,"A",{id:!0,class:!0,href:!0});var R_=l(de);Nl=n(R_,"SPAN",{});var F_=l(Nl);d(Ea.$$.fragment,F_),F_.forEach(a),R_.forEach(a),Um=h(Yh),Il=n(Yh,"SPAN",{});var H_=l(Il);Mm=o(H_,"Feature extractor"),H_.forEach(a),Yh.forEach(a),dc=h(s),fe=n(s,"P",{});var Gh=l(fe);Vm=o(Gh,"Load the feature extractor with "),Pt=n(Gh,"A",{href:!0});var B_=l(Pt);Km=o(B_,"AutoFeatureExtractor.from_pretrained()"),B_.forEach(a),Qm=o(Gh,":"),Gh.forEach(a),fc=h(s),d(wa.$$.fragment,s),bc=h(s),bs=n(s,"H3",{class:!0});var Uh=l(bs);be=n(Uh,"A",{id:!0,class:!0,href:!0});var J_=l(be);Rl=n(J_,"SPAN",{});var W_=l(Rl);d($a.$$.fragment,W_),W_.forEach(a),J_.forEach(a),Xm=h(Uh),Fl=n(Uh,"SPAN",{});var Y_=l(Fl);Zm=o(Y_,"Data augmentation"),Y_.forEach(a),Uh.forEach(a),jc=h(s),je=n(s,"P",{});var Mh=l(je);sd=o(Mh,"For vision tasks, it is common to add some type of data augmentation to the images as a part of preprocessing. You can add augmentations with any library you\u2019d like, but in this tutorial, you will use torchvision\u2019s "),ya=n(Mh,"A",{href:!0,rel:!0});var G_=l(ya);Hl=n(G_,"CODE",{});var U_=l(Hl);ed=o(U_,"transforms"),U_.forEach(a),G_.forEach(a),ad=o(Mh," module."),Mh.forEach(a),_c=h(s),Ct=n(s,"OL",{});var M_=l(Ct);S=n(M_,"LI",{});var Ae=l(S);td=o(Ae,"Normalize the image and use "),ka=n(Ae,"A",{href:!0,rel:!0});var V_=l(ka);Bl=n(V_,"CODE",{});var K_=l(Bl);nd=o(K_,"Compose"),K_.forEach(a),V_.forEach(a),ld=o(Ae," to chain some transforms - "),xa=n(Ae,"A",{href:!0,rel:!0});var Q_=l(xa);Jl=n(Q_,"CODE",{});var X_=l(Jl);rd=o(X_,"RandomResizedCrop"),X_.forEach(a),Q_.forEach(a),od=o(Ae," and "),Ta=n(Ae,"A",{href:!0,rel:!0});var Z_=l(Ta);Wl=n(Z_,"CODE",{});var sg=l(Wl);pd=o(sg,"ColorJitter"),sg.forEach(a),Z_.forEach(a),cd=o(Ae," - together:"),Ae.forEach(a),M_.forEach(a),gc=h(s),d(qa.$$.fragment,s),vc=h(s),Da=n(s,"OL",{start:!0});var eg=l(Da);js=n(eg,"LI",{});var hn=l(js);hd=o(hn,"The model accepts "),Ot=n(hn,"A",{href:!0});var ag=l(Ot);Yl=n(ag,"CODE",{});var tg=l(Yl);id=o(tg,"pixel_values"),tg.forEach(a),ag.forEach(a),ud=o(hn," as it\u2019s input. This value is generated by the feature extractor. Create a function that generates "),Gl=n(hn,"CODE",{});var ng=l(Gl);md=o(ng,"pixel_values"),ng.forEach(a),dd=o(hn," from the transforms:"),hn.forEach(a),eg.forEach(a),Ec=h(s),d(Aa.$$.fragment,s),wc=h(s),za=n(s,"OL",{start:!0});var lg=l(za);Pa=n(lg,"LI",{});var Vh=l(Pa);fd=o(Vh,"Then use \u{1F917} Datasets "),Ca=n(Vh,"A",{href:!0,rel:!0});var rg=l(Ca);Ul=n(rg,"CODE",{});var og=l(Ul);bd=o(og,"set_transform"),og.forEach(a),rg.forEach(a),jd=o(Vh," to apply the transforms on-the-fly:"),Vh.forEach(a),lg.forEach(a),$c=h(s),d(Oa.$$.fragment,s),yc=h(s),La=n(s,"OL",{start:!0});var pg=l(La);Sa=n(pg,"LI",{});var Kh=l(Sa);_d=o(Kh,"Now when you access the image, you will notice the feature extractor has added the model input "),Ml=n(Kh,"CODE",{});var cg=l(Ml);gd=o(cg,"pixel_values"),cg.forEach(a),vd=o(Kh,":"),Kh.forEach(a),pg.forEach(a),kc=h(s),d(Na.$$.fragment,s),xc=h(s),Lt=n(s,"P",{});var hg=l(Lt);Ed=o(hg,"Here is what the image looks like after you preprocess it. Just as you\u2019d expect from the applied transforms, the image has been randomly cropped and it\u2019s color properties are different."),hg.forEach(a),Tc=h(s),d(Ia.$$.fragment,s),qc=h(s),St=n(s,"P",{});var ig=l(St);Nt=n(ig,"IMG",{src:!0,alt:!0}),ig.forEach(a),Dc=h(s),_s=n(s,"H2",{class:!0});var Qh=l(_s);_e=n(Qh,"A",{id:!0,class:!0,href:!0});var ug=l(_e);Vl=n(ug,"SPAN",{});var mg=l(Vl);d(Ra.$$.fragment,mg),mg.forEach(a),ug.forEach(a),wd=h(Qh),Kl=n(Qh,"SPAN",{});var dg=l(Kl);$d=o(dg,"Multimodal"),dg.forEach(a),Qh.forEach(a),Ac=h(s),It=n(s,"P",{});var fg=l(It);yd=o(fg,"For multimodal tasks. you will use a combination of everything you\u2019ve learned so far and apply your skills to a automatic speech recognition (ASR) task. This means you will need a:"),fg.forEach(a),zc=h(s),ge=n(s,"UL",{});var Xh=l(ge);Ql=n(Xh,"LI",{});var bg=l(Ql);kd=o(bg,"Feature extractor to preprocess the audio data."),bg.forEach(a),xd=h(Xh),Xl=n(Xh,"LI",{});var jg=l(Xl);Td=o(jg,"Tokenizer to process the text."),jg.forEach(a),Xh.forEach(a),Pc=h(s),ve=n(s,"P",{});var Zh=l(ve);qd=o(Zh,"Let\u2019s return to the "),Fa=n(Zh,"A",{href:!0,rel:!0});var _g=l(Fa);Dd=o(_g,"LJ Speech"),_g.forEach(a),Ad=o(Zh," dataset:"),Zh.forEach(a),Cc=h(s),d(Ha.$$.fragment,s),Oc=h(s),Z=n(s,"P",{});var un=l(Z);zd=o(un,"Since you are mainly interested in the "),Zl=n(un,"CODE",{});var gg=l(Zl);Pd=o(gg,"audio"),gg.forEach(a),Cd=o(un," and "),sr=n(un,"CODE",{});var vg=l(sr);Od=o(vg,"text"),vg.forEach(a),Ld=o(un," column, remove the other columns:"),un.forEach(a),Lc=h(s),d(Ba.$$.fragment,s),Sc=h(s),ss=n(s,"P",{});var mn=l(ss);Sd=o(mn,"Now take a look at the "),er=n(mn,"CODE",{});var Eg=l(er);Nd=o(Eg,"audio"),Eg.forEach(a),Id=o(mn," and "),ar=n(mn,"CODE",{});var wg=l(ar);Rd=o(wg,"text"),wg.forEach(a),Fd=o(mn," columns:"),mn.forEach(a),Nc=h(s),d(Ja.$$.fragment,s),Ic=h(s),Ee=n(s,"P",{});var si=l(Ee);Hd=o(si,"Remember from the earlier section on processing audio data, you should always "),Rt=n(si,"A",{href:!0});var $g=l(Rt);Bd=o($g,"resample"),$g.forEach(a),Jd=o(si," your audio data\u2019s sampling rate to match the sampling rate of the dataset used to pretrain a model:"),si.forEach(a),Rc=h(s),d(Wa.$$.fragment,s),Fc=h(s),gs=n(s,"H3",{class:!0});var ei=l(gs);we=n(ei,"A",{id:!0,class:!0,href:!0});var yg=l(we);tr=n(yg,"SPAN",{});var kg=l(tr);d(Ya.$$.fragment,kg),kg.forEach(a),yg.forEach(a),Wd=h(ei),nr=n(ei,"SPAN",{});var xg=l(nr);Yd=o(xg,"Processor"),xg.forEach(a),ei.forEach(a),Hc=h(s),Ft=n(s,"P",{});var Tg=l(Ft);Gd=o(Tg,"A processor combines a feature extractor and tokenizer. Load a processor with [`AutoProcessor.from_pretrained]:"),Tg.forEach(a),Bc=h(s),d(Ga.$$.fragment,s),Jc=h(s),Ht=n(s,"OL",{});var qg=l(Ht);vs=n(qg,"LI",{});var dn=l(vs);Ud=o(dn,"Create a function to process the audio data to "),lr=n(dn,"CODE",{});var Dg=l(lr);Md=o(Dg,"input_values"),Dg.forEach(a),Vd=o(dn,", and tokenizes the text to "),rr=n(dn,"CODE",{});var Ag=l(rr);Kd=o(Ag,"labels"),Ag.forEach(a),Qd=o(dn,". These are your inputs to the model:"),dn.forEach(a),qg.forEach(a),Wc=h(s),d(Ua.$$.fragment,s),Yc=h(s),Ma=n(s,"OL",{start:!0});var zg=l(Ma);Va=n(zg,"LI",{});var ai=l(Va);Xd=o(ai,"Apply the "),or=n(ai,"CODE",{});var Pg=l(or);Zd=o(Pg,"prepare_dataset"),Pg.forEach(a),sf=o(ai," function to a sample:"),ai.forEach(a),zg.forEach(a),Gc=h(s),d(Ka.$$.fragment,s),Uc=h(s),es=n(s,"P",{});var fn=l(es);ef=o(fn,"Notice the processor has added "),pr=n(fn,"CODE",{});var Cg=l(pr);af=o(Cg,"input_values"),Cg.forEach(a),tf=o(fn," and "),cr=n(fn,"CODE",{});var Og=l(cr);nf=o(Og,"labels"),Og.forEach(a),lf=o(fn,". The sampling rate has also been correctly downsampled to 16kHz."),fn.forEach(a),Mc=h(s),Bt=n(s,"P",{});var Lg=l(Bt);rf=o(Lg,"Awesome, you should now be able to preprocess data for any modality and even combine different modalities! In the next tutorial, learn how to fine-tune a model on your newly preprocessed data."),Lg.forEach(a),Vc=h(s),Es=n(s,"H2",{class:!0});var ti=l(Es);$e=n(ti,"A",{id:!0,class:!0,href:!0});var Sg=l($e);hr=n(Sg,"SPAN",{});var Ng=l(hr);d(Qa.$$.fragment,Ng),Ng.forEach(a),Sg.forEach(a),of=h(ti),ir=n(ti,"SPAN",{});var Ig=l(ir);pf=o(Ig,"Everything you always wanted to know about padding and truncation"),Ig.forEach(a),ti.forEach(a),Kc=h(s),z=n(s,"P",{});var ze=l(z);cf=o(ze,`We have seen the commands that will work for most cases (pad your batch to the length of the maximum sentence and
truncate to the maximum length the model can accept). However, the API supports more strategies if you need them. The
three arguments you need to know for this are `),ur=n(ze,"CODE",{});var Rg=l(ur);hf=o(Rg,"padding"),Rg.forEach(a),uf=o(ze,", "),mr=n(ze,"CODE",{});var Fg=l(mr);mf=o(Fg,"truncation"),Fg.forEach(a),df=o(ze," and "),dr=n(ze,"CODE",{});var Hg=l(dr);ff=o(Hg,"max_length"),Hg.forEach(a),bf=o(ze,"."),ze.forEach(a),Qc=h(s),as=n(s,"UL",{});var bn=l(as);Xa=n(bn,"LI",{});var ni=l(Xa);Jt=n(ni,"P",{});var F1=l(Jt);fr=n(F1,"CODE",{});var Bg=l(fr);jf=o(Bg,"padding"),Bg.forEach(a),_f=o(F1," controls the padding. It can be a boolean or a string which should be:"),F1.forEach(a),gf=h(ni),ws=n(ni,"UL",{});var jn=l(ws);ye=n(jn,"LI",{});var Wo=l(ye);br=n(Wo,"CODE",{});var Jg=l(br);vf=o(Jg,"True"),Jg.forEach(a),Ef=o(Wo," or "),jr=n(Wo,"CODE",{});var Wg=l(jr);wf=o(Wg,"'longest'"),Wg.forEach(a),$f=o(Wo,` to pad to the longest sequence in the batch (doing no padding if you only provide
a single sequence).`),Wo.forEach(a),yf=h(jn),P=n(jn,"LI",{});var Js=l(P);_r=n(Js,"CODE",{});var Yg=l(_r);kf=o(Yg,"'max_length'"),Yg.forEach(a),xf=o(Js," to pad to a length specified by the "),gr=n(Js,"CODE",{});var Gg=l(gr);Tf=o(Gg,"max_length"),Gg.forEach(a),qf=o(Js,` argument or the maximum length accepted
by the model if no `),vr=n(Js,"CODE",{});var Ug=l(vr);Df=o(Ug,"max_length"),Ug.forEach(a),Af=o(Js," is provided ("),Er=n(Js,"CODE",{});var Mg=l(Er);zf=o(Mg,"max_length=None"),Mg.forEach(a),Pf=o(Js,`). If you only provide a single sequence,
padding will still be applied to it.`),Js.forEach(a),Cf=h(jn),ke=n(jn,"LI",{});var Yo=l(ke);wr=n(Yo,"CODE",{});var Vg=l(wr);Of=o(Vg,"False"),Vg.forEach(a),Lf=o(Yo," or "),$r=n(Yo,"CODE",{});var Kg=l($r);Sf=o(Kg,"'do_not_pad'"),Kg.forEach(a),Nf=o(Yo,` to not pad the sequences. As we have seen before, this is the default
behavior.`),Yo.forEach(a),jn.forEach(a),ni.forEach(a),If=h(bn),Za=n(bn,"LI",{});var li=l(Za);Wt=n(li,"P",{});var H1=l(Wt);yr=n(H1,"CODE",{});var Qg=l(yr);Rf=o(Qg,"truncation"),Qg.forEach(a),Ff=o(H1," controls the truncation. It can be a boolean or a string which should be:"),H1.forEach(a),Hf=h(li),N=n(li,"UL",{});var Pe=l(N);T=n(Pe,"LI",{});var I=l(T);kr=n(I,"CODE",{});var Xg=l(kr);Bf=o(Xg,"True"),Xg.forEach(a),Jf=o(I," or "),xr=n(I,"CODE",{});var Zg=l(xr);Wf=o(Zg,"'longest_first'"),Zg.forEach(a),Yf=o(I," truncate to a maximum length specified by the "),Tr=n(I,"CODE",{});var s2=l(Tr);Gf=o(s2,"max_length"),s2.forEach(a),Uf=o(I,` argument or
the maximum length accepted by the model if no `),qr=n(I,"CODE",{});var e2=l(qr);Mf=o(e2,"max_length"),e2.forEach(a),Vf=o(I," is provided ("),Dr=n(I,"CODE",{});var a2=l(Dr);Kf=o(a2,"max_length=None"),a2.forEach(a),Qf=o(I,`). This will
truncate token by token, removing a token from the longest sequence in the pair until the proper length is
reached.`),I.forEach(a),Xf=h(Pe),C=n(Pe,"LI",{});var Ws=l(C);Ar=n(Ws,"CODE",{});var t2=l(Ar);Zf=o(t2,"'only_second'"),t2.forEach(a),sb=o(Ws," truncate to a maximum length specified by the "),zr=n(Ws,"CODE",{});var n2=l(zr);eb=o(n2,"max_length"),n2.forEach(a),ab=o(Ws,` argument or the maximum
length accepted by the model if no `),Pr=n(Ws,"CODE",{});var l2=l(Pr);tb=o(l2,"max_length"),l2.forEach(a),nb=o(Ws," is provided ("),Cr=n(Ws,"CODE",{});var r2=l(Cr);lb=o(r2,"max_length=None"),r2.forEach(a),rb=o(Ws,`). This will only truncate
the second sentence of a pair if a pair of sequence (or a batch of pairs of sequences) is provided.`),Ws.forEach(a),ob=h(Pe),O=n(Pe,"LI",{});var Ys=l(O);Or=n(Ys,"CODE",{});var o2=l(Or);pb=o(o2,"'only_first'"),o2.forEach(a),cb=o(Ys," truncate to a maximum length specified by the "),Lr=n(Ys,"CODE",{});var p2=l(Lr);hb=o(p2,"max_length"),p2.forEach(a),ib=o(Ys,` argument or the maximum
length accepted by the model if no `),Sr=n(Ys,"CODE",{});var c2=l(Sr);ub=o(c2,"max_length"),c2.forEach(a),mb=o(Ys," is provided ("),Nr=n(Ys,"CODE",{});var h2=l(Nr);db=o(h2,"max_length=None"),h2.forEach(a),fb=o(Ys,`). This will only truncate
the first sentence of a pair if a pair of sequence (or a batch of pairs of sequences) is provided.`),Ys.forEach(a),bb=h(Pe),xe=n(Pe,"LI",{});var Go=l(xe);Ir=n(Go,"CODE",{});var i2=l(Ir);jb=o(i2,"False"),i2.forEach(a),_b=o(Go," or "),Rr=n(Go,"CODE",{});var u2=l(Rr);gb=o(u2,"'do_not_truncate'"),u2.forEach(a),vb=o(Go,` to not truncate the sequences. As we have seen before, this is the
default behavior.`),Go.forEach(a),Pe.forEach(a),li.forEach(a),Eb=h(bn),Fr=n(bn,"LI",{});var m2=l(Fr);ts=n(m2,"P",{});var et=l(ts);Hr=n(et,"CODE",{});var d2=l(Hr);wb=o(d2,"max_length"),d2.forEach(a),$b=o(et," to control the length of the padding/truncation. It can be an integer or "),Br=n(et,"CODE",{});var f2=l(Br);yb=o(f2,"None"),f2.forEach(a),kb=o(et,`, in which case
it will default to the maximum length the model can accept. If the model has no specific maximum input length,
truncation/padding to `),Jr=n(et,"CODE",{});var b2=l(Jr);xb=o(b2,"max_length"),b2.forEach(a),Tb=o(et," is deactivated."),et.forEach(a),m2.forEach(a),bn.forEach(a),Xc=h(s),y=n(s,"P",{});var L=l(y);qb=o(L,`Here is a table summarizing the recommend way to setup padding and truncation. If you use pair of inputs sequence in
any of the following examples, you can replace `),Wr=n(L,"CODE",{});var j2=l(Wr);Db=o(j2,"truncation=True"),j2.forEach(a),Ab=o(L," by a "),Yr=n(L,"CODE",{});var _2=l(Yr);zb=o(_2,"STRATEGY"),_2.forEach(a),Pb=o(L,` selected in
`),Gr=n(L,"CODE",{});var g2=l(Gr);Cb=o(g2,"['only_first', 'only_second', 'longest_first']"),g2.forEach(a),Ob=o(L,", i.e. "),Ur=n(L,"CODE",{});var v2=l(Ur);Lb=o(v2,"truncation='only_second'"),v2.forEach(a),Sb=o(L," or "),Mr=n(L,"CODE",{});var E2=l(Mr);Nb=o(E2,"truncation= 'longest_first'"),E2.forEach(a),Ib=o(L," to control how both sequence in the pair are truncated as detailed before."),L.forEach(a),Zc=h(s),Te=n(s,"TABLE",{});var ri=l(Te);Vr=n(ri,"THEAD",{});var w2=l(Vr);$s=n(w2,"TR",{});var _n=l($s);Kr=n(_n,"TH",{});var $2=l(Kr);Rb=o($2,"Truncation"),$2.forEach(a),Fb=h(_n),Qr=n(_n,"TH",{});var y2=l(Qr);Hb=o(y2,"Padding"),y2.forEach(a),Bb=h(_n),Xr=n(_n,"TH",{});var k2=l(Xr);Jb=o(k2,"Instruction"),k2.forEach(a),_n.forEach(a),w2.forEach(a),Wb=h(ri),g=n(ri,"TBODY",{});var v=l(g);ys=n(v,"TR",{});var gn=l(ys);Zr=n(gn,"TD",{});var x2=l(Zr);Yb=o(x2,"no truncation"),x2.forEach(a),Gb=h(gn),so=n(gn,"TD",{});var T2=l(so);Ub=o(T2,"no padding"),T2.forEach(a),Mb=h(gn),eo=n(gn,"TD",{});var q2=l(eo);ao=n(q2,"CODE",{});var D2=l(ao);Vb=o(D2,"tokenizer(batch_sentences)"),D2.forEach(a),q2.forEach(a),gn.forEach(a),Kb=h(v),ks=n(v,"TR",{});var vn=l(ks);sh=n(vn,"TD",{}),l(sh).forEach(a),Qb=h(vn),to=n(vn,"TD",{});var A2=l(to);Xb=o(A2,"padding to max sequence in batch"),A2.forEach(a),Zb=h(vn),Yt=n(vn,"TD",{});var B1=l(Yt);no=n(B1,"CODE",{});var z2=l(no);sj=o(z2,"tokenizer(batch_sentences, padding=True)"),z2.forEach(a),ej=o(B1," or"),B1.forEach(a),vn.forEach(a),aj=h(v),xs=n(v,"TR",{});var En=l(xs);eh=n(En,"TD",{}),l(eh).forEach(a),tj=h(En),ah=n(En,"TD",{}),l(ah).forEach(a),nj=h(En),lo=n(En,"TD",{});var P2=l(lo);ro=n(P2,"CODE",{});var C2=l(ro);lj=o(C2,"tokenizer(batch_sentences, padding='longest')"),C2.forEach(a),P2.forEach(a),En.forEach(a),rj=h(v),Ts=n(v,"TR",{});var wn=l(Ts);th=n(wn,"TD",{}),l(th).forEach(a),oj=h(wn),oo=n(wn,"TD",{});var O2=l(oo);pj=o(O2,"padding to max model input length"),O2.forEach(a),cj=h(wn),po=n(wn,"TD",{});var L2=l(po);co=n(L2,"CODE",{});var S2=l(co);hj=o(S2,"tokenizer(batch_sentences, padding='max_length')"),S2.forEach(a),L2.forEach(a),wn.forEach(a),ij=h(v),qs=n(v,"TR",{});var $n=l(qs);nh=n($n,"TD",{}),l(nh).forEach(a),uj=h($n),ho=n($n,"TD",{});var N2=l(ho);mj=o(N2,"padding to specific length"),N2.forEach(a),dj=h($n),io=n($n,"TD",{});var I2=l(io);uo=n(I2,"CODE",{});var R2=l(uo);fj=o(R2,"tokenizer(batch_sentences, padding='max_length', max_length=42)"),R2.forEach(a),I2.forEach(a),$n.forEach(a),bj=h(v),Ds=n(v,"TR",{});var yn=l(Ds);mo=n(yn,"TD",{});var F2=l(mo);jj=o(F2,"truncation to max model input length"),F2.forEach(a),_j=h(yn),fo=n(yn,"TD",{});var H2=l(fo);gj=o(H2,"no padding"),H2.forEach(a),vj=h(yn),Gt=n(yn,"TD",{});var J1=l(Gt);bo=n(J1,"CODE",{});var B2=l(bo);Ej=o(B2,"tokenizer(batch_sentences, truncation=True)"),B2.forEach(a),wj=o(J1," or"),J1.forEach(a),yn.forEach(a),$j=h(v),As=n(v,"TR",{});var kn=l(As);lh=n(kn,"TD",{}),l(lh).forEach(a),yj=h(kn),rh=n(kn,"TD",{}),l(rh).forEach(a),kj=h(kn),jo=n(kn,"TD",{});var J2=l(jo);_o=n(J2,"CODE",{});var W2=l(_o);xj=o(W2,"tokenizer(batch_sentences, truncation=STRATEGY)"),W2.forEach(a),J2.forEach(a),kn.forEach(a),Tj=h(v),zs=n(v,"TR",{});var xn=l(zs);oh=n(xn,"TD",{}),l(oh).forEach(a),qj=h(xn),go=n(xn,"TD",{});var Y2=l(go);Dj=o(Y2,"padding to max sequence in batch"),Y2.forEach(a),Aj=h(xn),Ut=n(xn,"TD",{});var W1=l(Ut);vo=n(W1,"CODE",{});var G2=l(vo);zj=o(G2,"tokenizer(batch_sentences, padding=True, truncation=True)"),G2.forEach(a),Pj=o(W1," or"),W1.forEach(a),xn.forEach(a),Cj=h(v),Ps=n(v,"TR",{});var Tn=l(Ps);ph=n(Tn,"TD",{}),l(ph).forEach(a),Oj=h(Tn),ch=n(Tn,"TD",{}),l(ch).forEach(a),Lj=h(Tn),Eo=n(Tn,"TD",{});var U2=l(Eo);wo=n(U2,"CODE",{});var M2=l(wo);Sj=o(M2,"tokenizer(batch_sentences, padding=True, truncation=STRATEGY)"),M2.forEach(a),U2.forEach(a),Tn.forEach(a),Nj=h(v),Cs=n(v,"TR",{});var qn=l(Cs);hh=n(qn,"TD",{}),l(hh).forEach(a),Ij=h(qn),$o=n(qn,"TD",{});var V2=l($o);Rj=o(V2,"padding to max model input length"),V2.forEach(a),Fj=h(qn),Mt=n(qn,"TD",{});var Y1=l(Mt);yo=n(Y1,"CODE",{});var K2=l(yo);Hj=o(K2,"tokenizer(batch_sentences, padding='max_length', truncation=True)"),K2.forEach(a),Bj=o(Y1," or"),Y1.forEach(a),qn.forEach(a),Jj=h(v),Os=n(v,"TR",{});var Dn=l(Os);ih=n(Dn,"TD",{}),l(ih).forEach(a),Wj=h(Dn),uh=n(Dn,"TD",{}),l(uh).forEach(a),Yj=h(Dn),ko=n(Dn,"TD",{});var Q2=l(ko);xo=n(Q2,"CODE",{});var X2=l(xo);Gj=o(X2,"tokenizer(batch_sentences, padding='max_length', truncation=STRATEGY)"),X2.forEach(a),Q2.forEach(a),Dn.forEach(a),Uj=h(v),Ls=n(v,"TR",{});var An=l(Ls);mh=n(An,"TD",{}),l(mh).forEach(a),Mj=h(An),To=n(An,"TD",{});var Z2=l(To);Vj=o(Z2,"padding to specific length"),Z2.forEach(a),Kj=h(An),qo=n(An,"TD",{});var sv=l(qo);Qj=o(sv,"Not possible"),sv.forEach(a),An.forEach(a),Xj=h(v),Ss=n(v,"TR",{});var zn=l(Ss);Do=n(zn,"TD",{});var ev=l(Do);Zj=o(ev,"truncation to specific length"),ev.forEach(a),s1=h(zn),Ao=n(zn,"TD",{});var av=l(Ao);e1=o(av,"no padding"),av.forEach(a),a1=h(zn),Vt=n(zn,"TD",{});var G1=l(Vt);zo=n(G1,"CODE",{});var tv=l(zo);t1=o(tv,"tokenizer(batch_sentences, truncation=True, max_length=42)"),tv.forEach(a),n1=o(G1," or"),G1.forEach(a),zn.forEach(a),l1=h(v),Ns=n(v,"TR",{});var Pn=l(Ns);dh=n(Pn,"TD",{}),l(dh).forEach(a),r1=h(Pn),fh=n(Pn,"TD",{}),l(fh).forEach(a),o1=h(Pn),Po=n(Pn,"TD",{});var nv=l(Po);Co=n(nv,"CODE",{});var lv=l(Co);p1=o(lv,"tokenizer(batch_sentences, truncation=STRATEGY, max_length=42)"),lv.forEach(a),nv.forEach(a),Pn.forEach(a),c1=h(v),Is=n(v,"TR",{});var Cn=l(Is);bh=n(Cn,"TD",{}),l(bh).forEach(a),h1=h(Cn),Oo=n(Cn,"TD",{});var rv=l(Oo);i1=o(rv,"padding to max sequence in batch"),rv.forEach(a),u1=h(Cn),Kt=n(Cn,"TD",{});var U1=l(Kt);Lo=n(U1,"CODE",{});var ov=l(Lo);m1=o(ov,"tokenizer(batch_sentences, padding=True, truncation=True, max_length=42)"),ov.forEach(a),d1=o(U1," or"),U1.forEach(a),Cn.forEach(a),f1=h(v),Rs=n(v,"TR",{});var On=l(Rs);jh=n(On,"TD",{}),l(jh).forEach(a),b1=h(On),_h=n(On,"TD",{}),l(_h).forEach(a),j1=h(On),So=n(On,"TD",{});var pv=l(So);No=n(pv,"CODE",{});var cv=l(No);_1=o(cv,"tokenizer(batch_sentences, padding=True, truncation=STRATEGY, max_length=42)"),cv.forEach(a),pv.forEach(a),On.forEach(a),g1=h(v),Fs=n(v,"TR",{});var Ln=l(Fs);gh=n(Ln,"TD",{}),l(gh).forEach(a),v1=h(Ln),Io=n(Ln,"TD",{});var hv=l(Io);E1=o(hv,"padding to max model input length"),hv.forEach(a),w1=h(Ln),Ro=n(Ln,"TD",{});var iv=l(Ro);$1=o(iv,"Not possible"),iv.forEach(a),Ln.forEach(a),y1=h(v),Hs=n(v,"TR",{});var Sn=l(Hs);vh=n(Sn,"TD",{}),l(vh).forEach(a),k1=h(Sn),Fo=n(Sn,"TD",{});var uv=l(Fo);x1=o(uv,"padding to specific length"),uv.forEach(a),T1=h(Sn),Qt=n(Sn,"TD",{});var M1=l(Qt);Ho=n(M1,"CODE",{});var mv=l(Ho);q1=o(mv,"tokenizer(batch_sentences, padding='max_length', truncation=True, max_length=42)"),mv.forEach(a),D1=o(M1," or"),M1.forEach(a),Sn.forEach(a),A1=h(v),Bs=n(v,"TR",{});var Nn=l(Bs);Eh=n(Nn,"TD",{}),l(Eh).forEach(a),z1=h(Nn),wh=n(Nn,"TD",{}),l(wh).forEach(a),P1=h(Nn),Bo=n(Nn,"TD",{});var dv=l(Bo);Jo=n(dv,"CODE",{});var fv=l(Jo);C1=o(fv,"tokenizer(batch_sentences, padding='max_length', truncation=STRATEGY, max_length=42)"),fv.forEach(a),dv.forEach(a),Nn.forEach(a),v.forEach(a),ri.forEach(a),this.h()},h(){u(w,"name","hf:doc:metadata"),u(w,"content",JSON.stringify(Tv)),u(x,"id","preprocess"),u(x,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),u(x,"href","#preprocess"),u($,"class","relative group"),u(Us,"id","nlp"),u(Us,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),u(Us,"href","#nlp"),u(ls,"class","relative group"),u(tt,"href","main_classes/tokenizer"),u(nt,"href","/docs/transformers/pr_15839/en/model_doc/auto#transformers.AutoTokenizer"),u(Vs,"id","tokenize"),u(Vs,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),u(Vs,"href","#tokenize"),u(rs,"class","relative group"),u(lt,"href","/docs/transformers/pr_15839/en/model_doc/auto#transformers.AutoTokenizer.from_pretrained"),u(ct,"href","glossary#input-ids"),u(it,"href","glossary#attention-mask"),u(mt,"href","glossary#token-type-ids"),u(Xs,"id","pad"),u(Xs,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),u(Xs,"href","#pad"),u(os,"class","relative group"),u(ee,"id","truncation"),u(ee,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),u(ee,"href","#truncation"),u(ps,"class","relative group"),u(ae,"id","build-tensors"),u(ae,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),u(ae,"href","#build-tensors"),u(cs,"class","relative group"),u(te,"id","audio"),u(te,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),u(te,"href","#audio"),u(hs,"class","relative group"),u(jt,"href","main_classes/feature_extractor"),u(Ve,"href","https://huggingface.co/datasets/superb"),u(Ve,"rel","nofollow"),u(Ke,"href","https://huggingface.co/docs/datasets/load_hub.html"),u(Ke,"rel","nofollow"),u(le,"id","resample"),u(le,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),u(le,"href","#resample"),u(is,"class","relative group"),u(sa,"href","https://huggingface.co/facebook/wav2vec2-base"),u(sa,"rel","nofollow"),u(ea,"href","https://huggingface.co/datasets/lj_speech"),u(ea,"rel","nofollow"),u(na,"href","https://huggingface.co/docs/datasets/package_reference/main_classes.html#datasets.Dataset.cast_column"),u(na,"rel","nofollow"),u(ra,"start","2"),u(ce,"id","feature-extractor"),u(ce,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),u(ce,"href","#feature-extractor"),u(us,"class","relative group"),u($t,"href","/docs/transformers/pr_15839/en/model_doc/auto#transformers.AutoFeatureExtractor.from_pretrained"),u(ie,"id","pad-and-truncate"),u(ie,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),u(ie,"href","#pad-and-truncate"),u(ms,"class","relative group"),u(ue,"id","vision"),u(ue,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),u(ue,"href","#vision"),u(ds,"class","relative group"),u(ja,"href","https://huggingface.co/datasets/food101"),u(ja,"rel","nofollow"),u(ga,"href","https://huggingface.co/docs/datasets/package_reference/main_classes.html?highlight=image#datasets.Image"),u(ga,"rel","nofollow"),bv(zt.src,V1="https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/vision-preprocess-tutorial.png")||u(zt,"src",V1),u(zt,"alt","vision-preprocess-tutorial.png"),u(de,"id","feature-extractor"),u(de,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),u(de,"href","#feature-extractor"),u(fs,"class","relative group"),u(Pt,"href","/docs/transformers/pr_15839/en/model_doc/auto#transformers.AutoFeatureExtractor.from_pretrained"),u(be,"id","data-augmentation"),u(be,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),u(be,"href","#data-augmentation"),u(bs,"class","relative group"),u(ya,"href","https://pytorch.org/vision/stable/transforms.html"),u(ya,"rel","nofollow"),u(ka,"href","https://pytorch.org/vision/master/generated/torchvision.transforms.Compose.html"),u(ka,"rel","nofollow"),u(xa,"href","https://pytorch.org/vision/main/generated/torchvision.transforms.RandomResizedCrop.html"),u(xa,"rel","nofollow"),u(Ta,"href","https://pytorch.org/vision/main/generated/torchvision.transforms.ColorJitter.html"),u(Ta,"rel","nofollow"),u(Ot,"href","model_doc/visionencoderdecoder#transformers.VisionEncoderDecoderModel.forward.pixel_values"),u(Da,"start","2"),u(Ca,"href","https://huggingface.co/docs/datasets/process.html#format-transform"),u(Ca,"rel","nofollow"),u(za,"start","3"),u(La,"start","4"),bv(Nt.src,K1="https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/preprocessed_image.png")||u(Nt,"src",K1),u(Nt,"alt","preprocessed_image"),u(_e,"id","multimodal"),u(_e,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),u(_e,"href","#multimodal"),u(_s,"class","relative group"),u(Fa,"href","https://huggingface.co/datasets/lj_speech"),u(Fa,"rel","nofollow"),u(Rt,"href","preprocessing#audio"),u(we,"id","processor"),u(we,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),u(we,"href","#processor"),u(gs,"class","relative group"),u(Ma,"start","2"),u($e,"id","everything-you-always-wanted-to-know-about-padding-and-truncation"),u($e,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),u($e,"href","#everything-you-always-wanted-to-know-about-padding-and-truncation"),u(Es,"class","relative group")},m(s,p){e(document.head,w),i(s,R,p),i(s,$,p),e($,x),e(x,ns),f(q,ns,null),e($,F),e($,Gs),e(Gs,oi),i(s,Uo,p),f(Ce,s,p),i(s,Mo,p),i(s,at,p),e(at,pi),i(s,Vo,p),i(s,H,p),e(H,Rn),e(Rn,ci),e(H,hi),e(H,Fn),e(Fn,ii),e(H,ui),e(H,Hn),e(Hn,mi),i(s,Ko,p),i(s,ls,p),e(ls,Us),e(Us,Bn),f(Oe,Bn,null),e(ls,di),e(ls,Jn),e(Jn,fi),i(s,Qo,p),f(Le,s,p),i(s,Xo,p),i(s,B,p),e(B,bi),e(B,tt),e(tt,ji),e(B,_i),e(B,Wn),e(Wn,gi),e(B,vi),i(s,Zo,p),f(Ms,s,p),i(s,sp,p),i(s,J,p),e(J,Ei),e(J,nt),e(nt,wi),e(J,$i),e(J,Yn),e(Yn,yi),e(J,ki),i(s,ep,p),i(s,rs,p),e(rs,Vs),e(Vs,Gn),f(Se,Gn,null),e(rs,xi),e(rs,Un),e(Un,Ti),i(s,ap,p),i(s,Ks,p),e(Ks,qi),e(Ks,lt),e(lt,Di),e(Ks,Ai),i(s,tp,p),f(Ne,s,p),i(s,np,p),i(s,rt,p),e(rt,zi),i(s,lp,p),f(Ie,s,p),i(s,rp,p),i(s,ot,p),e(ot,Pi),i(s,op,p),i(s,W,p),e(W,pt),e(pt,ct),e(ct,Ci),e(pt,Oi),e(W,Li),e(W,ht),e(ht,it),e(it,Si),e(ht,Ni),e(W,Ii),e(W,ut),e(ut,mt),e(mt,Ri),e(ut,Fi),i(s,pp,p),i(s,Qs,p),e(Qs,Hi),e(Qs,Mn),e(Mn,Bi),e(Qs,Ji),i(s,cp,p),f(Re,s,p),i(s,hp,p),i(s,Y,p),e(Y,Wi),e(Y,Vn),e(Vn,Yi),e(Y,Gi),e(Y,Kn),e(Kn,Ui),e(Y,Mi),i(s,ip,p),i(s,dt,p),e(dt,Vi),i(s,up,p),f(Fe,s,p),i(s,mp,p),i(s,os,p),e(os,Xs),e(Xs,Qn),f(He,Qn,null),e(os,Ki),e(os,Xn),e(Xn,Qi),i(s,dp,p),i(s,Zs,p),e(Zs,Xi),e(Zs,Zn),e(Zn,Zi),e(Zs,su),i(s,fp,p),i(s,G,p),e(G,eu),e(G,sl),e(sl,au),e(G,tu),e(G,el),e(el,nu),e(G,lu),i(s,bp,p),f(Be,s,p),i(s,jp,p),i(s,se,p),e(se,ru),e(se,al),e(al,ou),e(se,pu),i(s,_p,p),i(s,ps,p),e(ps,ee),e(ee,tl),f(Je,tl,null),e(ps,cu),e(ps,nl),e(nl,hu),i(s,gp,p),i(s,ft,p),e(ft,iu),i(s,vp,p),i(s,U,p),e(U,uu),e(U,ll),e(ll,mu),e(U,du),e(U,rl),e(rl,fu),e(U,bu),i(s,Ep,p),f(We,s,p),i(s,wp,p),i(s,cs,p),e(cs,ae),e(ae,ol),f(Ye,ol,null),e(cs,ju),e(cs,pl),e(pl,_u),i(s,$p,p),i(s,bt,p),e(bt,gu),i(s,yp,p),i(s,D,p),e(D,vu),e(D,cl),e(cl,Eu),e(D,wu),e(D,hl),e(hl,$u),e(D,yu),e(D,il),e(il,ku),e(D,xu),i(s,kp,p),f(Ge,s,p),i(s,xp,p),i(s,hs,p),e(hs,te),e(te,ul),f(Ue,ul,null),e(hs,Tu),e(hs,ml),e(ml,qu),i(s,Tp,p),i(s,ne,p),e(ne,Du),e(ne,jt),e(jt,Au),e(ne,zu),i(s,qp,p),f(Me,s,p),i(s,Dp,p),i(s,M,p),e(M,Pu),e(M,Ve),e(Ve,Cu),e(M,Ou),e(M,Ke),e(Ke,Lu),e(M,Su),i(s,Ap,p),f(Qe,s,p),i(s,zp,p),i(s,V,p),e(V,Nu),e(V,dl),e(dl,Iu),e(V,Ru),e(V,fl),e(fl,Fu),e(V,Hu),i(s,Pp,p),f(Xe,s,p),i(s,Cp,p),i(s,_t,p),e(_t,Bu),i(s,Op,p),i(s,K,p),e(K,gt),e(gt,bl),e(bl,Ju),e(gt,Wu),e(K,Yu),e(K,vt),e(vt,jl),e(jl,Gu),e(vt,Uu),e(K,Mu),e(K,Et),e(Et,_l),e(_l,Vu),e(Et,Ku),i(s,Lp,p),i(s,is,p),e(is,le),e(le,gl),f(Ze,gl,null),e(is,Qu),e(is,vl),e(vl,Xu),i(s,Sp,p),i(s,re,p),e(re,Zu),e(re,sa),e(sa,sm),e(re,em),i(s,Np,p),i(s,oe,p),e(oe,am),e(oe,ea),e(ea,tm),e(oe,nm),i(s,Ip,p),f(aa,s,p),i(s,Rp,p),i(s,wt,p),e(wt,ta),e(ta,lm),e(ta,na),e(na,El),e(El,rm),e(ta,om),i(s,Fp,p),f(la,s,p),i(s,Hp,p),i(s,ra,p),e(ra,wl),e(wl,pm),i(s,Bp,p),f(oa,s,p),i(s,Jp,p),i(s,pe,p),e(pe,cm),e(pe,$l),e($l,hm),e(pe,im),i(s,Wp,p),i(s,us,p),e(us,ce),e(ce,yl),f(pa,yl,null),e(us,um),e(us,kl),e(kl,mm),i(s,Yp,p),i(s,A,p),e(A,dm),e(A,xl),e(xl,fm),e(A,bm),e(A,Tl),e(Tl,jm),e(A,_m),e(A,ql),e(ql,gm),e(A,vm),i(s,Gp,p),i(s,he,p),e(he,Em),e(he,$t),e($t,wm),e(he,$m),i(s,Up,p),f(ca,s,p),i(s,Mp,p),i(s,Q,p),e(Q,ym),e(Q,Dl),e(Dl,km),e(Q,xm),e(Q,Al),e(Al,Tm),e(Q,qm),i(s,Vp,p),f(ha,s,p),i(s,Kp,p),i(s,ms,p),e(ms,ie),e(ie,zl),f(ia,zl,null),e(ms,Dm),e(ms,Pl),e(Pl,Am),i(s,Qp,p),i(s,yt,p),e(yt,zm),i(s,Xp,p),f(ua,s,p),i(s,Zp,p),i(s,kt,p),e(kt,Pm),i(s,sc,p),f(ma,s,p),i(s,ec,p),i(s,xt,p),e(xt,Cm),i(s,ac,p),f(da,s,p),i(s,tc,p),i(s,Tt,p),e(Tt,Om),i(s,nc,p),f(fa,s,p),i(s,lc,p),i(s,qt,p),e(qt,Lm),i(s,rc,p),i(s,ds,p),e(ds,ue),e(ue,Cl),f(ba,Cl,null),e(ds,Sm),e(ds,Ol),e(Ol,Nm),i(s,oc,p),i(s,Dt,p),e(Dt,Im),i(s,pc,p),i(s,X,p),e(X,Rm),e(X,ja),e(ja,Fm),e(X,Hm),e(X,Ll),e(Ll,Bm),e(X,Jm),i(s,cc,p),f(_a,s,p),i(s,hc,p),i(s,me,p),e(me,Wm),e(me,ga),e(ga,Sl),e(Sl,Ym),e(me,Gm),i(s,ic,p),f(va,s,p),i(s,uc,p),i(s,At,p),e(At,zt),i(s,mc,p),i(s,fs,p),e(fs,de),e(de,Nl),f(Ea,Nl,null),e(fs,Um),e(fs,Il),e(Il,Mm),i(s,dc,p),i(s,fe,p),e(fe,Vm),e(fe,Pt),e(Pt,Km),e(fe,Qm),i(s,fc,p),f(wa,s,p),i(s,bc,p),i(s,bs,p),e(bs,be),e(be,Rl),f($a,Rl,null),e(bs,Xm),e(bs,Fl),e(Fl,Zm),i(s,jc,p),i(s,je,p),e(je,sd),e(je,ya),e(ya,Hl),e(Hl,ed),e(je,ad),i(s,_c,p),i(s,Ct,p),e(Ct,S),e(S,td),e(S,ka),e(ka,Bl),e(Bl,nd),e(S,ld),e(S,xa),e(xa,Jl),e(Jl,rd),e(S,od),e(S,Ta),e(Ta,Wl),e(Wl,pd),e(S,cd),i(s,gc,p),f(qa,s,p),i(s,vc,p),i(s,Da,p),e(Da,js),e(js,hd),e(js,Ot),e(Ot,Yl),e(Yl,id),e(js,ud),e(js,Gl),e(Gl,md),e(js,dd),i(s,Ec,p),f(Aa,s,p),i(s,wc,p),i(s,za,p),e(za,Pa),e(Pa,fd),e(Pa,Ca),e(Ca,Ul),e(Ul,bd),e(Pa,jd),i(s,$c,p),f(Oa,s,p),i(s,yc,p),i(s,La,p),e(La,Sa),e(Sa,_d),e(Sa,Ml),e(Ml,gd),e(Sa,vd),i(s,kc,p),f(Na,s,p),i(s,xc,p),i(s,Lt,p),e(Lt,Ed),i(s,Tc,p),f(Ia,s,p),i(s,qc,p),i(s,St,p),e(St,Nt),i(s,Dc,p),i(s,_s,p),e(_s,_e),e(_e,Vl),f(Ra,Vl,null),e(_s,wd),e(_s,Kl),e(Kl,$d),i(s,Ac,p),i(s,It,p),e(It,yd),i(s,zc,p),i(s,ge,p),e(ge,Ql),e(Ql,kd),e(ge,xd),e(ge,Xl),e(Xl,Td),i(s,Pc,p),i(s,ve,p),e(ve,qd),e(ve,Fa),e(Fa,Dd),e(ve,Ad),i(s,Cc,p),f(Ha,s,p),i(s,Oc,p),i(s,Z,p),e(Z,zd),e(Z,Zl),e(Zl,Pd),e(Z,Cd),e(Z,sr),e(sr,Od),e(Z,Ld),i(s,Lc,p),f(Ba,s,p),i(s,Sc,p),i(s,ss,p),e(ss,Sd),e(ss,er),e(er,Nd),e(ss,Id),e(ss,ar),e(ar,Rd),e(ss,Fd),i(s,Nc,p),f(Ja,s,p),i(s,Ic,p),i(s,Ee,p),e(Ee,Hd),e(Ee,Rt),e(Rt,Bd),e(Ee,Jd),i(s,Rc,p),f(Wa,s,p),i(s,Fc,p),i(s,gs,p),e(gs,we),e(we,tr),f(Ya,tr,null),e(gs,Wd),e(gs,nr),e(nr,Yd),i(s,Hc,p),i(s,Ft,p),e(Ft,Gd),i(s,Bc,p),f(Ga,s,p),i(s,Jc,p),i(s,Ht,p),e(Ht,vs),e(vs,Ud),e(vs,lr),e(lr,Md),e(vs,Vd),e(vs,rr),e(rr,Kd),e(vs,Qd),i(s,Wc,p),f(Ua,s,p),i(s,Yc,p),i(s,Ma,p),e(Ma,Va),e(Va,Xd),e(Va,or),e(or,Zd),e(Va,sf),i(s,Gc,p),f(Ka,s,p),i(s,Uc,p),i(s,es,p),e(es,ef),e(es,pr),e(pr,af),e(es,tf),e(es,cr),e(cr,nf),e(es,lf),i(s,Mc,p),i(s,Bt,p),e(Bt,rf),i(s,Vc,p),i(s,Es,p),e(Es,$e),e($e,hr),f(Qa,hr,null),e(Es,of),e(Es,ir),e(ir,pf),i(s,Kc,p),i(s,z,p),e(z,cf),e(z,ur),e(ur,hf),e(z,uf),e(z,mr),e(mr,mf),e(z,df),e(z,dr),e(dr,ff),e(z,bf),i(s,Qc,p),i(s,as,p),e(as,Xa),e(Xa,Jt),e(Jt,fr),e(fr,jf),e(Jt,_f),e(Xa,gf),e(Xa,ws),e(ws,ye),e(ye,br),e(br,vf),e(ye,Ef),e(ye,jr),e(jr,wf),e(ye,$f),e(ws,yf),e(ws,P),e(P,_r),e(_r,kf),e(P,xf),e(P,gr),e(gr,Tf),e(P,qf),e(P,vr),e(vr,Df),e(P,Af),e(P,Er),e(Er,zf),e(P,Pf),e(ws,Cf),e(ws,ke),e(ke,wr),e(wr,Of),e(ke,Lf),e(ke,$r),e($r,Sf),e(ke,Nf),e(as,If),e(as,Za),e(Za,Wt),e(Wt,yr),e(yr,Rf),e(Wt,Ff),e(Za,Hf),e(Za,N),e(N,T),e(T,kr),e(kr,Bf),e(T,Jf),e(T,xr),e(xr,Wf),e(T,Yf),e(T,Tr),e(Tr,Gf),e(T,Uf),e(T,qr),e(qr,Mf),e(T,Vf),e(T,Dr),e(Dr,Kf),e(T,Qf),e(N,Xf),e(N,C),e(C,Ar),e(Ar,Zf),e(C,sb),e(C,zr),e(zr,eb),e(C,ab),e(C,Pr),e(Pr,tb),e(C,nb),e(C,Cr),e(Cr,lb),e(C,rb),e(N,ob),e(N,O),e(O,Or),e(Or,pb),e(O,cb),e(O,Lr),e(Lr,hb),e(O,ib),e(O,Sr),e(Sr,ub),e(O,mb),e(O,Nr),e(Nr,db),e(O,fb),e(N,bb),e(N,xe),e(xe,Ir),e(Ir,jb),e(xe,_b),e(xe,Rr),e(Rr,gb),e(xe,vb),e(as,Eb),e(as,Fr),e(Fr,ts),e(ts,Hr),e(Hr,wb),e(ts,$b),e(ts,Br),e(Br,yb),e(ts,kb),e(ts,Jr),e(Jr,xb),e(ts,Tb),i(s,Xc,p),i(s,y,p),e(y,qb),e(y,Wr),e(Wr,Db),e(y,Ab),e(y,Yr),e(Yr,zb),e(y,Pb),e(y,Gr),e(Gr,Cb),e(y,Ob),e(y,Ur),e(Ur,Lb),e(y,Sb),e(y,Mr),e(Mr,Nb),e(y,Ib),i(s,Zc,p),i(s,Te,p),e(Te,Vr),e(Vr,$s),e($s,Kr),e(Kr,Rb),e($s,Fb),e($s,Qr),e(Qr,Hb),e($s,Bb),e($s,Xr),e(Xr,Jb),e(Te,Wb),e(Te,g),e(g,ys),e(ys,Zr),e(Zr,Yb),e(ys,Gb),e(ys,so),e(so,Ub),e(ys,Mb),e(ys,eo),e(eo,ao),e(ao,Vb),e(g,Kb),e(g,ks),e(ks,sh),e(ks,Qb),e(ks,to),e(to,Xb),e(ks,Zb),e(ks,Yt),e(Yt,no),e(no,sj),e(Yt,ej),e(g,aj),e(g,xs),e(xs,eh),e(xs,tj),e(xs,ah),e(xs,nj),e(xs,lo),e(lo,ro),e(ro,lj),e(g,rj),e(g,Ts),e(Ts,th),e(Ts,oj),e(Ts,oo),e(oo,pj),e(Ts,cj),e(Ts,po),e(po,co),e(co,hj),e(g,ij),e(g,qs),e(qs,nh),e(qs,uj),e(qs,ho),e(ho,mj),e(qs,dj),e(qs,io),e(io,uo),e(uo,fj),e(g,bj),e(g,Ds),e(Ds,mo),e(mo,jj),e(Ds,_j),e(Ds,fo),e(fo,gj),e(Ds,vj),e(Ds,Gt),e(Gt,bo),e(bo,Ej),e(Gt,wj),e(g,$j),e(g,As),e(As,lh),e(As,yj),e(As,rh),e(As,kj),e(As,jo),e(jo,_o),e(_o,xj),e(g,Tj),e(g,zs),e(zs,oh),e(zs,qj),e(zs,go),e(go,Dj),e(zs,Aj),e(zs,Ut),e(Ut,vo),e(vo,zj),e(Ut,Pj),e(g,Cj),e(g,Ps),e(Ps,ph),e(Ps,Oj),e(Ps,ch),e(Ps,Lj),e(Ps,Eo),e(Eo,wo),e(wo,Sj),e(g,Nj),e(g,Cs),e(Cs,hh),e(Cs,Ij),e(Cs,$o),e($o,Rj),e(Cs,Fj),e(Cs,Mt),e(Mt,yo),e(yo,Hj),e(Mt,Bj),e(g,Jj),e(g,Os),e(Os,ih),e(Os,Wj),e(Os,uh),e(Os,Yj),e(Os,ko),e(ko,xo),e(xo,Gj),e(g,Uj),e(g,Ls),e(Ls,mh),e(Ls,Mj),e(Ls,To),e(To,Vj),e(Ls,Kj),e(Ls,qo),e(qo,Qj),e(g,Xj),e(g,Ss),e(Ss,Do),e(Do,Zj),e(Ss,s1),e(Ss,Ao),e(Ao,e1),e(Ss,a1),e(Ss,Vt),e(Vt,zo),e(zo,t1),e(Vt,n1),e(g,l1),e(g,Ns),e(Ns,dh),e(Ns,r1),e(Ns,fh),e(Ns,o1),e(Ns,Po),e(Po,Co),e(Co,p1),e(g,c1),e(g,Is),e(Is,bh),e(Is,h1),e(Is,Oo),e(Oo,i1),e(Is,u1),e(Is,Kt),e(Kt,Lo),e(Lo,m1),e(Kt,d1),e(g,f1),e(g,Rs),e(Rs,jh),e(Rs,b1),e(Rs,_h),e(Rs,j1),e(Rs,So),e(So,No),e(No,_1),e(g,g1),e(g,Fs),e(Fs,gh),e(Fs,v1),e(Fs,Io),e(Io,E1),e(Fs,w1),e(Fs,Ro),e(Ro,$1),e(g,y1),e(g,Hs),e(Hs,vh),e(Hs,k1),e(Hs,Fo),e(Fo,x1),e(Hs,T1),e(Hs,Qt),e(Qt,Ho),e(Ho,q1),e(Qt,D1),e(g,A1),e(g,Bs),e(Bs,Eh),e(Bs,z1),e(Bs,wh),e(Bs,P1),e(Bs,Bo),e(Bo,Jo),e(Jo,C1),$h=!0},p(s,[p]){const st={};p&2&&(st.$$scope={dirty:p,ctx:s}),Ms.$set(st)},i(s){$h||(b(q.$$.fragment,s),b(Ce.$$.fragment,s),b(Oe.$$.fragment,s),b(Le.$$.fragment,s),b(Ms.$$.fragment,s),b(Se.$$.fragment,s),b(Ne.$$.fragment,s),b(Ie.$$.fragment,s),b(Re.$$.fragment,s),b(Fe.$$.fragment,s),b(He.$$.fragment,s),b(Be.$$.fragment,s),b(Je.$$.fragment,s),b(We.$$.fragment,s),b(Ye.$$.fragment,s),b(Ge.$$.fragment,s),b(Ue.$$.fragment,s),b(Me.$$.fragment,s),b(Qe.$$.fragment,s),b(Xe.$$.fragment,s),b(Ze.$$.fragment,s),b(aa.$$.fragment,s),b(la.$$.fragment,s),b(oa.$$.fragment,s),b(pa.$$.fragment,s),b(ca.$$.fragment,s),b(ha.$$.fragment,s),b(ia.$$.fragment,s),b(ua.$$.fragment,s),b(ma.$$.fragment,s),b(da.$$.fragment,s),b(fa.$$.fragment,s),b(ba.$$.fragment,s),b(_a.$$.fragment,s),b(va.$$.fragment,s),b(Ea.$$.fragment,s),b(wa.$$.fragment,s),b($a.$$.fragment,s),b(qa.$$.fragment,s),b(Aa.$$.fragment,s),b(Oa.$$.fragment,s),b(Na.$$.fragment,s),b(Ia.$$.fragment,s),b(Ra.$$.fragment,s),b(Ha.$$.fragment,s),b(Ba.$$.fragment,s),b(Ja.$$.fragment,s),b(Wa.$$.fragment,s),b(Ya.$$.fragment,s),b(Ga.$$.fragment,s),b(Ua.$$.fragment,s),b(Ka.$$.fragment,s),b(Qa.$$.fragment,s),$h=!0)},o(s){j(q.$$.fragment,s),j(Ce.$$.fragment,s),j(Oe.$$.fragment,s),j(Le.$$.fragment,s),j(Ms.$$.fragment,s),j(Se.$$.fragment,s),j(Ne.$$.fragment,s),j(Ie.$$.fragment,s),j(Re.$$.fragment,s),j(Fe.$$.fragment,s),j(He.$$.fragment,s),j(Be.$$.fragment,s),j(Je.$$.fragment,s),j(We.$$.fragment,s),j(Ye.$$.fragment,s),j(Ge.$$.fragment,s),j(Ue.$$.fragment,s),j(Me.$$.fragment,s),j(Qe.$$.fragment,s),j(Xe.$$.fragment,s),j(Ze.$$.fragment,s),j(aa.$$.fragment,s),j(la.$$.fragment,s),j(oa.$$.fragment,s),j(pa.$$.fragment,s),j(ca.$$.fragment,s),j(ha.$$.fragment,s),j(ia.$$.fragment,s),j(ua.$$.fragment,s),j(ma.$$.fragment,s),j(da.$$.fragment,s),j(fa.$$.fragment,s),j(ba.$$.fragment,s),j(_a.$$.fragment,s),j(va.$$.fragment,s),j(Ea.$$.fragment,s),j(wa.$$.fragment,s),j($a.$$.fragment,s),j(qa.$$.fragment,s),j(Aa.$$.fragment,s),j(Oa.$$.fragment,s),j(Na.$$.fragment,s),j(Ia.$$.fragment,s),j(Ra.$$.fragment,s),j(Ha.$$.fragment,s),j(Ba.$$.fragment,s),j(Ja.$$.fragment,s),j(Wa.$$.fragment,s),j(Ya.$$.fragment,s),j(Ga.$$.fragment,s),j(Ua.$$.fragment,s),j(Ka.$$.fragment,s),j(Qa.$$.fragment,s),$h=!1},d(s){a(w),s&&a(R),s&&a($),_(q),s&&a(Uo),_(Ce,s),s&&a(Mo),s&&a(at),s&&a(Vo),s&&a(H),s&&a(Ko),s&&a(ls),_(Oe),s&&a(Qo),_(Le,s),s&&a(Xo),s&&a(B),s&&a(Zo),_(Ms,s),s&&a(sp),s&&a(J),s&&a(ep),s&&a(rs),_(Se),s&&a(ap),s&&a(Ks),s&&a(tp),_(Ne,s),s&&a(np),s&&a(rt),s&&a(lp),_(Ie,s),s&&a(rp),s&&a(ot),s&&a(op),s&&a(W),s&&a(pp),s&&a(Qs),s&&a(cp),_(Re,s),s&&a(hp),s&&a(Y),s&&a(ip),s&&a(dt),s&&a(up),_(Fe,s),s&&a(mp),s&&a(os),_(He),s&&a(dp),s&&a(Zs),s&&a(fp),s&&a(G),s&&a(bp),_(Be,s),s&&a(jp),s&&a(se),s&&a(_p),s&&a(ps),_(Je),s&&a(gp),s&&a(ft),s&&a(vp),s&&a(U),s&&a(Ep),_(We,s),s&&a(wp),s&&a(cs),_(Ye),s&&a($p),s&&a(bt),s&&a(yp),s&&a(D),s&&a(kp),_(Ge,s),s&&a(xp),s&&a(hs),_(Ue),s&&a(Tp),s&&a(ne),s&&a(qp),_(Me,s),s&&a(Dp),s&&a(M),s&&a(Ap),_(Qe,s),s&&a(zp),s&&a(V),s&&a(Pp),_(Xe,s),s&&a(Cp),s&&a(_t),s&&a(Op),s&&a(K),s&&a(Lp),s&&a(is),_(Ze),s&&a(Sp),s&&a(re),s&&a(Np),s&&a(oe),s&&a(Ip),_(aa,s),s&&a(Rp),s&&a(wt),s&&a(Fp),_(la,s),s&&a(Hp),s&&a(ra),s&&a(Bp),_(oa,s),s&&a(Jp),s&&a(pe),s&&a(Wp),s&&a(us),_(pa),s&&a(Yp),s&&a(A),s&&a(Gp),s&&a(he),s&&a(Up),_(ca,s),s&&a(Mp),s&&a(Q),s&&a(Vp),_(ha,s),s&&a(Kp),s&&a(ms),_(ia),s&&a(Qp),s&&a(yt),s&&a(Xp),_(ua,s),s&&a(Zp),s&&a(kt),s&&a(sc),_(ma,s),s&&a(ec),s&&a(xt),s&&a(ac),_(da,s),s&&a(tc),s&&a(Tt),s&&a(nc),_(fa,s),s&&a(lc),s&&a(qt),s&&a(rc),s&&a(ds),_(ba),s&&a(oc),s&&a(Dt),s&&a(pc),s&&a(X),s&&a(cc),_(_a,s),s&&a(hc),s&&a(me),s&&a(ic),_(va,s),s&&a(uc),s&&a(At),s&&a(mc),s&&a(fs),_(Ea),s&&a(dc),s&&a(fe),s&&a(fc),_(wa,s),s&&a(bc),s&&a(bs),_($a),s&&a(jc),s&&a(je),s&&a(_c),s&&a(Ct),s&&a(gc),_(qa,s),s&&a(vc),s&&a(Da),s&&a(Ec),_(Aa,s),s&&a(wc),s&&a(za),s&&a($c),_(Oa,s),s&&a(yc),s&&a(La),s&&a(kc),_(Na,s),s&&a(xc),s&&a(Lt),s&&a(Tc),_(Ia,s),s&&a(qc),s&&a(St),s&&a(Dc),s&&a(_s),_(Ra),s&&a(Ac),s&&a(It),s&&a(zc),s&&a(ge),s&&a(Pc),s&&a(ve),s&&a(Cc),_(Ha,s),s&&a(Oc),s&&a(Z),s&&a(Lc),_(Ba,s),s&&a(Sc),s&&a(ss),s&&a(Nc),_(Ja,s),s&&a(Ic),s&&a(Ee),s&&a(Rc),_(Wa,s),s&&a(Fc),s&&a(gs),_(Ya),s&&a(Hc),s&&a(Ft),s&&a(Bc),_(Ga,s),s&&a(Jc),s&&a(Ht),s&&a(Wc),_(Ua,s),s&&a(Yc),s&&a(Ma),s&&a(Gc),_(Ka,s),s&&a(Uc),s&&a(es),s&&a(Mc),s&&a(Bt),s&&a(Vc),s&&a(Es),_(Qa),s&&a(Kc),s&&a(z),s&&a(Qc),s&&a(as),s&&a(Xc),s&&a(y),s&&a(Zc),s&&a(Te)}}}const Tv={local:"preprocess",sections:[{local:"nlp",sections:[{local:"tokenize",title:"Tokenize"},{local:"pad",title:"Pad"},{local:"truncation",title:"Truncation"},{local:"build-tensors",title:"Build tensors"}],title:"NLP"},{local:"audio",sections:[{local:"resample",title:"Resample"},{local:"feature-extractor",title:"Feature extractor"},{local:"pad-and-truncate",title:"Pad and truncate"}],title:"Audio"},{local:"vision",sections:[{local:"feature-extractor",title:"Feature extractor"},{local:"data-augmentation",title:"Data augmentation"}],title:"Vision"},{local:"multimodal",sections:[{local:"processor",title:"Processor"}],title:"Multimodal"},{local:"everything-you-always-wanted-to-know-about-padding-and-truncation",title:"Everything you always wanted to know about padding and truncation"}],title:"Preprocess"};function qv(In,w,R){let{fw:$}=w;return In.$$set=x=>{"fw"in x&&R(0,$=x.fw)},[$]}class Nv extends jv{constructor(w){super();_v(this,w,qv,xv,gv,{fw:0})}}export{Nv as default,Tv as metadata};
