import{S as db,i as gb,s as _b,e as t,k as u,w as b,t as r,M as $b,c as l,d as a,m as c,a as p,x as f,h as o,b as m,N as bb,G as e,g as h,y as j,q as d,o as g,B as _,v as vb,L as fb}from"../chunks/vendor-hf-doc-builder.js";import{T as wb}from"../chunks/Tip-hf-doc-builder.js";import{Y as kb}from"../chunks/Youtube-hf-doc-builder.js";import{I as x}from"../chunks/IconCopyLink-hf-doc-builder.js";import{C as v}from"../chunks/CodeBlock-hf-doc-builder.js";import{D as yb}from"../chunks/DocNotebookDropdown-hf-doc-builder.js";import{F as xb,M as jb}from"../chunks/Markdown-hf-doc-builder.js";function Eb(P){let $,w,i,k,y;return{c(){$=t("p"),w=r("If you plan on using a pretrained model, it\u2019s important to use the associated pretrained tokenizer. This ensures the text is split the same way as the pretraining corpus, and uses the same corresponding tokens-to-index (usually referrred to as the "),i=t("em"),k=r("vocab"),y=r(") during pretraining.")},l(E){$=l(E,"P",{});var T=p($);w=o(T,"If you plan on using a pretrained model, it\u2019s important to use the associated pretrained tokenizer. This ensures the text is split the same way as the pretraining corpus, and uses the same corresponding tokens-to-index (usually referrred to as the "),i=l(T,"EM",{});var hs=p(i);k=o(hs,"vocab"),hs.forEach(a),y=o(T,") during pretraining."),T.forEach(a)},m(E,T){h(E,$,T),e($,w),e($,i),e(i,k),e($,y)},d(E){E&&a($)}}}function qb(P){let $,w;return $=new v({props:{code:`batch_sentences = [
    "But what about second breakfast?",
    "Don't think he knows about second breakfast, Pip.",
    "What about elevensies?",
]
encoded_input = tokenizer(batch_sentences, padding=True, truncation=True, return_tensors="pt")
print(encoded_input)`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>batch_sentences = [
<span class="hljs-meta">... </span>    <span class="hljs-string">&quot;But what about second breakfast?&quot;</span>,
<span class="hljs-meta">... </span>    <span class="hljs-string">&quot;Don&#x27;t think he knows about second breakfast, Pip.&quot;</span>,
<span class="hljs-meta">... </span>    <span class="hljs-string">&quot;What about elevensies?&quot;</span>,
<span class="hljs-meta">... </span>]
<span class="hljs-meta">&gt;&gt;&gt; </span>encoded_input = tokenizer(batch_sentences, padding=<span class="hljs-literal">True</span>, truncation=<span class="hljs-literal">True</span>, return_tensors=<span class="hljs-string">&quot;pt&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-built_in">print</span>(encoded_input)
{<span class="hljs-string">&#x27;input_ids&#x27;</span>: tensor([[<span class="hljs-number">101</span>, <span class="hljs-number">1252</span>, <span class="hljs-number">1184</span>, <span class="hljs-number">1164</span>, <span class="hljs-number">1248</span>, <span class="hljs-number">6462</span>, <span class="hljs-number">136</span>, <span class="hljs-number">102</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>],
                      [<span class="hljs-number">101</span>, <span class="hljs-number">1790</span>, <span class="hljs-number">112</span>, <span class="hljs-number">189</span>, <span class="hljs-number">1341</span>, <span class="hljs-number">1119</span>, <span class="hljs-number">3520</span>, <span class="hljs-number">1164</span>, <span class="hljs-number">1248</span>, <span class="hljs-number">6462</span>, <span class="hljs-number">117</span>, <span class="hljs-number">21902</span>, <span class="hljs-number">1643</span>, <span class="hljs-number">119</span>, <span class="hljs-number">102</span>],
                      [<span class="hljs-number">101</span>, <span class="hljs-number">1327</span>, <span class="hljs-number">1164</span>, <span class="hljs-number">5450</span>, <span class="hljs-number">23434</span>, <span class="hljs-number">136</span>, <span class="hljs-number">102</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>]]), 
 <span class="hljs-string">&#x27;token_type_ids&#x27;</span>: tensor([[<span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>],
                           [<span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>],
                           [<span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>]]), 
 <span class="hljs-string">&#x27;attention_mask&#x27;</span>: tensor([[<span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>],
                           [<span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>],
                           [<span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>]])}`}}),{c(){b($.$$.fragment)},l(i){f($.$$.fragment,i)},m(i,k){j($,i,k),w=!0},p:fb,i(i){w||(d($.$$.fragment,i),w=!0)},o(i){g($.$$.fragment,i),w=!1},d(i){_($,i)}}}function Ab(P){let $,w;return $=new jb({props:{$$slots:{default:[qb]},$$scope:{ctx:P}}}),{c(){b($.$$.fragment)},l(i){f($.$$.fragment,i)},m(i,k){j($,i,k),w=!0},p(i,k){const y={};k&2&&(y.$$scope={dirty:k,ctx:i}),$.$set(y)},i(i){w||(d($.$$.fragment,i),w=!0)},o(i){g($.$$.fragment,i),w=!1},d(i){_($,i)}}}function Pb(P){let $,w;return $=new v({props:{code:`batch_sentences = [
    "But what about second breakfast?",
    "Don't think he knows about second breakfast, Pip.",
    "What about elevensies?",
]
encoded_input = tokenizer(batch_sentences, padding=True, truncation=True, return_tensors="tf")
print(encoded_input)`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>batch_sentences = [
<span class="hljs-meta">... </span>    <span class="hljs-string">&quot;But what about second breakfast?&quot;</span>,
<span class="hljs-meta">... </span>    <span class="hljs-string">&quot;Don&#x27;t think he knows about second breakfast, Pip.&quot;</span>,
<span class="hljs-meta">... </span>    <span class="hljs-string">&quot;What about elevensies?&quot;</span>,
<span class="hljs-meta">... </span>]
<span class="hljs-meta">&gt;&gt;&gt; </span>encoded_input = tokenizer(batch_sentences, padding=<span class="hljs-literal">True</span>, truncation=<span class="hljs-literal">True</span>, return_tensors=<span class="hljs-string">&quot;tf&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-built_in">print</span>(encoded_input)
{<span class="hljs-string">&#x27;input_ids&#x27;</span>: &lt;tf.Tensor: shape=(<span class="hljs-number">2</span>, <span class="hljs-number">9</span>), dtype=int32, numpy=
array([[<span class="hljs-number">101</span>, <span class="hljs-number">1252</span>, <span class="hljs-number">1184</span>, <span class="hljs-number">1164</span>, <span class="hljs-number">1248</span>, <span class="hljs-number">6462</span>, <span class="hljs-number">136</span>, <span class="hljs-number">102</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>],
       [<span class="hljs-number">101</span>, <span class="hljs-number">1790</span>, <span class="hljs-number">112</span>, <span class="hljs-number">189</span>, <span class="hljs-number">1341</span>, <span class="hljs-number">1119</span>, <span class="hljs-number">3520</span>, <span class="hljs-number">1164</span>, <span class="hljs-number">1248</span>, <span class="hljs-number">6462</span>, <span class="hljs-number">117</span>, <span class="hljs-number">21902</span>, <span class="hljs-number">1643</span>, <span class="hljs-number">119</span>, <span class="hljs-number">102</span>],
       [<span class="hljs-number">101</span>, <span class="hljs-number">1327</span>, <span class="hljs-number">1164</span>, <span class="hljs-number">5450</span>, <span class="hljs-number">23434</span>, <span class="hljs-number">136</span>, <span class="hljs-number">102</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>]],
      dtype=int32)&gt;, 
 <span class="hljs-string">&#x27;token_type_ids&#x27;</span>: &lt;tf.Tensor: shape=(<span class="hljs-number">2</span>, <span class="hljs-number">9</span>), dtype=int32, numpy=
array([[<span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>],
       [<span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>],
       [<span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>]], dtype=int32)&gt;, 
 <span class="hljs-string">&#x27;attention_mask&#x27;</span>: &lt;tf.Tensor: shape=(<span class="hljs-number">2</span>, <span class="hljs-number">9</span>), dtype=int32, numpy=
array([[<span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>],
       [<span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>],
       [<span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>]], dtype=int32)&gt;}`}}),{c(){b($.$$.fragment)},l(i){f($.$$.fragment,i)},m(i,k){j($,i,k),w=!0},p:fb,i(i){w||(d($.$$.fragment,i),w=!0)},o(i){g($.$$.fragment,i),w=!1},d(i){_($,i)}}}function zb(P){let $,w;return $=new jb({props:{$$slots:{default:[Pb]},$$scope:{ctx:P}}}),{c(){b($.$$.fragment)},l(i){f($.$$.fragment,i)},m(i,k){j($,i,k),w=!0},p(i,k){const y={};k&2&&(y.$$scope={dirty:k,ctx:i}),$.$set(y)},i(i){w||(d($.$$.fragment,i),w=!0)},o(i){g($.$$.fragment,i),w=!1},d(i){_($,i)}}}function Tb(P){let $,w,i,k,y,E,T,hs,vr,Ft,Js,Ht,oe,wr,Rt,C,mn,kr,yr,bn,xr,Er,fn,qr,Bt,V,us,jn,Ms,Ar,dn,Pr,Wt,Us,Jt,D,zr,he,Tr,Cr,gn,Dr,Sr,Mt,cs,Ut,S,Lr,ue,Nr,Or,_n,Ir,Fr,Vt,G,ms,$n,Vs,Hr,vn,Rr,Gt,is,Br,ce,Wr,Jr,Yt,Gs,Kt,me,Mr,Qt,Ys,Xt,ie,Ur,Zt,L,be,fe,Vr,Gr,Yr,je,de,Kr,Qr,Xr,ge,_e,Zr,so,sl,bs,ao,wn,eo,no,al,Ks,el,N,to,kn,lo,po,yn,ro,oo,nl,$e,ho,tl,Qs,ll,Y,fs,xn,Xs,uo,En,co,pl,js,mo,qn,io,bo,rl,O,fo,An,jo,go,Pn,_o,$o,ol,Zs,hl,ds,vo,zn,wo,ko,ul,K,gs,Tn,sa,yo,Cn,xo,cl,ve,Eo,ml,I,qo,Dn,Ao,Po,Sn,zo,To,il,aa,bl,Q,_s,Ln,ea,Co,Nn,Do,fl,we,So,jl,q,Lo,On,No,Oo,In,Io,Fo,Fn,Ho,Ro,dl,$s,gl,X,vs,Hn,na,Bo,Rn,Wo,_l,ws,Jo,ke,Mo,Uo,$l,ta,vl,F,Vo,la,Go,Yo,pa,Ko,Qo,wl,ra,kl,H,Xo,Bn,Zo,sh,Wn,ah,eh,yl,oa,xl,ye,nh,El,R,xe,Jn,th,lh,ph,Ee,Mn,rh,oh,hh,qe,Un,uh,ch,ql,Z,ks,Vn,ha,mh,Gn,ih,Al,ys,bh,ua,fh,jh,Pl,xs,dh,ca,gh,_h,zl,ma,Tl,Ae,ia,$h,ba,Yn,vh,wh,Cl,fa,Dl,ja,Kn,kh,Sl,da,Ll,Es,yh,Qn,xh,Eh,Nl,ss,qs,Xn,ga,qh,Zn,Ah,Ol,A,Ph,st,zh,Th,at,Ch,Dh,et,Sh,Lh,Il,As,Nh,Pe,Oh,Ih,Fl,_a,Hl,B,Fh,nt,Hh,Rh,tt,Bh,Wh,Rl,$a,Bl,as,Ps,lt,va,Jh,pt,Mh,Wl,ze,Uh,Jl,wa,Ml,Te,Vh,Ul,ka,Vl,Ce,Gh,Gl,ya,Yl,De,Yh,Kl,xa,Ql,Se,Kh,Xl,es,zs,rt,Ea,Qh,ot,Xh,Zl,Le,Zh,sp,W,su,qa,au,eu,ht,nu,tu,ap,Aa,ep,Ts,lu,Pa,ut,pu,ru,np,za,tp,Ne,Oe,Ac,lp,ns,Cs,ct,Ta,ou,mt,hu,pp,Ds,uu,Ie,cu,mu,rp,Ca,op,ts,Ss,it,Da,iu,bt,bu,hp,Ls,fu,Sa,ft,ju,du,up,Fe,z,gu,La,jt,_u,$u,Na,dt,vu,wu,Oa,gt,ku,yu,cp,Ia,mp,Fa,ls,xu,He,_t,Eu,qu,$t,Au,Pu,ip,Ha,bp,Ra,Ba,zu,Wa,vt,Tu,Cu,fp,Ja,jp,Ma,Ua,Du,wt,Su,Lu,dp,Va,gp,Re,Nu,_p,Ga,$p,Be,We,Pc,vp,ps,Ns,kt,Ya,Ou,yt,Iu,wp,Je,Fu,kp,Os,xt,Hu,Ru,Et,Bu,yp,Is,Wu,Ka,Ju,Mu,xp,Qa,Ep,J,Uu,qt,Vu,Gu,At,Yu,Ku,qp,Xa,Ap,M,Qu,Pt,Xu,Zu,zt,sc,ac,Pp,Za,zp,Fs,ec,Me,nc,tc,Tp,se,Cp,rs,Hs,Tt,ae,lc,Ct,pc,Dp,Ue,rc,Sp,ee,Lp,Ve,os,oc,Dt,hc,uc,St,cc,mc,Np,ne,Op,te,le,ic,Lt,bc,fc,Ip,pe,Fp,U,jc,Nt,dc,gc,Ot,_c,$c,Hp,Ge,vc,Rp;return E=new x({}),Js=new yb({props:{classNames:"absolute z-10 right-0 top-0",options:[{label:"Mixed",value:"https://colab.research.google.com/github/huggingface/notebooks/blob/main/transformers_doc/en/preprocessing.ipynb"},{label:"PyTorch",value:"https://colab.research.google.com/github/huggingface/notebooks/blob/main/transformers_doc/en/pytorch/preprocessing.ipynb"},{label:"TensorFlow",value:"https://colab.research.google.com/github/huggingface/notebooks/blob/main/transformers_doc/en/tensorflow/preprocessing.ipynb"},{label:"Mixed",value:"https://studiolab.sagemaker.aws/import/github/huggingface/notebooks/blob/main/transformers_doc/en/preprocessing.ipynb"},{label:"PyTorch",value:"https://studiolab.sagemaker.aws/import/github/huggingface/notebooks/blob/main/transformers_doc/en/pytorch/preprocessing.ipynb"},{label:"TensorFlow",value:"https://studiolab.sagemaker.aws/import/github/huggingface/notebooks/blob/main/transformers_doc/en/tensorflow/preprocessing.ipynb"}]}}),Ms=new x({}),Us=new kb({props:{id:"Yffk5aydLzg"}}),cs=new wb({props:{$$slots:{default:[Eb]},$$scope:{ctx:P}}}),Vs=new x({}),Gs=new v({props:{code:`from transformers import AutoTokenizer

tokenizer = AutoTokenizer.from_pretrained("bert-base-cased")`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoTokenizer

<span class="hljs-meta">&gt;&gt;&gt; </span>tokenizer = AutoTokenizer.from_pretrained(<span class="hljs-string">&quot;bert-base-cased&quot;</span>)`}}),Ys=new v({props:{code:`encoded_input = tokenizer("Do not meddle in the affairs of wizards, for they are subtle and quick to anger.")
print(encoded_input)`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>encoded_input = tokenizer(<span class="hljs-string">&quot;Do not meddle in the affairs of wizards, for they are subtle and quick to anger.&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-built_in">print</span>(encoded_input)
{<span class="hljs-string">&#x27;input_ids&#x27;</span>: [<span class="hljs-number">101</span>, <span class="hljs-number">2079</span>, <span class="hljs-number">2025</span>, <span class="hljs-number">19960</span>, <span class="hljs-number">10362</span>, <span class="hljs-number">1999</span>, <span class="hljs-number">1996</span>, <span class="hljs-number">3821</span>, <span class="hljs-number">1997</span>, <span class="hljs-number">16657</span>, <span class="hljs-number">1010</span>, <span class="hljs-number">2005</span>, <span class="hljs-number">2027</span>, <span class="hljs-number">2024</span>, <span class="hljs-number">11259</span>, <span class="hljs-number">1998</span>, <span class="hljs-number">4248</span>, <span class="hljs-number">2000</span>, <span class="hljs-number">4963</span>, <span class="hljs-number">1012</span>, <span class="hljs-number">102</span>], 
 <span class="hljs-string">&#x27;token_type_ids&#x27;</span>: [<span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>], 
 <span class="hljs-string">&#x27;attention_mask&#x27;</span>: [<span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>]}`}}),Ks=new v({props:{code:'tokenizer.decode(encoded_input["input_ids"])',highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>tokenizer.decode(encoded_input[<span class="hljs-string">&quot;input_ids&quot;</span>])
<span class="hljs-string">&#x27;[CLS] Do not meddle in the affairs of wizards, for they are subtle and quick to anger. [SEP]&#x27;</span>`}}),Qs=new v({props:{code:`batch_sentences = [
    "But what about second breakfast?",
    "Don't think he knows about second breakfast, Pip.",
    "What about elevensies?",
]
encoded_inputs = tokenizer(batch_sentences)
print(encoded_inputs)`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>batch_sentences = [
<span class="hljs-meta">... </span>    <span class="hljs-string">&quot;But what about second breakfast?&quot;</span>,
<span class="hljs-meta">... </span>    <span class="hljs-string">&quot;Don&#x27;t think he knows about second breakfast, Pip.&quot;</span>,
<span class="hljs-meta">... </span>    <span class="hljs-string">&quot;What about elevensies?&quot;</span>,
<span class="hljs-meta">... </span>]
<span class="hljs-meta">&gt;&gt;&gt; </span>encoded_inputs = tokenizer(batch_sentences)
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-built_in">print</span>(encoded_inputs)
{<span class="hljs-string">&#x27;input_ids&#x27;</span>: [[<span class="hljs-number">101</span>, <span class="hljs-number">1252</span>, <span class="hljs-number">1184</span>, <span class="hljs-number">1164</span>, <span class="hljs-number">1248</span>, <span class="hljs-number">6462</span>, <span class="hljs-number">136</span>, <span class="hljs-number">102</span>], 
               [<span class="hljs-number">101</span>, <span class="hljs-number">1790</span>, <span class="hljs-number">112</span>, <span class="hljs-number">189</span>, <span class="hljs-number">1341</span>, <span class="hljs-number">1119</span>, <span class="hljs-number">3520</span>, <span class="hljs-number">1164</span>, <span class="hljs-number">1248</span>, <span class="hljs-number">6462</span>, <span class="hljs-number">117</span>, <span class="hljs-number">21902</span>, <span class="hljs-number">1643</span>, <span class="hljs-number">119</span>, <span class="hljs-number">102</span>], 
               [<span class="hljs-number">101</span>, <span class="hljs-number">1327</span>, <span class="hljs-number">1164</span>, <span class="hljs-number">5450</span>, <span class="hljs-number">23434</span>, <span class="hljs-number">136</span>, <span class="hljs-number">102</span>]], 
 <span class="hljs-string">&#x27;token_type_ids&#x27;</span>: [[<span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>], 
                    [<span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>], 
                    [<span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>]], 
 <span class="hljs-string">&#x27;attention_mask&#x27;</span>: [[<span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>], 
                    [<span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>], 
                    [<span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>]]}`}}),Xs=new x({}),Zs=new v({props:{code:`batch_sentences = [
    "But what about second breakfast?",
    "Don't think he knows about second breakfast, Pip.",
    "What about elevensies?",
]
encoded_input = tokenizer(batch_sentences, padding=True)
print(encoded_input)`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>batch_sentences = [
<span class="hljs-meta">... </span>    <span class="hljs-string">&quot;But what about second breakfast?&quot;</span>,
<span class="hljs-meta">... </span>    <span class="hljs-string">&quot;Don&#x27;t think he knows about second breakfast, Pip.&quot;</span>,
<span class="hljs-meta">... </span>    <span class="hljs-string">&quot;What about elevensies?&quot;</span>,
<span class="hljs-meta">... </span>]
<span class="hljs-meta">&gt;&gt;&gt; </span>encoded_input = tokenizer(batch_sentences, padding=<span class="hljs-literal">True</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-built_in">print</span>(encoded_input)
{<span class="hljs-string">&#x27;input_ids&#x27;</span>: [[<span class="hljs-number">101</span>, <span class="hljs-number">1252</span>, <span class="hljs-number">1184</span>, <span class="hljs-number">1164</span>, <span class="hljs-number">1248</span>, <span class="hljs-number">6462</span>, <span class="hljs-number">136</span>, <span class="hljs-number">102</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>], 
               [<span class="hljs-number">101</span>, <span class="hljs-number">1790</span>, <span class="hljs-number">112</span>, <span class="hljs-number">189</span>, <span class="hljs-number">1341</span>, <span class="hljs-number">1119</span>, <span class="hljs-number">3520</span>, <span class="hljs-number">1164</span>, <span class="hljs-number">1248</span>, <span class="hljs-number">6462</span>, <span class="hljs-number">117</span>, <span class="hljs-number">21902</span>, <span class="hljs-number">1643</span>, <span class="hljs-number">119</span>, <span class="hljs-number">102</span>], 
               [<span class="hljs-number">101</span>, <span class="hljs-number">1327</span>, <span class="hljs-number">1164</span>, <span class="hljs-number">5450</span>, <span class="hljs-number">23434</span>, <span class="hljs-number">136</span>, <span class="hljs-number">102</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>]], 
 <span class="hljs-string">&#x27;token_type_ids&#x27;</span>: [[<span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>], 
                    [<span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>], 
                    [<span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>]], 
 <span class="hljs-string">&#x27;attention_mask&#x27;</span>: [[<span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>], 
                    [<span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>], 
                    [<span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>]]}`}}),sa=new x({}),aa=new v({props:{code:`batch_sentences = [
    "But what about second breakfast?",
    "Don't think he knows about second breakfast, Pip.",
    "What about elevensies?",
]
encoded_input = tokenizer(batch_sentences, padding=True, truncation=True)
print(encoded_input)`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>batch_sentences = [
<span class="hljs-meta">... </span>    <span class="hljs-string">&quot;But what about second breakfast?&quot;</span>,
<span class="hljs-meta">... </span>    <span class="hljs-string">&quot;Don&#x27;t think he knows about second breakfast, Pip.&quot;</span>,
<span class="hljs-meta">... </span>    <span class="hljs-string">&quot;What about elevensies?&quot;</span>,
<span class="hljs-meta">... </span>]
<span class="hljs-meta">&gt;&gt;&gt; </span>encoded_input = tokenizer(batch_sentences, padding=<span class="hljs-literal">True</span>, truncation=<span class="hljs-literal">True</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-built_in">print</span>(encoded_input)
{<span class="hljs-string">&#x27;input_ids&#x27;</span>: [[<span class="hljs-number">101</span>, <span class="hljs-number">1252</span>, <span class="hljs-number">1184</span>, <span class="hljs-number">1164</span>, <span class="hljs-number">1248</span>, <span class="hljs-number">6462</span>, <span class="hljs-number">136</span>, <span class="hljs-number">102</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>], 
               [<span class="hljs-number">101</span>, <span class="hljs-number">1790</span>, <span class="hljs-number">112</span>, <span class="hljs-number">189</span>, <span class="hljs-number">1341</span>, <span class="hljs-number">1119</span>, <span class="hljs-number">3520</span>, <span class="hljs-number">1164</span>, <span class="hljs-number">1248</span>, <span class="hljs-number">6462</span>, <span class="hljs-number">117</span>, <span class="hljs-number">21902</span>, <span class="hljs-number">1643</span>, <span class="hljs-number">119</span>, <span class="hljs-number">102</span>], 
               [<span class="hljs-number">101</span>, <span class="hljs-number">1327</span>, <span class="hljs-number">1164</span>, <span class="hljs-number">5450</span>, <span class="hljs-number">23434</span>, <span class="hljs-number">136</span>, <span class="hljs-number">102</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>]], 
 <span class="hljs-string">&#x27;token_type_ids&#x27;</span>: [[<span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>], 
                    [<span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>], 
                    [<span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>]], 
 <span class="hljs-string">&#x27;attention_mask&#x27;</span>: [[<span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>], 
                    [<span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>], 
                    [<span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>]]}`}}),ea=new x({}),$s=new xb({props:{pytorch:!0,tensorflow:!0,jax:!1,$$slots:{tensorflow:[zb],pytorch:[Ab]},$$scope:{ctx:P}}}),na=new x({}),ta=new v({props:{code:"pip install datasets",highlighted:"pip install datasets"}}),ra=new v({props:{code:`from datasets import load_dataset, Audio

dataset = load_dataset("PolyAI/minds14", name="en-US", split="train")`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> datasets <span class="hljs-keyword">import</span> load_dataset, Audio

<span class="hljs-meta">&gt;&gt;&gt; </span>dataset = load_dataset(<span class="hljs-string">&quot;PolyAI/minds14&quot;</span>, name=<span class="hljs-string">&quot;en-US&quot;</span>, split=<span class="hljs-string">&quot;train&quot;</span>)`}}),oa=new v({props:{code:'dataset[0]["audio"]',highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>dataset[<span class="hljs-number">0</span>][<span class="hljs-string">&quot;audio&quot;</span>]
{<span class="hljs-string">&#x27;array&#x27;</span>: array([ <span class="hljs-number">0.</span>        ,  <span class="hljs-number">0.00024414</span>, -<span class="hljs-number">0.00024414</span>, ..., -<span class="hljs-number">0.00024414</span>,
         <span class="hljs-number">0.</span>        ,  <span class="hljs-number">0.</span>        ], dtype=float32),
 <span class="hljs-string">&#x27;path&#x27;</span>: <span class="hljs-string">&#x27;/root/.cache/huggingface/datasets/downloads/extracted/f14948e0e84be638dd7943ac36518a4cf3324e8b7aa331c5ab11541518e9368c/en-US~JOINT_ACCOUNT/602ba55abb1e6d0fbce92065.wav&#x27;</span>,
 <span class="hljs-string">&#x27;sampling_rate&#x27;</span>: <span class="hljs-number">8000</span>}`}}),ha=new x({}),ma=new v({props:{code:`dataset = load_dataset("PolyAI/minds14", name="en-US", split="train")
dataset[0]["audio"]`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>dataset = load_dataset(<span class="hljs-string">&quot;PolyAI/minds14&quot;</span>, name=<span class="hljs-string">&quot;en-US&quot;</span>, split=<span class="hljs-string">&quot;train&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>dataset[<span class="hljs-number">0</span>][<span class="hljs-string">&quot;audio&quot;</span>]
{<span class="hljs-string">&#x27;array&#x27;</span>: array([ <span class="hljs-number">0.</span>        ,  <span class="hljs-number">0.00024414</span>, -<span class="hljs-number">0.00024414</span>, ..., -<span class="hljs-number">0.00024414</span>,
         <span class="hljs-number">0.</span>        ,  <span class="hljs-number">0.</span>        ], dtype=float32),
 <span class="hljs-string">&#x27;path&#x27;</span>: <span class="hljs-string">&#x27;/root/.cache/huggingface/datasets/downloads/extracted/f14948e0e84be638dd7943ac36518a4cf3324e8b7aa331c5ab11541518e9368c/en-US~JOINT_ACCOUNT/602ba55abb1e6d0fbce92065.wav&#x27;</span>,
 <span class="hljs-string">&#x27;sampling_rate&#x27;</span>: <span class="hljs-number">8000</span>}`}}),fa=new v({props:{code:'dataset = dataset.cast_column("audio", Audio(sampling_rate=16_000))',highlighted:'<span class="hljs-meta">&gt;&gt;&gt; </span>dataset = dataset.cast_column(<span class="hljs-string">&quot;audio&quot;</span>, Audio(sampling_rate=<span class="hljs-number">16_000</span>))'}}),da=new v({props:{code:'dataset[0]["audio"]',highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>dataset[<span class="hljs-number">0</span>][<span class="hljs-string">&quot;audio&quot;</span>]
{<span class="hljs-string">&#x27;array&#x27;</span>: array([ <span class="hljs-number">2.3443763e-05</span>,  <span class="hljs-number">2.1729663e-04</span>,  <span class="hljs-number">2.2145823e-04</span>, ...,
         <span class="hljs-number">3.8356509e-05</span>, -<span class="hljs-number">7.3497440e-06</span>, -<span class="hljs-number">2.1754686e-05</span>], dtype=float32),
 <span class="hljs-string">&#x27;path&#x27;</span>: <span class="hljs-string">&#x27;/root/.cache/huggingface/datasets/downloads/extracted/f14948e0e84be638dd7943ac36518a4cf3324e8b7aa331c5ab11541518e9368c/en-US~JOINT_ACCOUNT/602ba55abb1e6d0fbce92065.wav&#x27;</span>,
 <span class="hljs-string">&#x27;sampling_rate&#x27;</span>: <span class="hljs-number">16000</span>}`}}),ga=new x({}),_a=new v({props:{code:`from transformers import AutoFeatureExtractor

feature_extractor = AutoFeatureExtractor.from_pretrained("facebook/wav2vec2-base")`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoFeatureExtractor

<span class="hljs-meta">&gt;&gt;&gt; </span>feature_extractor = AutoFeatureExtractor.from_pretrained(<span class="hljs-string">&quot;facebook/wav2vec2-base&quot;</span>)`}}),$a=new v({props:{code:`audio_input = [dataset[0]["audio"]["array"]]
feature_extractor(audio_input, sampling_rate=16000)`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>audio_input = [dataset[<span class="hljs-number">0</span>][<span class="hljs-string">&quot;audio&quot;</span>][<span class="hljs-string">&quot;array&quot;</span>]]
<span class="hljs-meta">&gt;&gt;&gt; </span>feature_extractor(audio_input, sampling_rate=<span class="hljs-number">16000</span>)
{<span class="hljs-string">&#x27;input_values&#x27;</span>: [array([ <span class="hljs-number">3.8106556e-04</span>,  <span class="hljs-number">2.7506407e-03</span>,  <span class="hljs-number">2.8015103e-03</span>, ...,
        <span class="hljs-number">5.6335266e-04</span>,  <span class="hljs-number">4.6588284e-06</span>, -<span class="hljs-number">1.7142107e-04</span>], dtype=float32)]}`}}),va=new x({}),wa=new v({props:{code:`dataset[0]["audio"]["array"].shape

dataset[1]["audio"]["array"].shape`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>dataset[<span class="hljs-number">0</span>][<span class="hljs-string">&quot;audio&quot;</span>][<span class="hljs-string">&quot;array&quot;</span>].shape
(<span class="hljs-number">173398</span>,)

<span class="hljs-meta">&gt;&gt;&gt; </span>dataset[<span class="hljs-number">1</span>][<span class="hljs-string">&quot;audio&quot;</span>][<span class="hljs-string">&quot;array&quot;</span>].shape
(<span class="hljs-number">106496</span>,)`}}),ka=new v({props:{code:`def preprocess_function(examples):
    audio_arrays = [x["array"] for x in examples["audio"]]
    inputs = feature_extractor(
        audio_arrays,
        sampling_rate=16000,
        padding=True,
        max_length=100000,
        truncation=True,
    )
    return inputs`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">def</span> <span class="hljs-title function_">preprocess_function</span>(<span class="hljs-params">examples</span>):
<span class="hljs-meta">... </span>    audio_arrays = [x[<span class="hljs-string">&quot;array&quot;</span>] <span class="hljs-keyword">for</span> x <span class="hljs-keyword">in</span> examples[<span class="hljs-string">&quot;audio&quot;</span>]]
<span class="hljs-meta">... </span>    inputs = feature_extractor(
<span class="hljs-meta">... </span>        audio_arrays,
<span class="hljs-meta">... </span>        sampling_rate=<span class="hljs-number">16000</span>,
<span class="hljs-meta">... </span>        padding=<span class="hljs-literal">True</span>,
<span class="hljs-meta">... </span>        max_length=<span class="hljs-number">100000</span>,
<span class="hljs-meta">... </span>        truncation=<span class="hljs-literal">True</span>,
<span class="hljs-meta">... </span>    )
<span class="hljs-meta">... </span>    <span class="hljs-keyword">return</span> inputs`}}),ya=new v({props:{code:"processed_dataset = preprocess_function(dataset[:5])",highlighted:'<span class="hljs-meta">&gt;&gt;&gt; </span>processed_dataset = preprocess_function(dataset[:<span class="hljs-number">5</span>])'}}),xa=new v({props:{code:`processed_dataset["input_values"][0].shape

processed_dataset["input_values"][1].shape`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>processed_dataset[<span class="hljs-string">&quot;input_values&quot;</span>][<span class="hljs-number">0</span>].shape
(<span class="hljs-number">100000</span>,)

<span class="hljs-meta">&gt;&gt;&gt; </span>processed_dataset[<span class="hljs-string">&quot;input_values&quot;</span>][<span class="hljs-number">1</span>].shape
(<span class="hljs-number">100000</span>,)`}}),Ea=new x({}),Aa=new v({props:{code:`from datasets import load_dataset

dataset = load_dataset("food101", split="train[:100]")`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> datasets <span class="hljs-keyword">import</span> load_dataset

<span class="hljs-meta">&gt;&gt;&gt; </span>dataset = load_dataset(<span class="hljs-string">&quot;food101&quot;</span>, split=<span class="hljs-string">&quot;train[:100]&quot;</span>)`}}),za=new v({props:{code:'dataset[0]["image"]',highlighted:'<span class="hljs-meta">&gt;&gt;&gt; </span>dataset[<span class="hljs-number">0</span>][<span class="hljs-string">&quot;image&quot;</span>]'}}),Ta=new x({}),Ca=new v({props:{code:`from transformers import AutoFeatureExtractor

feature_extractor = AutoFeatureExtractor.from_pretrained("google/vit-base-patch16-224")`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoFeatureExtractor

<span class="hljs-meta">&gt;&gt;&gt; </span>feature_extractor = AutoFeatureExtractor.from_pretrained(<span class="hljs-string">&quot;google/vit-base-patch16-224&quot;</span>)`}}),Da=new x({}),Ia=new v({props:{code:`from torchvision.transforms import Compose, Normalize, RandomResizedCrop, ColorJitter, ToTensor

normalize = Normalize(mean=feature_extractor.image_mean, std=feature_extractor.image_std)
_transforms = Compose(
    [RandomResizedCrop(feature_extractor.size), ColorJitter(brightness=0.5, hue=0.5), ToTensor(), normalize]
)`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> torchvision.transforms <span class="hljs-keyword">import</span> Compose, Normalize, RandomResizedCrop, ColorJitter, ToTensor

<span class="hljs-meta">&gt;&gt;&gt; </span>normalize = Normalize(mean=feature_extractor.image_mean, std=feature_extractor.image_std)
<span class="hljs-meta">&gt;&gt;&gt; </span>_transforms = Compose(
<span class="hljs-meta">... </span>    [RandomResizedCrop(feature_extractor.size), ColorJitter(brightness=<span class="hljs-number">0.5</span>, hue=<span class="hljs-number">0.5</span>), ToTensor(), normalize]
<span class="hljs-meta">... </span>)`}}),Ha=new v({props:{code:`def transforms(examples):
    examples["pixel_values"] = [_transforms(image.convert("RGB")) for image in examples["image"]]
    return examples`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">def</span> <span class="hljs-title function_">transforms</span>(<span class="hljs-params">examples</span>):
<span class="hljs-meta">... </span>    examples[<span class="hljs-string">&quot;pixel_values&quot;</span>] = [_transforms(image.convert(<span class="hljs-string">&quot;RGB&quot;</span>)) <span class="hljs-keyword">for</span> image <span class="hljs-keyword">in</span> examples[<span class="hljs-string">&quot;image&quot;</span>]]
<span class="hljs-meta">... </span>    <span class="hljs-keyword">return</span> examples`}}),Ja=new v({props:{code:"dataset.set_transform(transforms)",highlighted:'<span class="hljs-meta">&gt;&gt;&gt; </span>dataset.set_transform(transforms)'}}),Va=new v({props:{code:'dataset[0]["image"]',highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>dataset[<span class="hljs-number">0</span>][<span class="hljs-string">&quot;image&quot;</span>]
{<span class="hljs-string">&#x27;image&#x27;</span>: &lt;PIL.JpegImagePlugin.JpegImageFile image mode=RGB size=384x512 at <span class="hljs-number">0x7F1A7B0630D0</span>&gt;,
 <span class="hljs-string">&#x27;label&#x27;</span>: <span class="hljs-number">6</span>,
 <span class="hljs-string">&#x27;pixel_values&#x27;</span>: tensor([[[ <span class="hljs-number">0.0353</span>,  <span class="hljs-number">0.0745</span>,  <span class="hljs-number">0.1216</span>,  ..., -<span class="hljs-number">0.9922</span>, -<span class="hljs-number">0.9922</span>, -<span class="hljs-number">0.9922</span>],
          [-<span class="hljs-number">0.0196</span>,  <span class="hljs-number">0.0667</span>,  <span class="hljs-number">0.1294</span>,  ..., -<span class="hljs-number">0.9765</span>, -<span class="hljs-number">0.9843</span>, -<span class="hljs-number">0.9922</span>],
          [ <span class="hljs-number">0.0196</span>,  <span class="hljs-number">0.0824</span>,  <span class="hljs-number">0.1137</span>,  ..., -<span class="hljs-number">0.9765</span>, -<span class="hljs-number">0.9686</span>, -<span class="hljs-number">0.8667</span>],
          ...,
          [ <span class="hljs-number">0.0275</span>,  <span class="hljs-number">0.0745</span>,  <span class="hljs-number">0.0510</span>,  ..., -<span class="hljs-number">0.1137</span>, -<span class="hljs-number">0.1216</span>, -<span class="hljs-number">0.0824</span>],
          [ <span class="hljs-number">0.0667</span>,  <span class="hljs-number">0.0824</span>,  <span class="hljs-number">0.0667</span>,  ..., -<span class="hljs-number">0.0588</span>, -<span class="hljs-number">0.0745</span>, -<span class="hljs-number">0.0980</span>],
          [ <span class="hljs-number">0.0353</span>,  <span class="hljs-number">0.0353</span>,  <span class="hljs-number">0.0431</span>,  ..., -<span class="hljs-number">0.0039</span>, -<span class="hljs-number">0.0039</span>, -<span class="hljs-number">0.0588</span>]],
 
         [[ <span class="hljs-number">0.2078</span>,  <span class="hljs-number">0.2471</span>,  <span class="hljs-number">0.2863</span>,  ..., -<span class="hljs-number">0.9451</span>, -<span class="hljs-number">0.9373</span>, -<span class="hljs-number">0.9451</span>],
          [ <span class="hljs-number">0.1608</span>,  <span class="hljs-number">0.2471</span>,  <span class="hljs-number">0.3098</span>,  ..., -<span class="hljs-number">0.9373</span>, -<span class="hljs-number">0.9451</span>, -<span class="hljs-number">0.9373</span>],
          [ <span class="hljs-number">0.2078</span>,  <span class="hljs-number">0.2706</span>,  <span class="hljs-number">0.3020</span>,  ..., -<span class="hljs-number">0.9608</span>, -<span class="hljs-number">0.9373</span>, -<span class="hljs-number">0.8275</span>],
          ...,
          [-<span class="hljs-number">0.0353</span>,  <span class="hljs-number">0.0118</span>, -<span class="hljs-number">0.0039</span>,  ..., -<span class="hljs-number">0.2392</span>, -<span class="hljs-number">0.2471</span>, -<span class="hljs-number">0.2078</span>],
          [ <span class="hljs-number">0.0196</span>,  <span class="hljs-number">0.0353</span>,  <span class="hljs-number">0.0196</span>,  ..., -<span class="hljs-number">0.1843</span>, -<span class="hljs-number">0.2000</span>, -<span class="hljs-number">0.2235</span>],
          [-<span class="hljs-number">0.0118</span>, -<span class="hljs-number">0.0039</span>, -<span class="hljs-number">0.0039</span>,  ..., -<span class="hljs-number">0.0980</span>, -<span class="hljs-number">0.0980</span>, -<span class="hljs-number">0.1529</span>]],
 
         [[ <span class="hljs-number">0.3961</span>,  <span class="hljs-number">0.4431</span>,  <span class="hljs-number">0.4980</span>,  ..., -<span class="hljs-number">0.9216</span>, -<span class="hljs-number">0.9137</span>, -<span class="hljs-number">0.9216</span>],
          [ <span class="hljs-number">0.3569</span>,  <span class="hljs-number">0.4510</span>,  <span class="hljs-number">0.5216</span>,  ..., -<span class="hljs-number">0.9059</span>, -<span class="hljs-number">0.9137</span>, -<span class="hljs-number">0.9137</span>],
          [ <span class="hljs-number">0.4118</span>,  <span class="hljs-number">0.4745</span>,  <span class="hljs-number">0.5216</span>,  ..., -<span class="hljs-number">0.9137</span>, -<span class="hljs-number">0.8902</span>, -<span class="hljs-number">0.7804</span>],
          ...,
          [-<span class="hljs-number">0.2314</span>, -<span class="hljs-number">0.1922</span>, -<span class="hljs-number">0.2078</span>,  ..., -<span class="hljs-number">0.4196</span>, -<span class="hljs-number">0.4275</span>, -<span class="hljs-number">0.3882</span>],
          [-<span class="hljs-number">0.1843</span>, -<span class="hljs-number">0.1686</span>, -<span class="hljs-number">0.2000</span>,  ..., -<span class="hljs-number">0.3647</span>, -<span class="hljs-number">0.3804</span>, -<span class="hljs-number">0.4039</span>],
          [-<span class="hljs-number">0.1922</span>, -<span class="hljs-number">0.1922</span>, -<span class="hljs-number">0.1922</span>,  ..., -<span class="hljs-number">0.2941</span>, -<span class="hljs-number">0.2863</span>, -<span class="hljs-number">0.3412</span>]]])}`}}),Ga=new v({props:{code:`import numpy as np
import matplotlib.pyplot as plt

img = dataset[0]["pixel_values"]
plt.imshow(img.permute(1, 2, 0))`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">import</span> numpy <span class="hljs-keyword">as</span> np
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">import</span> matplotlib.pyplot <span class="hljs-keyword">as</span> plt

<span class="hljs-meta">&gt;&gt;&gt; </span>img = dataset[<span class="hljs-number">0</span>][<span class="hljs-string">&quot;pixel_values&quot;</span>]
<span class="hljs-meta">&gt;&gt;&gt; </span>plt.imshow(img.permute(<span class="hljs-number">1</span>, <span class="hljs-number">2</span>, <span class="hljs-number">0</span>))`}}),Ya=new x({}),Qa=new v({props:{code:`from datasets import load_dataset

lj_speech = load_dataset("lj_speech", split="train")`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> datasets <span class="hljs-keyword">import</span> load_dataset

<span class="hljs-meta">&gt;&gt;&gt; </span>lj_speech = load_dataset(<span class="hljs-string">&quot;lj_speech&quot;</span>, split=<span class="hljs-string">&quot;train&quot;</span>)`}}),Xa=new v({props:{code:'lj_speech = lj_speech.map(remove_columns=["file", "id", "normalized_text"])',highlighted:'<span class="hljs-meta">&gt;&gt;&gt; </span>lj_speech = lj_speech.<span class="hljs-built_in">map</span>(remove_columns=[<span class="hljs-string">&quot;file&quot;</span>, <span class="hljs-string">&quot;id&quot;</span>, <span class="hljs-string">&quot;normalized_text&quot;</span>])'}}),Za=new v({props:{code:`lj_speech[0]["audio"]

lj_speech[0]["text"]`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>lj_speech[<span class="hljs-number">0</span>][<span class="hljs-string">&quot;audio&quot;</span>]
{<span class="hljs-string">&#x27;array&#x27;</span>: array([-<span class="hljs-number">7.3242188e-04</span>, -<span class="hljs-number">7.6293945e-04</span>, -<span class="hljs-number">6.4086914e-04</span>, ...,
         <span class="hljs-number">7.3242188e-04</span>,  <span class="hljs-number">2.1362305e-04</span>,  <span class="hljs-number">6.1035156e-05</span>], dtype=float32),
 <span class="hljs-string">&#x27;path&#x27;</span>: <span class="hljs-string">&#x27;/root/.cache/huggingface/datasets/downloads/extracted/917ece08c95cf0c4115e45294e3cd0dee724a1165b7fc11798369308a465bd26/LJSpeech-1.1/wavs/LJ001-0001.wav&#x27;</span>,
 <span class="hljs-string">&#x27;sampling_rate&#x27;</span>: <span class="hljs-number">22050</span>}

<span class="hljs-meta">&gt;&gt;&gt; </span>lj_speech[<span class="hljs-number">0</span>][<span class="hljs-string">&quot;text&quot;</span>]
<span class="hljs-string">&#x27;Printing, in the only sense with which we are at present concerned, differs from most if not from all the arts and crafts represented in the Exhibition&#x27;</span>`}}),se=new v({props:{code:'lj_speech = lj_speech.cast_column("audio", Audio(sampling_rate=16_000))',highlighted:'<span class="hljs-meta">&gt;&gt;&gt; </span>lj_speech = lj_speech.cast_column(<span class="hljs-string">&quot;audio&quot;</span>, Audio(sampling_rate=<span class="hljs-number">16_000</span>))'}}),ae=new x({}),ee=new v({props:{code:`from transformers import AutoProcessor

processor = AutoProcessor.from_pretrained("facebook/wav2vec2-base-960h")`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoProcessor

<span class="hljs-meta">&gt;&gt;&gt; </span>processor = AutoProcessor.from_pretrained(<span class="hljs-string">&quot;facebook/wav2vec2-base-960h&quot;</span>)`}}),ne=new v({props:{code:`def prepare_dataset(example):
    audio = example["audio"]

    example["input_values"] = processor(audio["array"], sampling_rate=16000)

    with processor.as_target_processor():
        example["labels"] = processor(example["text"]).input_ids
    return example`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">def</span> <span class="hljs-title function_">prepare_dataset</span>(<span class="hljs-params">example</span>):
<span class="hljs-meta">... </span>    audio = example[<span class="hljs-string">&quot;audio&quot;</span>]

<span class="hljs-meta">... </span>    example[<span class="hljs-string">&quot;input_values&quot;</span>] = processor(audio[<span class="hljs-string">&quot;array&quot;</span>], sampling_rate=<span class="hljs-number">16000</span>)

<span class="hljs-meta">... </span>    <span class="hljs-keyword">with</span> processor.as_target_processor():
<span class="hljs-meta">... </span>        example[<span class="hljs-string">&quot;labels&quot;</span>] = processor(example[<span class="hljs-string">&quot;text&quot;</span>]).input_ids
<span class="hljs-meta">... </span>    <span class="hljs-keyword">return</span> example`}}),pe=new v({props:{code:"prepare_dataset(lj_speech[0])",highlighted:'<span class="hljs-meta">&gt;&gt;&gt; </span>prepare_dataset(lj_speech[<span class="hljs-number">0</span>])'}}),{c(){$=t("meta"),w=u(),i=t("h1"),k=t("a"),y=t("span"),b(E.$$.fragment),T=u(),hs=t("span"),vr=r("Preprocess"),Ft=u(),b(Js.$$.fragment),Ht=u(),oe=t("p"),wr=r("Before you can use your data in a model, the data needs to be processed into an acceptable format for the model. A model does not understand raw text, images or audio. These inputs need to be converted into numbers and assembled into tensors. In this tutorial, you will:"),Rt=u(),C=t("ul"),mn=t("li"),kr=r("Preprocess textual data with a tokenizer."),yr=u(),bn=t("li"),xr=r("Preprocess image or audio data with a feature extractor."),Er=u(),fn=t("li"),qr=r("Preprocess data for a multimodal task with a processor."),Bt=u(),V=t("h2"),us=t("a"),jn=t("span"),b(Ms.$$.fragment),Ar=u(),dn=t("span"),Pr=r("NLP"),Wt=u(),b(Us.$$.fragment),Jt=u(),D=t("p"),zr=r("The main tool for processing textual data is a "),he=t("a"),Tr=r("tokenizer"),Cr=r(". A tokenizer starts by splitting text into "),gn=t("em"),Dr=r("tokens"),Sr=r(" according to a set of rules. The tokens are converted into numbers, which are used to build tensors as input to a model. Any additional inputs required by a model are also added by the tokenizer."),Mt=u(),b(cs.$$.fragment),Ut=u(),S=t("p"),Lr=r("Get started quickly by loading a pretrained tokenizer with the "),ue=t("a"),Nr=r("AutoTokenizer"),Or=r(" class. This downloads the "),_n=t("em"),Ir=r("vocab"),Fr=r(" used when a model is pretrained."),Vt=u(),G=t("h3"),ms=t("a"),$n=t("span"),b(Vs.$$.fragment),Hr=u(),vn=t("span"),Rr=r("Tokenize"),Gt=u(),is=t("p"),Br=r("Load a pretrained tokenizer with "),ce=t("a"),Wr=r("AutoTokenizer.from_pretrained()"),Jr=r(":"),Yt=u(),b(Gs.$$.fragment),Kt=u(),me=t("p"),Mr=r("Then pass your sentence to the tokenizer:"),Qt=u(),b(Ys.$$.fragment),Xt=u(),ie=t("p"),Ur=r("The tokenizer returns a dictionary with three important itmes:"),Zt=u(),L=t("ul"),be=t("li"),fe=t("a"),Vr=r("input_ids"),Gr=r(" are the indices corresponding to each token in the sentence."),Yr=u(),je=t("li"),de=t("a"),Kr=r("attention_mask"),Qr=r(" indicates whether a token should be attended to or not."),Xr=u(),ge=t("li"),_e=t("a"),Zr=r("token_type_ids"),so=r(" identifies which sequence a token belongs to when there is more than one sequence."),sl=u(),bs=t("p"),ao=r("You can decode the "),wn=t("code"),eo=r("input_ids"),no=r(" to return the original input:"),al=u(),b(Ks.$$.fragment),el=u(),N=t("p"),to=r("As you can see, the tokenizer added two special tokens - "),kn=t("code"),lo=r("CLS"),po=r(" and "),yn=t("code"),ro=r("SEP"),oo=r(` (classifier and separator) - to the sentence. Not all models need
special tokens, but if they do, the tokenizer will automatically add them for you.`),nl=u(),$e=t("p"),ho=r("If there are several sentences you want to process, pass the sentences as a list to the tokenizer:"),tl=u(),b(Qs.$$.fragment),ll=u(),Y=t("h3"),fs=t("a"),xn=t("span"),b(Xs.$$.fragment),uo=u(),En=t("span"),co=r("Pad"),pl=u(),js=t("p"),mo=r("This brings us to an important topic. When you process a batch of sentences, they aren\u2019t always the same length. This is a problem because tensors, the input to the model, need to have a uniform shape. Padding is a strategy for ensuring tensors are rectangular by adding a special "),qn=t("em"),io=r("padding token"),bo=r(" to sentences with fewer tokens."),rl=u(),O=t("p"),fo=r("Set the "),An=t("code"),jo=r("padding"),go=r(" parameter to "),Pn=t("code"),_o=r("True"),$o=r(" to pad the shorter sequences in the batch to match the longest sequence:"),ol=u(),b(Zs.$$.fragment),hl=u(),ds=t("p"),vo=r("Notice the tokenizer padded the first and third sentences with a "),zn=t("code"),wo=r("0"),ko=r(" because they are shorter!"),ul=u(),K=t("h3"),gs=t("a"),Tn=t("span"),b(sa.$$.fragment),yo=u(),Cn=t("span"),xo=r("Truncation"),cl=u(),ve=t("p"),Eo=r("On the other end of the spectrum, sometimes a sequence may be too long for a model to handle. In this case, you will need to truncate the sequence to a shorter length."),ml=u(),I=t("p"),qo=r("Set the "),Dn=t("code"),Ao=r("truncation"),Po=r(" parameter to "),Sn=t("code"),zo=r("True"),To=r(" to truncate a sequence to the maximum length accepted by the model:"),il=u(),b(aa.$$.fragment),bl=u(),Q=t("h3"),_s=t("a"),Ln=t("span"),b(ea.$$.fragment),Co=u(),Nn=t("span"),Do=r("Build tensors"),fl=u(),we=t("p"),So=r("Finally, you want the tokenizer to return the actual tensors that are fed to the model."),jl=u(),q=t("p"),Lo=r("Set the "),On=t("code"),No=r("return_tensors"),Oo=r(" parameter to either "),In=t("code"),Io=r("pt"),Fo=r(" for PyTorch, or "),Fn=t("code"),Ho=r("tf"),Ro=r(" for TensorFlow:"),dl=u(),b($s.$$.fragment),gl=u(),X=t("h2"),vs=t("a"),Hn=t("span"),b(na.$$.fragment),Bo=u(),Rn=t("span"),Wo=r("Audio"),_l=u(),ws=t("p"),Jo=r("Audio inputs are preprocessed differently than textual inputs, but the end goal remains the same: create numerical sequences the model can understand. A "),ke=t("a"),Mo=r("feature extractor"),Uo=r(" is designed for the express purpose of extracting features from raw image or audio data and converting them into tensors. Before you begin, install \u{1F917} Datasets to load an audio dataset to experiment with:"),$l=u(),b(ta.$$.fragment),vl=u(),F=t("p"),Vo=r("Load the "),la=t("a"),Go=r("MInDS-14"),Yo=r(" dataset (see the \u{1F917} "),pa=t("a"),Ko=r("Datasets tutorial"),Qo=r(" for more details on how to load a dataset):"),wl=u(),b(ra.$$.fragment),kl=u(),H=t("p"),Xo=r("Access the first element of the "),Bn=t("code"),Zo=r("audio"),sh=r(" column to take a look at the input. Calling the "),Wn=t("code"),ah=r("audio"),eh=r(" column will automatically load and resample the audio file:"),yl=u(),b(oa.$$.fragment),xl=u(),ye=t("p"),nh=r("This returns three items:"),El=u(),R=t("ul"),xe=t("li"),Jn=t("code"),th=r("array"),lh=r(" is the speech signal loaded - and potentially resampled - as a 1D array."),ph=u(),Ee=t("li"),Mn=t("code"),rh=r("path"),oh=r(" points to the location of the audio file."),hh=u(),qe=t("li"),Un=t("code"),uh=r("sampling_rate"),ch=r(" refers to how many data points in the speech signal are measured per second."),ql=u(),Z=t("h3"),ks=t("a"),Vn=t("span"),b(ha.$$.fragment),mh=u(),Gn=t("span"),ih=r("Resample"),Al=u(),ys=t("p"),bh=r("For this tutorial, you will use the "),ua=t("a"),fh=r("Wav2Vec2"),jh=r(" model. As you can see from the model card, the Wav2Vec2 model is pretrained on 16kHz sampled speech audio. It is important your audio data\u2019s sampling rate matches the sampling rate of the dataset used to pretrain the model. If your data\u2019s sampling rate isn\u2019t the same, then you need to resample your audio data."),Pl=u(),xs=t("p"),dh=r("For example, the "),ca=t("a"),gh=r("MInDS-14"),_h=r(" dataset has a sampling rate of 8000kHz. In order to use the Wav2Vec2 model with this dataset, upsample the sampling rate to 16kHz:"),zl=u(),b(ma.$$.fragment),Tl=u(),Ae=t("ol"),ia=t("li"),$h=r("Use \u{1F917} Datasets\u2019 "),ba=t("a"),Yn=t("code"),vh=r("cast_column"),wh=r(" method to upsample the sampling rate to 16kHz:"),Cl=u(),b(fa.$$.fragment),Dl=u(),ja=t("ol"),Kn=t("li"),kh=r("Load the audio file:"),Sl=u(),b(da.$$.fragment),Ll=u(),Es=t("p"),yh=r("As you can see, the "),Qn=t("code"),xh=r("sampling_rate"),Eh=r(" is now 16kHz!"),Nl=u(),ss=t("h3"),qs=t("a"),Xn=t("span"),b(ga.$$.fragment),qh=u(),Zn=t("span"),Ah=r("Feature extractor"),Ol=u(),A=t("p"),Ph=r("The next step is to load a feature extractor to normalize and pad the input. When padding textual data, a "),st=t("code"),zh=r("0"),Th=r(" is added for shorter sequences. The same idea applies to audio data, and the audio feature extractor will add a "),at=t("code"),Ch=r("0"),Dh=r(" - interpreted as silence - to "),et=t("code"),Sh=r("array"),Lh=r("."),Il=u(),As=t("p"),Nh=r("Load the feature extractor with "),Pe=t("a"),Oh=r("AutoFeatureExtractor.from_pretrained()"),Ih=r(":"),Fl=u(),b(_a.$$.fragment),Hl=u(),B=t("p"),Fh=r("Pass the audio "),nt=t("code"),Hh=r("array"),Rh=r(" to the feature extractor. We also recommend adding the "),tt=t("code"),Bh=r("sampling_rate"),Wh=r(" argument in the feature extractor in order to better debug any silent errors that may occur."),Rl=u(),b($a.$$.fragment),Bl=u(),as=t("h3"),Ps=t("a"),lt=t("span"),b(va.$$.fragment),Jh=u(),pt=t("span"),Mh=r("Pad and truncate"),Wl=u(),ze=t("p"),Uh=r("Just like the tokenizer, you can apply padding or truncation to handle variable sequences in a batch. Take a look at the sequence length of these two audio samples:"),Jl=u(),b(wa.$$.fragment),Ml=u(),Te=t("p"),Vh=r("As you can see, the first sample has a longer sequence than the second sample. Let\u2019s create a function that will preprocess the dataset. Specify a maximum sample length, and the feature extractor will either pad or truncate the sequences to match it:"),Ul=u(),b(ka.$$.fragment),Vl=u(),Ce=t("p"),Gh=r("Apply the function to the the first few examples in the dataset:"),Gl=u(),b(ya.$$.fragment),Yl=u(),De=t("p"),Yh=r("Now take another look at the processed sample lengths:"),Kl=u(),b(xa.$$.fragment),Ql=u(),Se=t("p"),Kh=r("The lengths of the first two samples now match the maximum length you specified."),Xl=u(),es=t("h2"),zs=t("a"),rt=t("span"),b(Ea.$$.fragment),Qh=u(),ot=t("span"),Xh=r("Vision"),Zl=u(),Le=t("p"),Zh=r("A feature extractor is also used to process images for vision tasks. Once again, the goal is to convert the raw image into a batch of tensors as input."),sp=u(),W=t("p"),su=r("Let\u2019s load the "),qa=t("a"),au=r("food101"),eu=r(" dataset for this tutorial. Use \u{1F917} Datasets "),ht=t("code"),nu=r("split"),tu=r(" parameter to only load a small sample from the training split since the dataset is quite large:"),ap=u(),b(Aa.$$.fragment),ep=u(),Ts=t("p"),lu=r("Next, take a look at the image with \u{1F917} Datasets "),Pa=t("a"),ut=t("code"),pu=r("Image"),ru=r(" feature:"),np=u(),b(za.$$.fragment),tp=u(),Ne=t("p"),Oe=t("img"),lp=u(),ns=t("h3"),Cs=t("a"),ct=t("span"),b(Ta.$$.fragment),ou=u(),mt=t("span"),hu=r("Feature extractor"),pp=u(),Ds=t("p"),uu=r("Load the feature extractor with "),Ie=t("a"),cu=r("AutoFeatureExtractor.from_pretrained()"),mu=r(":"),rp=u(),b(Ca.$$.fragment),op=u(),ts=t("h3"),Ss=t("a"),it=t("span"),b(Da.$$.fragment),iu=u(),bt=t("span"),bu=r("Data augmentation"),hp=u(),Ls=t("p"),fu=r("For vision tasks, it is common to add some type of data augmentation to the images as a part of preprocessing. You can add augmentations with any library you\u2019d like, but in this tutorial, you will use torchvision\u2019s "),Sa=t("a"),ft=t("code"),ju=r("transforms"),du=r(" module."),up=u(),Fe=t("ol"),z=t("li"),gu=r("Normalize the image and use "),La=t("a"),jt=t("code"),_u=r("Compose"),$u=r(" to chain some transforms - "),Na=t("a"),dt=t("code"),vu=r("RandomResizedCrop"),wu=r(" and "),Oa=t("a"),gt=t("code"),ku=r("ColorJitter"),yu=r(" - together:"),cp=u(),b(Ia.$$.fragment),mp=u(),Fa=t("ol"),ls=t("li"),xu=r("The model accepts "),He=t("a"),_t=t("code"),Eu=r("pixel_values"),qu=r(" as it\u2019s input. This value is generated by the feature extractor. Create a function that generates "),$t=t("code"),Au=r("pixel_values"),Pu=r(" from the transforms:"),ip=u(),b(Ha.$$.fragment),bp=u(),Ra=t("ol"),Ba=t("li"),zu=r("Then use \u{1F917} Datasets "),Wa=t("a"),vt=t("code"),Tu=r("set_transform"),Cu=r(" to apply the transforms on-the-fly:"),fp=u(),b(Ja.$$.fragment),jp=u(),Ma=t("ol"),Ua=t("li"),Du=r("Now when you access the image, you will notice the feature extractor has added the model input "),wt=t("code"),Su=r("pixel_values"),Lu=r(":"),dp=u(),b(Va.$$.fragment),gp=u(),Re=t("p"),Nu=r("Here is what the image looks like after you preprocess it. Just as you\u2019d expect from the applied transforms, the image has been randomly cropped and it\u2019s color properties are different."),_p=u(),b(Ga.$$.fragment),$p=u(),Be=t("p"),We=t("img"),vp=u(),ps=t("h2"),Ns=t("a"),kt=t("span"),b(Ya.$$.fragment),Ou=u(),yt=t("span"),Iu=r("Multimodal"),wp=u(),Je=t("p"),Fu=r("For multimodal tasks. you will use a combination of everything you\u2019ve learned so far and apply your skills to a automatic speech recognition (ASR) task. This means you will need a:"),kp=u(),Os=t("ul"),xt=t("li"),Hu=r("Feature extractor to preprocess the audio data."),Ru=u(),Et=t("li"),Bu=r("Tokenizer to process the text."),yp=u(),Is=t("p"),Wu=r("Let\u2019s return to the "),Ka=t("a"),Ju=r("LJ Speech"),Mu=r(" dataset:"),xp=u(),b(Qa.$$.fragment),Ep=u(),J=t("p"),Uu=r("Since you are mainly interested in the "),qt=t("code"),Vu=r("audio"),Gu=r(" and "),At=t("code"),Yu=r("text"),Ku=r(" column, remove the other columns:"),qp=u(),b(Xa.$$.fragment),Ap=u(),M=t("p"),Qu=r("Now take a look at the "),Pt=t("code"),Xu=r("audio"),Zu=r(" and "),zt=t("code"),sc=r("text"),ac=r(" columns:"),Pp=u(),b(Za.$$.fragment),zp=u(),Fs=t("p"),ec=r("Remember from the earlier section on processing audio data, you should always "),Me=t("a"),nc=r("resample"),tc=r(" your audio data\u2019s sampling rate to match the sampling rate of the dataset used to pretrain a model:"),Tp=u(),b(se.$$.fragment),Cp=u(),rs=t("h3"),Hs=t("a"),Tt=t("span"),b(ae.$$.fragment),lc=u(),Ct=t("span"),pc=r("Processor"),Dp=u(),Ue=t("p"),rc=r("A processor combines a feature extractor and tokenizer. Load a processor with [`AutoProcessor.from_pretrained]:"),Sp=u(),b(ee.$$.fragment),Lp=u(),Ve=t("ol"),os=t("li"),oc=r("Create a function to process the audio data to "),Dt=t("code"),hc=r("input_values"),uc=r(", and tokenizes the text to "),St=t("code"),cc=r("labels"),mc=r(". These are your inputs to the model:"),Np=u(),b(ne.$$.fragment),Op=u(),te=t("ol"),le=t("li"),ic=r("Apply the "),Lt=t("code"),bc=r("prepare_dataset"),fc=r(" function to a sample:"),Ip=u(),b(pe.$$.fragment),Fp=u(),U=t("p"),jc=r("Notice the processor has added "),Nt=t("code"),dc=r("input_values"),gc=r(" and "),Ot=t("code"),_c=r("labels"),$c=r(". The sampling rate has also been correctly downsampled to 16kHz."),Hp=u(),Ge=t("p"),vc=r("Awesome, you should now be able to preprocess data for any modality and even combine different modalities! In the next tutorial, learn how to fine-tune a model on your newly preprocessed data."),this.h()},l(s){const n=$b('[data-svelte="svelte-1phssyn"]',document.head);$=l(n,"META",{name:!0,content:!0}),n.forEach(a),w=c(s),i=l(s,"H1",{class:!0});var re=p(i);k=l(re,"A",{id:!0,class:!0,href:!0});var It=p(k);y=l(It,"SPAN",{});var zc=p(y);f(E.$$.fragment,zc),zc.forEach(a),It.forEach(a),T=c(re),hs=l(re,"SPAN",{});var Tc=p(hs);vr=o(Tc,"Preprocess"),Tc.forEach(a),re.forEach(a),Ft=c(s),f(Js.$$.fragment,s),Ht=c(s),oe=l(s,"P",{});var Cc=p(oe);wr=o(Cc,"Before you can use your data in a model, the data needs to be processed into an acceptable format for the model. A model does not understand raw text, images or audio. These inputs need to be converted into numbers and assembled into tensors. In this tutorial, you will:"),Cc.forEach(a),Rt=c(s),C=l(s,"UL",{});var Ye=p(C);mn=l(Ye,"LI",{});var Dc=p(mn);kr=o(Dc,"Preprocess textual data with a tokenizer."),Dc.forEach(a),yr=c(Ye),bn=l(Ye,"LI",{});var Sc=p(bn);xr=o(Sc,"Preprocess image or audio data with a feature extractor."),Sc.forEach(a),Er=c(Ye),fn=l(Ye,"LI",{});var Lc=p(fn);qr=o(Lc,"Preprocess data for a multimodal task with a processor."),Lc.forEach(a),Ye.forEach(a),Bt=c(s),V=l(s,"H2",{class:!0});var Bp=p(V);us=l(Bp,"A",{id:!0,class:!0,href:!0});var Nc=p(us);jn=l(Nc,"SPAN",{});var Oc=p(jn);f(Ms.$$.fragment,Oc),Oc.forEach(a),Nc.forEach(a),Ar=c(Bp),dn=l(Bp,"SPAN",{});var Ic=p(dn);Pr=o(Ic,"NLP"),Ic.forEach(a),Bp.forEach(a),Wt=c(s),f(Us.$$.fragment,s),Jt=c(s),D=l(s,"P",{});var Ke=p(D);zr=o(Ke,"The main tool for processing textual data is a "),he=l(Ke,"A",{href:!0});var Fc=p(he);Tr=o(Fc,"tokenizer"),Fc.forEach(a),Cr=o(Ke,". A tokenizer starts by splitting text into "),gn=l(Ke,"EM",{});var Hc=p(gn);Dr=o(Hc,"tokens"),Hc.forEach(a),Sr=o(Ke," according to a set of rules. The tokens are converted into numbers, which are used to build tensors as input to a model. Any additional inputs required by a model are also added by the tokenizer."),Ke.forEach(a),Mt=c(s),f(cs.$$.fragment,s),Ut=c(s),S=l(s,"P",{});var Qe=p(S);Lr=o(Qe,"Get started quickly by loading a pretrained tokenizer with the "),ue=l(Qe,"A",{href:!0});var Rc=p(ue);Nr=o(Rc,"AutoTokenizer"),Rc.forEach(a),Or=o(Qe," class. This downloads the "),_n=l(Qe,"EM",{});var Bc=p(_n);Ir=o(Bc,"vocab"),Bc.forEach(a),Fr=o(Qe," used when a model is pretrained."),Qe.forEach(a),Vt=c(s),G=l(s,"H3",{class:!0});var Wp=p(G);ms=l(Wp,"A",{id:!0,class:!0,href:!0});var Wc=p(ms);$n=l(Wc,"SPAN",{});var Jc=p($n);f(Vs.$$.fragment,Jc),Jc.forEach(a),Wc.forEach(a),Hr=c(Wp),vn=l(Wp,"SPAN",{});var Mc=p(vn);Rr=o(Mc,"Tokenize"),Mc.forEach(a),Wp.forEach(a),Gt=c(s),is=l(s,"P",{});var Jp=p(is);Br=o(Jp,"Load a pretrained tokenizer with "),ce=l(Jp,"A",{href:!0});var Uc=p(ce);Wr=o(Uc,"AutoTokenizer.from_pretrained()"),Uc.forEach(a),Jr=o(Jp,":"),Jp.forEach(a),Yt=c(s),f(Gs.$$.fragment,s),Kt=c(s),me=l(s,"P",{});var Vc=p(me);Mr=o(Vc,"Then pass your sentence to the tokenizer:"),Vc.forEach(a),Qt=c(s),f(Ys.$$.fragment,s),Xt=c(s),ie=l(s,"P",{});var Gc=p(ie);Ur=o(Gc,"The tokenizer returns a dictionary with three important itmes:"),Gc.forEach(a),Zt=c(s),L=l(s,"UL",{});var Xe=p(L);be=l(Xe,"LI",{});var wc=p(be);fe=l(wc,"A",{href:!0});var Yc=p(fe);Vr=o(Yc,"input_ids"),Yc.forEach(a),Gr=o(wc," are the indices corresponding to each token in the sentence."),wc.forEach(a),Yr=c(Xe),je=l(Xe,"LI",{});var kc=p(je);de=l(kc,"A",{href:!0});var Kc=p(de);Kr=o(Kc,"attention_mask"),Kc.forEach(a),Qr=o(kc," indicates whether a token should be attended to or not."),kc.forEach(a),Xr=c(Xe),ge=l(Xe,"LI",{});var yc=p(ge);_e=l(yc,"A",{href:!0});var Qc=p(_e);Zr=o(Qc,"token_type_ids"),Qc.forEach(a),so=o(yc," identifies which sequence a token belongs to when there is more than one sequence."),yc.forEach(a),Xe.forEach(a),sl=c(s),bs=l(s,"P",{});var Mp=p(bs);ao=o(Mp,"You can decode the "),wn=l(Mp,"CODE",{});var Xc=p(wn);eo=o(Xc,"input_ids"),Xc.forEach(a),no=o(Mp," to return the original input:"),Mp.forEach(a),al=c(s),f(Ks.$$.fragment,s),el=c(s),N=l(s,"P",{});var Ze=p(N);to=o(Ze,"As you can see, the tokenizer added two special tokens - "),kn=l(Ze,"CODE",{});var Zc=p(kn);lo=o(Zc,"CLS"),Zc.forEach(a),po=o(Ze," and "),yn=l(Ze,"CODE",{});var sm=p(yn);ro=o(sm,"SEP"),sm.forEach(a),oo=o(Ze,` (classifier and separator) - to the sentence. Not all models need
special tokens, but if they do, the tokenizer will automatically add them for you.`),Ze.forEach(a),nl=c(s),$e=l(s,"P",{});var am=p($e);ho=o(am,"If there are several sentences you want to process, pass the sentences as a list to the tokenizer:"),am.forEach(a),tl=c(s),f(Qs.$$.fragment,s),ll=c(s),Y=l(s,"H3",{class:!0});var Up=p(Y);fs=l(Up,"A",{id:!0,class:!0,href:!0});var em=p(fs);xn=l(em,"SPAN",{});var nm=p(xn);f(Xs.$$.fragment,nm),nm.forEach(a),em.forEach(a),uo=c(Up),En=l(Up,"SPAN",{});var tm=p(En);co=o(tm,"Pad"),tm.forEach(a),Up.forEach(a),pl=c(s),js=l(s,"P",{});var Vp=p(js);mo=o(Vp,"This brings us to an important topic. When you process a batch of sentences, they aren\u2019t always the same length. This is a problem because tensors, the input to the model, need to have a uniform shape. Padding is a strategy for ensuring tensors are rectangular by adding a special "),qn=l(Vp,"EM",{});var lm=p(qn);io=o(lm,"padding token"),lm.forEach(a),bo=o(Vp," to sentences with fewer tokens."),Vp.forEach(a),rl=c(s),O=l(s,"P",{});var sn=p(O);fo=o(sn,"Set the "),An=l(sn,"CODE",{});var pm=p(An);jo=o(pm,"padding"),pm.forEach(a),go=o(sn," parameter to "),Pn=l(sn,"CODE",{});var rm=p(Pn);_o=o(rm,"True"),rm.forEach(a),$o=o(sn," to pad the shorter sequences in the batch to match the longest sequence:"),sn.forEach(a),ol=c(s),f(Zs.$$.fragment,s),hl=c(s),ds=l(s,"P",{});var Gp=p(ds);vo=o(Gp,"Notice the tokenizer padded the first and third sentences with a "),zn=l(Gp,"CODE",{});var om=p(zn);wo=o(om,"0"),om.forEach(a),ko=o(Gp," because they are shorter!"),Gp.forEach(a),ul=c(s),K=l(s,"H3",{class:!0});var Yp=p(K);gs=l(Yp,"A",{id:!0,class:!0,href:!0});var hm=p(gs);Tn=l(hm,"SPAN",{});var um=p(Tn);f(sa.$$.fragment,um),um.forEach(a),hm.forEach(a),yo=c(Yp),Cn=l(Yp,"SPAN",{});var cm=p(Cn);xo=o(cm,"Truncation"),cm.forEach(a),Yp.forEach(a),cl=c(s),ve=l(s,"P",{});var mm=p(ve);Eo=o(mm,"On the other end of the spectrum, sometimes a sequence may be too long for a model to handle. In this case, you will need to truncate the sequence to a shorter length."),mm.forEach(a),ml=c(s),I=l(s,"P",{});var an=p(I);qo=o(an,"Set the "),Dn=l(an,"CODE",{});var im=p(Dn);Ao=o(im,"truncation"),im.forEach(a),Po=o(an," parameter to "),Sn=l(an,"CODE",{});var bm=p(Sn);zo=o(bm,"True"),bm.forEach(a),To=o(an," to truncate a sequence to the maximum length accepted by the model:"),an.forEach(a),il=c(s),f(aa.$$.fragment,s),bl=c(s),Q=l(s,"H3",{class:!0});var Kp=p(Q);_s=l(Kp,"A",{id:!0,class:!0,href:!0});var fm=p(_s);Ln=l(fm,"SPAN",{});var jm=p(Ln);f(ea.$$.fragment,jm),jm.forEach(a),fm.forEach(a),Co=c(Kp),Nn=l(Kp,"SPAN",{});var dm=p(Nn);Do=o(dm,"Build tensors"),dm.forEach(a),Kp.forEach(a),fl=c(s),we=l(s,"P",{});var gm=p(we);So=o(gm,"Finally, you want the tokenizer to return the actual tensors that are fed to the model."),gm.forEach(a),jl=c(s),q=l(s,"P",{});var Rs=p(q);Lo=o(Rs,"Set the "),On=l(Rs,"CODE",{});var _m=p(On);No=o(_m,"return_tensors"),_m.forEach(a),Oo=o(Rs," parameter to either "),In=l(Rs,"CODE",{});var $m=p(In);Io=o($m,"pt"),$m.forEach(a),Fo=o(Rs," for PyTorch, or "),Fn=l(Rs,"CODE",{});var vm=p(Fn);Ho=o(vm,"tf"),vm.forEach(a),Ro=o(Rs," for TensorFlow:"),Rs.forEach(a),dl=c(s),f($s.$$.fragment,s),gl=c(s),X=l(s,"H2",{class:!0});var Qp=p(X);vs=l(Qp,"A",{id:!0,class:!0,href:!0});var wm=p(vs);Hn=l(wm,"SPAN",{});var km=p(Hn);f(na.$$.fragment,km),km.forEach(a),wm.forEach(a),Bo=c(Qp),Rn=l(Qp,"SPAN",{});var ym=p(Rn);Wo=o(ym,"Audio"),ym.forEach(a),Qp.forEach(a),_l=c(s),ws=l(s,"P",{});var Xp=p(ws);Jo=o(Xp,"Audio inputs are preprocessed differently than textual inputs, but the end goal remains the same: create numerical sequences the model can understand. A "),ke=l(Xp,"A",{href:!0});var xm=p(ke);Mo=o(xm,"feature extractor"),xm.forEach(a),Uo=o(Xp," is designed for the express purpose of extracting features from raw image or audio data and converting them into tensors. Before you begin, install \u{1F917} Datasets to load an audio dataset to experiment with:"),Xp.forEach(a),$l=c(s),f(ta.$$.fragment,s),vl=c(s),F=l(s,"P",{});var en=p(F);Vo=o(en,"Load the "),la=l(en,"A",{href:!0,rel:!0});var Em=p(la);Go=o(Em,"MInDS-14"),Em.forEach(a),Yo=o(en," dataset (see the \u{1F917} "),pa=l(en,"A",{href:!0,rel:!0});var qm=p(pa);Ko=o(qm,"Datasets tutorial"),qm.forEach(a),Qo=o(en," for more details on how to load a dataset):"),en.forEach(a),wl=c(s),f(ra.$$.fragment,s),kl=c(s),H=l(s,"P",{});var nn=p(H);Xo=o(nn,"Access the first element of the "),Bn=l(nn,"CODE",{});var Am=p(Bn);Zo=o(Am,"audio"),Am.forEach(a),sh=o(nn," column to take a look at the input. Calling the "),Wn=l(nn,"CODE",{});var Pm=p(Wn);ah=o(Pm,"audio"),Pm.forEach(a),eh=o(nn," column will automatically load and resample the audio file:"),nn.forEach(a),yl=c(s),f(oa.$$.fragment,s),xl=c(s),ye=l(s,"P",{});var zm=p(ye);nh=o(zm,"This returns three items:"),zm.forEach(a),El=c(s),R=l(s,"UL",{});var tn=p(R);xe=l(tn,"LI",{});var xc=p(xe);Jn=l(xc,"CODE",{});var Tm=p(Jn);th=o(Tm,"array"),Tm.forEach(a),lh=o(xc," is the speech signal loaded - and potentially resampled - as a 1D array."),xc.forEach(a),ph=c(tn),Ee=l(tn,"LI",{});var Ec=p(Ee);Mn=l(Ec,"CODE",{});var Cm=p(Mn);rh=o(Cm,"path"),Cm.forEach(a),oh=o(Ec," points to the location of the audio file."),Ec.forEach(a),hh=c(tn),qe=l(tn,"LI",{});var qc=p(qe);Un=l(qc,"CODE",{});var Dm=p(Un);uh=o(Dm,"sampling_rate"),Dm.forEach(a),ch=o(qc," refers to how many data points in the speech signal are measured per second."),qc.forEach(a),tn.forEach(a),ql=c(s),Z=l(s,"H3",{class:!0});var Zp=p(Z);ks=l(Zp,"A",{id:!0,class:!0,href:!0});var Sm=p(ks);Vn=l(Sm,"SPAN",{});var Lm=p(Vn);f(ha.$$.fragment,Lm),Lm.forEach(a),Sm.forEach(a),mh=c(Zp),Gn=l(Zp,"SPAN",{});var Nm=p(Gn);ih=o(Nm,"Resample"),Nm.forEach(a),Zp.forEach(a),Al=c(s),ys=l(s,"P",{});var sr=p(ys);bh=o(sr,"For this tutorial, you will use the "),ua=l(sr,"A",{href:!0,rel:!0});var Om=p(ua);fh=o(Om,"Wav2Vec2"),Om.forEach(a),jh=o(sr," model. As you can see from the model card, the Wav2Vec2 model is pretrained on 16kHz sampled speech audio. It is important your audio data\u2019s sampling rate matches the sampling rate of the dataset used to pretrain the model. If your data\u2019s sampling rate isn\u2019t the same, then you need to resample your audio data."),sr.forEach(a),Pl=c(s),xs=l(s,"P",{});var ar=p(xs);dh=o(ar,"For example, the "),ca=l(ar,"A",{href:!0,rel:!0});var Im=p(ca);gh=o(Im,"MInDS-14"),Im.forEach(a),_h=o(ar," dataset has a sampling rate of 8000kHz. In order to use the Wav2Vec2 model with this dataset, upsample the sampling rate to 16kHz:"),ar.forEach(a),zl=c(s),f(ma.$$.fragment,s),Tl=c(s),Ae=l(s,"OL",{});var Fm=p(Ae);ia=l(Fm,"LI",{});var er=p(ia);$h=o(er,"Use \u{1F917} Datasets\u2019 "),ba=l(er,"A",{href:!0,rel:!0});var Hm=p(ba);Yn=l(Hm,"CODE",{});var Rm=p(Yn);vh=o(Rm,"cast_column"),Rm.forEach(a),Hm.forEach(a),wh=o(er," method to upsample the sampling rate to 16kHz:"),er.forEach(a),Fm.forEach(a),Cl=c(s),f(fa.$$.fragment,s),Dl=c(s),ja=l(s,"OL",{start:!0});var Bm=p(ja);Kn=l(Bm,"LI",{});var Wm=p(Kn);kh=o(Wm,"Load the audio file:"),Wm.forEach(a),Bm.forEach(a),Sl=c(s),f(da.$$.fragment,s),Ll=c(s),Es=l(s,"P",{});var nr=p(Es);yh=o(nr,"As you can see, the "),Qn=l(nr,"CODE",{});var Jm=p(Qn);xh=o(Jm,"sampling_rate"),Jm.forEach(a),Eh=o(nr," is now 16kHz!"),nr.forEach(a),Nl=c(s),ss=l(s,"H3",{class:!0});var tr=p(ss);qs=l(tr,"A",{id:!0,class:!0,href:!0});var Mm=p(qs);Xn=l(Mm,"SPAN",{});var Um=p(Xn);f(ga.$$.fragment,Um),Um.forEach(a),Mm.forEach(a),qh=c(tr),Zn=l(tr,"SPAN",{});var Vm=p(Zn);Ah=o(Vm,"Feature extractor"),Vm.forEach(a),tr.forEach(a),Ol=c(s),A=l(s,"P",{});var Bs=p(A);Ph=o(Bs,"The next step is to load a feature extractor to normalize and pad the input. When padding textual data, a "),st=l(Bs,"CODE",{});var Gm=p(st);zh=o(Gm,"0"),Gm.forEach(a),Th=o(Bs," is added for shorter sequences. The same idea applies to audio data, and the audio feature extractor will add a "),at=l(Bs,"CODE",{});var Ym=p(at);Ch=o(Ym,"0"),Ym.forEach(a),Dh=o(Bs," - interpreted as silence - to "),et=l(Bs,"CODE",{});var Km=p(et);Sh=o(Km,"array"),Km.forEach(a),Lh=o(Bs,"."),Bs.forEach(a),Il=c(s),As=l(s,"P",{});var lr=p(As);Nh=o(lr,"Load the feature extractor with "),Pe=l(lr,"A",{href:!0});var Qm=p(Pe);Oh=o(Qm,"AutoFeatureExtractor.from_pretrained()"),Qm.forEach(a),Ih=o(lr,":"),lr.forEach(a),Fl=c(s),f(_a.$$.fragment,s),Hl=c(s),B=l(s,"P",{});var ln=p(B);Fh=o(ln,"Pass the audio "),nt=l(ln,"CODE",{});var Xm=p(nt);Hh=o(Xm,"array"),Xm.forEach(a),Rh=o(ln," to the feature extractor. We also recommend adding the "),tt=l(ln,"CODE",{});var Zm=p(tt);Bh=o(Zm,"sampling_rate"),Zm.forEach(a),Wh=o(ln," argument in the feature extractor in order to better debug any silent errors that may occur."),ln.forEach(a),Rl=c(s),f($a.$$.fragment,s),Bl=c(s),as=l(s,"H3",{class:!0});var pr=p(as);Ps=l(pr,"A",{id:!0,class:!0,href:!0});var si=p(Ps);lt=l(si,"SPAN",{});var ai=p(lt);f(va.$$.fragment,ai),ai.forEach(a),si.forEach(a),Jh=c(pr),pt=l(pr,"SPAN",{});var ei=p(pt);Mh=o(ei,"Pad and truncate"),ei.forEach(a),pr.forEach(a),Wl=c(s),ze=l(s,"P",{});var ni=p(ze);Uh=o(ni,"Just like the tokenizer, you can apply padding or truncation to handle variable sequences in a batch. Take a look at the sequence length of these two audio samples:"),ni.forEach(a),Jl=c(s),f(wa.$$.fragment,s),Ml=c(s),Te=l(s,"P",{});var ti=p(Te);Vh=o(ti,"As you can see, the first sample has a longer sequence than the second sample. Let\u2019s create a function that will preprocess the dataset. Specify a maximum sample length, and the feature extractor will either pad or truncate the sequences to match it:"),ti.forEach(a),Ul=c(s),f(ka.$$.fragment,s),Vl=c(s),Ce=l(s,"P",{});var li=p(Ce);Gh=o(li,"Apply the function to the the first few examples in the dataset:"),li.forEach(a),Gl=c(s),f(ya.$$.fragment,s),Yl=c(s),De=l(s,"P",{});var pi=p(De);Yh=o(pi,"Now take another look at the processed sample lengths:"),pi.forEach(a),Kl=c(s),f(xa.$$.fragment,s),Ql=c(s),Se=l(s,"P",{});var ri=p(Se);Kh=o(ri,"The lengths of the first two samples now match the maximum length you specified."),ri.forEach(a),Xl=c(s),es=l(s,"H2",{class:!0});var rr=p(es);zs=l(rr,"A",{id:!0,class:!0,href:!0});var oi=p(zs);rt=l(oi,"SPAN",{});var hi=p(rt);f(Ea.$$.fragment,hi),hi.forEach(a),oi.forEach(a),Qh=c(rr),ot=l(rr,"SPAN",{});var ui=p(ot);Xh=o(ui,"Vision"),ui.forEach(a),rr.forEach(a),Zl=c(s),Le=l(s,"P",{});var ci=p(Le);Zh=o(ci,"A feature extractor is also used to process images for vision tasks. Once again, the goal is to convert the raw image into a batch of tensors as input."),ci.forEach(a),sp=c(s),W=l(s,"P",{});var pn=p(W);su=o(pn,"Let\u2019s load the "),qa=l(pn,"A",{href:!0,rel:!0});var mi=p(qa);au=o(mi,"food101"),mi.forEach(a),eu=o(pn," dataset for this tutorial. Use \u{1F917} Datasets "),ht=l(pn,"CODE",{});var ii=p(ht);nu=o(ii,"split"),ii.forEach(a),tu=o(pn," parameter to only load a small sample from the training split since the dataset is quite large:"),pn.forEach(a),ap=c(s),f(Aa.$$.fragment,s),ep=c(s),Ts=l(s,"P",{});var or=p(Ts);lu=o(or,"Next, take a look at the image with \u{1F917} Datasets "),Pa=l(or,"A",{href:!0,rel:!0});var bi=p(Pa);ut=l(bi,"CODE",{});var fi=p(ut);pu=o(fi,"Image"),fi.forEach(a),bi.forEach(a),ru=o(or," feature:"),or.forEach(a),np=c(s),f(za.$$.fragment,s),tp=c(s),Ne=l(s,"P",{});var ji=p(Ne);Oe=l(ji,"IMG",{src:!0,alt:!0}),ji.forEach(a),lp=c(s),ns=l(s,"H3",{class:!0});var hr=p(ns);Cs=l(hr,"A",{id:!0,class:!0,href:!0});var di=p(Cs);ct=l(di,"SPAN",{});var gi=p(ct);f(Ta.$$.fragment,gi),gi.forEach(a),di.forEach(a),ou=c(hr),mt=l(hr,"SPAN",{});var _i=p(mt);hu=o(_i,"Feature extractor"),_i.forEach(a),hr.forEach(a),pp=c(s),Ds=l(s,"P",{});var ur=p(Ds);uu=o(ur,"Load the feature extractor with "),Ie=l(ur,"A",{href:!0});var $i=p(Ie);cu=o($i,"AutoFeatureExtractor.from_pretrained()"),$i.forEach(a),mu=o(ur,":"),ur.forEach(a),rp=c(s),f(Ca.$$.fragment,s),op=c(s),ts=l(s,"H3",{class:!0});var cr=p(ts);Ss=l(cr,"A",{id:!0,class:!0,href:!0});var vi=p(Ss);it=l(vi,"SPAN",{});var wi=p(it);f(Da.$$.fragment,wi),wi.forEach(a),vi.forEach(a),iu=c(cr),bt=l(cr,"SPAN",{});var ki=p(bt);bu=o(ki,"Data augmentation"),ki.forEach(a),cr.forEach(a),hp=c(s),Ls=l(s,"P",{});var mr=p(Ls);fu=o(mr,"For vision tasks, it is common to add some type of data augmentation to the images as a part of preprocessing. You can add augmentations with any library you\u2019d like, but in this tutorial, you will use torchvision\u2019s "),Sa=l(mr,"A",{href:!0,rel:!0});var yi=p(Sa);ft=l(yi,"CODE",{});var xi=p(ft);ju=o(xi,"transforms"),xi.forEach(a),yi.forEach(a),du=o(mr," module."),mr.forEach(a),up=c(s),Fe=l(s,"OL",{});var Ei=p(Fe);z=l(Ei,"LI",{});var Ws=p(z);gu=o(Ws,"Normalize the image and use "),La=l(Ws,"A",{href:!0,rel:!0});var qi=p(La);jt=l(qi,"CODE",{});var Ai=p(jt);_u=o(Ai,"Compose"),Ai.forEach(a),qi.forEach(a),$u=o(Ws," to chain some transforms - "),Na=l(Ws,"A",{href:!0,rel:!0});var Pi=p(Na);dt=l(Pi,"CODE",{});var zi=p(dt);vu=o(zi,"RandomResizedCrop"),zi.forEach(a),Pi.forEach(a),wu=o(Ws," and "),Oa=l(Ws,"A",{href:!0,rel:!0});var Ti=p(Oa);gt=l(Ti,"CODE",{});var Ci=p(gt);ku=o(Ci,"ColorJitter"),Ci.forEach(a),Ti.forEach(a),yu=o(Ws," - together:"),Ws.forEach(a),Ei.forEach(a),cp=c(s),f(Ia.$$.fragment,s),mp=c(s),Fa=l(s,"OL",{start:!0});var Di=p(Fa);ls=l(Di,"LI",{});var rn=p(ls);xu=o(rn,"The model accepts "),He=l(rn,"A",{href:!0});var Si=p(He);_t=l(Si,"CODE",{});var Li=p(_t);Eu=o(Li,"pixel_values"),Li.forEach(a),Si.forEach(a),qu=o(rn," as it\u2019s input. This value is generated by the feature extractor. Create a function that generates "),$t=l(rn,"CODE",{});var Ni=p($t);Au=o(Ni,"pixel_values"),Ni.forEach(a),Pu=o(rn," from the transforms:"),rn.forEach(a),Di.forEach(a),ip=c(s),f(Ha.$$.fragment,s),bp=c(s),Ra=l(s,"OL",{start:!0});var Oi=p(Ra);Ba=l(Oi,"LI",{});var ir=p(Ba);zu=o(ir,"Then use \u{1F917} Datasets "),Wa=l(ir,"A",{href:!0,rel:!0});var Ii=p(Wa);vt=l(Ii,"CODE",{});var Fi=p(vt);Tu=o(Fi,"set_transform"),Fi.forEach(a),Ii.forEach(a),Cu=o(ir," to apply the transforms on-the-fly:"),ir.forEach(a),Oi.forEach(a),fp=c(s),f(Ja.$$.fragment,s),jp=c(s),Ma=l(s,"OL",{start:!0});var Hi=p(Ma);Ua=l(Hi,"LI",{});var br=p(Ua);Du=o(br,"Now when you access the image, you will notice the feature extractor has added the model input "),wt=l(br,"CODE",{});var Ri=p(wt);Su=o(Ri,"pixel_values"),Ri.forEach(a),Lu=o(br,":"),br.forEach(a),Hi.forEach(a),dp=c(s),f(Va.$$.fragment,s),gp=c(s),Re=l(s,"P",{});var Bi=p(Re);Nu=o(Bi,"Here is what the image looks like after you preprocess it. Just as you\u2019d expect from the applied transforms, the image has been randomly cropped and it\u2019s color properties are different."),Bi.forEach(a),_p=c(s),f(Ga.$$.fragment,s),$p=c(s),Be=l(s,"P",{});var Wi=p(Be);We=l(Wi,"IMG",{src:!0,alt:!0}),Wi.forEach(a),vp=c(s),ps=l(s,"H2",{class:!0});var fr=p(ps);Ns=l(fr,"A",{id:!0,class:!0,href:!0});var Ji=p(Ns);kt=l(Ji,"SPAN",{});var Mi=p(kt);f(Ya.$$.fragment,Mi),Mi.forEach(a),Ji.forEach(a),Ou=c(fr),yt=l(fr,"SPAN",{});var Ui=p(yt);Iu=o(Ui,"Multimodal"),Ui.forEach(a),fr.forEach(a),wp=c(s),Je=l(s,"P",{});var Vi=p(Je);Fu=o(Vi,"For multimodal tasks. you will use a combination of everything you\u2019ve learned so far and apply your skills to a automatic speech recognition (ASR) task. This means you will need a:"),Vi.forEach(a),kp=c(s),Os=l(s,"UL",{});var jr=p(Os);xt=l(jr,"LI",{});var Gi=p(xt);Hu=o(Gi,"Feature extractor to preprocess the audio data."),Gi.forEach(a),Ru=c(jr),Et=l(jr,"LI",{});var Yi=p(Et);Bu=o(Yi,"Tokenizer to process the text."),Yi.forEach(a),jr.forEach(a),yp=c(s),Is=l(s,"P",{});var dr=p(Is);Wu=o(dr,"Let\u2019s return to the "),Ka=l(dr,"A",{href:!0,rel:!0});var Ki=p(Ka);Ju=o(Ki,"LJ Speech"),Ki.forEach(a),Mu=o(dr," dataset:"),dr.forEach(a),xp=c(s),f(Qa.$$.fragment,s),Ep=c(s),J=l(s,"P",{});var on=p(J);Uu=o(on,"Since you are mainly interested in the "),qt=l(on,"CODE",{});var Qi=p(qt);Vu=o(Qi,"audio"),Qi.forEach(a),Gu=o(on," and "),At=l(on,"CODE",{});var Xi=p(At);Yu=o(Xi,"text"),Xi.forEach(a),Ku=o(on," column, remove the other columns:"),on.forEach(a),qp=c(s),f(Xa.$$.fragment,s),Ap=c(s),M=l(s,"P",{});var hn=p(M);Qu=o(hn,"Now take a look at the "),Pt=l(hn,"CODE",{});var Zi=p(Pt);Xu=o(Zi,"audio"),Zi.forEach(a),Zu=o(hn," and "),zt=l(hn,"CODE",{});var sb=p(zt);sc=o(sb,"text"),sb.forEach(a),ac=o(hn," columns:"),hn.forEach(a),Pp=c(s),f(Za.$$.fragment,s),zp=c(s),Fs=l(s,"P",{});var gr=p(Fs);ec=o(gr,"Remember from the earlier section on processing audio data, you should always "),Me=l(gr,"A",{href:!0});var ab=p(Me);nc=o(ab,"resample"),ab.forEach(a),tc=o(gr," your audio data\u2019s sampling rate to match the sampling rate of the dataset used to pretrain a model:"),gr.forEach(a),Tp=c(s),f(se.$$.fragment,s),Cp=c(s),rs=l(s,"H3",{class:!0});var _r=p(rs);Hs=l(_r,"A",{id:!0,class:!0,href:!0});var eb=p(Hs);Tt=l(eb,"SPAN",{});var nb=p(Tt);f(ae.$$.fragment,nb),nb.forEach(a),eb.forEach(a),lc=c(_r),Ct=l(_r,"SPAN",{});var tb=p(Ct);pc=o(tb,"Processor"),tb.forEach(a),_r.forEach(a),Dp=c(s),Ue=l(s,"P",{});var lb=p(Ue);rc=o(lb,"A processor combines a feature extractor and tokenizer. Load a processor with [`AutoProcessor.from_pretrained]:"),lb.forEach(a),Sp=c(s),f(ee.$$.fragment,s),Lp=c(s),Ve=l(s,"OL",{});var pb=p(Ve);os=l(pb,"LI",{});var un=p(os);oc=o(un,"Create a function to process the audio data to "),Dt=l(un,"CODE",{});var rb=p(Dt);hc=o(rb,"input_values"),rb.forEach(a),uc=o(un,", and tokenizes the text to "),St=l(un,"CODE",{});var ob=p(St);cc=o(ob,"labels"),ob.forEach(a),mc=o(un,". These are your inputs to the model:"),un.forEach(a),pb.forEach(a),Np=c(s),f(ne.$$.fragment,s),Op=c(s),te=l(s,"OL",{start:!0});var hb=p(te);le=l(hb,"LI",{});var $r=p(le);ic=o($r,"Apply the "),Lt=l($r,"CODE",{});var ub=p(Lt);bc=o(ub,"prepare_dataset"),ub.forEach(a),fc=o($r," function to a sample:"),$r.forEach(a),hb.forEach(a),Ip=c(s),f(pe.$$.fragment,s),Fp=c(s),U=l(s,"P",{});var cn=p(U);jc=o(cn,"Notice the processor has added "),Nt=l(cn,"CODE",{});var cb=p(Nt);dc=o(cb,"input_values"),cb.forEach(a),gc=o(cn," and "),Ot=l(cn,"CODE",{});var mb=p(Ot);_c=o(mb,"labels"),mb.forEach(a),$c=o(cn,". The sampling rate has also been correctly downsampled to 16kHz."),cn.forEach(a),Hp=c(s),Ge=l(s,"P",{});var ib=p(Ge);vc=o(ib,"Awesome, you should now be able to preprocess data for any modality and even combine different modalities! In the next tutorial, learn how to fine-tune a model on your newly preprocessed data."),ib.forEach(a),this.h()},h(){m($,"name","hf:doc:metadata"),m($,"content",JSON.stringify(Cb)),m(k,"id","preprocess"),m(k,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),m(k,"href","#preprocess"),m(i,"class","relative group"),m(us,"id","nlp"),m(us,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),m(us,"href","#nlp"),m(V,"class","relative group"),m(he,"href","main_classes/tokenizer"),m(ue,"href","/docs/transformers/pr_17548/en/model_doc/auto#transformers.AutoTokenizer"),m(ms,"id","tokenize"),m(ms,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),m(ms,"href","#tokenize"),m(G,"class","relative group"),m(ce,"href","/docs/transformers/pr_17548/en/model_doc/auto#transformers.AutoTokenizer.from_pretrained"),m(fe,"href","glossary#input-ids"),m(de,"href","glossary#attention-mask"),m(_e,"href","glossary#token-type-ids"),m(fs,"id","pad"),m(fs,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),m(fs,"href","#pad"),m(Y,"class","relative group"),m(gs,"id","truncation"),m(gs,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),m(gs,"href","#truncation"),m(K,"class","relative group"),m(_s,"id","build-tensors"),m(_s,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),m(_s,"href","#build-tensors"),m(Q,"class","relative group"),m(vs,"id","audio"),m(vs,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),m(vs,"href","#audio"),m(X,"class","relative group"),m(ke,"href","main_classes/feature_extractor"),m(la,"href","https://huggingface.co/datasets/PolyAI/minds14"),m(la,"rel","nofollow"),m(pa,"href","https://huggingface.co/docs/datasets/load_hub.html"),m(pa,"rel","nofollow"),m(ks,"id","resample"),m(ks,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),m(ks,"href","#resample"),m(Z,"class","relative group"),m(ua,"href","https://huggingface.co/facebook/wav2vec2-base"),m(ua,"rel","nofollow"),m(ca,"href","https://huggingface.co/datasets/PolyAI/minds14"),m(ca,"rel","nofollow"),m(ba,"href","https://huggingface.co/docs/datasets/package_reference/main_classes.html#datasets.Dataset.cast_column"),m(ba,"rel","nofollow"),m(ja,"start","2"),m(qs,"id","feature-extractor"),m(qs,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),m(qs,"href","#feature-extractor"),m(ss,"class","relative group"),m(Pe,"href","/docs/transformers/pr_17548/en/model_doc/auto#transformers.AutoFeatureExtractor.from_pretrained"),m(Ps,"id","pad-and-truncate"),m(Ps,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),m(Ps,"href","#pad-and-truncate"),m(as,"class","relative group"),m(zs,"id","vision"),m(zs,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),m(zs,"href","#vision"),m(es,"class","relative group"),m(qa,"href","https://huggingface.co/datasets/food101"),m(qa,"rel","nofollow"),m(Pa,"href","https://huggingface.co/docs/datasets/package_reference/main_classes.html?highlight=image#datasets.Image"),m(Pa,"rel","nofollow"),bb(Oe.src,Ac="https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/vision-preprocess-tutorial.png")||m(Oe,"src",Ac),m(Oe,"alt","vision-preprocess-tutorial.png"),m(Cs,"id","feature-extractor"),m(Cs,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),m(Cs,"href","#feature-extractor"),m(ns,"class","relative group"),m(Ie,"href","/docs/transformers/pr_17548/en/model_doc/auto#transformers.AutoFeatureExtractor.from_pretrained"),m(Ss,"id","data-augmentation"),m(Ss,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),m(Ss,"href","#data-augmentation"),m(ts,"class","relative group"),m(Sa,"href","https://pytorch.org/vision/stable/transforms.html"),m(Sa,"rel","nofollow"),m(La,"href","https://pytorch.org/vision/master/generated/torchvision.transforms.Compose.html"),m(La,"rel","nofollow"),m(Na,"href","https://pytorch.org/vision/main/generated/torchvision.transforms.RandomResizedCrop.html"),m(Na,"rel","nofollow"),m(Oa,"href","https://pytorch.org/vision/main/generated/torchvision.transforms.ColorJitter.html"),m(Oa,"rel","nofollow"),m(He,"href","model_doc/visionencoderdecoder#transformers.VisionEncoderDecoderModel.forward.pixel_values"),m(Fa,"start","2"),m(Wa,"href","https://huggingface.co/docs/datasets/process.html#format-transform"),m(Wa,"rel","nofollow"),m(Ra,"start","3"),m(Ma,"start","4"),bb(We.src,Pc="https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/preprocessed_image.png")||m(We,"src",Pc),m(We,"alt","preprocessed_image"),m(Ns,"id","multimodal"),m(Ns,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),m(Ns,"href","#multimodal"),m(ps,"class","relative group"),m(Ka,"href","https://huggingface.co/datasets/lj_speech"),m(Ka,"rel","nofollow"),m(Me,"href","preprocessing#audio"),m(Hs,"id","processor"),m(Hs,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),m(Hs,"href","#processor"),m(rs,"class","relative group"),m(te,"start","2")},m(s,n){e(document.head,$),h(s,w,n),h(s,i,n),e(i,k),e(k,y),j(E,y,null),e(i,T),e(i,hs),e(hs,vr),h(s,Ft,n),j(Js,s,n),h(s,Ht,n),h(s,oe,n),e(oe,wr),h(s,Rt,n),h(s,C,n),e(C,mn),e(mn,kr),e(C,yr),e(C,bn),e(bn,xr),e(C,Er),e(C,fn),e(fn,qr),h(s,Bt,n),h(s,V,n),e(V,us),e(us,jn),j(Ms,jn,null),e(V,Ar),e(V,dn),e(dn,Pr),h(s,Wt,n),j(Us,s,n),h(s,Jt,n),h(s,D,n),e(D,zr),e(D,he),e(he,Tr),e(D,Cr),e(D,gn),e(gn,Dr),e(D,Sr),h(s,Mt,n),j(cs,s,n),h(s,Ut,n),h(s,S,n),e(S,Lr),e(S,ue),e(ue,Nr),e(S,Or),e(S,_n),e(_n,Ir),e(S,Fr),h(s,Vt,n),h(s,G,n),e(G,ms),e(ms,$n),j(Vs,$n,null),e(G,Hr),e(G,vn),e(vn,Rr),h(s,Gt,n),h(s,is,n),e(is,Br),e(is,ce),e(ce,Wr),e(is,Jr),h(s,Yt,n),j(Gs,s,n),h(s,Kt,n),h(s,me,n),e(me,Mr),h(s,Qt,n),j(Ys,s,n),h(s,Xt,n),h(s,ie,n),e(ie,Ur),h(s,Zt,n),h(s,L,n),e(L,be),e(be,fe),e(fe,Vr),e(be,Gr),e(L,Yr),e(L,je),e(je,de),e(de,Kr),e(je,Qr),e(L,Xr),e(L,ge),e(ge,_e),e(_e,Zr),e(ge,so),h(s,sl,n),h(s,bs,n),e(bs,ao),e(bs,wn),e(wn,eo),e(bs,no),h(s,al,n),j(Ks,s,n),h(s,el,n),h(s,N,n),e(N,to),e(N,kn),e(kn,lo),e(N,po),e(N,yn),e(yn,ro),e(N,oo),h(s,nl,n),h(s,$e,n),e($e,ho),h(s,tl,n),j(Qs,s,n),h(s,ll,n),h(s,Y,n),e(Y,fs),e(fs,xn),j(Xs,xn,null),e(Y,uo),e(Y,En),e(En,co),h(s,pl,n),h(s,js,n),e(js,mo),e(js,qn),e(qn,io),e(js,bo),h(s,rl,n),h(s,O,n),e(O,fo),e(O,An),e(An,jo),e(O,go),e(O,Pn),e(Pn,_o),e(O,$o),h(s,ol,n),j(Zs,s,n),h(s,hl,n),h(s,ds,n),e(ds,vo),e(ds,zn),e(zn,wo),e(ds,ko),h(s,ul,n),h(s,K,n),e(K,gs),e(gs,Tn),j(sa,Tn,null),e(K,yo),e(K,Cn),e(Cn,xo),h(s,cl,n),h(s,ve,n),e(ve,Eo),h(s,ml,n),h(s,I,n),e(I,qo),e(I,Dn),e(Dn,Ao),e(I,Po),e(I,Sn),e(Sn,zo),e(I,To),h(s,il,n),j(aa,s,n),h(s,bl,n),h(s,Q,n),e(Q,_s),e(_s,Ln),j(ea,Ln,null),e(Q,Co),e(Q,Nn),e(Nn,Do),h(s,fl,n),h(s,we,n),e(we,So),h(s,jl,n),h(s,q,n),e(q,Lo),e(q,On),e(On,No),e(q,Oo),e(q,In),e(In,Io),e(q,Fo),e(q,Fn),e(Fn,Ho),e(q,Ro),h(s,dl,n),j($s,s,n),h(s,gl,n),h(s,X,n),e(X,vs),e(vs,Hn),j(na,Hn,null),e(X,Bo),e(X,Rn),e(Rn,Wo),h(s,_l,n),h(s,ws,n),e(ws,Jo),e(ws,ke),e(ke,Mo),e(ws,Uo),h(s,$l,n),j(ta,s,n),h(s,vl,n),h(s,F,n),e(F,Vo),e(F,la),e(la,Go),e(F,Yo),e(F,pa),e(pa,Ko),e(F,Qo),h(s,wl,n),j(ra,s,n),h(s,kl,n),h(s,H,n),e(H,Xo),e(H,Bn),e(Bn,Zo),e(H,sh),e(H,Wn),e(Wn,ah),e(H,eh),h(s,yl,n),j(oa,s,n),h(s,xl,n),h(s,ye,n),e(ye,nh),h(s,El,n),h(s,R,n),e(R,xe),e(xe,Jn),e(Jn,th),e(xe,lh),e(R,ph),e(R,Ee),e(Ee,Mn),e(Mn,rh),e(Ee,oh),e(R,hh),e(R,qe),e(qe,Un),e(Un,uh),e(qe,ch),h(s,ql,n),h(s,Z,n),e(Z,ks),e(ks,Vn),j(ha,Vn,null),e(Z,mh),e(Z,Gn),e(Gn,ih),h(s,Al,n),h(s,ys,n),e(ys,bh),e(ys,ua),e(ua,fh),e(ys,jh),h(s,Pl,n),h(s,xs,n),e(xs,dh),e(xs,ca),e(ca,gh),e(xs,_h),h(s,zl,n),j(ma,s,n),h(s,Tl,n),h(s,Ae,n),e(Ae,ia),e(ia,$h),e(ia,ba),e(ba,Yn),e(Yn,vh),e(ia,wh),h(s,Cl,n),j(fa,s,n),h(s,Dl,n),h(s,ja,n),e(ja,Kn),e(Kn,kh),h(s,Sl,n),j(da,s,n),h(s,Ll,n),h(s,Es,n),e(Es,yh),e(Es,Qn),e(Qn,xh),e(Es,Eh),h(s,Nl,n),h(s,ss,n),e(ss,qs),e(qs,Xn),j(ga,Xn,null),e(ss,qh),e(ss,Zn),e(Zn,Ah),h(s,Ol,n),h(s,A,n),e(A,Ph),e(A,st),e(st,zh),e(A,Th),e(A,at),e(at,Ch),e(A,Dh),e(A,et),e(et,Sh),e(A,Lh),h(s,Il,n),h(s,As,n),e(As,Nh),e(As,Pe),e(Pe,Oh),e(As,Ih),h(s,Fl,n),j(_a,s,n),h(s,Hl,n),h(s,B,n),e(B,Fh),e(B,nt),e(nt,Hh),e(B,Rh),e(B,tt),e(tt,Bh),e(B,Wh),h(s,Rl,n),j($a,s,n),h(s,Bl,n),h(s,as,n),e(as,Ps),e(Ps,lt),j(va,lt,null),e(as,Jh),e(as,pt),e(pt,Mh),h(s,Wl,n),h(s,ze,n),e(ze,Uh),h(s,Jl,n),j(wa,s,n),h(s,Ml,n),h(s,Te,n),e(Te,Vh),h(s,Ul,n),j(ka,s,n),h(s,Vl,n),h(s,Ce,n),e(Ce,Gh),h(s,Gl,n),j(ya,s,n),h(s,Yl,n),h(s,De,n),e(De,Yh),h(s,Kl,n),j(xa,s,n),h(s,Ql,n),h(s,Se,n),e(Se,Kh),h(s,Xl,n),h(s,es,n),e(es,zs),e(zs,rt),j(Ea,rt,null),e(es,Qh),e(es,ot),e(ot,Xh),h(s,Zl,n),h(s,Le,n),e(Le,Zh),h(s,sp,n),h(s,W,n),e(W,su),e(W,qa),e(qa,au),e(W,eu),e(W,ht),e(ht,nu),e(W,tu),h(s,ap,n),j(Aa,s,n),h(s,ep,n),h(s,Ts,n),e(Ts,lu),e(Ts,Pa),e(Pa,ut),e(ut,pu),e(Ts,ru),h(s,np,n),j(za,s,n),h(s,tp,n),h(s,Ne,n),e(Ne,Oe),h(s,lp,n),h(s,ns,n),e(ns,Cs),e(Cs,ct),j(Ta,ct,null),e(ns,ou),e(ns,mt),e(mt,hu),h(s,pp,n),h(s,Ds,n),e(Ds,uu),e(Ds,Ie),e(Ie,cu),e(Ds,mu),h(s,rp,n),j(Ca,s,n),h(s,op,n),h(s,ts,n),e(ts,Ss),e(Ss,it),j(Da,it,null),e(ts,iu),e(ts,bt),e(bt,bu),h(s,hp,n),h(s,Ls,n),e(Ls,fu),e(Ls,Sa),e(Sa,ft),e(ft,ju),e(Ls,du),h(s,up,n),h(s,Fe,n),e(Fe,z),e(z,gu),e(z,La),e(La,jt),e(jt,_u),e(z,$u),e(z,Na),e(Na,dt),e(dt,vu),e(z,wu),e(z,Oa),e(Oa,gt),e(gt,ku),e(z,yu),h(s,cp,n),j(Ia,s,n),h(s,mp,n),h(s,Fa,n),e(Fa,ls),e(ls,xu),e(ls,He),e(He,_t),e(_t,Eu),e(ls,qu),e(ls,$t),e($t,Au),e(ls,Pu),h(s,ip,n),j(Ha,s,n),h(s,bp,n),h(s,Ra,n),e(Ra,Ba),e(Ba,zu),e(Ba,Wa),e(Wa,vt),e(vt,Tu),e(Ba,Cu),h(s,fp,n),j(Ja,s,n),h(s,jp,n),h(s,Ma,n),e(Ma,Ua),e(Ua,Du),e(Ua,wt),e(wt,Su),e(Ua,Lu),h(s,dp,n),j(Va,s,n),h(s,gp,n),h(s,Re,n),e(Re,Nu),h(s,_p,n),j(Ga,s,n),h(s,$p,n),h(s,Be,n),e(Be,We),h(s,vp,n),h(s,ps,n),e(ps,Ns),e(Ns,kt),j(Ya,kt,null),e(ps,Ou),e(ps,yt),e(yt,Iu),h(s,wp,n),h(s,Je,n),e(Je,Fu),h(s,kp,n),h(s,Os,n),e(Os,xt),e(xt,Hu),e(Os,Ru),e(Os,Et),e(Et,Bu),h(s,yp,n),h(s,Is,n),e(Is,Wu),e(Is,Ka),e(Ka,Ju),e(Is,Mu),h(s,xp,n),j(Qa,s,n),h(s,Ep,n),h(s,J,n),e(J,Uu),e(J,qt),e(qt,Vu),e(J,Gu),e(J,At),e(At,Yu),e(J,Ku),h(s,qp,n),j(Xa,s,n),h(s,Ap,n),h(s,M,n),e(M,Qu),e(M,Pt),e(Pt,Xu),e(M,Zu),e(M,zt),e(zt,sc),e(M,ac),h(s,Pp,n),j(Za,s,n),h(s,zp,n),h(s,Fs,n),e(Fs,ec),e(Fs,Me),e(Me,nc),e(Fs,tc),h(s,Tp,n),j(se,s,n),h(s,Cp,n),h(s,rs,n),e(rs,Hs),e(Hs,Tt),j(ae,Tt,null),e(rs,lc),e(rs,Ct),e(Ct,pc),h(s,Dp,n),h(s,Ue,n),e(Ue,rc),h(s,Sp,n),j(ee,s,n),h(s,Lp,n),h(s,Ve,n),e(Ve,os),e(os,oc),e(os,Dt),e(Dt,hc),e(os,uc),e(os,St),e(St,cc),e(os,mc),h(s,Np,n),j(ne,s,n),h(s,Op,n),h(s,te,n),e(te,le),e(le,ic),e(le,Lt),e(Lt,bc),e(le,fc),h(s,Ip,n),j(pe,s,n),h(s,Fp,n),h(s,U,n),e(U,jc),e(U,Nt),e(Nt,dc),e(U,gc),e(U,Ot),e(Ot,_c),e(U,$c),h(s,Hp,n),h(s,Ge,n),e(Ge,vc),Rp=!0},p(s,[n]){const re={};n&2&&(re.$$scope={dirty:n,ctx:s}),cs.$set(re);const It={};n&2&&(It.$$scope={dirty:n,ctx:s}),$s.$set(It)},i(s){Rp||(d(E.$$.fragment,s),d(Js.$$.fragment,s),d(Ms.$$.fragment,s),d(Us.$$.fragment,s),d(cs.$$.fragment,s),d(Vs.$$.fragment,s),d(Gs.$$.fragment,s),d(Ys.$$.fragment,s),d(Ks.$$.fragment,s),d(Qs.$$.fragment,s),d(Xs.$$.fragment,s),d(Zs.$$.fragment,s),d(sa.$$.fragment,s),d(aa.$$.fragment,s),d(ea.$$.fragment,s),d($s.$$.fragment,s),d(na.$$.fragment,s),d(ta.$$.fragment,s),d(ra.$$.fragment,s),d(oa.$$.fragment,s),d(ha.$$.fragment,s),d(ma.$$.fragment,s),d(fa.$$.fragment,s),d(da.$$.fragment,s),d(ga.$$.fragment,s),d(_a.$$.fragment,s),d($a.$$.fragment,s),d(va.$$.fragment,s),d(wa.$$.fragment,s),d(ka.$$.fragment,s),d(ya.$$.fragment,s),d(xa.$$.fragment,s),d(Ea.$$.fragment,s),d(Aa.$$.fragment,s),d(za.$$.fragment,s),d(Ta.$$.fragment,s),d(Ca.$$.fragment,s),d(Da.$$.fragment,s),d(Ia.$$.fragment,s),d(Ha.$$.fragment,s),d(Ja.$$.fragment,s),d(Va.$$.fragment,s),d(Ga.$$.fragment,s),d(Ya.$$.fragment,s),d(Qa.$$.fragment,s),d(Xa.$$.fragment,s),d(Za.$$.fragment,s),d(se.$$.fragment,s),d(ae.$$.fragment,s),d(ee.$$.fragment,s),d(ne.$$.fragment,s),d(pe.$$.fragment,s),Rp=!0)},o(s){g(E.$$.fragment,s),g(Js.$$.fragment,s),g(Ms.$$.fragment,s),g(Us.$$.fragment,s),g(cs.$$.fragment,s),g(Vs.$$.fragment,s),g(Gs.$$.fragment,s),g(Ys.$$.fragment,s),g(Ks.$$.fragment,s),g(Qs.$$.fragment,s),g(Xs.$$.fragment,s),g(Zs.$$.fragment,s),g(sa.$$.fragment,s),g(aa.$$.fragment,s),g(ea.$$.fragment,s),g($s.$$.fragment,s),g(na.$$.fragment,s),g(ta.$$.fragment,s),g(ra.$$.fragment,s),g(oa.$$.fragment,s),g(ha.$$.fragment,s),g(ma.$$.fragment,s),g(fa.$$.fragment,s),g(da.$$.fragment,s),g(ga.$$.fragment,s),g(_a.$$.fragment,s),g($a.$$.fragment,s),g(va.$$.fragment,s),g(wa.$$.fragment,s),g(ka.$$.fragment,s),g(ya.$$.fragment,s),g(xa.$$.fragment,s),g(Ea.$$.fragment,s),g(Aa.$$.fragment,s),g(za.$$.fragment,s),g(Ta.$$.fragment,s),g(Ca.$$.fragment,s),g(Da.$$.fragment,s),g(Ia.$$.fragment,s),g(Ha.$$.fragment,s),g(Ja.$$.fragment,s),g(Va.$$.fragment,s),g(Ga.$$.fragment,s),g(Ya.$$.fragment,s),g(Qa.$$.fragment,s),g(Xa.$$.fragment,s),g(Za.$$.fragment,s),g(se.$$.fragment,s),g(ae.$$.fragment,s),g(ee.$$.fragment,s),g(ne.$$.fragment,s),g(pe.$$.fragment,s),Rp=!1},d(s){a($),s&&a(w),s&&a(i),_(E),s&&a(Ft),_(Js,s),s&&a(Ht),s&&a(oe),s&&a(Rt),s&&a(C),s&&a(Bt),s&&a(V),_(Ms),s&&a(Wt),_(Us,s),s&&a(Jt),s&&a(D),s&&a(Mt),_(cs,s),s&&a(Ut),s&&a(S),s&&a(Vt),s&&a(G),_(Vs),s&&a(Gt),s&&a(is),s&&a(Yt),_(Gs,s),s&&a(Kt),s&&a(me),s&&a(Qt),_(Ys,s),s&&a(Xt),s&&a(ie),s&&a(Zt),s&&a(L),s&&a(sl),s&&a(bs),s&&a(al),_(Ks,s),s&&a(el),s&&a(N),s&&a(nl),s&&a($e),s&&a(tl),_(Qs,s),s&&a(ll),s&&a(Y),_(Xs),s&&a(pl),s&&a(js),s&&a(rl),s&&a(O),s&&a(ol),_(Zs,s),s&&a(hl),s&&a(ds),s&&a(ul),s&&a(K),_(sa),s&&a(cl),s&&a(ve),s&&a(ml),s&&a(I),s&&a(il),_(aa,s),s&&a(bl),s&&a(Q),_(ea),s&&a(fl),s&&a(we),s&&a(jl),s&&a(q),s&&a(dl),_($s,s),s&&a(gl),s&&a(X),_(na),s&&a(_l),s&&a(ws),s&&a($l),_(ta,s),s&&a(vl),s&&a(F),s&&a(wl),_(ra,s),s&&a(kl),s&&a(H),s&&a(yl),_(oa,s),s&&a(xl),s&&a(ye),s&&a(El),s&&a(R),s&&a(ql),s&&a(Z),_(ha),s&&a(Al),s&&a(ys),s&&a(Pl),s&&a(xs),s&&a(zl),_(ma,s),s&&a(Tl),s&&a(Ae),s&&a(Cl),_(fa,s),s&&a(Dl),s&&a(ja),s&&a(Sl),_(da,s),s&&a(Ll),s&&a(Es),s&&a(Nl),s&&a(ss),_(ga),s&&a(Ol),s&&a(A),s&&a(Il),s&&a(As),s&&a(Fl),_(_a,s),s&&a(Hl),s&&a(B),s&&a(Rl),_($a,s),s&&a(Bl),s&&a(as),_(va),s&&a(Wl),s&&a(ze),s&&a(Jl),_(wa,s),s&&a(Ml),s&&a(Te),s&&a(Ul),_(ka,s),s&&a(Vl),s&&a(Ce),s&&a(Gl),_(ya,s),s&&a(Yl),s&&a(De),s&&a(Kl),_(xa,s),s&&a(Ql),s&&a(Se),s&&a(Xl),s&&a(es),_(Ea),s&&a(Zl),s&&a(Le),s&&a(sp),s&&a(W),s&&a(ap),_(Aa,s),s&&a(ep),s&&a(Ts),s&&a(np),_(za,s),s&&a(tp),s&&a(Ne),s&&a(lp),s&&a(ns),_(Ta),s&&a(pp),s&&a(Ds),s&&a(rp),_(Ca,s),s&&a(op),s&&a(ts),_(Da),s&&a(hp),s&&a(Ls),s&&a(up),s&&a(Fe),s&&a(cp),_(Ia,s),s&&a(mp),s&&a(Fa),s&&a(ip),_(Ha,s),s&&a(bp),s&&a(Ra),s&&a(fp),_(Ja,s),s&&a(jp),s&&a(Ma),s&&a(dp),_(Va,s),s&&a(gp),s&&a(Re),s&&a(_p),_(Ga,s),s&&a($p),s&&a(Be),s&&a(vp),s&&a(ps),_(Ya),s&&a(wp),s&&a(Je),s&&a(kp),s&&a(Os),s&&a(yp),s&&a(Is),s&&a(xp),_(Qa,s),s&&a(Ep),s&&a(J),s&&a(qp),_(Xa,s),s&&a(Ap),s&&a(M),s&&a(Pp),_(Za,s),s&&a(zp),s&&a(Fs),s&&a(Tp),_(se,s),s&&a(Cp),s&&a(rs),_(ae),s&&a(Dp),s&&a(Ue),s&&a(Sp),_(ee,s),s&&a(Lp),s&&a(Ve),s&&a(Np),_(ne,s),s&&a(Op),s&&a(te),s&&a(Ip),_(pe,s),s&&a(Fp),s&&a(U),s&&a(Hp),s&&a(Ge)}}}const Cb={local:"preprocess",sections:[{local:"nlp",sections:[{local:"tokenize",title:"Tokenize"},{local:"pad",title:"Pad"},{local:"truncation",title:"Truncation"},{local:"build-tensors",title:"Build tensors"}],title:"NLP"},{local:"audio",sections:[{local:"resample",title:"Resample"},{local:"feature-extractor",title:"Feature extractor"},{local:"pad-and-truncate",title:"Pad and truncate"}],title:"Audio"},{local:"vision",sections:[{local:"feature-extractor",title:"Feature extractor"},{local:"data-augmentation",title:"Data augmentation"}],title:"Vision"},{local:"multimodal",sections:[{local:"processor",title:"Processor"}],title:"Multimodal"}],title:"Preprocess"};function Db(P){return vb(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class Rb extends db{constructor($){super();gb(this,$,Db,Tb,_b,{})}}export{Rb as default,Cb as metadata};
