import{S as ti,i as li,s as pi,e as p,k as c,w as f,t,M as ri,c as r,d as a,m as h,a as o,x as j,h as l,b as m,N as ai,G as e,g as u,y as d,q as g,o as _,B as $,v as oi,L as ei}from"../chunks/vendor-hf-doc-builder.js";import{T as Lp}from"../chunks/Tip-hf-doc-builder.js";import{Y as ui}from"../chunks/Youtube-hf-doc-builder.js";import{I as Is}from"../chunks/IconCopyLink-hf-doc-builder.js";import{C as w}from"../chunks/CodeBlock-hf-doc-builder.js";import{D as ci}from"../chunks/DocNotebookDropdown-hf-doc-builder.js";import{F as hi,M as ni}from"../chunks/Markdown-hf-doc-builder.js";function mi(A){let i,v,b,y,k,x,E;return{c(){i=p("p"),v=p("code"),b=t("AutoProcessor"),y=c(),k=p("strong"),x=t("always"),E=t(" works and automatically chooses the correct class for the model you\u2019re using, whether you\u2019re using a tokenizer, feature extractor or processor.")},l(q){i=r(q,"P",{});var D=o(i);v=r(D,"CODE",{});var Ss=o(v);b=l(Ss,"AutoProcessor"),Ss.forEach(a),y=h(D),k=r(D,"STRONG",{});var L=o(k);x=l(L,"always"),L.forEach(a),E=l(D," works and automatically chooses the correct class for the model you\u2019re using, whether you\u2019re using a tokenizer, feature extractor or processor."),D.forEach(a)},m(q,D){u(q,i,D),e(i,v),e(v,b),e(i,y),e(i,k),e(k,x),e(i,E)},d(q){q&&a(i)}}}function ii(A){let i,v,b,y,k;return{c(){i=p("p"),v=t("If you plan on using a pretrained model, it\u2019s important to use the associated pretrained tokenizer. This ensures the text is split the same way as the pretraining corpus, and uses the same corresponding tokens-to-index (usually referrred to as the "),b=p("em"),y=t("vocab"),k=t(") during pretraining.")},l(x){i=r(x,"P",{});var E=o(i);v=l(E,"If you plan on using a pretrained model, it\u2019s important to use the associated pretrained tokenizer. This ensures the text is split the same way as the pretraining corpus, and uses the same corresponding tokens-to-index (usually referrred to as the "),b=r(E,"EM",{});var q=o(b);y=l(q,"vocab"),q.forEach(a),k=l(E,") during pretraining."),E.forEach(a)},m(x,E){u(x,i,E),e(i,v),e(i,b),e(b,y),e(i,k)},d(x){x&&a(i)}}}function bi(A){let i,v,b,y,k;return{c(){i=p("p"),v=t("Check out the "),b=p("a"),y=t("Padding and truncation"),k=t(" concept guide to learn more different padding and truncation arguments."),this.h()},l(x){i=r(x,"P",{});var E=o(i);v=l(E,"Check out the "),b=r(E,"A",{href:!0});var q=o(b);y=l(q,"Padding and truncation"),q.forEach(a),k=l(E," concept guide to learn more different padding and truncation arguments."),E.forEach(a),this.h()},h(){m(b,"href","./pad_truncation")},m(x,E){u(x,i,E),e(i,v),e(i,b),e(b,y),e(i,k)},d(x){x&&a(i)}}}function fi(A){let i,v;return i=new w({props:{code:`batch_sentences = [
    "But what about second breakfast?",
    "Don't think he knows about second breakfast, Pip.",
    "What about elevensies?",
]
encoded_input = tokenizer(batch_sentences, padding=True, truncation=True, return_tensors="pt")
print(encoded_input)`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>batch_sentences = [
<span class="hljs-meta">... </span>    <span class="hljs-string">&quot;But what about second breakfast?&quot;</span>,
<span class="hljs-meta">... </span>    <span class="hljs-string">&quot;Don&#x27;t think he knows about second breakfast, Pip.&quot;</span>,
<span class="hljs-meta">... </span>    <span class="hljs-string">&quot;What about elevensies?&quot;</span>,
<span class="hljs-meta">... </span>]
<span class="hljs-meta">&gt;&gt;&gt; </span>encoded_input = tokenizer(batch_sentences, padding=<span class="hljs-literal">True</span>, truncation=<span class="hljs-literal">True</span>, return_tensors=<span class="hljs-string">&quot;pt&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-built_in">print</span>(encoded_input)
{<span class="hljs-string">&#x27;input_ids&#x27;</span>: tensor([[<span class="hljs-number">101</span>, <span class="hljs-number">1252</span>, <span class="hljs-number">1184</span>, <span class="hljs-number">1164</span>, <span class="hljs-number">1248</span>, <span class="hljs-number">6462</span>, <span class="hljs-number">136</span>, <span class="hljs-number">102</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>],
                      [<span class="hljs-number">101</span>, <span class="hljs-number">1790</span>, <span class="hljs-number">112</span>, <span class="hljs-number">189</span>, <span class="hljs-number">1341</span>, <span class="hljs-number">1119</span>, <span class="hljs-number">3520</span>, <span class="hljs-number">1164</span>, <span class="hljs-number">1248</span>, <span class="hljs-number">6462</span>, <span class="hljs-number">117</span>, <span class="hljs-number">21902</span>, <span class="hljs-number">1643</span>, <span class="hljs-number">119</span>, <span class="hljs-number">102</span>],
                      [<span class="hljs-number">101</span>, <span class="hljs-number">1327</span>, <span class="hljs-number">1164</span>, <span class="hljs-number">5450</span>, <span class="hljs-number">23434</span>, <span class="hljs-number">136</span>, <span class="hljs-number">102</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>]]), 
 <span class="hljs-string">&#x27;token_type_ids&#x27;</span>: tensor([[<span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>],
                           [<span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>],
                           [<span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>]]), 
 <span class="hljs-string">&#x27;attention_mask&#x27;</span>: tensor([[<span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>],
                           [<span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>],
                           [<span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>]])}`}}),{c(){f(i.$$.fragment)},l(b){j(i.$$.fragment,b)},m(b,y){d(i,b,y),v=!0},p:ei,i(b){v||(g(i.$$.fragment,b),v=!0)},o(b){_(i.$$.fragment,b),v=!1},d(b){$(i,b)}}}function ji(A){let i,v;return i=new ni({props:{$$slots:{default:[fi]},$$scope:{ctx:A}}}),{c(){f(i.$$.fragment)},l(b){j(i.$$.fragment,b)},m(b,y){d(i,b,y),v=!0},p(b,y){const k={};y&2&&(k.$$scope={dirty:y,ctx:b}),i.$set(k)},i(b){v||(g(i.$$.fragment,b),v=!0)},o(b){_(i.$$.fragment,b),v=!1},d(b){$(i,b)}}}function di(A){let i,v;return i=new w({props:{code:`batch_sentences = [
    "But what about second breakfast?",
    "Don't think he knows about second breakfast, Pip.",
    "What about elevensies?",
]
encoded_input = tokenizer(batch_sentences, padding=True, truncation=True, return_tensors="tf")
print(encoded_input)`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>batch_sentences = [
<span class="hljs-meta">... </span>    <span class="hljs-string">&quot;But what about second breakfast?&quot;</span>,
<span class="hljs-meta">... </span>    <span class="hljs-string">&quot;Don&#x27;t think he knows about second breakfast, Pip.&quot;</span>,
<span class="hljs-meta">... </span>    <span class="hljs-string">&quot;What about elevensies?&quot;</span>,
<span class="hljs-meta">... </span>]
<span class="hljs-meta">&gt;&gt;&gt; </span>encoded_input = tokenizer(batch_sentences, padding=<span class="hljs-literal">True</span>, truncation=<span class="hljs-literal">True</span>, return_tensors=<span class="hljs-string">&quot;tf&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-built_in">print</span>(encoded_input)
{<span class="hljs-string">&#x27;input_ids&#x27;</span>: &lt;tf.Tensor: shape=(<span class="hljs-number">2</span>, <span class="hljs-number">9</span>), dtype=int32, numpy=
array([[<span class="hljs-number">101</span>, <span class="hljs-number">1252</span>, <span class="hljs-number">1184</span>, <span class="hljs-number">1164</span>, <span class="hljs-number">1248</span>, <span class="hljs-number">6462</span>, <span class="hljs-number">136</span>, <span class="hljs-number">102</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>],
       [<span class="hljs-number">101</span>, <span class="hljs-number">1790</span>, <span class="hljs-number">112</span>, <span class="hljs-number">189</span>, <span class="hljs-number">1341</span>, <span class="hljs-number">1119</span>, <span class="hljs-number">3520</span>, <span class="hljs-number">1164</span>, <span class="hljs-number">1248</span>, <span class="hljs-number">6462</span>, <span class="hljs-number">117</span>, <span class="hljs-number">21902</span>, <span class="hljs-number">1643</span>, <span class="hljs-number">119</span>, <span class="hljs-number">102</span>],
       [<span class="hljs-number">101</span>, <span class="hljs-number">1327</span>, <span class="hljs-number">1164</span>, <span class="hljs-number">5450</span>, <span class="hljs-number">23434</span>, <span class="hljs-number">136</span>, <span class="hljs-number">102</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>]],
      dtype=int32)&gt;, 
 <span class="hljs-string">&#x27;token_type_ids&#x27;</span>: &lt;tf.Tensor: shape=(<span class="hljs-number">2</span>, <span class="hljs-number">9</span>), dtype=int32, numpy=
array([[<span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>],
       [<span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>],
       [<span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>]], dtype=int32)&gt;, 
 <span class="hljs-string">&#x27;attention_mask&#x27;</span>: &lt;tf.Tensor: shape=(<span class="hljs-number">2</span>, <span class="hljs-number">9</span>), dtype=int32, numpy=
array([[<span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>],
       [<span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>],
       [<span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>]], dtype=int32)&gt;}`}}),{c(){f(i.$$.fragment)},l(b){j(i.$$.fragment,b)},m(b,y){d(i,b,y),v=!0},p:ei,i(b){v||(g(i.$$.fragment,b),v=!0)},o(b){_(i.$$.fragment,b),v=!1},d(b){$(i,b)}}}function gi(A){let i,v;return i=new ni({props:{$$slots:{default:[di]},$$scope:{ctx:A}}}),{c(){f(i.$$.fragment)},l(b){j(i.$$.fragment,b)},m(b,y){d(i,b,y),v=!0},p(b,y){const k={};y&2&&(k.$$scope={dirty:y,ctx:b}),i.$set(k)},i(b){v||(g(i.$$.fragment,b),v=!0)},o(b){_(i.$$.fragment,b),v=!1},d(b){$(i,b)}}}function _i(A){let i,v,b,y,k;return{c(){i=p("p"),v=t("Use \u{1F917} Datasets "),b=p("code"),y=t("split"),k=t(" parameter to only load a small sample from the training split since the dataset is quite large!")},l(x){i=r(x,"P",{});var E=o(i);v=l(E,"Use \u{1F917} Datasets "),b=r(E,"CODE",{});var q=o(b);y=l(q,"split"),q.forEach(a),k=l(E," parameter to only load a small sample from the training split since the dataset is quite large!"),E.forEach(a)},m(x,E){u(x,i,E),e(i,v),e(i,b),e(b,y),e(i,k)},d(x){x&&a(i)}}}function $i(A){let i,v,b,y,k,x,E,q,D,Ss,L,ft,ee,Ip,jt,I,Ns,Sp,ne,Np,Fp,Rp,Fs,Bp,te,Mp,Wp,Jp,Rs,Hp,le,Up,Yp,dt,rs,gt,pe,Gp,_t,Bs,$t,Z,os,en,Ms,Vp,nn,Kp,vt,Ws,yt,S,Qp,re,Xp,Zp,tn,sr,ar,wt,us,kt,N,er,oe,nr,tr,ln,lr,pr,xt,Js,Et,ue,rr,qt,Hs,At,ce,or,Pt,F,he,me,ur,cr,hr,ie,be,mr,ir,br,fe,je,fr,jr,Tt,cs,dr,pn,gr,_r,Ct,Us,zt,R,$r,rn,vr,yr,on,wr,kr,Dt,de,xr,Ot,Ys,Lt,ss,hs,un,Gs,Er,cn,qr,It,ms,Ar,hn,Pr,Tr,St,B,Cr,mn,zr,Dr,bn,Or,Lr,Nt,Vs,Ft,is,Ir,fn,Sr,Nr,Rt,as,bs,jn,Ks,Fr,dn,Rr,Bt,ge,Br,Mt,M,Mr,gn,Wr,Jr,_n,Hr,Ur,Wt,Qs,Jt,fs,Ht,es,js,$n,Xs,Yr,vn,Gr,Ut,_e,Vr,Yt,T,Kr,yn,Qr,Xr,wn,Zr,so,kn,ao,eo,Gt,ds,Vt,ns,gs,xn,Zs,no,En,to,Kt,_s,lo,$e,po,ro,Qt,W,oo,sa,uo,co,aa,ho,mo,Xt,ea,Zt,J,io,qn,bo,fo,An,jo,go,sl,na,al,ve,_o,el,H,ye,Pn,$o,vo,yo,we,Tn,wo,ko,xo,ke,Cn,Eo,qo,nl,$s,Ao,ta,Po,To,tl,xe,la,Co,pa,zo,Do,ll,ra,pl,oa,ua,Oo,zn,Lo,Io,rl,ca,ol,C,So,Dn,No,Fo,On,Ro,Bo,Ln,Mo,Wo,ul,vs,Jo,Ee,Ho,Uo,cl,ha,hl,U,Yo,In,Go,Vo,Sn,Ko,Qo,ml,ma,il,qe,Xo,bl,ia,fl,Ae,Zo,jl,ba,dl,ys,su,Nn,au,eu,gl,fa,_l,Pe,nu,$l,ja,vl,ts,ws,Fn,da,tu,Rn,lu,yl,ks,pu,Te,ru,ou,wl,Y,uu,ga,cu,hu,_a,mu,iu,kl,xs,xl,$a,El,Es,bu,va,Bn,fu,ju,ql,ya,Al,wa,Mn,Bc,Pl,qs,du,Ce,gu,_u,Tl,ka,Cl,z,$u,xa,Wn,vu,yu,Ea,wu,ku,qa,xu,Eu,zl,ze,O,qu,Aa,Jn,Au,Pu,Pa,Hn,Tu,Cu,Ta,Un,zu,Du,Dl,Ca,Ol,za,ls,Ou,De,Yn,Lu,Iu,Gn,Su,Nu,Ll,Da,Il,Oa,La,Fu,Ia,Vn,Ru,Bu,Sl,Sa,Nl,Na,Fa,Mu,Kn,Wu,Ju,Fl,Ra,Rl,Oe,Hu,Bl,Ba,Ml,Ma,Qn,Mc,Wl,ps,As,Xn,Wa,Uu,Zn,Yu,Jl,Ps,Gu,Le,Vu,Ku,Hl,G,Qu,Ja,Xu,Zu,Ha,sc,ac,Ul,Ua,Yl,V,ec,st,nc,tc,at,lc,pc,Gl,Ya,Vl,K,rc,et,oc,uc,nt,cc,hc,Kl,Ga,Ql,Ts,mc,Ie,ic,bc,Xl,Va,Zl,Cs,fc,Se,jc,dc,sp,Ka,ap,Ne,P,gc,tt,_c,$c,lt,vc,yc,pt,wc,kc,rt,xc,Ec,ep,Qa,np,Xa,Za,qc,ot,Ac,Pc,tp,se,lp,Q,Tc,ut,Cc,zc,ct,Dc,Oc,pp;return x=new Is({}),L=new ci({props:{classNames:"absolute z-10 right-0 top-0",options:[{label:"Mixed",value:"https://colab.research.google.com/github/huggingface/notebooks/blob/main/transformers_doc/en/preprocessing.ipynb"},{label:"PyTorch",value:"https://colab.research.google.com/github/huggingface/notebooks/blob/main/transformers_doc/en/pytorch/preprocessing.ipynb"},{label:"TensorFlow",value:"https://colab.research.google.com/github/huggingface/notebooks/blob/main/transformers_doc/en/tensorflow/preprocessing.ipynb"},{label:"Mixed",value:"https://studiolab.sagemaker.aws/import/github/huggingface/notebooks/blob/main/transformers_doc/en/preprocessing.ipynb"},{label:"PyTorch",value:"https://studiolab.sagemaker.aws/import/github/huggingface/notebooks/blob/main/transformers_doc/en/pytorch/preprocessing.ipynb"},{label:"TensorFlow",value:"https://studiolab.sagemaker.aws/import/github/huggingface/notebooks/blob/main/transformers_doc/en/tensorflow/preprocessing.ipynb"}]}}),rs=new Lp({props:{$$slots:{default:[mi]},$$scope:{ctx:A}}}),Bs=new w({props:{code:"pip install datasets",highlighted:"pip install datasets"}}),Ms=new Is({}),Ws=new ui({props:{id:"Yffk5aydLzg"}}),us=new Lp({props:{$$slots:{default:[ii]},$$scope:{ctx:A}}}),Js=new w({props:{code:`from transformers import AutoTokenizer

tokenizer = AutoTokenizer.from_pretrained("bert-base-cased")`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoTokenizer

<span class="hljs-meta">&gt;&gt;&gt; </span>tokenizer = AutoTokenizer.from_pretrained(<span class="hljs-string">&quot;bert-base-cased&quot;</span>)`}}),Hs=new w({props:{code:`encoded_input = tokenizer("Do not meddle in the affairs of wizards, for they are subtle and quick to anger.")
print(encoded_input)`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>encoded_input = tokenizer(<span class="hljs-string">&quot;Do not meddle in the affairs of wizards, for they are subtle and quick to anger.&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-built_in">print</span>(encoded_input)
{<span class="hljs-string">&#x27;input_ids&#x27;</span>: [<span class="hljs-number">101</span>, <span class="hljs-number">2079</span>, <span class="hljs-number">2025</span>, <span class="hljs-number">19960</span>, <span class="hljs-number">10362</span>, <span class="hljs-number">1999</span>, <span class="hljs-number">1996</span>, <span class="hljs-number">3821</span>, <span class="hljs-number">1997</span>, <span class="hljs-number">16657</span>, <span class="hljs-number">1010</span>, <span class="hljs-number">2005</span>, <span class="hljs-number">2027</span>, <span class="hljs-number">2024</span>, <span class="hljs-number">11259</span>, <span class="hljs-number">1998</span>, <span class="hljs-number">4248</span>, <span class="hljs-number">2000</span>, <span class="hljs-number">4963</span>, <span class="hljs-number">1012</span>, <span class="hljs-number">102</span>], 
 <span class="hljs-string">&#x27;token_type_ids&#x27;</span>: [<span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>], 
 <span class="hljs-string">&#x27;attention_mask&#x27;</span>: [<span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>]}`}}),Us=new w({props:{code:'tokenizer.decode(encoded_input["input_ids"])',highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>tokenizer.decode(encoded_input[<span class="hljs-string">&quot;input_ids&quot;</span>])
<span class="hljs-string">&#x27;[CLS] Do not meddle in the affairs of wizards, for they are subtle and quick to anger. [SEP]&#x27;</span>`}}),Ys=new w({props:{code:`batch_sentences = [
    "But what about second breakfast?",
    "Don't think he knows about second breakfast, Pip.",
    "What about elevensies?",
]
encoded_inputs = tokenizer(batch_sentences)
print(encoded_inputs)`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>batch_sentences = [
<span class="hljs-meta">... </span>    <span class="hljs-string">&quot;But what about second breakfast?&quot;</span>,
<span class="hljs-meta">... </span>    <span class="hljs-string">&quot;Don&#x27;t think he knows about second breakfast, Pip.&quot;</span>,
<span class="hljs-meta">... </span>    <span class="hljs-string">&quot;What about elevensies?&quot;</span>,
<span class="hljs-meta">... </span>]
<span class="hljs-meta">&gt;&gt;&gt; </span>encoded_inputs = tokenizer(batch_sentences)
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-built_in">print</span>(encoded_inputs)
{<span class="hljs-string">&#x27;input_ids&#x27;</span>: [[<span class="hljs-number">101</span>, <span class="hljs-number">1252</span>, <span class="hljs-number">1184</span>, <span class="hljs-number">1164</span>, <span class="hljs-number">1248</span>, <span class="hljs-number">6462</span>, <span class="hljs-number">136</span>, <span class="hljs-number">102</span>], 
               [<span class="hljs-number">101</span>, <span class="hljs-number">1790</span>, <span class="hljs-number">112</span>, <span class="hljs-number">189</span>, <span class="hljs-number">1341</span>, <span class="hljs-number">1119</span>, <span class="hljs-number">3520</span>, <span class="hljs-number">1164</span>, <span class="hljs-number">1248</span>, <span class="hljs-number">6462</span>, <span class="hljs-number">117</span>, <span class="hljs-number">21902</span>, <span class="hljs-number">1643</span>, <span class="hljs-number">119</span>, <span class="hljs-number">102</span>], 
               [<span class="hljs-number">101</span>, <span class="hljs-number">1327</span>, <span class="hljs-number">1164</span>, <span class="hljs-number">5450</span>, <span class="hljs-number">23434</span>, <span class="hljs-number">136</span>, <span class="hljs-number">102</span>]], 
 <span class="hljs-string">&#x27;token_type_ids&#x27;</span>: [[<span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>], 
                    [<span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>], 
                    [<span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>]], 
 <span class="hljs-string">&#x27;attention_mask&#x27;</span>: [[<span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>], 
                    [<span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>], 
                    [<span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>]]}`}}),Gs=new Is({}),Vs=new w({props:{code:`batch_sentences = [
    "But what about second breakfast?",
    "Don't think he knows about second breakfast, Pip.",
    "What about elevensies?",
]
encoded_input = tokenizer(batch_sentences, padding=True)
print(encoded_input)`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>batch_sentences = [
<span class="hljs-meta">... </span>    <span class="hljs-string">&quot;But what about second breakfast?&quot;</span>,
<span class="hljs-meta">... </span>    <span class="hljs-string">&quot;Don&#x27;t think he knows about second breakfast, Pip.&quot;</span>,
<span class="hljs-meta">... </span>    <span class="hljs-string">&quot;What about elevensies?&quot;</span>,
<span class="hljs-meta">... </span>]
<span class="hljs-meta">&gt;&gt;&gt; </span>encoded_input = tokenizer(batch_sentences, padding=<span class="hljs-literal">True</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-built_in">print</span>(encoded_input)
{<span class="hljs-string">&#x27;input_ids&#x27;</span>: [[<span class="hljs-number">101</span>, <span class="hljs-number">1252</span>, <span class="hljs-number">1184</span>, <span class="hljs-number">1164</span>, <span class="hljs-number">1248</span>, <span class="hljs-number">6462</span>, <span class="hljs-number">136</span>, <span class="hljs-number">102</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>], 
               [<span class="hljs-number">101</span>, <span class="hljs-number">1790</span>, <span class="hljs-number">112</span>, <span class="hljs-number">189</span>, <span class="hljs-number">1341</span>, <span class="hljs-number">1119</span>, <span class="hljs-number">3520</span>, <span class="hljs-number">1164</span>, <span class="hljs-number">1248</span>, <span class="hljs-number">6462</span>, <span class="hljs-number">117</span>, <span class="hljs-number">21902</span>, <span class="hljs-number">1643</span>, <span class="hljs-number">119</span>, <span class="hljs-number">102</span>], 
               [<span class="hljs-number">101</span>, <span class="hljs-number">1327</span>, <span class="hljs-number">1164</span>, <span class="hljs-number">5450</span>, <span class="hljs-number">23434</span>, <span class="hljs-number">136</span>, <span class="hljs-number">102</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>]], 
 <span class="hljs-string">&#x27;token_type_ids&#x27;</span>: [[<span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>], 
                    [<span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>], 
                    [<span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>]], 
 <span class="hljs-string">&#x27;attention_mask&#x27;</span>: [[<span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>], 
                    [<span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>], 
                    [<span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>]]}`}}),Ks=new Is({}),Qs=new w({props:{code:`batch_sentences = [
    "But what about second breakfast?",
    "Don't think he knows about second breakfast, Pip.",
    "What about elevensies?",
]
encoded_input = tokenizer(batch_sentences, padding=True, truncation=True)
print(encoded_input)`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>batch_sentences = [
<span class="hljs-meta">... </span>    <span class="hljs-string">&quot;But what about second breakfast?&quot;</span>,
<span class="hljs-meta">... </span>    <span class="hljs-string">&quot;Don&#x27;t think he knows about second breakfast, Pip.&quot;</span>,
<span class="hljs-meta">... </span>    <span class="hljs-string">&quot;What about elevensies?&quot;</span>,
<span class="hljs-meta">... </span>]
<span class="hljs-meta">&gt;&gt;&gt; </span>encoded_input = tokenizer(batch_sentences, padding=<span class="hljs-literal">True</span>, truncation=<span class="hljs-literal">True</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-built_in">print</span>(encoded_input)
{<span class="hljs-string">&#x27;input_ids&#x27;</span>: [[<span class="hljs-number">101</span>, <span class="hljs-number">1252</span>, <span class="hljs-number">1184</span>, <span class="hljs-number">1164</span>, <span class="hljs-number">1248</span>, <span class="hljs-number">6462</span>, <span class="hljs-number">136</span>, <span class="hljs-number">102</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>], 
               [<span class="hljs-number">101</span>, <span class="hljs-number">1790</span>, <span class="hljs-number">112</span>, <span class="hljs-number">189</span>, <span class="hljs-number">1341</span>, <span class="hljs-number">1119</span>, <span class="hljs-number">3520</span>, <span class="hljs-number">1164</span>, <span class="hljs-number">1248</span>, <span class="hljs-number">6462</span>, <span class="hljs-number">117</span>, <span class="hljs-number">21902</span>, <span class="hljs-number">1643</span>, <span class="hljs-number">119</span>, <span class="hljs-number">102</span>], 
               [<span class="hljs-number">101</span>, <span class="hljs-number">1327</span>, <span class="hljs-number">1164</span>, <span class="hljs-number">5450</span>, <span class="hljs-number">23434</span>, <span class="hljs-number">136</span>, <span class="hljs-number">102</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>]], 
 <span class="hljs-string">&#x27;token_type_ids&#x27;</span>: [[<span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>], 
                    [<span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>], 
                    [<span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>]], 
 <span class="hljs-string">&#x27;attention_mask&#x27;</span>: [[<span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>], 
                    [<span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>], 
                    [<span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>]]}`}}),fs=new Lp({props:{$$slots:{default:[bi]},$$scope:{ctx:A}}}),Xs=new Is({}),ds=new hi({props:{pytorch:!0,tensorflow:!0,jax:!1,$$slots:{tensorflow:[gi],pytorch:[ji]},$$scope:{ctx:A}}}),Zs=new Is({}),ea=new w({props:{code:`from datasets import load_dataset, Audio

dataset = load_dataset("PolyAI/minds14", name="en-US", split="train")`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> datasets <span class="hljs-keyword">import</span> load_dataset, Audio

<span class="hljs-meta">&gt;&gt;&gt; </span>dataset = load_dataset(<span class="hljs-string">&quot;PolyAI/minds14&quot;</span>, name=<span class="hljs-string">&quot;en-US&quot;</span>, split=<span class="hljs-string">&quot;train&quot;</span>)`}}),na=new w({props:{code:'dataset[0]["audio"]',highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>dataset[<span class="hljs-number">0</span>][<span class="hljs-string">&quot;audio&quot;</span>]
{<span class="hljs-string">&#x27;array&#x27;</span>: array([ <span class="hljs-number">0.</span>        ,  <span class="hljs-number">0.00024414</span>, -<span class="hljs-number">0.00024414</span>, ..., -<span class="hljs-number">0.00024414</span>,
         <span class="hljs-number">0.</span>        ,  <span class="hljs-number">0.</span>        ], dtype=float32),
 <span class="hljs-string">&#x27;path&#x27;</span>: <span class="hljs-string">&#x27;/root/.cache/huggingface/datasets/downloads/extracted/f14948e0e84be638dd7943ac36518a4cf3324e8b7aa331c5ab11541518e9368c/en-US~JOINT_ACCOUNT/602ba55abb1e6d0fbce92065.wav&#x27;</span>,
 <span class="hljs-string">&#x27;sampling_rate&#x27;</span>: <span class="hljs-number">8000</span>}`}}),ra=new w({props:{code:'dataset = dataset.cast_column("audio", Audio(sampling_rate=16_000))',highlighted:'<span class="hljs-meta">&gt;&gt;&gt; </span>dataset = dataset.cast_column(<span class="hljs-string">&quot;audio&quot;</span>, Audio(sampling_rate=<span class="hljs-number">16_000</span>))'}}),ca=new w({props:{code:'dataset[0]["audio"]',highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>dataset[<span class="hljs-number">0</span>][<span class="hljs-string">&quot;audio&quot;</span>]
{<span class="hljs-string">&#x27;array&#x27;</span>: array([ <span class="hljs-number">2.3443763e-05</span>,  <span class="hljs-number">2.1729663e-04</span>,  <span class="hljs-number">2.2145823e-04</span>, ...,
         <span class="hljs-number">3.8356509e-05</span>, -<span class="hljs-number">7.3497440e-06</span>, -<span class="hljs-number">2.1754686e-05</span>], dtype=float32),
 <span class="hljs-string">&#x27;path&#x27;</span>: <span class="hljs-string">&#x27;/root/.cache/huggingface/datasets/downloads/extracted/f14948e0e84be638dd7943ac36518a4cf3324e8b7aa331c5ab11541518e9368c/en-US~JOINT_ACCOUNT/602ba55abb1e6d0fbce92065.wav&#x27;</span>,
 <span class="hljs-string">&#x27;sampling_rate&#x27;</span>: <span class="hljs-number">16000</span>}`}}),ha=new w({props:{code:`from transformers import AutoFeatureExtractor

feature_extractor = AutoFeatureExtractor.from_pretrained("facebook/wav2vec2-base")`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoFeatureExtractor

<span class="hljs-meta">&gt;&gt;&gt; </span>feature_extractor = AutoFeatureExtractor.from_pretrained(<span class="hljs-string">&quot;facebook/wav2vec2-base&quot;</span>)`}}),ma=new w({props:{code:`audio_input = [dataset[0]["audio"]["array"]]
feature_extractor(audio_input, sampling_rate=16000)`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>audio_input = [dataset[<span class="hljs-number">0</span>][<span class="hljs-string">&quot;audio&quot;</span>][<span class="hljs-string">&quot;array&quot;</span>]]
<span class="hljs-meta">&gt;&gt;&gt; </span>feature_extractor(audio_input, sampling_rate=<span class="hljs-number">16000</span>)
{<span class="hljs-string">&#x27;input_values&#x27;</span>: [array([ <span class="hljs-number">3.8106556e-04</span>,  <span class="hljs-number">2.7506407e-03</span>,  <span class="hljs-number">2.8015103e-03</span>, ...,
        <span class="hljs-number">5.6335266e-04</span>,  <span class="hljs-number">4.6588284e-06</span>, -<span class="hljs-number">1.7142107e-04</span>], dtype=float32)]}`}}),ia=new w({props:{code:`dataset[0]["audio"]["array"].shape

dataset[1]["audio"]["array"].shape`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>dataset[<span class="hljs-number">0</span>][<span class="hljs-string">&quot;audio&quot;</span>][<span class="hljs-string">&quot;array&quot;</span>].shape
(<span class="hljs-number">173398</span>,)

<span class="hljs-meta">&gt;&gt;&gt; </span>dataset[<span class="hljs-number">1</span>][<span class="hljs-string">&quot;audio&quot;</span>][<span class="hljs-string">&quot;array&quot;</span>].shape
(<span class="hljs-number">106496</span>,)`}}),ba=new w({props:{code:`def preprocess_function(examples):
    audio_arrays = [x["array"] for x in examples["audio"]]
    inputs = feature_extractor(
        audio_arrays,
        sampling_rate=16000,
        padding=True,
        max_length=100000,
        truncation=True,
    )
    return inputs`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">def</span> <span class="hljs-title function_">preprocess_function</span>(<span class="hljs-params">examples</span>):
<span class="hljs-meta">... </span>    audio_arrays = [x[<span class="hljs-string">&quot;array&quot;</span>] <span class="hljs-keyword">for</span> x <span class="hljs-keyword">in</span> examples[<span class="hljs-string">&quot;audio&quot;</span>]]
<span class="hljs-meta">... </span>    inputs = feature_extractor(
<span class="hljs-meta">... </span>        audio_arrays,
<span class="hljs-meta">... </span>        sampling_rate=<span class="hljs-number">16000</span>,
<span class="hljs-meta">... </span>        padding=<span class="hljs-literal">True</span>,
<span class="hljs-meta">... </span>        max_length=<span class="hljs-number">100000</span>,
<span class="hljs-meta">... </span>        truncation=<span class="hljs-literal">True</span>,
<span class="hljs-meta">... </span>    )
<span class="hljs-meta">... </span>    <span class="hljs-keyword">return</span> inputs`}}),fa=new w({props:{code:"processed_dataset = preprocess_function(dataset[:5])",highlighted:'<span class="hljs-meta">&gt;&gt;&gt; </span>processed_dataset = preprocess_function(dataset[:<span class="hljs-number">5</span>])'}}),ja=new w({props:{code:`processed_dataset["input_values"][0].shape

processed_dataset["input_values"][1].shape`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>processed_dataset[<span class="hljs-string">&quot;input_values&quot;</span>][<span class="hljs-number">0</span>].shape
(<span class="hljs-number">100000</span>,)

<span class="hljs-meta">&gt;&gt;&gt; </span>processed_dataset[<span class="hljs-string">&quot;input_values&quot;</span>][<span class="hljs-number">1</span>].shape
(<span class="hljs-number">100000</span>,)`}}),da=new Is({}),xs=new Lp({props:{$$slots:{default:[_i]},$$scope:{ctx:A}}}),$a=new w({props:{code:`from datasets import load_dataset

dataset = load_dataset("food101", split="train[:100]")`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> datasets <span class="hljs-keyword">import</span> load_dataset

<span class="hljs-meta">&gt;&gt;&gt; </span>dataset = load_dataset(<span class="hljs-string">&quot;food101&quot;</span>, split=<span class="hljs-string">&quot;train[:100]&quot;</span>)`}}),ya=new w({props:{code:'dataset[0]["image"]',highlighted:'<span class="hljs-meta">&gt;&gt;&gt; </span>dataset[<span class="hljs-number">0</span>][<span class="hljs-string">&quot;image&quot;</span>]'}}),ka=new w({props:{code:`from transformers import AutoFeatureExtractor

feature_extractor = AutoFeatureExtractor.from_pretrained("google/vit-base-patch16-224")`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoFeatureExtractor

<span class="hljs-meta">&gt;&gt;&gt; </span>feature_extractor = AutoFeatureExtractor.from_pretrained(<span class="hljs-string">&quot;google/vit-base-patch16-224&quot;</span>)`}}),Ca=new w({props:{code:`from torchvision.transforms import Compose, Normalize, RandomResizedCrop, ColorJitter, ToTensor

normalize = Normalize(mean=feature_extractor.image_mean, std=feature_extractor.image_std)
_transforms = Compose(
    [RandomResizedCrop(feature_extractor.size), ColorJitter(brightness=0.5, hue=0.5), ToTensor(), normalize]
)`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> torchvision.transforms <span class="hljs-keyword">import</span> Compose, Normalize, RandomResizedCrop, ColorJitter, ToTensor

<span class="hljs-meta">&gt;&gt;&gt; </span>normalize = Normalize(mean=feature_extractor.image_mean, std=feature_extractor.image_std)
<span class="hljs-meta">&gt;&gt;&gt; </span>_transforms = Compose(
<span class="hljs-meta">... </span>    [RandomResizedCrop(feature_extractor.size), ColorJitter(brightness=<span class="hljs-number">0.5</span>, hue=<span class="hljs-number">0.5</span>), ToTensor(), normalize]
<span class="hljs-meta">... </span>)`}}),Da=new w({props:{code:`def transforms(examples):
    examples["pixel_values"] = [_transforms(image.convert("RGB")) for image in examples["image"]]
    return examples`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">def</span> <span class="hljs-title function_">transforms</span>(<span class="hljs-params">examples</span>):
<span class="hljs-meta">... </span>    examples[<span class="hljs-string">&quot;pixel_values&quot;</span>] = [_transforms(image.convert(<span class="hljs-string">&quot;RGB&quot;</span>)) <span class="hljs-keyword">for</span> image <span class="hljs-keyword">in</span> examples[<span class="hljs-string">&quot;image&quot;</span>]]
<span class="hljs-meta">... </span>    <span class="hljs-keyword">return</span> examples`}}),Sa=new w({props:{code:"dataset.set_transform(transforms)",highlighted:'<span class="hljs-meta">&gt;&gt;&gt; </span>dataset.set_transform(transforms)'}}),Ra=new w({props:{code:'dataset[0]["image"]',highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>dataset[<span class="hljs-number">0</span>][<span class="hljs-string">&quot;image&quot;</span>]
{<span class="hljs-string">&#x27;image&#x27;</span>: &lt;PIL.JpegImagePlugin.JpegImageFile image mode=RGB size=384x512 at <span class="hljs-number">0x7F1A7B0630D0</span>&gt;,
 <span class="hljs-string">&#x27;label&#x27;</span>: <span class="hljs-number">6</span>,
 <span class="hljs-string">&#x27;pixel_values&#x27;</span>: tensor([[[ <span class="hljs-number">0.0353</span>,  <span class="hljs-number">0.0745</span>,  <span class="hljs-number">0.1216</span>,  ..., -<span class="hljs-number">0.9922</span>, -<span class="hljs-number">0.9922</span>, -<span class="hljs-number">0.9922</span>],
          [-<span class="hljs-number">0.0196</span>,  <span class="hljs-number">0.0667</span>,  <span class="hljs-number">0.1294</span>,  ..., -<span class="hljs-number">0.9765</span>, -<span class="hljs-number">0.9843</span>, -<span class="hljs-number">0.9922</span>],
          [ <span class="hljs-number">0.0196</span>,  <span class="hljs-number">0.0824</span>,  <span class="hljs-number">0.1137</span>,  ..., -<span class="hljs-number">0.9765</span>, -<span class="hljs-number">0.9686</span>, -<span class="hljs-number">0.8667</span>],
          ...,
          [ <span class="hljs-number">0.0275</span>,  <span class="hljs-number">0.0745</span>,  <span class="hljs-number">0.0510</span>,  ..., -<span class="hljs-number">0.1137</span>, -<span class="hljs-number">0.1216</span>, -<span class="hljs-number">0.0824</span>],
          [ <span class="hljs-number">0.0667</span>,  <span class="hljs-number">0.0824</span>,  <span class="hljs-number">0.0667</span>,  ..., -<span class="hljs-number">0.0588</span>, -<span class="hljs-number">0.0745</span>, -<span class="hljs-number">0.0980</span>],
          [ <span class="hljs-number">0.0353</span>,  <span class="hljs-number">0.0353</span>,  <span class="hljs-number">0.0431</span>,  ..., -<span class="hljs-number">0.0039</span>, -<span class="hljs-number">0.0039</span>, -<span class="hljs-number">0.0588</span>]],
 
         [[ <span class="hljs-number">0.2078</span>,  <span class="hljs-number">0.2471</span>,  <span class="hljs-number">0.2863</span>,  ..., -<span class="hljs-number">0.9451</span>, -<span class="hljs-number">0.9373</span>, -<span class="hljs-number">0.9451</span>],
          [ <span class="hljs-number">0.1608</span>,  <span class="hljs-number">0.2471</span>,  <span class="hljs-number">0.3098</span>,  ..., -<span class="hljs-number">0.9373</span>, -<span class="hljs-number">0.9451</span>, -<span class="hljs-number">0.9373</span>],
          [ <span class="hljs-number">0.2078</span>,  <span class="hljs-number">0.2706</span>,  <span class="hljs-number">0.3020</span>,  ..., -<span class="hljs-number">0.9608</span>, -<span class="hljs-number">0.9373</span>, -<span class="hljs-number">0.8275</span>],
          ...,
          [-<span class="hljs-number">0.0353</span>,  <span class="hljs-number">0.0118</span>, -<span class="hljs-number">0.0039</span>,  ..., -<span class="hljs-number">0.2392</span>, -<span class="hljs-number">0.2471</span>, -<span class="hljs-number">0.2078</span>],
          [ <span class="hljs-number">0.0196</span>,  <span class="hljs-number">0.0353</span>,  <span class="hljs-number">0.0196</span>,  ..., -<span class="hljs-number">0.1843</span>, -<span class="hljs-number">0.2000</span>, -<span class="hljs-number">0.2235</span>],
          [-<span class="hljs-number">0.0118</span>, -<span class="hljs-number">0.0039</span>, -<span class="hljs-number">0.0039</span>,  ..., -<span class="hljs-number">0.0980</span>, -<span class="hljs-number">0.0980</span>, -<span class="hljs-number">0.1529</span>]],
 
         [[ <span class="hljs-number">0.3961</span>,  <span class="hljs-number">0.4431</span>,  <span class="hljs-number">0.4980</span>,  ..., -<span class="hljs-number">0.9216</span>, -<span class="hljs-number">0.9137</span>, -<span class="hljs-number">0.9216</span>],
          [ <span class="hljs-number">0.3569</span>,  <span class="hljs-number">0.4510</span>,  <span class="hljs-number">0.5216</span>,  ..., -<span class="hljs-number">0.9059</span>, -<span class="hljs-number">0.9137</span>, -<span class="hljs-number">0.9137</span>],
          [ <span class="hljs-number">0.4118</span>,  <span class="hljs-number">0.4745</span>,  <span class="hljs-number">0.5216</span>,  ..., -<span class="hljs-number">0.9137</span>, -<span class="hljs-number">0.8902</span>, -<span class="hljs-number">0.7804</span>],
          ...,
          [-<span class="hljs-number">0.2314</span>, -<span class="hljs-number">0.1922</span>, -<span class="hljs-number">0.2078</span>,  ..., -<span class="hljs-number">0.4196</span>, -<span class="hljs-number">0.4275</span>, -<span class="hljs-number">0.3882</span>],
          [-<span class="hljs-number">0.1843</span>, -<span class="hljs-number">0.1686</span>, -<span class="hljs-number">0.2000</span>,  ..., -<span class="hljs-number">0.3647</span>, -<span class="hljs-number">0.3804</span>, -<span class="hljs-number">0.4039</span>],
          [-<span class="hljs-number">0.1922</span>, -<span class="hljs-number">0.1922</span>, -<span class="hljs-number">0.1922</span>,  ..., -<span class="hljs-number">0.2941</span>, -<span class="hljs-number">0.2863</span>, -<span class="hljs-number">0.3412</span>]]])}`}}),Ba=new w({props:{code:`import numpy as np
import matplotlib.pyplot as plt

img = dataset[0]["pixel_values"]
plt.imshow(img.permute(1, 2, 0))`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">import</span> numpy <span class="hljs-keyword">as</span> np
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">import</span> matplotlib.pyplot <span class="hljs-keyword">as</span> plt

<span class="hljs-meta">&gt;&gt;&gt; </span>img = dataset[<span class="hljs-number">0</span>][<span class="hljs-string">&quot;pixel_values&quot;</span>]
<span class="hljs-meta">&gt;&gt;&gt; </span>plt.imshow(img.permute(<span class="hljs-number">1</span>, <span class="hljs-number">2</span>, <span class="hljs-number">0</span>))`}}),Wa=new Is({}),Ua=new w({props:{code:`from datasets import load_dataset

lj_speech = load_dataset("lj_speech", split="train")`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> datasets <span class="hljs-keyword">import</span> load_dataset

<span class="hljs-meta">&gt;&gt;&gt; </span>lj_speech = load_dataset(<span class="hljs-string">&quot;lj_speech&quot;</span>, split=<span class="hljs-string">&quot;train&quot;</span>)`}}),Ya=new w({props:{code:'lj_speech = lj_speech.map(remove_columns=["file", "id", "normalized_text"])',highlighted:'<span class="hljs-meta">&gt;&gt;&gt; </span>lj_speech = lj_speech.<span class="hljs-built_in">map</span>(remove_columns=[<span class="hljs-string">&quot;file&quot;</span>, <span class="hljs-string">&quot;id&quot;</span>, <span class="hljs-string">&quot;normalized_text&quot;</span>])'}}),Ga=new w({props:{code:`lj_speech[0]["audio"]

lj_speech[0]["text"]`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>lj_speech[<span class="hljs-number">0</span>][<span class="hljs-string">&quot;audio&quot;</span>]
{<span class="hljs-string">&#x27;array&#x27;</span>: array([-<span class="hljs-number">7.3242188e-04</span>, -<span class="hljs-number">7.6293945e-04</span>, -<span class="hljs-number">6.4086914e-04</span>, ...,
         <span class="hljs-number">7.3242188e-04</span>,  <span class="hljs-number">2.1362305e-04</span>,  <span class="hljs-number">6.1035156e-05</span>], dtype=float32),
 <span class="hljs-string">&#x27;path&#x27;</span>: <span class="hljs-string">&#x27;/root/.cache/huggingface/datasets/downloads/extracted/917ece08c95cf0c4115e45294e3cd0dee724a1165b7fc11798369308a465bd26/LJSpeech-1.1/wavs/LJ001-0001.wav&#x27;</span>,
 <span class="hljs-string">&#x27;sampling_rate&#x27;</span>: <span class="hljs-number">22050</span>}

<span class="hljs-meta">&gt;&gt;&gt; </span>lj_speech[<span class="hljs-number">0</span>][<span class="hljs-string">&quot;text&quot;</span>]
<span class="hljs-string">&#x27;Printing, in the only sense with which we are at present concerned, differs from most if not from all the arts and crafts represented in the Exhibition&#x27;</span>`}}),Va=new w({props:{code:'lj_speech = lj_speech.cast_column("audio", Audio(sampling_rate=16_000))',highlighted:'<span class="hljs-meta">&gt;&gt;&gt; </span>lj_speech = lj_speech.cast_column(<span class="hljs-string">&quot;audio&quot;</span>, Audio(sampling_rate=<span class="hljs-number">16_000</span>))'}}),Ka=new w({props:{code:`from transformers import AutoProcessor

processor = AutoProcessor.from_pretrained("facebook/wav2vec2-base-960h")`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoProcessor

<span class="hljs-meta">&gt;&gt;&gt; </span>processor = AutoProcessor.from_pretrained(<span class="hljs-string">&quot;facebook/wav2vec2-base-960h&quot;</span>)`}}),Qa=new w({props:{code:`def prepare_dataset(example):
    audio = example["audio"]

    example.update(processor(audio=audio["array"], text=example["text"], sampling_rate=16000))

    return example`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">def</span> <span class="hljs-title function_">prepare_dataset</span>(<span class="hljs-params">example</span>):
<span class="hljs-meta">... </span>    audio = example[<span class="hljs-string">&quot;audio&quot;</span>]

<span class="hljs-meta">... </span>    example.update(processor(audio=audio[<span class="hljs-string">&quot;array&quot;</span>], text=example[<span class="hljs-string">&quot;text&quot;</span>], sampling_rate=<span class="hljs-number">16000</span>))

<span class="hljs-meta">... </span>    <span class="hljs-keyword">return</span> example`}}),se=new w({props:{code:"prepare_dataset(lj_speech[0])",highlighted:'<span class="hljs-meta">&gt;&gt;&gt; </span>prepare_dataset(lj_speech[<span class="hljs-number">0</span>])'}}),{c(){i=p("meta"),v=c(),b=p("h1"),y=p("a"),k=p("span"),f(x.$$.fragment),E=c(),q=p("span"),D=t("Preprocess"),Ss=c(),f(L.$$.fragment),ft=c(),ee=p("p"),Ip=t("Before you can train a model on a dataset, it needs to be preprocessed into the expected model input format. Whether your data is text, images, or audio, they need to be converted and assembled into batches of tensors. \u{1F917} Transformers provides a set of preprocessing classes to help prepare your data for the model. In this tutorial, you\u2019ll learn that for:"),jt=c(),I=p("ul"),Ns=p("li"),Sp=t("Text, use a "),ne=p("a"),Np=t("Tokenizer"),Fp=t(" to convert text into a sequence of tokens, create a numerical representation of the tokens, and assemble them into tensors."),Rp=c(),Fs=p("li"),Bp=t("Computer vision and speech, use a "),te=p("a"),Mp=t("Feature extractor"),Wp=t(" to extract sequential features from audio waveforms and images and convert them into tensors."),Jp=c(),Rs=p("li"),Hp=t("Multimodal inputs, use a "),le=p("a"),Up=t("Processor"),Yp=t(" to combine a tokenizer and a feature extractor."),dt=c(),f(rs.$$.fragment),gt=c(),pe=p("p"),Gp=t("Before you begin, install \u{1F917} Datasets so you can load some datasets to experiment with:"),_t=c(),f(Bs.$$.fragment),$t=c(),Z=p("h2"),os=p("a"),en=p("span"),f(Ms.$$.fragment),Vp=c(),nn=p("span"),Kp=t("Natural Language Processing"),vt=c(),f(Ws.$$.fragment),yt=c(),S=p("p"),Qp=t("The main tool for preprocessing textual data is a "),re=p("a"),Xp=t("tokenizer"),Zp=t(". A tokenizer splits text into "),tn=p("em"),sr=t("tokens"),ar=t(" according to a set of rules. The tokens are converted into numbers and then tensors, which become the model inputs. Any additional inputs required by the model are added by the tokenizer."),wt=c(),f(us.$$.fragment),kt=c(),N=p("p"),er=t("Get started by loading a pretrained tokenizer with the "),oe=p("a"),nr=t("AutoTokenizer.from_pretrained()"),tr=t(" method. This downloads the "),ln=p("em"),lr=t("vocab"),pr=t(" a model was pretrained with:"),xt=c(),f(Js.$$.fragment),Et=c(),ue=p("p"),rr=t("Then pass your text to the tokenizer:"),qt=c(),f(Hs.$$.fragment),At=c(),ce=p("p"),or=t("The tokenizer returns a dictionary with three important items:"),Pt=c(),F=p("ul"),he=p("li"),me=p("a"),ur=t("input_ids"),cr=t(" are the indices corresponding to each token in the sentence."),hr=c(),ie=p("li"),be=p("a"),mr=t("attention_mask"),ir=t(" indicates whether a token should be attended to or not."),br=c(),fe=p("li"),je=p("a"),fr=t("token_type_ids"),jr=t(" identifies which sequence a token belongs to when there is more than one sequence."),Tt=c(),cs=p("p"),dr=t("Return your input by decoding the "),pn=p("code"),gr=t("input_ids"),_r=t(":"),Ct=c(),f(Us.$$.fragment),zt=c(),R=p("p"),$r=t("As you can see, the tokenizer added two special tokens - "),rn=p("code"),vr=t("CLS"),yr=t(" and "),on=p("code"),wr=t("SEP"),kr=t(` (classifier and separator) - to the sentence. Not all models need
special tokens, but if they do, the tokenizer automatically adds them for you.`),Dt=c(),de=p("p"),xr=t("If there are several sentences you want to preprocess, pass them as a list to the tokenizer:"),Ot=c(),f(Ys.$$.fragment),Lt=c(),ss=p("h3"),hs=p("a"),un=p("span"),f(Gs.$$.fragment),Er=c(),cn=p("span"),qr=t("Pad"),It=c(),ms=p("p"),Ar=t("Sentences aren\u2019t always the same length which can be an issue because tensors, the model inputs, need to have a uniform shape. Padding is a strategy for ensuring tensors are rectangular by adding a special "),hn=p("em"),Pr=t("padding token"),Tr=t(" to shorter sentences."),St=c(),B=p("p"),Cr=t("Set the "),mn=p("code"),zr=t("padding"),Dr=t(" parameter to "),bn=p("code"),Or=t("True"),Lr=t(" to pad the shorter sequences in the batch to match the longest sequence:"),Nt=c(),f(Vs.$$.fragment),Ft=c(),is=p("p"),Ir=t("The first and third sentences are now padded with "),fn=p("code"),Sr=t("0"),Nr=t("\u2019s because they are shorter."),Rt=c(),as=p("h3"),bs=p("a"),jn=p("span"),f(Ks.$$.fragment),Fr=c(),dn=p("span"),Rr=t("Truncation"),Bt=c(),ge=p("p"),Br=t("On the other end of the spectrum, sometimes a sequence may be too long for a model to handle. In this case, you\u2019ll need to truncate the sequence to a shorter length."),Mt=c(),M=p("p"),Mr=t("Set the "),gn=p("code"),Wr=t("truncation"),Jr=t(" parameter to "),_n=p("code"),Hr=t("True"),Ur=t(" to truncate a sequence to the maximum length accepted by the model:"),Wt=c(),f(Qs.$$.fragment),Jt=c(),f(fs.$$.fragment),Ht=c(),es=p("h3"),js=p("a"),$n=p("span"),f(Xs.$$.fragment),Yr=c(),vn=p("span"),Gr=t("Build tensors"),Ut=c(),_e=p("p"),Vr=t("Finally, you want the tokenizer to return the actual tensors that get fed to the model."),Yt=c(),T=p("p"),Kr=t("Set the "),yn=p("code"),Qr=t("return_tensors"),Xr=t(" parameter to either "),wn=p("code"),Zr=t("pt"),so=t(" for PyTorch, or "),kn=p("code"),ao=t("tf"),eo=t(" for TensorFlow:"),Gt=c(),f(ds.$$.fragment),Vt=c(),ns=p("h2"),gs=p("a"),xn=p("span"),f(Zs.$$.fragment),no=c(),En=p("span"),to=t("Audio"),Kt=c(),_s=p("p"),lo=t("For audio tasks, you\u2019ll need a "),$e=p("a"),po=t("feature extractor"),ro=t(" to prepare your dataset for the model. The feature extractor is designed to extract features from raw audio data, and convert them into tensors."),Qt=c(),W=p("p"),oo=t("Load the "),sa=p("a"),uo=t("MInDS-14"),co=t(" dataset (see the \u{1F917} "),aa=p("a"),ho=t("Datasets tutorial"),mo=t(" for more details on how to load a dataset) to see how you can use a feature extractor with audio datasets:"),Xt=c(),f(ea.$$.fragment),Zt=c(),J=p("p"),io=t("Access the first element of the "),qn=p("code"),bo=t("audio"),fo=t(" column to take a look at the input. Calling the "),An=p("code"),jo=t("audio"),go=t(" column automatically loads and resamples the audio file:"),sl=c(),f(na.$$.fragment),al=c(),ve=p("p"),_o=t("This returns three items:"),el=c(),H=p("ul"),ye=p("li"),Pn=p("code"),$o=t("array"),vo=t(" is the speech signal loaded - and potentially resampled - as a 1D array."),yo=c(),we=p("li"),Tn=p("code"),wo=t("path"),ko=t(" points to the location of the audio file."),xo=c(),ke=p("li"),Cn=p("code"),Eo=t("sampling_rate"),qo=t(" refers to how many data points in the speech signal are measured per second."),nl=c(),$s=p("p"),Ao=t("For this tutorial, you\u2019ll use the "),ta=p("a"),Po=t("Wav2Vec2"),To=t(" model. Take a look at the model card, and you\u2019ll learn Wav2Vec2 is pretrained on 16kHz sampled speech audio. It is important your audio data\u2019s sampling rate matches the sampling rate of the dataset used to pretrain the model. If your data\u2019s sampling rate isn\u2019t the same, then you need to resample your data."),tl=c(),xe=p("ol"),la=p("li"),Co=t("Use \u{1F917} Datasets\u2019 "),pa=p("a"),zo=t("cast_column"),Do=t(" method to upsample the sampling rate to 16kHz:"),ll=c(),f(ra.$$.fragment),pl=c(),oa=p("ol"),ua=p("li"),Oo=t("Call the "),zn=p("code"),Lo=t("audio"),Io=t(" column again to resample the audio file:"),rl=c(),f(ca.$$.fragment),ol=c(),C=p("p"),So=t("Next, load a feature extractor to normalize and pad the input. When padding textual data, a "),Dn=p("code"),No=t("0"),Fo=t(" is added for shorter sequences. The same idea applies to audio data. The feature extractor adds a "),On=p("code"),Ro=t("0"),Bo=t(" - interpreted as silence - to "),Ln=p("code"),Mo=t("array"),Wo=t("."),ul=c(),vs=p("p"),Jo=t("Load the feature extractor with "),Ee=p("a"),Ho=t("AutoFeatureExtractor.from_pretrained()"),Uo=t(":"),cl=c(),f(ha.$$.fragment),hl=c(),U=p("p"),Yo=t("Pass the audio "),In=p("code"),Go=t("array"),Vo=t(" to the feature extractor. We also recommend adding the "),Sn=p("code"),Ko=t("sampling_rate"),Qo=t(" argument in the feature extractor in order to better debug any silent errors that may occur."),ml=c(),f(ma.$$.fragment),il=c(),qe=p("p"),Xo=t("Just like the tokenizer, you can apply padding or truncation to handle variable sequences in a batch. Take a look at the sequence length of these two audio samples:"),bl=c(),f(ia.$$.fragment),fl=c(),Ae=p("p"),Zo=t("Create a function to preprocess the dataset so the audio samples are the same lengths. Specify a maximum sample length, and the feature extractor will either pad or truncate the sequences to match it:"),jl=c(),f(ba.$$.fragment),dl=c(),ys=p("p"),su=t("Apply the "),Nn=p("code"),au=t("preprocess_function"),eu=t(" to the the first few examples in the dataset:"),gl=c(),f(fa.$$.fragment),_l=c(),Pe=p("p"),nu=t("The sample lengths are now the same and match the specified maximum length. You can pass your processed dataset to the model now!"),$l=c(),f(ja.$$.fragment),vl=c(),ts=p("h2"),ws=p("a"),Fn=p("span"),f(da.$$.fragment),tu=c(),Rn=p("span"),lu=t("Computer vision"),yl=c(),ks=p("p"),pu=t("For computer vision tasks, you\u2019ll need a "),Te=p("a"),ru=t("feature extractor"),ou=t(" to prepare your dataset for the model. The feature extractor is designed to extract features from images, and convert them into tensors."),wl=c(),Y=p("p"),uu=t("Load the "),ga=p("a"),cu=t("food101"),hu=t(" dataset (see the \u{1F917} "),_a=p("a"),mu=t("Datasets tutorial"),iu=t(" for more details on how to load a dataset) to see how you can use a feature extractor with computer vision datasets:"),kl=c(),f(xs.$$.fragment),xl=c(),f($a.$$.fragment),El=c(),Es=p("p"),bu=t("Next, take a look at the image with \u{1F917} Datasets "),va=p("a"),Bn=p("code"),fu=t("Image"),ju=t(" feature:"),ql=c(),f(ya.$$.fragment),Al=c(),wa=p("div"),Mn=p("img"),Pl=c(),qs=p("p"),du=t("Load the feature extractor with "),Ce=p("a"),gu=t("AutoFeatureExtractor.from_pretrained()"),_u=t(":"),Tl=c(),f(ka.$$.fragment),Cl=c(),z=p("p"),$u=t("For computer vision tasks, it is common to add some type of data augmentation to the images as a part of preprocessing. You can add augmentations with any library you\u2019d like, but in this tutorial, you\u2019ll use torchvision\u2019s "),xa=p("a"),Wn=p("code"),vu=t("transforms"),yu=t(" module. If you\u2019re interested in using another data augmentation library, learn how in the "),Ea=p("a"),wu=t("Albumentations"),ku=t(" or "),qa=p("a"),xu=t("Kornia notebooks"),Eu=t("."),zl=c(),ze=p("ol"),O=p("li"),qu=t("Normalize the image with the feature extractor and use "),Aa=p("a"),Jn=p("code"),Au=t("Compose"),Pu=t(" to chain some transforms - "),Pa=p("a"),Hn=p("code"),Tu=t("RandomResizedCrop"),Cu=t(" and "),Ta=p("a"),Un=p("code"),zu=t("ColorJitter"),Du=t(" - together:"),Dl=c(),f(Ca.$$.fragment),Ol=c(),za=p("ol"),ls=p("li"),Ou=t("The model accepts "),De=p("a"),Yn=p("code"),Lu=t("pixel_values"),Iu=t(" as its input, which is generated by the feature extractor. Create a function that generates "),Gn=p("code"),Su=t("pixel_values"),Nu=t(" from the transforms:"),Ll=c(),f(Da.$$.fragment),Il=c(),Oa=p("ol"),La=p("li"),Fu=t("Then use \u{1F917} Datasets "),Ia=p("a"),Vn=p("code"),Ru=t("set_transform"),Bu=t(" to apply the transforms on the fly:"),Sl=c(),f(Sa.$$.fragment),Nl=c(),Na=p("ol"),Fa=p("li"),Mu=t("Now when you access the image, you\u2019ll notice the feature extractor has added "),Kn=p("code"),Wu=t("pixel_values"),Ju=t(". You can pass your processed dataset to the model now!"),Fl=c(),f(Ra.$$.fragment),Rl=c(),Oe=p("p"),Hu=t("Here is what the image looks like after the transforms are applied. The image has been randomly cropped and it\u2019s color properties are different."),Bl=c(),f(Ba.$$.fragment),Ml=c(),Ma=p("div"),Qn=p("img"),Wl=c(),ps=p("h2"),As=p("a"),Xn=p("span"),f(Wa.$$.fragment),Uu=c(),Zn=p("span"),Yu=t("Multimodal"),Jl=c(),Ps=p("p"),Gu=t("For tasks involving multimodal inputs, you\u2019ll need a "),Le=p("a"),Vu=t("processor"),Ku=t(" to prepare your dataset for the model. A processor couples a tokenizer and feature extractor."),Hl=c(),G=p("p"),Qu=t("Load the "),Ja=p("a"),Xu=t("LJ Speech"),Zu=t(" dataset (see the \u{1F917} "),Ha=p("a"),sc=t("Datasets tutorial"),ac=t(" for more details on how to load a dataset) to see how you can use a processor for automatic speech recognition (ASR):"),Ul=c(),f(Ua.$$.fragment),Yl=c(),V=p("p"),ec=t("For ASR, you\u2019re mainly focused on "),st=p("code"),nc=t("audio"),tc=t(" and "),at=p("code"),lc=t("text"),pc=t(" so you can remove the other columns:"),Gl=c(),f(Ya.$$.fragment),Vl=c(),K=p("p"),rc=t("Now take a look at the "),et=p("code"),oc=t("audio"),uc=t(" and "),nt=p("code"),cc=t("text"),hc=t(" columns:"),Kl=c(),f(Ga.$$.fragment),Ql=c(),Ts=p("p"),mc=t("Remember you should always "),Ie=p("a"),ic=t("resample"),bc=t(" your audio dataset\u2019s sampling rate to match the sampling rate of the dataset used to pretrain a model!"),Xl=c(),f(Va.$$.fragment),Zl=c(),Cs=p("p"),fc=t("Load a processor with "),Se=p("a"),jc=t("AutoProcessor.from_pretrained()"),dc=t(":"),sp=c(),f(Ka.$$.fragment),ap=c(),Ne=p("ol"),P=p("li"),gc=t("Create a function to process the audio data contained in "),tt=p("code"),_c=t("array"),$c=t(" to "),lt=p("code"),vc=t("input_values"),yc=t(", and tokenize "),pt=p("code"),wc=t("text"),kc=t(" to "),rt=p("code"),xc=t("labels"),Ec=t(". These are the inputs to the model:"),ep=c(),f(Qa.$$.fragment),np=c(),Xa=p("ol"),Za=p("li"),qc=t("Apply the "),ot=p("code"),Ac=t("prepare_dataset"),Pc=t(" function to a sample:"),tp=c(),f(se.$$.fragment),lp=c(),Q=p("p"),Tc=t("The processor has now added "),ut=p("code"),Cc=t("input_values"),zc=t(" and "),ct=p("code"),Dc=t("labels"),Oc=t(", and the sampling rate has also been correctly downsampled to 16kHz. You can pass your processed dataset to the model now!"),this.h()},l(s){const n=ri('[data-svelte="svelte-1phssyn"]',document.head);i=r(n,"META",{name:!0,content:!0}),n.forEach(a),v=h(s),b=r(s,"H1",{class:!0});var ae=o(b);y=r(ae,"A",{id:!0,class:!0,href:!0});var ht=o(y);k=r(ht,"SPAN",{});var mt=o(k);j(x.$$.fragment,mt),mt.forEach(a),ht.forEach(a),E=h(ae),q=r(ae,"SPAN",{});var it=o(q);D=l(it,"Preprocess"),it.forEach(a),ae.forEach(a),Ss=h(s),j(L.$$.fragment,s),ft=h(s),ee=r(s,"P",{});var bt=o(ee);Ip=l(bt,"Before you can train a model on a dataset, it needs to be preprocessed into the expected model input format. Whether your data is text, images, or audio, they need to be converted and assembled into batches of tensors. \u{1F917} Transformers provides a set of preprocessing classes to help prepare your data for the model. In this tutorial, you\u2019ll learn that for:"),bt.forEach(a),jt=h(s),I=r(s,"UL",{});var Fe=o(I);Ns=r(Fe,"LI",{});var rp=o(Ns);Sp=l(rp,"Text, use a "),ne=r(rp,"A",{href:!0});var Wc=o(ne);Np=l(Wc,"Tokenizer"),Wc.forEach(a),Fp=l(rp," to convert text into a sequence of tokens, create a numerical representation of the tokens, and assemble them into tensors."),rp.forEach(a),Rp=h(Fe),Fs=r(Fe,"LI",{});var op=o(Fs);Bp=l(op,"Computer vision and speech, use a "),te=r(op,"A",{href:!0});var Jc=o(te);Mp=l(Jc,"Feature extractor"),Jc.forEach(a),Wp=l(op," to extract sequential features from audio waveforms and images and convert them into tensors."),op.forEach(a),Jp=h(Fe),Rs=r(Fe,"LI",{});var up=o(Rs);Hp=l(up,"Multimodal inputs, use a "),le=r(up,"A",{href:!0});var Hc=o(le);Up=l(Hc,"Processor"),Hc.forEach(a),Yp=l(up," to combine a tokenizer and a feature extractor."),up.forEach(a),Fe.forEach(a),dt=h(s),j(rs.$$.fragment,s),gt=h(s),pe=r(s,"P",{});var Uc=o(pe);Gp=l(Uc,"Before you begin, install \u{1F917} Datasets so you can load some datasets to experiment with:"),Uc.forEach(a),_t=h(s),j(Bs.$$.fragment,s),$t=h(s),Z=r(s,"H2",{class:!0});var cp=o(Z);os=r(cp,"A",{id:!0,class:!0,href:!0});var Yc=o(os);en=r(Yc,"SPAN",{});var Gc=o(en);j(Ms.$$.fragment,Gc),Gc.forEach(a),Yc.forEach(a),Vp=h(cp),nn=r(cp,"SPAN",{});var Vc=o(nn);Kp=l(Vc,"Natural Language Processing"),Vc.forEach(a),cp.forEach(a),vt=h(s),j(Ws.$$.fragment,s),yt=h(s),S=r(s,"P",{});var Re=o(S);Qp=l(Re,"The main tool for preprocessing textual data is a "),re=r(Re,"A",{href:!0});var Kc=o(re);Xp=l(Kc,"tokenizer"),Kc.forEach(a),Zp=l(Re,". A tokenizer splits text into "),tn=r(Re,"EM",{});var Qc=o(tn);sr=l(Qc,"tokens"),Qc.forEach(a),ar=l(Re," according to a set of rules. The tokens are converted into numbers and then tensors, which become the model inputs. Any additional inputs required by the model are added by the tokenizer."),Re.forEach(a),wt=h(s),j(us.$$.fragment,s),kt=h(s),N=r(s,"P",{});var Be=o(N);er=l(Be,"Get started by loading a pretrained tokenizer with the "),oe=r(Be,"A",{href:!0});var Xc=o(oe);nr=l(Xc,"AutoTokenizer.from_pretrained()"),Xc.forEach(a),tr=l(Be," method. This downloads the "),ln=r(Be,"EM",{});var Zc=o(ln);lr=l(Zc,"vocab"),Zc.forEach(a),pr=l(Be," a model was pretrained with:"),Be.forEach(a),xt=h(s),j(Js.$$.fragment,s),Et=h(s),ue=r(s,"P",{});var sh=o(ue);rr=l(sh,"Then pass your text to the tokenizer:"),sh.forEach(a),qt=h(s),j(Hs.$$.fragment,s),At=h(s),ce=r(s,"P",{});var ah=o(ce);or=l(ah,"The tokenizer returns a dictionary with three important items:"),ah.forEach(a),Pt=h(s),F=r(s,"UL",{});var Me=o(F);he=r(Me,"LI",{});var Lc=o(he);me=r(Lc,"A",{href:!0});var eh=o(me);ur=l(eh,"input_ids"),eh.forEach(a),cr=l(Lc," are the indices corresponding to each token in the sentence."),Lc.forEach(a),hr=h(Me),ie=r(Me,"LI",{});var Ic=o(ie);be=r(Ic,"A",{href:!0});var nh=o(be);mr=l(nh,"attention_mask"),nh.forEach(a),ir=l(Ic," indicates whether a token should be attended to or not."),Ic.forEach(a),br=h(Me),fe=r(Me,"LI",{});var Sc=o(fe);je=r(Sc,"A",{href:!0});var th=o(je);fr=l(th,"token_type_ids"),th.forEach(a),jr=l(Sc," identifies which sequence a token belongs to when there is more than one sequence."),Sc.forEach(a),Me.forEach(a),Tt=h(s),cs=r(s,"P",{});var hp=o(cs);dr=l(hp,"Return your input by decoding the "),pn=r(hp,"CODE",{});var lh=o(pn);gr=l(lh,"input_ids"),lh.forEach(a),_r=l(hp,":"),hp.forEach(a),Ct=h(s),j(Us.$$.fragment,s),zt=h(s),R=r(s,"P",{});var We=o(R);$r=l(We,"As you can see, the tokenizer added two special tokens - "),rn=r(We,"CODE",{});var ph=o(rn);vr=l(ph,"CLS"),ph.forEach(a),yr=l(We," and "),on=r(We,"CODE",{});var rh=o(on);wr=l(rh,"SEP"),rh.forEach(a),kr=l(We,` (classifier and separator) - to the sentence. Not all models need
special tokens, but if they do, the tokenizer automatically adds them for you.`),We.forEach(a),Dt=h(s),de=r(s,"P",{});var oh=o(de);xr=l(oh,"If there are several sentences you want to preprocess, pass them as a list to the tokenizer:"),oh.forEach(a),Ot=h(s),j(Ys.$$.fragment,s),Lt=h(s),ss=r(s,"H3",{class:!0});var mp=o(ss);hs=r(mp,"A",{id:!0,class:!0,href:!0});var uh=o(hs);un=r(uh,"SPAN",{});var ch=o(un);j(Gs.$$.fragment,ch),ch.forEach(a),uh.forEach(a),Er=h(mp),cn=r(mp,"SPAN",{});var hh=o(cn);qr=l(hh,"Pad"),hh.forEach(a),mp.forEach(a),It=h(s),ms=r(s,"P",{});var ip=o(ms);Ar=l(ip,"Sentences aren\u2019t always the same length which can be an issue because tensors, the model inputs, need to have a uniform shape. Padding is a strategy for ensuring tensors are rectangular by adding a special "),hn=r(ip,"EM",{});var mh=o(hn);Pr=l(mh,"padding token"),mh.forEach(a),Tr=l(ip," to shorter sentences."),ip.forEach(a),St=h(s),B=r(s,"P",{});var Je=o(B);Cr=l(Je,"Set the "),mn=r(Je,"CODE",{});var ih=o(mn);zr=l(ih,"padding"),ih.forEach(a),Dr=l(Je," parameter to "),bn=r(Je,"CODE",{});var bh=o(bn);Or=l(bh,"True"),bh.forEach(a),Lr=l(Je," to pad the shorter sequences in the batch to match the longest sequence:"),Je.forEach(a),Nt=h(s),j(Vs.$$.fragment,s),Ft=h(s),is=r(s,"P",{});var bp=o(is);Ir=l(bp,"The first and third sentences are now padded with "),fn=r(bp,"CODE",{});var fh=o(fn);Sr=l(fh,"0"),fh.forEach(a),Nr=l(bp,"\u2019s because they are shorter."),bp.forEach(a),Rt=h(s),as=r(s,"H3",{class:!0});var fp=o(as);bs=r(fp,"A",{id:!0,class:!0,href:!0});var jh=o(bs);jn=r(jh,"SPAN",{});var dh=o(jn);j(Ks.$$.fragment,dh),dh.forEach(a),jh.forEach(a),Fr=h(fp),dn=r(fp,"SPAN",{});var gh=o(dn);Rr=l(gh,"Truncation"),gh.forEach(a),fp.forEach(a),Bt=h(s),ge=r(s,"P",{});var _h=o(ge);Br=l(_h,"On the other end of the spectrum, sometimes a sequence may be too long for a model to handle. In this case, you\u2019ll need to truncate the sequence to a shorter length."),_h.forEach(a),Mt=h(s),M=r(s,"P",{});var He=o(M);Mr=l(He,"Set the "),gn=r(He,"CODE",{});var $h=o(gn);Wr=l($h,"truncation"),$h.forEach(a),Jr=l(He," parameter to "),_n=r(He,"CODE",{});var vh=o(_n);Hr=l(vh,"True"),vh.forEach(a),Ur=l(He," to truncate a sequence to the maximum length accepted by the model:"),He.forEach(a),Wt=h(s),j(Qs.$$.fragment,s),Jt=h(s),j(fs.$$.fragment,s),Ht=h(s),es=r(s,"H3",{class:!0});var jp=o(es);js=r(jp,"A",{id:!0,class:!0,href:!0});var yh=o(js);$n=r(yh,"SPAN",{});var wh=o($n);j(Xs.$$.fragment,wh),wh.forEach(a),yh.forEach(a),Yr=h(jp),vn=r(jp,"SPAN",{});var kh=o(vn);Gr=l(kh,"Build tensors"),kh.forEach(a),jp.forEach(a),Ut=h(s),_e=r(s,"P",{});var xh=o(_e);Vr=l(xh,"Finally, you want the tokenizer to return the actual tensors that get fed to the model."),xh.forEach(a),Yt=h(s),T=r(s,"P",{});var zs=o(T);Kr=l(zs,"Set the "),yn=r(zs,"CODE",{});var Eh=o(yn);Qr=l(Eh,"return_tensors"),Eh.forEach(a),Xr=l(zs," parameter to either "),wn=r(zs,"CODE",{});var qh=o(wn);Zr=l(qh,"pt"),qh.forEach(a),so=l(zs," for PyTorch, or "),kn=r(zs,"CODE",{});var Ah=o(kn);ao=l(Ah,"tf"),Ah.forEach(a),eo=l(zs," for TensorFlow:"),zs.forEach(a),Gt=h(s),j(ds.$$.fragment,s),Vt=h(s),ns=r(s,"H2",{class:!0});var dp=o(ns);gs=r(dp,"A",{id:!0,class:!0,href:!0});var Ph=o(gs);xn=r(Ph,"SPAN",{});var Th=o(xn);j(Zs.$$.fragment,Th),Th.forEach(a),Ph.forEach(a),no=h(dp),En=r(dp,"SPAN",{});var Ch=o(En);to=l(Ch,"Audio"),Ch.forEach(a),dp.forEach(a),Kt=h(s),_s=r(s,"P",{});var gp=o(_s);lo=l(gp,"For audio tasks, you\u2019ll need a "),$e=r(gp,"A",{href:!0});var zh=o($e);po=l(zh,"feature extractor"),zh.forEach(a),ro=l(gp," to prepare your dataset for the model. The feature extractor is designed to extract features from raw audio data, and convert them into tensors."),gp.forEach(a),Qt=h(s),W=r(s,"P",{});var Ue=o(W);oo=l(Ue,"Load the "),sa=r(Ue,"A",{href:!0,rel:!0});var Dh=o(sa);uo=l(Dh,"MInDS-14"),Dh.forEach(a),co=l(Ue," dataset (see the \u{1F917} "),aa=r(Ue,"A",{href:!0,rel:!0});var Oh=o(aa);ho=l(Oh,"Datasets tutorial"),Oh.forEach(a),mo=l(Ue," for more details on how to load a dataset) to see how you can use a feature extractor with audio datasets:"),Ue.forEach(a),Xt=h(s),j(ea.$$.fragment,s),Zt=h(s),J=r(s,"P",{});var Ye=o(J);io=l(Ye,"Access the first element of the "),qn=r(Ye,"CODE",{});var Lh=o(qn);bo=l(Lh,"audio"),Lh.forEach(a),fo=l(Ye," column to take a look at the input. Calling the "),An=r(Ye,"CODE",{});var Ih=o(An);jo=l(Ih,"audio"),Ih.forEach(a),go=l(Ye," column automatically loads and resamples the audio file:"),Ye.forEach(a),sl=h(s),j(na.$$.fragment,s),al=h(s),ve=r(s,"P",{});var Sh=o(ve);_o=l(Sh,"This returns three items:"),Sh.forEach(a),el=h(s),H=r(s,"UL",{});var Ge=o(H);ye=r(Ge,"LI",{});var Nc=o(ye);Pn=r(Nc,"CODE",{});var Nh=o(Pn);$o=l(Nh,"array"),Nh.forEach(a),vo=l(Nc," is the speech signal loaded - and potentially resampled - as a 1D array."),Nc.forEach(a),yo=h(Ge),we=r(Ge,"LI",{});var Fc=o(we);Tn=r(Fc,"CODE",{});var Fh=o(Tn);wo=l(Fh,"path"),Fh.forEach(a),ko=l(Fc," points to the location of the audio file."),Fc.forEach(a),xo=h(Ge),ke=r(Ge,"LI",{});var Rc=o(ke);Cn=r(Rc,"CODE",{});var Rh=o(Cn);Eo=l(Rh,"sampling_rate"),Rh.forEach(a),qo=l(Rc," refers to how many data points in the speech signal are measured per second."),Rc.forEach(a),Ge.forEach(a),nl=h(s),$s=r(s,"P",{});var _p=o($s);Ao=l(_p,"For this tutorial, you\u2019ll use the "),ta=r(_p,"A",{href:!0,rel:!0});var Bh=o(ta);Po=l(Bh,"Wav2Vec2"),Bh.forEach(a),To=l(_p," model. Take a look at the model card, and you\u2019ll learn Wav2Vec2 is pretrained on 16kHz sampled speech audio. It is important your audio data\u2019s sampling rate matches the sampling rate of the dataset used to pretrain the model. If your data\u2019s sampling rate isn\u2019t the same, then you need to resample your data."),_p.forEach(a),tl=h(s),xe=r(s,"OL",{});var Mh=o(xe);la=r(Mh,"LI",{});var $p=o(la);Co=l($p,"Use \u{1F917} Datasets\u2019 "),pa=r($p,"A",{href:!0,rel:!0});var Wh=o(pa);zo=l(Wh,"cast_column"),Wh.forEach(a),Do=l($p," method to upsample the sampling rate to 16kHz:"),$p.forEach(a),Mh.forEach(a),ll=h(s),j(ra.$$.fragment,s),pl=h(s),oa=r(s,"OL",{start:!0});var Jh=o(oa);ua=r(Jh,"LI",{});var vp=o(ua);Oo=l(vp,"Call the "),zn=r(vp,"CODE",{});var Hh=o(zn);Lo=l(Hh,"audio"),Hh.forEach(a),Io=l(vp," column again to resample the audio file:"),vp.forEach(a),Jh.forEach(a),rl=h(s),j(ca.$$.fragment,s),ol=h(s),C=r(s,"P",{});var Ds=o(C);So=l(Ds,"Next, load a feature extractor to normalize and pad the input. When padding textual data, a "),Dn=r(Ds,"CODE",{});var Uh=o(Dn);No=l(Uh,"0"),Uh.forEach(a),Fo=l(Ds," is added for shorter sequences. The same idea applies to audio data. The feature extractor adds a "),On=r(Ds,"CODE",{});var Yh=o(On);Ro=l(Yh,"0"),Yh.forEach(a),Bo=l(Ds," - interpreted as silence - to "),Ln=r(Ds,"CODE",{});var Gh=o(Ln);Mo=l(Gh,"array"),Gh.forEach(a),Wo=l(Ds,"."),Ds.forEach(a),ul=h(s),vs=r(s,"P",{});var yp=o(vs);Jo=l(yp,"Load the feature extractor with "),Ee=r(yp,"A",{href:!0});var Vh=o(Ee);Ho=l(Vh,"AutoFeatureExtractor.from_pretrained()"),Vh.forEach(a),Uo=l(yp,":"),yp.forEach(a),cl=h(s),j(ha.$$.fragment,s),hl=h(s),U=r(s,"P",{});var Ve=o(U);Yo=l(Ve,"Pass the audio "),In=r(Ve,"CODE",{});var Kh=o(In);Go=l(Kh,"array"),Kh.forEach(a),Vo=l(Ve," to the feature extractor. We also recommend adding the "),Sn=r(Ve,"CODE",{});var Qh=o(Sn);Ko=l(Qh,"sampling_rate"),Qh.forEach(a),Qo=l(Ve," argument in the feature extractor in order to better debug any silent errors that may occur."),Ve.forEach(a),ml=h(s),j(ma.$$.fragment,s),il=h(s),qe=r(s,"P",{});var Xh=o(qe);Xo=l(Xh,"Just like the tokenizer, you can apply padding or truncation to handle variable sequences in a batch. Take a look at the sequence length of these two audio samples:"),Xh.forEach(a),bl=h(s),j(ia.$$.fragment,s),fl=h(s),Ae=r(s,"P",{});var Zh=o(Ae);Zo=l(Zh,"Create a function to preprocess the dataset so the audio samples are the same lengths. Specify a maximum sample length, and the feature extractor will either pad or truncate the sequences to match it:"),Zh.forEach(a),jl=h(s),j(ba.$$.fragment,s),dl=h(s),ys=r(s,"P",{});var wp=o(ys);su=l(wp,"Apply the "),Nn=r(wp,"CODE",{});var sm=o(Nn);au=l(sm,"preprocess_function"),sm.forEach(a),eu=l(wp," to the the first few examples in the dataset:"),wp.forEach(a),gl=h(s),j(fa.$$.fragment,s),_l=h(s),Pe=r(s,"P",{});var am=o(Pe);nu=l(am,"The sample lengths are now the same and match the specified maximum length. You can pass your processed dataset to the model now!"),am.forEach(a),$l=h(s),j(ja.$$.fragment,s),vl=h(s),ts=r(s,"H2",{class:!0});var kp=o(ts);ws=r(kp,"A",{id:!0,class:!0,href:!0});var em=o(ws);Fn=r(em,"SPAN",{});var nm=o(Fn);j(da.$$.fragment,nm),nm.forEach(a),em.forEach(a),tu=h(kp),Rn=r(kp,"SPAN",{});var tm=o(Rn);lu=l(tm,"Computer vision"),tm.forEach(a),kp.forEach(a),yl=h(s),ks=r(s,"P",{});var xp=o(ks);pu=l(xp,"For computer vision tasks, you\u2019ll need a "),Te=r(xp,"A",{href:!0});var lm=o(Te);ru=l(lm,"feature extractor"),lm.forEach(a),ou=l(xp," to prepare your dataset for the model. The feature extractor is designed to extract features from images, and convert them into tensors."),xp.forEach(a),wl=h(s),Y=r(s,"P",{});var Ke=o(Y);uu=l(Ke,"Load the "),ga=r(Ke,"A",{href:!0,rel:!0});var pm=o(ga);cu=l(pm,"food101"),pm.forEach(a),hu=l(Ke," dataset (see the \u{1F917} "),_a=r(Ke,"A",{href:!0,rel:!0});var rm=o(_a);mu=l(rm,"Datasets tutorial"),rm.forEach(a),iu=l(Ke," for more details on how to load a dataset) to see how you can use a feature extractor with computer vision datasets:"),Ke.forEach(a),kl=h(s),j(xs.$$.fragment,s),xl=h(s),j($a.$$.fragment,s),El=h(s),Es=r(s,"P",{});var Ep=o(Es);bu=l(Ep,"Next, take a look at the image with \u{1F917} Datasets "),va=r(Ep,"A",{href:!0,rel:!0});var om=o(va);Bn=r(om,"CODE",{});var um=o(Bn);fu=l(um,"Image"),um.forEach(a),om.forEach(a),ju=l(Ep," feature:"),Ep.forEach(a),ql=h(s),j(ya.$$.fragment,s),Al=h(s),wa=r(s,"DIV",{class:!0});var cm=o(wa);Mn=r(cm,"IMG",{src:!0}),cm.forEach(a),Pl=h(s),qs=r(s,"P",{});var qp=o(qs);du=l(qp,"Load the feature extractor with "),Ce=r(qp,"A",{href:!0});var hm=o(Ce);gu=l(hm,"AutoFeatureExtractor.from_pretrained()"),hm.forEach(a),_u=l(qp,":"),qp.forEach(a),Tl=h(s),j(ka.$$.fragment,s),Cl=h(s),z=r(s,"P",{});var Os=o(z);$u=l(Os,"For computer vision tasks, it is common to add some type of data augmentation to the images as a part of preprocessing. You can add augmentations with any library you\u2019d like, but in this tutorial, you\u2019ll use torchvision\u2019s "),xa=r(Os,"A",{href:!0,rel:!0});var mm=o(xa);Wn=r(mm,"CODE",{});var im=o(Wn);vu=l(im,"transforms"),im.forEach(a),mm.forEach(a),yu=l(Os," module. If you\u2019re interested in using another data augmentation library, learn how in the "),Ea=r(Os,"A",{href:!0,rel:!0});var bm=o(Ea);wu=l(bm,"Albumentations"),bm.forEach(a),ku=l(Os," or "),qa=r(Os,"A",{href:!0,rel:!0});var fm=o(qa);xu=l(fm,"Kornia notebooks"),fm.forEach(a),Eu=l(Os,"."),Os.forEach(a),zl=h(s),ze=r(s,"OL",{});var jm=o(ze);O=r(jm,"LI",{});var Ls=o(O);qu=l(Ls,"Normalize the image with the feature extractor and use "),Aa=r(Ls,"A",{href:!0,rel:!0});var dm=o(Aa);Jn=r(dm,"CODE",{});var gm=o(Jn);Au=l(gm,"Compose"),gm.forEach(a),dm.forEach(a),Pu=l(Ls," to chain some transforms - "),Pa=r(Ls,"A",{href:!0,rel:!0});var _m=o(Pa);Hn=r(_m,"CODE",{});var $m=o(Hn);Tu=l($m,"RandomResizedCrop"),$m.forEach(a),_m.forEach(a),Cu=l(Ls," and "),Ta=r(Ls,"A",{href:!0,rel:!0});var vm=o(Ta);Un=r(vm,"CODE",{});var ym=o(Un);zu=l(ym,"ColorJitter"),ym.forEach(a),vm.forEach(a),Du=l(Ls," - together:"),Ls.forEach(a),jm.forEach(a),Dl=h(s),j(Ca.$$.fragment,s),Ol=h(s),za=r(s,"OL",{start:!0});var wm=o(za);ls=r(wm,"LI",{});var Qe=o(ls);Ou=l(Qe,"The model accepts "),De=r(Qe,"A",{href:!0});var km=o(De);Yn=r(km,"CODE",{});var xm=o(Yn);Lu=l(xm,"pixel_values"),xm.forEach(a),km.forEach(a),Iu=l(Qe," as its input, which is generated by the feature extractor. Create a function that generates "),Gn=r(Qe,"CODE",{});var Em=o(Gn);Su=l(Em,"pixel_values"),Em.forEach(a),Nu=l(Qe," from the transforms:"),Qe.forEach(a),wm.forEach(a),Ll=h(s),j(Da.$$.fragment,s),Il=h(s),Oa=r(s,"OL",{start:!0});var qm=o(Oa);La=r(qm,"LI",{});var Ap=o(La);Fu=l(Ap,"Then use \u{1F917} Datasets "),Ia=r(Ap,"A",{href:!0,rel:!0});var Am=o(Ia);Vn=r(Am,"CODE",{});var Pm=o(Vn);Ru=l(Pm,"set_transform"),Pm.forEach(a),Am.forEach(a),Bu=l(Ap," to apply the transforms on the fly:"),Ap.forEach(a),qm.forEach(a),Sl=h(s),j(Sa.$$.fragment,s),Nl=h(s),Na=r(s,"OL",{start:!0});var Tm=o(Na);Fa=r(Tm,"LI",{});var Pp=o(Fa);Mu=l(Pp,"Now when you access the image, you\u2019ll notice the feature extractor has added "),Kn=r(Pp,"CODE",{});var Cm=o(Kn);Wu=l(Cm,"pixel_values"),Cm.forEach(a),Ju=l(Pp,". You can pass your processed dataset to the model now!"),Pp.forEach(a),Tm.forEach(a),Fl=h(s),j(Ra.$$.fragment,s),Rl=h(s),Oe=r(s,"P",{});var zm=o(Oe);Hu=l(zm,"Here is what the image looks like after the transforms are applied. The image has been randomly cropped and it\u2019s color properties are different."),zm.forEach(a),Bl=h(s),j(Ba.$$.fragment,s),Ml=h(s),Ma=r(s,"DIV",{class:!0});var Dm=o(Ma);Qn=r(Dm,"IMG",{src:!0}),Dm.forEach(a),Wl=h(s),ps=r(s,"H2",{class:!0});var Tp=o(ps);As=r(Tp,"A",{id:!0,class:!0,href:!0});var Om=o(As);Xn=r(Om,"SPAN",{});var Lm=o(Xn);j(Wa.$$.fragment,Lm),Lm.forEach(a),Om.forEach(a),Uu=h(Tp),Zn=r(Tp,"SPAN",{});var Im=o(Zn);Yu=l(Im,"Multimodal"),Im.forEach(a),Tp.forEach(a),Jl=h(s),Ps=r(s,"P",{});var Cp=o(Ps);Gu=l(Cp,"For tasks involving multimodal inputs, you\u2019ll need a "),Le=r(Cp,"A",{href:!0});var Sm=o(Le);Vu=l(Sm,"processor"),Sm.forEach(a),Ku=l(Cp," to prepare your dataset for the model. A processor couples a tokenizer and feature extractor."),Cp.forEach(a),Hl=h(s),G=r(s,"P",{});var Xe=o(G);Qu=l(Xe,"Load the "),Ja=r(Xe,"A",{href:!0,rel:!0});var Nm=o(Ja);Xu=l(Nm,"LJ Speech"),Nm.forEach(a),Zu=l(Xe," dataset (see the \u{1F917} "),Ha=r(Xe,"A",{href:!0,rel:!0});var Fm=o(Ha);sc=l(Fm,"Datasets tutorial"),Fm.forEach(a),ac=l(Xe," for more details on how to load a dataset) to see how you can use a processor for automatic speech recognition (ASR):"),Xe.forEach(a),Ul=h(s),j(Ua.$$.fragment,s),Yl=h(s),V=r(s,"P",{});var Ze=o(V);ec=l(Ze,"For ASR, you\u2019re mainly focused on "),st=r(Ze,"CODE",{});var Rm=o(st);nc=l(Rm,"audio"),Rm.forEach(a),tc=l(Ze," and "),at=r(Ze,"CODE",{});var Bm=o(at);lc=l(Bm,"text"),Bm.forEach(a),pc=l(Ze," so you can remove the other columns:"),Ze.forEach(a),Gl=h(s),j(Ya.$$.fragment,s),Vl=h(s),K=r(s,"P",{});var sn=o(K);rc=l(sn,"Now take a look at the "),et=r(sn,"CODE",{});var Mm=o(et);oc=l(Mm,"audio"),Mm.forEach(a),uc=l(sn," and "),nt=r(sn,"CODE",{});var Wm=o(nt);cc=l(Wm,"text"),Wm.forEach(a),hc=l(sn," columns:"),sn.forEach(a),Kl=h(s),j(Ga.$$.fragment,s),Ql=h(s),Ts=r(s,"P",{});var zp=o(Ts);mc=l(zp,"Remember you should always "),Ie=r(zp,"A",{href:!0});var Jm=o(Ie);ic=l(Jm,"resample"),Jm.forEach(a),bc=l(zp," your audio dataset\u2019s sampling rate to match the sampling rate of the dataset used to pretrain a model!"),zp.forEach(a),Xl=h(s),j(Va.$$.fragment,s),Zl=h(s),Cs=r(s,"P",{});var Dp=o(Cs);fc=l(Dp,"Load a processor with "),Se=r(Dp,"A",{href:!0});var Hm=o(Se);jc=l(Hm,"AutoProcessor.from_pretrained()"),Hm.forEach(a),dc=l(Dp,":"),Dp.forEach(a),sp=h(s),j(Ka.$$.fragment,s),ap=h(s),Ne=r(s,"OL",{});var Um=o(Ne);P=r(Um,"LI",{});var X=o(P);gc=l(X,"Create a function to process the audio data contained in "),tt=r(X,"CODE",{});var Ym=o(tt);_c=l(Ym,"array"),Ym.forEach(a),$c=l(X," to "),lt=r(X,"CODE",{});var Gm=o(lt);vc=l(Gm,"input_values"),Gm.forEach(a),yc=l(X,", and tokenize "),pt=r(X,"CODE",{});var Vm=o(pt);wc=l(Vm,"text"),Vm.forEach(a),kc=l(X," to "),rt=r(X,"CODE",{});var Km=o(rt);xc=l(Km,"labels"),Km.forEach(a),Ec=l(X,". These are the inputs to the model:"),X.forEach(a),Um.forEach(a),ep=h(s),j(Qa.$$.fragment,s),np=h(s),Xa=r(s,"OL",{start:!0});var Qm=o(Xa);Za=r(Qm,"LI",{});var Op=o(Za);qc=l(Op,"Apply the "),ot=r(Op,"CODE",{});var Xm=o(ot);Ac=l(Xm,"prepare_dataset"),Xm.forEach(a),Pc=l(Op," function to a sample:"),Op.forEach(a),Qm.forEach(a),tp=h(s),j(se.$$.fragment,s),lp=h(s),Q=r(s,"P",{});var an=o(Q);Tc=l(an,"The processor has now added "),ut=r(an,"CODE",{});var Zm=o(ut);Cc=l(Zm,"input_values"),Zm.forEach(a),zc=l(an," and "),ct=r(an,"CODE",{});var si=o(ct);Dc=l(si,"labels"),si.forEach(a),Oc=l(an,", and the sampling rate has also been correctly downsampled to 16kHz. You can pass your processed dataset to the model now!"),an.forEach(a),this.h()},h(){m(i,"name","hf:doc:metadata"),m(i,"content",JSON.stringify(vi)),m(y,"id","preprocess"),m(y,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),m(y,"href","#preprocess"),m(b,"class","relative group"),m(ne,"href","./main_classes/tokenizer"),m(te,"href","./main_classes/feature_extractor"),m(le,"href","./main_classes/processors"),m(os,"id","natural-language-processing"),m(os,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),m(os,"href","#natural-language-processing"),m(Z,"class","relative group"),m(re,"href","main_classes/tokenizer"),m(oe,"href","/docs/transformers/pr_19907/en/model_doc/auto#transformers.AutoTokenizer.from_pretrained"),m(me,"href","glossary#input-ids"),m(be,"href","glossary#attention-mask"),m(je,"href","glossary#token-type-ids"),m(hs,"id","pad"),m(hs,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),m(hs,"href","#pad"),m(ss,"class","relative group"),m(bs,"id","truncation"),m(bs,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),m(bs,"href","#truncation"),m(as,"class","relative group"),m(js,"id","build-tensors"),m(js,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),m(js,"href","#build-tensors"),m(es,"class","relative group"),m(gs,"id","audio"),m(gs,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),m(gs,"href","#audio"),m(ns,"class","relative group"),m($e,"href","main_classes/feature_extractor"),m(sa,"href","https://huggingface.co/datasets/PolyAI/minds14"),m(sa,"rel","nofollow"),m(aa,"href","https://huggingface.co/docs/datasets/load_hub.html"),m(aa,"rel","nofollow"),m(ta,"href","https://huggingface.co/facebook/wav2vec2-base"),m(ta,"rel","nofollow"),m(pa,"href","https://huggingface.co/docs/datasets/main/en/package_reference/main_classes#datasets.Dataset.cast_column"),m(pa,"rel","nofollow"),m(oa,"start","2"),m(Ee,"href","/docs/transformers/pr_19907/en/model_doc/auto#transformers.AutoFeatureExtractor.from_pretrained"),m(ws,"id","computer-vision"),m(ws,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),m(ws,"href","#computer-vision"),m(ts,"class","relative group"),m(Te,"href","main_classes/feature_extractor"),m(ga,"href","https://huggingface.co/datasets/food101"),m(ga,"rel","nofollow"),m(_a,"href","https://huggingface.co/docs/datasets/load_hub.html"),m(_a,"rel","nofollow"),m(va,"href","https://huggingface.co/docs/datasets/package_reference/main_classes.html?highlight=image#datasets.Image"),m(va,"rel","nofollow"),ai(Mn.src,Bc="https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/vision-preprocess-tutorial.png")||m(Mn,"src",Bc),m(wa,"class","flex justify-center"),m(Ce,"href","/docs/transformers/pr_19907/en/model_doc/auto#transformers.AutoFeatureExtractor.from_pretrained"),m(xa,"href","https://pytorch.org/vision/stable/transforms.html"),m(xa,"rel","nofollow"),m(Ea,"href","https://colab.research.google.com/github/huggingface/notebooks/blob/main/examples/image_classification_albumentations.ipynb"),m(Ea,"rel","nofollow"),m(qa,"href","https://colab.research.google.com/github/huggingface/notebooks/blob/main/examples/image_classification_kornia.ipynb"),m(qa,"rel","nofollow"),m(Aa,"href","https://pytorch.org/vision/master/generated/torchvision.transforms.Compose.html"),m(Aa,"rel","nofollow"),m(Pa,"href","https://pytorch.org/vision/main/generated/torchvision.transforms.RandomResizedCrop.html"),m(Pa,"rel","nofollow"),m(Ta,"href","https://pytorch.org/vision/main/generated/torchvision.transforms.ColorJitter.html"),m(Ta,"rel","nofollow"),m(De,"href","model_doc/visionencoderdecoder#transformers.VisionEncoderDecoderModel.forward.pixel_values"),m(za,"start","2"),m(Ia,"href","https://huggingface.co/docs/datasets/process.html#format-transform"),m(Ia,"rel","nofollow"),m(Oa,"start","3"),m(Na,"start","4"),ai(Qn.src,Mc="https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/preprocessed_image.png")||m(Qn,"src",Mc),m(Ma,"class","flex justify-center"),m(As,"id","multimodal"),m(As,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),m(As,"href","#multimodal"),m(ps,"class","relative group"),m(Le,"href","main_classes/processors"),m(Ja,"href","https://huggingface.co/datasets/lj_speech"),m(Ja,"rel","nofollow"),m(Ha,"href","https://huggingface.co/docs/datasets/load_hub.html"),m(Ha,"rel","nofollow"),m(Ie,"href","preprocessing#audio"),m(Se,"href","/docs/transformers/pr_19907/en/model_doc/auto#transformers.AutoProcessor.from_pretrained"),m(Xa,"start","2")},m(s,n){e(document.head,i),u(s,v,n),u(s,b,n),e(b,y),e(y,k),d(x,k,null),e(b,E),e(b,q),e(q,D),u(s,Ss,n),d(L,s,n),u(s,ft,n),u(s,ee,n),e(ee,Ip),u(s,jt,n),u(s,I,n),e(I,Ns),e(Ns,Sp),e(Ns,ne),e(ne,Np),e(Ns,Fp),e(I,Rp),e(I,Fs),e(Fs,Bp),e(Fs,te),e(te,Mp),e(Fs,Wp),e(I,Jp),e(I,Rs),e(Rs,Hp),e(Rs,le),e(le,Up),e(Rs,Yp),u(s,dt,n),d(rs,s,n),u(s,gt,n),u(s,pe,n),e(pe,Gp),u(s,_t,n),d(Bs,s,n),u(s,$t,n),u(s,Z,n),e(Z,os),e(os,en),d(Ms,en,null),e(Z,Vp),e(Z,nn),e(nn,Kp),u(s,vt,n),d(Ws,s,n),u(s,yt,n),u(s,S,n),e(S,Qp),e(S,re),e(re,Xp),e(S,Zp),e(S,tn),e(tn,sr),e(S,ar),u(s,wt,n),d(us,s,n),u(s,kt,n),u(s,N,n),e(N,er),e(N,oe),e(oe,nr),e(N,tr),e(N,ln),e(ln,lr),e(N,pr),u(s,xt,n),d(Js,s,n),u(s,Et,n),u(s,ue,n),e(ue,rr),u(s,qt,n),d(Hs,s,n),u(s,At,n),u(s,ce,n),e(ce,or),u(s,Pt,n),u(s,F,n),e(F,he),e(he,me),e(me,ur),e(he,cr),e(F,hr),e(F,ie),e(ie,be),e(be,mr),e(ie,ir),e(F,br),e(F,fe),e(fe,je),e(je,fr),e(fe,jr),u(s,Tt,n),u(s,cs,n),e(cs,dr),e(cs,pn),e(pn,gr),e(cs,_r),u(s,Ct,n),d(Us,s,n),u(s,zt,n),u(s,R,n),e(R,$r),e(R,rn),e(rn,vr),e(R,yr),e(R,on),e(on,wr),e(R,kr),u(s,Dt,n),u(s,de,n),e(de,xr),u(s,Ot,n),d(Ys,s,n),u(s,Lt,n),u(s,ss,n),e(ss,hs),e(hs,un),d(Gs,un,null),e(ss,Er),e(ss,cn),e(cn,qr),u(s,It,n),u(s,ms,n),e(ms,Ar),e(ms,hn),e(hn,Pr),e(ms,Tr),u(s,St,n),u(s,B,n),e(B,Cr),e(B,mn),e(mn,zr),e(B,Dr),e(B,bn),e(bn,Or),e(B,Lr),u(s,Nt,n),d(Vs,s,n),u(s,Ft,n),u(s,is,n),e(is,Ir),e(is,fn),e(fn,Sr),e(is,Nr),u(s,Rt,n),u(s,as,n),e(as,bs),e(bs,jn),d(Ks,jn,null),e(as,Fr),e(as,dn),e(dn,Rr),u(s,Bt,n),u(s,ge,n),e(ge,Br),u(s,Mt,n),u(s,M,n),e(M,Mr),e(M,gn),e(gn,Wr),e(M,Jr),e(M,_n),e(_n,Hr),e(M,Ur),u(s,Wt,n),d(Qs,s,n),u(s,Jt,n),d(fs,s,n),u(s,Ht,n),u(s,es,n),e(es,js),e(js,$n),d(Xs,$n,null),e(es,Yr),e(es,vn),e(vn,Gr),u(s,Ut,n),u(s,_e,n),e(_e,Vr),u(s,Yt,n),u(s,T,n),e(T,Kr),e(T,yn),e(yn,Qr),e(T,Xr),e(T,wn),e(wn,Zr),e(T,so),e(T,kn),e(kn,ao),e(T,eo),u(s,Gt,n),d(ds,s,n),u(s,Vt,n),u(s,ns,n),e(ns,gs),e(gs,xn),d(Zs,xn,null),e(ns,no),e(ns,En),e(En,to),u(s,Kt,n),u(s,_s,n),e(_s,lo),e(_s,$e),e($e,po),e(_s,ro),u(s,Qt,n),u(s,W,n),e(W,oo),e(W,sa),e(sa,uo),e(W,co),e(W,aa),e(aa,ho),e(W,mo),u(s,Xt,n),d(ea,s,n),u(s,Zt,n),u(s,J,n),e(J,io),e(J,qn),e(qn,bo),e(J,fo),e(J,An),e(An,jo),e(J,go),u(s,sl,n),d(na,s,n),u(s,al,n),u(s,ve,n),e(ve,_o),u(s,el,n),u(s,H,n),e(H,ye),e(ye,Pn),e(Pn,$o),e(ye,vo),e(H,yo),e(H,we),e(we,Tn),e(Tn,wo),e(we,ko),e(H,xo),e(H,ke),e(ke,Cn),e(Cn,Eo),e(ke,qo),u(s,nl,n),u(s,$s,n),e($s,Ao),e($s,ta),e(ta,Po),e($s,To),u(s,tl,n),u(s,xe,n),e(xe,la),e(la,Co),e(la,pa),e(pa,zo),e(la,Do),u(s,ll,n),d(ra,s,n),u(s,pl,n),u(s,oa,n),e(oa,ua),e(ua,Oo),e(ua,zn),e(zn,Lo),e(ua,Io),u(s,rl,n),d(ca,s,n),u(s,ol,n),u(s,C,n),e(C,So),e(C,Dn),e(Dn,No),e(C,Fo),e(C,On),e(On,Ro),e(C,Bo),e(C,Ln),e(Ln,Mo),e(C,Wo),u(s,ul,n),u(s,vs,n),e(vs,Jo),e(vs,Ee),e(Ee,Ho),e(vs,Uo),u(s,cl,n),d(ha,s,n),u(s,hl,n),u(s,U,n),e(U,Yo),e(U,In),e(In,Go),e(U,Vo),e(U,Sn),e(Sn,Ko),e(U,Qo),u(s,ml,n),d(ma,s,n),u(s,il,n),u(s,qe,n),e(qe,Xo),u(s,bl,n),d(ia,s,n),u(s,fl,n),u(s,Ae,n),e(Ae,Zo),u(s,jl,n),d(ba,s,n),u(s,dl,n),u(s,ys,n),e(ys,su),e(ys,Nn),e(Nn,au),e(ys,eu),u(s,gl,n),d(fa,s,n),u(s,_l,n),u(s,Pe,n),e(Pe,nu),u(s,$l,n),d(ja,s,n),u(s,vl,n),u(s,ts,n),e(ts,ws),e(ws,Fn),d(da,Fn,null),e(ts,tu),e(ts,Rn),e(Rn,lu),u(s,yl,n),u(s,ks,n),e(ks,pu),e(ks,Te),e(Te,ru),e(ks,ou),u(s,wl,n),u(s,Y,n),e(Y,uu),e(Y,ga),e(ga,cu),e(Y,hu),e(Y,_a),e(_a,mu),e(Y,iu),u(s,kl,n),d(xs,s,n),u(s,xl,n),d($a,s,n),u(s,El,n),u(s,Es,n),e(Es,bu),e(Es,va),e(va,Bn),e(Bn,fu),e(Es,ju),u(s,ql,n),d(ya,s,n),u(s,Al,n),u(s,wa,n),e(wa,Mn),u(s,Pl,n),u(s,qs,n),e(qs,du),e(qs,Ce),e(Ce,gu),e(qs,_u),u(s,Tl,n),d(ka,s,n),u(s,Cl,n),u(s,z,n),e(z,$u),e(z,xa),e(xa,Wn),e(Wn,vu),e(z,yu),e(z,Ea),e(Ea,wu),e(z,ku),e(z,qa),e(qa,xu),e(z,Eu),u(s,zl,n),u(s,ze,n),e(ze,O),e(O,qu),e(O,Aa),e(Aa,Jn),e(Jn,Au),e(O,Pu),e(O,Pa),e(Pa,Hn),e(Hn,Tu),e(O,Cu),e(O,Ta),e(Ta,Un),e(Un,zu),e(O,Du),u(s,Dl,n),d(Ca,s,n),u(s,Ol,n),u(s,za,n),e(za,ls),e(ls,Ou),e(ls,De),e(De,Yn),e(Yn,Lu),e(ls,Iu),e(ls,Gn),e(Gn,Su),e(ls,Nu),u(s,Ll,n),d(Da,s,n),u(s,Il,n),u(s,Oa,n),e(Oa,La),e(La,Fu),e(La,Ia),e(Ia,Vn),e(Vn,Ru),e(La,Bu),u(s,Sl,n),d(Sa,s,n),u(s,Nl,n),u(s,Na,n),e(Na,Fa),e(Fa,Mu),e(Fa,Kn),e(Kn,Wu),e(Fa,Ju),u(s,Fl,n),d(Ra,s,n),u(s,Rl,n),u(s,Oe,n),e(Oe,Hu),u(s,Bl,n),d(Ba,s,n),u(s,Ml,n),u(s,Ma,n),e(Ma,Qn),u(s,Wl,n),u(s,ps,n),e(ps,As),e(As,Xn),d(Wa,Xn,null),e(ps,Uu),e(ps,Zn),e(Zn,Yu),u(s,Jl,n),u(s,Ps,n),e(Ps,Gu),e(Ps,Le),e(Le,Vu),e(Ps,Ku),u(s,Hl,n),u(s,G,n),e(G,Qu),e(G,Ja),e(Ja,Xu),e(G,Zu),e(G,Ha),e(Ha,sc),e(G,ac),u(s,Ul,n),d(Ua,s,n),u(s,Yl,n),u(s,V,n),e(V,ec),e(V,st),e(st,nc),e(V,tc),e(V,at),e(at,lc),e(V,pc),u(s,Gl,n),d(Ya,s,n),u(s,Vl,n),u(s,K,n),e(K,rc),e(K,et),e(et,oc),e(K,uc),e(K,nt),e(nt,cc),e(K,hc),u(s,Kl,n),d(Ga,s,n),u(s,Ql,n),u(s,Ts,n),e(Ts,mc),e(Ts,Ie),e(Ie,ic),e(Ts,bc),u(s,Xl,n),d(Va,s,n),u(s,Zl,n),u(s,Cs,n),e(Cs,fc),e(Cs,Se),e(Se,jc),e(Cs,dc),u(s,sp,n),d(Ka,s,n),u(s,ap,n),u(s,Ne,n),e(Ne,P),e(P,gc),e(P,tt),e(tt,_c),e(P,$c),e(P,lt),e(lt,vc),e(P,yc),e(P,pt),e(pt,wc),e(P,kc),e(P,rt),e(rt,xc),e(P,Ec),u(s,ep,n),d(Qa,s,n),u(s,np,n),u(s,Xa,n),e(Xa,Za),e(Za,qc),e(Za,ot),e(ot,Ac),e(Za,Pc),u(s,tp,n),d(se,s,n),u(s,lp,n),u(s,Q,n),e(Q,Tc),e(Q,ut),e(ut,Cc),e(Q,zc),e(Q,ct),e(ct,Dc),e(Q,Oc),pp=!0},p(s,[n]){const ae={};n&2&&(ae.$$scope={dirty:n,ctx:s}),rs.$set(ae);const ht={};n&2&&(ht.$$scope={dirty:n,ctx:s}),us.$set(ht);const mt={};n&2&&(mt.$$scope={dirty:n,ctx:s}),fs.$set(mt);const it={};n&2&&(it.$$scope={dirty:n,ctx:s}),ds.$set(it);const bt={};n&2&&(bt.$$scope={dirty:n,ctx:s}),xs.$set(bt)},i(s){pp||(g(x.$$.fragment,s),g(L.$$.fragment,s),g(rs.$$.fragment,s),g(Bs.$$.fragment,s),g(Ms.$$.fragment,s),g(Ws.$$.fragment,s),g(us.$$.fragment,s),g(Js.$$.fragment,s),g(Hs.$$.fragment,s),g(Us.$$.fragment,s),g(Ys.$$.fragment,s),g(Gs.$$.fragment,s),g(Vs.$$.fragment,s),g(Ks.$$.fragment,s),g(Qs.$$.fragment,s),g(fs.$$.fragment,s),g(Xs.$$.fragment,s),g(ds.$$.fragment,s),g(Zs.$$.fragment,s),g(ea.$$.fragment,s),g(na.$$.fragment,s),g(ra.$$.fragment,s),g(ca.$$.fragment,s),g(ha.$$.fragment,s),g(ma.$$.fragment,s),g(ia.$$.fragment,s),g(ba.$$.fragment,s),g(fa.$$.fragment,s),g(ja.$$.fragment,s),g(da.$$.fragment,s),g(xs.$$.fragment,s),g($a.$$.fragment,s),g(ya.$$.fragment,s),g(ka.$$.fragment,s),g(Ca.$$.fragment,s),g(Da.$$.fragment,s),g(Sa.$$.fragment,s),g(Ra.$$.fragment,s),g(Ba.$$.fragment,s),g(Wa.$$.fragment,s),g(Ua.$$.fragment,s),g(Ya.$$.fragment,s),g(Ga.$$.fragment,s),g(Va.$$.fragment,s),g(Ka.$$.fragment,s),g(Qa.$$.fragment,s),g(se.$$.fragment,s),pp=!0)},o(s){_(x.$$.fragment,s),_(L.$$.fragment,s),_(rs.$$.fragment,s),_(Bs.$$.fragment,s),_(Ms.$$.fragment,s),_(Ws.$$.fragment,s),_(us.$$.fragment,s),_(Js.$$.fragment,s),_(Hs.$$.fragment,s),_(Us.$$.fragment,s),_(Ys.$$.fragment,s),_(Gs.$$.fragment,s),_(Vs.$$.fragment,s),_(Ks.$$.fragment,s),_(Qs.$$.fragment,s),_(fs.$$.fragment,s),_(Xs.$$.fragment,s),_(ds.$$.fragment,s),_(Zs.$$.fragment,s),_(ea.$$.fragment,s),_(na.$$.fragment,s),_(ra.$$.fragment,s),_(ca.$$.fragment,s),_(ha.$$.fragment,s),_(ma.$$.fragment,s),_(ia.$$.fragment,s),_(ba.$$.fragment,s),_(fa.$$.fragment,s),_(ja.$$.fragment,s),_(da.$$.fragment,s),_(xs.$$.fragment,s),_($a.$$.fragment,s),_(ya.$$.fragment,s),_(ka.$$.fragment,s),_(Ca.$$.fragment,s),_(Da.$$.fragment,s),_(Sa.$$.fragment,s),_(Ra.$$.fragment,s),_(Ba.$$.fragment,s),_(Wa.$$.fragment,s),_(Ua.$$.fragment,s),_(Ya.$$.fragment,s),_(Ga.$$.fragment,s),_(Va.$$.fragment,s),_(Ka.$$.fragment,s),_(Qa.$$.fragment,s),_(se.$$.fragment,s),pp=!1},d(s){a(i),s&&a(v),s&&a(b),$(x),s&&a(Ss),$(L,s),s&&a(ft),s&&a(ee),s&&a(jt),s&&a(I),s&&a(dt),$(rs,s),s&&a(gt),s&&a(pe),s&&a(_t),$(Bs,s),s&&a($t),s&&a(Z),$(Ms),s&&a(vt),$(Ws,s),s&&a(yt),s&&a(S),s&&a(wt),$(us,s),s&&a(kt),s&&a(N),s&&a(xt),$(Js,s),s&&a(Et),s&&a(ue),s&&a(qt),$(Hs,s),s&&a(At),s&&a(ce),s&&a(Pt),s&&a(F),s&&a(Tt),s&&a(cs),s&&a(Ct),$(Us,s),s&&a(zt),s&&a(R),s&&a(Dt),s&&a(de),s&&a(Ot),$(Ys,s),s&&a(Lt),s&&a(ss),$(Gs),s&&a(It),s&&a(ms),s&&a(St),s&&a(B),s&&a(Nt),$(Vs,s),s&&a(Ft),s&&a(is),s&&a(Rt),s&&a(as),$(Ks),s&&a(Bt),s&&a(ge),s&&a(Mt),s&&a(M),s&&a(Wt),$(Qs,s),s&&a(Jt),$(fs,s),s&&a(Ht),s&&a(es),$(Xs),s&&a(Ut),s&&a(_e),s&&a(Yt),s&&a(T),s&&a(Gt),$(ds,s),s&&a(Vt),s&&a(ns),$(Zs),s&&a(Kt),s&&a(_s),s&&a(Qt),s&&a(W),s&&a(Xt),$(ea,s),s&&a(Zt),s&&a(J),s&&a(sl),$(na,s),s&&a(al),s&&a(ve),s&&a(el),s&&a(H),s&&a(nl),s&&a($s),s&&a(tl),s&&a(xe),s&&a(ll),$(ra,s),s&&a(pl),s&&a(oa),s&&a(rl),$(ca,s),s&&a(ol),s&&a(C),s&&a(ul),s&&a(vs),s&&a(cl),$(ha,s),s&&a(hl),s&&a(U),s&&a(ml),$(ma,s),s&&a(il),s&&a(qe),s&&a(bl),$(ia,s),s&&a(fl),s&&a(Ae),s&&a(jl),$(ba,s),s&&a(dl),s&&a(ys),s&&a(gl),$(fa,s),s&&a(_l),s&&a(Pe),s&&a($l),$(ja,s),s&&a(vl),s&&a(ts),$(da),s&&a(yl),s&&a(ks),s&&a(wl),s&&a(Y),s&&a(kl),$(xs,s),s&&a(xl),$($a,s),s&&a(El),s&&a(Es),s&&a(ql),$(ya,s),s&&a(Al),s&&a(wa),s&&a(Pl),s&&a(qs),s&&a(Tl),$(ka,s),s&&a(Cl),s&&a(z),s&&a(zl),s&&a(ze),s&&a(Dl),$(Ca,s),s&&a(Ol),s&&a(za),s&&a(Ll),$(Da,s),s&&a(Il),s&&a(Oa),s&&a(Sl),$(Sa,s),s&&a(Nl),s&&a(Na),s&&a(Fl),$(Ra,s),s&&a(Rl),s&&a(Oe),s&&a(Bl),$(Ba,s),s&&a(Ml),s&&a(Ma),s&&a(Wl),s&&a(ps),$(Wa),s&&a(Jl),s&&a(Ps),s&&a(Hl),s&&a(G),s&&a(Ul),$(Ua,s),s&&a(Yl),s&&a(V),s&&a(Gl),$(Ya,s),s&&a(Vl),s&&a(K),s&&a(Kl),$(Ga,s),s&&a(Ql),s&&a(Ts),s&&a(Xl),$(Va,s),s&&a(Zl),s&&a(Cs),s&&a(sp),$(Ka,s),s&&a(ap),s&&a(Ne),s&&a(ep),$(Qa,s),s&&a(np),s&&a(Xa),s&&a(tp),$(se,s),s&&a(lp),s&&a(Q)}}}const vi={local:"preprocess",sections:[{local:"natural-language-processing",sections:[{local:"pad",title:"Pad"},{local:"truncation",title:"Truncation"},{local:"build-tensors",title:"Build tensors"}],title:"Natural Language Processing"},{local:"audio",title:"Audio"},{local:"computer-vision",title:"Computer vision"},{local:"multimodal",title:"Multimodal"}],title:"Preprocess"};function yi(A){return oi(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class Ti extends ti{constructor(i){super();li(this,i,yi,$i,pi,{})}}export{Ti as default,vi as metadata};
