import{S as Yo,i as Qo,s as Xo,e as l,k as h,w as b,t as i,M as sc,c as r,d as e,m,a as t,x as d,h as u,b as o,N as Uo,G as a,g as p,y as j,q as f,o as g,B as v,v as ec,L as Go}from"../chunks/vendor-hf-doc-builder.js";import{T as ac}from"../chunks/Tip-hf-doc-builder.js";import{Y as nc}from"../chunks/Youtube-hf-doc-builder.js";import{I as E}from"../chunks/IconCopyLink-hf-doc-builder.js";import{C as k}from"../chunks/CodeBlock-hf-doc-builder.js";import{D as lc}from"../chunks/DocNotebookDropdown-hf-doc-builder.js";import{F as rc,M as Zo}from"../chunks/Markdown-hf-doc-builder.js";function tc(y){let _,$,c,w,z;return{c(){_=l("p"),$=i("Wenn Sie ein vortrainiertes Modell verwenden m\xF6chten, ist es wichtig, den zugeh\xF6rigen vortrainierten Tokenizer zu verwenden. Dadurch wird sichergestellt, dass der Text auf die gleiche Weise aufgeteilt wird wie das Pretraining-Korpus und die gleichen entsprechenden Token-zu-Index (in der Regel als "),c=l("em"),w=i("vocab"),z=i(" bezeichnet) w\xE4hrend des Pretrainings verwendet werden.")},l(S){_=r(S,"P",{});var q=t(_);$=u(q,"Wenn Sie ein vortrainiertes Modell verwenden m\xF6chten, ist es wichtig, den zugeh\xF6rigen vortrainierten Tokenizer zu verwenden. Dadurch wird sichergestellt, dass der Text auf die gleiche Weise aufgeteilt wird wie das Pretraining-Korpus und die gleichen entsprechenden Token-zu-Index (in der Regel als "),c=r(q,"EM",{});var as=t(c);w=u(as,"vocab"),as.forEach(e),z=u(q," bezeichnet) w\xE4hrend des Pretrainings verwendet werden."),q.forEach(e)},m(S,q){p(S,_,q),a(_,$),a(_,c),a(c,w),a(_,z)},d(S){S&&e(_)}}}function pc(y){let _,$;return _=new k({props:{code:`batch_sentences = [
    "But what about second breakfast?",
    "Don't think he knows about second breakfast, Pip.",
    "What about elevensies?",
]
encoded_input = tokenizer(batch_sentences, padding=True, truncation=True, return_tensors="pt")
print(encoded_input)`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>batch_sentences = [
<span class="hljs-meta">... </span>    <span class="hljs-string">&quot;But what about second breakfast?&quot;</span>,
<span class="hljs-meta">... </span>    <span class="hljs-string">&quot;Don&#x27;t think he knows about second breakfast, Pip.&quot;</span>,
<span class="hljs-meta">... </span>    <span class="hljs-string">&quot;What about elevensies?&quot;</span>,
<span class="hljs-meta">... </span>]
<span class="hljs-meta">&gt;&gt;&gt; </span>encoded_input = tokenizer(batch_sentences, padding=<span class="hljs-literal">True</span>, truncation=<span class="hljs-literal">True</span>, return_tensors=<span class="hljs-string">&quot;pt&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-built_in">print</span>(encoded_input)
{<span class="hljs-string">&#x27;input_ids&#x27;</span>: tensor([[<span class="hljs-number">101</span>, <span class="hljs-number">1252</span>, <span class="hljs-number">1184</span>, <span class="hljs-number">1164</span>, <span class="hljs-number">1248</span>, <span class="hljs-number">6462</span>, <span class="hljs-number">136</span>, <span class="hljs-number">102</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>],
                      [<span class="hljs-number">101</span>, <span class="hljs-number">1790</span>, <span class="hljs-number">112</span>, <span class="hljs-number">189</span>, <span class="hljs-number">1341</span>, <span class="hljs-number">1119</span>, <span class="hljs-number">3520</span>, <span class="hljs-number">1164</span>, <span class="hljs-number">1248</span>, <span class="hljs-number">6462</span>, <span class="hljs-number">117</span>, <span class="hljs-number">21902</span>, <span class="hljs-number">1643</span>, <span class="hljs-number">119</span>, <span class="hljs-number">102</span>],
                      [<span class="hljs-number">101</span>, <span class="hljs-number">1327</span>, <span class="hljs-number">1164</span>, <span class="hljs-number">5450</span>, <span class="hljs-number">23434</span>, <span class="hljs-number">136</span>, <span class="hljs-number">102</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>]]), 
 <span class="hljs-string">&#x27;token_type_ids&#x27;</span>: tensor([[<span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>],
                           [<span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>],
                           [<span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>]]), 
 <span class="hljs-string">&#x27;attention_mask&#x27;</span>: tensor([[<span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>],
                           [<span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>],
                           [<span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>]])}`}}),{c(){b(_.$$.fragment)},l(c){d(_.$$.fragment,c)},m(c,w){j(_,c,w),$=!0},p:Go,i(c){$||(f(_.$$.fragment,c),$=!0)},o(c){g(_.$$.fragment,c),$=!1},d(c){v(_,c)}}}function ic(y){let _,$;return _=new Zo({props:{$$slots:{default:[pc]},$$scope:{ctx:y}}}),{c(){b(_.$$.fragment)},l(c){d(_.$$.fragment,c)},m(c,w){j(_,c,w),$=!0},p(c,w){const z={};w&2&&(z.$$scope={dirty:w,ctx:c}),_.$set(z)},i(c){$||(f(_.$$.fragment,c),$=!0)},o(c){g(_.$$.fragment,c),$=!1},d(c){v(_,c)}}}function uc(y){let _,$;return _=new k({props:{code:`batch_sentences = [
    "But what about second breakfast?",
    "Don't think he knows about second breakfast, Pip.",
    "What about elevensies?",
]
encoded_input = tokenizer(batch_sentences, padding=True, truncation=True, return_tensors="tf")
print(encoded_input)`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>batch_sentences = [
<span class="hljs-meta">... </span>    <span class="hljs-string">&quot;But what about second breakfast?&quot;</span>,
<span class="hljs-meta">... </span>    <span class="hljs-string">&quot;Don&#x27;t think he knows about second breakfast, Pip.&quot;</span>,
<span class="hljs-meta">... </span>    <span class="hljs-string">&quot;What about elevensies?&quot;</span>,
<span class="hljs-meta">... </span>]
<span class="hljs-meta">&gt;&gt;&gt; </span>encoded_input = tokenizer(batch_sentences, padding=<span class="hljs-literal">True</span>, truncation=<span class="hljs-literal">True</span>, return_tensors=<span class="hljs-string">&quot;tf&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-built_in">print</span>(encoded_input)
{<span class="hljs-string">&#x27;input_ids&#x27;</span>: &lt;tf.Tensor: shape=(<span class="hljs-number">2</span>, <span class="hljs-number">9</span>), dtype=int32, numpy=
array([[<span class="hljs-number">101</span>, <span class="hljs-number">1252</span>, <span class="hljs-number">1184</span>, <span class="hljs-number">1164</span>, <span class="hljs-number">1248</span>, <span class="hljs-number">6462</span>, <span class="hljs-number">136</span>, <span class="hljs-number">102</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>],
       [<span class="hljs-number">101</span>, <span class="hljs-number">1790</span>, <span class="hljs-number">112</span>, <span class="hljs-number">189</span>, <span class="hljs-number">1341</span>, <span class="hljs-number">1119</span>, <span class="hljs-number">3520</span>, <span class="hljs-number">1164</span>, <span class="hljs-number">1248</span>, <span class="hljs-number">6462</span>, <span class="hljs-number">117</span>, <span class="hljs-number">21902</span>, <span class="hljs-number">1643</span>, <span class="hljs-number">119</span>, <span class="hljs-number">102</span>],
       [<span class="hljs-number">101</span>, <span class="hljs-number">1327</span>, <span class="hljs-number">1164</span>, <span class="hljs-number">5450</span>, <span class="hljs-number">23434</span>, <span class="hljs-number">136</span>, <span class="hljs-number">102</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>]],
      dtype=int32)&gt;, 
 <span class="hljs-string">&#x27;token_type_ids&#x27;</span>: &lt;tf.Tensor: shape=(<span class="hljs-number">2</span>, <span class="hljs-number">9</span>), dtype=int32, numpy=
array([[<span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>],
       [<span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>],
       [<span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>]], dtype=int32)&gt;, 
 <span class="hljs-string">&#x27;attention_mask&#x27;</span>: &lt;tf.Tensor: shape=(<span class="hljs-number">2</span>, <span class="hljs-number">9</span>), dtype=int32, numpy=
array([[<span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>],
       [<span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>],
       [<span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>]], dtype=int32)&gt;}`}}),{c(){b(_.$$.fragment)},l(c){d(_.$$.fragment,c)},m(c,w){j(_,c,w),$=!0},p:Go,i(c){$||(f(_.$$.fragment,c),$=!0)},o(c){g(_.$$.fragment,c),$=!1},d(c){v(_,c)}}}function hc(y){let _,$;return _=new Zo({props:{$$slots:{default:[uc]},$$scope:{ctx:y}}}),{c(){b(_.$$.fragment)},l(c){d(_.$$.fragment,c)},m(c,w){j(_,c,w),$=!0},p(c,w){const z={};w&2&&(z.$$scope={dirty:w,ctx:c}),_.$set(z)},i(c){$||(f(_.$$.fragment,c),$=!0)},o(c){g(_.$$.fragment,c),$=!1},d(c){v(_,c)}}}function mc(y){let _,$,c,w,z,S,q,as,Kt,ol,Fs,cl,Qe,Jt,bl,P,Ga,Ut,Gt,Za,Zt,Yt,Ya,Qt,dl,N,ns,Qa,Cs,Xt,Xa,sp,jl,Ns,fl,D,ep,Xe,ap,np,sn,lp,rp,gl,ls,vl,rs,tp,en,pp,ip,_l,W,ts,an,Ws,up,nn,hp,kl,ps,mp,ln,op,cp,$l,Os,wl,sa,bp,zl,Rs,El,ea,dp,Sl,M,aa,na,jp,fp,gp,la,ra,vp,_p,kp,ta,pa,$p,wp,xl,is,zp,rn,Ep,Sp,Al,Hs,yl,L,xp,tn,Ap,yp,pn,Tp,qp,Tl,ia,Pp,ql,Vs,Pl,O,us,un,Ks,Dp,hn,Mp,Dl,hs,Lp,mn,Bp,Ip,Ml,ua,Fp,Ll,Js,Bl,ha,Cp,Il,R,ms,on,Us,Np,cn,Wp,Fl,ma,Op,Cl,oa,Rp,Nl,Gs,Wl,H,os,bn,Zs,Hp,dn,Vp,Ol,ca,Kp,Rl,x,Jp,jn,Up,Gp,fn,Zp,Yp,gn,Qp,Xp,Hl,cs,Vl,V,bs,vn,Ys,si,_n,ei,Kl,ds,ai,ba,ni,li,Jl,Qs,Ul,B,ri,Xs,ti,pi,se,ii,ui,Gl,ee,Zl,js,hi,kn,mi,oi,Yl,ae,Ql,da,ci,Xl,I,$n,bi,di,wn,ji,fi,ja,zn,gi,vi,sr,K,fs,En,ne,_i,Sn,ki,er,gs,$i,le,wi,zi,ar,vs,Ei,re,Si,xi,nr,te,lr,fa,xn,Ai,rr,pe,tr,ie,An,yi,pr,ue,ir,ga,Ti,ur,J,_s,yn,he,qi,Tn,Pi,hr,A,Di,qn,Mi,Li,Pn,Bi,Ii,Dn,Fi,Ci,mr,ks,Ni,Mn,Wi,Oi,or,me,cr,$s,Ri,Ln,Hi,Vi,br,oe,dr,U,ws,Bn,ce,Ki,In,Ji,jr,va,Ui,fr,be,gr,_a,Gi,vr,de,_r,ka,Zi,kr,je,$r,$a,Yi,wr,fe,zr,wa,Qi,Er,G,zs,Fn,ge,Xi,Cn,su,Sr,za,eu,xr,F,au,ve,nu,lu,Nn,ru,tu,Ar,_e,yr,Es,pu,ke,iu,uu,Tr,$e,qr,Ea,Sa,jh,Pr,Z,Ss,Wn,we,hu,On,mu,Dr,xs,ou,Rn,cu,bu,Mr,ze,Lr,Y,As,Hn,Ee,du,Vn,ju,Br,ys,fu,Se,Kn,gu,vu,Ir,xa,T,_u,xe,Jn,ku,$u,Ae,Un,wu,zu,ye,Gn,Eu,Su,Fr,Te,Cr,qe,Q,xu,Aa,Zn,Au,yu,Yn,Tu,qu,Nr,Pe,Wr,De,Me,Pu,Le,Qn,Du,Mu,Or,Be,Rr,Ie,Xn,Lu,Hr,Fe,Vr,ya,Bu,Kr,Ce,Jr,Ta,qa,fh,Ur,X,Ts,sl,Ne,Iu,el,Fu,Gr,Pa,Cu,Zr,qs,al,Nu,Wu,nl,Ou,Yr,Ps,Ru,We,Hu,Vu,Qr,Oe,Xr,Da,Ku,st,Re,et,Ma,Ju,at,He,nt,Ds,Uu,La,Gu,Zu,lt,Ve,rt,ss,Ms,ll,Ke,Yu,rl,Qu,tt,Ba,Xu,pt,Je,it,Ia,es,sh,tl,eh,ah,pl,nh,lh,ut,Ue,ht,Ge,il,rh,mt,Ze,ot,C,th,ul,ph,ih,hl,uh,hh,ct,Fa,mh,bt;return S=new E({}),Fs=new lc({props:{classNames:"absolute z-10 right-0 top-0",options:[{label:"Mixed",value:"https://colab.research.google.com/github/huggingface/notebooks/blob/main/transformers_doc/de/preprocessing.ipynb"},{label:"PyTorch",value:"https://colab.research.google.com/github/huggingface/notebooks/blob/main/transformers_doc/de/pytorch/preprocessing.ipynb"},{label:"TensorFlow",value:"https://colab.research.google.com/github/huggingface/notebooks/blob/main/transformers_doc/de/tensorflow/preprocessing.ipynb"},{label:"Mixed",value:"https://studiolab.sagemaker.aws/import/github/huggingface/notebooks/blob/main/transformers_doc/de/preprocessing.ipynb"},{label:"PyTorch",value:"https://studiolab.sagemaker.aws/import/github/huggingface/notebooks/blob/main/transformers_doc/de/pytorch/preprocessing.ipynb"},{label:"TensorFlow",value:"https://studiolab.sagemaker.aws/import/github/huggingface/notebooks/blob/main/transformers_doc/de/tensorflow/preprocessing.ipynb"}]}}),Cs=new E({}),Ns=new nc({props:{id:"Yffk5aydLzg"}}),ls=new ac({props:{$$slots:{default:[tc]},$$scope:{ctx:y}}}),Ws=new E({}),Os=new k({props:{code:`from transformers import AutoTokenizer

tokenizer = AutoTokenizer.from_pretrained("bert-base-cased")`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoTokenizer

<span class="hljs-meta">&gt;&gt;&gt; </span>tokenizer = AutoTokenizer.from_pretrained(<span class="hljs-string">&quot;bert-base-cased&quot;</span>)`}}),Rs=new k({props:{code:`encoded_input = tokenizer("Do not meddle in the affairs of wizards, for they are subtle and quick to anger.")
print(encoded_input)`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>encoded_input = tokenizer(<span class="hljs-string">&quot;Do not meddle in the affairs of wizards, for they are subtle and quick to anger.&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-built_in">print</span>(encoded_input)
{<span class="hljs-string">&#x27;input_ids&#x27;</span>: [<span class="hljs-number">101</span>, <span class="hljs-number">2079</span>, <span class="hljs-number">2025</span>, <span class="hljs-number">19960</span>, <span class="hljs-number">10362</span>, <span class="hljs-number">1999</span>, <span class="hljs-number">1996</span>, <span class="hljs-number">3821</span>, <span class="hljs-number">1997</span>, <span class="hljs-number">16657</span>, <span class="hljs-number">1010</span>, <span class="hljs-number">2005</span>, <span class="hljs-number">2027</span>, <span class="hljs-number">2024</span>, <span class="hljs-number">11259</span>, <span class="hljs-number">1998</span>, <span class="hljs-number">4248</span>, <span class="hljs-number">2000</span>, <span class="hljs-number">4963</span>, <span class="hljs-number">1012</span>, <span class="hljs-number">102</span>], 
 <span class="hljs-string">&#x27;token_type_ids&#x27;</span>: [<span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>], 
 <span class="hljs-string">&#x27;attention_mask&#x27;</span>: [<span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>]}`}}),Hs=new k({props:{code:'tokenizer.decode(encoded_input["input_ids"])',highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>tokenizer.decode(encoded_input[<span class="hljs-string">&quot;input_ids&quot;</span>])
<span class="hljs-string">&#x27;[CLS] Do not meddle in the affairs of wizards, for they are subtle and quick to anger. [SEP]&#x27;</span>`}}),Vs=new k({props:{code:`batch_sentences = [
    "But what about second breakfast?",
    "Don't think he knows about second breakfast, Pip.",
    "What about elevensies?",
]
encoded_inputs = tokenizer(batch_sentences)
print(encoded_inputs)`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>batch_sentences = [
<span class="hljs-meta">... </span>    <span class="hljs-string">&quot;But what about second breakfast?&quot;</span>,
<span class="hljs-meta">... </span>    <span class="hljs-string">&quot;Don&#x27;t think he knows about second breakfast, Pip.&quot;</span>,
<span class="hljs-meta">... </span>    <span class="hljs-string">&quot;What about elevensies?&quot;</span>,
<span class="hljs-meta">... </span>]
<span class="hljs-meta">&gt;&gt;&gt; </span>encoded_inputs = tokenizer(batch_sentences)
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-built_in">print</span>(encoded_inputs)
{<span class="hljs-string">&#x27;input_ids&#x27;</span>: [[<span class="hljs-number">101</span>, <span class="hljs-number">1252</span>, <span class="hljs-number">1184</span>, <span class="hljs-number">1164</span>, <span class="hljs-number">1248</span>, <span class="hljs-number">6462</span>, <span class="hljs-number">136</span>, <span class="hljs-number">102</span>], 
               [<span class="hljs-number">101</span>, <span class="hljs-number">1790</span>, <span class="hljs-number">112</span>, <span class="hljs-number">189</span>, <span class="hljs-number">1341</span>, <span class="hljs-number">1119</span>, <span class="hljs-number">3520</span>, <span class="hljs-number">1164</span>, <span class="hljs-number">1248</span>, <span class="hljs-number">6462</span>, <span class="hljs-number">117</span>, <span class="hljs-number">21902</span>, <span class="hljs-number">1643</span>, <span class="hljs-number">119</span>, <span class="hljs-number">102</span>], 
               [<span class="hljs-number">101</span>, <span class="hljs-number">1327</span>, <span class="hljs-number">1164</span>, <span class="hljs-number">5450</span>, <span class="hljs-number">23434</span>, <span class="hljs-number">136</span>, <span class="hljs-number">102</span>]], 
 <span class="hljs-string">&#x27;token_type_ids&#x27;</span>: [[<span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>], 
                    [<span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>], 
                    [<span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>]], 
 <span class="hljs-string">&#x27;attention_mask&#x27;</span>: [[<span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>], 
                    [<span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>], 
                    [<span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>]]}`}}),Ks=new E({}),Js=new k({props:{code:`batch_sentences = [
    "But what about second breakfast?",
    "Don't think he knows about second breakfast, Pip.",
    "What about elevensies?",
]
encoded_input = tokenizer(batch_sentences, padding=True)
print(encoded_input)`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>batch_sentences = [
<span class="hljs-meta">... </span>    <span class="hljs-string">&quot;But what about second breakfast?&quot;</span>,
<span class="hljs-meta">... </span>    <span class="hljs-string">&quot;Don&#x27;t think he knows about second breakfast, Pip.&quot;</span>,
<span class="hljs-meta">... </span>    <span class="hljs-string">&quot;What about elevensies?&quot;</span>,
<span class="hljs-meta">... </span>]
<span class="hljs-meta">&gt;&gt;&gt; </span>encoded_input = tokenizer(batch_sentences, padding=<span class="hljs-literal">True</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-built_in">print</span>(encoded_input)
{<span class="hljs-string">&#x27;input_ids&#x27;</span>: [[<span class="hljs-number">101</span>, <span class="hljs-number">1252</span>, <span class="hljs-number">1184</span>, <span class="hljs-number">1164</span>, <span class="hljs-number">1248</span>, <span class="hljs-number">6462</span>, <span class="hljs-number">136</span>, <span class="hljs-number">102</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>], 
               [<span class="hljs-number">101</span>, <span class="hljs-number">1790</span>, <span class="hljs-number">112</span>, <span class="hljs-number">189</span>, <span class="hljs-number">1341</span>, <span class="hljs-number">1119</span>, <span class="hljs-number">3520</span>, <span class="hljs-number">1164</span>, <span class="hljs-number">1248</span>, <span class="hljs-number">6462</span>, <span class="hljs-number">117</span>, <span class="hljs-number">21902</span>, <span class="hljs-number">1643</span>, <span class="hljs-number">119</span>, <span class="hljs-number">102</span>], 
               [<span class="hljs-number">101</span>, <span class="hljs-number">1327</span>, <span class="hljs-number">1164</span>, <span class="hljs-number">5450</span>, <span class="hljs-number">23434</span>, <span class="hljs-number">136</span>, <span class="hljs-number">102</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>]], 
 <span class="hljs-string">&#x27;token_type_ids&#x27;</span>: [[<span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>], 
                    [<span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>], 
                    [<span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>]], 
 <span class="hljs-string">&#x27;attention_mask&#x27;</span>: [[<span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>], 
                    [<span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>], 
                    [<span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>]]}`}}),Us=new E({}),Gs=new k({props:{code:`batch_sentences = [
    "But what about second breakfast?",
    "Don't think he knows about second breakfast, Pip.",
    "What about elevensies?",
]
encoded_input = tokenizer(batch_sentences, padding=True, truncation=True)
print(encoded_input)`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>batch_sentences = [
<span class="hljs-meta">... </span>    <span class="hljs-string">&quot;But what about second breakfast?&quot;</span>,
<span class="hljs-meta">... </span>    <span class="hljs-string">&quot;Don&#x27;t think he knows about second breakfast, Pip.&quot;</span>,
<span class="hljs-meta">... </span>    <span class="hljs-string">&quot;What about elevensies?&quot;</span>,
<span class="hljs-meta">... </span>]
<span class="hljs-meta">&gt;&gt;&gt; </span>encoded_input = tokenizer(batch_sentences, padding=<span class="hljs-literal">True</span>, truncation=<span class="hljs-literal">True</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-built_in">print</span>(encoded_input)
{<span class="hljs-string">&#x27;input_ids&#x27;</span>: [[<span class="hljs-number">101</span>, <span class="hljs-number">1252</span>, <span class="hljs-number">1184</span>, <span class="hljs-number">1164</span>, <span class="hljs-number">1248</span>, <span class="hljs-number">6462</span>, <span class="hljs-number">136</span>, <span class="hljs-number">102</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>], 
               [<span class="hljs-number">101</span>, <span class="hljs-number">1790</span>, <span class="hljs-number">112</span>, <span class="hljs-number">189</span>, <span class="hljs-number">1341</span>, <span class="hljs-number">1119</span>, <span class="hljs-number">3520</span>, <span class="hljs-number">1164</span>, <span class="hljs-number">1248</span>, <span class="hljs-number">6462</span>, <span class="hljs-number">117</span>, <span class="hljs-number">21902</span>, <span class="hljs-number">1643</span>, <span class="hljs-number">119</span>, <span class="hljs-number">102</span>], 
               [<span class="hljs-number">101</span>, <span class="hljs-number">1327</span>, <span class="hljs-number">1164</span>, <span class="hljs-number">5450</span>, <span class="hljs-number">23434</span>, <span class="hljs-number">136</span>, <span class="hljs-number">102</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>]], 
 <span class="hljs-string">&#x27;token_type_ids&#x27;</span>: [[<span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>], 
                    [<span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>], 
                    [<span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>]], 
 <span class="hljs-string">&#x27;attention_mask&#x27;</span>: [[<span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>], 
                    [<span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>], 
                    [<span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>]]}`}}),Zs=new E({}),cs=new rc({props:{pytorch:!0,tensorflow:!0,jax:!1,$$slots:{tensorflow:[hc],pytorch:[ic]},$$scope:{ctx:y}}}),Ys=new E({}),Qs=new k({props:{code:"pip install datasets",highlighted:"pip install datasets"}}),ee=new k({props:{code:`from datasets import load_dataset, Audio

dataset = load_dataset("PolyAI/minds14", name="en-US", split="train")`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> datasets <span class="hljs-keyword">import</span> load_dataset, Audio

<span class="hljs-meta">&gt;&gt;&gt; </span>dataset = load_dataset(<span class="hljs-string">&quot;PolyAI/minds14&quot;</span>, name=<span class="hljs-string">&quot;en-US&quot;</span>, split=<span class="hljs-string">&quot;train&quot;</span>)`}}),ae=new k({props:{code:'dataset[0]["audio"]',highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>dataset[<span class="hljs-number">0</span>][<span class="hljs-string">&quot;audio&quot;</span>]
{<span class="hljs-string">&#x27;array&#x27;</span>: array([ <span class="hljs-number">0.</span>        ,  <span class="hljs-number">0.00024414</span>, -<span class="hljs-number">0.00024414</span>, ..., -<span class="hljs-number">0.00024414</span>,
         <span class="hljs-number">0.</span>        ,  <span class="hljs-number">0.</span>        ], dtype=float32),
 <span class="hljs-string">&#x27;path&#x27;</span>: <span class="hljs-string">&#x27;/root/.cache/huggingface/datasets/downloads/extracted/f14948e0e84be638dd7943ac36518a4cf3324e8b7aa331c5ab11541518e9368c/en-US~JOINT_ACCOUNT/602ba55abb1e6d0fbce92065.wav&#x27;</span>,
 <span class="hljs-string">&#x27;sampling_rate&#x27;</span>: <span class="hljs-number">8000</span>}`}}),ne=new E({}),te=new k({props:{code:`dataset = load_dataset("PolyAI/minds14", name="en-US", split="train")
dataset[0]["audio"]`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>dataset = load_dataset(<span class="hljs-string">&quot;PolyAI/minds14&quot;</span>, name=<span class="hljs-string">&quot;en-US&quot;</span>, split=<span class="hljs-string">&quot;train&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>dataset[<span class="hljs-number">0</span>][<span class="hljs-string">&quot;audio&quot;</span>]
{<span class="hljs-string">&#x27;array&#x27;</span>: array([ <span class="hljs-number">0.</span>        ,  <span class="hljs-number">0.00024414</span>, -<span class="hljs-number">0.00024414</span>, ..., -<span class="hljs-number">0.00024414</span>,
         <span class="hljs-number">0.</span>        ,  <span class="hljs-number">0.</span>        ], dtype=float32),
 <span class="hljs-string">&#x27;path&#x27;</span>: <span class="hljs-string">&#x27;/root/.cache/huggingface/datasets/downloads/extracted/f14948e0e84be638dd7943ac36518a4cf3324e8b7aa331c5ab11541518e9368c/en-US~JOINT_ACCOUNT/602ba55abb1e6d0fbce92065.wav&#x27;</span>,
 <span class="hljs-string">&#x27;sampling_rate&#x27;</span>: <span class="hljs-number">8000</span>}`}}),pe=new k({props:{code:'dataset = dataset.cast_column("audio", Audio(sampling_rate=16_000))',highlighted:'<span class="hljs-meta">&gt;&gt;&gt; </span>dataset = dataset.cast_column(<span class="hljs-string">&quot;audio&quot;</span>, Audio(sampling_rate=<span class="hljs-number">16_000</span>))'}}),ue=new k({props:{code:'dataset[0]["audio"]',highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>dataset[<span class="hljs-number">0</span>][<span class="hljs-string">&quot;audio&quot;</span>]
{<span class="hljs-string">&#x27;array&#x27;</span>: array([ <span class="hljs-number">2.3443763e-05</span>,  <span class="hljs-number">2.1729663e-04</span>,  <span class="hljs-number">2.2145823e-04</span>, ...,
         <span class="hljs-number">3.8356509e-05</span>, -<span class="hljs-number">7.3497440e-06</span>, -<span class="hljs-number">2.1754686e-05</span>], dtype=float32),
 <span class="hljs-string">&#x27;path&#x27;</span>: <span class="hljs-string">&#x27;/root/.cache/huggingface/datasets/downloads/extracted/f14948e0e84be638dd7943ac36518a4cf3324e8b7aa331c5ab11541518e9368c/en-US~JOINT_ACCOUNT/602ba55abb1e6d0fbce92065.wav&#x27;</span>,
 <span class="hljs-string">&#x27;sampling_rate&#x27;</span>: <span class="hljs-number">16000</span>}`}}),he=new E({}),me=new k({props:{code:`from transformers import AutoFeatureExtractor

feature_extractor = AutoFeatureExtractor.from_pretrained("facebook/wav2vec2-base")`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoFeatureExtractor

<span class="hljs-meta">&gt;&gt;&gt; </span>feature_extractor = AutoFeatureExtractor.from_pretrained(<span class="hljs-string">&quot;facebook/wav2vec2-base&quot;</span>)`}}),oe=new k({props:{code:`audio_input = [dataset[0]["audio"]["array"]]
feature_extractor(audio_input, sampling_rate=16000)`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>audio_input = [dataset[<span class="hljs-number">0</span>][<span class="hljs-string">&quot;audio&quot;</span>][<span class="hljs-string">&quot;array&quot;</span>]]
<span class="hljs-meta">&gt;&gt;&gt; </span>feature_extractor(audio_input, sampling_rate=<span class="hljs-number">16000</span>)
{<span class="hljs-string">&#x27;input_values&#x27;</span>: [array([ <span class="hljs-number">3.8106556e-04</span>,  <span class="hljs-number">2.7506407e-03</span>,  <span class="hljs-number">2.8015103e-03</span>, ...,
        <span class="hljs-number">5.6335266e-04</span>,  <span class="hljs-number">4.6588284e-06</span>, -<span class="hljs-number">1.7142107e-04</span>], dtype=float32)]}`}}),ce=new E({}),be=new k({props:{code:`dataset[0]["audio"]["array"].shape

dataset[1]["audio"]["array"].shape`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>dataset[<span class="hljs-number">0</span>][<span class="hljs-string">&quot;audio&quot;</span>][<span class="hljs-string">&quot;array&quot;</span>].shape
(<span class="hljs-number">173398</span>,)

<span class="hljs-meta">&gt;&gt;&gt; </span>dataset[<span class="hljs-number">1</span>][<span class="hljs-string">&quot;audio&quot;</span>][<span class="hljs-string">&quot;array&quot;</span>].shape
(<span class="hljs-number">106496</span>,)`}}),de=new k({props:{code:`def preprocess_function(examples):
    audio_arrays = [x["array"] for x in examples["audio"]]
    inputs = feature_extractor(
        audio_arrays,
        sampling_rate=16000,
        padding=True,
        max_length=100000,
        truncation=True,
    )
    return inputs`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">def</span> <span class="hljs-title function_">preprocess_function</span>(<span class="hljs-params">examples</span>):
<span class="hljs-meta">... </span>    audio_arrays = [x[<span class="hljs-string">&quot;array&quot;</span>] <span class="hljs-keyword">for</span> x <span class="hljs-keyword">in</span> examples[<span class="hljs-string">&quot;audio&quot;</span>]]
<span class="hljs-meta">... </span>    inputs = feature_extractor(
<span class="hljs-meta">... </span>        audio_arrays,
<span class="hljs-meta">... </span>        sampling_rate=<span class="hljs-number">16000</span>,
<span class="hljs-meta">... </span>        padding=<span class="hljs-literal">True</span>,
<span class="hljs-meta">... </span>        max_length=<span class="hljs-number">100000</span>,
<span class="hljs-meta">... </span>        truncation=<span class="hljs-literal">True</span>,
<span class="hljs-meta">... </span>    )
<span class="hljs-meta">... </span>    <span class="hljs-keyword">return</span> inputs`}}),je=new k({props:{code:"processed_dataset = preprocess_function(dataset[:5])",highlighted:'<span class="hljs-meta">&gt;&gt;&gt; </span>processed_dataset = preprocess_function(dataset[:<span class="hljs-number">5</span>])'}}),fe=new k({props:{code:`processed_dataset["input_values"][0].shape

processed_dataset["input_values"][1].shape`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>processed_dataset[<span class="hljs-string">&quot;input_values&quot;</span>][<span class="hljs-number">0</span>].shape
(<span class="hljs-number">100000</span>,)

<span class="hljs-meta">&gt;&gt;&gt; </span>processed_dataset[<span class="hljs-string">&quot;input_values&quot;</span>][<span class="hljs-number">1</span>].shape
(<span class="hljs-number">100000</span>,)`}}),ge=new E({}),_e=new k({props:{code:`from datasets import load_dataset

dataset = load_dataset("food101", split="train[:100]")`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> datasets <span class="hljs-keyword">import</span> load_dataset

<span class="hljs-meta">&gt;&gt;&gt; </span>dataset = load_dataset(<span class="hljs-string">&quot;food101&quot;</span>, split=<span class="hljs-string">&quot;train[:100]&quot;</span>)`}}),$e=new k({props:{code:'dataset[0]["image"]',highlighted:'<span class="hljs-meta">&gt;&gt;&gt; </span>dataset[<span class="hljs-number">0</span>][<span class="hljs-string">&quot;image&quot;</span>]'}}),we=new E({}),ze=new k({props:{code:`from transformers import AutoFeatureExtractor

feature_extractor = AutoFeatureExtractor.from_pretrained("google/vit-base-patch16-224")`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoFeatureExtractor

<span class="hljs-meta">&gt;&gt;&gt; </span>feature_extractor = AutoFeatureExtractor.from_pretrained(<span class="hljs-string">&quot;google/vit-base-patch16-224&quot;</span>)`}}),Ee=new E({}),Te=new k({props:{code:`from torchvision.transforms import Compose, Normalize, RandomResizedCrop, ColorJitter, ToTensor

normalize = Normalize(mean=feature_extractor.image_mean, std=feature_extractor.image_std)
_transforms = Compose(
    [RandomResizedCrop(feature_extractor.size), ColorJitter(brightness=0.5, hue=0.5), ToTensor(), normalize]
)`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> torchvision.transforms <span class="hljs-keyword">import</span> Compose, Normalize, RandomResizedCrop, ColorJitter, ToTensor

<span class="hljs-meta">&gt;&gt;&gt; </span>normalize = Normalize(mean=feature_extractor.image_mean, std=feature_extractor.image_std)
<span class="hljs-meta">&gt;&gt;&gt; </span>_transforms = Compose(
<span class="hljs-meta">... </span>    [RandomResizedCrop(feature_extractor.size), ColorJitter(brightness=<span class="hljs-number">0.5</span>, hue=<span class="hljs-number">0.5</span>), ToTensor(), normalize]
<span class="hljs-meta">... </span>)`}}),Pe=new k({props:{code:`def transforms(examples):
    examples["pixel_values"] = [_transforms(image.convert("RGB")) for image in examples["image"]]
    return examples`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">def</span> <span class="hljs-title function_">transforms</span>(<span class="hljs-params">examples</span>):
<span class="hljs-meta">... </span>    examples[<span class="hljs-string">&quot;pixel_values&quot;</span>] = [_transforms(image.convert(<span class="hljs-string">&quot;RGB&quot;</span>)) <span class="hljs-keyword">for</span> image <span class="hljs-keyword">in</span> examples[<span class="hljs-string">&quot;image&quot;</span>]]
<span class="hljs-meta">... </span>    <span class="hljs-keyword">return</span> examples`}}),Be=new k({props:{code:"dataset.set_transform(transforms)",highlighted:'<span class="hljs-meta">&gt;&gt;&gt; </span>dataset.set_transform(transforms)'}}),Fe=new k({props:{code:'dataset[0]["image"]',highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>dataset[<span class="hljs-number">0</span>][<span class="hljs-string">&quot;image&quot;</span>]
{<span class="hljs-string">&#x27;image&#x27;</span>: &lt;PIL.JpegImagePlugin.JpegImageFile image mode=RGB size=384x512 at <span class="hljs-number">0x7F1A7B0630D0</span>&gt;,
 <span class="hljs-string">&#x27;label&#x27;</span>: <span class="hljs-number">6</span>,
 <span class="hljs-string">&#x27;pixel_values&#x27;</span>: tensor([[[ <span class="hljs-number">0.0353</span>,  <span class="hljs-number">0.0745</span>,  <span class="hljs-number">0.1216</span>,  ..., -<span class="hljs-number">0.9922</span>, -<span class="hljs-number">0.9922</span>, -<span class="hljs-number">0.9922</span>],
          [-<span class="hljs-number">0.0196</span>,  <span class="hljs-number">0.0667</span>,  <span class="hljs-number">0.1294</span>,  ..., -<span class="hljs-number">0.9765</span>, -<span class="hljs-number">0.9843</span>, -<span class="hljs-number">0.9922</span>],
          [ <span class="hljs-number">0.0196</span>,  <span class="hljs-number">0.0824</span>,  <span class="hljs-number">0.1137</span>,  ..., -<span class="hljs-number">0.9765</span>, -<span class="hljs-number">0.9686</span>, -<span class="hljs-number">0.8667</span>],
          ...,
          [ <span class="hljs-number">0.0275</span>,  <span class="hljs-number">0.0745</span>,  <span class="hljs-number">0.0510</span>,  ..., -<span class="hljs-number">0.1137</span>, -<span class="hljs-number">0.1216</span>, -<span class="hljs-number">0.0824</span>],
          [ <span class="hljs-number">0.0667</span>,  <span class="hljs-number">0.0824</span>,  <span class="hljs-number">0.0667</span>,  ..., -<span class="hljs-number">0.0588</span>, -<span class="hljs-number">0.0745</span>, -<span class="hljs-number">0.0980</span>],
          [ <span class="hljs-number">0.0353</span>,  <span class="hljs-number">0.0353</span>,  <span class="hljs-number">0.0431</span>,  ..., -<span class="hljs-number">0.0039</span>, -<span class="hljs-number">0.0039</span>, -<span class="hljs-number">0.0588</span>]],
 
         [[ <span class="hljs-number">0.2078</span>,  <span class="hljs-number">0.2471</span>,  <span class="hljs-number">0.2863</span>,  ..., -<span class="hljs-number">0.9451</span>, -<span class="hljs-number">0.9373</span>, -<span class="hljs-number">0.9451</span>],
          [ <span class="hljs-number">0.1608</span>,  <span class="hljs-number">0.2471</span>,  <span class="hljs-number">0.3098</span>,  ..., -<span class="hljs-number">0.9373</span>, -<span class="hljs-number">0.9451</span>, -<span class="hljs-number">0.9373</span>],
          [ <span class="hljs-number">0.2078</span>,  <span class="hljs-number">0.2706</span>,  <span class="hljs-number">0.3020</span>,  ..., -<span class="hljs-number">0.9608</span>, -<span class="hljs-number">0.9373</span>, -<span class="hljs-number">0.8275</span>],
          ...,
          [-<span class="hljs-number">0.0353</span>,  <span class="hljs-number">0.0118</span>, -<span class="hljs-number">0.0039</span>,  ..., -<span class="hljs-number">0.2392</span>, -<span class="hljs-number">0.2471</span>, -<span class="hljs-number">0.2078</span>],
          [ <span class="hljs-number">0.0196</span>,  <span class="hljs-number">0.0353</span>,  <span class="hljs-number">0.0196</span>,  ..., -<span class="hljs-number">0.1843</span>, -<span class="hljs-number">0.2000</span>, -<span class="hljs-number">0.2235</span>],
          [-<span class="hljs-number">0.0118</span>, -<span class="hljs-number">0.0039</span>, -<span class="hljs-number">0.0039</span>,  ..., -<span class="hljs-number">0.0980</span>, -<span class="hljs-number">0.0980</span>, -<span class="hljs-number">0.1529</span>]],
 
         [[ <span class="hljs-number">0.3961</span>,  <span class="hljs-number">0.4431</span>,  <span class="hljs-number">0.4980</span>,  ..., -<span class="hljs-number">0.9216</span>, -<span class="hljs-number">0.9137</span>, -<span class="hljs-number">0.9216</span>],
          [ <span class="hljs-number">0.3569</span>,  <span class="hljs-number">0.4510</span>,  <span class="hljs-number">0.5216</span>,  ..., -<span class="hljs-number">0.9059</span>, -<span class="hljs-number">0.9137</span>, -<span class="hljs-number">0.9137</span>],
          [ <span class="hljs-number">0.4118</span>,  <span class="hljs-number">0.4745</span>,  <span class="hljs-number">0.5216</span>,  ..., -<span class="hljs-number">0.9137</span>, -<span class="hljs-number">0.8902</span>, -<span class="hljs-number">0.7804</span>],
          ...,
          [-<span class="hljs-number">0.2314</span>, -<span class="hljs-number">0.1922</span>, -<span class="hljs-number">0.2078</span>,  ..., -<span class="hljs-number">0.4196</span>, -<span class="hljs-number">0.4275</span>, -<span class="hljs-number">0.3882</span>],
          [-<span class="hljs-number">0.1843</span>, -<span class="hljs-number">0.1686</span>, -<span class="hljs-number">0.2000</span>,  ..., -<span class="hljs-number">0.3647</span>, -<span class="hljs-number">0.3804</span>, -<span class="hljs-number">0.4039</span>],
          [-<span class="hljs-number">0.1922</span>, -<span class="hljs-number">0.1922</span>, -<span class="hljs-number">0.1922</span>,  ..., -<span class="hljs-number">0.2941</span>, -<span class="hljs-number">0.2863</span>, -<span class="hljs-number">0.3412</span>]]])}`}}),Ce=new k({props:{code:`import numpy as np
import matplotlib.pyplot as plt

img = dataset[0]["pixel_values"]
plt.imshow(img.permute(1, 2, 0))`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">import</span> numpy <span class="hljs-keyword">as</span> np
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">import</span> matplotlib.pyplot <span class="hljs-keyword">as</span> plt

<span class="hljs-meta">&gt;&gt;&gt; </span>img = dataset[<span class="hljs-number">0</span>][<span class="hljs-string">&quot;pixel_values&quot;</span>]
<span class="hljs-meta">&gt;&gt;&gt; </span>plt.imshow(img.permute(<span class="hljs-number">1</span>, <span class="hljs-number">2</span>, <span class="hljs-number">0</span>))`}}),Ne=new E({}),Oe=new k({props:{code:`from datasets import load_dataset

lj_speech = load_dataset("lj_speech", split="train")`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> datasets <span class="hljs-keyword">import</span> load_dataset

<span class="hljs-meta">&gt;&gt;&gt; </span>lj_speech = load_dataset(<span class="hljs-string">&quot;lj_speech&quot;</span>, split=<span class="hljs-string">&quot;train&quot;</span>)`}}),Re=new k({props:{code:'lj_speech = lj_speech.map(remove_columns=["file", "id", "normalized_text"])',highlighted:'<span class="hljs-meta">&gt;&gt;&gt; </span>lj_speech = lj_speech.<span class="hljs-built_in">map</span>(remove_columns=[<span class="hljs-string">&quot;file&quot;</span>, <span class="hljs-string">&quot;id&quot;</span>, <span class="hljs-string">&quot;normalized_text&quot;</span>])'}}),He=new k({props:{code:`lj_speech[0]["audio"]

lj_speech[0]["text"]`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>lj_speech[<span class="hljs-number">0</span>][<span class="hljs-string">&quot;audio&quot;</span>]
{<span class="hljs-string">&#x27;array&#x27;</span>: array([-<span class="hljs-number">7.3242188e-04</span>, -<span class="hljs-number">7.6293945e-04</span>, -<span class="hljs-number">6.4086914e-04</span>, ...,
         <span class="hljs-number">7.3242188e-04</span>,  <span class="hljs-number">2.1362305e-04</span>,  <span class="hljs-number">6.1035156e-05</span>], dtype=float32),
 <span class="hljs-string">&#x27;path&#x27;</span>: <span class="hljs-string">&#x27;/root/.cache/huggingface/datasets/downloads/extracted/917ece08c95cf0c4115e45294e3cd0dee724a1165b7fc11798369308a465bd26/LJSpeech-1.1/wavs/LJ001-0001.wav&#x27;</span>,
 <span class="hljs-string">&#x27;sampling_rate&#x27;</span>: <span class="hljs-number">22050</span>}

<span class="hljs-meta">&gt;&gt;&gt; </span>lj_speech[<span class="hljs-number">0</span>][<span class="hljs-string">&quot;text&quot;</span>]
<span class="hljs-string">&#x27;Printing, in the only sense with which we are at present concerned, differs from most if not from all the arts and crafts represented in the Exhibition&#x27;</span>`}}),Ve=new k({props:{code:'lj_speech = lj_speech.cast_column("audio", Audio(sampling_rate=16_000))',highlighted:'<span class="hljs-meta">&gt;&gt;&gt; </span>lj_speech = lj_speech.cast_column(<span class="hljs-string">&quot;audio&quot;</span>, Audio(sampling_rate=<span class="hljs-number">16_000</span>))'}}),Ke=new E({}),Je=new k({props:{code:`from transformers import AutoProcessor

processor = AutoProcessor.from_pretrained("facebook/wav2vec2-base-960h")`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoProcessor

<span class="hljs-meta">&gt;&gt;&gt; </span>processor = AutoProcessor.from_pretrained(<span class="hljs-string">&quot;facebook/wav2vec2-base-960h&quot;</span>)`}}),Ue=new k({props:{code:`def prepare_dataset(example):
    audio = example["audio"]

    example.update(processor(audio=audio["array"], text=example["text"], sampling_rate=16000))

    return example`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">def</span> <span class="hljs-title function_">prepare_dataset</span>(<span class="hljs-params">example</span>):
<span class="hljs-meta">... </span>    audio = example[<span class="hljs-string">&quot;audio&quot;</span>]

<span class="hljs-meta">... </span>    example.update(processor(audio=audio[<span class="hljs-string">&quot;array&quot;</span>], text=example[<span class="hljs-string">&quot;text&quot;</span>], sampling_rate=<span class="hljs-number">16000</span>))

<span class="hljs-meta">... </span>    <span class="hljs-keyword">return</span> example`}}),Ze=new k({props:{code:"prepare_dataset(lj_speech[0])",highlighted:'<span class="hljs-meta">&gt;&gt;&gt; </span>prepare_dataset(lj_speech[<span class="hljs-number">0</span>])'}}),{c(){_=l("meta"),$=h(),c=l("h1"),w=l("a"),z=l("span"),b(S.$$.fragment),q=h(),as=l("span"),Kt=i("Vorverarbeiten"),ol=h(),b(Fs.$$.fragment),cl=h(),Qe=l("p"),Jt=i("Bevor Sie Ihre Daten in einem Modell verwenden k\xF6nnen, m\xFCssen die Daten in ein f\xFCr das Modell akzeptables Format gebracht werden. Ein Modell versteht keine Rohtexte, Bilder oder Audiodaten. Diese Eingaben m\xFCssen in Zahlen umgewandelt und zu Tensoren zusammengesetzt werden. In dieser Anleitung werden Sie:"),bl=h(),P=l("ul"),Ga=l("li"),Ut=i("Textdaten mit einem Tokenizer vorverarbeiten."),Gt=h(),Za=l("li"),Zt=i("Bild- oder Audiodaten mit einem Feature Extractor vorverarbeiten."),Yt=h(),Ya=l("li"),Qt=i("Daten f\xFCr eine multimodale Aufgabe mit einem Prozessor vorverarbeiten."),dl=h(),N=l("h2"),ns=l("a"),Qa=l("span"),b(Cs.$$.fragment),Xt=h(),Xa=l("span"),sp=i("NLP"),jl=h(),b(Ns.$$.fragment),fl=h(),D=l("p"),ep=i("Das wichtigste Werkzeug zur Verarbeitung von Textdaten ist ein "),Xe=l("a"),ap=i("Tokenizer"),np=i(". Ein Tokenizer zerlegt Text zun\xE4chst nach einer Reihe von Regeln in "),sn=l("em"),lp=i("Token"),rp=i(". Die Token werden in Zahlen umgewandelt, die zum Aufbau von Tensoren als Eingabe f\xFCr ein Modell verwendet werden. Alle zus\xE4tzlichen Eingaben, die ein Modell ben\xF6tigt, werden ebenfalls vom Tokenizer hinzugef\xFCgt."),gl=h(),b(ls.$$.fragment),vl=h(),rs=l("p"),tp=i("Laden Sie einen vortrainierten Tokenizer mit der Klasse [AutoTokenizer], um schnell loszulegen. Damit wird das "),en=l("em"),pp=i("vocab"),ip=i(" heruntergeladen, das verwendet wird, wenn ein Modell vortrainiert wird."),_l=h(),W=l("h3"),ts=l("a"),an=l("span"),b(Ws.$$.fragment),up=h(),nn=l("span"),hp=i("Tokenize"),kl=h(),ps=l("p"),mp=i("Laden Sie einen vortrainierten Tokenizer mit "),ln=l("code"),op=i("AutoTokenizer.from_pretrained()"),cp=i(":"),$l=h(),b(Os.$$.fragment),wl=h(),sa=l("p"),bp=i("Dann \xFCbergeben Sie Ihren Satz an den Tokenizer:"),zl=h(),b(Rs.$$.fragment),El=h(),ea=l("p"),dp=i("Der Tokenizer gibt ein W\xF6rterbuch mit drei wichtigen Elementen zur\xFCck:"),Sl=h(),M=l("ul"),aa=l("li"),na=l("a"),jp=i("input_ids"),fp=i(" sind die Indizes, die den einzelnen Token im Satz entsprechen."),gp=h(),la=l("li"),ra=l("a"),vp=i("attention_mask"),_p=i(" gibt an, ob ein Token beachtet werden soll oder nicht."),kp=h(),ta=l("li"),pa=l("a"),$p=i("token_type_ids"),wp=i(" gibt an, zu welcher Sequenz ein Token geh\xF6rt, wenn es mehr als eine Sequenz gibt."),xl=h(),is=l("p"),zp=i("Sie k\xF6nnen die "),rn=l("code"),Ep=i("input_ids"),Sp=i(" dekodieren, um die urspr\xFCngliche Eingabe zur\xFCckzugeben:"),Al=h(),b(Hs.$$.fragment),yl=h(),L=l("p"),xp=i("Wie Sie sehen k\xF6nnen, hat der Tokenisierer zwei spezielle Token - "),tn=l("code"),Ap=i("CLS"),yp=i(" und "),pn=l("code"),Tp=i("SEP"),qp=i(` (Klassifikator und Separator) - zum Satz hinzugef\xFCgt. Nicht alle Modelle ben\xF6tigen
spezielle Token, aber wenn dies der Fall ist, f\xFCgt der Tokenisierer sie automatisch f\xFCr Sie hinzu.`),Tl=h(),ia=l("p"),Pp=i("Wenn Sie mehrere S\xE4tze verarbeiten wollen, \xFCbergeben Sie die S\xE4tze als Liste an den Tokenizer:"),ql=h(),b(Vs.$$.fragment),Pl=h(),O=l("h3"),us=l("a"),un=l("span"),b(Ks.$$.fragment),Dp=h(),hn=l("span"),Mp=i("Pad"),Dl=h(),hs=l("p"),Lp=i("Dies bringt uns zu einem wichtigen Thema. Wenn Sie einen Haufen von S\xE4tzen verarbeiten, sind diese nicht immer gleich lang. Das ist ein Problem, weil Tensoren, die Eingabe f\xFCr das Modell, eine einheitliche Form haben m\xFCssen. Padding ist eine Strategie, die sicherstellt, dass Tensoren rechteckig sind, indem ein spezielles "),mn=l("em"),Bp=i("Padding-Token"),Ip=i(" zu S\xE4tzen mit weniger Token hinzugef\xFCgt wird."),Ml=h(),ua=l("p"),Fp=i("Setzen Sie den Parameter \u201Cpadding\u201D auf \u201Ctrue\u201D, um die k\xFCrzeren Sequenzen im Stapel so aufzuf\xFCllen, dass sie der l\xE4ngsten Sequenz entsprechen:"),Ll=h(),b(Js.$$.fragment),Bl=h(),ha=l("p"),Cp=i("Beachten Sie, dass der Tokenizer den ersten und den dritten Satz mit einer \u201C0\u201D aufgef\xFCllt hat, weil sie k\xFCrzer sind!"),Il=h(),R=l("h3"),ms=l("a"),on=l("span"),b(Us.$$.fragment),Np=h(),cn=l("span"),Wp=i("K\xFCrzung"),Fl=h(),ma=l("p"),Op=i("Auf der anderen Seite des Spektrums kann es vorkommen, dass eine Sequenz zu lang f\xFCr ein Modell ist. In diesem Fall m\xFCssen Sie die Sequenz auf eine k\xFCrzere L\xE4nge k\xFCrzen."),Cl=h(),oa=l("p"),Rp=i("Setzen Sie den Parameter \u201Ctruncation\u201D auf \u201Ctrue\u201D, um eine Sequenz auf die vom Modell akzeptierte H\xF6chstl\xE4nge zu k\xFCrzen:"),Nl=h(),b(Gs.$$.fragment),Wl=h(),H=l("h3"),os=l("a"),bn=l("span"),b(Zs.$$.fragment),Hp=h(),dn=l("span"),Vp=i("Tensoren erstellen"),Ol=h(),ca=l("p"),Kp=i("Schlie\xDFlich m\xF6chten Sie, dass der Tokenizer die tats\xE4chlichen Tensoren zur\xFCckgibt, die dem Modell zugef\xFChrt werden."),Rl=h(),x=l("p"),Jp=i("Setzen Sie den Parameter "),jn=l("code"),Up=i("return_tensors"),Gp=i(" entweder auf "),fn=l("code"),Zp=i("pt"),Yp=i(" f\xFCr PyTorch, oder "),gn=l("code"),Qp=i("tf"),Xp=i(" f\xFCr TensorFlow:"),Hl=h(),b(cs.$$.fragment),Vl=h(),V=l("h2"),bs=l("a"),vn=l("span"),b(Ys.$$.fragment),si=h(),_n=l("span"),ei=i("Audio"),Kl=h(),ds=l("p"),ai=i("Audioeingaben werden anders vorverarbeitet als Texteingaben, aber das Endziel bleibt dasselbe: numerische Sequenzen zu erstellen, die das Modell verstehen kann. Ein "),ba=l("a"),ni=i("feature extractor"),li=i(" dient dem ausdr\xFCcklichen Zweck, Merkmale aus Rohbild- oder Audiodaten zu extrahieren und in Tensoren zu konvertieren. Bevor Sie beginnen, installieren Sie \u{1F917} Datasets, um einen Audio-Datensatz zu laden, mit dem Sie experimentieren k\xF6nnen:"),Jl=h(),b(Qs.$$.fragment),Ul=h(),B=l("p"),ri=i("Laden Sie den "),Xs=l("a"),ti=i("MInDS-14"),pi=i(" Datensatz (weitere Informationen zum Laden eines Datensatzes finden Sie im \u{1F917} "),se=l("a"),ii=i("Datasets tutorial"),ui=i("):"),Gl=h(),b(ee.$$.fragment),Zl=h(),js=l("p"),hi=i("Greifen Sie auf das erste Element der "),kn=l("code"),mi=i("audio"),oi=i("-Spalte zu, um einen Blick auf die Eingabe zu werfen. Durch den Aufruf der Spalte \u201Caudio\u201D wird die Audiodatei automatisch geladen und neu gesampelt:"),Yl=h(),b(ae.$$.fragment),Ql=h(),da=l("p"),ci=i("Dies gibt drei Elemente zur\xFCck:"),Xl=h(),I=l("ul"),$n=l("li"),bi=i("\u201Carray\u201D ist das Sprachsignal, das als 1D-Array geladen - und m\xF6glicherweise neu gesampelt - wurde."),di=h(),wn=l("li"),ji=i("Pfad\u201D zeigt auf den Speicherort der Audiodatei."),fi=h(),ja=l("li"),zn=l("code"),gi=i("sampling_rate"),vi=i(" bezieht sich darauf, wie viele Datenpunkte im Sprachsignal pro Sekunde gemessen werden."),sr=h(),K=l("h3"),fs=l("a"),En=l("span"),b(ne.$$.fragment),_i=h(),Sn=l("span"),ki=i("Resample"),er=h(),gs=l("p"),$i=i("F\xFCr dieses Tutorial werden Sie das Modell "),le=l("a"),wi=i("Wav2Vec2"),zi=i(" verwenden. Wie Sie aus der Modellkarte ersehen k\xF6nnen, ist das Wav2Vec2-Modell auf 16kHz abgetastetes Sprachaudio vortrainiert. Es ist wichtig, dass die Abtastrate Ihrer Audiodaten mit der Abtastrate des Datensatzes \xFCbereinstimmt, der f\xFCr das Pre-Training des Modells verwendet wurde. Wenn die Abtastrate Ihrer Daten nicht dieselbe ist, m\xFCssen Sie Ihre Audiodaten neu abtasten."),ar=h(),vs=l("p"),Ei=i("Der Datensatz "),re=l("a"),Si=i("MInDS-14"),xi=i(" hat zum Beispiel eine Abtastrate von 8000 kHz. Um das Wav2Vec2-Modell mit diesem Datensatz verwenden zu k\xF6nnen, m\xFCssen Sie die Abtastrate auf 16 kHz erh\xF6hen:"),nr=h(),b(te.$$.fragment),lr=h(),fa=l("ol"),xn=l("li"),Ai=i("Verwenden Sie die Methode [~datasets.Dataset.cast_column] von \u{1F917} Datasets, um die Abtastrate auf 16kHz zu erh\xF6hen:"),rr=h(),b(pe.$$.fragment),tr=h(),ie=l("ol"),An=l("li"),yi=i("Laden Sie die Audiodatei:"),pr=h(),b(ue.$$.fragment),ir=h(),ga=l("p"),Ti=i("Wie Sie sehen k\xF6nnen, ist die Abtastrate jetzt 16kHz!"),ur=h(),J=l("h3"),_s=l("a"),yn=l("span"),b(he.$$.fragment),qi=h(),Tn=l("span"),Pi=i("Merkmalsextraktor"),hr=h(),A=l("p"),Di=i("Der n\xE4chste Schritt ist das Laden eines Merkmalsextraktors, um die Eingabe zu normalisieren und aufzuf\xFCllen. Beim Auff\xFCllen von Textdaten wird f\xFCr k\xFCrzere Sequenzen ein "),qn=l("code"),Mi=i("0"),Li=i(" hinzugef\xFCgt. Die gleiche Idee gilt f\xFCr Audiodaten, und der Audio-Feature-Extraktor f\xFCgt eine "),Pn=l("code"),Bi=i("0"),Ii=i(" - interpretiert als Stille - zu "),Dn=l("code"),Fi=i("array"),Ci=i(" hinzu."),mr=h(),ks=l("p"),Ni=i("Laden Sie den Merkmalsextraktor mit "),Mn=l("code"),Wi=i("AutoFeatureExtractor.from_pretrained()"),Oi=i(":"),or=h(),b(me.$$.fragment),cr=h(),$s=l("p"),Ri=i("\xDCbergeben Sie das Audio-\u201DArray\u201D an den Feature-Extraktor. Wir empfehlen auch, das Argument "),Ln=l("code"),Hi=i("sampling_rate"),Vi=i(" im Feature Extractor hinzuzuf\xFCgen, um eventuell auftretende stille Fehler besser zu beheben."),br=h(),b(oe.$$.fragment),dr=h(),U=l("h3"),ws=l("a"),Bn=l("span"),b(ce.$$.fragment),Ki=h(),In=l("span"),Ji=i("Auff\xFCllen und K\xFCrzen"),jr=h(),va=l("p"),Ui=i("Genau wie beim Tokenizer k\xF6nnen Sie variable Sequenzen in einem Stapel durch Auff\xFCllen oder Abschneiden behandeln. Werfen Sie einen Blick auf die Sequenzl\xE4nge dieser beiden Audiobeispiele:"),fr=h(),b(be.$$.fragment),gr=h(),_a=l("p"),Gi=i("Wie Sie sehen k\xF6nnen, hat das erste Beispiel eine l\xE4ngere Sequenz als das zweite Beispiel. Lassen Sie uns eine Funktion erstellen, die den Datensatz vorverarbeitet. Geben Sie eine maximale L\xE4nge der Probe an, und der Feature-Extraktor wird die Sequenzen entweder auff\xFCllen oder abschneiden, damit sie dieser L\xE4nge entsprechen:"),vr=h(),b(de.$$.fragment),_r=h(),ka=l("p"),Zi=i("Wenden Sie die Funktion auf die ersten paar Beispiele im Datensatz an:"),kr=h(),b(je.$$.fragment),$r=h(),$a=l("p"),Yi=i("Schauen Sie sich nun noch einmal die verarbeiteten Beispiel-L\xE4ngen an:"),wr=h(),b(fe.$$.fragment),zr=h(),wa=l("p"),Qi=i("Die L\xE4nge der ersten beiden Beispiele entspricht nun der von Ihnen angegebenen Maximall\xE4nge."),Er=h(),G=l("h2"),zs=l("a"),Fn=l("span"),b(ge.$$.fragment),Xi=h(),Cn=l("span"),su=i("Bildverarbeitung"),Sr=h(),za=l("p"),eu=i("Ein Merkmalsextraktor wird auch verwendet, um Bilder f\xFCr Bildverarbeitungsaufgaben zu verarbeiten. Auch hier besteht das Ziel darin, das Rohbild in eine Reihe von Tensoren als Eingabe zu konvertieren."),xr=h(),F=l("p"),au=i("Laden wir den "),ve=l("a"),nu=i("food101"),lu=i(" Datensatz f\xFCr dieses Tutorial. Verwenden Sie den Parameter \u{1F917} Datasets "),Nn=l("code"),ru=i("split"),tu=i(", um nur eine kleine Stichprobe aus dem Trainingssplit zu laden, da der Datensatz recht gro\xDF ist:"),Ar=h(),b(_e.$$.fragment),yr=h(),Es=l("p"),pu=i("Als N\xE4chstes sehen Sie sich das Bild mit dem Merkmal \u{1F917} Datens\xE4tze [Bild] ("),ke=l("a"),iu=i("https://huggingface.co/docs/datasets/package_reference/main_classes.html?highlight=image#datasets.Image"),uu=i(") an:"),Tr=h(),b($e.$$.fragment),qr=h(),Ea=l("p"),Sa=l("img"),Pr=h(),Z=l("h3"),Ss=l("a"),Wn=l("span"),b(we.$$.fragment),hu=h(),On=l("span"),mu=i("Merkmalsextraktor"),Dr=h(),xs=l("p"),ou=i("Laden Sie den Merkmalsextraktor mit "),Rn=l("code"),cu=i("AutoFeatureExtractor.from_pretrained()"),bu=i(":"),Mr=h(),b(ze.$$.fragment),Lr=h(),Y=l("h3"),As=l("a"),Hn=l("span"),b(Ee.$$.fragment),du=h(),Vn=l("span"),ju=i("Datenerweiterung"),Br=h(),ys=l("p"),fu=i("Bei Bildverarbeitungsaufgaben ist es \xFCblich, den Bildern als Teil der Vorverarbeitung eine Art von Datenerweiterung hinzuzuf\xFCgen. Sie k\xF6nnen Erweiterungen mit jeder beliebigen Bibliothek hinzuf\xFCgen, aber in diesem Tutorial werden Sie das Modul "),Se=l("a"),Kn=l("code"),gu=i("transforms"),vu=i(" von torchvision verwenden."),Ir=h(),xa=l("ol"),T=l("li"),_u=i("Normalisieren Sie das Bild und verwenden Sie "),xe=l("a"),Jn=l("code"),ku=i("Compose"),$u=i(", um einige Transformationen - "),Ae=l("a"),Un=l("code"),wu=i("RandomResizedCrop"),zu=i(" und "),ye=l("a"),Gn=l("code"),Eu=i("ColorJitter"),Su=i(" - miteinander zu verkn\xFCpfen:"),Fr=h(),b(Te.$$.fragment),Cr=h(),qe=l("ol"),Q=l("li"),xu=i("Das Modell akzeptiert "),Aa=l("a"),Zn=l("code"),Au=i("pixel_values"),yu=i(" als Eingabe. Dieser Wert wird vom Merkmalsextraktor erzeugt. Erstellen Sie eine Funktion, die "),Yn=l("code"),Tu=i("pixel_values"),qu=i(" aus den Transformationen erzeugt:"),Nr=h(),b(Pe.$$.fragment),Wr=h(),De=l("ol"),Me=l("li"),Pu=i("Dann verwenden Sie \u{1F917} Datasets "),Le=l("a"),Qn=l("code"),Du=i("set_transform"),Mu=i(", um die Transformationen im laufenden Betrieb anzuwenden:"),Or=h(),b(Be.$$.fragment),Rr=h(),Ie=l("ol"),Xn=l("li"),Lu=i("Wenn Sie nun auf das Bild zugreifen, werden Sie feststellen, dass der Feature Extractor die Modelleingabe \u201Cpixel_values\u201D hinzugef\xFCgt hat:"),Hr=h(),b(Fe.$$.fragment),Vr=h(),ya=l("p"),Bu=i("Hier sehen Sie, wie das Bild nach der Vorverarbeitung aussieht. Wie von den angewandten Transformationen zu erwarten, wurde das Bild willk\xFCrlich beschnitten und seine Farbeigenschaften sind anders."),Kr=h(),b(Ce.$$.fragment),Jr=h(),Ta=l("p"),qa=l("img"),Ur=h(),X=l("h2"),Ts=l("a"),sl=l("span"),b(Ne.$$.fragment),Iu=h(),el=l("span"),Fu=i("Multimodal"),Gr=h(),Pa=l("p"),Cu=i("F\xFCr multimodale Aufgaben werden Sie eine Kombination aus allem, was Sie bisher gelernt haben, verwenden und Ihre F\xE4higkeiten auf eine Aufgabe der automatischen Spracherkennung (ASR) anwenden. Dies bedeutet, dass Sie einen:"),Zr=h(),qs=l("ul"),al=l("li"),Nu=i("Feature Extractor zur Vorverarbeitung der Audiodaten."),Wu=h(),nl=l("li"),Ou=i("Tokenizer, um den Text zu verarbeiten."),Yr=h(),Ps=l("p"),Ru=i("Kehren wir zum "),We=l("a"),Hu=i("LJ Speech"),Vu=i(" Datensatz zur\xFCck:"),Qr=h(),b(Oe.$$.fragment),Xr=h(),Da=l("p"),Ku=i("Da Sie haupts\xE4chlich an den Spalten \u201CAudio\u201D und \u201CText\u201D interessiert sind, entfernen Sie die anderen Spalten:"),st=h(),b(Re.$$.fragment),et=h(),Ma=l("p"),Ju=i("Schauen Sie sich nun die Spalten \u201CAudio\u201D und \u201CText\u201D an:"),at=h(),b(He.$$.fragment),nt=h(),Ds=l("p"),Uu=i("Erinnern Sie sich an den fr\xFCheren Abschnitt \xFCber die Verarbeitung von Audiodaten: Sie sollten immer die Abtastrate Ihrer Audiodaten "),La=l("a"),Gu=i("resample"),Zu=i(", damit sie mit der Abtastrate des Datensatzes \xFCbereinstimmt, der f\xFCr das Vortraining eines Modells verwendet wird:"),lt=h(),b(Ve.$$.fragment),rt=h(),ss=l("h3"),Ms=l("a"),ll=l("span"),b(Ke.$$.fragment),Yu=h(),rl=l("span"),Qu=i("Prozessor"),tt=h(),Ba=l("p"),Xu=i("Ein Processor kombiniert einen Feature-Extraktor und einen Tokenizer. Laden Sie einen Processor mit [`AutoProcessor.from_pretrained]:"),pt=h(),b(Je.$$.fragment),it=h(),Ia=l("ol"),es=l("li"),sh=i("Erstellen Sie eine Funktion, die die Audiodaten zu "),tl=l("code"),eh=i("input_values"),ah=i(" verarbeitet und den Text zu "),pl=l("code"),nh=i("labels"),lh=i(" tokenisiert. Dies sind Ihre Eingaben f\xFCr das Modell:"),ut=h(),b(Ue.$$.fragment),ht=h(),Ge=l("ol"),il=l("li"),rh=i("Wenden Sie die Funktion \u201Cprepare_dataset\u201D auf ein Beispiel an:"),mt=h(),b(Ze.$$.fragment),ot=h(),C=l("p"),th=i("Beachten Sie, dass der Processor "),ul=l("code"),ph=i("input_values"),ih=i(" und "),hl=l("code"),uh=i("labels"),hh=i(" hinzugef\xFCgt hat. Auch die Abtastrate wurde korrekt auf 16kHz heruntergerechnet."),ct=h(),Fa=l("p"),mh=i("Toll, Sie sollten jetzt in der Lage sein, Daten f\xFCr jede Modalit\xE4t vorzuverarbeiten und sogar verschiedene Modalit\xE4ten zu kombinieren! Im n\xE4chsten Kurs lernen Sie, wie Sie ein Modell mit Ihren neu aufbereiteten Daten feinabstimmen k\xF6nnen."),this.h()},l(s){const n=sc('[data-svelte="svelte-1phssyn"]',document.head);_=r(n,"META",{name:!0,content:!0}),n.forEach(e),$=m(s),c=r(s,"H1",{class:!0});var Ye=t(c);w=r(Ye,"A",{id:!0,class:!0,href:!0});var ml=t(w);z=r(ml,"SPAN",{});var gh=t(z);d(S.$$.fragment,gh),gh.forEach(e),ml.forEach(e),q=m(Ye),as=r(Ye,"SPAN",{});var vh=t(as);Kt=u(vh,"Vorverarbeiten"),vh.forEach(e),Ye.forEach(e),ol=m(s),d(Fs.$$.fragment,s),cl=m(s),Qe=r(s,"P",{});var _h=t(Qe);Jt=u(_h,"Bevor Sie Ihre Daten in einem Modell verwenden k\xF6nnen, m\xFCssen die Daten in ein f\xFCr das Modell akzeptables Format gebracht werden. Ein Modell versteht keine Rohtexte, Bilder oder Audiodaten. Diese Eingaben m\xFCssen in Zahlen umgewandelt und zu Tensoren zusammengesetzt werden. In dieser Anleitung werden Sie:"),_h.forEach(e),bl=m(s),P=r(s,"UL",{});var Ca=t(P);Ga=r(Ca,"LI",{});var kh=t(Ga);Ut=u(kh,"Textdaten mit einem Tokenizer vorverarbeiten."),kh.forEach(e),Gt=m(Ca),Za=r(Ca,"LI",{});var $h=t(Za);Zt=u($h,"Bild- oder Audiodaten mit einem Feature Extractor vorverarbeiten."),$h.forEach(e),Yt=m(Ca),Ya=r(Ca,"LI",{});var wh=t(Ya);Qt=u(wh,"Daten f\xFCr eine multimodale Aufgabe mit einem Prozessor vorverarbeiten."),wh.forEach(e),Ca.forEach(e),dl=m(s),N=r(s,"H2",{class:!0});var dt=t(N);ns=r(dt,"A",{id:!0,class:!0,href:!0});var zh=t(ns);Qa=r(zh,"SPAN",{});var Eh=t(Qa);d(Cs.$$.fragment,Eh),Eh.forEach(e),zh.forEach(e),Xt=m(dt),Xa=r(dt,"SPAN",{});var Sh=t(Xa);sp=u(Sh,"NLP"),Sh.forEach(e),dt.forEach(e),jl=m(s),d(Ns.$$.fragment,s),fl=m(s),D=r(s,"P",{});var Na=t(D);ep=u(Na,"Das wichtigste Werkzeug zur Verarbeitung von Textdaten ist ein "),Xe=r(Na,"A",{href:!0});var xh=t(Xe);ap=u(xh,"Tokenizer"),xh.forEach(e),np=u(Na,". Ein Tokenizer zerlegt Text zun\xE4chst nach einer Reihe von Regeln in "),sn=r(Na,"EM",{});var Ah=t(sn);lp=u(Ah,"Token"),Ah.forEach(e),rp=u(Na,". Die Token werden in Zahlen umgewandelt, die zum Aufbau von Tensoren als Eingabe f\xFCr ein Modell verwendet werden. Alle zus\xE4tzlichen Eingaben, die ein Modell ben\xF6tigt, werden ebenfalls vom Tokenizer hinzugef\xFCgt."),Na.forEach(e),gl=m(s),d(ls.$$.fragment,s),vl=m(s),rs=r(s,"P",{});var jt=t(rs);tp=u(jt,"Laden Sie einen vortrainierten Tokenizer mit der Klasse [AutoTokenizer], um schnell loszulegen. Damit wird das "),en=r(jt,"EM",{});var yh=t(en);pp=u(yh,"vocab"),yh.forEach(e),ip=u(jt," heruntergeladen, das verwendet wird, wenn ein Modell vortrainiert wird."),jt.forEach(e),_l=m(s),W=r(s,"H3",{class:!0});var ft=t(W);ts=r(ft,"A",{id:!0,class:!0,href:!0});var Th=t(ts);an=r(Th,"SPAN",{});var qh=t(an);d(Ws.$$.fragment,qh),qh.forEach(e),Th.forEach(e),up=m(ft),nn=r(ft,"SPAN",{});var Ph=t(nn);hp=u(Ph,"Tokenize"),Ph.forEach(e),ft.forEach(e),kl=m(s),ps=r(s,"P",{});var gt=t(ps);mp=u(gt,"Laden Sie einen vortrainierten Tokenizer mit "),ln=r(gt,"CODE",{});var Dh=t(ln);op=u(Dh,"AutoTokenizer.from_pretrained()"),Dh.forEach(e),cp=u(gt,":"),gt.forEach(e),$l=m(s),d(Os.$$.fragment,s),wl=m(s),sa=r(s,"P",{});var Mh=t(sa);bp=u(Mh,"Dann \xFCbergeben Sie Ihren Satz an den Tokenizer:"),Mh.forEach(e),zl=m(s),d(Rs.$$.fragment,s),El=m(s),ea=r(s,"P",{});var Lh=t(ea);dp=u(Lh,"Der Tokenizer gibt ein W\xF6rterbuch mit drei wichtigen Elementen zur\xFCck:"),Lh.forEach(e),Sl=m(s),M=r(s,"UL",{});var Wa=t(M);aa=r(Wa,"LI",{});var oh=t(aa);na=r(oh,"A",{href:!0});var Bh=t(na);jp=u(Bh,"input_ids"),Bh.forEach(e),fp=u(oh," sind die Indizes, die den einzelnen Token im Satz entsprechen."),oh.forEach(e),gp=m(Wa),la=r(Wa,"LI",{});var ch=t(la);ra=r(ch,"A",{href:!0});var Ih=t(ra);vp=u(Ih,"attention_mask"),Ih.forEach(e),_p=u(ch," gibt an, ob ein Token beachtet werden soll oder nicht."),ch.forEach(e),kp=m(Wa),ta=r(Wa,"LI",{});var bh=t(ta);pa=r(bh,"A",{href:!0});var Fh=t(pa);$p=u(Fh,"token_type_ids"),Fh.forEach(e),wp=u(bh," gibt an, zu welcher Sequenz ein Token geh\xF6rt, wenn es mehr als eine Sequenz gibt."),bh.forEach(e),Wa.forEach(e),xl=m(s),is=r(s,"P",{});var vt=t(is);zp=u(vt,"Sie k\xF6nnen die "),rn=r(vt,"CODE",{});var Ch=t(rn);Ep=u(Ch,"input_ids"),Ch.forEach(e),Sp=u(vt," dekodieren, um die urspr\xFCngliche Eingabe zur\xFCckzugeben:"),vt.forEach(e),Al=m(s),d(Hs.$$.fragment,s),yl=m(s),L=r(s,"P",{});var Oa=t(L);xp=u(Oa,"Wie Sie sehen k\xF6nnen, hat der Tokenisierer zwei spezielle Token - "),tn=r(Oa,"CODE",{});var Nh=t(tn);Ap=u(Nh,"CLS"),Nh.forEach(e),yp=u(Oa," und "),pn=r(Oa,"CODE",{});var Wh=t(pn);Tp=u(Wh,"SEP"),Wh.forEach(e),qp=u(Oa,` (Klassifikator und Separator) - zum Satz hinzugef\xFCgt. Nicht alle Modelle ben\xF6tigen
spezielle Token, aber wenn dies der Fall ist, f\xFCgt der Tokenisierer sie automatisch f\xFCr Sie hinzu.`),Oa.forEach(e),Tl=m(s),ia=r(s,"P",{});var Oh=t(ia);Pp=u(Oh,"Wenn Sie mehrere S\xE4tze verarbeiten wollen, \xFCbergeben Sie die S\xE4tze als Liste an den Tokenizer:"),Oh.forEach(e),ql=m(s),d(Vs.$$.fragment,s),Pl=m(s),O=r(s,"H3",{class:!0});var _t=t(O);us=r(_t,"A",{id:!0,class:!0,href:!0});var Rh=t(us);un=r(Rh,"SPAN",{});var Hh=t(un);d(Ks.$$.fragment,Hh),Hh.forEach(e),Rh.forEach(e),Dp=m(_t),hn=r(_t,"SPAN",{});var Vh=t(hn);Mp=u(Vh,"Pad"),Vh.forEach(e),_t.forEach(e),Dl=m(s),hs=r(s,"P",{});var kt=t(hs);Lp=u(kt,"Dies bringt uns zu einem wichtigen Thema. Wenn Sie einen Haufen von S\xE4tzen verarbeiten, sind diese nicht immer gleich lang. Das ist ein Problem, weil Tensoren, die Eingabe f\xFCr das Modell, eine einheitliche Form haben m\xFCssen. Padding ist eine Strategie, die sicherstellt, dass Tensoren rechteckig sind, indem ein spezielles "),mn=r(kt,"EM",{});var Kh=t(mn);Bp=u(Kh,"Padding-Token"),Kh.forEach(e),Ip=u(kt," zu S\xE4tzen mit weniger Token hinzugef\xFCgt wird."),kt.forEach(e),Ml=m(s),ua=r(s,"P",{});var Jh=t(ua);Fp=u(Jh,"Setzen Sie den Parameter \u201Cpadding\u201D auf \u201Ctrue\u201D, um die k\xFCrzeren Sequenzen im Stapel so aufzuf\xFCllen, dass sie der l\xE4ngsten Sequenz entsprechen:"),Jh.forEach(e),Ll=m(s),d(Js.$$.fragment,s),Bl=m(s),ha=r(s,"P",{});var Uh=t(ha);Cp=u(Uh,"Beachten Sie, dass der Tokenizer den ersten und den dritten Satz mit einer \u201C0\u201D aufgef\xFCllt hat, weil sie k\xFCrzer sind!"),Uh.forEach(e),Il=m(s),R=r(s,"H3",{class:!0});var $t=t(R);ms=r($t,"A",{id:!0,class:!0,href:!0});var Gh=t(ms);on=r(Gh,"SPAN",{});var Zh=t(on);d(Us.$$.fragment,Zh),Zh.forEach(e),Gh.forEach(e),Np=m($t),cn=r($t,"SPAN",{});var Yh=t(cn);Wp=u(Yh,"K\xFCrzung"),Yh.forEach(e),$t.forEach(e),Fl=m(s),ma=r(s,"P",{});var Qh=t(ma);Op=u(Qh,"Auf der anderen Seite des Spektrums kann es vorkommen, dass eine Sequenz zu lang f\xFCr ein Modell ist. In diesem Fall m\xFCssen Sie die Sequenz auf eine k\xFCrzere L\xE4nge k\xFCrzen."),Qh.forEach(e),Cl=m(s),oa=r(s,"P",{});var Xh=t(oa);Rp=u(Xh,"Setzen Sie den Parameter \u201Ctruncation\u201D auf \u201Ctrue\u201D, um eine Sequenz auf die vom Modell akzeptierte H\xF6chstl\xE4nge zu k\xFCrzen:"),Xh.forEach(e),Nl=m(s),d(Gs.$$.fragment,s),Wl=m(s),H=r(s,"H3",{class:!0});var wt=t(H);os=r(wt,"A",{id:!0,class:!0,href:!0});var sm=t(os);bn=r(sm,"SPAN",{});var em=t(bn);d(Zs.$$.fragment,em),em.forEach(e),sm.forEach(e),Hp=m(wt),dn=r(wt,"SPAN",{});var am=t(dn);Vp=u(am,"Tensoren erstellen"),am.forEach(e),wt.forEach(e),Ol=m(s),ca=r(s,"P",{});var nm=t(ca);Kp=u(nm,"Schlie\xDFlich m\xF6chten Sie, dass der Tokenizer die tats\xE4chlichen Tensoren zur\xFCckgibt, die dem Modell zugef\xFChrt werden."),nm.forEach(e),Rl=m(s),x=r(s,"P",{});var Ls=t(x);Jp=u(Ls,"Setzen Sie den Parameter "),jn=r(Ls,"CODE",{});var lm=t(jn);Up=u(lm,"return_tensors"),lm.forEach(e),Gp=u(Ls," entweder auf "),fn=r(Ls,"CODE",{});var rm=t(fn);Zp=u(rm,"pt"),rm.forEach(e),Yp=u(Ls," f\xFCr PyTorch, oder "),gn=r(Ls,"CODE",{});var tm=t(gn);Qp=u(tm,"tf"),tm.forEach(e),Xp=u(Ls," f\xFCr TensorFlow:"),Ls.forEach(e),Hl=m(s),d(cs.$$.fragment,s),Vl=m(s),V=r(s,"H2",{class:!0});var zt=t(V);bs=r(zt,"A",{id:!0,class:!0,href:!0});var pm=t(bs);vn=r(pm,"SPAN",{});var im=t(vn);d(Ys.$$.fragment,im),im.forEach(e),pm.forEach(e),si=m(zt),_n=r(zt,"SPAN",{});var um=t(_n);ei=u(um,"Audio"),um.forEach(e),zt.forEach(e),Kl=m(s),ds=r(s,"P",{});var Et=t(ds);ai=u(Et,"Audioeingaben werden anders vorverarbeitet als Texteingaben, aber das Endziel bleibt dasselbe: numerische Sequenzen zu erstellen, die das Modell verstehen kann. Ein "),ba=r(Et,"A",{href:!0});var hm=t(ba);ni=u(hm,"feature extractor"),hm.forEach(e),li=u(Et," dient dem ausdr\xFCcklichen Zweck, Merkmale aus Rohbild- oder Audiodaten zu extrahieren und in Tensoren zu konvertieren. Bevor Sie beginnen, installieren Sie \u{1F917} Datasets, um einen Audio-Datensatz zu laden, mit dem Sie experimentieren k\xF6nnen:"),Et.forEach(e),Jl=m(s),d(Qs.$$.fragment,s),Ul=m(s),B=r(s,"P",{});var Ra=t(B);ri=u(Ra,"Laden Sie den "),Xs=r(Ra,"A",{href:!0,rel:!0});var mm=t(Xs);ti=u(mm,"MInDS-14"),mm.forEach(e),pi=u(Ra," Datensatz (weitere Informationen zum Laden eines Datensatzes finden Sie im \u{1F917} "),se=r(Ra,"A",{href:!0,rel:!0});var om=t(se);ii=u(om,"Datasets tutorial"),om.forEach(e),ui=u(Ra,"):"),Ra.forEach(e),Gl=m(s),d(ee.$$.fragment,s),Zl=m(s),js=r(s,"P",{});var St=t(js);hi=u(St,"Greifen Sie auf das erste Element der "),kn=r(St,"CODE",{});var cm=t(kn);mi=u(cm,"audio"),cm.forEach(e),oi=u(St,"-Spalte zu, um einen Blick auf die Eingabe zu werfen. Durch den Aufruf der Spalte \u201Caudio\u201D wird die Audiodatei automatisch geladen und neu gesampelt:"),St.forEach(e),Yl=m(s),d(ae.$$.fragment,s),Ql=m(s),da=r(s,"P",{});var bm=t(da);ci=u(bm,"Dies gibt drei Elemente zur\xFCck:"),bm.forEach(e),Xl=m(s),I=r(s,"UL",{});var Ha=t(I);$n=r(Ha,"LI",{});var dm=t($n);bi=u(dm,"\u201Carray\u201D ist das Sprachsignal, das als 1D-Array geladen - und m\xF6glicherweise neu gesampelt - wurde."),dm.forEach(e),di=m(Ha),wn=r(Ha,"LI",{});var jm=t(wn);ji=u(jm,"Pfad\u201D zeigt auf den Speicherort der Audiodatei."),jm.forEach(e),fi=m(Ha),ja=r(Ha,"LI",{});var dh=t(ja);zn=r(dh,"CODE",{});var fm=t(zn);gi=u(fm,"sampling_rate"),fm.forEach(e),vi=u(dh," bezieht sich darauf, wie viele Datenpunkte im Sprachsignal pro Sekunde gemessen werden."),dh.forEach(e),Ha.forEach(e),sr=m(s),K=r(s,"H3",{class:!0});var xt=t(K);fs=r(xt,"A",{id:!0,class:!0,href:!0});var gm=t(fs);En=r(gm,"SPAN",{});var vm=t(En);d(ne.$$.fragment,vm),vm.forEach(e),gm.forEach(e),_i=m(xt),Sn=r(xt,"SPAN",{});var _m=t(Sn);ki=u(_m,"Resample"),_m.forEach(e),xt.forEach(e),er=m(s),gs=r(s,"P",{});var At=t(gs);$i=u(At,"F\xFCr dieses Tutorial werden Sie das Modell "),le=r(At,"A",{href:!0,rel:!0});var km=t(le);wi=u(km,"Wav2Vec2"),km.forEach(e),zi=u(At," verwenden. Wie Sie aus der Modellkarte ersehen k\xF6nnen, ist das Wav2Vec2-Modell auf 16kHz abgetastetes Sprachaudio vortrainiert. Es ist wichtig, dass die Abtastrate Ihrer Audiodaten mit der Abtastrate des Datensatzes \xFCbereinstimmt, der f\xFCr das Pre-Training des Modells verwendet wurde. Wenn die Abtastrate Ihrer Daten nicht dieselbe ist, m\xFCssen Sie Ihre Audiodaten neu abtasten."),At.forEach(e),ar=m(s),vs=r(s,"P",{});var yt=t(vs);Ei=u(yt,"Der Datensatz "),re=r(yt,"A",{href:!0,rel:!0});var $m=t(re);Si=u($m,"MInDS-14"),$m.forEach(e),xi=u(yt," hat zum Beispiel eine Abtastrate von 8000 kHz. Um das Wav2Vec2-Modell mit diesem Datensatz verwenden zu k\xF6nnen, m\xFCssen Sie die Abtastrate auf 16 kHz erh\xF6hen:"),yt.forEach(e),nr=m(s),d(te.$$.fragment,s),lr=m(s),fa=r(s,"OL",{});var wm=t(fa);xn=r(wm,"LI",{});var zm=t(xn);Ai=u(zm,"Verwenden Sie die Methode [~datasets.Dataset.cast_column] von \u{1F917} Datasets, um die Abtastrate auf 16kHz zu erh\xF6hen:"),zm.forEach(e),wm.forEach(e),rr=m(s),d(pe.$$.fragment,s),tr=m(s),ie=r(s,"OL",{start:!0});var Em=t(ie);An=r(Em,"LI",{});var Sm=t(An);yi=u(Sm,"Laden Sie die Audiodatei:"),Sm.forEach(e),Em.forEach(e),pr=m(s),d(ue.$$.fragment,s),ir=m(s),ga=r(s,"P",{});var xm=t(ga);Ti=u(xm,"Wie Sie sehen k\xF6nnen, ist die Abtastrate jetzt 16kHz!"),xm.forEach(e),ur=m(s),J=r(s,"H3",{class:!0});var Tt=t(J);_s=r(Tt,"A",{id:!0,class:!0,href:!0});var Am=t(_s);yn=r(Am,"SPAN",{});var ym=t(yn);d(he.$$.fragment,ym),ym.forEach(e),Am.forEach(e),qi=m(Tt),Tn=r(Tt,"SPAN",{});var Tm=t(Tn);Pi=u(Tm,"Merkmalsextraktor"),Tm.forEach(e),Tt.forEach(e),hr=m(s),A=r(s,"P",{});var Bs=t(A);Di=u(Bs,"Der n\xE4chste Schritt ist das Laden eines Merkmalsextraktors, um die Eingabe zu normalisieren und aufzuf\xFCllen. Beim Auff\xFCllen von Textdaten wird f\xFCr k\xFCrzere Sequenzen ein "),qn=r(Bs,"CODE",{});var qm=t(qn);Mi=u(qm,"0"),qm.forEach(e),Li=u(Bs," hinzugef\xFCgt. Die gleiche Idee gilt f\xFCr Audiodaten, und der Audio-Feature-Extraktor f\xFCgt eine "),Pn=r(Bs,"CODE",{});var Pm=t(Pn);Bi=u(Pm,"0"),Pm.forEach(e),Ii=u(Bs," - interpretiert als Stille - zu "),Dn=r(Bs,"CODE",{});var Dm=t(Dn);Fi=u(Dm,"array"),Dm.forEach(e),Ci=u(Bs," hinzu."),Bs.forEach(e),mr=m(s),ks=r(s,"P",{});var qt=t(ks);Ni=u(qt,"Laden Sie den Merkmalsextraktor mit "),Mn=r(qt,"CODE",{});var Mm=t(Mn);Wi=u(Mm,"AutoFeatureExtractor.from_pretrained()"),Mm.forEach(e),Oi=u(qt,":"),qt.forEach(e),or=m(s),d(me.$$.fragment,s),cr=m(s),$s=r(s,"P",{});var Pt=t($s);Ri=u(Pt,"\xDCbergeben Sie das Audio-\u201DArray\u201D an den Feature-Extraktor. Wir empfehlen auch, das Argument "),Ln=r(Pt,"CODE",{});var Lm=t(Ln);Hi=u(Lm,"sampling_rate"),Lm.forEach(e),Vi=u(Pt," im Feature Extractor hinzuzuf\xFCgen, um eventuell auftretende stille Fehler besser zu beheben."),Pt.forEach(e),br=m(s),d(oe.$$.fragment,s),dr=m(s),U=r(s,"H3",{class:!0});var Dt=t(U);ws=r(Dt,"A",{id:!0,class:!0,href:!0});var Bm=t(ws);Bn=r(Bm,"SPAN",{});var Im=t(Bn);d(ce.$$.fragment,Im),Im.forEach(e),Bm.forEach(e),Ki=m(Dt),In=r(Dt,"SPAN",{});var Fm=t(In);Ji=u(Fm,"Auff\xFCllen und K\xFCrzen"),Fm.forEach(e),Dt.forEach(e),jr=m(s),va=r(s,"P",{});var Cm=t(va);Ui=u(Cm,"Genau wie beim Tokenizer k\xF6nnen Sie variable Sequenzen in einem Stapel durch Auff\xFCllen oder Abschneiden behandeln. Werfen Sie einen Blick auf die Sequenzl\xE4nge dieser beiden Audiobeispiele:"),Cm.forEach(e),fr=m(s),d(be.$$.fragment,s),gr=m(s),_a=r(s,"P",{});var Nm=t(_a);Gi=u(Nm,"Wie Sie sehen k\xF6nnen, hat das erste Beispiel eine l\xE4ngere Sequenz als das zweite Beispiel. Lassen Sie uns eine Funktion erstellen, die den Datensatz vorverarbeitet. Geben Sie eine maximale L\xE4nge der Probe an, und der Feature-Extraktor wird die Sequenzen entweder auff\xFCllen oder abschneiden, damit sie dieser L\xE4nge entsprechen:"),Nm.forEach(e),vr=m(s),d(de.$$.fragment,s),_r=m(s),ka=r(s,"P",{});var Wm=t(ka);Zi=u(Wm,"Wenden Sie die Funktion auf die ersten paar Beispiele im Datensatz an:"),Wm.forEach(e),kr=m(s),d(je.$$.fragment,s),$r=m(s),$a=r(s,"P",{});var Om=t($a);Yi=u(Om,"Schauen Sie sich nun noch einmal die verarbeiteten Beispiel-L\xE4ngen an:"),Om.forEach(e),wr=m(s),d(fe.$$.fragment,s),zr=m(s),wa=r(s,"P",{});var Rm=t(wa);Qi=u(Rm,"Die L\xE4nge der ersten beiden Beispiele entspricht nun der von Ihnen angegebenen Maximall\xE4nge."),Rm.forEach(e),Er=m(s),G=r(s,"H2",{class:!0});var Mt=t(G);zs=r(Mt,"A",{id:!0,class:!0,href:!0});var Hm=t(zs);Fn=r(Hm,"SPAN",{});var Vm=t(Fn);d(ge.$$.fragment,Vm),Vm.forEach(e),Hm.forEach(e),Xi=m(Mt),Cn=r(Mt,"SPAN",{});var Km=t(Cn);su=u(Km,"Bildverarbeitung"),Km.forEach(e),Mt.forEach(e),Sr=m(s),za=r(s,"P",{});var Jm=t(za);eu=u(Jm,"Ein Merkmalsextraktor wird auch verwendet, um Bilder f\xFCr Bildverarbeitungsaufgaben zu verarbeiten. Auch hier besteht das Ziel darin, das Rohbild in eine Reihe von Tensoren als Eingabe zu konvertieren."),Jm.forEach(e),xr=m(s),F=r(s,"P",{});var Va=t(F);au=u(Va,"Laden wir den "),ve=r(Va,"A",{href:!0,rel:!0});var Um=t(ve);nu=u(Um,"food101"),Um.forEach(e),lu=u(Va," Datensatz f\xFCr dieses Tutorial. Verwenden Sie den Parameter \u{1F917} Datasets "),Nn=r(Va,"CODE",{});var Gm=t(Nn);ru=u(Gm,"split"),Gm.forEach(e),tu=u(Va,", um nur eine kleine Stichprobe aus dem Trainingssplit zu laden, da der Datensatz recht gro\xDF ist:"),Va.forEach(e),Ar=m(s),d(_e.$$.fragment,s),yr=m(s),Es=r(s,"P",{});var Lt=t(Es);pu=u(Lt,"Als N\xE4chstes sehen Sie sich das Bild mit dem Merkmal \u{1F917} Datens\xE4tze [Bild] ("),ke=r(Lt,"A",{href:!0,rel:!0});var Zm=t(ke);iu=u(Zm,"https://huggingface.co/docs/datasets/package_reference/main_classes.html?highlight=image#datasets.Image"),Zm.forEach(e),uu=u(Lt,") an:"),Lt.forEach(e),Tr=m(s),d($e.$$.fragment,s),qr=m(s),Ea=r(s,"P",{});var Ym=t(Ea);Sa=r(Ym,"IMG",{src:!0,alt:!0}),Ym.forEach(e),Pr=m(s),Z=r(s,"H3",{class:!0});var Bt=t(Z);Ss=r(Bt,"A",{id:!0,class:!0,href:!0});var Qm=t(Ss);Wn=r(Qm,"SPAN",{});var Xm=t(Wn);d(we.$$.fragment,Xm),Xm.forEach(e),Qm.forEach(e),hu=m(Bt),On=r(Bt,"SPAN",{});var so=t(On);mu=u(so,"Merkmalsextraktor"),so.forEach(e),Bt.forEach(e),Dr=m(s),xs=r(s,"P",{});var It=t(xs);ou=u(It,"Laden Sie den Merkmalsextraktor mit "),Rn=r(It,"CODE",{});var eo=t(Rn);cu=u(eo,"AutoFeatureExtractor.from_pretrained()"),eo.forEach(e),bu=u(It,":"),It.forEach(e),Mr=m(s),d(ze.$$.fragment,s),Lr=m(s),Y=r(s,"H3",{class:!0});var Ft=t(Y);As=r(Ft,"A",{id:!0,class:!0,href:!0});var ao=t(As);Hn=r(ao,"SPAN",{});var no=t(Hn);d(Ee.$$.fragment,no),no.forEach(e),ao.forEach(e),du=m(Ft),Vn=r(Ft,"SPAN",{});var lo=t(Vn);ju=u(lo,"Datenerweiterung"),lo.forEach(e),Ft.forEach(e),Br=m(s),ys=r(s,"P",{});var Ct=t(ys);fu=u(Ct,"Bei Bildverarbeitungsaufgaben ist es \xFCblich, den Bildern als Teil der Vorverarbeitung eine Art von Datenerweiterung hinzuzuf\xFCgen. Sie k\xF6nnen Erweiterungen mit jeder beliebigen Bibliothek hinzuf\xFCgen, aber in diesem Tutorial werden Sie das Modul "),Se=r(Ct,"A",{href:!0,rel:!0});var ro=t(Se);Kn=r(ro,"CODE",{});var to=t(Kn);gu=u(to,"transforms"),to.forEach(e),ro.forEach(e),vu=u(Ct," von torchvision verwenden."),Ct.forEach(e),Ir=m(s),xa=r(s,"OL",{});var po=t(xa);T=r(po,"LI",{});var Is=t(T);_u=u(Is,"Normalisieren Sie das Bild und verwenden Sie "),xe=r(Is,"A",{href:!0,rel:!0});var io=t(xe);Jn=r(io,"CODE",{});var uo=t(Jn);ku=u(uo,"Compose"),uo.forEach(e),io.forEach(e),$u=u(Is,", um einige Transformationen - "),Ae=r(Is,"A",{href:!0,rel:!0});var ho=t(Ae);Un=r(ho,"CODE",{});var mo=t(Un);wu=u(mo,"RandomResizedCrop"),mo.forEach(e),ho.forEach(e),zu=u(Is," und "),ye=r(Is,"A",{href:!0,rel:!0});var oo=t(ye);Gn=r(oo,"CODE",{});var co=t(Gn);Eu=u(co,"ColorJitter"),co.forEach(e),oo.forEach(e),Su=u(Is," - miteinander zu verkn\xFCpfen:"),Is.forEach(e),po.forEach(e),Fr=m(s),d(Te.$$.fragment,s),Cr=m(s),qe=r(s,"OL",{start:!0});var bo=t(qe);Q=r(bo,"LI",{});var Ka=t(Q);xu=u(Ka,"Das Modell akzeptiert "),Aa=r(Ka,"A",{href:!0});var jo=t(Aa);Zn=r(jo,"CODE",{});var fo=t(Zn);Au=u(fo,"pixel_values"),fo.forEach(e),jo.forEach(e),yu=u(Ka," als Eingabe. Dieser Wert wird vom Merkmalsextraktor erzeugt. Erstellen Sie eine Funktion, die "),Yn=r(Ka,"CODE",{});var go=t(Yn);Tu=u(go,"pixel_values"),go.forEach(e),qu=u(Ka," aus den Transformationen erzeugt:"),Ka.forEach(e),bo.forEach(e),Nr=m(s),d(Pe.$$.fragment,s),Wr=m(s),De=r(s,"OL",{start:!0});var vo=t(De);Me=r(vo,"LI",{});var Nt=t(Me);Pu=u(Nt,"Dann verwenden Sie \u{1F917} Datasets "),Le=r(Nt,"A",{href:!0,rel:!0});var _o=t(Le);Qn=r(_o,"CODE",{});var ko=t(Qn);Du=u(ko,"set_transform"),ko.forEach(e),_o.forEach(e),Mu=u(Nt,", um die Transformationen im laufenden Betrieb anzuwenden:"),Nt.forEach(e),vo.forEach(e),Or=m(s),d(Be.$$.fragment,s),Rr=m(s),Ie=r(s,"OL",{start:!0});var $o=t(Ie);Xn=r($o,"LI",{});var wo=t(Xn);Lu=u(wo,"Wenn Sie nun auf das Bild zugreifen, werden Sie feststellen, dass der Feature Extractor die Modelleingabe \u201Cpixel_values\u201D hinzugef\xFCgt hat:"),wo.forEach(e),$o.forEach(e),Hr=m(s),d(Fe.$$.fragment,s),Vr=m(s),ya=r(s,"P",{});var zo=t(ya);Bu=u(zo,"Hier sehen Sie, wie das Bild nach der Vorverarbeitung aussieht. Wie von den angewandten Transformationen zu erwarten, wurde das Bild willk\xFCrlich beschnitten und seine Farbeigenschaften sind anders."),zo.forEach(e),Kr=m(s),d(Ce.$$.fragment,s),Jr=m(s),Ta=r(s,"P",{});var Eo=t(Ta);qa=r(Eo,"IMG",{src:!0,alt:!0}),Eo.forEach(e),Ur=m(s),X=r(s,"H2",{class:!0});var Wt=t(X);Ts=r(Wt,"A",{id:!0,class:!0,href:!0});var So=t(Ts);sl=r(So,"SPAN",{});var xo=t(sl);d(Ne.$$.fragment,xo),xo.forEach(e),So.forEach(e),Iu=m(Wt),el=r(Wt,"SPAN",{});var Ao=t(el);Fu=u(Ao,"Multimodal"),Ao.forEach(e),Wt.forEach(e),Gr=m(s),Pa=r(s,"P",{});var yo=t(Pa);Cu=u(yo,"F\xFCr multimodale Aufgaben werden Sie eine Kombination aus allem, was Sie bisher gelernt haben, verwenden und Ihre F\xE4higkeiten auf eine Aufgabe der automatischen Spracherkennung (ASR) anwenden. Dies bedeutet, dass Sie einen:"),yo.forEach(e),Zr=m(s),qs=r(s,"UL",{});var Ot=t(qs);al=r(Ot,"LI",{});var To=t(al);Nu=u(To,"Feature Extractor zur Vorverarbeitung der Audiodaten."),To.forEach(e),Wu=m(Ot),nl=r(Ot,"LI",{});var qo=t(nl);Ou=u(qo,"Tokenizer, um den Text zu verarbeiten."),qo.forEach(e),Ot.forEach(e),Yr=m(s),Ps=r(s,"P",{});var Rt=t(Ps);Ru=u(Rt,"Kehren wir zum "),We=r(Rt,"A",{href:!0,rel:!0});var Po=t(We);Hu=u(Po,"LJ Speech"),Po.forEach(e),Vu=u(Rt," Datensatz zur\xFCck:"),Rt.forEach(e),Qr=m(s),d(Oe.$$.fragment,s),Xr=m(s),Da=r(s,"P",{});var Do=t(Da);Ku=u(Do,"Da Sie haupts\xE4chlich an den Spalten \u201CAudio\u201D und \u201CText\u201D interessiert sind, entfernen Sie die anderen Spalten:"),Do.forEach(e),st=m(s),d(Re.$$.fragment,s),et=m(s),Ma=r(s,"P",{});var Mo=t(Ma);Ju=u(Mo,"Schauen Sie sich nun die Spalten \u201CAudio\u201D und \u201CText\u201D an:"),Mo.forEach(e),at=m(s),d(He.$$.fragment,s),nt=m(s),Ds=r(s,"P",{});var Ht=t(Ds);Uu=u(Ht,"Erinnern Sie sich an den fr\xFCheren Abschnitt \xFCber die Verarbeitung von Audiodaten: Sie sollten immer die Abtastrate Ihrer Audiodaten "),La=r(Ht,"A",{href:!0});var Lo=t(La);Gu=u(Lo,"resample"),Lo.forEach(e),Zu=u(Ht,", damit sie mit der Abtastrate des Datensatzes \xFCbereinstimmt, der f\xFCr das Vortraining eines Modells verwendet wird:"),Ht.forEach(e),lt=m(s),d(Ve.$$.fragment,s),rt=m(s),ss=r(s,"H3",{class:!0});var Vt=t(ss);Ms=r(Vt,"A",{id:!0,class:!0,href:!0});var Bo=t(Ms);ll=r(Bo,"SPAN",{});var Io=t(ll);d(Ke.$$.fragment,Io),Io.forEach(e),Bo.forEach(e),Yu=m(Vt),rl=r(Vt,"SPAN",{});var Fo=t(rl);Qu=u(Fo,"Prozessor"),Fo.forEach(e),Vt.forEach(e),tt=m(s),Ba=r(s,"P",{});var Co=t(Ba);Xu=u(Co,"Ein Processor kombiniert einen Feature-Extraktor und einen Tokenizer. Laden Sie einen Processor mit [`AutoProcessor.from_pretrained]:"),Co.forEach(e),pt=m(s),d(Je.$$.fragment,s),it=m(s),Ia=r(s,"OL",{});var No=t(Ia);es=r(No,"LI",{});var Ja=t(es);sh=u(Ja,"Erstellen Sie eine Funktion, die die Audiodaten zu "),tl=r(Ja,"CODE",{});var Wo=t(tl);eh=u(Wo,"input_values"),Wo.forEach(e),ah=u(Ja," verarbeitet und den Text zu "),pl=r(Ja,"CODE",{});var Oo=t(pl);nh=u(Oo,"labels"),Oo.forEach(e),lh=u(Ja," tokenisiert. Dies sind Ihre Eingaben f\xFCr das Modell:"),Ja.forEach(e),No.forEach(e),ut=m(s),d(Ue.$$.fragment,s),ht=m(s),Ge=r(s,"OL",{start:!0});var Ro=t(Ge);il=r(Ro,"LI",{});var Ho=t(il);rh=u(Ho,"Wenden Sie die Funktion \u201Cprepare_dataset\u201D auf ein Beispiel an:"),Ho.forEach(e),Ro.forEach(e),mt=m(s),d(Ze.$$.fragment,s),ot=m(s),C=r(s,"P",{});var Ua=t(C);th=u(Ua,"Beachten Sie, dass der Processor "),ul=r(Ua,"CODE",{});var Vo=t(ul);ph=u(Vo,"input_values"),Vo.forEach(e),ih=u(Ua," und "),hl=r(Ua,"CODE",{});var Ko=t(hl);uh=u(Ko,"labels"),Ko.forEach(e),hh=u(Ua," hinzugef\xFCgt hat. Auch die Abtastrate wurde korrekt auf 16kHz heruntergerechnet."),Ua.forEach(e),ct=m(s),Fa=r(s,"P",{});var Jo=t(Fa);mh=u(Jo,"Toll, Sie sollten jetzt in der Lage sein, Daten f\xFCr jede Modalit\xE4t vorzuverarbeiten und sogar verschiedene Modalit\xE4ten zu kombinieren! Im n\xE4chsten Kurs lernen Sie, wie Sie ein Modell mit Ihren neu aufbereiteten Daten feinabstimmen k\xF6nnen."),Jo.forEach(e),this.h()},h(){o(_,"name","hf:doc:metadata"),o(_,"content",JSON.stringify(oc)),o(w,"id","vorverarbeiten"),o(w,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),o(w,"href","#vorverarbeiten"),o(c,"class","relative group"),o(ns,"id","nlp"),o(ns,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),o(ns,"href","#nlp"),o(N,"class","relative group"),o(Xe,"href","main_classes/tokenizer"),o(ts,"id","tokenize"),o(ts,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),o(ts,"href","#tokenize"),o(W,"class","relative group"),o(na,"href","glossary#input-ids"),o(ra,"href","glossary#attention-mask"),o(pa,"href","glossary#token-type-ids"),o(us,"id","pad"),o(us,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),o(us,"href","#pad"),o(O,"class","relative group"),o(ms,"id","krzung"),o(ms,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),o(ms,"href","#krzung"),o(R,"class","relative group"),o(os,"id","tensoren-erstellen"),o(os,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),o(os,"href","#tensoren-erstellen"),o(H,"class","relative group"),o(bs,"id","audio"),o(bs,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),o(bs,"href","#audio"),o(V,"class","relative group"),o(ba,"href","main_classes/feature_extractor"),o(Xs,"href","https://huggingface.co/datasets/PolyAI/minds14"),o(Xs,"rel","nofollow"),o(se,"href","https://huggingface.co/docs/datasets/load_hub.html"),o(se,"rel","nofollow"),o(fs,"id","resample"),o(fs,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),o(fs,"href","#resample"),o(K,"class","relative group"),o(le,"href","https://huggingface.co/facebook/wav2vec2-base"),o(le,"rel","nofollow"),o(re,"href","https://huggingface.co/datasets/PolyAI/minds14"),o(re,"rel","nofollow"),o(ie,"start","2"),o(_s,"id","merkmalsextraktor"),o(_s,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),o(_s,"href","#merkmalsextraktor"),o(J,"class","relative group"),o(ws,"id","auffllen-und-krzen"),o(ws,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),o(ws,"href","#auffllen-und-krzen"),o(U,"class","relative group"),o(zs,"id","bildverarbeitung"),o(zs,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),o(zs,"href","#bildverarbeitung"),o(G,"class","relative group"),o(ve,"href","https://huggingface.co/datasets/food101"),o(ve,"rel","nofollow"),o(ke,"href","https://huggingface.co/docs/datasets/package_reference/main_classes.html?highlight=image#datasets.Image"),o(ke,"rel","nofollow"),Uo(Sa.src,jh="https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/vision-preprocess-tutorial.png")||o(Sa,"src",jh),o(Sa,"alt","vision-preprocess-tutorial.png"),o(Ss,"id","merkmalsextraktor"),o(Ss,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),o(Ss,"href","#merkmalsextraktor"),o(Z,"class","relative group"),o(As,"id","datenerweiterung"),o(As,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),o(As,"href","#datenerweiterung"),o(Y,"class","relative group"),o(Se,"href","https://pytorch.org/vision/stable/transforms.html"),o(Se,"rel","nofollow"),o(xe,"href","https://pytorch.org/vision/master/generated/torchvision.transforms.Compose.html"),o(xe,"rel","nofollow"),o(Ae,"href","https://pytorch.org/vision/main/generated/torchvision.transforms.RandomResizedCrop.html"),o(Ae,"rel","nofollow"),o(ye,"href","https://pytorch.org/vision/main/generated/torchvision.transforms.ColorJitter.html"),o(ye,"rel","nofollow"),o(Aa,"href","model_doc/visionencoderdecoder#transformers.VisionEncoderDecoderModel.forward.pixel_values"),o(qe,"start","2"),o(Le,"href","https://huggingface.co/docs/datasets/process.html#format-transform"),o(Le,"rel","nofollow"),o(De,"start","3"),o(Ie,"start","4"),Uo(qa.src,fh="https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/preprocessed_image.png")||o(qa,"src",fh),o(qa,"alt","preprocessed_image"),o(Ts,"id","multimodal"),o(Ts,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),o(Ts,"href","#multimodal"),o(X,"class","relative group"),o(We,"href","https://huggingface.co/datasets/lj_speech"),o(We,"rel","nofollow"),o(La,"href","preprocessing#audio"),o(Ms,"id","prozessor"),o(Ms,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),o(Ms,"href","#prozessor"),o(ss,"class","relative group"),o(Ge,"start","2")},m(s,n){a(document.head,_),p(s,$,n),p(s,c,n),a(c,w),a(w,z),j(S,z,null),a(c,q),a(c,as),a(as,Kt),p(s,ol,n),j(Fs,s,n),p(s,cl,n),p(s,Qe,n),a(Qe,Jt),p(s,bl,n),p(s,P,n),a(P,Ga),a(Ga,Ut),a(P,Gt),a(P,Za),a(Za,Zt),a(P,Yt),a(P,Ya),a(Ya,Qt),p(s,dl,n),p(s,N,n),a(N,ns),a(ns,Qa),j(Cs,Qa,null),a(N,Xt),a(N,Xa),a(Xa,sp),p(s,jl,n),j(Ns,s,n),p(s,fl,n),p(s,D,n),a(D,ep),a(D,Xe),a(Xe,ap),a(D,np),a(D,sn),a(sn,lp),a(D,rp),p(s,gl,n),j(ls,s,n),p(s,vl,n),p(s,rs,n),a(rs,tp),a(rs,en),a(en,pp),a(rs,ip),p(s,_l,n),p(s,W,n),a(W,ts),a(ts,an),j(Ws,an,null),a(W,up),a(W,nn),a(nn,hp),p(s,kl,n),p(s,ps,n),a(ps,mp),a(ps,ln),a(ln,op),a(ps,cp),p(s,$l,n),j(Os,s,n),p(s,wl,n),p(s,sa,n),a(sa,bp),p(s,zl,n),j(Rs,s,n),p(s,El,n),p(s,ea,n),a(ea,dp),p(s,Sl,n),p(s,M,n),a(M,aa),a(aa,na),a(na,jp),a(aa,fp),a(M,gp),a(M,la),a(la,ra),a(ra,vp),a(la,_p),a(M,kp),a(M,ta),a(ta,pa),a(pa,$p),a(ta,wp),p(s,xl,n),p(s,is,n),a(is,zp),a(is,rn),a(rn,Ep),a(is,Sp),p(s,Al,n),j(Hs,s,n),p(s,yl,n),p(s,L,n),a(L,xp),a(L,tn),a(tn,Ap),a(L,yp),a(L,pn),a(pn,Tp),a(L,qp),p(s,Tl,n),p(s,ia,n),a(ia,Pp),p(s,ql,n),j(Vs,s,n),p(s,Pl,n),p(s,O,n),a(O,us),a(us,un),j(Ks,un,null),a(O,Dp),a(O,hn),a(hn,Mp),p(s,Dl,n),p(s,hs,n),a(hs,Lp),a(hs,mn),a(mn,Bp),a(hs,Ip),p(s,Ml,n),p(s,ua,n),a(ua,Fp),p(s,Ll,n),j(Js,s,n),p(s,Bl,n),p(s,ha,n),a(ha,Cp),p(s,Il,n),p(s,R,n),a(R,ms),a(ms,on),j(Us,on,null),a(R,Np),a(R,cn),a(cn,Wp),p(s,Fl,n),p(s,ma,n),a(ma,Op),p(s,Cl,n),p(s,oa,n),a(oa,Rp),p(s,Nl,n),j(Gs,s,n),p(s,Wl,n),p(s,H,n),a(H,os),a(os,bn),j(Zs,bn,null),a(H,Hp),a(H,dn),a(dn,Vp),p(s,Ol,n),p(s,ca,n),a(ca,Kp),p(s,Rl,n),p(s,x,n),a(x,Jp),a(x,jn),a(jn,Up),a(x,Gp),a(x,fn),a(fn,Zp),a(x,Yp),a(x,gn),a(gn,Qp),a(x,Xp),p(s,Hl,n),j(cs,s,n),p(s,Vl,n),p(s,V,n),a(V,bs),a(bs,vn),j(Ys,vn,null),a(V,si),a(V,_n),a(_n,ei),p(s,Kl,n),p(s,ds,n),a(ds,ai),a(ds,ba),a(ba,ni),a(ds,li),p(s,Jl,n),j(Qs,s,n),p(s,Ul,n),p(s,B,n),a(B,ri),a(B,Xs),a(Xs,ti),a(B,pi),a(B,se),a(se,ii),a(B,ui),p(s,Gl,n),j(ee,s,n),p(s,Zl,n),p(s,js,n),a(js,hi),a(js,kn),a(kn,mi),a(js,oi),p(s,Yl,n),j(ae,s,n),p(s,Ql,n),p(s,da,n),a(da,ci),p(s,Xl,n),p(s,I,n),a(I,$n),a($n,bi),a(I,di),a(I,wn),a(wn,ji),a(I,fi),a(I,ja),a(ja,zn),a(zn,gi),a(ja,vi),p(s,sr,n),p(s,K,n),a(K,fs),a(fs,En),j(ne,En,null),a(K,_i),a(K,Sn),a(Sn,ki),p(s,er,n),p(s,gs,n),a(gs,$i),a(gs,le),a(le,wi),a(gs,zi),p(s,ar,n),p(s,vs,n),a(vs,Ei),a(vs,re),a(re,Si),a(vs,xi),p(s,nr,n),j(te,s,n),p(s,lr,n),p(s,fa,n),a(fa,xn),a(xn,Ai),p(s,rr,n),j(pe,s,n),p(s,tr,n),p(s,ie,n),a(ie,An),a(An,yi),p(s,pr,n),j(ue,s,n),p(s,ir,n),p(s,ga,n),a(ga,Ti),p(s,ur,n),p(s,J,n),a(J,_s),a(_s,yn),j(he,yn,null),a(J,qi),a(J,Tn),a(Tn,Pi),p(s,hr,n),p(s,A,n),a(A,Di),a(A,qn),a(qn,Mi),a(A,Li),a(A,Pn),a(Pn,Bi),a(A,Ii),a(A,Dn),a(Dn,Fi),a(A,Ci),p(s,mr,n),p(s,ks,n),a(ks,Ni),a(ks,Mn),a(Mn,Wi),a(ks,Oi),p(s,or,n),j(me,s,n),p(s,cr,n),p(s,$s,n),a($s,Ri),a($s,Ln),a(Ln,Hi),a($s,Vi),p(s,br,n),j(oe,s,n),p(s,dr,n),p(s,U,n),a(U,ws),a(ws,Bn),j(ce,Bn,null),a(U,Ki),a(U,In),a(In,Ji),p(s,jr,n),p(s,va,n),a(va,Ui),p(s,fr,n),j(be,s,n),p(s,gr,n),p(s,_a,n),a(_a,Gi),p(s,vr,n),j(de,s,n),p(s,_r,n),p(s,ka,n),a(ka,Zi),p(s,kr,n),j(je,s,n),p(s,$r,n),p(s,$a,n),a($a,Yi),p(s,wr,n),j(fe,s,n),p(s,zr,n),p(s,wa,n),a(wa,Qi),p(s,Er,n),p(s,G,n),a(G,zs),a(zs,Fn),j(ge,Fn,null),a(G,Xi),a(G,Cn),a(Cn,su),p(s,Sr,n),p(s,za,n),a(za,eu),p(s,xr,n),p(s,F,n),a(F,au),a(F,ve),a(ve,nu),a(F,lu),a(F,Nn),a(Nn,ru),a(F,tu),p(s,Ar,n),j(_e,s,n),p(s,yr,n),p(s,Es,n),a(Es,pu),a(Es,ke),a(ke,iu),a(Es,uu),p(s,Tr,n),j($e,s,n),p(s,qr,n),p(s,Ea,n),a(Ea,Sa),p(s,Pr,n),p(s,Z,n),a(Z,Ss),a(Ss,Wn),j(we,Wn,null),a(Z,hu),a(Z,On),a(On,mu),p(s,Dr,n),p(s,xs,n),a(xs,ou),a(xs,Rn),a(Rn,cu),a(xs,bu),p(s,Mr,n),j(ze,s,n),p(s,Lr,n),p(s,Y,n),a(Y,As),a(As,Hn),j(Ee,Hn,null),a(Y,du),a(Y,Vn),a(Vn,ju),p(s,Br,n),p(s,ys,n),a(ys,fu),a(ys,Se),a(Se,Kn),a(Kn,gu),a(ys,vu),p(s,Ir,n),p(s,xa,n),a(xa,T),a(T,_u),a(T,xe),a(xe,Jn),a(Jn,ku),a(T,$u),a(T,Ae),a(Ae,Un),a(Un,wu),a(T,zu),a(T,ye),a(ye,Gn),a(Gn,Eu),a(T,Su),p(s,Fr,n),j(Te,s,n),p(s,Cr,n),p(s,qe,n),a(qe,Q),a(Q,xu),a(Q,Aa),a(Aa,Zn),a(Zn,Au),a(Q,yu),a(Q,Yn),a(Yn,Tu),a(Q,qu),p(s,Nr,n),j(Pe,s,n),p(s,Wr,n),p(s,De,n),a(De,Me),a(Me,Pu),a(Me,Le),a(Le,Qn),a(Qn,Du),a(Me,Mu),p(s,Or,n),j(Be,s,n),p(s,Rr,n),p(s,Ie,n),a(Ie,Xn),a(Xn,Lu),p(s,Hr,n),j(Fe,s,n),p(s,Vr,n),p(s,ya,n),a(ya,Bu),p(s,Kr,n),j(Ce,s,n),p(s,Jr,n),p(s,Ta,n),a(Ta,qa),p(s,Ur,n),p(s,X,n),a(X,Ts),a(Ts,sl),j(Ne,sl,null),a(X,Iu),a(X,el),a(el,Fu),p(s,Gr,n),p(s,Pa,n),a(Pa,Cu),p(s,Zr,n),p(s,qs,n),a(qs,al),a(al,Nu),a(qs,Wu),a(qs,nl),a(nl,Ou),p(s,Yr,n),p(s,Ps,n),a(Ps,Ru),a(Ps,We),a(We,Hu),a(Ps,Vu),p(s,Qr,n),j(Oe,s,n),p(s,Xr,n),p(s,Da,n),a(Da,Ku),p(s,st,n),j(Re,s,n),p(s,et,n),p(s,Ma,n),a(Ma,Ju),p(s,at,n),j(He,s,n),p(s,nt,n),p(s,Ds,n),a(Ds,Uu),a(Ds,La),a(La,Gu),a(Ds,Zu),p(s,lt,n),j(Ve,s,n),p(s,rt,n),p(s,ss,n),a(ss,Ms),a(Ms,ll),j(Ke,ll,null),a(ss,Yu),a(ss,rl),a(rl,Qu),p(s,tt,n),p(s,Ba,n),a(Ba,Xu),p(s,pt,n),j(Je,s,n),p(s,it,n),p(s,Ia,n),a(Ia,es),a(es,sh),a(es,tl),a(tl,eh),a(es,ah),a(es,pl),a(pl,nh),a(es,lh),p(s,ut,n),j(Ue,s,n),p(s,ht,n),p(s,Ge,n),a(Ge,il),a(il,rh),p(s,mt,n),j(Ze,s,n),p(s,ot,n),p(s,C,n),a(C,th),a(C,ul),a(ul,ph),a(C,ih),a(C,hl),a(hl,uh),a(C,hh),p(s,ct,n),p(s,Fa,n),a(Fa,mh),bt=!0},p(s,[n]){const Ye={};n&2&&(Ye.$$scope={dirty:n,ctx:s}),ls.$set(Ye);const ml={};n&2&&(ml.$$scope={dirty:n,ctx:s}),cs.$set(ml)},i(s){bt||(f(S.$$.fragment,s),f(Fs.$$.fragment,s),f(Cs.$$.fragment,s),f(Ns.$$.fragment,s),f(ls.$$.fragment,s),f(Ws.$$.fragment,s),f(Os.$$.fragment,s),f(Rs.$$.fragment,s),f(Hs.$$.fragment,s),f(Vs.$$.fragment,s),f(Ks.$$.fragment,s),f(Js.$$.fragment,s),f(Us.$$.fragment,s),f(Gs.$$.fragment,s),f(Zs.$$.fragment,s),f(cs.$$.fragment,s),f(Ys.$$.fragment,s),f(Qs.$$.fragment,s),f(ee.$$.fragment,s),f(ae.$$.fragment,s),f(ne.$$.fragment,s),f(te.$$.fragment,s),f(pe.$$.fragment,s),f(ue.$$.fragment,s),f(he.$$.fragment,s),f(me.$$.fragment,s),f(oe.$$.fragment,s),f(ce.$$.fragment,s),f(be.$$.fragment,s),f(de.$$.fragment,s),f(je.$$.fragment,s),f(fe.$$.fragment,s),f(ge.$$.fragment,s),f(_e.$$.fragment,s),f($e.$$.fragment,s),f(we.$$.fragment,s),f(ze.$$.fragment,s),f(Ee.$$.fragment,s),f(Te.$$.fragment,s),f(Pe.$$.fragment,s),f(Be.$$.fragment,s),f(Fe.$$.fragment,s),f(Ce.$$.fragment,s),f(Ne.$$.fragment,s),f(Oe.$$.fragment,s),f(Re.$$.fragment,s),f(He.$$.fragment,s),f(Ve.$$.fragment,s),f(Ke.$$.fragment,s),f(Je.$$.fragment,s),f(Ue.$$.fragment,s),f(Ze.$$.fragment,s),bt=!0)},o(s){g(S.$$.fragment,s),g(Fs.$$.fragment,s),g(Cs.$$.fragment,s),g(Ns.$$.fragment,s),g(ls.$$.fragment,s),g(Ws.$$.fragment,s),g(Os.$$.fragment,s),g(Rs.$$.fragment,s),g(Hs.$$.fragment,s),g(Vs.$$.fragment,s),g(Ks.$$.fragment,s),g(Js.$$.fragment,s),g(Us.$$.fragment,s),g(Gs.$$.fragment,s),g(Zs.$$.fragment,s),g(cs.$$.fragment,s),g(Ys.$$.fragment,s),g(Qs.$$.fragment,s),g(ee.$$.fragment,s),g(ae.$$.fragment,s),g(ne.$$.fragment,s),g(te.$$.fragment,s),g(pe.$$.fragment,s),g(ue.$$.fragment,s),g(he.$$.fragment,s),g(me.$$.fragment,s),g(oe.$$.fragment,s),g(ce.$$.fragment,s),g(be.$$.fragment,s),g(de.$$.fragment,s),g(je.$$.fragment,s),g(fe.$$.fragment,s),g(ge.$$.fragment,s),g(_e.$$.fragment,s),g($e.$$.fragment,s),g(we.$$.fragment,s),g(ze.$$.fragment,s),g(Ee.$$.fragment,s),g(Te.$$.fragment,s),g(Pe.$$.fragment,s),g(Be.$$.fragment,s),g(Fe.$$.fragment,s),g(Ce.$$.fragment,s),g(Ne.$$.fragment,s),g(Oe.$$.fragment,s),g(Re.$$.fragment,s),g(He.$$.fragment,s),g(Ve.$$.fragment,s),g(Ke.$$.fragment,s),g(Je.$$.fragment,s),g(Ue.$$.fragment,s),g(Ze.$$.fragment,s),bt=!1},d(s){e(_),s&&e($),s&&e(c),v(S),s&&e(ol),v(Fs,s),s&&e(cl),s&&e(Qe),s&&e(bl),s&&e(P),s&&e(dl),s&&e(N),v(Cs),s&&e(jl),v(Ns,s),s&&e(fl),s&&e(D),s&&e(gl),v(ls,s),s&&e(vl),s&&e(rs),s&&e(_l),s&&e(W),v(Ws),s&&e(kl),s&&e(ps),s&&e($l),v(Os,s),s&&e(wl),s&&e(sa),s&&e(zl),v(Rs,s),s&&e(El),s&&e(ea),s&&e(Sl),s&&e(M),s&&e(xl),s&&e(is),s&&e(Al),v(Hs,s),s&&e(yl),s&&e(L),s&&e(Tl),s&&e(ia),s&&e(ql),v(Vs,s),s&&e(Pl),s&&e(O),v(Ks),s&&e(Dl),s&&e(hs),s&&e(Ml),s&&e(ua),s&&e(Ll),v(Js,s),s&&e(Bl),s&&e(ha),s&&e(Il),s&&e(R),v(Us),s&&e(Fl),s&&e(ma),s&&e(Cl),s&&e(oa),s&&e(Nl),v(Gs,s),s&&e(Wl),s&&e(H),v(Zs),s&&e(Ol),s&&e(ca),s&&e(Rl),s&&e(x),s&&e(Hl),v(cs,s),s&&e(Vl),s&&e(V),v(Ys),s&&e(Kl),s&&e(ds),s&&e(Jl),v(Qs,s),s&&e(Ul),s&&e(B),s&&e(Gl),v(ee,s),s&&e(Zl),s&&e(js),s&&e(Yl),v(ae,s),s&&e(Ql),s&&e(da),s&&e(Xl),s&&e(I),s&&e(sr),s&&e(K),v(ne),s&&e(er),s&&e(gs),s&&e(ar),s&&e(vs),s&&e(nr),v(te,s),s&&e(lr),s&&e(fa),s&&e(rr),v(pe,s),s&&e(tr),s&&e(ie),s&&e(pr),v(ue,s),s&&e(ir),s&&e(ga),s&&e(ur),s&&e(J),v(he),s&&e(hr),s&&e(A),s&&e(mr),s&&e(ks),s&&e(or),v(me,s),s&&e(cr),s&&e($s),s&&e(br),v(oe,s),s&&e(dr),s&&e(U),v(ce),s&&e(jr),s&&e(va),s&&e(fr),v(be,s),s&&e(gr),s&&e(_a),s&&e(vr),v(de,s),s&&e(_r),s&&e(ka),s&&e(kr),v(je,s),s&&e($r),s&&e($a),s&&e(wr),v(fe,s),s&&e(zr),s&&e(wa),s&&e(Er),s&&e(G),v(ge),s&&e(Sr),s&&e(za),s&&e(xr),s&&e(F),s&&e(Ar),v(_e,s),s&&e(yr),s&&e(Es),s&&e(Tr),v($e,s),s&&e(qr),s&&e(Ea),s&&e(Pr),s&&e(Z),v(we),s&&e(Dr),s&&e(xs),s&&e(Mr),v(ze,s),s&&e(Lr),s&&e(Y),v(Ee),s&&e(Br),s&&e(ys),s&&e(Ir),s&&e(xa),s&&e(Fr),v(Te,s),s&&e(Cr),s&&e(qe),s&&e(Nr),v(Pe,s),s&&e(Wr),s&&e(De),s&&e(Or),v(Be,s),s&&e(Rr),s&&e(Ie),s&&e(Hr),v(Fe,s),s&&e(Vr),s&&e(ya),s&&e(Kr),v(Ce,s),s&&e(Jr),s&&e(Ta),s&&e(Ur),s&&e(X),v(Ne),s&&e(Gr),s&&e(Pa),s&&e(Zr),s&&e(qs),s&&e(Yr),s&&e(Ps),s&&e(Qr),v(Oe,s),s&&e(Xr),s&&e(Da),s&&e(st),v(Re,s),s&&e(et),s&&e(Ma),s&&e(at),v(He,s),s&&e(nt),s&&e(Ds),s&&e(lt),v(Ve,s),s&&e(rt),s&&e(ss),v(Ke),s&&e(tt),s&&e(Ba),s&&e(pt),v(Je,s),s&&e(it),s&&e(Ia),s&&e(ut),v(Ue,s),s&&e(ht),s&&e(Ge),s&&e(mt),v(Ze,s),s&&e(ot),s&&e(C),s&&e(ct),s&&e(Fa)}}}const oc={local:"vorverarbeiten",sections:[{local:"nlp",sections:[{local:"tokenize",title:"Tokenize"},{local:"pad",title:"Pad"},{local:"krzung",title:"K\xFCrzung"},{local:"tensoren-erstellen",title:"Tensoren erstellen"}],title:"NLP"},{local:"audio",sections:[{local:"resample",title:"Resample"},{local:"merkmalsextraktor",title:"Merkmalsextraktor"},{local:"auffllen-und-krzen",title:"Auff\xFCllen und K\xFCrzen"}],title:"Audio"},{local:"bildverarbeitung",sections:[{local:"merkmalsextraktor",title:"Merkmalsextraktor"},{local:"datenerweiterung",title:"Datenerweiterung"}],title:"Bildverarbeitung"},{local:"multimodal",sections:[{local:"prozessor",title:"Prozessor"}],title:"Multimodal"}],title:"Vorverarbeiten"};function cc(y){return ec(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class kc extends Yo{constructor(_){super();Qo(this,_,cc,mc,Xo,{})}}export{kc as default,oc as metadata};
