import{S as Au,i as Cu,s as Du,e as l,k as c,t as p,c as t,a as r,m as u,h as o,d as a,b as m,g as i,G as e,Q as Bl,q as g,l as vd,n as Ep,o as _,B as E,p as zp,w as z,y as w,j as Cd,K as Dd,U as wd,x as y,V as Id,T as Sd,Y as qd,Z as yd,M as Nd,N as $d,v as Td}from"../chunks/vendor-hf-doc-builder.js";import{T as Od}from"../chunks/Tip-hf-doc-builder.js";import{Y as Ld}from"../chunks/Youtube-hf-doc-builder.js";import{I as S}from"../chunks/IconCopyLink-hf-doc-builder.js";import{a as xd,C as D}from"../chunks/CodeBlock-hf-doc-builder.js";import{b as Pd,I as Ud,a as Rd}from"../chunks/IconTensorflow-hf-doc-builder.js";import{D as Fd}from"../chunks/DocNotebookDropdown-hf-doc-builder.js";function kd(x,h,f){const b=x.slice();return b[8]=h[f],b[10]=f,b}function Ed(x){let h,f,b;var v=x[8].icon;function j(q){return{props:{classNames:"mr-1.5"}}}return v&&(h=new v(j())),{c(){h&&z(h.$$.fragment),f=vd()},l(q){h&&y(h.$$.fragment,q),f=vd()},m(q,$){h&&w(h,q,$),i(q,f,$),b=!0},p(q,$){if(v!==(v=q[8].icon)){if(h){Ep();const k=h;_(k.$$.fragment,1,0,()=>{E(k,1)}),zp()}v?(h=new v(j()),z(h.$$.fragment),g(h.$$.fragment,1),w(h,f.parentNode,f)):h=null}},i(q){b||(h&&g(h.$$.fragment,q),b=!0)},o(q){h&&_(h.$$.fragment,q),b=!1},d(q){q&&a(f),h&&E(h,q)}}}function zd(x){let h,f,b,v=x[8].name+"",j,q,$,k,d,P,A,C=x[8].icon&&Ed(x);function bs(){return x[6](x[8])}return{c(){h=l("button"),C&&C.c(),f=c(),b=l("p"),j=p(v),$=c(),this.h()},l(N){h=t(N,"BUTTON",{class:!0});var I=r(h);C&&C.l(I),f=u(I),b=t(I,"P",{class:!0});var T=r(b);j=o(T,v),T.forEach(a),$=u(I),I.forEach(a),this.h()},h(){m(b,"class",q="!m-0 "+x[8].classNames),m(h,"class",k="flex justify-center py-1.5 px-2.5 focus:outline-none rounded-"+(x[10]?"r":"l")+" "+(x[8].group!==x[1]&&"text-gray-500 filter grayscale"))},m(N,I){i(N,h,I),C&&C.m(h,null),e(h,f),e(h,b),e(b,j),e(h,$),d=!0,P||(A=Bl(h,"click",bs),P=!0)},p(N,I){x=N,x[8].icon?C?(C.p(x,I),I&1&&g(C,1)):(C=Ed(x),C.c(),g(C,1),C.m(h,f)):C&&(Ep(),_(C,1,1,()=>{C=null}),zp()),(!d||I&1)&&v!==(v=x[8].name+"")&&Cd(j,v),(!d||I&1&&q!==(q="!m-0 "+x[8].classNames))&&m(b,"class",q),(!d||I&3&&k!==(k="flex justify-center py-1.5 px-2.5 focus:outline-none rounded-"+(x[10]?"r":"l")+" "+(x[8].group!==x[1]&&"text-gray-500 filter grayscale")))&&m(h,"class",k)},i(N){d||(g(C),d=!0)},o(N){_(C),d=!1},d(N){N&&a(h),C&&C.d(),P=!1,A()}}}function Hd(x){let h,f,b,v=x[3].filter(x[5]),j=[];for(let $=0;$<v.length;$+=1)j[$]=zd(kd(x,v,$));const q=$=>_(j[$],1,1,()=>{j[$]=null});return{c(){h=l("div"),f=l("div");for(let $=0;$<j.length;$+=1)j[$].c();this.h()},l($){h=t($,"DIV",{});var k=r(h);f=t(k,"DIV",{class:!0});var d=r(f);for(let P=0;P<j.length;P+=1)j[P].l(d);d.forEach(a),k.forEach(a),this.h()},h(){m(f,"class","bg-white leading-none border border-gray-100 rounded-lg inline-flex p-0.5 text-sm mb-4 select-none")},m($,k){i($,h,k),e(h,f);for(let d=0;d<j.length;d+=1)j[d].m(f,null);b=!0},p($,[k]){if(k&27){v=$[3].filter($[5]);let d;for(d=0;d<v.length;d+=1){const P=kd($,v,d);j[d]?(j[d].p(P,k),g(j[d],1)):(j[d]=zd(P),j[d].c(),g(j[d],1),j[d].m(f,null))}for(Ep(),d=v.length;d<j.length;d+=1)q(d);zp()}},i($){if(!b){for(let k=0;k<v.length;k+=1)g(j[k]);b=!0}},o($){j=j.filter(Boolean);for(let k=0;k<j.length;k+=1)_(j[k]);b=!1},d($){$&&a(h),Dd(j,$)}}}function Bd(x,h,f){let b,{ids:v}=h;const j=v.join("-"),q=Pd(j);wd(x,q,A=>f(1,b=A));const $=[{id:"pt",classNames:"",icon:Ud,name:"Pytorch",group:"group1"},{id:"tf",classNames:"",icon:Rd,name:"TensorFlow",group:"group2"},{id:"stringapi",classNames:"text-blue-600",name:"String API",group:"group1"},{id:"readinstruction",classNames:"text-blue-600",name:"ReadInstruction",group:"group2"}];function k(A){Id(q,b=A,b)}const d=A=>v.includes(A.id),P=A=>k(A.group);return x.$$set=A=>{"ids"in A&&f(0,v=A.ids)},[v,b,q,$,k,d,P]}class Ad extends Au{constructor(h){super();Cu(this,h,Bd,Hd,Du,{ids:0})}}function Md(x){let h,f,b,v,j,q,$=x[1].highlighted+"",k;return f=new xd({props:{classNames:"transition duration-200 ease-in-out "+(x[2]&&"opacity-0"),title:"Copy code excerpt to clipboard",value:x[1].code}}),j=new Ad({props:{ids:x[4]}}),{c(){h=l("div"),z(f.$$.fragment),b=c(),v=l("pre"),z(j.$$.fragment),q=new qd,this.h()},l(d){h=t(d,"DIV",{class:!0});var P=r(h);y(f.$$.fragment,P),P.forEach(a),b=u(d),v=t(d,"PRE",{});var A=r(v);y(j.$$.fragment,A),q=yd(A),A.forEach(a),this.h()},h(){m(h,"class","absolute top-2.5 right-4"),q.a=null},m(d,P){i(d,h,P),w(f,h,null),i(d,b,P),i(d,v,P),w(j,v,null),q.m($,v),k=!0},p(d,P){const A={};P&4&&(A.classNames="transition duration-200 ease-in-out "+(d[2]&&"opacity-0")),P&2&&(A.value=d[1].code),f.$set(A),(!k||P&2)&&$!==($=d[1].highlighted+"")&&q.p($)},i(d){k||(g(f.$$.fragment,d),g(j.$$.fragment,d),k=!0)},o(d){_(f.$$.fragment,d),_(j.$$.fragment,d),k=!1},d(d){d&&a(h),E(f),d&&a(b),d&&a(v),E(j)}}}function Qd(x){let h,f,b,v,j,q,$=x[0].highlighted+"",k;return f=new xd({props:{classNames:"transition duration-200 ease-in-out "+(x[2]&&"opacity-0"),title:"Copy code excerpt to clipboard",value:x[0].code}}),j=new Ad({props:{ids:x[4]}}),{c(){h=l("div"),z(f.$$.fragment),b=c(),v=l("pre"),z(j.$$.fragment),q=new qd,this.h()},l(d){h=t(d,"DIV",{class:!0});var P=r(h);y(f.$$.fragment,P),P.forEach(a),b=u(d),v=t(d,"PRE",{});var A=r(v);y(j.$$.fragment,A),q=yd(A),A.forEach(a),this.h()},h(){m(h,"class","absolute top-2.5 right-4"),q.a=null},m(d,P){i(d,h,P),w(f,h,null),i(d,b,P),i(d,v,P),w(j,v,null),q.m($,v),k=!0},p(d,P){const A={};P&4&&(A.classNames="transition duration-200 ease-in-out "+(d[2]&&"opacity-0")),P&1&&(A.value=d[0].code),f.$set(A),(!k||P&1)&&$!==($=d[0].highlighted+"")&&q.p($)},i(d){k||(g(f.$$.fragment,d),g(j.$$.fragment,d),k=!0)},o(d){_(f.$$.fragment,d),_(j.$$.fragment,d),k=!1},d(d){d&&a(h),E(f),d&&a(b),d&&a(v),E(j)}}}function Vd(x){let h,f,b,v,j,q;const $=[Qd,Md],k=[];function d(P,A){return P[3]==="group1"?0:1}return f=d(x),b=k[f]=$[f](x),{c(){h=l("div"),b.c(),this.h()},l(P){h=t(P,"DIV",{class:!0});var A=r(h);b.l(A),A.forEach(a),this.h()},h(){m(h,"class","code-block relative")},m(P,A){i(P,h,A),k[f].m(h,null),v=!0,j||(q=[Bl(h,"mouseover",x[6]),Bl(h,"focus",x[6]),Bl(h,"mouseout",x[7]),Bl(h,"focus",x[7])],j=!0)},p(P,[A]){let C=f;f=d(P),f===C?k[f].p(P,A):(Ep(),_(k[C],1,1,()=>{k[C]=null}),zp(),b=k[f],b?b.p(P,A):(b=k[f]=$[f](P),b.c()),g(b,1),b.m(h,null))},i(P){v||(g(b),v=!0)},o(P){_(b),v=!1},d(P){P&&a(h),k[f].d(),j=!1,Sd(q)}}}function Jd(x,h,f){let b,{group1:v}=h,{group2:j}=h;const q=[v.id,j.id],$=q.join("-"),k=Pd($);wd(x,k,C=>f(3,b=C));let d=!0;function P(){f(2,d=!1)}function A(){f(2,d=!0)}return x.$$set=C=>{"group1"in C&&f(0,v=C.group1),"group2"in C&&f(1,j=C.group2)},[v,j,d,b,q,k,P,A]}class Wd extends Au{constructor(h){super();Cu(this,h,Jd,Vd,Du,{group1:0,group2:1})}}function Gd(x){let h,f,b,v,j;return{c(){h=l("p"),f=p("Se stai pensando si utilizzare un modello preaddestrato, \xE8 importante utilizzare il tokenizer preaddestrato associato. Questo assicura che il testo sia separato allo stesso modo che nel corpus usato per l\u2019addestramento, e venga usata la stessa mappatura tokens-to-index (solitamente indicato come il "),b=l("em"),v=p("vocabolario"),j=p(") come nel preaddestramento.")},l(q){h=t(q,"P",{});var $=r(h);f=o($,"Se stai pensando si utilizzare un modello preaddestrato, \xE8 importante utilizzare il tokenizer preaddestrato associato. Questo assicura che il testo sia separato allo stesso modo che nel corpus usato per l\u2019addestramento, e venga usata la stessa mappatura tokens-to-index (solitamente indicato come il "),b=t($,"EM",{});var k=r(b);v=o(k,"vocabolario"),k.forEach(a),j=o($,") come nel preaddestramento."),$.forEach(a)},m(q,$){i(q,h,$),e(h,f),e(h,b),e(b,v),e(h,j)},d(q){q&&a(h)}}}function Yd(x){let h,f,b,v,j,q,$,k,d,P,A,C,bs,N,I,T,dn,wp,qp,fn,yp,xp,bn,Pp,Ml,ss,js,jn,Ys,Ap,gn,Cp,Ql,Ks,Vl,R,Dp,de,Ip,Sp,_n,Np,Tp,Jl,gs,Wl,F,Op,vn,Lp,Up,$n,Rp,Fp,Gl,as,_s,kn,Zs,Hp,En,Bp,Yl,vs,Mp,zn,Qp,Vp,Kl,Xs,Zl,fe,Jp,Xl,sa,st,be,Wp,at,H,je,ge,Gp,Yp,Kp,_e,ve,Zp,Xp,so,$e,ke,ao,eo,et,$s,no,wn,lo,to,nt,aa,lt,B,ro,qn,po,oo,yn,io,co,tt,Ee,uo,rt,ea,pt,es,ks,xn,na,mo,Pn,ho,ot,Es,fo,An,bo,jo,it,M,go,Cn,_o,vo,Dn,$o,ko,ct,la,ut,zs,Eo,In,zo,wo,mt,ns,ws,Sn,ta,qo,Nn,yo,ht,ze,xo,dt,Q,Po,Tn,Ao,Co,On,Do,Io,ft,ra,bt,ls,qs,Ln,pa,So,Un,No,jt,we,To,gt,O,Oo,Rn,Lo,Uo,Fn,Ro,Fo,Hn,Ho,Bo,_t,oa,vt,ts,ys,Bn,ia,Mo,Mn,Qo,$t,xs,Vo,qe,Jo,Wo,kt,ca,Et,V,Go,ua,Yo,Ko,ma,Zo,Xo,zt,ha,wt,J,si,Qn,ai,ei,Vn,ni,li,qt,da,yt,ye,ti,xt,W,xe,Jn,ri,pi,oi,Pe,Wn,ii,ci,ui,Ae,Gn,mi,hi,Pt,rs,Ps,Yn,fa,di,Kn,fi,At,As,bi,ba,ji,gi,Ct,Cs,_i,ja,vi,$i,Dt,ga,It,Ce,_a,ki,va,Zn,Ei,zi,St,$a,Nt,ka,Xn,wi,Tt,Ea,Ot,Ds,qi,sl,yi,xi,Lt,ps,Is,al,za,Pi,el,Ai,Ut,L,Ci,nl,Di,Ii,ll,Si,Ni,tl,Ti,Oi,Rt,Ss,Li,rl,Ui,Ri,Ft,wa,Ht,G,Fi,pl,Hi,Bi,ol,Mi,Qi,Bt,qa,Mt,os,Ns,il,ya,Vi,cl,Ji,Qt,De,Wi,Vt,xa,Jt,Ie,Gi,Wt,Pa,Gt,Se,Yi,Yt,Aa,Kt,Ne,Ki,Zt,Ca,Xt,Te,Zi,sr,is,Ts,ul,Da,Xi,ml,sc,ar,Oe,ac,er,Y,ec,Ia,nc,lc,hl,tc,rc,nr,Sa,lr,Os,pc,Na,dl,oc,ic,tr,Ta,rr,Le,Ue,Iu,pr,cs,Ls,fl,Oa,cc,bl,uc,or,Us,mc,jl,hc,dc,ir,La,cr,us,Rs,gl,Ua,fc,_l,bc,ur,Fs,jc,Ra,vl,gc,_c,mr,Re,U,vc,Fa,$l,$c,kc,Ha,kl,Ec,zc,Ba,El,wc,qc,hr,Ma,dr,Qa,ms,yc,Fe,zl,xc,Pc,wl,Ac,Cc,fr,Va,br,Ja,Wa,Dc,Ga,ql,Ic,Sc,jr,Ya,gr,Ka,Za,Nc,yl,Tc,Oc,_r,Xa,vr,He,Lc,$r,se,kr,Be,Me,Su,Er,hs,Hs,xl,ae,Uc,Pl,Rc,zr,Qe,Fc,wr,Bs,Al,Hc,Bc,Cl,Mc,qr,Ms,Qc,ee,Vc,Jc,yr,ne,xr,K,Wc,Dl,Gc,Yc,Il,Kc,Zc,Pr,le,Ar,Z,Xc,Sl,su,au,Nl,eu,nu,Cr,te,Dr,Qs,lu,Ve,tu,ru,Ir,re,Sr,ds,Vs,Tl,pe,pu,Ol,ou,Nr,Je,iu,Tr,oe,Or,We,fs,cu,Ll,uu,mu,Ul,hu,du,Lr,ie,Ur,ce,ue,fu,Rl,bu,ju,Rr,me,Fr,X,gu,Fl,_u,vu,Hl,$u,ku,Hr,Ge,Eu,Br;return q=new S({}),A=new Fd({props:{classNames:"absolute z-10 right-0 top-0",options:[{label:"Mixed",value:"https://colab.research.google.com/github/huggingface/notebooks/blob/main/transformers_doc/it/preprocessing.ipynb"},{label:"PyTorch",value:"https://colab.research.google.com/github/huggingface/notebooks/blob/main/transformers_doc/it/pytorch/preprocessing.ipynb"},{label:"TensorFlow",value:"https://colab.research.google.com/github/huggingface/notebooks/blob/main/transformers_doc/it/tensorflow/preprocessing.ipynb"},{label:"Mixed",value:"https://studiolab.sagemaker.aws/import/github/huggingface/notebooks/blob/main/transformers_doc/it/preprocessing.ipynb"},{label:"PyTorch",value:"https://studiolab.sagemaker.aws/import/github/huggingface/notebooks/blob/main/transformers_doc/it/pytorch/preprocessing.ipynb"},{label:"TensorFlow",value:"https://studiolab.sagemaker.aws/import/github/huggingface/notebooks/blob/main/transformers_doc/it/tensorflow/preprocessing.ipynb"}]}}),Ys=new S({}),Ks=new Ld({props:{id:"Yffk5aydLzg"}}),gs=new Od({props:{$$slots:{default:[Gd]},$$scope:{ctx:x}}}),Zs=new S({}),Xs=new D({props:{code:`from transformers import AutoTokenizer

tokenizer = AutoTokenizer.from_pretrained("bert-base-cased")`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoTokenizer

<span class="hljs-meta">&gt;&gt;&gt; </span>tokenizer = AutoTokenizer.from_pretrained(<span class="hljs-string">&quot;bert-base-cased&quot;</span>)`}}),sa=new D({props:{code:`encoded_input = tokenizer("Do not meddle in the affairs of wizards, for they are subtle and quick to anger.")
print(encoded_input)`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>encoded_input = tokenizer(<span class="hljs-string">&quot;Do not meddle in the affairs of wizards, for they are subtle and quick to anger.&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-built_in">print</span>(encoded_input)
{<span class="hljs-string">&#x27;input_ids&#x27;</span>: [<span class="hljs-number">101</span>, <span class="hljs-number">2079</span>, <span class="hljs-number">2025</span>, <span class="hljs-number">19960</span>, <span class="hljs-number">10362</span>, <span class="hljs-number">1999</span>, <span class="hljs-number">1996</span>, <span class="hljs-number">3821</span>, <span class="hljs-number">1997</span>, <span class="hljs-number">16657</span>, <span class="hljs-number">1010</span>, <span class="hljs-number">2005</span>, <span class="hljs-number">2027</span>, <span class="hljs-number">2024</span>, <span class="hljs-number">11259</span>, <span class="hljs-number">1998</span>, <span class="hljs-number">4248</span>, <span class="hljs-number">2000</span>, <span class="hljs-number">4963</span>, <span class="hljs-number">1012</span>, <span class="hljs-number">102</span>], 
 <span class="hljs-string">&#x27;token_type_ids&#x27;</span>: [<span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>], 
 <span class="hljs-string">&#x27;attention_mask&#x27;</span>: [<span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>]}`}}),aa=new D({props:{code:'tokenizer.decode(encoded_input["input_ids"])',highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>tokenizer.decode(encoded_input[<span class="hljs-string">&quot;input_ids&quot;</span>])
<span class="hljs-string">&#x27;[CLS] Do not meddle in the affairs of wizards, for they are subtle and quick to anger. [SEP]&#x27;</span>`}}),ea=new D({props:{code:`batch_sentences = [
    "But what about second breakfast?",
    "Don't think he knows about second breakfast, Pip.",
    "What about elevensies?",
]
encoded_inputs = tokenizer(batch_sentences)
print(encoded_inputs)`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>batch_sentences = [
<span class="hljs-meta">... </span>    <span class="hljs-string">&quot;But what about second breakfast?&quot;</span>,
<span class="hljs-meta">... </span>    <span class="hljs-string">&quot;Don&#x27;t think he knows about second breakfast, Pip.&quot;</span>,
<span class="hljs-meta">... </span>    <span class="hljs-string">&quot;What about elevensies?&quot;</span>,
<span class="hljs-meta">... </span>]
<span class="hljs-meta">&gt;&gt;&gt; </span>encoded_inputs = tokenizer(batch_sentences)
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-built_in">print</span>(encoded_inputs)
{<span class="hljs-string">&#x27;input_ids&#x27;</span>: [[<span class="hljs-number">101</span>, <span class="hljs-number">1252</span>, <span class="hljs-number">1184</span>, <span class="hljs-number">1164</span>, <span class="hljs-number">1248</span>, <span class="hljs-number">6462</span>, <span class="hljs-number">136</span>, <span class="hljs-number">102</span>], 
               [<span class="hljs-number">101</span>, <span class="hljs-number">1790</span>, <span class="hljs-number">112</span>, <span class="hljs-number">189</span>, <span class="hljs-number">1341</span>, <span class="hljs-number">1119</span>, <span class="hljs-number">3520</span>, <span class="hljs-number">1164</span>, <span class="hljs-number">1248</span>, <span class="hljs-number">6462</span>, <span class="hljs-number">117</span>, <span class="hljs-number">21902</span>, <span class="hljs-number">1643</span>, <span class="hljs-number">119</span>, <span class="hljs-number">102</span>], 
               [<span class="hljs-number">101</span>, <span class="hljs-number">1327</span>, <span class="hljs-number">1164</span>, <span class="hljs-number">5450</span>, <span class="hljs-number">23434</span>, <span class="hljs-number">136</span>, <span class="hljs-number">102</span>]], 
 <span class="hljs-string">&#x27;token_type_ids&#x27;</span>: [[<span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>], 
                    [<span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>], 
                    [<span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>]], 
 <span class="hljs-string">&#x27;attention_mask&#x27;</span>: [[<span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>], 
                    [<span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>], 
                    [<span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>]]}`}}),na=new S({}),la=new D({props:{code:`batch_sentences = [
    "But what about second breakfast?",
    "Don't think he knows about second breakfast, Pip.",
    "What about elevensies?",
]
encoded_input = tokenizer(batch_sentences, padding=True)
print(encoded_input)`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>batch_sentences = [
<span class="hljs-meta">... </span>    <span class="hljs-string">&quot;But what about second breakfast?&quot;</span>,
<span class="hljs-meta">... </span>    <span class="hljs-string">&quot;Don&#x27;t think he knows about second breakfast, Pip.&quot;</span>,
<span class="hljs-meta">... </span>    <span class="hljs-string">&quot;What about elevensies?&quot;</span>,
<span class="hljs-meta">... </span>]
<span class="hljs-meta">&gt;&gt;&gt; </span>encoded_input = tokenizer(batch_sentences, padding=<span class="hljs-literal">True</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-built_in">print</span>(encoded_input)
{<span class="hljs-string">&#x27;input_ids&#x27;</span>: [[<span class="hljs-number">101</span>, <span class="hljs-number">1252</span>, <span class="hljs-number">1184</span>, <span class="hljs-number">1164</span>, <span class="hljs-number">1248</span>, <span class="hljs-number">6462</span>, <span class="hljs-number">136</span>, <span class="hljs-number">102</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>], 
               [<span class="hljs-number">101</span>, <span class="hljs-number">1790</span>, <span class="hljs-number">112</span>, <span class="hljs-number">189</span>, <span class="hljs-number">1341</span>, <span class="hljs-number">1119</span>, <span class="hljs-number">3520</span>, <span class="hljs-number">1164</span>, <span class="hljs-number">1248</span>, <span class="hljs-number">6462</span>, <span class="hljs-number">117</span>, <span class="hljs-number">21902</span>, <span class="hljs-number">1643</span>, <span class="hljs-number">119</span>, <span class="hljs-number">102</span>], 
               [<span class="hljs-number">101</span>, <span class="hljs-number">1327</span>, <span class="hljs-number">1164</span>, <span class="hljs-number">5450</span>, <span class="hljs-number">23434</span>, <span class="hljs-number">136</span>, <span class="hljs-number">102</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>]], 
 <span class="hljs-string">&#x27;token_type_ids&#x27;</span>: [[<span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>], 
                    [<span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>], 
                    [<span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>]], 
 <span class="hljs-string">&#x27;attention_mask&#x27;</span>: [[<span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>], 
                    [<span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>], 
                    [<span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>]]}`}}),ta=new S({}),ra=new D({props:{code:`batch_sentences = [
    "But what about second breakfast?",
    "Don't think he knows about second breakfast, Pip.",
    "What about elevensies?",
]
encoded_input = tokenizer(batch_sentences, padding=True, truncation=True)
print(encoded_input)`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>batch_sentences = [
<span class="hljs-meta">... </span>    <span class="hljs-string">&quot;But what about second breakfast?&quot;</span>,
<span class="hljs-meta">... </span>    <span class="hljs-string">&quot;Don&#x27;t think he knows about second breakfast, Pip.&quot;</span>,
<span class="hljs-meta">... </span>    <span class="hljs-string">&quot;What about elevensies?&quot;</span>,
<span class="hljs-meta">... </span>]
<span class="hljs-meta">&gt;&gt;&gt; </span>encoded_input = tokenizer(batch_sentences, padding=<span class="hljs-literal">True</span>, truncation=<span class="hljs-literal">True</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-built_in">print</span>(encoded_input)
{<span class="hljs-string">&#x27;input_ids&#x27;</span>: [[<span class="hljs-number">101</span>, <span class="hljs-number">1252</span>, <span class="hljs-number">1184</span>, <span class="hljs-number">1164</span>, <span class="hljs-number">1248</span>, <span class="hljs-number">6462</span>, <span class="hljs-number">136</span>, <span class="hljs-number">102</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>], 
               [<span class="hljs-number">101</span>, <span class="hljs-number">1790</span>, <span class="hljs-number">112</span>, <span class="hljs-number">189</span>, <span class="hljs-number">1341</span>, <span class="hljs-number">1119</span>, <span class="hljs-number">3520</span>, <span class="hljs-number">1164</span>, <span class="hljs-number">1248</span>, <span class="hljs-number">6462</span>, <span class="hljs-number">117</span>, <span class="hljs-number">21902</span>, <span class="hljs-number">1643</span>, <span class="hljs-number">119</span>, <span class="hljs-number">102</span>], 
               [<span class="hljs-number">101</span>, <span class="hljs-number">1327</span>, <span class="hljs-number">1164</span>, <span class="hljs-number">5450</span>, <span class="hljs-number">23434</span>, <span class="hljs-number">136</span>, <span class="hljs-number">102</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>]], 
 <span class="hljs-string">&#x27;token_type_ids&#x27;</span>: [[<span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>], 
                    [<span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>], 
                    [<span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>]], 
 <span class="hljs-string">&#x27;attention_mask&#x27;</span>: [[<span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>], 
                    [<span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>], 
                    [<span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>]]}`}}),pa=new S({}),oa=new Wd({props:{group1:{id:"pt",code:`batch_sentences = [
    "But what about second breakfast?",
    "Don't think he knows about second breakfast, Pip.",
    "What about elevensies?",
]
encoded_input = tokenizer(batch, padding=True, truncation=True, return_tensors="pt")
print(encoded_input)`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>batch_sentences = [
<span class="hljs-meta">... </span>    <span class="hljs-string">&quot;But what about second breakfast?&quot;</span>,
<span class="hljs-meta">... </span>    <span class="hljs-string">&quot;Don&#x27;t think he knows about second breakfast, Pip.&quot;</span>,
<span class="hljs-meta">... </span>    <span class="hljs-string">&quot;What about elevensies?&quot;</span>,
<span class="hljs-meta">... </span>]
<span class="hljs-meta">&gt;&gt;&gt; </span>encoded_input = tokenizer(batch, padding=<span class="hljs-literal">True</span>, truncation=<span class="hljs-literal">True</span>, return_tensors=<span class="hljs-string">&quot;pt&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-built_in">print</span>(encoded_input)
{<span class="hljs-string">&#x27;input_ids&#x27;</span>: tensor([[  <span class="hljs-number">101</span>,   <span class="hljs-number">153</span>,  <span class="hljs-number">7719</span>, <span class="hljs-number">21490</span>,  <span class="hljs-number">1122</span>,  <span class="hljs-number">1114</span>,  <span class="hljs-number">9582</span>,  <span class="hljs-number">1623</span>,   <span class="hljs-number">102</span>],
                      [  <span class="hljs-number">101</span>,  <span class="hljs-number">5226</span>,  <span class="hljs-number">1122</span>,  <span class="hljs-number">9649</span>,  <span class="hljs-number">1199</span>,  <span class="hljs-number">2610</span>,  <span class="hljs-number">1236</span>,   <span class="hljs-number">102</span>,     <span class="hljs-number">0</span>]]), 
 <span class="hljs-string">&#x27;token_type_ids&#x27;</span>: tensor([[<span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>],
                           [<span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>]]), 
 <span class="hljs-string">&#x27;attention_mask&#x27;</span>: tensor([[<span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>],
                           [<span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">0</span>]])}`},group2:{id:"tf",code:`batch_sentences = [
    "But what about second breakfast?",
    "Don't think he knows about second breakfast, Pip.",
    "What about elevensies?",
]
encoded_input = tokenizer(batch, padding=True, truncation=True, return_tensors="tf")
print(encoded_input)`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>batch_sentences = [
<span class="hljs-meta">... </span>    <span class="hljs-string">&quot;But what about second breakfast?&quot;</span>,
<span class="hljs-meta">... </span>    <span class="hljs-string">&quot;Don&#x27;t think he knows about second breakfast, Pip.&quot;</span>,
<span class="hljs-meta">... </span>    <span class="hljs-string">&quot;What about elevensies?&quot;</span>,
<span class="hljs-meta">... </span>]
<span class="hljs-meta">&gt;&gt;&gt; </span>encoded_input = tokenizer(batch, padding=<span class="hljs-literal">True</span>, truncation=<span class="hljs-literal">True</span>, return_tensors=<span class="hljs-string">&quot;tf&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-built_in">print</span>(encoded_input)
{<span class="hljs-string">&#x27;input_ids&#x27;</span>: &lt;tf.Tensor: shape=(<span class="hljs-number">2</span>, <span class="hljs-number">9</span>), dtype=int32, numpy=
array([[  <span class="hljs-number">101</span>,   <span class="hljs-number">153</span>,  <span class="hljs-number">7719</span>, <span class="hljs-number">21490</span>,  <span class="hljs-number">1122</span>,  <span class="hljs-number">1114</span>,  <span class="hljs-number">9582</span>,  <span class="hljs-number">1623</span>,   <span class="hljs-number">102</span>],
       [  <span class="hljs-number">101</span>,  <span class="hljs-number">5226</span>,  <span class="hljs-number">1122</span>,  <span class="hljs-number">9649</span>,  <span class="hljs-number">1199</span>,  <span class="hljs-number">2610</span>,  <span class="hljs-number">1236</span>,   <span class="hljs-number">102</span>,     <span class="hljs-number">0</span>]],
      dtype=int32)&gt;, 
 <span class="hljs-string">&#x27;token_type_ids&#x27;</span>: &lt;tf.Tensor: shape=(<span class="hljs-number">2</span>, <span class="hljs-number">9</span>), dtype=int32, numpy=
array([[<span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>],
       [<span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>]], dtype=int32)&gt;, 
 <span class="hljs-string">&#x27;attention_mask&#x27;</span>: &lt;tf.Tensor: shape=(<span class="hljs-number">2</span>, <span class="hljs-number">9</span>), dtype=int32, numpy=
array([[<span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>],
       [<span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">0</span>]], dtype=int32)&gt;}`}}}),ia=new S({}),ca=new D({props:{code:"pip install datasets",highlighted:"pip install datasets"}}),ha=new D({props:{code:`from datasets import load_dataset, Audio

dataset = load_dataset("PolyAI/minds14", name="en-US", split="train")`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> datasets <span class="hljs-keyword">import</span> load_dataset, Audio

<span class="hljs-meta">&gt;&gt;&gt; </span>dataset = load_dataset(<span class="hljs-string">&quot;PolyAI/minds14&quot;</span>, name=<span class="hljs-string">&quot;en-US&quot;</span>, split=<span class="hljs-string">&quot;train&quot;</span>)`}}),da=new D({props:{code:'dataset[0]["audio"]',highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>dataset[<span class="hljs-number">0</span>][<span class="hljs-string">&quot;audio&quot;</span>]
{<span class="hljs-string">&#x27;array&#x27;</span>: array([ <span class="hljs-number">0.</span>        ,  <span class="hljs-number">0.00024414</span>, -<span class="hljs-number">0.00024414</span>, ..., -<span class="hljs-number">0.00024414</span>,
         <span class="hljs-number">0.</span>        ,  <span class="hljs-number">0.</span>        ], dtype=float32),
 <span class="hljs-string">&#x27;path&#x27;</span>: <span class="hljs-string">&#x27;/root/.cache/huggingface/datasets/downloads/extracted/f14948e0e84be638dd7943ac36518a4cf3324e8b7aa331c5ab11541518e9368c/en-US~JOINT_ACCOUNT/602ba55abb1e6d0fbce92065.wav&#x27;</span>,
 <span class="hljs-string">&#x27;sampling_rate&#x27;</span>: <span class="hljs-number">8000</span>}`}}),fa=new S({}),ga=new D({props:{code:`dataset = load_dataset("PolyAI/minds14", name="en-US", split="train")
dataset[0]["audio"]`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>dataset = load_dataset(<span class="hljs-string">&quot;PolyAI/minds14&quot;</span>, name=<span class="hljs-string">&quot;en-US&quot;</span>, split=<span class="hljs-string">&quot;train&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>dataset[<span class="hljs-number">0</span>][<span class="hljs-string">&quot;audio&quot;</span>]
{<span class="hljs-string">&#x27;array&#x27;</span>: array([ <span class="hljs-number">0.</span>        ,  <span class="hljs-number">0.00024414</span>, -<span class="hljs-number">0.00024414</span>, ..., -<span class="hljs-number">0.00024414</span>,
         <span class="hljs-number">0.</span>        ,  <span class="hljs-number">0.</span>        ], dtype=float32),
 <span class="hljs-string">&#x27;path&#x27;</span>: <span class="hljs-string">&#x27;/root/.cache/huggingface/datasets/downloads/extracted/f14948e0e84be638dd7943ac36518a4cf3324e8b7aa331c5ab11541518e9368c/en-US~JOINT_ACCOUNT/602ba55abb1e6d0fbce92065.wav&#x27;</span>,
 <span class="hljs-string">&#x27;sampling_rate&#x27;</span>: <span class="hljs-number">8000</span>}`}}),$a=new D({props:{code:'dataset = dataset.cast_column("audio", Audio(sampling_rate=16_000))',highlighted:'<span class="hljs-meta">&gt;&gt;&gt; </span>dataset = dataset.cast_column(<span class="hljs-string">&quot;audio&quot;</span>, Audio(sampling_rate=<span class="hljs-number">16_000</span>))'}}),Ea=new D({props:{code:'dataset[0]["audio"]',highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>dataset[<span class="hljs-number">0</span>][<span class="hljs-string">&quot;audio&quot;</span>]
{<span class="hljs-string">&#x27;array&#x27;</span>: array([ <span class="hljs-number">2.3443763e-05</span>,  <span class="hljs-number">2.1729663e-04</span>,  <span class="hljs-number">2.2145823e-04</span>, ...,
         <span class="hljs-number">3.8356509e-05</span>, -<span class="hljs-number">7.3497440e-06</span>, -<span class="hljs-number">2.1754686e-05</span>], dtype=float32),
 <span class="hljs-string">&#x27;path&#x27;</span>: <span class="hljs-string">&#x27;/root/.cache/huggingface/datasets/downloads/extracted/f14948e0e84be638dd7943ac36518a4cf3324e8b7aa331c5ab11541518e9368c/en-US~JOINT_ACCOUNT/602ba55abb1e6d0fbce92065.wav&#x27;</span>,
 <span class="hljs-string">&#x27;sampling_rate&#x27;</span>: <span class="hljs-number">16000</span>}`}}),za=new S({}),wa=new D({props:{code:`from transformers import AutoFeatureExtractor

feature_extractor = AutoFeatureExtractor.from_pretrained("facebook/wav2vec2-base")`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoFeatureExtractor

<span class="hljs-meta">&gt;&gt;&gt; </span>feature_extractor = AutoFeatureExtractor.from_pretrained(<span class="hljs-string">&quot;facebook/wav2vec2-base&quot;</span>)`}}),qa=new D({props:{code:`audio_input = [dataset[0]["audio"]["array"]]
feature_extractor(audio_input, sampling_rate=16000)`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>audio_input = [dataset[<span class="hljs-number">0</span>][<span class="hljs-string">&quot;audio&quot;</span>][<span class="hljs-string">&quot;array&quot;</span>]]
<span class="hljs-meta">&gt;&gt;&gt; </span>feature_extractor(audio_input, sampling_rate=<span class="hljs-number">16000</span>)
{<span class="hljs-string">&#x27;input_values&#x27;</span>: [array([ <span class="hljs-number">3.8106556e-04</span>,  <span class="hljs-number">2.7506407e-03</span>,  <span class="hljs-number">2.8015103e-03</span>, ...,
        <span class="hljs-number">5.6335266e-04</span>,  <span class="hljs-number">4.6588284e-06</span>, -<span class="hljs-number">1.7142107e-04</span>], dtype=float32)]}`}}),ya=new S({}),xa=new D({props:{code:`dataset[0]["audio"]["array"].shape

dataset[1]["audio"]["array"].shape`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>dataset[<span class="hljs-number">0</span>][<span class="hljs-string">&quot;audio&quot;</span>][<span class="hljs-string">&quot;array&quot;</span>].shape
(<span class="hljs-number">173398</span>,)

<span class="hljs-meta">&gt;&gt;&gt; </span>dataset[<span class="hljs-number">1</span>][<span class="hljs-string">&quot;audio&quot;</span>][<span class="hljs-string">&quot;array&quot;</span>].shape
(<span class="hljs-number">106496</span>,)`}}),Pa=new D({props:{code:`def preprocess_function(examples):
    audio_arrays = [x["array"] for x in examples["audio"]]
    inputs = feature_extractor(
        audio_arrays,
        sampling_rate=16000,
        padding=True,
        max_length=100000,
        truncation=True,
    )
    return inputs`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">def</span> <span class="hljs-title function_">preprocess_function</span>(<span class="hljs-params">examples</span>):
<span class="hljs-meta">... </span>    audio_arrays = [x[<span class="hljs-string">&quot;array&quot;</span>] <span class="hljs-keyword">for</span> x <span class="hljs-keyword">in</span> examples[<span class="hljs-string">&quot;audio&quot;</span>]]
<span class="hljs-meta">... </span>    inputs = feature_extractor(
<span class="hljs-meta">... </span>        audio_arrays,
<span class="hljs-meta">... </span>        sampling_rate=<span class="hljs-number">16000</span>,
<span class="hljs-meta">... </span>        padding=<span class="hljs-literal">True</span>,
<span class="hljs-meta">... </span>        max_length=<span class="hljs-number">100000</span>,
<span class="hljs-meta">... </span>        truncation=<span class="hljs-literal">True</span>,
<span class="hljs-meta">... </span>    )
<span class="hljs-meta">... </span>    <span class="hljs-keyword">return</span> inputs`}}),Aa=new D({props:{code:"processed_dataset = preprocess_function(dataset[:5])",highlighted:'<span class="hljs-meta">&gt;&gt;&gt; </span>processed_dataset = preprocess_function(dataset[:<span class="hljs-number">5</span>])'}}),Ca=new D({props:{code:`processed_dataset["input_values"][0].shape

processed_dataset["input_values"][1].shape`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>processed_dataset[<span class="hljs-string">&quot;input_values&quot;</span>][<span class="hljs-number">0</span>].shape
(<span class="hljs-number">100000</span>,)

<span class="hljs-meta">&gt;&gt;&gt; </span>processed_dataset[<span class="hljs-string">&quot;input_values&quot;</span>][<span class="hljs-number">1</span>].shape
(<span class="hljs-number">100000</span>,)`}}),Da=new S({}),Sa=new D({props:{code:`from datasets import load_dataset

dataset = load_dataset("food101", split="train[:100]")`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> datasets <span class="hljs-keyword">import</span> load_dataset

<span class="hljs-meta">&gt;&gt;&gt; </span>dataset = load_dataset(<span class="hljs-string">&quot;food101&quot;</span>, split=<span class="hljs-string">&quot;train[:100]&quot;</span>)`}}),Ta=new D({props:{code:'dataset[0]["image"]',highlighted:'<span class="hljs-meta">&gt;&gt;&gt; </span>dataset[<span class="hljs-number">0</span>][<span class="hljs-string">&quot;image&quot;</span>]'}}),Oa=new S({}),La=new D({props:{code:`from transformers import AutoFeatureExtractor

feature_extractor = AutoFeatureExtractor.from_pretrained("google/vit-base-patch16-224")`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoFeatureExtractor

<span class="hljs-meta">&gt;&gt;&gt; </span>feature_extractor = AutoFeatureExtractor.from_pretrained(<span class="hljs-string">&quot;google/vit-base-patch16-224&quot;</span>)`}}),Ua=new S({}),Ma=new D({props:{code:`from torchvision.transforms import Compose, Normalize, RandomResizedCrop, ColorJitter, ToTensor

normalize = Normalize(mean=feature_extractor.image_mean, std=feature_extractor.image_std)
_transforms = Compose(
    [RandomResizedCrop(feature_extractor.size), ColorJitter(brightness=0.5, hue=0.5), ToTensor(), normalize]
)`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> torchvision.transforms <span class="hljs-keyword">import</span> Compose, Normalize, RandomResizedCrop, ColorJitter, ToTensor

<span class="hljs-meta">&gt;&gt;&gt; </span>normalize = Normalize(mean=feature_extractor.image_mean, std=feature_extractor.image_std)
<span class="hljs-meta">&gt;&gt;&gt; </span>_transforms = Compose(
<span class="hljs-meta">... </span>    [RandomResizedCrop(feature_extractor.size), ColorJitter(brightness=<span class="hljs-number">0.5</span>, hue=<span class="hljs-number">0.5</span>), ToTensor(), normalize]
<span class="hljs-meta">... </span>)`}}),Va=new D({props:{code:`def transforms(examples):
    examples["pixel_values"] = [_transforms(image.convert("RGB")) for image in examples["image"]]
    return examples`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">def</span> <span class="hljs-title function_">transforms</span>(<span class="hljs-params">examples</span>):
<span class="hljs-meta">... </span>    examples[<span class="hljs-string">&quot;pixel_values&quot;</span>] = [_transforms(image.convert(<span class="hljs-string">&quot;RGB&quot;</span>)) <span class="hljs-keyword">for</span> image <span class="hljs-keyword">in</span> examples[<span class="hljs-string">&quot;image&quot;</span>]]
<span class="hljs-meta">... </span>    <span class="hljs-keyword">return</span> examples`}}),Ya=new D({props:{code:"dataset.set_transform(transforms)",highlighted:'<span class="hljs-meta">&gt;&gt;&gt; </span>dataset.set_transform(transforms)'}}),Xa=new D({props:{code:'dataset[0]["image"]',highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>dataset[<span class="hljs-number">0</span>][<span class="hljs-string">&quot;image&quot;</span>]
{<span class="hljs-string">&#x27;image&#x27;</span>: &lt;PIL.JpegImagePlugin.JpegImageFile image mode=RGB size=384x512 at <span class="hljs-number">0x7F1A7B0630D0</span>&gt;,
 <span class="hljs-string">&#x27;label&#x27;</span>: <span class="hljs-number">6</span>,
 <span class="hljs-string">&#x27;pixel_values&#x27;</span>: tensor([[[ <span class="hljs-number">0.0353</span>,  <span class="hljs-number">0.0745</span>,  <span class="hljs-number">0.1216</span>,  ..., -<span class="hljs-number">0.9922</span>, -<span class="hljs-number">0.9922</span>, -<span class="hljs-number">0.9922</span>],
          [-<span class="hljs-number">0.0196</span>,  <span class="hljs-number">0.0667</span>,  <span class="hljs-number">0.1294</span>,  ..., -<span class="hljs-number">0.9765</span>, -<span class="hljs-number">0.9843</span>, -<span class="hljs-number">0.9922</span>],
          [ <span class="hljs-number">0.0196</span>,  <span class="hljs-number">0.0824</span>,  <span class="hljs-number">0.1137</span>,  ..., -<span class="hljs-number">0.9765</span>, -<span class="hljs-number">0.9686</span>, -<span class="hljs-number">0.8667</span>],
          ...,
          [ <span class="hljs-number">0.0275</span>,  <span class="hljs-number">0.0745</span>,  <span class="hljs-number">0.0510</span>,  ..., -<span class="hljs-number">0.1137</span>, -<span class="hljs-number">0.1216</span>, -<span class="hljs-number">0.0824</span>],
          [ <span class="hljs-number">0.0667</span>,  <span class="hljs-number">0.0824</span>,  <span class="hljs-number">0.0667</span>,  ..., -<span class="hljs-number">0.0588</span>, -<span class="hljs-number">0.0745</span>, -<span class="hljs-number">0.0980</span>],
          [ <span class="hljs-number">0.0353</span>,  <span class="hljs-number">0.0353</span>,  <span class="hljs-number">0.0431</span>,  ..., -<span class="hljs-number">0.0039</span>, -<span class="hljs-number">0.0039</span>, -<span class="hljs-number">0.0588</span>]],
 
         [[ <span class="hljs-number">0.2078</span>,  <span class="hljs-number">0.2471</span>,  <span class="hljs-number">0.2863</span>,  ..., -<span class="hljs-number">0.9451</span>, -<span class="hljs-number">0.9373</span>, -<span class="hljs-number">0.9451</span>],
          [ <span class="hljs-number">0.1608</span>,  <span class="hljs-number">0.2471</span>,  <span class="hljs-number">0.3098</span>,  ..., -<span class="hljs-number">0.9373</span>, -<span class="hljs-number">0.9451</span>, -<span class="hljs-number">0.9373</span>],
          [ <span class="hljs-number">0.2078</span>,  <span class="hljs-number">0.2706</span>,  <span class="hljs-number">0.3020</span>,  ..., -<span class="hljs-number">0.9608</span>, -<span class="hljs-number">0.9373</span>, -<span class="hljs-number">0.8275</span>],
          ...,
          [-<span class="hljs-number">0.0353</span>,  <span class="hljs-number">0.0118</span>, -<span class="hljs-number">0.0039</span>,  ..., -<span class="hljs-number">0.2392</span>, -<span class="hljs-number">0.2471</span>, -<span class="hljs-number">0.2078</span>],
          [ <span class="hljs-number">0.0196</span>,  <span class="hljs-number">0.0353</span>,  <span class="hljs-number">0.0196</span>,  ..., -<span class="hljs-number">0.1843</span>, -<span class="hljs-number">0.2000</span>, -<span class="hljs-number">0.2235</span>],
          [-<span class="hljs-number">0.0118</span>, -<span class="hljs-number">0.0039</span>, -<span class="hljs-number">0.0039</span>,  ..., -<span class="hljs-number">0.0980</span>, -<span class="hljs-number">0.0980</span>, -<span class="hljs-number">0.1529</span>]],
 
         [[ <span class="hljs-number">0.3961</span>,  <span class="hljs-number">0.4431</span>,  <span class="hljs-number">0.4980</span>,  ..., -<span class="hljs-number">0.9216</span>, -<span class="hljs-number">0.9137</span>, -<span class="hljs-number">0.9216</span>],
          [ <span class="hljs-number">0.3569</span>,  <span class="hljs-number">0.4510</span>,  <span class="hljs-number">0.5216</span>,  ..., -<span class="hljs-number">0.9059</span>, -<span class="hljs-number">0.9137</span>, -<span class="hljs-number">0.9137</span>],
          [ <span class="hljs-number">0.4118</span>,  <span class="hljs-number">0.4745</span>,  <span class="hljs-number">0.5216</span>,  ..., -<span class="hljs-number">0.9137</span>, -<span class="hljs-number">0.8902</span>, -<span class="hljs-number">0.7804</span>],
          ...,
          [-<span class="hljs-number">0.2314</span>, -<span class="hljs-number">0.1922</span>, -<span class="hljs-number">0.2078</span>,  ..., -<span class="hljs-number">0.4196</span>, -<span class="hljs-number">0.4275</span>, -<span class="hljs-number">0.3882</span>],
          [-<span class="hljs-number">0.1843</span>, -<span class="hljs-number">0.1686</span>, -<span class="hljs-number">0.2000</span>,  ..., -<span class="hljs-number">0.3647</span>, -<span class="hljs-number">0.3804</span>, -<span class="hljs-number">0.4039</span>],
          [-<span class="hljs-number">0.1922</span>, -<span class="hljs-number">0.1922</span>, -<span class="hljs-number">0.1922</span>,  ..., -<span class="hljs-number">0.2941</span>, -<span class="hljs-number">0.2863</span>, -<span class="hljs-number">0.3412</span>]]])}`}}),se=new D({props:{code:`import numpy as np
import matplotlib.pyplot as plt

img = dataset[0]["pixel_values"]
plt.imshow(img.permute(1, 2, 0))`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">import</span> numpy <span class="hljs-keyword">as</span> np
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">import</span> matplotlib.pyplot <span class="hljs-keyword">as</span> plt

<span class="hljs-meta">&gt;&gt;&gt; </span>img = dataset[<span class="hljs-number">0</span>][<span class="hljs-string">&quot;pixel_values&quot;</span>]
<span class="hljs-meta">&gt;&gt;&gt; </span>plt.imshow(img.permute(<span class="hljs-number">1</span>, <span class="hljs-number">2</span>, <span class="hljs-number">0</span>))`}}),ae=new S({}),ne=new D({props:{code:`from datasets import load_dataset

lj_speech = load_dataset("lj_speech", split="train")`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> datasets <span class="hljs-keyword">import</span> load_dataset

<span class="hljs-meta">&gt;&gt;&gt; </span>lj_speech = load_dataset(<span class="hljs-string">&quot;lj_speech&quot;</span>, split=<span class="hljs-string">&quot;train&quot;</span>)`}}),le=new D({props:{code:'lj_speech = lj_speech.map(remove_columns=["file", "id", "normalized_text"])',highlighted:'<span class="hljs-meta">&gt;&gt;&gt; </span>lj_speech = lj_speech.<span class="hljs-built_in">map</span>(remove_columns=[<span class="hljs-string">&quot;file&quot;</span>, <span class="hljs-string">&quot;id&quot;</span>, <span class="hljs-string">&quot;normalized_text&quot;</span>])'}}),te=new D({props:{code:`lj_speech[0]["audio"]

lj_speech[0]["text"]`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>lj_speech[<span class="hljs-number">0</span>][<span class="hljs-string">&quot;audio&quot;</span>]
{<span class="hljs-string">&#x27;array&#x27;</span>: array([-<span class="hljs-number">7.3242188e-04</span>, -<span class="hljs-number">7.6293945e-04</span>, -<span class="hljs-number">6.4086914e-04</span>, ...,
         <span class="hljs-number">7.3242188e-04</span>,  <span class="hljs-number">2.1362305e-04</span>,  <span class="hljs-number">6.1035156e-05</span>], dtype=float32),
 <span class="hljs-string">&#x27;path&#x27;</span>: <span class="hljs-string">&#x27;/root/.cache/huggingface/datasets/downloads/extracted/917ece08c95cf0c4115e45294e3cd0dee724a1165b7fc11798369308a465bd26/LJSpeech-1.1/wavs/LJ001-0001.wav&#x27;</span>,
 <span class="hljs-string">&#x27;sampling_rate&#x27;</span>: <span class="hljs-number">22050</span>}

<span class="hljs-meta">&gt;&gt;&gt; </span>lj_speech[<span class="hljs-number">0</span>][<span class="hljs-string">&quot;text&quot;</span>]
<span class="hljs-string">&#x27;Printing, in the only sense with which we are at present concerned, differs from most if not from all the arts and crafts represented in the Exhibition&#x27;</span>`}}),re=new D({props:{code:'lj_speech = lj_speech.cast_column("audio", Audio(sampling_rate=16_000))',highlighted:'<span class="hljs-meta">&gt;&gt;&gt; </span>lj_speech = lj_speech.cast_column(<span class="hljs-string">&quot;audio&quot;</span>, Audio(sampling_rate=<span class="hljs-number">16_000</span>))'}}),pe=new S({}),oe=new D({props:{code:`from transformers import AutoProcessor

processor = AutoProcessor.from_pretrained("facebook/wav2vec2-base-960h")`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoProcessor

<span class="hljs-meta">&gt;&gt;&gt; </span>processor = AutoProcessor.from_pretrained(<span class="hljs-string">&quot;facebook/wav2vec2-base-960h&quot;</span>)`}}),ie=new D({props:{code:`def prepare_dataset(example):
    audio = example["audio"]

    example.update(processor(audio=audio["array"], text=example["text"], sampling_rate=16000))

    return example`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">def</span> <span class="hljs-title function_">prepare_dataset</span>(<span class="hljs-params">example</span>):
<span class="hljs-meta">... </span>    audio = example[<span class="hljs-string">&quot;audio&quot;</span>]

<span class="hljs-meta">... </span>    example.update(processor(audio=audio[<span class="hljs-string">&quot;array&quot;</span>], text=example[<span class="hljs-string">&quot;text&quot;</span>], sampling_rate=<span class="hljs-number">16000</span>))

<span class="hljs-meta">... </span>    <span class="hljs-keyword">return</span> example`}}),me=new D({props:{code:"prepare_dataset(lj_speech[0])",highlighted:'<span class="hljs-meta">&gt;&gt;&gt; </span>prepare_dataset(lj_speech[<span class="hljs-number">0</span>])'}}),{c(){h=l("meta"),f=c(),b=l("h1"),v=l("a"),j=l("span"),z(q.$$.fragment),$=c(),k=l("span"),d=p("Preprocess"),P=c(),z(A.$$.fragment),C=c(),bs=l("p"),N=p("Prima di poter usare i dati in un modello, bisogna processarli in un formato accettabile per quest\u2019ultimo. Un modello non comprende il testo grezzo, le immagini o l\u2019audio. Bisogna convertire questi input in numeri e assemblarli all\u2019interno di tensori. In questa esercitazione, tu potrai:"),I=c(),T=l("ul"),dn=l("li"),wp=p("Preprocessare dati testuali con un tokenizer."),qp=c(),fn=l("li"),yp=p("Preprocessare immagini o dati audio con un estrattore di caratteristiche."),xp=c(),bn=l("li"),Pp=p("Preprocessare dati per attivit\xE0 multimodali mediante un processore."),Ml=c(),ss=l("h2"),js=l("a"),jn=l("span"),z(Ys.$$.fragment),Ap=c(),gn=l("span"),Cp=p("NLP"),Ql=c(),z(Ks.$$.fragment),Vl=c(),R=l("p"),Dp=p("Lo strumento principale per processare dati testuali \xE8 un "),de=l("a"),Ip=p("tokenizer"),Sp=p(". Un tokenizer inizia separando il testo in "),_n=l("em"),Np=p("tokens"),Tp=p(" secondo una serie di regole. I tokens sono convertiti in numeri, questi vengono utilizzati per costruire i tensori di input del modello. Anche altri input addizionali se richiesti dal modello vengono aggiunti dal tokenizer."),Jl=c(),z(gs.$$.fragment),Wl=c(),F=l("p"),Op=p("Iniziamo subito caricando un tokenizer preaddestrato con la classe "),vn=l("code"),Lp=p("AutoTokenizer"),Up=p(". Questo scarica il "),$n=l("em"),Rp=p("vocabolario"),Fp=p(" usato quando il modello \xE8 stato preaddestrato."),Gl=c(),as=l("h3"),_s=l("a"),kn=l("span"),z(Zs.$$.fragment),Hp=c(),En=l("span"),Bp=p("Tokenize"),Yl=c(),vs=l("p"),Mp=p("Carica un tokenizer preaddestrato con "),zn=l("code"),Qp=p("AutoTokenizer.from_pretrained()"),Vp=p(":"),Kl=c(),z(Xs.$$.fragment),Zl=c(),fe=l("p"),Jp=p("Poi inserisci le tue frasi nel tokenizer:"),Xl=c(),z(sa.$$.fragment),st=c(),be=l("p"),Wp=p("Il tokenizer restituisce un dizionario contenente tre oggetti importanti:"),at=c(),H=l("ul"),je=l("li"),ge=l("a"),Gp=p("input_ids"),Yp=p(" sono gli indici che corrispondono ad ogni token nella frase."),Kp=c(),_e=l("li"),ve=l("a"),Zp=p("attention_mask"),Xp=p(" indicata se un token deve essere elaborato o no."),so=c(),$e=l("li"),ke=l("a"),ao=p("token_type_ids"),eo=p(" identifica a quale sequenza appartiene un token se \xE8 presente pi\xF9 di una sequenza."),et=c(),$s=l("p"),no=p("Si possono decodificare gli "),wn=l("code"),lo=p("input_ids"),to=p(" per farsi restituire l\u2019input originale:"),nt=c(),z(aa.$$.fragment),lt=c(),B=l("p"),ro=p("Come si pu\xF2 vedere, il tokenizer aggiunge due token speciali - "),qn=l("code"),po=p("CLS"),oo=p(" e "),yn=l("code"),io=p("SEP"),co=p(" (classificatore e separatore) - alla frase. Non tutti i modelli hanno bisogno dei token speciali, ma se servono, il tokenizer li aggiunger\xE0 automaticamente."),tt=c(),Ee=l("p"),uo=p("Se ci sono pi\xF9 frasi che vuoi processare, passale come una lista al tokenizer:"),rt=c(),z(ea.$$.fragment),pt=c(),es=l("h3"),ks=l("a"),xn=l("span"),z(na.$$.fragment),mo=c(),Pn=l("span"),ho=p("Pad"),ot=c(),Es=l("p"),fo=p("Questo \xE8 un argomento importante. Quando processi un insieme di frasi potrebbero non avere tutte la stessa lunghezza. Questo \xE8 un problema perch\xE8 i tensori, in input del modello, devono avere dimensioni uniformi. Il padding \xE8 una strategia per assicurarsi che i tensori siano rettangolari aggiungendo uno speciale "),An=l("em"),bo=p("padding token"),jo=p(" alle frasi pi\xF9 corte."),it=c(),M=l("p"),go=p("Imposta il parametro "),Cn=l("code"),_o=p("padding"),vo=p(" a "),Dn=l("code"),$o=p("True"),ko=p(" per imbottire le frasi pi\xF9 corte nel gruppo in modo che combacino con la massima lunghezza presente:"),ct=c(),z(la.$$.fragment),ut=c(),zs=l("p"),Eo=p("Nota che il tokenizer aggiunge alle sequenze degli "),In=l("code"),zo=p("0"),wo=p(" perch\xE8 sono troppo corte!"),mt=c(),ns=l("h3"),ws=l("a"),Sn=l("span"),z(ta.$$.fragment),qo=c(),Nn=l("span"),yo=p("Truncation"),ht=c(),ze=l("p"),xo=p("L\u2019altra faccia della medaglia \xE8 che avolte le sequenze possono essere troppo lunghe per essere gestite dal modello. In questo caso, avrai bisogno di troncare la sequenza per avere una lunghezza minore."),dt=c(),Q=l("p"),Po=p("Imposta il parametro "),Tn=l("code"),Ao=p("truncation"),Co=p(" a "),On=l("code"),Do=p("True"),Io=p(" per troncare una sequenza alla massima lunghezza accettata dal modello:"),ft=c(),z(ra.$$.fragment),bt=c(),ls=l("h3"),qs=l("a"),Ln=l("span"),z(pa.$$.fragment),So=c(),Un=l("span"),No=p("Costruire i tensori"),jt=c(),we=l("p"),To=p("Infine, vuoi che il tokenizer restituisca i tensori prodotti dal modello."),gt=c(),O=l("p"),Oo=p("Imposta il parametro "),Rn=l("code"),Lo=p("return_tensors"),Uo=p(" su "),Fn=l("code"),Ro=p("pt"),Fo=p(" per PyTorch, o "),Hn=l("code"),Ho=p("tf"),Bo=p(" per TensorFlow:"),_t=c(),z(oa.$$.fragment),vt=c(),ts=l("h2"),ys=l("a"),Bn=l("span"),z(ia.$$.fragment),Mo=c(),Mn=l("span"),Qo=p("Audio"),$t=c(),xs=l("p"),Vo=p("Gli input audio sono processati in modo differente rispetto al testo, ma l\u2019obiettivo rimane lo stesso: creare sequenze numeriche che il modello pu\xF2 capire. Un "),qe=l("a"),Jo=p("estrattore di caratteristiche"),Wo=p(" \xE8 progettato con lo scopo preciso di estrarre caratteristiche da immagini o dati audio grezzi e convertirli in tensori. Prima di iniziare, installa \u{1F917} Datasets per caricare un dataset audio e sperimentare:"),kt=c(),z(ca.$$.fragment),Et=c(),V=l("p"),Go=p("Carica il dataset "),ua=l("a"),Yo=p("MInDS-14"),Ko=p(" (vedi il \u{1F917} "),ma=l("a"),Zo=p("Datasets tutorial"),Xo=p(" per avere maggiori dettagli su come caricare un dataset):"),zt=c(),z(ha.$$.fragment),wt=c(),J=l("p"),si=p("Accedi al primo elemento della colonna "),Qn=l("code"),ai=p("audio"),ei=p(" per dare uno sguardo all\u2019input. Richiamando la colonna "),Vn=l("code"),ni=p("audio"),li=p(" sar\xE0 caricato automaticamente e ricampionato il file audio:"),qt=c(),z(da.$$.fragment),yt=c(),ye=l("p"),ti=p("Questo restituisce tre oggetti:"),xt=c(),W=l("ul"),xe=l("li"),Jn=l("code"),ri=p("array"),pi=p(" \xE8 il segnale vocale caricato - e potenzialmente ricampionato - come vettore 1D."),oi=c(),Pe=l("li"),Wn=l("code"),ii=p("path"),ci=p(" il percorso del file audio."),ui=c(),Ae=l("li"),Gn=l("code"),mi=p("sampling_rate"),hi=p(" si riferisce al numero di campioni del segnale vocale misurati al secondo."),Pt=c(),rs=l("h3"),Ps=l("a"),Yn=l("span"),z(fa.$$.fragment),di=c(),Kn=l("span"),fi=p("Ricampionamento"),At=c(),As=l("p"),bi=p("Per questo tutorial, puoi usare il modello "),ba=l("a"),ji=p("Wav2Vec2"),gi=p(". Come puoi vedere dalla model card, il modello Wav2Vec2 \xE8 preaddestrato su un campionamento vocale a 16kHz.\xC8 importante che la frequenza di campionamento dei tuoi dati audio combaci con la frequenza di campionamento del dataset usato per preaddestrare il modello. Se la frequenza di campionamento dei tuoi dati non \xE8 uguale dovrai ricampionare i tuoi dati audio."),Ct=c(),Cs=l("p"),_i=p("Per esempio, il dataset "),ja=l("a"),vi=p("MInDS-14"),$i=p(" ha una frequenza di campionamento di 8000kHz. Utilizzando il modello Wav2Vec2 su questo dataset, alzala a 16kHz:"),Dt=c(),z(ga.$$.fragment),It=c(),Ce=l("ol"),_a=l("li"),ki=p("Usa il metodo di \u{1F917} Datasets\u2019 "),va=l("a"),Zn=l("code"),Ei=p("cast_column"),zi=p(" per alzare la frequenza di campionamento a 16kHz:"),St=c(),z($a.$$.fragment),Nt=c(),ka=l("ol"),Xn=l("li"),wi=p("Carica il file audio:"),Tt=c(),z(Ea.$$.fragment),Ot=c(),Ds=l("p"),qi=p("Come puoi notare, la "),sl=l("code"),yi=p("sampling_rate"),xi=p(" adesso \xE8 16kHz!"),Lt=c(),ps=l("h3"),Is=l("a"),al=l("span"),z(za.$$.fragment),Pi=c(),el=l("span"),Ai=p("Feature extractor"),Ut=c(),L=l("p"),Ci=p("Il prossimo passo \xE8 caricare un estrattore di caratteristiche per normalizzare e fare padding sull\u2019input. Quando applichiamo il padding sui dati testuali, uno "),nl=l("code"),Di=p("0"),Ii=p(" \xE8 aggiunto alle sequenze pi\xF9 brevi. La stessa idea si applica ai dati audio, l\u2019estrattore di caratteristiche per gli audio aggiunger\xE0 uno "),ll=l("code"),Si=p("0"),Ni=p(" - interpretato come silenzio - agli "),tl=l("code"),Ti=p("array"),Oi=p("."),Rt=c(),Ss=l("p"),Li=p("Carica l\u2019estrattore delle caratteristiche con "),rl=l("code"),Ui=p("AutoFeatureExtractor.from_pretrained()"),Ri=p(":"),Ft=c(),z(wa.$$.fragment),Ht=c(),G=l("p"),Fi=p("Inserisci l\u2019 "),pl=l("code"),Hi=p("array"),Bi=p(" audio nell\u2019estrattore delle caratteristiche. Noi raccomandiamo sempre di aggiungere il parametro "),ol=l("code"),Mi=p("sampling_rate"),Qi=p(" nell\u2019estrattore delle caratteristiche per correggere meglio qualche errore, dovuto ai silenzi, che potrebbe verificarsi."),Bt=c(),z(qa.$$.fragment),Mt=c(),os=l("h3"),Ns=l("a"),il=l("span"),z(ya.$$.fragment),Vi=c(),cl=l("span"),Ji=p("Pad e truncate"),Qt=c(),De=l("p"),Wi=p("Come per il tokenizer, puoi applicare le operazioni padding o truncation per manipolare sequenze di variabili a lotti. Dai uno sguaro alla lunghezza delle sequenze di questi due campioni audio:"),Vt=c(),z(xa.$$.fragment),Jt=c(),Ie=l("p"),Gi=p("Come puoi vedere, il primo campione ha una sequenza pi\xF9 lunga del secondo. Crea una funzione che preprocesser\xE0 il dataset. Specifica una lunghezza massima del campione, e l\u2019estrattore di features si occuper\xE0 di riempire o troncare la sequenza per coincidervi:"),Wt=c(),z(Pa.$$.fragment),Gt=c(),Se=l("p"),Yi=p("Applica la funzione ai primi esempi nel dataset:"),Yt=c(),z(Aa.$$.fragment),Kt=c(),Ne=l("p"),Ki=p("Adesso guarda la lunghezza dei campioni elaborati:"),Zt=c(),z(Ca.$$.fragment),Xt=c(),Te=l("p"),Zi=p("La lunghezza dei campioni adesso coincide con la massima lunghezza impostata nelle funzione."),sr=c(),is=l("h2"),Ts=l("a"),ul=l("span"),z(Da.$$.fragment),Xi=c(),ml=l("span"),sc=p("Vision"),ar=c(),Oe=l("p"),ac=p("Un estrattore di caratteristiche si pu\xF2 usare anche per processare immagini e per compiti di visione. Ancora una volta, l\u2019obiettivo \xE8 convertire l\u2019immagine grezza in un lotto di tensori come input."),er=c(),Y=l("p"),ec=p("Carica il dataset "),Ia=l("a"),nc=p("food101"),lc=p(" per questa esercitazione. Usa il parametro "),hl=l("code"),tc=p("split"),rc=p(" di \u{1F917} Datasets  per caricare solo un piccolo campione dal dataset di addestramento poich\xE8 il set di dati \xE8 molto grande:"),nr=c(),z(Sa.$$.fragment),lr=c(),Os=l("p"),pc=p("Secondo passo, dai uno sguardo alle immagini usando la caratteristica "),Na=l("a"),dl=l("code"),oc=p("Image"),ic=p(" di \u{1F917} Datasets:"),tr=c(),z(Ta.$$.fragment),rr=c(),Le=l("p"),Ue=l("img"),pr=c(),cs=l("h3"),Ls=l("a"),fl=l("span"),z(Oa.$$.fragment),cc=c(),bl=l("span"),uc=p("Feature extractor"),or=c(),Us=l("p"),mc=p("Carica l\u2019estrattore di caratteristiche "),jl=l("code"),hc=p("AutoFeatureExtractor.from_pretrained()"),dc=p(":"),ir=c(),z(La.$$.fragment),cr=c(),us=l("h3"),Rs=l("a"),gl=l("span"),z(Ua.$$.fragment),fc=c(),_l=l("span"),bc=p("Data augmentation"),ur=c(),Fs=l("p"),jc=p("Per le attivit\xE0 di visione, \xE8 usuale aggiungere alcuni tipi di data augmentation alle immagini come parte del preprocessing. Puoi aggiungere augmentations con qualsiasi libreria che preferisci, ma in questa esercitazione, userai il modulo "),Ra=l("a"),vl=l("code"),gc=p("transforms"),_c=p(" di torchvision."),mr=c(),Re=l("ol"),U=l("li"),vc=p("Normalizza l\u2019immagine e usa "),Fa=l("a"),$l=l("code"),$c=p("Compose"),kc=p(" per concatenare alcune trasformazioni - "),Ha=l("a"),kl=l("code"),Ec=p("RandomResizedCrop"),zc=p(" e "),Ba=l("a"),El=l("code"),wc=p("ColorJitter"),qc=p(" - insieme:"),hr=c(),z(Ma.$$.fragment),dr=c(),Qa=l("ol"),ms=l("li"),yc=p("Il modello accetta "),Fe=l("a"),zl=l("code"),xc=p("pixel_values"),Pc=p(" come input. Questo valore \xE8 generato dall\u2019estrattore di caratteristiche. Crea una funzione che genera "),wl=l("code"),Ac=p("pixel_values"),Cc=p(" dai transforms:"),fr=c(),z(Va.$$.fragment),br=c(),Ja=l("ol"),Wa=l("li"),Dc=p("Poi utilizza \u{1F917} Datasets "),Ga=l("a"),ql=l("code"),Ic=p("set_transform"),Sc=p("per applicare al volo la trasformazione:"),jr=c(),z(Ya.$$.fragment),gr=c(),Ka=l("ol"),Za=l("li"),Nc=p("Adesso quando accedi all\u2019immagine, puoi notare che l\u2019estrattore di caratteristiche ha aggiunto "),yl=l("code"),Tc=p("pixel_values"),Oc=p(" allo schema di input:"),_r=c(),z(Xa.$$.fragment),vr=c(),He=l("p"),Lc=p("Di seguito come si vede l\u2019immagine dopo la fase di preprocessing. Come ci si aspetterebbe dalle trasformazioni applicate, l\u2019immagine \xE8 stata ritagliata in modo casuale e le propriet\xE0 del colore sono diverse."),$r=c(),z(se.$$.fragment),kr=c(),Be=l("p"),Me=l("img"),Er=c(),hs=l("h2"),Hs=l("a"),xl=l("span"),z(ae.$$.fragment),Uc=c(),Pl=l("span"),Rc=p("Multimodal"),zr=c(),Qe=l("p"),Fc=p("Per attivit\xE0 multimodali userai una combinazione di tutto quello che hai imparato poco fa e applicherai le tue competenze alla comprensione automatica del parlato (Automatic Speech Recognition -  ASR). Questo significa che avrai bisogno di:"),wr=c(),Bs=l("ul"),Al=l("li"),Hc=p("Un estrattore delle caratteristiche per processare i dati audio."),Bc=c(),Cl=l("li"),Mc=p("Il Tokenizer per processare i testi."),qr=c(),Ms=l("p"),Qc=p("Ritorna sul datasere "),ee=l("a"),Vc=p("LJ Speech"),Jc=p(":"),yr=c(),z(ne.$$.fragment),xr=c(),K=l("p"),Wc=p("Visto che sei interessato solo alle colonne "),Dl=l("code"),Gc=p("audio"),Yc=p(" e "),Il=l("code"),Kc=p("text"),Zc=p(", elimina tutte le altre:"),Pr=c(),z(le.$$.fragment),Ar=c(),Z=l("p"),Xc=p("Adesso guarda le colonne "),Sl=l("code"),su=p("audio"),au=p(" e "),Nl=l("code"),eu=p("text"),nu=p(":"),Cr=c(),z(te.$$.fragment),Dr=c(),Qs=l("p"),lu=p("Ricorda dalla sezione precedente sull\u2019elaborazione dei dati audio, tu dovresti sempre "),Ve=l("a"),tu=p("ricampionare"),ru=p(" la frequenza di campionamento dei tuoi dati audio per farla coincidere con quella del dataset usato dal modello preaddestrato:"),Ir=c(),z(re.$$.fragment),Sr=c(),ds=l("h3"),Vs=l("a"),Tl=l("span"),z(pe.$$.fragment),pu=c(),Ol=l("span"),ou=p("Processor"),Nr=c(),Je=l("p"),iu=p("Un processor combina un estrattore di caratteristiche e un tokenizer. Carica un processor con [`AutoProcessor.from_pretrained]:"),Tr=c(),z(oe.$$.fragment),Or=c(),We=l("ol"),fs=l("li"),cu=p("Crea una funzione che processi i dati audio in "),Ll=l("code"),uu=p("input_values"),mu=p(", e tokenizza il testo in "),Ul=l("code"),hu=p("labels"),du=p(". Questi sono i tuoi input per il modello:"),Lr=c(),z(ie.$$.fragment),Ur=c(),ce=l("ol"),ue=l("li"),fu=p("Applica la funzione "),Rl=l("code"),bu=p("prepare_dataset"),ju=p(" ad un campione:"),Rr=c(),z(me.$$.fragment),Fr=c(),X=l("p"),gu=p("Nota che il processor ha aggiunto "),Fl=l("code"),_u=p("input_values"),vu=p(" e "),Hl=l("code"),$u=p("labels"),ku=p(". La frequenza di campionamento \xE8 stata corretta riducendola a 16kHz."),Hr=c(),Ge=l("p"),Eu=p("Fantastico, ora dovresti essere in grado di preelaborare i dati per qualsiasi modalit\xE0 e persino di combinare modalit\xE0 diverse! Nella prossima esercitazione, impareremo a mettere a punto un modello sui dati appena pre-elaborati."),this.h()},l(s){const n=Nd('[data-svelte="svelte-1phssyn"]',document.head);h=t(n,"META",{name:!0,content:!0}),n.forEach(a),f=u(s),b=t(s,"H1",{class:!0});var he=r(b);v=t(he,"A",{id:!0,class:!0,href:!0});var Nu=r(v);j=t(Nu,"SPAN",{});var Tu=r(j);y(q.$$.fragment,Tu),Tu.forEach(a),Nu.forEach(a),$=u(he),k=t(he,"SPAN",{});var Ou=r(k);d=o(Ou,"Preprocess"),Ou.forEach(a),he.forEach(a),P=u(s),y(A.$$.fragment,s),C=u(s),bs=t(s,"P",{});var Lu=r(bs);N=o(Lu,"Prima di poter usare i dati in un modello, bisogna processarli in un formato accettabile per quest\u2019ultimo. Un modello non comprende il testo grezzo, le immagini o l\u2019audio. Bisogna convertire questi input in numeri e assemblarli all\u2019interno di tensori. In questa esercitazione, tu potrai:"),Lu.forEach(a),I=u(s),T=t(s,"UL",{});var Ye=r(T);dn=t(Ye,"LI",{});var Uu=r(dn);wp=o(Uu,"Preprocessare dati testuali con un tokenizer."),Uu.forEach(a),qp=u(Ye),fn=t(Ye,"LI",{});var Ru=r(fn);yp=o(Ru,"Preprocessare immagini o dati audio con un estrattore di caratteristiche."),Ru.forEach(a),xp=u(Ye),bn=t(Ye,"LI",{});var Fu=r(bn);Pp=o(Fu,"Preprocessare dati per attivit\xE0 multimodali mediante un processore."),Fu.forEach(a),Ye.forEach(a),Ml=u(s),ss=t(s,"H2",{class:!0});var Mr=r(ss);js=t(Mr,"A",{id:!0,class:!0,href:!0});var Hu=r(js);jn=t(Hu,"SPAN",{});var Bu=r(jn);y(Ys.$$.fragment,Bu),Bu.forEach(a),Hu.forEach(a),Ap=u(Mr),gn=t(Mr,"SPAN",{});var Mu=r(gn);Cp=o(Mu,"NLP"),Mu.forEach(a),Mr.forEach(a),Ql=u(s),y(Ks.$$.fragment,s),Vl=u(s),R=t(s,"P",{});var Ke=r(R);Dp=o(Ke,"Lo strumento principale per processare dati testuali \xE8 un "),de=t(Ke,"A",{href:!0});var Qu=r(de);Ip=o(Qu,"tokenizer"),Qu.forEach(a),Sp=o(Ke,". Un tokenizer inizia separando il testo in "),_n=t(Ke,"EM",{});var Vu=r(_n);Np=o(Vu,"tokens"),Vu.forEach(a),Tp=o(Ke," secondo una serie di regole. I tokens sono convertiti in numeri, questi vengono utilizzati per costruire i tensori di input del modello. Anche altri input addizionali se richiesti dal modello vengono aggiunti dal tokenizer."),Ke.forEach(a),Jl=u(s),y(gs.$$.fragment,s),Wl=u(s),F=t(s,"P",{});var Ze=r(F);Op=o(Ze,"Iniziamo subito caricando un tokenizer preaddestrato con la classe "),vn=t(Ze,"CODE",{});var Ju=r(vn);Lp=o(Ju,"AutoTokenizer"),Ju.forEach(a),Up=o(Ze,". Questo scarica il "),$n=t(Ze,"EM",{});var Wu=r($n);Rp=o(Wu,"vocabolario"),Wu.forEach(a),Fp=o(Ze," usato quando il modello \xE8 stato preaddestrato."),Ze.forEach(a),Gl=u(s),as=t(s,"H3",{class:!0});var Qr=r(as);_s=t(Qr,"A",{id:!0,class:!0,href:!0});var Gu=r(_s);kn=t(Gu,"SPAN",{});var Yu=r(kn);y(Zs.$$.fragment,Yu),Yu.forEach(a),Gu.forEach(a),Hp=u(Qr),En=t(Qr,"SPAN",{});var Ku=r(En);Bp=o(Ku,"Tokenize"),Ku.forEach(a),Qr.forEach(a),Yl=u(s),vs=t(s,"P",{});var Vr=r(vs);Mp=o(Vr,"Carica un tokenizer preaddestrato con "),zn=t(Vr,"CODE",{});var Zu=r(zn);Qp=o(Zu,"AutoTokenizer.from_pretrained()"),Zu.forEach(a),Vp=o(Vr,":"),Vr.forEach(a),Kl=u(s),y(Xs.$$.fragment,s),Zl=u(s),fe=t(s,"P",{});var Xu=r(fe);Jp=o(Xu,"Poi inserisci le tue frasi nel tokenizer:"),Xu.forEach(a),Xl=u(s),y(sa.$$.fragment,s),st=u(s),be=t(s,"P",{});var sm=r(be);Wp=o(sm,"Il tokenizer restituisce un dizionario contenente tre oggetti importanti:"),sm.forEach(a),at=u(s),H=t(s,"UL",{});var Xe=r(H);je=t(Xe,"LI",{});var zu=r(je);ge=t(zu,"A",{href:!0});var am=r(ge);Gp=o(am,"input_ids"),am.forEach(a),Yp=o(zu," sono gli indici che corrispondono ad ogni token nella frase."),zu.forEach(a),Kp=u(Xe),_e=t(Xe,"LI",{});var wu=r(_e);ve=t(wu,"A",{href:!0});var em=r(ve);Zp=o(em,"attention_mask"),em.forEach(a),Xp=o(wu," indicata se un token deve essere elaborato o no."),wu.forEach(a),so=u(Xe),$e=t(Xe,"LI",{});var qu=r($e);ke=t(qu,"A",{href:!0});var nm=r(ke);ao=o(nm,"token_type_ids"),nm.forEach(a),eo=o(qu," identifica a quale sequenza appartiene un token se \xE8 presente pi\xF9 di una sequenza."),qu.forEach(a),Xe.forEach(a),et=u(s),$s=t(s,"P",{});var Jr=r($s);no=o(Jr,"Si possono decodificare gli "),wn=t(Jr,"CODE",{});var lm=r(wn);lo=o(lm,"input_ids"),lm.forEach(a),to=o(Jr," per farsi restituire l\u2019input originale:"),Jr.forEach(a),nt=u(s),y(aa.$$.fragment,s),lt=u(s),B=t(s,"P",{});var sn=r(B);ro=o(sn,"Come si pu\xF2 vedere, il tokenizer aggiunge due token speciali - "),qn=t(sn,"CODE",{});var tm=r(qn);po=o(tm,"CLS"),tm.forEach(a),oo=o(sn," e "),yn=t(sn,"CODE",{});var rm=r(yn);io=o(rm,"SEP"),rm.forEach(a),co=o(sn," (classificatore e separatore) - alla frase. Non tutti i modelli hanno bisogno dei token speciali, ma se servono, il tokenizer li aggiunger\xE0 automaticamente."),sn.forEach(a),tt=u(s),Ee=t(s,"P",{});var pm=r(Ee);uo=o(pm,"Se ci sono pi\xF9 frasi che vuoi processare, passale come una lista al tokenizer:"),pm.forEach(a),rt=u(s),y(ea.$$.fragment,s),pt=u(s),es=t(s,"H3",{class:!0});var Wr=r(es);ks=t(Wr,"A",{id:!0,class:!0,href:!0});var om=r(ks);xn=t(om,"SPAN",{});var im=r(xn);y(na.$$.fragment,im),im.forEach(a),om.forEach(a),mo=u(Wr),Pn=t(Wr,"SPAN",{});var cm=r(Pn);ho=o(cm,"Pad"),cm.forEach(a),Wr.forEach(a),ot=u(s),Es=t(s,"P",{});var Gr=r(Es);fo=o(Gr,"Questo \xE8 un argomento importante. Quando processi un insieme di frasi potrebbero non avere tutte la stessa lunghezza. Questo \xE8 un problema perch\xE8 i tensori, in input del modello, devono avere dimensioni uniformi. Il padding \xE8 una strategia per assicurarsi che i tensori siano rettangolari aggiungendo uno speciale "),An=t(Gr,"EM",{});var um=r(An);bo=o(um,"padding token"),um.forEach(a),jo=o(Gr," alle frasi pi\xF9 corte."),Gr.forEach(a),it=u(s),M=t(s,"P",{});var an=r(M);go=o(an,"Imposta il parametro "),Cn=t(an,"CODE",{});var mm=r(Cn);_o=o(mm,"padding"),mm.forEach(a),vo=o(an," a "),Dn=t(an,"CODE",{});var hm=r(Dn);$o=o(hm,"True"),hm.forEach(a),ko=o(an," per imbottire le frasi pi\xF9 corte nel gruppo in modo che combacino con la massima lunghezza presente:"),an.forEach(a),ct=u(s),y(la.$$.fragment,s),ut=u(s),zs=t(s,"P",{});var Yr=r(zs);Eo=o(Yr,"Nota che il tokenizer aggiunge alle sequenze degli "),In=t(Yr,"CODE",{});var dm=r(In);zo=o(dm,"0"),dm.forEach(a),wo=o(Yr," perch\xE8 sono troppo corte!"),Yr.forEach(a),mt=u(s),ns=t(s,"H3",{class:!0});var Kr=r(ns);ws=t(Kr,"A",{id:!0,class:!0,href:!0});var fm=r(ws);Sn=t(fm,"SPAN",{});var bm=r(Sn);y(ta.$$.fragment,bm),bm.forEach(a),fm.forEach(a),qo=u(Kr),Nn=t(Kr,"SPAN",{});var jm=r(Nn);yo=o(jm,"Truncation"),jm.forEach(a),Kr.forEach(a),ht=u(s),ze=t(s,"P",{});var gm=r(ze);xo=o(gm,"L\u2019altra faccia della medaglia \xE8 che avolte le sequenze possono essere troppo lunghe per essere gestite dal modello. In questo caso, avrai bisogno di troncare la sequenza per avere una lunghezza minore."),gm.forEach(a),dt=u(s),Q=t(s,"P",{});var en=r(Q);Po=o(en,"Imposta il parametro "),Tn=t(en,"CODE",{});var _m=r(Tn);Ao=o(_m,"truncation"),_m.forEach(a),Co=o(en," a "),On=t(en,"CODE",{});var vm=r(On);Do=o(vm,"True"),vm.forEach(a),Io=o(en," per troncare una sequenza alla massima lunghezza accettata dal modello:"),en.forEach(a),ft=u(s),y(ra.$$.fragment,s),bt=u(s),ls=t(s,"H3",{class:!0});var Zr=r(ls);qs=t(Zr,"A",{id:!0,class:!0,href:!0});var $m=r(qs);Ln=t($m,"SPAN",{});var km=r(Ln);y(pa.$$.fragment,km),km.forEach(a),$m.forEach(a),So=u(Zr),Un=t(Zr,"SPAN",{});var Em=r(Un);No=o(Em,"Costruire i tensori"),Em.forEach(a),Zr.forEach(a),jt=u(s),we=t(s,"P",{});var zm=r(we);To=o(zm,"Infine, vuoi che il tokenizer restituisca i tensori prodotti dal modello."),zm.forEach(a),gt=u(s),O=t(s,"P",{});var Js=r(O);Oo=o(Js,"Imposta il parametro "),Rn=t(Js,"CODE",{});var wm=r(Rn);Lo=o(wm,"return_tensors"),wm.forEach(a),Uo=o(Js," su "),Fn=t(Js,"CODE",{});var qm=r(Fn);Ro=o(qm,"pt"),qm.forEach(a),Fo=o(Js," per PyTorch, o "),Hn=t(Js,"CODE",{});var ym=r(Hn);Ho=o(ym,"tf"),ym.forEach(a),Bo=o(Js," per TensorFlow:"),Js.forEach(a),_t=u(s),y(oa.$$.fragment,s),vt=u(s),ts=t(s,"H2",{class:!0});var Xr=r(ts);ys=t(Xr,"A",{id:!0,class:!0,href:!0});var xm=r(ys);Bn=t(xm,"SPAN",{});var Pm=r(Bn);y(ia.$$.fragment,Pm),Pm.forEach(a),xm.forEach(a),Mo=u(Xr),Mn=t(Xr,"SPAN",{});var Am=r(Mn);Qo=o(Am,"Audio"),Am.forEach(a),Xr.forEach(a),$t=u(s),xs=t(s,"P",{});var sp=r(xs);Vo=o(sp,"Gli input audio sono processati in modo differente rispetto al testo, ma l\u2019obiettivo rimane lo stesso: creare sequenze numeriche che il modello pu\xF2 capire. Un "),qe=t(sp,"A",{href:!0});var Cm=r(qe);Jo=o(Cm,"estrattore di caratteristiche"),Cm.forEach(a),Wo=o(sp," \xE8 progettato con lo scopo preciso di estrarre caratteristiche da immagini o dati audio grezzi e convertirli in tensori. Prima di iniziare, installa \u{1F917} Datasets per caricare un dataset audio e sperimentare:"),sp.forEach(a),kt=u(s),y(ca.$$.fragment,s),Et=u(s),V=t(s,"P",{});var nn=r(V);Go=o(nn,"Carica il dataset "),ua=t(nn,"A",{href:!0,rel:!0});var Dm=r(ua);Yo=o(Dm,"MInDS-14"),Dm.forEach(a),Ko=o(nn," (vedi il \u{1F917} "),ma=t(nn,"A",{href:!0,rel:!0});var Im=r(ma);Zo=o(Im,"Datasets tutorial"),Im.forEach(a),Xo=o(nn," per avere maggiori dettagli su come caricare un dataset):"),nn.forEach(a),zt=u(s),y(ha.$$.fragment,s),wt=u(s),J=t(s,"P",{});var ln=r(J);si=o(ln,"Accedi al primo elemento della colonna "),Qn=t(ln,"CODE",{});var Sm=r(Qn);ai=o(Sm,"audio"),Sm.forEach(a),ei=o(ln," per dare uno sguardo all\u2019input. Richiamando la colonna "),Vn=t(ln,"CODE",{});var Nm=r(Vn);ni=o(Nm,"audio"),Nm.forEach(a),li=o(ln," sar\xE0 caricato automaticamente e ricampionato il file audio:"),ln.forEach(a),qt=u(s),y(da.$$.fragment,s),yt=u(s),ye=t(s,"P",{});var Tm=r(ye);ti=o(Tm,"Questo restituisce tre oggetti:"),Tm.forEach(a),xt=u(s),W=t(s,"UL",{});var tn=r(W);xe=t(tn,"LI",{});var yu=r(xe);Jn=t(yu,"CODE",{});var Om=r(Jn);ri=o(Om,"array"),Om.forEach(a),pi=o(yu," \xE8 il segnale vocale caricato - e potenzialmente ricampionato - come vettore 1D."),yu.forEach(a),oi=u(tn),Pe=t(tn,"LI",{});var xu=r(Pe);Wn=t(xu,"CODE",{});var Lm=r(Wn);ii=o(Lm,"path"),Lm.forEach(a),ci=o(xu," il percorso del file audio."),xu.forEach(a),ui=u(tn),Ae=t(tn,"LI",{});var Pu=r(Ae);Gn=t(Pu,"CODE",{});var Um=r(Gn);mi=o(Um,"sampling_rate"),Um.forEach(a),hi=o(Pu," si riferisce al numero di campioni del segnale vocale misurati al secondo."),Pu.forEach(a),tn.forEach(a),Pt=u(s),rs=t(s,"H3",{class:!0});var ap=r(rs);Ps=t(ap,"A",{id:!0,class:!0,href:!0});var Rm=r(Ps);Yn=t(Rm,"SPAN",{});var Fm=r(Yn);y(fa.$$.fragment,Fm),Fm.forEach(a),Rm.forEach(a),di=u(ap),Kn=t(ap,"SPAN",{});var Hm=r(Kn);fi=o(Hm,"Ricampionamento"),Hm.forEach(a),ap.forEach(a),At=u(s),As=t(s,"P",{});var ep=r(As);bi=o(ep,"Per questo tutorial, puoi usare il modello "),ba=t(ep,"A",{href:!0,rel:!0});var Bm=r(ba);ji=o(Bm,"Wav2Vec2"),Bm.forEach(a),gi=o(ep,". Come puoi vedere dalla model card, il modello Wav2Vec2 \xE8 preaddestrato su un campionamento vocale a 16kHz.\xC8 importante che la frequenza di campionamento dei tuoi dati audio combaci con la frequenza di campionamento del dataset usato per preaddestrare il modello. Se la frequenza di campionamento dei tuoi dati non \xE8 uguale dovrai ricampionare i tuoi dati audio."),ep.forEach(a),Ct=u(s),Cs=t(s,"P",{});var np=r(Cs);_i=o(np,"Per esempio, il dataset "),ja=t(np,"A",{href:!0,rel:!0});var Mm=r(ja);vi=o(Mm,"MInDS-14"),Mm.forEach(a),$i=o(np," ha una frequenza di campionamento di 8000kHz. Utilizzando il modello Wav2Vec2 su questo dataset, alzala a 16kHz:"),np.forEach(a),Dt=u(s),y(ga.$$.fragment,s),It=u(s),Ce=t(s,"OL",{});var Qm=r(Ce);_a=t(Qm,"LI",{});var lp=r(_a);ki=o(lp,"Usa il metodo di \u{1F917} Datasets\u2019 "),va=t(lp,"A",{href:!0,rel:!0});var Vm=r(va);Zn=t(Vm,"CODE",{});var Jm=r(Zn);Ei=o(Jm,"cast_column"),Jm.forEach(a),Vm.forEach(a),zi=o(lp," per alzare la frequenza di campionamento a 16kHz:"),lp.forEach(a),Qm.forEach(a),St=u(s),y($a.$$.fragment,s),Nt=u(s),ka=t(s,"OL",{start:!0});var Wm=r(ka);Xn=t(Wm,"LI",{});var Gm=r(Xn);wi=o(Gm,"Carica il file audio:"),Gm.forEach(a),Wm.forEach(a),Tt=u(s),y(Ea.$$.fragment,s),Ot=u(s),Ds=t(s,"P",{});var tp=r(Ds);qi=o(tp,"Come puoi notare, la "),sl=t(tp,"CODE",{});var Ym=r(sl);yi=o(Ym,"sampling_rate"),Ym.forEach(a),xi=o(tp," adesso \xE8 16kHz!"),tp.forEach(a),Lt=u(s),ps=t(s,"H3",{class:!0});var rp=r(ps);Is=t(rp,"A",{id:!0,class:!0,href:!0});var Km=r(Is);al=t(Km,"SPAN",{});var Zm=r(al);y(za.$$.fragment,Zm),Zm.forEach(a),Km.forEach(a),Pi=u(rp),el=t(rp,"SPAN",{});var Xm=r(el);Ai=o(Xm,"Feature extractor"),Xm.forEach(a),rp.forEach(a),Ut=u(s),L=t(s,"P",{});var Ws=r(L);Ci=o(Ws,"Il prossimo passo \xE8 caricare un estrattore di caratteristiche per normalizzare e fare padding sull\u2019input. Quando applichiamo il padding sui dati testuali, uno "),nl=t(Ws,"CODE",{});var sh=r(nl);Di=o(sh,"0"),sh.forEach(a),Ii=o(Ws," \xE8 aggiunto alle sequenze pi\xF9 brevi. La stessa idea si applica ai dati audio, l\u2019estrattore di caratteristiche per gli audio aggiunger\xE0 uno "),ll=t(Ws,"CODE",{});var ah=r(ll);Si=o(ah,"0"),ah.forEach(a),Ni=o(Ws," - interpretato come silenzio - agli "),tl=t(Ws,"CODE",{});var eh=r(tl);Ti=o(eh,"array"),eh.forEach(a),Oi=o(Ws,"."),Ws.forEach(a),Rt=u(s),Ss=t(s,"P",{});var pp=r(Ss);Li=o(pp,"Carica l\u2019estrattore delle caratteristiche con "),rl=t(pp,"CODE",{});var nh=r(rl);Ui=o(nh,"AutoFeatureExtractor.from_pretrained()"),nh.forEach(a),Ri=o(pp,":"),pp.forEach(a),Ft=u(s),y(wa.$$.fragment,s),Ht=u(s),G=t(s,"P",{});var rn=r(G);Fi=o(rn,"Inserisci l\u2019 "),pl=t(rn,"CODE",{});var lh=r(pl);Hi=o(lh,"array"),lh.forEach(a),Bi=o(rn," audio nell\u2019estrattore delle caratteristiche. Noi raccomandiamo sempre di aggiungere il parametro "),ol=t(rn,"CODE",{});var th=r(ol);Mi=o(th,"sampling_rate"),th.forEach(a),Qi=o(rn," nell\u2019estrattore delle caratteristiche per correggere meglio qualche errore, dovuto ai silenzi, che potrebbe verificarsi."),rn.forEach(a),Bt=u(s),y(qa.$$.fragment,s),Mt=u(s),os=t(s,"H3",{class:!0});var op=r(os);Ns=t(op,"A",{id:!0,class:!0,href:!0});var rh=r(Ns);il=t(rh,"SPAN",{});var ph=r(il);y(ya.$$.fragment,ph),ph.forEach(a),rh.forEach(a),Vi=u(op),cl=t(op,"SPAN",{});var oh=r(cl);Ji=o(oh,"Pad e truncate"),oh.forEach(a),op.forEach(a),Qt=u(s),De=t(s,"P",{});var ih=r(De);Wi=o(ih,"Come per il tokenizer, puoi applicare le operazioni padding o truncation per manipolare sequenze di variabili a lotti. Dai uno sguaro alla lunghezza delle sequenze di questi due campioni audio:"),ih.forEach(a),Vt=u(s),y(xa.$$.fragment,s),Jt=u(s),Ie=t(s,"P",{});var ch=r(Ie);Gi=o(ch,"Come puoi vedere, il primo campione ha una sequenza pi\xF9 lunga del secondo. Crea una funzione che preprocesser\xE0 il dataset. Specifica una lunghezza massima del campione, e l\u2019estrattore di features si occuper\xE0 di riempire o troncare la sequenza per coincidervi:"),ch.forEach(a),Wt=u(s),y(Pa.$$.fragment,s),Gt=u(s),Se=t(s,"P",{});var uh=r(Se);Yi=o(uh,"Applica la funzione ai primi esempi nel dataset:"),uh.forEach(a),Yt=u(s),y(Aa.$$.fragment,s),Kt=u(s),Ne=t(s,"P",{});var mh=r(Ne);Ki=o(mh,"Adesso guarda la lunghezza dei campioni elaborati:"),mh.forEach(a),Zt=u(s),y(Ca.$$.fragment,s),Xt=u(s),Te=t(s,"P",{});var hh=r(Te);Zi=o(hh,"La lunghezza dei campioni adesso coincide con la massima lunghezza impostata nelle funzione."),hh.forEach(a),sr=u(s),is=t(s,"H2",{class:!0});var ip=r(is);Ts=t(ip,"A",{id:!0,class:!0,href:!0});var dh=r(Ts);ul=t(dh,"SPAN",{});var fh=r(ul);y(Da.$$.fragment,fh),fh.forEach(a),dh.forEach(a),Xi=u(ip),ml=t(ip,"SPAN",{});var bh=r(ml);sc=o(bh,"Vision"),bh.forEach(a),ip.forEach(a),ar=u(s),Oe=t(s,"P",{});var jh=r(Oe);ac=o(jh,"Un estrattore di caratteristiche si pu\xF2 usare anche per processare immagini e per compiti di visione. Ancora una volta, l\u2019obiettivo \xE8 convertire l\u2019immagine grezza in un lotto di tensori come input."),jh.forEach(a),er=u(s),Y=t(s,"P",{});var pn=r(Y);ec=o(pn,"Carica il dataset "),Ia=t(pn,"A",{href:!0,rel:!0});var gh=r(Ia);nc=o(gh,"food101"),gh.forEach(a),lc=o(pn," per questa esercitazione. Usa il parametro "),hl=t(pn,"CODE",{});var _h=r(hl);tc=o(_h,"split"),_h.forEach(a),rc=o(pn," di \u{1F917} Datasets  per caricare solo un piccolo campione dal dataset di addestramento poich\xE8 il set di dati \xE8 molto grande:"),pn.forEach(a),nr=u(s),y(Sa.$$.fragment,s),lr=u(s),Os=t(s,"P",{});var cp=r(Os);pc=o(cp,"Secondo passo, dai uno sguardo alle immagini usando la caratteristica "),Na=t(cp,"A",{href:!0,rel:!0});var vh=r(Na);dl=t(vh,"CODE",{});var $h=r(dl);oc=o($h,"Image"),$h.forEach(a),vh.forEach(a),ic=o(cp," di \u{1F917} Datasets:"),cp.forEach(a),tr=u(s),y(Ta.$$.fragment,s),rr=u(s),Le=t(s,"P",{});var kh=r(Le);Ue=t(kh,"IMG",{src:!0,alt:!0}),kh.forEach(a),pr=u(s),cs=t(s,"H3",{class:!0});var up=r(cs);Ls=t(up,"A",{id:!0,class:!0,href:!0});var Eh=r(Ls);fl=t(Eh,"SPAN",{});var zh=r(fl);y(Oa.$$.fragment,zh),zh.forEach(a),Eh.forEach(a),cc=u(up),bl=t(up,"SPAN",{});var wh=r(bl);uc=o(wh,"Feature extractor"),wh.forEach(a),up.forEach(a),or=u(s),Us=t(s,"P",{});var mp=r(Us);mc=o(mp,"Carica l\u2019estrattore di caratteristiche "),jl=t(mp,"CODE",{});var qh=r(jl);hc=o(qh,"AutoFeatureExtractor.from_pretrained()"),qh.forEach(a),dc=o(mp,":"),mp.forEach(a),ir=u(s),y(La.$$.fragment,s),cr=u(s),us=t(s,"H3",{class:!0});var hp=r(us);Rs=t(hp,"A",{id:!0,class:!0,href:!0});var yh=r(Rs);gl=t(yh,"SPAN",{});var xh=r(gl);y(Ua.$$.fragment,xh),xh.forEach(a),yh.forEach(a),fc=u(hp),_l=t(hp,"SPAN",{});var Ph=r(_l);bc=o(Ph,"Data augmentation"),Ph.forEach(a),hp.forEach(a),ur=u(s),Fs=t(s,"P",{});var dp=r(Fs);jc=o(dp,"Per le attivit\xE0 di visione, \xE8 usuale aggiungere alcuni tipi di data augmentation alle immagini come parte del preprocessing. Puoi aggiungere augmentations con qualsiasi libreria che preferisci, ma in questa esercitazione, userai il modulo "),Ra=t(dp,"A",{href:!0,rel:!0});var Ah=r(Ra);vl=t(Ah,"CODE",{});var Ch=r(vl);gc=o(Ch,"transforms"),Ch.forEach(a),Ah.forEach(a),_c=o(dp," di torchvision."),dp.forEach(a),mr=u(s),Re=t(s,"OL",{});var Dh=r(Re);U=t(Dh,"LI",{});var Gs=r(U);vc=o(Gs,"Normalizza l\u2019immagine e usa "),Fa=t(Gs,"A",{href:!0,rel:!0});var Ih=r(Fa);$l=t(Ih,"CODE",{});var Sh=r($l);$c=o(Sh,"Compose"),Sh.forEach(a),Ih.forEach(a),kc=o(Gs," per concatenare alcune trasformazioni - "),Ha=t(Gs,"A",{href:!0,rel:!0});var Nh=r(Ha);kl=t(Nh,"CODE",{});var Th=r(kl);Ec=o(Th,"RandomResizedCrop"),Th.forEach(a),Nh.forEach(a),zc=o(Gs," e "),Ba=t(Gs,"A",{href:!0,rel:!0});var Oh=r(Ba);El=t(Oh,"CODE",{});var Lh=r(El);wc=o(Lh,"ColorJitter"),Lh.forEach(a),Oh.forEach(a),qc=o(Gs," - insieme:"),Gs.forEach(a),Dh.forEach(a),hr=u(s),y(Ma.$$.fragment,s),dr=u(s),Qa=t(s,"OL",{start:!0});var Uh=r(Qa);ms=t(Uh,"LI",{});var on=r(ms);yc=o(on,"Il modello accetta "),Fe=t(on,"A",{href:!0});var Rh=r(Fe);zl=t(Rh,"CODE",{});var Fh=r(zl);xc=o(Fh,"pixel_values"),Fh.forEach(a),Rh.forEach(a),Pc=o(on," come input. Questo valore \xE8 generato dall\u2019estrattore di caratteristiche. Crea una funzione che genera "),wl=t(on,"CODE",{});var Hh=r(wl);Ac=o(Hh,"pixel_values"),Hh.forEach(a),Cc=o(on," dai transforms:"),on.forEach(a),Uh.forEach(a),fr=u(s),y(Va.$$.fragment,s),br=u(s),Ja=t(s,"OL",{start:!0});var Bh=r(Ja);Wa=t(Bh,"LI",{});var fp=r(Wa);Dc=o(fp,"Poi utilizza \u{1F917} Datasets "),Ga=t(fp,"A",{href:!0,rel:!0});var Mh=r(Ga);ql=t(Mh,"CODE",{});var Qh=r(ql);Ic=o(Qh,"set_transform"),Qh.forEach(a),Mh.forEach(a),Sc=o(fp,"per applicare al volo la trasformazione:"),fp.forEach(a),Bh.forEach(a),jr=u(s),y(Ya.$$.fragment,s),gr=u(s),Ka=t(s,"OL",{start:!0});var Vh=r(Ka);Za=t(Vh,"LI",{});var bp=r(Za);Nc=o(bp,"Adesso quando accedi all\u2019immagine, puoi notare che l\u2019estrattore di caratteristiche ha aggiunto "),yl=t(bp,"CODE",{});var Jh=r(yl);Tc=o(Jh,"pixel_values"),Jh.forEach(a),Oc=o(bp," allo schema di input:"),bp.forEach(a),Vh.forEach(a),_r=u(s),y(Xa.$$.fragment,s),vr=u(s),He=t(s,"P",{});var Wh=r(He);Lc=o(Wh,"Di seguito come si vede l\u2019immagine dopo la fase di preprocessing. Come ci si aspetterebbe dalle trasformazioni applicate, l\u2019immagine \xE8 stata ritagliata in modo casuale e le propriet\xE0 del colore sono diverse."),Wh.forEach(a),$r=u(s),y(se.$$.fragment,s),kr=u(s),Be=t(s,"P",{});var Gh=r(Be);Me=t(Gh,"IMG",{src:!0,alt:!0}),Gh.forEach(a),Er=u(s),hs=t(s,"H2",{class:!0});var jp=r(hs);Hs=t(jp,"A",{id:!0,class:!0,href:!0});var Yh=r(Hs);xl=t(Yh,"SPAN",{});var Kh=r(xl);y(ae.$$.fragment,Kh),Kh.forEach(a),Yh.forEach(a),Uc=u(jp),Pl=t(jp,"SPAN",{});var Zh=r(Pl);Rc=o(Zh,"Multimodal"),Zh.forEach(a),jp.forEach(a),zr=u(s),Qe=t(s,"P",{});var Xh=r(Qe);Fc=o(Xh,"Per attivit\xE0 multimodali userai una combinazione di tutto quello che hai imparato poco fa e applicherai le tue competenze alla comprensione automatica del parlato (Automatic Speech Recognition -  ASR). Questo significa che avrai bisogno di:"),Xh.forEach(a),wr=u(s),Bs=t(s,"UL",{});var gp=r(Bs);Al=t(gp,"LI",{});var sd=r(Al);Hc=o(sd,"Un estrattore delle caratteristiche per processare i dati audio."),sd.forEach(a),Bc=u(gp),Cl=t(gp,"LI",{});var ad=r(Cl);Mc=o(ad,"Il Tokenizer per processare i testi."),ad.forEach(a),gp.forEach(a),qr=u(s),Ms=t(s,"P",{});var _p=r(Ms);Qc=o(_p,"Ritorna sul datasere "),ee=t(_p,"A",{href:!0,rel:!0});var ed=r(ee);Vc=o(ed,"LJ Speech"),ed.forEach(a),Jc=o(_p,":"),_p.forEach(a),yr=u(s),y(ne.$$.fragment,s),xr=u(s),K=t(s,"P",{});var cn=r(K);Wc=o(cn,"Visto che sei interessato solo alle colonne "),Dl=t(cn,"CODE",{});var nd=r(Dl);Gc=o(nd,"audio"),nd.forEach(a),Yc=o(cn," e "),Il=t(cn,"CODE",{});var ld=r(Il);Kc=o(ld,"text"),ld.forEach(a),Zc=o(cn,", elimina tutte le altre:"),cn.forEach(a),Pr=u(s),y(le.$$.fragment,s),Ar=u(s),Z=t(s,"P",{});var un=r(Z);Xc=o(un,"Adesso guarda le colonne "),Sl=t(un,"CODE",{});var td=r(Sl);su=o(td,"audio"),td.forEach(a),au=o(un," e "),Nl=t(un,"CODE",{});var rd=r(Nl);eu=o(rd,"text"),rd.forEach(a),nu=o(un,":"),un.forEach(a),Cr=u(s),y(te.$$.fragment,s),Dr=u(s),Qs=t(s,"P",{});var vp=r(Qs);lu=o(vp,"Ricorda dalla sezione precedente sull\u2019elaborazione dei dati audio, tu dovresti sempre "),Ve=t(vp,"A",{href:!0});var pd=r(Ve);tu=o(pd,"ricampionare"),pd.forEach(a),ru=o(vp," la frequenza di campionamento dei tuoi dati audio per farla coincidere con quella del dataset usato dal modello preaddestrato:"),vp.forEach(a),Ir=u(s),y(re.$$.fragment,s),Sr=u(s),ds=t(s,"H3",{class:!0});var $p=r(ds);Vs=t($p,"A",{id:!0,class:!0,href:!0});var od=r(Vs);Tl=t(od,"SPAN",{});var id=r(Tl);y(pe.$$.fragment,id),id.forEach(a),od.forEach(a),pu=u($p),Ol=t($p,"SPAN",{});var cd=r(Ol);ou=o(cd,"Processor"),cd.forEach(a),$p.forEach(a),Nr=u(s),Je=t(s,"P",{});var ud=r(Je);iu=o(ud,"Un processor combina un estrattore di caratteristiche e un tokenizer. Carica un processor con [`AutoProcessor.from_pretrained]:"),ud.forEach(a),Tr=u(s),y(oe.$$.fragment,s),Or=u(s),We=t(s,"OL",{});var md=r(We);fs=t(md,"LI",{});var mn=r(fs);cu=o(mn,"Crea una funzione che processi i dati audio in "),Ll=t(mn,"CODE",{});var hd=r(Ll);uu=o(hd,"input_values"),hd.forEach(a),mu=o(mn,", e tokenizza il testo in "),Ul=t(mn,"CODE",{});var dd=r(Ul);hu=o(dd,"labels"),dd.forEach(a),du=o(mn,". Questi sono i tuoi input per il modello:"),mn.forEach(a),md.forEach(a),Lr=u(s),y(ie.$$.fragment,s),Ur=u(s),ce=t(s,"OL",{start:!0});var fd=r(ce);ue=t(fd,"LI",{});var kp=r(ue);fu=o(kp,"Applica la funzione "),Rl=t(kp,"CODE",{});var bd=r(Rl);bu=o(bd,"prepare_dataset"),bd.forEach(a),ju=o(kp," ad un campione:"),kp.forEach(a),fd.forEach(a),Rr=u(s),y(me.$$.fragment,s),Fr=u(s),X=t(s,"P",{});var hn=r(X);gu=o(hn,"Nota che il processor ha aggiunto "),Fl=t(hn,"CODE",{});var jd=r(Fl);_u=o(jd,"input_values"),jd.forEach(a),vu=o(hn," e "),Hl=t(hn,"CODE",{});var gd=r(Hl);$u=o(gd,"labels"),gd.forEach(a),ku=o(hn,". La frequenza di campionamento \xE8 stata corretta riducendola a 16kHz."),hn.forEach(a),Hr=u(s),Ge=t(s,"P",{});var _d=r(Ge);Eu=o(_d,"Fantastico, ora dovresti essere in grado di preelaborare i dati per qualsiasi modalit\xE0 e persino di combinare modalit\xE0 diverse! Nella prossima esercitazione, impareremo a mettere a punto un modello sui dati appena pre-elaborati."),_d.forEach(a),this.h()},h(){m(h,"name","hf:doc:metadata"),m(h,"content",JSON.stringify(Kd)),m(v,"id","preprocess"),m(v,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),m(v,"href","#preprocess"),m(b,"class","relative group"),m(js,"id","nlp"),m(js,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),m(js,"href","#nlp"),m(ss,"class","relative group"),m(de,"href","main_classes/tokenizer"),m(_s,"id","tokenize"),m(_s,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),m(_s,"href","#tokenize"),m(as,"class","relative group"),m(ge,"href","glossary#input-ids"),m(ve,"href","glossary#attention-mask"),m(ke,"href","glossary#token-type-ids"),m(ks,"id","pad"),m(ks,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),m(ks,"href","#pad"),m(es,"class","relative group"),m(ws,"id","truncation"),m(ws,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),m(ws,"href","#truncation"),m(ns,"class","relative group"),m(qs,"id","costruire-i-tensori"),m(qs,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),m(qs,"href","#costruire-i-tensori"),m(ls,"class","relative group"),m(ys,"id","audio"),m(ys,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),m(ys,"href","#audio"),m(ts,"class","relative group"),m(qe,"href","main_classes/feature_extractor"),m(ua,"href","https://huggingface.co/datasets/PolyAI/minds14"),m(ua,"rel","nofollow"),m(ma,"href","https://huggingface.co/docs/datasets/load_hub.html"),m(ma,"rel","nofollow"),m(Ps,"id","ricampionamento"),m(Ps,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),m(Ps,"href","#ricampionamento"),m(rs,"class","relative group"),m(ba,"href","https://huggingface.co/facebook/wav2vec2-base"),m(ba,"rel","nofollow"),m(ja,"href","https://huggingface.co/datasets/PolyAI/minds14"),m(ja,"rel","nofollow"),m(va,"href","https://huggingface.co/docs/datasets/package_reference/main_classes.html#datasets.Dataset.cast_column"),m(va,"rel","nofollow"),m(ka,"start","2"),m(Is,"id","feature-extractor"),m(Is,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),m(Is,"href","#feature-extractor"),m(ps,"class","relative group"),m(Ns,"id","pad-e-truncate"),m(Ns,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),m(Ns,"href","#pad-e-truncate"),m(os,"class","relative group"),m(Ts,"id","vision"),m(Ts,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),m(Ts,"href","#vision"),m(is,"class","relative group"),m(Ia,"href","https://huggingface.co/datasets/food101"),m(Ia,"rel","nofollow"),m(Na,"href","https://huggingface.co/docs/datasets/package_reference/main_classes.html?highlight=image#datasets.Image"),m(Na,"rel","nofollow"),$d(Ue.src,Iu="https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/vision-preprocess-tutorial.png")||m(Ue,"src",Iu),m(Ue,"alt","vision-preprocess-tutorial.png"),m(Ls,"id","feature-extractor"),m(Ls,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),m(Ls,"href","#feature-extractor"),m(cs,"class","relative group"),m(Rs,"id","data-augmentation"),m(Rs,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),m(Rs,"href","#data-augmentation"),m(us,"class","relative group"),m(Ra,"href","https://pytorch.org/vision/stable/transforms.html"),m(Ra,"rel","nofollow"),m(Fa,"href","https://pytorch.org/vision/master/generated/torchvision.transforms.Compose.html"),m(Fa,"rel","nofollow"),m(Ha,"href","https://pytorch.org/vision/main/generated/torchvision.transforms.RandomResizedCrop.html"),m(Ha,"rel","nofollow"),m(Ba,"href","https://pytorch.org/vision/main/generated/torchvision.transforms.ColorJitter.html"),m(Ba,"rel","nofollow"),m(Fe,"href","model_doc/visionencoderdecoder#transformers.VisionEncoderDecoderModel.forward.pixel_values"),m(Qa,"start","2"),m(Ga,"href","https://huggingface.co/docs/datasets/process.html#format-transform"),m(Ga,"rel","nofollow"),m(Ja,"start","3"),m(Ka,"start","4"),$d(Me.src,Su="https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/preprocessed_image.png")||m(Me,"src",Su),m(Me,"alt","preprocessed_image"),m(Hs,"id","multimodal"),m(Hs,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),m(Hs,"href","#multimodal"),m(hs,"class","relative group"),m(ee,"href","https://huggingface.co/datasets/lj_speech"),m(ee,"rel","nofollow"),m(Ve,"href","preprocessing#audio"),m(Vs,"id","processor"),m(Vs,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),m(Vs,"href","#processor"),m(ds,"class","relative group"),m(ce,"start","2")},m(s,n){e(document.head,h),i(s,f,n),i(s,b,n),e(b,v),e(v,j),w(q,j,null),e(b,$),e(b,k),e(k,d),i(s,P,n),w(A,s,n),i(s,C,n),i(s,bs,n),e(bs,N),i(s,I,n),i(s,T,n),e(T,dn),e(dn,wp),e(T,qp),e(T,fn),e(fn,yp),e(T,xp),e(T,bn),e(bn,Pp),i(s,Ml,n),i(s,ss,n),e(ss,js),e(js,jn),w(Ys,jn,null),e(ss,Ap),e(ss,gn),e(gn,Cp),i(s,Ql,n),w(Ks,s,n),i(s,Vl,n),i(s,R,n),e(R,Dp),e(R,de),e(de,Ip),e(R,Sp),e(R,_n),e(_n,Np),e(R,Tp),i(s,Jl,n),w(gs,s,n),i(s,Wl,n),i(s,F,n),e(F,Op),e(F,vn),e(vn,Lp),e(F,Up),e(F,$n),e($n,Rp),e(F,Fp),i(s,Gl,n),i(s,as,n),e(as,_s),e(_s,kn),w(Zs,kn,null),e(as,Hp),e(as,En),e(En,Bp),i(s,Yl,n),i(s,vs,n),e(vs,Mp),e(vs,zn),e(zn,Qp),e(vs,Vp),i(s,Kl,n),w(Xs,s,n),i(s,Zl,n),i(s,fe,n),e(fe,Jp),i(s,Xl,n),w(sa,s,n),i(s,st,n),i(s,be,n),e(be,Wp),i(s,at,n),i(s,H,n),e(H,je),e(je,ge),e(ge,Gp),e(je,Yp),e(H,Kp),e(H,_e),e(_e,ve),e(ve,Zp),e(_e,Xp),e(H,so),e(H,$e),e($e,ke),e(ke,ao),e($e,eo),i(s,et,n),i(s,$s,n),e($s,no),e($s,wn),e(wn,lo),e($s,to),i(s,nt,n),w(aa,s,n),i(s,lt,n),i(s,B,n),e(B,ro),e(B,qn),e(qn,po),e(B,oo),e(B,yn),e(yn,io),e(B,co),i(s,tt,n),i(s,Ee,n),e(Ee,uo),i(s,rt,n),w(ea,s,n),i(s,pt,n),i(s,es,n),e(es,ks),e(ks,xn),w(na,xn,null),e(es,mo),e(es,Pn),e(Pn,ho),i(s,ot,n),i(s,Es,n),e(Es,fo),e(Es,An),e(An,bo),e(Es,jo),i(s,it,n),i(s,M,n),e(M,go),e(M,Cn),e(Cn,_o),e(M,vo),e(M,Dn),e(Dn,$o),e(M,ko),i(s,ct,n),w(la,s,n),i(s,ut,n),i(s,zs,n),e(zs,Eo),e(zs,In),e(In,zo),e(zs,wo),i(s,mt,n),i(s,ns,n),e(ns,ws),e(ws,Sn),w(ta,Sn,null),e(ns,qo),e(ns,Nn),e(Nn,yo),i(s,ht,n),i(s,ze,n),e(ze,xo),i(s,dt,n),i(s,Q,n),e(Q,Po),e(Q,Tn),e(Tn,Ao),e(Q,Co),e(Q,On),e(On,Do),e(Q,Io),i(s,ft,n),w(ra,s,n),i(s,bt,n),i(s,ls,n),e(ls,qs),e(qs,Ln),w(pa,Ln,null),e(ls,So),e(ls,Un),e(Un,No),i(s,jt,n),i(s,we,n),e(we,To),i(s,gt,n),i(s,O,n),e(O,Oo),e(O,Rn),e(Rn,Lo),e(O,Uo),e(O,Fn),e(Fn,Ro),e(O,Fo),e(O,Hn),e(Hn,Ho),e(O,Bo),i(s,_t,n),w(oa,s,n),i(s,vt,n),i(s,ts,n),e(ts,ys),e(ys,Bn),w(ia,Bn,null),e(ts,Mo),e(ts,Mn),e(Mn,Qo),i(s,$t,n),i(s,xs,n),e(xs,Vo),e(xs,qe),e(qe,Jo),e(xs,Wo),i(s,kt,n),w(ca,s,n),i(s,Et,n),i(s,V,n),e(V,Go),e(V,ua),e(ua,Yo),e(V,Ko),e(V,ma),e(ma,Zo),e(V,Xo),i(s,zt,n),w(ha,s,n),i(s,wt,n),i(s,J,n),e(J,si),e(J,Qn),e(Qn,ai),e(J,ei),e(J,Vn),e(Vn,ni),e(J,li),i(s,qt,n),w(da,s,n),i(s,yt,n),i(s,ye,n),e(ye,ti),i(s,xt,n),i(s,W,n),e(W,xe),e(xe,Jn),e(Jn,ri),e(xe,pi),e(W,oi),e(W,Pe),e(Pe,Wn),e(Wn,ii),e(Pe,ci),e(W,ui),e(W,Ae),e(Ae,Gn),e(Gn,mi),e(Ae,hi),i(s,Pt,n),i(s,rs,n),e(rs,Ps),e(Ps,Yn),w(fa,Yn,null),e(rs,di),e(rs,Kn),e(Kn,fi),i(s,At,n),i(s,As,n),e(As,bi),e(As,ba),e(ba,ji),e(As,gi),i(s,Ct,n),i(s,Cs,n),e(Cs,_i),e(Cs,ja),e(ja,vi),e(Cs,$i),i(s,Dt,n),w(ga,s,n),i(s,It,n),i(s,Ce,n),e(Ce,_a),e(_a,ki),e(_a,va),e(va,Zn),e(Zn,Ei),e(_a,zi),i(s,St,n),w($a,s,n),i(s,Nt,n),i(s,ka,n),e(ka,Xn),e(Xn,wi),i(s,Tt,n),w(Ea,s,n),i(s,Ot,n),i(s,Ds,n),e(Ds,qi),e(Ds,sl),e(sl,yi),e(Ds,xi),i(s,Lt,n),i(s,ps,n),e(ps,Is),e(Is,al),w(za,al,null),e(ps,Pi),e(ps,el),e(el,Ai),i(s,Ut,n),i(s,L,n),e(L,Ci),e(L,nl),e(nl,Di),e(L,Ii),e(L,ll),e(ll,Si),e(L,Ni),e(L,tl),e(tl,Ti),e(L,Oi),i(s,Rt,n),i(s,Ss,n),e(Ss,Li),e(Ss,rl),e(rl,Ui),e(Ss,Ri),i(s,Ft,n),w(wa,s,n),i(s,Ht,n),i(s,G,n),e(G,Fi),e(G,pl),e(pl,Hi),e(G,Bi),e(G,ol),e(ol,Mi),e(G,Qi),i(s,Bt,n),w(qa,s,n),i(s,Mt,n),i(s,os,n),e(os,Ns),e(Ns,il),w(ya,il,null),e(os,Vi),e(os,cl),e(cl,Ji),i(s,Qt,n),i(s,De,n),e(De,Wi),i(s,Vt,n),w(xa,s,n),i(s,Jt,n),i(s,Ie,n),e(Ie,Gi),i(s,Wt,n),w(Pa,s,n),i(s,Gt,n),i(s,Se,n),e(Se,Yi),i(s,Yt,n),w(Aa,s,n),i(s,Kt,n),i(s,Ne,n),e(Ne,Ki),i(s,Zt,n),w(Ca,s,n),i(s,Xt,n),i(s,Te,n),e(Te,Zi),i(s,sr,n),i(s,is,n),e(is,Ts),e(Ts,ul),w(Da,ul,null),e(is,Xi),e(is,ml),e(ml,sc),i(s,ar,n),i(s,Oe,n),e(Oe,ac),i(s,er,n),i(s,Y,n),e(Y,ec),e(Y,Ia),e(Ia,nc),e(Y,lc),e(Y,hl),e(hl,tc),e(Y,rc),i(s,nr,n),w(Sa,s,n),i(s,lr,n),i(s,Os,n),e(Os,pc),e(Os,Na),e(Na,dl),e(dl,oc),e(Os,ic),i(s,tr,n),w(Ta,s,n),i(s,rr,n),i(s,Le,n),e(Le,Ue),i(s,pr,n),i(s,cs,n),e(cs,Ls),e(Ls,fl),w(Oa,fl,null),e(cs,cc),e(cs,bl),e(bl,uc),i(s,or,n),i(s,Us,n),e(Us,mc),e(Us,jl),e(jl,hc),e(Us,dc),i(s,ir,n),w(La,s,n),i(s,cr,n),i(s,us,n),e(us,Rs),e(Rs,gl),w(Ua,gl,null),e(us,fc),e(us,_l),e(_l,bc),i(s,ur,n),i(s,Fs,n),e(Fs,jc),e(Fs,Ra),e(Ra,vl),e(vl,gc),e(Fs,_c),i(s,mr,n),i(s,Re,n),e(Re,U),e(U,vc),e(U,Fa),e(Fa,$l),e($l,$c),e(U,kc),e(U,Ha),e(Ha,kl),e(kl,Ec),e(U,zc),e(U,Ba),e(Ba,El),e(El,wc),e(U,qc),i(s,hr,n),w(Ma,s,n),i(s,dr,n),i(s,Qa,n),e(Qa,ms),e(ms,yc),e(ms,Fe),e(Fe,zl),e(zl,xc),e(ms,Pc),e(ms,wl),e(wl,Ac),e(ms,Cc),i(s,fr,n),w(Va,s,n),i(s,br,n),i(s,Ja,n),e(Ja,Wa),e(Wa,Dc),e(Wa,Ga),e(Ga,ql),e(ql,Ic),e(Wa,Sc),i(s,jr,n),w(Ya,s,n),i(s,gr,n),i(s,Ka,n),e(Ka,Za),e(Za,Nc),e(Za,yl),e(yl,Tc),e(Za,Oc),i(s,_r,n),w(Xa,s,n),i(s,vr,n),i(s,He,n),e(He,Lc),i(s,$r,n),w(se,s,n),i(s,kr,n),i(s,Be,n),e(Be,Me),i(s,Er,n),i(s,hs,n),e(hs,Hs),e(Hs,xl),w(ae,xl,null),e(hs,Uc),e(hs,Pl),e(Pl,Rc),i(s,zr,n),i(s,Qe,n),e(Qe,Fc),i(s,wr,n),i(s,Bs,n),e(Bs,Al),e(Al,Hc),e(Bs,Bc),e(Bs,Cl),e(Cl,Mc),i(s,qr,n),i(s,Ms,n),e(Ms,Qc),e(Ms,ee),e(ee,Vc),e(Ms,Jc),i(s,yr,n),w(ne,s,n),i(s,xr,n),i(s,K,n),e(K,Wc),e(K,Dl),e(Dl,Gc),e(K,Yc),e(K,Il),e(Il,Kc),e(K,Zc),i(s,Pr,n),w(le,s,n),i(s,Ar,n),i(s,Z,n),e(Z,Xc),e(Z,Sl),e(Sl,su),e(Z,au),e(Z,Nl),e(Nl,eu),e(Z,nu),i(s,Cr,n),w(te,s,n),i(s,Dr,n),i(s,Qs,n),e(Qs,lu),e(Qs,Ve),e(Ve,tu),e(Qs,ru),i(s,Ir,n),w(re,s,n),i(s,Sr,n),i(s,ds,n),e(ds,Vs),e(Vs,Tl),w(pe,Tl,null),e(ds,pu),e(ds,Ol),e(Ol,ou),i(s,Nr,n),i(s,Je,n),e(Je,iu),i(s,Tr,n),w(oe,s,n),i(s,Or,n),i(s,We,n),e(We,fs),e(fs,cu),e(fs,Ll),e(Ll,uu),e(fs,mu),e(fs,Ul),e(Ul,hu),e(fs,du),i(s,Lr,n),w(ie,s,n),i(s,Ur,n),i(s,ce,n),e(ce,ue),e(ue,fu),e(ue,Rl),e(Rl,bu),e(ue,ju),i(s,Rr,n),w(me,s,n),i(s,Fr,n),i(s,X,n),e(X,gu),e(X,Fl),e(Fl,_u),e(X,vu),e(X,Hl),e(Hl,$u),e(X,ku),i(s,Hr,n),i(s,Ge,n),e(Ge,Eu),Br=!0},p(s,[n]){const he={};n&2&&(he.$$scope={dirty:n,ctx:s}),gs.$set(he)},i(s){Br||(g(q.$$.fragment,s),g(A.$$.fragment,s),g(Ys.$$.fragment,s),g(Ks.$$.fragment,s),g(gs.$$.fragment,s),g(Zs.$$.fragment,s),g(Xs.$$.fragment,s),g(sa.$$.fragment,s),g(aa.$$.fragment,s),g(ea.$$.fragment,s),g(na.$$.fragment,s),g(la.$$.fragment,s),g(ta.$$.fragment,s),g(ra.$$.fragment,s),g(pa.$$.fragment,s),g(oa.$$.fragment,s),g(ia.$$.fragment,s),g(ca.$$.fragment,s),g(ha.$$.fragment,s),g(da.$$.fragment,s),g(fa.$$.fragment,s),g(ga.$$.fragment,s),g($a.$$.fragment,s),g(Ea.$$.fragment,s),g(za.$$.fragment,s),g(wa.$$.fragment,s),g(qa.$$.fragment,s),g(ya.$$.fragment,s),g(xa.$$.fragment,s),g(Pa.$$.fragment,s),g(Aa.$$.fragment,s),g(Ca.$$.fragment,s),g(Da.$$.fragment,s),g(Sa.$$.fragment,s),g(Ta.$$.fragment,s),g(Oa.$$.fragment,s),g(La.$$.fragment,s),g(Ua.$$.fragment,s),g(Ma.$$.fragment,s),g(Va.$$.fragment,s),g(Ya.$$.fragment,s),g(Xa.$$.fragment,s),g(se.$$.fragment,s),g(ae.$$.fragment,s),g(ne.$$.fragment,s),g(le.$$.fragment,s),g(te.$$.fragment,s),g(re.$$.fragment,s),g(pe.$$.fragment,s),g(oe.$$.fragment,s),g(ie.$$.fragment,s),g(me.$$.fragment,s),Br=!0)},o(s){_(q.$$.fragment,s),_(A.$$.fragment,s),_(Ys.$$.fragment,s),_(Ks.$$.fragment,s),_(gs.$$.fragment,s),_(Zs.$$.fragment,s),_(Xs.$$.fragment,s),_(sa.$$.fragment,s),_(aa.$$.fragment,s),_(ea.$$.fragment,s),_(na.$$.fragment,s),_(la.$$.fragment,s),_(ta.$$.fragment,s),_(ra.$$.fragment,s),_(pa.$$.fragment,s),_(oa.$$.fragment,s),_(ia.$$.fragment,s),_(ca.$$.fragment,s),_(ha.$$.fragment,s),_(da.$$.fragment,s),_(fa.$$.fragment,s),_(ga.$$.fragment,s),_($a.$$.fragment,s),_(Ea.$$.fragment,s),_(za.$$.fragment,s),_(wa.$$.fragment,s),_(qa.$$.fragment,s),_(ya.$$.fragment,s),_(xa.$$.fragment,s),_(Pa.$$.fragment,s),_(Aa.$$.fragment,s),_(Ca.$$.fragment,s),_(Da.$$.fragment,s),_(Sa.$$.fragment,s),_(Ta.$$.fragment,s),_(Oa.$$.fragment,s),_(La.$$.fragment,s),_(Ua.$$.fragment,s),_(Ma.$$.fragment,s),_(Va.$$.fragment,s),_(Ya.$$.fragment,s),_(Xa.$$.fragment,s),_(se.$$.fragment,s),_(ae.$$.fragment,s),_(ne.$$.fragment,s),_(le.$$.fragment,s),_(te.$$.fragment,s),_(re.$$.fragment,s),_(pe.$$.fragment,s),_(oe.$$.fragment,s),_(ie.$$.fragment,s),_(me.$$.fragment,s),Br=!1},d(s){a(h),s&&a(f),s&&a(b),E(q),s&&a(P),E(A,s),s&&a(C),s&&a(bs),s&&a(I),s&&a(T),s&&a(Ml),s&&a(ss),E(Ys),s&&a(Ql),E(Ks,s),s&&a(Vl),s&&a(R),s&&a(Jl),E(gs,s),s&&a(Wl),s&&a(F),s&&a(Gl),s&&a(as),E(Zs),s&&a(Yl),s&&a(vs),s&&a(Kl),E(Xs,s),s&&a(Zl),s&&a(fe),s&&a(Xl),E(sa,s),s&&a(st),s&&a(be),s&&a(at),s&&a(H),s&&a(et),s&&a($s),s&&a(nt),E(aa,s),s&&a(lt),s&&a(B),s&&a(tt),s&&a(Ee),s&&a(rt),E(ea,s),s&&a(pt),s&&a(es),E(na),s&&a(ot),s&&a(Es),s&&a(it),s&&a(M),s&&a(ct),E(la,s),s&&a(ut),s&&a(zs),s&&a(mt),s&&a(ns),E(ta),s&&a(ht),s&&a(ze),s&&a(dt),s&&a(Q),s&&a(ft),E(ra,s),s&&a(bt),s&&a(ls),E(pa),s&&a(jt),s&&a(we),s&&a(gt),s&&a(O),s&&a(_t),E(oa,s),s&&a(vt),s&&a(ts),E(ia),s&&a($t),s&&a(xs),s&&a(kt),E(ca,s),s&&a(Et),s&&a(V),s&&a(zt),E(ha,s),s&&a(wt),s&&a(J),s&&a(qt),E(da,s),s&&a(yt),s&&a(ye),s&&a(xt),s&&a(W),s&&a(Pt),s&&a(rs),E(fa),s&&a(At),s&&a(As),s&&a(Ct),s&&a(Cs),s&&a(Dt),E(ga,s),s&&a(It),s&&a(Ce),s&&a(St),E($a,s),s&&a(Nt),s&&a(ka),s&&a(Tt),E(Ea,s),s&&a(Ot),s&&a(Ds),s&&a(Lt),s&&a(ps),E(za),s&&a(Ut),s&&a(L),s&&a(Rt),s&&a(Ss),s&&a(Ft),E(wa,s),s&&a(Ht),s&&a(G),s&&a(Bt),E(qa,s),s&&a(Mt),s&&a(os),E(ya),s&&a(Qt),s&&a(De),s&&a(Vt),E(xa,s),s&&a(Jt),s&&a(Ie),s&&a(Wt),E(Pa,s),s&&a(Gt),s&&a(Se),s&&a(Yt),E(Aa,s),s&&a(Kt),s&&a(Ne),s&&a(Zt),E(Ca,s),s&&a(Xt),s&&a(Te),s&&a(sr),s&&a(is),E(Da),s&&a(ar),s&&a(Oe),s&&a(er),s&&a(Y),s&&a(nr),E(Sa,s),s&&a(lr),s&&a(Os),s&&a(tr),E(Ta,s),s&&a(rr),s&&a(Le),s&&a(pr),s&&a(cs),E(Oa),s&&a(or),s&&a(Us),s&&a(ir),E(La,s),s&&a(cr),s&&a(us),E(Ua),s&&a(ur),s&&a(Fs),s&&a(mr),s&&a(Re),s&&a(hr),E(Ma,s),s&&a(dr),s&&a(Qa),s&&a(fr),E(Va,s),s&&a(br),s&&a(Ja),s&&a(jr),E(Ya,s),s&&a(gr),s&&a(Ka),s&&a(_r),E(Xa,s),s&&a(vr),s&&a(He),s&&a($r),E(se,s),s&&a(kr),s&&a(Be),s&&a(Er),s&&a(hs),E(ae),s&&a(zr),s&&a(Qe),s&&a(wr),s&&a(Bs),s&&a(qr),s&&a(Ms),s&&a(yr),E(ne,s),s&&a(xr),s&&a(K),s&&a(Pr),E(le,s),s&&a(Ar),s&&a(Z),s&&a(Cr),E(te,s),s&&a(Dr),s&&a(Qs),s&&a(Ir),E(re,s),s&&a(Sr),s&&a(ds),E(pe),s&&a(Nr),s&&a(Je),s&&a(Tr),E(oe,s),s&&a(Or),s&&a(We),s&&a(Lr),E(ie,s),s&&a(Ur),s&&a(ce),s&&a(Rr),E(me,s),s&&a(Fr),s&&a(X),s&&a(Hr),s&&a(Ge)}}}const Kd={local:"preprocess",sections:[{local:"nlp",sections:[{local:"tokenize",title:"Tokenize"},{local:"pad",title:"Pad"},{local:"truncation",title:"Truncation"},{local:"costruire-i-tensori",title:"Costruire i tensori"}],title:"NLP"},{local:"audio",sections:[{local:"ricampionamento",title:"Ricampionamento"},{local:"feature-extractor",title:"Feature extractor"},{local:"pad-e-truncate",title:"Pad e truncate"}],title:"Audio"},{local:"vision",sections:[{local:"feature-extractor",title:"Feature extractor"},{local:"data-augmentation",title:"Data augmentation"}],title:"Vision"},{local:"multimodal",sections:[{local:"processor",title:"Processor"}],title:"Multimodal"}],title:"Preprocess"};function Zd(x){return Td(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class rf extends Au{constructor(h){super();Cu(this,h,Zd,Yd,Du,{})}}export{rf as default,Kd as metadata};
