import{S as BE,i as YE,s as GE,e as o,k as d,w as u,t as a,M as JE,c as r,d as s,m as h,a as l,x as m,h as n,b as f,f as UE,G as t,g as p,y as w,q as g,o as _,B as v,v as VE}from"../../chunks/vendor-hf-doc-builder.js";import{T as fs}from"../../chunks/Tip-hf-doc-builder.js";import{Y as ab}from"../../chunks/Youtube-hf-doc-builder.js";import{I as Ga}from"../../chunks/IconCopyLink-hf-doc-builder.js";import{C as b}from"../../chunks/CodeBlock-hf-doc-builder.js";import{C as XE}from"../../chunks/CourseFloatingBanner-hf-doc-builder.js";function KE(L){let c,q,$,k,D,y,j,C;return{c(){c=o("p"),q=a("\u270F\uFE0F "),$=o("strong"),k=a("Try it out!"),D=a(" Use the "),y=o("code"),j=a("Dataset.unique()"),C=a(" function to find the number of unique drugs and conditions in the training and test sets.")},l(E){c=r(E,"P",{});var x=l(c);q=n(x,"\u270F\uFE0F "),$=r(x,"STRONG",{});var z=l($);k=n(z,"Try it out!"),z.forEach(s),D=n(x," Use the "),y=r(x,"CODE",{});var T=l(y);j=n(T,"Dataset.unique()"),T.forEach(s),C=n(x," function to find the number of unique drugs and conditions in the training and test sets."),x.forEach(s)},m(E,x){p(E,c,x),t(c,q),t(c,$),t($,k),t(c,D),t(c,y),t(y,j),t(c,C)},d(E){E&&s(c)}}}function QE(L){let c,q,$,k,D,y,j,C;return{c(){c=o("p"),q=a("\u{1F64B} An alternative way to add new columns to a dataset is with the "),$=o("code"),k=a("Dataset.add_column()"),D=a(" function. This allows you to provide the column as a Python list or NumPy array and can be handy in situations where "),y=o("code"),j=a("Dataset.map()"),C=a(" is not well suited for your analysis.")},l(E){c=r(E,"P",{});var x=l(c);q=n(x,"\u{1F64B} An alternative way to add new columns to a dataset is with the "),$=r(x,"CODE",{});var z=l($);k=n(z,"Dataset.add_column()"),z.forEach(s),D=n(x," function. This allows you to provide the column as a Python list or NumPy array and can be handy in situations where "),y=r(x,"CODE",{});var T=l(y);j=n(T,"Dataset.map()"),T.forEach(s),C=n(x," is not well suited for your analysis."),x.forEach(s)},m(E,x){p(E,c,x),t(c,q),t(c,$),t($,k),t(c,D),t(c,y),t(y,j),t(c,C)},d(E){E&&s(c)}}}function ZE(L){let c,q,$,k,D,y,j,C,E,x,z;return{c(){c=o("p"),q=a("\u270F\uFE0F "),$=o("strong"),k=a("Try it out!"),D=a(" Use the "),y=o("code"),j=a("Dataset.sort()"),C=a(" function to inspect the reviews with the largest numbers of words. See the "),E=o("a"),x=a("documentation"),z=a(" to see which argument you need to use sort the reviews by length in descending order."),this.h()},l(T){c=r(T,"P",{});var O=l(c);q=n(O,"\u270F\uFE0F "),$=r(O,"STRONG",{});var G=l($);k=n(G,"Try it out!"),G.forEach(s),D=n(O," Use the "),y=r(O,"CODE",{});var N=l(y);j=n(N,"Dataset.sort()"),N.forEach(s),C=n(O," function to inspect the reviews with the largest numbers of words. See the "),E=r(O,"A",{href:!0,rel:!0});var P=l(E);x=n(P,"documentation"),P.forEach(s),z=n(O," to see which argument you need to use sort the reviews by length in descending order."),O.forEach(s),this.h()},h(){f(E,"href","https://huggingface.co/docs/datasets/package_reference/main_classes.html#datasets.Dataset.sort"),f(E,"rel","nofollow")},m(T,O){p(T,c,O),t(c,q),t(c,$),t($,k),t(c,D),t(c,y),t(y,j),t(c,C),t(c,E),t(E,x),t(c,z)},d(T){T&&s(c)}}}function ex(L){let c,q,$,k,D,y,j,C,E,x,z,T,O,G;return{c(){c=o("p"),q=a("\u270F\uFE0F "),$=o("strong"),k=a("Try it out!"),D=a(" Execute the same instruction with and without "),y=o("code"),j=a("batched=True"),C=a(", then try it with a slow tokenizer (add "),E=o("code"),x=a("use_fast=False"),z=a(" in the "),T=o("code"),O=a("AutoTokenizer.from_pretrained()"),G=a(" method) so you can see what numbers you get on your hardware.")},l(N){c=r(N,"P",{});var P=l(c);q=n(P,"\u270F\uFE0F "),$=r(P,"STRONG",{});var ge=l($);k=n(ge,"Try it out!"),ge.forEach(s),D=n(P," Execute the same instruction with and without "),y=r(P,"CODE",{});var M=l(y);j=n(M,"batched=True"),M.forEach(s),C=n(P,", then try it with a slow tokenizer (add "),E=r(P,"CODE",{});var X=l(E);x=n(X,"use_fast=False"),X.forEach(s),z=n(P," in the "),T=r(P,"CODE",{});var ne=l(T);O=n(ne,"AutoTokenizer.from_pretrained()"),ne.forEach(s),G=n(P," method) so you can see what numbers you get on your hardware."),P.forEach(s)},m(N,P){p(N,c,P),t(c,q),t(c,$),t($,k),t(c,D),t(c,y),t(y,j),t(c,C),t(c,E),t(E,x),t(c,z),t(c,T),t(T,O),t(c,G)},d(N){N&&s(c)}}}function tx(L){let c,q,$,k,D;return{c(){c=o("p"),q=a("Using "),$=o("code"),k=a("num_proc"),D=a(" to speed up your processing is usually a great idea, as long as the function you are using is not already doing some kind of multiprocessing of its own.")},l(y){c=r(y,"P",{});var j=l(c);q=n(j,"Using "),$=r(j,"CODE",{});var C=l($);k=n(C,"num_proc"),C.forEach(s),D=n(j," to speed up your processing is usually a great idea, as long as the function you are using is not already doing some kind of multiprocessing of its own."),j.forEach(s)},m(y,j){p(y,c,j),t(c,q),t(c,$),t($,k),t(c,D)},d(y){y&&s(c)}}}function sx(L){let c,q,$,k,D,y,j,C,E,x,z;return{c(){c=o("p"),q=a("\u{1F4A1} In machine learning, an "),$=o("em"),k=a("example"),D=a(" is usually defined as the set of "),y=o("em"),j=a("features"),C=a(" that we feed to the model. In some contexts, these features will be the set of columns in a "),E=o("code"),x=a("Dataset"),z=a(", but in others (like here and for question answering), multiple features can be extracted from a single example and belong to a single column.")},l(T){c=r(T,"P",{});var O=l(c);q=n(O,"\u{1F4A1} In machine learning, an "),$=r(O,"EM",{});var G=l($);k=n(G,"example"),G.forEach(s),D=n(O," is usually defined as the set of "),y=r(O,"EM",{});var N=l(y);j=n(N,"features"),N.forEach(s),C=n(O," that we feed to the model. In some contexts, these features will be the set of columns in a "),E=r(O,"CODE",{});var P=l(E);x=n(P,"Dataset"),P.forEach(s),z=n(O,", but in others (like here and for question answering), multiple features can be extracted from a single example and belong to a single column."),O.forEach(s)},m(T,O){p(T,c,O),t(c,q),t(c,$),t($,k),t(c,D),t(c,y),t(y,j),t(c,C),t(c,E),t(E,x),t(c,z)},d(T){T&&s(c)}}}function ax(L){let c,q,$,k,D,y,j,C,E,x,z,T,O,G,N,P,ge,M,X,ne,se,us,Je,Ve,bt,R;return{c(){c=o("p"),q=a("\u{1F6A8} Under the hood, "),$=o("code"),k=a("Dataset.set_format()"),D=a(" changes the return format for the dataset\u2019s "),y=o("code"),j=a("__getitem__()"),C=a(" dunder method. This means that when we want to create a new object like "),E=o("code"),x=a("train_df"),z=a(" from a "),T=o("code"),O=a("Dataset"),G=a(" in the "),N=o("code"),P=a('"pandas"'),ge=a(" format, we need to slice the whole dataset to obtain a "),M=o("code"),X=a("pandas.DataFrame"),ne=a(". You can verify for yourself that the type of "),se=o("code"),us=a('drug_dataset["train"]'),Je=a(" is "),Ve=o("code"),bt=a("Dataset"),R=a(", irrespective of the output format.")},l(Xe){c=r(Xe,"P",{});var A=l(c);q=n(A,"\u{1F6A8} Under the hood, "),$=r(A,"CODE",{});var Ja=l($);k=n(Ja,"Dataset.set_format()"),Ja.forEach(s),D=n(A," changes the return format for the dataset\u2019s "),y=r(A,"CODE",{});var Va=l(y);j=n(Va,"__getitem__()"),Va.forEach(s),C=n(A," dunder method. This means that when we want to create a new object like "),E=r(A,"CODE",{});var $t=l(E);x=n($t,"train_df"),$t.forEach(s),z=n(A," from a "),T=r(A,"CODE",{});var Xa=l(T);O=n(Xa,"Dataset"),Xa.forEach(s),G=n(A," in the "),N=r(A,"CODE",{});var Ka=l(N);P=n(Ka,'"pandas"'),Ka.forEach(s),ge=n(A," format, we need to slice the whole dataset to obtain a "),M=r(A,"CODE",{});var yt=l(M);X=n(yt,"pandas.DataFrame"),yt.forEach(s),ne=n(A,". You can verify for yourself that the type of "),se=r(A,"CODE",{});var Qa=l(se);us=n(Qa,'drug_dataset["train"]'),Qa.forEach(s),Je=n(A," is "),Ve=r(A,"CODE",{});var Za=l(Ve);bt=n(Za,"Dataset"),Za.forEach(s),R=n(A,", irrespective of the output format."),A.forEach(s)},m(Xe,A){p(Xe,c,A),t(c,q),t(c,$),t($,k),t(c,D),t(c,y),t(y,j),t(c,C),t(c,E),t(E,x),t(c,z),t(c,T),t(T,O),t(c,G),t(c,N),t(N,P),t(c,ge),t(c,M),t(M,X),t(c,ne),t(c,se),t(se,us),t(c,Je),t(c,Ve),t(Ve,bt),t(c,R)},d(Xe){Xe&&s(c)}}}function nx(L){let c,q,$,k,D,y,j,C;return{c(){c=o("p"),q=a("\u270F\uFE0F "),$=o("strong"),k=a("Try it out!"),D=a(" Compute the average rating per drug and store the result in a new "),y=o("code"),j=a("Dataset"),C=a(".")},l(E){c=r(E,"P",{});var x=l(c);q=n(x,"\u270F\uFE0F "),$=r(x,"STRONG",{});var z=l($);k=n(z,"Try it out!"),z.forEach(s),D=n(x," Compute the average rating per drug and store the result in a new "),y=r(x,"CODE",{});var T=l(y);j=n(T,"Dataset"),T.forEach(s),C=n(x,"."),x.forEach(s)},m(E,x){p(E,c,x),t(c,q),t(c,$),t($,k),t(c,D),t(c,y),t(y,j),t(c,C)},d(E){E&&s(c)}}}function ox(L){let c,q,$,k,D,y,j,C,E,x,z,T,O,G,N,P,ge,M,X,ne,se,us,Je,Ve,bt,R,Xe,A,Ja,Va,$t,Xa,Ka,yt,Qa,Za,en,Uc,Bc,wd,be,Yc,ms,Gc,Jc,ws,Vc,Xc,gd,$e,Kc,Wo,Qc,Zc,Uo,ef,tf,_d,gs,vd,oe,sf,Bo,af,nf,Yo,of,rf,Go,lf,df,bd,_s,$d,ye,hf,Jo,pf,cf,Vo,ff,uf,yd,vs,Ed,bs,xd,re,mf,Xo,wf,gf,Ko,_f,vf,Qo,bf,$f,kd,Ee,$s,yf,Zo,Ef,xf,kf,ys,jf,er,Df,Tf,qf,Ke,Cf,tr,Of,zf,sr,Pf,Af,jd,xe,Nf,ar,If,Sf,nr,Hf,Ff,Dd,Es,Td,ke,Lf,or,Mf,Rf,rr,Wf,Uf,qd,xs,Cd,ks,Od,Et,zd,K,Bf,lr,Yf,Gf,ir,Jf,Vf,tn,Xf,Kf,dr,Qf,Zf,Pd,js,Ad,Ds,Nd,Q,eu,hr,tu,su,pr,au,nu,cr,ou,ru,fr,lu,iu,Id,Ts,Sd,je,du,ur,hu,pu,mr,cu,fu,Hd,qs,Fd,Z,uu,wr,mu,wu,Cs,gu,_u,gr,vu,bu,_r,$u,yu,Ld,Os,Md,sn,Eu,Rd,zs,Wd,Ps,Ud,an,xu,Bd,As,Yd,Ns,Gd,De,ku,Is,ju,Du,vr,Tu,qu,Jd,Ss,Vd,Te,Cu,br,Ou,zu,$r,Pu,Au,Xd,Hs,Kd,Fs,Qd,nn,Nu,Zd,Qe,xt,yr,Ls,Iu,Er,Su,eh,on,Hu,th,rn,Fu,sh,Ms,ah,J,Lu,xr,Mu,Ru,kr,Wu,Uu,jr,Bu,Yu,Dr,Gu,Ju,Tr,Vu,Xu,nh,Rs,oh,Ws,rh,qe,Ku,qr,Qu,Zu,Cr,em,tm,lh,Us,ih,Bs,dh,ln,sm,hh,kt,ph,Ce,am,Or,nm,om,zr,rm,lm,ch,Ys,fh,Gs,uh,dn,im,mh,jt,wh,Dt,dm,Pr,hm,pm,gh,Js,_h,Vs,vh,Tt,cm,Ar,fm,um,bh,Xs,$h,qt,mm,Nr,wm,gm,yh,Ze,Ct,Ir,Ks,_m,Qs,vm,Sr,bm,$m,Eh,le,ym,Hr,Em,xm,Fr,km,jm,Lr,Dm,Tm,xh,ee,qm,Mr,Cm,Om,Rr,zm,Pm,Wr,Am,Nm,Ur,Im,Sm,kh,Zs,jh,Oe,Hm,Br,Fm,Lm,Yr,Mm,Rm,Dh,ie,Wm,Gr,Um,Bm,Jr,Ym,Gm,hn,Jm,Vm,Th,ea,qh,de,Xm,pn,Km,Qm,Vr,Zm,ew,Xr,tw,sw,Ch,ta,Oh,Ot,aw,Kr,nw,ow,zh,zt,Ph,cn,rw,Ah,Pt,Qr,et,fn,lw,iw,un,dw,hw,mn,pw,cw,sa,tt,wn,Zr,fw,uw,gn,mw,ww,_n,gw,_w,st,vn,el,vw,bw,bn,$w,yw,$n,Ew,Nh,ze,xw,tl,kw,jw,sl,Dw,Tw,Ih,yn,qw,Sh,_e,al,Cw,Ow,nl,zw,Pw,ol,Aw,Nw,Hh,aa,Fh,En,Iw,Lh,At,rl,at,xn,Sw,Hw,kn,Fw,Lw,jn,Mw,Rw,ve,nt,Dn,ll,Ww,Uw,Tn,Bw,Yw,qn,Gw,Jw,ot,Cn,il,Vw,Xw,On,Kw,Qw,zn,Zw,eg,rt,Nt,dl,tg,sg,hl,ag,ng,Pn,og,rg,An,lg,ig,lt,It,pl,dg,hg,cl,pg,cg,Nn,fg,ug,In,mg,Mh,he,wg,fl,gg,_g,ul,vg,bg,ml,$g,yg,Rh,St,Wh,pe,Eg,wl,xg,kg,gl,jg,Dg,Sn,Tg,qg,Uh,Ht,Bh,Pe,Cg,_l,Og,zg,vl,Pg,Ag,Yh,na,Gh,Ft,Ng,bl,Ig,Sg,Jh,oa,Vh,ra,Xh,Hn,Hg,Kh,la,Qh,ia,Zh,Ae,Fg,$l,Lg,Mg,da,Rg,Wg,ep,te,Ug,yl,Bg,Yg,El,Gg,Jg,xl,Vg,Xg,kl,Kg,Qg,tp,ha,sp,Fn,Zg,ap,pa,np,ca,op,Ne,e_,jl,t_,s_,Dl,a_,n_,rp,fa,lp,Lt,o_,Tl,r_,l_,ip,ua,dp,ma,hp,Ln,i_,pp,Mt,d_,ql,h_,p_,cp,it,Rt,Cl,wa,c_,dt,f_,Ol,u_,m_,zl,w_,g_,fp,ga,up,ce,__,Pl,v_,b_,Al,$_,y_,Nl,E_,x_,mp,_a,wp,Wt,k_,Il,j_,D_,gp,va,_p,Ie,Sl,I,vp,T_,Hl,q_,C_,Fl,O_,z_,Ll,P_,A_,Ml,N_,I_,Rl,S_,H_,Wl,F_,L_,Ul,M_,R_,Bl,W_,U_,ht,S,Yl,B_,Y_,Gl,G_,J_,Jl,V_,X_,Vl,K_,Q_,Xl,Z_,ev,Kl,tv,sv,Ql,av,nv,Zl,ov,rv,ei,lv,iv,H,ti,dv,hv,si,pv,cv,ai,fv,uv,ni,mv,wv,oi,gv,_v,ri,vv,bv,li,$v,yv,ii,Ev,xv,di,kv,jv,F,hi,Dv,Tv,pi,qv,Cv,ci,Ov,zv,fi,Pv,Av,ui,Nv,Iv,mi,Sv,Hv,wi,Fv,Lv,gi,Mv,Rv,_i,Wv,bp,Se,Uv,vi,Bv,Yv,bi,Gv,Jv,$p,ba,yp,Ut,Ep,Bt,Vv,$i,Xv,Kv,xp,$a,kp,He,yi,Fe,jp,Qv,Ei,Zv,e2,xi,t2,s2,ae,pt,ki,a2,n2,ji,o2,r2,Di,l2,i2,ct,Ti,d2,h2,qi,p2,c2,Ci,f2,u2,ft,Oi,m2,w2,zi,g2,_2,Pi,v2,b2,ut,Ai,$2,y2,Ni,E2,x2,Ii,k2,j2,mt,Si,D2,T2,Hi,q2,C2,Fi,O2,Dp,Le,z2,Li,P2,A2,Mi,N2,I2,Tp,ya,qp,Ea,Cp,Yt,Op,fe,S2,Ri,H2,F2,Wi,L2,M2,Ui,R2,W2,zp,xa,Pp,wt,Gt,Bi,ka,U2,Yi,B2,Ap,Mn,Y2,Np,V,G2,Gi,J2,V2,Ji,X2,K2,Vi,Q2,Z2,Xi,e1,t1,Ki,s1,a1,Ip,ja,Sp,Da,Hp,Jt,n1,Rn,o1,r1,Fp,gt,Vt,Qi,Ta,l1,Zi,i1,Lp,qa,Mp,Wn,d1,Rp,Xt,ed,Ca,Un,h1,p1,Bn,c1,f1,_t,Oa,Yn,u1,m1,Gn,td,w1,g1,za,Jn,_1,v1,Vn,sd,b1,$1,Pa,Xn,y1,E1,Kn,ad,x1,Wp,Qn,k1,Up,Aa,Bp,Zn,j1,Yp,Na,Gp,ue,D1,nd,T1,q1,od,C1,O1,rd,z1,P1,Jp,Kt,A1,ld,N1,I1,Vp,Ia,Xp,Sa,Kp,Qt,S1,id,H1,F1,Qp,Ha,Zp,Zt,L1,Fa,M1,R1,ec,La,tc,Ma,sc,es,W1,eo,U1,B1,ac,Ra,nc,to,Y1,oc,ts,Wa,G1,so,J1,V1,X1,vt,K1,dd,Q1,Z1,ao,eb,tb,rc,no,sb,lc;return y=new Ga({}),z=new XE({props:{chapter:5,classNames:"absolute z-10 right-0 top-0",notebooks:[{label:"Google Colab",value:"https://colab.research.google.com/github/huggingface/notebooks/blob/master/course/en/chapter5/section3.ipynb"},{label:"Aws Studio",value:"https://studiolab.sagemaker.aws/import/github/huggingface/notebooks/blob/master/course/en/chapter5/section3.ipynb"}]}}),P=new ab({props:{id:"tqfSFcPMgOI"}}),se=new Ga({}),gs=new b({props:{code:`!wget "https://archive.ics.uci.edu/ml/machine-learning-databases/00462/drugsCom_raw.zip"
!unzip drugsCom_raw.zip`,highlighted:`!wget <span class="hljs-string">&quot;https://archive.ics.uci.edu/ml/machine-learning-databases/00462/drugsCom_raw.zip&quot;</span>
!unzip drugsCom_raw.<span class="hljs-built_in">zip</span>`}}),_s=new b({props:{code:`from datasets import load_dataset

data_files = {"train": "drugsComTrain_raw.tsv", "test": "drugsComTest_raw.tsv"}
# \\t is the tab character in Python
drug_dataset = load_dataset("csv", data_files=data_files, delimiter="\\t")`,highlighted:`<span class="hljs-keyword">from</span> datasets <span class="hljs-keyword">import</span> load_dataset

data_files = {<span class="hljs-string">&quot;train&quot;</span>: <span class="hljs-string">&quot;drugsComTrain_raw.tsv&quot;</span>, <span class="hljs-string">&quot;test&quot;</span>: <span class="hljs-string">&quot;drugsComTest_raw.tsv&quot;</span>}
<span class="hljs-comment"># \\t is the tab character in Python</span>
drug_dataset = load_dataset(<span class="hljs-string">&quot;csv&quot;</span>, data_files=data_files, delimiter=<span class="hljs-string">&quot;\\t&quot;</span>)`}}),vs=new b({props:{code:`drug_sample = drug_dataset["train"].shuffle(seed=42).select(range(1000))
# Peek at the first few examples
drug_sample[:3]`,highlighted:`drug_sample = drug_dataset[<span class="hljs-string">&quot;train&quot;</span>].shuffle(seed=<span class="hljs-number">42</span>).select(<span class="hljs-built_in">range</span>(<span class="hljs-number">1000</span>))
<span class="hljs-comment"># Peek at the first few examples</span>
drug_sample[:<span class="hljs-number">3</span>]`}}),bs=new b({props:{code:`{'Unnamed: 0': [87571, 178045, 80482],
 'drugName': ['Naproxen', 'Duloxetine', 'Mobic'],
 'condition': ['Gout, Acute', 'ibromyalgia', 'Inflammatory Conditions'],
 'review': ['"like the previous person mention, I&#039;m a strong believer of aleve, it works faster for my gout than the prescription meds I take. No more going to the doctor for refills.....Aleve works!"',
  '"I have taken Cymbalta for about a year and a half for fibromyalgia pain. It is great\\r\\nas a pain reducer and an anti-depressant, however, the side effects outweighed \\r\\nany benefit I got from it. I had trouble with restlessness, being tired constantly,\\r\\ndizziness, dry mouth, numbness and tingling in my feet, and horrible sweating. I am\\r\\nbeing weaned off of it now. Went from 60 mg to 30mg and now to 15 mg. I will be\\r\\noff completely in about a week. The fibro pain is coming back, but I would rather deal with it than the side effects."',
  '"I have been taking Mobic for over a year with no side effects other than an elevated blood pressure.  I had severe knee and ankle pain which completely went away after taking Mobic.  I attempted to stop the medication however pain returned after a few days."'],
 'rating': [9.0, 3.0, 10.0],
 'date': ['September 2, 2015', 'November 7, 2011', 'June 5, 2013'],
 'usefulCount': [36, 13, 128]}`,highlighted:`{<span class="hljs-string">&#x27;Unnamed: 0&#x27;</span>: [<span class="hljs-number">87571</span>, <span class="hljs-number">178045</span>, <span class="hljs-number">80482</span>],
 <span class="hljs-string">&#x27;drugName&#x27;</span>: [<span class="hljs-string">&#x27;Naproxen&#x27;</span>, <span class="hljs-string">&#x27;Duloxetine&#x27;</span>, <span class="hljs-string">&#x27;Mobic&#x27;</span>],
 <span class="hljs-string">&#x27;condition&#x27;</span>: [<span class="hljs-string">&#x27;Gout, Acute&#x27;</span>, <span class="hljs-string">&#x27;ibromyalgia&#x27;</span>, <span class="hljs-string">&#x27;Inflammatory Conditions&#x27;</span>],
 <span class="hljs-string">&#x27;review&#x27;</span>: [<span class="hljs-string">&#x27;&quot;like the previous person mention, I&amp;#039;m a strong believer of aleve, it works faster for my gout than the prescription meds I take. No more going to the doctor for refills.....Aleve works!&quot;&#x27;</span>,
  <span class="hljs-string">&#x27;&quot;I have taken Cymbalta for about a year and a half for fibromyalgia pain. It is great\\r\\nas a pain reducer and an anti-depressant, however, the side effects outweighed \\r\\nany benefit I got from it. I had trouble with restlessness, being tired constantly,\\r\\ndizziness, dry mouth, numbness and tingling in my feet, and horrible sweating. I am\\r\\nbeing weaned off of it now. Went from 60 mg to 30mg and now to 15 mg. I will be\\r\\noff completely in about a week. The fibro pain is coming back, but I would rather deal with it than the side effects.&quot;&#x27;</span>,
  <span class="hljs-string">&#x27;&quot;I have been taking Mobic for over a year with no side effects other than an elevated blood pressure.  I had severe knee and ankle pain which completely went away after taking Mobic.  I attempted to stop the medication however pain returned after a few days.&quot;&#x27;</span>],
 <span class="hljs-string">&#x27;rating&#x27;</span>: [<span class="hljs-number">9.0</span>, <span class="hljs-number">3.0</span>, <span class="hljs-number">10.0</span>],
 <span class="hljs-string">&#x27;date&#x27;</span>: [<span class="hljs-string">&#x27;September 2, 2015&#x27;</span>, <span class="hljs-string">&#x27;November 7, 2011&#x27;</span>, <span class="hljs-string">&#x27;June 5, 2013&#x27;</span>],
 <span class="hljs-string">&#x27;usefulCount&#x27;</span>: [<span class="hljs-number">36</span>, <span class="hljs-number">13</span>, <span class="hljs-number">128</span>]}`}}),Es=new b({props:{code:`for split in drug_dataset.keys():
    assert len(drug_dataset[split]) == len(drug_dataset[split].unique("Unnamed: 0"))`,highlighted:`<span class="hljs-keyword">for</span> split <span class="hljs-keyword">in</span> drug_dataset.keys():
    <span class="hljs-keyword">assert</span> <span class="hljs-built_in">len</span>(drug_dataset[split]) == <span class="hljs-built_in">len</span>(drug_dataset[split].unique(<span class="hljs-string">&quot;Unnamed: 0&quot;</span>))`}}),xs=new b({props:{code:`drug_dataset = drug_dataset.rename_column(
    original_column_name="Unnamed: 0", new_column_name="patient_id"
)
drug_dataset`,highlighted:`drug_dataset = drug_dataset.rename_column(
    original_column_name=<span class="hljs-string">&quot;Unnamed: 0&quot;</span>, new_column_name=<span class="hljs-string">&quot;patient_id&quot;</span>
)
drug_dataset`}}),ks=new b({props:{code:`DatasetDict({
    train: Dataset({
        features: ['patient_id', 'drugName', 'condition', 'review', 'rating', 'date', 'usefulCount'],
        num_rows: 161297
    })
    test: Dataset({
        features: ['patient_id', 'drugName', 'condition', 'review', 'rating', 'date', 'usefulCount'],
        num_rows: 53766
    })
})`,highlighted:`DatasetDict({
    train: Dataset({
        features: [<span class="hljs-string">&#x27;patient_id&#x27;</span>, <span class="hljs-string">&#x27;drugName&#x27;</span>, <span class="hljs-string">&#x27;condition&#x27;</span>, <span class="hljs-string">&#x27;review&#x27;</span>, <span class="hljs-string">&#x27;rating&#x27;</span>, <span class="hljs-string">&#x27;date&#x27;</span>, <span class="hljs-string">&#x27;usefulCount&#x27;</span>],
        num_rows: <span class="hljs-number">161297</span>
    })
    test: Dataset({
        features: [<span class="hljs-string">&#x27;patient_id&#x27;</span>, <span class="hljs-string">&#x27;drugName&#x27;</span>, <span class="hljs-string">&#x27;condition&#x27;</span>, <span class="hljs-string">&#x27;review&#x27;</span>, <span class="hljs-string">&#x27;rating&#x27;</span>, <span class="hljs-string">&#x27;date&#x27;</span>, <span class="hljs-string">&#x27;usefulCount&#x27;</span>],
        num_rows: <span class="hljs-number">53766</span>
    })
})`}}),Et=new fs({props:{$$slots:{default:[KE]},$$scope:{ctx:L}}}),js=new b({props:{code:`def lowercase_condition(example):
    return {"condition": example["condition"].lower()}


drug_dataset.map(lowercase_condition)`,highlighted:`<span class="hljs-keyword">def</span> <span class="hljs-title function_">lowercase_condition</span>(<span class="hljs-params">example</span>):
    <span class="hljs-keyword">return</span> {<span class="hljs-string">&quot;condition&quot;</span>: example[<span class="hljs-string">&quot;condition&quot;</span>].lower()}


drug_dataset.<span class="hljs-built_in">map</span>(lowercase_condition)`}}),Ds=new b({props:{code:"AttributeError: 'NoneType' object has no attribute 'lower'",highlighted:'AttributeError: <span class="hljs-string">&#x27;NoneType&#x27;</span> <span class="hljs-built_in">object</span> has no attribute <span class="hljs-string">&#x27;lower&#x27;</span>'}}),Ts=new b({props:{code:`def filter_nones(x):
    return x["condition"] is not None`,highlighted:`<span class="hljs-keyword">def</span> <span class="hljs-title function_">filter_nones</span>(<span class="hljs-params">x</span>):
    <span class="hljs-keyword">return</span> x[<span class="hljs-string">&quot;condition&quot;</span>] <span class="hljs-keyword">is</span> <span class="hljs-keyword">not</span> <span class="hljs-literal">None</span>`}}),qs=new b({props:{code:"lambda <arguments> : <expression>",highlighted:'lambda <span class="hljs-tag">&lt;<span class="hljs-name">arguments</span>&gt;</span> : <span class="hljs-tag">&lt;<span class="hljs-name">expression</span>&gt;</span>'}}),Os=new b({props:{code:"lambda x : x * x",highlighted:'lambda <span class="hljs-keyword">x</span> : <span class="hljs-keyword">x</span> * <span class="hljs-keyword">x</span>'}}),zs=new b({props:{code:"(lambda x: x * x)(3)",highlighted:'(<span class="hljs-keyword">lambda</span> x: x * x)(<span class="hljs-number">3</span>)'}}),Ps=new b({props:{code:"9",highlighted:'<span class="hljs-number">9</span>'}}),As=new b({props:{code:"(lambda base, height: 0.5 * base * height)(4, 8)",highlighted:'(<span class="hljs-keyword">lambda</span> base, height: <span class="hljs-number">0.5</span> * base * height)(<span class="hljs-number">4</span>, <span class="hljs-number">8</span>)'}}),Ns=new b({props:{code:"16.0",highlighted:'<span class="hljs-number">16.0</span>'}}),Ss=new b({props:{code:'drug_dataset = drug_dataset.filter(lambda x: x["condition"] is not None)',highlighted:'drug_dataset = drug_dataset.<span class="hljs-built_in">filter</span>(<span class="hljs-keyword">lambda</span> x: x[<span class="hljs-string">&quot;condition&quot;</span>] <span class="hljs-keyword">is</span> <span class="hljs-keyword">not</span> <span class="hljs-literal">None</span>)'}}),Hs=new b({props:{code:`drug_dataset = drug_dataset.map(lowercase_condition)
# Check that lowercasing worked
drug_dataset["train"]["condition"][:3]`,highlighted:`drug_dataset = drug_dataset.<span class="hljs-built_in">map</span>(lowercase_condition)
<span class="hljs-comment"># Check that lowercasing worked</span>
drug_dataset[<span class="hljs-string">&quot;train&quot;</span>][<span class="hljs-string">&quot;condition&quot;</span>][:<span class="hljs-number">3</span>]`}}),Fs=new b({props:{code:"['left ventricular dysfunction', 'adhd', 'birth control']",highlighted:'[<span class="hljs-string">&#x27;left ventricular dysfunction&#x27;</span>, <span class="hljs-string">&#x27;adhd&#x27;</span>, <span class="hljs-string">&#x27;birth control&#x27;</span>]'}}),Ls=new Ga({}),Ms=new b({props:{code:`def compute_review_length(example):
    return {"review_length": len(example["review"].split())}`,highlighted:`<span class="hljs-keyword">def</span> <span class="hljs-title function_">compute_review_length</span>(<span class="hljs-params">example</span>):
    <span class="hljs-keyword">return</span> {<span class="hljs-string">&quot;review_length&quot;</span>: <span class="hljs-built_in">len</span>(example[<span class="hljs-string">&quot;review&quot;</span>].split())}`}}),Rs=new b({props:{code:`drug_dataset = drug_dataset.map(compute_review_length)
# Inspect the first training example
drug_dataset["train"][0]`,highlighted:`drug_dataset = drug_dataset.<span class="hljs-built_in">map</span>(compute_review_length)
<span class="hljs-comment"># Inspect the first training example</span>
drug_dataset[<span class="hljs-string">&quot;train&quot;</span>][<span class="hljs-number">0</span>]`}}),Ws=new b({props:{code:`{'patient_id': 206461,
 'drugName': 'Valsartan',
 'condition': 'left ventricular dysfunction',
 'review': '"It has no side effect, I take it in combination of Bystolic 5 Mg and Fish Oil"',
 'rating': 9.0,
 'date': 'May 20, 2012',
 'usefulCount': 27,
 'review_length': 17}`,highlighted:`{<span class="hljs-string">&#x27;patient_id&#x27;</span>: <span class="hljs-number">206461</span>,
 <span class="hljs-string">&#x27;drugName&#x27;</span>: <span class="hljs-string">&#x27;Valsartan&#x27;</span>,
 <span class="hljs-string">&#x27;condition&#x27;</span>: <span class="hljs-string">&#x27;left ventricular dysfunction&#x27;</span>,
 <span class="hljs-string">&#x27;review&#x27;</span>: <span class="hljs-string">&#x27;&quot;It has no side effect, I take it in combination of Bystolic 5 Mg and Fish Oil&quot;&#x27;</span>,
 <span class="hljs-string">&#x27;rating&#x27;</span>: <span class="hljs-number">9.0</span>,
 <span class="hljs-string">&#x27;date&#x27;</span>: <span class="hljs-string">&#x27;May 20, 2012&#x27;</span>,
 <span class="hljs-string">&#x27;usefulCount&#x27;</span>: <span class="hljs-number">27</span>,
 <span class="hljs-string">&#x27;review_length&#x27;</span>: <span class="hljs-number">17</span>}`}}),Us=new b({props:{code:'drug_dataset["train"].sort("review_length")[:3]',highlighted:'drug_dataset[<span class="hljs-string">&quot;train&quot;</span>].sort(<span class="hljs-string">&quot;review_length&quot;</span>)[:<span class="hljs-number">3</span>]'}}),Bs=new b({props:{code:`{'patient_id': [103488, 23627, 20558],
 'drugName': ['Loestrin 21 1 / 20', 'Chlorzoxazone', 'Nucynta'],
 'condition': ['birth control', 'muscle spasm', 'pain'],
 'review': ['"Excellent."', '"useless"', '"ok"'],
 'rating': [10.0, 1.0, 6.0],
 'date': ['November 4, 2008', 'March 24, 2017', 'August 20, 2016'],
 'usefulCount': [5, 2, 10],
 'review_length': [1, 1, 1]}`,highlighted:`{<span class="hljs-string">&#x27;patient_id&#x27;</span>: [<span class="hljs-number">103488</span>, <span class="hljs-number">23627</span>, <span class="hljs-number">20558</span>],
 <span class="hljs-string">&#x27;drugName&#x27;</span>: [<span class="hljs-string">&#x27;Loestrin 21 1 / 20&#x27;</span>, <span class="hljs-string">&#x27;Chlorzoxazone&#x27;</span>, <span class="hljs-string">&#x27;Nucynta&#x27;</span>],
 <span class="hljs-string">&#x27;condition&#x27;</span>: [<span class="hljs-string">&#x27;birth control&#x27;</span>, <span class="hljs-string">&#x27;muscle spasm&#x27;</span>, <span class="hljs-string">&#x27;pain&#x27;</span>],
 <span class="hljs-string">&#x27;review&#x27;</span>: [<span class="hljs-string">&#x27;&quot;Excellent.&quot;&#x27;</span>, <span class="hljs-string">&#x27;&quot;useless&quot;&#x27;</span>, <span class="hljs-string">&#x27;&quot;ok&quot;&#x27;</span>],
 <span class="hljs-string">&#x27;rating&#x27;</span>: [<span class="hljs-number">10.0</span>, <span class="hljs-number">1.0</span>, <span class="hljs-number">6.0</span>],
 <span class="hljs-string">&#x27;date&#x27;</span>: [<span class="hljs-string">&#x27;November 4, 2008&#x27;</span>, <span class="hljs-string">&#x27;March 24, 2017&#x27;</span>, <span class="hljs-string">&#x27;August 20, 2016&#x27;</span>],
 <span class="hljs-string">&#x27;usefulCount&#x27;</span>: [<span class="hljs-number">5</span>, <span class="hljs-number">2</span>, <span class="hljs-number">10</span>],
 <span class="hljs-string">&#x27;review_length&#x27;</span>: [<span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>]}`}}),kt=new fs({props:{$$slots:{default:[QE]},$$scope:{ctx:L}}}),Ys=new b({props:{code:`drug_dataset = drug_dataset.filter(lambda x: x["review_length"] > 30)
print(drug_dataset.num_rows)`,highlighted:`drug_dataset = drug_dataset.<span class="hljs-built_in">filter</span>(<span class="hljs-keyword">lambda</span> x: x[<span class="hljs-string">&quot;review_length&quot;</span>] &gt; <span class="hljs-number">30</span>)
<span class="hljs-built_in">print</span>(drug_dataset.num_rows)`}}),Gs=new b({props:{code:"{'train': 138514, 'test': 46108}",highlighted:'{<span class="hljs-string">&#x27;train&#x27;</span>: <span class="hljs-number">138514</span>, <span class="hljs-string">&#x27;test&#x27;</span>: <span class="hljs-number">46108</span>}'}}),jt=new fs({props:{$$slots:{default:[ZE]},$$scope:{ctx:L}}}),Js=new b({props:{code:`import html

text = "I&#039;m a transformer called BERT"
html.unescape(text)`,highlighted:`<span class="hljs-keyword">import</span> html

text = <span class="hljs-string">&quot;I&amp;#039;m a transformer called BERT&quot;</span>
html.unescape(text)`}}),Vs=new b({props:{code:`"I'm a transformer called BERT"`,highlighted:'<span class="hljs-string">&quot;I&#x27;m a transformer called BERT&quot;</span>'}}),Xs=new b({props:{code:'drug_dataset = drug_dataset.map(lambda x: {"review": html.unescape(x["review"])})',highlighted:'drug_dataset = drug_dataset.<span class="hljs-built_in">map</span>(<span class="hljs-keyword">lambda</span> x: {<span class="hljs-string">&quot;review&quot;</span>: html.unescape(x[<span class="hljs-string">&quot;review&quot;</span>])})'}}),Ks=new Ga({}),Zs=new b({props:{code:`new_drug_dataset = drug_dataset.map(
    lambda x: {"review": [html.unescape(o) for o in x["review"]]}, batched=True
)`,highlighted:`new_drug_dataset = drug_dataset.<span class="hljs-built_in">map</span>(
    <span class="hljs-keyword">lambda</span> x: {<span class="hljs-string">&quot;review&quot;</span>: [html.unescape(o) <span class="hljs-keyword">for</span> o <span class="hljs-keyword">in</span> x[<span class="hljs-string">&quot;review&quot;</span>]]}, batched=<span class="hljs-literal">True</span>
)`}}),ea=new b({props:{code:`from transformers import AutoTokenizer

tokenizer = AutoTokenizer.from_pretrained("bert-base-cased")


def tokenize_function(examples):
    return tokenizer(examples["review"], truncation=True)`,highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoTokenizer

tokenizer = AutoTokenizer.from_pretrained(<span class="hljs-string">&quot;bert-base-cased&quot;</span>)


<span class="hljs-keyword">def</span> <span class="hljs-title function_">tokenize_function</span>(<span class="hljs-params">examples</span>):
    <span class="hljs-keyword">return</span> tokenizer(examples[<span class="hljs-string">&quot;review&quot;</span>], truncation=<span class="hljs-literal">True</span>)`}}),ta=new b({props:{code:"%time tokenized_dataset = drug_dataset.map(tokenize_function, batched=True)",highlighted:'%time tokenized_dataset = drug_dataset.<span class="hljs-built_in">map</span>(tokenize_function, batched=<span class="hljs-literal">True</span>)'}}),zt=new fs({props:{$$slots:{default:[ex]},$$scope:{ctx:L}}}),aa=new b({props:{code:`slow_tokenizer = AutoTokenizer.from_pretrained("bert-base-cased", use_fast=False)


def slow_tokenize_function(examples):
    return slow_tokenizer(examples["review"], truncation=True)


tokenized_dataset = drug_dataset.map(slow_tokenize_function, batched=True, num_proc=8)`,highlighted:`slow_tokenizer = AutoTokenizer.from_pretrained(<span class="hljs-string">&quot;bert-base-cased&quot;</span>, use_fast=<span class="hljs-literal">False</span>)


<span class="hljs-keyword">def</span> <span class="hljs-title function_">slow_tokenize_function</span>(<span class="hljs-params">examples</span>):
    <span class="hljs-keyword">return</span> slow_tokenizer(examples[<span class="hljs-string">&quot;review&quot;</span>], truncation=<span class="hljs-literal">True</span>)


tokenized_dataset = drug_dataset.<span class="hljs-built_in">map</span>(slow_tokenize_function, batched=<span class="hljs-literal">True</span>, num_proc=<span class="hljs-number">8</span>)`}}),St=new fs({props:{$$slots:{default:[tx]},$$scope:{ctx:L}}}),Ht=new fs({props:{$$slots:{default:[sx]},$$scope:{ctx:L}}}),na=new b({props:{code:`def tokenize_and_split(examples):
    return tokenizer(
        examples["review"],
        truncation=True,
        max_length=128,
        return_overflowing_tokens=True,
    )`,highlighted:`<span class="hljs-keyword">def</span> <span class="hljs-title function_">tokenize_and_split</span>(<span class="hljs-params">examples</span>):
    <span class="hljs-keyword">return</span> tokenizer(
        examples[<span class="hljs-string">&quot;review&quot;</span>],
        truncation=<span class="hljs-literal">True</span>,
        max_length=<span class="hljs-number">128</span>,
        return_overflowing_tokens=<span class="hljs-literal">True</span>,
    )`}}),oa=new b({props:{code:`result = tokenize_and_split(drug_dataset["train"][0])
[len(inp) for inp in result["input_ids"]]`,highlighted:`result = tokenize_and_split(drug_dataset[<span class="hljs-string">&quot;train&quot;</span>][<span class="hljs-number">0</span>])
[<span class="hljs-built_in">len</span>(inp) <span class="hljs-keyword">for</span> inp <span class="hljs-keyword">in</span> result[<span class="hljs-string">&quot;input_ids&quot;</span>]]`}}),ra=new b({props:{code:"[128, 49]",highlighted:'[<span class="hljs-number">128</span>, <span class="hljs-number">49</span>]'}}),la=new b({props:{code:"tokenized_dataset = drug_dataset.map(tokenize_and_split, batched=True)",highlighted:'tokenized_dataset = drug_dataset.<span class="hljs-built_in">map</span>(tokenize_and_split, batched=<span class="hljs-literal">True</span>)'}}),ia=new b({props:{code:"ArrowInvalid: Column 1 named condition expected length 1463 but got length 1000",highlighted:'ArrowInvalid: Column <span class="hljs-number">1</span> named condition expected length <span class="hljs-number">1463</span> but got length <span class="hljs-number">1000</span>'}}),ha=new b({props:{code:`tokenized_dataset = drug_dataset.map(
    tokenize_and_split, batched=True, remove_columns=drug_dataset["train"].column_names
)`,highlighted:`tokenized_dataset = drug_dataset.<span class="hljs-built_in">map</span>(
    tokenize_and_split, batched=<span class="hljs-literal">True</span>, remove_columns=drug_dataset[<span class="hljs-string">&quot;train&quot;</span>].column_names
)`}}),pa=new b({props:{code:'len(tokenized_dataset["train"]), len(drug_dataset["train"])',highlighted:'<span class="hljs-built_in">len</span>(tokenized_dataset[<span class="hljs-string">&quot;train&quot;</span>]), <span class="hljs-built_in">len</span>(drug_dataset[<span class="hljs-string">&quot;train&quot;</span>])'}}),ca=new b({props:{code:"(206772, 138514)",highlighted:'(<span class="hljs-number">206772</span>, <span class="hljs-number">138514</span>)'}}),fa=new b({props:{code:`def tokenize_and_split(examples):
    result = tokenizer(
        examples["review"],
        truncation=True,
        max_length=128,
        return_overflowing_tokens=True,
    )
    # Extract mapping between new and old indices
    sample_map = result.pop("overflow_to_sample_mapping")
    for key, values in examples.items():
        result[key] = [values[i] for i in sample_map]
    return result`,highlighted:`<span class="hljs-keyword">def</span> <span class="hljs-title function_">tokenize_and_split</span>(<span class="hljs-params">examples</span>):
    result = tokenizer(
        examples[<span class="hljs-string">&quot;review&quot;</span>],
        truncation=<span class="hljs-literal">True</span>,
        max_length=<span class="hljs-number">128</span>,
        return_overflowing_tokens=<span class="hljs-literal">True</span>,
    )
    <span class="hljs-comment"># Extract mapping between new and old indices</span>
    sample_map = result.pop(<span class="hljs-string">&quot;overflow_to_sample_mapping&quot;</span>)
    <span class="hljs-keyword">for</span> key, values <span class="hljs-keyword">in</span> examples.items():
        result[key] = [values[i] <span class="hljs-keyword">for</span> i <span class="hljs-keyword">in</span> sample_map]
    <span class="hljs-keyword">return</span> result`}}),ua=new b({props:{code:`tokenized_dataset = drug_dataset.map(tokenize_and_split, batched=True)
tokenized_dataset`,highlighted:`tokenized_dataset = drug_dataset.<span class="hljs-built_in">map</span>(tokenize_and_split, batched=<span class="hljs-literal">True</span>)
tokenized_dataset`}}),ma=new b({props:{code:`DatasetDict({
    train: Dataset({
        features: ['attention_mask', 'condition', 'date', 'drugName', 'input_ids', 'patient_id', 'rating', 'review', 'review_length', 'token_type_ids', 'usefulCount'],
        num_rows: 206772
    })
    test: Dataset({
        features: ['attention_mask', 'condition', 'date', 'drugName', 'input_ids', 'patient_id', 'rating', 'review', 'review_length', 'token_type_ids', 'usefulCount'],
        num_rows: 68876
    })
})`,highlighted:`DatasetDict({
    train: Dataset({
        features: [<span class="hljs-string">&#x27;attention_mask&#x27;</span>, <span class="hljs-string">&#x27;condition&#x27;</span>, <span class="hljs-string">&#x27;date&#x27;</span>, <span class="hljs-string">&#x27;drugName&#x27;</span>, <span class="hljs-string">&#x27;input_ids&#x27;</span>, <span class="hljs-string">&#x27;patient_id&#x27;</span>, <span class="hljs-string">&#x27;rating&#x27;</span>, <span class="hljs-string">&#x27;review&#x27;</span>, <span class="hljs-string">&#x27;review_length&#x27;</span>, <span class="hljs-string">&#x27;token_type_ids&#x27;</span>, <span class="hljs-string">&#x27;usefulCount&#x27;</span>],
        num_rows: <span class="hljs-number">206772</span>
    })
    test: Dataset({
        features: [<span class="hljs-string">&#x27;attention_mask&#x27;</span>, <span class="hljs-string">&#x27;condition&#x27;</span>, <span class="hljs-string">&#x27;date&#x27;</span>, <span class="hljs-string">&#x27;drugName&#x27;</span>, <span class="hljs-string">&#x27;input_ids&#x27;</span>, <span class="hljs-string">&#x27;patient_id&#x27;</span>, <span class="hljs-string">&#x27;rating&#x27;</span>, <span class="hljs-string">&#x27;review&#x27;</span>, <span class="hljs-string">&#x27;review_length&#x27;</span>, <span class="hljs-string">&#x27;token_type_ids&#x27;</span>, <span class="hljs-string">&#x27;usefulCount&#x27;</span>],
        num_rows: <span class="hljs-number">68876</span>
    })
})`}}),wa=new Ga({}),ga=new ab({props:{id:"tfcY1067A5Q"}}),_a=new b({props:{code:'drug_dataset.set_format("pandas")',highlighted:'drug_dataset.set_format(<span class="hljs-string">&quot;pandas&quot;</span>)'}}),va=new b({props:{code:'drug_dataset["train"][:3]',highlighted:'drug_dataset[<span class="hljs-string">&quot;train&quot;</span>][:<span class="hljs-number">3</span>]'}}),ba=new b({props:{code:'train_df = drug_dataset["train"][:]',highlighted:'train_df = drug_dataset[<span class="hljs-string">&quot;train&quot;</span>][:]'}}),Ut=new fs({props:{$$slots:{default:[ax]},$$scope:{ctx:L}}}),$a=new b({props:{code:`frequencies = (
    train_df["condition"]
    .value_counts()
    .to_frame()
    .reset_index()
    .rename(columns={"index": "condition", "condition": "frequency"})
)
frequencies.head()`,highlighted:`frequencies = (
    train_df[<span class="hljs-string">&quot;condition&quot;</span>]
    .value_counts()
    .to_frame()
    .reset_index()
    .rename(columns={<span class="hljs-string">&quot;index&quot;</span>: <span class="hljs-string">&quot;condition&quot;</span>, <span class="hljs-string">&quot;condition&quot;</span>: <span class="hljs-string">&quot;frequency&quot;</span>})
)
frequencies.head()`}}),ya=new b({props:{code:`from datasets import Dataset

freq_dataset = Dataset.from_pandas(frequencies)
freq_dataset`,highlighted:`<span class="hljs-keyword">from</span> datasets <span class="hljs-keyword">import</span> Dataset

freq_dataset = Dataset.from_pandas(frequencies)
freq_dataset`}}),Ea=new b({props:{code:`Dataset({
    features: ['condition', 'frequency'],
    num_rows: 819
})`,highlighted:`Dataset({
    features: [<span class="hljs-string">&#x27;condition&#x27;</span>, <span class="hljs-string">&#x27;frequency&#x27;</span>],
    num_rows: <span class="hljs-number">819</span>
})`}}),Yt=new fs({props:{$$slots:{default:[nx]},$$scope:{ctx:L}}}),xa=new b({props:{code:"drug_dataset.reset_format()",highlighted:"drug_dataset.reset_format()"}}),ka=new Ga({}),ja=new b({props:{code:`drug_dataset_clean = drug_dataset["train"].train_test_split(train_size=0.8, seed=42)
# Rename the default "test" split to "validation"
drug_dataset_clean["validation"] = drug_dataset_clean.pop("test")
# Add the "test" set to our \`DatasetDict\`
drug_dataset_clean["test"] = drug_dataset["test"]
drug_dataset_clean`,highlighted:`drug_dataset_clean = drug_dataset[<span class="hljs-string">&quot;train&quot;</span>].train_test_split(train_size=<span class="hljs-number">0.8</span>, seed=<span class="hljs-number">42</span>)
<span class="hljs-comment"># Rename the default &quot;test&quot; split to &quot;validation&quot;</span>
drug_dataset_clean[<span class="hljs-string">&quot;validation&quot;</span>] = drug_dataset_clean.pop(<span class="hljs-string">&quot;test&quot;</span>)
<span class="hljs-comment"># Add the &quot;test&quot; set to our \`DatasetDict\`</span>
drug_dataset_clean[<span class="hljs-string">&quot;test&quot;</span>] = drug_dataset[<span class="hljs-string">&quot;test&quot;</span>]
drug_dataset_clean`}}),Da=new b({props:{code:`DatasetDict({
    train: Dataset({
        features: ['patient_id', 'drugName', 'condition', 'review', 'rating', 'date', 'usefulCount', 'review_length', 'review_clean'],
        num_rows: 110811
    })
    validation: Dataset({
        features: ['patient_id', 'drugName', 'condition', 'review', 'rating', 'date', 'usefulCount', 'review_length', 'review_clean'],
        num_rows: 27703
    })
    test: Dataset({
        features: ['patient_id', 'drugName', 'condition', 'review', 'rating', 'date', 'usefulCount', 'review_length', 'review_clean'],
        num_rows: 46108
    })
})`,highlighted:`DatasetDict({
    train: Dataset({
        features: [<span class="hljs-string">&#x27;patient_id&#x27;</span>, <span class="hljs-string">&#x27;drugName&#x27;</span>, <span class="hljs-string">&#x27;condition&#x27;</span>, <span class="hljs-string">&#x27;review&#x27;</span>, <span class="hljs-string">&#x27;rating&#x27;</span>, <span class="hljs-string">&#x27;date&#x27;</span>, <span class="hljs-string">&#x27;usefulCount&#x27;</span>, <span class="hljs-string">&#x27;review_length&#x27;</span>, <span class="hljs-string">&#x27;review_clean&#x27;</span>],
        num_rows: <span class="hljs-number">110811</span>
    })
    validation: Dataset({
        features: [<span class="hljs-string">&#x27;patient_id&#x27;</span>, <span class="hljs-string">&#x27;drugName&#x27;</span>, <span class="hljs-string">&#x27;condition&#x27;</span>, <span class="hljs-string">&#x27;review&#x27;</span>, <span class="hljs-string">&#x27;rating&#x27;</span>, <span class="hljs-string">&#x27;date&#x27;</span>, <span class="hljs-string">&#x27;usefulCount&#x27;</span>, <span class="hljs-string">&#x27;review_length&#x27;</span>, <span class="hljs-string">&#x27;review_clean&#x27;</span>],
        num_rows: <span class="hljs-number">27703</span>
    })
    test: Dataset({
        features: [<span class="hljs-string">&#x27;patient_id&#x27;</span>, <span class="hljs-string">&#x27;drugName&#x27;</span>, <span class="hljs-string">&#x27;condition&#x27;</span>, <span class="hljs-string">&#x27;review&#x27;</span>, <span class="hljs-string">&#x27;rating&#x27;</span>, <span class="hljs-string">&#x27;date&#x27;</span>, <span class="hljs-string">&#x27;usefulCount&#x27;</span>, <span class="hljs-string">&#x27;review_length&#x27;</span>, <span class="hljs-string">&#x27;review_clean&#x27;</span>],
        num_rows: <span class="hljs-number">46108</span>
    })
})`}}),Ta=new Ga({}),qa=new ab({props:{id:"blF9uxYcKHo"}}),Aa=new b({props:{code:'drug_dataset_clean.save_to_disk("drug-reviews")',highlighted:'drug_dataset_clean.save_to_disk(<span class="hljs-string">&quot;drug-reviews&quot;</span>)'}}),Na=new b({props:{code:`drug-reviews/
\u251C\u2500\u2500 dataset_dict.json
\u251C\u2500\u2500 test
\u2502   \u251C\u2500\u2500 dataset.arrow
\u2502   \u251C\u2500\u2500 dataset_info.json
\u2502   \u2514\u2500\u2500 state.json
\u251C\u2500\u2500 train
\u2502   \u251C\u2500\u2500 dataset.arrow
\u2502   \u251C\u2500\u2500 dataset_info.json
\u2502   \u251C\u2500\u2500 indices.arrow
\u2502   \u2514\u2500\u2500 state.json
\u2514\u2500\u2500 validation
    \u251C\u2500\u2500 dataset.arrow
    \u251C\u2500\u2500 dataset_info.json
    \u251C\u2500\u2500 indices.arrow
    \u2514\u2500\u2500 state.json`,highlighted:`drug-reviews/
\u251C\u2500\u2500 dataset_dict.json
\u251C\u2500\u2500 test
\u2502   \u251C\u2500\u2500 dataset.arrow
\u2502   \u251C\u2500\u2500 dataset_info.json
\u2502   \u2514\u2500\u2500 <span class="hljs-keyword">state</span>.json
\u251C\u2500\u2500 train
\u2502   \u251C\u2500\u2500 dataset.arrow
\u2502   \u251C\u2500\u2500 dataset_info.json
\u2502   \u251C\u2500\u2500 indices.arrow
\u2502   \u2514\u2500\u2500 <span class="hljs-keyword">state</span>.json
\u2514\u2500\u2500 validation
    \u251C\u2500\u2500 dataset.arrow
    \u251C\u2500\u2500 dataset_info.json
    \u251C\u2500\u2500 indices.arrow
    \u2514\u2500\u2500 <span class="hljs-keyword">state</span>.json`}}),Ia=new b({props:{code:`from datasets import load_from_disk

drug_dataset_reloaded = load_from_disk("drug-reviews")
drug_dataset_reloaded`,highlighted:`<span class="hljs-keyword">from</span> datasets <span class="hljs-keyword">import</span> load_from_disk

drug_dataset_reloaded = load_from_disk(<span class="hljs-string">&quot;drug-reviews&quot;</span>)
drug_dataset_reloaded`}}),Sa=new b({props:{code:`DatasetDict({
    train: Dataset({
        features: ['patient_id', 'drugName', 'condition', 'review', 'rating', 'date', 'usefulCount', 'review_length'],
        num_rows: 110811
    })
    validation: Dataset({
        features: ['patient_id', 'drugName', 'condition', 'review', 'rating', 'date', 'usefulCount', 'review_length'],
        num_rows: 27703
    })
    test: Dataset({
        features: ['patient_id', 'drugName', 'condition', 'review', 'rating', 'date', 'usefulCount', 'review_length'],
        num_rows: 46108
    })
})`,highlighted:`DatasetDict({
    train: Dataset({
        features: [<span class="hljs-string">&#x27;patient_id&#x27;</span>, <span class="hljs-string">&#x27;drugName&#x27;</span>, <span class="hljs-string">&#x27;condition&#x27;</span>, <span class="hljs-string">&#x27;review&#x27;</span>, <span class="hljs-string">&#x27;rating&#x27;</span>, <span class="hljs-string">&#x27;date&#x27;</span>, <span class="hljs-string">&#x27;usefulCount&#x27;</span>, <span class="hljs-string">&#x27;review_length&#x27;</span>],
        num_rows: <span class="hljs-number">110811</span>
    })
    validation: Dataset({
        features: [<span class="hljs-string">&#x27;patient_id&#x27;</span>, <span class="hljs-string">&#x27;drugName&#x27;</span>, <span class="hljs-string">&#x27;condition&#x27;</span>, <span class="hljs-string">&#x27;review&#x27;</span>, <span class="hljs-string">&#x27;rating&#x27;</span>, <span class="hljs-string">&#x27;date&#x27;</span>, <span class="hljs-string">&#x27;usefulCount&#x27;</span>, <span class="hljs-string">&#x27;review_length&#x27;</span>],
        num_rows: <span class="hljs-number">27703</span>
    })
    test: Dataset({
        features: [<span class="hljs-string">&#x27;patient_id&#x27;</span>, <span class="hljs-string">&#x27;drugName&#x27;</span>, <span class="hljs-string">&#x27;condition&#x27;</span>, <span class="hljs-string">&#x27;review&#x27;</span>, <span class="hljs-string">&#x27;rating&#x27;</span>, <span class="hljs-string">&#x27;date&#x27;</span>, <span class="hljs-string">&#x27;usefulCount&#x27;</span>, <span class="hljs-string">&#x27;review_length&#x27;</span>],
        num_rows: <span class="hljs-number">46108</span>
    })
})`}}),Ha=new b({props:{code:`for split, dataset in drug_dataset_clean.items():
    dataset.to_json(f"drug-reviews-{split}.jsonl")`,highlighted:`<span class="hljs-keyword">for</span> split, dataset <span class="hljs-keyword">in</span> drug_dataset_clean.items():
    dataset.to_json(<span class="hljs-string">f&quot;drug-reviews-<span class="hljs-subst">{split}</span>.jsonl&quot;</span>)`}}),La=new b({props:{code:"!head -n 1 drug-reviews-train.jsonl",highlighted:'!head -n <span class="hljs-number">1</span> drug-reviews-train.jsonl'}}),Ma=new b({props:{code:`{"patient_id":141780,"drugName":"Escitalopram","condition":"depression","review":"\\"I seemed to experience the regular side effects of LEXAPRO, insomnia, low sex drive, sleepiness during the day. I am taking it at night because my doctor said if it made me tired to take it at night. I assumed it would and started out taking it at night. Strange dreams, some pleasant. I was diagnosed with fibromyalgia. Seems to be helping with the pain. Have had anxiety and depression in my family, and have tried quite a few other medications that haven't worked. Only have been on it for two weeks but feel more positive in my mind, want to accomplish more in my life. Hopefully the side effects will dwindle away, worth it to stick with it from hearing others responses. Great medication.\\"","rating":9.0,"date":"May 29, 2011","usefulCount":10,"review_length":125}`,highlighted:'{<span class="hljs-string">&quot;patient_id&quot;</span>:<span class="hljs-number">141780</span>,<span class="hljs-string">&quot;drugName&quot;</span>:<span class="hljs-string">&quot;Escitalopram&quot;</span>,<span class="hljs-string">&quot;condition&quot;</span>:<span class="hljs-string">&quot;depression&quot;</span>,<span class="hljs-string">&quot;review&quot;</span>:<span class="hljs-string">&quot;\\&quot;I seemed to experience the regular side effects of LEXAPRO, insomnia, low sex drive, sleepiness during the day. I am taking it at night because my doctor said if it made me tired to take it at night. I assumed it would and started out taking it at night. Strange dreams, some pleasant. I was diagnosed with fibromyalgia. Seems to be helping with the pain. Have had anxiety and depression in my family, and have tried quite a few other medications that haven&#x27;t worked. Only have been on it for two weeks but feel more positive in my mind, want to accomplish more in my life. Hopefully the side effects will dwindle away, worth it to stick with it from hearing others responses. Great medication.\\&quot;&quot;</span>,<span class="hljs-string">&quot;rating&quot;</span>:<span class="hljs-number">9.0</span>,<span class="hljs-string">&quot;date&quot;</span>:<span class="hljs-string">&quot;May 29, 2011&quot;</span>,<span class="hljs-string">&quot;usefulCount&quot;</span>:<span class="hljs-number">10</span>,<span class="hljs-string">&quot;review_length&quot;</span>:<span class="hljs-number">125</span>}'}}),Ra=new b({props:{code:`data_files = {
    "train": "drug-reviews-train.jsonl",
    "validation": "drug-reviews-validation.jsonl",
    "test": "drug-reviews-test.jsonl",
}
drug_dataset_reloaded = load_dataset("json", data_files=data_files)`,highlighted:`data_files = {
    <span class="hljs-string">&quot;train&quot;</span>: <span class="hljs-string">&quot;drug-reviews-train.jsonl&quot;</span>,
    <span class="hljs-string">&quot;validation&quot;</span>: <span class="hljs-string">&quot;drug-reviews-validation.jsonl&quot;</span>,
    <span class="hljs-string">&quot;test&quot;</span>: <span class="hljs-string">&quot;drug-reviews-test.jsonl&quot;</span>,
}
drug_dataset_reloaded = load_dataset(<span class="hljs-string">&quot;json&quot;</span>, data_files=data_files)`}}),{c(){c=o("meta"),q=d(),$=o("h1"),k=o("a"),D=o("span"),u(y.$$.fragment),j=d(),C=o("span"),E=a("Time to slice and dice"),x=d(),u(z.$$.fragment),T=d(),O=o("p"),G=a("Most of the time, the data you work with won\u2019t be perfectly prepared for training models. In this section we\u2019ll explore the various features that \u{1F917} Datasets provides to clean up your datasets."),N=d(),u(P.$$.fragment),ge=d(),M=o("h2"),X=o("a"),ne=o("span"),u(se.$$.fragment),us=d(),Je=o("span"),Ve=a("Slicing and dicing our data"),bt=d(),R=o("p"),Xe=a("Similar to Pandas, \u{1F917} Datasets provides several functions to manipulate the contents of "),A=o("code"),Ja=a("Dataset"),Va=a(" and "),$t=o("code"),Xa=a("DatasetDict"),Ka=a(" objects. We already encountered the "),yt=o("code"),Qa=a("Dataset.map()"),Za=a(" method in "),en=o("a"),Uc=a("Chapter 3"),Bc=a(", and in this section we\u2019ll explore some of the other functions at our disposal."),wd=d(),be=o("p"),Yc=a("For this example we\u2019ll use the "),ms=o("a"),Gc=a("Drug Review Dataset"),Jc=a(" that\u2019s hosted on the "),ws=o("a"),Vc=a("UC Irvine Machine Learning Repository"),Xc=a(", which contains patient reviews on various drugs, along with the condition being treated and a 10-star rating of the patient\u2019s satisfaction."),gd=d(),$e=o("p"),Kc=a("First we need to download and extract the data, which can be done with the "),Wo=o("code"),Qc=a("wget"),Zc=a(" and "),Uo=o("code"),ef=a("unzip"),tf=a(" commands:"),_d=d(),u(gs.$$.fragment),vd=d(),oe=o("p"),sf=a("Since TSV is just a variant of CSV that uses tabs instead of commas as the separator, we can load these files by using the "),Bo=o("code"),af=a("csv"),nf=a(" loading script and specifying the "),Yo=o("code"),of=a("delimiter"),rf=a(" argument in the "),Go=o("code"),lf=a("load_dataset()"),df=a(" function as follows:"),bd=d(),u(_s.$$.fragment),$d=d(),ye=o("p"),hf=a("A good practice when doing any sort of data analysis is to grab a small random sample to get a quick feel for the type of data you\u2019re working with. In \u{1F917} Datasets, we can create a random sample by chaining the "),Jo=o("code"),pf=a("Dataset.shuffle()"),cf=a(" and "),Vo=o("code"),ff=a("Dataset.select()"),uf=a(" functions together:"),yd=d(),u(vs.$$.fragment),Ed=d(),u(bs.$$.fragment),xd=d(),re=o("p"),mf=a("Note that we\u2019ve fixed the seed in "),Xo=o("code"),wf=a("Dataset.shuffle()"),gf=a(" for reproducibility purposes. "),Ko=o("code"),_f=a("Dataset.select()"),vf=a(" expects an iterable of indices, so we\u2019ve passed "),Qo=o("code"),bf=a("range(1000)"),$f=a(" to grab the first 1,000 examples from the shuffled dataset. From this sample we can already see a few quirks in our dataset:"),kd=d(),Ee=o("ul"),$s=o("li"),yf=a("The "),Zo=o("code"),Ef=a("Unnamed: 0"),xf=a(" column looks suspiciously like an anonymized ID for each patient."),kf=d(),ys=o("li"),jf=a("The "),er=o("code"),Df=a("condition"),Tf=a(" column includes a mix of uppercase and lowercase labels."),qf=d(),Ke=o("li"),Cf=a("The reviews are of varying length and contain a mix of Python line separators ("),tr=o("code"),Of=a("\\r\\n"),zf=a(") as well as HTML character codes like "),sr=o("code"),Pf=a("&\\#039;"),Af=a("."),jd=d(),xe=o("p"),Nf=a("Let\u2019s see how we can use \u{1F917} Datasets to deal with each of these issues. To test the patient ID hypothesis for the "),ar=o("code"),If=a("Unnamed: 0"),Sf=a(" column, we can use the "),nr=o("code"),Hf=a("Dataset.unique()"),Ff=a(" function to verify that the number of IDs matches the number of rows in each split:"),Dd=d(),u(Es.$$.fragment),Td=d(),ke=o("p"),Lf=a("This seems to confirm our hypothesis, so let\u2019s clean up the dataset a bit by renaming the "),or=o("code"),Mf=a("Unnamed: 0"),Rf=a(" column to something a bit more interpretable. We can use the "),rr=o("code"),Wf=a("DatasetDict.rename_column()"),Uf=a(" function to rename the column across both splits in one go:"),qd=d(),u(xs.$$.fragment),Cd=d(),u(ks.$$.fragment),Od=d(),u(Et.$$.fragment),zd=d(),K=o("p"),Bf=a("Next, let\u2019s normalize all the "),lr=o("code"),Yf=a("condition"),Gf=a(" labels using "),ir=o("code"),Jf=a("Dataset.map()"),Vf=a(". As we did with tokenization in "),tn=o("a"),Xf=a("Chapter 3"),Kf=a(", we can define a simple function that can be applied across all the rows of each split in "),dr=o("code"),Qf=a("drug_dataset"),Zf=a(":"),Pd=d(),u(js.$$.fragment),Ad=d(),u(Ds.$$.fragment),Nd=d(),Q=o("p"),eu=a("Oh no, we\u2019ve run into a problem with our map function! From the error we can infer that some of the entries in the "),hr=o("code"),tu=a("condition"),su=a(" column are "),pr=o("code"),au=a("None"),nu=a(", which cannot be lowercased as they\u2019re not strings. Let\u2019s drop these rows using "),cr=o("code"),ou=a("Dataset.filter()"),ru=a(", which works in a similar way to "),fr=o("code"),lu=a("Dataset.map()"),iu=a(" and expects a function that receives a single example of the dataset. Instead of writing an explicit function like:"),Id=d(),u(Ts.$$.fragment),Sd=d(),je=o("p"),du=a("and then running "),ur=o("code"),hu=a("drug_dataset.filter(filter_nones)"),pu=a(", we can do this in one line using a "),mr=o("em"),cu=a("lambda function"),fu=a(". In Python, lambda functions are small functions that you can define without explicitly naming them. They take the general form:"),Hd=d(),u(qs.$$.fragment),Fd=d(),Z=o("p"),uu=a("where "),wr=o("code"),mu=a("lambda"),wu=a(" is one of Python\u2019s special "),Cs=o("a"),gu=a("keywords"),_u=a(", "),gr=o("code"),vu=a("<arguments>"),bu=a(" is a list/set of comma-separated values that define the inputs to the function, and "),_r=o("code"),$u=a("<expression>"),yu=a(" represents the operations you wish to execute. For example, we can define a simple lambda function that squares a number as follows:"),Ld=d(),u(Os.$$.fragment),Md=d(),sn=o("p"),Eu=a("To apply this function to an input, we need to wrap it and the input in parentheses:"),Rd=d(),u(zs.$$.fragment),Wd=d(),u(Ps.$$.fragment),Ud=d(),an=o("p"),xu=a("Similarly, we can define lambda functions with multiple arguments by separating them with commas. For example, we can compute the area of a triangle as follows:"),Bd=d(),u(As.$$.fragment),Yd=d(),u(Ns.$$.fragment),Gd=d(),De=o("p"),ku=a("Lambda functions are handy when you want to define small, single-use functions (for more information about them, we recommend reading the excellent "),Is=o("a"),ju=a("Real Python tutorial"),Du=a(" by Andre Burgaud). In the \u{1F917} Datasets context, we can use lambda functions to define simple map and filter operations, so let\u2019s use this trick to eliminate the "),vr=o("code"),Tu=a("None"),qu=a(" entries in our dataset:"),Jd=d(),u(Ss.$$.fragment),Vd=d(),Te=o("p"),Cu=a("With the "),br=o("code"),Ou=a("None"),zu=a(" entries removed, we can normalize our "),$r=o("code"),Pu=a("condition"),Au=a(" column:"),Xd=d(),u(Hs.$$.fragment),Kd=d(),u(Fs.$$.fragment),Qd=d(),nn=o("p"),Nu=a("It works! Now that we\u2019ve cleaned up the labels, let\u2019s take a look at cleaning up the reviews themselves."),Zd=d(),Qe=o("h2"),xt=o("a"),yr=o("span"),u(Ls.$$.fragment),Iu=d(),Er=o("span"),Su=a("Creating new columns"),eh=d(),on=o("p"),Hu=a("Whenever you\u2019re dealing with customer reviews, a good practice is to check the number of words in each review. A review might be just a single word like \u201CGreat!\u201D or a full-blown essay with thousands of words, and depending on the use case you\u2019ll need to handle these extremes differently. To compute the number of words in each review, we\u2019ll use a rough heuristic based on splitting each text by whitespace."),th=d(),rn=o("p"),Fu=a("Let\u2019s define a simple function that counts the number of words in each review:"),sh=d(),u(Ms.$$.fragment),ah=d(),J=o("p"),Lu=a("Unlike our "),xr=o("code"),Mu=a("lowercase_condition()"),Ru=a(" function, "),kr=o("code"),Wu=a("compute_review_length()"),Uu=a(" returns a dictionary whose key does not correspond to one of the column names in the dataset. In this case, when "),jr=o("code"),Bu=a("compute_review_length()"),Yu=a(" is passed to "),Dr=o("code"),Gu=a("Dataset.map()"),Ju=a(", it will be applied to all the rows in the dataset to create a new "),Tr=o("code"),Vu=a("review_length"),Xu=a(" column:"),nh=d(),u(Rs.$$.fragment),oh=d(),u(Ws.$$.fragment),rh=d(),qe=o("p"),Ku=a("As expected, we can see a "),qr=o("code"),Qu=a("review_length"),Zu=a(" column has been added to our training set. We can sort this new column with "),Cr=o("code"),em=a("Dataset.sort()"),tm=a(" to see what the extreme values look like:"),lh=d(),u(Us.$$.fragment),ih=d(),u(Bs.$$.fragment),dh=d(),ln=o("p"),sm=a("As we suspected, some reviews contain just a single word, which, although it may be okay for sentiment analysis, would not be informative if we want to predict the condition."),hh=d(),u(kt.$$.fragment),ph=d(),Ce=o("p"),am=a("Let\u2019s use the "),Or=o("code"),nm=a("Dataset.filter()"),om=a(" function to remove reviews that contain fewer than 30 words. Similarly to what we did with the "),zr=o("code"),rm=a("condition"),lm=a(" column, we can filter out the very short reviews by requiring that the reviews have a length above this threshold:"),ch=d(),u(Ys.$$.fragment),fh=d(),u(Gs.$$.fragment),uh=d(),dn=o("p"),im=a("As you can see, this has removed around 15% of the reviews from our original training and test sets."),mh=d(),u(jt.$$.fragment),wh=d(),Dt=o("p"),dm=a("The last thing we need to deal with is the presence of HTML character codes in our reviews. We can use Python\u2019s "),Pr=o("code"),hm=a("html"),pm=a(" module to unescape these characters, like so:"),gh=d(),u(Js.$$.fragment),_h=d(),u(Vs.$$.fragment),vh=d(),Tt=o("p"),cm=a("We\u2019ll use "),Ar=o("code"),fm=a("Dataset.map()"),um=a(" to unescape all the HTML characters in our corpus:"),bh=d(),u(Xs.$$.fragment),$h=d(),qt=o("p"),mm=a("As you can see, the "),Nr=o("code"),wm=a("Dataset.map()"),gm=a(" method is quite useful for processing data \u2014 and we haven\u2019t even scratched the surface of everything it can do!"),yh=d(),Ze=o("h2"),Ct=o("a"),Ir=o("span"),u(Ks.$$.fragment),_m=d(),Qs=o("span"),vm=a("The "),Sr=o("code"),bm=a("map()"),$m=a(" method's superpowers"),Eh=d(),le=o("p"),ym=a("The "),Hr=o("code"),Em=a("Dataset.map()"),xm=a(" method takes a "),Fr=o("code"),km=a("batched"),jm=a(" argument that, if set to "),Lr=o("code"),Dm=a("True"),Tm=a(", causes it to send a batch of examples to the map function at once (the batch size is configurable but defaults to 1,000). For instance, the previous map function that unescaped all the HTML took a bit of time to run (you can read the time taken from the progress bars). We can speed this up by processing several elements at the same time using a list comprehension."),xh=d(),ee=o("p"),qm=a("When you specify "),Mr=o("code"),Cm=a("batched=True"),Om=a(" the function receives a dictionary with the fields of the dataset, but each value is now a "),Rr=o("em"),zm=a("list of values"),Pm=a(", and not just a single value. The return value of "),Wr=o("code"),Am=a("Dataset.map()"),Nm=a(" should be the same: a dictionary with the fields we want to update or add to our dataset, and a list of values. For example, here is another way to unescape all HTML characters, but using "),Ur=o("code"),Im=a("batched=True"),Sm=a(":"),kh=d(),u(Zs.$$.fragment),jh=d(),Oe=o("p"),Hm=a("If you\u2019re running this code in a notebook, you\u2019ll see that this command executes way faster than the previous one. And it\u2019s not because our reviews have already been HTML-unescaped \u2014 if you re-execute the instruction from the previous section (without "),Br=o("code"),Fm=a("batched=True"),Lm=a("), it will take the same amount of time as before. This is because list comprehensions are usually faster than executing the same code in a "),Yr=o("code"),Mm=a("for"),Rm=a(" loop, and we also gain some performance by accessing lots of elements at the same time instead of one by one."),Dh=d(),ie=o("p"),Wm=a("Using "),Gr=o("code"),Um=a("Dataset.map()"),Bm=a(" with "),Jr=o("code"),Ym=a("batched=True"),Gm=a(" will be essential to unlock the speed of the \u201Cfast\u201D tokenizers that we\u2019ll encounter in "),hn=o("a"),Jm=a("Chapter 6"),Vm=a(", which can quickly tokenize big lists of texts. For instance, to tokenize all the drug reviews with a fast tokenizer, we could use a function like this:"),Th=d(),u(ea.$$.fragment),qh=d(),de=o("p"),Xm=a("As you saw in "),pn=o("a"),Km=a("Chapter 3"),Qm=a(", we can pass one or several examples to the tokenizer, so we can use this function with or without "),Vr=o("code"),Zm=a("batched=True"),ew=a(". Let\u2019s take this opportunity to compare the performance of the different options. In a notebook, you can time a one-line instruction by adding "),Xr=o("code"),tw=a("%time"),sw=a(" before the line of code you wish to measure:"),Ch=d(),u(ta.$$.fragment),Oh=d(),Ot=o("p"),aw=a("You can also time a whole cell by putting "),Kr=o("code"),nw=a("%%time"),ow=a(" at the beginning of the cell. On the hardware we executed this on, it showed 10.8s for this instruction (it\u2019s the number written after \u201CWall time\u201D)."),zh=d(),u(zt.$$.fragment),Ph=d(),cn=o("p"),rw=a("Here are the results we obtained with and without batching, with a fast and a slow tokenizer:"),Ah=d(),Pt=o("table"),Qr=o("thead"),et=o("tr"),fn=o("th"),lw=a("Options"),iw=d(),un=o("th"),dw=a("Fast tokenizer"),hw=d(),mn=o("th"),pw=a("Slow tokenizer"),cw=d(),sa=o("tbody"),tt=o("tr"),wn=o("td"),Zr=o("code"),fw=a("batched=True"),uw=d(),gn=o("td"),mw=a("10.8s"),ww=d(),_n=o("td"),gw=a("4min41s"),_w=d(),st=o("tr"),vn=o("td"),el=o("code"),vw=a("batched=False"),bw=d(),bn=o("td"),$w=a("59.2s"),yw=d(),$n=o("td"),Ew=a("5min3s"),Nh=d(),ze=o("p"),xw=a("This means that using a fast tokenizer with the "),tl=o("code"),kw=a("batched=True"),jw=a(" option is 30 times faster than its slow counterpart with no batching \u2014 this is truly amazing! That\u2019s the main reason why fast tokenizers are the default when using "),sl=o("code"),Dw=a("AutoTokenizer"),Tw=a(" (and why they are called \u201Cfast\u201D). They\u2019re able to achieve such a speedup because behind the scenes the tokenization code is executed in Rust, which is a language that makes it easy to parallelize code execution."),Ih=d(),yn=o("p"),qw=a("Parallelization is also the reason for the nearly 6x speedup the fast tokenizer achieves with batching: you can\u2019t parallelize a single tokenization operation, but when you want to tokenize lots of texts at the same time you can just split the execution across several processes, each responsible for its own texts."),Sh=d(),_e=o("p"),al=o("code"),Cw=a("Dataset.map()"),Ow=a(" also has some parallelization capabilities of its own. Since they are not backed by Rust, they won\u2019t let a slow tokenizer catch up with a fast one, but they can still be helpful (especially if you\u2019re using a tokenizer that doesn\u2019t have a fast version). To enable multiprocessing, use the "),nl=o("code"),zw=a("num_proc"),Pw=a(" argument and specify the number of processes to use in your call to "),ol=o("code"),Aw=a("Dataset.map()"),Nw=a(":"),Hh=d(),u(aa.$$.fragment),Fh=d(),En=o("p"),Iw=a("You can experiment a little with timing to determine the optimal number of processes to use; in our case 8 seemed to produce the best speed gain. Here are the numbers we got with and without multiprocessing:"),Lh=d(),At=o("table"),rl=o("thead"),at=o("tr"),xn=o("th"),Sw=a("Options"),Hw=d(),kn=o("th"),Fw=a("Fast tokenizer"),Lw=d(),jn=o("th"),Mw=a("Slow tokenizer"),Rw=d(),ve=o("tbody"),nt=o("tr"),Dn=o("td"),ll=o("code"),Ww=a("batched=True"),Uw=d(),Tn=o("td"),Bw=a("10.8s"),Yw=d(),qn=o("td"),Gw=a("4min41s"),Jw=d(),ot=o("tr"),Cn=o("td"),il=o("code"),Vw=a("batched=False"),Xw=d(),On=o("td"),Kw=a("59.2s"),Qw=d(),zn=o("td"),Zw=a("5min3s"),eg=d(),rt=o("tr"),Nt=o("td"),dl=o("code"),tg=a("batched=True"),sg=a(", "),hl=o("code"),ag=a("num_proc=8"),ng=d(),Pn=o("td"),og=a("6.52s"),rg=d(),An=o("td"),lg=a("41.3s"),ig=d(),lt=o("tr"),It=o("td"),pl=o("code"),dg=a("batched=False"),hg=a(", "),cl=o("code"),pg=a("num_proc=8"),cg=d(),Nn=o("td"),fg=a("9.49s"),ug=d(),In=o("td"),mg=a("45.2s"),Mh=d(),he=o("p"),wg=a("Those are much more reasonable results for the slow tokenizer, but the performance of the fast tokenizer was also substantially improved. Note, however, that won\u2019t always be the case \u2014 for values of "),fl=o("code"),gg=a("num_proc"),_g=a(" other than 8, our tests showed that it was faster to use "),ul=o("code"),vg=a("batched=True"),bg=a(" without that option. In general, we don\u2019t recommend using Python multiprocessing for fast tokenizers with "),ml=o("code"),$g=a("batched=True"),yg=a("."),Rh=d(),u(St.$$.fragment),Wh=d(),pe=o("p"),Eg=a("All of this functionality condensed into a single method is already pretty amazing, but there\u2019s more! With "),wl=o("code"),xg=a("Dataset.map()"),kg=a(" and "),gl=o("code"),jg=a("batched=True"),Dg=a(" you can change the number of elements in your dataset. This is super useful in many situations where you want to create several training features from one example, and we will need to do this as part of the preprocessing for several of the NLP tasks we\u2019ll undertake in "),Sn=o("a"),Tg=a("Chapter 7"),qg=a("."),Uh=d(),u(Ht.$$.fragment),Bh=d(),Pe=o("p"),Cg=a("Let\u2019s have a look at how it works! Here we will tokenize our examples and truncate them to a maximum length of 128, but we will ask the tokenizer to return "),_l=o("em"),Og=a("all"),zg=a(" the chunks of the texts instead of just the first one. This can be done with "),vl=o("code"),Pg=a("return_overflowing_tokens=True"),Ag=a(":"),Yh=d(),u(na.$$.fragment),Gh=d(),Ft=o("p"),Ng=a("Let\u2019s test this on one example before using "),bl=o("code"),Ig=a("Dataset.map()"),Sg=a(" on the whole dataset:"),Jh=d(),u(oa.$$.fragment),Vh=d(),u(ra.$$.fragment),Xh=d(),Hn=o("p"),Hg=a("So, our first example in the training set became two features because it was tokenized to more than the maximum number of tokens we specified: the first one of length 128 and the second one of length 49. Now let\u2019s do this for all elements of the dataset!"),Kh=d(),u(la.$$.fragment),Qh=d(),u(ia.$$.fragment),Zh=d(),Ae=o("p"),Fg=a("Oh no! That didn\u2019t work! Why not? Looking at the error message will give us a clue: there is a mismatch in the lengths of one of the columns, one being of length 1,463 and the other of length 1,000. If you\u2019ve looked at the "),$l=o("code"),Lg=a("Dataset.map()"),Mg=d(),da=o("a"),Rg=a("documentation"),Wg=a(", you may recall that it\u2019s the number of samples passed to the function that we are mapping; here those 1,000 examples gave 1,463 new features, resulting in a shape error."),ep=d(),te=o("p"),Ug=a("The problem is that we\u2019re trying to mix two different datasets of different sizes: the "),yl=o("code"),Bg=a("drug_dataset"),Yg=a(" columns will have a certain number of examples (the 1,000 in our error), but the "),El=o("code"),Gg=a("tokenized_dataset"),Jg=a(" we are building will have more (the 1,463 in the error message). That doesn\u2019t work for a "),xl=o("code"),Vg=a("Dataset"),Xg=a(", so we need to either remove the columns from the old dataset or make them the same size as they are in the new dataset. We can do the former with the "),kl=o("code"),Kg=a("remove_columns"),Qg=a(" argument:"),tp=d(),u(ha.$$.fragment),sp=d(),Fn=o("p"),Zg=a("Now this works without error. We can check that our new dataset has many more elements than the original dataset by comparing the lengths:"),ap=d(),u(pa.$$.fragment),np=d(),u(ca.$$.fragment),op=d(),Ne=o("p"),e_=a("We mentioned that we can also deal with the mismatched length problem by making the old columns the same size as the new ones. To do this, we will need the "),jl=o("code"),t_=a("overflow_to_sample_mapping"),s_=a(" field the tokenizer returns when we set "),Dl=o("code"),a_=a("return_overflowing_tokens=True"),n_=a(". It gives us a mapping from a new feature index to the index of the sample it originated from. Using this, we can associate each key present in our original dataset with a list of values of the right size by repeating the values of each example as many times as it generates new features:"),rp=d(),u(fa.$$.fragment),lp=d(),Lt=o("p"),o_=a("We can see it works with "),Tl=o("code"),r_=a("Dataset.map()"),l_=a(" without us needing to remove the old columns:"),ip=d(),u(ua.$$.fragment),dp=d(),u(ma.$$.fragment),hp=d(),Ln=o("p"),i_=a("We get the same number of training features as before, but here we\u2019ve kept all the old fields. If you need them for some post-processing after applying your model, you might want to use this approach."),pp=d(),Mt=o("p"),d_=a(`You\u2019ve now seen how \u{1F917} Datasets can be used to preprocess a dataset in various ways. Although the processing functions of \u{1F917} Datasets will cover most of your model training needs,
there may be times when you\u2019ll need to switch to Pandas to access more powerful features, like `),ql=o("code"),h_=a("DataFrame.groupby()"),p_=a(" or high-level APIs for visualization. Fortunately, \u{1F917} Datasets is designed to be interoperable with libraries such as Pandas, NumPy, PyTorch, TensorFlow, and JAX. Let\u2019s take a look at how this works."),cp=d(),it=o("h2"),Rt=o("a"),Cl=o("span"),u(wa.$$.fragment),c_=d(),dt=o("span"),f_=a("From "),Ol=o("code"),u_=a("Dataset"),m_=a("s to "),zl=o("code"),w_=a("DataFrame"),g_=a("s and back"),fp=d(),u(ga.$$.fragment),up=d(),ce=o("p"),__=a("To enable the conversion between various third-party libraries, \u{1F917} Datasets provides a "),Pl=o("code"),v_=a("Dataset.set_format()"),b_=a(" function. This function only changes the "),Al=o("em"),$_=a("output format"),y_=a(" of the dataset, so you can easily switch to another format without affecting the underlying "),Nl=o("em"),E_=a("data format"),x_=a(", which is Apache Arrow. The formatting is done in place. To demonstrate, let\u2019s convert our dataset to Pandas:"),mp=d(),u(_a.$$.fragment),wp=d(),Wt=o("p"),k_=a("Now when we access elements of the dataset we get a "),Il=o("code"),j_=a("pandas.DataFrame"),D_=a(" instead of a dictionary:"),gp=d(),u(va.$$.fragment),_p=d(),Ie=o("table"),Sl=o("thead"),I=o("tr"),vp=o("th"),T_=d(),Hl=o("th"),q_=a("patient_id"),C_=d(),Fl=o("th"),O_=a("drugName"),z_=d(),Ll=o("th"),P_=a("condition"),A_=d(),Ml=o("th"),N_=a("review"),I_=d(),Rl=o("th"),S_=a("rating"),H_=d(),Wl=o("th"),F_=a("date"),L_=d(),Ul=o("th"),M_=a("usefulCount"),R_=d(),Bl=o("th"),W_=a("review_length"),U_=d(),ht=o("tbody"),S=o("tr"),Yl=o("th"),B_=a("0"),Y_=d(),Gl=o("td"),G_=a("95260"),J_=d(),Jl=o("td"),V_=a("Guanfacine"),X_=d(),Vl=o("td"),K_=a("adhd"),Q_=d(),Xl=o("td"),Z_=a('"My son is halfway through his fourth week of Intuniv..."'),ev=d(),Kl=o("td"),tv=a("8.0"),sv=d(),Ql=o("td"),av=a("April 27, 2010"),nv=d(),Zl=o("td"),ov=a("192"),rv=d(),ei=o("td"),lv=a("141"),iv=d(),H=o("tr"),ti=o("th"),dv=a("1"),hv=d(),si=o("td"),pv=a("92703"),cv=d(),ai=o("td"),fv=a("Lybrel"),uv=d(),ni=o("td"),mv=a("birth control"),wv=d(),oi=o("td"),gv=a('"I used to take another oral contraceptive, which had 21 pill cycle, and was very happy- very light periods, max 5 days, no other side effects..."'),_v=d(),ri=o("td"),vv=a("5.0"),bv=d(),li=o("td"),$v=a("December 14, 2009"),yv=d(),ii=o("td"),Ev=a("17"),xv=d(),di=o("td"),kv=a("134"),jv=d(),F=o("tr"),hi=o("th"),Dv=a("2"),Tv=d(),pi=o("td"),qv=a("138000"),Cv=d(),ci=o("td"),Ov=a("Ortho Evra"),zv=d(),fi=o("td"),Pv=a("birth control"),Av=d(),ui=o("td"),Nv=a('"This is my first time using any form of birth control..."'),Iv=d(),mi=o("td"),Sv=a("8.0"),Hv=d(),wi=o("td"),Fv=a("November 3, 2015"),Lv=d(),gi=o("td"),Mv=a("10"),Rv=d(),_i=o("td"),Wv=a("89"),bp=d(),Se=o("p"),Uv=a("Let\u2019s create a "),vi=o("code"),Bv=a("pandas.DataFrame"),Yv=a(" for the whole training set by selecting all the elements of "),bi=o("code"),Gv=a('drug_dataset["train"]'),Jv=a(":"),$p=d(),u(ba.$$.fragment),yp=d(),u(Ut.$$.fragment),Ep=d(),Bt=o("p"),Vv=a("From here we can use all the Pandas functionality that we want. For example, we can do fancy chaining to compute the class distribution among the "),$i=o("code"),Xv=a("condition"),Kv=a(" entries:"),xp=d(),u($a.$$.fragment),kp=d(),He=o("table"),yi=o("thead"),Fe=o("tr"),jp=o("th"),Qv=d(),Ei=o("th"),Zv=a("condition"),e2=d(),xi=o("th"),t2=a("frequency"),s2=d(),ae=o("tbody"),pt=o("tr"),ki=o("th"),a2=a("0"),n2=d(),ji=o("td"),o2=a("birth control"),r2=d(),Di=o("td"),l2=a("27655"),i2=d(),ct=o("tr"),Ti=o("th"),d2=a("1"),h2=d(),qi=o("td"),p2=a("depression"),c2=d(),Ci=o("td"),f2=a("8023"),u2=d(),ft=o("tr"),Oi=o("th"),m2=a("2"),w2=d(),zi=o("td"),g2=a("acne"),_2=d(),Pi=o("td"),v2=a("5209"),b2=d(),ut=o("tr"),Ai=o("th"),$2=a("3"),y2=d(),Ni=o("td"),E2=a("anxiety"),x2=d(),Ii=o("td"),k2=a("4991"),j2=d(),mt=o("tr"),Si=o("th"),D2=a("4"),T2=d(),Hi=o("td"),q2=a("pain"),C2=d(),Fi=o("td"),O2=a("4744"),Dp=d(),Le=o("p"),z2=a("And once we\u2019re done with our Pandas analysis, we can always create a new "),Li=o("code"),P2=a("Dataset"),A2=a(" object by using the "),Mi=o("code"),N2=a("Dataset.from_pandas()"),I2=a(" function as follows:"),Tp=d(),u(ya.$$.fragment),qp=d(),u(Ea.$$.fragment),Cp=d(),u(Yt.$$.fragment),Op=d(),fe=o("p"),S2=a("This wraps up our tour of the various preprocessing techniques available in \u{1F917} Datasets. To round out the section, let\u2019s create a validation set to prepare the dataset for training a classifier on. Before doing so, we\u2019ll reset the output format of "),Ri=o("code"),H2=a("drug_dataset"),F2=a(" from "),Wi=o("code"),L2=a('"pandas"'),M2=a(" to "),Ui=o("code"),R2=a('"arrow"'),W2=a(":"),zp=d(),u(xa.$$.fragment),Pp=d(),wt=o("h2"),Gt=o("a"),Bi=o("span"),u(ka.$$.fragment),U2=d(),Yi=o("span"),B2=a("Creating a validation set"),Ap=d(),Mn=o("p"),Y2=a("Although we have a test set we could use for evaluation, it\u2019s a good practice to leave the test set untouched and create a separate validation set during development. Once you are happy with the performance of your models on the validation set, you can do a final sanity check on the test set. This process helps mitigate the risk that you\u2019ll overfit to the test set and deploy a model that fails on real-world data."),Np=d(),V=o("p"),G2=a("\u{1F917} Datasets provides a "),Gi=o("code"),J2=a("Dataset.train_test_split()"),V2=a(" function that is based on the famous functionality from "),Ji=o("code"),X2=a("scikit-learn"),K2=a(". Let\u2019s use it to split our training set into "),Vi=o("code"),Q2=a("train"),Z2=a(" and "),Xi=o("code"),e1=a("validation"),t1=a(" splits (we set the "),Ki=o("code"),s1=a("seed"),a1=a(" argument for reproducibility):"),Ip=d(),u(ja.$$.fragment),Sp=d(),u(Da.$$.fragment),Hp=d(),Jt=o("p"),n1=a("Great, we\u2019ve now prepared a dataset that\u2019s ready for training some models on! In "),Rn=o("a"),o1=a("section 5"),r1=a(" we\u2019ll show you how to upload datasets to the Hugging Face Hub, but for now let\u2019s cap off our analysis by looking at a few ways you can save datasets on your local machine."),Fp=d(),gt=o("h2"),Vt=o("a"),Qi=o("span"),u(Ta.$$.fragment),l1=d(),Zi=o("span"),i1=a("Saving a dataset"),Lp=d(),u(qa.$$.fragment),Mp=d(),Wn=o("p"),d1=a("Although \u{1F917} Datasets will cache every downloaded dataset and the operations performed on it, there are times when you\u2019ll want to save a dataset to disk (e.g., in case the cache gets deleted). As shown in the table below, \u{1F917} Datasets provides three main functions to save your dataset in different formats:"),Rp=d(),Xt=o("table"),ed=o("thead"),Ca=o("tr"),Un=o("th"),h1=a("Data format"),p1=d(),Bn=o("th"),c1=a("Function"),f1=d(),_t=o("tbody"),Oa=o("tr"),Yn=o("td"),u1=a("Arrow"),m1=d(),Gn=o("td"),td=o("code"),w1=a("Dataset.save_to_disk()"),g1=d(),za=o("tr"),Jn=o("td"),_1=a("CSV"),v1=d(),Vn=o("td"),sd=o("code"),b1=a("Dataset.to_csv()"),$1=d(),Pa=o("tr"),Xn=o("td"),y1=a("JSON"),E1=d(),Kn=o("td"),ad=o("code"),x1=a("Dataset.to_json()"),Wp=d(),Qn=o("p"),k1=a("For example, let\u2019s save our cleaned dataset in the Arrow format:"),Up=d(),u(Aa.$$.fragment),Bp=d(),Zn=o("p"),j1=a("This will create a directory with the following structure:"),Yp=d(),u(Na.$$.fragment),Gp=d(),ue=o("p"),D1=a("where we can see that each split is associated with its own "),nd=o("em"),T1=a("dataset.arrow"),q1=a(" table, and some metadata in "),od=o("em"),C1=a("dataset_info.json"),O1=a(" and "),rd=o("em"),z1=a("state.json"),P1=a(". You can think of the Arrow format as a fancy table of columns and rows that is optimized for building high-performance applications that process and transport large datasets."),Jp=d(),Kt=o("p"),A1=a("Once the dataset is saved, we can load it by using the "),ld=o("code"),N1=a("load_from_disk()"),I1=a(" function as follows:"),Vp=d(),u(Ia.$$.fragment),Xp=d(),u(Sa.$$.fragment),Kp=d(),Qt=o("p"),S1=a("For the CSV and JSON formats, we have to store each split as a separate file. One way to do this is by iterating over the keys and values in the "),id=o("code"),H1=a("DatasetDict"),F1=a(" object:"),Qp=d(),u(Ha.$$.fragment),Zp=d(),Zt=o("p"),L1=a("This saves each split in "),Fa=o("a"),M1=a("JSON Lines format"),R1=a(", where each row in the dataset is stored as a single line of JSON. Here\u2019s what the first example looks like:"),ec=d(),u(La.$$.fragment),tc=d(),u(Ma.$$.fragment),sc=d(),es=o("p"),W1=a("We can then use the techniques from "),eo=o("a"),U1=a("section 2"),B1=a(" to load the JSON files as follows:"),ac=d(),u(Ra.$$.fragment),nc=d(),to=o("p"),Y1=a("And that\u2019s it for our excursion into data wrangling with \u{1F917} Datasets! Now that we have a cleaned dataset for training a model on, here are a few ideas that you could try out:"),oc=d(),ts=o("ol"),Wa=o("li"),G1=a("Use the techniques from "),so=o("a"),J1=a("Chapter 3"),V1=a(" to train a classifier that can predict the patient condition based on the drug review."),X1=d(),vt=o("li"),K1=a("Use the "),dd=o("code"),Q1=a("summarization"),Z1=a(" pipeline from "),ao=o("a"),eb=a("Chapter 1"),tb=a(" to generate summaries of the reviews."),rc=d(),no=o("p"),sb=a("Next, we\u2019ll take a look at how \u{1F917} Datasets can enable you to work with huge datasets without blowing up your laptop!"),this.h()},l(e){const i=JE('[data-svelte="svelte-1phssyn"]',document.head);c=r(i,"META",{name:!0,content:!0}),i.forEach(s),q=h(e),$=r(e,"H1",{class:!0});var Ua=l($);k=r(Ua,"A",{id:!0,class:!0,href:!0});var hd=l(k);D=r(hd,"SPAN",{});var pd=l(D);m(y.$$.fragment,pd),pd.forEach(s),hd.forEach(s),j=h(Ua),C=r(Ua,"SPAN",{});var cd=l(C);E=n(cd,"Time to slice and dice"),cd.forEach(s),Ua.forEach(s),x=h(e),m(z.$$.fragment,e),T=h(e),O=r(e,"P",{});var fd=l(O);G=n(fd,"Most of the time, the data you work with won\u2019t be perfectly prepared for training models. In this section we\u2019ll explore the various features that \u{1F917} Datasets provides to clean up your datasets."),fd.forEach(s),N=h(e),m(P.$$.fragment,e),ge=h(e),M=r(e,"H2",{class:!0});var Ba=l(M);X=r(Ba,"A",{id:!0,class:!0,href:!0});var ud=l(X);ne=r(ud,"SPAN",{});var md=l(ne);m(se.$$.fragment,md),md.forEach(s),ud.forEach(s),us=h(Ba),Je=r(Ba,"SPAN",{});var nb=l(Je);Ve=n(nb,"Slicing and dicing our data"),nb.forEach(s),Ba.forEach(s),bt=h(e),R=r(e,"P",{});var Me=l(R);Xe=n(Me,"Similar to Pandas, \u{1F917} Datasets provides several functions to manipulate the contents of "),A=r(Me,"CODE",{});var ob=l(A);Ja=n(ob,"Dataset"),ob.forEach(s),Va=n(Me," and "),$t=r(Me,"CODE",{});var rb=l($t);Xa=n(rb,"DatasetDict"),rb.forEach(s),Ka=n(Me," objects. We already encountered the "),yt=r(Me,"CODE",{});var lb=l(yt);Qa=n(lb,"Dataset.map()"),lb.forEach(s),Za=n(Me," method in "),en=r(Me,"A",{href:!0});var ib=l(en);Uc=n(ib,"Chapter 3"),ib.forEach(s),Bc=n(Me,", and in this section we\u2019ll explore some of the other functions at our disposal."),Me.forEach(s),wd=h(e),be=r(e,"P",{});var oo=l(be);Yc=n(oo,"For this example we\u2019ll use the "),ms=r(oo,"A",{href:!0,rel:!0});var db=l(ms);Gc=n(db,"Drug Review Dataset"),db.forEach(s),Jc=n(oo," that\u2019s hosted on the "),ws=r(oo,"A",{href:!0,rel:!0});var hb=l(ws);Vc=n(hb,"UC Irvine Machine Learning Repository"),hb.forEach(s),Xc=n(oo,", which contains patient reviews on various drugs, along with the condition being treated and a 10-star rating of the patient\u2019s satisfaction."),oo.forEach(s),gd=h(e),$e=r(e,"P",{});var ro=l($e);Kc=n(ro,"First we need to download and extract the data, which can be done with the "),Wo=r(ro,"CODE",{});var pb=l(Wo);Qc=n(pb,"wget"),pb.forEach(s),Zc=n(ro," and "),Uo=r(ro,"CODE",{});var cb=l(Uo);ef=n(cb,"unzip"),cb.forEach(s),tf=n(ro," commands:"),ro.forEach(s),_d=h(e),m(gs.$$.fragment,e),vd=h(e),oe=r(e,"P",{});var ss=l(oe);sf=n(ss,"Since TSV is just a variant of CSV that uses tabs instead of commas as the separator, we can load these files by using the "),Bo=r(ss,"CODE",{});var fb=l(Bo);af=n(fb,"csv"),fb.forEach(s),nf=n(ss," loading script and specifying the "),Yo=r(ss,"CODE",{});var ub=l(Yo);of=n(ub,"delimiter"),ub.forEach(s),rf=n(ss," argument in the "),Go=r(ss,"CODE",{});var mb=l(Go);lf=n(mb,"load_dataset()"),mb.forEach(s),df=n(ss," function as follows:"),ss.forEach(s),bd=h(e),m(_s.$$.fragment,e),$d=h(e),ye=r(e,"P",{});var lo=l(ye);hf=n(lo,"A good practice when doing any sort of data analysis is to grab a small random sample to get a quick feel for the type of data you\u2019re working with. In \u{1F917} Datasets, we can create a random sample by chaining the "),Jo=r(lo,"CODE",{});var wb=l(Jo);pf=n(wb,"Dataset.shuffle()"),wb.forEach(s),cf=n(lo," and "),Vo=r(lo,"CODE",{});var gb=l(Vo);ff=n(gb,"Dataset.select()"),gb.forEach(s),uf=n(lo," functions together:"),lo.forEach(s),yd=h(e),m(vs.$$.fragment,e),Ed=h(e),m(bs.$$.fragment,e),xd=h(e),re=r(e,"P",{});var as=l(re);mf=n(as,"Note that we\u2019ve fixed the seed in "),Xo=r(as,"CODE",{});var _b=l(Xo);wf=n(_b,"Dataset.shuffle()"),_b.forEach(s),gf=n(as," for reproducibility purposes. "),Ko=r(as,"CODE",{});var vb=l(Ko);_f=n(vb,"Dataset.select()"),vb.forEach(s),vf=n(as," expects an iterable of indices, so we\u2019ve passed "),Qo=r(as,"CODE",{});var bb=l(Qo);bf=n(bb,"range(1000)"),bb.forEach(s),$f=n(as," to grab the first 1,000 examples from the shuffled dataset. From this sample we can already see a few quirks in our dataset:"),as.forEach(s),kd=h(e),Ee=r(e,"UL",{});var io=l(Ee);$s=r(io,"LI",{});var ic=l($s);yf=n(ic,"The "),Zo=r(ic,"CODE",{});var $b=l(Zo);Ef=n($b,"Unnamed: 0"),$b.forEach(s),xf=n(ic," column looks suspiciously like an anonymized ID for each patient."),ic.forEach(s),kf=h(io),ys=r(io,"LI",{});var dc=l(ys);jf=n(dc,"The "),er=r(dc,"CODE",{});var yb=l(er);Df=n(yb,"condition"),yb.forEach(s),Tf=n(dc," column includes a mix of uppercase and lowercase labels."),dc.forEach(s),qf=h(io),Ke=r(io,"LI",{});var ho=l(Ke);Cf=n(ho,"The reviews are of varying length and contain a mix of Python line separators ("),tr=r(ho,"CODE",{});var Eb=l(tr);Of=n(Eb,"\\r\\n"),Eb.forEach(s),zf=n(ho,") as well as HTML character codes like "),sr=r(ho,"CODE",{});var xb=l(sr);Pf=n(xb,"&\\#039;"),xb.forEach(s),Af=n(ho,"."),ho.forEach(s),io.forEach(s),jd=h(e),xe=r(e,"P",{});var po=l(xe);Nf=n(po,"Let\u2019s see how we can use \u{1F917} Datasets to deal with each of these issues. To test the patient ID hypothesis for the "),ar=r(po,"CODE",{});var kb=l(ar);If=n(kb,"Unnamed: 0"),kb.forEach(s),Sf=n(po," column, we can use the "),nr=r(po,"CODE",{});var jb=l(nr);Hf=n(jb,"Dataset.unique()"),jb.forEach(s),Ff=n(po," function to verify that the number of IDs matches the number of rows in each split:"),po.forEach(s),Dd=h(e),m(Es.$$.fragment,e),Td=h(e),ke=r(e,"P",{});var co=l(ke);Lf=n(co,"This seems to confirm our hypothesis, so let\u2019s clean up the dataset a bit by renaming the "),or=r(co,"CODE",{});var Db=l(or);Mf=n(Db,"Unnamed: 0"),Db.forEach(s),Rf=n(co," column to something a bit more interpretable. We can use the "),rr=r(co,"CODE",{});var Tb=l(rr);Wf=n(Tb,"DatasetDict.rename_column()"),Tb.forEach(s),Uf=n(co," function to rename the column across both splits in one go:"),co.forEach(s),qd=h(e),m(xs.$$.fragment,e),Cd=h(e),m(ks.$$.fragment,e),Od=h(e),m(Et.$$.fragment,e),zd=h(e),K=r(e,"P",{});var Re=l(K);Bf=n(Re,"Next, let\u2019s normalize all the "),lr=r(Re,"CODE",{});var qb=l(lr);Yf=n(qb,"condition"),qb.forEach(s),Gf=n(Re," labels using "),ir=r(Re,"CODE",{});var Cb=l(ir);Jf=n(Cb,"Dataset.map()"),Cb.forEach(s),Vf=n(Re,". As we did with tokenization in "),tn=r(Re,"A",{href:!0});var Ob=l(tn);Xf=n(Ob,"Chapter 3"),Ob.forEach(s),Kf=n(Re,", we can define a simple function that can be applied across all the rows of each split in "),dr=r(Re,"CODE",{});var zb=l(dr);Qf=n(zb,"drug_dataset"),zb.forEach(s),Zf=n(Re,":"),Re.forEach(s),Pd=h(e),m(js.$$.fragment,e),Ad=h(e),m(Ds.$$.fragment,e),Nd=h(e),Q=r(e,"P",{});var We=l(Q);eu=n(We,"Oh no, we\u2019ve run into a problem with our map function! From the error we can infer that some of the entries in the "),hr=r(We,"CODE",{});var Pb=l(hr);tu=n(Pb,"condition"),Pb.forEach(s),su=n(We," column are "),pr=r(We,"CODE",{});var Ab=l(pr);au=n(Ab,"None"),Ab.forEach(s),nu=n(We,", which cannot be lowercased as they\u2019re not strings. Let\u2019s drop these rows using "),cr=r(We,"CODE",{});var Nb=l(cr);ou=n(Nb,"Dataset.filter()"),Nb.forEach(s),ru=n(We,", which works in a similar way to "),fr=r(We,"CODE",{});var Ib=l(fr);lu=n(Ib,"Dataset.map()"),Ib.forEach(s),iu=n(We," and expects a function that receives a single example of the dataset. Instead of writing an explicit function like:"),We.forEach(s),Id=h(e),m(Ts.$$.fragment,e),Sd=h(e),je=r(e,"P",{});var fo=l(je);du=n(fo,"and then running "),ur=r(fo,"CODE",{});var Sb=l(ur);hu=n(Sb,"drug_dataset.filter(filter_nones)"),Sb.forEach(s),pu=n(fo,", we can do this in one line using a "),mr=r(fo,"EM",{});var Hb=l(mr);cu=n(Hb,"lambda function"),Hb.forEach(s),fu=n(fo,". In Python, lambda functions are small functions that you can define without explicitly naming them. They take the general form:"),fo.forEach(s),Hd=h(e),m(qs.$$.fragment,e),Fd=h(e),Z=r(e,"P",{});var Ue=l(Z);uu=n(Ue,"where "),wr=r(Ue,"CODE",{});var Fb=l(wr);mu=n(Fb,"lambda"),Fb.forEach(s),wu=n(Ue," is one of Python\u2019s special "),Cs=r(Ue,"A",{href:!0,rel:!0});var Lb=l(Cs);gu=n(Lb,"keywords"),Lb.forEach(s),_u=n(Ue,", "),gr=r(Ue,"CODE",{});var Mb=l(gr);vu=n(Mb,"<arguments>"),Mb.forEach(s),bu=n(Ue," is a list/set of comma-separated values that define the inputs to the function, and "),_r=r(Ue,"CODE",{});var Rb=l(_r);$u=n(Rb,"<expression>"),Rb.forEach(s),yu=n(Ue," represents the operations you wish to execute. For example, we can define a simple lambda function that squares a number as follows:"),Ue.forEach(s),Ld=h(e),m(Os.$$.fragment,e),Md=h(e),sn=r(e,"P",{});var Wb=l(sn);Eu=n(Wb,"To apply this function to an input, we need to wrap it and the input in parentheses:"),Wb.forEach(s),Rd=h(e),m(zs.$$.fragment,e),Wd=h(e),m(Ps.$$.fragment,e),Ud=h(e),an=r(e,"P",{});var Ub=l(an);xu=n(Ub,"Similarly, we can define lambda functions with multiple arguments by separating them with commas. For example, we can compute the area of a triangle as follows:"),Ub.forEach(s),Bd=h(e),m(As.$$.fragment,e),Yd=h(e),m(Ns.$$.fragment,e),Gd=h(e),De=r(e,"P",{});var uo=l(De);ku=n(uo,"Lambda functions are handy when you want to define small, single-use functions (for more information about them, we recommend reading the excellent "),Is=r(uo,"A",{href:!0,rel:!0});var Bb=l(Is);ju=n(Bb,"Real Python tutorial"),Bb.forEach(s),Du=n(uo," by Andre Burgaud). In the \u{1F917} Datasets context, we can use lambda functions to define simple map and filter operations, so let\u2019s use this trick to eliminate the "),vr=r(uo,"CODE",{});var Yb=l(vr);Tu=n(Yb,"None"),Yb.forEach(s),qu=n(uo," entries in our dataset:"),uo.forEach(s),Jd=h(e),m(Ss.$$.fragment,e),Vd=h(e),Te=r(e,"P",{});var mo=l(Te);Cu=n(mo,"With the "),br=r(mo,"CODE",{});var Gb=l(br);Ou=n(Gb,"None"),Gb.forEach(s),zu=n(mo," entries removed, we can normalize our "),$r=r(mo,"CODE",{});var Jb=l($r);Pu=n(Jb,"condition"),Jb.forEach(s),Au=n(mo," column:"),mo.forEach(s),Xd=h(e),m(Hs.$$.fragment,e),Kd=h(e),m(Fs.$$.fragment,e),Qd=h(e),nn=r(e,"P",{});var Vb=l(nn);Nu=n(Vb,"It works! Now that we\u2019ve cleaned up the labels, let\u2019s take a look at cleaning up the reviews themselves."),Vb.forEach(s),Zd=h(e),Qe=r(e,"H2",{class:!0});var hc=l(Qe);xt=r(hc,"A",{id:!0,class:!0,href:!0});var Xb=l(xt);yr=r(Xb,"SPAN",{});var Kb=l(yr);m(Ls.$$.fragment,Kb),Kb.forEach(s),Xb.forEach(s),Iu=h(hc),Er=r(hc,"SPAN",{});var Qb=l(Er);Su=n(Qb,"Creating new columns"),Qb.forEach(s),hc.forEach(s),eh=h(e),on=r(e,"P",{});var Zb=l(on);Hu=n(Zb,"Whenever you\u2019re dealing with customer reviews, a good practice is to check the number of words in each review. A review might be just a single word like \u201CGreat!\u201D or a full-blown essay with thousands of words, and depending on the use case you\u2019ll need to handle these extremes differently. To compute the number of words in each review, we\u2019ll use a rough heuristic based on splitting each text by whitespace."),Zb.forEach(s),th=h(e),rn=r(e,"P",{});var e$=l(rn);Fu=n(e$,"Let\u2019s define a simple function that counts the number of words in each review:"),e$.forEach(s),sh=h(e),m(Ms.$$.fragment,e),ah=h(e),J=r(e,"P",{});var me=l(J);Lu=n(me,"Unlike our "),xr=r(me,"CODE",{});var t$=l(xr);Mu=n(t$,"lowercase_condition()"),t$.forEach(s),Ru=n(me," function, "),kr=r(me,"CODE",{});var s$=l(kr);Wu=n(s$,"compute_review_length()"),s$.forEach(s),Uu=n(me," returns a dictionary whose key does not correspond to one of the column names in the dataset. In this case, when "),jr=r(me,"CODE",{});var a$=l(jr);Bu=n(a$,"compute_review_length()"),a$.forEach(s),Yu=n(me," is passed to "),Dr=r(me,"CODE",{});var n$=l(Dr);Gu=n(n$,"Dataset.map()"),n$.forEach(s),Ju=n(me,", it will be applied to all the rows in the dataset to create a new "),Tr=r(me,"CODE",{});var o$=l(Tr);Vu=n(o$,"review_length"),o$.forEach(s),Xu=n(me," column:"),me.forEach(s),nh=h(e),m(Rs.$$.fragment,e),oh=h(e),m(Ws.$$.fragment,e),rh=h(e),qe=r(e,"P",{});var wo=l(qe);Ku=n(wo,"As expected, we can see a "),qr=r(wo,"CODE",{});var r$=l(qr);Qu=n(r$,"review_length"),r$.forEach(s),Zu=n(wo," column has been added to our training set. We can sort this new column with "),Cr=r(wo,"CODE",{});var l$=l(Cr);em=n(l$,"Dataset.sort()"),l$.forEach(s),tm=n(wo," to see what the extreme values look like:"),wo.forEach(s),lh=h(e),m(Us.$$.fragment,e),ih=h(e),m(Bs.$$.fragment,e),dh=h(e),ln=r(e,"P",{});var i$=l(ln);sm=n(i$,"As we suspected, some reviews contain just a single word, which, although it may be okay for sentiment analysis, would not be informative if we want to predict the condition."),i$.forEach(s),hh=h(e),m(kt.$$.fragment,e),ph=h(e),Ce=r(e,"P",{});var go=l(Ce);am=n(go,"Let\u2019s use the "),Or=r(go,"CODE",{});var d$=l(Or);nm=n(d$,"Dataset.filter()"),d$.forEach(s),om=n(go," function to remove reviews that contain fewer than 30 words. Similarly to what we did with the "),zr=r(go,"CODE",{});var h$=l(zr);rm=n(h$,"condition"),h$.forEach(s),lm=n(go," column, we can filter out the very short reviews by requiring that the reviews have a length above this threshold:"),go.forEach(s),ch=h(e),m(Ys.$$.fragment,e),fh=h(e),m(Gs.$$.fragment,e),uh=h(e),dn=r(e,"P",{});var p$=l(dn);im=n(p$,"As you can see, this has removed around 15% of the reviews from our original training and test sets."),p$.forEach(s),mh=h(e),m(jt.$$.fragment,e),wh=h(e),Dt=r(e,"P",{});var pc=l(Dt);dm=n(pc,"The last thing we need to deal with is the presence of HTML character codes in our reviews. We can use Python\u2019s "),Pr=r(pc,"CODE",{});var c$=l(Pr);hm=n(c$,"html"),c$.forEach(s),pm=n(pc," module to unescape these characters, like so:"),pc.forEach(s),gh=h(e),m(Js.$$.fragment,e),_h=h(e),m(Vs.$$.fragment,e),vh=h(e),Tt=r(e,"P",{});var cc=l(Tt);cm=n(cc,"We\u2019ll use "),Ar=r(cc,"CODE",{});var f$=l(Ar);fm=n(f$,"Dataset.map()"),f$.forEach(s),um=n(cc," to unescape all the HTML characters in our corpus:"),cc.forEach(s),bh=h(e),m(Xs.$$.fragment,e),$h=h(e),qt=r(e,"P",{});var fc=l(qt);mm=n(fc,"As you can see, the "),Nr=r(fc,"CODE",{});var u$=l(Nr);wm=n(u$,"Dataset.map()"),u$.forEach(s),gm=n(fc," method is quite useful for processing data \u2014 and we haven\u2019t even scratched the surface of everything it can do!"),fc.forEach(s),yh=h(e),Ze=r(e,"H2",{class:!0});var uc=l(Ze);Ct=r(uc,"A",{id:!0,class:!0,href:!0});var m$=l(Ct);Ir=r(m$,"SPAN",{});var w$=l(Ir);m(Ks.$$.fragment,w$),w$.forEach(s),m$.forEach(s),_m=h(uc),Qs=r(uc,"SPAN",{});var mc=l(Qs);vm=n(mc,"The "),Sr=r(mc,"CODE",{});var g$=l(Sr);bm=n(g$,"map()"),g$.forEach(s),$m=n(mc," method's superpowers"),mc.forEach(s),uc.forEach(s),Eh=h(e),le=r(e,"P",{});var ns=l(le);ym=n(ns,"The "),Hr=r(ns,"CODE",{});var _$=l(Hr);Em=n(_$,"Dataset.map()"),_$.forEach(s),xm=n(ns," method takes a "),Fr=r(ns,"CODE",{});var v$=l(Fr);km=n(v$,"batched"),v$.forEach(s),jm=n(ns," argument that, if set to "),Lr=r(ns,"CODE",{});var b$=l(Lr);Dm=n(b$,"True"),b$.forEach(s),Tm=n(ns,", causes it to send a batch of examples to the map function at once (the batch size is configurable but defaults to 1,000). For instance, the previous map function that unescaped all the HTML took a bit of time to run (you can read the time taken from the progress bars). We can speed this up by processing several elements at the same time using a list comprehension."),ns.forEach(s),xh=h(e),ee=r(e,"P",{});var Be=l(ee);qm=n(Be,"When you specify "),Mr=r(Be,"CODE",{});var $$=l(Mr);Cm=n($$,"batched=True"),$$.forEach(s),Om=n(Be," the function receives a dictionary with the fields of the dataset, but each value is now a "),Rr=r(Be,"EM",{});var y$=l(Rr);zm=n(y$,"list of values"),y$.forEach(s),Pm=n(Be,", and not just a single value. The return value of "),Wr=r(Be,"CODE",{});var E$=l(Wr);Am=n(E$,"Dataset.map()"),E$.forEach(s),Nm=n(Be," should be the same: a dictionary with the fields we want to update or add to our dataset, and a list of values. For example, here is another way to unescape all HTML characters, but using "),Ur=r(Be,"CODE",{});var x$=l(Ur);Im=n(x$,"batched=True"),x$.forEach(s),Sm=n(Be,":"),Be.forEach(s),kh=h(e),m(Zs.$$.fragment,e),jh=h(e),Oe=r(e,"P",{});var _o=l(Oe);Hm=n(_o,"If you\u2019re running this code in a notebook, you\u2019ll see that this command executes way faster than the previous one. And it\u2019s not because our reviews have already been HTML-unescaped \u2014 if you re-execute the instruction from the previous section (without "),Br=r(_o,"CODE",{});var k$=l(Br);Fm=n(k$,"batched=True"),k$.forEach(s),Lm=n(_o,"), it will take the same amount of time as before. This is because list comprehensions are usually faster than executing the same code in a "),Yr=r(_o,"CODE",{});var j$=l(Yr);Mm=n(j$,"for"),j$.forEach(s),Rm=n(_o," loop, and we also gain some performance by accessing lots of elements at the same time instead of one by one."),_o.forEach(s),Dh=h(e),ie=r(e,"P",{});var os=l(ie);Wm=n(os,"Using "),Gr=r(os,"CODE",{});var D$=l(Gr);Um=n(D$,"Dataset.map()"),D$.forEach(s),Bm=n(os," with "),Jr=r(os,"CODE",{});var T$=l(Jr);Ym=n(T$,"batched=True"),T$.forEach(s),Gm=n(os," will be essential to unlock the speed of the \u201Cfast\u201D tokenizers that we\u2019ll encounter in "),hn=r(os,"A",{href:!0});var q$=l(hn);Jm=n(q$,"Chapter 6"),q$.forEach(s),Vm=n(os,", which can quickly tokenize big lists of texts. For instance, to tokenize all the drug reviews with a fast tokenizer, we could use a function like this:"),os.forEach(s),Th=h(e),m(ea.$$.fragment,e),qh=h(e),de=r(e,"P",{});var rs=l(de);Xm=n(rs,"As you saw in "),pn=r(rs,"A",{href:!0});var C$=l(pn);Km=n(C$,"Chapter 3"),C$.forEach(s),Qm=n(rs,", we can pass one or several examples to the tokenizer, so we can use this function with or without "),Vr=r(rs,"CODE",{});var O$=l(Vr);Zm=n(O$,"batched=True"),O$.forEach(s),ew=n(rs,". Let\u2019s take this opportunity to compare the performance of the different options. In a notebook, you can time a one-line instruction by adding "),Xr=r(rs,"CODE",{});var z$=l(Xr);tw=n(z$,"%time"),z$.forEach(s),sw=n(rs," before the line of code you wish to measure:"),rs.forEach(s),Ch=h(e),m(ta.$$.fragment,e),Oh=h(e),Ot=r(e,"P",{});var wc=l(Ot);aw=n(wc,"You can also time a whole cell by putting "),Kr=r(wc,"CODE",{});var P$=l(Kr);nw=n(P$,"%%time"),P$.forEach(s),ow=n(wc," at the beginning of the cell. On the hardware we executed this on, it showed 10.8s for this instruction (it\u2019s the number written after \u201CWall time\u201D)."),wc.forEach(s),zh=h(e),m(zt.$$.fragment,e),Ph=h(e),cn=r(e,"P",{});var A$=l(cn);rw=n(A$,"Here are the results we obtained with and without batching, with a fast and a slow tokenizer:"),A$.forEach(s),Ah=h(e),Pt=r(e,"TABLE",{});var gc=l(Pt);Qr=r(gc,"THEAD",{});var N$=l(Qr);et=r(N$,"TR",{});var vo=l(et);fn=r(vo,"TH",{align:!0});var I$=l(fn);lw=n(I$,"Options"),I$.forEach(s),iw=h(vo),un=r(vo,"TH",{align:!0});var S$=l(un);dw=n(S$,"Fast tokenizer"),S$.forEach(s),hw=h(vo),mn=r(vo,"TH",{align:!0});var H$=l(mn);pw=n(H$,"Slow tokenizer"),H$.forEach(s),vo.forEach(s),N$.forEach(s),cw=h(gc),sa=r(gc,"TBODY",{});var _c=l(sa);tt=r(_c,"TR",{});var bo=l(tt);wn=r(bo,"TD",{align:!0});var F$=l(wn);Zr=r(F$,"CODE",{});var L$=l(Zr);fw=n(L$,"batched=True"),L$.forEach(s),F$.forEach(s),uw=h(bo),gn=r(bo,"TD",{align:!0});var M$=l(gn);mw=n(M$,"10.8s"),M$.forEach(s),ww=h(bo),_n=r(bo,"TD",{align:!0});var R$=l(_n);gw=n(R$,"4min41s"),R$.forEach(s),bo.forEach(s),_w=h(_c),st=r(_c,"TR",{});var $o=l(st);vn=r($o,"TD",{align:!0});var W$=l(vn);el=r(W$,"CODE",{});var U$=l(el);vw=n(U$,"batched=False"),U$.forEach(s),W$.forEach(s),bw=h($o),bn=r($o,"TD",{align:!0});var B$=l(bn);$w=n(B$,"59.2s"),B$.forEach(s),yw=h($o),$n=r($o,"TD",{align:!0});var Y$=l($n);Ew=n(Y$,"5min3s"),Y$.forEach(s),$o.forEach(s),_c.forEach(s),gc.forEach(s),Nh=h(e),ze=r(e,"P",{});var yo=l(ze);xw=n(yo,"This means that using a fast tokenizer with the "),tl=r(yo,"CODE",{});var G$=l(tl);kw=n(G$,"batched=True"),G$.forEach(s),jw=n(yo," option is 30 times faster than its slow counterpart with no batching \u2014 this is truly amazing! That\u2019s the main reason why fast tokenizers are the default when using "),sl=r(yo,"CODE",{});var J$=l(sl);Dw=n(J$,"AutoTokenizer"),J$.forEach(s),Tw=n(yo," (and why they are called \u201Cfast\u201D). They\u2019re able to achieve such a speedup because behind the scenes the tokenization code is executed in Rust, which is a language that makes it easy to parallelize code execution."),yo.forEach(s),Ih=h(e),yn=r(e,"P",{});var V$=l(yn);qw=n(V$,"Parallelization is also the reason for the nearly 6x speedup the fast tokenizer achieves with batching: you can\u2019t parallelize a single tokenization operation, but when you want to tokenize lots of texts at the same time you can just split the execution across several processes, each responsible for its own texts."),V$.forEach(s),Sh=h(e),_e=r(e,"P",{});var Ya=l(_e);al=r(Ya,"CODE",{});var X$=l(al);Cw=n(X$,"Dataset.map()"),X$.forEach(s),Ow=n(Ya," also has some parallelization capabilities of its own. Since they are not backed by Rust, they won\u2019t let a slow tokenizer catch up with a fast one, but they can still be helpful (especially if you\u2019re using a tokenizer that doesn\u2019t have a fast version). To enable multiprocessing, use the "),nl=r(Ya,"CODE",{});var K$=l(nl);zw=n(K$,"num_proc"),K$.forEach(s),Pw=n(Ya," argument and specify the number of processes to use in your call to "),ol=r(Ya,"CODE",{});var Q$=l(ol);Aw=n(Q$,"Dataset.map()"),Q$.forEach(s),Nw=n(Ya,":"),Ya.forEach(s),Hh=h(e),m(aa.$$.fragment,e),Fh=h(e),En=r(e,"P",{});var Z$=l(En);Iw=n(Z$,"You can experiment a little with timing to determine the optimal number of processes to use; in our case 8 seemed to produce the best speed gain. Here are the numbers we got with and without multiprocessing:"),Z$.forEach(s),Lh=h(e),At=r(e,"TABLE",{});var vc=l(At);rl=r(vc,"THEAD",{});var ey=l(rl);at=r(ey,"TR",{});var Eo=l(at);xn=r(Eo,"TH",{align:!0});var ty=l(xn);Sw=n(ty,"Options"),ty.forEach(s),Hw=h(Eo),kn=r(Eo,"TH",{align:!0});var sy=l(kn);Fw=n(sy,"Fast tokenizer"),sy.forEach(s),Lw=h(Eo),jn=r(Eo,"TH",{align:!0});var ay=l(jn);Mw=n(ay,"Slow tokenizer"),ay.forEach(s),Eo.forEach(s),ey.forEach(s),Rw=h(vc),ve=r(vc,"TBODY",{});var ls=l(ve);nt=r(ls,"TR",{});var xo=l(nt);Dn=r(xo,"TD",{align:!0});var ny=l(Dn);ll=r(ny,"CODE",{});var oy=l(ll);Ww=n(oy,"batched=True"),oy.forEach(s),ny.forEach(s),Uw=h(xo),Tn=r(xo,"TD",{align:!0});var ry=l(Tn);Bw=n(ry,"10.8s"),ry.forEach(s),Yw=h(xo),qn=r(xo,"TD",{align:!0});var ly=l(qn);Gw=n(ly,"4min41s"),ly.forEach(s),xo.forEach(s),Jw=h(ls),ot=r(ls,"TR",{});var ko=l(ot);Cn=r(ko,"TD",{align:!0});var iy=l(Cn);il=r(iy,"CODE",{});var dy=l(il);Vw=n(dy,"batched=False"),dy.forEach(s),iy.forEach(s),Xw=h(ko),On=r(ko,"TD",{align:!0});var hy=l(On);Kw=n(hy,"59.2s"),hy.forEach(s),Qw=h(ko),zn=r(ko,"TD",{align:!0});var py=l(zn);Zw=n(py,"5min3s"),py.forEach(s),ko.forEach(s),eg=h(ls),rt=r(ls,"TR",{});var jo=l(rt);Nt=r(jo,"TD",{align:!0});var bc=l(Nt);dl=r(bc,"CODE",{});var cy=l(dl);tg=n(cy,"batched=True"),cy.forEach(s),sg=n(bc,", "),hl=r(bc,"CODE",{});var fy=l(hl);ag=n(fy,"num_proc=8"),fy.forEach(s),bc.forEach(s),ng=h(jo),Pn=r(jo,"TD",{align:!0});var uy=l(Pn);og=n(uy,"6.52s"),uy.forEach(s),rg=h(jo),An=r(jo,"TD",{align:!0});var my=l(An);lg=n(my,"41.3s"),my.forEach(s),jo.forEach(s),ig=h(ls),lt=r(ls,"TR",{});var Do=l(lt);It=r(Do,"TD",{align:!0});var $c=l(It);pl=r($c,"CODE",{});var wy=l(pl);dg=n(wy,"batched=False"),wy.forEach(s),hg=n($c,", "),cl=r($c,"CODE",{});var gy=l(cl);pg=n(gy,"num_proc=8"),gy.forEach(s),$c.forEach(s),cg=h(Do),Nn=r(Do,"TD",{align:!0});var _y=l(Nn);fg=n(_y,"9.49s"),_y.forEach(s),ug=h(Do),In=r(Do,"TD",{align:!0});var vy=l(In);mg=n(vy,"45.2s"),vy.forEach(s),Do.forEach(s),ls.forEach(s),vc.forEach(s),Mh=h(e),he=r(e,"P",{});var is=l(he);wg=n(is,"Those are much more reasonable results for the slow tokenizer, but the performance of the fast tokenizer was also substantially improved. Note, however, that won\u2019t always be the case \u2014 for values of "),fl=r(is,"CODE",{});var by=l(fl);gg=n(by,"num_proc"),by.forEach(s),_g=n(is," other than 8, our tests showed that it was faster to use "),ul=r(is,"CODE",{});var $y=l(ul);vg=n($y,"batched=True"),$y.forEach(s),bg=n(is," without that option. In general, we don\u2019t recommend using Python multiprocessing for fast tokenizers with "),ml=r(is,"CODE",{});var yy=l(ml);$g=n(yy,"batched=True"),yy.forEach(s),yg=n(is,"."),is.forEach(s),Rh=h(e),m(St.$$.fragment,e),Wh=h(e),pe=r(e,"P",{});var ds=l(pe);Eg=n(ds,"All of this functionality condensed into a single method is already pretty amazing, but there\u2019s more! With "),wl=r(ds,"CODE",{});var Ey=l(wl);xg=n(Ey,"Dataset.map()"),Ey.forEach(s),kg=n(ds," and "),gl=r(ds,"CODE",{});var xy=l(gl);jg=n(xy,"batched=True"),xy.forEach(s),Dg=n(ds," you can change the number of elements in your dataset. This is super useful in many situations where you want to create several training features from one example, and we will need to do this as part of the preprocessing for several of the NLP tasks we\u2019ll undertake in "),Sn=r(ds,"A",{href:!0});var ky=l(Sn);Tg=n(ky,"Chapter 7"),ky.forEach(s),qg=n(ds,"."),ds.forEach(s),Uh=h(e),m(Ht.$$.fragment,e),Bh=h(e),Pe=r(e,"P",{});var To=l(Pe);Cg=n(To,"Let\u2019s have a look at how it works! Here we will tokenize our examples and truncate them to a maximum length of 128, but we will ask the tokenizer to return "),_l=r(To,"EM",{});var jy=l(_l);Og=n(jy,"all"),jy.forEach(s),zg=n(To," the chunks of the texts instead of just the first one. This can be done with "),vl=r(To,"CODE",{});var Dy=l(vl);Pg=n(Dy,"return_overflowing_tokens=True"),Dy.forEach(s),Ag=n(To,":"),To.forEach(s),Yh=h(e),m(na.$$.fragment,e),Gh=h(e),Ft=r(e,"P",{});var yc=l(Ft);Ng=n(yc,"Let\u2019s test this on one example before using "),bl=r(yc,"CODE",{});var Ty=l(bl);Ig=n(Ty,"Dataset.map()"),Ty.forEach(s),Sg=n(yc," on the whole dataset:"),yc.forEach(s),Jh=h(e),m(oa.$$.fragment,e),Vh=h(e),m(ra.$$.fragment,e),Xh=h(e),Hn=r(e,"P",{});var qy=l(Hn);Hg=n(qy,"So, our first example in the training set became two features because it was tokenized to more than the maximum number of tokens we specified: the first one of length 128 and the second one of length 49. Now let\u2019s do this for all elements of the dataset!"),qy.forEach(s),Kh=h(e),m(la.$$.fragment,e),Qh=h(e),m(ia.$$.fragment,e),Zh=h(e),Ae=r(e,"P",{});var qo=l(Ae);Fg=n(qo,"Oh no! That didn\u2019t work! Why not? Looking at the error message will give us a clue: there is a mismatch in the lengths of one of the columns, one being of length 1,463 and the other of length 1,000. If you\u2019ve looked at the "),$l=r(qo,"CODE",{});var Cy=l($l);Lg=n(Cy,"Dataset.map()"),Cy.forEach(s),Mg=h(qo),da=r(qo,"A",{href:!0,rel:!0});var Oy=l(da);Rg=n(Oy,"documentation"),Oy.forEach(s),Wg=n(qo,", you may recall that it\u2019s the number of samples passed to the function that we are mapping; here those 1,000 examples gave 1,463 new features, resulting in a shape error."),qo.forEach(s),ep=h(e),te=r(e,"P",{});var Ye=l(te);Ug=n(Ye,"The problem is that we\u2019re trying to mix two different datasets of different sizes: the "),yl=r(Ye,"CODE",{});var zy=l(yl);Bg=n(zy,"drug_dataset"),zy.forEach(s),Yg=n(Ye," columns will have a certain number of examples (the 1,000 in our error), but the "),El=r(Ye,"CODE",{});var Py=l(El);Gg=n(Py,"tokenized_dataset"),Py.forEach(s),Jg=n(Ye," we are building will have more (the 1,463 in the error message). That doesn\u2019t work for a "),xl=r(Ye,"CODE",{});var Ay=l(xl);Vg=n(Ay,"Dataset"),Ay.forEach(s),Xg=n(Ye,", so we need to either remove the columns from the old dataset or make them the same size as they are in the new dataset. We can do the former with the "),kl=r(Ye,"CODE",{});var Ny=l(kl);Kg=n(Ny,"remove_columns"),Ny.forEach(s),Qg=n(Ye," argument:"),Ye.forEach(s),tp=h(e),m(ha.$$.fragment,e),sp=h(e),Fn=r(e,"P",{});var Iy=l(Fn);Zg=n(Iy,"Now this works without error. We can check that our new dataset has many more elements than the original dataset by comparing the lengths:"),Iy.forEach(s),ap=h(e),m(pa.$$.fragment,e),np=h(e),m(ca.$$.fragment,e),op=h(e),Ne=r(e,"P",{});var Co=l(Ne);e_=n(Co,"We mentioned that we can also deal with the mismatched length problem by making the old columns the same size as the new ones. To do this, we will need the "),jl=r(Co,"CODE",{});var Sy=l(jl);t_=n(Sy,"overflow_to_sample_mapping"),Sy.forEach(s),s_=n(Co," field the tokenizer returns when we set "),Dl=r(Co,"CODE",{});var Hy=l(Dl);a_=n(Hy,"return_overflowing_tokens=True"),Hy.forEach(s),n_=n(Co,". It gives us a mapping from a new feature index to the index of the sample it originated from. Using this, we can associate each key present in our original dataset with a list of values of the right size by repeating the values of each example as many times as it generates new features:"),Co.forEach(s),rp=h(e),m(fa.$$.fragment,e),lp=h(e),Lt=r(e,"P",{});var Ec=l(Lt);o_=n(Ec,"We can see it works with "),Tl=r(Ec,"CODE",{});var Fy=l(Tl);r_=n(Fy,"Dataset.map()"),Fy.forEach(s),l_=n(Ec," without us needing to remove the old columns:"),Ec.forEach(s),ip=h(e),m(ua.$$.fragment,e),dp=h(e),m(ma.$$.fragment,e),hp=h(e),Ln=r(e,"P",{});var Ly=l(Ln);i_=n(Ly,"We get the same number of training features as before, but here we\u2019ve kept all the old fields. If you need them for some post-processing after applying your model, you might want to use this approach."),Ly.forEach(s),pp=h(e),Mt=r(e,"P",{});var xc=l(Mt);d_=n(xc,`You\u2019ve now seen how \u{1F917} Datasets can be used to preprocess a dataset in various ways. Although the processing functions of \u{1F917} Datasets will cover most of your model training needs,
there may be times when you\u2019ll need to switch to Pandas to access more powerful features, like `),ql=r(xc,"CODE",{});var My=l(ql);h_=n(My,"DataFrame.groupby()"),My.forEach(s),p_=n(xc," or high-level APIs for visualization. Fortunately, \u{1F917} Datasets is designed to be interoperable with libraries such as Pandas, NumPy, PyTorch, TensorFlow, and JAX. Let\u2019s take a look at how this works."),xc.forEach(s),cp=h(e),it=r(e,"H2",{class:!0});var kc=l(it);Rt=r(kc,"A",{id:!0,class:!0,href:!0});var Ry=l(Rt);Cl=r(Ry,"SPAN",{});var Wy=l(Cl);m(wa.$$.fragment,Wy),Wy.forEach(s),Ry.forEach(s),c_=h(kc),dt=r(kc,"SPAN",{});var Oo=l(dt);f_=n(Oo,"From "),Ol=r(Oo,"CODE",{});var Uy=l(Ol);u_=n(Uy,"Dataset"),Uy.forEach(s),m_=n(Oo,"s to "),zl=r(Oo,"CODE",{});var By=l(zl);w_=n(By,"DataFrame"),By.forEach(s),g_=n(Oo,"s and back"),Oo.forEach(s),kc.forEach(s),fp=h(e),m(ga.$$.fragment,e),up=h(e),ce=r(e,"P",{});var hs=l(ce);__=n(hs,"To enable the conversion between various third-party libraries, \u{1F917} Datasets provides a "),Pl=r(hs,"CODE",{});var Yy=l(Pl);v_=n(Yy,"Dataset.set_format()"),Yy.forEach(s),b_=n(hs," function. This function only changes the "),Al=r(hs,"EM",{});var Gy=l(Al);$_=n(Gy,"output format"),Gy.forEach(s),y_=n(hs," of the dataset, so you can easily switch to another format without affecting the underlying "),Nl=r(hs,"EM",{});var Jy=l(Nl);E_=n(Jy,"data format"),Jy.forEach(s),x_=n(hs,", which is Apache Arrow. The formatting is done in place. To demonstrate, let\u2019s convert our dataset to Pandas:"),hs.forEach(s),mp=h(e),m(_a.$$.fragment,e),wp=h(e),Wt=r(e,"P",{});var jc=l(Wt);k_=n(jc,"Now when we access elements of the dataset we get a "),Il=r(jc,"CODE",{});var Vy=l(Il);j_=n(Vy,"pandas.DataFrame"),Vy.forEach(s),D_=n(jc," instead of a dictionary:"),jc.forEach(s),gp=h(e),m(va.$$.fragment,e),_p=h(e),Ie=r(e,"TABLE",{border:!0,class:!0});var Dc=l(Ie);Sl=r(Dc,"THEAD",{});var Xy=l(Sl);I=r(Xy,"TR",{style:!0});var W=l(I);vp=r(W,"TH",{}),l(vp).forEach(s),T_=h(W),Hl=r(W,"TH",{});var Ky=l(Hl);q_=n(Ky,"patient_id"),Ky.forEach(s),C_=h(W),Fl=r(W,"TH",{});var Qy=l(Fl);O_=n(Qy,"drugName"),Qy.forEach(s),z_=h(W),Ll=r(W,"TH",{});var Zy=l(Ll);P_=n(Zy,"condition"),Zy.forEach(s),A_=h(W),Ml=r(W,"TH",{});var e7=l(Ml);N_=n(e7,"review"),e7.forEach(s),I_=h(W),Rl=r(W,"TH",{});var t7=l(Rl);S_=n(t7,"rating"),t7.forEach(s),H_=h(W),Wl=r(W,"TH",{});var s7=l(Wl);F_=n(s7,"date"),s7.forEach(s),L_=h(W),Ul=r(W,"TH",{});var a7=l(Ul);M_=n(a7,"usefulCount"),a7.forEach(s),R_=h(W),Bl=r(W,"TH",{});var n7=l(Bl);W_=n(n7,"review_length"),n7.forEach(s),W.forEach(s),Xy.forEach(s),U_=h(Dc),ht=r(Dc,"TBODY",{});var zo=l(ht);S=r(zo,"TR",{});var U=l(S);Yl=r(U,"TH",{});var o7=l(Yl);B_=n(o7,"0"),o7.forEach(s),Y_=h(U),Gl=r(U,"TD",{});var r7=l(Gl);G_=n(r7,"95260"),r7.forEach(s),J_=h(U),Jl=r(U,"TD",{});var l7=l(Jl);V_=n(l7,"Guanfacine"),l7.forEach(s),X_=h(U),Vl=r(U,"TD",{});var i7=l(Vl);K_=n(i7,"adhd"),i7.forEach(s),Q_=h(U),Xl=r(U,"TD",{});var d7=l(Xl);Z_=n(d7,'"My son is halfway through his fourth week of Intuniv..."'),d7.forEach(s),ev=h(U),Kl=r(U,"TD",{});var h7=l(Kl);tv=n(h7,"8.0"),h7.forEach(s),sv=h(U),Ql=r(U,"TD",{});var p7=l(Ql);av=n(p7,"April 27, 2010"),p7.forEach(s),nv=h(U),Zl=r(U,"TD",{});var c7=l(Zl);ov=n(c7,"192"),c7.forEach(s),rv=h(U),ei=r(U,"TD",{});var f7=l(ei);lv=n(f7,"141"),f7.forEach(s),U.forEach(s),iv=h(zo),H=r(zo,"TR",{});var B=l(H);ti=r(B,"TH",{});var u7=l(ti);dv=n(u7,"1"),u7.forEach(s),hv=h(B),si=r(B,"TD",{});var m7=l(si);pv=n(m7,"92703"),m7.forEach(s),cv=h(B),ai=r(B,"TD",{});var w7=l(ai);fv=n(w7,"Lybrel"),w7.forEach(s),uv=h(B),ni=r(B,"TD",{});var g7=l(ni);mv=n(g7,"birth control"),g7.forEach(s),wv=h(B),oi=r(B,"TD",{});var _7=l(oi);gv=n(_7,'"I used to take another oral contraceptive, which had 21 pill cycle, and was very happy- very light periods, max 5 days, no other side effects..."'),_7.forEach(s),_v=h(B),ri=r(B,"TD",{});var v7=l(ri);vv=n(v7,"5.0"),v7.forEach(s),bv=h(B),li=r(B,"TD",{});var b7=l(li);$v=n(b7,"December 14, 2009"),b7.forEach(s),yv=h(B),ii=r(B,"TD",{});var $7=l(ii);Ev=n($7,"17"),$7.forEach(s),xv=h(B),di=r(B,"TD",{});var y7=l(di);kv=n(y7,"134"),y7.forEach(s),B.forEach(s),jv=h(zo),F=r(zo,"TR",{});var Y=l(F);hi=r(Y,"TH",{});var E7=l(hi);Dv=n(E7,"2"),E7.forEach(s),Tv=h(Y),pi=r(Y,"TD",{});var x7=l(pi);qv=n(x7,"138000"),x7.forEach(s),Cv=h(Y),ci=r(Y,"TD",{});var k7=l(ci);Ov=n(k7,"Ortho Evra"),k7.forEach(s),zv=h(Y),fi=r(Y,"TD",{});var j7=l(fi);Pv=n(j7,"birth control"),j7.forEach(s),Av=h(Y),ui=r(Y,"TD",{});var D7=l(ui);Nv=n(D7,'"This is my first time using any form of birth control..."'),D7.forEach(s),Iv=h(Y),mi=r(Y,"TD",{});var T7=l(mi);Sv=n(T7,"8.0"),T7.forEach(s),Hv=h(Y),wi=r(Y,"TD",{});var q7=l(wi);Fv=n(q7,"November 3, 2015"),q7.forEach(s),Lv=h(Y),gi=r(Y,"TD",{});var C7=l(gi);Mv=n(C7,"10"),C7.forEach(s),Rv=h(Y),_i=r(Y,"TD",{});var O7=l(_i);Wv=n(O7,"89"),O7.forEach(s),Y.forEach(s),zo.forEach(s),Dc.forEach(s),bp=h(e),Se=r(e,"P",{});var Po=l(Se);Uv=n(Po,"Let\u2019s create a "),vi=r(Po,"CODE",{});var z7=l(vi);Bv=n(z7,"pandas.DataFrame"),z7.forEach(s),Yv=n(Po," for the whole training set by selecting all the elements of "),bi=r(Po,"CODE",{});var P7=l(bi);Gv=n(P7,'drug_dataset["train"]'),P7.forEach(s),Jv=n(Po,":"),Po.forEach(s),$p=h(e),m(ba.$$.fragment,e),yp=h(e),m(Ut.$$.fragment,e),Ep=h(e),Bt=r(e,"P",{});var Tc=l(Bt);Vv=n(Tc,"From here we can use all the Pandas functionality that we want. For example, we can do fancy chaining to compute the class distribution among the "),$i=r(Tc,"CODE",{});var A7=l($i);Xv=n(A7,"condition"),A7.forEach(s),Kv=n(Tc," entries:"),Tc.forEach(s),xp=h(e),m($a.$$.fragment,e),kp=h(e),He=r(e,"TABLE",{border:!0,class:!0});var qc=l(He);yi=r(qc,"THEAD",{});var N7=l(yi);Fe=r(N7,"TR",{style:!0});var Ao=l(Fe);jp=r(Ao,"TH",{}),l(jp).forEach(s),Qv=h(Ao),Ei=r(Ao,"TH",{});var I7=l(Ei);Zv=n(I7,"condition"),I7.forEach(s),e2=h(Ao),xi=r(Ao,"TH",{});var S7=l(xi);t2=n(S7,"frequency"),S7.forEach(s),Ao.forEach(s),N7.forEach(s),s2=h(qc),ae=r(qc,"TBODY",{});var Ge=l(ae);pt=r(Ge,"TR",{});var No=l(pt);ki=r(No,"TH",{});var H7=l(ki);a2=n(H7,"0"),H7.forEach(s),n2=h(No),ji=r(No,"TD",{});var F7=l(ji);o2=n(F7,"birth control"),F7.forEach(s),r2=h(No),Di=r(No,"TD",{});var L7=l(Di);l2=n(L7,"27655"),L7.forEach(s),No.forEach(s),i2=h(Ge),ct=r(Ge,"TR",{});var Io=l(ct);Ti=r(Io,"TH",{});var M7=l(Ti);d2=n(M7,"1"),M7.forEach(s),h2=h(Io),qi=r(Io,"TD",{});var R7=l(qi);p2=n(R7,"depression"),R7.forEach(s),c2=h(Io),Ci=r(Io,"TD",{});var W7=l(Ci);f2=n(W7,"8023"),W7.forEach(s),Io.forEach(s),u2=h(Ge),ft=r(Ge,"TR",{});var So=l(ft);Oi=r(So,"TH",{});var U7=l(Oi);m2=n(U7,"2"),U7.forEach(s),w2=h(So),zi=r(So,"TD",{});var B7=l(zi);g2=n(B7,"acne"),B7.forEach(s),_2=h(So),Pi=r(So,"TD",{});var Y7=l(Pi);v2=n(Y7,"5209"),Y7.forEach(s),So.forEach(s),b2=h(Ge),ut=r(Ge,"TR",{});var Ho=l(ut);Ai=r(Ho,"TH",{});var G7=l(Ai);$2=n(G7,"3"),G7.forEach(s),y2=h(Ho),Ni=r(Ho,"TD",{});var J7=l(Ni);E2=n(J7,"anxiety"),J7.forEach(s),x2=h(Ho),Ii=r(Ho,"TD",{});var V7=l(Ii);k2=n(V7,"4991"),V7.forEach(s),Ho.forEach(s),j2=h(Ge),mt=r(Ge,"TR",{});var Fo=l(mt);Si=r(Fo,"TH",{});var X7=l(Si);D2=n(X7,"4"),X7.forEach(s),T2=h(Fo),Hi=r(Fo,"TD",{});var K7=l(Hi);q2=n(K7,"pain"),K7.forEach(s),C2=h(Fo),Fi=r(Fo,"TD",{});var Q7=l(Fi);O2=n(Q7,"4744"),Q7.forEach(s),Fo.forEach(s),Ge.forEach(s),qc.forEach(s),Dp=h(e),Le=r(e,"P",{});var Lo=l(Le);z2=n(Lo,"And once we\u2019re done with our Pandas analysis, we can always create a new "),Li=r(Lo,"CODE",{});var Z7=l(Li);P2=n(Z7,"Dataset"),Z7.forEach(s),A2=n(Lo," object by using the "),Mi=r(Lo,"CODE",{});var eE=l(Mi);N2=n(eE,"Dataset.from_pandas()"),eE.forEach(s),I2=n(Lo," function as follows:"),Lo.forEach(s),Tp=h(e),m(ya.$$.fragment,e),qp=h(e),m(Ea.$$.fragment,e),Cp=h(e),m(Yt.$$.fragment,e),Op=h(e),fe=r(e,"P",{});var ps=l(fe);S2=n(ps,"This wraps up our tour of the various preprocessing techniques available in \u{1F917} Datasets. To round out the section, let\u2019s create a validation set to prepare the dataset for training a classifier on. Before doing so, we\u2019ll reset the output format of "),Ri=r(ps,"CODE",{});var tE=l(Ri);H2=n(tE,"drug_dataset"),tE.forEach(s),F2=n(ps," from "),Wi=r(ps,"CODE",{});var sE=l(Wi);L2=n(sE,'"pandas"'),sE.forEach(s),M2=n(ps," to "),Ui=r(ps,"CODE",{});var aE=l(Ui);R2=n(aE,'"arrow"'),aE.forEach(s),W2=n(ps,":"),ps.forEach(s),zp=h(e),m(xa.$$.fragment,e),Pp=h(e),wt=r(e,"H2",{class:!0});var Cc=l(wt);Gt=r(Cc,"A",{id:!0,class:!0,href:!0});var nE=l(Gt);Bi=r(nE,"SPAN",{});var oE=l(Bi);m(ka.$$.fragment,oE),oE.forEach(s),nE.forEach(s),U2=h(Cc),Yi=r(Cc,"SPAN",{});var rE=l(Yi);B2=n(rE,"Creating a validation set"),rE.forEach(s),Cc.forEach(s),Ap=h(e),Mn=r(e,"P",{});var lE=l(Mn);Y2=n(lE,"Although we have a test set we could use for evaluation, it\u2019s a good practice to leave the test set untouched and create a separate validation set during development. Once you are happy with the performance of your models on the validation set, you can do a final sanity check on the test set. This process helps mitigate the risk that you\u2019ll overfit to the test set and deploy a model that fails on real-world data."),lE.forEach(s),Np=h(e),V=r(e,"P",{});var we=l(V);G2=n(we,"\u{1F917} Datasets provides a "),Gi=r(we,"CODE",{});var iE=l(Gi);J2=n(iE,"Dataset.train_test_split()"),iE.forEach(s),V2=n(we," function that is based on the famous functionality from "),Ji=r(we,"CODE",{});var dE=l(Ji);X2=n(dE,"scikit-learn"),dE.forEach(s),K2=n(we,". Let\u2019s use it to split our training set into "),Vi=r(we,"CODE",{});var hE=l(Vi);Q2=n(hE,"train"),hE.forEach(s),Z2=n(we," and "),Xi=r(we,"CODE",{});var pE=l(Xi);e1=n(pE,"validation"),pE.forEach(s),t1=n(we," splits (we set the "),Ki=r(we,"CODE",{});var cE=l(Ki);s1=n(cE,"seed"),cE.forEach(s),a1=n(we," argument for reproducibility):"),we.forEach(s),Ip=h(e),m(ja.$$.fragment,e),Sp=h(e),m(Da.$$.fragment,e),Hp=h(e),Jt=r(e,"P",{});var Oc=l(Jt);n1=n(Oc,"Great, we\u2019ve now prepared a dataset that\u2019s ready for training some models on! In "),Rn=r(Oc,"A",{href:!0});var fE=l(Rn);o1=n(fE,"section 5"),fE.forEach(s),r1=n(Oc," we\u2019ll show you how to upload datasets to the Hugging Face Hub, but for now let\u2019s cap off our analysis by looking at a few ways you can save datasets on your local machine."),Oc.forEach(s),Fp=h(e),gt=r(e,"H2",{class:!0});var zc=l(gt);Vt=r(zc,"A",{id:!0,class:!0,href:!0});var uE=l(Vt);Qi=r(uE,"SPAN",{});var mE=l(Qi);m(Ta.$$.fragment,mE),mE.forEach(s),uE.forEach(s),l1=h(zc),Zi=r(zc,"SPAN",{});var wE=l(Zi);i1=n(wE,"Saving a dataset"),wE.forEach(s),zc.forEach(s),Lp=h(e),m(qa.$$.fragment,e),Mp=h(e),Wn=r(e,"P",{});var gE=l(Wn);d1=n(gE,"Although \u{1F917} Datasets will cache every downloaded dataset and the operations performed on it, there are times when you\u2019ll want to save a dataset to disk (e.g., in case the cache gets deleted). As shown in the table below, \u{1F917} Datasets provides three main functions to save your dataset in different formats:"),gE.forEach(s),Rp=h(e),Xt=r(e,"TABLE",{});var Pc=l(Xt);ed=r(Pc,"THEAD",{});var _E=l(ed);Ca=r(_E,"TR",{});var Ac=l(Ca);Un=r(Ac,"TH",{align:!0});var vE=l(Un);h1=n(vE,"Data format"),vE.forEach(s),p1=h(Ac),Bn=r(Ac,"TH",{align:!0});var bE=l(Bn);c1=n(bE,"Function"),bE.forEach(s),Ac.forEach(s),_E.forEach(s),f1=h(Pc),_t=r(Pc,"TBODY",{});var Mo=l(_t);Oa=r(Mo,"TR",{});var Nc=l(Oa);Yn=r(Nc,"TD",{align:!0});var $E=l(Yn);u1=n($E,"Arrow"),$E.forEach(s),m1=h(Nc),Gn=r(Nc,"TD",{align:!0});var yE=l(Gn);td=r(yE,"CODE",{});var EE=l(td);w1=n(EE,"Dataset.save_to_disk()"),EE.forEach(s),yE.forEach(s),Nc.forEach(s),g1=h(Mo),za=r(Mo,"TR",{});var Ic=l(za);Jn=r(Ic,"TD",{align:!0});var xE=l(Jn);_1=n(xE,"CSV"),xE.forEach(s),v1=h(Ic),Vn=r(Ic,"TD",{align:!0});var kE=l(Vn);sd=r(kE,"CODE",{});var jE=l(sd);b1=n(jE,"Dataset.to_csv()"),jE.forEach(s),kE.forEach(s),Ic.forEach(s),$1=h(Mo),Pa=r(Mo,"TR",{});var Sc=l(Pa);Xn=r(Sc,"TD",{align:!0});var DE=l(Xn);y1=n(DE,"JSON"),DE.forEach(s),E1=h(Sc),Kn=r(Sc,"TD",{align:!0});var TE=l(Kn);ad=r(TE,"CODE",{});var qE=l(ad);x1=n(qE,"Dataset.to_json()"),qE.forEach(s),TE.forEach(s),Sc.forEach(s),Mo.forEach(s),Pc.forEach(s),Wp=h(e),Qn=r(e,"P",{});var CE=l(Qn);k1=n(CE,"For example, let\u2019s save our cleaned dataset in the Arrow format:"),CE.forEach(s),Up=h(e),m(Aa.$$.fragment,e),Bp=h(e),Zn=r(e,"P",{});var OE=l(Zn);j1=n(OE,"This will create a directory with the following structure:"),OE.forEach(s),Yp=h(e),m(Na.$$.fragment,e),Gp=h(e),ue=r(e,"P",{});var cs=l(ue);D1=n(cs,"where we can see that each split is associated with its own "),nd=r(cs,"EM",{});var zE=l(nd);T1=n(zE,"dataset.arrow"),zE.forEach(s),q1=n(cs," table, and some metadata in "),od=r(cs,"EM",{});var PE=l(od);C1=n(PE,"dataset_info.json"),PE.forEach(s),O1=n(cs," and "),rd=r(cs,"EM",{});var AE=l(rd);z1=n(AE,"state.json"),AE.forEach(s),P1=n(cs,". You can think of the Arrow format as a fancy table of columns and rows that is optimized for building high-performance applications that process and transport large datasets."),cs.forEach(s),Jp=h(e),Kt=r(e,"P",{});var Hc=l(Kt);A1=n(Hc,"Once the dataset is saved, we can load it by using the "),ld=r(Hc,"CODE",{});var NE=l(ld);N1=n(NE,"load_from_disk()"),NE.forEach(s),I1=n(Hc," function as follows:"),Hc.forEach(s),Vp=h(e),m(Ia.$$.fragment,e),Xp=h(e),m(Sa.$$.fragment,e),Kp=h(e),Qt=r(e,"P",{});var Fc=l(Qt);S1=n(Fc,"For the CSV and JSON formats, we have to store each split as a separate file. One way to do this is by iterating over the keys and values in the "),id=r(Fc,"CODE",{});var IE=l(id);H1=n(IE,"DatasetDict"),IE.forEach(s),F1=n(Fc," object:"),Fc.forEach(s),Qp=h(e),m(Ha.$$.fragment,e),Zp=h(e),Zt=r(e,"P",{});var Lc=l(Zt);L1=n(Lc,"This saves each split in "),Fa=r(Lc,"A",{href:!0,rel:!0});var SE=l(Fa);M1=n(SE,"JSON Lines format"),SE.forEach(s),R1=n(Lc,", where each row in the dataset is stored as a single line of JSON. Here\u2019s what the first example looks like:"),Lc.forEach(s),ec=h(e),m(La.$$.fragment,e),tc=h(e),m(Ma.$$.fragment,e),sc=h(e),es=r(e,"P",{});var Mc=l(es);W1=n(Mc,"We can then use the techniques from "),eo=r(Mc,"A",{href:!0});var HE=l(eo);U1=n(HE,"section 2"),HE.forEach(s),B1=n(Mc," to load the JSON files as follows:"),Mc.forEach(s),ac=h(e),m(Ra.$$.fragment,e),nc=h(e),to=r(e,"P",{});var FE=l(to);Y1=n(FE,"And that\u2019s it for our excursion into data wrangling with \u{1F917} Datasets! Now that we have a cleaned dataset for training a model on, here are a few ideas that you could try out:"),FE.forEach(s),oc=h(e),ts=r(e,"OL",{});var Rc=l(ts);Wa=r(Rc,"LI",{});var Wc=l(Wa);G1=n(Wc,"Use the techniques from "),so=r(Wc,"A",{href:!0});var LE=l(so);J1=n(LE,"Chapter 3"),LE.forEach(s),V1=n(Wc," to train a classifier that can predict the patient condition based on the drug review."),Wc.forEach(s),X1=h(Rc),vt=r(Rc,"LI",{});var Ro=l(vt);K1=n(Ro,"Use the "),dd=r(Ro,"CODE",{});var ME=l(dd);Q1=n(ME,"summarization"),ME.forEach(s),Z1=n(Ro," pipeline from "),ao=r(Ro,"A",{href:!0});var RE=l(ao);eb=n(RE,"Chapter 1"),RE.forEach(s),tb=n(Ro," to generate summaries of the reviews."),Ro.forEach(s),Rc.forEach(s),rc=h(e),no=r(e,"P",{});var WE=l(no);sb=n(WE,"Next, we\u2019ll take a look at how \u{1F917} Datasets can enable you to work with huge datasets without blowing up your laptop!"),WE.forEach(s),this.h()},h(){f(c,"name","hf:doc:metadata"),f(c,"content",JSON.stringify(rx)),f(k,"id","time-to-slice-and-dice"),f(k,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),f(k,"href","#time-to-slice-and-dice"),f($,"class","relative group"),f(X,"id","slicing-and-dicing-our-data"),f(X,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),f(X,"href","#slicing-and-dicing-our-data"),f(M,"class","relative group"),f(en,"href","/course/chapter3"),f(ms,"href","https://archive.ics.uci.edu/ml/datasets/Drug+Review+Dataset+%28Drugs.com%29"),f(ms,"rel","nofollow"),f(ws,"href","https://archive.ics.uci.edu/ml/index.php"),f(ws,"rel","nofollow"),f(tn,"href","/course/chapter3"),f(Cs,"href","https://docs.python.org/3/reference/lexical_analysis.html#keywords"),f(Cs,"rel","nofollow"),f(Is,"href","https://realpython.com/python-lambda/"),f(Is,"rel","nofollow"),f(xt,"id","creating-new-columns"),f(xt,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),f(xt,"href","#creating-new-columns"),f(Qe,"class","relative group"),f(Ct,"id","the-map-methods-superpowers"),f(Ct,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),f(Ct,"href","#the-map-methods-superpowers"),f(Ze,"class","relative group"),f(hn,"href","/course/chapter6"),f(pn,"href","/course/chapter3"),f(fn,"align","center"),f(un,"align","center"),f(mn,"align","center"),f(wn,"align","center"),f(gn,"align","center"),f(_n,"align","center"),f(vn,"align","center"),f(bn,"align","center"),f($n,"align","center"),f(xn,"align","center"),f(kn,"align","center"),f(jn,"align","center"),f(Dn,"align","center"),f(Tn,"align","center"),f(qn,"align","center"),f(Cn,"align","center"),f(On,"align","center"),f(zn,"align","center"),f(Nt,"align","center"),f(Pn,"align","center"),f(An,"align","center"),f(It,"align","center"),f(Nn,"align","center"),f(In,"align","center"),f(Sn,"href","/course/chapter7"),f(da,"href","https://huggingface.co/docs/datasets/package_reference/main_classes.html#datasets.Dataset.map"),f(da,"rel","nofollow"),f(Rt,"id","from-datasets-to-dataframes-and-back"),f(Rt,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),f(Rt,"href","#from-datasets-to-dataframes-and-back"),f(it,"class","relative group"),UE(I,"text-align","right"),f(Ie,"border","1"),f(Ie,"class","dataframe"),UE(Fe,"text-align","right"),f(He,"border","1"),f(He,"class","dataframe"),f(Gt,"id","creating-a-validation-set"),f(Gt,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),f(Gt,"href","#creating-a-validation-set"),f(wt,"class","relative group"),f(Rn,"href","/course/chapter5/5"),f(Vt,"id","saving-a-dataset"),f(Vt,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),f(Vt,"href","#saving-a-dataset"),f(gt,"class","relative group"),f(Un,"align","center"),f(Bn,"align","center"),f(Yn,"align","center"),f(Gn,"align","center"),f(Jn,"align","center"),f(Vn,"align","center"),f(Xn,"align","center"),f(Kn,"align","center"),f(Fa,"href","https://jsonlines.org"),f(Fa,"rel","nofollow"),f(eo,"href","/course/chapter5/2"),f(so,"href","/course/chapter3"),f(ao,"href","/course/chapter1")},m(e,i){t(document.head,c),p(e,q,i),p(e,$,i),t($,k),t(k,D),w(y,D,null),t($,j),t($,C),t(C,E),p(e,x,i),w(z,e,i),p(e,T,i),p(e,O,i),t(O,G),p(e,N,i),w(P,e,i),p(e,ge,i),p(e,M,i),t(M,X),t(X,ne),w(se,ne,null),t(M,us),t(M,Je),t(Je,Ve),p(e,bt,i),p(e,R,i),t(R,Xe),t(R,A),t(A,Ja),t(R,Va),t(R,$t),t($t,Xa),t(R,Ka),t(R,yt),t(yt,Qa),t(R,Za),t(R,en),t(en,Uc),t(R,Bc),p(e,wd,i),p(e,be,i),t(be,Yc),t(be,ms),t(ms,Gc),t(be,Jc),t(be,ws),t(ws,Vc),t(be,Xc),p(e,gd,i),p(e,$e,i),t($e,Kc),t($e,Wo),t(Wo,Qc),t($e,Zc),t($e,Uo),t(Uo,ef),t($e,tf),p(e,_d,i),w(gs,e,i),p(e,vd,i),p(e,oe,i),t(oe,sf),t(oe,Bo),t(Bo,af),t(oe,nf),t(oe,Yo),t(Yo,of),t(oe,rf),t(oe,Go),t(Go,lf),t(oe,df),p(e,bd,i),w(_s,e,i),p(e,$d,i),p(e,ye,i),t(ye,hf),t(ye,Jo),t(Jo,pf),t(ye,cf),t(ye,Vo),t(Vo,ff),t(ye,uf),p(e,yd,i),w(vs,e,i),p(e,Ed,i),w(bs,e,i),p(e,xd,i),p(e,re,i),t(re,mf),t(re,Xo),t(Xo,wf),t(re,gf),t(re,Ko),t(Ko,_f),t(re,vf),t(re,Qo),t(Qo,bf),t(re,$f),p(e,kd,i),p(e,Ee,i),t(Ee,$s),t($s,yf),t($s,Zo),t(Zo,Ef),t($s,xf),t(Ee,kf),t(Ee,ys),t(ys,jf),t(ys,er),t(er,Df),t(ys,Tf),t(Ee,qf),t(Ee,Ke),t(Ke,Cf),t(Ke,tr),t(tr,Of),t(Ke,zf),t(Ke,sr),t(sr,Pf),t(Ke,Af),p(e,jd,i),p(e,xe,i),t(xe,Nf),t(xe,ar),t(ar,If),t(xe,Sf),t(xe,nr),t(nr,Hf),t(xe,Ff),p(e,Dd,i),w(Es,e,i),p(e,Td,i),p(e,ke,i),t(ke,Lf),t(ke,or),t(or,Mf),t(ke,Rf),t(ke,rr),t(rr,Wf),t(ke,Uf),p(e,qd,i),w(xs,e,i),p(e,Cd,i),w(ks,e,i),p(e,Od,i),w(Et,e,i),p(e,zd,i),p(e,K,i),t(K,Bf),t(K,lr),t(lr,Yf),t(K,Gf),t(K,ir),t(ir,Jf),t(K,Vf),t(K,tn),t(tn,Xf),t(K,Kf),t(K,dr),t(dr,Qf),t(K,Zf),p(e,Pd,i),w(js,e,i),p(e,Ad,i),w(Ds,e,i),p(e,Nd,i),p(e,Q,i),t(Q,eu),t(Q,hr),t(hr,tu),t(Q,su),t(Q,pr),t(pr,au),t(Q,nu),t(Q,cr),t(cr,ou),t(Q,ru),t(Q,fr),t(fr,lu),t(Q,iu),p(e,Id,i),w(Ts,e,i),p(e,Sd,i),p(e,je,i),t(je,du),t(je,ur),t(ur,hu),t(je,pu),t(je,mr),t(mr,cu),t(je,fu),p(e,Hd,i),w(qs,e,i),p(e,Fd,i),p(e,Z,i),t(Z,uu),t(Z,wr),t(wr,mu),t(Z,wu),t(Z,Cs),t(Cs,gu),t(Z,_u),t(Z,gr),t(gr,vu),t(Z,bu),t(Z,_r),t(_r,$u),t(Z,yu),p(e,Ld,i),w(Os,e,i),p(e,Md,i),p(e,sn,i),t(sn,Eu),p(e,Rd,i),w(zs,e,i),p(e,Wd,i),w(Ps,e,i),p(e,Ud,i),p(e,an,i),t(an,xu),p(e,Bd,i),w(As,e,i),p(e,Yd,i),w(Ns,e,i),p(e,Gd,i),p(e,De,i),t(De,ku),t(De,Is),t(Is,ju),t(De,Du),t(De,vr),t(vr,Tu),t(De,qu),p(e,Jd,i),w(Ss,e,i),p(e,Vd,i),p(e,Te,i),t(Te,Cu),t(Te,br),t(br,Ou),t(Te,zu),t(Te,$r),t($r,Pu),t(Te,Au),p(e,Xd,i),w(Hs,e,i),p(e,Kd,i),w(Fs,e,i),p(e,Qd,i),p(e,nn,i),t(nn,Nu),p(e,Zd,i),p(e,Qe,i),t(Qe,xt),t(xt,yr),w(Ls,yr,null),t(Qe,Iu),t(Qe,Er),t(Er,Su),p(e,eh,i),p(e,on,i),t(on,Hu),p(e,th,i),p(e,rn,i),t(rn,Fu),p(e,sh,i),w(Ms,e,i),p(e,ah,i),p(e,J,i),t(J,Lu),t(J,xr),t(xr,Mu),t(J,Ru),t(J,kr),t(kr,Wu),t(J,Uu),t(J,jr),t(jr,Bu),t(J,Yu),t(J,Dr),t(Dr,Gu),t(J,Ju),t(J,Tr),t(Tr,Vu),t(J,Xu),p(e,nh,i),w(Rs,e,i),p(e,oh,i),w(Ws,e,i),p(e,rh,i),p(e,qe,i),t(qe,Ku),t(qe,qr),t(qr,Qu),t(qe,Zu),t(qe,Cr),t(Cr,em),t(qe,tm),p(e,lh,i),w(Us,e,i),p(e,ih,i),w(Bs,e,i),p(e,dh,i),p(e,ln,i),t(ln,sm),p(e,hh,i),w(kt,e,i),p(e,ph,i),p(e,Ce,i),t(Ce,am),t(Ce,Or),t(Or,nm),t(Ce,om),t(Ce,zr),t(zr,rm),t(Ce,lm),p(e,ch,i),w(Ys,e,i),p(e,fh,i),w(Gs,e,i),p(e,uh,i),p(e,dn,i),t(dn,im),p(e,mh,i),w(jt,e,i),p(e,wh,i),p(e,Dt,i),t(Dt,dm),t(Dt,Pr),t(Pr,hm),t(Dt,pm),p(e,gh,i),w(Js,e,i),p(e,_h,i),w(Vs,e,i),p(e,vh,i),p(e,Tt,i),t(Tt,cm),t(Tt,Ar),t(Ar,fm),t(Tt,um),p(e,bh,i),w(Xs,e,i),p(e,$h,i),p(e,qt,i),t(qt,mm),t(qt,Nr),t(Nr,wm),t(qt,gm),p(e,yh,i),p(e,Ze,i),t(Ze,Ct),t(Ct,Ir),w(Ks,Ir,null),t(Ze,_m),t(Ze,Qs),t(Qs,vm),t(Qs,Sr),t(Sr,bm),t(Qs,$m),p(e,Eh,i),p(e,le,i),t(le,ym),t(le,Hr),t(Hr,Em),t(le,xm),t(le,Fr),t(Fr,km),t(le,jm),t(le,Lr),t(Lr,Dm),t(le,Tm),p(e,xh,i),p(e,ee,i),t(ee,qm),t(ee,Mr),t(Mr,Cm),t(ee,Om),t(ee,Rr),t(Rr,zm),t(ee,Pm),t(ee,Wr),t(Wr,Am),t(ee,Nm),t(ee,Ur),t(Ur,Im),t(ee,Sm),p(e,kh,i),w(Zs,e,i),p(e,jh,i),p(e,Oe,i),t(Oe,Hm),t(Oe,Br),t(Br,Fm),t(Oe,Lm),t(Oe,Yr),t(Yr,Mm),t(Oe,Rm),p(e,Dh,i),p(e,ie,i),t(ie,Wm),t(ie,Gr),t(Gr,Um),t(ie,Bm),t(ie,Jr),t(Jr,Ym),t(ie,Gm),t(ie,hn),t(hn,Jm),t(ie,Vm),p(e,Th,i),w(ea,e,i),p(e,qh,i),p(e,de,i),t(de,Xm),t(de,pn),t(pn,Km),t(de,Qm),t(de,Vr),t(Vr,Zm),t(de,ew),t(de,Xr),t(Xr,tw),t(de,sw),p(e,Ch,i),w(ta,e,i),p(e,Oh,i),p(e,Ot,i),t(Ot,aw),t(Ot,Kr),t(Kr,nw),t(Ot,ow),p(e,zh,i),w(zt,e,i),p(e,Ph,i),p(e,cn,i),t(cn,rw),p(e,Ah,i),p(e,Pt,i),t(Pt,Qr),t(Qr,et),t(et,fn),t(fn,lw),t(et,iw),t(et,un),t(un,dw),t(et,hw),t(et,mn),t(mn,pw),t(Pt,cw),t(Pt,sa),t(sa,tt),t(tt,wn),t(wn,Zr),t(Zr,fw),t(tt,uw),t(tt,gn),t(gn,mw),t(tt,ww),t(tt,_n),t(_n,gw),t(sa,_w),t(sa,st),t(st,vn),t(vn,el),t(el,vw),t(st,bw),t(st,bn),t(bn,$w),t(st,yw),t(st,$n),t($n,Ew),p(e,Nh,i),p(e,ze,i),t(ze,xw),t(ze,tl),t(tl,kw),t(ze,jw),t(ze,sl),t(sl,Dw),t(ze,Tw),p(e,Ih,i),p(e,yn,i),t(yn,qw),p(e,Sh,i),p(e,_e,i),t(_e,al),t(al,Cw),t(_e,Ow),t(_e,nl),t(nl,zw),t(_e,Pw),t(_e,ol),t(ol,Aw),t(_e,Nw),p(e,Hh,i),w(aa,e,i),p(e,Fh,i),p(e,En,i),t(En,Iw),p(e,Lh,i),p(e,At,i),t(At,rl),t(rl,at),t(at,xn),t(xn,Sw),t(at,Hw),t(at,kn),t(kn,Fw),t(at,Lw),t(at,jn),t(jn,Mw),t(At,Rw),t(At,ve),t(ve,nt),t(nt,Dn),t(Dn,ll),t(ll,Ww),t(nt,Uw),t(nt,Tn),t(Tn,Bw),t(nt,Yw),t(nt,qn),t(qn,Gw),t(ve,Jw),t(ve,ot),t(ot,Cn),t(Cn,il),t(il,Vw),t(ot,Xw),t(ot,On),t(On,Kw),t(ot,Qw),t(ot,zn),t(zn,Zw),t(ve,eg),t(ve,rt),t(rt,Nt),t(Nt,dl),t(dl,tg),t(Nt,sg),t(Nt,hl),t(hl,ag),t(rt,ng),t(rt,Pn),t(Pn,og),t(rt,rg),t(rt,An),t(An,lg),t(ve,ig),t(ve,lt),t(lt,It),t(It,pl),t(pl,dg),t(It,hg),t(It,cl),t(cl,pg),t(lt,cg),t(lt,Nn),t(Nn,fg),t(lt,ug),t(lt,In),t(In,mg),p(e,Mh,i),p(e,he,i),t(he,wg),t(he,fl),t(fl,gg),t(he,_g),t(he,ul),t(ul,vg),t(he,bg),t(he,ml),t(ml,$g),t(he,yg),p(e,Rh,i),w(St,e,i),p(e,Wh,i),p(e,pe,i),t(pe,Eg),t(pe,wl),t(wl,xg),t(pe,kg),t(pe,gl),t(gl,jg),t(pe,Dg),t(pe,Sn),t(Sn,Tg),t(pe,qg),p(e,Uh,i),w(Ht,e,i),p(e,Bh,i),p(e,Pe,i),t(Pe,Cg),t(Pe,_l),t(_l,Og),t(Pe,zg),t(Pe,vl),t(vl,Pg),t(Pe,Ag),p(e,Yh,i),w(na,e,i),p(e,Gh,i),p(e,Ft,i),t(Ft,Ng),t(Ft,bl),t(bl,Ig),t(Ft,Sg),p(e,Jh,i),w(oa,e,i),p(e,Vh,i),w(ra,e,i),p(e,Xh,i),p(e,Hn,i),t(Hn,Hg),p(e,Kh,i),w(la,e,i),p(e,Qh,i),w(ia,e,i),p(e,Zh,i),p(e,Ae,i),t(Ae,Fg),t(Ae,$l),t($l,Lg),t(Ae,Mg),t(Ae,da),t(da,Rg),t(Ae,Wg),p(e,ep,i),p(e,te,i),t(te,Ug),t(te,yl),t(yl,Bg),t(te,Yg),t(te,El),t(El,Gg),t(te,Jg),t(te,xl),t(xl,Vg),t(te,Xg),t(te,kl),t(kl,Kg),t(te,Qg),p(e,tp,i),w(ha,e,i),p(e,sp,i),p(e,Fn,i),t(Fn,Zg),p(e,ap,i),w(pa,e,i),p(e,np,i),w(ca,e,i),p(e,op,i),p(e,Ne,i),t(Ne,e_),t(Ne,jl),t(jl,t_),t(Ne,s_),t(Ne,Dl),t(Dl,a_),t(Ne,n_),p(e,rp,i),w(fa,e,i),p(e,lp,i),p(e,Lt,i),t(Lt,o_),t(Lt,Tl),t(Tl,r_),t(Lt,l_),p(e,ip,i),w(ua,e,i),p(e,dp,i),w(ma,e,i),p(e,hp,i),p(e,Ln,i),t(Ln,i_),p(e,pp,i),p(e,Mt,i),t(Mt,d_),t(Mt,ql),t(ql,h_),t(Mt,p_),p(e,cp,i),p(e,it,i),t(it,Rt),t(Rt,Cl),w(wa,Cl,null),t(it,c_),t(it,dt),t(dt,f_),t(dt,Ol),t(Ol,u_),t(dt,m_),t(dt,zl),t(zl,w_),t(dt,g_),p(e,fp,i),w(ga,e,i),p(e,up,i),p(e,ce,i),t(ce,__),t(ce,Pl),t(Pl,v_),t(ce,b_),t(ce,Al),t(Al,$_),t(ce,y_),t(ce,Nl),t(Nl,E_),t(ce,x_),p(e,mp,i),w(_a,e,i),p(e,wp,i),p(e,Wt,i),t(Wt,k_),t(Wt,Il),t(Il,j_),t(Wt,D_),p(e,gp,i),w(va,e,i),p(e,_p,i),p(e,Ie,i),t(Ie,Sl),t(Sl,I),t(I,vp),t(I,T_),t(I,Hl),t(Hl,q_),t(I,C_),t(I,Fl),t(Fl,O_),t(I,z_),t(I,Ll),t(Ll,P_),t(I,A_),t(I,Ml),t(Ml,N_),t(I,I_),t(I,Rl),t(Rl,S_),t(I,H_),t(I,Wl),t(Wl,F_),t(I,L_),t(I,Ul),t(Ul,M_),t(I,R_),t(I,Bl),t(Bl,W_),t(Ie,U_),t(Ie,ht),t(ht,S),t(S,Yl),t(Yl,B_),t(S,Y_),t(S,Gl),t(Gl,G_),t(S,J_),t(S,Jl),t(Jl,V_),t(S,X_),t(S,Vl),t(Vl,K_),t(S,Q_),t(S,Xl),t(Xl,Z_),t(S,ev),t(S,Kl),t(Kl,tv),t(S,sv),t(S,Ql),t(Ql,av),t(S,nv),t(S,Zl),t(Zl,ov),t(S,rv),t(S,ei),t(ei,lv),t(ht,iv),t(ht,H),t(H,ti),t(ti,dv),t(H,hv),t(H,si),t(si,pv),t(H,cv),t(H,ai),t(ai,fv),t(H,uv),t(H,ni),t(ni,mv),t(H,wv),t(H,oi),t(oi,gv),t(H,_v),t(H,ri),t(ri,vv),t(H,bv),t(H,li),t(li,$v),t(H,yv),t(H,ii),t(ii,Ev),t(H,xv),t(H,di),t(di,kv),t(ht,jv),t(ht,F),t(F,hi),t(hi,Dv),t(F,Tv),t(F,pi),t(pi,qv),t(F,Cv),t(F,ci),t(ci,Ov),t(F,zv),t(F,fi),t(fi,Pv),t(F,Av),t(F,ui),t(ui,Nv),t(F,Iv),t(F,mi),t(mi,Sv),t(F,Hv),t(F,wi),t(wi,Fv),t(F,Lv),t(F,gi),t(gi,Mv),t(F,Rv),t(F,_i),t(_i,Wv),p(e,bp,i),p(e,Se,i),t(Se,Uv),t(Se,vi),t(vi,Bv),t(Se,Yv),t(Se,bi),t(bi,Gv),t(Se,Jv),p(e,$p,i),w(ba,e,i),p(e,yp,i),w(Ut,e,i),p(e,Ep,i),p(e,Bt,i),t(Bt,Vv),t(Bt,$i),t($i,Xv),t(Bt,Kv),p(e,xp,i),w($a,e,i),p(e,kp,i),p(e,He,i),t(He,yi),t(yi,Fe),t(Fe,jp),t(Fe,Qv),t(Fe,Ei),t(Ei,Zv),t(Fe,e2),t(Fe,xi),t(xi,t2),t(He,s2),t(He,ae),t(ae,pt),t(pt,ki),t(ki,a2),t(pt,n2),t(pt,ji),t(ji,o2),t(pt,r2),t(pt,Di),t(Di,l2),t(ae,i2),t(ae,ct),t(ct,Ti),t(Ti,d2),t(ct,h2),t(ct,qi),t(qi,p2),t(ct,c2),t(ct,Ci),t(Ci,f2),t(ae,u2),t(ae,ft),t(ft,Oi),t(Oi,m2),t(ft,w2),t(ft,zi),t(zi,g2),t(ft,_2),t(ft,Pi),t(Pi,v2),t(ae,b2),t(ae,ut),t(ut,Ai),t(Ai,$2),t(ut,y2),t(ut,Ni),t(Ni,E2),t(ut,x2),t(ut,Ii),t(Ii,k2),t(ae,j2),t(ae,mt),t(mt,Si),t(Si,D2),t(mt,T2),t(mt,Hi),t(Hi,q2),t(mt,C2),t(mt,Fi),t(Fi,O2),p(e,Dp,i),p(e,Le,i),t(Le,z2),t(Le,Li),t(Li,P2),t(Le,A2),t(Le,Mi),t(Mi,N2),t(Le,I2),p(e,Tp,i),w(ya,e,i),p(e,qp,i),w(Ea,e,i),p(e,Cp,i),w(Yt,e,i),p(e,Op,i),p(e,fe,i),t(fe,S2),t(fe,Ri),t(Ri,H2),t(fe,F2),t(fe,Wi),t(Wi,L2),t(fe,M2),t(fe,Ui),t(Ui,R2),t(fe,W2),p(e,zp,i),w(xa,e,i),p(e,Pp,i),p(e,wt,i),t(wt,Gt),t(Gt,Bi),w(ka,Bi,null),t(wt,U2),t(wt,Yi),t(Yi,B2),p(e,Ap,i),p(e,Mn,i),t(Mn,Y2),p(e,Np,i),p(e,V,i),t(V,G2),t(V,Gi),t(Gi,J2),t(V,V2),t(V,Ji),t(Ji,X2),t(V,K2),t(V,Vi),t(Vi,Q2),t(V,Z2),t(V,Xi),t(Xi,e1),t(V,t1),t(V,Ki),t(Ki,s1),t(V,a1),p(e,Ip,i),w(ja,e,i),p(e,Sp,i),w(Da,e,i),p(e,Hp,i),p(e,Jt,i),t(Jt,n1),t(Jt,Rn),t(Rn,o1),t(Jt,r1),p(e,Fp,i),p(e,gt,i),t(gt,Vt),t(Vt,Qi),w(Ta,Qi,null),t(gt,l1),t(gt,Zi),t(Zi,i1),p(e,Lp,i),w(qa,e,i),p(e,Mp,i),p(e,Wn,i),t(Wn,d1),p(e,Rp,i),p(e,Xt,i),t(Xt,ed),t(ed,Ca),t(Ca,Un),t(Un,h1),t(Ca,p1),t(Ca,Bn),t(Bn,c1),t(Xt,f1),t(Xt,_t),t(_t,Oa),t(Oa,Yn),t(Yn,u1),t(Oa,m1),t(Oa,Gn),t(Gn,td),t(td,w1),t(_t,g1),t(_t,za),t(za,Jn),t(Jn,_1),t(za,v1),t(za,Vn),t(Vn,sd),t(sd,b1),t(_t,$1),t(_t,Pa),t(Pa,Xn),t(Xn,y1),t(Pa,E1),t(Pa,Kn),t(Kn,ad),t(ad,x1),p(e,Wp,i),p(e,Qn,i),t(Qn,k1),p(e,Up,i),w(Aa,e,i),p(e,Bp,i),p(e,Zn,i),t(Zn,j1),p(e,Yp,i),w(Na,e,i),p(e,Gp,i),p(e,ue,i),t(ue,D1),t(ue,nd),t(nd,T1),t(ue,q1),t(ue,od),t(od,C1),t(ue,O1),t(ue,rd),t(rd,z1),t(ue,P1),p(e,Jp,i),p(e,Kt,i),t(Kt,A1),t(Kt,ld),t(ld,N1),t(Kt,I1),p(e,Vp,i),w(Ia,e,i),p(e,Xp,i),w(Sa,e,i),p(e,Kp,i),p(e,Qt,i),t(Qt,S1),t(Qt,id),t(id,H1),t(Qt,F1),p(e,Qp,i),w(Ha,e,i),p(e,Zp,i),p(e,Zt,i),t(Zt,L1),t(Zt,Fa),t(Fa,M1),t(Zt,R1),p(e,ec,i),w(La,e,i),p(e,tc,i),w(Ma,e,i),p(e,sc,i),p(e,es,i),t(es,W1),t(es,eo),t(eo,U1),t(es,B1),p(e,ac,i),w(Ra,e,i),p(e,nc,i),p(e,to,i),t(to,Y1),p(e,oc,i),p(e,ts,i),t(ts,Wa),t(Wa,G1),t(Wa,so),t(so,J1),t(Wa,V1),t(ts,X1),t(ts,vt),t(vt,K1),t(vt,dd),t(dd,Q1),t(vt,Z1),t(vt,ao),t(ao,eb),t(vt,tb),p(e,rc,i),p(e,no,i),t(no,sb),lc=!0},p(e,[i]){const Ua={};i&2&&(Ua.$$scope={dirty:i,ctx:e}),Et.$set(Ua);const hd={};i&2&&(hd.$$scope={dirty:i,ctx:e}),kt.$set(hd);const pd={};i&2&&(pd.$$scope={dirty:i,ctx:e}),jt.$set(pd);const cd={};i&2&&(cd.$$scope={dirty:i,ctx:e}),zt.$set(cd);const fd={};i&2&&(fd.$$scope={dirty:i,ctx:e}),St.$set(fd);const Ba={};i&2&&(Ba.$$scope={dirty:i,ctx:e}),Ht.$set(Ba);const ud={};i&2&&(ud.$$scope={dirty:i,ctx:e}),Ut.$set(ud);const md={};i&2&&(md.$$scope={dirty:i,ctx:e}),Yt.$set(md)},i(e){lc||(g(y.$$.fragment,e),g(z.$$.fragment,e),g(P.$$.fragment,e),g(se.$$.fragment,e),g(gs.$$.fragment,e),g(_s.$$.fragment,e),g(vs.$$.fragment,e),g(bs.$$.fragment,e),g(Es.$$.fragment,e),g(xs.$$.fragment,e),g(ks.$$.fragment,e),g(Et.$$.fragment,e),g(js.$$.fragment,e),g(Ds.$$.fragment,e),g(Ts.$$.fragment,e),g(qs.$$.fragment,e),g(Os.$$.fragment,e),g(zs.$$.fragment,e),g(Ps.$$.fragment,e),g(As.$$.fragment,e),g(Ns.$$.fragment,e),g(Ss.$$.fragment,e),g(Hs.$$.fragment,e),g(Fs.$$.fragment,e),g(Ls.$$.fragment,e),g(Ms.$$.fragment,e),g(Rs.$$.fragment,e),g(Ws.$$.fragment,e),g(Us.$$.fragment,e),g(Bs.$$.fragment,e),g(kt.$$.fragment,e),g(Ys.$$.fragment,e),g(Gs.$$.fragment,e),g(jt.$$.fragment,e),g(Js.$$.fragment,e),g(Vs.$$.fragment,e),g(Xs.$$.fragment,e),g(Ks.$$.fragment,e),g(Zs.$$.fragment,e),g(ea.$$.fragment,e),g(ta.$$.fragment,e),g(zt.$$.fragment,e),g(aa.$$.fragment,e),g(St.$$.fragment,e),g(Ht.$$.fragment,e),g(na.$$.fragment,e),g(oa.$$.fragment,e),g(ra.$$.fragment,e),g(la.$$.fragment,e),g(ia.$$.fragment,e),g(ha.$$.fragment,e),g(pa.$$.fragment,e),g(ca.$$.fragment,e),g(fa.$$.fragment,e),g(ua.$$.fragment,e),g(ma.$$.fragment,e),g(wa.$$.fragment,e),g(ga.$$.fragment,e),g(_a.$$.fragment,e),g(va.$$.fragment,e),g(ba.$$.fragment,e),g(Ut.$$.fragment,e),g($a.$$.fragment,e),g(ya.$$.fragment,e),g(Ea.$$.fragment,e),g(Yt.$$.fragment,e),g(xa.$$.fragment,e),g(ka.$$.fragment,e),g(ja.$$.fragment,e),g(Da.$$.fragment,e),g(Ta.$$.fragment,e),g(qa.$$.fragment,e),g(Aa.$$.fragment,e),g(Na.$$.fragment,e),g(Ia.$$.fragment,e),g(Sa.$$.fragment,e),g(Ha.$$.fragment,e),g(La.$$.fragment,e),g(Ma.$$.fragment,e),g(Ra.$$.fragment,e),lc=!0)},o(e){_(y.$$.fragment,e),_(z.$$.fragment,e),_(P.$$.fragment,e),_(se.$$.fragment,e),_(gs.$$.fragment,e),_(_s.$$.fragment,e),_(vs.$$.fragment,e),_(bs.$$.fragment,e),_(Es.$$.fragment,e),_(xs.$$.fragment,e),_(ks.$$.fragment,e),_(Et.$$.fragment,e),_(js.$$.fragment,e),_(Ds.$$.fragment,e),_(Ts.$$.fragment,e),_(qs.$$.fragment,e),_(Os.$$.fragment,e),_(zs.$$.fragment,e),_(Ps.$$.fragment,e),_(As.$$.fragment,e),_(Ns.$$.fragment,e),_(Ss.$$.fragment,e),_(Hs.$$.fragment,e),_(Fs.$$.fragment,e),_(Ls.$$.fragment,e),_(Ms.$$.fragment,e),_(Rs.$$.fragment,e),_(Ws.$$.fragment,e),_(Us.$$.fragment,e),_(Bs.$$.fragment,e),_(kt.$$.fragment,e),_(Ys.$$.fragment,e),_(Gs.$$.fragment,e),_(jt.$$.fragment,e),_(Js.$$.fragment,e),_(Vs.$$.fragment,e),_(Xs.$$.fragment,e),_(Ks.$$.fragment,e),_(Zs.$$.fragment,e),_(ea.$$.fragment,e),_(ta.$$.fragment,e),_(zt.$$.fragment,e),_(aa.$$.fragment,e),_(St.$$.fragment,e),_(Ht.$$.fragment,e),_(na.$$.fragment,e),_(oa.$$.fragment,e),_(ra.$$.fragment,e),_(la.$$.fragment,e),_(ia.$$.fragment,e),_(ha.$$.fragment,e),_(pa.$$.fragment,e),_(ca.$$.fragment,e),_(fa.$$.fragment,e),_(ua.$$.fragment,e),_(ma.$$.fragment,e),_(wa.$$.fragment,e),_(ga.$$.fragment,e),_(_a.$$.fragment,e),_(va.$$.fragment,e),_(ba.$$.fragment,e),_(Ut.$$.fragment,e),_($a.$$.fragment,e),_(ya.$$.fragment,e),_(Ea.$$.fragment,e),_(Yt.$$.fragment,e),_(xa.$$.fragment,e),_(ka.$$.fragment,e),_(ja.$$.fragment,e),_(Da.$$.fragment,e),_(Ta.$$.fragment,e),_(qa.$$.fragment,e),_(Aa.$$.fragment,e),_(Na.$$.fragment,e),_(Ia.$$.fragment,e),_(Sa.$$.fragment,e),_(Ha.$$.fragment,e),_(La.$$.fragment,e),_(Ma.$$.fragment,e),_(Ra.$$.fragment,e),lc=!1},d(e){s(c),e&&s(q),e&&s($),v(y),e&&s(x),v(z,e),e&&s(T),e&&s(O),e&&s(N),v(P,e),e&&s(ge),e&&s(M),v(se),e&&s(bt),e&&s(R),e&&s(wd),e&&s(be),e&&s(gd),e&&s($e),e&&s(_d),v(gs,e),e&&s(vd),e&&s(oe),e&&s(bd),v(_s,e),e&&s($d),e&&s(ye),e&&s(yd),v(vs,e),e&&s(Ed),v(bs,e),e&&s(xd),e&&s(re),e&&s(kd),e&&s(Ee),e&&s(jd),e&&s(xe),e&&s(Dd),v(Es,e),e&&s(Td),e&&s(ke),e&&s(qd),v(xs,e),e&&s(Cd),v(ks,e),e&&s(Od),v(Et,e),e&&s(zd),e&&s(K),e&&s(Pd),v(js,e),e&&s(Ad),v(Ds,e),e&&s(Nd),e&&s(Q),e&&s(Id),v(Ts,e),e&&s(Sd),e&&s(je),e&&s(Hd),v(qs,e),e&&s(Fd),e&&s(Z),e&&s(Ld),v(Os,e),e&&s(Md),e&&s(sn),e&&s(Rd),v(zs,e),e&&s(Wd),v(Ps,e),e&&s(Ud),e&&s(an),e&&s(Bd),v(As,e),e&&s(Yd),v(Ns,e),e&&s(Gd),e&&s(De),e&&s(Jd),v(Ss,e),e&&s(Vd),e&&s(Te),e&&s(Xd),v(Hs,e),e&&s(Kd),v(Fs,e),e&&s(Qd),e&&s(nn),e&&s(Zd),e&&s(Qe),v(Ls),e&&s(eh),e&&s(on),e&&s(th),e&&s(rn),e&&s(sh),v(Ms,e),e&&s(ah),e&&s(J),e&&s(nh),v(Rs,e),e&&s(oh),v(Ws,e),e&&s(rh),e&&s(qe),e&&s(lh),v(Us,e),e&&s(ih),v(Bs,e),e&&s(dh),e&&s(ln),e&&s(hh),v(kt,e),e&&s(ph),e&&s(Ce),e&&s(ch),v(Ys,e),e&&s(fh),v(Gs,e),e&&s(uh),e&&s(dn),e&&s(mh),v(jt,e),e&&s(wh),e&&s(Dt),e&&s(gh),v(Js,e),e&&s(_h),v(Vs,e),e&&s(vh),e&&s(Tt),e&&s(bh),v(Xs,e),e&&s($h),e&&s(qt),e&&s(yh),e&&s(Ze),v(Ks),e&&s(Eh),e&&s(le),e&&s(xh),e&&s(ee),e&&s(kh),v(Zs,e),e&&s(jh),e&&s(Oe),e&&s(Dh),e&&s(ie),e&&s(Th),v(ea,e),e&&s(qh),e&&s(de),e&&s(Ch),v(ta,e),e&&s(Oh),e&&s(Ot),e&&s(zh),v(zt,e),e&&s(Ph),e&&s(cn),e&&s(Ah),e&&s(Pt),e&&s(Nh),e&&s(ze),e&&s(Ih),e&&s(yn),e&&s(Sh),e&&s(_e),e&&s(Hh),v(aa,e),e&&s(Fh),e&&s(En),e&&s(Lh),e&&s(At),e&&s(Mh),e&&s(he),e&&s(Rh),v(St,e),e&&s(Wh),e&&s(pe),e&&s(Uh),v(Ht,e),e&&s(Bh),e&&s(Pe),e&&s(Yh),v(na,e),e&&s(Gh),e&&s(Ft),e&&s(Jh),v(oa,e),e&&s(Vh),v(ra,e),e&&s(Xh),e&&s(Hn),e&&s(Kh),v(la,e),e&&s(Qh),v(ia,e),e&&s(Zh),e&&s(Ae),e&&s(ep),e&&s(te),e&&s(tp),v(ha,e),e&&s(sp),e&&s(Fn),e&&s(ap),v(pa,e),e&&s(np),v(ca,e),e&&s(op),e&&s(Ne),e&&s(rp),v(fa,e),e&&s(lp),e&&s(Lt),e&&s(ip),v(ua,e),e&&s(dp),v(ma,e),e&&s(hp),e&&s(Ln),e&&s(pp),e&&s(Mt),e&&s(cp),e&&s(it),v(wa),e&&s(fp),v(ga,e),e&&s(up),e&&s(ce),e&&s(mp),v(_a,e),e&&s(wp),e&&s(Wt),e&&s(gp),v(va,e),e&&s(_p),e&&s(Ie),e&&s(bp),e&&s(Se),e&&s($p),v(ba,e),e&&s(yp),v(Ut,e),e&&s(Ep),e&&s(Bt),e&&s(xp),v($a,e),e&&s(kp),e&&s(He),e&&s(Dp),e&&s(Le),e&&s(Tp),v(ya,e),e&&s(qp),v(Ea,e),e&&s(Cp),v(Yt,e),e&&s(Op),e&&s(fe),e&&s(zp),v(xa,e),e&&s(Pp),e&&s(wt),v(ka),e&&s(Ap),e&&s(Mn),e&&s(Np),e&&s(V),e&&s(Ip),v(ja,e),e&&s(Sp),v(Da,e),e&&s(Hp),e&&s(Jt),e&&s(Fp),e&&s(gt),v(Ta),e&&s(Lp),v(qa,e),e&&s(Mp),e&&s(Wn),e&&s(Rp),e&&s(Xt),e&&s(Wp),e&&s(Qn),e&&s(Up),v(Aa,e),e&&s(Bp),e&&s(Zn),e&&s(Yp),v(Na,e),e&&s(Gp),e&&s(ue),e&&s(Jp),e&&s(Kt),e&&s(Vp),v(Ia,e),e&&s(Xp),v(Sa,e),e&&s(Kp),e&&s(Qt),e&&s(Qp),v(Ha,e),e&&s(Zp),e&&s(Zt),e&&s(ec),v(La,e),e&&s(tc),v(Ma,e),e&&s(sc),e&&s(es),e&&s(ac),v(Ra,e),e&&s(nc),e&&s(to),e&&s(oc),e&&s(ts),e&&s(rc),e&&s(no)}}}const rx={local:"time-to-slice-and-dice",sections:[{local:"slicing-and-dicing-our-data",title:"Slicing and dicing our data"},{local:"creating-new-columns",title:"Creating new columns"},{local:"the-map-methods-superpowers",title:"The `map()` method's superpowers"},{local:"from-datasets-to-dataframes-and-back",title:"From `Dataset`s to `DataFrame`s and back"},{local:"creating-a-validation-set",title:"Creating a validation set"},{local:"saving-a-dataset",title:"Saving a dataset"}],title:"Time to slice and dice"};function lx(L){return VE(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class ux extends BE{constructor(c){super();YE(this,c,lx,ox,GE,{})}}export{ux as default,rx as metadata};
