import{S as li,i as oi,s as ii,e as o,k as d,w as g,t as a,M as ri,c as i,d as s,m as f,a as r,x as b,h as n,b as c,G as t,g as p,y as w,q as _,o as y,B as $,v as pi}from"../../chunks/vendor-hf-doc-builder.js";import{T as ws}from"../../chunks/Tip-hf-doc-builder.js";import{Y as hi}from"../../chunks/Youtube-hf-doc-builder.js";import{I as Sa}from"../../chunks/IconCopyLink-hf-doc-builder.js";import{C as E}from"../../chunks/CodeBlock-hf-doc-builder.js";import{D as di}from"../../chunks/DocNotebookDropdown-hf-doc-builder.js";function fi(L){let h,O,m,j,D,u,P,k,x,v,A,I,T,R;return{c(){h=o("p"),O=a("\u270E By default, \u{1F917} Datasets will decompress the files needed to load a dataset. If you want to preserve hard drive space, you can pass "),m=o("code"),j=a("DownloadConfig(delete_extracted=True)"),D=a(" to the "),u=o("code"),P=a("download_config"),k=a(" argument of "),x=o("code"),v=a("load_dataset()"),A=a(". See the "),I=o("a"),T=a("documentation"),R=a(" for more details."),this.h()},l(z){h=i(z,"P",{});var C=r(h);O=n(C,"\u270E By default, \u{1F917} Datasets will decompress the files needed to load a dataset. If you want to preserve hard drive space, you can pass "),m=i(C,"CODE",{});var te=r(m);j=n(te,"DownloadConfig(delete_extracted=True)"),te.forEach(s),D=n(C," to the "),u=i(C,"CODE",{});var W=r(u);P=n(W,"download_config"),W.forEach(s),k=n(C," argument of "),x=i(C,"CODE",{});var q=r(x);v=n(q,"load_dataset()"),q.forEach(s),A=n(C,". See the "),I=i(C,"A",{href:!0,rel:!0});var se=r(I);T=n(se,"documentation"),se.forEach(s),R=n(C," for more details."),C.forEach(s),this.h()},h(){c(I,"href","https://huggingface.co/docs/datasets/package_reference/builder_classes.html?#datasets.utils.DownloadConfig"),c(I,"rel","nofollow")},m(z,C){p(z,h,C),t(h,O),t(h,m),t(m,j),t(h,D),t(h,u),t(u,P),t(h,k),t(h,x),t(x,v),t(h,A),t(h,I),t(I,T),t(h,R)},d(z){z&&s(h)}}}function ci(L){let h,O,m,j,D,u,P,k,x,v,A;return{c(){h=o("p"),O=a("\u270F\uFE0F "),m=o("strong"),j=a("Try it out!"),D=a(" Pick one of the "),u=o("a"),P=a("subsets"),k=a(" from the Pile that is larger than your laptop or desktop\u2019s RAM, load it with \u{1F917} Datasets, and measure the amount of RAM used. Note that to get an accurate measurement, you\u2019ll want to do this in a new process. You can find the decompressed sizes of each subset in Table 1 of "),x=o("a"),v=a("the Pile paper"),A=a("."),this.h()},l(I){h=i(I,"P",{});var T=r(h);O=n(T,"\u270F\uFE0F "),m=i(T,"STRONG",{});var R=r(m);j=n(R,"Try it out!"),R.forEach(s),D=n(T," Pick one of the "),u=i(T,"A",{href:!0,rel:!0});var z=r(u);P=n(z,"subsets"),z.forEach(s),k=n(T," from the Pile that is larger than your laptop or desktop\u2019s RAM, load it with \u{1F917} Datasets, and measure the amount of RAM used. Note that to get an accurate measurement, you\u2019ll want to do this in a new process. You can find the decompressed sizes of each subset in Table 1 of "),x=i(T,"A",{href:!0,rel:!0});var C=r(x);v=n(C,"the Pile paper"),C.forEach(s),A=n(T,"."),T.forEach(s),this.h()},h(){c(u,"href","https://mystic.the-eye.eu/public/AI/pile_preliminary_components/"),c(u,"rel","nofollow"),c(x,"href","https://arxiv.org/abs/2101.00027"),c(x,"rel","nofollow")},m(I,T){p(I,h,T),t(h,O),t(h,m),t(m,j),t(h,D),t(h,u),t(u,P),t(h,k),t(h,x),t(x,v),t(h,A)},d(I){I&&s(h)}}}function mi(L){let h,O,m,j,D,u,P;return{c(){h=o("p"),O=a("\u{1F4A1} In Jupyter notebooks you can also time cells using the "),m=o("a"),j=o("code"),D=a("%%timeit"),u=a(" magic function"),P=a("."),this.h()},l(k){h=i(k,"P",{});var x=r(h);O=n(x,"\u{1F4A1} In Jupyter notebooks you can also time cells using the "),m=i(x,"A",{href:!0,rel:!0});var v=r(m);j=i(v,"CODE",{});var A=r(j);D=n(A,"%%timeit"),A.forEach(s),u=n(v," magic function"),v.forEach(s),P=n(x,"."),x.forEach(s),this.h()},h(){c(m,"href","https://ipython.readthedocs.io/en/stable/interactive/magics.html#magic-timeit"),c(m,"rel","nofollow")},m(k,x){p(k,h,x),t(h,O),t(h,m),t(m,j),t(j,D),t(m,u),t(h,P)},d(k){k&&s(h)}}}function ui(L){let h,O,m,j,D,u,P,k;return{c(){h=o("p"),O=a("\u{1F4A1} To speed up tokenization with streaming you can pass "),m=o("code"),j=a("batched=True"),D=a(", as we saw in the last section. It will process the examples batch by batch; the default batch size is 1,000 and can be specified with the "),u=o("code"),P=a("batch_size"),k=a(" argument.")},l(x){h=i(x,"P",{});var v=r(h);O=n(v,"\u{1F4A1} To speed up tokenization with streaming you can pass "),m=i(v,"CODE",{});var A=r(m);j=n(A,"batched=True"),A.forEach(s),D=n(v,", as we saw in the last section. It will process the examples batch by batch; the default batch size is 1,000 and can be specified with the "),u=i(v,"CODE",{});var I=r(u);P=n(I,"batch_size"),I.forEach(s),k=n(v," argument."),v.forEach(s)},m(x,v){p(x,h,v),t(h,O),t(h,m),t(m,j),t(h,D),t(h,u),t(u,P),t(h,k)},d(x){x&&s(h)}}}function gi(L){let h,O,m,j,D,u,P,k,x,v,A,I,T;return{c(){h=o("p"),O=a("\u270F\uFE0F "),m=o("strong"),j=a("Try it out!"),D=a(" Use one of the large Common Crawl corpora like "),u=o("a"),P=o("code"),k=a("mc4"),x=a(" or "),v=o("a"),A=o("code"),I=a("oscar"),T=a(" to create a streaming multilingual dataset that represents the spoken proportions of languages in a country of your choice. For example, the four national languages in Switzerland are German, French, Italian, and Romansh, so you could try creating a Swiss corpus by sampling the Oscar subsets according to their spoken proportion."),this.h()},l(R){h=i(R,"P",{});var z=r(h);O=n(z,"\u270F\uFE0F "),m=i(z,"STRONG",{});var C=r(m);j=n(C,"Try it out!"),C.forEach(s),D=n(z," Use one of the large Common Crawl corpora like "),u=i(z,"A",{href:!0,rel:!0});var te=r(u);P=i(te,"CODE",{});var W=r(P);k=n(W,"mc4"),W.forEach(s),te.forEach(s),x=n(z," or "),v=i(z,"A",{href:!0,rel:!0});var q=r(v);A=i(q,"CODE",{});var se=r(A);I=n(se,"oscar"),se.forEach(s),q.forEach(s),T=n(z," to create a streaming multilingual dataset that represents the spoken proportions of languages in a country of your choice. For example, the four national languages in Switzerland are German, French, Italian, and Romansh, so you could try creating a Swiss corpus by sampling the Oscar subsets according to their spoken proportion."),z.forEach(s),this.h()},h(){c(u,"href","https://huggingface.co/datasets/mc4"),c(u,"rel","nofollow"),c(v,"href","https://huggingface.co/datasets/oscar"),c(v,"rel","nofollow")},m(R,z){p(R,h,z),t(h,O),t(h,m),t(m,j),t(h,D),t(h,u),t(u,P),t(P,k),t(h,x),t(h,v),t(v,A),t(A,I),t(h,T)},d(R){R&&s(h)}}}function bi(L){let h,O,m,j,D,u,P,k,x,v,A,I,T,R,z,C,te,W,q,se,Pt,Ra,qa,It,Ma,Na,_s,ve,ys,oe,Ga,je,La,Ba,$s,ae,ie,zt,Ee,Fa,Ot,Ha,xs,S,Ua,ke,Wa,Ja,Ae,Ya,Ka,Te,Za,Qa,De,Va,Xa,Pe,en,tn,Ct,sn,an,vs,Ie,js,re,nn,ut,ln,on,Es,ze,ks,Oe,As,gt,rn,Ts,pe,Ds,bt,pn,Ps,Ce,Is,Se,zs,wt,hn,Os,ne,he,St,Re,dn,Rt,fn,Cs,J,cn,qe,qt,mn,un,Mt,gn,bn,Ss,Me,Rs,de,wn,Nt,_n,yn,qs,Ne,Ms,Ge,Ns,B,$n,Gt,xn,vn,Lt,jn,En,Bt,kn,An,Gs,Le,Ls,Be,Bs,_t,Tn,Fs,fe,Hs,Y,Dn,Fe,Pn,In,He,zn,On,Us,M,Cn,Ft,Sn,Rn,Ue,qn,Mn,We,Ht,Nn,Gn,Je,Ln,Bn,Ws,Ye,Js,Ke,Ys,K,Fn,Ut,Hn,Un,Wt,Wn,Jn,Ks,ce,Zs,le,me,Jt,Ze,Yn,Yt,Kn,Qs,Z,Zn,Kt,Qn,Vn,Zt,Xn,el,Vs,Qe,Xs,N,tl,Qt,sl,al,Vt,nl,ll,Xt,ol,il,es,rl,pl,ea,Ve,ta,Xe,sa,Q,hl,ts,dl,fl,yt,cl,ml,aa,et,na,tt,la,ue,oa,F,ul,ss,gl,bl,as,wl,_l,ns,yl,$l,ia,st,ra,at,pa,H,xl,ls,vl,jl,os,El,kl,is,Al,Tl,ha,nt,da,lt,fa,ge,Dl,rs,Pl,Il,ca,ot,ma,U,zl,ps,Ol,Cl,hs,Sl,Rl,ds,ql,Ml,ua,it,ga,rt,ba,be,Nl,fs,Gl,Ll,wa,pt,_a,ht,ya,V,Bl,cs,Fl,Hl,ms,Ul,Wl,$a,$t,Jl,xa,dt,va,ft,ja,we,Ea,xt,Yl,ka;return u=new Sa({}),A=new di({props:{classNames:"absolute z-10 right-0 top-0",options:[{label:"Google Colab",value:"https://colab.research.google.com/github/huggingface/notebooks/blob/master/course/chapter5/section4.ipynb"},{label:"Aws Studio",value:"https://studiolab.sagemaker.aws/import/github/huggingface/notebooks/blob/master/course/chapter5/section4.ipynb"}]}}),ve=new hi({props:{id:"JwISwTCPPWo"}}),Ee=new Sa({}),Ie=new E({props:{code:"!pip install zstandard",highlighted:"!pip install zstandard"}}),ze=new E({props:{code:`from datasets import load_dataset

# This takes a few minutes to run, so go grab a tea or coffee while you wait :)
data_files = "https://mystic.the-eye.eu/public/AI/pile_preliminary_components/PUBMED_title_abstracts_2019_baseline.jsonl.zst"
pubmed_dataset = load_dataset("json", data_files=data_files, split="train")
pubmed_dataset`,highlighted:`<span class="hljs-keyword">from</span> datasets <span class="hljs-keyword">import</span> load_dataset

<span class="hljs-comment"># This takes a few minutes to run, so go grab a tea or coffee while you wait :)</span>
data_files = <span class="hljs-string">&quot;https://mystic.the-eye.eu/public/AI/pile_preliminary_components/PUBMED_title_abstracts_2019_baseline.jsonl.zst&quot;</span>
pubmed_dataset = load_dataset(<span class="hljs-string">&quot;json&quot;</span>, data_files=data_files, split=<span class="hljs-string">&quot;train&quot;</span>)
pubmed_dataset`}}),Oe=new E({props:{code:`Dataset({
    features: ['meta', 'text'],
    num_rows: 15518009
})`,highlighted:`Dataset({
    features: [<span class="hljs-string">&#x27;meta&#x27;</span>, <span class="hljs-string">&#x27;text&#x27;</span>],
    num_rows: <span class="hljs-number">15518009</span>
})`}}),pe=new ws({props:{$$slots:{default:[fi]},$$scope:{ctx:L}}}),Ce=new E({props:{code:"pubmed_dataset[0]",highlighted:'pubmed_dataset[<span class="hljs-number">0</span>]'}}),Se=new E({props:{code:`{'meta': {'pmid': 11409574, 'language': 'eng'},
 'text': 'Epidemiology of hypoxaemia in children with acute lower respiratory infection.\\nTo determine the prevalence of hypoxaemia in children aged under 5 years suffering acute lower respiratory infections (ALRI), the risk factors for hypoxaemia in children under 5 years of age with ALRI, and the association of hypoxaemia with an increased risk of dying in children of the same age ...'}`,highlighted:`{<span class="hljs-string">&#x27;meta&#x27;</span>: {<span class="hljs-string">&#x27;pmid&#x27;</span>: <span class="hljs-number">11409574</span>, <span class="hljs-string">&#x27;language&#x27;</span>: <span class="hljs-string">&#x27;eng&#x27;</span>},
 <span class="hljs-string">&#x27;text&#x27;</span>: <span class="hljs-string">&#x27;Epidemiology of hypoxaemia in children with acute lower respiratory infection.\\nTo determine the prevalence of hypoxaemia in children aged under 5 years suffering acute lower respiratory infections (ALRI), the risk factors for hypoxaemia in children under 5 years of age with ALRI, and the association of hypoxaemia with an increased risk of dying in children of the same age ...&#x27;</span>}`}}),Re=new Sa({}),Me=new E({props:{code:"!pip install psutil",highlighted:"!pip install psutil"}}),Ne=new E({props:{code:`import psutil

# Process.memory_info is expressed in bytes, so convert to megabytes
print(f"RAM used: {psutil.Process().memory_info().rss / (1024 * 1024):.2f} MB")`,highlighted:`<span class="hljs-keyword">import</span> psutil

<span class="hljs-comment"># Process.memory_info is expressed in bytes, so convert to megabytes</span>
<span class="hljs-built_in">print</span>(<span class="hljs-string">f&quot;RAM used: <span class="hljs-subst">{psutil.Process().memory_info().rss / (<span class="hljs-number">1024</span> * <span class="hljs-number">1024</span>):<span class="hljs-number">.2</span>f}</span> MB&quot;</span>)`}}),Ge=new E({props:{code:"RAM used: 5678.33 MB",highlighted:'RAM used: <span class="hljs-number">5678.33</span> MB'}}),Le=new E({props:{code:`print(f"Number of files in dataset : {pubmed_dataset.dataset_size}")
size_gb = pubmed_dataset.dataset_size / (1024**3)
print(f"Dataset size (cache file) : {size_gb:.2f} GB")`,highlighted:`<span class="hljs-built_in">print</span>(<span class="hljs-string">f&quot;Number of files in dataset : <span class="hljs-subst">{pubmed_dataset.dataset_size}</span>&quot;</span>)
size_gb = pubmed_dataset.dataset_size / (<span class="hljs-number">1024</span>**<span class="hljs-number">3</span>)
<span class="hljs-built_in">print</span>(<span class="hljs-string">f&quot;Dataset size (cache file) : <span class="hljs-subst">{size_gb:<span class="hljs-number">.2</span>f}</span> GB&quot;</span>)`}}),Be=new E({props:{code:`Number of files in dataset : 20979437051
Dataset size (cache file) : 19.54 GB`,highlighted:`Number of files <span class="hljs-keyword">in</span> dataset : <span class="hljs-number">20979437051</span>
Dataset size (cache file) : <span class="hljs-number">19.54</span> GB`}}),fe=new ws({props:{$$slots:{default:[ci]},$$scope:{ctx:L}}}),Ye=new E({props:{code:`import timeit

code_snippet = """batch_size = 1000

for idx in range(0, len(pubmed_dataset), batch_size):
    _ = pubmed_dataset[idx:idx + batch_size]
"""

time = timeit.timeit(stmt=code_snippet, number=1, globals=globals())
print(
    f"Iterated over {len(pubmed_dataset)} examples (about {size_gb:.1f} GB) in "
    f"{time:.1f}s, i.e. {size_gb/time:.3f} GB/s"
)`,highlighted:`<span class="hljs-keyword">import</span> timeit

code_snippet = <span class="hljs-string">&quot;&quot;&quot;batch_size = 1000

for idx in range(0, len(pubmed_dataset), batch_size):
    _ = pubmed_dataset[idx:idx + batch_size]
&quot;&quot;&quot;</span>

time = timeit.timeit(stmt=code_snippet, number=<span class="hljs-number">1</span>, <span class="hljs-built_in">globals</span>=<span class="hljs-built_in">globals</span>())
<span class="hljs-built_in">print</span>(
    <span class="hljs-string">f&quot;Iterated over <span class="hljs-subst">{<span class="hljs-built_in">len</span>(pubmed_dataset)}</span> examples (about <span class="hljs-subst">{size_gb:<span class="hljs-number">.1</span>f}</span> GB) in &quot;</span>
    <span class="hljs-string">f&quot;<span class="hljs-subst">{time:<span class="hljs-number">.1</span>f}</span>s, i.e. <span class="hljs-subst">{size_gb/time:<span class="hljs-number">.3</span>f}</span> GB/s&quot;</span>
)`}}),Ke=new E({props:{code:"'Iterated over 15518009 examples (about 19.5 GB) in 64.2s, i.e. 0.304 GB/s'",highlighted:'<span class="hljs-string">&#x27;Iterated over 15518009 examples (about 19.5 GB) in 64.2s, i.e. 0.304 GB/s&#x27;</span>'}}),ce=new ws({props:{$$slots:{default:[mi]},$$scope:{ctx:L}}}),Ze=new Sa({}),Qe=new E({props:{code:`pubmed_dataset_streamed = load_dataset(
    "json", data_files=data_files, split="train", streaming=True
)`,highlighted:`pubmed_dataset_streamed = load_dataset(
    <span class="hljs-string">&quot;json&quot;</span>, data_files=data_files, split=<span class="hljs-string">&quot;train&quot;</span>, streaming=<span class="hljs-literal">True</span>
)`}}),Ve=new E({props:{code:"next(iter(pubmed_dataset_streamed))",highlighted:'<span class="hljs-built_in">next</span>(<span class="hljs-built_in">iter</span>(pubmed_dataset_streamed))'}}),Xe=new E({props:{code:`{'meta': {'pmid': 11409574, 'language': 'eng'},
 'text': 'Epidemiology of hypoxaemia in children with acute lower respiratory infection.\\nTo determine the prevalence of hypoxaemia in children aged under 5 years suffering acute lower respiratory infections (ALRI), the risk factors for hypoxaemia in children under 5 years of age with ALRI, and the association of hypoxaemia with an increased risk of dying in children of the same age ...'}`,highlighted:`{<span class="hljs-string">&#x27;meta&#x27;</span>: {<span class="hljs-string">&#x27;pmid&#x27;</span>: <span class="hljs-number">11409574</span>, <span class="hljs-string">&#x27;language&#x27;</span>: <span class="hljs-string">&#x27;eng&#x27;</span>},
 <span class="hljs-string">&#x27;text&#x27;</span>: <span class="hljs-string">&#x27;Epidemiology of hypoxaemia in children with acute lower respiratory infection.\\nTo determine the prevalence of hypoxaemia in children aged under 5 years suffering acute lower respiratory infections (ALRI), the risk factors for hypoxaemia in children under 5 years of age with ALRI, and the association of hypoxaemia with an increased risk of dying in children of the same age ...&#x27;</span>}`}}),et=new E({props:{code:`from transformers import AutoTokenizer

tokenizer = AutoTokenizer.from_pretrained("distilbert-base-uncased")
tokenized_dataset = pubmed_dataset_streamed.map(lambda x: tokenizer(x["text"]))
next(iter(tokenized_dataset))`,highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoTokenizer

tokenizer = AutoTokenizer.from_pretrained(<span class="hljs-string">&quot;distilbert-base-uncased&quot;</span>)
tokenized_dataset = pubmed_dataset_streamed.<span class="hljs-built_in">map</span>(<span class="hljs-keyword">lambda</span> x: tokenizer(x[<span class="hljs-string">&quot;text&quot;</span>]))
<span class="hljs-built_in">next</span>(<span class="hljs-built_in">iter</span>(tokenized_dataset))`}}),tt=new E({props:{code:"{'input_ids': [101, 4958, 5178, 4328, 6779, ...], 'attention_mask': [1, 1, 1, 1, 1, ...]}",highlighted:'{<span class="hljs-string">&#x27;input_ids&#x27;</span>: [<span class="hljs-number">101</span>, <span class="hljs-number">4958</span>, <span class="hljs-number">5178</span>, <span class="hljs-number">4328</span>, <span class="hljs-number">6779</span>, ...], <span class="hljs-string">&#x27;attention_mask&#x27;</span>: [<span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, ...]}'}}),ue=new ws({props:{$$slots:{default:[ui]},$$scope:{ctx:L}}}),st=new E({props:{code:`shuffled_dataset = pubmed_dataset_streamed.shuffle(buffer_size=10_000, seed=42)
next(iter(shuffled_dataset))`,highlighted:`shuffled_dataset = pubmed_dataset_streamed.shuffle(buffer_size=<span class="hljs-number">10_000</span>, seed=<span class="hljs-number">42</span>)
<span class="hljs-built_in">next</span>(<span class="hljs-built_in">iter</span>(shuffled_dataset))`}}),at=new E({props:{code:`{'meta': {'pmid': 11410799, 'language': 'eng'},
 'text': 'Randomized study of dose or schedule modification of granulocyte colony-stimulating factor in platinum-based chemotherapy for elderly patients with lung cancer ...'}`,highlighted:`{<span class="hljs-string">&#x27;meta&#x27;</span>: {<span class="hljs-string">&#x27;pmid&#x27;</span>: <span class="hljs-number">11410799</span>, <span class="hljs-string">&#x27;language&#x27;</span>: <span class="hljs-string">&#x27;eng&#x27;</span>},
 <span class="hljs-string">&#x27;text&#x27;</span>: <span class="hljs-string">&#x27;Randomized study of dose or schedule modification of granulocyte colony-stimulating factor in platinum-based chemotherapy for elderly patients with lung cancer ...&#x27;</span>}`}}),nt=new E({props:{code:`dataset_head = pubmed_dataset_streamed.take(5)
list(dataset_head)`,highlighted:`dataset_head = pubmed_dataset_streamed.take(<span class="hljs-number">5</span>)
<span class="hljs-built_in">list</span>(dataset_head)`}}),lt=new E({props:{code:`[{'meta': {'pmid': 11409574, 'language': 'eng'},
  'text': 'Epidemiology of hypoxaemia in children with acute lower respiratory infection ...'},
 {'meta': {'pmid': 11409575, 'language': 'eng'},
  'text': 'Clinical signs of hypoxaemia in children with acute lower respiratory infection: indicators of oxygen therapy ...'},
 {'meta': {'pmid': 11409576, 'language': 'eng'},
  'text': "Hypoxaemia in children with severe pneumonia in Papua New Guinea ..."},
 {'meta': {'pmid': 11409577, 'language': 'eng'},
  'text': 'Oxygen concentrators and cylinders ...'},
 {'meta': {'pmid': 11409578, 'language': 'eng'},
  'text': 'Oxygen supply in rural africa: a personal experience ...'}]`,highlighted:`[{<span class="hljs-string">&#x27;meta&#x27;</span>: {<span class="hljs-string">&#x27;pmid&#x27;</span>: <span class="hljs-number">11409574</span>, <span class="hljs-string">&#x27;language&#x27;</span>: <span class="hljs-string">&#x27;eng&#x27;</span>},
  <span class="hljs-string">&#x27;text&#x27;</span>: <span class="hljs-string">&#x27;Epidemiology of hypoxaemia in children with acute lower respiratory infection ...&#x27;</span>},
 {<span class="hljs-string">&#x27;meta&#x27;</span>: {<span class="hljs-string">&#x27;pmid&#x27;</span>: <span class="hljs-number">11409575</span>, <span class="hljs-string">&#x27;language&#x27;</span>: <span class="hljs-string">&#x27;eng&#x27;</span>},
  <span class="hljs-string">&#x27;text&#x27;</span>: <span class="hljs-string">&#x27;Clinical signs of hypoxaemia in children with acute lower respiratory infection: indicators of oxygen therapy ...&#x27;</span>},
 {<span class="hljs-string">&#x27;meta&#x27;</span>: {<span class="hljs-string">&#x27;pmid&#x27;</span>: <span class="hljs-number">11409576</span>, <span class="hljs-string">&#x27;language&#x27;</span>: <span class="hljs-string">&#x27;eng&#x27;</span>},
  <span class="hljs-string">&#x27;text&#x27;</span>: <span class="hljs-string">&quot;Hypoxaemia in children with severe pneumonia in Papua New Guinea ...&quot;</span>},
 {<span class="hljs-string">&#x27;meta&#x27;</span>: {<span class="hljs-string">&#x27;pmid&#x27;</span>: <span class="hljs-number">11409577</span>, <span class="hljs-string">&#x27;language&#x27;</span>: <span class="hljs-string">&#x27;eng&#x27;</span>},
  <span class="hljs-string">&#x27;text&#x27;</span>: <span class="hljs-string">&#x27;Oxygen concentrators and cylinders ...&#x27;</span>},
 {<span class="hljs-string">&#x27;meta&#x27;</span>: {<span class="hljs-string">&#x27;pmid&#x27;</span>: <span class="hljs-number">11409578</span>, <span class="hljs-string">&#x27;language&#x27;</span>: <span class="hljs-string">&#x27;eng&#x27;</span>},
  <span class="hljs-string">&#x27;text&#x27;</span>: <span class="hljs-string">&#x27;Oxygen supply in rural africa: a personal experience ...&#x27;</span>}]`}}),ot=new E({props:{code:`# Skip the first 1,000 examples and include the rest in the training set
train_dataset = shuffled_dataset.skip(1000)
# Take the first 1,000 examples for the validation set
validation_dataset = shuffled_dataset.take(1000)`,highlighted:`<span class="hljs-comment"># Skip the first 1,000 examples and include the rest in the training set</span>
train_dataset = shuffled_dataset.skip(<span class="hljs-number">1000</span>)
<span class="hljs-comment"># Take the first 1,000 examples for the validation set</span>
validation_dataset = shuffled_dataset.take(<span class="hljs-number">1000</span>)`}}),it=new E({props:{code:`law_dataset_streamed = load_dataset(
    "json",
    data_files="https://mystic.the-eye.eu/public/AI/pile_preliminary_components/FreeLaw_Opinions.jsonl.zst",
    split="train",
    streaming=True,
)
next(iter(law_dataset_streamed))`,highlighted:`law_dataset_streamed = load_dataset(
    <span class="hljs-string">&quot;json&quot;</span>,
    data_files=<span class="hljs-string">&quot;https://mystic.the-eye.eu/public/AI/pile_preliminary_components/FreeLaw_Opinions.jsonl.zst&quot;</span>,
    split=<span class="hljs-string">&quot;train&quot;</span>,
    streaming=<span class="hljs-literal">True</span>,
)
<span class="hljs-built_in">next</span>(<span class="hljs-built_in">iter</span>(law_dataset_streamed))`}}),rt=new E({props:{code:`{'meta': {'case_ID': '110921.json',
  'case_jurisdiction': 'scotus.tar.gz',
  'date_created': '2010-04-28T17:12:49Z'},
 'text': '\\n461 U.S. 238 (1983)\\nOLIM ET AL.\\nv.\\nWAKINEKONA\\nNo. 81-1581.\\nSupreme Court of United States.\\nArgued January 19, 1983.\\nDecided April 26, 1983.\\nCERTIORARI TO THE UNITED STATES COURT OF APPEALS FOR THE NINTH CIRCUIT\\n*239 Michael A. Lilly, First Deputy Attorney General of Hawaii, argued the cause for petitioners. With him on the brief was James H. Dannenberg, Deputy Attorney General...'}`,highlighted:`{<span class="hljs-string">&#x27;meta&#x27;</span>: {<span class="hljs-string">&#x27;case_ID&#x27;</span>: <span class="hljs-string">&#x27;110921.json&#x27;</span>,
  <span class="hljs-string">&#x27;case_jurisdiction&#x27;</span>: <span class="hljs-string">&#x27;scotus.tar.gz&#x27;</span>,
  <span class="hljs-string">&#x27;date_created&#x27;</span>: <span class="hljs-string">&#x27;2010-04-28T17:12:49Z&#x27;</span>},
 <span class="hljs-string">&#x27;text&#x27;</span>: <span class="hljs-string">&#x27;\\n461 U.S. 238 (1983)\\nOLIM ET AL.\\nv.\\nWAKINEKONA\\nNo. 81-1581.\\nSupreme Court of United States.\\nArgued January 19, 1983.\\nDecided April 26, 1983.\\nCERTIORARI TO THE UNITED STATES COURT OF APPEALS FOR THE NINTH CIRCUIT\\n*239 Michael A. Lilly, First Deputy Attorney General of Hawaii, argued the cause for petitioners. With him on the brief was James H. Dannenberg, Deputy Attorney General...&#x27;</span>}`}}),pt=new E({props:{code:`from itertools import islice
from datasets import interleave_datasets

combined_dataset = interleave_datasets([pubmed_dataset_streamed, law_dataset_streamed])
list(islice(combined_dataset, 2))`,highlighted:`<span class="hljs-keyword">from</span> itertools <span class="hljs-keyword">import</span> islice
<span class="hljs-keyword">from</span> datasets <span class="hljs-keyword">import</span> interleave_datasets

combined_dataset = interleave_datasets([pubmed_dataset_streamed, law_dataset_streamed])
<span class="hljs-built_in">list</span>(islice(combined_dataset, <span class="hljs-number">2</span>))`}}),ht=new E({props:{code:`[{'meta': {'pmid': 11409574, 'language': 'eng'},
  'text': 'Epidemiology of hypoxaemia in children with acute lower respiratory infection ...'},
 {'meta': {'case_ID': '110921.json',
   'case_jurisdiction': 'scotus.tar.gz',
   'date_created': '2010-04-28T17:12:49Z'},
  'text': '\\n461 U.S. 238 (1983)\\nOLIM ET AL.\\nv.\\nWAKINEKONA\\nNo. 81-1581.\\nSupreme Court of United States.\\nArgued January 19, 1983.\\nDecided April 26, 1983.\\nCERTIORARI TO THE UNITED STATES COURT OF APPEALS FOR THE NINTH CIRCUIT\\n*239 Michael A. Lilly, First Deputy Attorney General of Hawaii, argued the cause for petitioners. With him on the brief was James H. Dannenberg, Deputy Attorney General...'}]`,highlighted:`[{<span class="hljs-string">&#x27;meta&#x27;</span>: {<span class="hljs-string">&#x27;pmid&#x27;</span>: <span class="hljs-number">11409574</span>, <span class="hljs-string">&#x27;language&#x27;</span>: <span class="hljs-string">&#x27;eng&#x27;</span>},
  <span class="hljs-string">&#x27;text&#x27;</span>: <span class="hljs-string">&#x27;Epidemiology of hypoxaemia in children with acute lower respiratory infection ...&#x27;</span>},
 {<span class="hljs-string">&#x27;meta&#x27;</span>: {<span class="hljs-string">&#x27;case_ID&#x27;</span>: <span class="hljs-string">&#x27;110921.json&#x27;</span>,
   <span class="hljs-string">&#x27;case_jurisdiction&#x27;</span>: <span class="hljs-string">&#x27;scotus.tar.gz&#x27;</span>,
   <span class="hljs-string">&#x27;date_created&#x27;</span>: <span class="hljs-string">&#x27;2010-04-28T17:12:49Z&#x27;</span>},
  <span class="hljs-string">&#x27;text&#x27;</span>: <span class="hljs-string">&#x27;\\n461 U.S. 238 (1983)\\nOLIM ET AL.\\nv.\\nWAKINEKONA\\nNo. 81-1581.\\nSupreme Court of United States.\\nArgued January 19, 1983.\\nDecided April 26, 1983.\\nCERTIORARI TO THE UNITED STATES COURT OF APPEALS FOR THE NINTH CIRCUIT\\n*239 Michael A. Lilly, First Deputy Attorney General of Hawaii, argued the cause for petitioners. With him on the brief was James H. Dannenberg, Deputy Attorney General...&#x27;</span>}]`}}),dt=new E({props:{code:`base_url = "https://mystic.the-eye.eu/public/AI/pile/"
data_files = {
    "train": [base_url + "train/" + f"{idx:02d}.jsonl.zst" for idx in range(30)],
    "validation": base_url + "val.jsonl.zst",
    "test": base_url + "test.jsonl.zst",
}
pile_dataset = load_dataset("json", data_files=data_files, streaming=True)
next(iter(pile_dataset["train"]))`,highlighted:`base_url = <span class="hljs-string">&quot;https://mystic.the-eye.eu/public/AI/pile/&quot;</span>
data_files = {
    <span class="hljs-string">&quot;train&quot;</span>: [base_url + <span class="hljs-string">&quot;train/&quot;</span> + <span class="hljs-string">f&quot;<span class="hljs-subst">{idx:02d}</span>.jsonl.zst&quot;</span> <span class="hljs-keyword">for</span> idx <span class="hljs-keyword">in</span> <span class="hljs-built_in">range</span>(<span class="hljs-number">30</span>)],
    <span class="hljs-string">&quot;validation&quot;</span>: base_url + <span class="hljs-string">&quot;val.jsonl.zst&quot;</span>,
    <span class="hljs-string">&quot;test&quot;</span>: base_url + <span class="hljs-string">&quot;test.jsonl.zst&quot;</span>,
}
pile_dataset = load_dataset(<span class="hljs-string">&quot;json&quot;</span>, data_files=data_files, streaming=<span class="hljs-literal">True</span>)
<span class="hljs-built_in">next</span>(<span class="hljs-built_in">iter</span>(pile_dataset[<span class="hljs-string">&quot;train&quot;</span>]))`}}),ft=new E({props:{code:`{'meta': {'pile_set_name': 'Pile-CC'},
 'text': 'It is done, and submitted. You can play \u201CSurvival of the Tastiest\u201D on Android, and on the web...'}`,highlighted:`{<span class="hljs-string">&#x27;meta&#x27;</span>: {<span class="hljs-string">&#x27;pile_set_name&#x27;</span>: <span class="hljs-string">&#x27;Pile-CC&#x27;</span>},
 <span class="hljs-string">&#x27;text&#x27;</span>: <span class="hljs-string">&#x27;It is done, and submitted. You can play \u201CSurvival of the Tastiest\u201D on Android, and on the web...&#x27;</span>}`}}),we=new ws({props:{$$slots:{default:[gi]},$$scope:{ctx:L}}}),{c(){h=o("meta"),O=d(),m=o("h1"),j=o("a"),D=o("span"),g(u.$$.fragment),P=d(),k=o("span"),x=a("Big data? \u{1F917} Datasets to the rescue!"),v=d(),g(A.$$.fragment),I=d(),T=o("p"),R=a("Nowadays it is not uncommon to find yourself working with multi-gigabyte datasets, especially if you\u2019re planning to pretrain a transformer like BERT or GPT-2 from scratch. In these cases, even "),z=o("em"),C=a("loading"),te=a(" the data can be a challenge. For example, the WebText corpus used to pretrain GPT-2 consists of over 8 million documents and 40 GB of text \u2014 loading this into your laptop\u2019s RAM is likely to give it a heart attack!"),W=d(),q=o("p"),se=a("Fortunately, \u{1F917} Datasets has been designed to overcome these limitations. It frees you from memory management problems by treating datasets as "),Pt=o("em"),Ra=a("memory-mapped"),qa=a(" files, and from hard drive limits by "),It=o("em"),Ma=a("streaming"),Na=a(" the entries in a corpus."),_s=d(),g(ve.$$.fragment),ys=d(),oe=o("p"),Ga=a("In this section we\u2019ll explore these features of \u{1F917} Datasets with a huge 825 GB corpus known as "),je=o("a"),La=a("the Pile"),Ba=a(". Let\u2019s get started!"),$s=d(),ae=o("h2"),ie=o("a"),zt=o("span"),g(Ee.$$.fragment),Fa=d(),Ot=o("span"),Ha=a("What is the Pile?"),xs=d(),S=o("p"),Ua=a("The Pile is an English text corpus that was created by "),ke=o("a"),Wa=a("EleutherAI"),Ja=a(" for training large-scale language models. It includes a diverse range of datasets, spanning scientific articles, GitHub code repositories, and filtered web text. The training corpus is available in "),Ae=o("a"),Ya=a("14 GB chunks"),Ka=a(", and you can also download several of the "),Te=o("a"),Za=a("individual components"),Qa=a(". Let\u2019s start by taking a look at the PubMed Abstracts dataset, which is a corpus of abstracts from 15 million biomedical publications on "),De=o("a"),Va=a("PubMed"),Xa=a(". The dataset is in "),Pe=o("a"),en=a("JSON Lines format"),tn=a(" and is compressed using the "),Ct=o("code"),sn=a("zstandard"),an=a(" library, so first we need to install that:"),vs=d(),g(Ie.$$.fragment),js=d(),re=o("p"),nn=a("Next, we can load the dataset using the method for remote files that we learned in "),ut=o("a"),ln=a("section 2"),on=a(":"),Es=d(),g(ze.$$.fragment),ks=d(),g(Oe.$$.fragment),As=d(),gt=o("p"),rn=a("We can see that there are 15,518,009 rows and 2 columns in our dataset \u2014 that\u2019s a lot!"),Ts=d(),g(pe.$$.fragment),Ds=d(),bt=o("p"),pn=a("Let\u2019s inspect the contents of the first example:"),Ps=d(),g(Ce.$$.fragment),Is=d(),g(Se.$$.fragment),zs=d(),wt=o("p"),hn=a("Okay, this looks like the abstract from a medical article. Now let\u2019s see how much RAM we\u2019ve used to load the dataset!"),Os=d(),ne=o("h2"),he=o("a"),St=o("span"),g(Re.$$.fragment),dn=d(),Rt=o("span"),fn=a("The magic of memory mapping"),Cs=d(),J=o("p"),cn=a("A simple way to measure memory usage in Python is with the "),qe=o("a"),qt=o("code"),mn=a("psutil"),un=a(" library, which can be installed with "),Mt=o("code"),gn=a("pip"),bn=a(" as follows:"),Ss=d(),g(Me.$$.fragment),Rs=d(),de=o("p"),wn=a("It provides a "),Nt=o("code"),_n=a("Process"),yn=a(" class that allows us to check the memory usage of the current process as follows:"),qs=d(),g(Ne.$$.fragment),Ms=d(),g(Ge.$$.fragment),Ns=d(),B=o("p"),$n=a("Here the "),Gt=o("code"),xn=a("rss"),vn=a(" attribute refers to the "),Lt=o("em"),jn=a("resident set size"),En=a(", which is the fraction of memory that a process occupies in RAM. This measurement also includes the memory used by the Python interpreter and the libraries we\u2019ve loaded, so the actual amount of memory used to load the dataset is a bit smaller. For comparison, let\u2019s see how large the dataset is on disk, using the "),Bt=o("code"),kn=a("dataset_size"),An=a(" attribute. Since the result is expressed in bytes like before, we need to manually convert it to gigabytes:"),Gs=d(),g(Le.$$.fragment),Ls=d(),g(Be.$$.fragment),Bs=d(),_t=o("p"),Tn=a("Nice \u2014 despite it being almost 20 GB large, we\u2019re able to load and access the dataset with much less RAM!"),Fs=d(),g(fe.$$.fragment),Hs=d(),Y=o("p"),Dn=a("If you\u2019re familiar with Pandas, this result might come as a surprise because of Wes Kinney\u2019s famous "),Fe=o("a"),Pn=a("rule of thumb"),In=a(" that you typically need 5 to 10 times as much RAM as the size of your dataset. So how does \u{1F917} Datasets solve this memory management problem? \u{1F917} Datasets treats each dataset as a "),He=o("a"),zn=a("memory-mapped file"),On=a(", which provides a mapping between RAM and filesystem storage that allows the library to access and operate on elements of the dataset without needing to fully load it into memory."),Us=d(),M=o("p"),Cn=a("Memory-mapped files can also be shared across multiple processes, which enables methods like "),Ft=o("code"),Sn=a("Dataset.map()"),Rn=a(" to be parallelized without needing to move or copy the dataset. Under the hood, these capabilities are all realized by the "),Ue=o("a"),qn=a("Apache Arrow"),Mn=a(" memory format and "),We=o("a"),Ht=o("code"),Nn=a("pyarrow"),Gn=a(" library, which make the data loading and processing lightning fast. (For more details about Apache Arrow and comparisons to Pandas, check out "),Je=o("a"),Ln=a("Dejan Simic\u2019s blog post"),Bn=a(".) To see this in action, let\u2019s run a little speed test by iterating over all the elements in the PubMed Abstracts dataset:"),Ws=d(),g(Ye.$$.fragment),Js=d(),g(Ke.$$.fragment),Ys=d(),K=o("p"),Fn=a("Here we\u2019ve used Python\u2019s "),Ut=o("code"),Hn=a("timeit"),Un=a(" module to measure the execution time taken by "),Wt=o("code"),Wn=a("code_snippet"),Jn=a(". You\u2019ll typically be able to iterate over a dataset at speed of a few tenths of a GB/s to several GB/s. This works great for the vast majority of applications, but sometimes you\u2019ll have to work with a dataset that is too large to even store on your laptop\u2019s hard drive. For example, if we tried to download the Pile in its entirety, we\u2019d need 825 GB of free disk space! To handle these cases, \u{1F917} Datasets provides a streaming feature that allows us to download and access elements on the fly, without needing to download the whole dataset. Let\u2019s take a look at how this works."),Ks=d(),g(ce.$$.fragment),Zs=d(),le=o("h2"),me=o("a"),Jt=o("span"),g(Ze.$$.fragment),Yn=d(),Yt=o("span"),Kn=a("Streaming datasets"),Qs=d(),Z=o("p"),Zn=a("To enable dataset streaming you just need to pass the "),Kt=o("code"),Qn=a("streaming=True"),Vn=a(" argument to the "),Zt=o("code"),Xn=a("load_dataset()"),el=a(" function. For example, let\u2019s load the PubMed Abstracts dataset again, but in streaming mode:"),Vs=d(),g(Qe.$$.fragment),Xs=d(),N=o("p"),tl=a("Instead of the familiar "),Qt=o("code"),sl=a("Dataset"),al=a(" that we\u2019ve encountered elsewhere in this chapter, the object returned with "),Vt=o("code"),nl=a("streaming=True"),ll=a(" is an "),Xt=o("code"),ol=a("IterableDataset"),il=a(". As the name suggests, to access the elements of an "),es=o("code"),rl=a("IterableDataset"),pl=a(" we need to iterate over it. We can access the first element of our streamed dataset as follows:"),ea=d(),g(Ve.$$.fragment),ta=d(),g(Xe.$$.fragment),sa=d(),Q=o("p"),hl=a("The elements from a streamed dataset can be processed on the fly using "),ts=o("code"),dl=a("IterableDataset.map()"),fl=a(", which is useful during training if you need to tokenize the inputs. The process is exactly the same as the one we used to tokenize our dataset in "),yt=o("a"),cl=a("Chapter 3"),ml=a(", with the only difference being that outputs are returned one by one:"),aa=d(),g(et.$$.fragment),na=d(),g(tt.$$.fragment),la=d(),g(ue.$$.fragment),oa=d(),F=o("p"),ul=a("You can also shuffle a streamed dataset using "),ss=o("code"),gl=a("IterableDataset.shuffle()"),bl=a(", but unlike "),as=o("code"),wl=a("Dataset.shuffle()"),_l=a(" this only shuffles the elements in a predefined "),ns=o("code"),yl=a("buffer_size"),$l=a(":"),ia=d(),g(st.$$.fragment),ra=d(),g(at.$$.fragment),pa=d(),H=o("p"),xl=a("In this example, we selected a random example from the first 10,000 examples in the buffer. Once an example is accessed, its spot in the buffer is filled with the next example in the corpus (i.e., the 10,001st example in the case above). You can also select elements from a streamed dataset using the "),ls=o("code"),vl=a("IterableDataset.take()"),jl=a(" and "),os=o("code"),El=a("IterableDataset.skip()"),kl=a(" functions, which act in a similar way to "),is=o("code"),Al=a("Dataset.select()"),Tl=a(". For example, to select the first 5 examples in the PubMed Abstracts dataset we can do the following:"),ha=d(),g(nt.$$.fragment),da=d(),g(lt.$$.fragment),fa=d(),ge=o("p"),Dl=a("Similarly, you can use the "),rs=o("code"),Pl=a("IterableDataset.skip()"),Il=a(" function to create training and validation splits from a shuffled dataset as follows:"),ca=d(),g(ot.$$.fragment),ma=d(),U=o("p"),zl=a("Let\u2019s round out our exploration of dataset streaming with a common application: combining multiple datasets together to create a single corpus. \u{1F917} Datasets provides an "),ps=o("code"),Ol=a("interleave_datasets()"),Cl=a(" function that converts a list of "),hs=o("code"),Sl=a("IterableDataset"),Rl=a(" objects into a single "),ds=o("code"),ql=a("IterableDataset"),Ml=a(", where the elements of the new dataset are obtained by alternating among the source examples. This function is especially useful when you\u2019re trying to combine large datasets, so as an example let\u2019s stream the FreeLaw subset of the Pile, which is a 51 GB dataset of legal opinions from US courts:"),ua=d(),g(it.$$.fragment),ga=d(),g(rt.$$.fragment),ba=d(),be=o("p"),Nl=a("This dataset is large enough to stress the RAM of most laptops, yet we\u2019ve been able to load and access it without breaking a sweat! Let\u2019s now combine the examples from the FreeLaw and PubMed Abstracts datasets with the "),fs=o("code"),Gl=a("interleave_datasets()"),Ll=a(" function:"),wa=d(),g(pt.$$.fragment),_a=d(),g(ht.$$.fragment),ya=d(),V=o("p"),Bl=a("Here we\u2019ve used the "),cs=o("code"),Fl=a("islice()"),Hl=a(" function from Python\u2019s "),ms=o("code"),Ul=a("itertools"),Wl=a(" module to select the first two examples from the combined dataset, and we can see that they match the first examples from each of the two source datasets."),$a=d(),$t=o("p"),Jl=a("Finally, if you want to stream the Pile in its 825 GB entirety, you can grab all the prepared files as follows:"),xa=d(),g(dt.$$.fragment),va=d(),g(ft.$$.fragment),ja=d(),g(we.$$.fragment),Ea=d(),xt=o("p"),Yl=a("You now have all the tools you need to load and process datasets of all shapes and sizes \u2014 but unless you\u2019re exceptionally lucky, there will come a point in your NLP journey where you\u2019ll have to actually create a dataset to solve the problem at hand. That\u2019s the topic of the next section!"),this.h()},l(e){const l=ri('[data-svelte="svelte-1phssyn"]',document.head);h=i(l,"META",{name:!0,content:!0}),l.forEach(s),O=f(e),m=i(e,"H1",{class:!0});var ct=r(m);j=i(ct,"A",{id:!0,class:!0,href:!0});var us=r(j);D=i(us,"SPAN",{});var gs=r(D);b(u.$$.fragment,gs),gs.forEach(s),us.forEach(s),P=f(ct),k=i(ct,"SPAN",{});var bs=r(k);x=n(bs,"Big data? \u{1F917} Datasets to the rescue!"),bs.forEach(s),ct.forEach(s),v=f(e),b(A.$$.fragment,e),I=f(e),T=i(e,"P",{});var mt=r(T);R=n(mt,"Nowadays it is not uncommon to find yourself working with multi-gigabyte datasets, especially if you\u2019re planning to pretrain a transformer like BERT or GPT-2 from scratch. In these cases, even "),z=i(mt,"EM",{});var Kl=r(z);C=n(Kl,"loading"),Kl.forEach(s),te=n(mt," the data can be a challenge. For example, the WebText corpus used to pretrain GPT-2 consists of over 8 million documents and 40 GB of text \u2014 loading this into your laptop\u2019s RAM is likely to give it a heart attack!"),mt.forEach(s),W=f(e),q=i(e,"P",{});var vt=r(q);se=n(vt,"Fortunately, \u{1F917} Datasets has been designed to overcome these limitations. It frees you from memory management problems by treating datasets as "),Pt=i(vt,"EM",{});var Zl=r(Pt);Ra=n(Zl,"memory-mapped"),Zl.forEach(s),qa=n(vt," files, and from hard drive limits by "),It=i(vt,"EM",{});var Ql=r(It);Ma=n(Ql,"streaming"),Ql.forEach(s),Na=n(vt," the entries in a corpus."),vt.forEach(s),_s=f(e),b(ve.$$.fragment,e),ys=f(e),oe=i(e,"P",{});var Aa=r(oe);Ga=n(Aa,"In this section we\u2019ll explore these features of \u{1F917} Datasets with a huge 825 GB corpus known as "),je=i(Aa,"A",{href:!0,rel:!0});var Vl=r(je);La=n(Vl,"the Pile"),Vl.forEach(s),Ba=n(Aa,". Let\u2019s get started!"),Aa.forEach(s),$s=f(e),ae=i(e,"H2",{class:!0});var Ta=r(ae);ie=i(Ta,"A",{id:!0,class:!0,href:!0});var Xl=r(ie);zt=i(Xl,"SPAN",{});var eo=r(zt);b(Ee.$$.fragment,eo),eo.forEach(s),Xl.forEach(s),Fa=f(Ta),Ot=i(Ta,"SPAN",{});var to=r(Ot);Ha=n(to,"What is the Pile?"),to.forEach(s),Ta.forEach(s),xs=f(e),S=i(e,"P",{});var G=r(S);Ua=n(G,"The Pile is an English text corpus that was created by "),ke=i(G,"A",{href:!0,rel:!0});var so=r(ke);Wa=n(so,"EleutherAI"),so.forEach(s),Ja=n(G," for training large-scale language models. It includes a diverse range of datasets, spanning scientific articles, GitHub code repositories, and filtered web text. The training corpus is available in "),Ae=i(G,"A",{href:!0,rel:!0});var ao=r(Ae);Ya=n(ao,"14 GB chunks"),ao.forEach(s),Ka=n(G,", and you can also download several of the "),Te=i(G,"A",{href:!0,rel:!0});var no=r(Te);Za=n(no,"individual components"),no.forEach(s),Qa=n(G,". Let\u2019s start by taking a look at the PubMed Abstracts dataset, which is a corpus of abstracts from 15 million biomedical publications on "),De=i(G,"A",{href:!0,rel:!0});var lo=r(De);Va=n(lo,"PubMed"),lo.forEach(s),Xa=n(G,". The dataset is in "),Pe=i(G,"A",{href:!0,rel:!0});var oo=r(Pe);en=n(oo,"JSON Lines format"),oo.forEach(s),tn=n(G," and is compressed using the "),Ct=i(G,"CODE",{});var io=r(Ct);sn=n(io,"zstandard"),io.forEach(s),an=n(G," library, so first we need to install that:"),G.forEach(s),vs=f(e),b(Ie.$$.fragment,e),js=f(e),re=i(e,"P",{});var Da=r(re);nn=n(Da,"Next, we can load the dataset using the method for remote files that we learned in "),ut=i(Da,"A",{href:!0});var ro=r(ut);ln=n(ro,"section 2"),ro.forEach(s),on=n(Da,":"),Da.forEach(s),Es=f(e),b(ze.$$.fragment,e),ks=f(e),b(Oe.$$.fragment,e),As=f(e),gt=i(e,"P",{});var po=r(gt);rn=n(po,"We can see that there are 15,518,009 rows and 2 columns in our dataset \u2014 that\u2019s a lot!"),po.forEach(s),Ts=f(e),b(pe.$$.fragment,e),Ds=f(e),bt=i(e,"P",{});var ho=r(bt);pn=n(ho,"Let\u2019s inspect the contents of the first example:"),ho.forEach(s),Ps=f(e),b(Ce.$$.fragment,e),Is=f(e),b(Se.$$.fragment,e),zs=f(e),wt=i(e,"P",{});var fo=r(wt);hn=n(fo,"Okay, this looks like the abstract from a medical article. Now let\u2019s see how much RAM we\u2019ve used to load the dataset!"),fo.forEach(s),Os=f(e),ne=i(e,"H2",{class:!0});var Pa=r(ne);he=i(Pa,"A",{id:!0,class:!0,href:!0});var co=r(he);St=i(co,"SPAN",{});var mo=r(St);b(Re.$$.fragment,mo),mo.forEach(s),co.forEach(s),dn=f(Pa),Rt=i(Pa,"SPAN",{});var uo=r(Rt);fn=n(uo,"The magic of memory mapping"),uo.forEach(s),Pa.forEach(s),Cs=f(e),J=i(e,"P",{});var jt=r(J);cn=n(jt,"A simple way to measure memory usage in Python is with the "),qe=i(jt,"A",{href:!0,rel:!0});var go=r(qe);qt=i(go,"CODE",{});var bo=r(qt);mn=n(bo,"psutil"),bo.forEach(s),go.forEach(s),un=n(jt," library, which can be installed with "),Mt=i(jt,"CODE",{});var wo=r(Mt);gn=n(wo,"pip"),wo.forEach(s),bn=n(jt," as follows:"),jt.forEach(s),Ss=f(e),b(Me.$$.fragment,e),Rs=f(e),de=i(e,"P",{});var Ia=r(de);wn=n(Ia,"It provides a "),Nt=i(Ia,"CODE",{});var _o=r(Nt);_n=n(_o,"Process"),_o.forEach(s),yn=n(Ia," class that allows us to check the memory usage of the current process as follows:"),Ia.forEach(s),qs=f(e),b(Ne.$$.fragment,e),Ms=f(e),b(Ge.$$.fragment,e),Ns=f(e),B=i(e,"P",{});var _e=r(B);$n=n(_e,"Here the "),Gt=i(_e,"CODE",{});var yo=r(Gt);xn=n(yo,"rss"),yo.forEach(s),vn=n(_e," attribute refers to the "),Lt=i(_e,"EM",{});var $o=r(Lt);jn=n($o,"resident set size"),$o.forEach(s),En=n(_e,", which is the fraction of memory that a process occupies in RAM. This measurement also includes the memory used by the Python interpreter and the libraries we\u2019ve loaded, so the actual amount of memory used to load the dataset is a bit smaller. For comparison, let\u2019s see how large the dataset is on disk, using the "),Bt=i(_e,"CODE",{});var xo=r(Bt);kn=n(xo,"dataset_size"),xo.forEach(s),An=n(_e," attribute. Since the result is expressed in bytes like before, we need to manually convert it to gigabytes:"),_e.forEach(s),Gs=f(e),b(Le.$$.fragment,e),Ls=f(e),b(Be.$$.fragment,e),Bs=f(e),_t=i(e,"P",{});var vo=r(_t);Tn=n(vo,"Nice \u2014 despite it being almost 20 GB large, we\u2019re able to load and access the dataset with much less RAM!"),vo.forEach(s),Fs=f(e),b(fe.$$.fragment,e),Hs=f(e),Y=i(e,"P",{});var Et=r(Y);Dn=n(Et,"If you\u2019re familiar with Pandas, this result might come as a surprise because of Wes Kinney\u2019s famous "),Fe=i(Et,"A",{href:!0,rel:!0});var jo=r(Fe);Pn=n(jo,"rule of thumb"),jo.forEach(s),In=n(Et," that you typically need 5 to 10 times as much RAM as the size of your dataset. So how does \u{1F917} Datasets solve this memory management problem? \u{1F917} Datasets treats each dataset as a "),He=i(Et,"A",{href:!0,rel:!0});var Eo=r(He);zn=n(Eo,"memory-mapped file"),Eo.forEach(s),On=n(Et,", which provides a mapping between RAM and filesystem storage that allows the library to access and operate on elements of the dataset without needing to fully load it into memory."),Et.forEach(s),Us=f(e),M=i(e,"P",{});var X=r(M);Cn=n(X,"Memory-mapped files can also be shared across multiple processes, which enables methods like "),Ft=i(X,"CODE",{});var ko=r(Ft);Sn=n(ko,"Dataset.map()"),ko.forEach(s),Rn=n(X," to be parallelized without needing to move or copy the dataset. Under the hood, these capabilities are all realized by the "),Ue=i(X,"A",{href:!0,rel:!0});var Ao=r(Ue);qn=n(Ao,"Apache Arrow"),Ao.forEach(s),Mn=n(X," memory format and "),We=i(X,"A",{href:!0,rel:!0});var To=r(We);Ht=i(To,"CODE",{});var Do=r(Ht);Nn=n(Do,"pyarrow"),Do.forEach(s),To.forEach(s),Gn=n(X," library, which make the data loading and processing lightning fast. (For more details about Apache Arrow and comparisons to Pandas, check out "),Je=i(X,"A",{href:!0,rel:!0});var Po=r(Je);Ln=n(Po,"Dejan Simic\u2019s blog post"),Po.forEach(s),Bn=n(X,".) To see this in action, let\u2019s run a little speed test by iterating over all the elements in the PubMed Abstracts dataset:"),X.forEach(s),Ws=f(e),b(Ye.$$.fragment,e),Js=f(e),b(Ke.$$.fragment,e),Ys=f(e),K=i(e,"P",{});var kt=r(K);Fn=n(kt,"Here we\u2019ve used Python\u2019s "),Ut=i(kt,"CODE",{});var Io=r(Ut);Hn=n(Io,"timeit"),Io.forEach(s),Un=n(kt," module to measure the execution time taken by "),Wt=i(kt,"CODE",{});var zo=r(Wt);Wn=n(zo,"code_snippet"),zo.forEach(s),Jn=n(kt,". You\u2019ll typically be able to iterate over a dataset at speed of a few tenths of a GB/s to several GB/s. This works great for the vast majority of applications, but sometimes you\u2019ll have to work with a dataset that is too large to even store on your laptop\u2019s hard drive. For example, if we tried to download the Pile in its entirety, we\u2019d need 825 GB of free disk space! To handle these cases, \u{1F917} Datasets provides a streaming feature that allows us to download and access elements on the fly, without needing to download the whole dataset. Let\u2019s take a look at how this works."),kt.forEach(s),Ks=f(e),b(ce.$$.fragment,e),Zs=f(e),le=i(e,"H2",{class:!0});var za=r(le);me=i(za,"A",{id:!0,class:!0,href:!0});var Oo=r(me);Jt=i(Oo,"SPAN",{});var Co=r(Jt);b(Ze.$$.fragment,Co),Co.forEach(s),Oo.forEach(s),Yn=f(za),Yt=i(za,"SPAN",{});var So=r(Yt);Kn=n(So,"Streaming datasets"),So.forEach(s),za.forEach(s),Qs=f(e),Z=i(e,"P",{});var At=r(Z);Zn=n(At,"To enable dataset streaming you just need to pass the "),Kt=i(At,"CODE",{});var Ro=r(Kt);Qn=n(Ro,"streaming=True"),Ro.forEach(s),Vn=n(At," argument to the "),Zt=i(At,"CODE",{});var qo=r(Zt);Xn=n(qo,"load_dataset()"),qo.forEach(s),el=n(At," function. For example, let\u2019s load the PubMed Abstracts dataset again, but in streaming mode:"),At.forEach(s),Vs=f(e),b(Qe.$$.fragment,e),Xs=f(e),N=i(e,"P",{});var ee=r(N);tl=n(ee,"Instead of the familiar "),Qt=i(ee,"CODE",{});var Mo=r(Qt);sl=n(Mo,"Dataset"),Mo.forEach(s),al=n(ee," that we\u2019ve encountered elsewhere in this chapter, the object returned with "),Vt=i(ee,"CODE",{});var No=r(Vt);nl=n(No,"streaming=True"),No.forEach(s),ll=n(ee," is an "),Xt=i(ee,"CODE",{});var Go=r(Xt);ol=n(Go,"IterableDataset"),Go.forEach(s),il=n(ee,". As the name suggests, to access the elements of an "),es=i(ee,"CODE",{});var Lo=r(es);rl=n(Lo,"IterableDataset"),Lo.forEach(s),pl=n(ee," we need to iterate over it. We can access the first element of our streamed dataset as follows:"),ee.forEach(s),ea=f(e),b(Ve.$$.fragment,e),ta=f(e),b(Xe.$$.fragment,e),sa=f(e),Q=i(e,"P",{});var Tt=r(Q);hl=n(Tt,"The elements from a streamed dataset can be processed on the fly using "),ts=i(Tt,"CODE",{});var Bo=r(ts);dl=n(Bo,"IterableDataset.map()"),Bo.forEach(s),fl=n(Tt,", which is useful during training if you need to tokenize the inputs. The process is exactly the same as the one we used to tokenize our dataset in "),yt=i(Tt,"A",{href:!0});var Fo=r(yt);cl=n(Fo,"Chapter 3"),Fo.forEach(s),ml=n(Tt,", with the only difference being that outputs are returned one by one:"),Tt.forEach(s),aa=f(e),b(et.$$.fragment,e),na=f(e),b(tt.$$.fragment,e),la=f(e),b(ue.$$.fragment,e),oa=f(e),F=i(e,"P",{});var ye=r(F);ul=n(ye,"You can also shuffle a streamed dataset using "),ss=i(ye,"CODE",{});var Ho=r(ss);gl=n(Ho,"IterableDataset.shuffle()"),Ho.forEach(s),bl=n(ye,", but unlike "),as=i(ye,"CODE",{});var Uo=r(as);wl=n(Uo,"Dataset.shuffle()"),Uo.forEach(s),_l=n(ye," this only shuffles the elements in a predefined "),ns=i(ye,"CODE",{});var Wo=r(ns);yl=n(Wo,"buffer_size"),Wo.forEach(s),$l=n(ye,":"),ye.forEach(s),ia=f(e),b(st.$$.fragment,e),ra=f(e),b(at.$$.fragment,e),pa=f(e),H=i(e,"P",{});var $e=r(H);xl=n($e,"In this example, we selected a random example from the first 10,000 examples in the buffer. Once an example is accessed, its spot in the buffer is filled with the next example in the corpus (i.e., the 10,001st example in the case above). You can also select elements from a streamed dataset using the "),ls=i($e,"CODE",{});var Jo=r(ls);vl=n(Jo,"IterableDataset.take()"),Jo.forEach(s),jl=n($e," and "),os=i($e,"CODE",{});var Yo=r(os);El=n(Yo,"IterableDataset.skip()"),Yo.forEach(s),kl=n($e," functions, which act in a similar way to "),is=i($e,"CODE",{});var Ko=r(is);Al=n(Ko,"Dataset.select()"),Ko.forEach(s),Tl=n($e,". For example, to select the first 5 examples in the PubMed Abstracts dataset we can do the following:"),$e.forEach(s),ha=f(e),b(nt.$$.fragment,e),da=f(e),b(lt.$$.fragment,e),fa=f(e),ge=i(e,"P",{});var Oa=r(ge);Dl=n(Oa,"Similarly, you can use the "),rs=i(Oa,"CODE",{});var Zo=r(rs);Pl=n(Zo,"IterableDataset.skip()"),Zo.forEach(s),Il=n(Oa," function to create training and validation splits from a shuffled dataset as follows:"),Oa.forEach(s),ca=f(e),b(ot.$$.fragment,e),ma=f(e),U=i(e,"P",{});var xe=r(U);zl=n(xe,"Let\u2019s round out our exploration of dataset streaming with a common application: combining multiple datasets together to create a single corpus. \u{1F917} Datasets provides an "),ps=i(xe,"CODE",{});var Qo=r(ps);Ol=n(Qo,"interleave_datasets()"),Qo.forEach(s),Cl=n(xe," function that converts a list of "),hs=i(xe,"CODE",{});var Vo=r(hs);Sl=n(Vo,"IterableDataset"),Vo.forEach(s),Rl=n(xe," objects into a single "),ds=i(xe,"CODE",{});var Xo=r(ds);ql=n(Xo,"IterableDataset"),Xo.forEach(s),Ml=n(xe,", where the elements of the new dataset are obtained by alternating among the source examples. This function is especially useful when you\u2019re trying to combine large datasets, so as an example let\u2019s stream the FreeLaw subset of the Pile, which is a 51 GB dataset of legal opinions from US courts:"),xe.forEach(s),ua=f(e),b(it.$$.fragment,e),ga=f(e),b(rt.$$.fragment,e),ba=f(e),be=i(e,"P",{});var Ca=r(be);Nl=n(Ca,"This dataset is large enough to stress the RAM of most laptops, yet we\u2019ve been able to load and access it without breaking a sweat! Let\u2019s now combine the examples from the FreeLaw and PubMed Abstracts datasets with the "),fs=i(Ca,"CODE",{});var ei=r(fs);Gl=n(ei,"interleave_datasets()"),ei.forEach(s),Ll=n(Ca," function:"),Ca.forEach(s),wa=f(e),b(pt.$$.fragment,e),_a=f(e),b(ht.$$.fragment,e),ya=f(e),V=i(e,"P",{});var Dt=r(V);Bl=n(Dt,"Here we\u2019ve used the "),cs=i(Dt,"CODE",{});var ti=r(cs);Fl=n(ti,"islice()"),ti.forEach(s),Hl=n(Dt," function from Python\u2019s "),ms=i(Dt,"CODE",{});var si=r(ms);Ul=n(si,"itertools"),si.forEach(s),Wl=n(Dt," module to select the first two examples from the combined dataset, and we can see that they match the first examples from each of the two source datasets."),Dt.forEach(s),$a=f(e),$t=i(e,"P",{});var ai=r($t);Jl=n(ai,"Finally, if you want to stream the Pile in its 825 GB entirety, you can grab all the prepared files as follows:"),ai.forEach(s),xa=f(e),b(dt.$$.fragment,e),va=f(e),b(ft.$$.fragment,e),ja=f(e),b(we.$$.fragment,e),Ea=f(e),xt=i(e,"P",{});var ni=r(xt);Yl=n(ni,"You now have all the tools you need to load and process datasets of all shapes and sizes \u2014 but unless you\u2019re exceptionally lucky, there will come a point in your NLP journey where you\u2019ll have to actually create a dataset to solve the problem at hand. That\u2019s the topic of the next section!"),ni.forEach(s),this.h()},h(){c(h,"name","hf:doc:metadata"),c(h,"content",JSON.stringify(wi)),c(j,"id","big-data-datasets-to-the-rescue"),c(j,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),c(j,"href","#big-data-datasets-to-the-rescue"),c(m,"class","relative group"),c(je,"href","https://pile.eleuther.ai"),c(je,"rel","nofollow"),c(ie,"id","what-is-the-pile"),c(ie,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),c(ie,"href","#what-is-the-pile"),c(ae,"class","relative group"),c(ke,"href","https://www.eleuther.ai"),c(ke,"rel","nofollow"),c(Ae,"href","https://mystic.the-eye.eu/public/AI/pile/"),c(Ae,"rel","nofollow"),c(Te,"href","https://mystic.the-eye.eu/public/AI/pile_preliminary_components/"),c(Te,"rel","nofollow"),c(De,"href","https://pubmed.ncbi.nlm.nih.gov/"),c(De,"rel","nofollow"),c(Pe,"href","https://jsonlines.org"),c(Pe,"rel","nofollow"),c(ut,"href","/course/chapter5/2"),c(he,"id","the-magic-of-memory-mapping"),c(he,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),c(he,"href","#the-magic-of-memory-mapping"),c(ne,"class","relative group"),c(qe,"href","https://psutil.readthedocs.io/en/latest/"),c(qe,"rel","nofollow"),c(Fe,"href","https://wesmckinney.com/blog/apache-arrow-pandas-internals/"),c(Fe,"rel","nofollow"),c(He,"href","https://en.wikipedia.org/wiki/Memory-mapped_file"),c(He,"rel","nofollow"),c(Ue,"href","https://arrow.apache.org"),c(Ue,"rel","nofollow"),c(We,"href","https://arrow.apache.org/docs/python/index.html"),c(We,"rel","nofollow"),c(Je,"href","https://towardsdatascience.com/apache-arrow-read-dataframe-with-zero-memory-69634092b1a"),c(Je,"rel","nofollow"),c(me,"id","streaming-datasets"),c(me,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),c(me,"href","#streaming-datasets"),c(le,"class","relative group"),c(yt,"href","/course/chapter3")},m(e,l){t(document.head,h),p(e,O,l),p(e,m,l),t(m,j),t(j,D),w(u,D,null),t(m,P),t(m,k),t(k,x),p(e,v,l),w(A,e,l),p(e,I,l),p(e,T,l),t(T,R),t(T,z),t(z,C),t(T,te),p(e,W,l),p(e,q,l),t(q,se),t(q,Pt),t(Pt,Ra),t(q,qa),t(q,It),t(It,Ma),t(q,Na),p(e,_s,l),w(ve,e,l),p(e,ys,l),p(e,oe,l),t(oe,Ga),t(oe,je),t(je,La),t(oe,Ba),p(e,$s,l),p(e,ae,l),t(ae,ie),t(ie,zt),w(Ee,zt,null),t(ae,Fa),t(ae,Ot),t(Ot,Ha),p(e,xs,l),p(e,S,l),t(S,Ua),t(S,ke),t(ke,Wa),t(S,Ja),t(S,Ae),t(Ae,Ya),t(S,Ka),t(S,Te),t(Te,Za),t(S,Qa),t(S,De),t(De,Va),t(S,Xa),t(S,Pe),t(Pe,en),t(S,tn),t(S,Ct),t(Ct,sn),t(S,an),p(e,vs,l),w(Ie,e,l),p(e,js,l),p(e,re,l),t(re,nn),t(re,ut),t(ut,ln),t(re,on),p(e,Es,l),w(ze,e,l),p(e,ks,l),w(Oe,e,l),p(e,As,l),p(e,gt,l),t(gt,rn),p(e,Ts,l),w(pe,e,l),p(e,Ds,l),p(e,bt,l),t(bt,pn),p(e,Ps,l),w(Ce,e,l),p(e,Is,l),w(Se,e,l),p(e,zs,l),p(e,wt,l),t(wt,hn),p(e,Os,l),p(e,ne,l),t(ne,he),t(he,St),w(Re,St,null),t(ne,dn),t(ne,Rt),t(Rt,fn),p(e,Cs,l),p(e,J,l),t(J,cn),t(J,qe),t(qe,qt),t(qt,mn),t(J,un),t(J,Mt),t(Mt,gn),t(J,bn),p(e,Ss,l),w(Me,e,l),p(e,Rs,l),p(e,de,l),t(de,wn),t(de,Nt),t(Nt,_n),t(de,yn),p(e,qs,l),w(Ne,e,l),p(e,Ms,l),w(Ge,e,l),p(e,Ns,l),p(e,B,l),t(B,$n),t(B,Gt),t(Gt,xn),t(B,vn),t(B,Lt),t(Lt,jn),t(B,En),t(B,Bt),t(Bt,kn),t(B,An),p(e,Gs,l),w(Le,e,l),p(e,Ls,l),w(Be,e,l),p(e,Bs,l),p(e,_t,l),t(_t,Tn),p(e,Fs,l),w(fe,e,l),p(e,Hs,l),p(e,Y,l),t(Y,Dn),t(Y,Fe),t(Fe,Pn),t(Y,In),t(Y,He),t(He,zn),t(Y,On),p(e,Us,l),p(e,M,l),t(M,Cn),t(M,Ft),t(Ft,Sn),t(M,Rn),t(M,Ue),t(Ue,qn),t(M,Mn),t(M,We),t(We,Ht),t(Ht,Nn),t(M,Gn),t(M,Je),t(Je,Ln),t(M,Bn),p(e,Ws,l),w(Ye,e,l),p(e,Js,l),w(Ke,e,l),p(e,Ys,l),p(e,K,l),t(K,Fn),t(K,Ut),t(Ut,Hn),t(K,Un),t(K,Wt),t(Wt,Wn),t(K,Jn),p(e,Ks,l),w(ce,e,l),p(e,Zs,l),p(e,le,l),t(le,me),t(me,Jt),w(Ze,Jt,null),t(le,Yn),t(le,Yt),t(Yt,Kn),p(e,Qs,l),p(e,Z,l),t(Z,Zn),t(Z,Kt),t(Kt,Qn),t(Z,Vn),t(Z,Zt),t(Zt,Xn),t(Z,el),p(e,Vs,l),w(Qe,e,l),p(e,Xs,l),p(e,N,l),t(N,tl),t(N,Qt),t(Qt,sl),t(N,al),t(N,Vt),t(Vt,nl),t(N,ll),t(N,Xt),t(Xt,ol),t(N,il),t(N,es),t(es,rl),t(N,pl),p(e,ea,l),w(Ve,e,l),p(e,ta,l),w(Xe,e,l),p(e,sa,l),p(e,Q,l),t(Q,hl),t(Q,ts),t(ts,dl),t(Q,fl),t(Q,yt),t(yt,cl),t(Q,ml),p(e,aa,l),w(et,e,l),p(e,na,l),w(tt,e,l),p(e,la,l),w(ue,e,l),p(e,oa,l),p(e,F,l),t(F,ul),t(F,ss),t(ss,gl),t(F,bl),t(F,as),t(as,wl),t(F,_l),t(F,ns),t(ns,yl),t(F,$l),p(e,ia,l),w(st,e,l),p(e,ra,l),w(at,e,l),p(e,pa,l),p(e,H,l),t(H,xl),t(H,ls),t(ls,vl),t(H,jl),t(H,os),t(os,El),t(H,kl),t(H,is),t(is,Al),t(H,Tl),p(e,ha,l),w(nt,e,l),p(e,da,l),w(lt,e,l),p(e,fa,l),p(e,ge,l),t(ge,Dl),t(ge,rs),t(rs,Pl),t(ge,Il),p(e,ca,l),w(ot,e,l),p(e,ma,l),p(e,U,l),t(U,zl),t(U,ps),t(ps,Ol),t(U,Cl),t(U,hs),t(hs,Sl),t(U,Rl),t(U,ds),t(ds,ql),t(U,Ml),p(e,ua,l),w(it,e,l),p(e,ga,l),w(rt,e,l),p(e,ba,l),p(e,be,l),t(be,Nl),t(be,fs),t(fs,Gl),t(be,Ll),p(e,wa,l),w(pt,e,l),p(e,_a,l),w(ht,e,l),p(e,ya,l),p(e,V,l),t(V,Bl),t(V,cs),t(cs,Fl),t(V,Hl),t(V,ms),t(ms,Ul),t(V,Wl),p(e,$a,l),p(e,$t,l),t($t,Jl),p(e,xa,l),w(dt,e,l),p(e,va,l),w(ft,e,l),p(e,ja,l),w(we,e,l),p(e,Ea,l),p(e,xt,l),t(xt,Yl),ka=!0},p(e,[l]){const ct={};l&2&&(ct.$$scope={dirty:l,ctx:e}),pe.$set(ct);const us={};l&2&&(us.$$scope={dirty:l,ctx:e}),fe.$set(us);const gs={};l&2&&(gs.$$scope={dirty:l,ctx:e}),ce.$set(gs);const bs={};l&2&&(bs.$$scope={dirty:l,ctx:e}),ue.$set(bs);const mt={};l&2&&(mt.$$scope={dirty:l,ctx:e}),we.$set(mt)},i(e){ka||(_(u.$$.fragment,e),_(A.$$.fragment,e),_(ve.$$.fragment,e),_(Ee.$$.fragment,e),_(Ie.$$.fragment,e),_(ze.$$.fragment,e),_(Oe.$$.fragment,e),_(pe.$$.fragment,e),_(Ce.$$.fragment,e),_(Se.$$.fragment,e),_(Re.$$.fragment,e),_(Me.$$.fragment,e),_(Ne.$$.fragment,e),_(Ge.$$.fragment,e),_(Le.$$.fragment,e),_(Be.$$.fragment,e),_(fe.$$.fragment,e),_(Ye.$$.fragment,e),_(Ke.$$.fragment,e),_(ce.$$.fragment,e),_(Ze.$$.fragment,e),_(Qe.$$.fragment,e),_(Ve.$$.fragment,e),_(Xe.$$.fragment,e),_(et.$$.fragment,e),_(tt.$$.fragment,e),_(ue.$$.fragment,e),_(st.$$.fragment,e),_(at.$$.fragment,e),_(nt.$$.fragment,e),_(lt.$$.fragment,e),_(ot.$$.fragment,e),_(it.$$.fragment,e),_(rt.$$.fragment,e),_(pt.$$.fragment,e),_(ht.$$.fragment,e),_(dt.$$.fragment,e),_(ft.$$.fragment,e),_(we.$$.fragment,e),ka=!0)},o(e){y(u.$$.fragment,e),y(A.$$.fragment,e),y(ve.$$.fragment,e),y(Ee.$$.fragment,e),y(Ie.$$.fragment,e),y(ze.$$.fragment,e),y(Oe.$$.fragment,e),y(pe.$$.fragment,e),y(Ce.$$.fragment,e),y(Se.$$.fragment,e),y(Re.$$.fragment,e),y(Me.$$.fragment,e),y(Ne.$$.fragment,e),y(Ge.$$.fragment,e),y(Le.$$.fragment,e),y(Be.$$.fragment,e),y(fe.$$.fragment,e),y(Ye.$$.fragment,e),y(Ke.$$.fragment,e),y(ce.$$.fragment,e),y(Ze.$$.fragment,e),y(Qe.$$.fragment,e),y(Ve.$$.fragment,e),y(Xe.$$.fragment,e),y(et.$$.fragment,e),y(tt.$$.fragment,e),y(ue.$$.fragment,e),y(st.$$.fragment,e),y(at.$$.fragment,e),y(nt.$$.fragment,e),y(lt.$$.fragment,e),y(ot.$$.fragment,e),y(it.$$.fragment,e),y(rt.$$.fragment,e),y(pt.$$.fragment,e),y(ht.$$.fragment,e),y(dt.$$.fragment,e),y(ft.$$.fragment,e),y(we.$$.fragment,e),ka=!1},d(e){s(h),e&&s(O),e&&s(m),$(u),e&&s(v),$(A,e),e&&s(I),e&&s(T),e&&s(W),e&&s(q),e&&s(_s),$(ve,e),e&&s(ys),e&&s(oe),e&&s($s),e&&s(ae),$(Ee),e&&s(xs),e&&s(S),e&&s(vs),$(Ie,e),e&&s(js),e&&s(re),e&&s(Es),$(ze,e),e&&s(ks),$(Oe,e),e&&s(As),e&&s(gt),e&&s(Ts),$(pe,e),e&&s(Ds),e&&s(bt),e&&s(Ps),$(Ce,e),e&&s(Is),$(Se,e),e&&s(zs),e&&s(wt),e&&s(Os),e&&s(ne),$(Re),e&&s(Cs),e&&s(J),e&&s(Ss),$(Me,e),e&&s(Rs),e&&s(de),e&&s(qs),$(Ne,e),e&&s(Ms),$(Ge,e),e&&s(Ns),e&&s(B),e&&s(Gs),$(Le,e),e&&s(Ls),$(Be,e),e&&s(Bs),e&&s(_t),e&&s(Fs),$(fe,e),e&&s(Hs),e&&s(Y),e&&s(Us),e&&s(M),e&&s(Ws),$(Ye,e),e&&s(Js),$(Ke,e),e&&s(Ys),e&&s(K),e&&s(Ks),$(ce,e),e&&s(Zs),e&&s(le),$(Ze),e&&s(Qs),e&&s(Z),e&&s(Vs),$(Qe,e),e&&s(Xs),e&&s(N),e&&s(ea),$(Ve,e),e&&s(ta),$(Xe,e),e&&s(sa),e&&s(Q),e&&s(aa),$(et,e),e&&s(na),$(tt,e),e&&s(la),$(ue,e),e&&s(oa),e&&s(F),e&&s(ia),$(st,e),e&&s(ra),$(at,e),e&&s(pa),e&&s(H),e&&s(ha),$(nt,e),e&&s(da),$(lt,e),e&&s(fa),e&&s(ge),e&&s(ca),$(ot,e),e&&s(ma),e&&s(U),e&&s(ua),$(it,e),e&&s(ga),$(rt,e),e&&s(ba),e&&s(be),e&&s(wa),$(pt,e),e&&s(_a),$(ht,e),e&&s(ya),e&&s(V),e&&s($a),e&&s($t),e&&s(xa),$(dt,e),e&&s(va),$(ft,e),e&&s(ja),$(we,e),e&&s(Ea),e&&s(xt)}}}const wi={local:"big-data-datasets-to-the-rescue",sections:[{local:"what-is-the-pile",title:"What is the Pile?"},{local:"the-magic-of-memory-mapping",title:"The magic of memory mapping"},{local:"streaming-datasets",title:"Streaming datasets"}],title:"Big data? \u{1F917} Datasets to the rescue!"};function _i(L){return pi(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class ki extends li{constructor(h){super();oi(this,h,_i,bi,ii,{})}}export{ki as default,wi as metadata};
