import{S as Mr,i as Br,s as Nr,e as r,k as d,w as _,t as a,M as Sr,c as i,d as t,m as c,a as p,x as $,h as l,b as m,G as s,g as o,y as b,q as g,o as x,B as j,v as Lr}from"../../chunks/vendor-hf-doc-builder.js";import{T as pt}from"../../chunks/Tip-hf-doc-builder.js";import{Y as Hr}from"../../chunks/Youtube-hf-doc-builder.js";import{I as Gr}from"../../chunks/IconCopyLink-hf-doc-builder.js";import{C as A}from"../../chunks/CodeBlock-hf-doc-builder.js";import{C as Ur}from"../../chunks/CourseFloatingBanner-hf-doc-builder.js";import"../../chunks/DocNotebookDropdown-hf-doc-builder.js";function Fr(B){let f,O,u,v,T,h,C,D,E,w,P,I,k;return{c(){f=r("p"),O=a("\u270E \u9ED8\u8BA4\u60C5\u51B5\u4E0B, \u{1F917} Datasets \u4F1A\u81EA\u52A8\u89E3\u538B\u52A0\u8F7D\u6570\u636E\u96C6\u6240\u9700\u7684\u6587\u4EF6\u3002 \u5982\u679C\u4F60\u60F3\u4FDD\u7559\u786C\u76D8\u7A7A\u95F4, \u4F60\u53EF\u4EE5\u4F20\u9012 "),u=r("code"),v=a("DownloadConfig(delete_extracted=True)"),T=a(" \u5230 "),h=r("code"),C=a("download_config"),D=a(" \u7684 "),E=r("code"),w=a("load_dataset()"),P=a("\u53C2\u6570. \u6709\u5173\u66F4\u591A\u8BE6\u7EC6\u4FE1\u606F, \u8BF7\u53C2\u9605\u6587\u6863]("),I=r("a"),k=a("https://huggingface.co/docs/datasets/package_reference/builder_classes.html?#datasets.utils.DownloadConfig)\u3002"),this.h()},l(q){f=i(q,"P",{});var y=p(f);O=l(y,"\u270E \u9ED8\u8BA4\u60C5\u51B5\u4E0B, \u{1F917} Datasets \u4F1A\u81EA\u52A8\u89E3\u538B\u52A0\u8F7D\u6570\u636E\u96C6\u6240\u9700\u7684\u6587\u4EF6\u3002 \u5982\u679C\u4F60\u60F3\u4FDD\u7559\u786C\u76D8\u7A7A\u95F4, \u4F60\u53EF\u4EE5\u4F20\u9012 "),u=i(y,"CODE",{});var N=p(u);v=l(N,"DownloadConfig(delete_extracted=True)"),N.forEach(t),T=l(y," \u5230 "),h=i(y,"CODE",{});var se=p(h);C=l(se,"download_config"),se.forEach(t),D=l(y," \u7684 "),E=i(y,"CODE",{});var W=p(E);w=l(W,"load_dataset()"),W.forEach(t),P=l(y,"\u53C2\u6570. \u6709\u5173\u66F4\u591A\u8BE6\u7EC6\u4FE1\u606F, \u8BF7\u53C2\u9605\u6587\u6863]("),I=i(y,"A",{href:!0,rel:!0});var S=p(I);k=l(S,"https://huggingface.co/docs/datasets/package_reference/builder_classes.html?#datasets.utils.DownloadConfig)\u3002"),S.forEach(t),y.forEach(t),this.h()},h(){m(I,"href","https://huggingface.co/docs/datasets/package_reference/builder_classes.html?#datasets.utils.DownloadConfig)%E3%80%82"),m(I,"rel","nofollow")},m(q,y){o(q,f,y),s(f,O),s(f,u),s(u,v),s(f,T),s(f,h),s(h,C),s(f,D),s(f,E),s(E,w),s(f,P),s(f,I),s(I,k)},d(q){q&&t(f)}}}function Wr(B){let f,O,u,v,T,h,C,D,E,w,P;return{c(){f=r("p"),O=a("\u270F\uFE0F "),u=r("strong"),v=a("\u8BD5\u8BD5\u770B!"),T=a(" \u4ECE"),h=r("a"),C=a("subsets"),D=a("\u4E2D\u9009\u62E9\u4E00\u4E2A\u5927\u4E8E\u4F60\u7684\u7B14\u8BB0\u672C\u6216\u8005\u53F0\u5F0F\u673A\u7684RAM\u5927\u5C0F\u7684\u5B50\u96C6, \u7528 \u{1F917} Datasets\u52A0\u8F7D\u8FD9\u4E2A\u6570\u636E\u96C6, \u5E76\u4E14\u6D4B\u91CFRAM\u7684\u4F7F\u7528\u91CF\u3002 \u8BF7\u6CE8\u610F, \u8981\u83B7\u5F97\u51C6\u786E\u7684\u6D4B\u91CF\u7ED3\u679C, \u4F60\u9700\u8981\u5728\u53E6\u4E00\u4E2A\u8FDB\u7A0B\u4E2D\u6267\u884C\u8FD9\u4E2A\u64CD\u4F5C\u3002\u4F60\u53EF\u4EE5\u5728 "),E=r("a"),w=a("the Pile paper"),P=a("\u7684\u8868\u4E00\u4E2D\u627E\u5230\u6BCF\u4E2A\u5B50\u96C6\u89E3\u538B\u540E\u7684\u5927\u5C0F\u3002"),this.h()},l(I){f=i(I,"P",{});var k=p(f);O=l(k,"\u270F\uFE0F "),u=i(k,"STRONG",{});var q=p(u);v=l(q,"\u8BD5\u8BD5\u770B!"),q.forEach(t),T=l(k," \u4ECE"),h=i(k,"A",{href:!0,rel:!0});var y=p(h);C=l(y,"subsets"),y.forEach(t),D=l(k,"\u4E2D\u9009\u62E9\u4E00\u4E2A\u5927\u4E8E\u4F60\u7684\u7B14\u8BB0\u672C\u6216\u8005\u53F0\u5F0F\u673A\u7684RAM\u5927\u5C0F\u7684\u5B50\u96C6, \u7528 \u{1F917} Datasets\u52A0\u8F7D\u8FD9\u4E2A\u6570\u636E\u96C6, \u5E76\u4E14\u6D4B\u91CFRAM\u7684\u4F7F\u7528\u91CF\u3002 \u8BF7\u6CE8\u610F, \u8981\u83B7\u5F97\u51C6\u786E\u7684\u6D4B\u91CF\u7ED3\u679C, \u4F60\u9700\u8981\u5728\u53E6\u4E00\u4E2A\u8FDB\u7A0B\u4E2D\u6267\u884C\u8FD9\u4E2A\u64CD\u4F5C\u3002\u4F60\u53EF\u4EE5\u5728 "),E=i(k,"A",{href:!0,rel:!0});var N=p(E);w=l(N,"the Pile paper"),N.forEach(t),P=l(k,"\u7684\u8868\u4E00\u4E2D\u627E\u5230\u6BCF\u4E2A\u5B50\u96C6\u89E3\u538B\u540E\u7684\u5927\u5C0F\u3002"),k.forEach(t),this.h()},h(){m(h,"href","https://the-eye.eu/public/AI/pile_preliminary_components/"),m(h,"rel","nofollow"),m(E,"href","https://arxiv.org/abs/2101.00027"),m(E,"rel","nofollow")},m(I,k){o(I,f,k),s(f,O),s(f,u),s(u,v),s(f,T),s(f,h),s(h,C),s(f,D),s(f,E),s(E,w),s(f,P)},d(I){I&&t(f)}}}function Jr(B){let f,O,u,v,T,h,C;return{c(){f=r("p"),O=a("\u{1F4A1}\u5728 Jupyter \u7B14\u8BB0\u4E2D\u4F60\u8FD8\u53EF\u4EE5\u4F7F\u7528"),u=r("a"),v=r("code"),T=a("%%timeit"),h=a(" magic function"),C=a("\u4E3A\u5355\u5143\u683C\u8BA1\u65F6\u3002"),this.h()},l(D){f=i(D,"P",{});var E=p(f);O=l(E,"\u{1F4A1}\u5728 Jupyter \u7B14\u8BB0\u4E2D\u4F60\u8FD8\u53EF\u4EE5\u4F7F\u7528"),u=i(E,"A",{href:!0,rel:!0});var w=p(u);v=i(w,"CODE",{});var P=p(v);T=l(P,"%%timeit"),P.forEach(t),h=l(w," magic function"),w.forEach(t),C=l(E,"\u4E3A\u5355\u5143\u683C\u8BA1\u65F6\u3002"),E.forEach(t),this.h()},h(){m(u,"href","https://ipython.readthedocs.io/en/stable/interactive/magics.html#magic-timeit"),m(u,"rel","nofollow")},m(D,E){o(D,f,E),s(f,O),s(f,u),s(u,v),s(v,T),s(u,h),s(f,C)},d(D){D&&t(f)}}}function Kr(B){let f,O,u,v,T,h,C,D;return{c(){f=r("p"),O=a("\u{1F4A1} \u4F60\u53EF\u4EE5\u4F20\u9012 "),u=r("code"),v=a("batched=True"),T=a(" \u6765\u901A\u8FC7\u6D41\u5F0F\u52A0\u901F\u6807\u8BB0\u5316, \u5982\u540C\u6211\u4EEC\u5728\u4E0A\u4E00\u8282\u770B\u5230\u7684\u90A3\u6837\u3002\u5B83\u5C06\u9010\u6279\u5904\u7406\u793A\u4F8B; \u9ED8\u8BA4\u7684\u6279\u91CF\u5927\u5C0F\u4E3A 1,000, \u53EF\u4EE5\u4F7F\u7528 "),h=r("code"),C=a("batch_size"),D=a(" \u53C2\u6570\u6307\u5B9A\u6279\u91CF\u5927\u5C0F\u3002")},l(E){f=i(E,"P",{});var w=p(f);O=l(w,"\u{1F4A1} \u4F60\u53EF\u4EE5\u4F20\u9012 "),u=i(w,"CODE",{});var P=p(u);v=l(P,"batched=True"),P.forEach(t),T=l(w," \u6765\u901A\u8FC7\u6D41\u5F0F\u52A0\u901F\u6807\u8BB0\u5316, \u5982\u540C\u6211\u4EEC\u5728\u4E0A\u4E00\u8282\u770B\u5230\u7684\u90A3\u6837\u3002\u5B83\u5C06\u9010\u6279\u5904\u7406\u793A\u4F8B; \u9ED8\u8BA4\u7684\u6279\u91CF\u5927\u5C0F\u4E3A 1,000, \u53EF\u4EE5\u4F7F\u7528 "),h=i(w,"CODE",{});var I=p(h);C=l(I,"batch_size"),I.forEach(t),D=l(w," \u53C2\u6570\u6307\u5B9A\u6279\u91CF\u5927\u5C0F\u3002"),w.forEach(t)},m(E,w){o(E,f,w),s(f,O),s(f,u),s(u,v),s(f,T),s(f,h),s(h,C),s(f,D)},d(E){E&&t(f)}}}function Yr(B){let f,O,u,v,T,h,C,D,E,w,P,I,k;return{c(){f=r("p"),O=a("\u270F\uFE0F "),u=r("strong"),v=a("\u8BD5\u8BD5\u770B!"),T=a(" \u4F7F\u7528\u50CF"),h=r("a"),C=r("code"),D=a("mc4"),E=a(" \u6216\u8005 "),w=r("a"),P=r("code"),I=a("oscar"),k=a("\u8FD9\u6837\u7684\u5927\u578B Common Crawl \u8BED\u6599\u5E93\u6765\u521B\u5EFA\u4E00\u4E2A\u6D41\u5F0F\u591A\u8BED\u8A00\u6570\u636E\u96C6, \u8BE5\u6570\u636E\u96C6\u4EE3\u8868\u4F60\u9009\u62E9\u7684\u56FD\u5BB6/\u5730\u533A\u8BED\u8A00\u7684\u53E3\u8BED\u6BD4\u4F8B\u3002\u4F8B\u5982, \u745E\u58EB\u7684\u56DB\u79CD\u6C11\u65CF\u8BED\u8A00\u5206\u522B\u662F\u5FB7\u8BED\u3001\u6CD5\u8BED\u3001\u610F\u5927\u5229\u8BED\u548C\u7F57\u66FC\u4EC0\u8BED, \u56E0\u6B64\u4F60\u53EF\u4EE5\u5C1D\u8BD5\u6839\u636E\u6839\u636E\u53E3\u8BED\u6BD4\u4F8B\u5BF9Oscar\u5B50\u96C6\u8FDB\u884C\u91C7\u7528\u6765\u521B\u5EFA\u745E\u58EB\u8BED\u6599\u5E93\u3002"),this.h()},l(q){f=i(q,"P",{});var y=p(f);O=l(y,"\u270F\uFE0F "),u=i(y,"STRONG",{});var N=p(u);v=l(N,"\u8BD5\u8BD5\u770B!"),N.forEach(t),T=l(y," \u4F7F\u7528\u50CF"),h=i(y,"A",{href:!0,rel:!0});var se=p(h);C=i(se,"CODE",{});var W=p(C);D=l(W,"mc4"),W.forEach(t),se.forEach(t),E=l(y," \u6216\u8005 "),w=i(y,"A",{href:!0,rel:!0});var S=p(w);P=i(S,"CODE",{});var fs=p(P);I=l(fs,"oscar"),fs.forEach(t),S.forEach(t),k=l(y,"\u8FD9\u6837\u7684\u5927\u578B Common Crawl \u8BED\u6599\u5E93\u6765\u521B\u5EFA\u4E00\u4E2A\u6D41\u5F0F\u591A\u8BED\u8A00\u6570\u636E\u96C6, \u8BE5\u6570\u636E\u96C6\u4EE3\u8868\u4F60\u9009\u62E9\u7684\u56FD\u5BB6/\u5730\u533A\u8BED\u8A00\u7684\u53E3\u8BED\u6BD4\u4F8B\u3002\u4F8B\u5982, \u745E\u58EB\u7684\u56DB\u79CD\u6C11\u65CF\u8BED\u8A00\u5206\u522B\u662F\u5FB7\u8BED\u3001\u6CD5\u8BED\u3001\u610F\u5927\u5229\u8BED\u548C\u7F57\u66FC\u4EC0\u8BED, \u56E0\u6B64\u4F60\u53EF\u4EE5\u5C1D\u8BD5\u6839\u636E\u6839\u636E\u53E3\u8BED\u6BD4\u4F8B\u5BF9Oscar\u5B50\u96C6\u8FDB\u884C\u91C7\u7528\u6765\u521B\u5EFA\u745E\u58EB\u8BED\u6599\u5E93\u3002"),y.forEach(t),this.h()},h(){m(h,"href","https://huggingface.co/datasets/mc4"),m(h,"rel","nofollow"),m(w,"href","https://huggingface.co/datasets/oscar"),m(w,"rel","nofollow")},m(q,y){o(q,f,y),s(f,O),s(f,u),s(u,v),s(f,T),s(f,h),s(h,C),s(C,D),s(f,E),s(f,w),s(w,P),s(P,I),s(f,k)},d(q){q&&t(f)}}}function Zr(B){let f,O,u,v,T,h,C,D,E,w,P,I,k,q,y,N,se,W,S,fs,ot,be,ft,ae,wa,ge,ya,va,dt,te,le,vs,xe,Aa,As,Da,ct,z,Pa,je,ka,Ta,Ee,Ca,Ia,we,Oa,za,ye,qa,Ra,ve,Ga,Ma,Ds,Ba,Na,Ps,Sa,La,mt,Ae,ut,ne,Ha,ds,Ua,Fa,ht,De,_t,Pe,$t,cs,Wa,bt,re,gt,ms,Ja,xt,ke,jt,Te,Et,us,Ka,wt,Ce,Ya,yt,J,Za,Ie,ks,Qa,Va,Ts,Xa,el,vt,Oe,At,ie,sl,Cs,tl,al,Dt,ze,Pt,qe,kt,L,ll,Is,nl,rl,Os,il,pl,zs,ol,fl,Tt,Re,Ct,Ge,It,hs,dl,Ot,pe,zt,K,cl,Me,ml,ul,Be,hl,_l,qt,G,$l,qs,bl,gl,Ne,xl,jl,Se,Rs,El,wl,Le,yl,vl,Rt,He,Gt,Ue,Mt,Y,Al,Gs,Dl,Pl,Ms,kl,Tl,Bt,oe,Nt,Fe,Cl,St,Z,Il,Bs,Ol,zl,Ns,ql,Rl,Lt,We,Ht,M,Gl,Ss,Ml,Bl,Ls,Nl,Sl,Hs,Ll,Hl,Us,Ul,Fl,Ut,Je,Ft,Ke,Wt,Q,Wl,Fs,Jl,Kl,_s,Yl,Zl,Jt,Ye,Kt,Ze,Yt,fe,Zt,H,Ql,Ws,Vl,Xl,Js,en,sn,Ks,tn,an,Qt,Qe,Vt,Ve,Xt,U,ln,Ys,nn,rn,Zs,pn,on,Qs,fn,dn,ea,Xe,sa,es,ta,de,cn,Vs,mn,un,aa,ss,la,F,hn,Xs,_n,$n,et,bn,gn,st,xn,jn,na,ts,ra,as,ia,ce,En,tt,wn,yn,pa,ls,oa,ns,fa,V,vn,at,An,Dn,lt,Pn,kn,da,$s,Tn,ca,rs,ma,is,ua,me,ha,bs,Cn,_a;return h=new Gr({}),P=new Ur({props:{chapter:5,classNames:"absolute z-10 right-0 top-0",notebooks:[{label:"Google Colab",value:"https://colab.research.google.com/github/huggingface/notebooks/blob/master/course/zh-CN/chapter5/section4.ipynb"},{label:"Aws Studio",value:"https://studiolab.sagemaker.aws/import/github/huggingface/notebooks/blob/master/course/zh-CN/chapter5/section4.ipynb"}]}}),be=new Hr({props:{id:"JwISwTCPPWo"}}),xe=new Gr({}),Ae=new A({props:{code:"!pip install zstandard",highlighted:"!pip install zstandard"}}),De=new A({props:{code:`from datasets import load_dataset

# This takes a few minutes to run, so go grab a tea or coffee while you wait :)
data_files = "https://the-eye.eu/public/AI/pile_preliminary_components/PUBMED_title_abstracts_2019_baseline.jsonl.zst"
pubmed_dataset = load_dataset("json", data_files=data_files, split="train")
pubmed_dataset`,highlighted:`<span class="hljs-keyword">from</span> datasets <span class="hljs-keyword">import</span> load_dataset

<span class="hljs-comment"># This takes a few minutes to run, so go grab a tea or coffee while you wait :)</span>
data_files = <span class="hljs-string">&quot;https://the-eye.eu/public/AI/pile_preliminary_components/PUBMED_title_abstracts_2019_baseline.jsonl.zst&quot;</span>
pubmed_dataset = load_dataset(<span class="hljs-string">&quot;json&quot;</span>, data_files=data_files, split=<span class="hljs-string">&quot;train&quot;</span>)
pubmed_dataset`}}),Pe=new A({props:{code:`Dataset({
    features: ['meta', 'text'],
    num_rows: 15518009
})`,highlighted:`Dataset({
    features: [<span class="hljs-string">&#x27;meta&#x27;</span>, <span class="hljs-string">&#x27;text&#x27;</span>],
    num_rows: <span class="hljs-number">15518009</span>
})`}}),re=new pt({props:{$$slots:{default:[Fr]},$$scope:{ctx:B}}}),ke=new A({props:{code:"pubmed_dataset[0]",highlighted:'pubmed_dataset[<span class="hljs-number">0</span>]'}}),Te=new A({props:{code:`{'meta': {'pmid': 11409574, 'language': 'eng'},
 'text': 'Epidemiology of hypoxaemia in children with acute lower respiratory infection.\\nTo determine the prevalence of hypoxaemia in children aged under 5 years suffering acute lower respiratory infections (ALRI), the risk factors for hypoxaemia in children under 5 years of age with ALRI, and the association of hypoxaemia with an increased risk of dying in children of the same age ...'}`,highlighted:`{<span class="hljs-string">&#x27;meta&#x27;</span>: {<span class="hljs-string">&#x27;pmid&#x27;</span>: <span class="hljs-number">11409574</span>, <span class="hljs-string">&#x27;language&#x27;</span>: <span class="hljs-string">&#x27;eng&#x27;</span>},
 <span class="hljs-string">&#x27;text&#x27;</span>: <span class="hljs-string">&#x27;Epidemiology of hypoxaemia in children with acute lower respiratory infection.\\nTo determine the prevalence of hypoxaemia in children aged under 5 years suffering acute lower respiratory infections (ALRI), the risk factors for hypoxaemia in children under 5 years of age with ALRI, and the association of hypoxaemia with an increased risk of dying in children of the same age ...&#x27;</span>}`}}),Oe=new A({props:{code:"!pip install psutil",highlighted:"!pip install psutil"}}),ze=new A({props:{code:`import psutil

# Process.memory_info is expressed in bytes, so convert to megabytes
print(f"RAM used: {psutil.Process().memory_info().rss / (1024 * 1024):.2f} MB")`,highlighted:`<span class="hljs-keyword">import</span> psutil

<span class="hljs-comment"># Process.memory_info is expressed in bytes, so convert to megabytes</span>
<span class="hljs-built_in">print</span>(<span class="hljs-string">f&quot;RAM used: <span class="hljs-subst">{psutil.Process().memory_info().rss / (<span class="hljs-number">1024</span> * <span class="hljs-number">1024</span>):<span class="hljs-number">.2</span>f}</span> MB&quot;</span>)`}}),qe=new A({props:{code:"RAM used: 5678.33 MB",highlighted:'RAM used: <span class="hljs-number">5678.33</span> MB'}}),Re=new A({props:{code:`print(f"Number of files in dataset : {pubmed_dataset.dataset_size}")
size_gb = pubmed_dataset.dataset_size / (1024**3)
print(f"Dataset size (cache file) : {size_gb:.2f} GB")`,highlighted:`<span class="hljs-built_in">print</span>(<span class="hljs-string">f&quot;Number of files in dataset : <span class="hljs-subst">{pubmed_dataset.dataset_size}</span>&quot;</span>)
size_gb = pubmed_dataset.dataset_size / (<span class="hljs-number">1024</span>**<span class="hljs-number">3</span>)
<span class="hljs-built_in">print</span>(<span class="hljs-string">f&quot;Dataset size (cache file) : <span class="hljs-subst">{size_gb:<span class="hljs-number">.2</span>f}</span> GB&quot;</span>)`}}),Ge=new A({props:{code:`Number of files in dataset : 20979437051
Dataset size (cache file) : 19.54 GB`,highlighted:`Number of files <span class="hljs-keyword">in</span> dataset : <span class="hljs-number">20979437051</span>
Dataset size (cache file) : <span class="hljs-number">19.54</span> GB`}}),pe=new pt({props:{$$slots:{default:[Wr]},$$scope:{ctx:B}}}),He=new A({props:{code:`import timeit

code_snippet = """batch_size = 1000

for idx in range(0, len(pubmed_dataset), batch_size):
    _ = pubmed_dataset[idx:idx + batch_size]
"""

time = timeit.timeit(stmt=code_snippet, number=1, globals=globals())
print(
    f"Iterated over {len(pubmed_dataset)} examples (about {size_gb:.1f} GB) in "
    f"{time:.1f}s, i.e. {size_gb/time:.3f} GB/s"
)`,highlighted:`<span class="hljs-keyword">import</span> timeit

code_snippet = <span class="hljs-string">&quot;&quot;&quot;batch_size = 1000

for idx in range(0, len(pubmed_dataset), batch_size):
    _ = pubmed_dataset[idx:idx + batch_size]
&quot;&quot;&quot;</span>

time = timeit.timeit(stmt=code_snippet, number=<span class="hljs-number">1</span>, <span class="hljs-built_in">globals</span>=<span class="hljs-built_in">globals</span>())
<span class="hljs-built_in">print</span>(
    <span class="hljs-string">f&quot;Iterated over <span class="hljs-subst">{<span class="hljs-built_in">len</span>(pubmed_dataset)}</span> examples (about <span class="hljs-subst">{size_gb:<span class="hljs-number">.1</span>f}</span> GB) in &quot;</span>
    <span class="hljs-string">f&quot;<span class="hljs-subst">{time:<span class="hljs-number">.1</span>f}</span>s, i.e. <span class="hljs-subst">{size_gb/time:<span class="hljs-number">.3</span>f}</span> GB/s&quot;</span>
)`}}),Ue=new A({props:{code:"'Iterated over 15518009 examples (about 19.5 GB) in 64.2s, i.e. 0.304 GB/s'",highlighted:'<span class="hljs-string">&#x27;Iterated over 15518009 examples (about 19.5 GB) in 64.2s, i.e. 0.304 GB/s&#x27;</span>'}}),oe=new pt({props:{$$slots:{default:[Jr]},$$scope:{ctx:B}}}),We=new A({props:{code:`pubmed_dataset_streamed = load_dataset(
    "json", data_files=data_files, split="train", streaming=True
)`,highlighted:`pubmed_dataset_streamed = load_dataset(
    <span class="hljs-string">&quot;json&quot;</span>, data_files=data_files, split=<span class="hljs-string">&quot;train&quot;</span>, streaming=<span class="hljs-literal">True</span>
)`}}),Je=new A({props:{code:"next(iter(pubmed_dataset_streamed))",highlighted:'<span class="hljs-built_in">next</span>(<span class="hljs-built_in">iter</span>(pubmed_dataset_streamed))'}}),Ke=new A({props:{code:`{'meta': {'pmid': 11409574, 'language': 'eng'},
 'text': 'Epidemiology of hypoxaemia in children with acute lower respiratory infection.\\nTo determine the prevalence of hypoxaemia in children aged under 5 years suffering acute lower respiratory infections (ALRI), the risk factors for hypoxaemia in children under 5 years of age with ALRI, and the association of hypoxaemia with an increased risk of dying in children of the same age ...'}`,highlighted:`{<span class="hljs-string">&#x27;meta&#x27;</span>: {<span class="hljs-string">&#x27;pmid&#x27;</span>: <span class="hljs-number">11409574</span>, <span class="hljs-string">&#x27;language&#x27;</span>: <span class="hljs-string">&#x27;eng&#x27;</span>},
 <span class="hljs-string">&#x27;text&#x27;</span>: <span class="hljs-string">&#x27;Epidemiology of hypoxaemia in children with acute lower respiratory infection.\\nTo determine the prevalence of hypoxaemia in children aged under 5 years suffering acute lower respiratory infections (ALRI), the risk factors for hypoxaemia in children under 5 years of age with ALRI, and the association of hypoxaemia with an increased risk of dying in children of the same age ...&#x27;</span>}`}}),Ye=new A({props:{code:`from transformers import AutoTokenizer

tokenizer = AutoTokenizer.from_pretrained("distilbert-base-uncased")
tokenized_dataset = pubmed_dataset_streamed.map(lambda x: tokenizer(x["text"]))
next(iter(tokenized_dataset))`,highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoTokenizer

tokenizer = AutoTokenizer.from_pretrained(<span class="hljs-string">&quot;distilbert-base-uncased&quot;</span>)
tokenized_dataset = pubmed_dataset_streamed.<span class="hljs-built_in">map</span>(<span class="hljs-keyword">lambda</span> x: tokenizer(x[<span class="hljs-string">&quot;text&quot;</span>]))
<span class="hljs-built_in">next</span>(<span class="hljs-built_in">iter</span>(tokenized_dataset))`}}),Ze=new A({props:{code:"{'input_ids': [101, 4958, 5178, 4328, 6779, ...], 'attention_mask': [1, 1, 1, 1, 1, ...]}",highlighted:'{<span class="hljs-string">&#x27;input_ids&#x27;</span>: [<span class="hljs-number">101</span>, <span class="hljs-number">4958</span>, <span class="hljs-number">5178</span>, <span class="hljs-number">4328</span>, <span class="hljs-number">6779</span>, ...], <span class="hljs-string">&#x27;attention_mask&#x27;</span>: [<span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, ...]}'}}),fe=new pt({props:{$$slots:{default:[Kr]},$$scope:{ctx:B}}}),Qe=new A({props:{code:`shuffled_dataset = pubmed_dataset_streamed.shuffle(buffer_size=10_000, seed=42)
next(iter(shuffled_dataset))`,highlighted:`shuffled_dataset = pubmed_dataset_streamed.shuffle(buffer_size=<span class="hljs-number">10_000</span>, seed=<span class="hljs-number">42</span>)
<span class="hljs-built_in">next</span>(<span class="hljs-built_in">iter</span>(shuffled_dataset))`}}),Ve=new A({props:{code:`{'meta': {'pmid': 11410799, 'language': 'eng'},
 'text': 'Randomized study of dose or schedule modification of granulocyte colony-stimulating factor in platinum-based chemotherapy for elderly patients with lung cancer ...'}`,highlighted:`{<span class="hljs-string">&#x27;meta&#x27;</span>: {<span class="hljs-string">&#x27;pmid&#x27;</span>: <span class="hljs-number">11410799</span>, <span class="hljs-string">&#x27;language&#x27;</span>: <span class="hljs-string">&#x27;eng&#x27;</span>},
 <span class="hljs-string">&#x27;text&#x27;</span>: <span class="hljs-string">&#x27;Randomized study of dose or schedule modification of granulocyte colony-stimulating factor in platinum-based chemotherapy for elderly patients with lung cancer ...&#x27;</span>}`}}),Xe=new A({props:{code:`dataset_head = pubmed_dataset_streamed.take(5)
list(dataset_head)`,highlighted:`dataset_head = pubmed_dataset_streamed.take(<span class="hljs-number">5</span>)
<span class="hljs-built_in">list</span>(dataset_head)`}}),es=new A({props:{code:`[{'meta': {'pmid': 11409574, 'language': 'eng'},
  'text': 'Epidemiology of hypoxaemia in children with acute lower respiratory infection ...'},
 {'meta': {'pmid': 11409575, 'language': 'eng'},
  'text': 'Clinical signs of hypoxaemia in children with acute lower respiratory infection: indicators of oxygen therapy ...'},
 {'meta': {'pmid': 11409576, 'language': 'eng'},
  'text': "Hypoxaemia in children with severe pneumonia in Papua New Guinea ..."},
 {'meta': {'pmid': 11409577, 'language': 'eng'},
  'text': 'Oxygen concentrators and cylinders ...'},
 {'meta': {'pmid': 11409578, 'language': 'eng'},
  'text': 'Oxygen supply in rural africa: a personal experience ...'}]`,highlighted:`[{<span class="hljs-string">&#x27;meta&#x27;</span>: {<span class="hljs-string">&#x27;pmid&#x27;</span>: <span class="hljs-number">11409574</span>, <span class="hljs-string">&#x27;language&#x27;</span>: <span class="hljs-string">&#x27;eng&#x27;</span>},
  <span class="hljs-string">&#x27;text&#x27;</span>: <span class="hljs-string">&#x27;Epidemiology of hypoxaemia in children with acute lower respiratory infection ...&#x27;</span>},
 {<span class="hljs-string">&#x27;meta&#x27;</span>: {<span class="hljs-string">&#x27;pmid&#x27;</span>: <span class="hljs-number">11409575</span>, <span class="hljs-string">&#x27;language&#x27;</span>: <span class="hljs-string">&#x27;eng&#x27;</span>},
  <span class="hljs-string">&#x27;text&#x27;</span>: <span class="hljs-string">&#x27;Clinical signs of hypoxaemia in children with acute lower respiratory infection: indicators of oxygen therapy ...&#x27;</span>},
 {<span class="hljs-string">&#x27;meta&#x27;</span>: {<span class="hljs-string">&#x27;pmid&#x27;</span>: <span class="hljs-number">11409576</span>, <span class="hljs-string">&#x27;language&#x27;</span>: <span class="hljs-string">&#x27;eng&#x27;</span>},
  <span class="hljs-string">&#x27;text&#x27;</span>: <span class="hljs-string">&quot;Hypoxaemia in children with severe pneumonia in Papua New Guinea ...&quot;</span>},
 {<span class="hljs-string">&#x27;meta&#x27;</span>: {<span class="hljs-string">&#x27;pmid&#x27;</span>: <span class="hljs-number">11409577</span>, <span class="hljs-string">&#x27;language&#x27;</span>: <span class="hljs-string">&#x27;eng&#x27;</span>},
  <span class="hljs-string">&#x27;text&#x27;</span>: <span class="hljs-string">&#x27;Oxygen concentrators and cylinders ...&#x27;</span>},
 {<span class="hljs-string">&#x27;meta&#x27;</span>: {<span class="hljs-string">&#x27;pmid&#x27;</span>: <span class="hljs-number">11409578</span>, <span class="hljs-string">&#x27;language&#x27;</span>: <span class="hljs-string">&#x27;eng&#x27;</span>},
  <span class="hljs-string">&#x27;text&#x27;</span>: <span class="hljs-string">&#x27;Oxygen supply in rural africa: a personal experience ...&#x27;</span>}]`}}),ss=new A({props:{code:`# Skip the first 1,000 examples and include the rest in the training set
train_dataset = shuffled_dataset.skip(1000)
# Take the first 1,000 examples for the validation set
validation_dataset = shuffled_dataset.take(1000)`,highlighted:`<span class="hljs-comment"># Skip the first 1,000 examples and include the rest in the training set</span>
train_dataset = shuffled_dataset.skip(<span class="hljs-number">1000</span>)
<span class="hljs-comment"># Take the first 1,000 examples for the validation set</span>
validation_dataset = shuffled_dataset.take(<span class="hljs-number">1000</span>)`}}),ts=new A({props:{code:`law_dataset_streamed = load_dataset(
    "json",
    data_files="https://the-eye.eu/public/AI/pile_preliminary_components/FreeLaw_Opinions.jsonl.zst",
    split="train",
    streaming=True,
)
next(iter(law_dataset_streamed))`,highlighted:`law_dataset_streamed = load_dataset(
    <span class="hljs-string">&quot;json&quot;</span>,
    data_files=<span class="hljs-string">&quot;https://the-eye.eu/public/AI/pile_preliminary_components/FreeLaw_Opinions.jsonl.zst&quot;</span>,
    split=<span class="hljs-string">&quot;train&quot;</span>,
    streaming=<span class="hljs-literal">True</span>,
)
<span class="hljs-built_in">next</span>(<span class="hljs-built_in">iter</span>(law_dataset_streamed))`}}),as=new A({props:{code:`{'meta': {'case_ID': '110921.json',
  'case_jurisdiction': 'scotus.tar.gz',
  'date_created': '2010-04-28T17:12:49Z'},
 'text': '\\n461 U.S. 238 (1983)\\nOLIM ET AL.\\nv.\\nWAKINEKONA\\nNo. 81-1581.\\nSupreme Court of United States.\\nArgued January 19, 1983.\\nDecided April 26, 1983.\\nCERTIORARI TO THE UNITED STATES COURT OF APPEALS FOR THE NINTH CIRCUIT\\n*239 Michael A. Lilly, First Deputy Attorney General of Hawaii, argued the cause for petitioners. With him on the brief was James H. Dannenberg, Deputy Attorney General...'}`,highlighted:`{<span class="hljs-string">&#x27;meta&#x27;</span>: {<span class="hljs-string">&#x27;case_ID&#x27;</span>: <span class="hljs-string">&#x27;110921.json&#x27;</span>,
  <span class="hljs-string">&#x27;case_jurisdiction&#x27;</span>: <span class="hljs-string">&#x27;scotus.tar.gz&#x27;</span>,
  <span class="hljs-string">&#x27;date_created&#x27;</span>: <span class="hljs-string">&#x27;2010-04-28T17:12:49Z&#x27;</span>},
 <span class="hljs-string">&#x27;text&#x27;</span>: <span class="hljs-string">&#x27;\\n461 U.S. 238 (1983)\\nOLIM ET AL.\\nv.\\nWAKINEKONA\\nNo. 81-1581.\\nSupreme Court of United States.\\nArgued January 19, 1983.\\nDecided April 26, 1983.\\nCERTIORARI TO THE UNITED STATES COURT OF APPEALS FOR THE NINTH CIRCUIT\\n*239 Michael A. Lilly, First Deputy Attorney General of Hawaii, argued the cause for petitioners. With him on the brief was James H. Dannenberg, Deputy Attorney General...&#x27;</span>}`}}),ls=new A({props:{code:`from itertools import islice
from datasets import interleave_datasets

combined_dataset = interleave_datasets([pubmed_dataset_streamed, law_dataset_streamed])
list(islice(combined_dataset, 2))`,highlighted:`<span class="hljs-keyword">from</span> itertools <span class="hljs-keyword">import</span> islice
<span class="hljs-keyword">from</span> datasets <span class="hljs-keyword">import</span> interleave_datasets

combined_dataset = interleave_datasets([pubmed_dataset_streamed, law_dataset_streamed])
<span class="hljs-built_in">list</span>(islice(combined_dataset, <span class="hljs-number">2</span>))`}}),ns=new A({props:{code:`[{'meta': {'pmid': 11409574, 'language': 'eng'},
  'text': 'Epidemiology of hypoxaemia in children with acute lower respiratory infection ...'},
 {'meta': {'case_ID': '110921.json',
   'case_jurisdiction': 'scotus.tar.gz',
   'date_created': '2010-04-28T17:12:49Z'},
  'text': '\\n461 U.S. 238 (1983)\\nOLIM ET AL.\\nv.\\nWAKINEKONA\\nNo. 81-1581.\\nSupreme Court of United States.\\nArgued January 19, 1983.\\nDecided April 26, 1983.\\nCERTIORARI TO THE UNITED STATES COURT OF APPEALS FOR THE NINTH CIRCUIT\\n*239 Michael A. Lilly, First Deputy Attorney General of Hawaii, argued the cause for petitioners. With him on the brief was James H. Dannenberg, Deputy Attorney General...'}]`,highlighted:`[{<span class="hljs-string">&#x27;meta&#x27;</span>: {<span class="hljs-string">&#x27;pmid&#x27;</span>: <span class="hljs-number">11409574</span>, <span class="hljs-string">&#x27;language&#x27;</span>: <span class="hljs-string">&#x27;eng&#x27;</span>},
  <span class="hljs-string">&#x27;text&#x27;</span>: <span class="hljs-string">&#x27;Epidemiology of hypoxaemia in children with acute lower respiratory infection ...&#x27;</span>},
 {<span class="hljs-string">&#x27;meta&#x27;</span>: {<span class="hljs-string">&#x27;case_ID&#x27;</span>: <span class="hljs-string">&#x27;110921.json&#x27;</span>,
   <span class="hljs-string">&#x27;case_jurisdiction&#x27;</span>: <span class="hljs-string">&#x27;scotus.tar.gz&#x27;</span>,
   <span class="hljs-string">&#x27;date_created&#x27;</span>: <span class="hljs-string">&#x27;2010-04-28T17:12:49Z&#x27;</span>},
  <span class="hljs-string">&#x27;text&#x27;</span>: <span class="hljs-string">&#x27;\\n461 U.S. 238 (1983)\\nOLIM ET AL.\\nv.\\nWAKINEKONA\\nNo. 81-1581.\\nSupreme Court of United States.\\nArgued January 19, 1983.\\nDecided April 26, 1983.\\nCERTIORARI TO THE UNITED STATES COURT OF APPEALS FOR THE NINTH CIRCUIT\\n*239 Michael A. Lilly, First Deputy Attorney General of Hawaii, argued the cause for petitioners. With him on the brief was James H. Dannenberg, Deputy Attorney General...&#x27;</span>}]`}}),rs=new A({props:{code:`base_url = "https://the-eye.eu/public/AI/pile/"
data_files = {
    "train": [base_url + "train/" + f"{idx:02d}.jsonl.zst" for idx in range(30)],
    "validation": base_url + "val.jsonl.zst",
    "test": base_url + "test.jsonl.zst",
}
pile_dataset = load_dataset("json", data_files=data_files, streaming=True)
next(iter(pile_dataset["train"]))`,highlighted:`base_url = <span class="hljs-string">&quot;https://the-eye.eu/public/AI/pile/&quot;</span>
data_files = {
    <span class="hljs-string">&quot;train&quot;</span>: [base_url + <span class="hljs-string">&quot;train/&quot;</span> + <span class="hljs-string">f&quot;<span class="hljs-subst">{idx:02d}</span>.jsonl.zst&quot;</span> <span class="hljs-keyword">for</span> idx <span class="hljs-keyword">in</span> <span class="hljs-built_in">range</span>(<span class="hljs-number">30</span>)],
    <span class="hljs-string">&quot;validation&quot;</span>: base_url + <span class="hljs-string">&quot;val.jsonl.zst&quot;</span>,
    <span class="hljs-string">&quot;test&quot;</span>: base_url + <span class="hljs-string">&quot;test.jsonl.zst&quot;</span>,
}
pile_dataset = load_dataset(<span class="hljs-string">&quot;json&quot;</span>, data_files=data_files, streaming=<span class="hljs-literal">True</span>)
<span class="hljs-built_in">next</span>(<span class="hljs-built_in">iter</span>(pile_dataset[<span class="hljs-string">&quot;train&quot;</span>]))`}}),is=new A({props:{code:`{'meta': {'pile_set_name': 'Pile-CC'},
 'text': 'It is done, and submitted. You can play \u201CSurvival of the Tastiest\u201D on Android, and on the web...'}`,highlighted:`{<span class="hljs-string">&#x27;meta&#x27;</span>: {<span class="hljs-string">&#x27;pile_set_name&#x27;</span>: <span class="hljs-string">&#x27;Pile-CC&#x27;</span>},
 <span class="hljs-string">&#x27;text&#x27;</span>: <span class="hljs-string">&#x27;It is done, and submitted. You can play \u201CSurvival of the Tastiest\u201D on Android, and on the web...&#x27;</span>}`}}),me=new pt({props:{$$slots:{default:[Yr]},$$scope:{ctx:B}}}),{c(){f=r("meta"),O=d(),u=r("h1"),v=r("a"),T=r("span"),_(h.$$.fragment),C=d(),D=r("span"),E=a("\u5927\u6570\u636E? \u{1F917} Datasets \u6765\u6551\u63F4!"),w=d(),_(P.$$.fragment),I=d(),k=r("p"),q=a("\u5982\u4ECA,\u4E0D\u96BE\u53D1\u73B0\u6211\u4EEC\u7ECF\u5E38\u4F7F\u7528\u6570GB\u7684\u6570\u636E\u96C6, \u7279\u522B\u662F\u5982\u679C\u4F60\u6253\u7B97\u4ECE\u5934\u5F00\u59CB\u9884\u8BAD\u7EC3\u50CF BERT \u6216\u8005 GPT-2 \u8FD9\u6837\u7684\u8F6C\u6362\u5668\u3002 \u5728\u8FD9\u79CD\u60C5\u51B5\u4E0B, "),y=r("em"),N=a("\u52A0\u8F7D"),se=a(" \u6570\u636E\u96C6\u5C31\u662F\u4E00\u4E2A\u6311\u6218\u3002\u4F8B\u5982, \u7528\u4E8E\u9884\u8BAD\u7EC3 GPT-2 \u7684 WebText \u8BED\u6599\u5E93\u5305\u542B\u8D85\u8FC7 800 \u4E07\u4E2A\u6587\u6863\u548C 40 GB \u7684\u6587\u672C \u2014 \u5C06\u5176\u52A0\u8F7D\u5230\u7B14\u8BB0\u672C\u7535\u8111\u7684 RAM \u4E2D\u53EF\u80FD\u4F1A\u8BA9\u5B83\u6293\u72C2\uFF01"),W=d(),S=r("p"),fs=a("\u5E78\u8FD0\u7684\u662F, \u{1F917} Datasets \u65E8\u5728\u514B\u670D\u8FD9\u4E9B\u9650\u5236\u3002\u5B83\u901A\u8FC7\u5C06\u6570\u636E\u96C6\u4F5C\u4E3A\u5185\u5B58\u6620\u5C04\u6587\u4EF6\u6765\u5904\u7406\uFF0C\u5E76\u901A\u8FC7\u5728\u8BED\u6599\u5E93\u4E2D\u6D41\u5316\u6761\u76EE\u6765\u6446\u8131\u786C\u76D8\u9650\u5236, \u4ECE\u800C\u4F7F\u4F60\u907F\u514D\u5185\u5B58\u7BA1\u7406\u95EE\u9898\u3002"),ot=d(),_(be.$$.fragment),ft=d(),ae=r("p"),wa=a("\u5728\u672C\u8282\u4E2D, \u6211\u4EEC\u5C06\u63A2\u7D22\u{1F917} Datasets \u7684\u7279\u6027\u3002\u5B83\u6709\u4E00\u4E2A\u79F0\u4E3A "),ge=r("a"),ya=a("the Pile"),va=a("\u7684825 GB\u7684\u8BED\u6599\u5E93\u3002 \u8BA9\u6211\u4EEC\u5F00\u59CB\u5427!"),dt=d(),te=r("h2"),le=r("a"),vs=r("span"),_(xe.$$.fragment),Aa=d(),As=r("span"),Da=a("\u4EC0\u4E48\u662FPile?"),ct=d(),z=r("p"),Pa=a("The Pile \u662F\u7531"),je=r("a"),ka=a("EleutherAI"),Ta=a("\u521B\u5EFA\u7684\u4E00\u4E2A\u82F1\u8BED\u6587\u672C\u8BED\u6599\u5E93, \u7528\u4E8E\u8BAD\u7EC3\u5927\u89C4\u6A21\u8BED\u8A00\u6A21\u578B\u3002\u5B83\u5305\u542B\u5404\u79CD\u5404\u6837\u7684\u6570\u636E\u96C6, \u6DB5\u76D6\u79D1\u5B66\u6587\u7AE0, GitHub \u4EE3\u7801\u5E93\u4EE5\u53CA\u8FC7\u6EE4\u7684Web\u6587\u672C\u3002\u8BAD\u7EC3\u8BED\u6599\u5E93\u5728"),Ee=r("a"),Ca=a("14 GB chunks"),Ia=a(", \u5E76\u4E14\u4F60\u4E5F\u53EF\u4EE5\u4E0B\u8F7D\u51E0\u4E2A"),we=r("a"),Oa=a("\u5355\u72EC\u7684\u7EC4\u4EF6"),za=a("\u3002 \u8BA9\u6211\u4EEC\u5148\u6765\u770B\u770B PubMed Abstracts \u6570\u636E\u96C6, \u5B83\u662F"),ye=r("a"),qa=a("PubMed"),Ra=a("\u4E0A\u76841500\u4E07\u7BC7\u751F\u7269\u533B\u5B66\u51FA\u7248\u7269\u7684\u6458\u8981\u7684\u8BED\u6599\u5E93\u3002 \u6570\u636E\u96C6\u91C7\u7528"),ve=r("a"),Ga=a("JSON\u884C\u683C\u5F0F"),Ma=a(" \u5E76\u4F7F\u7528"),Ds=r("code"),Ba=a("zstandard"),Na=a("\u5E93\u8FDB\u884C\u538B\u7F29, \u6240\u4EE5\u6211\u4EEC\u9996\u5148\u9700\u8981\u5148\u5B89\u88C5"),Ps=r("code"),Sa=a("zstandard"),La=a("\u5E93:"),mt=d(),_(Ae.$$.fragment),ut=d(),ne=r("p"),Ha=a("\u63A5\u4E0B\u6765, \u6211\u4EEC\u53EF\u4EE5\u4F7F\u7528"),ds=r("a"),Ua=a("\u7B2C\u4E8C\u8282"),Fa=a("\u4E2D\u6240\u5B66\u7684\u52A0\u8F7D\u8FDC\u7A0B\u6570\u636E\u96C6\u7684\u65B9\u6CD5\u52A0\u8F7D\u6570\u636E\u96C6:"),ht=d(),_(De.$$.fragment),_t=d(),_(Pe.$$.fragment),$t=d(),cs=r("p"),Wa=a("\u6211\u4EEC\u53EF\u4EE5\u770B\u5230\u6211\u4EEC\u7684\u6570\u636E\u96C6\u4E2D\u6709 15,518,009 \u884C\u548C 2 \u5217 \u2014 \u8FD9\u662F\u975E\u5E38\u591A\u7684!"),bt=d(),_(re.$$.fragment),gt=d(),ms=r("p"),Ja=a("\u8BA9\u6211\u4EEC\u770B\u770B\u6570\u636E\u96C6\u7684\u7B2C\u4E00\u4E2A\u5143\u7D20\u7684\u5185\u5BB9:"),xt=d(),_(ke.$$.fragment),jt=d(),_(Te.$$.fragment),Et=d(),us=r("p"),Ka=a("\u53EF\u4EE5\u770B\u5230, \u8FD9\u770B\u8D77\u6765\u50CF\u662F\u533B\u5B66\u6587\u7AE0\u7684\u6458\u8981\u3002 \u73B0\u5728\u8BA9\u6211\u4EEC\u770B\u770B\u6211\u4EEC\u4F7F\u7528\u4E86RAM\u7684\u591A\u5C11\u5B58\u50A8\u7A7A\u95F4\u6765\u52A0\u8F7D\u6570\u636E\u96C6!"),wt=d(),Ce=r("h2"),Ya=a("\u5185\u5B58\u6620\u5C04\u7684\u9B54\u529B"),yt=d(),J=r("p"),Za=a("\u5728 Python \u4E2D\u6D4B\u91CF\u5185\u5B58\u4F7F\u7528\u60C5\u51B5\u7684\u4E00\u4E2A\u7B80\u5355\u7684\u65B9\u6CD5\u662F\u4F7F\u7528"),Ie=r("a"),ks=r("code"),Qa=a("psutil"),Va=a("\u5E93,\u5B83\u53EF\u4EE5\u4F7F\u7528 "),Ts=r("code"),Xa=a("pip"),el=a("\u5B89\u88C5, \u5982\u4E0B\u6240\u793A:"),vt=d(),_(Oe.$$.fragment),At=d(),ie=r("p"),sl=a("\u5B83\u63D0\u4F9B\u4E86\u4E00\u4E2A "),Cs=r("code"),tl=a("Process"),al=a(" \u7C7B,\u8FD9\u4E2A\u7C7B\u5141\u8BB8\u6211\u4EEC\u68C0\u67E5\u5F53\u524D\u8FDB\u7A0B\u7684\u5185\u5B58\u4F7F\u7528\u60C5\u51B5, \u5982\u4E0B\u6240\u793A:"),Dt=d(),_(ze.$$.fragment),Pt=d(),_(qe.$$.fragment),kt=d(),L=r("p"),ll=a("\u8FD9\u91CC\u7684"),Is=r("code"),nl=a("rss"),rl=a("\u5C5E\u6027\u662F\u6307 "),Os=r("em"),il=a("\u5E38\u9A7B\u96C6"),pl=a(" \u7684\u5927\u5C0F, \u5B83\u662F\u8FDB\u7A0B\u5728RAM\u4E2D\u5360\u7528\u7684\u5185\u5B58\u6BD4\u4F8B\u3002 \u8FD9\u4E2A\u6D4B\u91CF\u7ED3\u679C\u4E5F\u5305\u62EC\u4E86 Python \u7F16\u8BD1\u5668\u548C\u6211\u4EEC\u52A0\u8F7D\u7684\u5E93\u6240\u4F7F\u7528\u7684\u5185\u5B58, \u6240\u4EE5\u5B9E\u9645\u4E0A\u7528\u4E8E\u52A0\u8F7D\u6570\u636E\u96C6\u7684\u5185\u5B58\u4F1A\u66F4\u5C0F\u4E00\u4E9B\u3002\u4E3A\u4E86\u6BD4\u8F83, \u8BA9\u6211\u4EEC\u4F7F\u7528 "),zs=r("code"),ol=a("dataset_size"),fl=a(" \u5C5E\u6027\u770B\u770B\u6570\u636E\u96C6\u5728\u78C1\u76D8\u4E0A\u6709\u591A\u5927\u3002 \u7531\u4E8E\u7ED3\u679C\u50CF\u4E4B\u524D\u4E00\u6837\u7528\u5B57\u8282\u8868\u793A, \u6211\u4EEC\u9700\u8981\u624B\u52A8\u5C06\u5176\u8F6C\u6362\u4E3AGB:"),Tt=d(),_(Re.$$.fragment),Ct=d(),_(Ge.$$.fragment),It=d(),hs=r("p"),dl=a("\u975E\u5E38\u68D2 \u2014 \u5C3D\u7BA1\u5B83\u5C06\u8FD120GB, \u4F46\u6211\u4EEC\u80FD\u591F\u5360\u7528\u5F88\u5C11\u7684RAM\u7A7A\u95F4\u52A0\u8F7D\u548C\u8BBF\u95EE\u6570\u636E\u96C6!"),Ot=d(),_(pe.$$.fragment),zt=d(),K=r("p"),cl=a("\u5982\u679C\u4F60\u719F\u6089 Pandas, \u8FD9\u4E2A\u7ED3\u679C\u53EF\u80FD\u4F1A\u8BA9\u4EBA\u611F\u5230\u5F88\u610F\u5916\u3002\u56E0\u4E3A Wes Kinney \u7684\u8457\u540D\u7684"),Me=r("a"),ml=a("\u7ECF\u9A8C\u6CD5\u5219"),ul=a(" \u662F\u4F60\u9700\u8981\u7684RAM\u5E94\u8BE5\u662F\u6570\u636E\u96C6\u7684\u5927\u5C0F\u76845\u500D\u523010\u500D\u3002 \u90A3\u4E48 \u{1F917} Datasets \u662F\u5982\u4F55\u89E3\u51B3\u8FD9\u4E2A\u5185\u5B58\u7BA1\u7406\u95EE\u9898\u7684\u5462? \u{1F917} Datasets \u5C06\u6BCF\u4E00\u4E2A\u6570\u636E\u96C6\u770B\u4F5C\u4E00\u4E2A"),Be=r("a"),hl=a("\u5185\u5B58\u6620\u5C04\u6587\u4EF6"),_l=a(", \u5B83\u63D0\u4F9B\u4E86RAM\u548C\u6587\u4EF6\u7CFB\u7EDF\u5B58\u50A8\u4E4B\u95F4\u7684\u6620\u5C04, \u8BE5\u6620\u5C04\u5141\u8BB8\u5E93\u8BBF\u95EE\u548C\u64CD\u4F5C\u6570\u636E\u96C6\u7684\u5143\u7D20, \u800C\u4E14\u65E0\u9700\u5C06\u5176\u5B8C\u5168\u52A0\u8F7D\u5230\u5185\u5B58\u4E2D\u3002"),qt=d(),G=r("p"),$l=a("\u5185\u5B58\u6620\u5C04\u6587\u4EF6\u4E5F\u4E00\u4E2A\u5728\u591A\u4E2A\u8FDB\u7A0B\u4E4B\u95F4\u5171\u4EAB, \u8FD9\u4F7F\u5F97\u50CF "),qs=r("code"),bl=a("Dataset.map()"),gl=a("\u4E4B\u7C7B\u7684\u65B9\u6CD5\u53EF\u4EE5\u5E76\u884C\u5316, \u5E76\u4E14\u65E0\u9700\u79FB\u52A8\u6216\u8005\u8D4B\u503C\u6570\u636E\u96C6\u3002\u5728\u5E95\u5C42, \u8FD9\u4E9B\u529F\u80FD\u90FD\u662F\u7531"),Ne=r("a"),xl=a("Apache Arrow"),jl=a("\u5185\u5B58\u683C\u5F0F\u548C"),Se=r("a"),Rs=r("code"),El=a("pyarrow"),wl=a("\u5E93\u63D0\u4F9B\u7684\u652F\u6301, \u4F7F\u5F97\u6570\u636E\u52A0\u8F7D\u548C\u5904\u7406\u901F\u5EA6\u5FEB\u5982\u95EA\u7535\u3002 (\u66F4\u591A\u6709\u5173Apache Arrow\u7684\u8BE6\u7EC6\u4FE1\u606F\u4EE5\u53CA\u4E0EPandas\u7684\u6BD4\u8F83, \u8BF7\u67E5\u770B"),Le=r("a"),yl=a("Dejan Simic\u2019s blog post"),vl=a(".) \u4E3A\u4E86\u66F4\u6E05\u6670\u5730\u770B\u5230\u8FD9\u4E2A\u8FC7\u7A0B, \u8BA9\u6211\u4EEC\u901A\u8FC7\u8FED\u4EE3PubMed Abstracts\u6570\u636E\u96C6\u4E2D\u7684\u6240\u6709\u5143\u7D20\u6765\u8FD0\u884C\u4E00\u4E2A\u901F\u5EA6\u6D4B\u8BD5\u5C0F\u7A0B\u5E8F:"),Rt=d(),_(He.$$.fragment),Gt=d(),_(Ue.$$.fragment),Mt=d(),Y=r("p"),Al=a("\u8FD9\u91CC\u6211\u4EEC\u4F7F\u7528\u4E86 Python\u7684 "),Gs=r("code"),Dl=a("timeit"),Pl=a(" \u6A21\u5757\u6765\u6D4B\u91CF\u6267\u884C "),Ms=r("code"),kl=a("code_snippet"),Tl=a("\u6240\u8017\u7684\u65F6\u95F4\u3002 \u4F60\u901A\u5E38\u80FD\u4EE5\u5341\u5206\u4E4B\u51E0GB/s\u5230\u51E0GB/s\u7684\u901F\u5EA6\u8FED\u4EE3\u6570\u636E\u96C6\u3002\u901A\u8FC7\u4E0A\u8FF0\u7684\u65B9\u6CD5\u5C31\u5DF2\u7ECF\u80FD\u591F\u89E3\u51B3\u5927\u591A\u6570\u5927\u6570\u636E\u96C6\u52A0\u8F7D\u7684\u9650\u5236, \u4F46\u662F\u6709\u65F6\u5019\u4F60\u4E0D\u5F97\u4E0D\u4F7F\u7528\u4E00\u4E2A\u5F88\u5927\u7684\u6570\u636E\u96C6, \u5B83\u751A\u81F3\u90FD\u4E0D\u80FD\u5B58\u50A8\u5728\u7B14\u8BB0\u672C\u7535\u8111\u7684\u786C\u76D8\u4E0A\u3002\u4F8B\u5982, \u5982\u679C\u6211\u4EEC\u5C1D\u8BD5\u4E0B\u8F7D\u6574\u4E2A Pile, \u6211\u4EEC\u9700\u8981825GB\u7684\u53EF\u7528\u78C1\u76D8\u7A7A\u95F4! \u4E3A\u4E86\u5904\u7406\u8FD9\u79CD\u60C5\u51B5, \u{1F917} Datasets \u63D0\u4F9B\u4E86\u4E00\u4E2A\u6D41\u5F0F\u529F\u80FD, \u8FD9\u4E2A\u529F\u80FD\u5141\u8BB8\u6211\u4EEC\u52A8\u6001\u4E0B\u8F7D\u548C\u8BBF\u95EE\u5143\u7D20, \u5E76\u4E14\u4E0D\u9700\u8981\u4E0B\u8F7D\u6574\u4E2A\u6570\u636E\u96C6\u3002\u8BA9\u6211\u4EEC\u6765\u770B\u770B\u8FD9\u4E2A\u529F\u80FD\u662F\u5982\u4F55\u5DE5\u4F5C\u7684\u3002"),Bt=d(),_(oe.$$.fragment),Nt=d(),Fe=r("h2"),Cl=a("\u6D41\u5F0F\u6570\u636E\u96C6"),St=d(),Z=r("p"),Il=a("\u8981\u4F7F\u7528\u6570\u636E\u96C6\u6D41, \u4F60\u53EA\u9700\u8981\u5C06 "),Bs=r("code"),Ol=a("streaming=True"),zl=a(" \u53C2\u6570\u4F20\u9012\u7ED9 "),Ns=r("code"),ql=a("load_dataset()"),Rl=a(" \u51FD\u6570\u3002\u63A5\u4E0B\u6765, \u8BA9\u6211\u4EEC\u518D\u6B21\u52A0\u8F7D PubMed Abstracts \u6570\u636E\u96C6, \u4F46\u662F\u91C7\u7528\u6D41\u6A21\u5F0F:"),Lt=d(),_(We.$$.fragment),Ht=d(),M=r("p"),Gl=a("\u4E0E\u6211\u4EEC\u5728\u672C\u7AE0\u5176\u4ED6\u5730\u65B9\u9047\u5230\u7684\u719F\u6089\u7684 "),Ss=r("code"),Ml=a("Dataset"),Bl=a(" \u4E0D\u540C,  "),Ls=r("code"),Nl=a("streaming=True"),Sl=a(" \u8FD4\u56DE\u7684\u5BF9\u8C61\u662F\u4E00\u4E2A "),Hs=r("code"),Ll=a("IterableDataset"),Hl=a("\u3002 \u987E\u540D\u601D\u4E49, \u8981\u8BBF\u95EE "),Us=r("code"),Ul=a("IterableDataset"),Fl=a(" , \u6211\u4EEC\u9700\u8981\u8FED\u4EE3\u5B83\u3002\u6211\u4EEC\u53EF\u4EE5\u6309\u7167\u5982\u4E0B\u65B9\u5F0F\u8BBF\u95EE\u6D41\u5F0F\u6570\u636E\u96C6\u7684\u7B2C\u4E00\u4E2A\u5143\u7D20:"),Ut=d(),_(Je.$$.fragment),Ft=d(),_(Ke.$$.fragment),Wt=d(),Q=r("p"),Wl=a("\u5982\u679C\u60A8\u9700\u8981\u5728\u8BAD\u7EC3\u671F\u95F4\u6807\u8BB0\u6D41\u5F0F\u6570\u636E\u96C6\u4E2D\u7684\u5143\u7D20\u53EF\u4EE5\u4F7F\u7528 "),Fs=r("code"),Jl=a("IterableDataset.map()"),Kl=a("\u8FDB\u884C\u52A8\u6001\u5904\u7406\u3002\u8BE5\u8FC7\u7A0B\u4E0E\u6211\u4EEC\u5728"),_s=r("a"),Yl=a("\u7B2C\u4E09\u7AE0"),Zl=a("\u4E2D\u6807\u8BB0\u6570\u636E\u96C6\u7684\u8FC7\u7A0B\u5B8C\u5168\u76F8\u540C, \u552F\u4E00\u7684\u533A\u522B\u662F\u8F93\u51FA\u662F\u9010\u4E2A\u8FD4\u56DE\u7684:"),Jt=d(),_(Ye.$$.fragment),Kt=d(),_(Ze.$$.fragment),Yt=d(),_(fe.$$.fragment),Zt=d(),H=r("p"),Ql=a("\u4F60\u8FD8\u53EF\u4EE5\u4F7F\u7528 "),Ws=r("code"),Vl=a("IterableDataset.shuffle()"),Xl=a(" \u6253\u4E71\u6D41\u5F0F\u6570\u636E\u96C6, \u4F46\u4E0E "),Js=r("code"),en=a("Dataset.shuffle()"),sn=a(" \u4E0D\u540C\u7684\u662F\u8FD9\u53EA\u4F1A\u6253\u4E71\u9884\u5B9A\u4E49 "),Ks=r("code"),tn=a("buffer_size"),an=a(" \u4E2D\u7684\u5143\u7D20:"),Qt=d(),_(Qe.$$.fragment),Vt=d(),_(Ve.$$.fragment),Xt=d(),U=r("p"),ln=a("\u5728\u8FD9\u4E2A\u793A\u4F8B\u4E2D, \u6211\u4EEC\u4ECE\u7F13\u51B2\u533A\u7684\u524D 10,000 \u4E2A\u793A\u4F8B\u4E2D\u968F\u673A\u9009\u62E9\u4E86\u4E00\u4E2A\u793A\u4F8B\u3002\u4E00\u65E6\u8BBF\u95EE\u4E86\u4E00\u4E2A\u793A\u4F8B, \u5B83\u5728\u7F13\u51B2\u533A\u4E2D\u7684\u4F4D\u7F6E\u5C31\u4F1A\u88AB\u8BED\u6599\u5E93\u4E2D\u7684\u4E0B\u4E00\u4E2A\u793A\u4F8B\u586B\u5145 (\u5373, \u4E0A\u8FF0\u6848\u4F8B\u4E2D\u7684\u7B2C 10,001\u4E2A\u793A\u4F8B)\u3002\u4F60\u8FD8\u53EF\u4EE5\u4F7F\u7528 "),Ys=r("code"),nn=a("IterableDataset.take()"),rn=a(" \u548C "),Zs=r("code"),pn=a("IterableDataset.skip()"),on=a(" \u51FD\u6570\u4ECE\u6D41\u5F0F\u6570\u636E\u96C6\u4E2D\u9009\u62E9\u5143\u7D20, \u5B83\u7684\u4F5C\u7528\u7C7B\u4F3C\u4E8E "),Qs=r("code"),fn=a("Dataset.select()"),dn=a("\u3002\u4F8B\u5982, \u8981\u9009\u62E9 PubMed Abstracts \u6570\u636E\u96C6\u7684\u524D5\u4E2A\u793A\u4F8B, \u6211\u4EEC\u53EF\u4EE5\u6267\u884C\u4EE5\u4E0B\u64CD\u4F5C:"),ea=d(),_(Xe.$$.fragment),sa=d(),_(es.$$.fragment),ta=d(),de=r("p"),cn=a("\u540C\u6837, \u4F60\u53EF\u4EE5\u4F7F\u7528 "),Vs=r("code"),mn=a("IterableDataset.skip()"),un=a(" \u51FD\u6570\u5C06\u6253\u4E71\u7684\u6570\u636E\u96C6\u62C6\u5206\u4E3A\u8BAD\u7EC3\u96C6\u548C\u9A8C\u8BC1\u96C6, \u5982\u4E0B\u6240\u793A:"),aa=d(),_(ss.$$.fragment),la=d(),F=r("p"),hn=a("\u8BA9\u6211\u4EEC\u7528\u4E00\u4E2A\u5E38\u89C1\u7684\u4EFB\u52A1\u6765\u8FDB\u884C\u6211\u4EEC\u5BF9\u6570\u636E\u96C6\u6D41\u7684\u6700\u540E\u63A2\u7D22: \u5C06\u591A\u4E2A\u6570\u636E\u96C6\u7EC4\u5408\u5728\u4E00\u8D77\u521B\u5EFA\u4E00\u4E2A\u5FC3\u5F97\u8BED\u6599\u5E93\u3002 \u{1F917} Datasets \u63D0\u4F9B\u4E86\u4E00\u4E2A "),Xs=r("code"),_n=a("interleave_datasets()"),$n=a(" \u51FD\u6570, \u5B83\u5C06\u4E00\u4E2A "),et=r("code"),bn=a("IterableDataset"),gn=a(" \u5BF9\u8C61\u5217\u8868\u7EC4\u5408\u4E3A\u5355\u4E2A\u7684 "),st=r("code"),xn=a("IterableDataset"),jn=a(", \u5176\u4E2D\u65B0\u6570\u636E\u96C6\u7684\u5143\u7D20\u662F\u901A\u8FC7\u5728\u5217\u8868\u4E2D\u7684\u5BF9\u8C61\u4EA4\u66FF\u83B7\u5F97\u7684\u3002\u5F53\u4F60\u8BD5\u56FE\u7EC4\u5408\u5927\u578B\u6570\u636E\u96C6\u65F6, \u8FD9\u4E2A\u51FD\u6570\u7279\u522B\u6709\u7528, \u8BA9\u6211\u4EEC\u901A\u8FC7\u4E0B\u9762\u8FD9\u4E2A\u4F8B\u5B50\u6765\u8BD5\u7740\u7EC4\u5408 Pile\u7684\u81EA\u7531\u6CD5\u5F8B\u6570\u636E\u96C6,\u5B83\u662F\u6765\u81EA\u7F8E\u56FD\u6CD5\u9662\u768451 GB\u7684\u6CD5\u5F8B\u610F\u89C1\u6570\u636E\u96C6:"),na=d(),_(ts.$$.fragment),ra=d(),_(as.$$.fragment),ia=d(),ce=r("p"),En=a("\u8FD9\u4E2A\u6570\u636E\u96C6\u8DB3\u591F\u5927, \u53EF\u4EE5\u5BF9\u5927\u591A\u6570\u7B14\u8BB0\u672C\u7535\u8111\u7684RAM\u6709\u8DB3\u591F\u7684\u538B\u529B, \u4F46\u662F\u6211\u4EEC\u5DF2\u7ECF\u80FD\u591F\u6BEB\u4E0D\u8D39\u529B\u5730\u52A0\u8F7D\u548C\u8BBF\u95EE\u5B83! \u73B0\u5728\u6211\u4EEC\u4F7F\u7528 "),tt=r("code"),wn=a("interleave_datasets()"),yn=a(" \u51FD\u6570\u52A0\u8F7D\u6765\u81EA FreeLaw \u548C PubMed Abstracts \u7684\u6570\u636E\u96C6:"),pa=d(),_(ls.$$.fragment),oa=d(),_(ns.$$.fragment),fa=d(),V=r("p"),vn=a("\u8FD9\u91CC\u6211\u4EEC\u4F7F\u7528\u4E86\u6765\u81EAPython\u7684 "),at=r("code"),An=a("itertools"),Dn=a(" \u6A21\u5757\u7684 "),lt=r("code"),Pn=a("islice()"),kn=a(" \u51FD\u6570\u4ECE\u5408\u5E76\u7684\u6570\u636E\u96C6\u4E2D\u9009\u62E9\u524D\u4E24\u4E2A\u793A\u4F8B, \u5E76\u4E14\u6211\u4EEC\u53EF\u4EE5\u770B\u5230\u5B83\u4EEC\u5B9E\u9645\u4E0A\u5C31\u662F\u4E24\u4E2A\u6E90\u6570\u636E\u96C6\u4E2D\u7684\u524D\u4E24\u4E2A\u793A\u4F8B\u62FC\u5728\u4E00\u8D77\u5F62\u6210\u7684:"),da=d(),$s=r("p"),Tn=a("\u6700\u540E, \u5982\u679C\u4F60\u60F3\u6D41\u5F0F\u4F20\u8F93\u6574\u4E2A825GB\u7684 Pile, \u4F60\u53EF\u4EE5\u6309\u7167\u5982\u4E0B\u65B9\u5F0F\u83B7\u53D6\u6240\u6709\u51C6\u5907\u597D\u7684\u6587\u4EF6:"),ca=d(),_(rs.$$.fragment),ma=d(),_(is.$$.fragment),ua=d(),_(me.$$.fragment),ha=d(),bs=r("p"),Cn=a("\u4F60\u73B0\u5728\u62E5\u6709\u52A0\u8F7D\u548C\u5904\u7406\u5404\u79CD\u7C7B\u578B\u548C\u5927\u5C0F\u7684\u6570\u636E\u96C6\u7684\u6240\u9700\u7684\u6240\u6709\u5DE5\u5177 \u2014 \u4F46\u662F\u9664\u975E\u4F60\u975E\u5E38\u5E78\u8FD0, \u5426\u5219\u5728\u4F60\u7684NLP\u4E4B\u65C5\u4E2D\u4F1A\u6709\u4E00\u4E2A\u96BE\u9898, \u4F60\u5C06\u4E0D\u5F97\u4E0D\u521B\u5EFA\u4E00\u4E2A\u6570\u636E\u96C6\u6765\u89E3\u51B3\u624B\u5934\u7684\u95EE\u9898\u3002\u8FD9\u5C31\u662F\u4E0B\u4E00\u8282\u7684\u4E3B\u9898!"),this.h()},l(e){const n=Sr('[data-svelte="svelte-1phssyn"]',document.head);f=i(n,"META",{name:!0,content:!0}),n.forEach(t),O=c(e),u=i(e,"H1",{class:!0});var ps=p(u);v=i(ps,"A",{id:!0,class:!0,href:!0});var nt=p(v);T=i(nt,"SPAN",{});var rt=p(T);$(h.$$.fragment,rt),rt.forEach(t),nt.forEach(t),C=c(ps),D=i(ps,"SPAN",{});var it=p(D);E=l(it,"\u5927\u6570\u636E? \u{1F917} Datasets \u6765\u6551\u63F4!"),it.forEach(t),ps.forEach(t),w=c(e),$(P.$$.fragment,e),I=c(e),k=i(e,"P",{});var os=p(k);q=l(os,"\u5982\u4ECA,\u4E0D\u96BE\u53D1\u73B0\u6211\u4EEC\u7ECF\u5E38\u4F7F\u7528\u6570GB\u7684\u6570\u636E\u96C6, \u7279\u522B\u662F\u5982\u679C\u4F60\u6253\u7B97\u4ECE\u5934\u5F00\u59CB\u9884\u8BAD\u7EC3\u50CF BERT \u6216\u8005 GPT-2 \u8FD9\u6837\u7684\u8F6C\u6362\u5668\u3002 \u5728\u8FD9\u79CD\u60C5\u51B5\u4E0B, "),y=i(os,"EM",{});var In=p(y);N=l(In,"\u52A0\u8F7D"),In.forEach(t),se=l(os," \u6570\u636E\u96C6\u5C31\u662F\u4E00\u4E2A\u6311\u6218\u3002\u4F8B\u5982, \u7528\u4E8E\u9884\u8BAD\u7EC3 GPT-2 \u7684 WebText \u8BED\u6599\u5E93\u5305\u542B\u8D85\u8FC7 800 \u4E07\u4E2A\u6587\u6863\u548C 40 GB \u7684\u6587\u672C \u2014 \u5C06\u5176\u52A0\u8F7D\u5230\u7B14\u8BB0\u672C\u7535\u8111\u7684 RAM \u4E2D\u53EF\u80FD\u4F1A\u8BA9\u5B83\u6293\u72C2\uFF01"),os.forEach(t),W=c(e),S=i(e,"P",{});var On=p(S);fs=l(On,"\u5E78\u8FD0\u7684\u662F, \u{1F917} Datasets \u65E8\u5728\u514B\u670D\u8FD9\u4E9B\u9650\u5236\u3002\u5B83\u901A\u8FC7\u5C06\u6570\u636E\u96C6\u4F5C\u4E3A\u5185\u5B58\u6620\u5C04\u6587\u4EF6\u6765\u5904\u7406\uFF0C\u5E76\u901A\u8FC7\u5728\u8BED\u6599\u5E93\u4E2D\u6D41\u5316\u6761\u76EE\u6765\u6446\u8131\u786C\u76D8\u9650\u5236, \u4ECE\u800C\u4F7F\u4F60\u907F\u514D\u5185\u5B58\u7BA1\u7406\u95EE\u9898\u3002"),On.forEach(t),ot=c(e),$(be.$$.fragment,e),ft=c(e),ae=i(e,"P",{});var $a=p(ae);wa=l($a,"\u5728\u672C\u8282\u4E2D, \u6211\u4EEC\u5C06\u63A2\u7D22\u{1F917} Datasets \u7684\u7279\u6027\u3002\u5B83\u6709\u4E00\u4E2A\u79F0\u4E3A "),ge=i($a,"A",{href:!0,rel:!0});var zn=p(ge);ya=l(zn,"the Pile"),zn.forEach(t),va=l($a,"\u7684825 GB\u7684\u8BED\u6599\u5E93\u3002 \u8BA9\u6211\u4EEC\u5F00\u59CB\u5427!"),$a.forEach(t),dt=c(e),te=i(e,"H2",{class:!0});var ba=p(te);le=i(ba,"A",{id:!0,class:!0,href:!0});var qn=p(le);vs=i(qn,"SPAN",{});var Rn=p(vs);$(xe.$$.fragment,Rn),Rn.forEach(t),qn.forEach(t),Aa=c(ba),As=i(ba,"SPAN",{});var Gn=p(As);Da=l(Gn,"\u4EC0\u4E48\u662FPile?"),Gn.forEach(t),ba.forEach(t),ct=c(e),z=i(e,"P",{});var R=p(z);Pa=l(R,"The Pile \u662F\u7531"),je=i(R,"A",{href:!0,rel:!0});var Mn=p(je);ka=l(Mn,"EleutherAI"),Mn.forEach(t),Ta=l(R,"\u521B\u5EFA\u7684\u4E00\u4E2A\u82F1\u8BED\u6587\u672C\u8BED\u6599\u5E93, \u7528\u4E8E\u8BAD\u7EC3\u5927\u89C4\u6A21\u8BED\u8A00\u6A21\u578B\u3002\u5B83\u5305\u542B\u5404\u79CD\u5404\u6837\u7684\u6570\u636E\u96C6, \u6DB5\u76D6\u79D1\u5B66\u6587\u7AE0, GitHub \u4EE3\u7801\u5E93\u4EE5\u53CA\u8FC7\u6EE4\u7684Web\u6587\u672C\u3002\u8BAD\u7EC3\u8BED\u6599\u5E93\u5728"),Ee=i(R,"A",{href:!0,rel:!0});var Bn=p(Ee);Ca=l(Bn,"14 GB chunks"),Bn.forEach(t),Ia=l(R,", \u5E76\u4E14\u4F60\u4E5F\u53EF\u4EE5\u4E0B\u8F7D\u51E0\u4E2A"),we=i(R,"A",{href:!0,rel:!0});var Nn=p(we);Oa=l(Nn,"\u5355\u72EC\u7684\u7EC4\u4EF6"),Nn.forEach(t),za=l(R,"\u3002 \u8BA9\u6211\u4EEC\u5148\u6765\u770B\u770B PubMed Abstracts \u6570\u636E\u96C6, \u5B83\u662F"),ye=i(R,"A",{href:!0,rel:!0});var Sn=p(ye);qa=l(Sn,"PubMed"),Sn.forEach(t),Ra=l(R,"\u4E0A\u76841500\u4E07\u7BC7\u751F\u7269\u533B\u5B66\u51FA\u7248\u7269\u7684\u6458\u8981\u7684\u8BED\u6599\u5E93\u3002 \u6570\u636E\u96C6\u91C7\u7528"),ve=i(R,"A",{href:!0,rel:!0});var Ln=p(ve);Ga=l(Ln,"JSON\u884C\u683C\u5F0F"),Ln.forEach(t),Ma=l(R," \u5E76\u4F7F\u7528"),Ds=i(R,"CODE",{});var Hn=p(Ds);Ba=l(Hn,"zstandard"),Hn.forEach(t),Na=l(R,"\u5E93\u8FDB\u884C\u538B\u7F29, \u6240\u4EE5\u6211\u4EEC\u9996\u5148\u9700\u8981\u5148\u5B89\u88C5"),Ps=i(R,"CODE",{});var Un=p(Ps);Sa=l(Un,"zstandard"),Un.forEach(t),La=l(R,"\u5E93:"),R.forEach(t),mt=c(e),$(Ae.$$.fragment,e),ut=c(e),ne=i(e,"P",{});var ga=p(ne);Ha=l(ga,"\u63A5\u4E0B\u6765, \u6211\u4EEC\u53EF\u4EE5\u4F7F\u7528"),ds=i(ga,"A",{href:!0});var Fn=p(ds);Ua=l(Fn,"\u7B2C\u4E8C\u8282"),Fn.forEach(t),Fa=l(ga,"\u4E2D\u6240\u5B66\u7684\u52A0\u8F7D\u8FDC\u7A0B\u6570\u636E\u96C6\u7684\u65B9\u6CD5\u52A0\u8F7D\u6570\u636E\u96C6:"),ga.forEach(t),ht=c(e),$(De.$$.fragment,e),_t=c(e),$(Pe.$$.fragment,e),$t=c(e),cs=i(e,"P",{});var Wn=p(cs);Wa=l(Wn,"\u6211\u4EEC\u53EF\u4EE5\u770B\u5230\u6211\u4EEC\u7684\u6570\u636E\u96C6\u4E2D\u6709 15,518,009 \u884C\u548C 2 \u5217 \u2014 \u8FD9\u662F\u975E\u5E38\u591A\u7684!"),Wn.forEach(t),bt=c(e),$(re.$$.fragment,e),gt=c(e),ms=i(e,"P",{});var Jn=p(ms);Ja=l(Jn,"\u8BA9\u6211\u4EEC\u770B\u770B\u6570\u636E\u96C6\u7684\u7B2C\u4E00\u4E2A\u5143\u7D20\u7684\u5185\u5BB9:"),Jn.forEach(t),xt=c(e),$(ke.$$.fragment,e),jt=c(e),$(Te.$$.fragment,e),Et=c(e),us=i(e,"P",{});var Kn=p(us);Ka=l(Kn,"\u53EF\u4EE5\u770B\u5230, \u8FD9\u770B\u8D77\u6765\u50CF\u662F\u533B\u5B66\u6587\u7AE0\u7684\u6458\u8981\u3002 \u73B0\u5728\u8BA9\u6211\u4EEC\u770B\u770B\u6211\u4EEC\u4F7F\u7528\u4E86RAM\u7684\u591A\u5C11\u5B58\u50A8\u7A7A\u95F4\u6765\u52A0\u8F7D\u6570\u636E\u96C6!"),Kn.forEach(t),wt=c(e),Ce=i(e,"H2",{id:!0});var Yn=p(Ce);Ya=l(Yn,"\u5185\u5B58\u6620\u5C04\u7684\u9B54\u529B"),Yn.forEach(t),yt=c(e),J=i(e,"P",{});var gs=p(J);Za=l(gs,"\u5728 Python \u4E2D\u6D4B\u91CF\u5185\u5B58\u4F7F\u7528\u60C5\u51B5\u7684\u4E00\u4E2A\u7B80\u5355\u7684\u65B9\u6CD5\u662F\u4F7F\u7528"),Ie=i(gs,"A",{href:!0,rel:!0});var Zn=p(Ie);ks=i(Zn,"CODE",{});var Qn=p(ks);Qa=l(Qn,"psutil"),Qn.forEach(t),Zn.forEach(t),Va=l(gs,"\u5E93,\u5B83\u53EF\u4EE5\u4F7F\u7528 "),Ts=i(gs,"CODE",{});var Vn=p(Ts);Xa=l(Vn,"pip"),Vn.forEach(t),el=l(gs,"\u5B89\u88C5, \u5982\u4E0B\u6240\u793A:"),gs.forEach(t),vt=c(e),$(Oe.$$.fragment,e),At=c(e),ie=i(e,"P",{});var xa=p(ie);sl=l(xa,"\u5B83\u63D0\u4F9B\u4E86\u4E00\u4E2A "),Cs=i(xa,"CODE",{});var Xn=p(Cs);tl=l(Xn,"Process"),Xn.forEach(t),al=l(xa," \u7C7B,\u8FD9\u4E2A\u7C7B\u5141\u8BB8\u6211\u4EEC\u68C0\u67E5\u5F53\u524D\u8FDB\u7A0B\u7684\u5185\u5B58\u4F7F\u7528\u60C5\u51B5, \u5982\u4E0B\u6240\u793A:"),xa.forEach(t),Dt=c(e),$(ze.$$.fragment,e),Pt=c(e),$(qe.$$.fragment,e),kt=c(e),L=i(e,"P",{});var ue=p(L);ll=l(ue,"\u8FD9\u91CC\u7684"),Is=i(ue,"CODE",{});var er=p(Is);nl=l(er,"rss"),er.forEach(t),rl=l(ue,"\u5C5E\u6027\u662F\u6307 "),Os=i(ue,"EM",{});var sr=p(Os);il=l(sr,"\u5E38\u9A7B\u96C6"),sr.forEach(t),pl=l(ue," \u7684\u5927\u5C0F, \u5B83\u662F\u8FDB\u7A0B\u5728RAM\u4E2D\u5360\u7528\u7684\u5185\u5B58\u6BD4\u4F8B\u3002 \u8FD9\u4E2A\u6D4B\u91CF\u7ED3\u679C\u4E5F\u5305\u62EC\u4E86 Python \u7F16\u8BD1\u5668\u548C\u6211\u4EEC\u52A0\u8F7D\u7684\u5E93\u6240\u4F7F\u7528\u7684\u5185\u5B58, \u6240\u4EE5\u5B9E\u9645\u4E0A\u7528\u4E8E\u52A0\u8F7D\u6570\u636E\u96C6\u7684\u5185\u5B58\u4F1A\u66F4\u5C0F\u4E00\u4E9B\u3002\u4E3A\u4E86\u6BD4\u8F83, \u8BA9\u6211\u4EEC\u4F7F\u7528 "),zs=i(ue,"CODE",{});var tr=p(zs);ol=l(tr,"dataset_size"),tr.forEach(t),fl=l(ue," \u5C5E\u6027\u770B\u770B\u6570\u636E\u96C6\u5728\u78C1\u76D8\u4E0A\u6709\u591A\u5927\u3002 \u7531\u4E8E\u7ED3\u679C\u50CF\u4E4B\u524D\u4E00\u6837\u7528\u5B57\u8282\u8868\u793A, \u6211\u4EEC\u9700\u8981\u624B\u52A8\u5C06\u5176\u8F6C\u6362\u4E3AGB:"),ue.forEach(t),Tt=c(e),$(Re.$$.fragment,e),Ct=c(e),$(Ge.$$.fragment,e),It=c(e),hs=i(e,"P",{});var ar=p(hs);dl=l(ar,"\u975E\u5E38\u68D2 \u2014 \u5C3D\u7BA1\u5B83\u5C06\u8FD120GB, \u4F46\u6211\u4EEC\u80FD\u591F\u5360\u7528\u5F88\u5C11\u7684RAM\u7A7A\u95F4\u52A0\u8F7D\u548C\u8BBF\u95EE\u6570\u636E\u96C6!"),ar.forEach(t),Ot=c(e),$(pe.$$.fragment,e),zt=c(e),K=i(e,"P",{});var xs=p(K);cl=l(xs,"\u5982\u679C\u4F60\u719F\u6089 Pandas, \u8FD9\u4E2A\u7ED3\u679C\u53EF\u80FD\u4F1A\u8BA9\u4EBA\u611F\u5230\u5F88\u610F\u5916\u3002\u56E0\u4E3A Wes Kinney \u7684\u8457\u540D\u7684"),Me=i(xs,"A",{href:!0,rel:!0});var lr=p(Me);ml=l(lr,"\u7ECF\u9A8C\u6CD5\u5219"),lr.forEach(t),ul=l(xs," \u662F\u4F60\u9700\u8981\u7684RAM\u5E94\u8BE5\u662F\u6570\u636E\u96C6\u7684\u5927\u5C0F\u76845\u500D\u523010\u500D\u3002 \u90A3\u4E48 \u{1F917} Datasets \u662F\u5982\u4F55\u89E3\u51B3\u8FD9\u4E2A\u5185\u5B58\u7BA1\u7406\u95EE\u9898\u7684\u5462? \u{1F917} Datasets \u5C06\u6BCF\u4E00\u4E2A\u6570\u636E\u96C6\u770B\u4F5C\u4E00\u4E2A"),Be=i(xs,"A",{href:!0,rel:!0});var nr=p(Be);hl=l(nr,"\u5185\u5B58\u6620\u5C04\u6587\u4EF6"),nr.forEach(t),_l=l(xs,", \u5B83\u63D0\u4F9B\u4E86RAM\u548C\u6587\u4EF6\u7CFB\u7EDF\u5B58\u50A8\u4E4B\u95F4\u7684\u6620\u5C04, \u8BE5\u6620\u5C04\u5141\u8BB8\u5E93\u8BBF\u95EE\u548C\u64CD\u4F5C\u6570\u636E\u96C6\u7684\u5143\u7D20, \u800C\u4E14\u65E0\u9700\u5C06\u5176\u5B8C\u5168\u52A0\u8F7D\u5230\u5185\u5B58\u4E2D\u3002"),xs.forEach(t),qt=c(e),G=i(e,"P",{});var X=p(G);$l=l(X,"\u5185\u5B58\u6620\u5C04\u6587\u4EF6\u4E5F\u4E00\u4E2A\u5728\u591A\u4E2A\u8FDB\u7A0B\u4E4B\u95F4\u5171\u4EAB, \u8FD9\u4F7F\u5F97\u50CF "),qs=i(X,"CODE",{});var rr=p(qs);bl=l(rr,"Dataset.map()"),rr.forEach(t),gl=l(X,"\u4E4B\u7C7B\u7684\u65B9\u6CD5\u53EF\u4EE5\u5E76\u884C\u5316, \u5E76\u4E14\u65E0\u9700\u79FB\u52A8\u6216\u8005\u8D4B\u503C\u6570\u636E\u96C6\u3002\u5728\u5E95\u5C42, \u8FD9\u4E9B\u529F\u80FD\u90FD\u662F\u7531"),Ne=i(X,"A",{href:!0,rel:!0});var ir=p(Ne);xl=l(ir,"Apache Arrow"),ir.forEach(t),jl=l(X,"\u5185\u5B58\u683C\u5F0F\u548C"),Se=i(X,"A",{href:!0,rel:!0});var pr=p(Se);Rs=i(pr,"CODE",{});var or=p(Rs);El=l(or,"pyarrow"),or.forEach(t),pr.forEach(t),wl=l(X,"\u5E93\u63D0\u4F9B\u7684\u652F\u6301, \u4F7F\u5F97\u6570\u636E\u52A0\u8F7D\u548C\u5904\u7406\u901F\u5EA6\u5FEB\u5982\u95EA\u7535\u3002 (\u66F4\u591A\u6709\u5173Apache Arrow\u7684\u8BE6\u7EC6\u4FE1\u606F\u4EE5\u53CA\u4E0EPandas\u7684\u6BD4\u8F83, \u8BF7\u67E5\u770B"),Le=i(X,"A",{href:!0,rel:!0});var fr=p(Le);yl=l(fr,"Dejan Simic\u2019s blog post"),fr.forEach(t),vl=l(X,".) \u4E3A\u4E86\u66F4\u6E05\u6670\u5730\u770B\u5230\u8FD9\u4E2A\u8FC7\u7A0B, \u8BA9\u6211\u4EEC\u901A\u8FC7\u8FED\u4EE3PubMed Abstracts\u6570\u636E\u96C6\u4E2D\u7684\u6240\u6709\u5143\u7D20\u6765\u8FD0\u884C\u4E00\u4E2A\u901F\u5EA6\u6D4B\u8BD5\u5C0F\u7A0B\u5E8F:"),X.forEach(t),Rt=c(e),$(He.$$.fragment,e),Gt=c(e),$(Ue.$$.fragment,e),Mt=c(e),Y=i(e,"P",{});var js=p(Y);Al=l(js,"\u8FD9\u91CC\u6211\u4EEC\u4F7F\u7528\u4E86 Python\u7684 "),Gs=i(js,"CODE",{});var dr=p(Gs);Dl=l(dr,"timeit"),dr.forEach(t),Pl=l(js," \u6A21\u5757\u6765\u6D4B\u91CF\u6267\u884C "),Ms=i(js,"CODE",{});var cr=p(Ms);kl=l(cr,"code_snippet"),cr.forEach(t),Tl=l(js,"\u6240\u8017\u7684\u65F6\u95F4\u3002 \u4F60\u901A\u5E38\u80FD\u4EE5\u5341\u5206\u4E4B\u51E0GB/s\u5230\u51E0GB/s\u7684\u901F\u5EA6\u8FED\u4EE3\u6570\u636E\u96C6\u3002\u901A\u8FC7\u4E0A\u8FF0\u7684\u65B9\u6CD5\u5C31\u5DF2\u7ECF\u80FD\u591F\u89E3\u51B3\u5927\u591A\u6570\u5927\u6570\u636E\u96C6\u52A0\u8F7D\u7684\u9650\u5236, \u4F46\u662F\u6709\u65F6\u5019\u4F60\u4E0D\u5F97\u4E0D\u4F7F\u7528\u4E00\u4E2A\u5F88\u5927\u7684\u6570\u636E\u96C6, \u5B83\u751A\u81F3\u90FD\u4E0D\u80FD\u5B58\u50A8\u5728\u7B14\u8BB0\u672C\u7535\u8111\u7684\u786C\u76D8\u4E0A\u3002\u4F8B\u5982, \u5982\u679C\u6211\u4EEC\u5C1D\u8BD5\u4E0B\u8F7D\u6574\u4E2A Pile, \u6211\u4EEC\u9700\u8981825GB\u7684\u53EF\u7528\u78C1\u76D8\u7A7A\u95F4! \u4E3A\u4E86\u5904\u7406\u8FD9\u79CD\u60C5\u51B5, \u{1F917} Datasets \u63D0\u4F9B\u4E86\u4E00\u4E2A\u6D41\u5F0F\u529F\u80FD, \u8FD9\u4E2A\u529F\u80FD\u5141\u8BB8\u6211\u4EEC\u52A8\u6001\u4E0B\u8F7D\u548C\u8BBF\u95EE\u5143\u7D20, \u5E76\u4E14\u4E0D\u9700\u8981\u4E0B\u8F7D\u6574\u4E2A\u6570\u636E\u96C6\u3002\u8BA9\u6211\u4EEC\u6765\u770B\u770B\u8FD9\u4E2A\u529F\u80FD\u662F\u5982\u4F55\u5DE5\u4F5C\u7684\u3002"),js.forEach(t),Bt=c(e),$(oe.$$.fragment,e),Nt=c(e),Fe=i(e,"H2",{id:!0});var mr=p(Fe);Cl=l(mr,"\u6D41\u5F0F\u6570\u636E\u96C6"),mr.forEach(t),St=c(e),Z=i(e,"P",{});var Es=p(Z);Il=l(Es,"\u8981\u4F7F\u7528\u6570\u636E\u96C6\u6D41, \u4F60\u53EA\u9700\u8981\u5C06 "),Bs=i(Es,"CODE",{});var ur=p(Bs);Ol=l(ur,"streaming=True"),ur.forEach(t),zl=l(Es," \u53C2\u6570\u4F20\u9012\u7ED9 "),Ns=i(Es,"CODE",{});var hr=p(Ns);ql=l(hr,"load_dataset()"),hr.forEach(t),Rl=l(Es," \u51FD\u6570\u3002\u63A5\u4E0B\u6765, \u8BA9\u6211\u4EEC\u518D\u6B21\u52A0\u8F7D PubMed Abstracts \u6570\u636E\u96C6, \u4F46\u662F\u91C7\u7528\u6D41\u6A21\u5F0F:"),Es.forEach(t),Lt=c(e),$(We.$$.fragment,e),Ht=c(e),M=i(e,"P",{});var ee=p(M);Gl=l(ee,"\u4E0E\u6211\u4EEC\u5728\u672C\u7AE0\u5176\u4ED6\u5730\u65B9\u9047\u5230\u7684\u719F\u6089\u7684 "),Ss=i(ee,"CODE",{});var _r=p(Ss);Ml=l(_r,"Dataset"),_r.forEach(t),Bl=l(ee," \u4E0D\u540C,  "),Ls=i(ee,"CODE",{});var $r=p(Ls);Nl=l($r,"streaming=True"),$r.forEach(t),Sl=l(ee," \u8FD4\u56DE\u7684\u5BF9\u8C61\u662F\u4E00\u4E2A "),Hs=i(ee,"CODE",{});var br=p(Hs);Ll=l(br,"IterableDataset"),br.forEach(t),Hl=l(ee,"\u3002 \u987E\u540D\u601D\u4E49, \u8981\u8BBF\u95EE "),Us=i(ee,"CODE",{});var gr=p(Us);Ul=l(gr,"IterableDataset"),gr.forEach(t),Fl=l(ee," , \u6211\u4EEC\u9700\u8981\u8FED\u4EE3\u5B83\u3002\u6211\u4EEC\u53EF\u4EE5\u6309\u7167\u5982\u4E0B\u65B9\u5F0F\u8BBF\u95EE\u6D41\u5F0F\u6570\u636E\u96C6\u7684\u7B2C\u4E00\u4E2A\u5143\u7D20:"),ee.forEach(t),Ut=c(e),$(Je.$$.fragment,e),Ft=c(e),$(Ke.$$.fragment,e),Wt=c(e),Q=i(e,"P",{});var ws=p(Q);Wl=l(ws,"\u5982\u679C\u60A8\u9700\u8981\u5728\u8BAD\u7EC3\u671F\u95F4\u6807\u8BB0\u6D41\u5F0F\u6570\u636E\u96C6\u4E2D\u7684\u5143\u7D20\u53EF\u4EE5\u4F7F\u7528 "),Fs=i(ws,"CODE",{});var xr=p(Fs);Jl=l(xr,"IterableDataset.map()"),xr.forEach(t),Kl=l(ws,"\u8FDB\u884C\u52A8\u6001\u5904\u7406\u3002\u8BE5\u8FC7\u7A0B\u4E0E\u6211\u4EEC\u5728"),_s=i(ws,"A",{href:!0});var jr=p(_s);Yl=l(jr,"\u7B2C\u4E09\u7AE0"),jr.forEach(t),Zl=l(ws,"\u4E2D\u6807\u8BB0\u6570\u636E\u96C6\u7684\u8FC7\u7A0B\u5B8C\u5168\u76F8\u540C, \u552F\u4E00\u7684\u533A\u522B\u662F\u8F93\u51FA\u662F\u9010\u4E2A\u8FD4\u56DE\u7684:"),ws.forEach(t),Jt=c(e),$(Ye.$$.fragment,e),Kt=c(e),$(Ze.$$.fragment,e),Yt=c(e),$(fe.$$.fragment,e),Zt=c(e),H=i(e,"P",{});var he=p(H);Ql=l(he,"\u4F60\u8FD8\u53EF\u4EE5\u4F7F\u7528 "),Ws=i(he,"CODE",{});var Er=p(Ws);Vl=l(Er,"IterableDataset.shuffle()"),Er.forEach(t),Xl=l(he," \u6253\u4E71\u6D41\u5F0F\u6570\u636E\u96C6, \u4F46\u4E0E "),Js=i(he,"CODE",{});var wr=p(Js);en=l(wr,"Dataset.shuffle()"),wr.forEach(t),sn=l(he," \u4E0D\u540C\u7684\u662F\u8FD9\u53EA\u4F1A\u6253\u4E71\u9884\u5B9A\u4E49 "),Ks=i(he,"CODE",{});var yr=p(Ks);tn=l(yr,"buffer_size"),yr.forEach(t),an=l(he," \u4E2D\u7684\u5143\u7D20:"),he.forEach(t),Qt=c(e),$(Qe.$$.fragment,e),Vt=c(e),$(Ve.$$.fragment,e),Xt=c(e),U=i(e,"P",{});var _e=p(U);ln=l(_e,"\u5728\u8FD9\u4E2A\u793A\u4F8B\u4E2D, \u6211\u4EEC\u4ECE\u7F13\u51B2\u533A\u7684\u524D 10,000 \u4E2A\u793A\u4F8B\u4E2D\u968F\u673A\u9009\u62E9\u4E86\u4E00\u4E2A\u793A\u4F8B\u3002\u4E00\u65E6\u8BBF\u95EE\u4E86\u4E00\u4E2A\u793A\u4F8B, \u5B83\u5728\u7F13\u51B2\u533A\u4E2D\u7684\u4F4D\u7F6E\u5C31\u4F1A\u88AB\u8BED\u6599\u5E93\u4E2D\u7684\u4E0B\u4E00\u4E2A\u793A\u4F8B\u586B\u5145 (\u5373, \u4E0A\u8FF0\u6848\u4F8B\u4E2D\u7684\u7B2C 10,001\u4E2A\u793A\u4F8B)\u3002\u4F60\u8FD8\u53EF\u4EE5\u4F7F\u7528 "),Ys=i(_e,"CODE",{});var vr=p(Ys);nn=l(vr,"IterableDataset.take()"),vr.forEach(t),rn=l(_e," \u548C "),Zs=i(_e,"CODE",{});var Ar=p(Zs);pn=l(Ar,"IterableDataset.skip()"),Ar.forEach(t),on=l(_e," \u51FD\u6570\u4ECE\u6D41\u5F0F\u6570\u636E\u96C6\u4E2D\u9009\u62E9\u5143\u7D20, \u5B83\u7684\u4F5C\u7528\u7C7B\u4F3C\u4E8E "),Qs=i(_e,"CODE",{});var Dr=p(Qs);fn=l(Dr,"Dataset.select()"),Dr.forEach(t),dn=l(_e,"\u3002\u4F8B\u5982, \u8981\u9009\u62E9 PubMed Abstracts \u6570\u636E\u96C6\u7684\u524D5\u4E2A\u793A\u4F8B, \u6211\u4EEC\u53EF\u4EE5\u6267\u884C\u4EE5\u4E0B\u64CD\u4F5C:"),_e.forEach(t),ea=c(e),$(Xe.$$.fragment,e),sa=c(e),$(es.$$.fragment,e),ta=c(e),de=i(e,"P",{});var ja=p(de);cn=l(ja,"\u540C\u6837, \u4F60\u53EF\u4EE5\u4F7F\u7528 "),Vs=i(ja,"CODE",{});var Pr=p(Vs);mn=l(Pr,"IterableDataset.skip()"),Pr.forEach(t),un=l(ja," \u51FD\u6570\u5C06\u6253\u4E71\u7684\u6570\u636E\u96C6\u62C6\u5206\u4E3A\u8BAD\u7EC3\u96C6\u548C\u9A8C\u8BC1\u96C6, \u5982\u4E0B\u6240\u793A:"),ja.forEach(t),aa=c(e),$(ss.$$.fragment,e),la=c(e),F=i(e,"P",{});var $e=p(F);hn=l($e,"\u8BA9\u6211\u4EEC\u7528\u4E00\u4E2A\u5E38\u89C1\u7684\u4EFB\u52A1\u6765\u8FDB\u884C\u6211\u4EEC\u5BF9\u6570\u636E\u96C6\u6D41\u7684\u6700\u540E\u63A2\u7D22: \u5C06\u591A\u4E2A\u6570\u636E\u96C6\u7EC4\u5408\u5728\u4E00\u8D77\u521B\u5EFA\u4E00\u4E2A\u5FC3\u5F97\u8BED\u6599\u5E93\u3002 \u{1F917} Datasets \u63D0\u4F9B\u4E86\u4E00\u4E2A "),Xs=i($e,"CODE",{});var kr=p(Xs);_n=l(kr,"interleave_datasets()"),kr.forEach(t),$n=l($e," \u51FD\u6570, \u5B83\u5C06\u4E00\u4E2A "),et=i($e,"CODE",{});var Tr=p(et);bn=l(Tr,"IterableDataset"),Tr.forEach(t),gn=l($e," \u5BF9\u8C61\u5217\u8868\u7EC4\u5408\u4E3A\u5355\u4E2A\u7684 "),st=i($e,"CODE",{});var Cr=p(st);xn=l(Cr,"IterableDataset"),Cr.forEach(t),jn=l($e,", \u5176\u4E2D\u65B0\u6570\u636E\u96C6\u7684\u5143\u7D20\u662F\u901A\u8FC7\u5728\u5217\u8868\u4E2D\u7684\u5BF9\u8C61\u4EA4\u66FF\u83B7\u5F97\u7684\u3002\u5F53\u4F60\u8BD5\u56FE\u7EC4\u5408\u5927\u578B\u6570\u636E\u96C6\u65F6, \u8FD9\u4E2A\u51FD\u6570\u7279\u522B\u6709\u7528, \u8BA9\u6211\u4EEC\u901A\u8FC7\u4E0B\u9762\u8FD9\u4E2A\u4F8B\u5B50\u6765\u8BD5\u7740\u7EC4\u5408 Pile\u7684\u81EA\u7531\u6CD5\u5F8B\u6570\u636E\u96C6,\u5B83\u662F\u6765\u81EA\u7F8E\u56FD\u6CD5\u9662\u768451 GB\u7684\u6CD5\u5F8B\u610F\u89C1\u6570\u636E\u96C6:"),$e.forEach(t),na=c(e),$(ts.$$.fragment,e),ra=c(e),$(as.$$.fragment,e),ia=c(e),ce=i(e,"P",{});var Ea=p(ce);En=l(Ea,"\u8FD9\u4E2A\u6570\u636E\u96C6\u8DB3\u591F\u5927, \u53EF\u4EE5\u5BF9\u5927\u591A\u6570\u7B14\u8BB0\u672C\u7535\u8111\u7684RAM\u6709\u8DB3\u591F\u7684\u538B\u529B, \u4F46\u662F\u6211\u4EEC\u5DF2\u7ECF\u80FD\u591F\u6BEB\u4E0D\u8D39\u529B\u5730\u52A0\u8F7D\u548C\u8BBF\u95EE\u5B83! \u73B0\u5728\u6211\u4EEC\u4F7F\u7528 "),tt=i(Ea,"CODE",{});var Ir=p(tt);wn=l(Ir,"interleave_datasets()"),Ir.forEach(t),yn=l(Ea," \u51FD\u6570\u52A0\u8F7D\u6765\u81EA FreeLaw \u548C PubMed Abstracts \u7684\u6570\u636E\u96C6:"),Ea.forEach(t),pa=c(e),$(ls.$$.fragment,e),oa=c(e),$(ns.$$.fragment,e),fa=c(e),V=i(e,"P",{});var ys=p(V);vn=l(ys,"\u8FD9\u91CC\u6211\u4EEC\u4F7F\u7528\u4E86\u6765\u81EAPython\u7684 "),at=i(ys,"CODE",{});var Or=p(at);An=l(Or,"itertools"),Or.forEach(t),Dn=l(ys," \u6A21\u5757\u7684 "),lt=i(ys,"CODE",{});var zr=p(lt);Pn=l(zr,"islice()"),zr.forEach(t),kn=l(ys," \u51FD\u6570\u4ECE\u5408\u5E76\u7684\u6570\u636E\u96C6\u4E2D\u9009\u62E9\u524D\u4E24\u4E2A\u793A\u4F8B, \u5E76\u4E14\u6211\u4EEC\u53EF\u4EE5\u770B\u5230\u5B83\u4EEC\u5B9E\u9645\u4E0A\u5C31\u662F\u4E24\u4E2A\u6E90\u6570\u636E\u96C6\u4E2D\u7684\u524D\u4E24\u4E2A\u793A\u4F8B\u62FC\u5728\u4E00\u8D77\u5F62\u6210\u7684:"),ys.forEach(t),da=c(e),$s=i(e,"P",{});var qr=p($s);Tn=l(qr,"\u6700\u540E, \u5982\u679C\u4F60\u60F3\u6D41\u5F0F\u4F20\u8F93\u6574\u4E2A825GB\u7684 Pile, \u4F60\u53EF\u4EE5\u6309\u7167\u5982\u4E0B\u65B9\u5F0F\u83B7\u53D6\u6240\u6709\u51C6\u5907\u597D\u7684\u6587\u4EF6:"),qr.forEach(t),ca=c(e),$(rs.$$.fragment,e),ma=c(e),$(is.$$.fragment,e),ua=c(e),$(me.$$.fragment,e),ha=c(e),bs=i(e,"P",{});var Rr=p(bs);Cn=l(Rr,"\u4F60\u73B0\u5728\u62E5\u6709\u52A0\u8F7D\u548C\u5904\u7406\u5404\u79CD\u7C7B\u578B\u548C\u5927\u5C0F\u7684\u6570\u636E\u96C6\u7684\u6240\u9700\u7684\u6240\u6709\u5DE5\u5177 \u2014 \u4F46\u662F\u9664\u975E\u4F60\u975E\u5E38\u5E78\u8FD0, \u5426\u5219\u5728\u4F60\u7684NLP\u4E4B\u65C5\u4E2D\u4F1A\u6709\u4E00\u4E2A\u96BE\u9898, \u4F60\u5C06\u4E0D\u5F97\u4E0D\u521B\u5EFA\u4E00\u4E2A\u6570\u636E\u96C6\u6765\u89E3\u51B3\u624B\u5934\u7684\u95EE\u9898\u3002\u8FD9\u5C31\u662F\u4E0B\u4E00\u8282\u7684\u4E3B\u9898!"),Rr.forEach(t),this.h()},h(){m(f,"name","hf:doc:metadata"),m(f,"content",JSON.stringify(Qr)),m(v,"id","datasets"),m(v,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),m(v,"href","#datasets"),m(u,"class","relative group"),m(ge,"href","https://pile.eleuther.ai"),m(ge,"rel","nofollow"),m(le,"id","pile"),m(le,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),m(le,"href","#pile"),m(te,"class","relative group"),m(je,"href","https://www.eleuther.ai"),m(je,"rel","nofollow"),m(Ee,"href","https://the-eye.eu/public/AI/pile/"),m(Ee,"rel","nofollow"),m(we,"href","https://the-eye.eu/public/AI/pile_preliminary_components/"),m(we,"rel","nofollow"),m(ye,"href","https://pubmed.ncbi.nlm.nih.gov/"),m(ye,"rel","nofollow"),m(ve,"href","https://jsonlines.org"),m(ve,"rel","nofollow"),m(ds,"href","/course/chapter5/2"),m(Ce,"id",""),m(Ie,"href","https://psutil.readthedocs.io/en/latest/"),m(Ie,"rel","nofollow"),m(Me,"href","https://wesmckinney.com/blog/apache-arrow-pandas-internals/"),m(Me,"rel","nofollow"),m(Be,"href","https://en.wikipedia.org/wiki/Memory-mapped_file"),m(Be,"rel","nofollow"),m(Ne,"href","https://arrow.apache.org"),m(Ne,"rel","nofollow"),m(Se,"href","https://arrow.apache.org/docs/python/index.html"),m(Se,"rel","nofollow"),m(Le,"href","https://towardsdatascience.com/apache-arrow-read-dataframe-with-zero-memory-69634092b1a"),m(Le,"rel","nofollow"),m(Fe,"id",""),m(_s,"href","/course/chapter3")},m(e,n){s(document.head,f),o(e,O,n),o(e,u,n),s(u,v),s(v,T),b(h,T,null),s(u,C),s(u,D),s(D,E),o(e,w,n),b(P,e,n),o(e,I,n),o(e,k,n),s(k,q),s(k,y),s(y,N),s(k,se),o(e,W,n),o(e,S,n),s(S,fs),o(e,ot,n),b(be,e,n),o(e,ft,n),o(e,ae,n),s(ae,wa),s(ae,ge),s(ge,ya),s(ae,va),o(e,dt,n),o(e,te,n),s(te,le),s(le,vs),b(xe,vs,null),s(te,Aa),s(te,As),s(As,Da),o(e,ct,n),o(e,z,n),s(z,Pa),s(z,je),s(je,ka),s(z,Ta),s(z,Ee),s(Ee,Ca),s(z,Ia),s(z,we),s(we,Oa),s(z,za),s(z,ye),s(ye,qa),s(z,Ra),s(z,ve),s(ve,Ga),s(z,Ma),s(z,Ds),s(Ds,Ba),s(z,Na),s(z,Ps),s(Ps,Sa),s(z,La),o(e,mt,n),b(Ae,e,n),o(e,ut,n),o(e,ne,n),s(ne,Ha),s(ne,ds),s(ds,Ua),s(ne,Fa),o(e,ht,n),b(De,e,n),o(e,_t,n),b(Pe,e,n),o(e,$t,n),o(e,cs,n),s(cs,Wa),o(e,bt,n),b(re,e,n),o(e,gt,n),o(e,ms,n),s(ms,Ja),o(e,xt,n),b(ke,e,n),o(e,jt,n),b(Te,e,n),o(e,Et,n),o(e,us,n),s(us,Ka),o(e,wt,n),o(e,Ce,n),s(Ce,Ya),o(e,yt,n),o(e,J,n),s(J,Za),s(J,Ie),s(Ie,ks),s(ks,Qa),s(J,Va),s(J,Ts),s(Ts,Xa),s(J,el),o(e,vt,n),b(Oe,e,n),o(e,At,n),o(e,ie,n),s(ie,sl),s(ie,Cs),s(Cs,tl),s(ie,al),o(e,Dt,n),b(ze,e,n),o(e,Pt,n),b(qe,e,n),o(e,kt,n),o(e,L,n),s(L,ll),s(L,Is),s(Is,nl),s(L,rl),s(L,Os),s(Os,il),s(L,pl),s(L,zs),s(zs,ol),s(L,fl),o(e,Tt,n),b(Re,e,n),o(e,Ct,n),b(Ge,e,n),o(e,It,n),o(e,hs,n),s(hs,dl),o(e,Ot,n),b(pe,e,n),o(e,zt,n),o(e,K,n),s(K,cl),s(K,Me),s(Me,ml),s(K,ul),s(K,Be),s(Be,hl),s(K,_l),o(e,qt,n),o(e,G,n),s(G,$l),s(G,qs),s(qs,bl),s(G,gl),s(G,Ne),s(Ne,xl),s(G,jl),s(G,Se),s(Se,Rs),s(Rs,El),s(G,wl),s(G,Le),s(Le,yl),s(G,vl),o(e,Rt,n),b(He,e,n),o(e,Gt,n),b(Ue,e,n),o(e,Mt,n),o(e,Y,n),s(Y,Al),s(Y,Gs),s(Gs,Dl),s(Y,Pl),s(Y,Ms),s(Ms,kl),s(Y,Tl),o(e,Bt,n),b(oe,e,n),o(e,Nt,n),o(e,Fe,n),s(Fe,Cl),o(e,St,n),o(e,Z,n),s(Z,Il),s(Z,Bs),s(Bs,Ol),s(Z,zl),s(Z,Ns),s(Ns,ql),s(Z,Rl),o(e,Lt,n),b(We,e,n),o(e,Ht,n),o(e,M,n),s(M,Gl),s(M,Ss),s(Ss,Ml),s(M,Bl),s(M,Ls),s(Ls,Nl),s(M,Sl),s(M,Hs),s(Hs,Ll),s(M,Hl),s(M,Us),s(Us,Ul),s(M,Fl),o(e,Ut,n),b(Je,e,n),o(e,Ft,n),b(Ke,e,n),o(e,Wt,n),o(e,Q,n),s(Q,Wl),s(Q,Fs),s(Fs,Jl),s(Q,Kl),s(Q,_s),s(_s,Yl),s(Q,Zl),o(e,Jt,n),b(Ye,e,n),o(e,Kt,n),b(Ze,e,n),o(e,Yt,n),b(fe,e,n),o(e,Zt,n),o(e,H,n),s(H,Ql),s(H,Ws),s(Ws,Vl),s(H,Xl),s(H,Js),s(Js,en),s(H,sn),s(H,Ks),s(Ks,tn),s(H,an),o(e,Qt,n),b(Qe,e,n),o(e,Vt,n),b(Ve,e,n),o(e,Xt,n),o(e,U,n),s(U,ln),s(U,Ys),s(Ys,nn),s(U,rn),s(U,Zs),s(Zs,pn),s(U,on),s(U,Qs),s(Qs,fn),s(U,dn),o(e,ea,n),b(Xe,e,n),o(e,sa,n),b(es,e,n),o(e,ta,n),o(e,de,n),s(de,cn),s(de,Vs),s(Vs,mn),s(de,un),o(e,aa,n),b(ss,e,n),o(e,la,n),o(e,F,n),s(F,hn),s(F,Xs),s(Xs,_n),s(F,$n),s(F,et),s(et,bn),s(F,gn),s(F,st),s(st,xn),s(F,jn),o(e,na,n),b(ts,e,n),o(e,ra,n),b(as,e,n),o(e,ia,n),o(e,ce,n),s(ce,En),s(ce,tt),s(tt,wn),s(ce,yn),o(e,pa,n),b(ls,e,n),o(e,oa,n),b(ns,e,n),o(e,fa,n),o(e,V,n),s(V,vn),s(V,at),s(at,An),s(V,Dn),s(V,lt),s(lt,Pn),s(V,kn),o(e,da,n),o(e,$s,n),s($s,Tn),o(e,ca,n),b(rs,e,n),o(e,ma,n),b(is,e,n),o(e,ua,n),b(me,e,n),o(e,ha,n),o(e,bs,n),s(bs,Cn),_a=!0},p(e,[n]){const ps={};n&2&&(ps.$$scope={dirty:n,ctx:e}),re.$set(ps);const nt={};n&2&&(nt.$$scope={dirty:n,ctx:e}),pe.$set(nt);const rt={};n&2&&(rt.$$scope={dirty:n,ctx:e}),oe.$set(rt);const it={};n&2&&(it.$$scope={dirty:n,ctx:e}),fe.$set(it);const os={};n&2&&(os.$$scope={dirty:n,ctx:e}),me.$set(os)},i(e){_a||(g(h.$$.fragment,e),g(P.$$.fragment,e),g(be.$$.fragment,e),g(xe.$$.fragment,e),g(Ae.$$.fragment,e),g(De.$$.fragment,e),g(Pe.$$.fragment,e),g(re.$$.fragment,e),g(ke.$$.fragment,e),g(Te.$$.fragment,e),g(Oe.$$.fragment,e),g(ze.$$.fragment,e),g(qe.$$.fragment,e),g(Re.$$.fragment,e),g(Ge.$$.fragment,e),g(pe.$$.fragment,e),g(He.$$.fragment,e),g(Ue.$$.fragment,e),g(oe.$$.fragment,e),g(We.$$.fragment,e),g(Je.$$.fragment,e),g(Ke.$$.fragment,e),g(Ye.$$.fragment,e),g(Ze.$$.fragment,e),g(fe.$$.fragment,e),g(Qe.$$.fragment,e),g(Ve.$$.fragment,e),g(Xe.$$.fragment,e),g(es.$$.fragment,e),g(ss.$$.fragment,e),g(ts.$$.fragment,e),g(as.$$.fragment,e),g(ls.$$.fragment,e),g(ns.$$.fragment,e),g(rs.$$.fragment,e),g(is.$$.fragment,e),g(me.$$.fragment,e),_a=!0)},o(e){x(h.$$.fragment,e),x(P.$$.fragment,e),x(be.$$.fragment,e),x(xe.$$.fragment,e),x(Ae.$$.fragment,e),x(De.$$.fragment,e),x(Pe.$$.fragment,e),x(re.$$.fragment,e),x(ke.$$.fragment,e),x(Te.$$.fragment,e),x(Oe.$$.fragment,e),x(ze.$$.fragment,e),x(qe.$$.fragment,e),x(Re.$$.fragment,e),x(Ge.$$.fragment,e),x(pe.$$.fragment,e),x(He.$$.fragment,e),x(Ue.$$.fragment,e),x(oe.$$.fragment,e),x(We.$$.fragment,e),x(Je.$$.fragment,e),x(Ke.$$.fragment,e),x(Ye.$$.fragment,e),x(Ze.$$.fragment,e),x(fe.$$.fragment,e),x(Qe.$$.fragment,e),x(Ve.$$.fragment,e),x(Xe.$$.fragment,e),x(es.$$.fragment,e),x(ss.$$.fragment,e),x(ts.$$.fragment,e),x(as.$$.fragment,e),x(ls.$$.fragment,e),x(ns.$$.fragment,e),x(rs.$$.fragment,e),x(is.$$.fragment,e),x(me.$$.fragment,e),_a=!1},d(e){t(f),e&&t(O),e&&t(u),j(h),e&&t(w),j(P,e),e&&t(I),e&&t(k),e&&t(W),e&&t(S),e&&t(ot),j(be,e),e&&t(ft),e&&t(ae),e&&t(dt),e&&t(te),j(xe),e&&t(ct),e&&t(z),e&&t(mt),j(Ae,e),e&&t(ut),e&&t(ne),e&&t(ht),j(De,e),e&&t(_t),j(Pe,e),e&&t($t),e&&t(cs),e&&t(bt),j(re,e),e&&t(gt),e&&t(ms),e&&t(xt),j(ke,e),e&&t(jt),j(Te,e),e&&t(Et),e&&t(us),e&&t(wt),e&&t(Ce),e&&t(yt),e&&t(J),e&&t(vt),j(Oe,e),e&&t(At),e&&t(ie),e&&t(Dt),j(ze,e),e&&t(Pt),j(qe,e),e&&t(kt),e&&t(L),e&&t(Tt),j(Re,e),e&&t(Ct),j(Ge,e),e&&t(It),e&&t(hs),e&&t(Ot),j(pe,e),e&&t(zt),e&&t(K),e&&t(qt),e&&t(G),e&&t(Rt),j(He,e),e&&t(Gt),j(Ue,e),e&&t(Mt),e&&t(Y),e&&t(Bt),j(oe,e),e&&t(Nt),e&&t(Fe),e&&t(St),e&&t(Z),e&&t(Lt),j(We,e),e&&t(Ht),e&&t(M),e&&t(Ut),j(Je,e),e&&t(Ft),j(Ke,e),e&&t(Wt),e&&t(Q),e&&t(Jt),j(Ye,e),e&&t(Kt),j(Ze,e),e&&t(Yt),j(fe,e),e&&t(Zt),e&&t(H),e&&t(Qt),j(Qe,e),e&&t(Vt),j(Ve,e),e&&t(Xt),e&&t(U),e&&t(ea),j(Xe,e),e&&t(sa),j(es,e),e&&t(ta),e&&t(de),e&&t(aa),j(ss,e),e&&t(la),e&&t(F),e&&t(na),j(ts,e),e&&t(ra),j(as,e),e&&t(ia),e&&t(ce),e&&t(pa),j(ls,e),e&&t(oa),j(ns,e),e&&t(fa),e&&t(V),e&&t(da),e&&t($s),e&&t(ca),j(rs,e),e&&t(ma),j(is,e),e&&t(ua),j(me,e),e&&t(ha),e&&t(bs)}}}const Qr={local:"datasets",sections:[{local:"pile",title:"\u4EC0\u4E48\u662FPile?"},{local:"",title:"\u5185\u5B58\u6620\u5C04\u7684\u9B54\u529B"},{local:"",title:"\u6D41\u5F0F\u6570\u636E\u96C6"}],title:"\u5927\u6570\u636E? \u{1F917} Datasets \u6765\u6551\u63F4!"};function Vr(B){return Lr(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class ri extends Mr{constructor(f){super();Br(this,f,Vr,Zr,Nr,{})}}export{ri as default,Qr as metadata};
