import{S as Oh,i as Ph,s as Ah,e as i,k as p,w as g,t as a,M as Nh,c as o,d as t,m as h,a as r,x as _,h as n,b as d,Z as Ct,G as e,g as u,y as b,q as v,o as w,B as x,v as Ch}from"../../chunks/vendor-hf-doc-builder.js";import{T as lt}from"../../chunks/Tip-hf-doc-builder.js";import{Y as Gh}from"../../chunks/Youtube-hf-doc-builder.js";import{I as Gt}from"../../chunks/IconCopyLink-hf-doc-builder.js";import{C as P}from"../../chunks/CodeBlock-hf-doc-builder.js";import{D as Ih}from"../../chunks/DocNotebookDropdown-hf-doc-builder.js";function Rh(F){let c,D,f,$,k;return{c(){c=i("p"),D=a("\u270F\uFE0F "),f=i("strong"),$=a("Try it out!"),k=a(" Click on a few of the URLs in the JSON payload above to get a feel for what type of information each GitHub issue is linked to.")},l(m){c=o(m,"P",{});var E=r(c);D=n(E,"\u270F\uFE0F "),f=o(E,"STRONG",{});var T=r(f);$=n(T,"Try it out!"),T.forEach(t),k=n(E," Click on a few of the URLs in the JSON payload above to get a feel for what type of information each GitHub issue is linked to."),E.forEach(t)},m(m,E){u(m,c,E),e(c,D),e(c,f),e(f,$),e(c,k)},d(m){m&&t(c)}}}function Sh(F){let c,D,f,$,k,m,E,T,y,j,A,H,O;return{c(){c=i("p"),D=a("\u26A0\uFE0F Do not share a notebook with your "),f=i("code"),$=a("GITHUB_TOKEN"),k=a(" pasted in it. We recommend you delete the last cell once you have executed it to avoid leaking this information accidentally. Even better, store the token in a "),m=i("em"),E=a(".env"),T=a(" file and use the "),y=i("a"),j=i("code"),A=a("python-dotenv"),H=a(" library"),O=a(" to load it automatically for you as an environment variable."),this.h()},l(C){c=o(C,"P",{});var q=r(c);D=n(q,"\u26A0\uFE0F Do not share a notebook with your "),f=o(q,"CODE",{});var N=r(f);$=n(N,"GITHUB_TOKEN"),N.forEach(t),k=n(q," pasted in it. We recommend you delete the last cell once you have executed it to avoid leaking this information accidentally. Even better, store the token in a "),m=o(q,"EM",{});var W=r(m);E=n(W,".env"),W.forEach(t),T=n(q," file and use the "),y=o(q,"A",{href:!0,rel:!0});var S=r(y);j=o(S,"CODE",{});var G=r(j);A=n(G,"python-dotenv"),G.forEach(t),H=n(S," library"),S.forEach(t),O=n(q," to load it automatically for you as an environment variable."),q.forEach(t),this.h()},h(){d(y,"href","https://github.com/theskumar/python-dotenv"),d(y,"rel","nofollow")},m(C,q){u(C,c,q),e(c,D),e(c,f),e(f,$),e(c,k),e(c,m),e(m,E),e(c,T),e(c,y),e(y,j),e(j,A),e(y,H),e(c,O)},d(C){C&&t(c)}}}function Fh(F){let c,D,f,$,k,m,E,T,y,j,A,H,O,C,q,N,W,S,G,B;return{c(){c=i("p"),D=a("\u270F\uFE0F "),f=i("strong"),$=a("Try it out!"),k=a(" Calculate the average time it takes to close issues in \u{1F917} Datasets. You may find the "),m=i("code"),E=a("Dataset.filter()"),T=a(" function useful to filter out the pull requests and open issues, and you can use the "),y=i("code"),j=a("Dataset.set_format()"),A=a(" function to convert the dataset to a "),H=i("code"),O=a("DataFrame"),C=a(" so you can easily manipulate the "),q=i("code"),N=a("created_at"),W=a(" and "),S=i("code"),G=a("closed_at"),B=a(" timestamps. For bonus points, calculate the average time it takes to close pull requests.")},l(us){c=o(us,"P",{});var I=r(c);D=n(I,"\u270F\uFE0F "),f=o(I,"STRONG",{});var Z=r(f);$=n(Z,"Try it out!"),Z.forEach(t),k=n(I," Calculate the average time it takes to close issues in \u{1F917} Datasets. You may find the "),m=o(I,"CODE",{});var it=r(m);E=n(it,"Dataset.filter()"),it.forEach(t),T=n(I," function useful to filter out the pull requests and open issues, and you can use the "),y=o(I,"CODE",{});var bs=r(y);j=n(bs,"Dataset.set_format()"),bs.forEach(t),A=n(I," function to convert the dataset to a "),H=o(I,"CODE",{});var ot=r(H);O=n(ot,"DataFrame"),ot.forEach(t),C=n(I," so you can easily manipulate the "),q=o(I,"CODE",{});var rt=r(q);N=n(rt,"created_at"),rt.forEach(t),W=n(I," and "),S=o(I,"CODE",{});var ut=r(S);G=n(ut,"closed_at"),ut.forEach(t),B=n(I," timestamps. For bonus points, calculate the average time it takes to close pull requests."),I.forEach(t)},m(us,I){u(us,c,I),e(c,D),e(c,f),e(f,$),e(c,k),e(c,m),e(m,E),e(c,T),e(c,y),e(y,j),e(c,A),e(c,H),e(H,O),e(c,C),e(c,q),e(q,N),e(c,W),e(c,S),e(S,G),e(c,B)},d(us){us&&t(c)}}}function Lh(F){let c,D,f,$,k,m,E,T,y,j,A;return{c(){c=i("p"),D=a("\u270F\uFE0F "),f=i("strong"),$=a("Try it out!"),k=a(" Use your Hugging Face Hub username and password to obtain a token and create an empty repository called "),m=i("code"),E=a("github-issues"),T=a(". Remember to "),y=i("strong"),j=a("never save your credentials"),A=a(" in Colab or any other repository, as this information can be exploited by bad actors.")},l(H){c=o(H,"P",{});var O=r(c);D=n(O,"\u270F\uFE0F "),f=o(O,"STRONG",{});var C=r(f);$=n(C,"Try it out!"),C.forEach(t),k=n(O," Use your Hugging Face Hub username and password to obtain a token and create an empty repository called "),m=o(O,"CODE",{});var q=r(m);E=n(q,"github-issues"),q.forEach(t),T=n(O,". Remember to "),y=o(O,"STRONG",{});var N=r(y);j=n(N,"never save your credentials"),N.forEach(t),A=n(O," in Colab or any other repository, as this information can be exploited by bad actors."),O.forEach(t)},m(H,O){u(H,c,O),e(c,D),e(c,f),e(f,$),e(c,k),e(c,m),e(m,E),e(c,T),e(c,y),e(y,j),e(c,A)},d(H){H&&t(c)}}}function Uh(F){let c,D,f,$,k,m,E,T;return{c(){c=i("p"),D=a("\u{1F4A1} You can also upload a dataset to the Hugging Face Hub directly from the terminal by using "),f=i("code"),$=a("huggingface-cli"),k=a(" and a bit of Git magic. See the "),m=i("a"),E=a("\u{1F917} Datasets guide"),T=a(" for details on how to do this."),this.h()},l(y){c=o(y,"P",{});var j=r(c);D=n(j,"\u{1F4A1} You can also upload a dataset to the Hugging Face Hub directly from the terminal by using "),f=o(j,"CODE",{});var A=r(f);$=n(A,"huggingface-cli"),A.forEach(t),k=n(j," and a bit of Git magic. See the "),m=o(j,"A",{href:!0,rel:!0});var H=r(m);E=n(H,"\u{1F917} Datasets guide"),H.forEach(t),T=n(j," for details on how to do this."),j.forEach(t),this.h()},h(){d(m,"href","https://huggingface.co/docs/datasets/share.html#add-a-community-dataset"),d(m,"rel","nofollow")},m(y,j){u(y,c,j),e(c,D),e(c,f),e(f,$),e(c,k),e(c,m),e(m,E),e(c,T)},d(y){y&&t(c)}}}function Mh(F){let c,D,f,$,k,m,E,T,y,j,A,H,O,C;return{c(){c=i("p"),D=a("\u270F\uFE0F "),f=i("strong"),$=a("Try it out!"),k=a(" Use the "),m=i("code"),E=a("dataset-tagging"),T=a(" application and "),y=i("a"),j=a("\u{1F917} Datasets guide"),A=a(" to complete the "),H=i("em"),O=a("README.md"),C=a(" file for your GitHub issues dataset."),this.h()},l(q){c=o(q,"P",{});var N=r(c);D=n(N,"\u270F\uFE0F "),f=o(N,"STRONG",{});var W=r(f);$=n(W,"Try it out!"),W.forEach(t),k=n(N," Use the "),m=o(N,"CODE",{});var S=r(m);E=n(S,"dataset-tagging"),S.forEach(t),T=n(N," application and "),y=o(N,"A",{href:!0,rel:!0});var G=r(y);j=n(G,"\u{1F917} Datasets guide"),G.forEach(t),A=n(N," to complete the "),H=o(N,"EM",{});var B=r(H);O=n(B,"README.md"),B.forEach(t),C=n(N," file for your GitHub issues dataset."),N.forEach(t),this.h()},h(){d(y,"href","https://github.com/huggingface/datasets/blob/master/templates/README_guide.md"),d(y,"rel","nofollow")},m(q,N){u(q,c,N),e(c,D),e(c,f),e(f,$),e(c,k),e(c,m),e(m,E),e(c,T),e(c,y),e(y,j),e(c,A),e(c,H),e(H,O),e(c,C)},d(q){q&&t(c)}}}function zh(F){let c,D,f,$,k,m,E,T;return{c(){c=i("p"),D=a("\u270F\uFE0F "),f=i("strong"),$=a("Try it out!"),k=a(" Go through the steps we took in this section to create a dataset of GitHub issues for your favorite open source library (pick something other than \u{1F917} Datasets, of course!). For bonus points, fine-tune a multilabel classifier to predict the tags present in the "),m=i("code"),E=a("labels"),T=a(" field.")},l(y){c=o(y,"P",{});var j=r(c);D=n(j,"\u270F\uFE0F "),f=o(j,"STRONG",{});var A=r(f);$=n(A,"Try it out!"),A.forEach(t),k=n(j," Go through the steps we took in this section to create a dataset of GitHub issues for your favorite open source library (pick something other than \u{1F917} Datasets, of course!). For bonus points, fine-tune a multilabel classifier to predict the tags present in the "),m=o(j,"CODE",{});var H=r(m);E=n(H,"labels"),H.forEach(t),T=n(j," field."),j.forEach(t)},m(y,j){u(y,c,j),e(c,D),e(c,f),e(f,$),e(c,k),e(c,m),e(m,E),e(c,T)},d(y){y&&t(c)}}}function Wh(F){let c,D,f,$,k,m,E,T,y,j,A,H,O,C,q,N,W,S,G,B,us,I,Z,it,bs,ot,rt,ut,It,li,Ka,pt,ii,Va,ps,vs,Rt,Ks,oi,St,ri,Xa,ws,ui,Vs,pi,hi,sn,Xs,se,Pu,en,ht,ci,tn,ee,te,Au,an,K,di,ae,fi,mi,xs,Ft,gi,_i,bi,nn,ys,vi,Lt,wi,xi,ln,ne,on,V,yi,Ut,ji,$i,Mt,Ei,ki,rn,le,un,js,qi,zt,Di,Ti,pn,ie,hn,oe,cn,Y,Hi,Wt,Oi,Pi,re,Ai,Ni,Bt,Ci,Gi,dn,ue,fn,pe,mn,Q,Ii,Yt,Ri,Si,Qt,Fi,Li,Jt,Ui,Mi,gn,$s,_n,L,zi,he,Wi,Bi,Zt,Yi,Qi,ce,Ji,Zi,Kt,Ki,Vi,bn,de,vn,Es,wn,ct,Xi,xn,fe,yn,X,so,Vt,eo,to,Xt,ao,no,jn,me,$n,ks,lo,dt,io,oo,En,ge,kn,_e,qn,ss,ro,be,uo,po,ve,ho,co,Dn,ft,hs,fo,sa,mo,go,ea,_o,bo,Tn,mt,vo,Hn,cs,qs,ta,we,wo,aa,xo,On,R,yo,na,jo,$o,gt,Eo,ko,la,qo,Do,ia,To,Ho,oa,Oo,Po,ra,Ao,No,Pn,xe,An,ye,Nn,U,Co,ua,Go,Io,pa,Ro,So,ha,Fo,Lo,ca,Uo,Mo,Cn,je,Gn,Ds,In,_t,zo,Rn,bt,Wo,Sn,ds,Ts,da,$e,Bo,fa,Yo,Fn,vt,Qo,Ln,Ee,ke,Nu,Un,Hs,Jo,Os,ma,Zo,Ko,Vo,Mn,qe,zn,De,Wn,J,Xo,ga,sr,er,_a,tr,ar,ba,nr,lr,Bn,Te,Yn,He,Qn,es,ir,va,or,rr,wa,ur,pr,Jn,Oe,Zn,wt,hr,Kn,Pe,Vn,fs,Ps,xa,Ae,cr,ya,dr,Xn,Ne,sl,ts,fr,Ce,mr,gr,ja,_r,br,el,Ge,tl,Ie,al,As,vr,$a,wr,xr,nl,Ns,yr,Ea,jr,$r,ll,Re,il,Cs,Er,ka,kr,qr,ol,Se,rl,Gs,Dr,qa,Tr,Hr,ul,Fe,pl,Le,hl,as,Or,Da,Pr,Ar,Ta,Nr,Cr,cl,Is,dl,Rs,Gr,Ha,Ir,Rr,fl,Ue,ml,M,Sr,Oa,Fr,Lr,Pa,Ur,Mr,Aa,zr,Wr,Na,Br,Yr,gl,Me,_l,Ss,Qr,Ca,Jr,Zr,bl,ze,vl,Fs,Kr,Ga,Vr,Xr,wl,We,Be,Cu,xl,ns,su,Ia,eu,tu,Ra,au,nu,yl,Ye,jl,Qe,$l,Ls,lu,Sa,iu,ou,El,Us,kl,ms,Ms,Fa,Je,ru,La,uu,ql,xt,pu,Dl,zs,hu,Ua,cu,du,Tl,yt,gs,fu,Ws,Ma,mu,gu,_u,za,bu,vu,Hl,Ze,Ke,Gu,Ol,Ve,Xe,wu,st,xu,yu,Pl,ls,ju,Wa,$u,Eu,Ba,ku,qu,Al,et,tt,Iu,Nl,Bs,Cl,jt,Du,Gl,Ys,Il;return m=new Gt({}),A=new Ih({props:{classNames:"absolute z-10 right-0 top-0",options:[{label:"Google Colab",value:"https://colab.research.google.com/github/huggingface/notebooks/blob/master/course/chapter5/section5.ipynb"},{label:"Aws Studio",value:"https://studiolab.sagemaker.aws/import/github/huggingface/notebooks/blob/master/course/chapter5/section5.ipynb"}]}}),Ks=new Gt({}),ne=new P({props:{code:"!pip install requests",highlighted:"!pip install requests"}}),le=new P({props:{code:`import requests

url = "https://api.github.com/repos/huggingface/datasets/issues?page=1&per_page=1"
response = requests.get(url)`,highlighted:`<span class="hljs-keyword">import</span> requests

url = <span class="hljs-string">&quot;https://api.github.com/repos/huggingface/datasets/issues?page=1&amp;per_page=1&quot;</span>
response = requests.get(url)`}}),ie=new P({props:{code:"response.status_code",highlighted:"response.status_code"}}),oe=new P({props:{code:"200",highlighted:'<span class="hljs-number">200</span>'}}),ue=new P({props:{code:"response.json()",highlighted:"response.json()"}}),pe=new P({props:{code:`[{'url': 'https://api.github.com/repos/huggingface/datasets/issues/2792',
  'repository_url': 'https://api.github.com/repos/huggingface/datasets',
  'labels_url': 'https://api.github.com/repos/huggingface/datasets/issues/2792/labels{/name}',
  'comments_url': 'https://api.github.com/repos/huggingface/datasets/issues/2792/comments',
  'events_url': 'https://api.github.com/repos/huggingface/datasets/issues/2792/events',
  'html_url': 'https://github.com/huggingface/datasets/pull/2792',
  'id': 968650274,
  'node_id': 'MDExOlB1bGxSZXF1ZXN0NzEwNzUyMjc0',
  'number': 2792,
  'title': 'Update GooAQ',
  'user': {'login': 'bhavitvyamalik',
   'id': 19718818,
   'node_id': 'MDQ6VXNlcjE5NzE4ODE4',
   'avatar_url': 'https://avatars.githubusercontent.com/u/19718818?v=4',
   'gravatar_id': '',
   'url': 'https://api.github.com/users/bhavitvyamalik',
   'html_url': 'https://github.com/bhavitvyamalik',
   'followers_url': 'https://api.github.com/users/bhavitvyamalik/followers',
   'following_url': 'https://api.github.com/users/bhavitvyamalik/following{/other_user}',
   'gists_url': 'https://api.github.com/users/bhavitvyamalik/gists{/gist_id}',
   'starred_url': 'https://api.github.com/users/bhavitvyamalik/starred{/owner}{/repo}',
   'subscriptions_url': 'https://api.github.com/users/bhavitvyamalik/subscriptions',
   'organizations_url': 'https://api.github.com/users/bhavitvyamalik/orgs',
   'repos_url': 'https://api.github.com/users/bhavitvyamalik/repos',
   'events_url': 'https://api.github.com/users/bhavitvyamalik/events{/privacy}',
   'received_events_url': 'https://api.github.com/users/bhavitvyamalik/received_events',
   'type': 'User',
   'site_admin': False},
  'labels': [],
  'state': 'open',
  'locked': False,
  'assignee': None,
  'assignees': [],
  'milestone': None,
  'comments': 1,
  'created_at': '2021-08-12T11:40:18Z',
  'updated_at': '2021-08-12T12:31:17Z',
  'closed_at': None,
  'author_association': 'CONTRIBUTOR',
  'active_lock_reason': None,
  'pull_request': {'url': 'https://api.github.com/repos/huggingface/datasets/pulls/2792',
   'html_url': 'https://github.com/huggingface/datasets/pull/2792',
   'diff_url': 'https://github.com/huggingface/datasets/pull/2792.diff',
   'patch_url': 'https://github.com/huggingface/datasets/pull/2792.patch'},
  'body': '[GooAQ](https://github.com/allenai/gooaq) dataset was recently updated after splits were added for the same. This PR contains new updated GooAQ with train/val/test splits and updated README as well.',
  'performed_via_github_app': None}]`,highlighted:`[{<span class="hljs-string">&#x27;url&#x27;</span>: <span class="hljs-string">&#x27;https://api.github.com/repos/huggingface/datasets/issues/2792&#x27;</span>,
  <span class="hljs-string">&#x27;repository_url&#x27;</span>: <span class="hljs-string">&#x27;https://api.github.com/repos/huggingface/datasets&#x27;</span>,
  <span class="hljs-string">&#x27;labels_url&#x27;</span>: <span class="hljs-string">&#x27;https://api.github.com/repos/huggingface/datasets/issues/2792/labels{/name}&#x27;</span>,
  <span class="hljs-string">&#x27;comments_url&#x27;</span>: <span class="hljs-string">&#x27;https://api.github.com/repos/huggingface/datasets/issues/2792/comments&#x27;</span>,
  <span class="hljs-string">&#x27;events_url&#x27;</span>: <span class="hljs-string">&#x27;https://api.github.com/repos/huggingface/datasets/issues/2792/events&#x27;</span>,
  <span class="hljs-string">&#x27;html_url&#x27;</span>: <span class="hljs-string">&#x27;https://github.com/huggingface/datasets/pull/2792&#x27;</span>,
  <span class="hljs-string">&#x27;id&#x27;</span>: <span class="hljs-number">968650274</span>,
  <span class="hljs-string">&#x27;node_id&#x27;</span>: <span class="hljs-string">&#x27;MDExOlB1bGxSZXF1ZXN0NzEwNzUyMjc0&#x27;</span>,
  <span class="hljs-string">&#x27;number&#x27;</span>: <span class="hljs-number">2792</span>,
  <span class="hljs-string">&#x27;title&#x27;</span>: <span class="hljs-string">&#x27;Update GooAQ&#x27;</span>,
  <span class="hljs-string">&#x27;user&#x27;</span>: {<span class="hljs-string">&#x27;login&#x27;</span>: <span class="hljs-string">&#x27;bhavitvyamalik&#x27;</span>,
   <span class="hljs-string">&#x27;id&#x27;</span>: <span class="hljs-number">19718818</span>,
   <span class="hljs-string">&#x27;node_id&#x27;</span>: <span class="hljs-string">&#x27;MDQ6VXNlcjE5NzE4ODE4&#x27;</span>,
   <span class="hljs-string">&#x27;avatar_url&#x27;</span>: <span class="hljs-string">&#x27;https://avatars.githubusercontent.com/u/19718818?v=4&#x27;</span>,
   <span class="hljs-string">&#x27;gravatar_id&#x27;</span>: <span class="hljs-string">&#x27;&#x27;</span>,
   <span class="hljs-string">&#x27;url&#x27;</span>: <span class="hljs-string">&#x27;https://api.github.com/users/bhavitvyamalik&#x27;</span>,
   <span class="hljs-string">&#x27;html_url&#x27;</span>: <span class="hljs-string">&#x27;https://github.com/bhavitvyamalik&#x27;</span>,
   <span class="hljs-string">&#x27;followers_url&#x27;</span>: <span class="hljs-string">&#x27;https://api.github.com/users/bhavitvyamalik/followers&#x27;</span>,
   <span class="hljs-string">&#x27;following_url&#x27;</span>: <span class="hljs-string">&#x27;https://api.github.com/users/bhavitvyamalik/following{/other_user}&#x27;</span>,
   <span class="hljs-string">&#x27;gists_url&#x27;</span>: <span class="hljs-string">&#x27;https://api.github.com/users/bhavitvyamalik/gists{/gist_id}&#x27;</span>,
   <span class="hljs-string">&#x27;starred_url&#x27;</span>: <span class="hljs-string">&#x27;https://api.github.com/users/bhavitvyamalik/starred{/owner}{/repo}&#x27;</span>,
   <span class="hljs-string">&#x27;subscriptions_url&#x27;</span>: <span class="hljs-string">&#x27;https://api.github.com/users/bhavitvyamalik/subscriptions&#x27;</span>,
   <span class="hljs-string">&#x27;organizations_url&#x27;</span>: <span class="hljs-string">&#x27;https://api.github.com/users/bhavitvyamalik/orgs&#x27;</span>,
   <span class="hljs-string">&#x27;repos_url&#x27;</span>: <span class="hljs-string">&#x27;https://api.github.com/users/bhavitvyamalik/repos&#x27;</span>,
   <span class="hljs-string">&#x27;events_url&#x27;</span>: <span class="hljs-string">&#x27;https://api.github.com/users/bhavitvyamalik/events{/privacy}&#x27;</span>,
   <span class="hljs-string">&#x27;received_events_url&#x27;</span>: <span class="hljs-string">&#x27;https://api.github.com/users/bhavitvyamalik/received_events&#x27;</span>,
   <span class="hljs-string">&#x27;type&#x27;</span>: <span class="hljs-string">&#x27;User&#x27;</span>,
   <span class="hljs-string">&#x27;site_admin&#x27;</span>: <span class="hljs-literal">False</span>},
  <span class="hljs-string">&#x27;labels&#x27;</span>: [],
  <span class="hljs-string">&#x27;state&#x27;</span>: <span class="hljs-string">&#x27;open&#x27;</span>,
  <span class="hljs-string">&#x27;locked&#x27;</span>: <span class="hljs-literal">False</span>,
  <span class="hljs-string">&#x27;assignee&#x27;</span>: <span class="hljs-literal">None</span>,
  <span class="hljs-string">&#x27;assignees&#x27;</span>: [],
  <span class="hljs-string">&#x27;milestone&#x27;</span>: <span class="hljs-literal">None</span>,
  <span class="hljs-string">&#x27;comments&#x27;</span>: <span class="hljs-number">1</span>,
  <span class="hljs-string">&#x27;created_at&#x27;</span>: <span class="hljs-string">&#x27;2021-08-12T11:40:18Z&#x27;</span>,
  <span class="hljs-string">&#x27;updated_at&#x27;</span>: <span class="hljs-string">&#x27;2021-08-12T12:31:17Z&#x27;</span>,
  <span class="hljs-string">&#x27;closed_at&#x27;</span>: <span class="hljs-literal">None</span>,
  <span class="hljs-string">&#x27;author_association&#x27;</span>: <span class="hljs-string">&#x27;CONTRIBUTOR&#x27;</span>,
  <span class="hljs-string">&#x27;active_lock_reason&#x27;</span>: <span class="hljs-literal">None</span>,
  <span class="hljs-string">&#x27;pull_request&#x27;</span>: {<span class="hljs-string">&#x27;url&#x27;</span>: <span class="hljs-string">&#x27;https://api.github.com/repos/huggingface/datasets/pulls/2792&#x27;</span>,
   <span class="hljs-string">&#x27;html_url&#x27;</span>: <span class="hljs-string">&#x27;https://github.com/huggingface/datasets/pull/2792&#x27;</span>,
   <span class="hljs-string">&#x27;diff_url&#x27;</span>: <span class="hljs-string">&#x27;https://github.com/huggingface/datasets/pull/2792.diff&#x27;</span>,
   <span class="hljs-string">&#x27;patch_url&#x27;</span>: <span class="hljs-string">&#x27;https://github.com/huggingface/datasets/pull/2792.patch&#x27;</span>},
  <span class="hljs-string">&#x27;body&#x27;</span>: <span class="hljs-string">&#x27;[GooAQ](https://github.com/allenai/gooaq) dataset was recently updated after splits were added for the same. This PR contains new updated GooAQ with train/val/test splits and updated README as well.&#x27;</span>,
  <span class="hljs-string">&#x27;performed_via_github_app&#x27;</span>: <span class="hljs-literal">None</span>}]`}}),$s=new lt({props:{$$slots:{default:[Rh]},$$scope:{ctx:F}}}),de=new P({props:{code:`GITHUB_TOKEN = xxx  # Copy your GitHub token here
headers = {"Authorization": f"token {GITHUB_TOKEN}"}`,highlighted:`GITHUB_TOKEN = xxx  <span class="hljs-comment"># Copy your GitHub token here</span>
headers = {<span class="hljs-string">&quot;Authorization&quot;</span>: <span class="hljs-string">f&quot;token <span class="hljs-subst">{GITHUB_TOKEN}</span>&quot;</span>}`}}),Es=new lt({props:{warning:!0,$$slots:{default:[Sh]},$$scope:{ctx:F}}}),fe=new P({props:{code:`import time
import math
from pathlib import Path
import pandas as pd
from tqdm.notebook import tqdm


def fetch_issues(
    owner="huggingface",
    repo="datasets",
    num_issues=10_000,
    rate_limit=5_000,
    issues_path=Path("."),
):
    if not issues_path.is_dir():
        issues_path.mkdir(exist_ok=True)

    batch = []
    all_issues = []
    per_page = 100  # Number of issues to return per page
    num_pages = math.ceil(num_issues / per_page)
    base_url = "https://api.github.com/repos"

    for page in tqdm(range(num_pages)):
        # Query with state=all to get both open and closed issues
        query = f"issues?page={page}&per_page={per_page}&state=all"
        issues = requests.get(f"{base_url}/{owner}/{repo}/{query}", headers=headers)
        batch.extend(issues.json())

        if len(batch) > rate_limit and len(all_issues) < num_issues:
            all_issues.extend(batch)
            batch = []  # Flush batch for next time period
            print(f"Reached GitHub rate limit. Sleeping for one hour ...")
            time.sleep(60 * 60 + 1)

    all_issues.extend(batch)
    df = pd.DataFrame.from_records(all_issues)
    df.to_json(f"{issues_path}/{repo}-issues.jsonl", orient="records", lines=True)
    print(
        f"Downloaded all the issues for {repo}! Dataset stored at {issues_path}/{repo}-issues.jsonl"
    )`,highlighted:`<span class="hljs-keyword">import</span> time
<span class="hljs-keyword">import</span> math
<span class="hljs-keyword">from</span> pathlib <span class="hljs-keyword">import</span> Path
<span class="hljs-keyword">import</span> pandas <span class="hljs-keyword">as</span> pd
<span class="hljs-keyword">from</span> tqdm.notebook <span class="hljs-keyword">import</span> tqdm


<span class="hljs-keyword">def</span> <span class="hljs-title function_">fetch_issues</span>(<span class="hljs-params">
    owner=<span class="hljs-string">&quot;huggingface&quot;</span>,
    repo=<span class="hljs-string">&quot;datasets&quot;</span>,
    num_issues=<span class="hljs-number">10_000</span>,
    rate_limit=<span class="hljs-number">5_000</span>,
    issues_path=Path(<span class="hljs-params"><span class="hljs-string">&quot;.&quot;</span></span>),
</span>):
    <span class="hljs-keyword">if</span> <span class="hljs-keyword">not</span> issues_path.is_dir():
        issues_path.mkdir(exist_ok=<span class="hljs-literal">True</span>)

    batch = []
    all_issues = []
    per_page = <span class="hljs-number">100</span>  <span class="hljs-comment"># Number of issues to return per page</span>
    num_pages = math.ceil(num_issues / per_page)
    base_url = <span class="hljs-string">&quot;https://api.github.com/repos&quot;</span>

    <span class="hljs-keyword">for</span> page <span class="hljs-keyword">in</span> tqdm(<span class="hljs-built_in">range</span>(num_pages)):
        <span class="hljs-comment"># Query with state=all to get both open and closed issues</span>
        query = <span class="hljs-string">f&quot;issues?page=<span class="hljs-subst">{page}</span>&amp;per_page=<span class="hljs-subst">{per_page}</span>&amp;state=all&quot;</span>
        issues = requests.get(<span class="hljs-string">f&quot;<span class="hljs-subst">{base_url}</span>/<span class="hljs-subst">{owner}</span>/<span class="hljs-subst">{repo}</span>/<span class="hljs-subst">{query}</span>&quot;</span>, headers=headers)
        batch.extend(issues.json())

        <span class="hljs-keyword">if</span> <span class="hljs-built_in">len</span>(batch) &gt; rate_limit <span class="hljs-keyword">and</span> <span class="hljs-built_in">len</span>(all_issues) &lt; num_issues:
            all_issues.extend(batch)
            batch = []  <span class="hljs-comment"># Flush batch for next time period</span>
            <span class="hljs-built_in">print</span>(<span class="hljs-string">f&quot;Reached GitHub rate limit. Sleeping for one hour ...&quot;</span>)
            time.sleep(<span class="hljs-number">60</span> * <span class="hljs-number">60</span> + <span class="hljs-number">1</span>)

    all_issues.extend(batch)
    df = pd.DataFrame.from_records(all_issues)
    df.to_json(<span class="hljs-string">f&quot;<span class="hljs-subst">{issues_path}</span>/<span class="hljs-subst">{repo}</span>-issues.jsonl&quot;</span>, orient=<span class="hljs-string">&quot;records&quot;</span>, lines=<span class="hljs-literal">True</span>)
    <span class="hljs-built_in">print</span>(
        <span class="hljs-string">f&quot;Downloaded all the issues for <span class="hljs-subst">{repo}</span>! Dataset stored at <span class="hljs-subst">{issues_path}</span>/<span class="hljs-subst">{repo}</span>-issues.jsonl&quot;</span>
    )`}}),me=new P({props:{code:`# Depending on your internet connection, this can take several minutes to run...
fetch_issues()`,highlighted:`<span class="hljs-comment"># Depending on your internet connection, this can take several minutes to run...</span>
fetch_issues()`}}),ge=new P({props:{code:`issues_dataset = load_dataset("json", data_files="datasets-issues.jsonl", split="train")
issues_dataset`,highlighted:`issues_dataset = load_dataset(<span class="hljs-string">&quot;json&quot;</span>, data_files=<span class="hljs-string">&quot;datasets-issues.jsonl&quot;</span>, split=<span class="hljs-string">&quot;train&quot;</span>)
issues_dataset`}}),_e=new P({props:{code:`Dataset({
    features: ['url', 'repository_url', 'labels_url', 'comments_url', 'events_url', 'html_url', 'id', 'node_id', 'number', 'title', 'user', 'labels', 'state', 'locked', 'assignee', 'assignees', 'milestone', 'comments', 'created_at', 'updated_at', 'closed_at', 'author_association', 'active_lock_reason', 'pull_request', 'body', 'timeline_url', 'performed_via_github_app'],
    num_rows: 3019
})`,highlighted:`Dataset({
    features: [<span class="hljs-string">&#x27;url&#x27;</span>, <span class="hljs-string">&#x27;repository_url&#x27;</span>, <span class="hljs-string">&#x27;labels_url&#x27;</span>, <span class="hljs-string">&#x27;comments_url&#x27;</span>, <span class="hljs-string">&#x27;events_url&#x27;</span>, <span class="hljs-string">&#x27;html_url&#x27;</span>, <span class="hljs-string">&#x27;id&#x27;</span>, <span class="hljs-string">&#x27;node_id&#x27;</span>, <span class="hljs-string">&#x27;number&#x27;</span>, <span class="hljs-string">&#x27;title&#x27;</span>, <span class="hljs-string">&#x27;user&#x27;</span>, <span class="hljs-string">&#x27;labels&#x27;</span>, <span class="hljs-string">&#x27;state&#x27;</span>, <span class="hljs-string">&#x27;locked&#x27;</span>, <span class="hljs-string">&#x27;assignee&#x27;</span>, <span class="hljs-string">&#x27;assignees&#x27;</span>, <span class="hljs-string">&#x27;milestone&#x27;</span>, <span class="hljs-string">&#x27;comments&#x27;</span>, <span class="hljs-string">&#x27;created_at&#x27;</span>, <span class="hljs-string">&#x27;updated_at&#x27;</span>, <span class="hljs-string">&#x27;closed_at&#x27;</span>, <span class="hljs-string">&#x27;author_association&#x27;</span>, <span class="hljs-string">&#x27;active_lock_reason&#x27;</span>, <span class="hljs-string">&#x27;pull_request&#x27;</span>, <span class="hljs-string">&#x27;body&#x27;</span>, <span class="hljs-string">&#x27;timeline_url&#x27;</span>, <span class="hljs-string">&#x27;performed_via_github_app&#x27;</span>],
    num_rows: <span class="hljs-number">3019</span>
})`}}),we=new Gt({}),xe=new P({props:{code:`sample = issues_dataset.shuffle(seed=666).select(range(3))

# Print out the URL and pull request entries
for url, pr in zip(sample["html_url"], sample["pull_request"]):
    print(f">> URL: {url}")
    print(f">> Pull request: {pr}\\n")`,highlighted:`sample = issues_dataset.shuffle(seed=<span class="hljs-number">666</span>).select(<span class="hljs-built_in">range</span>(<span class="hljs-number">3</span>))

<span class="hljs-comment"># Print out the URL and pull request entries</span>
<span class="hljs-keyword">for</span> url, pr <span class="hljs-keyword">in</span> <span class="hljs-built_in">zip</span>(sample[<span class="hljs-string">&quot;html_url&quot;</span>], sample[<span class="hljs-string">&quot;pull_request&quot;</span>]):
    <span class="hljs-built_in">print</span>(<span class="hljs-string">f&quot;&gt;&gt; URL: <span class="hljs-subst">{url}</span>&quot;</span>)
    <span class="hljs-built_in">print</span>(<span class="hljs-string">f&quot;&gt;&gt; Pull request: <span class="hljs-subst">{pr}</span>\\n&quot;</span>)`}}),ye=new P({props:{code:`>> URL: https://github.com/huggingface/datasets/pull/850
>> Pull request: {'url': 'https://api.github.com/repos/huggingface/datasets/pulls/850', 'html_url': 'https://github.com/huggingface/datasets/pull/850', 'diff_url': 'https://github.com/huggingface/datasets/pull/850.diff', 'patch_url': 'https://github.com/huggingface/datasets/pull/850.patch'}

>> URL: https://github.com/huggingface/datasets/issues/2773
>> Pull request: None

>> URL: https://github.com/huggingface/datasets/pull/783
>> Pull request: {'url': 'https://api.github.com/repos/huggingface/datasets/pulls/783', 'html_url': 'https://github.com/huggingface/datasets/pull/783', 'diff_url': 'https://github.com/huggingface/datasets/pull/783.diff', 'patch_url': 'https://github.com/huggingface/datasets/pull/783.patch'}`,highlighted:`&gt;&gt; URL: https://github.com/huggingface/datasets/pull/<span class="hljs-number">850</span>
&gt;&gt; Pull request: {<span class="hljs-string">&#x27;url&#x27;</span>: <span class="hljs-string">&#x27;https://api.github.com/repos/huggingface/datasets/pulls/850&#x27;</span>, <span class="hljs-string">&#x27;html_url&#x27;</span>: <span class="hljs-string">&#x27;https://github.com/huggingface/datasets/pull/850&#x27;</span>, <span class="hljs-string">&#x27;diff_url&#x27;</span>: <span class="hljs-string">&#x27;https://github.com/huggingface/datasets/pull/850.diff&#x27;</span>, <span class="hljs-string">&#x27;patch_url&#x27;</span>: <span class="hljs-string">&#x27;https://github.com/huggingface/datasets/pull/850.patch&#x27;</span>}

&gt;&gt; URL: https://github.com/huggingface/datasets/issues/<span class="hljs-number">2773</span>
&gt;&gt; Pull request: <span class="hljs-literal">None</span>

&gt;&gt; URL: https://github.com/huggingface/datasets/pull/<span class="hljs-number">783</span>
&gt;&gt; Pull request: {<span class="hljs-string">&#x27;url&#x27;</span>: <span class="hljs-string">&#x27;https://api.github.com/repos/huggingface/datasets/pulls/783&#x27;</span>, <span class="hljs-string">&#x27;html_url&#x27;</span>: <span class="hljs-string">&#x27;https://github.com/huggingface/datasets/pull/783&#x27;</span>, <span class="hljs-string">&#x27;diff_url&#x27;</span>: <span class="hljs-string">&#x27;https://github.com/huggingface/datasets/pull/783.diff&#x27;</span>, <span class="hljs-string">&#x27;patch_url&#x27;</span>: <span class="hljs-string">&#x27;https://github.com/huggingface/datasets/pull/783.patch&#x27;</span>}`}}),je=new P({props:{code:`issues_dataset = issues_dataset.map(
    lambda x: {"is_pull_request": False if x["pull_request"] is None else True}
)`,highlighted:`issues_dataset = issues_dataset.<span class="hljs-built_in">map</span>(
    <span class="hljs-keyword">lambda</span> x: {<span class="hljs-string">&quot;is_pull_request&quot;</span>: <span class="hljs-literal">False</span> <span class="hljs-keyword">if</span> x[<span class="hljs-string">&quot;pull_request&quot;</span>] <span class="hljs-keyword">is</span> <span class="hljs-literal">None</span> <span class="hljs-keyword">else</span> <span class="hljs-literal">True</span>}
)`}}),Ds=new lt({props:{$$slots:{default:[Fh]},$$scope:{ctx:F}}}),$e=new Gt({}),qe=new P({props:{code:`issue_number = 2792
url = f"https://api.github.com/repos/huggingface/datasets/issues/{issue_number}/comments"
response = requests.get(url, headers=headers)
response.json()`,highlighted:`issue_number = <span class="hljs-number">2792</span>
url = <span class="hljs-string">f&quot;https://api.github.com/repos/huggingface/datasets/issues/<span class="hljs-subst">{issue_number}</span>/comments&quot;</span>
response = requests.get(url, headers=headers)
response.json()`}}),De=new P({props:{code:`[{'url': 'https://api.github.com/repos/huggingface/datasets/issues/comments/897594128',
  'html_url': 'https://github.com/huggingface/datasets/pull/2792#issuecomment-897594128',
  'issue_url': 'https://api.github.com/repos/huggingface/datasets/issues/2792',
  'id': 897594128,
  'node_id': 'IC_kwDODunzps41gDMQ',
  'user': {'login': 'bhavitvyamalik',
   'id': 19718818,
   'node_id': 'MDQ6VXNlcjE5NzE4ODE4',
   'avatar_url': 'https://avatars.githubusercontent.com/u/19718818?v=4',
   'gravatar_id': '',
   'url': 'https://api.github.com/users/bhavitvyamalik',
   'html_url': 'https://github.com/bhavitvyamalik',
   'followers_url': 'https://api.github.com/users/bhavitvyamalik/followers',
   'following_url': 'https://api.github.com/users/bhavitvyamalik/following{/other_user}',
   'gists_url': 'https://api.github.com/users/bhavitvyamalik/gists{/gist_id}',
   'starred_url': 'https://api.github.com/users/bhavitvyamalik/starred{/owner}{/repo}',
   'subscriptions_url': 'https://api.github.com/users/bhavitvyamalik/subscriptions',
   'organizations_url': 'https://api.github.com/users/bhavitvyamalik/orgs',
   'repos_url': 'https://api.github.com/users/bhavitvyamalik/repos',
   'events_url': 'https://api.github.com/users/bhavitvyamalik/events{/privacy}',
   'received_events_url': 'https://api.github.com/users/bhavitvyamalik/received_events',
   'type': 'User',
   'site_admin': False},
  'created_at': '2021-08-12T12:21:52Z',
  'updated_at': '2021-08-12T12:31:17Z',
  'author_association': 'CONTRIBUTOR',
  'body': "@albertvillanova my tests are failing here:\\r\\n\`\`\`\\r\\ndataset_name = 'gooaq'\\r\\n\\r\\n    def test_load_dataset(self, dataset_name):\\r\\n        configs = self.dataset_tester.load_all_configs(dataset_name, is_local=True)[:1]\\r\\n>       self.dataset_tester.check_load_dataset(dataset_name, configs, is_local=True, use_local_dummy_data=True)\\r\\n\\r\\ntests/test_dataset_common.py:234: \\r\\n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \\r\\ntests/test_dataset_common.py:187: in check_load_dataset\\r\\n    self.parent.assertTrue(len(dataset[split]) > 0)\\r\\nE   AssertionError: False is not true\\r\\n\`\`\`\\r\\nWhen I try loading dataset on local machine it works fine. Any suggestions on how can I avoid this error?",
  'performed_via_github_app': None}]`,highlighted:`[{<span class="hljs-string">&#x27;url&#x27;</span>: <span class="hljs-string">&#x27;https://api.github.com/repos/huggingface/datasets/issues/comments/897594128&#x27;</span>,
  <span class="hljs-string">&#x27;html_url&#x27;</span>: <span class="hljs-string">&#x27;https://github.com/huggingface/datasets/pull/2792#issuecomment-897594128&#x27;</span>,
  <span class="hljs-string">&#x27;issue_url&#x27;</span>: <span class="hljs-string">&#x27;https://api.github.com/repos/huggingface/datasets/issues/2792&#x27;</span>,
  <span class="hljs-string">&#x27;id&#x27;</span>: <span class="hljs-number">897594128</span>,
  <span class="hljs-string">&#x27;node_id&#x27;</span>: <span class="hljs-string">&#x27;IC_kwDODunzps41gDMQ&#x27;</span>,
  <span class="hljs-string">&#x27;user&#x27;</span>: {<span class="hljs-string">&#x27;login&#x27;</span>: <span class="hljs-string">&#x27;bhavitvyamalik&#x27;</span>,
   <span class="hljs-string">&#x27;id&#x27;</span>: <span class="hljs-number">19718818</span>,
   <span class="hljs-string">&#x27;node_id&#x27;</span>: <span class="hljs-string">&#x27;MDQ6VXNlcjE5NzE4ODE4&#x27;</span>,
   <span class="hljs-string">&#x27;avatar_url&#x27;</span>: <span class="hljs-string">&#x27;https://avatars.githubusercontent.com/u/19718818?v=4&#x27;</span>,
   <span class="hljs-string">&#x27;gravatar_id&#x27;</span>: <span class="hljs-string">&#x27;&#x27;</span>,
   <span class="hljs-string">&#x27;url&#x27;</span>: <span class="hljs-string">&#x27;https://api.github.com/users/bhavitvyamalik&#x27;</span>,
   <span class="hljs-string">&#x27;html_url&#x27;</span>: <span class="hljs-string">&#x27;https://github.com/bhavitvyamalik&#x27;</span>,
   <span class="hljs-string">&#x27;followers_url&#x27;</span>: <span class="hljs-string">&#x27;https://api.github.com/users/bhavitvyamalik/followers&#x27;</span>,
   <span class="hljs-string">&#x27;following_url&#x27;</span>: <span class="hljs-string">&#x27;https://api.github.com/users/bhavitvyamalik/following{/other_user}&#x27;</span>,
   <span class="hljs-string">&#x27;gists_url&#x27;</span>: <span class="hljs-string">&#x27;https://api.github.com/users/bhavitvyamalik/gists{/gist_id}&#x27;</span>,
   <span class="hljs-string">&#x27;starred_url&#x27;</span>: <span class="hljs-string">&#x27;https://api.github.com/users/bhavitvyamalik/starred{/owner}{/repo}&#x27;</span>,
   <span class="hljs-string">&#x27;subscriptions_url&#x27;</span>: <span class="hljs-string">&#x27;https://api.github.com/users/bhavitvyamalik/subscriptions&#x27;</span>,
   <span class="hljs-string">&#x27;organizations_url&#x27;</span>: <span class="hljs-string">&#x27;https://api.github.com/users/bhavitvyamalik/orgs&#x27;</span>,
   <span class="hljs-string">&#x27;repos_url&#x27;</span>: <span class="hljs-string">&#x27;https://api.github.com/users/bhavitvyamalik/repos&#x27;</span>,
   <span class="hljs-string">&#x27;events_url&#x27;</span>: <span class="hljs-string">&#x27;https://api.github.com/users/bhavitvyamalik/events{/privacy}&#x27;</span>,
   <span class="hljs-string">&#x27;received_events_url&#x27;</span>: <span class="hljs-string">&#x27;https://api.github.com/users/bhavitvyamalik/received_events&#x27;</span>,
   <span class="hljs-string">&#x27;type&#x27;</span>: <span class="hljs-string">&#x27;User&#x27;</span>,
   <span class="hljs-string">&#x27;site_admin&#x27;</span>: <span class="hljs-literal">False</span>},
  <span class="hljs-string">&#x27;created_at&#x27;</span>: <span class="hljs-string">&#x27;2021-08-12T12:21:52Z&#x27;</span>,
  <span class="hljs-string">&#x27;updated_at&#x27;</span>: <span class="hljs-string">&#x27;2021-08-12T12:31:17Z&#x27;</span>,
  <span class="hljs-string">&#x27;author_association&#x27;</span>: <span class="hljs-string">&#x27;CONTRIBUTOR&#x27;</span>,
  <span class="hljs-string">&#x27;body&#x27;</span>: <span class="hljs-string">&quot;@albertvillanova my tests are failing here:\\r\\n\`\`\`\\r\\ndataset_name = &#x27;gooaq&#x27;\\r\\n\\r\\n    def test_load_dataset(self, dataset_name):\\r\\n        configs = self.dataset_tester.load_all_configs(dataset_name, is_local=True)[:1]\\r\\n&gt;       self.dataset_tester.check_load_dataset(dataset_name, configs, is_local=True, use_local_dummy_data=True)\\r\\n\\r\\ntests/test_dataset_common.py:234: \\r\\n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \\r\\ntests/test_dataset_common.py:187: in check_load_dataset\\r\\n    self.parent.assertTrue(len(dataset[split]) &gt; 0)\\r\\nE   AssertionError: False is not true\\r\\n\`\`\`\\r\\nWhen I try loading dataset on local machine it works fine. Any suggestions on how can I avoid this error?&quot;</span>,
  <span class="hljs-string">&#x27;performed_via_github_app&#x27;</span>: <span class="hljs-literal">None</span>}]`}}),Te=new P({props:{code:`def get_comments(issue_number):
    url = f"https://api.github.com/repos/huggingface/datasets/issues/{issue_number}/comments"
    response = requests.get(url, headers=headers)
    return [r["body"] for r in response.json()]


# Test our function works as expected
get_comments(2792)`,highlighted:`<span class="hljs-keyword">def</span> <span class="hljs-title function_">get_comments</span>(<span class="hljs-params">issue_number</span>):
    url = <span class="hljs-string">f&quot;https://api.github.com/repos/huggingface/datasets/issues/<span class="hljs-subst">{issue_number}</span>/comments&quot;</span>
    response = requests.get(url, headers=headers)
    <span class="hljs-keyword">return</span> [r[<span class="hljs-string">&quot;body&quot;</span>] <span class="hljs-keyword">for</span> r <span class="hljs-keyword">in</span> response.json()]


<span class="hljs-comment"># Test our function works as expected</span>
get_comments(<span class="hljs-number">2792</span>)`}}),He=new P({props:{code:"[\"@albertvillanova my tests are failing here:\\r\\n```\\r\\ndataset_name = 'gooaq'\\r\\n\\r\\n    def test_load_dataset(self, dataset_name):\\r\\n        configs = self.dataset_tester.load_all_configs(dataset_name, is_local=True)[:1]\\r\\n>       self.dataset_tester.check_load_dataset(dataset_name, configs, is_local=True, use_local_dummy_data=True)\\r\\n\\r\\ntests/test_dataset_common.py:234: \\r\\n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \\r\\ntests/test_dataset_common.py:187: in check_load_dataset\\r\\n    self.parent.assertTrue(len(dataset[split]) > 0)\\r\\nE   AssertionError: False is not true\\r\\n```\\r\\nWhen I try loading dataset on local machine it works fine. Any suggestions on how can I avoid this error?\"]",highlighted:'[<span class="hljs-string">&quot;@albertvillanova my tests are failing here:\\r\\n```\\r\\ndataset_name = &#x27;gooaq&#x27;\\r\\n\\r\\n    def test_load_dataset(self, dataset_name):\\r\\n        configs = self.dataset_tester.load_all_configs(dataset_name, is_local=True)[:1]\\r\\n&gt;       self.dataset_tester.check_load_dataset(dataset_name, configs, is_local=True, use_local_dummy_data=True)\\r\\n\\r\\ntests/test_dataset_common.py:234: \\r\\n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \\r\\ntests/test_dataset_common.py:187: in check_load_dataset\\r\\n    self.parent.assertTrue(len(dataset[split]) &gt; 0)\\r\\nE   AssertionError: False is not true\\r\\n```\\r\\nWhen I try loading dataset on local machine it works fine. Any suggestions on how can I avoid this error?&quot;</span>]'}}),Oe=new P({props:{code:`# Depending on your internet connection, this can take a few minutes...
issues_with_comments_dataset = issues_dataset.map(
    lambda x: {"comments": get_comments(x["number"])}
)`,highlighted:`<span class="hljs-comment"># Depending on your internet connection, this can take a few minutes...</span>
issues_with_comments_dataset = issues_dataset.<span class="hljs-built_in">map</span>(
    <span class="hljs-keyword">lambda</span> x: {<span class="hljs-string">&quot;comments&quot;</span>: get_comments(x[<span class="hljs-string">&quot;number&quot;</span>])}
)`}}),Pe=new P({props:{code:'issues_with_comments_dataset.to_json("issues-datasets-with-hf-doc-builder.jsonl")',highlighted:'issues_with_comments_dataset.to_json(<span class="hljs-string">&quot;issues-datasets-with-hf-doc-builder.jsonl&quot;</span>)'}}),Ae=new Gt({}),Ne=new Gh({props:{id:"HaN6qCr_Afc"}}),Ge=new P({props:{code:`from huggingface_hub import list_datasets

all_datasets = list_datasets()
print(f"Number of datasets on Hub: {len(all_datasets)}")
print(all_datasets[0])`,highlighted:`<span class="hljs-keyword">from</span> huggingface_hub <span class="hljs-keyword">import</span> list_datasets

all_datasets = list_datasets()
<span class="hljs-built_in">print</span>(<span class="hljs-string">f&quot;Number of datasets on Hub: <span class="hljs-subst">{<span class="hljs-built_in">len</span>(all_datasets)}</span>&quot;</span>)
<span class="hljs-built_in">print</span>(all_datasets[<span class="hljs-number">0</span>])`}}),Ie=new P({props:{code:`Number of datasets on Hub: 1487
Dataset Name: acronym_identification, Tags: ['annotations_creators:expert-generated', 'language_creators:found', 'languages:en', 'licenses:mit', 'multilinguality:monolingual', 'size_categories:10K<n<100K', 'source_datasets:original', 'task_categories:structure-prediction', 'task_ids:structure-prediction-other-acronym-identification']`,highlighted:`Number of datasets on Hub: <span class="hljs-number">1487</span>
Dataset Name: acronym_identification, Tags: [<span class="hljs-string">&#x27;annotations_creators:expert-generated&#x27;</span>, <span class="hljs-string">&#x27;language_creators:found&#x27;</span>, <span class="hljs-string">&#x27;languages:en&#x27;</span>, <span class="hljs-string">&#x27;licenses:mit&#x27;</span>, <span class="hljs-string">&#x27;multilinguality:monolingual&#x27;</span>, <span class="hljs-string">&#x27;size_categories:10K&lt;n&lt;100K&#x27;</span>, <span class="hljs-string">&#x27;source_datasets:original&#x27;</span>, <span class="hljs-string">&#x27;task_categories:structure-prediction&#x27;</span>, <span class="hljs-string">&#x27;task_ids:structure-prediction-other-acronym-identification&#x27;</span>]`}}),Re=new P({props:{code:`from huggingface_hub import notebook_login

notebook_login()`,highlighted:`<span class="hljs-keyword">from</span> huggingface_hub <span class="hljs-keyword">import</span> notebook_login

notebook_login()`}}),Se=new P({props:{code:"huggingface-cli login",highlighted:"huggingface-cli login"}}),Fe=new P({props:{code:`from huggingface_hub import create_repo

repo_url = create_repo(name="github-issues", repo_type="dataset")
repo_url`,highlighted:`<span class="hljs-keyword">from</span> huggingface_hub <span class="hljs-keyword">import</span> create_repo

repo_url = create_repo(name=<span class="hljs-string">&quot;github-issues&quot;</span>, repo_type=<span class="hljs-string">&quot;dataset&quot;</span>)
repo_url`}}),Le=new P({props:{code:"'https://huggingface.co/datasets/lewtun/github-issues'",highlighted:'<span class="hljs-string">&#x27;https://huggingface.co/datasets/lewtun/github-issues&#x27;</span>'}}),Is=new lt({props:{$$slots:{default:[Lh]},$$scope:{ctx:F}}}),Ue=new P({props:{code:`from huggingface_hub import Repository

repo = Repository(local_dir="github-issues", clone_from=repo_url)
!cp datasets-issues-with-hf-doc-builder.jsonl github-issues/`,highlighted:`<span class="hljs-keyword">from</span> huggingface_hub <span class="hljs-keyword">import</span> Repository

repo = Repository(local_dir=<span class="hljs-string">&quot;github-issues&quot;</span>, clone_from=repo_url)
!cp datasets-issues-<span class="hljs-keyword">with</span>-hf-doc-builder.jsonl github-issues/`}}),Me=new P({props:{code:'repo.lfs_track("*.jsonl")',highlighted:'repo.lfs_track(<span class="hljs-string">&quot;*.jsonl&quot;</span>)'}}),ze=new P({props:{code:"repo.push_to_hub()",highlighted:"repo.push_to_hub()"}}),Ye=new P({props:{code:`remote_dataset = load_dataset("lewtun/github-issues", split="train")
remote_dataset`,highlighted:`remote_dataset = load_dataset(<span class="hljs-string">&quot;lewtun/github-issues&quot;</span>, split=<span class="hljs-string">&quot;train&quot;</span>)
remote_dataset`}}),Qe=new P({props:{code:`Dataset({
    features: ['url', 'repository_url', 'labels_url', 'comments_url', 'events_url', 'html_url', 'id', 'node_id', 'number', 'title', 'user', 'labels', 'state', 'locked', 'assignee', 'assignees', 'milestone', 'comments', 'created_at', 'updated_at', 'closed_at', 'author_association', 'active_lock_reason', 'pull_request', 'body', 'performed_via_github_app', 'is_pull_request'],
    num_rows: 2855
})`,highlighted:`Dataset({
    features: [<span class="hljs-string">&#x27;url&#x27;</span>, <span class="hljs-string">&#x27;repository_url&#x27;</span>, <span class="hljs-string">&#x27;labels_url&#x27;</span>, <span class="hljs-string">&#x27;comments_url&#x27;</span>, <span class="hljs-string">&#x27;events_url&#x27;</span>, <span class="hljs-string">&#x27;html_url&#x27;</span>, <span class="hljs-string">&#x27;id&#x27;</span>, <span class="hljs-string">&#x27;node_id&#x27;</span>, <span class="hljs-string">&#x27;number&#x27;</span>, <span class="hljs-string">&#x27;title&#x27;</span>, <span class="hljs-string">&#x27;user&#x27;</span>, <span class="hljs-string">&#x27;labels&#x27;</span>, <span class="hljs-string">&#x27;state&#x27;</span>, <span class="hljs-string">&#x27;locked&#x27;</span>, <span class="hljs-string">&#x27;assignee&#x27;</span>, <span class="hljs-string">&#x27;assignees&#x27;</span>, <span class="hljs-string">&#x27;milestone&#x27;</span>, <span class="hljs-string">&#x27;comments&#x27;</span>, <span class="hljs-string">&#x27;created_at&#x27;</span>, <span class="hljs-string">&#x27;updated_at&#x27;</span>, <span class="hljs-string">&#x27;closed_at&#x27;</span>, <span class="hljs-string">&#x27;author_association&#x27;</span>, <span class="hljs-string">&#x27;active_lock_reason&#x27;</span>, <span class="hljs-string">&#x27;pull_request&#x27;</span>, <span class="hljs-string">&#x27;body&#x27;</span>, <span class="hljs-string">&#x27;performed_via_github_app&#x27;</span>, <span class="hljs-string">&#x27;is_pull_request&#x27;</span>],
    num_rows: <span class="hljs-number">2855</span>
})`}}),Us=new lt({props:{$$slots:{default:[Uh]},$$scope:{ctx:F}}}),Je=new Gt({}),Bs=new lt({props:{$$slots:{default:[Mh]},$$scope:{ctx:F}}}),Ys=new lt({props:{$$slots:{default:[zh]},$$scope:{ctx:F}}}),{c(){c=i("meta"),D=p(),f=i("h1"),$=i("a"),k=i("span"),g(m.$$.fragment),E=p(),T=i("span"),y=a("Creating your own dataset"),j=p(),g(A.$$.fragment),H=p(),O=i("p"),C=a("Sometimes the dataset that you need to build an NLP application doesn\u2019t exist, so you\u2019ll need to create it yourself. In this section we\u2019ll show you how to create a corpus of "),q=i("a"),N=a("GitHub issues"),W=a(", which are commonly used to track bugs or features in GitHub repositories. This corpus could be used for various purposes, including:"),S=p(),G=i("ul"),B=i("li"),us=a("Exploring how long it takes to close open issues or pull requests"),I=p(),Z=i("li"),it=a("Training a "),bs=i("em"),ot=a("multilabel classifier"),rt=a(" that can tag issues with metadata based on the issue\u2019s description (e.g., \u201Cbug,\u201D \u201Cenhancement,\u201D or \u201Cquestion\u201D)"),ut=p(),It=i("li"),li=a("Creating a semantic search engine to find which issues match a user\u2019s query"),Ka=p(),pt=i("p"),ii=a("Here we\u2019ll focus on creating the corpus, and in the next section we\u2019ll tackle the semantic search application. To keep things meta, we\u2019ll use the GitHub issues associated with a popular open source project: \u{1F917} Datasets! Let\u2019s take a look at how to get the data and explore the information contained in these issues."),Va=p(),ps=i("h2"),vs=i("a"),Rt=i("span"),g(Ks.$$.fragment),oi=p(),St=i("span"),ri=a("Getting the data"),Xa=p(),ws=i("p"),ui=a("You can find all the issues in \u{1F917} Datasets by navigating to the repository\u2019s "),Vs=i("a"),pi=a("Issues tab"),hi=a(". As shown in the following screenshot, at the time of writing there were 331 open issues and 668 closed ones."),sn=p(),Xs=i("div"),se=i("img"),en=p(),ht=i("p"),ci=a("If you click on one of these issues you\u2019ll find it contains a title, a description, and a set of labels that characterize the issue. An example is shown in the screenshot below."),tn=p(),ee=i("div"),te=i("img"),an=p(),K=i("p"),di=a("To download all the repository\u2019s issues, we\u2019ll use the "),ae=i("a"),fi=a("GitHub REST API"),mi=a(" to poll the "),xs=i("a"),Ft=i("code"),gi=a("Issues"),_i=a(" endpoint"),bi=a(". This endpoint returns a list of JSON objects, with each object containing a large number of fields that include the title and description as well as metadata about the status of the issue and so on."),nn=p(),ys=i("p"),vi=a("A convenient way to download the issues is via the "),Lt=i("code"),wi=a("requests"),xi=a(" library, which is the standard way for making HTTP requests in Python. You can install the library by running:"),ln=p(),g(ne.$$.fragment),on=p(),V=i("p"),yi=a("Once the library is installed, you can make GET requests to the "),Ut=i("code"),ji=a("Issues"),$i=a(" endpoint by invoking the "),Mt=i("code"),Ei=a("requests.get()"),ki=a(" function. For example, you can run the following command to retrieve the first issue on the first page:"),rn=p(),g(le.$$.fragment),un=p(),js=i("p"),qi=a("The "),zt=i("code"),Di=a("response"),Ti=a(" object contains a lot of useful information about the request, including the HTTP status code:"),pn=p(),g(ie.$$.fragment),hn=p(),g(oe.$$.fragment),cn=p(),Y=i("p"),Hi=a("where a "),Wt=i("code"),Oi=a("200"),Pi=a(" status means the request was successful (you can find a list of possible HTTP status codes "),re=i("a"),Ai=a("here"),Ni=a("). What we are really interested in, though, is the "),Bt=i("em"),Ci=a("payload"),Gi=a(", which can be accessed in various formats like bytes, strings, or JSON. Since we know our issues are in JSON format, let\u2019s inspect the payload as follows:"),dn=p(),g(ue.$$.fragment),fn=p(),g(pe.$$.fragment),mn=p(),Q=i("p"),Ii=a("Whoa, that\u2019s a lot of information! We can see useful fields like "),Yt=i("code"),Ri=a("title"),Si=a(", "),Qt=i("code"),Fi=a("body"),Li=a(", and "),Jt=i("code"),Ui=a("number"),Mi=a(" that describe the issue, as well as information about the GitHub user who opened the issue."),gn=p(),g($s.$$.fragment),_n=p(),L=i("p"),zi=a("As described in the GitHub "),he=i("a"),Wi=a("documentation"),Bi=a(", unauthenticated requests are limited to 60 requests per hour. Although you can increase the "),Zt=i("code"),Yi=a("per_page"),Qi=a(" query parameter to reduce the number of requests you make, you will still hit the rate limit on any repository that has more than a few thousand issues. So instead, you should follow GitHub\u2019s "),ce=i("a"),Ji=a("instructions"),Zi=a(" on creating a "),Kt=i("em"),Ki=a("personal access token"),Vi=a(" so that you can boost the rate limit to 5,000 requests per hour. Once you have your token, you can include it as part of the request header:"),bn=p(),g(de.$$.fragment),vn=p(),g(Es.$$.fragment),wn=p(),ct=i("p"),Xi=a("Now that we have our access token, let\u2019s create a function that can download all the issues from a GitHub repository:"),xn=p(),g(fe.$$.fragment),yn=p(),X=i("p"),so=a("Now when we call "),Vt=i("code"),eo=a("fetch_issues()"),to=a(" it will download all the issues in batches to avoid exceeding GitHub\u2019s limit on the number of requests per hour; the result will be stored in a "),Xt=i("em"),ao=a("repository_name-issues.jsonl"),no=a(" file, where each line is a JSON object the represents an issue. Let\u2019s use this function to grab all the issues from \u{1F917} Datasets:"),jn=p(),g(me.$$.fragment),$n=p(),ks=i("p"),lo=a("Once the issues are downloaded we can load them locally using our newfound skills from "),dt=i("a"),io=a("section 2"),oo=a(":"),En=p(),g(ge.$$.fragment),kn=p(),g(_e.$$.fragment),qn=p(),ss=i("p"),ro=a("Great, we\u2019ve created our first dataset from scratch! But why are there several thousand issues when the "),be=i("a"),uo=a("Issues tab"),po=a(" of the \u{1F917} Datasets repository only shows around 1,000 issues in total \u{1F914}? As described in the GitHub "),ve=i("a"),ho=a("documentation"),co=a(", that\u2019s because we\u2019ve downloaded all the pull requests as well:"),Dn=p(),ft=i("blockquote"),hs=i("p"),fo=a("GitHub\u2019s REST API v3 considers every pull request an issue, but not every issue is a pull request. For this reason, \u201CIssues\u201D endpoints may return both issues and pull requests in the response. You can identify pull requests by the "),sa=i("code"),mo=a("pull_request"),go=a(" key. Be aware that the "),ea=i("code"),_o=a("id"),bo=a(" of a pull request returned from \u201CIssues\u201D endpoints will be an issue id."),Tn=p(),mt=i("p"),vo=a("Since the contents of issues and pull requests are quite different, let\u2019s do some minor preprocessing to enable us to distinguish between them."),Hn=p(),cs=i("h2"),qs=i("a"),ta=i("span"),g(we.$$.fragment),wo=p(),aa=i("span"),xo=a("Cleaning up the data"),On=p(),R=i("p"),yo=a("The above snippet from GitHub\u2019s documentation tells us that the "),na=i("code"),jo=a("pull_request"),$o=a(" column can be used to differentiate between issues and pull requests. Let\u2019s look at a random sample to see what the difference is. As we did in "),gt=i("a"),Eo=a("section 3"),ko=a(", we\u2019ll chain "),la=i("code"),qo=a("Dataset.shuffle()"),Do=a(" and "),ia=i("code"),To=a("Dataset.select()"),Ho=a(" to create a random sample and then zip the "),oa=i("code"),Oo=a("html_url"),Po=a(" and "),ra=i("code"),Ao=a("pull_request"),No=a(" columns so we can compare the various URLs:"),Pn=p(),g(xe.$$.fragment),An=p(),g(ye.$$.fragment),Nn=p(),U=i("p"),Co=a("Here we can see that each pull request is associated with various URLs, while ordinary issues have a "),ua=i("code"),Go=a("None"),Io=a(" entry. We can use this distinction to create a new "),pa=i("code"),Ro=a("is_pull_request"),So=a(" column that checks whether the "),ha=i("code"),Fo=a("pull_request"),Lo=a(" field is "),ca=i("code"),Uo=a("None"),Mo=a(" or not:"),Cn=p(),g(je.$$.fragment),Gn=p(),g(Ds.$$.fragment),In=p(),_t=i("p"),zo=a("Although we could proceed to further clean up the dataset by dropping or renaming some columns, it is generally a good practice to keep the dataset as \u201Craw\u201D as possible at this stage so that it can be easily used in multiple applications."),Rn=p(),bt=i("p"),Wo=a("Before we push our dataset to the Hugging Face Hub, let\u2019s deal with one thing that\u2019s missing from it: the comments associated with each issue and pull request. We\u2019ll add them next with \u2014 you guessed it \u2014 the GitHub REST API!"),Sn=p(),ds=i("h2"),Ts=i("a"),da=i("span"),g($e.$$.fragment),Bo=p(),fa=i("span"),Yo=a("Augmenting the dataset"),Fn=p(),vt=i("p"),Qo=a("As shown in the following screenshot, the comments associated with an issue or pull request provide a rich source of information, especially if we\u2019re interested in building a search engine to answer user queries about the library."),Ln=p(),Ee=i("div"),ke=i("img"),Un=p(),Hs=i("p"),Jo=a("The GitHub REST API provides a "),Os=i("a"),ma=i("code"),Zo=a("Comments"),Ko=a(" endpoint"),Vo=a(" that returns all the comments associated with an issue number. Let\u2019s test the endpoint to see what it returns:"),Mn=p(),g(qe.$$.fragment),zn=p(),g(De.$$.fragment),Wn=p(),J=i("p"),Xo=a("We can see that the comment is stored in the "),ga=i("code"),sr=a("body"),er=a(" field, so let\u2019s write a simple function that returns all the comments associated with an issue by picking out the "),_a=i("code"),tr=a("body"),ar=a(" contents for each element in "),ba=i("code"),nr=a("response.json()"),lr=a(":"),Bn=p(),g(Te.$$.fragment),Yn=p(),g(He.$$.fragment),Qn=p(),es=i("p"),ir=a("This looks good, so let\u2019s use "),va=i("code"),or=a("Dataset.map()"),rr=a(" to add a new "),wa=i("code"),ur=a("comments"),pr=a(" column to each issue in our dataset:"),Jn=p(),g(Oe.$$.fragment),Zn=p(),wt=i("p"),hr=a("The final step is to save the augmented dataset alongside our raw data so we can push them both to the Hub:"),Kn=p(),g(Pe.$$.fragment),Vn=p(),fs=i("h2"),Ps=i("a"),xa=i("span"),g(Ae.$$.fragment),cr=p(),ya=i("span"),dr=a("Uploading the dataset to the Hugging Face Hub"),Xn=p(),g(Ne.$$.fragment),sl=p(),ts=i("p"),fr=a("Now that we have our augmented dataset, it\u2019s time to push it to the Hub so we can share it with the community! To upload the dataset we\u2019ll use the "),Ce=i("a"),mr=a("\u{1F917} Hub library"),gr=a(", which allows us to interact with the Hugging Face Hub through a Python API. \u{1F917} Hub comes preinstalled with \u{1F917} Transformers, so we can use it directly. For example, we can use the "),ja=i("code"),_r=a("list_datasets()"),br=a(" function to get information about all the public datasets currently hosted on the Hub:"),el=p(),g(Ge.$$.fragment),tl=p(),g(Ie.$$.fragment),al=p(),As=i("p"),vr=a("We can see that there are currently nearly 1,500 datasets on the Hub, and the "),$a=i("code"),wr=a("list_datasets()"),xr=a(" function also provides some basic metadata about each dataset repository."),nl=p(),Ns=i("p"),yr=a("For our purposes, the first thing we need to do is create a new dataset repository on the Hub. To do that we need an authentication token, which can be obtained by first logging into the Hugging Face Hub with the "),Ea=i("code"),jr=a("notebook_login()"),$r=a(" function:"),ll=p(),g(Re.$$.fragment),il=p(),Cs=i("p"),Er=a("This will create a widget where you can enter your username and password, and an API token will be saved in "),ka=i("em"),kr=a("~/.huggingface/token"),qr=a(". If you\u2019re running the code in a terminal, you can log in via the CLI instead:"),ol=p(),g(Se.$$.fragment),rl=p(),Gs=i("p"),Dr=a("Once we\u2019ve done this, we can create a new dataset repository with the "),qa=i("code"),Tr=a("create_repo()"),Hr=a(" function:"),ul=p(),g(Fe.$$.fragment),pl=p(),g(Le.$$.fragment),hl=p(),as=i("p"),Or=a("In this example, we\u2019ve created an empty dataset repository called "),Da=i("code"),Pr=a("github-issues"),Ar=a(" under the "),Ta=i("code"),Nr=a("lewtun"),Cr=a(" username (the username should be your Hub username when you\u2019re running this code!)."),cl=p(),g(Is.$$.fragment),dl=p(),Rs=i("p"),Gr=a("Next, let\u2019s clone the repository from the Hub to our local machine and copy our dataset file into it. \u{1F917} Hub provides a handy "),Ha=i("code"),Ir=a("Repository"),Rr=a(" class that wraps many of the common Git commands, so to clone the remote repository we simply need to provide the URL and local path we wish to clone to:"),fl=p(),g(Ue.$$.fragment),ml=p(),M=i("p"),Sr=a("By default, various file extensions (such as "),Oa=i("em"),Fr=a(".bin"),Lr=a(", "),Pa=i("em"),Ur=a(".gz"),Mr=a(", and "),Aa=i("em"),zr=a(".zip"),Wr=a(") are tracked with Git LFS so that large files can be versioned within the same Git workflow. You can find a list of tracked file extensions inside the repository\u2019s "),Na=i("em"),Br=a(".gitattributes"),Yr=a(" file. To include the JSON Lines format in the list, we can run the following command:"),gl=p(),g(Me.$$.fragment),_l=p(),Ss=i("p"),Qr=a("Then we can use "),Ca=i("code"),Jr=a("Repository.push_to_hub()"),Zr=a(" to push the dataset to the Hub:"),bl=p(),g(ze.$$.fragment),vl=p(),Fs=i("p"),Kr=a("If we navigate to the URL contained in "),Ga=i("code"),Vr=a("repo_url"),Xr=a(", we should now see that our dataset file has been uploaded."),wl=p(),We=i("div"),Be=i("img"),xl=p(),ns=i("p"),su=a("From here, anyone can download the dataset by simply providing "),Ia=i("code"),eu=a("load_dataset()"),tu=a(" with the repository ID as the "),Ra=i("code"),au=a("path"),nu=a(" argument:"),yl=p(),g(Ye.$$.fragment),jl=p(),g(Qe.$$.fragment),$l=p(),Ls=i("p"),lu=a("Cool, we\u2019ve pushed our dataset to the Hub and it\u2019s available for others to use! There\u2019s just one important thing left to do: adding a "),Sa=i("em"),iu=a("dataset card"),ou=a(" that explains how the corpus was created and provides other useful information for the community."),El=p(),g(Us.$$.fragment),kl=p(),ms=i("h2"),Ms=i("a"),Fa=i("span"),g(Je.$$.fragment),ru=p(),La=i("span"),uu=a("Creating a dataset card"),ql=p(),xt=i("p"),pu=a("Well-documented datasets are more likely to be useful to others (including your future self!), as they provide the context to enable users to decide whether the dataset is relevant to their task and to evaluate any potential biases in or risks associated with using the dataset."),Dl=p(),zs=i("p"),hu=a("On the Hugging Face Hub, this information is stored in each dataset repository\u2019s "),Ua=i("em"),cu=a("README.md"),du=a(" file. There are two main steps you should take before creating this file:"),Tl=p(),yt=i("ol"),gs=i("li"),fu=a("Use the "),Ws=i("a"),Ma=i("code"),mu=a("datasets-tagging"),gu=a(" application"),_u=a(" to create metadata tags in YAML format. These tags are used for a variety of search features on the Hugging Face Hub and ensure your dataset can be easily found by members of the community. Since we have created a custom dataset here, you\u2019ll need to clone the "),za=i("code"),bu=a("datasets-tagging"),vu=a(" repository and run the application locally. Here\u2019s what the interface looks like:"),Hl=p(),Ze=i("div"),Ke=i("img"),Ol=p(),Ve=i("ol"),Xe=i("li"),wu=a("Read the "),st=i("a"),xu=a("\u{1F917} Datasets guide"),yu=a(" on creating informative dataset cards and use it as a template."),Pl=p(),ls=i("p"),ju=a("You can create the "),Wa=i("em"),$u=a("README.md"),Eu=a(" file directly on the Hub, and you can find a template dataset card in the "),Ba=i("code"),ku=a("lewtun/github-issues"),qu=a(" dataset repository. A screenshot of the filled-out dataset card is shown below."),Al=p(),et=i("div"),tt=i("img"),Nl=p(),g(Bs.$$.fragment),Cl=p(),jt=i("p"),Du=a("That\u2019s it! We\u2019ve seen in this section that creating a good dataset can be quite involved, but fortunately uploading it and sharing it with the community is not. In the next section we\u2019ll use our new dataset to create a semantic search engine with \u{1F917} Datasets that can match questions to the most relevant issues and comments."),Gl=p(),g(Ys.$$.fragment),this.h()},l(s){const l=Nh('[data-svelte="svelte-1phssyn"]',document.head);c=o(l,"META",{name:!0,content:!0}),l.forEach(t),D=h(s),f=o(s,"H1",{class:!0});var at=r(f);$=o(at,"A",{id:!0,class:!0,href:!0});var Ya=r($);k=o(Ya,"SPAN",{});var Qa=r(k);_(m.$$.fragment,Qa),Qa.forEach(t),Ya.forEach(t),E=h(at),T=o(at,"SPAN",{});var Ja=r(T);y=n(Ja,"Creating your own dataset"),Ja.forEach(t),at.forEach(t),j=h(s),_(A.$$.fragment,s),H=h(s),O=o(s,"P",{});var nt=r(O);C=n(nt,"Sometimes the dataset that you need to build an NLP application doesn\u2019t exist, so you\u2019ll need to create it yourself. In this section we\u2019ll show you how to create a corpus of "),q=o(nt,"A",{href:!0,rel:!0});var Za=r(q);N=n(Za,"GitHub issues"),Za.forEach(t),W=n(nt,", which are commonly used to track bugs or features in GitHub repositories. This corpus could be used for various purposes, including:"),nt.forEach(t),S=h(s),G=o(s,"UL",{});var _s=r(G);B=o(_s,"LI",{});var Ru=r(B);us=n(Ru,"Exploring how long it takes to close open issues or pull requests"),Ru.forEach(t),I=h(_s),Z=o(_s,"LI",{});var Rl=r(Z);it=n(Rl,"Training a "),bs=o(Rl,"EM",{});var Su=r(bs);ot=n(Su,"multilabel classifier"),Su.forEach(t),rt=n(Rl," that can tag issues with metadata based on the issue\u2019s description (e.g., \u201Cbug,\u201D \u201Cenhancement,\u201D or \u201Cquestion\u201D)"),Rl.forEach(t),ut=h(_s),It=o(_s,"LI",{});var Fu=r(It);li=n(Fu,"Creating a semantic search engine to find which issues match a user\u2019s query"),Fu.forEach(t),_s.forEach(t),Ka=h(s),pt=o(s,"P",{});var Lu=r(pt);ii=n(Lu,"Here we\u2019ll focus on creating the corpus, and in the next section we\u2019ll tackle the semantic search application. To keep things meta, we\u2019ll use the GitHub issues associated with a popular open source project: \u{1F917} Datasets! Let\u2019s take a look at how to get the data and explore the information contained in these issues."),Lu.forEach(t),Va=h(s),ps=o(s,"H2",{class:!0});var Sl=r(ps);vs=o(Sl,"A",{id:!0,class:!0,href:!0});var Uu=r(vs);Rt=o(Uu,"SPAN",{});var Mu=r(Rt);_(Ks.$$.fragment,Mu),Mu.forEach(t),Uu.forEach(t),oi=h(Sl),St=o(Sl,"SPAN",{});var zu=r(St);ri=n(zu,"Getting the data"),zu.forEach(t),Sl.forEach(t),Xa=h(s),ws=o(s,"P",{});var Fl=r(ws);ui=n(Fl,"You can find all the issues in \u{1F917} Datasets by navigating to the repository\u2019s "),Vs=o(Fl,"A",{href:!0,rel:!0});var Wu=r(Vs);pi=n(Wu,"Issues tab"),Wu.forEach(t),hi=n(Fl,". As shown in the following screenshot, at the time of writing there were 331 open issues and 668 closed ones."),Fl.forEach(t),sn=h(s),Xs=o(s,"DIV",{class:!0});var Bu=r(Xs);se=o(Bu,"IMG",{src:!0,alt:!0,width:!0}),Bu.forEach(t),en=h(s),ht=o(s,"P",{});var Yu=r(ht);ci=n(Yu,"If you click on one of these issues you\u2019ll find it contains a title, a description, and a set of labels that characterize the issue. An example is shown in the screenshot below."),Yu.forEach(t),tn=h(s),ee=o(s,"DIV",{class:!0});var Qu=r(ee);te=o(Qu,"IMG",{src:!0,alt:!0,width:!0}),Qu.forEach(t),an=h(s),K=o(s,"P",{});var $t=r(K);di=n($t,"To download all the repository\u2019s issues, we\u2019ll use the "),ae=o($t,"A",{href:!0,rel:!0});var Ju=r(ae);fi=n(Ju,"GitHub REST API"),Ju.forEach(t),mi=n($t," to poll the "),xs=o($t,"A",{href:!0,rel:!0});var Tu=r(xs);Ft=o(Tu,"CODE",{});var Zu=r(Ft);gi=n(Zu,"Issues"),Zu.forEach(t),_i=n(Tu," endpoint"),Tu.forEach(t),bi=n($t,". This endpoint returns a list of JSON objects, with each object containing a large number of fields that include the title and description as well as metadata about the status of the issue and so on."),$t.forEach(t),nn=h(s),ys=o(s,"P",{});var Ll=r(ys);vi=n(Ll,"A convenient way to download the issues is via the "),Lt=o(Ll,"CODE",{});var Ku=r(Lt);wi=n(Ku,"requests"),Ku.forEach(t),xi=n(Ll," library, which is the standard way for making HTTP requests in Python. You can install the library by running:"),Ll.forEach(t),ln=h(s),_(ne.$$.fragment,s),on=h(s),V=o(s,"P",{});var Et=r(V);yi=n(Et,"Once the library is installed, you can make GET requests to the "),Ut=o(Et,"CODE",{});var Vu=r(Ut);ji=n(Vu,"Issues"),Vu.forEach(t),$i=n(Et," endpoint by invoking the "),Mt=o(Et,"CODE",{});var Xu=r(Mt);Ei=n(Xu,"requests.get()"),Xu.forEach(t),ki=n(Et," function. For example, you can run the following command to retrieve the first issue on the first page:"),Et.forEach(t),rn=h(s),_(le.$$.fragment,s),un=h(s),js=o(s,"P",{});var Ul=r(js);qi=n(Ul,"The "),zt=o(Ul,"CODE",{});var sp=r(zt);Di=n(sp,"response"),sp.forEach(t),Ti=n(Ul," object contains a lot of useful information about the request, including the HTTP status code:"),Ul.forEach(t),pn=h(s),_(ie.$$.fragment,s),hn=h(s),_(oe.$$.fragment,s),cn=h(s),Y=o(s,"P",{});var Qs=r(Y);Hi=n(Qs,"where a "),Wt=o(Qs,"CODE",{});var ep=r(Wt);Oi=n(ep,"200"),ep.forEach(t),Pi=n(Qs," status means the request was successful (you can find a list of possible HTTP status codes "),re=o(Qs,"A",{href:!0,rel:!0});var tp=r(re);Ai=n(tp,"here"),tp.forEach(t),Ni=n(Qs,"). What we are really interested in, though, is the "),Bt=o(Qs,"EM",{});var ap=r(Bt);Ci=n(ap,"payload"),ap.forEach(t),Gi=n(Qs,", which can be accessed in various formats like bytes, strings, or JSON. Since we know our issues are in JSON format, let\u2019s inspect the payload as follows:"),Qs.forEach(t),dn=h(s),_(ue.$$.fragment,s),fn=h(s),_(pe.$$.fragment,s),mn=h(s),Q=o(s,"P",{});var Js=r(Q);Ii=n(Js,"Whoa, that\u2019s a lot of information! We can see useful fields like "),Yt=o(Js,"CODE",{});var np=r(Yt);Ri=n(np,"title"),np.forEach(t),Si=n(Js,", "),Qt=o(Js,"CODE",{});var lp=r(Qt);Fi=n(lp,"body"),lp.forEach(t),Li=n(Js,", and "),Jt=o(Js,"CODE",{});var ip=r(Jt);Ui=n(ip,"number"),ip.forEach(t),Mi=n(Js," that describe the issue, as well as information about the GitHub user who opened the issue."),Js.forEach(t),gn=h(s),_($s.$$.fragment,s),_n=h(s),L=o(s,"P",{});var is=r(L);zi=n(is,"As described in the GitHub "),he=o(is,"A",{href:!0,rel:!0});var op=r(he);Wi=n(op,"documentation"),op.forEach(t),Bi=n(is,", unauthenticated requests are limited to 60 requests per hour. Although you can increase the "),Zt=o(is,"CODE",{});var rp=r(Zt);Yi=n(rp,"per_page"),rp.forEach(t),Qi=n(is," query parameter to reduce the number of requests you make, you will still hit the rate limit on any repository that has more than a few thousand issues. So instead, you should follow GitHub\u2019s "),ce=o(is,"A",{href:!0,rel:!0});var up=r(ce);Ji=n(up,"instructions"),up.forEach(t),Zi=n(is," on creating a "),Kt=o(is,"EM",{});var pp=r(Kt);Ki=n(pp,"personal access token"),pp.forEach(t),Vi=n(is," so that you can boost the rate limit to 5,000 requests per hour. Once you have your token, you can include it as part of the request header:"),is.forEach(t),bn=h(s),_(de.$$.fragment,s),vn=h(s),_(Es.$$.fragment,s),wn=h(s),ct=o(s,"P",{});var hp=r(ct);Xi=n(hp,"Now that we have our access token, let\u2019s create a function that can download all the issues from a GitHub repository:"),hp.forEach(t),xn=h(s),_(fe.$$.fragment,s),yn=h(s),X=o(s,"P",{});var kt=r(X);so=n(kt,"Now when we call "),Vt=o(kt,"CODE",{});var cp=r(Vt);eo=n(cp,"fetch_issues()"),cp.forEach(t),to=n(kt," it will download all the issues in batches to avoid exceeding GitHub\u2019s limit on the number of requests per hour; the result will be stored in a "),Xt=o(kt,"EM",{});var dp=r(Xt);ao=n(dp,"repository_name-issues.jsonl"),dp.forEach(t),no=n(kt," file, where each line is a JSON object the represents an issue. Let\u2019s use this function to grab all the issues from \u{1F917} Datasets:"),kt.forEach(t),jn=h(s),_(me.$$.fragment,s),$n=h(s),ks=o(s,"P",{});var Ml=r(ks);lo=n(Ml,"Once the issues are downloaded we can load them locally using our newfound skills from "),dt=o(Ml,"A",{href:!0});var fp=r(dt);io=n(fp,"section 2"),fp.forEach(t),oo=n(Ml,":"),Ml.forEach(t),En=h(s),_(ge.$$.fragment,s),kn=h(s),_(_e.$$.fragment,s),qn=h(s),ss=o(s,"P",{});var qt=r(ss);ro=n(qt,"Great, we\u2019ve created our first dataset from scratch! But why are there several thousand issues when the "),be=o(qt,"A",{href:!0,rel:!0});var mp=r(be);uo=n(mp,"Issues tab"),mp.forEach(t),po=n(qt," of the \u{1F917} Datasets repository only shows around 1,000 issues in total \u{1F914}? As described in the GitHub "),ve=o(qt,"A",{href:!0,rel:!0});var gp=r(ve);ho=n(gp,"documentation"),gp.forEach(t),co=n(qt,", that\u2019s because we\u2019ve downloaded all the pull requests as well:"),qt.forEach(t),Dn=h(s),ft=o(s,"BLOCKQUOTE",{});var _p=r(ft);hs=o(_p,"P",{});var Dt=r(hs);fo=n(Dt,"GitHub\u2019s REST API v3 considers every pull request an issue, but not every issue is a pull request. For this reason, \u201CIssues\u201D endpoints may return both issues and pull requests in the response. You can identify pull requests by the "),sa=o(Dt,"CODE",{});var bp=r(sa);mo=n(bp,"pull_request"),bp.forEach(t),go=n(Dt," key. Be aware that the "),ea=o(Dt,"CODE",{});var vp=r(ea);_o=n(vp,"id"),vp.forEach(t),bo=n(Dt," of a pull request returned from \u201CIssues\u201D endpoints will be an issue id."),Dt.forEach(t),_p.forEach(t),Tn=h(s),mt=o(s,"P",{});var wp=r(mt);vo=n(wp,"Since the contents of issues and pull requests are quite different, let\u2019s do some minor preprocessing to enable us to distinguish between them."),wp.forEach(t),Hn=h(s),cs=o(s,"H2",{class:!0});var zl=r(cs);qs=o(zl,"A",{id:!0,class:!0,href:!0});var xp=r(qs);ta=o(xp,"SPAN",{});var yp=r(ta);_(we.$$.fragment,yp),yp.forEach(t),xp.forEach(t),wo=h(zl),aa=o(zl,"SPAN",{});var jp=r(aa);xo=n(jp,"Cleaning up the data"),jp.forEach(t),zl.forEach(t),On=h(s),R=o(s,"P",{});var z=r(R);yo=n(z,"The above snippet from GitHub\u2019s documentation tells us that the "),na=o(z,"CODE",{});var $p=r(na);jo=n($p,"pull_request"),$p.forEach(t),$o=n(z," column can be used to differentiate between issues and pull requests. Let\u2019s look at a random sample to see what the difference is. As we did in "),gt=o(z,"A",{href:!0});var Ep=r(gt);Eo=n(Ep,"section 3"),Ep.forEach(t),ko=n(z,", we\u2019ll chain "),la=o(z,"CODE",{});var kp=r(la);qo=n(kp,"Dataset.shuffle()"),kp.forEach(t),Do=n(z," and "),ia=o(z,"CODE",{});var qp=r(ia);To=n(qp,"Dataset.select()"),qp.forEach(t),Ho=n(z," to create a random sample and then zip the "),oa=o(z,"CODE",{});var Dp=r(oa);Oo=n(Dp,"html_url"),Dp.forEach(t),Po=n(z," and "),ra=o(z,"CODE",{});var Tp=r(ra);Ao=n(Tp,"pull_request"),Tp.forEach(t),No=n(z," columns so we can compare the various URLs:"),z.forEach(t),Pn=h(s),_(xe.$$.fragment,s),An=h(s),_(ye.$$.fragment,s),Nn=h(s),U=o(s,"P",{});var os=r(U);Co=n(os,"Here we can see that each pull request is associated with various URLs, while ordinary issues have a "),ua=o(os,"CODE",{});var Hp=r(ua);Go=n(Hp,"None"),Hp.forEach(t),Io=n(os," entry. We can use this distinction to create a new "),pa=o(os,"CODE",{});var Op=r(pa);Ro=n(Op,"is_pull_request"),Op.forEach(t),So=n(os," column that checks whether the "),ha=o(os,"CODE",{});var Pp=r(ha);Fo=n(Pp,"pull_request"),Pp.forEach(t),Lo=n(os," field is "),ca=o(os,"CODE",{});var Ap=r(ca);Uo=n(Ap,"None"),Ap.forEach(t),Mo=n(os," or not:"),os.forEach(t),Cn=h(s),_(je.$$.fragment,s),Gn=h(s),_(Ds.$$.fragment,s),In=h(s),_t=o(s,"P",{});var Np=r(_t);zo=n(Np,"Although we could proceed to further clean up the dataset by dropping or renaming some columns, it is generally a good practice to keep the dataset as \u201Craw\u201D as possible at this stage so that it can be easily used in multiple applications."),Np.forEach(t),Rn=h(s),bt=o(s,"P",{});var Cp=r(bt);Wo=n(Cp,"Before we push our dataset to the Hugging Face Hub, let\u2019s deal with one thing that\u2019s missing from it: the comments associated with each issue and pull request. We\u2019ll add them next with \u2014 you guessed it \u2014 the GitHub REST API!"),Cp.forEach(t),Sn=h(s),ds=o(s,"H2",{class:!0});var Wl=r(ds);Ts=o(Wl,"A",{id:!0,class:!0,href:!0});var Gp=r(Ts);da=o(Gp,"SPAN",{});var Ip=r(da);_($e.$$.fragment,Ip),Ip.forEach(t),Gp.forEach(t),Bo=h(Wl),fa=o(Wl,"SPAN",{});var Rp=r(fa);Yo=n(Rp,"Augmenting the dataset"),Rp.forEach(t),Wl.forEach(t),Fn=h(s),vt=o(s,"P",{});var Sp=r(vt);Qo=n(Sp,"As shown in the following screenshot, the comments associated with an issue or pull request provide a rich source of information, especially if we\u2019re interested in building a search engine to answer user queries about the library."),Sp.forEach(t),Ln=h(s),Ee=o(s,"DIV",{class:!0});var Fp=r(Ee);ke=o(Fp,"IMG",{src:!0,alt:!0,width:!0}),Fp.forEach(t),Un=h(s),Hs=o(s,"P",{});var Bl=r(Hs);Jo=n(Bl,"The GitHub REST API provides a "),Os=o(Bl,"A",{href:!0,rel:!0});var Hu=r(Os);ma=o(Hu,"CODE",{});var Lp=r(ma);Zo=n(Lp,"Comments"),Lp.forEach(t),Ko=n(Hu," endpoint"),Hu.forEach(t),Vo=n(Bl," that returns all the comments associated with an issue number. Let\u2019s test the endpoint to see what it returns:"),Bl.forEach(t),Mn=h(s),_(qe.$$.fragment,s),zn=h(s),_(De.$$.fragment,s),Wn=h(s),J=o(s,"P",{});var Zs=r(J);Xo=n(Zs,"We can see that the comment is stored in the "),ga=o(Zs,"CODE",{});var Up=r(ga);sr=n(Up,"body"),Up.forEach(t),er=n(Zs," field, so let\u2019s write a simple function that returns all the comments associated with an issue by picking out the "),_a=o(Zs,"CODE",{});var Mp=r(_a);tr=n(Mp,"body"),Mp.forEach(t),ar=n(Zs," contents for each element in "),ba=o(Zs,"CODE",{});var zp=r(ba);nr=n(zp,"response.json()"),zp.forEach(t),lr=n(Zs,":"),Zs.forEach(t),Bn=h(s),_(Te.$$.fragment,s),Yn=h(s),_(He.$$.fragment,s),Qn=h(s),es=o(s,"P",{});var Tt=r(es);ir=n(Tt,"This looks good, so let\u2019s use "),va=o(Tt,"CODE",{});var Wp=r(va);or=n(Wp,"Dataset.map()"),Wp.forEach(t),rr=n(Tt," to add a new "),wa=o(Tt,"CODE",{});var Bp=r(wa);ur=n(Bp,"comments"),Bp.forEach(t),pr=n(Tt," column to each issue in our dataset:"),Tt.forEach(t),Jn=h(s),_(Oe.$$.fragment,s),Zn=h(s),wt=o(s,"P",{});var Yp=r(wt);hr=n(Yp,"The final step is to save the augmented dataset alongside our raw data so we can push them both to the Hub:"),Yp.forEach(t),Kn=h(s),_(Pe.$$.fragment,s),Vn=h(s),fs=o(s,"H2",{class:!0});var Yl=r(fs);Ps=o(Yl,"A",{id:!0,class:!0,href:!0});var Qp=r(Ps);xa=o(Qp,"SPAN",{});var Jp=r(xa);_(Ae.$$.fragment,Jp),Jp.forEach(t),Qp.forEach(t),cr=h(Yl),ya=o(Yl,"SPAN",{});var Zp=r(ya);dr=n(Zp,"Uploading the dataset to the Hugging Face Hub"),Zp.forEach(t),Yl.forEach(t),Xn=h(s),_(Ne.$$.fragment,s),sl=h(s),ts=o(s,"P",{});var Ht=r(ts);fr=n(Ht,"Now that we have our augmented dataset, it\u2019s time to push it to the Hub so we can share it with the community! To upload the dataset we\u2019ll use the "),Ce=o(Ht,"A",{href:!0,rel:!0});var Kp=r(Ce);mr=n(Kp,"\u{1F917} Hub library"),Kp.forEach(t),gr=n(Ht,", which allows us to interact with the Hugging Face Hub through a Python API. \u{1F917} Hub comes preinstalled with \u{1F917} Transformers, so we can use it directly. For example, we can use the "),ja=o(Ht,"CODE",{});var Vp=r(ja);_r=n(Vp,"list_datasets()"),Vp.forEach(t),br=n(Ht," function to get information about all the public datasets currently hosted on the Hub:"),Ht.forEach(t),el=h(s),_(Ge.$$.fragment,s),tl=h(s),_(Ie.$$.fragment,s),al=h(s),As=o(s,"P",{});var Ql=r(As);vr=n(Ql,"We can see that there are currently nearly 1,500 datasets on the Hub, and the "),$a=o(Ql,"CODE",{});var Xp=r($a);wr=n(Xp,"list_datasets()"),Xp.forEach(t),xr=n(Ql," function also provides some basic metadata about each dataset repository."),Ql.forEach(t),nl=h(s),Ns=o(s,"P",{});var Jl=r(Ns);yr=n(Jl,"For our purposes, the first thing we need to do is create a new dataset repository on the Hub. To do that we need an authentication token, which can be obtained by first logging into the Hugging Face Hub with the "),Ea=o(Jl,"CODE",{});var sh=r(Ea);jr=n(sh,"notebook_login()"),sh.forEach(t),$r=n(Jl," function:"),Jl.forEach(t),ll=h(s),_(Re.$$.fragment,s),il=h(s),Cs=o(s,"P",{});var Zl=r(Cs);Er=n(Zl,"This will create a widget where you can enter your username and password, and an API token will be saved in "),ka=o(Zl,"EM",{});var eh=r(ka);kr=n(eh,"~/.huggingface/token"),eh.forEach(t),qr=n(Zl,". If you\u2019re running the code in a terminal, you can log in via the CLI instead:"),Zl.forEach(t),ol=h(s),_(Se.$$.fragment,s),rl=h(s),Gs=o(s,"P",{});var Kl=r(Gs);Dr=n(Kl,"Once we\u2019ve done this, we can create a new dataset repository with the "),qa=o(Kl,"CODE",{});var th=r(qa);Tr=n(th,"create_repo()"),th.forEach(t),Hr=n(Kl," function:"),Kl.forEach(t),ul=h(s),_(Fe.$$.fragment,s),pl=h(s),_(Le.$$.fragment,s),hl=h(s),as=o(s,"P",{});var Ot=r(as);Or=n(Ot,"In this example, we\u2019ve created an empty dataset repository called "),Da=o(Ot,"CODE",{});var ah=r(Da);Pr=n(ah,"github-issues"),ah.forEach(t),Ar=n(Ot," under the "),Ta=o(Ot,"CODE",{});var nh=r(Ta);Nr=n(nh,"lewtun"),nh.forEach(t),Cr=n(Ot," username (the username should be your Hub username when you\u2019re running this code!)."),Ot.forEach(t),cl=h(s),_(Is.$$.fragment,s),dl=h(s),Rs=o(s,"P",{});var Vl=r(Rs);Gr=n(Vl,"Next, let\u2019s clone the repository from the Hub to our local machine and copy our dataset file into it. \u{1F917} Hub provides a handy "),Ha=o(Vl,"CODE",{});var lh=r(Ha);Ir=n(lh,"Repository"),lh.forEach(t),Rr=n(Vl," class that wraps many of the common Git commands, so to clone the remote repository we simply need to provide the URL and local path we wish to clone to:"),Vl.forEach(t),fl=h(s),_(Ue.$$.fragment,s),ml=h(s),M=o(s,"P",{});var rs=r(M);Sr=n(rs,"By default, various file extensions (such as "),Oa=o(rs,"EM",{});var ih=r(Oa);Fr=n(ih,".bin"),ih.forEach(t),Lr=n(rs,", "),Pa=o(rs,"EM",{});var oh=r(Pa);Ur=n(oh,".gz"),oh.forEach(t),Mr=n(rs,", and "),Aa=o(rs,"EM",{});var rh=r(Aa);zr=n(rh,".zip"),rh.forEach(t),Wr=n(rs,") are tracked with Git LFS so that large files can be versioned within the same Git workflow. You can find a list of tracked file extensions inside the repository\u2019s "),Na=o(rs,"EM",{});var uh=r(Na);Br=n(uh,".gitattributes"),uh.forEach(t),Yr=n(rs," file. To include the JSON Lines format in the list, we can run the following command:"),rs.forEach(t),gl=h(s),_(Me.$$.fragment,s),_l=h(s),Ss=o(s,"P",{});var Xl=r(Ss);Qr=n(Xl,"Then we can use "),Ca=o(Xl,"CODE",{});var ph=r(Ca);Jr=n(ph,"Repository.push_to_hub()"),ph.forEach(t),Zr=n(Xl," to push the dataset to the Hub:"),Xl.forEach(t),bl=h(s),_(ze.$$.fragment,s),vl=h(s),Fs=o(s,"P",{});var si=r(Fs);Kr=n(si,"If we navigate to the URL contained in "),Ga=o(si,"CODE",{});var hh=r(Ga);Vr=n(hh,"repo_url"),hh.forEach(t),Xr=n(si,", we should now see that our dataset file has been uploaded."),si.forEach(t),wl=h(s),We=o(s,"DIV",{class:!0});var ch=r(We);Be=o(ch,"IMG",{src:!0,alt:!0,width:!0}),ch.forEach(t),xl=h(s),ns=o(s,"P",{});var Pt=r(ns);su=n(Pt,"From here, anyone can download the dataset by simply providing "),Ia=o(Pt,"CODE",{});var dh=r(Ia);eu=n(dh,"load_dataset()"),dh.forEach(t),tu=n(Pt," with the repository ID as the "),Ra=o(Pt,"CODE",{});var fh=r(Ra);au=n(fh,"path"),fh.forEach(t),nu=n(Pt," argument:"),Pt.forEach(t),yl=h(s),_(Ye.$$.fragment,s),jl=h(s),_(Qe.$$.fragment,s),$l=h(s),Ls=o(s,"P",{});var ei=r(Ls);lu=n(ei,"Cool, we\u2019ve pushed our dataset to the Hub and it\u2019s available for others to use! There\u2019s just one important thing left to do: adding a "),Sa=o(ei,"EM",{});var mh=r(Sa);iu=n(mh,"dataset card"),mh.forEach(t),ou=n(ei," that explains how the corpus was created and provides other useful information for the community."),ei.forEach(t),El=h(s),_(Us.$$.fragment,s),kl=h(s),ms=o(s,"H2",{class:!0});var ti=r(ms);Ms=o(ti,"A",{id:!0,class:!0,href:!0});var gh=r(Ms);Fa=o(gh,"SPAN",{});var _h=r(Fa);_(Je.$$.fragment,_h),_h.forEach(t),gh.forEach(t),ru=h(ti),La=o(ti,"SPAN",{});var bh=r(La);uu=n(bh,"Creating a dataset card"),bh.forEach(t),ti.forEach(t),ql=h(s),xt=o(s,"P",{});var vh=r(xt);pu=n(vh,"Well-documented datasets are more likely to be useful to others (including your future self!), as they provide the context to enable users to decide whether the dataset is relevant to their task and to evaluate any potential biases in or risks associated with using the dataset."),vh.forEach(t),Dl=h(s),zs=o(s,"P",{});var ai=r(zs);hu=n(ai,"On the Hugging Face Hub, this information is stored in each dataset repository\u2019s "),Ua=o(ai,"EM",{});var wh=r(Ua);cu=n(wh,"README.md"),wh.forEach(t),du=n(ai," file. There are two main steps you should take before creating this file:"),ai.forEach(t),Tl=h(s),yt=o(s,"OL",{});var xh=r(yt);gs=o(xh,"LI",{});var At=r(gs);fu=n(At,"Use the "),Ws=o(At,"A",{href:!0,rel:!0});var Ou=r(Ws);Ma=o(Ou,"CODE",{});var yh=r(Ma);mu=n(yh,"datasets-tagging"),yh.forEach(t),gu=n(Ou," application"),Ou.forEach(t),_u=n(At," to create metadata tags in YAML format. These tags are used for a variety of search features on the Hugging Face Hub and ensure your dataset can be easily found by members of the community. Since we have created a custom dataset here, you\u2019ll need to clone the "),za=o(At,"CODE",{});var jh=r(za);bu=n(jh,"datasets-tagging"),jh.forEach(t),vu=n(At," repository and run the application locally. Here\u2019s what the interface looks like:"),At.forEach(t),xh.forEach(t),Hl=h(s),Ze=o(s,"DIV",{class:!0});var $h=r(Ze);Ke=o($h,"IMG",{src:!0,alt:!0,width:!0}),$h.forEach(t),Ol=h(s),Ve=o(s,"OL",{start:!0});var Eh=r(Ve);Xe=o(Eh,"LI",{});var ni=r(Xe);wu=n(ni,"Read the "),st=o(ni,"A",{href:!0,rel:!0});var kh=r(st);xu=n(kh,"\u{1F917} Datasets guide"),kh.forEach(t),yu=n(ni," on creating informative dataset cards and use it as a template."),ni.forEach(t),Eh.forEach(t),Pl=h(s),ls=o(s,"P",{});var Nt=r(ls);ju=n(Nt,"You can create the "),Wa=o(Nt,"EM",{});var qh=r(Wa);$u=n(qh,"README.md"),qh.forEach(t),Eu=n(Nt," file directly on the Hub, and you can find a template dataset card in the "),Ba=o(Nt,"CODE",{});var Dh=r(Ba);ku=n(Dh,"lewtun/github-issues"),Dh.forEach(t),qu=n(Nt," dataset repository. A screenshot of the filled-out dataset card is shown below."),Nt.forEach(t),Al=h(s),et=o(s,"DIV",{class:!0});var Th=r(et);tt=o(Th,"IMG",{src:!0,alt:!0,width:!0}),Th.forEach(t),Nl=h(s),_(Bs.$$.fragment,s),Cl=h(s),jt=o(s,"P",{});var Hh=r(jt);Du=n(Hh,"That\u2019s it! We\u2019ve seen in this section that creating a good dataset can be quite involved, but fortunately uploading it and sharing it with the community is not. In the next section we\u2019ll use our new dataset to create a semantic search engine with \u{1F917} Datasets that can match questions to the most relevant issues and comments."),Hh.forEach(t),Gl=h(s),_(Ys.$$.fragment,s),this.h()},h(){d(c,"name","hf:doc:metadata"),d(c,"content",JSON.stringify(Bh)),d($,"id","creating-your-own-dataset"),d($,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),d($,"href","#creating-your-own-dataset"),d(f,"class","relative group"),d(q,"href","https://github.com/features/issues/"),d(q,"rel","nofollow"),d(vs,"id","getting-the-data"),d(vs,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),d(vs,"href","#getting-the-data"),d(ps,"class","relative group"),d(Vs,"href","https://github.com/huggingface/datasets/issues"),d(Vs,"rel","nofollow"),Ct(se.src,Pu="https://huggingface.co/datasets/huggingface-course/documentation-images/resolve/main/en/chapter5/datasets-issues.png")||d(se,"src",Pu),d(se,"alt","The GitHub issues associated with \u{1F917} Datasets."),d(se,"width","80%"),d(Xs,"class","flex justify-center"),Ct(te.src,Au="https://huggingface.co/datasets/huggingface-course/documentation-images/resolve/main/en/chapter5/datasets-issues-single.png")||d(te,"src",Au),d(te,"alt","A typical GitHub issue in the \u{1F917} Datasets repository."),d(te,"width","80%"),d(ee,"class","flex justify-center"),d(ae,"href","https://docs.github.com/en/rest"),d(ae,"rel","nofollow"),d(xs,"href","https://docs.github.com/en/rest/reference/issues#list-repository-issues"),d(xs,"rel","nofollow"),d(re,"href","https://en.wikipedia.org/wiki/List_of_HTTP_status_codes"),d(re,"rel","nofollow"),d(he,"href","https://docs.github.com/en/rest/overview/resources-in-the-rest-api#rate-limiting"),d(he,"rel","nofollow"),d(ce,"href","https://docs.github.com/en/github/authenticating-to-github/creating-a-personal-access-token"),d(ce,"rel","nofollow"),d(dt,"href","/course/chaper5/2"),d(be,"href","https://github.com/huggingface/datasets/issues"),d(be,"rel","nofollow"),d(ve,"href","https://docs.github.com/en/rest/reference/issues#list-issues-assigned-to-the-authenticated-user"),d(ve,"rel","nofollow"),d(qs,"id","cleaning-up-the-data"),d(qs,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),d(qs,"href","#cleaning-up-the-data"),d(cs,"class","relative group"),d(gt,"href","/course/chapter5/3"),d(Ts,"id","augmenting-the-dataset"),d(Ts,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),d(Ts,"href","#augmenting-the-dataset"),d(ds,"class","relative group"),Ct(ke.src,Nu="https://huggingface.co/datasets/huggingface-course/documentation-images/resolve/main/en/chapter5/datasets-issues-comment.png")||d(ke,"src",Nu),d(ke,"alt","Comments associated with an issue about \u{1F917} Datasets."),d(ke,"width","80%"),d(Ee,"class","flex justify-center"),d(Os,"href","https://docs.github.com/en/rest/reference/issues#list-issue-comments"),d(Os,"rel","nofollow"),d(Ps,"id","uploading-the-dataset-to-the-hugging-face-hub"),d(Ps,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),d(Ps,"href","#uploading-the-dataset-to-the-hugging-face-hub"),d(fs,"class","relative group"),d(Ce,"href","https://github.com/huggingface/huggingface_hub"),d(Ce,"rel","nofollow"),Ct(Be.src,Cu="https://huggingface.co/datasets/huggingface-course/documentation-images/resolve/main/en/chapter5/hub-repo.png")||d(Be,"src",Cu),d(Be,"alt","Our dataset repository on the Hugging Face Hub."),d(Be,"width","80%"),d(We,"class","flex justify-center"),d(Ms,"id","creating-a-dataset-card"),d(Ms,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),d(Ms,"href","#creating-a-dataset-card"),d(ms,"class","relative group"),d(Ws,"href","https://huggingface.co/datasets/tagging/"),d(Ws,"rel","nofollow"),Ct(Ke.src,Gu="https://huggingface.co/datasets/huggingface-course/documentation-images/resolve/main/en/chapter5/datasets-tagger.png")||d(Ke,"src",Gu),d(Ke,"alt","The `datasets-tagging` interface."),d(Ke,"width","80%"),d(Ze,"class","flex justify-center"),d(st,"href","https://github.com/huggingface/datasets/blob/master/templates/README_guide.md"),d(st,"rel","nofollow"),d(Ve,"start","2"),Ct(tt.src,Iu="https://huggingface.co/datasets/huggingface-course/documentation-images/resolve/main/en/chapter5/dataset-card.png")||d(tt,"src",Iu),d(tt,"alt","A dataset card."),d(tt,"width","80%"),d(et,"class","flex justify-center")},m(s,l){e(document.head,c),u(s,D,l),u(s,f,l),e(f,$),e($,k),b(m,k,null),e(f,E),e(f,T),e(T,y),u(s,j,l),b(A,s,l),u(s,H,l),u(s,O,l),e(O,C),e(O,q),e(q,N),e(O,W),u(s,S,l),u(s,G,l),e(G,B),e(B,us),e(G,I),e(G,Z),e(Z,it),e(Z,bs),e(bs,ot),e(Z,rt),e(G,ut),e(G,It),e(It,li),u(s,Ka,l),u(s,pt,l),e(pt,ii),u(s,Va,l),u(s,ps,l),e(ps,vs),e(vs,Rt),b(Ks,Rt,null),e(ps,oi),e(ps,St),e(St,ri),u(s,Xa,l),u(s,ws,l),e(ws,ui),e(ws,Vs),e(Vs,pi),e(ws,hi),u(s,sn,l),u(s,Xs,l),e(Xs,se),u(s,en,l),u(s,ht,l),e(ht,ci),u(s,tn,l),u(s,ee,l),e(ee,te),u(s,an,l),u(s,K,l),e(K,di),e(K,ae),e(ae,fi),e(K,mi),e(K,xs),e(xs,Ft),e(Ft,gi),e(xs,_i),e(K,bi),u(s,nn,l),u(s,ys,l),e(ys,vi),e(ys,Lt),e(Lt,wi),e(ys,xi),u(s,ln,l),b(ne,s,l),u(s,on,l),u(s,V,l),e(V,yi),e(V,Ut),e(Ut,ji),e(V,$i),e(V,Mt),e(Mt,Ei),e(V,ki),u(s,rn,l),b(le,s,l),u(s,un,l),u(s,js,l),e(js,qi),e(js,zt),e(zt,Di),e(js,Ti),u(s,pn,l),b(ie,s,l),u(s,hn,l),b(oe,s,l),u(s,cn,l),u(s,Y,l),e(Y,Hi),e(Y,Wt),e(Wt,Oi),e(Y,Pi),e(Y,re),e(re,Ai),e(Y,Ni),e(Y,Bt),e(Bt,Ci),e(Y,Gi),u(s,dn,l),b(ue,s,l),u(s,fn,l),b(pe,s,l),u(s,mn,l),u(s,Q,l),e(Q,Ii),e(Q,Yt),e(Yt,Ri),e(Q,Si),e(Q,Qt),e(Qt,Fi),e(Q,Li),e(Q,Jt),e(Jt,Ui),e(Q,Mi),u(s,gn,l),b($s,s,l),u(s,_n,l),u(s,L,l),e(L,zi),e(L,he),e(he,Wi),e(L,Bi),e(L,Zt),e(Zt,Yi),e(L,Qi),e(L,ce),e(ce,Ji),e(L,Zi),e(L,Kt),e(Kt,Ki),e(L,Vi),u(s,bn,l),b(de,s,l),u(s,vn,l),b(Es,s,l),u(s,wn,l),u(s,ct,l),e(ct,Xi),u(s,xn,l),b(fe,s,l),u(s,yn,l),u(s,X,l),e(X,so),e(X,Vt),e(Vt,eo),e(X,to),e(X,Xt),e(Xt,ao),e(X,no),u(s,jn,l),b(me,s,l),u(s,$n,l),u(s,ks,l),e(ks,lo),e(ks,dt),e(dt,io),e(ks,oo),u(s,En,l),b(ge,s,l),u(s,kn,l),b(_e,s,l),u(s,qn,l),u(s,ss,l),e(ss,ro),e(ss,be),e(be,uo),e(ss,po),e(ss,ve),e(ve,ho),e(ss,co),u(s,Dn,l),u(s,ft,l),e(ft,hs),e(hs,fo),e(hs,sa),e(sa,mo),e(hs,go),e(hs,ea),e(ea,_o),e(hs,bo),u(s,Tn,l),u(s,mt,l),e(mt,vo),u(s,Hn,l),u(s,cs,l),e(cs,qs),e(qs,ta),b(we,ta,null),e(cs,wo),e(cs,aa),e(aa,xo),u(s,On,l),u(s,R,l),e(R,yo),e(R,na),e(na,jo),e(R,$o),e(R,gt),e(gt,Eo),e(R,ko),e(R,la),e(la,qo),e(R,Do),e(R,ia),e(ia,To),e(R,Ho),e(R,oa),e(oa,Oo),e(R,Po),e(R,ra),e(ra,Ao),e(R,No),u(s,Pn,l),b(xe,s,l),u(s,An,l),b(ye,s,l),u(s,Nn,l),u(s,U,l),e(U,Co),e(U,ua),e(ua,Go),e(U,Io),e(U,pa),e(pa,Ro),e(U,So),e(U,ha),e(ha,Fo),e(U,Lo),e(U,ca),e(ca,Uo),e(U,Mo),u(s,Cn,l),b(je,s,l),u(s,Gn,l),b(Ds,s,l),u(s,In,l),u(s,_t,l),e(_t,zo),u(s,Rn,l),u(s,bt,l),e(bt,Wo),u(s,Sn,l),u(s,ds,l),e(ds,Ts),e(Ts,da),b($e,da,null),e(ds,Bo),e(ds,fa),e(fa,Yo),u(s,Fn,l),u(s,vt,l),e(vt,Qo),u(s,Ln,l),u(s,Ee,l),e(Ee,ke),u(s,Un,l),u(s,Hs,l),e(Hs,Jo),e(Hs,Os),e(Os,ma),e(ma,Zo),e(Os,Ko),e(Hs,Vo),u(s,Mn,l),b(qe,s,l),u(s,zn,l),b(De,s,l),u(s,Wn,l),u(s,J,l),e(J,Xo),e(J,ga),e(ga,sr),e(J,er),e(J,_a),e(_a,tr),e(J,ar),e(J,ba),e(ba,nr),e(J,lr),u(s,Bn,l),b(Te,s,l),u(s,Yn,l),b(He,s,l),u(s,Qn,l),u(s,es,l),e(es,ir),e(es,va),e(va,or),e(es,rr),e(es,wa),e(wa,ur),e(es,pr),u(s,Jn,l),b(Oe,s,l),u(s,Zn,l),u(s,wt,l),e(wt,hr),u(s,Kn,l),b(Pe,s,l),u(s,Vn,l),u(s,fs,l),e(fs,Ps),e(Ps,xa),b(Ae,xa,null),e(fs,cr),e(fs,ya),e(ya,dr),u(s,Xn,l),b(Ne,s,l),u(s,sl,l),u(s,ts,l),e(ts,fr),e(ts,Ce),e(Ce,mr),e(ts,gr),e(ts,ja),e(ja,_r),e(ts,br),u(s,el,l),b(Ge,s,l),u(s,tl,l),b(Ie,s,l),u(s,al,l),u(s,As,l),e(As,vr),e(As,$a),e($a,wr),e(As,xr),u(s,nl,l),u(s,Ns,l),e(Ns,yr),e(Ns,Ea),e(Ea,jr),e(Ns,$r),u(s,ll,l),b(Re,s,l),u(s,il,l),u(s,Cs,l),e(Cs,Er),e(Cs,ka),e(ka,kr),e(Cs,qr),u(s,ol,l),b(Se,s,l),u(s,rl,l),u(s,Gs,l),e(Gs,Dr),e(Gs,qa),e(qa,Tr),e(Gs,Hr),u(s,ul,l),b(Fe,s,l),u(s,pl,l),b(Le,s,l),u(s,hl,l),u(s,as,l),e(as,Or),e(as,Da),e(Da,Pr),e(as,Ar),e(as,Ta),e(Ta,Nr),e(as,Cr),u(s,cl,l),b(Is,s,l),u(s,dl,l),u(s,Rs,l),e(Rs,Gr),e(Rs,Ha),e(Ha,Ir),e(Rs,Rr),u(s,fl,l),b(Ue,s,l),u(s,ml,l),u(s,M,l),e(M,Sr),e(M,Oa),e(Oa,Fr),e(M,Lr),e(M,Pa),e(Pa,Ur),e(M,Mr),e(M,Aa),e(Aa,zr),e(M,Wr),e(M,Na),e(Na,Br),e(M,Yr),u(s,gl,l),b(Me,s,l),u(s,_l,l),u(s,Ss,l),e(Ss,Qr),e(Ss,Ca),e(Ca,Jr),e(Ss,Zr),u(s,bl,l),b(ze,s,l),u(s,vl,l),u(s,Fs,l),e(Fs,Kr),e(Fs,Ga),e(Ga,Vr),e(Fs,Xr),u(s,wl,l),u(s,We,l),e(We,Be),u(s,xl,l),u(s,ns,l),e(ns,su),e(ns,Ia),e(Ia,eu),e(ns,tu),e(ns,Ra),e(Ra,au),e(ns,nu),u(s,yl,l),b(Ye,s,l),u(s,jl,l),b(Qe,s,l),u(s,$l,l),u(s,Ls,l),e(Ls,lu),e(Ls,Sa),e(Sa,iu),e(Ls,ou),u(s,El,l),b(Us,s,l),u(s,kl,l),u(s,ms,l),e(ms,Ms),e(Ms,Fa),b(Je,Fa,null),e(ms,ru),e(ms,La),e(La,uu),u(s,ql,l),u(s,xt,l),e(xt,pu),u(s,Dl,l),u(s,zs,l),e(zs,hu),e(zs,Ua),e(Ua,cu),e(zs,du),u(s,Tl,l),u(s,yt,l),e(yt,gs),e(gs,fu),e(gs,Ws),e(Ws,Ma),e(Ma,mu),e(Ws,gu),e(gs,_u),e(gs,za),e(za,bu),e(gs,vu),u(s,Hl,l),u(s,Ze,l),e(Ze,Ke),u(s,Ol,l),u(s,Ve,l),e(Ve,Xe),e(Xe,wu),e(Xe,st),e(st,xu),e(Xe,yu),u(s,Pl,l),u(s,ls,l),e(ls,ju),e(ls,Wa),e(Wa,$u),e(ls,Eu),e(ls,Ba),e(Ba,ku),e(ls,qu),u(s,Al,l),u(s,et,l),e(et,tt),u(s,Nl,l),b(Bs,s,l),u(s,Cl,l),u(s,jt,l),e(jt,Du),u(s,Gl,l),b(Ys,s,l),Il=!0},p(s,[l]){const at={};l&2&&(at.$$scope={dirty:l,ctx:s}),$s.$set(at);const Ya={};l&2&&(Ya.$$scope={dirty:l,ctx:s}),Es.$set(Ya);const Qa={};l&2&&(Qa.$$scope={dirty:l,ctx:s}),Ds.$set(Qa);const Ja={};l&2&&(Ja.$$scope={dirty:l,ctx:s}),Is.$set(Ja);const nt={};l&2&&(nt.$$scope={dirty:l,ctx:s}),Us.$set(nt);const Za={};l&2&&(Za.$$scope={dirty:l,ctx:s}),Bs.$set(Za);const _s={};l&2&&(_s.$$scope={dirty:l,ctx:s}),Ys.$set(_s)},i(s){Il||(v(m.$$.fragment,s),v(A.$$.fragment,s),v(Ks.$$.fragment,s),v(ne.$$.fragment,s),v(le.$$.fragment,s),v(ie.$$.fragment,s),v(oe.$$.fragment,s),v(ue.$$.fragment,s),v(pe.$$.fragment,s),v($s.$$.fragment,s),v(de.$$.fragment,s),v(Es.$$.fragment,s),v(fe.$$.fragment,s),v(me.$$.fragment,s),v(ge.$$.fragment,s),v(_e.$$.fragment,s),v(we.$$.fragment,s),v(xe.$$.fragment,s),v(ye.$$.fragment,s),v(je.$$.fragment,s),v(Ds.$$.fragment,s),v($e.$$.fragment,s),v(qe.$$.fragment,s),v(De.$$.fragment,s),v(Te.$$.fragment,s),v(He.$$.fragment,s),v(Oe.$$.fragment,s),v(Pe.$$.fragment,s),v(Ae.$$.fragment,s),v(Ne.$$.fragment,s),v(Ge.$$.fragment,s),v(Ie.$$.fragment,s),v(Re.$$.fragment,s),v(Se.$$.fragment,s),v(Fe.$$.fragment,s),v(Le.$$.fragment,s),v(Is.$$.fragment,s),v(Ue.$$.fragment,s),v(Me.$$.fragment,s),v(ze.$$.fragment,s),v(Ye.$$.fragment,s),v(Qe.$$.fragment,s),v(Us.$$.fragment,s),v(Je.$$.fragment,s),v(Bs.$$.fragment,s),v(Ys.$$.fragment,s),Il=!0)},o(s){w(m.$$.fragment,s),w(A.$$.fragment,s),w(Ks.$$.fragment,s),w(ne.$$.fragment,s),w(le.$$.fragment,s),w(ie.$$.fragment,s),w(oe.$$.fragment,s),w(ue.$$.fragment,s),w(pe.$$.fragment,s),w($s.$$.fragment,s),w(de.$$.fragment,s),w(Es.$$.fragment,s),w(fe.$$.fragment,s),w(me.$$.fragment,s),w(ge.$$.fragment,s),w(_e.$$.fragment,s),w(we.$$.fragment,s),w(xe.$$.fragment,s),w(ye.$$.fragment,s),w(je.$$.fragment,s),w(Ds.$$.fragment,s),w($e.$$.fragment,s),w(qe.$$.fragment,s),w(De.$$.fragment,s),w(Te.$$.fragment,s),w(He.$$.fragment,s),w(Oe.$$.fragment,s),w(Pe.$$.fragment,s),w(Ae.$$.fragment,s),w(Ne.$$.fragment,s),w(Ge.$$.fragment,s),w(Ie.$$.fragment,s),w(Re.$$.fragment,s),w(Se.$$.fragment,s),w(Fe.$$.fragment,s),w(Le.$$.fragment,s),w(Is.$$.fragment,s),w(Ue.$$.fragment,s),w(Me.$$.fragment,s),w(ze.$$.fragment,s),w(Ye.$$.fragment,s),w(Qe.$$.fragment,s),w(Us.$$.fragment,s),w(Je.$$.fragment,s),w(Bs.$$.fragment,s),w(Ys.$$.fragment,s),Il=!1},d(s){t(c),s&&t(D),s&&t(f),x(m),s&&t(j),x(A,s),s&&t(H),s&&t(O),s&&t(S),s&&t(G),s&&t(Ka),s&&t(pt),s&&t(Va),s&&t(ps),x(Ks),s&&t(Xa),s&&t(ws),s&&t(sn),s&&t(Xs),s&&t(en),s&&t(ht),s&&t(tn),s&&t(ee),s&&t(an),s&&t(K),s&&t(nn),s&&t(ys),s&&t(ln),x(ne,s),s&&t(on),s&&t(V),s&&t(rn),x(le,s),s&&t(un),s&&t(js),s&&t(pn),x(ie,s),s&&t(hn),x(oe,s),s&&t(cn),s&&t(Y),s&&t(dn),x(ue,s),s&&t(fn),x(pe,s),s&&t(mn),s&&t(Q),s&&t(gn),x($s,s),s&&t(_n),s&&t(L),s&&t(bn),x(de,s),s&&t(vn),x(Es,s),s&&t(wn),s&&t(ct),s&&t(xn),x(fe,s),s&&t(yn),s&&t(X),s&&t(jn),x(me,s),s&&t($n),s&&t(ks),s&&t(En),x(ge,s),s&&t(kn),x(_e,s),s&&t(qn),s&&t(ss),s&&t(Dn),s&&t(ft),s&&t(Tn),s&&t(mt),s&&t(Hn),s&&t(cs),x(we),s&&t(On),s&&t(R),s&&t(Pn),x(xe,s),s&&t(An),x(ye,s),s&&t(Nn),s&&t(U),s&&t(Cn),x(je,s),s&&t(Gn),x(Ds,s),s&&t(In),s&&t(_t),s&&t(Rn),s&&t(bt),s&&t(Sn),s&&t(ds),x($e),s&&t(Fn),s&&t(vt),s&&t(Ln),s&&t(Ee),s&&t(Un),s&&t(Hs),s&&t(Mn),x(qe,s),s&&t(zn),x(De,s),s&&t(Wn),s&&t(J),s&&t(Bn),x(Te,s),s&&t(Yn),x(He,s),s&&t(Qn),s&&t(es),s&&t(Jn),x(Oe,s),s&&t(Zn),s&&t(wt),s&&t(Kn),x(Pe,s),s&&t(Vn),s&&t(fs),x(Ae),s&&t(Xn),x(Ne,s),s&&t(sl),s&&t(ts),s&&t(el),x(Ge,s),s&&t(tl),x(Ie,s),s&&t(al),s&&t(As),s&&t(nl),s&&t(Ns),s&&t(ll),x(Re,s),s&&t(il),s&&t(Cs),s&&t(ol),x(Se,s),s&&t(rl),s&&t(Gs),s&&t(ul),x(Fe,s),s&&t(pl),x(Le,s),s&&t(hl),s&&t(as),s&&t(cl),x(Is,s),s&&t(dl),s&&t(Rs),s&&t(fl),x(Ue,s),s&&t(ml),s&&t(M),s&&t(gl),x(Me,s),s&&t(_l),s&&t(Ss),s&&t(bl),x(ze,s),s&&t(vl),s&&t(Fs),s&&t(wl),s&&t(We),s&&t(xl),s&&t(ns),s&&t(yl),x(Ye,s),s&&t(jl),x(Qe,s),s&&t($l),s&&t(Ls),s&&t(El),x(Us,s),s&&t(kl),s&&t(ms),x(Je),s&&t(ql),s&&t(xt),s&&t(Dl),s&&t(zs),s&&t(Tl),s&&t(yt),s&&t(Hl),s&&t(Ze),s&&t(Ol),s&&t(Ve),s&&t(Pl),s&&t(ls),s&&t(Al),s&&t(et),s&&t(Nl),x(Bs,s),s&&t(Cl),s&&t(jt),s&&t(Gl),x(Ys,s)}}}const Bh={local:"creating-your-own-dataset",sections:[{local:"getting-the-data",title:"Getting the data"},{local:"cleaning-up-the-data",title:"Cleaning up the data"},{local:"augmenting-the-dataset",title:"Augmenting the dataset"},{local:"uploading-the-dataset-to-the-hugging-face-hub",title:"Uploading the dataset to the Hugging Face Hub"},{local:"creating-a-dataset-card",title:"Creating a dataset card"}],title:"Creating your own dataset"};function Yh(F){return Ch(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class sc extends Oh{constructor(c){super();Ph(this,c,Yh,Wh,Ah,{})}}export{sc as default,Bh as metadata};
