import{S as Lu,i as Mu,s as zu,e as i,k as p,w as _,t as a,M as Bu,c as o,d as t,m as h,a as r,x as b,h as n,b as d,N as va,G as e,g as u,y as v,q as x,o as w,B as y,v as Wu}from"../../chunks/vendor-hf-doc-builder.js";import{T as ft}from"../../chunks/Tip-hf-doc-builder.js";import{Y as Yu}from"../../chunks/Youtube-hf-doc-builder.js";import{I as mt}from"../../chunks/IconCopyLink-hf-doc-builder.js";import{C as O}from"../../chunks/CodeBlock-hf-doc-builder.js";import{C as Qu}from"../../chunks/CourseFloatingBanner-hf-doc-builder.js";function Zu(U){let c,T,f,$,q;return{c(){c=i("p"),T=a("\u270F\uFE0F "),f=i("strong"),$=a("Try it out!"),q=a(" Click on a few of the URLs in the JSON payload above to get a feel for what type of information each GitHub issue is linked to.")},l(m){c=o(m,"P",{});var E=r(c);T=n(E,"\u270F\uFE0F "),f=o(E,"STRONG",{});var D=r(f);$=n(D,"Try it out!"),D.forEach(t),q=n(E," Click on a few of the URLs in the JSON payload above to get a feel for what type of information each GitHub issue is linked to."),E.forEach(t)},m(m,E){u(m,c,E),e(c,T),e(c,f),e(f,$),e(c,q)},d(m){m&&t(c)}}}function Ju(U){let c,T,f,$,q,m,E,D,g,j,H,A,C;return{c(){c=i("p"),T=a("\u26A0\uFE0F Do not share a notebook with your "),f=i("code"),$=a("GITHUB_TOKEN"),q=a(" pasted in it. We recommend you delete the last cell once you have executed it to avoid leaking this information accidentally. Even better, store the token in a "),m=i("em"),E=a(".env"),D=a(" file and use the "),g=i("a"),j=i("code"),H=a("python-dotenv"),A=a(" library"),C=a(" to load it automatically for you as an environment variable."),this.h()},l(S){c=o(S,"P",{});var k=r(c);T=n(k,"\u26A0\uFE0F Do not share a notebook with your "),f=o(k,"CODE",{});var P=r(f);$=n(P,"GITHUB_TOKEN"),P.forEach(t),q=n(k," pasted in it. We recommend you delete the last cell once you have executed it to avoid leaking this information accidentally. Even better, store the token in a "),m=o(k,"EM",{});var z=r(m);E=n(z,".env"),z.forEach(t),D=n(k," file and use the "),g=o(k,"A",{href:!0,rel:!0});var R=r(g);j=o(R,"CODE",{});var N=r(j);H=n(N,"python-dotenv"),N.forEach(t),A=n(R," library"),R.forEach(t),C=n(k," to load it automatically for you as an environment variable."),k.forEach(t),this.h()},h(){d(g,"href","https://github.com/theskumar/python-dotenv"),d(g,"rel","nofollow")},m(S,k){u(S,c,k),e(c,T),e(c,f),e(f,$),e(c,q),e(c,m),e(m,E),e(c,D),e(c,g),e(g,j),e(j,H),e(g,A),e(c,C)},d(S){S&&t(c)}}}function Vu(U){let c,T,f,$,q,m,E,D,g,j,H,A,C,S,k,P,z,R,N,B;return{c(){c=i("p"),T=a("\u270F\uFE0F "),f=i("strong"),$=a("Try it out!"),q=a(" Calculate the average time it takes to close issues in \u{1F917} Datasets. You may find the "),m=i("code"),E=a("Dataset.filter()"),D=a(" function useful to filter out the pull requests and open issues, and you can use the "),g=i("code"),j=a("Dataset.set_format()"),H=a(" function to convert the dataset to a "),A=i("code"),C=a("DataFrame"),S=a(" so you can easily manipulate the "),k=i("code"),P=a("created_at"),z=a(" and "),R=i("code"),N=a("closed_at"),B=a(" timestamps. For bonus points, calculate the average time it takes to close pull requests.")},l(is){c=o(is,"P",{});var G=r(c);T=n(G,"\u270F\uFE0F "),f=o(G,"STRONG",{});var Z=r(f);$=n(Z,"Try it out!"),Z.forEach(t),q=n(G," Calculate the average time it takes to close issues in \u{1F917} Datasets. You may find the "),m=o(G,"CODE",{});var Se=r(m);E=n(Se,"Dataset.filter()"),Se.forEach(t),D=n(G," function useful to filter out the pull requests and open issues, and you can use the "),g=o(G,"CODE",{});var fs=r(g);j=n(fs,"Dataset.set_format()"),fs.forEach(t),H=n(G," function to convert the dataset to a "),A=o(G,"CODE",{});var Re=r(A);C=n(Re,"DataFrame"),Re.forEach(t),S=n(G," so you can easily manipulate the "),k=o(G,"CODE",{});var Ue=r(k);P=n(Ue,"created_at"),Ue.forEach(t),z=n(G," and "),R=o(G,"CODE",{});var Fe=r(R);N=n(Fe,"closed_at"),Fe.forEach(t),B=n(G," timestamps. For bonus points, calculate the average time it takes to close pull requests."),G.forEach(t)},m(is,G){u(is,c,G),e(c,T),e(c,f),e(f,$),e(c,q),e(c,m),e(m,E),e(c,D),e(c,g),e(g,j),e(c,H),e(c,A),e(A,C),e(c,S),e(c,k),e(k,P),e(c,z),e(c,R),e(R,N),e(c,B)},d(is){is&&t(c)}}}function Xu(U){let c,T,f,$,q,m,E,D;return{c(){c=i("p"),T=a("\u{1F4A1} You can also upload a dataset to the Hugging Face Hub directly from the terminal by using "),f=i("code"),$=a("huggingface-cli"),q=a(" and a bit of Git magic. See the "),m=i("a"),E=a("\u{1F917} Datasets guide"),D=a(" for details on how to do this."),this.h()},l(g){c=o(g,"P",{});var j=r(c);T=n(j,"\u{1F4A1} You can also upload a dataset to the Hugging Face Hub directly from the terminal by using "),f=o(j,"CODE",{});var H=r(f);$=n(H,"huggingface-cli"),H.forEach(t),q=n(j," and a bit of Git magic. See the "),m=o(j,"A",{href:!0,rel:!0});var A=r(m);E=n(A,"\u{1F917} Datasets guide"),A.forEach(t),D=n(j," for details on how to do this."),j.forEach(t),this.h()},h(){d(m,"href","https://huggingface.co/docs/datasets/share.html#add-a-community-dataset"),d(m,"rel","nofollow")},m(g,j){u(g,c,j),e(c,T),e(c,f),e(f,$),e(c,q),e(c,m),e(m,E),e(c,D)},d(g){g&&t(c)}}}function Ku(U){let c,T,f,$,q,m,E,D,g,j,H,A,C,S;return{c(){c=i("p"),T=a("\u270F\uFE0F "),f=i("strong"),$=a("Try it out!"),q=a(" Use the "),m=i("code"),E=a("dataset-tagging"),D=a(" application and "),g=i("a"),j=a("\u{1F917} Datasets guide"),H=a(" to complete the "),A=i("em"),C=a("README.md"),S=a(" file for your GitHub issues dataset."),this.h()},l(k){c=o(k,"P",{});var P=r(c);T=n(P,"\u270F\uFE0F "),f=o(P,"STRONG",{});var z=r(f);$=n(z,"Try it out!"),z.forEach(t),q=n(P," Use the "),m=o(P,"CODE",{});var R=r(m);E=n(R,"dataset-tagging"),R.forEach(t),D=n(P," application and "),g=o(P,"A",{href:!0,rel:!0});var N=r(g);j=n(N,"\u{1F917} Datasets guide"),N.forEach(t),H=n(P," to complete the "),A=o(P,"EM",{});var B=r(A);C=n(B,"README.md"),B.forEach(t),S=n(P," file for your GitHub issues dataset."),P.forEach(t),this.h()},h(){d(g,"href","https://github.com/huggingface/datasets/blob/master/templates/README_guide.md"),d(g,"rel","nofollow")},m(k,P){u(k,c,P),e(c,T),e(c,f),e(f,$),e(c,q),e(c,m),e(m,E),e(c,D),e(c,g),e(g,j),e(c,H),e(c,A),e(A,C),e(c,S)},d(k){k&&t(c)}}}function sp(U){let c,T,f,$,q,m,E,D;return{c(){c=i("p"),T=a("\u270F\uFE0F "),f=i("strong"),$=a("Try it out!"),q=a(" Go through the steps we took in this section to create a dataset of GitHub issues for your favorite open source library (pick something other than \u{1F917} Datasets, of course!). For bonus points, fine-tune a multilabel classifier to predict the tags present in the "),m=i("code"),E=a("labels"),D=a(" field.")},l(g){c=o(g,"P",{});var j=r(c);T=n(j,"\u270F\uFE0F "),f=o(j,"STRONG",{});var H=r(f);$=n(H,"Try it out!"),H.forEach(t),q=n(j," Go through the steps we took in this section to create a dataset of GitHub issues for your favorite open source library (pick something other than \u{1F917} Datasets, of course!). For bonus points, fine-tune a multilabel classifier to predict the tags present in the "),m=o(j,"CODE",{});var A=r(m);E=n(A,"labels"),A.forEach(t),D=n(j," field."),j.forEach(t)},m(g,j){u(g,c,j),e(c,T),e(c,f),e(f,$),e(c,q),e(c,m),e(m,E),e(c,D)},d(g){g&&t(c)}}}function ep(U){let c,T,f,$,q,m,E,D,g,j,H,A,C,S,k,P,z,R,N,B,is,G,Z,Se,fs,Re,Ue,Fe,gt,il,xa,Le,ol,wa,os,ms,_t,Us,rl,bt,ul,ya,gs,pl,Fs,hl,cl,ja,Ls,Ms,tr,$a,Me,dl,Ea,zs,Bs,ar,ka,J,fl,Ws,ml,gl,_s,vt,_l,bl,vl,qa,bs,xl,xt,wl,yl,Ta,Ys,Da,V,jl,wt,$l,El,yt,kl,ql,Aa,Qs,Ha,vs,Tl,jt,Dl,Al,Oa,Zs,Pa,Js,Ca,W,Hl,$t,Ol,Pl,Vs,Cl,Nl,Et,Gl,Il,Na,Xs,Ga,Ks,Ia,Y,Sl,kt,Rl,Ul,qt,Fl,Ll,Tt,Ml,zl,Sa,xs,Ra,F,Bl,se,Wl,Yl,Dt,Ql,Zl,ee,Jl,Vl,At,Xl,Kl,Ua,te,Fa,ws,La,ze,si,Ma,ae,za,X,ei,Ht,ti,ai,Ot,ni,li,Ba,ne,Wa,ys,ii,Be,oi,ri,Ya,le,Qa,ie,Za,K,ui,oe,pi,hi,re,ci,di,Ja,We,rs,fi,Pt,mi,gi,Ct,_i,bi,Va,Ye,vi,Xa,us,js,Nt,ue,xi,Gt,wi,Ka,I,yi,It,ji,$i,Qe,Ei,ki,St,qi,Ti,Rt,Di,Ai,Ut,Hi,Oi,Ft,Pi,Ci,sn,pe,en,he,tn,L,Ni,Lt,Gi,Ii,Mt,Si,Ri,zt,Ui,Fi,Bt,Li,Mi,an,ce,nn,$s,ln,Ze,zi,on,Je,Bi,rn,ps,Es,Wt,de,Wi,Yt,Yi,un,Ve,Qi,pn,fe,me,nr,hn,ks,Zi,qs,Qt,Ji,Vi,Xi,cn,ge,dn,_e,fn,Q,Ki,Zt,so,eo,Jt,to,ao,Vt,no,lo,mn,be,gn,ve,_n,ss,io,Xt,oo,ro,Kt,uo,po,bn,xe,vn,Xe,ho,xn,hs,Ts,sa,we,co,ea,fo,wn,ye,yn,es,mo,ta,go,_o,aa,bo,vo,jn,je,$n,Ds,xo,na,wo,yo,En,$e,kn,Ke,jo,qn,Ee,Tn,ts,$o,la,Eo,ko,ia,qo,To,Dn,ke,An,qe,Hn,As,Do,oa,Ao,Ho,On,Hs,Pn,cs,Os,ra,Te,Oo,ua,Po,Cn,st,Co,Nn,Ps,No,pa,Go,Io,Gn,et,ds,So,Cs,ha,Ro,Uo,Fo,ca,Lo,Mo,In,De,Ae,lr,Sn,He,Oe,zo,Pe,Bo,Wo,Rn,as,Yo,da,Qo,Zo,fa,Jo,Vo,Un,Ce,Ne,ir,Fn,Ns,Ln,tt,Xo,Mn,Gs,zn;return m=new mt({}),H=new Qu({props:{chapter:5,classNames:"absolute z-10 right-0 top-0",notebooks:[{label:"Google Colab",value:"https://colab.research.google.com/github/huggingface/notebooks/blob/master/course/en/chapter5/section5.ipynb"},{label:"Aws Studio",value:"https://studiolab.sagemaker.aws/import/github/huggingface/notebooks/blob/master/course/en/chapter5/section5.ipynb"}]}}),Us=new mt({}),Ys=new O({props:{code:"!pip install requests",highlighted:"!pip install requests"}}),Qs=new O({props:{code:`import requests

url = "https://api.github.com/repos/huggingface/datasets/issues?page=1&per_page=1"
response = requests.get(url)`,highlighted:`<span class="hljs-keyword">import</span> requests

url = <span class="hljs-string">&quot;https://api.github.com/repos/huggingface/datasets/issues?page=1&amp;per_page=1&quot;</span>
response = requests.get(url)`}}),Zs=new O({props:{code:"response.status_code",highlighted:"response.status_code"}}),Js=new O({props:{code:"200",highlighted:'<span class="hljs-number">200</span>'}}),Xs=new O({props:{code:"response.json()",highlighted:"response.json()"}}),Ks=new O({props:{code:`[{'url': 'https://api.github.com/repos/huggingface/datasets/issues/2792',
  'repository_url': 'https://api.github.com/repos/huggingface/datasets',
  'labels_url': 'https://api.github.com/repos/huggingface/datasets/issues/2792/labels{/name}',
  'comments_url': 'https://api.github.com/repos/huggingface/datasets/issues/2792/comments',
  'events_url': 'https://api.github.com/repos/huggingface/datasets/issues/2792/events',
  'html_url': 'https://github.com/huggingface/datasets/pull/2792',
  'id': 968650274,
  'node_id': 'MDExOlB1bGxSZXF1ZXN0NzEwNzUyMjc0',
  'number': 2792,
  'title': 'Update GooAQ',
  'user': {'login': 'bhavitvyamalik',
   'id': 19718818,
   'node_id': 'MDQ6VXNlcjE5NzE4ODE4',
   'avatar_url': 'https://avatars.githubusercontent.com/u/19718818?v=4',
   'gravatar_id': '',
   'url': 'https://api.github.com/users/bhavitvyamalik',
   'html_url': 'https://github.com/bhavitvyamalik',
   'followers_url': 'https://api.github.com/users/bhavitvyamalik/followers',
   'following_url': 'https://api.github.com/users/bhavitvyamalik/following{/other_user}',
   'gists_url': 'https://api.github.com/users/bhavitvyamalik/gists{/gist_id}',
   'starred_url': 'https://api.github.com/users/bhavitvyamalik/starred{/owner}{/repo}',
   'subscriptions_url': 'https://api.github.com/users/bhavitvyamalik/subscriptions',
   'organizations_url': 'https://api.github.com/users/bhavitvyamalik/orgs',
   'repos_url': 'https://api.github.com/users/bhavitvyamalik/repos',
   'events_url': 'https://api.github.com/users/bhavitvyamalik/events{/privacy}',
   'received_events_url': 'https://api.github.com/users/bhavitvyamalik/received_events',
   'type': 'User',
   'site_admin': False},
  'labels': [],
  'state': 'open',
  'locked': False,
  'assignee': None,
  'assignees': [],
  'milestone': None,
  'comments': 1,
  'created_at': '2021-08-12T11:40:18Z',
  'updated_at': '2021-08-12T12:31:17Z',
  'closed_at': None,
  'author_association': 'CONTRIBUTOR',
  'active_lock_reason': None,
  'pull_request': {'url': 'https://api.github.com/repos/huggingface/datasets/pulls/2792',
   'html_url': 'https://github.com/huggingface/datasets/pull/2792',
   'diff_url': 'https://github.com/huggingface/datasets/pull/2792.diff',
   'patch_url': 'https://github.com/huggingface/datasets/pull/2792.patch'},
  'body': '[GooAQ](https://github.com/allenai/gooaq) dataset was recently updated after splits were added for the same. This PR contains new updated GooAQ with train/val/test splits and updated README as well.',
  'performed_via_github_app': None}]`,highlighted:`[{<span class="hljs-string">&#x27;url&#x27;</span>: <span class="hljs-string">&#x27;https://api.github.com/repos/huggingface/datasets/issues/2792&#x27;</span>,
  <span class="hljs-string">&#x27;repository_url&#x27;</span>: <span class="hljs-string">&#x27;https://api.github.com/repos/huggingface/datasets&#x27;</span>,
  <span class="hljs-string">&#x27;labels_url&#x27;</span>: <span class="hljs-string">&#x27;https://api.github.com/repos/huggingface/datasets/issues/2792/labels{/name}&#x27;</span>,
  <span class="hljs-string">&#x27;comments_url&#x27;</span>: <span class="hljs-string">&#x27;https://api.github.com/repos/huggingface/datasets/issues/2792/comments&#x27;</span>,
  <span class="hljs-string">&#x27;events_url&#x27;</span>: <span class="hljs-string">&#x27;https://api.github.com/repos/huggingface/datasets/issues/2792/events&#x27;</span>,
  <span class="hljs-string">&#x27;html_url&#x27;</span>: <span class="hljs-string">&#x27;https://github.com/huggingface/datasets/pull/2792&#x27;</span>,
  <span class="hljs-string">&#x27;id&#x27;</span>: <span class="hljs-number">968650274</span>,
  <span class="hljs-string">&#x27;node_id&#x27;</span>: <span class="hljs-string">&#x27;MDExOlB1bGxSZXF1ZXN0NzEwNzUyMjc0&#x27;</span>,
  <span class="hljs-string">&#x27;number&#x27;</span>: <span class="hljs-number">2792</span>,
  <span class="hljs-string">&#x27;title&#x27;</span>: <span class="hljs-string">&#x27;Update GooAQ&#x27;</span>,
  <span class="hljs-string">&#x27;user&#x27;</span>: {<span class="hljs-string">&#x27;login&#x27;</span>: <span class="hljs-string">&#x27;bhavitvyamalik&#x27;</span>,
   <span class="hljs-string">&#x27;id&#x27;</span>: <span class="hljs-number">19718818</span>,
   <span class="hljs-string">&#x27;node_id&#x27;</span>: <span class="hljs-string">&#x27;MDQ6VXNlcjE5NzE4ODE4&#x27;</span>,
   <span class="hljs-string">&#x27;avatar_url&#x27;</span>: <span class="hljs-string">&#x27;https://avatars.githubusercontent.com/u/19718818?v=4&#x27;</span>,
   <span class="hljs-string">&#x27;gravatar_id&#x27;</span>: <span class="hljs-string">&#x27;&#x27;</span>,
   <span class="hljs-string">&#x27;url&#x27;</span>: <span class="hljs-string">&#x27;https://api.github.com/users/bhavitvyamalik&#x27;</span>,
   <span class="hljs-string">&#x27;html_url&#x27;</span>: <span class="hljs-string">&#x27;https://github.com/bhavitvyamalik&#x27;</span>,
   <span class="hljs-string">&#x27;followers_url&#x27;</span>: <span class="hljs-string">&#x27;https://api.github.com/users/bhavitvyamalik/followers&#x27;</span>,
   <span class="hljs-string">&#x27;following_url&#x27;</span>: <span class="hljs-string">&#x27;https://api.github.com/users/bhavitvyamalik/following{/other_user}&#x27;</span>,
   <span class="hljs-string">&#x27;gists_url&#x27;</span>: <span class="hljs-string">&#x27;https://api.github.com/users/bhavitvyamalik/gists{/gist_id}&#x27;</span>,
   <span class="hljs-string">&#x27;starred_url&#x27;</span>: <span class="hljs-string">&#x27;https://api.github.com/users/bhavitvyamalik/starred{/owner}{/repo}&#x27;</span>,
   <span class="hljs-string">&#x27;subscriptions_url&#x27;</span>: <span class="hljs-string">&#x27;https://api.github.com/users/bhavitvyamalik/subscriptions&#x27;</span>,
   <span class="hljs-string">&#x27;organizations_url&#x27;</span>: <span class="hljs-string">&#x27;https://api.github.com/users/bhavitvyamalik/orgs&#x27;</span>,
   <span class="hljs-string">&#x27;repos_url&#x27;</span>: <span class="hljs-string">&#x27;https://api.github.com/users/bhavitvyamalik/repos&#x27;</span>,
   <span class="hljs-string">&#x27;events_url&#x27;</span>: <span class="hljs-string">&#x27;https://api.github.com/users/bhavitvyamalik/events{/privacy}&#x27;</span>,
   <span class="hljs-string">&#x27;received_events_url&#x27;</span>: <span class="hljs-string">&#x27;https://api.github.com/users/bhavitvyamalik/received_events&#x27;</span>,
   <span class="hljs-string">&#x27;type&#x27;</span>: <span class="hljs-string">&#x27;User&#x27;</span>,
   <span class="hljs-string">&#x27;site_admin&#x27;</span>: <span class="hljs-literal">False</span>},
  <span class="hljs-string">&#x27;labels&#x27;</span>: [],
  <span class="hljs-string">&#x27;state&#x27;</span>: <span class="hljs-string">&#x27;open&#x27;</span>,
  <span class="hljs-string">&#x27;locked&#x27;</span>: <span class="hljs-literal">False</span>,
  <span class="hljs-string">&#x27;assignee&#x27;</span>: <span class="hljs-literal">None</span>,
  <span class="hljs-string">&#x27;assignees&#x27;</span>: [],
  <span class="hljs-string">&#x27;milestone&#x27;</span>: <span class="hljs-literal">None</span>,
  <span class="hljs-string">&#x27;comments&#x27;</span>: <span class="hljs-number">1</span>,
  <span class="hljs-string">&#x27;created_at&#x27;</span>: <span class="hljs-string">&#x27;2021-08-12T11:40:18Z&#x27;</span>,
  <span class="hljs-string">&#x27;updated_at&#x27;</span>: <span class="hljs-string">&#x27;2021-08-12T12:31:17Z&#x27;</span>,
  <span class="hljs-string">&#x27;closed_at&#x27;</span>: <span class="hljs-literal">None</span>,
  <span class="hljs-string">&#x27;author_association&#x27;</span>: <span class="hljs-string">&#x27;CONTRIBUTOR&#x27;</span>,
  <span class="hljs-string">&#x27;active_lock_reason&#x27;</span>: <span class="hljs-literal">None</span>,
  <span class="hljs-string">&#x27;pull_request&#x27;</span>: {<span class="hljs-string">&#x27;url&#x27;</span>: <span class="hljs-string">&#x27;https://api.github.com/repos/huggingface/datasets/pulls/2792&#x27;</span>,
   <span class="hljs-string">&#x27;html_url&#x27;</span>: <span class="hljs-string">&#x27;https://github.com/huggingface/datasets/pull/2792&#x27;</span>,
   <span class="hljs-string">&#x27;diff_url&#x27;</span>: <span class="hljs-string">&#x27;https://github.com/huggingface/datasets/pull/2792.diff&#x27;</span>,
   <span class="hljs-string">&#x27;patch_url&#x27;</span>: <span class="hljs-string">&#x27;https://github.com/huggingface/datasets/pull/2792.patch&#x27;</span>},
  <span class="hljs-string">&#x27;body&#x27;</span>: <span class="hljs-string">&#x27;[GooAQ](https://github.com/allenai/gooaq) dataset was recently updated after splits were added for the same. This PR contains new updated GooAQ with train/val/test splits and updated README as well.&#x27;</span>,
  <span class="hljs-string">&#x27;performed_via_github_app&#x27;</span>: <span class="hljs-literal">None</span>}]`}}),xs=new ft({props:{$$slots:{default:[Zu]},$$scope:{ctx:U}}}),te=new O({props:{code:`GITHUB_TOKEN = xxx  # Copy your GitHub token here
headers = {"Authorization": f"token {GITHUB_TOKEN}"}`,highlighted:`GITHUB_TOKEN = xxx  <span class="hljs-comment"># Copy your GitHub token here</span>
headers = {<span class="hljs-string">&quot;Authorization&quot;</span>: <span class="hljs-string">f&quot;token <span class="hljs-subst">{GITHUB_TOKEN}</span>&quot;</span>}`}}),ws=new ft({props:{warning:!0,$$slots:{default:[Ju]},$$scope:{ctx:U}}}),ae=new O({props:{code:`import time
import math
from pathlib import Path
import pandas as pd
from tqdm.notebook import tqdm


def fetch_issues(
    owner="huggingface",
    repo="datasets",
    num_issues=10_000,
    rate_limit=5_000,
    issues_path=Path("."),
):
    if not issues_path.is_dir():
        issues_path.mkdir(exist_ok=True)

    batch = []
    all_issues = []
    per_page = 100  # Number of issues to return per page
    num_pages = math.ceil(num_issues / per_page)
    base_url = "https://api.github.com/repos"

    for page in tqdm(range(num_pages)):
        # Query with state=all to get both open and closed issues
        query = f"issues?page={page}&per_page={per_page}&state=all"
        issues = requests.get(f"{base_url}/{owner}/{repo}/{query}", headers=headers)
        batch.extend(issues.json())

        if len(batch) > rate_limit and len(all_issues) < num_issues:
            all_issues.extend(batch)
            batch = []  # Flush batch for next time period
            print(f"Reached GitHub rate limit. Sleeping for one hour ...")
            time.sleep(60 * 60 + 1)

    all_issues.extend(batch)
    df = pd.DataFrame.from_records(all_issues)
    df.to_json(f"{issues_path}/{repo}-issues.jsonl", orient="records", lines=True)
    print(
        f"Downloaded all the issues for {repo}! Dataset stored at {issues_path}/{repo}-issues.jsonl"
    )`,highlighted:`<span class="hljs-keyword">import</span> time
<span class="hljs-keyword">import</span> math
<span class="hljs-keyword">from</span> pathlib <span class="hljs-keyword">import</span> Path
<span class="hljs-keyword">import</span> pandas <span class="hljs-keyword">as</span> pd
<span class="hljs-keyword">from</span> tqdm.notebook <span class="hljs-keyword">import</span> tqdm


<span class="hljs-keyword">def</span> <span class="hljs-title function_">fetch_issues</span>(<span class="hljs-params">
    owner=<span class="hljs-string">&quot;huggingface&quot;</span>,
    repo=<span class="hljs-string">&quot;datasets&quot;</span>,
    num_issues=<span class="hljs-number">10_000</span>,
    rate_limit=<span class="hljs-number">5_000</span>,
    issues_path=Path(<span class="hljs-params"><span class="hljs-string">&quot;.&quot;</span></span>),
</span>):
    <span class="hljs-keyword">if</span> <span class="hljs-keyword">not</span> issues_path.is_dir():
        issues_path.mkdir(exist_ok=<span class="hljs-literal">True</span>)

    batch = []
    all_issues = []
    per_page = <span class="hljs-number">100</span>  <span class="hljs-comment"># Number of issues to return per page</span>
    num_pages = math.ceil(num_issues / per_page)
    base_url = <span class="hljs-string">&quot;https://api.github.com/repos&quot;</span>

    <span class="hljs-keyword">for</span> page <span class="hljs-keyword">in</span> tqdm(<span class="hljs-built_in">range</span>(num_pages)):
        <span class="hljs-comment"># Query with state=all to get both open and closed issues</span>
        query = <span class="hljs-string">f&quot;issues?page=<span class="hljs-subst">{page}</span>&amp;per_page=<span class="hljs-subst">{per_page}</span>&amp;state=all&quot;</span>
        issues = requests.get(<span class="hljs-string">f&quot;<span class="hljs-subst">{base_url}</span>/<span class="hljs-subst">{owner}</span>/<span class="hljs-subst">{repo}</span>/<span class="hljs-subst">{query}</span>&quot;</span>, headers=headers)
        batch.extend(issues.json())

        <span class="hljs-keyword">if</span> <span class="hljs-built_in">len</span>(batch) &gt; rate_limit <span class="hljs-keyword">and</span> <span class="hljs-built_in">len</span>(all_issues) &lt; num_issues:
            all_issues.extend(batch)
            batch = []  <span class="hljs-comment"># Flush batch for next time period</span>
            <span class="hljs-built_in">print</span>(<span class="hljs-string">f&quot;Reached GitHub rate limit. Sleeping for one hour ...&quot;</span>)
            time.sleep(<span class="hljs-number">60</span> * <span class="hljs-number">60</span> + <span class="hljs-number">1</span>)

    all_issues.extend(batch)
    df = pd.DataFrame.from_records(all_issues)
    df.to_json(<span class="hljs-string">f&quot;<span class="hljs-subst">{issues_path}</span>/<span class="hljs-subst">{repo}</span>-issues.jsonl&quot;</span>, orient=<span class="hljs-string">&quot;records&quot;</span>, lines=<span class="hljs-literal">True</span>)
    <span class="hljs-built_in">print</span>(
        <span class="hljs-string">f&quot;Downloaded all the issues for <span class="hljs-subst">{repo}</span>! Dataset stored at <span class="hljs-subst">{issues_path}</span>/<span class="hljs-subst">{repo}</span>-issues.jsonl&quot;</span>
    )`}}),ne=new O({props:{code:`# Depending on your internet connection, this can take several minutes to run...
fetch_issues()`,highlighted:`<span class="hljs-comment"># Depending on your internet connection, this can take several minutes to run...</span>
fetch_issues()`}}),le=new O({props:{code:`issues_dataset = load_dataset("json", data_files="datasets-issues.jsonl", split="train")
issues_dataset`,highlighted:`issues_dataset = load_dataset(<span class="hljs-string">&quot;json&quot;</span>, data_files=<span class="hljs-string">&quot;datasets-issues.jsonl&quot;</span>, split=<span class="hljs-string">&quot;train&quot;</span>)
issues_dataset`}}),ie=new O({props:{code:`Dataset({
    features: ['url', 'repository_url', 'labels_url', 'comments_url', 'events_url', 'html_url', 'id', 'node_id', 'number', 'title', 'user', 'labels', 'state', 'locked', 'assignee', 'assignees', 'milestone', 'comments', 'created_at', 'updated_at', 'closed_at', 'author_association', 'active_lock_reason', 'pull_request', 'body', 'timeline_url', 'performed_via_github_app'],
    num_rows: 3019
})`,highlighted:`Dataset({
    features: [<span class="hljs-string">&#x27;url&#x27;</span>, <span class="hljs-string">&#x27;repository_url&#x27;</span>, <span class="hljs-string">&#x27;labels_url&#x27;</span>, <span class="hljs-string">&#x27;comments_url&#x27;</span>, <span class="hljs-string">&#x27;events_url&#x27;</span>, <span class="hljs-string">&#x27;html_url&#x27;</span>, <span class="hljs-string">&#x27;id&#x27;</span>, <span class="hljs-string">&#x27;node_id&#x27;</span>, <span class="hljs-string">&#x27;number&#x27;</span>, <span class="hljs-string">&#x27;title&#x27;</span>, <span class="hljs-string">&#x27;user&#x27;</span>, <span class="hljs-string">&#x27;labels&#x27;</span>, <span class="hljs-string">&#x27;state&#x27;</span>, <span class="hljs-string">&#x27;locked&#x27;</span>, <span class="hljs-string">&#x27;assignee&#x27;</span>, <span class="hljs-string">&#x27;assignees&#x27;</span>, <span class="hljs-string">&#x27;milestone&#x27;</span>, <span class="hljs-string">&#x27;comments&#x27;</span>, <span class="hljs-string">&#x27;created_at&#x27;</span>, <span class="hljs-string">&#x27;updated_at&#x27;</span>, <span class="hljs-string">&#x27;closed_at&#x27;</span>, <span class="hljs-string">&#x27;author_association&#x27;</span>, <span class="hljs-string">&#x27;active_lock_reason&#x27;</span>, <span class="hljs-string">&#x27;pull_request&#x27;</span>, <span class="hljs-string">&#x27;body&#x27;</span>, <span class="hljs-string">&#x27;timeline_url&#x27;</span>, <span class="hljs-string">&#x27;performed_via_github_app&#x27;</span>],
    num_rows: <span class="hljs-number">3019</span>
})`}}),ue=new mt({}),pe=new O({props:{code:`sample = issues_dataset.shuffle(seed=666).select(range(3))

# Print out the URL and pull request entries
for url, pr in zip(sample["html_url"], sample["pull_request"]):
    print(f">> URL: {url}")
    print(f">> Pull request: {pr}\\n")`,highlighted:`sample = issues_dataset.shuffle(seed=<span class="hljs-number">666</span>).select(<span class="hljs-built_in">range</span>(<span class="hljs-number">3</span>))

<span class="hljs-comment"># Print out the URL and pull request entries</span>
<span class="hljs-keyword">for</span> url, pr <span class="hljs-keyword">in</span> <span class="hljs-built_in">zip</span>(sample[<span class="hljs-string">&quot;html_url&quot;</span>], sample[<span class="hljs-string">&quot;pull_request&quot;</span>]):
    <span class="hljs-built_in">print</span>(<span class="hljs-string">f&quot;&gt;&gt; URL: <span class="hljs-subst">{url}</span>&quot;</span>)
    <span class="hljs-built_in">print</span>(<span class="hljs-string">f&quot;&gt;&gt; Pull request: <span class="hljs-subst">{pr}</span>\\n&quot;</span>)`}}),he=new O({props:{code:`>> URL: https://github.com/huggingface/datasets/pull/850
>> Pull request: {'url': 'https://api.github.com/repos/huggingface/datasets/pulls/850', 'html_url': 'https://github.com/huggingface/datasets/pull/850', 'diff_url': 'https://github.com/huggingface/datasets/pull/850.diff', 'patch_url': 'https://github.com/huggingface/datasets/pull/850.patch'}

>> URL: https://github.com/huggingface/datasets/issues/2773
>> Pull request: None

>> URL: https://github.com/huggingface/datasets/pull/783
>> Pull request: {'url': 'https://api.github.com/repos/huggingface/datasets/pulls/783', 'html_url': 'https://github.com/huggingface/datasets/pull/783', 'diff_url': 'https://github.com/huggingface/datasets/pull/783.diff', 'patch_url': 'https://github.com/huggingface/datasets/pull/783.patch'}`,highlighted:`&gt;&gt; URL: https://github.com/huggingface/datasets/pull/<span class="hljs-number">850</span>
&gt;&gt; Pull request: {<span class="hljs-string">&#x27;url&#x27;</span>: <span class="hljs-string">&#x27;https://api.github.com/repos/huggingface/datasets/pulls/850&#x27;</span>, <span class="hljs-string">&#x27;html_url&#x27;</span>: <span class="hljs-string">&#x27;https://github.com/huggingface/datasets/pull/850&#x27;</span>, <span class="hljs-string">&#x27;diff_url&#x27;</span>: <span class="hljs-string">&#x27;https://github.com/huggingface/datasets/pull/850.diff&#x27;</span>, <span class="hljs-string">&#x27;patch_url&#x27;</span>: <span class="hljs-string">&#x27;https://github.com/huggingface/datasets/pull/850.patch&#x27;</span>}

&gt;&gt; URL: https://github.com/huggingface/datasets/issues/<span class="hljs-number">2773</span>
&gt;&gt; Pull request: <span class="hljs-literal">None</span>

&gt;&gt; URL: https://github.com/huggingface/datasets/pull/<span class="hljs-number">783</span>
&gt;&gt; Pull request: {<span class="hljs-string">&#x27;url&#x27;</span>: <span class="hljs-string">&#x27;https://api.github.com/repos/huggingface/datasets/pulls/783&#x27;</span>, <span class="hljs-string">&#x27;html_url&#x27;</span>: <span class="hljs-string">&#x27;https://github.com/huggingface/datasets/pull/783&#x27;</span>, <span class="hljs-string">&#x27;diff_url&#x27;</span>: <span class="hljs-string">&#x27;https://github.com/huggingface/datasets/pull/783.diff&#x27;</span>, <span class="hljs-string">&#x27;patch_url&#x27;</span>: <span class="hljs-string">&#x27;https://github.com/huggingface/datasets/pull/783.patch&#x27;</span>}`}}),ce=new O({props:{code:`issues_dataset = issues_dataset.map(
    lambda x: {"is_pull_request": False if x["pull_request"] is None else True}
)`,highlighted:`issues_dataset = issues_dataset.<span class="hljs-built_in">map</span>(
    <span class="hljs-keyword">lambda</span> x: {<span class="hljs-string">&quot;is_pull_request&quot;</span>: <span class="hljs-literal">False</span> <span class="hljs-keyword">if</span> x[<span class="hljs-string">&quot;pull_request&quot;</span>] <span class="hljs-keyword">is</span> <span class="hljs-literal">None</span> <span class="hljs-keyword">else</span> <span class="hljs-literal">True</span>}
)`}}),$s=new ft({props:{$$slots:{default:[Vu]},$$scope:{ctx:U}}}),de=new mt({}),ge=new O({props:{code:`issue_number = 2792
url = f"https://api.github.com/repos/huggingface/datasets/issues/{issue_number}/comments"
response = requests.get(url, headers=headers)
response.json()`,highlighted:`issue_number = <span class="hljs-number">2792</span>
url = <span class="hljs-string">f&quot;https://api.github.com/repos/huggingface/datasets/issues/<span class="hljs-subst">{issue_number}</span>/comments&quot;</span>
response = requests.get(url, headers=headers)
response.json()`}}),_e=new O({props:{code:`[{'url': 'https://api.github.com/repos/huggingface/datasets/issues/comments/897594128',
  'html_url': 'https://github.com/huggingface/datasets/pull/2792#issuecomment-897594128',
  'issue_url': 'https://api.github.com/repos/huggingface/datasets/issues/2792',
  'id': 897594128,
  'node_id': 'IC_kwDODunzps41gDMQ',
  'user': {'login': 'bhavitvyamalik',
   'id': 19718818,
   'node_id': 'MDQ6VXNlcjE5NzE4ODE4',
   'avatar_url': 'https://avatars.githubusercontent.com/u/19718818?v=4',
   'gravatar_id': '',
   'url': 'https://api.github.com/users/bhavitvyamalik',
   'html_url': 'https://github.com/bhavitvyamalik',
   'followers_url': 'https://api.github.com/users/bhavitvyamalik/followers',
   'following_url': 'https://api.github.com/users/bhavitvyamalik/following{/other_user}',
   'gists_url': 'https://api.github.com/users/bhavitvyamalik/gists{/gist_id}',
   'starred_url': 'https://api.github.com/users/bhavitvyamalik/starred{/owner}{/repo}',
   'subscriptions_url': 'https://api.github.com/users/bhavitvyamalik/subscriptions',
   'organizations_url': 'https://api.github.com/users/bhavitvyamalik/orgs',
   'repos_url': 'https://api.github.com/users/bhavitvyamalik/repos',
   'events_url': 'https://api.github.com/users/bhavitvyamalik/events{/privacy}',
   'received_events_url': 'https://api.github.com/users/bhavitvyamalik/received_events',
   'type': 'User',
   'site_admin': False},
  'created_at': '2021-08-12T12:21:52Z',
  'updated_at': '2021-08-12T12:31:17Z',
  'author_association': 'CONTRIBUTOR',
  'body': "@albertvillanova my tests are failing here:\\r\\n\`\`\`\\r\\ndataset_name = 'gooaq'\\r\\n\\r\\n    def test_load_dataset(self, dataset_name):\\r\\n        configs = self.dataset_tester.load_all_configs(dataset_name, is_local=True)[:1]\\r\\n>       self.dataset_tester.check_load_dataset(dataset_name, configs, is_local=True, use_local_dummy_data=True)\\r\\n\\r\\ntests/test_dataset_common.py:234: \\r\\n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \\r\\ntests/test_dataset_common.py:187: in check_load_dataset\\r\\n    self.parent.assertTrue(len(dataset[split]) > 0)\\r\\nE   AssertionError: False is not true\\r\\n\`\`\`\\r\\nWhen I try loading dataset on local machine it works fine. Any suggestions on how can I avoid this error?",
  'performed_via_github_app': None}]`,highlighted:`[{<span class="hljs-string">&#x27;url&#x27;</span>: <span class="hljs-string">&#x27;https://api.github.com/repos/huggingface/datasets/issues/comments/897594128&#x27;</span>,
  <span class="hljs-string">&#x27;html_url&#x27;</span>: <span class="hljs-string">&#x27;https://github.com/huggingface/datasets/pull/2792#issuecomment-897594128&#x27;</span>,
  <span class="hljs-string">&#x27;issue_url&#x27;</span>: <span class="hljs-string">&#x27;https://api.github.com/repos/huggingface/datasets/issues/2792&#x27;</span>,
  <span class="hljs-string">&#x27;id&#x27;</span>: <span class="hljs-number">897594128</span>,
  <span class="hljs-string">&#x27;node_id&#x27;</span>: <span class="hljs-string">&#x27;IC_kwDODunzps41gDMQ&#x27;</span>,
  <span class="hljs-string">&#x27;user&#x27;</span>: {<span class="hljs-string">&#x27;login&#x27;</span>: <span class="hljs-string">&#x27;bhavitvyamalik&#x27;</span>,
   <span class="hljs-string">&#x27;id&#x27;</span>: <span class="hljs-number">19718818</span>,
   <span class="hljs-string">&#x27;node_id&#x27;</span>: <span class="hljs-string">&#x27;MDQ6VXNlcjE5NzE4ODE4&#x27;</span>,
   <span class="hljs-string">&#x27;avatar_url&#x27;</span>: <span class="hljs-string">&#x27;https://avatars.githubusercontent.com/u/19718818?v=4&#x27;</span>,
   <span class="hljs-string">&#x27;gravatar_id&#x27;</span>: <span class="hljs-string">&#x27;&#x27;</span>,
   <span class="hljs-string">&#x27;url&#x27;</span>: <span class="hljs-string">&#x27;https://api.github.com/users/bhavitvyamalik&#x27;</span>,
   <span class="hljs-string">&#x27;html_url&#x27;</span>: <span class="hljs-string">&#x27;https://github.com/bhavitvyamalik&#x27;</span>,
   <span class="hljs-string">&#x27;followers_url&#x27;</span>: <span class="hljs-string">&#x27;https://api.github.com/users/bhavitvyamalik/followers&#x27;</span>,
   <span class="hljs-string">&#x27;following_url&#x27;</span>: <span class="hljs-string">&#x27;https://api.github.com/users/bhavitvyamalik/following{/other_user}&#x27;</span>,
   <span class="hljs-string">&#x27;gists_url&#x27;</span>: <span class="hljs-string">&#x27;https://api.github.com/users/bhavitvyamalik/gists{/gist_id}&#x27;</span>,
   <span class="hljs-string">&#x27;starred_url&#x27;</span>: <span class="hljs-string">&#x27;https://api.github.com/users/bhavitvyamalik/starred{/owner}{/repo}&#x27;</span>,
   <span class="hljs-string">&#x27;subscriptions_url&#x27;</span>: <span class="hljs-string">&#x27;https://api.github.com/users/bhavitvyamalik/subscriptions&#x27;</span>,
   <span class="hljs-string">&#x27;organizations_url&#x27;</span>: <span class="hljs-string">&#x27;https://api.github.com/users/bhavitvyamalik/orgs&#x27;</span>,
   <span class="hljs-string">&#x27;repos_url&#x27;</span>: <span class="hljs-string">&#x27;https://api.github.com/users/bhavitvyamalik/repos&#x27;</span>,
   <span class="hljs-string">&#x27;events_url&#x27;</span>: <span class="hljs-string">&#x27;https://api.github.com/users/bhavitvyamalik/events{/privacy}&#x27;</span>,
   <span class="hljs-string">&#x27;received_events_url&#x27;</span>: <span class="hljs-string">&#x27;https://api.github.com/users/bhavitvyamalik/received_events&#x27;</span>,
   <span class="hljs-string">&#x27;type&#x27;</span>: <span class="hljs-string">&#x27;User&#x27;</span>,
   <span class="hljs-string">&#x27;site_admin&#x27;</span>: <span class="hljs-literal">False</span>},
  <span class="hljs-string">&#x27;created_at&#x27;</span>: <span class="hljs-string">&#x27;2021-08-12T12:21:52Z&#x27;</span>,
  <span class="hljs-string">&#x27;updated_at&#x27;</span>: <span class="hljs-string">&#x27;2021-08-12T12:31:17Z&#x27;</span>,
  <span class="hljs-string">&#x27;author_association&#x27;</span>: <span class="hljs-string">&#x27;CONTRIBUTOR&#x27;</span>,
  <span class="hljs-string">&#x27;body&#x27;</span>: <span class="hljs-string">&quot;@albertvillanova my tests are failing here:\\r\\n\`\`\`\\r\\ndataset_name = &#x27;gooaq&#x27;\\r\\n\\r\\n    def test_load_dataset(self, dataset_name):\\r\\n        configs = self.dataset_tester.load_all_configs(dataset_name, is_local=True)[:1]\\r\\n&gt;       self.dataset_tester.check_load_dataset(dataset_name, configs, is_local=True, use_local_dummy_data=True)\\r\\n\\r\\ntests/test_dataset_common.py:234: \\r\\n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \\r\\ntests/test_dataset_common.py:187: in check_load_dataset\\r\\n    self.parent.assertTrue(len(dataset[split]) &gt; 0)\\r\\nE   AssertionError: False is not true\\r\\n\`\`\`\\r\\nWhen I try loading dataset on local machine it works fine. Any suggestions on how can I avoid this error?&quot;</span>,
  <span class="hljs-string">&#x27;performed_via_github_app&#x27;</span>: <span class="hljs-literal">None</span>}]`}}),be=new O({props:{code:`def get_comments(issue_number):
    url = f"https://api.github.com/repos/huggingface/datasets/issues/{issue_number}/comments"
    response = requests.get(url, headers=headers)
    return [r["body"] for r in response.json()]


# Test our function works as expected
get_comments(2792)`,highlighted:`<span class="hljs-keyword">def</span> <span class="hljs-title function_">get_comments</span>(<span class="hljs-params">issue_number</span>):
    url = <span class="hljs-string">f&quot;https://api.github.com/repos/huggingface/datasets/issues/<span class="hljs-subst">{issue_number}</span>/comments&quot;</span>
    response = requests.get(url, headers=headers)
    <span class="hljs-keyword">return</span> [r[<span class="hljs-string">&quot;body&quot;</span>] <span class="hljs-keyword">for</span> r <span class="hljs-keyword">in</span> response.json()]


<span class="hljs-comment"># Test our function works as expected</span>
get_comments(<span class="hljs-number">2792</span>)`}}),ve=new O({props:{code:"[\"@albertvillanova my tests are failing here:\\r\\n```\\r\\ndataset_name = 'gooaq'\\r\\n\\r\\n    def test_load_dataset(self, dataset_name):\\r\\n        configs = self.dataset_tester.load_all_configs(dataset_name, is_local=True)[:1]\\r\\n>       self.dataset_tester.check_load_dataset(dataset_name, configs, is_local=True, use_local_dummy_data=True)\\r\\n\\r\\ntests/test_dataset_common.py:234: \\r\\n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \\r\\ntests/test_dataset_common.py:187: in check_load_dataset\\r\\n    self.parent.assertTrue(len(dataset[split]) > 0)\\r\\nE   AssertionError: False is not true\\r\\n```\\r\\nWhen I try loading dataset on local machine it works fine. Any suggestions on how can I avoid this error?\"]",highlighted:'[<span class="hljs-string">&quot;@albertvillanova my tests are failing here:\\r\\n```\\r\\ndataset_name = &#x27;gooaq&#x27;\\r\\n\\r\\n    def test_load_dataset(self, dataset_name):\\r\\n        configs = self.dataset_tester.load_all_configs(dataset_name, is_local=True)[:1]\\r\\n&gt;       self.dataset_tester.check_load_dataset(dataset_name, configs, is_local=True, use_local_dummy_data=True)\\r\\n\\r\\ntests/test_dataset_common.py:234: \\r\\n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \\r\\ntests/test_dataset_common.py:187: in check_load_dataset\\r\\n    self.parent.assertTrue(len(dataset[split]) &gt; 0)\\r\\nE   AssertionError: False is not true\\r\\n```\\r\\nWhen I try loading dataset on local machine it works fine. Any suggestions on how can I avoid this error?&quot;</span>]'}}),xe=new O({props:{code:`# Depending on your internet connection, this can take a few minutes...
issues_with_comments_dataset = issues_dataset.map(
    lambda x: {"comments": get_comments(x["number"])}
)`,highlighted:`<span class="hljs-comment"># Depending on your internet connection, this can take a few minutes...</span>
issues_with_comments_dataset = issues_dataset.<span class="hljs-built_in">map</span>(
    <span class="hljs-keyword">lambda</span> x: {<span class="hljs-string">&quot;comments&quot;</span>: get_comments(x[<span class="hljs-string">&quot;number&quot;</span>])}
)`}}),we=new mt({}),ye=new Yu({props:{id:"HaN6qCr_Afc"}}),je=new O({props:{code:`from huggingface_hub import notebook_login

notebook_login()`,highlighted:`<span class="hljs-keyword">from</span> huggingface_hub <span class="hljs-keyword">import</span> notebook_login

notebook_login()`}}),$e=new O({props:{code:"huggingface-cli login",highlighted:"huggingface-cli login"}}),Ee=new O({props:{code:'issues_with_comments_dataset.push_to_hub("github-issues")',highlighted:'issues_with_comments_dataset.push_to_hub(<span class="hljs-string">&quot;github-issues&quot;</span>)'}}),ke=new O({props:{code:`remote_dataset = load_dataset("lewtun/github-issues", split="train")
remote_dataset`,highlighted:`remote_dataset = load_dataset(<span class="hljs-string">&quot;lewtun/github-issues&quot;</span>, split=<span class="hljs-string">&quot;train&quot;</span>)
remote_dataset`}}),qe=new O({props:{code:`Dataset({
    features: ['url', 'repository_url', 'labels_url', 'comments_url', 'events_url', 'html_url', 'id', 'node_id', 'number', 'title', 'user', 'labels', 'state', 'locked', 'assignee', 'assignees', 'milestone', 'comments', 'created_at', 'updated_at', 'closed_at', 'author_association', 'active_lock_reason', 'pull_request', 'body', 'performed_via_github_app', 'is_pull_request'],
    num_rows: 2855
})`,highlighted:`Dataset({
    features: [<span class="hljs-string">&#x27;url&#x27;</span>, <span class="hljs-string">&#x27;repository_url&#x27;</span>, <span class="hljs-string">&#x27;labels_url&#x27;</span>, <span class="hljs-string">&#x27;comments_url&#x27;</span>, <span class="hljs-string">&#x27;events_url&#x27;</span>, <span class="hljs-string">&#x27;html_url&#x27;</span>, <span class="hljs-string">&#x27;id&#x27;</span>, <span class="hljs-string">&#x27;node_id&#x27;</span>, <span class="hljs-string">&#x27;number&#x27;</span>, <span class="hljs-string">&#x27;title&#x27;</span>, <span class="hljs-string">&#x27;user&#x27;</span>, <span class="hljs-string">&#x27;labels&#x27;</span>, <span class="hljs-string">&#x27;state&#x27;</span>, <span class="hljs-string">&#x27;locked&#x27;</span>, <span class="hljs-string">&#x27;assignee&#x27;</span>, <span class="hljs-string">&#x27;assignees&#x27;</span>, <span class="hljs-string">&#x27;milestone&#x27;</span>, <span class="hljs-string">&#x27;comments&#x27;</span>, <span class="hljs-string">&#x27;created_at&#x27;</span>, <span class="hljs-string">&#x27;updated_at&#x27;</span>, <span class="hljs-string">&#x27;closed_at&#x27;</span>, <span class="hljs-string">&#x27;author_association&#x27;</span>, <span class="hljs-string">&#x27;active_lock_reason&#x27;</span>, <span class="hljs-string">&#x27;pull_request&#x27;</span>, <span class="hljs-string">&#x27;body&#x27;</span>, <span class="hljs-string">&#x27;performed_via_github_app&#x27;</span>, <span class="hljs-string">&#x27;is_pull_request&#x27;</span>],
    num_rows: <span class="hljs-number">2855</span>
})`}}),Hs=new ft({props:{$$slots:{default:[Xu]},$$scope:{ctx:U}}}),Te=new mt({}),Ns=new ft({props:{$$slots:{default:[Ku]},$$scope:{ctx:U}}}),Gs=new ft({props:{$$slots:{default:[sp]},$$scope:{ctx:U}}}),{c(){c=i("meta"),T=p(),f=i("h1"),$=i("a"),q=i("span"),_(m.$$.fragment),E=p(),D=i("span"),g=a("Creating your own dataset"),j=p(),_(H.$$.fragment),A=p(),C=i("p"),S=a("Sometimes the dataset that you need to build an NLP application doesn\u2019t exist, so you\u2019ll need to create it yourself. In this section we\u2019ll show you how to create a corpus of "),k=i("a"),P=a("GitHub issues"),z=a(", which are commonly used to track bugs or features in GitHub repositories. This corpus could be used for various purposes, including:"),R=p(),N=i("ul"),B=i("li"),is=a("Exploring how long it takes to close open issues or pull requests"),G=p(),Z=i("li"),Se=a("Training a "),fs=i("em"),Re=a("multilabel classifier"),Ue=a(" that can tag issues with metadata based on the issue\u2019s description (e.g., \u201Cbug,\u201D \u201Cenhancement,\u201D or \u201Cquestion\u201D)"),Fe=p(),gt=i("li"),il=a("Creating a semantic search engine to find which issues match a user\u2019s query"),xa=p(),Le=i("p"),ol=a("Here we\u2019ll focus on creating the corpus, and in the next section we\u2019ll tackle the semantic search application. To keep things meta, we\u2019ll use the GitHub issues associated with a popular open source project: \u{1F917} Datasets! Let\u2019s take a look at how to get the data and explore the information contained in these issues."),wa=p(),os=i("h2"),ms=i("a"),_t=i("span"),_(Us.$$.fragment),rl=p(),bt=i("span"),ul=a("Getting the data"),ya=p(),gs=i("p"),pl=a("You can find all the issues in \u{1F917} Datasets by navigating to the repository\u2019s "),Fs=i("a"),hl=a("Issues tab"),cl=a(". As shown in the following screenshot, at the time of writing there were 331 open issues and 668 closed ones."),ja=p(),Ls=i("div"),Ms=i("img"),$a=p(),Me=i("p"),dl=a("If you click on one of these issues you\u2019ll find it contains a title, a description, and a set of labels that characterize the issue. An example is shown in the screenshot below."),Ea=p(),zs=i("div"),Bs=i("img"),ka=p(),J=i("p"),fl=a("To download all the repository\u2019s issues, we\u2019ll use the "),Ws=i("a"),ml=a("GitHub REST API"),gl=a(" to poll the "),_s=i("a"),vt=i("code"),_l=a("Issues"),bl=a(" endpoint"),vl=a(". This endpoint returns a list of JSON objects, with each object containing a large number of fields that include the title and description as well as metadata about the status of the issue and so on."),qa=p(),bs=i("p"),xl=a("A convenient way to download the issues is via the "),xt=i("code"),wl=a("requests"),yl=a(" library, which is the standard way for making HTTP requests in Python. You can install the library by running:"),Ta=p(),_(Ys.$$.fragment),Da=p(),V=i("p"),jl=a("Once the library is installed, you can make GET requests to the "),wt=i("code"),$l=a("Issues"),El=a(" endpoint by invoking the "),yt=i("code"),kl=a("requests.get()"),ql=a(" function. For example, you can run the following command to retrieve the first issue on the first page:"),Aa=p(),_(Qs.$$.fragment),Ha=p(),vs=i("p"),Tl=a("The "),jt=i("code"),Dl=a("response"),Al=a(" object contains a lot of useful information about the request, including the HTTP status code:"),Oa=p(),_(Zs.$$.fragment),Pa=p(),_(Js.$$.fragment),Ca=p(),W=i("p"),Hl=a("where a "),$t=i("code"),Ol=a("200"),Pl=a(" status means the request was successful (you can find a list of possible HTTP status codes "),Vs=i("a"),Cl=a("here"),Nl=a("). What we are really interested in, though, is the "),Et=i("em"),Gl=a("payload"),Il=a(", which can be accessed in various formats like bytes, strings, or JSON. Since we know our issues are in JSON format, let\u2019s inspect the payload as follows:"),Na=p(),_(Xs.$$.fragment),Ga=p(),_(Ks.$$.fragment),Ia=p(),Y=i("p"),Sl=a("Whoa, that\u2019s a lot of information! We can see useful fields like "),kt=i("code"),Rl=a("title"),Ul=a(", "),qt=i("code"),Fl=a("body"),Ll=a(", and "),Tt=i("code"),Ml=a("number"),zl=a(" that describe the issue, as well as information about the GitHub user who opened the issue."),Sa=p(),_(xs.$$.fragment),Ra=p(),F=i("p"),Bl=a("As described in the GitHub "),se=i("a"),Wl=a("documentation"),Yl=a(", unauthenticated requests are limited to 60 requests per hour. Although you can increase the "),Dt=i("code"),Ql=a("per_page"),Zl=a(" query parameter to reduce the number of requests you make, you will still hit the rate limit on any repository that has more than a few thousand issues. So instead, you should follow GitHub\u2019s "),ee=i("a"),Jl=a("instructions"),Vl=a(" on creating a "),At=i("em"),Xl=a("personal access token"),Kl=a(" so that you can boost the rate limit to 5,000 requests per hour. Once you have your token, you can include it as part of the request header:"),Ua=p(),_(te.$$.fragment),Fa=p(),_(ws.$$.fragment),La=p(),ze=i("p"),si=a("Now that we have our access token, let\u2019s create a function that can download all the issues from a GitHub repository:"),Ma=p(),_(ae.$$.fragment),za=p(),X=i("p"),ei=a("Now when we call "),Ht=i("code"),ti=a("fetch_issues()"),ai=a(" it will download all the issues in batches to avoid exceeding GitHub\u2019s limit on the number of requests per hour; the result will be stored in a "),Ot=i("em"),ni=a("repository_name-issues.jsonl"),li=a(" file, where each line is a JSON object the represents an issue. Let\u2019s use this function to grab all the issues from \u{1F917} Datasets:"),Ba=p(),_(ne.$$.fragment),Wa=p(),ys=i("p"),ii=a("Once the issues are downloaded we can load them locally using our newfound skills from "),Be=i("a"),oi=a("section 2"),ri=a(":"),Ya=p(),_(le.$$.fragment),Qa=p(),_(ie.$$.fragment),Za=p(),K=i("p"),ui=a("Great, we\u2019ve created our first dataset from scratch! But why are there several thousand issues when the "),oe=i("a"),pi=a("Issues tab"),hi=a(" of the \u{1F917} Datasets repository only shows around 1,000 issues in total \u{1F914}? As described in the GitHub "),re=i("a"),ci=a("documentation"),di=a(", that\u2019s because we\u2019ve downloaded all the pull requests as well:"),Ja=p(),We=i("blockquote"),rs=i("p"),fi=a("GitHub\u2019s REST API v3 considers every pull request an issue, but not every issue is a pull request. For this reason, \u201CIssues\u201D endpoints may return both issues and pull requests in the response. You can identify pull requests by the "),Pt=i("code"),mi=a("pull_request"),gi=a(" key. Be aware that the "),Ct=i("code"),_i=a("id"),bi=a(" of a pull request returned from \u201CIssues\u201D endpoints will be an issue id."),Va=p(),Ye=i("p"),vi=a("Since the contents of issues and pull requests are quite different, let\u2019s do some minor preprocessing to enable us to distinguish between them."),Xa=p(),us=i("h2"),js=i("a"),Nt=i("span"),_(ue.$$.fragment),xi=p(),Gt=i("span"),wi=a("Cleaning up the data"),Ka=p(),I=i("p"),yi=a("The above snippet from GitHub\u2019s documentation tells us that the "),It=i("code"),ji=a("pull_request"),$i=a(" column can be used to differentiate between issues and pull requests. Let\u2019s look at a random sample to see what the difference is. As we did in "),Qe=i("a"),Ei=a("section 3"),ki=a(", we\u2019ll chain "),St=i("code"),qi=a("Dataset.shuffle()"),Ti=a(" and "),Rt=i("code"),Di=a("Dataset.select()"),Ai=a(" to create a random sample and then zip the "),Ut=i("code"),Hi=a("html_url"),Oi=a(" and "),Ft=i("code"),Pi=a("pull_request"),Ci=a(" columns so we can compare the various URLs:"),sn=p(),_(pe.$$.fragment),en=p(),_(he.$$.fragment),tn=p(),L=i("p"),Ni=a("Here we can see that each pull request is associated with various URLs, while ordinary issues have a "),Lt=i("code"),Gi=a("None"),Ii=a(" entry. We can use this distinction to create a new "),Mt=i("code"),Si=a("is_pull_request"),Ri=a(" column that checks whether the "),zt=i("code"),Ui=a("pull_request"),Fi=a(" field is "),Bt=i("code"),Li=a("None"),Mi=a(" or not:"),an=p(),_(ce.$$.fragment),nn=p(),_($s.$$.fragment),ln=p(),Ze=i("p"),zi=a("Although we could proceed to further clean up the dataset by dropping or renaming some columns, it is generally a good practice to keep the dataset as \u201Craw\u201D as possible at this stage so that it can be easily used in multiple applications."),on=p(),Je=i("p"),Bi=a("Before we push our dataset to the Hugging Face Hub, let\u2019s deal with one thing that\u2019s missing from it: the comments associated with each issue and pull request. We\u2019ll add them next with \u2014 you guessed it \u2014 the GitHub REST API!"),rn=p(),ps=i("h2"),Es=i("a"),Wt=i("span"),_(de.$$.fragment),Wi=p(),Yt=i("span"),Yi=a("Augmenting the dataset"),un=p(),Ve=i("p"),Qi=a("As shown in the following screenshot, the comments associated with an issue or pull request provide a rich source of information, especially if we\u2019re interested in building a search engine to answer user queries about the library."),pn=p(),fe=i("div"),me=i("img"),hn=p(),ks=i("p"),Zi=a("The GitHub REST API provides a "),qs=i("a"),Qt=i("code"),Ji=a("Comments"),Vi=a(" endpoint"),Xi=a(" that returns all the comments associated with an issue number. Let\u2019s test the endpoint to see what it returns:"),cn=p(),_(ge.$$.fragment),dn=p(),_(_e.$$.fragment),fn=p(),Q=i("p"),Ki=a("We can see that the comment is stored in the "),Zt=i("code"),so=a("body"),eo=a(" field, so let\u2019s write a simple function that returns all the comments associated with an issue by picking out the "),Jt=i("code"),to=a("body"),ao=a(" contents for each element in "),Vt=i("code"),no=a("response.json()"),lo=a(":"),mn=p(),_(be.$$.fragment),gn=p(),_(ve.$$.fragment),_n=p(),ss=i("p"),io=a("This looks good, so let\u2019s use "),Xt=i("code"),oo=a("Dataset.map()"),ro=a(" to add a new "),Kt=i("code"),uo=a("comments"),po=a(" column to each issue in our dataset:"),bn=p(),_(xe.$$.fragment),vn=p(),Xe=i("p"),ho=a("The final step is to push our dataset to the Hub. Let\u2019s take a look at how we can do that."),xn=p(),hs=i("h2"),Ts=i("a"),sa=i("span"),_(we.$$.fragment),co=p(),ea=i("span"),fo=a("Uploading the dataset to the Hugging Face Hub"),wn=p(),_(ye.$$.fragment),yn=p(),es=i("p"),mo=a("Now that we have our augmented dataset, it\u2019s time to push it to the Hub so we can share it with the community! Uploading a dataset is very simple: just like models and tokenizers from \u{1F917} Transformers, we can use a "),ta=i("code"),go=a("push_to_hub()"),_o=a(" method to push a dataset. To do that we need an authentication token, which can be obtained by first logging into the Hugging Face Hub with the "),aa=i("code"),bo=a("notebook_login()"),vo=a(" function:"),jn=p(),_(je.$$.fragment),$n=p(),Ds=i("p"),xo=a("This will create a widget where you can enter your username and password, and an API token will be saved in "),na=i("em"),wo=a("~/.huggingface/token"),yo=a(". If you\u2019re running the code in a terminal, you can log in via the CLI instead:"),En=p(),_($e.$$.fragment),kn=p(),Ke=i("p"),jo=a("Once we\u2019ve done this, we can upload our dataset by running:"),qn=p(),_(Ee.$$.fragment),Tn=p(),ts=i("p"),$o=a("From here, anyone can download the dataset by simply providing "),la=i("code"),Eo=a("load_dataset()"),ko=a(" with the repository ID as the "),ia=i("code"),qo=a("path"),To=a(" argument:"),Dn=p(),_(ke.$$.fragment),An=p(),_(qe.$$.fragment),Hn=p(),As=i("p"),Do=a("Cool, we\u2019ve pushed our dataset to the Hub and it\u2019s available for others to use! There\u2019s just one important thing left to do: adding a "),oa=i("em"),Ao=a("dataset card"),Ho=a(" that explains how the corpus was created and provides other useful information for the community."),On=p(),_(Hs.$$.fragment),Pn=p(),cs=i("h2"),Os=i("a"),ra=i("span"),_(Te.$$.fragment),Oo=p(),ua=i("span"),Po=a("Creating a dataset card"),Cn=p(),st=i("p"),Co=a("Well-documented datasets are more likely to be useful to others (including your future self!), as they provide the context to enable users to decide whether the dataset is relevant to their task and to evaluate any potential biases in or risks associated with using the dataset."),Nn=p(),Ps=i("p"),No=a("On the Hugging Face Hub, this information is stored in each dataset repository\u2019s "),pa=i("em"),Go=a("README.md"),Io=a(" file. There are two main steps you should take before creating this file:"),Gn=p(),et=i("ol"),ds=i("li"),So=a("Use the "),Cs=i("a"),ha=i("code"),Ro=a("datasets-tagging"),Uo=a(" application"),Fo=a(" to create metadata tags in YAML format. These tags are used for a variety of search features on the Hugging Face Hub and ensure your dataset can be easily found by members of the community. Since we have created a custom dataset here, you\u2019ll need to clone the "),ca=i("code"),Lo=a("datasets-tagging"),Mo=a(" repository and run the application locally. Here\u2019s what the interface looks like:"),In=p(),De=i("div"),Ae=i("img"),Sn=p(),He=i("ol"),Oe=i("li"),zo=a("Read the "),Pe=i("a"),Bo=a("\u{1F917} Datasets guide"),Wo=a(" on creating informative dataset cards and use it as a template."),Rn=p(),as=i("p"),Yo=a("You can create the "),da=i("em"),Qo=a("README.md"),Zo=a(" file directly on the Hub, and you can find a template dataset card in the "),fa=i("code"),Jo=a("lewtun/github-issues"),Vo=a(" dataset repository. A screenshot of the filled-out dataset card is shown below."),Un=p(),Ce=i("div"),Ne=i("img"),Fn=p(),_(Ns.$$.fragment),Ln=p(),tt=i("p"),Xo=a("That\u2019s it! We\u2019ve seen in this section that creating a good dataset can be quite involved, but fortunately uploading it and sharing it with the community is not. In the next section we\u2019ll use our new dataset to create a semantic search engine with \u{1F917} Datasets that can match questions to the most relevant issues and comments."),Mn=p(),_(Gs.$$.fragment),this.h()},l(s){const l=Bu('[data-svelte="svelte-1phssyn"]',document.head);c=o(l,"META",{name:!0,content:!0}),l.forEach(t),T=h(s),f=o(s,"H1",{class:!0});var Ge=r(f);$=o(Ge,"A",{id:!0,class:!0,href:!0});var ma=r($);q=o(ma,"SPAN",{});var ga=r(q);b(m.$$.fragment,ga),ga.forEach(t),ma.forEach(t),E=h(Ge),D=o(Ge,"SPAN",{});var _a=r(D);g=n(_a,"Creating your own dataset"),_a.forEach(t),Ge.forEach(t),j=h(s),b(H.$$.fragment,s),A=h(s),C=o(s,"P",{});var Ie=r(C);S=n(Ie,"Sometimes the dataset that you need to build an NLP application doesn\u2019t exist, so you\u2019ll need to create it yourself. In this section we\u2019ll show you how to create a corpus of "),k=o(Ie,"A",{href:!0,rel:!0});var ba=r(k);P=n(ba,"GitHub issues"),ba.forEach(t),z=n(Ie,", which are commonly used to track bugs or features in GitHub repositories. This corpus could be used for various purposes, including:"),Ie.forEach(t),R=h(s),N=o(s,"UL",{});var at=r(N);B=o(at,"LI",{});var or=r(B);is=n(or,"Exploring how long it takes to close open issues or pull requests"),or.forEach(t),G=h(at),Z=o(at,"LI",{});var Bn=r(Z);Se=n(Bn,"Training a "),fs=o(Bn,"EM",{});var rr=r(fs);Re=n(rr,"multilabel classifier"),rr.forEach(t),Ue=n(Bn," that can tag issues with metadata based on the issue\u2019s description (e.g., \u201Cbug,\u201D \u201Cenhancement,\u201D or \u201Cquestion\u201D)"),Bn.forEach(t),Fe=h(at),gt=o(at,"LI",{});var ur=r(gt);il=n(ur,"Creating a semantic search engine to find which issues match a user\u2019s query"),ur.forEach(t),at.forEach(t),xa=h(s),Le=o(s,"P",{});var pr=r(Le);ol=n(pr,"Here we\u2019ll focus on creating the corpus, and in the next section we\u2019ll tackle the semantic search application. To keep things meta, we\u2019ll use the GitHub issues associated with a popular open source project: \u{1F917} Datasets! Let\u2019s take a look at how to get the data and explore the information contained in these issues."),pr.forEach(t),wa=h(s),os=o(s,"H2",{class:!0});var Wn=r(os);ms=o(Wn,"A",{id:!0,class:!0,href:!0});var hr=r(ms);_t=o(hr,"SPAN",{});var cr=r(_t);b(Us.$$.fragment,cr),cr.forEach(t),hr.forEach(t),rl=h(Wn),bt=o(Wn,"SPAN",{});var dr=r(bt);ul=n(dr,"Getting the data"),dr.forEach(t),Wn.forEach(t),ya=h(s),gs=o(s,"P",{});var Yn=r(gs);pl=n(Yn,"You can find all the issues in \u{1F917} Datasets by navigating to the repository\u2019s "),Fs=o(Yn,"A",{href:!0,rel:!0});var fr=r(Fs);hl=n(fr,"Issues tab"),fr.forEach(t),cl=n(Yn,". As shown in the following screenshot, at the time of writing there were 331 open issues and 668 closed ones."),Yn.forEach(t),ja=h(s),Ls=o(s,"DIV",{class:!0});var mr=r(Ls);Ms=o(mr,"IMG",{src:!0,alt:!0,width:!0}),mr.forEach(t),$a=h(s),Me=o(s,"P",{});var gr=r(Me);dl=n(gr,"If you click on one of these issues you\u2019ll find it contains a title, a description, and a set of labels that characterize the issue. An example is shown in the screenshot below."),gr.forEach(t),Ea=h(s),zs=o(s,"DIV",{class:!0});var _r=r(zs);Bs=o(_r,"IMG",{src:!0,alt:!0,width:!0}),_r.forEach(t),ka=h(s),J=o(s,"P",{});var nt=r(J);fl=n(nt,"To download all the repository\u2019s issues, we\u2019ll use the "),Ws=o(nt,"A",{href:!0,rel:!0});var br=r(Ws);ml=n(br,"GitHub REST API"),br.forEach(t),gl=n(nt," to poll the "),_s=o(nt,"A",{href:!0,rel:!0});var Ko=r(_s);vt=o(Ko,"CODE",{});var vr=r(vt);_l=n(vr,"Issues"),vr.forEach(t),bl=n(Ko," endpoint"),Ko.forEach(t),vl=n(nt,". This endpoint returns a list of JSON objects, with each object containing a large number of fields that include the title and description as well as metadata about the status of the issue and so on."),nt.forEach(t),qa=h(s),bs=o(s,"P",{});var Qn=r(bs);xl=n(Qn,"A convenient way to download the issues is via the "),xt=o(Qn,"CODE",{});var xr=r(xt);wl=n(xr,"requests"),xr.forEach(t),yl=n(Qn," library, which is the standard way for making HTTP requests in Python. You can install the library by running:"),Qn.forEach(t),Ta=h(s),b(Ys.$$.fragment,s),Da=h(s),V=o(s,"P",{});var lt=r(V);jl=n(lt,"Once the library is installed, you can make GET requests to the "),wt=o(lt,"CODE",{});var wr=r(wt);$l=n(wr,"Issues"),wr.forEach(t),El=n(lt," endpoint by invoking the "),yt=o(lt,"CODE",{});var yr=r(yt);kl=n(yr,"requests.get()"),yr.forEach(t),ql=n(lt," function. For example, you can run the following command to retrieve the first issue on the first page:"),lt.forEach(t),Aa=h(s),b(Qs.$$.fragment,s),Ha=h(s),vs=o(s,"P",{});var Zn=r(vs);Tl=n(Zn,"The "),jt=o(Zn,"CODE",{});var jr=r(jt);Dl=n(jr,"response"),jr.forEach(t),Al=n(Zn," object contains a lot of useful information about the request, including the HTTP status code:"),Zn.forEach(t),Oa=h(s),b(Zs.$$.fragment,s),Pa=h(s),b(Js.$$.fragment,s),Ca=h(s),W=o(s,"P",{});var Is=r(W);Hl=n(Is,"where a "),$t=o(Is,"CODE",{});var $r=r($t);Ol=n($r,"200"),$r.forEach(t),Pl=n(Is," status means the request was successful (you can find a list of possible HTTP status codes "),Vs=o(Is,"A",{href:!0,rel:!0});var Er=r(Vs);Cl=n(Er,"here"),Er.forEach(t),Nl=n(Is,"). What we are really interested in, though, is the "),Et=o(Is,"EM",{});var kr=r(Et);Gl=n(kr,"payload"),kr.forEach(t),Il=n(Is,", which can be accessed in various formats like bytes, strings, or JSON. Since we know our issues are in JSON format, let\u2019s inspect the payload as follows:"),Is.forEach(t),Na=h(s),b(Xs.$$.fragment,s),Ga=h(s),b(Ks.$$.fragment,s),Ia=h(s),Y=o(s,"P",{});var Ss=r(Y);Sl=n(Ss,"Whoa, that\u2019s a lot of information! We can see useful fields like "),kt=o(Ss,"CODE",{});var qr=r(kt);Rl=n(qr,"title"),qr.forEach(t),Ul=n(Ss,", "),qt=o(Ss,"CODE",{});var Tr=r(qt);Fl=n(Tr,"body"),Tr.forEach(t),Ll=n(Ss,", and "),Tt=o(Ss,"CODE",{});var Dr=r(Tt);Ml=n(Dr,"number"),Dr.forEach(t),zl=n(Ss," that describe the issue, as well as information about the GitHub user who opened the issue."),Ss.forEach(t),Sa=h(s),b(xs.$$.fragment,s),Ra=h(s),F=o(s,"P",{});var ns=r(F);Bl=n(ns,"As described in the GitHub "),se=o(ns,"A",{href:!0,rel:!0});var Ar=r(se);Wl=n(Ar,"documentation"),Ar.forEach(t),Yl=n(ns,", unauthenticated requests are limited to 60 requests per hour. Although you can increase the "),Dt=o(ns,"CODE",{});var Hr=r(Dt);Ql=n(Hr,"per_page"),Hr.forEach(t),Zl=n(ns," query parameter to reduce the number of requests you make, you will still hit the rate limit on any repository that has more than a few thousand issues. So instead, you should follow GitHub\u2019s "),ee=o(ns,"A",{href:!0,rel:!0});var Or=r(ee);Jl=n(Or,"instructions"),Or.forEach(t),Vl=n(ns," on creating a "),At=o(ns,"EM",{});var Pr=r(At);Xl=n(Pr,"personal access token"),Pr.forEach(t),Kl=n(ns," so that you can boost the rate limit to 5,000 requests per hour. Once you have your token, you can include it as part of the request header:"),ns.forEach(t),Ua=h(s),b(te.$$.fragment,s),Fa=h(s),b(ws.$$.fragment,s),La=h(s),ze=o(s,"P",{});var Cr=r(ze);si=n(Cr,"Now that we have our access token, let\u2019s create a function that can download all the issues from a GitHub repository:"),Cr.forEach(t),Ma=h(s),b(ae.$$.fragment,s),za=h(s),X=o(s,"P",{});var it=r(X);ei=n(it,"Now when we call "),Ht=o(it,"CODE",{});var Nr=r(Ht);ti=n(Nr,"fetch_issues()"),Nr.forEach(t),ai=n(it," it will download all the issues in batches to avoid exceeding GitHub\u2019s limit on the number of requests per hour; the result will be stored in a "),Ot=o(it,"EM",{});var Gr=r(Ot);ni=n(Gr,"repository_name-issues.jsonl"),Gr.forEach(t),li=n(it," file, where each line is a JSON object the represents an issue. Let\u2019s use this function to grab all the issues from \u{1F917} Datasets:"),it.forEach(t),Ba=h(s),b(ne.$$.fragment,s),Wa=h(s),ys=o(s,"P",{});var Jn=r(ys);ii=n(Jn,"Once the issues are downloaded we can load them locally using our newfound skills from "),Be=o(Jn,"A",{href:!0});var Ir=r(Be);oi=n(Ir,"section 2"),Ir.forEach(t),ri=n(Jn,":"),Jn.forEach(t),Ya=h(s),b(le.$$.fragment,s),Qa=h(s),b(ie.$$.fragment,s),Za=h(s),K=o(s,"P",{});var ot=r(K);ui=n(ot,"Great, we\u2019ve created our first dataset from scratch! But why are there several thousand issues when the "),oe=o(ot,"A",{href:!0,rel:!0});var Sr=r(oe);pi=n(Sr,"Issues tab"),Sr.forEach(t),hi=n(ot," of the \u{1F917} Datasets repository only shows around 1,000 issues in total \u{1F914}? As described in the GitHub "),re=o(ot,"A",{href:!0,rel:!0});var Rr=r(re);ci=n(Rr,"documentation"),Rr.forEach(t),di=n(ot,", that\u2019s because we\u2019ve downloaded all the pull requests as well:"),ot.forEach(t),Ja=h(s),We=o(s,"BLOCKQUOTE",{});var Ur=r(We);rs=o(Ur,"P",{});var rt=r(rs);fi=n(rt,"GitHub\u2019s REST API v3 considers every pull request an issue, but not every issue is a pull request. For this reason, \u201CIssues\u201D endpoints may return both issues and pull requests in the response. You can identify pull requests by the "),Pt=o(rt,"CODE",{});var Fr=r(Pt);mi=n(Fr,"pull_request"),Fr.forEach(t),gi=n(rt," key. Be aware that the "),Ct=o(rt,"CODE",{});var Lr=r(Ct);_i=n(Lr,"id"),Lr.forEach(t),bi=n(rt," of a pull request returned from \u201CIssues\u201D endpoints will be an issue id."),rt.forEach(t),Ur.forEach(t),Va=h(s),Ye=o(s,"P",{});var Mr=r(Ye);vi=n(Mr,"Since the contents of issues and pull requests are quite different, let\u2019s do some minor preprocessing to enable us to distinguish between them."),Mr.forEach(t),Xa=h(s),us=o(s,"H2",{class:!0});var Vn=r(us);js=o(Vn,"A",{id:!0,class:!0,href:!0});var zr=r(js);Nt=o(zr,"SPAN",{});var Br=r(Nt);b(ue.$$.fragment,Br),Br.forEach(t),zr.forEach(t),xi=h(Vn),Gt=o(Vn,"SPAN",{});var Wr=r(Gt);wi=n(Wr,"Cleaning up the data"),Wr.forEach(t),Vn.forEach(t),Ka=h(s),I=o(s,"P",{});var M=r(I);yi=n(M,"The above snippet from GitHub\u2019s documentation tells us that the "),It=o(M,"CODE",{});var Yr=r(It);ji=n(Yr,"pull_request"),Yr.forEach(t),$i=n(M," column can be used to differentiate between issues and pull requests. Let\u2019s look at a random sample to see what the difference is. As we did in "),Qe=o(M,"A",{href:!0});var Qr=r(Qe);Ei=n(Qr,"section 3"),Qr.forEach(t),ki=n(M,", we\u2019ll chain "),St=o(M,"CODE",{});var Zr=r(St);qi=n(Zr,"Dataset.shuffle()"),Zr.forEach(t),Ti=n(M," and "),Rt=o(M,"CODE",{});var Jr=r(Rt);Di=n(Jr,"Dataset.select()"),Jr.forEach(t),Ai=n(M," to create a random sample and then zip the "),Ut=o(M,"CODE",{});var Vr=r(Ut);Hi=n(Vr,"html_url"),Vr.forEach(t),Oi=n(M," and "),Ft=o(M,"CODE",{});var Xr=r(Ft);Pi=n(Xr,"pull_request"),Xr.forEach(t),Ci=n(M," columns so we can compare the various URLs:"),M.forEach(t),sn=h(s),b(pe.$$.fragment,s),en=h(s),b(he.$$.fragment,s),tn=h(s),L=o(s,"P",{});var ls=r(L);Ni=n(ls,"Here we can see that each pull request is associated with various URLs, while ordinary issues have a "),Lt=o(ls,"CODE",{});var Kr=r(Lt);Gi=n(Kr,"None"),Kr.forEach(t),Ii=n(ls," entry. We can use this distinction to create a new "),Mt=o(ls,"CODE",{});var su=r(Mt);Si=n(su,"is_pull_request"),su.forEach(t),Ri=n(ls," column that checks whether the "),zt=o(ls,"CODE",{});var eu=r(zt);Ui=n(eu,"pull_request"),eu.forEach(t),Fi=n(ls," field is "),Bt=o(ls,"CODE",{});var tu=r(Bt);Li=n(tu,"None"),tu.forEach(t),Mi=n(ls," or not:"),ls.forEach(t),an=h(s),b(ce.$$.fragment,s),nn=h(s),b($s.$$.fragment,s),ln=h(s),Ze=o(s,"P",{});var au=r(Ze);zi=n(au,"Although we could proceed to further clean up the dataset by dropping or renaming some columns, it is generally a good practice to keep the dataset as \u201Craw\u201D as possible at this stage so that it can be easily used in multiple applications."),au.forEach(t),on=h(s),Je=o(s,"P",{});var nu=r(Je);Bi=n(nu,"Before we push our dataset to the Hugging Face Hub, let\u2019s deal with one thing that\u2019s missing from it: the comments associated with each issue and pull request. We\u2019ll add them next with \u2014 you guessed it \u2014 the GitHub REST API!"),nu.forEach(t),rn=h(s),ps=o(s,"H2",{class:!0});var Xn=r(ps);Es=o(Xn,"A",{id:!0,class:!0,href:!0});var lu=r(Es);Wt=o(lu,"SPAN",{});var iu=r(Wt);b(de.$$.fragment,iu),iu.forEach(t),lu.forEach(t),Wi=h(Xn),Yt=o(Xn,"SPAN",{});var ou=r(Yt);Yi=n(ou,"Augmenting the dataset"),ou.forEach(t),Xn.forEach(t),un=h(s),Ve=o(s,"P",{});var ru=r(Ve);Qi=n(ru,"As shown in the following screenshot, the comments associated with an issue or pull request provide a rich source of information, especially if we\u2019re interested in building a search engine to answer user queries about the library."),ru.forEach(t),pn=h(s),fe=o(s,"DIV",{class:!0});var uu=r(fe);me=o(uu,"IMG",{src:!0,alt:!0,width:!0}),uu.forEach(t),hn=h(s),ks=o(s,"P",{});var Kn=r(ks);Zi=n(Kn,"The GitHub REST API provides a "),qs=o(Kn,"A",{href:!0,rel:!0});var sr=r(qs);Qt=o(sr,"CODE",{});var pu=r(Qt);Ji=n(pu,"Comments"),pu.forEach(t),Vi=n(sr," endpoint"),sr.forEach(t),Xi=n(Kn," that returns all the comments associated with an issue number. Let\u2019s test the endpoint to see what it returns:"),Kn.forEach(t),cn=h(s),b(ge.$$.fragment,s),dn=h(s),b(_e.$$.fragment,s),fn=h(s),Q=o(s,"P",{});var Rs=r(Q);Ki=n(Rs,"We can see that the comment is stored in the "),Zt=o(Rs,"CODE",{});var hu=r(Zt);so=n(hu,"body"),hu.forEach(t),eo=n(Rs," field, so let\u2019s write a simple function that returns all the comments associated with an issue by picking out the "),Jt=o(Rs,"CODE",{});var cu=r(Jt);to=n(cu,"body"),cu.forEach(t),ao=n(Rs," contents for each element in "),Vt=o(Rs,"CODE",{});var du=r(Vt);no=n(du,"response.json()"),du.forEach(t),lo=n(Rs,":"),Rs.forEach(t),mn=h(s),b(be.$$.fragment,s),gn=h(s),b(ve.$$.fragment,s),_n=h(s),ss=o(s,"P",{});var ut=r(ss);io=n(ut,"This looks good, so let\u2019s use "),Xt=o(ut,"CODE",{});var fu=r(Xt);oo=n(fu,"Dataset.map()"),fu.forEach(t),ro=n(ut," to add a new "),Kt=o(ut,"CODE",{});var mu=r(Kt);uo=n(mu,"comments"),mu.forEach(t),po=n(ut," column to each issue in our dataset:"),ut.forEach(t),bn=h(s),b(xe.$$.fragment,s),vn=h(s),Xe=o(s,"P",{});var gu=r(Xe);ho=n(gu,"The final step is to push our dataset to the Hub. Let\u2019s take a look at how we can do that."),gu.forEach(t),xn=h(s),hs=o(s,"H2",{class:!0});var sl=r(hs);Ts=o(sl,"A",{id:!0,class:!0,href:!0});var _u=r(Ts);sa=o(_u,"SPAN",{});var bu=r(sa);b(we.$$.fragment,bu),bu.forEach(t),_u.forEach(t),co=h(sl),ea=o(sl,"SPAN",{});var vu=r(ea);fo=n(vu,"Uploading the dataset to the Hugging Face Hub"),vu.forEach(t),sl.forEach(t),wn=h(s),b(ye.$$.fragment,s),yn=h(s),es=o(s,"P",{});var pt=r(es);mo=n(pt,"Now that we have our augmented dataset, it\u2019s time to push it to the Hub so we can share it with the community! Uploading a dataset is very simple: just like models and tokenizers from \u{1F917} Transformers, we can use a "),ta=o(pt,"CODE",{});var xu=r(ta);go=n(xu,"push_to_hub()"),xu.forEach(t),_o=n(pt," method to push a dataset. To do that we need an authentication token, which can be obtained by first logging into the Hugging Face Hub with the "),aa=o(pt,"CODE",{});var wu=r(aa);bo=n(wu,"notebook_login()"),wu.forEach(t),vo=n(pt," function:"),pt.forEach(t),jn=h(s),b(je.$$.fragment,s),$n=h(s),Ds=o(s,"P",{});var el=r(Ds);xo=n(el,"This will create a widget where you can enter your username and password, and an API token will be saved in "),na=o(el,"EM",{});var yu=r(na);wo=n(yu,"~/.huggingface/token"),yu.forEach(t),yo=n(el,". If you\u2019re running the code in a terminal, you can log in via the CLI instead:"),el.forEach(t),En=h(s),b($e.$$.fragment,s),kn=h(s),Ke=o(s,"P",{});var ju=r(Ke);jo=n(ju,"Once we\u2019ve done this, we can upload our dataset by running:"),ju.forEach(t),qn=h(s),b(Ee.$$.fragment,s),Tn=h(s),ts=o(s,"P",{});var ht=r(ts);$o=n(ht,"From here, anyone can download the dataset by simply providing "),la=o(ht,"CODE",{});var $u=r(la);Eo=n($u,"load_dataset()"),$u.forEach(t),ko=n(ht," with the repository ID as the "),ia=o(ht,"CODE",{});var Eu=r(ia);qo=n(Eu,"path"),Eu.forEach(t),To=n(ht," argument:"),ht.forEach(t),Dn=h(s),b(ke.$$.fragment,s),An=h(s),b(qe.$$.fragment,s),Hn=h(s),As=o(s,"P",{});var tl=r(As);Do=n(tl,"Cool, we\u2019ve pushed our dataset to the Hub and it\u2019s available for others to use! There\u2019s just one important thing left to do: adding a "),oa=o(tl,"EM",{});var ku=r(oa);Ao=n(ku,"dataset card"),ku.forEach(t),Ho=n(tl," that explains how the corpus was created and provides other useful information for the community."),tl.forEach(t),On=h(s),b(Hs.$$.fragment,s),Pn=h(s),cs=o(s,"H2",{class:!0});var al=r(cs);Os=o(al,"A",{id:!0,class:!0,href:!0});var qu=r(Os);ra=o(qu,"SPAN",{});var Tu=r(ra);b(Te.$$.fragment,Tu),Tu.forEach(t),qu.forEach(t),Oo=h(al),ua=o(al,"SPAN",{});var Du=r(ua);Po=n(Du,"Creating a dataset card"),Du.forEach(t),al.forEach(t),Cn=h(s),st=o(s,"P",{});var Au=r(st);Co=n(Au,"Well-documented datasets are more likely to be useful to others (including your future self!), as they provide the context to enable users to decide whether the dataset is relevant to their task and to evaluate any potential biases in or risks associated with using the dataset."),Au.forEach(t),Nn=h(s),Ps=o(s,"P",{});var nl=r(Ps);No=n(nl,"On the Hugging Face Hub, this information is stored in each dataset repository\u2019s "),pa=o(nl,"EM",{});var Hu=r(pa);Go=n(Hu,"README.md"),Hu.forEach(t),Io=n(nl," file. There are two main steps you should take before creating this file:"),nl.forEach(t),Gn=h(s),et=o(s,"OL",{});var Ou=r(et);ds=o(Ou,"LI",{});var ct=r(ds);So=n(ct,"Use the "),Cs=o(ct,"A",{href:!0,rel:!0});var er=r(Cs);ha=o(er,"CODE",{});var Pu=r(ha);Ro=n(Pu,"datasets-tagging"),Pu.forEach(t),Uo=n(er," application"),er.forEach(t),Fo=n(ct," to create metadata tags in YAML format. These tags are used for a variety of search features on the Hugging Face Hub and ensure your dataset can be easily found by members of the community. Since we have created a custom dataset here, you\u2019ll need to clone the "),ca=o(ct,"CODE",{});var Cu=r(ca);Lo=n(Cu,"datasets-tagging"),Cu.forEach(t),Mo=n(ct," repository and run the application locally. Here\u2019s what the interface looks like:"),ct.forEach(t),Ou.forEach(t),In=h(s),De=o(s,"DIV",{class:!0});var Nu=r(De);Ae=o(Nu,"IMG",{src:!0,alt:!0,width:!0}),Nu.forEach(t),Sn=h(s),He=o(s,"OL",{start:!0});var Gu=r(He);Oe=o(Gu,"LI",{});var ll=r(Oe);zo=n(ll,"Read the "),Pe=o(ll,"A",{href:!0,rel:!0});var Iu=r(Pe);Bo=n(Iu,"\u{1F917} Datasets guide"),Iu.forEach(t),Wo=n(ll," on creating informative dataset cards and use it as a template."),ll.forEach(t),Gu.forEach(t),Rn=h(s),as=o(s,"P",{});var dt=r(as);Yo=n(dt,"You can create the "),da=o(dt,"EM",{});var Su=r(da);Qo=n(Su,"README.md"),Su.forEach(t),Zo=n(dt," file directly on the Hub, and you can find a template dataset card in the "),fa=o(dt,"CODE",{});var Ru=r(fa);Jo=n(Ru,"lewtun/github-issues"),Ru.forEach(t),Vo=n(dt," dataset repository. A screenshot of the filled-out dataset card is shown below."),dt.forEach(t),Un=h(s),Ce=o(s,"DIV",{class:!0});var Uu=r(Ce);Ne=o(Uu,"IMG",{src:!0,alt:!0,width:!0}),Uu.forEach(t),Fn=h(s),b(Ns.$$.fragment,s),Ln=h(s),tt=o(s,"P",{});var Fu=r(tt);Xo=n(Fu,"That\u2019s it! We\u2019ve seen in this section that creating a good dataset can be quite involved, but fortunately uploading it and sharing it with the community is not. In the next section we\u2019ll use our new dataset to create a semantic search engine with \u{1F917} Datasets that can match questions to the most relevant issues and comments."),Fu.forEach(t),Mn=h(s),b(Gs.$$.fragment,s),this.h()},h(){d(c,"name","hf:doc:metadata"),d(c,"content",JSON.stringify(tp)),d($,"id","creating-your-own-dataset"),d($,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),d($,"href","#creating-your-own-dataset"),d(f,"class","relative group"),d(k,"href","https://github.com/features/issues/"),d(k,"rel","nofollow"),d(ms,"id","getting-the-data"),d(ms,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),d(ms,"href","#getting-the-data"),d(os,"class","relative group"),d(Fs,"href","https://github.com/huggingface/datasets/issues"),d(Fs,"rel","nofollow"),va(Ms.src,tr="https://huggingface.co/datasets/huggingface-course/documentation-images/resolve/main/en/chapter5/datasets-issues.png")||d(Ms,"src",tr),d(Ms,"alt","The GitHub issues associated with \u{1F917} Datasets."),d(Ms,"width","80%"),d(Ls,"class","flex justify-center"),va(Bs.src,ar="https://huggingface.co/datasets/huggingface-course/documentation-images/resolve/main/en/chapter5/datasets-issues-single.png")||d(Bs,"src",ar),d(Bs,"alt","A typical GitHub issue in the \u{1F917} Datasets repository."),d(Bs,"width","80%"),d(zs,"class","flex justify-center"),d(Ws,"href","https://docs.github.com/en/rest"),d(Ws,"rel","nofollow"),d(_s,"href","https://docs.github.com/en/rest/reference/issues#list-repository-issues"),d(_s,"rel","nofollow"),d(Vs,"href","https://en.wikipedia.org/wiki/List_of_HTTP_status_codes"),d(Vs,"rel","nofollow"),d(se,"href","https://docs.github.com/en/rest/overview/resources-in-the-rest-api#rate-limiting"),d(se,"rel","nofollow"),d(ee,"href","https://docs.github.com/en/github/authenticating-to-github/creating-a-personal-access-token"),d(ee,"rel","nofollow"),d(Be,"href","/course/chaper5/2"),d(oe,"href","https://github.com/huggingface/datasets/issues"),d(oe,"rel","nofollow"),d(re,"href","https://docs.github.com/en/rest/reference/issues#list-issues-assigned-to-the-authenticated-user"),d(re,"rel","nofollow"),d(js,"id","cleaning-up-the-data"),d(js,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),d(js,"href","#cleaning-up-the-data"),d(us,"class","relative group"),d(Qe,"href","/course/chapter5/3"),d(Es,"id","augmenting-the-dataset"),d(Es,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),d(Es,"href","#augmenting-the-dataset"),d(ps,"class","relative group"),va(me.src,nr="https://huggingface.co/datasets/huggingface-course/documentation-images/resolve/main/en/chapter5/datasets-issues-comment.png")||d(me,"src",nr),d(me,"alt","Comments associated with an issue about \u{1F917} Datasets."),d(me,"width","80%"),d(fe,"class","flex justify-center"),d(qs,"href","https://docs.github.com/en/rest/reference/issues#list-issue-comments"),d(qs,"rel","nofollow"),d(Ts,"id","uploading-the-dataset-to-the-hugging-face-hub"),d(Ts,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),d(Ts,"href","#uploading-the-dataset-to-the-hugging-face-hub"),d(hs,"class","relative group"),d(Os,"id","creating-a-dataset-card"),d(Os,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),d(Os,"href","#creating-a-dataset-card"),d(cs,"class","relative group"),d(Cs,"href","https://huggingface.co/datasets/tagging/"),d(Cs,"rel","nofollow"),va(Ae.src,lr="https://huggingface.co/datasets/huggingface-course/documentation-images/resolve/main/en/chapter5/datasets-tagger.png")||d(Ae,"src",lr),d(Ae,"alt","The `datasets-tagging` interface."),d(Ae,"width","80%"),d(De,"class","flex justify-center"),d(Pe,"href","https://github.com/huggingface/datasets/blob/master/templates/README_guide.md"),d(Pe,"rel","nofollow"),d(He,"start","2"),va(Ne.src,ir="https://huggingface.co/datasets/huggingface-course/documentation-images/resolve/main/en/chapter5/dataset-card.png")||d(Ne,"src",ir),d(Ne,"alt","A dataset card."),d(Ne,"width","80%"),d(Ce,"class","flex justify-center")},m(s,l){e(document.head,c),u(s,T,l),u(s,f,l),e(f,$),e($,q),v(m,q,null),e(f,E),e(f,D),e(D,g),u(s,j,l),v(H,s,l),u(s,A,l),u(s,C,l),e(C,S),e(C,k),e(k,P),e(C,z),u(s,R,l),u(s,N,l),e(N,B),e(B,is),e(N,G),e(N,Z),e(Z,Se),e(Z,fs),e(fs,Re),e(Z,Ue),e(N,Fe),e(N,gt),e(gt,il),u(s,xa,l),u(s,Le,l),e(Le,ol),u(s,wa,l),u(s,os,l),e(os,ms),e(ms,_t),v(Us,_t,null),e(os,rl),e(os,bt),e(bt,ul),u(s,ya,l),u(s,gs,l),e(gs,pl),e(gs,Fs),e(Fs,hl),e(gs,cl),u(s,ja,l),u(s,Ls,l),e(Ls,Ms),u(s,$a,l),u(s,Me,l),e(Me,dl),u(s,Ea,l),u(s,zs,l),e(zs,Bs),u(s,ka,l),u(s,J,l),e(J,fl),e(J,Ws),e(Ws,ml),e(J,gl),e(J,_s),e(_s,vt),e(vt,_l),e(_s,bl),e(J,vl),u(s,qa,l),u(s,bs,l),e(bs,xl),e(bs,xt),e(xt,wl),e(bs,yl),u(s,Ta,l),v(Ys,s,l),u(s,Da,l),u(s,V,l),e(V,jl),e(V,wt),e(wt,$l),e(V,El),e(V,yt),e(yt,kl),e(V,ql),u(s,Aa,l),v(Qs,s,l),u(s,Ha,l),u(s,vs,l),e(vs,Tl),e(vs,jt),e(jt,Dl),e(vs,Al),u(s,Oa,l),v(Zs,s,l),u(s,Pa,l),v(Js,s,l),u(s,Ca,l),u(s,W,l),e(W,Hl),e(W,$t),e($t,Ol),e(W,Pl),e(W,Vs),e(Vs,Cl),e(W,Nl),e(W,Et),e(Et,Gl),e(W,Il),u(s,Na,l),v(Xs,s,l),u(s,Ga,l),v(Ks,s,l),u(s,Ia,l),u(s,Y,l),e(Y,Sl),e(Y,kt),e(kt,Rl),e(Y,Ul),e(Y,qt),e(qt,Fl),e(Y,Ll),e(Y,Tt),e(Tt,Ml),e(Y,zl),u(s,Sa,l),v(xs,s,l),u(s,Ra,l),u(s,F,l),e(F,Bl),e(F,se),e(se,Wl),e(F,Yl),e(F,Dt),e(Dt,Ql),e(F,Zl),e(F,ee),e(ee,Jl),e(F,Vl),e(F,At),e(At,Xl),e(F,Kl),u(s,Ua,l),v(te,s,l),u(s,Fa,l),v(ws,s,l),u(s,La,l),u(s,ze,l),e(ze,si),u(s,Ma,l),v(ae,s,l),u(s,za,l),u(s,X,l),e(X,ei),e(X,Ht),e(Ht,ti),e(X,ai),e(X,Ot),e(Ot,ni),e(X,li),u(s,Ba,l),v(ne,s,l),u(s,Wa,l),u(s,ys,l),e(ys,ii),e(ys,Be),e(Be,oi),e(ys,ri),u(s,Ya,l),v(le,s,l),u(s,Qa,l),v(ie,s,l),u(s,Za,l),u(s,K,l),e(K,ui),e(K,oe),e(oe,pi),e(K,hi),e(K,re),e(re,ci),e(K,di),u(s,Ja,l),u(s,We,l),e(We,rs),e(rs,fi),e(rs,Pt),e(Pt,mi),e(rs,gi),e(rs,Ct),e(Ct,_i),e(rs,bi),u(s,Va,l),u(s,Ye,l),e(Ye,vi),u(s,Xa,l),u(s,us,l),e(us,js),e(js,Nt),v(ue,Nt,null),e(us,xi),e(us,Gt),e(Gt,wi),u(s,Ka,l),u(s,I,l),e(I,yi),e(I,It),e(It,ji),e(I,$i),e(I,Qe),e(Qe,Ei),e(I,ki),e(I,St),e(St,qi),e(I,Ti),e(I,Rt),e(Rt,Di),e(I,Ai),e(I,Ut),e(Ut,Hi),e(I,Oi),e(I,Ft),e(Ft,Pi),e(I,Ci),u(s,sn,l),v(pe,s,l),u(s,en,l),v(he,s,l),u(s,tn,l),u(s,L,l),e(L,Ni),e(L,Lt),e(Lt,Gi),e(L,Ii),e(L,Mt),e(Mt,Si),e(L,Ri),e(L,zt),e(zt,Ui),e(L,Fi),e(L,Bt),e(Bt,Li),e(L,Mi),u(s,an,l),v(ce,s,l),u(s,nn,l),v($s,s,l),u(s,ln,l),u(s,Ze,l),e(Ze,zi),u(s,on,l),u(s,Je,l),e(Je,Bi),u(s,rn,l),u(s,ps,l),e(ps,Es),e(Es,Wt),v(de,Wt,null),e(ps,Wi),e(ps,Yt),e(Yt,Yi),u(s,un,l),u(s,Ve,l),e(Ve,Qi),u(s,pn,l),u(s,fe,l),e(fe,me),u(s,hn,l),u(s,ks,l),e(ks,Zi),e(ks,qs),e(qs,Qt),e(Qt,Ji),e(qs,Vi),e(ks,Xi),u(s,cn,l),v(ge,s,l),u(s,dn,l),v(_e,s,l),u(s,fn,l),u(s,Q,l),e(Q,Ki),e(Q,Zt),e(Zt,so),e(Q,eo),e(Q,Jt),e(Jt,to),e(Q,ao),e(Q,Vt),e(Vt,no),e(Q,lo),u(s,mn,l),v(be,s,l),u(s,gn,l),v(ve,s,l),u(s,_n,l),u(s,ss,l),e(ss,io),e(ss,Xt),e(Xt,oo),e(ss,ro),e(ss,Kt),e(Kt,uo),e(ss,po),u(s,bn,l),v(xe,s,l),u(s,vn,l),u(s,Xe,l),e(Xe,ho),u(s,xn,l),u(s,hs,l),e(hs,Ts),e(Ts,sa),v(we,sa,null),e(hs,co),e(hs,ea),e(ea,fo),u(s,wn,l),v(ye,s,l),u(s,yn,l),u(s,es,l),e(es,mo),e(es,ta),e(ta,go),e(es,_o),e(es,aa),e(aa,bo),e(es,vo),u(s,jn,l),v(je,s,l),u(s,$n,l),u(s,Ds,l),e(Ds,xo),e(Ds,na),e(na,wo),e(Ds,yo),u(s,En,l),v($e,s,l),u(s,kn,l),u(s,Ke,l),e(Ke,jo),u(s,qn,l),v(Ee,s,l),u(s,Tn,l),u(s,ts,l),e(ts,$o),e(ts,la),e(la,Eo),e(ts,ko),e(ts,ia),e(ia,qo),e(ts,To),u(s,Dn,l),v(ke,s,l),u(s,An,l),v(qe,s,l),u(s,Hn,l),u(s,As,l),e(As,Do),e(As,oa),e(oa,Ao),e(As,Ho),u(s,On,l),v(Hs,s,l),u(s,Pn,l),u(s,cs,l),e(cs,Os),e(Os,ra),v(Te,ra,null),e(cs,Oo),e(cs,ua),e(ua,Po),u(s,Cn,l),u(s,st,l),e(st,Co),u(s,Nn,l),u(s,Ps,l),e(Ps,No),e(Ps,pa),e(pa,Go),e(Ps,Io),u(s,Gn,l),u(s,et,l),e(et,ds),e(ds,So),e(ds,Cs),e(Cs,ha),e(ha,Ro),e(Cs,Uo),e(ds,Fo),e(ds,ca),e(ca,Lo),e(ds,Mo),u(s,In,l),u(s,De,l),e(De,Ae),u(s,Sn,l),u(s,He,l),e(He,Oe),e(Oe,zo),e(Oe,Pe),e(Pe,Bo),e(Oe,Wo),u(s,Rn,l),u(s,as,l),e(as,Yo),e(as,da),e(da,Qo),e(as,Zo),e(as,fa),e(fa,Jo),e(as,Vo),u(s,Un,l),u(s,Ce,l),e(Ce,Ne),u(s,Fn,l),v(Ns,s,l),u(s,Ln,l),u(s,tt,l),e(tt,Xo),u(s,Mn,l),v(Gs,s,l),zn=!0},p(s,[l]){const Ge={};l&2&&(Ge.$$scope={dirty:l,ctx:s}),xs.$set(Ge);const ma={};l&2&&(ma.$$scope={dirty:l,ctx:s}),ws.$set(ma);const ga={};l&2&&(ga.$$scope={dirty:l,ctx:s}),$s.$set(ga);const _a={};l&2&&(_a.$$scope={dirty:l,ctx:s}),Hs.$set(_a);const Ie={};l&2&&(Ie.$$scope={dirty:l,ctx:s}),Ns.$set(Ie);const ba={};l&2&&(ba.$$scope={dirty:l,ctx:s}),Gs.$set(ba)},i(s){zn||(x(m.$$.fragment,s),x(H.$$.fragment,s),x(Us.$$.fragment,s),x(Ys.$$.fragment,s),x(Qs.$$.fragment,s),x(Zs.$$.fragment,s),x(Js.$$.fragment,s),x(Xs.$$.fragment,s),x(Ks.$$.fragment,s),x(xs.$$.fragment,s),x(te.$$.fragment,s),x(ws.$$.fragment,s),x(ae.$$.fragment,s),x(ne.$$.fragment,s),x(le.$$.fragment,s),x(ie.$$.fragment,s),x(ue.$$.fragment,s),x(pe.$$.fragment,s),x(he.$$.fragment,s),x(ce.$$.fragment,s),x($s.$$.fragment,s),x(de.$$.fragment,s),x(ge.$$.fragment,s),x(_e.$$.fragment,s),x(be.$$.fragment,s),x(ve.$$.fragment,s),x(xe.$$.fragment,s),x(we.$$.fragment,s),x(ye.$$.fragment,s),x(je.$$.fragment,s),x($e.$$.fragment,s),x(Ee.$$.fragment,s),x(ke.$$.fragment,s),x(qe.$$.fragment,s),x(Hs.$$.fragment,s),x(Te.$$.fragment,s),x(Ns.$$.fragment,s),x(Gs.$$.fragment,s),zn=!0)},o(s){w(m.$$.fragment,s),w(H.$$.fragment,s),w(Us.$$.fragment,s),w(Ys.$$.fragment,s),w(Qs.$$.fragment,s),w(Zs.$$.fragment,s),w(Js.$$.fragment,s),w(Xs.$$.fragment,s),w(Ks.$$.fragment,s),w(xs.$$.fragment,s),w(te.$$.fragment,s),w(ws.$$.fragment,s),w(ae.$$.fragment,s),w(ne.$$.fragment,s),w(le.$$.fragment,s),w(ie.$$.fragment,s),w(ue.$$.fragment,s),w(pe.$$.fragment,s),w(he.$$.fragment,s),w(ce.$$.fragment,s),w($s.$$.fragment,s),w(de.$$.fragment,s),w(ge.$$.fragment,s),w(_e.$$.fragment,s),w(be.$$.fragment,s),w(ve.$$.fragment,s),w(xe.$$.fragment,s),w(we.$$.fragment,s),w(ye.$$.fragment,s),w(je.$$.fragment,s),w($e.$$.fragment,s),w(Ee.$$.fragment,s),w(ke.$$.fragment,s),w(qe.$$.fragment,s),w(Hs.$$.fragment,s),w(Te.$$.fragment,s),w(Ns.$$.fragment,s),w(Gs.$$.fragment,s),zn=!1},d(s){t(c),s&&t(T),s&&t(f),y(m),s&&t(j),y(H,s),s&&t(A),s&&t(C),s&&t(R),s&&t(N),s&&t(xa),s&&t(Le),s&&t(wa),s&&t(os),y(Us),s&&t(ya),s&&t(gs),s&&t(ja),s&&t(Ls),s&&t($a),s&&t(Me),s&&t(Ea),s&&t(zs),s&&t(ka),s&&t(J),s&&t(qa),s&&t(bs),s&&t(Ta),y(Ys,s),s&&t(Da),s&&t(V),s&&t(Aa),y(Qs,s),s&&t(Ha),s&&t(vs),s&&t(Oa),y(Zs,s),s&&t(Pa),y(Js,s),s&&t(Ca),s&&t(W),s&&t(Na),y(Xs,s),s&&t(Ga),y(Ks,s),s&&t(Ia),s&&t(Y),s&&t(Sa),y(xs,s),s&&t(Ra),s&&t(F),s&&t(Ua),y(te,s),s&&t(Fa),y(ws,s),s&&t(La),s&&t(ze),s&&t(Ma),y(ae,s),s&&t(za),s&&t(X),s&&t(Ba),y(ne,s),s&&t(Wa),s&&t(ys),s&&t(Ya),y(le,s),s&&t(Qa),y(ie,s),s&&t(Za),s&&t(K),s&&t(Ja),s&&t(We),s&&t(Va),s&&t(Ye),s&&t(Xa),s&&t(us),y(ue),s&&t(Ka),s&&t(I),s&&t(sn),y(pe,s),s&&t(en),y(he,s),s&&t(tn),s&&t(L),s&&t(an),y(ce,s),s&&t(nn),y($s,s),s&&t(ln),s&&t(Ze),s&&t(on),s&&t(Je),s&&t(rn),s&&t(ps),y(de),s&&t(un),s&&t(Ve),s&&t(pn),s&&t(fe),s&&t(hn),s&&t(ks),s&&t(cn),y(ge,s),s&&t(dn),y(_e,s),s&&t(fn),s&&t(Q),s&&t(mn),y(be,s),s&&t(gn),y(ve,s),s&&t(_n),s&&t(ss),s&&t(bn),y(xe,s),s&&t(vn),s&&t(Xe),s&&t(xn),s&&t(hs),y(we),s&&t(wn),y(ye,s),s&&t(yn),s&&t(es),s&&t(jn),y(je,s),s&&t($n),s&&t(Ds),s&&t(En),y($e,s),s&&t(kn),s&&t(Ke),s&&t(qn),y(Ee,s),s&&t(Tn),s&&t(ts),s&&t(Dn),y(ke,s),s&&t(An),y(qe,s),s&&t(Hn),s&&t(As),s&&t(On),y(Hs,s),s&&t(Pn),s&&t(cs),y(Te),s&&t(Cn),s&&t(st),s&&t(Nn),s&&t(Ps),s&&t(Gn),s&&t(et),s&&t(In),s&&t(De),s&&t(Sn),s&&t(He),s&&t(Rn),s&&t(as),s&&t(Un),s&&t(Ce),s&&t(Fn),y(Ns,s),s&&t(Ln),s&&t(tt),s&&t(Mn),y(Gs,s)}}}const tp={local:"creating-your-own-dataset",sections:[{local:"getting-the-data",title:"Getting the data"},{local:"cleaning-up-the-data",title:"Cleaning up the data"},{local:"augmenting-the-dataset",title:"Augmenting the dataset"},{local:"uploading-the-dataset-to-the-hugging-face-hub",title:"Uploading the dataset to the Hugging Face Hub"},{local:"creating-a-dataset-card",title:"Creating a dataset card"}],title:"Creating your own dataset"};function ap(U){return Wu(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class pp extends Lu{constructor(c){super();Mu(this,c,ap,ep,zu,{})}}export{pp as default,tp as metadata};
