import{S as Pd,i as Nd,s as Fd,e as n,k as c,w as v,t as a,M as Hd,c as r,d as s,m as u,x as y,a as l,h as o,b as E,Z as Sd,f as Ao,G as t,g as d,y as k,o as g,p as So,q as b,B as x,v as Rd,n as Oo}from"../../chunks/vendor-hf-doc-builder.js";import{T as Od}from"../../chunks/Tip-hf-doc-builder.js";import{Y as Ld}from"../../chunks/Youtube-hf-doc-builder.js";import{I as _a}from"../../chunks/IconCopyLink-hf-doc-builder.js";import{C as O}from"../../chunks/CodeBlock-hf-doc-builder.js";import{D as Id}from"../../chunks/DocNotebookDropdown-hf-doc-builder.js";import{F as Md}from"../../chunks/FrameworkSwitchCourse-hf-doc-builder.js";function Wd(W){let p,$;return p=new Id({props:{classNames:"absolute z-10 right-0 top-0",options:[{label:"Google Colab",value:"https://colab.research.google.com/github/huggingface/notebooks/blob/master/course/chapter5/section6_tf.ipynb"},{label:"Aws Studio",value:"https://studiolab.sagemaker.aws/import/github/huggingface/notebooks/blob/master/course/chapter5/section6_tf.ipynb"}]}}),{c(){v(p.$$.fragment)},l(h){y(p.$$.fragment,h)},m(h,j){k(p,h,j),$=!0},i(h){$||(b(p.$$.fragment,h),$=!0)},o(h){g(p.$$.fragment,h),$=!1},d(h){x(p,h)}}}function Ud(W){let p,$;return p=new Id({props:{classNames:"absolute z-10 right-0 top-0",options:[{label:"Google Colab",value:"https://colab.research.google.com/github/huggingface/notebooks/blob/master/course/chapter5/section6_pt.ipynb"},{label:"Aws Studio",value:"https://studiolab.sagemaker.aws/import/github/huggingface/notebooks/blob/master/course/chapter5/section6_pt.ipynb"}]}}),{c(){v(p.$$.fragment)},l(h){y(p.$$.fragment,h)},m(h,j){k(p,h,j),$=!0},i(h){$||(b(p.$$.fragment,h),$=!0)},o(h){g(p.$$.fragment,h),$=!1},d(h){x(p,h)}}}function zd(W){let p,$,h,j,f,w,I,_,A,q,M,P,S,N,F,R,z,D,H,B;return{c(){p=n("p"),$=a("\u270F\uFE0F "),h=n("strong"),j=a("Try it out!"),f=a(" See if you can use "),w=n("code"),I=a("Dataset.map()"),_=a(" to explode the "),A=n("code"),q=a("comments"),M=a(" column of "),P=n("code"),S=a("issues_dataset"),N=c(),F=n("em"),R=a("without"),z=a(" resorting to the use of Pandas. This is a little tricky; you might find the "),D=n("a"),H=a("\u201CBatch mapping\u201D"),B=a(" section of the \u{1F917} Datasets documentation useful for this task."),this.h()},l(U){p=r(U,"P",{});var T=l(p);$=o(T,"\u270F\uFE0F "),h=r(T,"STRONG",{});var G=l(h);j=o(G,"Try it out!"),G.forEach(s),f=o(T," See if you can use "),w=r(T,"CODE",{});var m=l(w);I=o(m,"Dataset.map()"),m.forEach(s),_=o(T," to explode the "),A=r(T,"CODE",{});var C=l(A);q=o(C,"comments"),C.forEach(s),M=o(T," column of "),P=r(T,"CODE",{});var L=l(P);S=o(L,"issues_dataset"),L.forEach(s),N=u(T),F=r(T,"EM",{});var V=l(F);R=o(V,"without"),V.forEach(s),z=o(T," resorting to the use of Pandas. This is a little tricky; you might find the "),D=r(T,"A",{href:!0,rel:!0});var ne=l(D);H=o(ne,"\u201CBatch mapping\u201D"),ne.forEach(s),B=o(T," section of the \u{1F917} Datasets documentation useful for this task."),T.forEach(s),this.h()},h(){E(D,"href","https://huggingface.co/docs/datasets/v1.12.1/about_map_batch.html?batch-mapping#batch-mapping"),E(D,"rel","nofollow")},m(U,T){d(U,p,T),t(p,$),t(p,h),t(h,j),t(p,f),t(p,w),t(w,I),t(p,_),t(p,A),t(A,q),t(p,M),t(p,P),t(P,S),t(p,N),t(p,F),t(F,R),t(p,z),t(p,D),t(D,H),t(p,B)},d(U){U&&s(p)}}}function Gd(W){let p,$,h,j,f,w,I,_,A,q,M,P,S,N,F,R,z;return p=new O({props:{code:`from transformers import AutoTokenizer, TFAutoModel

model_ckpt = "sentence-transformers/multi-qa-mpnet-base-dot-v1"
tokenizer = AutoTokenizer.from_pretrained(model_ckpt)
model = TFAutoModel.from_pretrained(model_ckpt, from_pt=True)`,highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoTokenizer, TFAutoModel

model_ckpt = <span class="hljs-string">&quot;sentence-transformers/multi-qa-mpnet-base-dot-v1&quot;</span>
tokenizer = AutoTokenizer.from_pretrained(model_ckpt)
model = TFAutoModel.from_pretrained(model_ckpt, from_pt=<span class="hljs-literal">True</span>)`}}),{c(){v(p.$$.fragment),$=c(),h=n("p"),j=a("Note that we\u2019ve set "),f=n("code"),w=a("from_pt=True"),I=a(" as an argument of the "),_=n("code"),A=a("from_pretrained()"),q=a(" method. That\u2019s because the "),M=n("code"),P=a("multi-qa-mpnet-base-dot-v1"),S=a(" checkpoint only has PyTorch weights, so setting "),N=n("code"),F=a("from_pt=True"),R=a(" will automatically convert them to the TensorFlow format for us. As you can see, it is very simple to switch between frameworks in \u{1F917} Transformers!")},l(D){y(p.$$.fragment,D),$=u(D),h=r(D,"P",{});var H=l(h);j=o(H,"Note that we\u2019ve set "),f=r(H,"CODE",{});var B=l(f);w=o(B,"from_pt=True"),B.forEach(s),I=o(H," as an argument of the "),_=r(H,"CODE",{});var U=l(_);A=o(U,"from_pretrained()"),U.forEach(s),q=o(H," method. That\u2019s because the "),M=r(H,"CODE",{});var T=l(M);P=o(T,"multi-qa-mpnet-base-dot-v1"),T.forEach(s),S=o(H," checkpoint only has PyTorch weights, so setting "),N=r(H,"CODE",{});var G=l(N);F=o(G,"from_pt=True"),G.forEach(s),R=o(H," will automatically convert them to the TensorFlow format for us. As you can see, it is very simple to switch between frameworks in \u{1F917} Transformers!"),H.forEach(s)},m(D,H){k(p,D,H),d(D,$,H),d(D,h,H),t(h,j),t(h,f),t(f,w),t(h,I),t(h,_),t(_,A),t(h,q),t(h,M),t(M,P),t(h,S),t(h,N),t(N,F),t(h,R),z=!0},i(D){z||(b(p.$$.fragment,D),z=!0)},o(D){g(p.$$.fragment,D),z=!1},d(D){x(p,D),D&&s($),D&&s(h)}}}function Yd(W){let p,$,h,j,f,w,I;return p=new O({props:{code:`from transformers import AutoTokenizer, AutoModel

model_ckpt = "sentence-transformers/multi-qa-mpnet-base-dot-v1"
tokenizer = AutoTokenizer.from_pretrained(model_ckpt)
model = AutoModel.from_pretrained(model_ckpt)`,highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoTokenizer, AutoModel

model_ckpt = <span class="hljs-string">&quot;sentence-transformers/multi-qa-mpnet-base-dot-v1&quot;</span>
tokenizer = AutoTokenizer.from_pretrained(model_ckpt)
model = AutoModel.from_pretrained(model_ckpt)`}}),w=new O({props:{code:`import torch

device = torch.device("cuda")
model.to(device)`,highlighted:`<span class="hljs-keyword">import</span> torch

device = torch.device(<span class="hljs-string">&quot;cuda&quot;</span>)
model.to(device)`}}),{c(){v(p.$$.fragment),$=c(),h=n("p"),j=a("To speed up the embedding process, it helps to place the model and inputs on a GPU device, so let\u2019s do that now:"),f=c(),v(w.$$.fragment)},l(_){y(p.$$.fragment,_),$=u(_),h=r(_,"P",{});var A=l(h);j=o(A,"To speed up the embedding process, it helps to place the model and inputs on a GPU device, so let\u2019s do that now:"),A.forEach(s),f=u(_),y(w.$$.fragment,_)},m(_,A){k(p,_,A),d(_,$,A),d(_,h,A),t(h,j),d(_,f,A),k(w,_,A),I=!0},i(_){I||(b(p.$$.fragment,_),b(w.$$.fragment,_),I=!0)},o(_){g(p.$$.fragment,_),g(w.$$.fragment,_),I=!1},d(_){x(p,_),_&&s($),_&&s(h),_&&s(f),x(w,_)}}}function Bd(W){let p,$,h,j,f,w,I,_,A,q,M,P,S,N,F,R,z,D,H,B,U,T,G;return p=new O({props:{code:`def get_embeddings(text_list):
    encoded_input = tokenizer(
        text_list, padding=True, truncation=True, return_tensors="tf"
    )
    encoded_input = {k: v for k, v in encoded_input.items()}
    model_output = model(**encoded_input)
    return cls_pooling(model_output)`,highlighted:`<span class="hljs-keyword">def</span> <span class="hljs-title function_">get_embeddings</span>(<span class="hljs-params">text_list</span>):
    encoded_input = tokenizer(
        text_list, padding=<span class="hljs-literal">True</span>, truncation=<span class="hljs-literal">True</span>, return_tensors=<span class="hljs-string">&quot;tf&quot;</span>
    )
    encoded_input = {k: v <span class="hljs-keyword">for</span> k, v <span class="hljs-keyword">in</span> encoded_input.items()}
    model_output = model(**encoded_input)
    <span class="hljs-keyword">return</span> cls_pooling(model_output)`}}),w=new O({props:{code:`embedding = get_embeddings(comments_dataset["text"][0])
embedding.shape`,highlighted:`embedding = get_embeddings(comments_dataset[<span class="hljs-string">&quot;text&quot;</span>][<span class="hljs-number">0</span>])
embedding.shape`}}),_=new O({props:{code:"TensorShape([1, 768])",highlighted:'TensorShape([<span class="hljs-number">1</span>, <span class="hljs-number">768</span>])'}}),T=new O({props:{code:`embeddings_dataset = comments_dataset.map(
    lambda x: {"embeddings": get_embeddings(x["text"]).numpy()[0]}
)`,highlighted:`embeddings_dataset = comments_dataset.<span class="hljs-built_in">map</span>(
    <span class="hljs-keyword">lambda</span> x: {<span class="hljs-string">&quot;embeddings&quot;</span>: get_embeddings(x[<span class="hljs-string">&quot;text&quot;</span>]).numpy()[<span class="hljs-number">0</span>]}
)`}}),{c(){v(p.$$.fragment),$=c(),h=n("p"),j=a("We can test the function works by feeding it the first text entry in our corpus and inspecting the output shape:"),f=c(),v(w.$$.fragment),I=c(),v(_.$$.fragment),A=c(),q=n("p"),M=a("Great, we\u2019ve converted the first entry in our corpus into a 768-dimensional vector! We can use "),P=n("code"),S=a("Dataset.map()"),N=a(" to apply our "),F=n("code"),R=a("get_embeddings()"),z=a(" function to each row in our corpus, so let\u2019s create a new "),D=n("code"),H=a("embeddings"),B=a(" column as follows:"),U=c(),v(T.$$.fragment)},l(m){y(p.$$.fragment,m),$=u(m),h=r(m,"P",{});var C=l(h);j=o(C,"We can test the function works by feeding it the first text entry in our corpus and inspecting the output shape:"),C.forEach(s),f=u(m),y(w.$$.fragment,m),I=u(m),y(_.$$.fragment,m),A=u(m),q=r(m,"P",{});var L=l(q);M=o(L,"Great, we\u2019ve converted the first entry in our corpus into a 768-dimensional vector! We can use "),P=r(L,"CODE",{});var V=l(P);S=o(V,"Dataset.map()"),V.forEach(s),N=o(L," to apply our "),F=r(L,"CODE",{});var ne=l(F);R=o(ne,"get_embeddings()"),ne.forEach(s),z=o(L," function to each row in our corpus, so let\u2019s create a new "),D=r(L,"CODE",{});var de=l(D);H=o(de,"embeddings"),de.forEach(s),B=o(L," column as follows:"),L.forEach(s),U=u(m),y(T.$$.fragment,m)},m(m,C){k(p,m,C),d(m,$,C),d(m,h,C),t(h,j),d(m,f,C),k(w,m,C),d(m,I,C),k(_,m,C),d(m,A,C),d(m,q,C),t(q,M),t(q,P),t(P,S),t(q,N),t(q,F),t(F,R),t(q,z),t(q,D),t(D,H),t(q,B),d(m,U,C),k(T,m,C),G=!0},i(m){G||(b(p.$$.fragment,m),b(w.$$.fragment,m),b(_.$$.fragment,m),b(T.$$.fragment,m),G=!0)},o(m){g(p.$$.fragment,m),g(w.$$.fragment,m),g(_.$$.fragment,m),g(T.$$.fragment,m),G=!1},d(m){x(p,m),m&&s($),m&&s(h),m&&s(f),x(w,m),m&&s(I),x(_,m),m&&s(A),m&&s(q),m&&s(U),x(T,m)}}}function Vd(W){let p,$,h,j,f,w,I,_,A,q,M,P,S,N,F,R,z,D,H,B,U,T,G;return p=new O({props:{code:`def get_embeddings(text_list):
    encoded_input = tokenizer(
        text_list, padding=True, truncation=True, return_tensors="pt"
    )
    encoded_input = {k: v.to(device) for k, v in encoded_input.items()}
    model_output = model(**encoded_input)
    return cls_pooling(model_output)`,highlighted:`<span class="hljs-keyword">def</span> <span class="hljs-title function_">get_embeddings</span>(<span class="hljs-params">text_list</span>):
    encoded_input = tokenizer(
        text_list, padding=<span class="hljs-literal">True</span>, truncation=<span class="hljs-literal">True</span>, return_tensors=<span class="hljs-string">&quot;pt&quot;</span>
    )
    encoded_input = {k: v.to(device) <span class="hljs-keyword">for</span> k, v <span class="hljs-keyword">in</span> encoded_input.items()}
    model_output = model(**encoded_input)
    <span class="hljs-keyword">return</span> cls_pooling(model_output)`}}),w=new O({props:{code:`embedding = get_embeddings(comments_dataset["text"][0])
embedding.shape`,highlighted:`embedding = get_embeddings(comments_dataset[<span class="hljs-string">&quot;text&quot;</span>][<span class="hljs-number">0</span>])
embedding.shape`}}),_=new O({props:{code:"torch.Size([1, 768])",highlighted:'torch.Size([<span class="hljs-number">1</span>, <span class="hljs-number">768</span>])'}}),T=new O({props:{code:`embeddings_dataset = comments_dataset.map(
    lambda x: {"embeddings": get_embeddings(x["text"]).detach().cpu().numpy()[0]}
)`,highlighted:`embeddings_dataset = comments_dataset.<span class="hljs-built_in">map</span>(
    <span class="hljs-keyword">lambda</span> x: {<span class="hljs-string">&quot;embeddings&quot;</span>: get_embeddings(x[<span class="hljs-string">&quot;text&quot;</span>]).detach().cpu().numpy()[<span class="hljs-number">0</span>]}
)`}}),{c(){v(p.$$.fragment),$=c(),h=n("p"),j=a("We can test the function works by feeding it the first text entry in our corpus and inspecting the output shape:"),f=c(),v(w.$$.fragment),I=c(),v(_.$$.fragment),A=c(),q=n("p"),M=a("Great, we\u2019ve converted the first entry in our corpus into a 768-dimensional vector! We can use "),P=n("code"),S=a("Dataset.map()"),N=a(" to apply our "),F=n("code"),R=a("get_embeddings()"),z=a(" function to each row in our corpus, so let\u2019s create a new "),D=n("code"),H=a("embeddings"),B=a(" column as follows:"),U=c(),v(T.$$.fragment)},l(m){y(p.$$.fragment,m),$=u(m),h=r(m,"P",{});var C=l(h);j=o(C,"We can test the function works by feeding it the first text entry in our corpus and inspecting the output shape:"),C.forEach(s),f=u(m),y(w.$$.fragment,m),I=u(m),y(_.$$.fragment,m),A=u(m),q=r(m,"P",{});var L=l(q);M=o(L,"Great, we\u2019ve converted the first entry in our corpus into a 768-dimensional vector! We can use "),P=r(L,"CODE",{});var V=l(P);S=o(V,"Dataset.map()"),V.forEach(s),N=o(L," to apply our "),F=r(L,"CODE",{});var ne=l(F);R=o(ne,"get_embeddings()"),ne.forEach(s),z=o(L," function to each row in our corpus, so let\u2019s create a new "),D=r(L,"CODE",{});var de=l(D);H=o(de,"embeddings"),de.forEach(s),B=o(L," column as follows:"),L.forEach(s),U=u(m),y(T.$$.fragment,m)},m(m,C){k(p,m,C),d(m,$,C),d(m,h,C),t(h,j),d(m,f,C),k(w,m,C),d(m,I,C),k(_,m,C),d(m,A,C),d(m,q,C),t(q,M),t(q,P),t(P,S),t(q,N),t(q,F),t(F,R),t(q,z),t(q,D),t(D,H),t(q,B),d(m,U,C),k(T,m,C),G=!0},i(m){G||(b(p.$$.fragment,m),b(w.$$.fragment,m),b(_.$$.fragment,m),b(T.$$.fragment,m),G=!0)},o(m){g(p.$$.fragment,m),g(w.$$.fragment,m),g(_.$$.fragment,m),g(T.$$.fragment,m),G=!1},d(m){x(p,m),m&&s($),m&&s(h),m&&s(f),x(w,m),m&&s(I),x(_,m),m&&s(A),m&&s(q),m&&s(U),x(T,m)}}}function Jd(W){let p,$,h,j;return p=new O({props:{code:`question = "How can I load a dataset offline?"
question_embedding = get_embeddings([question]).numpy()
question_embedding.shape`,highlighted:`question = <span class="hljs-string">&quot;How can I load a dataset offline?&quot;</span>
question_embedding = get_embeddings([question]).numpy()
question_embedding.shape`}}),h=new O({props:{code:"(1, 768)",highlighted:'(<span class="hljs-number">1</span>, <span class="hljs-number">768</span>)'}}),{c(){v(p.$$.fragment),$=c(),v(h.$$.fragment)},l(f){y(p.$$.fragment,f),$=u(f),y(h.$$.fragment,f)},m(f,w){k(p,f,w),d(f,$,w),k(h,f,w),j=!0},i(f){j||(b(p.$$.fragment,f),b(h.$$.fragment,f),j=!0)},o(f){g(p.$$.fragment,f),g(h.$$.fragment,f),j=!1},d(f){x(p,f),f&&s($),x(h,f)}}}function Qd(W){let p,$,h,j;return p=new O({props:{code:`question = "How can I load a dataset offline?"
question_embedding = get_embeddings([question]).cpu().detach().numpy()
question_embedding.shape`,highlighted:`question = <span class="hljs-string">&quot;How can I load a dataset offline?&quot;</span>
question_embedding = get_embeddings([question]).cpu().detach().numpy()
question_embedding.shape`}}),h=new O({props:{code:"torch.Size([1, 768])",highlighted:'torch.Size([<span class="hljs-number">1</span>, <span class="hljs-number">768</span>])'}}),{c(){v(p.$$.fragment),$=c(),v(h.$$.fragment)},l(f){y(p.$$.fragment,f),$=u(f),y(h.$$.fragment,f)},m(f,w){k(p,f,w),d(f,$,w),k(h,f,w),j=!0},i(f){j||(b(p.$$.fragment,f),b(h.$$.fragment,f),j=!0)},o(f){g(p.$$.fragment,f),g(h.$$.fragment,f),j=!1},d(f){x(p,f),f&&s($),x(h,f)}}}function Xd(W){let p,$,h,j,f,w,I,_,A,q,M;return{c(){p=n("p"),$=a("\u270F\uFE0F "),h=n("strong"),j=a("Try it out!"),f=a(" Create your own query and see whether you can find an answer in the retrieved documents. You might have to increase the "),w=n("code"),I=a("k"),_=a(" parameter in "),A=n("code"),q=a("Dataset.get_nearest_examples()"),M=a(" to broaden the search.")},l(P){p=r(P,"P",{});var S=l(p);$=o(S,"\u270F\uFE0F "),h=r(S,"STRONG",{});var N=l(h);j=o(N,"Try it out!"),N.forEach(s),f=o(S," Create your own query and see whether you can find an answer in the retrieved documents. You might have to increase the "),w=r(S,"CODE",{});var F=l(w);I=o(F,"k"),F.forEach(s),_=o(S," parameter in "),A=r(S,"CODE",{});var R=l(A);q=o(R,"Dataset.get_nearest_examples()"),R.forEach(s),M=o(S," to broaden the search."),S.forEach(s)},m(P,S){d(P,p,S),t(p,$),t(p,h),t(h,j),t(p,f),t(p,w),t(w,I),t(p,_),t(p,A),t(A,q),t(p,M)},d(P){P&&s(p)}}}function Zd(W){let p,$,h,j,f,w,I,_,A,q,M,P,S,N,F,R,z,D,H,B,U,T,G,m,C,L,V,ne,de,Io,ga,ge,Po,Ct,No,Fo,ss,Ho,Ro,ba,At,Lo,wa,Ce,Be,Ml,Mo,Ve,Wl,$a,Ae,Ie,as,Je,Wo,os,Uo,va,St,zo,ya,Qe,ka,be,Go,ns,Yo,Bo,Ot,Vo,Jo,xa,Xe,Ea,Ze,ja,J,Qo,rs,Xo,Zo,ls,Ko,en,is,tn,sn,ds,an,on,cs,nn,rn,qa,Ke,Ta,et,Da,Q,ln,us,dn,cn,ps,un,pn,ms,mn,hn,hs,fn,_n,fs,gn,bn,Ca,tt,Aa,st,Sa,Z,wn,_s,$n,vn,gs,yn,kn,Pe,bs,xn,En,jn,ws,qn,Tn,Oa,at,Ia,Ne,Dn,$s,Cn,An,Pa,ot,Na,nt,Fa,Fe,Sn,vs,On,In,Ha,rt,Ra,ee,ys,K,La,Pn,ks,Nn,Fn,xs,Hn,Rn,Es,Ln,Mn,js,Wn,Un,ce,te,qs,zn,Gn,Ts,Yn,Bn,Ds,Vn,Jn,Cs,Qn,Xn,As,Zn,Kn,se,Ss,er,tr,Os,sr,ar,Is,or,nr,Ps,rr,lr,Ns,ir,dr,ae,Fs,cr,ur,Hs,pr,mr,Rs,hr,fr,Ls,_r,gr,Ms,br,wr,oe,Ws,$r,vr,Us,yr,kr,zs,xr,Er,Gs,jr,qr,Ys,Tr,Ma,re,Dr,Bs,Cr,Ar,Vs,Sr,Or,Js,Ir,Pr,Wa,lt,Ua,it,za,It,Nr,Ga,He,Ya,Re,Fr,Qs,Hr,Rr,Ba,dt,Va,Pt,Lr,Ja,ct,Qa,ut,Xa,we,Mr,Xs,Wr,Ur,Zs,zr,Gr,Za,pt,Ka,Nt,Yr,eo,Se,Le,Ks,mt,Br,ea,Vr,to,Y,Jr,Ft,Qr,Xr,ta,Zr,Kr,sa,el,tl,ht,sl,al,aa,ol,nl,ft,rl,ll,oa,il,dl,so,ue,pe,Ht,$e,cl,na,ul,pl,ra,ml,hl,ao,_t,oo,Rt,fl,no,me,he,Lt,Mt,_l,ro,Oe,Me,la,gt,gl,ia,bl,lo,ve,wl,da,$l,vl,bt,yl,kl,io,ye,xl,ca,El,jl,ua,ql,Tl,co,wt,uo,We,Dl,pa,Cl,Al,po,fe,_e,Wt,Ut,Sl,mo,$t,ho,ke,Ol,ma,Il,Pl,ha,Nl,Fl,fo,vt,_o,zt,Hl,go,yt,bo,kt,wo,Gt,Rl,$o,Ue,vo;h=new Md({props:{fw:W[0]}}),_=new _a({});const Ul=[Ud,Wd],xt=[];function zl(e,i){return e[0]==="pt"?0:1}S=zl(W),N=xt[S]=Ul[S](W),T=new Ld({props:{id:"OATCgQtNX2o"}}),V=new _a({}),Je=new _a({}),Qe=new O({props:{code:`from huggingface_hub import hf_hub_url

data_files = hf_hub_url(
    repo_id="lewtun/github-issues",
    filename="datasets-issues-with-hf-doc-builder.jsonl",
    repo_type="dataset",
)`,highlighted:`<span class="hljs-keyword">from</span> huggingface_hub <span class="hljs-keyword">import</span> hf_hub_url

data_files = hf_hub_url(
    repo_id=<span class="hljs-string">&quot;lewtun/github-issues&quot;</span>,
    filename=<span class="hljs-string">&quot;datasets-issues-with-hf-doc-builder.jsonl&quot;</span>,
    repo_type=<span class="hljs-string">&quot;dataset&quot;</span>,
)`}}),Xe=new O({props:{code:`from datasets import load_dataset

issues_dataset = load_dataset("json", data_files=data_files, split="train")
issues_dataset`,highlighted:`<span class="hljs-keyword">from</span> datasets <span class="hljs-keyword">import</span> load_dataset

issues_dataset = load_dataset(<span class="hljs-string">&quot;json&quot;</span>, data_files=data_files, split=<span class="hljs-string">&quot;train&quot;</span>)
issues_dataset`}}),Ze=new O({props:{code:`Dataset({
    features: ['url', 'repository_url', 'labels_url', 'comments_url', 'events_url', 'html_url', 'id', 'node_id', 'number', 'title', 'user', 'labels', 'state', 'locked', 'assignee', 'assignees', 'milestone', 'comments', 'created_at', 'updated_at', 'closed_at', 'author_association', 'active_lock_reason', 'pull_request', 'body', 'performed_via_github_app', 'is_pull_request'],
    num_rows: 2855
})`,highlighted:`Dataset({
    features: [<span class="hljs-string">&#x27;url&#x27;</span>, <span class="hljs-string">&#x27;repository_url&#x27;</span>, <span class="hljs-string">&#x27;labels_url&#x27;</span>, <span class="hljs-string">&#x27;comments_url&#x27;</span>, <span class="hljs-string">&#x27;events_url&#x27;</span>, <span class="hljs-string">&#x27;html_url&#x27;</span>, <span class="hljs-string">&#x27;id&#x27;</span>, <span class="hljs-string">&#x27;node_id&#x27;</span>, <span class="hljs-string">&#x27;number&#x27;</span>, <span class="hljs-string">&#x27;title&#x27;</span>, <span class="hljs-string">&#x27;user&#x27;</span>, <span class="hljs-string">&#x27;labels&#x27;</span>, <span class="hljs-string">&#x27;state&#x27;</span>, <span class="hljs-string">&#x27;locked&#x27;</span>, <span class="hljs-string">&#x27;assignee&#x27;</span>, <span class="hljs-string">&#x27;assignees&#x27;</span>, <span class="hljs-string">&#x27;milestone&#x27;</span>, <span class="hljs-string">&#x27;comments&#x27;</span>, <span class="hljs-string">&#x27;created_at&#x27;</span>, <span class="hljs-string">&#x27;updated_at&#x27;</span>, <span class="hljs-string">&#x27;closed_at&#x27;</span>, <span class="hljs-string">&#x27;author_association&#x27;</span>, <span class="hljs-string">&#x27;active_lock_reason&#x27;</span>, <span class="hljs-string">&#x27;pull_request&#x27;</span>, <span class="hljs-string">&#x27;body&#x27;</span>, <span class="hljs-string">&#x27;performed_via_github_app&#x27;</span>, <span class="hljs-string">&#x27;is_pull_request&#x27;</span>],
    num_rows: <span class="hljs-number">2855</span>
})`}}),Ke=new O({props:{code:`issues_dataset = issues_dataset.filter(
    lambda x: (x["is_pull_request"] == False and len(x["comments"]) > 0)
)
issues_dataset`,highlighted:`issues_dataset = issues_dataset.<span class="hljs-built_in">filter</span>(
    <span class="hljs-keyword">lambda</span> x: (x[<span class="hljs-string">&quot;is_pull_request&quot;</span>] == <span class="hljs-literal">False</span> <span class="hljs-keyword">and</span> <span class="hljs-built_in">len</span>(x[<span class="hljs-string">&quot;comments&quot;</span>]) &gt; <span class="hljs-number">0</span>)
)
issues_dataset`}}),et=new O({props:{code:`Dataset({
    features: ['url', 'repository_url', 'labels_url', 'comments_url', 'events_url', 'html_url', 'id', 'node_id', 'number', 'title', 'user', 'labels', 'state', 'locked', 'assignee', 'assignees', 'milestone', 'comments', 'created_at', 'updated_at', 'closed_at', 'author_association', 'active_lock_reason', 'pull_request', 'body', 'performed_via_github_app', 'is_pull_request'],
    num_rows: 771
})`,highlighted:`Dataset({
    features: [<span class="hljs-string">&#x27;url&#x27;</span>, <span class="hljs-string">&#x27;repository_url&#x27;</span>, <span class="hljs-string">&#x27;labels_url&#x27;</span>, <span class="hljs-string">&#x27;comments_url&#x27;</span>, <span class="hljs-string">&#x27;events_url&#x27;</span>, <span class="hljs-string">&#x27;html_url&#x27;</span>, <span class="hljs-string">&#x27;id&#x27;</span>, <span class="hljs-string">&#x27;node_id&#x27;</span>, <span class="hljs-string">&#x27;number&#x27;</span>, <span class="hljs-string">&#x27;title&#x27;</span>, <span class="hljs-string">&#x27;user&#x27;</span>, <span class="hljs-string">&#x27;labels&#x27;</span>, <span class="hljs-string">&#x27;state&#x27;</span>, <span class="hljs-string">&#x27;locked&#x27;</span>, <span class="hljs-string">&#x27;assignee&#x27;</span>, <span class="hljs-string">&#x27;assignees&#x27;</span>, <span class="hljs-string">&#x27;milestone&#x27;</span>, <span class="hljs-string">&#x27;comments&#x27;</span>, <span class="hljs-string">&#x27;created_at&#x27;</span>, <span class="hljs-string">&#x27;updated_at&#x27;</span>, <span class="hljs-string">&#x27;closed_at&#x27;</span>, <span class="hljs-string">&#x27;author_association&#x27;</span>, <span class="hljs-string">&#x27;active_lock_reason&#x27;</span>, <span class="hljs-string">&#x27;pull_request&#x27;</span>, <span class="hljs-string">&#x27;body&#x27;</span>, <span class="hljs-string">&#x27;performed_via_github_app&#x27;</span>, <span class="hljs-string">&#x27;is_pull_request&#x27;</span>],
    num_rows: <span class="hljs-number">771</span>
})`}}),tt=new O({props:{code:`columns = issues_dataset.column_names
columns_to_keep = ["title", "body", "html_url", "comments"]
columns_to_remove = set(columns_to_keep).symmetric_difference(columns)
issues_dataset = issues_dataset.remove_columns(columns_to_remove)
issues_dataset`,highlighted:`columns = issues_dataset.column_names
columns_to_keep = [<span class="hljs-string">&quot;title&quot;</span>, <span class="hljs-string">&quot;body&quot;</span>, <span class="hljs-string">&quot;html_url&quot;</span>, <span class="hljs-string">&quot;comments&quot;</span>]
columns_to_remove = <span class="hljs-built_in">set</span>(columns_to_keep).symmetric_difference(columns)
issues_dataset = issues_dataset.remove_columns(columns_to_remove)
issues_dataset`}}),st=new O({props:{code:`Dataset({
    features: ['html_url', 'title', 'comments', 'body'],
    num_rows: 771
})`,highlighted:`Dataset({
    features: [<span class="hljs-string">&#x27;html_url&#x27;</span>, <span class="hljs-string">&#x27;title&#x27;</span>, <span class="hljs-string">&#x27;comments&#x27;</span>, <span class="hljs-string">&#x27;body&#x27;</span>],
    num_rows: <span class="hljs-number">771</span>
})`}}),at=new O({props:{code:`issues_dataset.set_format("pandas")
df = issues_dataset[:]`,highlighted:`issues_dataset.set_format(<span class="hljs-string">&quot;pandas&quot;</span>)
df = issues_dataset[:]`}}),ot=new O({props:{code:'df["comments"][0].tolist()',highlighted:'df[<span class="hljs-string">&quot;comments&quot;</span>][<span class="hljs-number">0</span>].tolist()'}}),nt=new O({props:{code:`['the bug code locate in \uFF1A\\r\\n    if data_args.task_name is not None:\\r\\n        # Downloading and loading a dataset from the hub.\\r\\n        datasets = load_dataset("glue", data_args.task_name, cache_dir=model_args.cache_dir)',
 'Hi @jinec,\\r\\n\\r\\nFrom time to time we get this kind of \`ConnectionError\` coming from the github.com website: https://raw.githubusercontent.com\\r\\n\\r\\nNormally, it should work if you wait a little and then retry.\\r\\n\\r\\nCould you please confirm if the problem persists?',
 'cannot connect\uFF0Ceven by Web browser\uFF0Cplease check that  there is some  problems\u3002',
 'I can access https://raw.githubusercontent.com/huggingface/datasets/1.7.0/datasets/glue/glue.py without problem...']`,highlighted:`[<span class="hljs-string">&#x27;the bug code locate in \uFF1A\\r\\n    if data_args.task_name is not None:\\r\\n        # Downloading and loading a dataset from the hub.\\r\\n        datasets = load_dataset(&quot;glue&quot;, data_args.task_name, cache_dir=model_args.cache_dir)&#x27;</span>,
 <span class="hljs-string">&#x27;Hi @jinec,\\r\\n\\r\\nFrom time to time we get this kind of \`ConnectionError\` coming from the github.com website: https://raw.githubusercontent.com\\r\\n\\r\\nNormally, it should work if you wait a little and then retry.\\r\\n\\r\\nCould you please confirm if the problem persists?&#x27;</span>,
 <span class="hljs-string">&#x27;cannot connect\uFF0Ceven by Web browser\uFF0Cplease check that  there is some  problems\u3002&#x27;</span>,
 <span class="hljs-string">&#x27;I can access https://raw.githubusercontent.com/huggingface/datasets/1.7.0/datasets/glue/glue.py without problem...&#x27;</span>]`}}),rt=new O({props:{code:`comments_df = df.explode("comments", ignore_index=True)
comments_df.head(4)`,highlighted:`comments_df = df.explode(<span class="hljs-string">&quot;comments&quot;</span>, ignore_index=<span class="hljs-literal">True</span>)
comments_df.head(<span class="hljs-number">4</span>)`}}),lt=new O({props:{code:`from datasets import Dataset

comments_dataset = Dataset.from_pandas(comments_df)
comments_dataset`,highlighted:`<span class="hljs-keyword">from</span> datasets <span class="hljs-keyword">import</span> Dataset

comments_dataset = Dataset.from_pandas(comments_df)
comments_dataset`}}),it=new O({props:{code:`Dataset({
    features: ['html_url', 'title', 'comments', 'body'],
    num_rows: 2842
})`,highlighted:`Dataset({
    features: [<span class="hljs-string">&#x27;html_url&#x27;</span>, <span class="hljs-string">&#x27;title&#x27;</span>, <span class="hljs-string">&#x27;comments&#x27;</span>, <span class="hljs-string">&#x27;body&#x27;</span>],
    num_rows: <span class="hljs-number">2842</span>
})`}}),He=new Od({props:{$$slots:{default:[zd]},$$scope:{ctx:W}}}),dt=new O({props:{code:`comments_dataset = comments_dataset.map(
    lambda x: {"comment_length": len(x["comments"].split())}
)`,highlighted:`comments_dataset = comments_dataset.<span class="hljs-built_in">map</span>(
    <span class="hljs-keyword">lambda</span> x: {<span class="hljs-string">&quot;comment_length&quot;</span>: <span class="hljs-built_in">len</span>(x[<span class="hljs-string">&quot;comments&quot;</span>].split())}
)`}}),ct=new O({props:{code:`comments_dataset = comments_dataset.filter(lambda x: x["comment_length"] > 15)
comments_dataset`,highlighted:`comments_dataset = comments_dataset.<span class="hljs-built_in">filter</span>(<span class="hljs-keyword">lambda</span> x: x[<span class="hljs-string">&quot;comment_length&quot;</span>] &gt; <span class="hljs-number">15</span>)
comments_dataset`}}),ut=new O({props:{code:`Dataset({
    features: ['html_url', 'title', 'comments', 'body', 'comment_length'],
    num_rows: 2098
})`,highlighted:`Dataset({
    features: [<span class="hljs-string">&#x27;html_url&#x27;</span>, <span class="hljs-string">&#x27;title&#x27;</span>, <span class="hljs-string">&#x27;comments&#x27;</span>, <span class="hljs-string">&#x27;body&#x27;</span>, <span class="hljs-string">&#x27;comment_length&#x27;</span>],
    num_rows: <span class="hljs-number">2098</span>
})`}}),pt=new O({props:{code:`def concatenate_text(examples):
    return {
        "text": examples["title"]
        + " \\n "
        + examples["body"]
        + " \\n "
        + examples["comments"]
    }


comments_dataset = comments_dataset.map(concatenate_text)`,highlighted:`<span class="hljs-keyword">def</span> <span class="hljs-title function_">concatenate_text</span>(<span class="hljs-params">examples</span>):
    <span class="hljs-keyword">return</span> {
        <span class="hljs-string">&quot;text&quot;</span>: examples[<span class="hljs-string">&quot;title&quot;</span>]
        + <span class="hljs-string">&quot; \\n &quot;</span>
        + examples[<span class="hljs-string">&quot;body&quot;</span>]
        + <span class="hljs-string">&quot; \\n &quot;</span>
        + examples[<span class="hljs-string">&quot;comments&quot;</span>]
    }


comments_dataset = comments_dataset.<span class="hljs-built_in">map</span>(concatenate_text)`}}),mt=new _a({});const Gl=[Yd,Gd],Et=[];function Yl(e,i){return e[0]==="pt"?0:1}ue=Yl(W),pe=Et[ue]=Gl[ue](W),_t=new O({props:{code:`def cls_pooling(model_output):
    return model_output.last_hidden_state[:, 0]`,highlighted:`<span class="hljs-keyword">def</span> <span class="hljs-title function_">cls_pooling</span>(<span class="hljs-params">model_output</span>):
    <span class="hljs-keyword">return</span> model_output.last_hidden_state[:, <span class="hljs-number">0</span>]`}});const Bl=[Vd,Bd],jt=[];function Vl(e,i){return e[0]==="pt"?0:1}me=Vl(W),he=jt[me]=Bl[me](W),gt=new _a({}),wt=new O({props:{code:'embeddings_dataset.add_faiss_index(column="embeddings")',highlighted:'embeddings_dataset.add_faiss_index(column=<span class="hljs-string">&quot;embeddings&quot;</span>)'}});const Jl=[Qd,Jd],qt=[];function Ql(e,i){return e[0]==="pt"?0:1}return fe=Ql(W),_e=qt[fe]=Jl[fe](W),$t=new O({props:{code:`scores, samples = embeddings_dataset.get_nearest_examples(
    "embeddings", question_embedding, k=5
)`,highlighted:`scores, samples = embeddings_dataset.get_nearest_examples(
    <span class="hljs-string">&quot;embeddings&quot;</span>, question_embedding, k=<span class="hljs-number">5</span>
)`}}),vt=new O({props:{code:`import pandas as pd

samples_df = pd.DataFrame.from_dict(samples)
samples_df["scores"] = scores
samples_df.sort_values("scores", ascending=False, inplace=True)`,highlighted:`<span class="hljs-keyword">import</span> pandas <span class="hljs-keyword">as</span> pd

samples_df = pd.DataFrame.from_dict(samples)
samples_df[<span class="hljs-string">&quot;scores&quot;</span>] = scores
samples_df.sort_values(<span class="hljs-string">&quot;scores&quot;</span>, ascending=<span class="hljs-literal">False</span>, inplace=<span class="hljs-literal">True</span>)`}}),yt=new O({props:{code:`for _, row in samples_df.iterrows():
    print(f"COMMENT: {row.comments}")
    print(f"SCORE: {row.scores}")
    print(f"TITLE: {row.title}")
    print(f"URL: {row.html_url}")
    print("=" * 50)
    print()`,highlighted:`<span class="hljs-keyword">for</span> _, row <span class="hljs-keyword">in</span> samples_df.iterrows():
    <span class="hljs-built_in">print</span>(<span class="hljs-string">f&quot;COMMENT: <span class="hljs-subst">{row.comments}</span>&quot;</span>)
    <span class="hljs-built_in">print</span>(<span class="hljs-string">f&quot;SCORE: <span class="hljs-subst">{row.scores}</span>&quot;</span>)
    <span class="hljs-built_in">print</span>(<span class="hljs-string">f&quot;TITLE: <span class="hljs-subst">{row.title}</span>&quot;</span>)
    <span class="hljs-built_in">print</span>(<span class="hljs-string">f&quot;URL: <span class="hljs-subst">{row.html_url}</span>&quot;</span>)
    <span class="hljs-built_in">print</span>(<span class="hljs-string">&quot;=&quot;</span> * <span class="hljs-number">50</span>)
    <span class="hljs-built_in">print</span>()`}}),kt=new O({props:{code:`"""
COMMENT: Requiring online connection is a deal breaker in some cases unfortunately so it'd be great if offline mode is added similar to how \`transformers\` loads models offline fine.

@mandubian's second bullet point suggests that there's a workaround allowing you to use your offline (custom?) dataset with \`datasets\`. Could you please elaborate on how that should look like?
SCORE: 25.505046844482422
TITLE: Discussion using datasets in offline mode
URL: https://github.com/huggingface/datasets/issues/824
==================================================

COMMENT: The local dataset builders (csv, text , json and pandas) are now part of the \`datasets\` package since #1726 :)
You can now use them offline
\\\`\\\`\\\`python
datasets = load_dataset("text", data_files=data_files)
\\\`\\\`\\\`

We'll do a new release soon
SCORE: 24.555509567260742
TITLE: Discussion using datasets in offline mode
URL: https://github.com/huggingface/datasets/issues/824
==================================================

COMMENT: I opened a PR that allows to reload modules that have already been loaded once even if there's no internet.

Let me know if you know other ways that can make the offline mode experience better. I'd be happy to add them :)

I already note the "freeze" modules option, to prevent local modules updates. It would be a cool feature.

----------

> @mandubian's second bullet point suggests that there's a workaround allowing you to use your offline (custom?) dataset with \`datasets\`. Could you please elaborate on how that should look like?

Indeed \`load_dataset\` allows to load remote dataset script (squad, glue, etc.) but also you own local ones.
For example if you have a dataset script at \`./my_dataset/my_dataset.py\` then you can do
\\\`\\\`\\\`python
load_dataset("./my_dataset")
\\\`\\\`\\\`
and the dataset script will generate your dataset once and for all.

----------

About I'm looking into having \`csv\`, \`json\`, \`text\`, \`pandas\` dataset builders already included in the \`datasets\` package, so that they are available offline by default, as opposed to the other datasets that require the script to be downloaded.
cf #1724
SCORE: 24.14896583557129
TITLE: Discussion using datasets in offline mode
URL: https://github.com/huggingface/datasets/issues/824
==================================================

COMMENT: > here is my way to load a dataset offline, but it **requires** an online machine
>
> 1. (online machine)
>
> \`\`\`
>
> import datasets
>
> data = datasets.load_dataset(...)
>
> data.save_to_disk(/YOUR/DATASET/DIR)
>
> \`\`\`
>
> 2. copy the dir from online to the offline machine
>
> 3. (offline machine)
>
> \`\`\`
>
> import datasets
>
> data = datasets.load_from_disk(/SAVED/DATA/DIR)
>
> \`\`\`
>
>
>
> HTH.


SCORE: 22.893993377685547
TITLE: Discussion using datasets in offline mode
URL: https://github.com/huggingface/datasets/issues/824
==================================================

COMMENT: here is my way to load a dataset offline, but it **requires** an online machine
1. (online machine)
\\\`\\\`\\\`
import datasets
data = datasets.load_dataset(...)
data.save_to_disk(/YOUR/DATASET/DIR)
\\\`\\\`\\\`
2. copy the dir from online to the offline machine
3. (offline machine)
\\\`\\\`\\\`
import datasets
data = datasets.load_from_disk(/SAVED/DATA/DIR)
\\\`\\\`\\\`

HTH.
SCORE: 22.406635284423828
TITLE: Discussion using datasets in offline mode
URL: https://github.com/huggingface/datasets/issues/824
==================================================
"""`,highlighted:`<span class="hljs-string">&quot;&quot;&quot;
COMMENT: Requiring online connection is a deal breaker in some cases unfortunately so it&#x27;d be great if offline mode is added similar to how \`transformers\` loads models offline fine.

@mandubian&#x27;s second bullet point suggests that there&#x27;s a workaround allowing you to use your offline (custom?) dataset with \`datasets\`. Could you please elaborate on how that should look like?
SCORE: 25.505046844482422
TITLE: Discussion using datasets in offline mode
URL: https://github.com/huggingface/datasets/issues/824
==================================================

COMMENT: The local dataset builders (csv, text , json and pandas) are now part of the \`datasets\` package since #1726 :)
You can now use them offline
\\\`\\\`\\\`python
datasets = load_dataset(&quot;text&quot;, data_files=data_files)
\\\`\\\`\\\`

We&#x27;ll do a new release soon
SCORE: 24.555509567260742
TITLE: Discussion using datasets in offline mode
URL: https://github.com/huggingface/datasets/issues/824
==================================================

COMMENT: I opened a PR that allows to reload modules that have already been loaded once even if there&#x27;s no internet.

Let me know if you know other ways that can make the offline mode experience better. I&#x27;d be happy to add them :)

I already note the &quot;freeze&quot; modules option, to prevent local modules updates. It would be a cool feature.

----------

&gt; @mandubian&#x27;s second bullet point suggests that there&#x27;s a workaround allowing you to use your offline (custom?) dataset with \`datasets\`. Could you please elaborate on how that should look like?

Indeed \`load_dataset\` allows to load remote dataset script (squad, glue, etc.) but also you own local ones.
For example if you have a dataset script at \`./my_dataset/my_dataset.py\` then you can do
\\\`\\\`\\\`python
load_dataset(&quot;./my_dataset&quot;)
\\\`\\\`\\\`
and the dataset script will generate your dataset once and for all.

----------

About I&#x27;m looking into having \`csv\`, \`json\`, \`text\`, \`pandas\` dataset builders already included in the \`datasets\` package, so that they are available offline by default, as opposed to the other datasets that require the script to be downloaded.
cf #1724
SCORE: 24.14896583557129
TITLE: Discussion using datasets in offline mode
URL: https://github.com/huggingface/datasets/issues/824
==================================================

COMMENT: &gt; here is my way to load a dataset offline, but it **requires** an online machine
&gt;
&gt; 1. (online machine)
&gt;
&gt; \`\`\`
&gt;
&gt; import datasets
&gt;
&gt; data = datasets.load_dataset(...)
&gt;
&gt; data.save_to_disk(/YOUR/DATASET/DIR)
&gt;
&gt; \`\`\`
&gt;
&gt; 2. copy the dir from online to the offline machine
&gt;
&gt; 3. (offline machine)
&gt;
&gt; \`\`\`
&gt;
&gt; import datasets
&gt;
&gt; data = datasets.load_from_disk(/SAVED/DATA/DIR)
&gt;
&gt; \`\`\`
&gt;
&gt;
&gt;
&gt; HTH.


SCORE: 22.893993377685547
TITLE: Discussion using datasets in offline mode
URL: https://github.com/huggingface/datasets/issues/824
==================================================

COMMENT: here is my way to load a dataset offline, but it **requires** an online machine
1. (online machine)
\\\`\\\`\\\`
import datasets
data = datasets.load_dataset(...)
data.save_to_disk(/YOUR/DATASET/DIR)
\\\`\\\`\\\`
2. copy the dir from online to the offline machine
3. (offline machine)
\\\`\\\`\\\`
import datasets
data = datasets.load_from_disk(/SAVED/DATA/DIR)
\\\`\\\`\\\`

HTH.
SCORE: 22.406635284423828
TITLE: Discussion using datasets in offline mode
URL: https://github.com/huggingface/datasets/issues/824
==================================================
&quot;&quot;&quot;</span>`}}),Ue=new Od({props:{$$slots:{default:[Xd]},$$scope:{ctx:W}}}),{c(){p=n("meta"),$=c(),v(h.$$.fragment),j=c(),f=n("h1"),w=n("a"),I=n("span"),v(_.$$.fragment),A=c(),q=n("span"),M=a("Semantic search with FAISS"),P=c(),N.c(),F=c(),R=n("p"),z=a("In "),D=n("a"),H=a("section 5"),B=a(", we created a dataset of GitHub issues and comments from the \u{1F917} Datasets repository. In this section we\u2019ll use this information to build a search engine that can help us find answers to our most pressing questions about the library!"),U=c(),v(T.$$.fragment),G=c(),m=n("h2"),C=n("a"),L=n("span"),v(V.$$.fragment),ne=c(),de=n("span"),Io=a("Using embeddings for semantic search"),ga=c(),ge=n("p"),Po=a("As we saw in "),Ct=n("a"),No=a("Chapter 1"),Fo=a(", Transformer-based language models represent each token in a span of text as an "),ss=n("em"),Ho=a("embedding vector"),Ro=a(". It turns out that one can \u201Cpool\u201D the individual embeddings to create a vector representation for whole sentences, paragraphs, or (in some cases) documents. These embeddings can then be used to find similar documents in the corpus by computing the dot-product similarity (or some other similarity metric) between each embedding and returning the documents with the greatest overlap."),ba=c(),At=n("p"),Lo=a("In this section we\u2019ll use embeddings to develop a semantic search engine. These search engines offer several advantages over conventional approaches that are based on matching keywords in a query with the documents."),wa=c(),Ce=n("div"),Be=n("img"),Mo=c(),Ve=n("img"),$a=c(),Ae=n("h2"),Ie=n("a"),as=n("span"),v(Je.$$.fragment),Wo=c(),os=n("span"),Uo=a("Loading and preparing the dataset"),va=c(),St=n("p"),zo=a("The first thing we need to do is download our dataset of GitHub issues, so let\u2019s use the \u{1F917} Hub library to resolve the URL where our file is stored on the Hugging Face Hub:"),ya=c(),v(Qe.$$.fragment),ka=c(),be=n("p"),Go=a("With the URL stored in "),ns=n("code"),Yo=a("data_files"),Bo=a(", we can then load the remote dataset using the method introduced in "),Ot=n("a"),Vo=a("section 2"),Jo=a(":"),xa=c(),v(Xe.$$.fragment),Ea=c(),v(Ze.$$.fragment),ja=c(),J=n("p"),Qo=a("Here we\u2019ve specified the default "),rs=n("code"),Xo=a("train"),Zo=a(" split in "),ls=n("code"),Ko=a("load_dataset()"),en=a(", so it returns a "),is=n("code"),tn=a("Dataset"),sn=a(" instead of a "),ds=n("code"),an=a("DatasetDict"),on=a(". The first order of business is to filter out the pull requests, as these tend to be rarely used for answering user queries and will introduce noise in our search engine. As should be familiar by now, we can use the "),cs=n("code"),nn=a("Dataset.filter()"),rn=a(" function to exclude these rows in our dataset. While we\u2019re at it, let\u2019s also filter out rows with no comments, since these provide no answers to user queries:"),qa=c(),v(Ke.$$.fragment),Ta=c(),v(et.$$.fragment),Da=c(),Q=n("p"),ln=a("We can see that there are a lot of columns in our dataset, most of which we don\u2019t need to build our search engine. From a search perspective, the most informative columns are "),us=n("code"),dn=a("title"),cn=a(", "),ps=n("code"),un=a("body"),pn=a(", and "),ms=n("code"),mn=a("comments"),hn=a(", while "),hs=n("code"),fn=a("html_url"),_n=a(" provides us with a link back to the source issue. Let\u2019s use the "),fs=n("code"),gn=a("Dataset.remove_columns()"),bn=a(" function to drop the rest:"),Ca=c(),v(tt.$$.fragment),Aa=c(),v(st.$$.fragment),Sa=c(),Z=n("p"),wn=a("To create our embeddings we\u2019ll augment each comment with the issue\u2019s title and body, since these fields often include useful contextual information. Because our "),_s=n("code"),$n=a("comments"),vn=a(" column is currently a list of comments for each issue, we need to \u201Cexplode\u201D the column so that each row consists of an "),gs=n("code"),yn=a("(html_url, title, body, comment)"),kn=a(" tuple. In Pandas we can do this with the "),Pe=n("a"),bs=n("code"),xn=a("DataFrame.explode()"),En=a(" function"),jn=a(", which creates a new row for each element in a list-like column, while replicating all the other column values. To see this in action, let\u2019s first switch to the Pandas  "),ws=n("code"),qn=a("DataFrame"),Tn=a(" format:"),Oa=c(),v(at.$$.fragment),Ia=c(),Ne=n("p"),Dn=a("If we inspect the first row in this "),$s=n("code"),Cn=a("DataFrame"),An=a(" we can see there are four comments associated with this issue:"),Pa=c(),v(ot.$$.fragment),Na=c(),v(nt.$$.fragment),Fa=c(),Fe=n("p"),Sn=a("When we explode "),vs=n("code"),On=a("df"),In=a(", we expect to get one row for each of these comments. Let\u2019s check if that\u2019s the case:"),Ha=c(),v(rt.$$.fragment),Ra=c(),ee=n("table"),ys=n("thead"),K=n("tr"),La=n("th"),Pn=c(),ks=n("th"),Nn=a("html_url"),Fn=c(),xs=n("th"),Hn=a("title"),Rn=c(),Es=n("th"),Ln=a("comments"),Mn=c(),js=n("th"),Wn=a("body"),Un=c(),ce=n("tbody"),te=n("tr"),qs=n("th"),zn=a("0"),Gn=c(),Ts=n("td"),Yn=a("https://github.com/huggingface/datasets/issues/2787"),Bn=c(),Ds=n("td"),Vn=a("ConnectionError: Couldn't reach https://raw.githubusercontent.com"),Jn=c(),Cs=n("td"),Qn=a("the bug code locate in \uFF1A\\r\\n    if data_args.task_name is not None..."),Xn=c(),As=n("td"),Zn=a("Hello,\\r\\nI am trying to run run_glue.py and it gives me this error..."),Kn=c(),se=n("tr"),Ss=n("th"),er=a("1"),tr=c(),Os=n("td"),sr=a("https://github.com/huggingface/datasets/issues/2787"),ar=c(),Is=n("td"),or=a("ConnectionError: Couldn't reach https://raw.githubusercontent.com"),nr=c(),Ps=n("td"),rr=a("Hi @jinec,\\r\\n\\r\\nFrom time to time we get this kind of `ConnectionError` coming from the github.com website: https://raw.githubusercontent.com..."),lr=c(),Ns=n("td"),ir=a("Hello,\\r\\nI am trying to run run_glue.py and it gives me this error..."),dr=c(),ae=n("tr"),Fs=n("th"),cr=a("2"),ur=c(),Hs=n("td"),pr=a("https://github.com/huggingface/datasets/issues/2787"),mr=c(),Rs=n("td"),hr=a("ConnectionError: Couldn't reach https://raw.githubusercontent.com"),fr=c(),Ls=n("td"),_r=a("cannot connect\uFF0Ceven by Web browser\uFF0Cplease check that  there is some  problems\u3002"),gr=c(),Ms=n("td"),br=a("Hello,\\r\\nI am trying to run run_glue.py and it gives me this error..."),wr=c(),oe=n("tr"),Ws=n("th"),$r=a("3"),vr=c(),Us=n("td"),yr=a("https://github.com/huggingface/datasets/issues/2787"),kr=c(),zs=n("td"),xr=a("ConnectionError: Couldn't reach https://raw.githubusercontent.com"),Er=c(),Gs=n("td"),jr=a("I can access https://raw.githubusercontent.com/huggingface/datasets/1.7.0/datasets/glue/glue.py without problem..."),qr=c(),Ys=n("td"),Tr=a("Hello,\\r\\nI am trying to run run_glue.py and it gives me this error..."),Ma=c(),re=n("p"),Dr=a("Great, we can see the rows have been replicated, with the "),Bs=n("code"),Cr=a("comments"),Ar=a(" column containing the individual comments! Now that we\u2019re finished with Pandas, we can quickly switch back to a "),Vs=n("code"),Sr=a("Dataset"),Or=a(" by loading the "),Js=n("code"),Ir=a("DataFrame"),Pr=a(" in memory:"),Wa=c(),v(lt.$$.fragment),Ua=c(),v(it.$$.fragment),za=c(),It=n("p"),Nr=a("Okay, this has given us a few thousand comments to work with!"),Ga=c(),v(He.$$.fragment),Ya=c(),Re=n("p"),Fr=a("Now that we have one comment per row, let\u2019s create a new "),Qs=n("code"),Hr=a("comments_length"),Rr=a(" column that contains the number of words per comment:"),Ba=c(),v(dt.$$.fragment),Va=c(),Pt=n("p"),Lr=a("We can use this new column to filter out short comments, which typically include things like \u201Ccc @lewtun\u201D or \u201CThanks!\u201D that are not relevant for our search engine. There\u2019s no precise number to select for the filter, but around 15 words seems like a good start:"),Ja=c(),v(ct.$$.fragment),Qa=c(),v(ut.$$.fragment),Xa=c(),we=n("p"),Mr=a("Having cleaned up our dataset a bit, let\u2019s concatenate the issue title, description, and comments together in a new "),Xs=n("code"),Wr=a("text"),Ur=a(" column. As usual, we\u2019ll write a simple function that we can pass to "),Zs=n("code"),zr=a("Dataset.map()"),Gr=a(":"),Za=c(),v(pt.$$.fragment),Ka=c(),Nt=n("p"),Yr=a("We\u2019re finally ready to create some embeddings! Let\u2019s take a look."),eo=c(),Se=n("h2"),Le=n("a"),Ks=n("span"),v(mt.$$.fragment),Br=c(),ea=n("span"),Vr=a("Creating text embeddings"),to=c(),Y=n("p"),Jr=a("We saw in "),Ft=n("a"),Qr=a("Chapter 2"),Xr=a(" that we can obtain token embeddings by using the "),ta=n("code"),Zr=a("AutoModel"),Kr=a(" class. All we need to do is pick a suitable checkpoint to load the model from. Fortunately, there\u2019s a library called "),sa=n("code"),el=a("sentence-transformers"),tl=a(" that is dedicated to creating embeddings. As described in the library\u2019s "),ht=n("a"),sl=a("documentation"),al=a(", our use case is an example of "),aa=n("em"),ol=a("asymmetric semantic search"),nl=a(" because we have a short query whose answer we\u2019d like to find in a longer document, like a an issue comment. The handy "),ft=n("a"),rl=a("model overview table"),ll=a(" in the documentation indicates that the "),oa=n("code"),il=a("multi-qa-mpnet-base-dot-v1"),dl=a(" checkpoint has the best performance for semantic search, so we\u2019ll use that for our application. We\u2019ll also load the tokenizer using the same checkpoint:"),so=c(),pe.c(),Ht=c(),$e=n("p"),cl=a("As we mentioned earlier, we\u2019d like to represent each entry in our GitHub issues corpus as a single vector, so we need to \u201Cpool\u201D or average our token embeddings in some way. One popular approach is to perform "),na=n("em"),ul=a("CLS pooling"),pl=a(" on our model\u2019s outputs, where we simply collect the last hidden state for the special "),ra=n("code"),ml=a("[CLS]"),hl=a(" token. The following function does the trick for us:"),ao=c(),v(_t.$$.fragment),oo=c(),Rt=n("p"),fl=a("Next, we\u2019ll create a helper function that will tokenize a list of documents, place the tensors on the GPU, feed them to the model, and finally apply CLS pooling to the outputs:"),no=c(),he.c(),Lt=c(),Mt=n("p"),_l=a("Notice that we\u2019ve converted the embeddings to NumPy arrays \u2014 that\u2019s because \u{1F917} Datasets requires this format when we try to index them with FAISS, which we\u2019ll do next."),ro=c(),Oe=n("h2"),Me=n("a"),la=n("span"),v(gt.$$.fragment),gl=c(),ia=n("span"),bl=a("Using FAISS for efficient similarity search"),lo=c(),ve=n("p"),wl=a("Now that we have a dataset of embeddings, we need some way to search over them. To do this, we\u2019ll use a special data structure in \u{1F917} Datasets called a "),da=n("em"),$l=a("FAISS index"),vl=a(". "),bt=n("a"),yl=a("FAISS"),kl=a(" (short for Facebook AI Similarity Search) is a library that provides efficient algorithms to quickly search and cluster embedding vectors."),io=c(),ye=n("p"),xl=a("The basic idea behind FAISS is to create a special data structure called an "),ca=n("em"),El=a("index"),jl=a(" that allows one to find which embeddings are similar to an input embedding. Creating a FAISS index in \u{1F917} Datasets is simple \u2014 we use the "),ua=n("code"),ql=a("Dataset.add_faiss_index()"),Tl=a(" function and specify which column of our dataset we\u2019d like to index:"),co=c(),v(wt.$$.fragment),uo=c(),We=n("p"),Dl=a("We can now perform queries on this index by doing a nearest neighbor lookup with the "),pa=n("code"),Cl=a("Dataset.get_nearest_examples()"),Al=a(" function. Let\u2019s test this out by first embedding a question as follows:"),po=c(),_e.c(),Wt=c(),Ut=n("p"),Sl=a("Just like with the documents, we now have a 768-dimensional vector representing the query, which we can compare against the whole corpus to find the most similar embeddings:"),mo=c(),v($t.$$.fragment),ho=c(),ke=n("p"),Ol=a("The "),ma=n("code"),Il=a("Dataset.get_nearest_examples()"),Pl=a(" function returns a tuple of scores that rank the overlap between the query and the document, and a corresponding set of samples (here, the 5 best matches). Let\u2019s collect these in a "),ha=n("code"),Nl=a("pandas.DataFrame"),Fl=a(" so we can easily sort them:"),fo=c(),v(vt.$$.fragment),_o=c(),zt=n("p"),Hl=a("Now we can iterate over the first few rows to see how well our query matched the available comments:"),go=c(),v(yt.$$.fragment),bo=c(),v(kt.$$.fragment),wo=c(),Gt=n("p"),Rl=a("Not bad! Our second hit seems to match the query."),$o=c(),v(Ue.$$.fragment),this.h()},l(e){const i=Hd('[data-svelte="svelte-1phssyn"]',document.head);p=r(i,"META",{name:!0,content:!0}),i.forEach(s),$=u(e),y(h.$$.fragment,e),j=u(e),f=r(e,"H1",{class:!0});var Tt=l(f);w=r(Tt,"A",{id:!0,class:!0,href:!0});var Yt=l(w);I=r(Yt,"SPAN",{});var fa=l(I);y(_.$$.fragment,fa),fa.forEach(s),Yt.forEach(s),A=u(Tt),q=r(Tt,"SPAN",{});var Bt=l(q);M=o(Bt,"Semantic search with FAISS"),Bt.forEach(s),Tt.forEach(s),P=u(e),N.l(e),F=u(e),R=r(e,"P",{});var ze=l(R);z=o(ze,"In "),D=r(ze,"A",{href:!0});var Vt=l(D);H=o(Vt,"section 5"),Vt.forEach(s),B=o(ze,", we created a dataset of GitHub issues and comments from the \u{1F917} Datasets repository. In this section we\u2019ll use this information to build a search engine that can help us find answers to our most pressing questions about the library!"),ze.forEach(s),U=u(e),y(T.$$.fragment,e),G=u(e),m=r(e,"H2",{class:!0});var Dt=l(m);C=r(Dt,"A",{id:!0,class:!0,href:!0});var Xl=l(C);L=r(Xl,"SPAN",{});var Zl=l(L);y(V.$$.fragment,Zl),Zl.forEach(s),Xl.forEach(s),ne=u(Dt),de=r(Dt,"SPAN",{});var Kl=l(de);Io=o(Kl,"Using embeddings for semantic search"),Kl.forEach(s),Dt.forEach(s),ga=u(e),ge=r(e,"P",{});var Jt=l(ge);Po=o(Jt,"As we saw in "),Ct=r(Jt,"A",{href:!0});var ei=l(Ct);No=o(ei,"Chapter 1"),ei.forEach(s),Fo=o(Jt,", Transformer-based language models represent each token in a span of text as an "),ss=r(Jt,"EM",{});var ti=l(ss);Ho=o(ti,"embedding vector"),ti.forEach(s),Ro=o(Jt,". It turns out that one can \u201Cpool\u201D the individual embeddings to create a vector representation for whole sentences, paragraphs, or (in some cases) documents. These embeddings can then be used to find similar documents in the corpus by computing the dot-product similarity (or some other similarity metric) between each embedding and returning the documents with the greatest overlap."),Jt.forEach(s),ba=u(e),At=r(e,"P",{});var si=l(At);Lo=o(si,"In this section we\u2019ll use embeddings to develop a semantic search engine. These search engines offer several advantages over conventional approaches that are based on matching keywords in a query with the documents."),si.forEach(s),wa=u(e),Ce=r(e,"DIV",{class:!0});var yo=l(Ce);Be=r(yo,"IMG",{class:!0,src:!0,alt:!0}),Mo=u(yo),Ve=r(yo,"IMG",{class:!0,src:!0,alt:!0}),yo.forEach(s),$a=u(e),Ae=r(e,"H2",{class:!0});var ko=l(Ae);Ie=r(ko,"A",{id:!0,class:!0,href:!0});var ai=l(Ie);as=r(ai,"SPAN",{});var oi=l(as);y(Je.$$.fragment,oi),oi.forEach(s),ai.forEach(s),Wo=u(ko),os=r(ko,"SPAN",{});var ni=l(os);Uo=o(ni,"Loading and preparing the dataset"),ni.forEach(s),ko.forEach(s),va=u(e),St=r(e,"P",{});var ri=l(St);zo=o(ri,"The first thing we need to do is download our dataset of GitHub issues, so let\u2019s use the \u{1F917} Hub library to resolve the URL where our file is stored on the Hugging Face Hub:"),ri.forEach(s),ya=u(e),y(Qe.$$.fragment,e),ka=u(e),be=r(e,"P",{});var Qt=l(be);Go=o(Qt,"With the URL stored in "),ns=r(Qt,"CODE",{});var li=l(ns);Yo=o(li,"data_files"),li.forEach(s),Bo=o(Qt,", we can then load the remote dataset using the method introduced in "),Ot=r(Qt,"A",{href:!0});var ii=l(Ot);Vo=o(ii,"section 2"),ii.forEach(s),Jo=o(Qt,":"),Qt.forEach(s),xa=u(e),y(Xe.$$.fragment,e),Ea=u(e),y(Ze.$$.fragment,e),ja=u(e),J=r(e,"P",{});var le=l(J);Qo=o(le,"Here we\u2019ve specified the default "),rs=r(le,"CODE",{});var di=l(rs);Xo=o(di,"train"),di.forEach(s),Zo=o(le," split in "),ls=r(le,"CODE",{});var ci=l(ls);Ko=o(ci,"load_dataset()"),ci.forEach(s),en=o(le,", so it returns a "),is=r(le,"CODE",{});var ui=l(is);tn=o(ui,"Dataset"),ui.forEach(s),sn=o(le," instead of a "),ds=r(le,"CODE",{});var pi=l(ds);an=o(pi,"DatasetDict"),pi.forEach(s),on=o(le,". The first order of business is to filter out the pull requests, as these tend to be rarely used for answering user queries and will introduce noise in our search engine. As should be familiar by now, we can use the "),cs=r(le,"CODE",{});var mi=l(cs);nn=o(mi,"Dataset.filter()"),mi.forEach(s),rn=o(le," function to exclude these rows in our dataset. While we\u2019re at it, let\u2019s also filter out rows with no comments, since these provide no answers to user queries:"),le.forEach(s),qa=u(e),y(Ke.$$.fragment,e),Ta=u(e),y(et.$$.fragment,e),Da=u(e),Q=r(e,"P",{});var ie=l(Q);ln=o(ie,"We can see that there are a lot of columns in our dataset, most of which we don\u2019t need to build our search engine. From a search perspective, the most informative columns are "),us=r(ie,"CODE",{});var hi=l(us);dn=o(hi,"title"),hi.forEach(s),cn=o(ie,", "),ps=r(ie,"CODE",{});var fi=l(ps);un=o(fi,"body"),fi.forEach(s),pn=o(ie,", and "),ms=r(ie,"CODE",{});var _i=l(ms);mn=o(_i,"comments"),_i.forEach(s),hn=o(ie,", while "),hs=r(ie,"CODE",{});var gi=l(hs);fn=o(gi,"html_url"),gi.forEach(s),_n=o(ie," provides us with a link back to the source issue. Let\u2019s use the "),fs=r(ie,"CODE",{});var bi=l(fs);gn=o(bi,"Dataset.remove_columns()"),bi.forEach(s),bn=o(ie," function to drop the rest:"),ie.forEach(s),Ca=u(e),y(tt.$$.fragment,e),Aa=u(e),y(st.$$.fragment,e),Sa=u(e),Z=r(e,"P",{});var xe=l(Z);wn=o(xe,"To create our embeddings we\u2019ll augment each comment with the issue\u2019s title and body, since these fields often include useful contextual information. Because our "),_s=r(xe,"CODE",{});var wi=l(_s);$n=o(wi,"comments"),wi.forEach(s),vn=o(xe," column is currently a list of comments for each issue, we need to \u201Cexplode\u201D the column so that each row consists of an "),gs=r(xe,"CODE",{});var $i=l(gs);yn=o($i,"(html_url, title, body, comment)"),$i.forEach(s),kn=o(xe," tuple. In Pandas we can do this with the "),Pe=r(xe,"A",{href:!0,rel:!0});var Ll=l(Pe);bs=r(Ll,"CODE",{});var vi=l(bs);xn=o(vi,"DataFrame.explode()"),vi.forEach(s),En=o(Ll," function"),Ll.forEach(s),jn=o(xe,", which creates a new row for each element in a list-like column, while replicating all the other column values. To see this in action, let\u2019s first switch to the Pandas  "),ws=r(xe,"CODE",{});var yi=l(ws);qn=o(yi,"DataFrame"),yi.forEach(s),Tn=o(xe," format:"),xe.forEach(s),Oa=u(e),y(at.$$.fragment,e),Ia=u(e),Ne=r(e,"P",{});var xo=l(Ne);Dn=o(xo,"If we inspect the first row in this "),$s=r(xo,"CODE",{});var ki=l($s);Cn=o(ki,"DataFrame"),ki.forEach(s),An=o(xo," we can see there are four comments associated with this issue:"),xo.forEach(s),Pa=u(e),y(ot.$$.fragment,e),Na=u(e),y(nt.$$.fragment,e),Fa=u(e),Fe=r(e,"P",{});var Eo=l(Fe);Sn=o(Eo,"When we explode "),vs=r(Eo,"CODE",{});var xi=l(vs);On=o(xi,"df"),xi.forEach(s),In=o(Eo,", we expect to get one row for each of these comments. Let\u2019s check if that\u2019s the case:"),Eo.forEach(s),Ha=u(e),y(rt.$$.fragment,e),Ra=u(e),ee=r(e,"TABLE",{border:!0,class:!0,style:!0});var jo=l(ee);ys=r(jo,"THEAD",{});var Ei=l(ys);K=r(Ei,"TR",{style:!0});var Ee=l(K);La=r(Ee,"TH",{}),l(La).forEach(s),Pn=u(Ee),ks=r(Ee,"TH",{});var ji=l(ks);Nn=o(ji,"html_url"),ji.forEach(s),Fn=u(Ee),xs=r(Ee,"TH",{});var qi=l(xs);Hn=o(qi,"title"),qi.forEach(s),Rn=u(Ee),Es=r(Ee,"TH",{});var Ti=l(Es);Ln=o(Ti,"comments"),Ti.forEach(s),Mn=u(Ee),js=r(Ee,"TH",{});var Di=l(js);Wn=o(Di,"body"),Di.forEach(s),Ee.forEach(s),Ei.forEach(s),Un=u(jo),ce=r(jo,"TBODY",{});var Ge=l(ce);te=r(Ge,"TR",{});var je=l(te);qs=r(je,"TH",{});var Ci=l(qs);zn=o(Ci,"0"),Ci.forEach(s),Gn=u(je),Ts=r(je,"TD",{});var Ai=l(Ts);Yn=o(Ai,"https://github.com/huggingface/datasets/issues/2787"),Ai.forEach(s),Bn=u(je),Ds=r(je,"TD",{});var Si=l(Ds);Vn=o(Si,"ConnectionError: Couldn't reach https://raw.githubusercontent.com"),Si.forEach(s),Jn=u(je),Cs=r(je,"TD",{});var Oi=l(Cs);Qn=o(Oi,"the bug code locate in \uFF1A\\r\\n    if data_args.task_name is not None..."),Oi.forEach(s),Xn=u(je),As=r(je,"TD",{});var Ii=l(As);Zn=o(Ii,"Hello,\\r\\nI am trying to run run_glue.py and it gives me this error..."),Ii.forEach(s),je.forEach(s),Kn=u(Ge),se=r(Ge,"TR",{});var qe=l(se);Ss=r(qe,"TH",{});var Pi=l(Ss);er=o(Pi,"1"),Pi.forEach(s),tr=u(qe),Os=r(qe,"TD",{});var Ni=l(Os);sr=o(Ni,"https://github.com/huggingface/datasets/issues/2787"),Ni.forEach(s),ar=u(qe),Is=r(qe,"TD",{});var Fi=l(Is);or=o(Fi,"ConnectionError: Couldn't reach https://raw.githubusercontent.com"),Fi.forEach(s),nr=u(qe),Ps=r(qe,"TD",{});var Hi=l(Ps);rr=o(Hi,"Hi @jinec,\\r\\n\\r\\nFrom time to time we get this kind of `ConnectionError` coming from the github.com website: https://raw.githubusercontent.com..."),Hi.forEach(s),lr=u(qe),Ns=r(qe,"TD",{});var Ri=l(Ns);ir=o(Ri,"Hello,\\r\\nI am trying to run run_glue.py and it gives me this error..."),Ri.forEach(s),qe.forEach(s),dr=u(Ge),ae=r(Ge,"TR",{});var Te=l(ae);Fs=r(Te,"TH",{});var Li=l(Fs);cr=o(Li,"2"),Li.forEach(s),ur=u(Te),Hs=r(Te,"TD",{});var Mi=l(Hs);pr=o(Mi,"https://github.com/huggingface/datasets/issues/2787"),Mi.forEach(s),mr=u(Te),Rs=r(Te,"TD",{});var Wi=l(Rs);hr=o(Wi,"ConnectionError: Couldn't reach https://raw.githubusercontent.com"),Wi.forEach(s),fr=u(Te),Ls=r(Te,"TD",{});var Ui=l(Ls);_r=o(Ui,"cannot connect\uFF0Ceven by Web browser\uFF0Cplease check that  there is some  problems\u3002"),Ui.forEach(s),gr=u(Te),Ms=r(Te,"TD",{});var zi=l(Ms);br=o(zi,"Hello,\\r\\nI am trying to run run_glue.py and it gives me this error..."),zi.forEach(s),Te.forEach(s),wr=u(Ge),oe=r(Ge,"TR",{});var De=l(oe);Ws=r(De,"TH",{});var Gi=l(Ws);$r=o(Gi,"3"),Gi.forEach(s),vr=u(De),Us=r(De,"TD",{});var Yi=l(Us);yr=o(Yi,"https://github.com/huggingface/datasets/issues/2787"),Yi.forEach(s),kr=u(De),zs=r(De,"TD",{});var Bi=l(zs);xr=o(Bi,"ConnectionError: Couldn't reach https://raw.githubusercontent.com"),Bi.forEach(s),Er=u(De),Gs=r(De,"TD",{});var Vi=l(Gs);jr=o(Vi,"I can access https://raw.githubusercontent.com/huggingface/datasets/1.7.0/datasets/glue/glue.py without problem..."),Vi.forEach(s),qr=u(De),Ys=r(De,"TD",{});var Ji=l(Ys);Tr=o(Ji,"Hello,\\r\\nI am trying to run run_glue.py and it gives me this error..."),Ji.forEach(s),De.forEach(s),Ge.forEach(s),jo.forEach(s),Ma=u(e),re=r(e,"P",{});var Ye=l(re);Dr=o(Ye,"Great, we can see the rows have been replicated, with the "),Bs=r(Ye,"CODE",{});var Qi=l(Bs);Cr=o(Qi,"comments"),Qi.forEach(s),Ar=o(Ye," column containing the individual comments! Now that we\u2019re finished with Pandas, we can quickly switch back to a "),Vs=r(Ye,"CODE",{});var Xi=l(Vs);Sr=o(Xi,"Dataset"),Xi.forEach(s),Or=o(Ye," by loading the "),Js=r(Ye,"CODE",{});var Zi=l(Js);Ir=o(Zi,"DataFrame"),Zi.forEach(s),Pr=o(Ye," in memory:"),Ye.forEach(s),Wa=u(e),y(lt.$$.fragment,e),Ua=u(e),y(it.$$.fragment,e),za=u(e),It=r(e,"P",{});var Ki=l(It);Nr=o(Ki,"Okay, this has given us a few thousand comments to work with!"),Ki.forEach(s),Ga=u(e),y(He.$$.fragment,e),Ya=u(e),Re=r(e,"P",{});var qo=l(Re);Fr=o(qo,"Now that we have one comment per row, let\u2019s create a new "),Qs=r(qo,"CODE",{});var ed=l(Qs);Hr=o(ed,"comments_length"),ed.forEach(s),Rr=o(qo," column that contains the number of words per comment:"),qo.forEach(s),Ba=u(e),y(dt.$$.fragment,e),Va=u(e),Pt=r(e,"P",{});var td=l(Pt);Lr=o(td,"We can use this new column to filter out short comments, which typically include things like \u201Ccc @lewtun\u201D or \u201CThanks!\u201D that are not relevant for our search engine. There\u2019s no precise number to select for the filter, but around 15 words seems like a good start:"),td.forEach(s),Ja=u(e),y(ct.$$.fragment,e),Qa=u(e),y(ut.$$.fragment,e),Xa=u(e),we=r(e,"P",{});var Xt=l(we);Mr=o(Xt,"Having cleaned up our dataset a bit, let\u2019s concatenate the issue title, description, and comments together in a new "),Xs=r(Xt,"CODE",{});var sd=l(Xs);Wr=o(sd,"text"),sd.forEach(s),Ur=o(Xt," column. As usual, we\u2019ll write a simple function that we can pass to "),Zs=r(Xt,"CODE",{});var ad=l(Zs);zr=o(ad,"Dataset.map()"),ad.forEach(s),Gr=o(Xt,":"),Xt.forEach(s),Za=u(e),y(pt.$$.fragment,e),Ka=u(e),Nt=r(e,"P",{});var od=l(Nt);Yr=o(od,"We\u2019re finally ready to create some embeddings! Let\u2019s take a look."),od.forEach(s),eo=u(e),Se=r(e,"H2",{class:!0});var To=l(Se);Le=r(To,"A",{id:!0,class:!0,href:!0});var nd=l(Le);Ks=r(nd,"SPAN",{});var rd=l(Ks);y(mt.$$.fragment,rd),rd.forEach(s),nd.forEach(s),Br=u(To),ea=r(To,"SPAN",{});var ld=l(ea);Vr=o(ld,"Creating text embeddings"),ld.forEach(s),To.forEach(s),to=u(e),Y=r(e,"P",{});var X=l(Y);Jr=o(X,"We saw in "),Ft=r(X,"A",{href:!0});var id=l(Ft);Qr=o(id,"Chapter 2"),id.forEach(s),Xr=o(X," that we can obtain token embeddings by using the "),ta=r(X,"CODE",{});var dd=l(ta);Zr=o(dd,"AutoModel"),dd.forEach(s),Kr=o(X," class. All we need to do is pick a suitable checkpoint to load the model from. Fortunately, there\u2019s a library called "),sa=r(X,"CODE",{});var cd=l(sa);el=o(cd,"sentence-transformers"),cd.forEach(s),tl=o(X," that is dedicated to creating embeddings. As described in the library\u2019s "),ht=r(X,"A",{href:!0,rel:!0});var ud=l(ht);sl=o(ud,"documentation"),ud.forEach(s),al=o(X,", our use case is an example of "),aa=r(X,"EM",{});var pd=l(aa);ol=o(pd,"asymmetric semantic search"),pd.forEach(s),nl=o(X," because we have a short query whose answer we\u2019d like to find in a longer document, like a an issue comment. The handy "),ft=r(X,"A",{href:!0,rel:!0});var md=l(ft);rl=o(md,"model overview table"),md.forEach(s),ll=o(X," in the documentation indicates that the "),oa=r(X,"CODE",{});var hd=l(oa);il=o(hd,"multi-qa-mpnet-base-dot-v1"),hd.forEach(s),dl=o(X," checkpoint has the best performance for semantic search, so we\u2019ll use that for our application. We\u2019ll also load the tokenizer using the same checkpoint:"),X.forEach(s),so=u(e),pe.l(e),Ht=u(e),$e=r(e,"P",{});var Zt=l($e);cl=o(Zt,"As we mentioned earlier, we\u2019d like to represent each entry in our GitHub issues corpus as a single vector, so we need to \u201Cpool\u201D or average our token embeddings in some way. One popular approach is to perform "),na=r(Zt,"EM",{});var fd=l(na);ul=o(fd,"CLS pooling"),fd.forEach(s),pl=o(Zt," on our model\u2019s outputs, where we simply collect the last hidden state for the special "),ra=r(Zt,"CODE",{});var _d=l(ra);ml=o(_d,"[CLS]"),_d.forEach(s),hl=o(Zt," token. The following function does the trick for us:"),Zt.forEach(s),ao=u(e),y(_t.$$.fragment,e),oo=u(e),Rt=r(e,"P",{});var gd=l(Rt);fl=o(gd,"Next, we\u2019ll create a helper function that will tokenize a list of documents, place the tensors on the GPU, feed them to the model, and finally apply CLS pooling to the outputs:"),gd.forEach(s),no=u(e),he.l(e),Lt=u(e),Mt=r(e,"P",{});var bd=l(Mt);_l=o(bd,"Notice that we\u2019ve converted the embeddings to NumPy arrays \u2014 that\u2019s because \u{1F917} Datasets requires this format when we try to index them with FAISS, which we\u2019ll do next."),bd.forEach(s),ro=u(e),Oe=r(e,"H2",{class:!0});var Do=l(Oe);Me=r(Do,"A",{id:!0,class:!0,href:!0});var wd=l(Me);la=r(wd,"SPAN",{});var $d=l(la);y(gt.$$.fragment,$d),$d.forEach(s),wd.forEach(s),gl=u(Do),ia=r(Do,"SPAN",{});var vd=l(ia);bl=o(vd,"Using FAISS for efficient similarity search"),vd.forEach(s),Do.forEach(s),lo=u(e),ve=r(e,"P",{});var Kt=l(ve);wl=o(Kt,"Now that we have a dataset of embeddings, we need some way to search over them. To do this, we\u2019ll use a special data structure in \u{1F917} Datasets called a "),da=r(Kt,"EM",{});var yd=l(da);$l=o(yd,"FAISS index"),yd.forEach(s),vl=o(Kt,". "),bt=r(Kt,"A",{href:!0,rel:!0});var kd=l(bt);yl=o(kd,"FAISS"),kd.forEach(s),kl=o(Kt," (short for Facebook AI Similarity Search) is a library that provides efficient algorithms to quickly search and cluster embedding vectors."),Kt.forEach(s),io=u(e),ye=r(e,"P",{});var es=l(ye);xl=o(es,"The basic idea behind FAISS is to create a special data structure called an "),ca=r(es,"EM",{});var xd=l(ca);El=o(xd,"index"),xd.forEach(s),jl=o(es," that allows one to find which embeddings are similar to an input embedding. Creating a FAISS index in \u{1F917} Datasets is simple \u2014 we use the "),ua=r(es,"CODE",{});var Ed=l(ua);ql=o(Ed,"Dataset.add_faiss_index()"),Ed.forEach(s),Tl=o(es," function and specify which column of our dataset we\u2019d like to index:"),es.forEach(s),co=u(e),y(wt.$$.fragment,e),uo=u(e),We=r(e,"P",{});var Co=l(We);Dl=o(Co,"We can now perform queries on this index by doing a nearest neighbor lookup with the "),pa=r(Co,"CODE",{});var jd=l(pa);Cl=o(jd,"Dataset.get_nearest_examples()"),jd.forEach(s),Al=o(Co," function. Let\u2019s test this out by first embedding a question as follows:"),Co.forEach(s),po=u(e),_e.l(e),Wt=u(e),Ut=r(e,"P",{});var qd=l(Ut);Sl=o(qd,"Just like with the documents, we now have a 768-dimensional vector representing the query, which we can compare against the whole corpus to find the most similar embeddings:"),qd.forEach(s),mo=u(e),y($t.$$.fragment,e),ho=u(e),ke=r(e,"P",{});var ts=l(ke);Ol=o(ts,"The "),ma=r(ts,"CODE",{});var Td=l(ma);Il=o(Td,"Dataset.get_nearest_examples()"),Td.forEach(s),Pl=o(ts," function returns a tuple of scores that rank the overlap between the query and the document, and a corresponding set of samples (here, the 5 best matches). Let\u2019s collect these in a "),ha=r(ts,"CODE",{});var Dd=l(ha);Nl=o(Dd,"pandas.DataFrame"),Dd.forEach(s),Fl=o(ts," so we can easily sort them:"),ts.forEach(s),fo=u(e),y(vt.$$.fragment,e),_o=u(e),zt=r(e,"P",{});var Cd=l(zt);Hl=o(Cd,"Now we can iterate over the first few rows to see how well our query matched the available comments:"),Cd.forEach(s),go=u(e),y(yt.$$.fragment,e),bo=u(e),y(kt.$$.fragment,e),wo=u(e),Gt=r(e,"P",{});var Ad=l(Gt);Rl=o(Ad,"Not bad! Our second hit seems to match the query."),Ad.forEach(s),$o=u(e),y(Ue.$$.fragment,e),this.h()},h(){E(p,"name","hf:doc:metadata"),E(p,"content",JSON.stringify(Kd)),E(w,"id","semantic-search-with-faiss"),E(w,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),E(w,"href","#semantic-search-with-faiss"),E(f,"class","relative group"),E(D,"href","/course/chapter5/5"),E(C,"id","using-embeddings-for-semantic-search"),E(C,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),E(C,"href","#using-embeddings-for-semantic-search"),E(m,"class","relative group"),E(Ct,"href","/course/chapter1"),E(Be,"class","block dark:hidden"),Sd(Be.src,Ml="https://huggingface.co/datasets/huggingface-course/documentation-images/resolve/main/en/chapter5/semantic-search.svg")||E(Be,"src",Ml),E(Be,"alt","Semantic search."),E(Ve,"class","hidden dark:block"),Sd(Ve.src,Wl="https://huggingface.co/datasets/huggingface-course/documentation-images/resolve/main/en/chapter5/semantic-search-dark.svg")||E(Ve,"src",Wl),E(Ve,"alt","Semantic search."),E(Ce,"class","flex justify-center"),E(Ie,"id","loading-and-preparing-the-dataset"),E(Ie,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),E(Ie,"href","#loading-and-preparing-the-dataset"),E(Ae,"class","relative group"),E(Ot,"href","/course/chapter5/2"),E(Pe,"href","https://pandas.pydata.org/pandas-docs/stable/reference/api/pandas.DataFrame.explode.html"),E(Pe,"rel","nofollow"),Ao(K,"text-align","right"),E(ee,"border","1"),E(ee,"class","dataframe"),Ao(ee,"table-layout","fixed"),Ao(ee,"word-wrap","break-word"),Ao(ee,"width","100%"),E(Le,"id","creating-text-embeddings"),E(Le,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),E(Le,"href","#creating-text-embeddings"),E(Se,"class","relative group"),E(Ft,"href","/course/chapter2"),E(ht,"href","https://www.sbert.net/examples/applications/semantic-search/README.html#symmetric-vs-asymmetric-semantic-search"),E(ht,"rel","nofollow"),E(ft,"href","https://www.sbert.net/docs/pretrained_models.html#model-overview"),E(ft,"rel","nofollow"),E(Me,"id","using-faiss-for-efficient-similarity-search"),E(Me,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),E(Me,"href","#using-faiss-for-efficient-similarity-search"),E(Oe,"class","relative group"),E(bt,"href","https://faiss.ai/"),E(bt,"rel","nofollow")},m(e,i){t(document.head,p),d(e,$,i),k(h,e,i),d(e,j,i),d(e,f,i),t(f,w),t(w,I),k(_,I,null),t(f,A),t(f,q),t(q,M),d(e,P,i),xt[S].m(e,i),d(e,F,i),d(e,R,i),t(R,z),t(R,D),t(D,H),t(R,B),d(e,U,i),k(T,e,i),d(e,G,i),d(e,m,i),t(m,C),t(C,L),k(V,L,null),t(m,ne),t(m,de),t(de,Io),d(e,ga,i),d(e,ge,i),t(ge,Po),t(ge,Ct),t(Ct,No),t(ge,Fo),t(ge,ss),t(ss,Ho),t(ge,Ro),d(e,ba,i),d(e,At,i),t(At,Lo),d(e,wa,i),d(e,Ce,i),t(Ce,Be),t(Ce,Mo),t(Ce,Ve),d(e,$a,i),d(e,Ae,i),t(Ae,Ie),t(Ie,as),k(Je,as,null),t(Ae,Wo),t(Ae,os),t(os,Uo),d(e,va,i),d(e,St,i),t(St,zo),d(e,ya,i),k(Qe,e,i),d(e,ka,i),d(e,be,i),t(be,Go),t(be,ns),t(ns,Yo),t(be,Bo),t(be,Ot),t(Ot,Vo),t(be,Jo),d(e,xa,i),k(Xe,e,i),d(e,Ea,i),k(Ze,e,i),d(e,ja,i),d(e,J,i),t(J,Qo),t(J,rs),t(rs,Xo),t(J,Zo),t(J,ls),t(ls,Ko),t(J,en),t(J,is),t(is,tn),t(J,sn),t(J,ds),t(ds,an),t(J,on),t(J,cs),t(cs,nn),t(J,rn),d(e,qa,i),k(Ke,e,i),d(e,Ta,i),k(et,e,i),d(e,Da,i),d(e,Q,i),t(Q,ln),t(Q,us),t(us,dn),t(Q,cn),t(Q,ps),t(ps,un),t(Q,pn),t(Q,ms),t(ms,mn),t(Q,hn),t(Q,hs),t(hs,fn),t(Q,_n),t(Q,fs),t(fs,gn),t(Q,bn),d(e,Ca,i),k(tt,e,i),d(e,Aa,i),k(st,e,i),d(e,Sa,i),d(e,Z,i),t(Z,wn),t(Z,_s),t(_s,$n),t(Z,vn),t(Z,gs),t(gs,yn),t(Z,kn),t(Z,Pe),t(Pe,bs),t(bs,xn),t(Pe,En),t(Z,jn),t(Z,ws),t(ws,qn),t(Z,Tn),d(e,Oa,i),k(at,e,i),d(e,Ia,i),d(e,Ne,i),t(Ne,Dn),t(Ne,$s),t($s,Cn),t(Ne,An),d(e,Pa,i),k(ot,e,i),d(e,Na,i),k(nt,e,i),d(e,Fa,i),d(e,Fe,i),t(Fe,Sn),t(Fe,vs),t(vs,On),t(Fe,In),d(e,Ha,i),k(rt,e,i),d(e,Ra,i),d(e,ee,i),t(ee,ys),t(ys,K),t(K,La),t(K,Pn),t(K,ks),t(ks,Nn),t(K,Fn),t(K,xs),t(xs,Hn),t(K,Rn),t(K,Es),t(Es,Ln),t(K,Mn),t(K,js),t(js,Wn),t(ee,Un),t(ee,ce),t(ce,te),t(te,qs),t(qs,zn),t(te,Gn),t(te,Ts),t(Ts,Yn),t(te,Bn),t(te,Ds),t(Ds,Vn),t(te,Jn),t(te,Cs),t(Cs,Qn),t(te,Xn),t(te,As),t(As,Zn),t(ce,Kn),t(ce,se),t(se,Ss),t(Ss,er),t(se,tr),t(se,Os),t(Os,sr),t(se,ar),t(se,Is),t(Is,or),t(se,nr),t(se,Ps),t(Ps,rr),t(se,lr),t(se,Ns),t(Ns,ir),t(ce,dr),t(ce,ae),t(ae,Fs),t(Fs,cr),t(ae,ur),t(ae,Hs),t(Hs,pr),t(ae,mr),t(ae,Rs),t(Rs,hr),t(ae,fr),t(ae,Ls),t(Ls,_r),t(ae,gr),t(ae,Ms),t(Ms,br),t(ce,wr),t(ce,oe),t(oe,Ws),t(Ws,$r),t(oe,vr),t(oe,Us),t(Us,yr),t(oe,kr),t(oe,zs),t(zs,xr),t(oe,Er),t(oe,Gs),t(Gs,jr),t(oe,qr),t(oe,Ys),t(Ys,Tr),d(e,Ma,i),d(e,re,i),t(re,Dr),t(re,Bs),t(Bs,Cr),t(re,Ar),t(re,Vs),t(Vs,Sr),t(re,Or),t(re,Js),t(Js,Ir),t(re,Pr),d(e,Wa,i),k(lt,e,i),d(e,Ua,i),k(it,e,i),d(e,za,i),d(e,It,i),t(It,Nr),d(e,Ga,i),k(He,e,i),d(e,Ya,i),d(e,Re,i),t(Re,Fr),t(Re,Qs),t(Qs,Hr),t(Re,Rr),d(e,Ba,i),k(dt,e,i),d(e,Va,i),d(e,Pt,i),t(Pt,Lr),d(e,Ja,i),k(ct,e,i),d(e,Qa,i),k(ut,e,i),d(e,Xa,i),d(e,we,i),t(we,Mr),t(we,Xs),t(Xs,Wr),t(we,Ur),t(we,Zs),t(Zs,zr),t(we,Gr),d(e,Za,i),k(pt,e,i),d(e,Ka,i),d(e,Nt,i),t(Nt,Yr),d(e,eo,i),d(e,Se,i),t(Se,Le),t(Le,Ks),k(mt,Ks,null),t(Se,Br),t(Se,ea),t(ea,Vr),d(e,to,i),d(e,Y,i),t(Y,Jr),t(Y,Ft),t(Ft,Qr),t(Y,Xr),t(Y,ta),t(ta,Zr),t(Y,Kr),t(Y,sa),t(sa,el),t(Y,tl),t(Y,ht),t(ht,sl),t(Y,al),t(Y,aa),t(aa,ol),t(Y,nl),t(Y,ft),t(ft,rl),t(Y,ll),t(Y,oa),t(oa,il),t(Y,dl),d(e,so,i),Et[ue].m(e,i),d(e,Ht,i),d(e,$e,i),t($e,cl),t($e,na),t(na,ul),t($e,pl),t($e,ra),t(ra,ml),t($e,hl),d(e,ao,i),k(_t,e,i),d(e,oo,i),d(e,Rt,i),t(Rt,fl),d(e,no,i),jt[me].m(e,i),d(e,Lt,i),d(e,Mt,i),t(Mt,_l),d(e,ro,i),d(e,Oe,i),t(Oe,Me),t(Me,la),k(gt,la,null),t(Oe,gl),t(Oe,ia),t(ia,bl),d(e,lo,i),d(e,ve,i),t(ve,wl),t(ve,da),t(da,$l),t(ve,vl),t(ve,bt),t(bt,yl),t(ve,kl),d(e,io,i),d(e,ye,i),t(ye,xl),t(ye,ca),t(ca,El),t(ye,jl),t(ye,ua),t(ua,ql),t(ye,Tl),d(e,co,i),k(wt,e,i),d(e,uo,i),d(e,We,i),t(We,Dl),t(We,pa),t(pa,Cl),t(We,Al),d(e,po,i),qt[fe].m(e,i),d(e,Wt,i),d(e,Ut,i),t(Ut,Sl),d(e,mo,i),k($t,e,i),d(e,ho,i),d(e,ke,i),t(ke,Ol),t(ke,ma),t(ma,Il),t(ke,Pl),t(ke,ha),t(ha,Nl),t(ke,Fl),d(e,fo,i),k(vt,e,i),d(e,_o,i),d(e,zt,i),t(zt,Hl),d(e,go,i),k(yt,e,i),d(e,bo,i),k(kt,e,i),d(e,wo,i),d(e,Gt,i),t(Gt,Rl),d(e,$o,i),k(Ue,e,i),vo=!0},p(e,[i]){const Tt={};i&1&&(Tt.fw=e[0]),h.$set(Tt);let Yt=S;S=zl(e),S!==Yt&&(Oo(),g(xt[Yt],1,1,()=>{xt[Yt]=null}),So(),N=xt[S],N||(N=xt[S]=Ul[S](e),N.c()),b(N,1),N.m(F.parentNode,F));const fa={};i&2&&(fa.$$scope={dirty:i,ctx:e}),He.$set(fa);let Bt=ue;ue=Yl(e),ue!==Bt&&(Oo(),g(Et[Bt],1,1,()=>{Et[Bt]=null}),So(),pe=Et[ue],pe||(pe=Et[ue]=Gl[ue](e),pe.c()),b(pe,1),pe.m(Ht.parentNode,Ht));let ze=me;me=Vl(e),me!==ze&&(Oo(),g(jt[ze],1,1,()=>{jt[ze]=null}),So(),he=jt[me],he||(he=jt[me]=Bl[me](e),he.c()),b(he,1),he.m(Lt.parentNode,Lt));let Vt=fe;fe=Ql(e),fe!==Vt&&(Oo(),g(qt[Vt],1,1,()=>{qt[Vt]=null}),So(),_e=qt[fe],_e||(_e=qt[fe]=Jl[fe](e),_e.c()),b(_e,1),_e.m(Wt.parentNode,Wt));const Dt={};i&2&&(Dt.$$scope={dirty:i,ctx:e}),Ue.$set(Dt)},i(e){vo||(b(h.$$.fragment,e),b(_.$$.fragment,e),b(N),b(T.$$.fragment,e),b(V.$$.fragment,e),b(Je.$$.fragment,e),b(Qe.$$.fragment,e),b(Xe.$$.fragment,e),b(Ze.$$.fragment,e),b(Ke.$$.fragment,e),b(et.$$.fragment,e),b(tt.$$.fragment,e),b(st.$$.fragment,e),b(at.$$.fragment,e),b(ot.$$.fragment,e),b(nt.$$.fragment,e),b(rt.$$.fragment,e),b(lt.$$.fragment,e),b(it.$$.fragment,e),b(He.$$.fragment,e),b(dt.$$.fragment,e),b(ct.$$.fragment,e),b(ut.$$.fragment,e),b(pt.$$.fragment,e),b(mt.$$.fragment,e),b(pe),b(_t.$$.fragment,e),b(he),b(gt.$$.fragment,e),b(wt.$$.fragment,e),b(_e),b($t.$$.fragment,e),b(vt.$$.fragment,e),b(yt.$$.fragment,e),b(kt.$$.fragment,e),b(Ue.$$.fragment,e),vo=!0)},o(e){g(h.$$.fragment,e),g(_.$$.fragment,e),g(N),g(T.$$.fragment,e),g(V.$$.fragment,e),g(Je.$$.fragment,e),g(Qe.$$.fragment,e),g(Xe.$$.fragment,e),g(Ze.$$.fragment,e),g(Ke.$$.fragment,e),g(et.$$.fragment,e),g(tt.$$.fragment,e),g(st.$$.fragment,e),g(at.$$.fragment,e),g(ot.$$.fragment,e),g(nt.$$.fragment,e),g(rt.$$.fragment,e),g(lt.$$.fragment,e),g(it.$$.fragment,e),g(He.$$.fragment,e),g(dt.$$.fragment,e),g(ct.$$.fragment,e),g(ut.$$.fragment,e),g(pt.$$.fragment,e),g(mt.$$.fragment,e),g(pe),g(_t.$$.fragment,e),g(he),g(gt.$$.fragment,e),g(wt.$$.fragment,e),g(_e),g($t.$$.fragment,e),g(vt.$$.fragment,e),g(yt.$$.fragment,e),g(kt.$$.fragment,e),g(Ue.$$.fragment,e),vo=!1},d(e){s(p),e&&s($),x(h,e),e&&s(j),e&&s(f),x(_),e&&s(P),xt[S].d(e),e&&s(F),e&&s(R),e&&s(U),x(T,e),e&&s(G),e&&s(m),x(V),e&&s(ga),e&&s(ge),e&&s(ba),e&&s(At),e&&s(wa),e&&s(Ce),e&&s($a),e&&s(Ae),x(Je),e&&s(va),e&&s(St),e&&s(ya),x(Qe,e),e&&s(ka),e&&s(be),e&&s(xa),x(Xe,e),e&&s(Ea),x(Ze,e),e&&s(ja),e&&s(J),e&&s(qa),x(Ke,e),e&&s(Ta),x(et,e),e&&s(Da),e&&s(Q),e&&s(Ca),x(tt,e),e&&s(Aa),x(st,e),e&&s(Sa),e&&s(Z),e&&s(Oa),x(at,e),e&&s(Ia),e&&s(Ne),e&&s(Pa),x(ot,e),e&&s(Na),x(nt,e),e&&s(Fa),e&&s(Fe),e&&s(Ha),x(rt,e),e&&s(Ra),e&&s(ee),e&&s(Ma),e&&s(re),e&&s(Wa),x(lt,e),e&&s(Ua),x(it,e),e&&s(za),e&&s(It),e&&s(Ga),x(He,e),e&&s(Ya),e&&s(Re),e&&s(Ba),x(dt,e),e&&s(Va),e&&s(Pt),e&&s(Ja),x(ct,e),e&&s(Qa),x(ut,e),e&&s(Xa),e&&s(we),e&&s(Za),x(pt,e),e&&s(Ka),e&&s(Nt),e&&s(eo),e&&s(Se),x(mt),e&&s(to),e&&s(Y),e&&s(so),Et[ue].d(e),e&&s(Ht),e&&s($e),e&&s(ao),x(_t,e),e&&s(oo),e&&s(Rt),e&&s(no),jt[me].d(e),e&&s(Lt),e&&s(Mt),e&&s(ro),e&&s(Oe),x(gt),e&&s(lo),e&&s(ve),e&&s(io),e&&s(ye),e&&s(co),x(wt,e),e&&s(uo),e&&s(We),e&&s(po),qt[fe].d(e),e&&s(Wt),e&&s(Ut),e&&s(mo),x($t,e),e&&s(ho),e&&s(ke),e&&s(fo),x(vt,e),e&&s(_o),e&&s(zt),e&&s(go),x(yt,e),e&&s(bo),x(kt,e),e&&s(wo),e&&s(Gt),e&&s($o),x(Ue,e)}}}const Kd={local:"semantic-search-with-faiss",sections:[{local:"using-embeddings-for-semantic-search",title:"Using embeddings for semantic search"},{local:"loading-and-preparing-the-dataset",title:"Loading and preparing the dataset"},{local:"creating-text-embeddings",title:"Creating text embeddings"},{local:"using-faiss-for-efficient-similarity-search",title:"Using FAISS for efficient similarity search"}],title:"Semantic search with FAISS"};function ec(W,p,$){let h="pt";return Rd(()=>{const j=new URLSearchParams(window.location.search);$(0,h=j.get("fw")||"pt")}),[h]}class ic extends Pd{constructor(p){super();Nd(this,p,ec,Zd,Fd,{})}}export{ic as default,Kd as metadata};
