import{S as Kc,i as Zc,s as ed,e as o,k as d,w as E,t as n,M as td,c as r,d as s,m as p,x,a as l,h as a,b as j,N as Jc,f as eo,F as t,g as u,y as k,o as v,p as to,q as b,B as w,v as sd,n as so}from"../../chunks/vendor-1e8b365d.js";import{T as Qc}from"../../chunks/Tip-62b14c6e.js";import{Y as nd}from"../../chunks/Youtube-c2a8cc39.js";import{I as Rn}from"../../chunks/IconCopyLink-483c28ba.js";import{C as A}from"../../chunks/CodeBlock-e5764662.js";import{D as Xc}from"../../chunks/DocNotebookDropdown-37d928d3.js";import{F as ad}from"../../chunks/FrameworkSwitchCourse-7f8f0f31.js";function od(G){let m,q;return m=new Xc({props:{classNames:"absolute z-10 right-0 top-0",options:[{label:"Google Colab",value:"https://colab.research.google.com/github/huggingface/notebooks/blob/master/course/chapter5/section6_tf.ipynb"},{label:"Aws Studio",value:"https://studiolab.sagemaker.aws/import/github/huggingface/notebooks/blob/master/course/chapter5/section6_tf.ipynb"}]}}),{c(){E(m.$$.fragment)},l(f){x(m.$$.fragment,f)},m(f,y){k(m,f,y),q=!0},i(f){q||(b(m.$$.fragment,f),q=!0)},o(f){v(m.$$.fragment,f),q=!1},d(f){w(m,f)}}}function rd(G){let m,q;return m=new Xc({props:{classNames:"absolute z-10 right-0 top-0",options:[{label:"Google Colab",value:"https://colab.research.google.com/github/huggingface/notebooks/blob/master/course/chapter5/section6_pt.ipynb"},{label:"Aws Studio",value:"https://studiolab.sagemaker.aws/import/github/huggingface/notebooks/blob/master/course/chapter5/section6_pt.ipynb"}]}}),{c(){E(m.$$.fragment)},l(f){x(m.$$.fragment,f)},m(f,y){k(m,f,y),q=!0},i(f){q||(b(m.$$.fragment,f),q=!0)},o(f){v(m.$$.fragment,f),q=!1},d(f){w(m,f)}}}function ld(G){let m,q,f,y,_,$,S,g,C,D,z,O,T,N,I,R,Y,P,Q,B,U,H,V;return{c(){m=o("p"),q=n("\u270F\uFE0F "),f=o("strong"),y=n("Essayez !"),_=n(" Voyez si vous pouvez utiliser "),$=o("code"),S=n("Dataset.map()"),g=n(" pour exploser la colonne "),C=o("code"),D=n("comments"),z=n(" de "),O=o("code"),T=n("issues_dataset"),N=d(),I=o("em"),R=n("sans"),Y=n(" recourir \xE0 l\u2019utilisation de Pandas. C\u2019est un peu d\xE9licat. La section "),P=o("a"),Q=n("\xAB Batch mapping \xBB"),B=n(" de la documentation \u{1F917} "),U=o("em"),H=n("Datasets"),V=n(" peut \xEAtre utile pour cette t\xE2che."),this.h()},l(c){m=r(c,"P",{});var h=l(m);q=a(h,"\u270F\uFE0F "),f=r(h,"STRONG",{});var L=l(f);y=a(L,"Essayez !"),L.forEach(s),_=a(h," Voyez si vous pouvez utiliser "),$=r(h,"CODE",{});var W=l($);S=a(W,"Dataset.map()"),W.forEach(s),g=a(h," pour exploser la colonne "),C=r(h,"CODE",{});var J=l(C);D=a(J,"comments"),J.forEach(s),z=a(h," de "),O=r(h,"CODE",{});var K=l(O);T=a(K,"issues_dataset"),K.forEach(s),N=p(h),I=r(h,"EM",{});var oe=l(I);R=a(oe,"sans"),oe.forEach(s),Y=a(h," recourir \xE0 l\u2019utilisation de Pandas. C\u2019est un peu d\xE9licat. La section "),P=r(h,"A",{href:!0,rel:!0});var Ie=l(P);Q=a(Ie,"\xAB Batch mapping \xBB"),Ie.forEach(s),B=a(h," de la documentation \u{1F917} "),U=r(h,"EM",{});var He=l(U);H=a(He,"Datasets"),He.forEach(s),V=a(h," peut \xEAtre utile pour cette t\xE2che."),h.forEach(s),this.h()},h(){j(P,"href","https://huggingface.co/docs/datasets/v1.12.1/about_map_batch.html?batch-mapping#batch-mapping"),j(P,"rel","nofollow")},m(c,h){u(c,m,h),t(m,q),t(m,f),t(f,y),t(m,_),t(m,$),t($,S),t(m,g),t(m,C),t(C,D),t(m,z),t(m,O),t(O,T),t(m,N),t(m,I),t(I,R),t(m,Y),t(m,P),t(P,Q),t(m,B),t(m,U),t(U,H),t(m,V)},d(c){c&&s(m)}}}function id(G){let m,q,f,y,_,$,S,g,C,D,z,O,T,N,I,R,Y,P,Q,B,U,H,V;return m=new A({props:{code:`from transformers import AutoTokenizer, TFAutoModel

model_ckpt = "sentence-transformers/multi-qa-mpnet-base-dot-v1"
tokenizer = AutoTokenizer.from_pretrained(model_ckpt)
model = TFAutoModel.from_pretrained(model_ckpt, from_pt=True)`,highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoTokenizer, TFAutoModel

model_ckpt = <span class="hljs-string">&quot;sentence-transformers/multi-qa-mpnet-base-dot-v1&quot;</span>
tokenizer = AutoTokenizer.from_pretrained(model_ckpt)
model = TFAutoModel.from_pretrained(model_ckpt, from_pt=<span class="hljs-literal">True</span>)`}}),{c(){E(m.$$.fragment),q=d(),f=o("p"),y=n("Notez que nous avons d\xE9fini "),_=o("code"),$=n("from_pt=True"),S=n(" comme argument de la m\xE9thode "),g=o("code"),C=n("from_pretrained()"),D=n(". C\u2019est parce que le point de contr\xF4le "),z=o("code"),O=n("multi-qa-mpnet-base-dot-v1"),T=n(" n\u2019a que des poids PyTorch. Donc d\xE9finir "),N=o("code"),I=n("from_pt=True"),R=n(" converti automatiquement au format TensorFlow pour nous. Comme vous pouvez le voir, il est tr\xE8s simple de passer d\u2019un "),Y=o("em"),P=n("framework"),Q=n(" \xE0 l\u2019autre dans \u{1F917} "),B=o("em"),U=n("Transformers"),H=n(" !")},l(c){x(m.$$.fragment,c),q=p(c),f=r(c,"P",{});var h=l(f);y=a(h,"Notez que nous avons d\xE9fini "),_=r(h,"CODE",{});var L=l(_);$=a(L,"from_pt=True"),L.forEach(s),S=a(h," comme argument de la m\xE9thode "),g=r(h,"CODE",{});var W=l(g);C=a(W,"from_pretrained()"),W.forEach(s),D=a(h,". C\u2019est parce que le point de contr\xF4le "),z=r(h,"CODE",{});var J=l(z);O=a(J,"multi-qa-mpnet-base-dot-v1"),J.forEach(s),T=a(h," n\u2019a que des poids PyTorch. Donc d\xE9finir "),N=r(h,"CODE",{});var K=l(N);I=a(K,"from_pt=True"),K.forEach(s),R=a(h," converti automatiquement au format TensorFlow pour nous. Comme vous pouvez le voir, il est tr\xE8s simple de passer d\u2019un "),Y=r(h,"EM",{});var oe=l(Y);P=a(oe,"framework"),oe.forEach(s),Q=a(h," \xE0 l\u2019autre dans \u{1F917} "),B=r(h,"EM",{});var Ie=l(B);U=a(Ie,"Transformers"),Ie.forEach(s),H=a(h," !"),h.forEach(s)},m(c,h){k(m,c,h),u(c,q,h),u(c,f,h),t(f,y),t(f,_),t(_,$),t(f,S),t(f,g),t(g,C),t(f,D),t(f,z),t(z,O),t(f,T),t(f,N),t(N,I),t(f,R),t(f,Y),t(Y,P),t(f,Q),t(f,B),t(B,U),t(f,H),V=!0},i(c){V||(b(m.$$.fragment,c),V=!0)},o(c){v(m.$$.fragment,c),V=!1},d(c){w(m,c),c&&s(q),c&&s(f)}}}function ud(G){let m,q,f,y,_,$,S;return m=new A({props:{code:`from transformers import AutoTokenizer, AutoModel

model_ckpt = "sentence-transformers/multi-qa-mpnet-base-dot-v1"
tokenizer = AutoTokenizer.from_pretrained(model_ckpt)
model = AutoModel.from_pretrained(model_ckpt)`,highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoTokenizer, AutoModel

model_ckpt = <span class="hljs-string">&quot;sentence-transformers/multi-qa-mpnet-base-dot-v1&quot;</span>
tokenizer = AutoTokenizer.from_pretrained(model_ckpt)
model = AutoModel.from_pretrained(model_ckpt)`}}),$=new A({props:{code:`import torch

device = torch.device("cuda")
model.to(device)`,highlighted:`<span class="hljs-keyword">import</span> torch

device = torch.device(<span class="hljs-string">&quot;cuda&quot;</span>)
model.to(device)`}}),{c(){E(m.$$.fragment),q=d(),f=o("p"),y=n("Pour acc\xE9l\xE9rer le processus, il est utile de placer le mod\xE8le et les entr\xE9es sur un p\xE9riph\xE9rique GPU, alors faisons-le maintenant :"),_=d(),E($.$$.fragment)},l(g){x(m.$$.fragment,g),q=p(g),f=r(g,"P",{});var C=l(f);y=a(C,"Pour acc\xE9l\xE9rer le processus, il est utile de placer le mod\xE8le et les entr\xE9es sur un p\xE9riph\xE9rique GPU, alors faisons-le maintenant :"),C.forEach(s),_=p(g),x($.$$.fragment,g)},m(g,C){k(m,g,C),u(g,q,C),u(g,f,C),t(f,y),u(g,_,C),k($,g,C),S=!0},i(g){S||(b(m.$$.fragment,g),b($.$$.fragment,g),S=!0)},o(g){v(m.$$.fragment,g),v($.$$.fragment,g),S=!1},d(g){w(m,g),g&&s(q),g&&s(f),g&&s(_),w($,g)}}}function cd(G){let m,q,f,y,_,$,S,g,C,D,z,O,T,N,I,R,Y,P,Q,B,U,H,V;return m=new A({props:{code:`def get_embeddings(text_list):
    encoded_input = tokenizer(
        text_list, padding=True, truncation=True, return_tensors="tf"
    )
    encoded_input = {k: v for k, v in encoded_input.items()}
    model_output = model(**encoded_input)
    return cls_pooling(model_output)`,highlighted:`<span class="hljs-keyword">def</span> <span class="hljs-title function_">get_embeddings</span>(<span class="hljs-params">text_list</span>):
    encoded_input = tokenizer(
        text_list, padding=<span class="hljs-literal">True</span>, truncation=<span class="hljs-literal">True</span>, return_tensors=<span class="hljs-string">&quot;tf&quot;</span>
    )
    encoded_input = {k: v <span class="hljs-keyword">for</span> k, v <span class="hljs-keyword">in</span> encoded_input.items()}
    model_output = model(**encoded_input)
    <span class="hljs-keyword">return</span> cls_pooling(model_output)`}}),$=new A({props:{code:`embedding = get_embeddings(comments_dataset["text"][0])
embedding.shape`,highlighted:`embedding = get_embeddings(comments_dataset[<span class="hljs-string">&quot;text&quot;</span>][<span class="hljs-number">0</span>])
embedding.shape`}}),g=new A({props:{code:"TensorShape([1, 768])",highlighted:'TensorShape([<span class="hljs-number">1</span>, <span class="hljs-number">768</span>])'}}),H=new A({props:{code:`embeddings_dataset = comments_dataset.map(
    lambda x: {"embeddings": get_embeddings(x["text"]).numpy()[0]}
)`,highlighted:`embeddings_dataset = comments_dataset.<span class="hljs-built_in">map</span>(
    <span class="hljs-keyword">lambda</span> x: {<span class="hljs-string">&quot;embeddings&quot;</span>: get_embeddings(x[<span class="hljs-string">&quot;text&quot;</span>]).numpy()[<span class="hljs-number">0</span>]}
)`}}),{c(){E(m.$$.fragment),q=d(),f=o("p"),y=n("Nous pouvons tester le fonctionnement de la fonction en lui donnant la premi\xE8re entr\xE9e textuelle de notre corpus et en inspectant la forme de sortie :"),_=d(),E($.$$.fragment),S=d(),E(g.$$.fragment),C=d(),D=o("p"),z=n("Super ! Nous avons converti la premi\xE8re entr\xE9e de notre corpus en un vecteur \xE0 768 dimensions. Nous pouvons utiliser "),O=o("code"),T=n("Dataset.map()"),N=n(" pour appliquer notre fonction "),I=o("code"),R=n("get_embeddings()"),Y=n(" \xE0 chaque ligne de notre corpus. Cr\xE9ons donc une nouvelle colonne "),P=o("code"),Q=n("embeddings"),B=n(" comme suit :"),U=d(),E(H.$$.fragment)},l(c){x(m.$$.fragment,c),q=p(c),f=r(c,"P",{});var h=l(f);y=a(h,"Nous pouvons tester le fonctionnement de la fonction en lui donnant la premi\xE8re entr\xE9e textuelle de notre corpus et en inspectant la forme de sortie :"),h.forEach(s),_=p(c),x($.$$.fragment,c),S=p(c),x(g.$$.fragment,c),C=p(c),D=r(c,"P",{});var L=l(D);z=a(L,"Super ! Nous avons converti la premi\xE8re entr\xE9e de notre corpus en un vecteur \xE0 768 dimensions. Nous pouvons utiliser "),O=r(L,"CODE",{});var W=l(O);T=a(W,"Dataset.map()"),W.forEach(s),N=a(L," pour appliquer notre fonction "),I=r(L,"CODE",{});var J=l(I);R=a(J,"get_embeddings()"),J.forEach(s),Y=a(L," \xE0 chaque ligne de notre corpus. Cr\xE9ons donc une nouvelle colonne "),P=r(L,"CODE",{});var K=l(P);Q=a(K,"embeddings"),K.forEach(s),B=a(L," comme suit :"),L.forEach(s),U=p(c),x(H.$$.fragment,c)},m(c,h){k(m,c,h),u(c,q,h),u(c,f,h),t(f,y),u(c,_,h),k($,c,h),u(c,S,h),k(g,c,h),u(c,C,h),u(c,D,h),t(D,z),t(D,O),t(O,T),t(D,N),t(D,I),t(I,R),t(D,Y),t(D,P),t(P,Q),t(D,B),u(c,U,h),k(H,c,h),V=!0},i(c){V||(b(m.$$.fragment,c),b($.$$.fragment,c),b(g.$$.fragment,c),b(H.$$.fragment,c),V=!0)},o(c){v(m.$$.fragment,c),v($.$$.fragment,c),v(g.$$.fragment,c),v(H.$$.fragment,c),V=!1},d(c){w(m,c),c&&s(q),c&&s(f),c&&s(_),w($,c),c&&s(S),w(g,c),c&&s(C),c&&s(D),c&&s(U),w(H,c)}}}function dd(G){let m,q,f,y,_,$,S,g,C,D,z,O,T,N,I,R,Y,P,Q,B,U,H,V;return m=new A({props:{code:`def get_embeddings(text_list):
    encoded_input = tokenizer(
        text_list, padding=True, truncation=True, return_tensors="pt"
    )
    encoded_input = {k: v.to(device) for k, v in encoded_input.items()}
    model_output = model(**encoded_input)
    return cls_pooling(model_output)`,highlighted:`<span class="hljs-keyword">def</span> <span class="hljs-title function_">get_embeddings</span>(<span class="hljs-params">text_list</span>):
    encoded_input = tokenizer(
        text_list, padding=<span class="hljs-literal">True</span>, truncation=<span class="hljs-literal">True</span>, return_tensors=<span class="hljs-string">&quot;pt&quot;</span>
    )
    encoded_input = {k: v.to(device) <span class="hljs-keyword">for</span> k, v <span class="hljs-keyword">in</span> encoded_input.items()}
    model_output = model(**encoded_input)
    <span class="hljs-keyword">return</span> cls_pooling(model_output)`}}),$=new A({props:{code:`embedding = get_embeddings(comments_dataset["text"][0])
embedding.shape`,highlighted:`embedding = get_embeddings(comments_dataset[<span class="hljs-string">&quot;text&quot;</span>][<span class="hljs-number">0</span>])
embedding.shape`}}),g=new A({props:{code:"torch.Size([1, 768])",highlighted:'torch.Size([<span class="hljs-number">1</span>, <span class="hljs-number">768</span>])'}}),H=new A({props:{code:`embeddings_dataset = comments_dataset.map(
    lambda x: {"embeddings": get_embeddings(x["text"]).detach().cpu().numpy()[0]}
)`,highlighted:`embeddings_dataset = comments_dataset.<span class="hljs-built_in">map</span>(
    <span class="hljs-keyword">lambda</span> x: {<span class="hljs-string">&quot;embeddings&quot;</span>: get_embeddings(x[<span class="hljs-string">&quot;text&quot;</span>]).detach().cpu().numpy()[<span class="hljs-number">0</span>]}
)`}}),{c(){E(m.$$.fragment),q=d(),f=o("p"),y=n("Nous pouvons tester le fonctionnement de la fonction en lui donnant la premi\xE8re entr\xE9e textuelle de notre corpus et en inspectant la forme de sortie :"),_=d(),E($.$$.fragment),S=d(),E(g.$$.fragment),C=d(),D=o("p"),z=n("Super ! Nous avons converti la premi\xE8re entr\xE9e de notre corpus en un vecteur \xE0 768 dimensions. Nous pouvons utiliser "),O=o("code"),T=n("Dataset.map()"),N=n(" pour appliquer notre fonction "),I=o("code"),R=n("get_embeddings()"),Y=n(" \xE0 chaque ligne de notre corpus. Cr\xE9ons donc une nouvelle colonne "),P=o("code"),Q=n("embeddings"),B=n(" comme suit :"),U=d(),E(H.$$.fragment)},l(c){x(m.$$.fragment,c),q=p(c),f=r(c,"P",{});var h=l(f);y=a(h,"Nous pouvons tester le fonctionnement de la fonction en lui donnant la premi\xE8re entr\xE9e textuelle de notre corpus et en inspectant la forme de sortie :"),h.forEach(s),_=p(c),x($.$$.fragment,c),S=p(c),x(g.$$.fragment,c),C=p(c),D=r(c,"P",{});var L=l(D);z=a(L,"Super ! Nous avons converti la premi\xE8re entr\xE9e de notre corpus en un vecteur \xE0 768 dimensions. Nous pouvons utiliser "),O=r(L,"CODE",{});var W=l(O);T=a(W,"Dataset.map()"),W.forEach(s),N=a(L," pour appliquer notre fonction "),I=r(L,"CODE",{});var J=l(I);R=a(J,"get_embeddings()"),J.forEach(s),Y=a(L," \xE0 chaque ligne de notre corpus. Cr\xE9ons donc une nouvelle colonne "),P=r(L,"CODE",{});var K=l(P);Q=a(K,"embeddings"),K.forEach(s),B=a(L," comme suit :"),L.forEach(s),U=p(c),x(H.$$.fragment,c)},m(c,h){k(m,c,h),u(c,q,h),u(c,f,h),t(f,y),u(c,_,h),k($,c,h),u(c,S,h),k(g,c,h),u(c,C,h),u(c,D,h),t(D,z),t(D,O),t(O,T),t(D,N),t(D,I),t(I,R),t(D,Y),t(D,P),t(P,Q),t(D,B),u(c,U,h),k(H,c,h),V=!0},i(c){V||(b(m.$$.fragment,c),b($.$$.fragment,c),b(g.$$.fragment,c),b(H.$$.fragment,c),V=!0)},o(c){v(m.$$.fragment,c),v($.$$.fragment,c),v(g.$$.fragment,c),v(H.$$.fragment,c),V=!1},d(c){w(m,c),c&&s(q),c&&s(f),c&&s(_),w($,c),c&&s(S),w(g,c),c&&s(C),c&&s(D),c&&s(U),w(H,c)}}}function pd(G){let m,q,f,y;return m=new A({props:{code:`question = "How can I load a dataset offline?"
question_embedding = get_embeddings([question]).numpy()
question_embedding.shape`,highlighted:`question = <span class="hljs-string">&quot;How can I load a dataset offline?&quot;</span>
question_embedding = get_embeddings([question]).numpy()
question_embedding.shape`}}),f=new A({props:{code:"(1, 768)",highlighted:'(<span class="hljs-number">1</span>, <span class="hljs-number">768</span>)'}}),{c(){E(m.$$.fragment),q=d(),E(f.$$.fragment)},l(_){x(m.$$.fragment,_),q=p(_),x(f.$$.fragment,_)},m(_,$){k(m,_,$),u(_,q,$),k(f,_,$),y=!0},i(_){y||(b(m.$$.fragment,_),b(f.$$.fragment,_),y=!0)},o(_){v(m.$$.fragment,_),v(f.$$.fragment,_),y=!1},d(_){w(m,_),_&&s(q),w(f,_)}}}function md(G){let m,q,f,y;return m=new A({props:{code:`question = "How can I load a dataset offline?"
question_embedding = get_embeddings([question]).cpu().detach().numpy()
question_embedding.shape`,highlighted:`question = <span class="hljs-string">&quot;How can I load a dataset offline?&quot;</span>
question_embedding = get_embeddings([question]).cpu().detach().numpy()
question_embedding.shape`}}),f=new A({props:{code:"torch.Size([1, 768])",highlighted:'torch.Size([<span class="hljs-number">1</span>, <span class="hljs-number">768</span>])'}}),{c(){E(m.$$.fragment),q=d(),E(f.$$.fragment)},l(_){x(m.$$.fragment,_),q=p(_),x(f.$$.fragment,_)},m(_,$){k(m,_,$),u(_,q,$),k(f,_,$),y=!0},i(_){y||(b(m.$$.fragment,_),b(f.$$.fragment,_),y=!0)},o(_){v(m.$$.fragment,_),v(f.$$.fragment,_),y=!1},d(_){w(m,_),_&&s(q),w(f,_)}}}function fd(G){let m,q,f,y,_,$,S,g,C,D,z;return{c(){m=o("p"),q=n("\u270F\uFE0F "),f=o("strong"),y=n("Essayez !"),_=n(" Cr\xE9ez votre propre requ\xEAte et voyez si vous pouvez trouver une r\xE9ponse dans les documents r\xE9cup\xE9r\xE9s. Vous devrez peut-\xEAtre augmenter le param\xE8tre "),$=o("code"),S=n("k"),g=n(" dans "),C=o("code"),D=n("Dataset.get_nearest_examples()"),z=n(" pour \xE9largir la recherche.")},l(O){m=r(O,"P",{});var T=l(m);q=a(T,"\u270F\uFE0F "),f=r(T,"STRONG",{});var N=l(f);y=a(N,"Essayez !"),N.forEach(s),_=a(T," Cr\xE9ez votre propre requ\xEAte et voyez si vous pouvez trouver une r\xE9ponse dans les documents r\xE9cup\xE9r\xE9s. Vous devrez peut-\xEAtre augmenter le param\xE8tre "),$=r(T,"CODE",{});var I=l($);S=a(I,"k"),I.forEach(s),g=a(T," dans "),C=r(T,"CODE",{});var R=l(C);D=a(R,"Dataset.get_nearest_examples()"),R.forEach(s),z=a(T," pour \xE9largir la recherche."),T.forEach(s)},m(O,T){u(O,m,T),t(m,q),t(m,f),t(f,y),t(m,_),t(m,$),t($,S),t(m,g),t(m,C),t(C,D),t(m,z)},d(O){O&&s(m)}}}function hd(G){let m,q,f,y,_,$,S,g,C,D,z,O,T,N,I,R,Y,P,Q,B,U,H,V,c,h,L,W,J,K,oe,Ie,He,no,Ln,te,ao,Lt,oo,ro,os,lo,io,rs,uo,co,ls,po,mo,Hn,Ht,fo,Fn,Pe,st,zi,ho,nt,Ui,zn,Me,Fe,is,at,_o,us,go,Un,ke,vo,cs,bo,$o,ds,qo,Eo,Gn,ot,Vn,we,xo,ps,ko,wo,Ft,jo,yo,Yn,rt,Bn,lt,Wn,X,Do,ms,Co,To,fs,Ao,So,hs,Oo,No,_s,Io,Po,gs,Mo,Ro,vs,Lo,Ho,Jn,it,Qn,ut,Xn,Z,Fo,bs,zo,Uo,$s,Go,Vo,qs,Yo,Bo,Es,Wo,Jo,xs,Qo,Xo,Kn,ct,Zn,dt,ea,ee,Ko,ks,Zo,er,ws,tr,sr,js,nr,ar,pt,ys,or,rr,Ds,lr,ir,ta,mt,sa,ze,ur,Cs,cr,dr,na,ft,aa,ht,oa,Ue,pr,Ts,mr,fr,ra,_t,la,re,As,se,ia,hr,Ss,_r,gr,Os,vr,br,Ns,$r,qr,Is,Er,xr,ge,le,Ps,kr,wr,Ms,jr,yr,Rs,Dr,Cr,Ls,Tr,Ar,Hs,Sr,Or,ie,Fs,Nr,Ir,zs,Pr,Mr,Us,Rr,Lr,Gs,Hr,Fr,Vs,zr,Ur,ue,Ys,Gr,Vr,Bs,Yr,Br,Ws,Wr,Jr,Js,Qr,Xr,Qs,Kr,Zr,ce,Xs,el,tl,Ks,sl,nl,Zs,al,ol,en,rl,ll,tn,il,ua,de,ul,sn,cl,dl,nn,pl,ml,an,fl,hl,ca,gt,da,vt,pa,zt,_l,ma,Ge,fa,Ve,gl,on,vl,bl,ha,bt,_a,Ut,$l,ga,$t,va,qt,ba,je,ql,rn,El,xl,ln,kl,wl,$a,Et,qa,Gt,jl,Ea,Re,Ye,un,xt,yl,cn,Dl,xa,M,Cl,Vt,Tl,Al,dn,Sl,Ol,pn,Nl,Il,mn,Pl,Ml,fn,Rl,Ll,kt,Hl,Fl,hn,zl,Ul,wt,Gl,Vl,_n,Yl,Bl,gn,Wl,Jl,vn,Ql,Xl,bn,Kl,Zl,ka,ve,be,Yt,ne,ei,$n,ti,si,qn,ni,ai,En,oi,ri,xn,li,ii,wa,jt,ja,Bt,ui,ya,$e,qe,Wt,Be,ci,kn,di,pi,Da,Le,We,wn,yt,mi,jn,fi,Ca,pe,hi,yn,_i,gi,Dn,vi,bi,Dt,$i,qi,Ta,me,Ei,Cn,xi,ki,Tn,wi,ji,An,yi,Di,Aa,Ct,Sa,Je,Ci,Sn,Ti,Ai,Oa,Ee,xe,Jt,Qt,Si,Na,Tt,Ia,fe,Oi,On,Ni,Ii,Nn,Pi,Mi,In,Ri,Li,Pa,At,Ma,Xt,Hi,Ra,St,La,Ot,Ha,Kt,Fi,Fa,Qe,za;f=new ad({props:{fw:G[0]}}),g=new Rn({});const Gi=[rd,od],Nt=[];function Vi(e,i){return e[0]==="pt"?0:1}T=Vi(G),N=Nt[T]=Gi[T](G),h=new nd({props:{id:"OATCgQtNX2o"}}),oe=new Rn({}),at=new Rn({}),ot=new A({props:{code:`from huggingface_hub import hf_hub_url

data_files = hf_hub_url(
    repo_id="lewtun/github-issues",
    filename="datasets-issues-with-comments.jsonl",
    repo_type="dataset",
)`,highlighted:`<span class="hljs-keyword">from</span> huggingface_hub <span class="hljs-keyword">import</span> hf_hub_url

data_files = hf_hub_url(
    repo_id=<span class="hljs-string">&quot;lewtun/github-issues&quot;</span>,
    filename=<span class="hljs-string">&quot;datasets-issues-with-comments.jsonl&quot;</span>,
    repo_type=<span class="hljs-string">&quot;dataset&quot;</span>,
)`}}),rt=new A({props:{code:`from datasets import load_dataset

issues_dataset = load_dataset("json", data_files=data_files, split="train")
issues_dataset`,highlighted:`<span class="hljs-keyword">from</span> datasets <span class="hljs-keyword">import</span> load_dataset

issues_dataset = load_dataset(<span class="hljs-string">&quot;json&quot;</span>, data_files=data_files, split=<span class="hljs-string">&quot;train&quot;</span>)
issues_dataset`}}),lt=new A({props:{code:`Dataset({
    features: ['url', 'repository_url', 'labels_url', 'comments_url', 'events_url', 'html_url', 'id', 'node_id', 'number', 'title', 'user', 'labels', 'state', 'locked', 'assignee', 'assignees', 'milestone', 'comments', 'created_at', 'updated_at', 'closed_at', 'author_association', 'active_lock_reason', 'pull_request', 'body', 'performed_via_github_app', 'is_pull_request'],
    num_rows: 2855
})`,highlighted:`Dataset({
    features: [<span class="hljs-string">&#x27;url&#x27;</span>, <span class="hljs-string">&#x27;repository_url&#x27;</span>, <span class="hljs-string">&#x27;labels_url&#x27;</span>, <span class="hljs-string">&#x27;comments_url&#x27;</span>, <span class="hljs-string">&#x27;events_url&#x27;</span>, <span class="hljs-string">&#x27;html_url&#x27;</span>, <span class="hljs-string">&#x27;id&#x27;</span>, <span class="hljs-string">&#x27;node_id&#x27;</span>, <span class="hljs-string">&#x27;number&#x27;</span>, <span class="hljs-string">&#x27;title&#x27;</span>, <span class="hljs-string">&#x27;user&#x27;</span>, <span class="hljs-string">&#x27;labels&#x27;</span>, <span class="hljs-string">&#x27;state&#x27;</span>, <span class="hljs-string">&#x27;locked&#x27;</span>, <span class="hljs-string">&#x27;assignee&#x27;</span>, <span class="hljs-string">&#x27;assignees&#x27;</span>, <span class="hljs-string">&#x27;milestone&#x27;</span>, <span class="hljs-string">&#x27;comments&#x27;</span>, <span class="hljs-string">&#x27;created_at&#x27;</span>, <span class="hljs-string">&#x27;updated_at&#x27;</span>, <span class="hljs-string">&#x27;closed_at&#x27;</span>, <span class="hljs-string">&#x27;author_association&#x27;</span>, <span class="hljs-string">&#x27;active_lock_reason&#x27;</span>, <span class="hljs-string">&#x27;pull_request&#x27;</span>, <span class="hljs-string">&#x27;body&#x27;</span>, <span class="hljs-string">&#x27;performed_via_github_app&#x27;</span>, <span class="hljs-string">&#x27;is_pull_request&#x27;</span>],
    num_rows: <span class="hljs-number">2855</span>
})`}}),it=new A({props:{code:`issues_dataset = issues_dataset.filter(
    lambda x: (x["is_pull_request"] == False and len(x["comments"]) > 0)
)
issues_dataset`,highlighted:`issues_dataset = issues_dataset.<span class="hljs-built_in">filter</span>(
    <span class="hljs-keyword">lambda</span> x: (x[<span class="hljs-string">&quot;is_pull_request&quot;</span>] == <span class="hljs-literal">False</span> <span class="hljs-keyword">and</span> <span class="hljs-built_in">len</span>(x[<span class="hljs-string">&quot;comments&quot;</span>]) &gt; <span class="hljs-number">0</span>)
)
issues_dataset`}}),ut=new A({props:{code:`Dataset({
    features: ['url', 'repository_url', 'labels_url', 'comments_url', 'events_url', 'html_url', 'id', 'node_id', 'number', 'title', 'user', 'labels', 'state', 'locked', 'assignee', 'assignees', 'milestone', 'comments', 'created_at', 'updated_at', 'closed_at', 'author_association', 'active_lock_reason', 'pull_request', 'body', 'performed_via_github_app', 'is_pull_request'],
    num_rows: 771
})`,highlighted:`Dataset({
    features: [<span class="hljs-string">&#x27;url&#x27;</span>, <span class="hljs-string">&#x27;repository_url&#x27;</span>, <span class="hljs-string">&#x27;labels_url&#x27;</span>, <span class="hljs-string">&#x27;comments_url&#x27;</span>, <span class="hljs-string">&#x27;events_url&#x27;</span>, <span class="hljs-string">&#x27;html_url&#x27;</span>, <span class="hljs-string">&#x27;id&#x27;</span>, <span class="hljs-string">&#x27;node_id&#x27;</span>, <span class="hljs-string">&#x27;number&#x27;</span>, <span class="hljs-string">&#x27;title&#x27;</span>, <span class="hljs-string">&#x27;user&#x27;</span>, <span class="hljs-string">&#x27;labels&#x27;</span>, <span class="hljs-string">&#x27;state&#x27;</span>, <span class="hljs-string">&#x27;locked&#x27;</span>, <span class="hljs-string">&#x27;assignee&#x27;</span>, <span class="hljs-string">&#x27;assignees&#x27;</span>, <span class="hljs-string">&#x27;milestone&#x27;</span>, <span class="hljs-string">&#x27;comments&#x27;</span>, <span class="hljs-string">&#x27;created_at&#x27;</span>, <span class="hljs-string">&#x27;updated_at&#x27;</span>, <span class="hljs-string">&#x27;closed_at&#x27;</span>, <span class="hljs-string">&#x27;author_association&#x27;</span>, <span class="hljs-string">&#x27;active_lock_reason&#x27;</span>, <span class="hljs-string">&#x27;pull_request&#x27;</span>, <span class="hljs-string">&#x27;body&#x27;</span>, <span class="hljs-string">&#x27;performed_via_github_app&#x27;</span>, <span class="hljs-string">&#x27;is_pull_request&#x27;</span>],
    num_rows: <span class="hljs-number">771</span>
})`}}),ct=new A({props:{code:`columns = issues_dataset.column_names
columns_to_keep = ["title", "body", "html_url", "comments"]
columns_to_remove = set(columns_to_keep).symmetric_difference(columns)
issues_dataset = issues_dataset.remove_columns(columns_to_remove)
issues_dataset`,highlighted:`columns = issues_dataset.column_names
columns_to_keep = [<span class="hljs-string">&quot;title&quot;</span>, <span class="hljs-string">&quot;body&quot;</span>, <span class="hljs-string">&quot;html_url&quot;</span>, <span class="hljs-string">&quot;comments&quot;</span>]
columns_to_remove = <span class="hljs-built_in">set</span>(columns_to_keep).symmetric_difference(columns)
issues_dataset = issues_dataset.remove_columns(columns_to_remove)
issues_dataset`}}),dt=new A({props:{code:`Dataset({
    features: ['html_url', 'title', 'comments', 'body'],
    num_rows: 771
})`,highlighted:`Dataset({
    features: [<span class="hljs-string">&#x27;html_url&#x27;</span>, <span class="hljs-string">&#x27;title&#x27;</span>, <span class="hljs-string">&#x27;comments&#x27;</span>, <span class="hljs-string">&#x27;body&#x27;</span>],
    num_rows: <span class="hljs-number">771</span>
})`}}),mt=new A({props:{code:`issues_dataset.set_format("pandas")
df = issues_dataset[:]`,highlighted:`issues_dataset.set_format(<span class="hljs-string">&quot;pandas&quot;</span>)
df = issues_dataset[:]`}}),ft=new A({props:{code:'df["comments"][0].tolist()',highlighted:'df[<span class="hljs-string">&quot;comments&quot;</span>][<span class="hljs-number">0</span>].tolist()'}}),ht=new A({props:{code:`['the bug code locate in \uFF1A\\r\\n    if data_args.task_name is not None:\\r\\n        # Downloading and loading a dataset from the hub.\\r\\n        datasets = load_dataset("glue", data_args.task_name, cache_dir=model_args.cache_dir)',
 'Hi @jinec,\\r\\n\\r\\nFrom time to time we get this kind of \`ConnectionError\` coming from the github.com website: https://raw.githubusercontent.com\\r\\n\\r\\nNormally, it should work if you wait a little and then retry.\\r\\n\\r\\nCould you please confirm if the problem persists?',
 'cannot connect\uFF0Ceven by Web browser\uFF0Cplease check that  there is some  problems\u3002',
 'I can access https://raw.githubusercontent.com/huggingface/datasets/1.7.0/datasets/glue/glue.py without problem...']`,highlighted:`[<span class="hljs-string">&#x27;the bug code locate in \uFF1A\\r\\n    if data_args.task_name is not None:\\r\\n        # Downloading and loading a dataset from the hub.\\r\\n        datasets = load_dataset(&quot;glue&quot;, data_args.task_name, cache_dir=model_args.cache_dir)&#x27;</span>,
 <span class="hljs-string">&#x27;Hi @jinec,\\r\\n\\r\\nFrom time to time we get this kind of \`ConnectionError\` coming from the github.com website: https://raw.githubusercontent.com\\r\\n\\r\\nNormally, it should work if you wait a little and then retry.\\r\\n\\r\\nCould you please confirm if the problem persists?&#x27;</span>,
 <span class="hljs-string">&#x27;cannot connect\uFF0Ceven by Web browser\uFF0Cplease check that  there is some  problems\u3002&#x27;</span>,
 <span class="hljs-string">&#x27;I can access https://raw.githubusercontent.com/huggingface/datasets/1.7.0/datasets/glue/glue.py without problem...&#x27;</span>]`}}),_t=new A({props:{code:`comments_df = df.explode("comments", ignore_index=True)
comments_df.head(4)`,highlighted:`comments_df = df.explode(<span class="hljs-string">&quot;comments&quot;</span>, ignore_index=<span class="hljs-literal">True</span>)
comments_df.head(<span class="hljs-number">4</span>)`}}),gt=new A({props:{code:`from datasets import Dataset

comments_dataset = Dataset.from_pandas(comments_df)
comments_dataset`,highlighted:`<span class="hljs-keyword">from</span> datasets <span class="hljs-keyword">import</span> Dataset

comments_dataset = Dataset.from_pandas(comments_df)
comments_dataset`}}),vt=new A({props:{code:`Dataset({
    features: ['html_url', 'title', 'comments', 'body'],
    num_rows: 2842
})`,highlighted:`Dataset({
    features: [<span class="hljs-string">&#x27;html_url&#x27;</span>, <span class="hljs-string">&#x27;title&#x27;</span>, <span class="hljs-string">&#x27;comments&#x27;</span>, <span class="hljs-string">&#x27;body&#x27;</span>],
    num_rows: <span class="hljs-number">2842</span>
})`}}),Ge=new Qc({props:{$$slots:{default:[ld]},$$scope:{ctx:G}}}),bt=new A({props:{code:`comments_dataset = comments_dataset.map(
    lambda x: {"comment_length": len(x["comments"].split())}
)`,highlighted:`comments_dataset = comments_dataset.<span class="hljs-built_in">map</span>(
    <span class="hljs-keyword">lambda</span> x: {<span class="hljs-string">&quot;comment_length&quot;</span>: <span class="hljs-built_in">len</span>(x[<span class="hljs-string">&quot;comments&quot;</span>].split())}
)`}}),$t=new A({props:{code:`comments_dataset = comments_dataset.filter(lambda x: x["comment_length"] > 15)
comments_dataset`,highlighted:`comments_dataset = comments_dataset.<span class="hljs-built_in">filter</span>(<span class="hljs-keyword">lambda</span> x: x[<span class="hljs-string">&quot;comment_length&quot;</span>] &gt; <span class="hljs-number">15</span>)
comments_dataset`}}),qt=new A({props:{code:`Dataset({
    features: ['html_url', 'title', 'comments', 'body', 'comment_length'],
    num_rows: 2098
})`,highlighted:`Dataset({
    features: [<span class="hljs-string">&#x27;html_url&#x27;</span>, <span class="hljs-string">&#x27;title&#x27;</span>, <span class="hljs-string">&#x27;comments&#x27;</span>, <span class="hljs-string">&#x27;body&#x27;</span>, <span class="hljs-string">&#x27;comment_length&#x27;</span>],
    num_rows: <span class="hljs-number">2098</span>
})`}}),Et=new A({props:{code:`def concatenate_text(examples):
    return {
        "text": examples["title"]
        + " \\n "
        + examples["body"]
        + " \\n "
        + examples["comments"]
    }


comments_dataset = comments_dataset.map(concatenate_text)`,highlighted:`<span class="hljs-keyword">def</span> <span class="hljs-title function_">concatenate_text</span>(<span class="hljs-params">examples</span>):
    <span class="hljs-keyword">return</span> {
        <span class="hljs-string">&quot;text&quot;</span>: examples[<span class="hljs-string">&quot;title&quot;</span>]
        + <span class="hljs-string">&quot; \\n &quot;</span>
        + examples[<span class="hljs-string">&quot;body&quot;</span>]
        + <span class="hljs-string">&quot; \\n &quot;</span>
        + examples[<span class="hljs-string">&quot;comments&quot;</span>]
    }


comments_dataset = comments_dataset.<span class="hljs-built_in">map</span>(concatenate_text)`}}),xt=new Rn({});const Yi=[ud,id],It=[];function Bi(e,i){return e[0]==="pt"?0:1}ve=Bi(G),be=It[ve]=Yi[ve](G),jt=new A({props:{code:`def cls_pooling(model_output):
    return model_output.last_hidden_state[:, 0]`,highlighted:`<span class="hljs-keyword">def</span> <span class="hljs-title function_">cls_pooling</span>(<span class="hljs-params">model_output</span>):
    <span class="hljs-keyword">return</span> model_output.last_hidden_state[:, <span class="hljs-number">0</span>]`}});const Wi=[dd,cd],Pt=[];function Ji(e,i){return e[0]==="pt"?0:1}$e=Ji(G),qe=Pt[$e]=Wi[$e](G),yt=new Rn({}),Ct=new A({props:{code:'embeddings_dataset.add_faiss_index(column="embeddings")',highlighted:'embeddings_dataset.add_faiss_index(column=<span class="hljs-string">&quot;embeddings&quot;</span>)'}});const Qi=[md,pd],Mt=[];function Xi(e,i){return e[0]==="pt"?0:1}return Ee=Xi(G),xe=Mt[Ee]=Qi[Ee](G),Tt=new A({props:{code:`scores, samples = embeddings_dataset.get_nearest_examples(
    "embeddings", question_embedding, k=5
)`,highlighted:`scores, samples = embeddings_dataset.get_nearest_examples(
    <span class="hljs-string">&quot;embeddings&quot;</span>, question_embedding, k=<span class="hljs-number">5</span>
)`}}),At=new A({props:{code:`import pandas as pd

samples_df = pd.DataFrame.from_dict(samples)
samples_df["scores"] = scores
samples_df.sort_values("scores", ascending=False, inplace=True)`,highlighted:`<span class="hljs-keyword">import</span> pandas <span class="hljs-keyword">as</span> pd

samples_df = pd.DataFrame.from_dict(samples)
samples_df[<span class="hljs-string">&quot;scores&quot;</span>] = scores
samples_df.sort_values(<span class="hljs-string">&quot;scores&quot;</span>, ascending=<span class="hljs-literal">False</span>, inplace=<span class="hljs-literal">True</span>)`}}),St=new A({props:{code:`for _, row in samples_df.iterrows():
    print(f"COMMENT: {row.comments}")
    print(f"SCORE: {row.scores}")
    print(f"TITLE: {row.title}")
    print(f"URL: {row.html_url}")
    print("=" * 50)
    print()`,highlighted:`<span class="hljs-keyword">for</span> _, row <span class="hljs-keyword">in</span> samples_df.iterrows():
    <span class="hljs-built_in">print</span>(<span class="hljs-string">f&quot;COMMENT: <span class="hljs-subst">{row.comments}</span>&quot;</span>)
    <span class="hljs-built_in">print</span>(<span class="hljs-string">f&quot;SCORE: <span class="hljs-subst">{row.scores}</span>&quot;</span>)
    <span class="hljs-built_in">print</span>(<span class="hljs-string">f&quot;TITLE: <span class="hljs-subst">{row.title}</span>&quot;</span>)
    <span class="hljs-built_in">print</span>(<span class="hljs-string">f&quot;URL: <span class="hljs-subst">{row.html_url}</span>&quot;</span>)
    <span class="hljs-built_in">print</span>(<span class="hljs-string">&quot;=&quot;</span> * <span class="hljs-number">50</span>)
    <span class="hljs-built_in">print</span>()`}}),Ot=new A({props:{code:`"""
COMMENT: Requiring online connection is a deal breaker in some cases unfortunately so it'd be great if offline mode is added similar to how \`transformers\` loads models offline fine.

@mandubian's second bullet point suggests that there's a workaround allowing you to use your offline (custom?) dataset with \`datasets\`. Could you please elaborate on how that should look like?
SCORE: 25.505046844482422
TITLE: Discussion using datasets in offline mode
URL: https://github.com/huggingface/datasets/issues/824
==================================================

COMMENT: The local dataset builders (csv, text , json and pandas) are now part of the \`datasets\` package since #1726 :)
You can now use them offline
\\\`\\\`\\\`python
datasets = load_dataset("text", data_files=data_files)
\\\`\\\`\\\`

We'll do a new release soon
SCORE: 24.555509567260742
TITLE: Discussion using datasets in offline mode
URL: https://github.com/huggingface/datasets/issues/824
==================================================

COMMENT: I opened a PR that allows to reload modules that have already been loaded once even if there's no internet.

Let me know if you know other ways that can make the offline mode experience better. I'd be happy to add them :)

I already note the "freeze" modules option, to prevent local modules updates. It would be a cool feature.

----------

> @mandubian's second bullet point suggests that there's a workaround allowing you to use your offline (custom?) dataset with \`datasets\`. Could you please elaborate on how that should look like?

Indeed \`load_dataset\` allows to load remote dataset script (squad, glue, etc.) but also you own local ones.
For example if you have a dataset script at \`./my_dataset/my_dataset.py\` then you can do
\\\`\\\`\\\`python
load_dataset("./my_dataset")
\\\`\\\`\\\`
and the dataset script will generate your dataset once and for all.

----------

About I'm looking into having \`csv\`, \`json\`, \`text\`, \`pandas\` dataset builders already included in the \`datasets\` package, so that they are available offline by default, as opposed to the other datasets that require the script to be downloaded.
cf #1724
SCORE: 24.14896583557129
TITLE: Discussion using datasets in offline mode
URL: https://github.com/huggingface/datasets/issues/824
==================================================

COMMENT: > here is my way to load a dataset offline, but it **requires** an online machine
>
> 1. (online machine)
>
> \`\`\`
>
> import datasets
>
> data = datasets.load_dataset(...)
>
> data.save_to_disk(/YOUR/DATASET/DIR)
>
> \`\`\`
>
> 2. copy the dir from online to the offline machine
>
> 3. (offline machine)
>
> \`\`\`
>
> import datasets
>
> data = datasets.load_from_disk(/SAVED/DATA/DIR)
>
> \`\`\`
>
>
>
> HTH.


SCORE: 22.893993377685547
TITLE: Discussion using datasets in offline mode
URL: https://github.com/huggingface/datasets/issues/824
==================================================

COMMENT: here is my way to load a dataset offline, but it **requires** an online machine
1. (online machine)
\\\`\\\`\\\`
import datasets
data = datasets.load_dataset(...)
data.save_to_disk(/YOUR/DATASET/DIR)
\\\`\\\`\\\`
2. copy the dir from online to the offline machine
3. (offline machine)
\\\`\\\`\\\`
import datasets
data = datasets.load_from_disk(/SAVED/DATA/DIR)
\\\`\\\`\\\`

HTH.
SCORE: 22.406635284423828
TITLE: Discussion using datasets in offline mode
URL: https://github.com/huggingface/datasets/issues/824
==================================================
"""`,highlighted:`<span class="hljs-string">&quot;&quot;&quot;
COMMENT: Requiring online connection is a deal breaker in some cases unfortunately so it&#x27;d be great if offline mode is added similar to how \`transformers\` loads models offline fine.

@mandubian&#x27;s second bullet point suggests that there&#x27;s a workaround allowing you to use your offline (custom?) dataset with \`datasets\`. Could you please elaborate on how that should look like?
SCORE: 25.505046844482422
TITLE: Discussion using datasets in offline mode
URL: https://github.com/huggingface/datasets/issues/824
==================================================

COMMENT: The local dataset builders (csv, text , json and pandas) are now part of the \`datasets\` package since #1726 :)
You can now use them offline
\\\`\\\`\\\`python
datasets = load_dataset(&quot;text&quot;, data_files=data_files)
\\\`\\\`\\\`

We&#x27;ll do a new release soon
SCORE: 24.555509567260742
TITLE: Discussion using datasets in offline mode
URL: https://github.com/huggingface/datasets/issues/824
==================================================

COMMENT: I opened a PR that allows to reload modules that have already been loaded once even if there&#x27;s no internet.

Let me know if you know other ways that can make the offline mode experience better. I&#x27;d be happy to add them :)

I already note the &quot;freeze&quot; modules option, to prevent local modules updates. It would be a cool feature.

----------

&gt; @mandubian&#x27;s second bullet point suggests that there&#x27;s a workaround allowing you to use your offline (custom?) dataset with \`datasets\`. Could you please elaborate on how that should look like?

Indeed \`load_dataset\` allows to load remote dataset script (squad, glue, etc.) but also you own local ones.
For example if you have a dataset script at \`./my_dataset/my_dataset.py\` then you can do
\\\`\\\`\\\`python
load_dataset(&quot;./my_dataset&quot;)
\\\`\\\`\\\`
and the dataset script will generate your dataset once and for all.

----------

About I&#x27;m looking into having \`csv\`, \`json\`, \`text\`, \`pandas\` dataset builders already included in the \`datasets\` package, so that they are available offline by default, as opposed to the other datasets that require the script to be downloaded.
cf #1724
SCORE: 24.14896583557129
TITLE: Discussion using datasets in offline mode
URL: https://github.com/huggingface/datasets/issues/824
==================================================

COMMENT: &gt; here is my way to load a dataset offline, but it **requires** an online machine
&gt;
&gt; 1. (online machine)
&gt;
&gt; \`\`\`
&gt;
&gt; import datasets
&gt;
&gt; data = datasets.load_dataset(...)
&gt;
&gt; data.save_to_disk(/YOUR/DATASET/DIR)
&gt;
&gt; \`\`\`
&gt;
&gt; 2. copy the dir from online to the offline machine
&gt;
&gt; 3. (offline machine)
&gt;
&gt; \`\`\`
&gt;
&gt; import datasets
&gt;
&gt; data = datasets.load_from_disk(/SAVED/DATA/DIR)
&gt;
&gt; \`\`\`
&gt;
&gt;
&gt;
&gt; HTH.


SCORE: 22.893993377685547
TITLE: Discussion using datasets in offline mode
URL: https://github.com/huggingface/datasets/issues/824
==================================================

COMMENT: here is my way to load a dataset offline, but it **requires** an online machine
1. (online machine)
\\\`\\\`\\\`
import datasets
data = datasets.load_dataset(...)
data.save_to_disk(/YOUR/DATASET/DIR)
\\\`\\\`\\\`
2. copy the dir from online to the offline machine
3. (offline machine)
\\\`\\\`\\\`
import datasets
data = datasets.load_from_disk(/SAVED/DATA/DIR)
\\\`\\\`\\\`

HTH.
SCORE: 22.406635284423828
TITLE: Discussion using datasets in offline mode
URL: https://github.com/huggingface/datasets/issues/824
==================================================
&quot;&quot;&quot;</span>`}}),Qe=new Qc({props:{$$slots:{default:[fd]},$$scope:{ctx:G}}}),{c(){m=o("meta"),q=d(),E(f.$$.fragment),y=d(),_=o("h1"),$=o("a"),S=o("span"),E(g.$$.fragment),C=d(),D=o("span"),z=n("Recherche s\xE9mantique avec FAISS"),O=d(),N.c(),I=d(),R=o("p"),Y=n("Dans "),P=o("a"),Q=n("section 5"),B=n(", nous avons cr\xE9\xE9 un jeu de donn\xE9es de probl\xE8mes et de commentaires GitHub \xE0 partir du d\xE9p\xF4t \u{1F917} "),U=o("em"),H=n("Datasets"),V=n(". Dans cette section, nous utilisons ces informations pour cr\xE9er un moteur de recherche qui peut nous aider \xE0 trouver des r\xE9ponses \xE0 nos questions les plus urgentes sur la biblioth\xE8que !"),c=d(),E(h.$$.fragment),L=d(),W=o("h2"),J=o("a"),K=o("span"),E(oe.$$.fragment),Ie=d(),He=o("span"),no=n("Utilisation des ench\xE2ssements pour la recherche s\xE9mantique"),Ln=d(),te=o("p"),ao=n("Comme nous l\u2019avons vu dans le "),Lt=o("a"),oo=n("Chapitre 1"),ro=n(", les mod\xE8les de langage bas\xE9s sur les "),os=o("em"),lo=n("transformers"),io=n(" repr\xE9sentent chaque "),rs=o("em"),uo=n("token"),co=n(" dans une \xE9tendue de texte sous la forme d\u2019un "),ls=o("em"),po=n("ench\xE2ssement"),mo=n(". Il s\u2019av\xE8re que l\u2019on peut regrouper les ench\xE2ssements individuels pour cr\xE9er une repr\xE9sentation vectorielle pour des phrases enti\xE8res, des paragraphes ou (dans certains cas) des documents. Ces ench\xE2ssements peuvent ensuite \xEAtre utilis\xE9s pour trouver des documents similaires dans le corpus en calculant la similarit\xE9 du produit scalaire (ou une autre m\xE9trique de similarit\xE9) entre chaque ench\xE2ssement et en renvoyant les documents avec le plus grand chevauchement."),Hn=d(),Ht=o("p"),fo=n("Dans cette section, nous utilisons les ench\xE2ssements pour d\xE9velopper un moteur de recherche s\xE9mantique. Ces moteurs de recherche offrent plusieurs avantages par rapport aux approches conventionnelles bas\xE9es sur la correspondance des mots-cl\xE9s dans une requ\xEAte avec les documents."),Fn=d(),Pe=o("div"),st=o("img"),ho=d(),nt=o("img"),zn=d(),Me=o("h2"),Fe=o("a"),is=o("span"),E(at.$$.fragment),_o=d(),us=o("span"),go=n("Chargement et pr\xE9paration du jeu de donn\xE9es"),Un=d(),ke=o("p"),vo=n("La premi\xE8re chose que nous devons faire est de t\xE9l\xE9charger notre jeu de donn\xE9es de probl\xE8mes GitHub. Utilisons la biblioth\xE8que \u{1F917} "),cs=o("em"),bo=n("Hub"),$o=n(" pour r\xE9soudre l\u2019URL o\xF9 notre fichier est stock\xE9 sur le "),ds=o("em"),qo=n("Hib"),Eo=n(" d\u2019Hugging Face :"),Gn=d(),E(ot.$$.fragment),Vn=d(),we=o("p"),xo=n("Avec l\u2019URL stock\xE9 dans "),ps=o("code"),ko=n("data_files"),wo=n(", nous pouvons ensuite charger le jeu de donn\xE9es distant en utilisant la m\xE9thode introduite dans "),Ft=o("a"),jo=n("section 2"),yo=n(" :"),Yn=d(),E(rt.$$.fragment),Bn=d(),E(lt.$$.fragment),Wn=d(),X=o("p"),Do=n("Ici, nous avons sp\xE9cifi\xE9 l\u2019\xE9chantillon "),ms=o("code"),Co=n("train"),To=n(" par d\xE9faut dans "),fs=o("code"),Ao=n("load_dataset()"),So=n(", de sorte que cela renvoie un "),hs=o("code"),Oo=n("Dataset"),No=n(" au lieu d\u2019un "),_s=o("code"),Io=n("DatasetDict"),Po=n(". La premi\xE8re chose \xE0 faire est de filtrer les "),gs=o("em"),Mo=n("pull requests"),Ro=n(" car celles-ci ont tendance \xE0 \xEAtre rarement utilis\xE9es pour r\xE9pondre aux requ\xEAtes des utilisateurs et introduiront du bruit dans notre moteur de recherche. Comme cela devrait \xEAtre familier maintenant, nous pouvons utiliser la fonction "),vs=o("code"),Lo=n("Dataset.filter()"),Ho=n(" pour exclure ces lignes de notre jeu de donn\xE9es. Pendant que nous y sommes, filtrons \xE9galement les lignes sans commentaires, car celles-ci ne fournissent aucune r\xE9ponse aux requ\xEAtes des utilisateurs :"),Jn=d(),E(it.$$.fragment),Qn=d(),E(ut.$$.fragment),Xn=d(),Z=o("p"),Fo=n("Nous pouvons voir qu\u2019il y a beaucoup de colonnes dans notre jeu de donn\xE9es, dont la plupart n\u2019ont pas besoin de construire notre moteur de recherche. Du point de vue de la recherche, les colonnes les plus informatives sont "),bs=o("code"),zo=n("title"),Uo=n(", "),$s=o("code"),Go=n("body"),Vo=n(" et "),qs=o("code"),Yo=n("comments"),Bo=n(", tandis que "),Es=o("code"),Wo=n("html_url"),Jo=n(" nous fournit un lien vers le probl\xE8me source. Utilisons la fonction "),xs=o("code"),Qo=n("Dataset.remove_columns()"),Xo=n(" pour supprimer le reste :"),Kn=d(),E(ct.$$.fragment),Zn=d(),E(dt.$$.fragment),ea=d(),ee=o("p"),Ko=n("Pour cr\xE9er nos ench\xE2ssements, nous ajoutons \xE0 chaque commentaire le titre et le corps du probl\xE8me, car ces champs contiennent des informations contextuelles utiles. \xC9tant donn\xE9 que notre colonne "),ks=o("code"),Zo=n("comments"),er=n(" est actuellement une liste de commentaires pour chaque probl\xE8me, nous devons \xAB \xE9clater \xBB la colonne afin que chaque ligne se compose d\u2019un "),ws=o("em"),tr=n("tuple"),sr=d(),js=o("code"),nr=n("(html_url, title, body, comment)"),ar=n(". Dans Pandas, nous pouvons le faire avec la fonction "),pt=o("a"),ys=o("code"),or=n("DataFrame.explode()"),rr=n(", qui cr\xE9e une nouvelle ligne pour chaque \xE9l\xE9ment dans une colonne de type liste, tout en r\xE9pliquant toutes les autres valeurs de colonne. Pour voir cela en action, passons d\u2019abord au format "),Ds=o("code"),lr=n("DataFrame"),ir=n(" de Pandas :"),ta=d(),E(mt.$$.fragment),sa=d(),ze=o("p"),ur=n("Si nous inspectons la premi\xE8re ligne de ce "),Cs=o("code"),cr=n("DataFrame"),dr=n(", nous pouvons voir qu\u2019il y a quatre commentaires associ\xE9s \xE0 ce probl\xE8me :"),na=d(),E(ft.$$.fragment),aa=d(),E(ht.$$.fragment),oa=d(),Ue=o("p"),pr=n("Lorsque nous d\xE9composons "),Ts=o("code"),mr=n("df"),fr=n(", nous nous attendons \xE0 obtenir une ligne pour chacun de ces commentaires. V\xE9rifions si c\u2019est le cas :"),ra=d(),E(_t.$$.fragment),la=d(),re=o("table"),As=o("thead"),se=o("tr"),ia=o("th"),hr=d(),Ss=o("th"),_r=n("html_url"),gr=d(),Os=o("th"),vr=n("title"),br=d(),Ns=o("th"),$r=n("comments"),qr=d(),Is=o("th"),Er=n("body"),xr=d(),ge=o("tbody"),le=o("tr"),Ps=o("th"),kr=n("0"),wr=d(),Ms=o("td"),jr=n("https://github.com/huggingface/datasets/issues/2787"),yr=d(),Rs=o("td"),Dr=n("ConnectionError: Couldn't reach https://raw.githubusercontent.com"),Cr=d(),Ls=o("td"),Tr=n("the bug code locate in \uFF1A\\r\\n    if data_args.task_name is not None..."),Ar=d(),Hs=o("td"),Sr=n("Hello,\\r\\nI am trying to run run_glue.py and it gives me this error..."),Or=d(),ie=o("tr"),Fs=o("th"),Nr=n("1"),Ir=d(),zs=o("td"),Pr=n("https://github.com/huggingface/datasets/issues/2787"),Mr=d(),Us=o("td"),Rr=n("ConnectionError: Couldn't reach https://raw.githubusercontent.com"),Lr=d(),Gs=o("td"),Hr=n("Hi @jinec,\\r\\n\\r\\nFrom time to time we get this kind of `ConnectionError` coming from the github.com website: https://raw.githubusercontent.com..."),Fr=d(),Vs=o("td"),zr=n("Hello,\\r\\nI am trying to run run_glue.py and it gives me this error..."),Ur=d(),ue=o("tr"),Ys=o("th"),Gr=n("2"),Vr=d(),Bs=o("td"),Yr=n("https://github.com/huggingface/datasets/issues/2787"),Br=d(),Ws=o("td"),Wr=n("ConnectionError: Couldn't reach https://raw.githubusercontent.com"),Jr=d(),Js=o("td"),Qr=n("cannot connect\uFF0Ceven by Web browser\uFF0Cplease check that  there is some  problems\u3002"),Xr=d(),Qs=o("td"),Kr=n("Hello,\\r\\nI am trying to run run_glue.py and it gives me this error..."),Zr=d(),ce=o("tr"),Xs=o("th"),el=n("3"),tl=d(),Ks=o("td"),sl=n("https://github.com/huggingface/datasets/issues/2787"),nl=d(),Zs=o("td"),al=n("ConnectionError: Couldn't reach https://raw.githubusercontent.com"),ol=d(),en=o("td"),rl=n("I can access https://raw.githubusercontent.com/huggingface/datasets/1.7.0/datasets/glue/glue.py without problem..."),ll=d(),tn=o("td"),il=n("Hello,\\r\\nI am trying to run run_glue.py and it gives me this error..."),ua=d(),de=o("p"),ul=n("G\xE9nial, nous pouvons voir que les lignes ont \xE9t\xE9 r\xE9pliqu\xE9es, avec la colonne "),sn=o("code"),cl=n("comments"),dl=n(" contenant les commentaires individuels ! Maintenant que nous en avons fini avec Pandas, nous pouvons rapidement revenir \xE0 un "),nn=o("code"),pl=n("Dataset"),ml=n(" en chargeant le "),an=o("code"),fl=n("DataFrame"),hl=n(" en m\xE9moire :"),ca=d(),E(gt.$$.fragment),da=d(),E(vt.$$.fragment),pa=d(),zt=o("p"),_l=n("D\u2019accord, cela nous a donn\xE9 quelques milliers de commentaires avec lesquels travailler !"),ma=d(),E(Ge.$$.fragment),fa=d(),Ve=o("p"),gl=n("Maintenant que nous avons un commentaire par ligne, cr\xE9ons une nouvelle colonne "),on=o("code"),vl=n("comments_length"),bl=n(" contenant le nombre de mots par commentaire :"),ha=d(),E(bt.$$.fragment),_a=d(),Ut=o("p"),$l=n("Nous pouvons utiliser cette nouvelle colonne pour filtrer les commentaires courts incluant g\xE9n\xE9ralement des \xE9l\xE9ments tels que \xAB cc @lewtun \xBB ou \xAB Merci ! \xBB qui ne sont pas pertinents pour notre moteur de recherche. Il n\u2019y a pas de nombre pr\xE9cis \xE0 s\xE9lectionner pour le filtre mais 15 mots semblent \xEAtre un bon d\xE9but :"),ga=d(),E($t.$$.fragment),va=d(),E(qt.$$.fragment),ba=d(),je=o("p"),ql=n("Apr\xE8s avoir un peu nettoy\xE9 notre jeu de donn\xE9es, concat\xE9nons le titre, la description et les commentaires du probl\xE8me dans une nouvelle colonne "),rn=o("code"),El=n("text"),xl=n(". Comme d\u2019habitude, nous allons \xE9crire une fonction simple que nous pouvons passer \xE0 "),ln=o("code"),kl=n("Dataset.map()"),wl=n(" :"),$a=d(),E(Et.$$.fragment),qa=d(),Gt=o("p"),jl=n("Nous sommes enfin pr\xEAts \xE0 cr\xE9er des ench\xE2ssements ! Jetons un coup d\u2019\u0153il."),Ea=d(),Re=o("h2"),Ye=o("a"),un=o("span"),E(xt.$$.fragment),yl=d(),cn=o("span"),Dl=n("Cr\xE9ation d\u2019ench\xE2ssements pour les textes"),xa=d(),M=o("p"),Cl=n("Nous avons vu dans "),Vt=o("a"),Tl=n("Chapitre 2"),Al=n(" que nous pouvons obtenir des ench\xE2ssements de "),dn=o("em"),Sl=n("tokens"),Ol=n(" en utilisant la classe "),pn=o("code"),Nl=n("AutoModel"),Il=n(". Tout ce que nous avons \xE0 faire est de choisir un "),mn=o("em"),Pl=n("checkpoint"),Ml=n(" appropri\xE9 \xE0 partir duquel charger le mod\xE8le. Heureusement, il existe une biblioth\xE8que appel\xE9e "),fn=o("code"),Rl=n("sentence-transformers"),Ll=n(" d\xE9di\xE9e \xE0 la cr\xE9ation d\u2019ench\xE2ssements. Comme d\xE9crit dans la "),kt=o("a"),Hl=n("documentation de la biblioth\xE8que"),Fl=n(", notre cas d\u2019utilisation est un exemple de "),hn=o("em"),zl=n("recherche s\xE9mantique asym\xE9trique"),Ul=n(". En effet, nous avons une requ\xEAte courte dont nous aimerions trouver la r\xE9ponse dans un document plus long, par exemple un commentaire \xE0 un probl\xE8me. Le "),wt=o("a"),Gl=n("tableau de pr\xE9sentation des mod\xE8les"),Vl=n(" de la documentation indique que le "),_n=o("em"),Yl=n("checkpoint"),Bl=d(),gn=o("code"),Wl=n("multi-qa-mpnet-base-dot-v1"),Jl=n(" a les meilleures performances pour la recherche s\xE9mantique. Utilisons donc le pour notre application. Nous allons \xE9galement charger le "),vn=o("em"),Ql=n("tokenizer"),Xl=n(" en utilisant le m\xEAme "),bn=o("em"),Kl=n("checkpoint"),Zl=n(" :"),ka=d(),be.c(),Yt=d(),ne=o("p"),ei=n("Comme nous l\u2019avons mentionn\xE9 pr\xE9c\xE9demment, nous aimerions repr\xE9senter chaque entr\xE9e dans notre corpus de probl\xE8mes GitHub comme un vecteur unique. Nous devons donc regrouper ou faire la moyenne de nos ench\xE2ssements de "),$n=o("em"),ti=n("tokens"),si=n(" d\u2019une mani\xE8re ou d\u2019une autre. Une approche populaire consiste \xE0 effectuer un "),qn=o("em"),ni=n("regroupement CLS"),ai=n(" sur les sorties de notre mod\xE8le, o\xF9 nous collectons simplement le dernier \xE9tat cach\xE9 pour le "),En=o("em"),oi=n("token"),ri=n(" sp\xE9cial "),xn=o("code"),li=n("[CLS]"),ii=n(". La fonction suivante fait \xE7a pour nous :"),wa=d(),E(jt.$$.fragment),ja=d(),Bt=o("p"),ui=n("Ensuite, nous allons cr\xE9er une fonction utile qui va tokeniser une liste de documents, placer les tenseurs dans le GPU, les donner au mod\xE8le et enfin appliquer le regroupement CLS aux sorties :"),ya=d(),qe.c(),Wt=d(),Be=o("p"),ci=n("Notez que nous avons converti les ench\xE2ssements en tableaux NumPy. C\u2019est parce que \u{1F917} "),kn=o("em"),di=n("Datasets"),pi=n(" n\xE9cessite ce format lorsque nous essayons de les indexer avec FAISS, ce que nous ferons ensuite."),Da=d(),Le=o("h2"),We=o("a"),wn=o("span"),E(yt.$$.fragment),mi=d(),jn=o("span"),fi=n("Utilisation de FAISS pour une recherche de similarit\xE9 efficace"),Ca=d(),pe=o("p"),hi=n("Maintenant que nous avons un jeu de donn\xE9es d\u2019incorporations, nous avons besoin d\u2019un moyen de les rechercher. Pour ce faire, nous utiliserons une structure de donn\xE9es sp\xE9ciale dans \u{1F917} "),yn=o("em"),_i=n("Datasets"),gi=n(" appel\xE9e "),Dn=o("em"),vi=n("FAISS index"),bi=n(". "),Dt=o("a"),$i=n("FAISS"),qi=n(" (abr\xE9viation de Facebook AI Similarity Search) est une biblioth\xE8que qui fournit des algorithmes efficaces pour rechercher et regrouper rapidement des vecteurs d\u2019int\xE9gration."),Ta=d(),me=o("p"),Ei=n("L\u2019id\xE9e de base derri\xE8re FAISS est de cr\xE9er une structure de donn\xE9es sp\xE9ciale appel\xE9e un "),Cn=o("em"),xi=n("index"),ki=n(" qui permet de trouver quels plongements sont similaires \xE0 un plongement d\u2019entr\xE9e. Cr\xE9er un index FAISS dans \u{1F917} "),Tn=o("em"),wi=n("Datasets"),ji=n(" est simple \u2014 nous utilisons la fonction "),An=o("code"),yi=n("Dataset.add_faiss_index()"),Di=n(" et sp\xE9cifions quelle colonne de notre jeu de donn\xE9es nous aimerions indexer :"),Aa=d(),E(Ct.$$.fragment),Sa=d(),Je=o("p"),Ci=n("Nous pouvons maintenant effectuer des requ\xEAtes sur cet index en effectuant une recherche des voisins les plus proches avec la fonction "),Sn=o("code"),Ti=n("Dataset.get_nearest_examples()"),Ai=n(". Testons cela en ench\xE2ssant d\u2019abord une question comme suit :"),Oa=d(),xe.c(),Jt=d(),Qt=o("p"),Si=n("Tout comme avec les documents, nous avons maintenant un vecteur de 768 dimensions repr\xE9sentant la requ\xEAte. Nous pouvons le comparer \xE0 l\u2019ensemble du corpus pour trouver les ench\xE2ssements les plus similaires :"),Na=d(),E(Tt.$$.fragment),Ia=d(),fe=o("p"),Oi=n("La fonction "),On=o("code"),Ni=n("Dataset.get_nearest_examples()"),Ii=n(" renvoie un "),Nn=o("em"),Pi=n("tuple"),Mi=n(" de scores qui classent le chevauchement entre la requ\xEAte et le document, et un jeu correspondant d\u2019\xE9chantillons (ici, les 5 meilleures correspondances). Collectons-les dans un "),In=o("code"),Ri=n("pandas.DataFrame"),Li=n(" afin de pouvoir les trier facilement :"),Pa=d(),E(At.$$.fragment),Ma=d(),Xt=o("p"),Hi=n("Nous pouvons maintenant parcourir les premi\xE8res lignes pour voir dans quelle mesure notre requ\xEAte correspond aux commentaires disponibles :"),Ra=d(),E(St.$$.fragment),La=d(),E(Ot.$$.fragment),Ha=d(),Kt=o("p"),Fi=n("Pas mal ! Notre deuxi\xE8me r\xE9sultat semble correspondre \xE0 la requ\xEAte."),Fa=d(),E(Qe.$$.fragment),this.h()},l(e){const i=td('[data-svelte="svelte-1phssyn"]',document.head);m=r(i,"META",{name:!0,content:!0}),i.forEach(s),q=p(e),x(f.$$.fragment,e),y=p(e),_=r(e,"H1",{class:!0});var Rt=l(_);$=r(Rt,"A",{id:!0,class:!0,href:!0});var Zt=l($);S=r(Zt,"SPAN",{});var Pn=l(S);x(g.$$.fragment,Pn),Pn.forEach(s),Zt.forEach(s),C=p(Rt),D=r(Rt,"SPAN",{});var es=l(D);z=a(es,"Recherche s\xE9mantique avec FAISS"),es.forEach(s),Rt.forEach(s),O=p(e),N.l(e),I=p(e),R=r(e,"P",{});var ye=l(R);Y=a(ye,"Dans "),P=r(ye,"A",{href:!0});var ts=l(P);Q=a(ts,"section 5"),ts.forEach(s),B=a(ye,", nous avons cr\xE9\xE9 un jeu de donn\xE9es de probl\xE8mes et de commentaires GitHub \xE0 partir du d\xE9p\xF4t \u{1F917} "),U=r(ye,"EM",{});var Mn=l(U);H=a(Mn,"Datasets"),Mn.forEach(s),V=a(ye,". Dans cette section, nous utilisons ces informations pour cr\xE9er un moteur de recherche qui peut nous aider \xE0 trouver des r\xE9ponses \xE0 nos questions les plus urgentes sur la biblioth\xE8que !"),ye.forEach(s),c=p(e),x(h.$$.fragment,e),L=p(e),W=r(e,"H2",{class:!0});var Ua=l(W);J=r(Ua,"A",{id:!0,class:!0,href:!0});var Ki=l(J);K=r(Ki,"SPAN",{});var Zi=l(K);x(oe.$$.fragment,Zi),Zi.forEach(s),Ki.forEach(s),Ie=p(Ua),He=r(Ua,"SPAN",{});var eu=l(He);no=a(eu,"Utilisation des ench\xE2ssements pour la recherche s\xE9mantique"),eu.forEach(s),Ua.forEach(s),Ln=p(e),te=r(e,"P",{});var De=l(te);ao=a(De,"Comme nous l\u2019avons vu dans le "),Lt=r(De,"A",{href:!0});var tu=l(Lt);oo=a(tu,"Chapitre 1"),tu.forEach(s),ro=a(De,", les mod\xE8les de langage bas\xE9s sur les "),os=r(De,"EM",{});var su=l(os);lo=a(su,"transformers"),su.forEach(s),io=a(De," repr\xE9sentent chaque "),rs=r(De,"EM",{});var nu=l(rs);uo=a(nu,"token"),nu.forEach(s),co=a(De," dans une \xE9tendue de texte sous la forme d\u2019un "),ls=r(De,"EM",{});var au=l(ls);po=a(au,"ench\xE2ssement"),au.forEach(s),mo=a(De,". Il s\u2019av\xE8re que l\u2019on peut regrouper les ench\xE2ssements individuels pour cr\xE9er une repr\xE9sentation vectorielle pour des phrases enti\xE8res, des paragraphes ou (dans certains cas) des documents. Ces ench\xE2ssements peuvent ensuite \xEAtre utilis\xE9s pour trouver des documents similaires dans le corpus en calculant la similarit\xE9 du produit scalaire (ou une autre m\xE9trique de similarit\xE9) entre chaque ench\xE2ssement et en renvoyant les documents avec le plus grand chevauchement."),De.forEach(s),Hn=p(e),Ht=r(e,"P",{});var ou=l(Ht);fo=a(ou,"Dans cette section, nous utilisons les ench\xE2ssements pour d\xE9velopper un moteur de recherche s\xE9mantique. Ces moteurs de recherche offrent plusieurs avantages par rapport aux approches conventionnelles bas\xE9es sur la correspondance des mots-cl\xE9s dans une requ\xEAte avec les documents."),ou.forEach(s),Fn=p(e),Pe=r(e,"DIV",{class:!0});var Ga=l(Pe);st=r(Ga,"IMG",{class:!0,src:!0,alt:!0}),ho=p(Ga),nt=r(Ga,"IMG",{class:!0,src:!0,alt:!0}),Ga.forEach(s),zn=p(e),Me=r(e,"H2",{class:!0});var Va=l(Me);Fe=r(Va,"A",{id:!0,class:!0,href:!0});var ru=l(Fe);is=r(ru,"SPAN",{});var lu=l(is);x(at.$$.fragment,lu),lu.forEach(s),ru.forEach(s),_o=p(Va),us=r(Va,"SPAN",{});var iu=l(us);go=a(iu,"Chargement et pr\xE9paration du jeu de donn\xE9es"),iu.forEach(s),Va.forEach(s),Un=p(e),ke=r(e,"P",{});var ss=l(ke);vo=a(ss,"La premi\xE8re chose que nous devons faire est de t\xE9l\xE9charger notre jeu de donn\xE9es de probl\xE8mes GitHub. Utilisons la biblioth\xE8que \u{1F917} "),cs=r(ss,"EM",{});var uu=l(cs);bo=a(uu,"Hub"),uu.forEach(s),$o=a(ss," pour r\xE9soudre l\u2019URL o\xF9 notre fichier est stock\xE9 sur le "),ds=r(ss,"EM",{});var cu=l(ds);qo=a(cu,"Hib"),cu.forEach(s),Eo=a(ss," d\u2019Hugging Face :"),ss.forEach(s),Gn=p(e),x(ot.$$.fragment,e),Vn=p(e),we=r(e,"P",{});var ns=l(we);xo=a(ns,"Avec l\u2019URL stock\xE9 dans "),ps=r(ns,"CODE",{});var du=l(ps);ko=a(du,"data_files"),du.forEach(s),wo=a(ns,", nous pouvons ensuite charger le jeu de donn\xE9es distant en utilisant la m\xE9thode introduite dans "),Ft=r(ns,"A",{href:!0});var pu=l(Ft);jo=a(pu,"section 2"),pu.forEach(s),yo=a(ns," :"),ns.forEach(s),Yn=p(e),x(rt.$$.fragment,e),Bn=p(e),x(lt.$$.fragment,e),Wn=p(e),X=r(e,"P",{});var ae=l(X);Do=a(ae,"Ici, nous avons sp\xE9cifi\xE9 l\u2019\xE9chantillon "),ms=r(ae,"CODE",{});var mu=l(ms);Co=a(mu,"train"),mu.forEach(s),To=a(ae," par d\xE9faut dans "),fs=r(ae,"CODE",{});var fu=l(fs);Ao=a(fu,"load_dataset()"),fu.forEach(s),So=a(ae,", de sorte que cela renvoie un "),hs=r(ae,"CODE",{});var hu=l(hs);Oo=a(hu,"Dataset"),hu.forEach(s),No=a(ae," au lieu d\u2019un "),_s=r(ae,"CODE",{});var _u=l(_s);Io=a(_u,"DatasetDict"),_u.forEach(s),Po=a(ae,". La premi\xE8re chose \xE0 faire est de filtrer les "),gs=r(ae,"EM",{});var gu=l(gs);Mo=a(gu,"pull requests"),gu.forEach(s),Ro=a(ae," car celles-ci ont tendance \xE0 \xEAtre rarement utilis\xE9es pour r\xE9pondre aux requ\xEAtes des utilisateurs et introduiront du bruit dans notre moteur de recherche. Comme cela devrait \xEAtre familier maintenant, nous pouvons utiliser la fonction "),vs=r(ae,"CODE",{});var vu=l(vs);Lo=a(vu,"Dataset.filter()"),vu.forEach(s),Ho=a(ae," pour exclure ces lignes de notre jeu de donn\xE9es. Pendant que nous y sommes, filtrons \xE9galement les lignes sans commentaires, car celles-ci ne fournissent aucune r\xE9ponse aux requ\xEAtes des utilisateurs :"),ae.forEach(s),Jn=p(e),x(it.$$.fragment,e),Qn=p(e),x(ut.$$.fragment,e),Xn=p(e),Z=r(e,"P",{});var he=l(Z);Fo=a(he,"Nous pouvons voir qu\u2019il y a beaucoup de colonnes dans notre jeu de donn\xE9es, dont la plupart n\u2019ont pas besoin de construire notre moteur de recherche. Du point de vue de la recherche, les colonnes les plus informatives sont "),bs=r(he,"CODE",{});var bu=l(bs);zo=a(bu,"title"),bu.forEach(s),Uo=a(he,", "),$s=r(he,"CODE",{});var $u=l($s);Go=a($u,"body"),$u.forEach(s),Vo=a(he," et "),qs=r(he,"CODE",{});var qu=l(qs);Yo=a(qu,"comments"),qu.forEach(s),Bo=a(he,", tandis que "),Es=r(he,"CODE",{});var Eu=l(Es);Wo=a(Eu,"html_url"),Eu.forEach(s),Jo=a(he," nous fournit un lien vers le probl\xE8me source. Utilisons la fonction "),xs=r(he,"CODE",{});var xu=l(xs);Qo=a(xu,"Dataset.remove_columns()"),xu.forEach(s),Xo=a(he," pour supprimer le reste :"),he.forEach(s),Kn=p(e),x(ct.$$.fragment,e),Zn=p(e),x(dt.$$.fragment,e),ea=p(e),ee=r(e,"P",{});var _e=l(ee);Ko=a(_e,"Pour cr\xE9er nos ench\xE2ssements, nous ajoutons \xE0 chaque commentaire le titre et le corps du probl\xE8me, car ces champs contiennent des informations contextuelles utiles. \xC9tant donn\xE9 que notre colonne "),ks=r(_e,"CODE",{});var ku=l(ks);Zo=a(ku,"comments"),ku.forEach(s),er=a(_e," est actuellement une liste de commentaires pour chaque probl\xE8me, nous devons \xAB \xE9clater \xBB la colonne afin que chaque ligne se compose d\u2019un "),ws=r(_e,"EM",{});var wu=l(ws);tr=a(wu,"tuple"),wu.forEach(s),sr=p(_e),js=r(_e,"CODE",{});var ju=l(js);nr=a(ju,"(html_url, title, body, comment)"),ju.forEach(s),ar=a(_e,". Dans Pandas, nous pouvons le faire avec la fonction "),pt=r(_e,"A",{href:!0,rel:!0});var yu=l(pt);ys=r(yu,"CODE",{});var Du=l(ys);or=a(Du,"DataFrame.explode()"),Du.forEach(s),yu.forEach(s),rr=a(_e,", qui cr\xE9e une nouvelle ligne pour chaque \xE9l\xE9ment dans une colonne de type liste, tout en r\xE9pliquant toutes les autres valeurs de colonne. Pour voir cela en action, passons d\u2019abord au format "),Ds=r(_e,"CODE",{});var Cu=l(Ds);lr=a(Cu,"DataFrame"),Cu.forEach(s),ir=a(_e," de Pandas :"),_e.forEach(s),ta=p(e),x(mt.$$.fragment,e),sa=p(e),ze=r(e,"P",{});var Ya=l(ze);ur=a(Ya,"Si nous inspectons la premi\xE8re ligne de ce "),Cs=r(Ya,"CODE",{});var Tu=l(Cs);cr=a(Tu,"DataFrame"),Tu.forEach(s),dr=a(Ya,", nous pouvons voir qu\u2019il y a quatre commentaires associ\xE9s \xE0 ce probl\xE8me :"),Ya.forEach(s),na=p(e),x(ft.$$.fragment,e),aa=p(e),x(ht.$$.fragment,e),oa=p(e),Ue=r(e,"P",{});var Ba=l(Ue);pr=a(Ba,"Lorsque nous d\xE9composons "),Ts=r(Ba,"CODE",{});var Au=l(Ts);mr=a(Au,"df"),Au.forEach(s),fr=a(Ba,", nous nous attendons \xE0 obtenir une ligne pour chacun de ces commentaires. V\xE9rifions si c\u2019est le cas :"),Ba.forEach(s),ra=p(e),x(_t.$$.fragment,e),la=p(e),re=r(e,"TABLE",{border:!0,class:!0,style:!0});var Wa=l(re);As=r(Wa,"THEAD",{});var Su=l(As);se=r(Su,"TR",{style:!0});var Ce=l(se);ia=r(Ce,"TH",{}),l(ia).forEach(s),hr=p(Ce),Ss=r(Ce,"TH",{});var Ou=l(Ss);_r=a(Ou,"html_url"),Ou.forEach(s),gr=p(Ce),Os=r(Ce,"TH",{});var Nu=l(Os);vr=a(Nu,"title"),Nu.forEach(s),br=p(Ce),Ns=r(Ce,"TH",{});var Iu=l(Ns);$r=a(Iu,"comments"),Iu.forEach(s),qr=p(Ce),Is=r(Ce,"TH",{});var Pu=l(Is);Er=a(Pu,"body"),Pu.forEach(s),Ce.forEach(s),Su.forEach(s),xr=p(Wa),ge=r(Wa,"TBODY",{});var Xe=l(ge);le=r(Xe,"TR",{});var Te=l(le);Ps=r(Te,"TH",{});var Mu=l(Ps);kr=a(Mu,"0"),Mu.forEach(s),wr=p(Te),Ms=r(Te,"TD",{});var Ru=l(Ms);jr=a(Ru,"https://github.com/huggingface/datasets/issues/2787"),Ru.forEach(s),yr=p(Te),Rs=r(Te,"TD",{});var Lu=l(Rs);Dr=a(Lu,"ConnectionError: Couldn't reach https://raw.githubusercontent.com"),Lu.forEach(s),Cr=p(Te),Ls=r(Te,"TD",{});var Hu=l(Ls);Tr=a(Hu,"the bug code locate in \uFF1A\\r\\n    if data_args.task_name is not None..."),Hu.forEach(s),Ar=p(Te),Hs=r(Te,"TD",{});var Fu=l(Hs);Sr=a(Fu,"Hello,\\r\\nI am trying to run run_glue.py and it gives me this error..."),Fu.forEach(s),Te.forEach(s),Or=p(Xe),ie=r(Xe,"TR",{});var Ae=l(ie);Fs=r(Ae,"TH",{});var zu=l(Fs);Nr=a(zu,"1"),zu.forEach(s),Ir=p(Ae),zs=r(Ae,"TD",{});var Uu=l(zs);Pr=a(Uu,"https://github.com/huggingface/datasets/issues/2787"),Uu.forEach(s),Mr=p(Ae),Us=r(Ae,"TD",{});var Gu=l(Us);Rr=a(Gu,"ConnectionError: Couldn't reach https://raw.githubusercontent.com"),Gu.forEach(s),Lr=p(Ae),Gs=r(Ae,"TD",{});var Vu=l(Gs);Hr=a(Vu,"Hi @jinec,\\r\\n\\r\\nFrom time to time we get this kind of `ConnectionError` coming from the github.com website: https://raw.githubusercontent.com..."),Vu.forEach(s),Fr=p(Ae),Vs=r(Ae,"TD",{});var Yu=l(Vs);zr=a(Yu,"Hello,\\r\\nI am trying to run run_glue.py and it gives me this error..."),Yu.forEach(s),Ae.forEach(s),Ur=p(Xe),ue=r(Xe,"TR",{});var Se=l(ue);Ys=r(Se,"TH",{});var Bu=l(Ys);Gr=a(Bu,"2"),Bu.forEach(s),Vr=p(Se),Bs=r(Se,"TD",{});var Wu=l(Bs);Yr=a(Wu,"https://github.com/huggingface/datasets/issues/2787"),Wu.forEach(s),Br=p(Se),Ws=r(Se,"TD",{});var Ju=l(Ws);Wr=a(Ju,"ConnectionError: Couldn't reach https://raw.githubusercontent.com"),Ju.forEach(s),Jr=p(Se),Js=r(Se,"TD",{});var Qu=l(Js);Qr=a(Qu,"cannot connect\uFF0Ceven by Web browser\uFF0Cplease check that  there is some  problems\u3002"),Qu.forEach(s),Xr=p(Se),Qs=r(Se,"TD",{});var Xu=l(Qs);Kr=a(Xu,"Hello,\\r\\nI am trying to run run_glue.py and it gives me this error..."),Xu.forEach(s),Se.forEach(s),Zr=p(Xe),ce=r(Xe,"TR",{});var Oe=l(ce);Xs=r(Oe,"TH",{});var Ku=l(Xs);el=a(Ku,"3"),Ku.forEach(s),tl=p(Oe),Ks=r(Oe,"TD",{});var Zu=l(Ks);sl=a(Zu,"https://github.com/huggingface/datasets/issues/2787"),Zu.forEach(s),nl=p(Oe),Zs=r(Oe,"TD",{});var ec=l(Zs);al=a(ec,"ConnectionError: Couldn't reach https://raw.githubusercontent.com"),ec.forEach(s),ol=p(Oe),en=r(Oe,"TD",{});var tc=l(en);rl=a(tc,"I can access https://raw.githubusercontent.com/huggingface/datasets/1.7.0/datasets/glue/glue.py without problem..."),tc.forEach(s),ll=p(Oe),tn=r(Oe,"TD",{});var sc=l(tn);il=a(sc,"Hello,\\r\\nI am trying to run run_glue.py and it gives me this error..."),sc.forEach(s),Oe.forEach(s),Xe.forEach(s),Wa.forEach(s),ua=p(e),de=r(e,"P",{});var Ke=l(de);ul=a(Ke,"G\xE9nial, nous pouvons voir que les lignes ont \xE9t\xE9 r\xE9pliqu\xE9es, avec la colonne "),sn=r(Ke,"CODE",{});var nc=l(sn);cl=a(nc,"comments"),nc.forEach(s),dl=a(Ke," contenant les commentaires individuels ! Maintenant que nous en avons fini avec Pandas, nous pouvons rapidement revenir \xE0 un "),nn=r(Ke,"CODE",{});var ac=l(nn);pl=a(ac,"Dataset"),ac.forEach(s),ml=a(Ke," en chargeant le "),an=r(Ke,"CODE",{});var oc=l(an);fl=a(oc,"DataFrame"),oc.forEach(s),hl=a(Ke," en m\xE9moire :"),Ke.forEach(s),ca=p(e),x(gt.$$.fragment,e),da=p(e),x(vt.$$.fragment,e),pa=p(e),zt=r(e,"P",{});var rc=l(zt);_l=a(rc,"D\u2019accord, cela nous a donn\xE9 quelques milliers de commentaires avec lesquels travailler !"),rc.forEach(s),ma=p(e),x(Ge.$$.fragment,e),fa=p(e),Ve=r(e,"P",{});var Ja=l(Ve);gl=a(Ja,"Maintenant que nous avons un commentaire par ligne, cr\xE9ons une nouvelle colonne "),on=r(Ja,"CODE",{});var lc=l(on);vl=a(lc,"comments_length"),lc.forEach(s),bl=a(Ja," contenant le nombre de mots par commentaire :"),Ja.forEach(s),ha=p(e),x(bt.$$.fragment,e),_a=p(e),Ut=r(e,"P",{});var ic=l(Ut);$l=a(ic,"Nous pouvons utiliser cette nouvelle colonne pour filtrer les commentaires courts incluant g\xE9n\xE9ralement des \xE9l\xE9ments tels que \xAB cc @lewtun \xBB ou \xAB Merci ! \xBB qui ne sont pas pertinents pour notre moteur de recherche. Il n\u2019y a pas de nombre pr\xE9cis \xE0 s\xE9lectionner pour le filtre mais 15 mots semblent \xEAtre un bon d\xE9but :"),ic.forEach(s),ga=p(e),x($t.$$.fragment,e),va=p(e),x(qt.$$.fragment,e),ba=p(e),je=r(e,"P",{});var as=l(je);ql=a(as,"Apr\xE8s avoir un peu nettoy\xE9 notre jeu de donn\xE9es, concat\xE9nons le titre, la description et les commentaires du probl\xE8me dans une nouvelle colonne "),rn=r(as,"CODE",{});var uc=l(rn);El=a(uc,"text"),uc.forEach(s),xl=a(as,". Comme d\u2019habitude, nous allons \xE9crire une fonction simple que nous pouvons passer \xE0 "),ln=r(as,"CODE",{});var cc=l(ln);kl=a(cc,"Dataset.map()"),cc.forEach(s),wl=a(as," :"),as.forEach(s),$a=p(e),x(Et.$$.fragment,e),qa=p(e),Gt=r(e,"P",{});var dc=l(Gt);jl=a(dc,"Nous sommes enfin pr\xEAts \xE0 cr\xE9er des ench\xE2ssements ! Jetons un coup d\u2019\u0153il."),dc.forEach(s),Ea=p(e),Re=r(e,"H2",{class:!0});var Qa=l(Re);Ye=r(Qa,"A",{id:!0,class:!0,href:!0});var pc=l(Ye);un=r(pc,"SPAN",{});var mc=l(un);x(xt.$$.fragment,mc),mc.forEach(s),pc.forEach(s),yl=p(Qa),cn=r(Qa,"SPAN",{});var fc=l(cn);Dl=a(fc,"Cr\xE9ation d\u2019ench\xE2ssements pour les textes"),fc.forEach(s),Qa.forEach(s),xa=p(e),M=r(e,"P",{});var F=l(M);Cl=a(F,"Nous avons vu dans "),Vt=r(F,"A",{href:!0});var hc=l(Vt);Tl=a(hc,"Chapitre 2"),hc.forEach(s),Al=a(F," que nous pouvons obtenir des ench\xE2ssements de "),dn=r(F,"EM",{});var _c=l(dn);Sl=a(_c,"tokens"),_c.forEach(s),Ol=a(F," en utilisant la classe "),pn=r(F,"CODE",{});var gc=l(pn);Nl=a(gc,"AutoModel"),gc.forEach(s),Il=a(F,". Tout ce que nous avons \xE0 faire est de choisir un "),mn=r(F,"EM",{});var vc=l(mn);Pl=a(vc,"checkpoint"),vc.forEach(s),Ml=a(F," appropri\xE9 \xE0 partir duquel charger le mod\xE8le. Heureusement, il existe une biblioth\xE8que appel\xE9e "),fn=r(F,"CODE",{});var bc=l(fn);Rl=a(bc,"sentence-transformers"),bc.forEach(s),Ll=a(F," d\xE9di\xE9e \xE0 la cr\xE9ation d\u2019ench\xE2ssements. Comme d\xE9crit dans la "),kt=r(F,"A",{href:!0,rel:!0});var $c=l(kt);Hl=a($c,"documentation de la biblioth\xE8que"),$c.forEach(s),Fl=a(F,", notre cas d\u2019utilisation est un exemple de "),hn=r(F,"EM",{});var qc=l(hn);zl=a(qc,"recherche s\xE9mantique asym\xE9trique"),qc.forEach(s),Ul=a(F,". En effet, nous avons une requ\xEAte courte dont nous aimerions trouver la r\xE9ponse dans un document plus long, par exemple un commentaire \xE0 un probl\xE8me. Le "),wt=r(F,"A",{href:!0,rel:!0});var Ec=l(wt);Gl=a(Ec,"tableau de pr\xE9sentation des mod\xE8les"),Ec.forEach(s),Vl=a(F," de la documentation indique que le "),_n=r(F,"EM",{});var xc=l(_n);Yl=a(xc,"checkpoint"),xc.forEach(s),Bl=p(F),gn=r(F,"CODE",{});var kc=l(gn);Wl=a(kc,"multi-qa-mpnet-base-dot-v1"),kc.forEach(s),Jl=a(F," a les meilleures performances pour la recherche s\xE9mantique. Utilisons donc le pour notre application. Nous allons \xE9galement charger le "),vn=r(F,"EM",{});var wc=l(vn);Ql=a(wc,"tokenizer"),wc.forEach(s),Xl=a(F," en utilisant le m\xEAme "),bn=r(F,"EM",{});var jc=l(bn);Kl=a(jc,"checkpoint"),jc.forEach(s),Zl=a(F," :"),F.forEach(s),ka=p(e),be.l(e),Yt=p(e),ne=r(e,"P",{});var Ne=l(ne);ei=a(Ne,"Comme nous l\u2019avons mentionn\xE9 pr\xE9c\xE9demment, nous aimerions repr\xE9senter chaque entr\xE9e dans notre corpus de probl\xE8mes GitHub comme un vecteur unique. Nous devons donc regrouper ou faire la moyenne de nos ench\xE2ssements de "),$n=r(Ne,"EM",{});var yc=l($n);ti=a(yc,"tokens"),yc.forEach(s),si=a(Ne," d\u2019une mani\xE8re ou d\u2019une autre. Une approche populaire consiste \xE0 effectuer un "),qn=r(Ne,"EM",{});var Dc=l(qn);ni=a(Dc,"regroupement CLS"),Dc.forEach(s),ai=a(Ne," sur les sorties de notre mod\xE8le, o\xF9 nous collectons simplement le dernier \xE9tat cach\xE9 pour le "),En=r(Ne,"EM",{});var Cc=l(En);oi=a(Cc,"token"),Cc.forEach(s),ri=a(Ne," sp\xE9cial "),xn=r(Ne,"CODE",{});var Tc=l(xn);li=a(Tc,"[CLS]"),Tc.forEach(s),ii=a(Ne,". La fonction suivante fait \xE7a pour nous :"),Ne.forEach(s),wa=p(e),x(jt.$$.fragment,e),ja=p(e),Bt=r(e,"P",{});var Ac=l(Bt);ui=a(Ac,"Ensuite, nous allons cr\xE9er une fonction utile qui va tokeniser une liste de documents, placer les tenseurs dans le GPU, les donner au mod\xE8le et enfin appliquer le regroupement CLS aux sorties :"),Ac.forEach(s),ya=p(e),qe.l(e),Wt=p(e),Be=r(e,"P",{});var Xa=l(Be);ci=a(Xa,"Notez que nous avons converti les ench\xE2ssements en tableaux NumPy. C\u2019est parce que \u{1F917} "),kn=r(Xa,"EM",{});var Sc=l(kn);di=a(Sc,"Datasets"),Sc.forEach(s),pi=a(Xa," n\xE9cessite ce format lorsque nous essayons de les indexer avec FAISS, ce que nous ferons ensuite."),Xa.forEach(s),Da=p(e),Le=r(e,"H2",{class:!0});var Ka=l(Le);We=r(Ka,"A",{id:!0,class:!0,href:!0});var Oc=l(We);wn=r(Oc,"SPAN",{});var Nc=l(wn);x(yt.$$.fragment,Nc),Nc.forEach(s),Oc.forEach(s),mi=p(Ka),jn=r(Ka,"SPAN",{});var Ic=l(jn);fi=a(Ic,"Utilisation de FAISS pour une recherche de similarit\xE9 efficace"),Ic.forEach(s),Ka.forEach(s),Ca=p(e),pe=r(e,"P",{});var Ze=l(pe);hi=a(Ze,"Maintenant que nous avons un jeu de donn\xE9es d\u2019incorporations, nous avons besoin d\u2019un moyen de les rechercher. Pour ce faire, nous utiliserons une structure de donn\xE9es sp\xE9ciale dans \u{1F917} "),yn=r(Ze,"EM",{});var Pc=l(yn);_i=a(Pc,"Datasets"),Pc.forEach(s),gi=a(Ze," appel\xE9e "),Dn=r(Ze,"EM",{});var Mc=l(Dn);vi=a(Mc,"FAISS index"),Mc.forEach(s),bi=a(Ze,". "),Dt=r(Ze,"A",{href:!0,rel:!0});var Rc=l(Dt);$i=a(Rc,"FAISS"),Rc.forEach(s),qi=a(Ze," (abr\xE9viation de Facebook AI Similarity Search) est une biblioth\xE8que qui fournit des algorithmes efficaces pour rechercher et regrouper rapidement des vecteurs d\u2019int\xE9gration."),Ze.forEach(s),Ta=p(e),me=r(e,"P",{});var et=l(me);Ei=a(et,"L\u2019id\xE9e de base derri\xE8re FAISS est de cr\xE9er une structure de donn\xE9es sp\xE9ciale appel\xE9e un "),Cn=r(et,"EM",{});var Lc=l(Cn);xi=a(Lc,"index"),Lc.forEach(s),ki=a(et," qui permet de trouver quels plongements sont similaires \xE0 un plongement d\u2019entr\xE9e. Cr\xE9er un index FAISS dans \u{1F917} "),Tn=r(et,"EM",{});var Hc=l(Tn);wi=a(Hc,"Datasets"),Hc.forEach(s),ji=a(et," est simple \u2014 nous utilisons la fonction "),An=r(et,"CODE",{});var Fc=l(An);yi=a(Fc,"Dataset.add_faiss_index()"),Fc.forEach(s),Di=a(et," et sp\xE9cifions quelle colonne de notre jeu de donn\xE9es nous aimerions indexer :"),et.forEach(s),Aa=p(e),x(Ct.$$.fragment,e),Sa=p(e),Je=r(e,"P",{});var Za=l(Je);Ci=a(Za,"Nous pouvons maintenant effectuer des requ\xEAtes sur cet index en effectuant une recherche des voisins les plus proches avec la fonction "),Sn=r(Za,"CODE",{});var zc=l(Sn);Ti=a(zc,"Dataset.get_nearest_examples()"),zc.forEach(s),Ai=a(Za,". Testons cela en ench\xE2ssant d\u2019abord une question comme suit :"),Za.forEach(s),Oa=p(e),xe.l(e),Jt=p(e),Qt=r(e,"P",{});var Uc=l(Qt);Si=a(Uc,"Tout comme avec les documents, nous avons maintenant un vecteur de 768 dimensions repr\xE9sentant la requ\xEAte. Nous pouvons le comparer \xE0 l\u2019ensemble du corpus pour trouver les ench\xE2ssements les plus similaires :"),Uc.forEach(s),Na=p(e),x(Tt.$$.fragment,e),Ia=p(e),fe=r(e,"P",{});var tt=l(fe);Oi=a(tt,"La fonction "),On=r(tt,"CODE",{});var Gc=l(On);Ni=a(Gc,"Dataset.get_nearest_examples()"),Gc.forEach(s),Ii=a(tt," renvoie un "),Nn=r(tt,"EM",{});var Vc=l(Nn);Pi=a(Vc,"tuple"),Vc.forEach(s),Mi=a(tt," de scores qui classent le chevauchement entre la requ\xEAte et le document, et un jeu correspondant d\u2019\xE9chantillons (ici, les 5 meilleures correspondances). Collectons-les dans un "),In=r(tt,"CODE",{});var Yc=l(In);Ri=a(Yc,"pandas.DataFrame"),Yc.forEach(s),Li=a(tt," afin de pouvoir les trier facilement :"),tt.forEach(s),Pa=p(e),x(At.$$.fragment,e),Ma=p(e),Xt=r(e,"P",{});var Bc=l(Xt);Hi=a(Bc,"Nous pouvons maintenant parcourir les premi\xE8res lignes pour voir dans quelle mesure notre requ\xEAte correspond aux commentaires disponibles :"),Bc.forEach(s),Ra=p(e),x(St.$$.fragment,e),La=p(e),x(Ot.$$.fragment,e),Ha=p(e),Kt=r(e,"P",{});var Wc=l(Kt);Fi=a(Wc,"Pas mal ! Notre deuxi\xE8me r\xE9sultat semble correspondre \xE0 la requ\xEAte."),Wc.forEach(s),Fa=p(e),x(Qe.$$.fragment,e),this.h()},h(){j(m,"name","hf:doc:metadata"),j(m,"content",JSON.stringify(_d)),j($,"id","recherche-smantique-avec-faiss"),j($,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),j($,"href","#recherche-smantique-avec-faiss"),j(_,"class","relative group"),j(P,"href","/course/fr/chapter5/5"),j(J,"id","utilisation-des-enchssements-pour-la-recherche-smantique"),j(J,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),j(J,"href","#utilisation-des-enchssements-pour-la-recherche-smantique"),j(W,"class","relative group"),j(Lt,"href","/course/fr/chapter1"),j(st,"class","block dark:hidden"),Jc(st.src,zi="https://huggingface.co/datasets/huggingface-course/documentation-images/resolve/main/en/chapter5/semantic-search.svg")||j(st,"src",zi),j(st,"alt","Recherche s\xE9mantique."),j(nt,"class","hidden dark:block"),Jc(nt.src,Ui="https://huggingface.co/datasets/huggingface-course/documentation-images/resolve/main/en/chapter5/semantic-search-dark.svg")||j(nt,"src",Ui),j(nt,"alt","Recherche s\xE9mantique."),j(Pe,"class","flex justify-center"),j(Fe,"id","chargement-et-prparation-du-jeu-de-donnes"),j(Fe,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),j(Fe,"href","#chargement-et-prparation-du-jeu-de-donnes"),j(Me,"class","relative group"),j(Ft,"href","/course/fr/chapter5/2"),j(pt,"href","https://pandas.pydata.org/pandas-docs/stable/reference/api/pandas.DataFrame.explode.html"),j(pt,"rel","nofollow"),eo(se,"text-align","right"),j(re,"border","1"),j(re,"class","dataframe"),eo(re,"table-layout","fixed"),eo(re,"word-wrap","break-word"),eo(re,"width","100%"),j(Ye,"id","cration-denchssements-pour-les-textes"),j(Ye,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),j(Ye,"href","#cration-denchssements-pour-les-textes"),j(Re,"class","relative group"),j(Vt,"href","/course/fr/chapter2"),j(kt,"href","https://www.sbert.net/examples/applications/semantic-search/README.html#symmetric-vs-asymmetric-semantic-search"),j(kt,"rel","nofollow"),j(wt,"href","https://www.sbert.net/docs/pretrained_models.html#model-overview"),j(wt,"rel","nofollow"),j(We,"id","utilisation-de-faiss-pour-une-recherche-de-similarit-efficace"),j(We,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),j(We,"href","#utilisation-de-faiss-pour-une-recherche-de-similarit-efficace"),j(Le,"class","relative group"),j(Dt,"href","https://faiss.ai/"),j(Dt,"rel","nofollow")},m(e,i){t(document.head,m),u(e,q,i),k(f,e,i),u(e,y,i),u(e,_,i),t(_,$),t($,S),k(g,S,null),t(_,C),t(_,D),t(D,z),u(e,O,i),Nt[T].m(e,i),u(e,I,i),u(e,R,i),t(R,Y),t(R,P),t(P,Q),t(R,B),t(R,U),t(U,H),t(R,V),u(e,c,i),k(h,e,i),u(e,L,i),u(e,W,i),t(W,J),t(J,K),k(oe,K,null),t(W,Ie),t(W,He),t(He,no),u(e,Ln,i),u(e,te,i),t(te,ao),t(te,Lt),t(Lt,oo),t(te,ro),t(te,os),t(os,lo),t(te,io),t(te,rs),t(rs,uo),t(te,co),t(te,ls),t(ls,po),t(te,mo),u(e,Hn,i),u(e,Ht,i),t(Ht,fo),u(e,Fn,i),u(e,Pe,i),t(Pe,st),t(Pe,ho),t(Pe,nt),u(e,zn,i),u(e,Me,i),t(Me,Fe),t(Fe,is),k(at,is,null),t(Me,_o),t(Me,us),t(us,go),u(e,Un,i),u(e,ke,i),t(ke,vo),t(ke,cs),t(cs,bo),t(ke,$o),t(ke,ds),t(ds,qo),t(ke,Eo),u(e,Gn,i),k(ot,e,i),u(e,Vn,i),u(e,we,i),t(we,xo),t(we,ps),t(ps,ko),t(we,wo),t(we,Ft),t(Ft,jo),t(we,yo),u(e,Yn,i),k(rt,e,i),u(e,Bn,i),k(lt,e,i),u(e,Wn,i),u(e,X,i),t(X,Do),t(X,ms),t(ms,Co),t(X,To),t(X,fs),t(fs,Ao),t(X,So),t(X,hs),t(hs,Oo),t(X,No),t(X,_s),t(_s,Io),t(X,Po),t(X,gs),t(gs,Mo),t(X,Ro),t(X,vs),t(vs,Lo),t(X,Ho),u(e,Jn,i),k(it,e,i),u(e,Qn,i),k(ut,e,i),u(e,Xn,i),u(e,Z,i),t(Z,Fo),t(Z,bs),t(bs,zo),t(Z,Uo),t(Z,$s),t($s,Go),t(Z,Vo),t(Z,qs),t(qs,Yo),t(Z,Bo),t(Z,Es),t(Es,Wo),t(Z,Jo),t(Z,xs),t(xs,Qo),t(Z,Xo),u(e,Kn,i),k(ct,e,i),u(e,Zn,i),k(dt,e,i),u(e,ea,i),u(e,ee,i),t(ee,Ko),t(ee,ks),t(ks,Zo),t(ee,er),t(ee,ws),t(ws,tr),t(ee,sr),t(ee,js),t(js,nr),t(ee,ar),t(ee,pt),t(pt,ys),t(ys,or),t(ee,rr),t(ee,Ds),t(Ds,lr),t(ee,ir),u(e,ta,i),k(mt,e,i),u(e,sa,i),u(e,ze,i),t(ze,ur),t(ze,Cs),t(Cs,cr),t(ze,dr),u(e,na,i),k(ft,e,i),u(e,aa,i),k(ht,e,i),u(e,oa,i),u(e,Ue,i),t(Ue,pr),t(Ue,Ts),t(Ts,mr),t(Ue,fr),u(e,ra,i),k(_t,e,i),u(e,la,i),u(e,re,i),t(re,As),t(As,se),t(se,ia),t(se,hr),t(se,Ss),t(Ss,_r),t(se,gr),t(se,Os),t(Os,vr),t(se,br),t(se,Ns),t(Ns,$r),t(se,qr),t(se,Is),t(Is,Er),t(re,xr),t(re,ge),t(ge,le),t(le,Ps),t(Ps,kr),t(le,wr),t(le,Ms),t(Ms,jr),t(le,yr),t(le,Rs),t(Rs,Dr),t(le,Cr),t(le,Ls),t(Ls,Tr),t(le,Ar),t(le,Hs),t(Hs,Sr),t(ge,Or),t(ge,ie),t(ie,Fs),t(Fs,Nr),t(ie,Ir),t(ie,zs),t(zs,Pr),t(ie,Mr),t(ie,Us),t(Us,Rr),t(ie,Lr),t(ie,Gs),t(Gs,Hr),t(ie,Fr),t(ie,Vs),t(Vs,zr),t(ge,Ur),t(ge,ue),t(ue,Ys),t(Ys,Gr),t(ue,Vr),t(ue,Bs),t(Bs,Yr),t(ue,Br),t(ue,Ws),t(Ws,Wr),t(ue,Jr),t(ue,Js),t(Js,Qr),t(ue,Xr),t(ue,Qs),t(Qs,Kr),t(ge,Zr),t(ge,ce),t(ce,Xs),t(Xs,el),t(ce,tl),t(ce,Ks),t(Ks,sl),t(ce,nl),t(ce,Zs),t(Zs,al),t(ce,ol),t(ce,en),t(en,rl),t(ce,ll),t(ce,tn),t(tn,il),u(e,ua,i),u(e,de,i),t(de,ul),t(de,sn),t(sn,cl),t(de,dl),t(de,nn),t(nn,pl),t(de,ml),t(de,an),t(an,fl),t(de,hl),u(e,ca,i),k(gt,e,i),u(e,da,i),k(vt,e,i),u(e,pa,i),u(e,zt,i),t(zt,_l),u(e,ma,i),k(Ge,e,i),u(e,fa,i),u(e,Ve,i),t(Ve,gl),t(Ve,on),t(on,vl),t(Ve,bl),u(e,ha,i),k(bt,e,i),u(e,_a,i),u(e,Ut,i),t(Ut,$l),u(e,ga,i),k($t,e,i),u(e,va,i),k(qt,e,i),u(e,ba,i),u(e,je,i),t(je,ql),t(je,rn),t(rn,El),t(je,xl),t(je,ln),t(ln,kl),t(je,wl),u(e,$a,i),k(Et,e,i),u(e,qa,i),u(e,Gt,i),t(Gt,jl),u(e,Ea,i),u(e,Re,i),t(Re,Ye),t(Ye,un),k(xt,un,null),t(Re,yl),t(Re,cn),t(cn,Dl),u(e,xa,i),u(e,M,i),t(M,Cl),t(M,Vt),t(Vt,Tl),t(M,Al),t(M,dn),t(dn,Sl),t(M,Ol),t(M,pn),t(pn,Nl),t(M,Il),t(M,mn),t(mn,Pl),t(M,Ml),t(M,fn),t(fn,Rl),t(M,Ll),t(M,kt),t(kt,Hl),t(M,Fl),t(M,hn),t(hn,zl),t(M,Ul),t(M,wt),t(wt,Gl),t(M,Vl),t(M,_n),t(_n,Yl),t(M,Bl),t(M,gn),t(gn,Wl),t(M,Jl),t(M,vn),t(vn,Ql),t(M,Xl),t(M,bn),t(bn,Kl),t(M,Zl),u(e,ka,i),It[ve].m(e,i),u(e,Yt,i),u(e,ne,i),t(ne,ei),t(ne,$n),t($n,ti),t(ne,si),t(ne,qn),t(qn,ni),t(ne,ai),t(ne,En),t(En,oi),t(ne,ri),t(ne,xn),t(xn,li),t(ne,ii),u(e,wa,i),k(jt,e,i),u(e,ja,i),u(e,Bt,i),t(Bt,ui),u(e,ya,i),Pt[$e].m(e,i),u(e,Wt,i),u(e,Be,i),t(Be,ci),t(Be,kn),t(kn,di),t(Be,pi),u(e,Da,i),u(e,Le,i),t(Le,We),t(We,wn),k(yt,wn,null),t(Le,mi),t(Le,jn),t(jn,fi),u(e,Ca,i),u(e,pe,i),t(pe,hi),t(pe,yn),t(yn,_i),t(pe,gi),t(pe,Dn),t(Dn,vi),t(pe,bi),t(pe,Dt),t(Dt,$i),t(pe,qi),u(e,Ta,i),u(e,me,i),t(me,Ei),t(me,Cn),t(Cn,xi),t(me,ki),t(me,Tn),t(Tn,wi),t(me,ji),t(me,An),t(An,yi),t(me,Di),u(e,Aa,i),k(Ct,e,i),u(e,Sa,i),u(e,Je,i),t(Je,Ci),t(Je,Sn),t(Sn,Ti),t(Je,Ai),u(e,Oa,i),Mt[Ee].m(e,i),u(e,Jt,i),u(e,Qt,i),t(Qt,Si),u(e,Na,i),k(Tt,e,i),u(e,Ia,i),u(e,fe,i),t(fe,Oi),t(fe,On),t(On,Ni),t(fe,Ii),t(fe,Nn),t(Nn,Pi),t(fe,Mi),t(fe,In),t(In,Ri),t(fe,Li),u(e,Pa,i),k(At,e,i),u(e,Ma,i),u(e,Xt,i),t(Xt,Hi),u(e,Ra,i),k(St,e,i),u(e,La,i),k(Ot,e,i),u(e,Ha,i),u(e,Kt,i),t(Kt,Fi),u(e,Fa,i),k(Qe,e,i),za=!0},p(e,[i]){const Rt={};i&1&&(Rt.fw=e[0]),f.$set(Rt);let Zt=T;T=Vi(e),T!==Zt&&(so(),v(Nt[Zt],1,1,()=>{Nt[Zt]=null}),to(),N=Nt[T],N||(N=Nt[T]=Gi[T](e),N.c()),b(N,1),N.m(I.parentNode,I));const Pn={};i&2&&(Pn.$$scope={dirty:i,ctx:e}),Ge.$set(Pn);let es=ve;ve=Bi(e),ve!==es&&(so(),v(It[es],1,1,()=>{It[es]=null}),to(),be=It[ve],be||(be=It[ve]=Yi[ve](e),be.c()),b(be,1),be.m(Yt.parentNode,Yt));let ye=$e;$e=Ji(e),$e!==ye&&(so(),v(Pt[ye],1,1,()=>{Pt[ye]=null}),to(),qe=Pt[$e],qe||(qe=Pt[$e]=Wi[$e](e),qe.c()),b(qe,1),qe.m(Wt.parentNode,Wt));let ts=Ee;Ee=Xi(e),Ee!==ts&&(so(),v(Mt[ts],1,1,()=>{Mt[ts]=null}),to(),xe=Mt[Ee],xe||(xe=Mt[Ee]=Qi[Ee](e),xe.c()),b(xe,1),xe.m(Jt.parentNode,Jt));const Mn={};i&2&&(Mn.$$scope={dirty:i,ctx:e}),Qe.$set(Mn)},i(e){za||(b(f.$$.fragment,e),b(g.$$.fragment,e),b(N),b(h.$$.fragment,e),b(oe.$$.fragment,e),b(at.$$.fragment,e),b(ot.$$.fragment,e),b(rt.$$.fragment,e),b(lt.$$.fragment,e),b(it.$$.fragment,e),b(ut.$$.fragment,e),b(ct.$$.fragment,e),b(dt.$$.fragment,e),b(mt.$$.fragment,e),b(ft.$$.fragment,e),b(ht.$$.fragment,e),b(_t.$$.fragment,e),b(gt.$$.fragment,e),b(vt.$$.fragment,e),b(Ge.$$.fragment,e),b(bt.$$.fragment,e),b($t.$$.fragment,e),b(qt.$$.fragment,e),b(Et.$$.fragment,e),b(xt.$$.fragment,e),b(be),b(jt.$$.fragment,e),b(qe),b(yt.$$.fragment,e),b(Ct.$$.fragment,e),b(xe),b(Tt.$$.fragment,e),b(At.$$.fragment,e),b(St.$$.fragment,e),b(Ot.$$.fragment,e),b(Qe.$$.fragment,e),za=!0)},o(e){v(f.$$.fragment,e),v(g.$$.fragment,e),v(N),v(h.$$.fragment,e),v(oe.$$.fragment,e),v(at.$$.fragment,e),v(ot.$$.fragment,e),v(rt.$$.fragment,e),v(lt.$$.fragment,e),v(it.$$.fragment,e),v(ut.$$.fragment,e),v(ct.$$.fragment,e),v(dt.$$.fragment,e),v(mt.$$.fragment,e),v(ft.$$.fragment,e),v(ht.$$.fragment,e),v(_t.$$.fragment,e),v(gt.$$.fragment,e),v(vt.$$.fragment,e),v(Ge.$$.fragment,e),v(bt.$$.fragment,e),v($t.$$.fragment,e),v(qt.$$.fragment,e),v(Et.$$.fragment,e),v(xt.$$.fragment,e),v(be),v(jt.$$.fragment,e),v(qe),v(yt.$$.fragment,e),v(Ct.$$.fragment,e),v(xe),v(Tt.$$.fragment,e),v(At.$$.fragment,e),v(St.$$.fragment,e),v(Ot.$$.fragment,e),v(Qe.$$.fragment,e),za=!1},d(e){s(m),e&&s(q),w(f,e),e&&s(y),e&&s(_),w(g),e&&s(O),Nt[T].d(e),e&&s(I),e&&s(R),e&&s(c),w(h,e),e&&s(L),e&&s(W),w(oe),e&&s(Ln),e&&s(te),e&&s(Hn),e&&s(Ht),e&&s(Fn),e&&s(Pe),e&&s(zn),e&&s(Me),w(at),e&&s(Un),e&&s(ke),e&&s(Gn),w(ot,e),e&&s(Vn),e&&s(we),e&&s(Yn),w(rt,e),e&&s(Bn),w(lt,e),e&&s(Wn),e&&s(X),e&&s(Jn),w(it,e),e&&s(Qn),w(ut,e),e&&s(Xn),e&&s(Z),e&&s(Kn),w(ct,e),e&&s(Zn),w(dt,e),e&&s(ea),e&&s(ee),e&&s(ta),w(mt,e),e&&s(sa),e&&s(ze),e&&s(na),w(ft,e),e&&s(aa),w(ht,e),e&&s(oa),e&&s(Ue),e&&s(ra),w(_t,e),e&&s(la),e&&s(re),e&&s(ua),e&&s(de),e&&s(ca),w(gt,e),e&&s(da),w(vt,e),e&&s(pa),e&&s(zt),e&&s(ma),w(Ge,e),e&&s(fa),e&&s(Ve),e&&s(ha),w(bt,e),e&&s(_a),e&&s(Ut),e&&s(ga),w($t,e),e&&s(va),w(qt,e),e&&s(ba),e&&s(je),e&&s($a),w(Et,e),e&&s(qa),e&&s(Gt),e&&s(Ea),e&&s(Re),w(xt),e&&s(xa),e&&s(M),e&&s(ka),It[ve].d(e),e&&s(Yt),e&&s(ne),e&&s(wa),w(jt,e),e&&s(ja),e&&s(Bt),e&&s(ya),Pt[$e].d(e),e&&s(Wt),e&&s(Be),e&&s(Da),e&&s(Le),w(yt),e&&s(Ca),e&&s(pe),e&&s(Ta),e&&s(me),e&&s(Aa),w(Ct,e),e&&s(Sa),e&&s(Je),e&&s(Oa),Mt[Ee].d(e),e&&s(Jt),e&&s(Qt),e&&s(Na),w(Tt,e),e&&s(Ia),e&&s(fe),e&&s(Pa),w(At,e),e&&s(Ma),e&&s(Xt),e&&s(Ra),w(St,e),e&&s(La),w(Ot,e),e&&s(Ha),e&&s(Kt),e&&s(Fa),w(Qe,e)}}}const _d={local:"recherche-smantique-avec-faiss",sections:[{local:"utilisation-des-enchssements-pour-la-recherche-smantique",title:"Utilisation des ench\xE2ssements pour la recherche s\xE9mantique"},{local:"chargement-et-prparation-du-jeu-de-donnes",title:"Chargement et pr\xE9paration du jeu de donn\xE9es"},{local:"cration-denchssements-pour-les-textes",title:"Cr\xE9ation d\u2019ench\xE2ssements pour les textes"},{local:"utilisation-de-faiss-pour-une-recherche-de-similarit-efficace",title:"Utilisation de FAISS pour une recherche de similarit\xE9 efficace"}],title:"Recherche s\xE9mantique avec FAISS"};function gd(G,m,q){let f="pt";return sd(()=>{const y=new URLSearchParams(window.location.search);q(0,f=y.get("fw")||"pt")}),[f]}class wd extends Kc{constructor(m){super();Zc(this,m,gd,hd,ed,{})}}export{wd as default,_d as metadata};
