import{S as pi,i as mi,s as ui,e as o,k as m,w as x,t as a,M as di,c as r,d as e,m as u,x as k,a as i,h as n,b as q,N as li,f as Xa,G as s,g as p,y as v,o as b,p as Ka,q as $,B as j,v as fi,n as Za}from"../../chunks/vendor-hf-doc-builder.js";import{T as oi}from"../../chunks/Tip-hf-doc-builder.js";import{Y as ci}from"../../chunks/Youtube-hf-doc-builder.js";import{I as ri}from"../../chunks/IconCopyLink-hf-doc-builder.js";import{C as T}from"../../chunks/CodeBlock-hf-doc-builder.js";import{C as ii}from"../../chunks/CourseFloatingBanner-hf-doc-builder.js";import{F as _i}from"../../chunks/FrameworkSwitchCourse-hf-doc-builder.js";import"../../chunks/DocNotebookDropdown-hf-doc-builder.js";function hi(R){let f,g;return f=new ii({props:{chapter:5,classNames:"absolute z-10 right-0 top-0",notebooks:[{label:"Google Colab",value:"https://colab.research.google.com/github/huggingface/notebooks/blob/master/course/zh-CN/chapter5/section6_tf.ipynb"},{label:"Aws Studio",value:"https://studiolab.sagemaker.aws/import/github/huggingface/notebooks/blob/master/course/zh-CN/chapter5/section6_tf.ipynb"}]}}),{c(){x(f.$$.fragment)},l(c){k(f.$$.fragment,c)},m(c,y){v(f,c,y),g=!0},i(c){g||($(f.$$.fragment,c),g=!0)},o(c){b(f.$$.fragment,c),g=!1},d(c){j(f,c)}}}function gi(R){let f,g;return f=new ii({props:{chapter:5,classNames:"absolute z-10 right-0 top-0",notebooks:[{label:"Google Colab",value:"https://colab.research.google.com/github/huggingface/notebooks/blob/master/course/zh-CN/chapter5/section6_pt.ipynb"},{label:"Aws Studio",value:"https://studiolab.sagemaker.aws/import/github/huggingface/notebooks/blob/master/course/zh-CN/chapter5/section6_pt.ipynb"}]}}),{c(){x(f.$$.fragment)},l(c){k(f.$$.fragment,c)},m(c,y){v(f,c,y),g=!0},i(c){g||($(f.$$.fragment,c),g=!0)},o(c){b(f.$$.fragment,c),g=!1},d(c){j(f,c)}}}function bi(R){let f,g,c,y,_,h,D,w;return{c(){f=o("p"),g=a("\u270F\uFE0F "),c=o("strong"),y=a("Try it out!"),_=a(" \u770B\u770B\u80FD\u4E0D\u80FD\u4E0D\u7528pandas\u5C31\u53EF\u4EE5\u5B8C\u6210\u5217\u7684\u6269\u5145\uFF1B \u8FD9\u6709\u70B9\u68D8\u624B\uFF1B \u4F60\u53EF\u80FD\u4F1A\u53D1\u73B0 \u{1F917} Datasets \u6587\u6863\u7684 "),h=o("a"),D=a("\u201CBatch mapping\u201D"),w=a(" \u5BF9\u8FD9\u4E2A\u4EFB\u52A1\u5F88\u6709\u7528\u3002"),this.h()},l(A){f=r(A,"P",{});var E=i(f);g=n(E,"\u270F\uFE0F "),c=r(E,"STRONG",{});var z=i(c);y=n(z,"Try it out!"),z.forEach(e),_=n(E," \u770B\u770B\u80FD\u4E0D\u80FD\u4E0D\u7528pandas\u5C31\u53EF\u4EE5\u5B8C\u6210\u5217\u7684\u6269\u5145\uFF1B \u8FD9\u6709\u70B9\u68D8\u624B\uFF1B \u4F60\u53EF\u80FD\u4F1A\u53D1\u73B0 \u{1F917} Datasets \u6587\u6863\u7684 "),h=r(E,"A",{href:!0,rel:!0});var N=i(h);D=n(N,"\u201CBatch mapping\u201D"),N.forEach(e),w=n(E," \u5BF9\u8FD9\u4E2A\u4EFB\u52A1\u5F88\u6709\u7528\u3002"),E.forEach(e),this.h()},h(){q(h,"href","https://huggingface.co/docs/datasets/v1.12.1/about_map_batch.html?batch-mapping#batch-mapping"),q(h,"rel","nofollow")},m(A,E){p(A,f,E),s(f,g),s(f,c),s(c,y),s(f,_),s(f,h),s(h,D),s(f,w)},d(A){A&&e(f)}}}function $i(R){let f,g,c,y,_;return f=new T({props:{code:`from transformers import AutoTokenizer, TFAutoModel

model_ckpt = "sentence-transformers/multi-qa-mpnet-base-dot-v1"
tokenizer = AutoTokenizer.from_pretrained(model_ckpt)
model = TFAutoModel.from_pretrained(model_ckpt, from_pt=True)`,highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoTokenizer, TFAutoModel

model_ckpt = <span class="hljs-string">&quot;sentence-transformers/multi-qa-mpnet-base-dot-v1&quot;</span>
tokenizer = AutoTokenizer.from_pretrained(model_ckpt)
model = TFAutoModel.from_pretrained(model_ckpt, from_pt=<span class="hljs-literal">True</span>)`}}),{c(){x(f.$$.fragment),g=m(),c=o("p"),y=a("\u8BF7\u6CE8\u610F\uFF0C\u6211\u4EEC\u5DF2\u5C06 from_pt=True \u8BBE\u7F6E\u4E3A  from_pretrained\uFF08\uFF09 \u65B9\u6CD5\u7684\u53C2\u6570\u3002\u8FD9\u662F\u56E0\u4E3A multi-qa-mpnet-base-dot-v1 \u68C0\u67E5\u70B9\u53EA\u6709PyTorch\u6743\u91CD\uFF0C\u56E0\u6B64\u8BBE\u7F6E from_pt=True \u4F1A\u81EA\u52A8\u5C06\u5B83\u4EEC\u8F6C\u6362\u4E3ATensorFlow\u683C\u5F0F\u3002\u5982\u60A8\u6240\u89C1\uFF0C\u5728Transformers\u4E2D\u7684\u{1F917}\u6846\u67B6\u4E4B\u95F4\u5207\u6362\u975E\u5E38\u7B80\u5355\uFF01")},l(h){k(f.$$.fragment,h),g=u(h),c=r(h,"P",{});var D=i(c);y=n(D,"\u8BF7\u6CE8\u610F\uFF0C\u6211\u4EEC\u5DF2\u5C06 from_pt=True \u8BBE\u7F6E\u4E3A  from_pretrained\uFF08\uFF09 \u65B9\u6CD5\u7684\u53C2\u6570\u3002\u8FD9\u662F\u56E0\u4E3A multi-qa-mpnet-base-dot-v1 \u68C0\u67E5\u70B9\u53EA\u6709PyTorch\u6743\u91CD\uFF0C\u56E0\u6B64\u8BBE\u7F6E from_pt=True \u4F1A\u81EA\u52A8\u5C06\u5B83\u4EEC\u8F6C\u6362\u4E3ATensorFlow\u683C\u5F0F\u3002\u5982\u60A8\u6240\u89C1\uFF0C\u5728Transformers\u4E2D\u7684\u{1F917}\u6846\u67B6\u4E4B\u95F4\u5207\u6362\u975E\u5E38\u7B80\u5355\uFF01"),D.forEach(e)},m(h,D){v(f,h,D),p(h,g,D),p(h,c,D),s(c,y),_=!0},i(h){_||($(f.$$.fragment,h),_=!0)},o(h){b(f.$$.fragment,h),_=!1},d(h){j(f,h),h&&e(g),h&&e(c)}}}function wi(R){let f,g,c,y,_,h,D;return f=new T({props:{code:`from transformers import AutoTokenizer, AutoModel

model_ckpt = "sentence-transformers/multi-qa-mpnet-base-dot-v1"
tokenizer = AutoTokenizer.from_pretrained(model_ckpt)
model = AutoModel.from_pretrained(model_ckpt)`,highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoTokenizer, AutoModel

model_ckpt = <span class="hljs-string">&quot;sentence-transformers/multi-qa-mpnet-base-dot-v1&quot;</span>
tokenizer = AutoTokenizer.from_pretrained(model_ckpt)
model = AutoModel.from_pretrained(model_ckpt)`}}),h=new T({props:{code:`import torch

device = torch.device("cuda")
model.to(device)`,highlighted:`<span class="hljs-keyword">import</span> torch

device = torch.device(<span class="hljs-string">&quot;cuda&quot;</span>)
model.to(device)`}}),{c(){x(f.$$.fragment),g=m(),c=o("p"),y=a("\u4E3A\u4E86\u52A0\u5FEB\u5D4C\u5165\u8FC7\u7A0B\uFF0C\u5C06\u6A21\u578B\u548C\u8F93\u5165\u653E\u5728 GPU \u8BBE\u5907\u4E0A\uFF0C\u6240\u4EE5\u73B0\u5728\u8BA9\u6211\u4EEC\u8FD9\u6837\u505A\uFF1A"),_=m(),x(h.$$.fragment)},l(w){k(f.$$.fragment,w),g=u(w),c=r(w,"P",{});var A=i(c);y=n(A,"\u4E3A\u4E86\u52A0\u5FEB\u5D4C\u5165\u8FC7\u7A0B\uFF0C\u5C06\u6A21\u578B\u548C\u8F93\u5165\u653E\u5728 GPU \u8BBE\u5907\u4E0A\uFF0C\u6240\u4EE5\u73B0\u5728\u8BA9\u6211\u4EEC\u8FD9\u6837\u505A\uFF1A"),A.forEach(e),_=u(w),k(h.$$.fragment,w)},m(w,A){v(f,w,A),p(w,g,A),p(w,c,A),s(c,y),p(w,_,A),v(h,w,A),D=!0},i(w){D||($(f.$$.fragment,w),$(h.$$.fragment,w),D=!0)},o(w){b(f.$$.fragment,w),b(h.$$.fragment,w),D=!1},d(w){j(f,w),w&&e(g),w&&e(c),w&&e(_),j(h,w)}}}function xi(R){let f,g,c,y,_,h,D,w,A,E,z,N,O,P,F,M,nt,H,lt,ot,U,I,Y;return f=new T({props:{code:`def get_embeddings(text_list):
    encoded_input = tokenizer(
        text_list, padding=True, truncation=True, return_tensors="tf"
    )
    encoded_input = {k: v for k, v in encoded_input.items()}
    model_output = model(**encoded_input)
    return cls_pooling(model_output)`,highlighted:`<span class="hljs-keyword">def</span> <span class="hljs-title function_">get_embeddings</span>(<span class="hljs-params">text_list</span>):
    encoded_input = tokenizer(
        text_list, padding=<span class="hljs-literal">True</span>, truncation=<span class="hljs-literal">True</span>, return_tensors=<span class="hljs-string">&quot;tf&quot;</span>
    )
    encoded_input = {k: v <span class="hljs-keyword">for</span> k, v <span class="hljs-keyword">in</span> encoded_input.items()}
    model_output = model(**encoded_input)
    <span class="hljs-keyword">return</span> cls_pooling(model_output)`}}),h=new T({props:{code:`embedding = get_embeddings(comments_dataset["text"][0])
embedding.shape`,highlighted:`embedding = get_embeddings(comments_dataset[<span class="hljs-string">&quot;text&quot;</span>][<span class="hljs-number">0</span>])
embedding.shape`}}),w=new T({props:{code:"TensorShape([1, 768])",highlighted:'TensorShape([<span class="hljs-number">1</span>, <span class="hljs-number">768</span>])'}}),I=new T({props:{code:`embeddings_dataset = comments_dataset.map(
    lambda x: {"embeddings": get_embeddings(x["text"]).numpy()[0]}
)`,highlighted:`embeddings_dataset = comments_dataset.<span class="hljs-built_in">map</span>(
    <span class="hljs-keyword">lambda</span> x: {<span class="hljs-string">&quot;embeddings&quot;</span>: get_embeddings(x[<span class="hljs-string">&quot;text&quot;</span>]).numpy()[<span class="hljs-number">0</span>]}
)`}}),{c(){x(f.$$.fragment),g=m(),c=o("p"),y=a("\u6211\u4EEC\u53EF\u4EE5\u901A\u8FC7\u5728\u6211\u4EEC\u7684\u8BED\u6599\u5E93\u4E2D\u8F93\u5165\u7B2C\u4E00\u4E2A\u6587\u672C\u6761\u76EE\u5E76\u68C0\u67E5\u8F93\u51FA\u7EF4\u5EA6\u6765\u6D4B\u8BD5\u8BE5\u51FD\u6570\u662F\u5426\u6709\u6548\uFF1A"),_=m(),x(h.$$.fragment),D=m(),x(w.$$.fragment),A=m(),E=o("p"),z=a("\u592A\u597D\u4E86\uFF0C\u6211\u4EEC\u5DF2\u7ECF\u5C06\u8BED\u6599\u5E93\u4E2D\u7684\u7B2C\u4E00\u4E2A\u6761\u76EE\u8F6C\u6362\u4E3A 768 \u7EF4\u5411\u91CF\uFF01\u6211\u4EEC\u53EF\u4EE5\u7528 "),N=o("strong"),O=a("Dataset.map()"),P=a(" \u5E94\u7528\u6211\u4EEC\u7684 "),F=o("strong"),M=a("get_embeddings()"),nt=a(" \u51FD\u6570\u5230\u6211\u4EEC\u8BED\u6599\u5E93\u4E2D\u7684\u6BCF\u4E00\u884C\uFF0C\u6240\u4EE5\u8BA9\u6211\u4EEC\u521B\u5EFA\u4E00\u4E2A\u65B0\u7684 "),H=o("strong"),lt=a("embeddings"),ot=a(" \u5217\u5982\u4E0B\uFF1A"),U=m(),x(I.$$.fragment)},l(d){k(f.$$.fragment,d),g=u(d),c=r(d,"P",{});var S=i(c);y=n(S,"\u6211\u4EEC\u53EF\u4EE5\u901A\u8FC7\u5728\u6211\u4EEC\u7684\u8BED\u6599\u5E93\u4E2D\u8F93\u5165\u7B2C\u4E00\u4E2A\u6587\u672C\u6761\u76EE\u5E76\u68C0\u67E5\u8F93\u51FA\u7EF4\u5EA6\u6765\u6D4B\u8BD5\u8BE5\u51FD\u6570\u662F\u5426\u6709\u6548\uFF1A"),S.forEach(e),_=u(d),k(h.$$.fragment,d),D=u(d),k(w.$$.fragment,d),A=u(d),E=r(d,"P",{});var C=i(E);z=n(C,"\u592A\u597D\u4E86\uFF0C\u6211\u4EEC\u5DF2\u7ECF\u5C06\u8BED\u6599\u5E93\u4E2D\u7684\u7B2C\u4E00\u4E2A\u6761\u76EE\u8F6C\u6362\u4E3A 768 \u7EF4\u5411\u91CF\uFF01\u6211\u4EEC\u53EF\u4EE5\u7528 "),N=r(C,"STRONG",{});var B=i(N);O=n(B,"Dataset.map()"),B.forEach(e),P=n(C," \u5E94\u7528\u6211\u4EEC\u7684 "),F=r(C,"STRONG",{});var vt=i(F);M=n(vt,"get_embeddings()"),vt.forEach(e),nt=n(C," \u51FD\u6570\u5230\u6211\u4EEC\u8BED\u6599\u5E93\u4E2D\u7684\u6BCF\u4E00\u884C\uFF0C\u6240\u4EE5\u8BA9\u6211\u4EEC\u521B\u5EFA\u4E00\u4E2A\u65B0\u7684 "),H=r(C,"STRONG",{});var st=i(H);lt=n(st,"embeddings"),st.forEach(e),ot=n(C," \u5217\u5982\u4E0B\uFF1A"),C.forEach(e),U=u(d),k(I.$$.fragment,d)},m(d,S){v(f,d,S),p(d,g,S),p(d,c,S),s(c,y),p(d,_,S),v(h,d,S),p(d,D,S),v(w,d,S),p(d,A,S),p(d,E,S),s(E,z),s(E,N),s(N,O),s(E,P),s(E,F),s(F,M),s(E,nt),s(E,H),s(H,lt),s(E,ot),p(d,U,S),v(I,d,S),Y=!0},i(d){Y||($(f.$$.fragment,d),$(h.$$.fragment,d),$(w.$$.fragment,d),$(I.$$.fragment,d),Y=!0)},o(d){b(f.$$.fragment,d),b(h.$$.fragment,d),b(w.$$.fragment,d),b(I.$$.fragment,d),Y=!1},d(d){j(f,d),d&&e(g),d&&e(c),d&&e(_),j(h,d),d&&e(D),j(w,d),d&&e(A),d&&e(E),d&&e(U),j(I,d)}}}function ki(R){let f,g,c,y,_,h,D,w,A,E,z,N,O,P,F,M,nt,H,lt,ot,U,I,Y;return f=new T({props:{code:`def get_embeddings(text_list):
    encoded_input = tokenizer(
        text_list, padding=True, truncation=True, return_tensors="pt"
    )
    encoded_input = {k: v.to(device) for k, v in encoded_input.items()}
    model_output = model(**encoded_input)
    return cls_pooling(model_output)`,highlighted:`<span class="hljs-keyword">def</span> <span class="hljs-title function_">get_embeddings</span>(<span class="hljs-params">text_list</span>):
    encoded_input = tokenizer(
        text_list, padding=<span class="hljs-literal">True</span>, truncation=<span class="hljs-literal">True</span>, return_tensors=<span class="hljs-string">&quot;pt&quot;</span>
    )
    encoded_input = {k: v.to(device) <span class="hljs-keyword">for</span> k, v <span class="hljs-keyword">in</span> encoded_input.items()}
    model_output = model(**encoded_input)
    <span class="hljs-keyword">return</span> cls_pooling(model_output)`}}),h=new T({props:{code:`embedding = get_embeddings(comments_dataset["text"][0])
embedding.shape`,highlighted:`embedding = get_embeddings(comments_dataset[<span class="hljs-string">&quot;text&quot;</span>][<span class="hljs-number">0</span>])
embedding.shape`}}),w=new T({props:{code:"torch.Size([1, 768])",highlighted:'torch.Size([<span class="hljs-number">1</span>, <span class="hljs-number">768</span>])'}}),I=new T({props:{code:`embeddings_dataset = comments_dataset.map(
    lambda x: {"embeddings": get_embeddings(x["text"]).detach().cpu().numpy()[0]}
)`,highlighted:`embeddings_dataset = comments_dataset.<span class="hljs-built_in">map</span>(
    <span class="hljs-keyword">lambda</span> x: {<span class="hljs-string">&quot;embeddings&quot;</span>: get_embeddings(x[<span class="hljs-string">&quot;text&quot;</span>]).detach().cpu().numpy()[<span class="hljs-number">0</span>]}
)`}}),{c(){x(f.$$.fragment),g=m(),c=o("p"),y=a("\u6211\u4EEC\u53EF\u4EE5\u901A\u8FC7\u5728\u6211\u4EEC\u7684\u8BED\u6599\u5E93\u4E2D\u8F93\u5165\u7B2C\u4E00\u4E2A\u6587\u672C\u6761\u76EE\u5E76\u68C0\u67E5\u8F93\u51FA\u7EF4\u5EA6\u6765\u6D4B\u8BD5\u8BE5\u51FD\u6570\u662F\u5426\u6709\u6548\uFF1A"),_=m(),x(h.$$.fragment),D=m(),x(w.$$.fragment),A=m(),E=o("p"),z=a("\u592A\u597D\u4E86\uFF0C\u6211\u4EEC\u5DF2\u7ECF\u5C06\u8BED\u6599\u5E93\u4E2D\u7684\u7B2C\u4E00\u4E2A\u6761\u76EE\u8F6C\u6362\u4E3A 768 \u7EF4\u5411\u91CF\uFF01\u6211\u4EEC\u53EF\u4EE5\u7528 "),N=o("strong"),O=a("Dataset.map()"),P=a(" \u5E94\u7528\u6211\u4EEC\u7684 "),F=o("strong"),M=a("get_embeddings()"),nt=a(" \u51FD\u6570\u5230\u6211\u4EEC\u8BED\u6599\u5E93\u4E2D\u7684\u6BCF\u4E00\u884C\uFF0C\u6240\u4EE5\u8BA9\u6211\u4EEC\u521B\u5EFA\u4E00\u4E2A\u65B0\u7684 "),H=o("strong"),lt=a("embeddings"),ot=a(" \u5217\u5982\u4E0B\uFF1A"),U=m(),x(I.$$.fragment)},l(d){k(f.$$.fragment,d),g=u(d),c=r(d,"P",{});var S=i(c);y=n(S,"\u6211\u4EEC\u53EF\u4EE5\u901A\u8FC7\u5728\u6211\u4EEC\u7684\u8BED\u6599\u5E93\u4E2D\u8F93\u5165\u7B2C\u4E00\u4E2A\u6587\u672C\u6761\u76EE\u5E76\u68C0\u67E5\u8F93\u51FA\u7EF4\u5EA6\u6765\u6D4B\u8BD5\u8BE5\u51FD\u6570\u662F\u5426\u6709\u6548\uFF1A"),S.forEach(e),_=u(d),k(h.$$.fragment,d),D=u(d),k(w.$$.fragment,d),A=u(d),E=r(d,"P",{});var C=i(E);z=n(C,"\u592A\u597D\u4E86\uFF0C\u6211\u4EEC\u5DF2\u7ECF\u5C06\u8BED\u6599\u5E93\u4E2D\u7684\u7B2C\u4E00\u4E2A\u6761\u76EE\u8F6C\u6362\u4E3A 768 \u7EF4\u5411\u91CF\uFF01\u6211\u4EEC\u53EF\u4EE5\u7528 "),N=r(C,"STRONG",{});var B=i(N);O=n(B,"Dataset.map()"),B.forEach(e),P=n(C," \u5E94\u7528\u6211\u4EEC\u7684 "),F=r(C,"STRONG",{});var vt=i(F);M=n(vt,"get_embeddings()"),vt.forEach(e),nt=n(C," \u51FD\u6570\u5230\u6211\u4EEC\u8BED\u6599\u5E93\u4E2D\u7684\u6BCF\u4E00\u884C\uFF0C\u6240\u4EE5\u8BA9\u6211\u4EEC\u521B\u5EFA\u4E00\u4E2A\u65B0\u7684 "),H=r(C,"STRONG",{});var st=i(H);lt=n(st,"embeddings"),st.forEach(e),ot=n(C," \u5217\u5982\u4E0B\uFF1A"),C.forEach(e),U=u(d),k(I.$$.fragment,d)},m(d,S){v(f,d,S),p(d,g,S),p(d,c,S),s(c,y),p(d,_,S),v(h,d,S),p(d,D,S),v(w,d,S),p(d,A,S),p(d,E,S),s(E,z),s(E,N),s(N,O),s(E,P),s(E,F),s(F,M),s(E,nt),s(E,H),s(H,lt),s(E,ot),p(d,U,S),v(I,d,S),Y=!0},i(d){Y||($(f.$$.fragment,d),$(h.$$.fragment,d),$(w.$$.fragment,d),$(I.$$.fragment,d),Y=!0)},o(d){b(f.$$.fragment,d),b(h.$$.fragment,d),b(w.$$.fragment,d),b(I.$$.fragment,d),Y=!1},d(d){j(f,d),d&&e(g),d&&e(c),d&&e(_),j(h,d),d&&e(D),j(w,d),d&&e(A),d&&e(E),d&&e(U),j(I,d)}}}function vi(R){let f,g,c,y;return f=new T({props:{code:`question = "How can I load a dataset offline?"
question_embedding = get_embeddings([question]).numpy()
question_embedding.shape`,highlighted:`question = <span class="hljs-string">&quot;How can I load a dataset offline?&quot;</span>
question_embedding = get_embeddings([question]).numpy()
question_embedding.shape`}}),c=new T({props:{code:"(1, 768)",highlighted:'(<span class="hljs-number">1</span>, <span class="hljs-number">768</span>)'}}),{c(){x(f.$$.fragment),g=m(),x(c.$$.fragment)},l(_){k(f.$$.fragment,_),g=u(_),k(c.$$.fragment,_)},m(_,h){v(f,_,h),p(_,g,h),v(c,_,h),y=!0},i(_){y||($(f.$$.fragment,_),$(c.$$.fragment,_),y=!0)},o(_){b(f.$$.fragment,_),b(c.$$.fragment,_),y=!1},d(_){j(f,_),_&&e(g),j(c,_)}}}function ji(R){let f,g,c,y;return f=new T({props:{code:`question = "How can I load a dataset offline?"
question_embedding = get_embeddings([question]).cpu().detach().numpy()
question_embedding.shape`,highlighted:`question = <span class="hljs-string">&quot;How can I load a dataset offline?&quot;</span>
question_embedding = get_embeddings([question]).cpu().detach().numpy()
question_embedding.shape`}}),c=new T({props:{code:"torch.Size([1, 768])",highlighted:'torch.Size([<span class="hljs-number">1</span>, <span class="hljs-number">768</span>])'}}),{c(){x(f.$$.fragment),g=m(),x(c.$$.fragment)},l(_){k(f.$$.fragment,_),g=u(_),k(c.$$.fragment,_)},m(_,h){v(f,_,h),p(_,g,h),v(c,_,h),y=!0},i(_){y||($(f.$$.fragment,_),$(c.$$.fragment,_),y=!0)},o(_){b(f.$$.fragment,_),b(c.$$.fragment,_),y=!1},d(_){j(f,_),_&&e(g),j(c,_)}}}function yi(R){let f,g;return{c(){f=o("p"),g=a("\u270F\uFE0F  \u8BD5\u8BD5\u770B\uFF01\u521B\u5EFA\u60A8\u81EA\u5DF1\u7684\u67E5\u8BE2\u5E76\u67E5\u770B\u60A8\u662F\u5426\u53EF\u4EE5\u5728\u68C0\u7D22\u5230\u7684\u6587\u6863\u4E2D\u627E\u5230\u7B54\u6848\u3002\u60A8\u53EF\u80FD\u9700\u8981\u589E\u52A0\u53C2\u6570k\u4EE5\u6269\u5927\u641C\u7D22\u8303\u56F4\u3002")},l(c){f=r(c,"P",{});var y=i(f);g=n(y,"\u270F\uFE0F  \u8BD5\u8BD5\u770B\uFF01\u521B\u5EFA\u60A8\u81EA\u5DF1\u7684\u67E5\u8BE2\u5E76\u67E5\u770B\u60A8\u662F\u5426\u53EF\u4EE5\u5728\u68C0\u7D22\u5230\u7684\u6587\u6863\u4E2D\u627E\u5230\u7B54\u6848\u3002\u60A8\u53EF\u80FD\u9700\u8981\u589E\u52A0\u53C2\u6570k\u4EE5\u6269\u5927\u641C\u7D22\u8303\u56F4\u3002"),y.forEach(e)},m(c,y){p(c,f,y),s(f,g)},d(c){c&&e(f)}}}function Ei(R){let f,g,c,y,_,h,D,w,A,E,z,N,O,P,F,M,nt,H,lt,ot,U,I,Y,d,S,C,B,vt,st,tn,sn,Le,jt,Pt,Ao,en,Ft,Ro,ze,Ht,an,Ue,ws,nn,Ye,Gt,Be,ct,ln,Ls,on,rn,xs,pn,mn,We,Mt,Ve,Lt,Je,ks,un,Qe,zt,Xe,Ut,Ke,L,dn,zs,fn,cn,Us,_n,hn,Ys,gn,bn,Bs,$n,wn,Ws,xn,kn,Ze,Yt,ta,Bt,sa,W,vn,Vs,jn,yn,Js,En,Tn,Wt,qn,Sn,Qs,Dn,An,ea,Vt,aa,Tt,Rn,Xs,In,Cn,na,Jt,la,Qt,oa,vs,Nn,ra,Xt,ia,Q,Ks,V,pa,On,Zs,Pn,Fn,te,Hn,Gn,se,Mn,Ln,ee,zn,Un,rt,X,ae,Yn,Bn,ne,Wn,Vn,le,Jn,Qn,oe,Xn,Kn,re,Zn,tl,K,ie,sl,el,pe,al,nl,me,ll,ol,ue,rl,il,de,pl,ml,Z,fe,ul,dl,ce,fl,cl,_e,_l,hl,he,gl,bl,ge,$l,wl,tt,be,xl,kl,$e,vl,jl,we,yl,El,xe,Tl,ql,ke,Sl,ma,et,Dl,ve,Al,Rl,je,Il,Cl,ye,Nl,Ol,ua,Kt,da,Zt,fa,js,Pl,ca,qt,_a,St,Fl,Ee,Hl,Gl,ha,ts,ga,ys,Ml,ba,ss,$a,es,wa,_t,Ll,Te,zl,Ul,qe,Yl,Bl,xa,as,ka,Es,Wl,va,ns,Vl,ja,G,Jl,Ts,Ql,Xl,Se,Kl,Zl,De,to,so,ls,eo,ao,os,no,lo,Ae,oo,ro,ya,it,pt,qs,Dt,io,Re,po,mo,Ea,rs,Ta,Ss,uo,qa,mt,ut,Ds,As,fo,Sa,yt,At,Ie,is,co,Ce,_o,Da,ht,ho,ps,go,bo,Ne,$o,wo,Aa,ms,Ra,Rt,xo,Oe,ko,vo,Ia,dt,ft,Rs,Is,jo,Ca,us,Na,Et,Pe,yo,Eo,Fe,To,qo,Oa,ds,Pa,Cs,So,Fa,fs,Ha,cs,Ga,Ns,Do,Ma,It,La;c=new _i({props:{fw:R[0]}}),w=new ri({});const Io=[gi,hi],_s=[];function Co(t,l){return t[0]==="pt"?0:1}O=Co(R),P=_s[O]=Io[O](R),I=new ci({props:{id:"OATCgQtNX2o"}}),Gt=new T({props:{code:`from huggingface_hub import hf_hub_url

data_files = hf_hub_url(
    repo_id="lewtun/github-issues",
    filename="datasets-issues-with-hf-doc-builder.jsonl",
    repo_type="dataset",
)`,highlighted:`<span class="hljs-keyword">from</span> huggingface_hub <span class="hljs-keyword">import</span> hf_hub_url

data_files = hf_hub_url(
    repo_id=<span class="hljs-string">&quot;lewtun/github-issues&quot;</span>,
    filename=<span class="hljs-string">&quot;datasets-issues-with-hf-doc-builder.jsonl&quot;</span>,
    repo_type=<span class="hljs-string">&quot;dataset&quot;</span>,
)`}}),Mt=new T({props:{code:`from datasets import load_dataset

issues_dataset = load_dataset("json", data_files=data_files, split="train")
issues_dataset`,highlighted:`<span class="hljs-keyword">from</span> datasets <span class="hljs-keyword">import</span> load_dataset

issues_dataset = load_dataset(<span class="hljs-string">&quot;json&quot;</span>, data_files=data_files, split=<span class="hljs-string">&quot;train&quot;</span>)
issues_dataset`}}),Lt=new T({props:{code:`Dataset({
    features: ['url', 'repository_url', 'labels_url', 'comments_url', 'events_url', 'html_url', 'id', 'node_id', 'number', 'title', 'user', 'labels', 'state', 'locked', 'assignee', 'assignees', 'milestone', 'comments', 'created_at', 'updated_at', 'closed_at', 'author_association', 'active_lock_reason', 'pull_request', 'body', 'performed_via_github_app', 'is_pull_request'],
    num_rows: 2855
})`,highlighted:`Dataset({
    features: [<span class="hljs-string">&#x27;url&#x27;</span>, <span class="hljs-string">&#x27;repository_url&#x27;</span>, <span class="hljs-string">&#x27;labels_url&#x27;</span>, <span class="hljs-string">&#x27;comments_url&#x27;</span>, <span class="hljs-string">&#x27;events_url&#x27;</span>, <span class="hljs-string">&#x27;html_url&#x27;</span>, <span class="hljs-string">&#x27;id&#x27;</span>, <span class="hljs-string">&#x27;node_id&#x27;</span>, <span class="hljs-string">&#x27;number&#x27;</span>, <span class="hljs-string">&#x27;title&#x27;</span>, <span class="hljs-string">&#x27;user&#x27;</span>, <span class="hljs-string">&#x27;labels&#x27;</span>, <span class="hljs-string">&#x27;state&#x27;</span>, <span class="hljs-string">&#x27;locked&#x27;</span>, <span class="hljs-string">&#x27;assignee&#x27;</span>, <span class="hljs-string">&#x27;assignees&#x27;</span>, <span class="hljs-string">&#x27;milestone&#x27;</span>, <span class="hljs-string">&#x27;comments&#x27;</span>, <span class="hljs-string">&#x27;created_at&#x27;</span>, <span class="hljs-string">&#x27;updated_at&#x27;</span>, <span class="hljs-string">&#x27;closed_at&#x27;</span>, <span class="hljs-string">&#x27;author_association&#x27;</span>, <span class="hljs-string">&#x27;active_lock_reason&#x27;</span>, <span class="hljs-string">&#x27;pull_request&#x27;</span>, <span class="hljs-string">&#x27;body&#x27;</span>, <span class="hljs-string">&#x27;performed_via_github_app&#x27;</span>, <span class="hljs-string">&#x27;is_pull_request&#x27;</span>],
    num_rows: <span class="hljs-number">2855</span>
})`}}),zt=new T({props:{code:`issues_dataset = issues_dataset.filter(
    lambda x: (x["is_pull_request"] == False and len(x["comments"]) > 0)
)
issues_dataset`,highlighted:`issues_dataset = issues_dataset.<span class="hljs-built_in">filter</span>(
    <span class="hljs-keyword">lambda</span> x: (x[<span class="hljs-string">&quot;is_pull_request&quot;</span>] == <span class="hljs-literal">False</span> <span class="hljs-keyword">and</span> <span class="hljs-built_in">len</span>(x[<span class="hljs-string">&quot;comments&quot;</span>]) &gt; <span class="hljs-number">0</span>)
)
issues_dataset`}}),Ut=new T({props:{code:`Dataset({
    features: ['url', 'repository_url', 'labels_url', 'comments_url', 'events_url', 'html_url', 'id', 'node_id', 'number', 'title', 'user', 'labels', 'state', 'locked', 'assignee', 'assignees', 'milestone', 'comments', 'created_at', 'updated_at', 'closed_at', 'author_association', 'active_lock_reason', 'pull_request', 'body', 'performed_via_github_app', 'is_pull_request'],
    num_rows: 771
})`,highlighted:`Dataset({
    features: [<span class="hljs-string">&#x27;url&#x27;</span>, <span class="hljs-string">&#x27;repository_url&#x27;</span>, <span class="hljs-string">&#x27;labels_url&#x27;</span>, <span class="hljs-string">&#x27;comments_url&#x27;</span>, <span class="hljs-string">&#x27;events_url&#x27;</span>, <span class="hljs-string">&#x27;html_url&#x27;</span>, <span class="hljs-string">&#x27;id&#x27;</span>, <span class="hljs-string">&#x27;node_id&#x27;</span>, <span class="hljs-string">&#x27;number&#x27;</span>, <span class="hljs-string">&#x27;title&#x27;</span>, <span class="hljs-string">&#x27;user&#x27;</span>, <span class="hljs-string">&#x27;labels&#x27;</span>, <span class="hljs-string">&#x27;state&#x27;</span>, <span class="hljs-string">&#x27;locked&#x27;</span>, <span class="hljs-string">&#x27;assignee&#x27;</span>, <span class="hljs-string">&#x27;assignees&#x27;</span>, <span class="hljs-string">&#x27;milestone&#x27;</span>, <span class="hljs-string">&#x27;comments&#x27;</span>, <span class="hljs-string">&#x27;created_at&#x27;</span>, <span class="hljs-string">&#x27;updated_at&#x27;</span>, <span class="hljs-string">&#x27;closed_at&#x27;</span>, <span class="hljs-string">&#x27;author_association&#x27;</span>, <span class="hljs-string">&#x27;active_lock_reason&#x27;</span>, <span class="hljs-string">&#x27;pull_request&#x27;</span>, <span class="hljs-string">&#x27;body&#x27;</span>, <span class="hljs-string">&#x27;performed_via_github_app&#x27;</span>, <span class="hljs-string">&#x27;is_pull_request&#x27;</span>],
    num_rows: <span class="hljs-number">771</span>
})`}}),Yt=new T({props:{code:`columns = issues_dataset.column_names
columns_to_keep = ["title", "body", "html_url", "comments"]
columns_to_remove = set(columns_to_keep).symmetric_difference(columns)
issues_dataset = issues_dataset.remove_columns(columns_to_remove)
issues_dataset`,highlighted:`columns = issues_dataset.column_names
columns_to_keep = [<span class="hljs-string">&quot;title&quot;</span>, <span class="hljs-string">&quot;body&quot;</span>, <span class="hljs-string">&quot;html_url&quot;</span>, <span class="hljs-string">&quot;comments&quot;</span>]
columns_to_remove = <span class="hljs-built_in">set</span>(columns_to_keep).symmetric_difference(columns)
issues_dataset = issues_dataset.remove_columns(columns_to_remove)
issues_dataset`}}),Bt=new T({props:{code:`Dataset({
    features: ['html_url', 'title', 'comments', 'body'],
    num_rows: 771
})`,highlighted:`Dataset({
    features: [<span class="hljs-string">&#x27;html_url&#x27;</span>, <span class="hljs-string">&#x27;title&#x27;</span>, <span class="hljs-string">&#x27;comments&#x27;</span>, <span class="hljs-string">&#x27;body&#x27;</span>],
    num_rows: <span class="hljs-number">771</span>
})`}}),Vt=new T({props:{code:`issues_dataset.set_format("pandas")
df = issues_dataset[:]`,highlighted:`issues_dataset.set_format(<span class="hljs-string">&quot;pandas&quot;</span>)
df = issues_dataset[:]`}}),Jt=new T({props:{code:'df["comments"][0].tolist()',highlighted:'df[<span class="hljs-string">&quot;comments&quot;</span>][<span class="hljs-number">0</span>].tolist()'}}),Qt=new T({props:{code:`['the bug code locate in \uFF1A\\r\\n    if data_args.task_name is not None:\\r\\n        # Downloading and loading a dataset from the hub.\\r\\n        datasets = load_dataset("glue", data_args.task_name, cache_dir=model_args.cache_dir)',
 'Hi @jinec,\\r\\n\\r\\nFrom time to time we get this kind of \`ConnectionError\` coming from the github.com website: https://raw.githubusercontent.com\\r\\n\\r\\nNormally, it should work if you wait a little and then retry.\\r\\n\\r\\nCould you please confirm if the problem persists?',
 'cannot connect\uFF0Ceven by Web browser\uFF0Cplease check that  there is some  problems\u3002',
 'I can access https://raw.githubusercontent.com/huggingface/datasets/1.7.0/datasets/glue/glue.py without problem...']`,highlighted:`[<span class="hljs-string">&#x27;the bug code locate in \uFF1A\\r\\n    if data_args.task_name is not None:\\r\\n        # Downloading and loading a dataset from the hub.\\r\\n        datasets = load_dataset(&quot;glue&quot;, data_args.task_name, cache_dir=model_args.cache_dir)&#x27;</span>,
 <span class="hljs-string">&#x27;Hi @jinec,\\r\\n\\r\\nFrom time to time we get this kind of \`ConnectionError\` coming from the github.com website: https://raw.githubusercontent.com\\r\\n\\r\\nNormally, it should work if you wait a little and then retry.\\r\\n\\r\\nCould you please confirm if the problem persists?&#x27;</span>,
 <span class="hljs-string">&#x27;cannot connect\uFF0Ceven by Web browser\uFF0Cplease check that  there is some  problems\u3002&#x27;</span>,
 <span class="hljs-string">&#x27;I can access https://raw.githubusercontent.com/huggingface/datasets/1.7.0/datasets/glue/glue.py without problem...&#x27;</span>]`}}),Xt=new T({props:{code:`comments_df = df.explode("comments", ignore_index=True)
comments_df.head(4)`,highlighted:`comments_df = df.explode(<span class="hljs-string">&quot;comments&quot;</span>, ignore_index=<span class="hljs-literal">True</span>)
comments_df.head(<span class="hljs-number">4</span>)`}}),Kt=new T({props:{code:`from datasets import Dataset

comments_dataset = Dataset.from_pandas(comments_df)
comments_dataset`,highlighted:`<span class="hljs-keyword">from</span> datasets <span class="hljs-keyword">import</span> Dataset

comments_dataset = Dataset.from_pandas(comments_df)
comments_dataset`}}),Zt=new T({props:{code:`Dataset({
    features: ['html_url', 'title', 'comments', 'body'],
    num_rows: 2842
})`,highlighted:`Dataset({
    features: [<span class="hljs-string">&#x27;html_url&#x27;</span>, <span class="hljs-string">&#x27;title&#x27;</span>, <span class="hljs-string">&#x27;comments&#x27;</span>, <span class="hljs-string">&#x27;body&#x27;</span>],
    num_rows: <span class="hljs-number">2842</span>
})`}}),qt=new oi({props:{$$slots:{default:[bi]},$$scope:{ctx:R}}}),ts=new T({props:{code:`comments_dataset = comments_dataset.map(
    lambda x: {"comment_length": len(x["comments"].split())}
)`,highlighted:`comments_dataset = comments_dataset.<span class="hljs-built_in">map</span>(
    <span class="hljs-keyword">lambda</span> x: {<span class="hljs-string">&quot;comment_length&quot;</span>: <span class="hljs-built_in">len</span>(x[<span class="hljs-string">&quot;comments&quot;</span>].split())}
)`}}),ss=new T({props:{code:`comments_dataset = comments_dataset.filter(lambda x: x["comment_length"] > 15)
comments_dataset`,highlighted:`comments_dataset = comments_dataset.<span class="hljs-built_in">filter</span>(<span class="hljs-keyword">lambda</span> x: x[<span class="hljs-string">&quot;comment_length&quot;</span>] &gt; <span class="hljs-number">15</span>)
comments_dataset`}}),es=new T({props:{code:`Dataset({
    features: ['html_url', 'title', 'comments', 'body', 'comment_length'],
    num_rows: 2098
})`,highlighted:`Dataset({
    features: [<span class="hljs-string">&#x27;html_url&#x27;</span>, <span class="hljs-string">&#x27;title&#x27;</span>, <span class="hljs-string">&#x27;comments&#x27;</span>, <span class="hljs-string">&#x27;body&#x27;</span>, <span class="hljs-string">&#x27;comment_length&#x27;</span>],
    num_rows: <span class="hljs-number">2098</span>
})`}}),as=new T({props:{code:`def concatenate_text(examples):
    return {
        "text": examples["title"]
        + " \\n "
        + examples["body"]
        + " \\n "
        + examples["comments"]
    }


comments_dataset = comments_dataset.map(concatenate_text)`,highlighted:`<span class="hljs-keyword">def</span> <span class="hljs-title function_">concatenate_text</span>(<span class="hljs-params">examples</span>):
    <span class="hljs-keyword">return</span> {
        <span class="hljs-string">&quot;text&quot;</span>: examples[<span class="hljs-string">&quot;title&quot;</span>]
        + <span class="hljs-string">&quot; \\n &quot;</span>
        + examples[<span class="hljs-string">&quot;body&quot;</span>]
        + <span class="hljs-string">&quot; \\n &quot;</span>
        + examples[<span class="hljs-string">&quot;comments&quot;</span>]
    }


comments_dataset = comments_dataset.<span class="hljs-built_in">map</span>(concatenate_text)`}});const No=[wi,$i],hs=[];function Oo(t,l){return t[0]==="pt"?0:1}it=Oo(R),pt=hs[it]=No[it](R),rs=new T({props:{code:`def cls_pooling(model_output):
    return model_output.last_hidden_state[:, 0]`,highlighted:`<span class="hljs-keyword">def</span> <span class="hljs-title function_">cls_pooling</span>(<span class="hljs-params">model_output</span>):
    <span class="hljs-keyword">return</span> model_output.last_hidden_state[:, <span class="hljs-number">0</span>]`}});const Po=[ki,xi],gs=[];function Fo(t,l){return t[0]==="pt"?0:1}mt=Fo(R),ut=gs[mt]=Po[mt](R),is=new ri({}),ms=new T({props:{code:'embeddings_dataset.add_faiss_index(column="embeddings")',highlighted:'embeddings_dataset.add_faiss_index(column=<span class="hljs-string">&quot;embeddings&quot;</span>)'}});const Ho=[ji,vi],bs=[];function Go(t,l){return t[0]==="pt"?0:1}return dt=Go(R),ft=bs[dt]=Ho[dt](R),us=new T({props:{code:`scores, samples = embeddings_dataset.get_nearest_examples(
    "embeddings", question_embedding, k=5
)`,highlighted:`scores, samples = embeddings_dataset.get_nearest_examples(
    <span class="hljs-string">&quot;embeddings&quot;</span>, question_embedding, k=<span class="hljs-number">5</span>
)`}}),ds=new T({props:{code:`import pandas as pd

samples_df = pd.DataFrame.from_dict(samples)
samples_df["scores"] = scores
samples_df.sort_values("scores", ascending=False, inplace=True)`,highlighted:`<span class="hljs-keyword">import</span> pandas <span class="hljs-keyword">as</span> pd

samples_df = pd.DataFrame.from_dict(samples)
samples_df[<span class="hljs-string">&quot;scores&quot;</span>] = scores
samples_df.sort_values(<span class="hljs-string">&quot;scores&quot;</span>, ascending=<span class="hljs-literal">False</span>, inplace=<span class="hljs-literal">True</span>)`}}),fs=new T({props:{code:`for _, row in samples_df.iterrows():
    print(f"COMMENT: {row.comments}")
    print(f"SCORE: {row.scores}")
    print(f"TITLE: {row.title}")
    print(f"URL: {row.html_url}")
    print("=" * 50)
    print()`,highlighted:`<span class="hljs-keyword">for</span> _, row <span class="hljs-keyword">in</span> samples_df.iterrows():
    <span class="hljs-built_in">print</span>(<span class="hljs-string">f&quot;COMMENT: <span class="hljs-subst">{row.comments}</span>&quot;</span>)
    <span class="hljs-built_in">print</span>(<span class="hljs-string">f&quot;SCORE: <span class="hljs-subst">{row.scores}</span>&quot;</span>)
    <span class="hljs-built_in">print</span>(<span class="hljs-string">f&quot;TITLE: <span class="hljs-subst">{row.title}</span>&quot;</span>)
    <span class="hljs-built_in">print</span>(<span class="hljs-string">f&quot;URL: <span class="hljs-subst">{row.html_url}</span>&quot;</span>)
    <span class="hljs-built_in">print</span>(<span class="hljs-string">&quot;=&quot;</span> * <span class="hljs-number">50</span>)
    <span class="hljs-built_in">print</span>()`}}),cs=new T({props:{code:`"""
COMMENT: Requiring online connection is a deal breaker in some cases unfortunately so it'd be great if offline mode is added similar to how \`transformers\` loads models offline fine.

@mandubian's second bullet point suggests that there's a workaround allowing you to use your offline (custom?) dataset with \`datasets\`. Could you please elaborate on how that should look like?
SCORE: 25.505046844482422
TITLE: Discussion using datasets in offline mode
URL: https://github.com/huggingface/datasets/issues/824
==================================================

COMMENT: The local dataset builders (csv, text , json and pandas) are now part of the \`datasets\` package since #1726 :)
You can now use them offline
\\\`\\\`\\\`python
datasets = load_dataset("text", data_files=data_files)
\\\`\\\`\\\`

We'll do a new release soon
SCORE: 24.555509567260742
TITLE: Discussion using datasets in offline mode
URL: https://github.com/huggingface/datasets/issues/824
==================================================

COMMENT: I opened a PR that allows to reload modules that have already been loaded once even if there's no internet.

Let me know if you know other ways that can make the offline mode experience better. I'd be happy to add them :)

I already note the "freeze" modules option, to prevent local modules updates. It would be a cool feature.

----------

> @mandubian's second bullet point suggests that there's a workaround allowing you to use your offline (custom?) dataset with \`datasets\`. Could you please elaborate on how that should look like?

Indeed \`load_dataset\` allows to load remote dataset script (squad, glue, etc.) but also you own local ones.
For example if you have a dataset script at \`./my_dataset/my_dataset.py\` then you can do
\\\`\\\`\\\`python
load_dataset("./my_dataset")
\\\`\\\`\\\`
and the dataset script will generate your dataset once and for all.

----------

About I'm looking into having \`csv\`, \`json\`, \`text\`, \`pandas\` dataset builders already included in the \`datasets\` package, so that they are available offline by default, as opposed to the other datasets that require the script to be downloaded.
cf #1724
SCORE: 24.14896583557129
TITLE: Discussion using datasets in offline mode
URL: https://github.com/huggingface/datasets/issues/824
==================================================

COMMENT: > here is my way to load a dataset offline, but it **requires** an online machine
>
> 1. (online machine)
>
> \`\`\`
>
> import datasets
>
> data = datasets.load_dataset(...)
>
> data.save_to_disk(/YOUR/DATASET/DIR)
>
> \`\`\`
>
> 2. copy the dir from online to the offline machine
>
> 3. (offline machine)
>
> \`\`\`
>
> import datasets
>
> data = datasets.load_from_disk(/SAVED/DATA/DIR)
>
> \`\`\`
>
>
>
> HTH.


SCORE: 22.893993377685547
TITLE: Discussion using datasets in offline mode
URL: https://github.com/huggingface/datasets/issues/824
==================================================

COMMENT: here is my way to load a dataset offline, but it **requires** an online machine
1. (online machine)
\\\`\\\`\\\`
import datasets
data = datasets.load_dataset(...)
data.save_to_disk(/YOUR/DATASET/DIR)
\\\`\\\`\\\`
2. copy the dir from online to the offline machine
3. (offline machine)
\\\`\\\`\\\`
import datasets
data = datasets.load_from_disk(/SAVED/DATA/DIR)
\\\`\\\`\\\`

HTH.
SCORE: 22.406635284423828
TITLE: Discussion using datasets in offline mode
URL: https://github.com/huggingface/datasets/issues/824
==================================================
"""`,highlighted:`<span class="hljs-string">&quot;&quot;&quot;
COMMENT: Requiring online connection is a deal breaker in some cases unfortunately so it&#x27;d be great if offline mode is added similar to how \`transformers\` loads models offline fine.

@mandubian&#x27;s second bullet point suggests that there&#x27;s a workaround allowing you to use your offline (custom?) dataset with \`datasets\`. Could you please elaborate on how that should look like?
SCORE: 25.505046844482422
TITLE: Discussion using datasets in offline mode
URL: https://github.com/huggingface/datasets/issues/824
==================================================

COMMENT: The local dataset builders (csv, text , json and pandas) are now part of the \`datasets\` package since #1726 :)
You can now use them offline
\\\`\\\`\\\`python
datasets = load_dataset(&quot;text&quot;, data_files=data_files)
\\\`\\\`\\\`

We&#x27;ll do a new release soon
SCORE: 24.555509567260742
TITLE: Discussion using datasets in offline mode
URL: https://github.com/huggingface/datasets/issues/824
==================================================

COMMENT: I opened a PR that allows to reload modules that have already been loaded once even if there&#x27;s no internet.

Let me know if you know other ways that can make the offline mode experience better. I&#x27;d be happy to add them :)

I already note the &quot;freeze&quot; modules option, to prevent local modules updates. It would be a cool feature.

----------

&gt; @mandubian&#x27;s second bullet point suggests that there&#x27;s a workaround allowing you to use your offline (custom?) dataset with \`datasets\`. Could you please elaborate on how that should look like?

Indeed \`load_dataset\` allows to load remote dataset script (squad, glue, etc.) but also you own local ones.
For example if you have a dataset script at \`./my_dataset/my_dataset.py\` then you can do
\\\`\\\`\\\`python
load_dataset(&quot;./my_dataset&quot;)
\\\`\\\`\\\`
and the dataset script will generate your dataset once and for all.

----------

About I&#x27;m looking into having \`csv\`, \`json\`, \`text\`, \`pandas\` dataset builders already included in the \`datasets\` package, so that they are available offline by default, as opposed to the other datasets that require the script to be downloaded.
cf #1724
SCORE: 24.14896583557129
TITLE: Discussion using datasets in offline mode
URL: https://github.com/huggingface/datasets/issues/824
==================================================

COMMENT: &gt; here is my way to load a dataset offline, but it **requires** an online machine
&gt;
&gt; 1. (online machine)
&gt;
&gt; \`\`\`
&gt;
&gt; import datasets
&gt;
&gt; data = datasets.load_dataset(...)
&gt;
&gt; data.save_to_disk(/YOUR/DATASET/DIR)
&gt;
&gt; \`\`\`
&gt;
&gt; 2. copy the dir from online to the offline machine
&gt;
&gt; 3. (offline machine)
&gt;
&gt; \`\`\`
&gt;
&gt; import datasets
&gt;
&gt; data = datasets.load_from_disk(/SAVED/DATA/DIR)
&gt;
&gt; \`\`\`
&gt;
&gt;
&gt;
&gt; HTH.


SCORE: 22.893993377685547
TITLE: Discussion using datasets in offline mode
URL: https://github.com/huggingface/datasets/issues/824
==================================================

COMMENT: here is my way to load a dataset offline, but it **requires** an online machine
1. (online machine)
\\\`\\\`\\\`
import datasets
data = datasets.load_dataset(...)
data.save_to_disk(/YOUR/DATASET/DIR)
\\\`\\\`\\\`
2. copy the dir from online to the offline machine
3. (offline machine)
\\\`\\\`\\\`
import datasets
data = datasets.load_from_disk(/SAVED/DATA/DIR)
\\\`\\\`\\\`

HTH.
SCORE: 22.406635284423828
TITLE: Discussion using datasets in offline mode
URL: https://github.com/huggingface/datasets/issues/824
==================================================
&quot;&quot;&quot;</span>`}}),It=new oi({props:{$$slots:{default:[yi]},$$scope:{ctx:R}}}),{c(){f=o("meta"),g=m(),x(c.$$.fragment),y=m(),_=o("h1"),h=o("a"),D=o("span"),x(w.$$.fragment),A=m(),E=o("span"),z=a("\u4F7F\u7528 FAISS \u8FDB\u884C\u8BED\u4E49\u641C\u7D22"),N=m(),P.c(),F=m(),M=o("p"),nt=a("\u5728"),H=o("a"),lt=a("\u7B2C\u4E94\u5C0F\u8282"),ot=a(", \u6211\u4EEC\u4ECE \u{1F917} Datasets \u5B58\u50A8\u5E93\u521B\u5EFA\u4E86\u4E00\u4E2A\u5305\u542B GitHub \u95EE\u9898\u548C\u8BC4\u8BBA\u7684\u6570\u636E\u96C6\u3002\u5728\u672C\u8282\u4E2D\uFF0C\u6211\u4EEC\u5C06\u4F7F\u7528\u8FD9\u4E9B\u4FE1\u606F\u6765\u6784\u5EFA\u4E00\u4E2A\u641C\u7D22\u5F15\u64CE\uFF0C\u5B83\u53EF\u4EE5\u5E2E\u52A9\u6211\u4EEC\u627E\u5230\u8FD9\u4E2A\u5E93\u6700\u7D27\u8FEB\u95EE\u9898\u7684\u7B54\u6848\uFF01"),U=m(),x(I.$$.fragment),Y=m(),d=o("h2"),S=a("\u4F7F\u7528\u5D4C\u5165\u8FDB\u884C\u8BED\u4E49\u641C\u7D22"),C=m(),B=o("p"),vt=a("\u6B63\u5982\u6211\u4EEC\u5728"),st=o("a"),tn=a("\u7B2C\u4E00\u7AE0"),sn=a(",\u5B66\u4E60\u7684, \u57FA\u4E8E Transformer \u7684\u8BED\u8A00\u6A21\u578B\u4F1A\u5C06\u6587\u672C\u4E2D\u7684\u6BCF\u4E2A\u6807\u8BB0\u8F6C\u6362\u4E3A\u5D4C\u5165\u5411\u91CF.\u4E8B\u5B9E\u8BC1\u660E\uFF0C\u53EF\u4EE5\u201C\u6C47\u96C6\u201D\u5404\u4E2A\u5D4C\u5165\u5411\u91CF\u6765\u521B\u5EFA\u6574\u4E2A\u53E5\u5B50\u3001\u6BB5\u843D\u6216\u6587\u6863\uFF08\u5728\u67D0\u4E9B\u60C5\u51B5\u4E0B\uFF09\u7684\u5411\u91CF\u8868\u793A\u3002\u7136\u540E\uFF0C\u901A\u8FC7\u8BA1\u7B97\u6BCF\u4E2A\u5D4C\u5165\u4E4B\u95F4\u7684\u70B9\u79EF\u76F8\u4F3C\u5EA6\uFF08\u6216\u5176\u4ED6\u4E00\u4E9B\u76F8\u4F3C\u5EA6\u5EA6\u91CF\uFF09\u5E76\u8FD4\u56DE\u76F8\u4F3C\u5EA6\u6700\u5927\u7684\u6587\u6863\uFF0C\u8FD9\u4E9B\u5D4C\u5165\u53EF\u7528\u4E8E\u5728\u8BED\u6599\u5E93\u4E2D\u627E\u5230\u76F8\u4F3C\u7684\u6587\u6863\u3002\u5728\u672C\u8282\u4E2D\uFF0C\u6211\u4EEC\u5C06\u4F7F\u7528\u5D4C\u5165\u6765\u5F00\u53D1\u8BED\u4E49\u641C\u7D22\u5F15\u64CE\u3002\u4E0E\u57FA\u4E8E\u5C06\u67E5\u8BE2\u4E2D\u7684\u5173\u952E\u5B57\u7684\u4F20\u7EDF\u65B9\u6CD5\u76F8\u6BD4\uFF0C\u8FD9\u4E9B\u641C\u7D22\u5F15\u64CE\u5177\u6709\u591A\u79CD\u4F18\u52BF\u3002"),Le=m(),jt=o("div"),Pt=o("img"),en=m(),Ft=o("img"),ze=m(),Ht=o("h2"),an=a("## \u52A0\u8F7D\u548C\u51C6\u5907\u6570\u636E\u96C6"),Ue=m(),ws=o("p"),nn=a("\u6211\u4EEC\u9700\u8981\u505A\u7684\u7B2C\u4E00\u4EF6\u4E8B\u662F\u4E0B\u8F7D\u6211\u4EEC\u7684 GitHub \u95EE\u9898\u6570\u636E\u96C6\uFF0C\u6240\u4EE5\u8BA9\u6211\u4EEC\u4F7F\u7528 \u{1F917} Hub \u5E93\u6765\u89E3\u6790\u6211\u4EEC\u7684\u6587\u4EF6\u5728 Hugging Face Hub \u4E0A\u5B58\u50A8\u7684\u6570\u636E\uFF1A"),Ye=m(),x(Gt.$$.fragment),Be=m(),ct=o("p"),ln=a("\u5C06 URL \u5B58\u50A8\u5728 "),Ls=o("strong"),on=a("data_files"),rn=a(" \uFF0C\u7136\u540E\u6211\u4EEC\u53EF\u4EE5\u4F7F\u7528"),xs=o("a"),pn=a("\u7B2C\u4E8C\u5C0F\u8282"),mn=a("\u4ECB\u7ECD\u7684\u65B9\u6CD5\u52A0\u8F7D\u8FDC\u7A0B\u6570\u636E\u96C6:"),We=m(),x(Mt.$$.fragment),Ve=m(),x(Lt.$$.fragment),Je=m(),ks=o("p"),un=a("\u8FD9\u91CC\u6211\u4EEC\u5728load_dataset()\u4E2D\u4F7F\u7528\u4E86\u9ED8\u8BA4\u7684\u8BAD\u7EC3\u96C6\u5206\u5272\uFF0C\u6240\u4EE5\u5B83\u8FD4\u56DE\u4E00\u4E2A\u6570\u636E\u96C6\u800C\u4E0D\u662F\u6570\u636E\u96C6\u5B57\u5178\u3002\u7B2C\u4E00\u9879\u4EFB\u52A1\u662F\u8FC7\u6EE4\u6389pull\u8BF7\u6C42\uFF0C\u56E0\u4E3A\u8FD9\u4E9B\u8BF7\u6C42\u5F88\u5C11\u7528\u4E8E\u56DE\u7B54\u7528\u6237\u63D0\u51FA\u7684\u95EE\u9898\uFF0C\u800C\u4E14\u4F1A\u7ED9\u6211\u4EEC\u7684\u641C\u7D22\u5F15\u64CE\u5E26\u6765\u566A\u58F0\u3002\u73B0\u5728\u5E94\u8BE5\u5F88\u719F\u6089\u4E86\uFF0C\u6211\u4EEC\u53EF\u4EE5\u4F7F\u7528dataset.filter()\u51FD\u6570\u6765\u6392\u9664\u6570\u636E\u96C6\u4E2D\u7684\u8FD9\u4E9B\u884C\u3002\u540C\u65F6\uFF0C\u8BA9\u6211\u4EEC\u4E5F\u8FC7\u6EE4\u6389\u6CA1\u6709\u6CE8\u91CA\u7684\u884C\uFF0C\u56E0\u4E3A\u8FD9\u4E9B\u884C\u4E0D\u4F1A\u662F\u7528\u6237\u63D0\u95EE\u7684\u7B54\u6848:"),Qe=m(),x(zt.$$.fragment),Xe=m(),x(Ut.$$.fragment),Ke=m(),L=o("p"),dn=a("\u6211\u4EEC\u53EF\u4EE5\u770B\u5230\u6211\u4EEC\u7684\u6570\u636E\u96C6\u4E2D\u6709\u5F88\u591A\u5217\uFF0C\u5176\u4E2D\u5927\u90E8\u5206\u6211\u4EEC\u4E0D\u9700\u8981\u6784\u5EFA\u6211\u4EEC\u7684\u641C\u7D22\u5F15\u64CE\u3002\u4ECE\u641C\u7D22\u7684\u89D2\u5EA6\u6765\u770B\uFF0C\u4FE1\u606F\u91CF\u6700\u5927\u7684\u5217\u662F "),zs=o("strong"),fn=a("title"),cn=a(" , "),Us=o("strong"),_n=a("body"),hn=a(" \uFF0C \u548C "),Ys=o("strong"),gn=a("comments"),bn=a(" \uFF0C\u800C "),Bs=o("strong"),$n=a("html_url"),wn=a(" \u4E3A\u6211\u4EEC\u63D0\u4F9B\u4E86\u4E00\u4E2A\u56DE\u5230\u6E90\u95EE\u9898\u7684\u94FE\u63A5\u3002\u8BA9\u6211\u4EEC\u4F7F\u7528 "),Ws=o("strong"),xn=a("Dataset.remove_columns()"),kn=a(" \u5220\u9664\u5176\u4F59\u90E8\u5206\u7684\u529F\u80FD\uFF1A"),Ze=m(),x(Yt.$$.fragment),ta=m(),x(Bt.$$.fragment),sa=m(),W=o("p"),vn=a("\u4E3A\u4E86\u521B\u5EFA\u6211\u4EEC\u7684\u5D4C\u5165\uFF0C\u6211\u4EEC\u5C06\u7528\u95EE\u9898\u7684\u6807\u9898\u548C\u6B63\u6587\u6765\u6269\u5145\u6BCF\u6761\u8BC4\u8BBA\uFF0C\u56E0\u4E3A\u8FD9\u4E9B\u5B57\u6BB5\u901A\u5E38\u5305\u542B\u6709\u7528\u7684\u4E0A\u4E0B\u6587\u4FE1\u606F\u3002\u56E0\u4E3A\u6211\u4EEC\u7684 "),Vs=o("strong"),jn=a("comments"),yn=a(" \u5217\u5F53\u524D\u662F\u6BCF\u4E2A\u95EE\u9898\u7684\u8BC4\u8BBA\u5217\u8868\uFF0C\u6211\u4EEC\u9700\u8981\u201C\u91CD\u65B0\u7EC4\u5408\u201D\u5217\uFF0C\u4EE5\u4FBF\u6BCF\u4E00\u6761\u8BC4\u8BBA\u90FD\u5305\u542B\u4E00\u4E2A "),Js=o("strong"),En=a("(html_url, title, body, comment)"),Tn=a(" \u5143\u7EC4\u3002\u5728 Pandas \u4E2D\uFF0C\u6211\u4EEC\u53EF\u4EE5\u4F7F\u7528 "),Wt=o("a"),qn=a("DataFrame.explode() \u51FD\u6570"),Sn=a(", \u5B83\u4E3A\u7C7B\u4F3C\u5217\u8868\u7684\u5217\u4E2D\u7684\u6BCF\u4E2A\u5143\u7D20\u521B\u5EFA\u4E00\u4E2A\u65B0\u884C\uFF0C\u540C\u65F6\u590D\u5236\u6240\u6709\u5176\u4ED6\u5217\u503C\u3002\u4E3A\u4E86\u770B\u5230\u5B83\u7684\u5B9E\u9645\u6548\u679C\uFF0C\u8BA9\u6211\u4EEC\u9996\u5148\u5207\u6362\u5230 Pandas\u7684"),Qs=o("strong"),Dn=a("DataFrame"),An=a(" \u683C\u5F0F\uFF1A"),ea=m(),x(Vt.$$.fragment),aa=m(),Tt=o("p"),Rn=a("\u5982\u679C\u6211\u4EEC\u68C0\u67E5\u8FD9\u91CC\u7684\u7B2C\u4E00\u884C "),Xs=o("strong"),In=a("DataFrame"),Cn=a(" \u6211\u4EEC\u53EF\u4EE5\u770B\u5230\u6709\u56DB\u4E2A\u8BC4\u8BBA\u4E0E\u8FD9\u4E2A\u95EE\u9898\u76F8\u5173\uFF1A"),na=m(),x(Jt.$$.fragment),la=m(),x(Qt.$$.fragment),oa=m(),vs=o("p"),Nn=a("\u6211\u4EEC\u5E0C\u671B\u8FD9\u4E9B\u8BC4\u8BBA\u4E2D\u7684\u6BCF\u4E00\u6761\u90FD\u5F97\u5230\u4E00\u884C\u3002\u8BA9\u6211\u4EEC\u68C0\u67E5\u662F\u5426\u662F\u8FD9\u79CD\u60C5\u51B5\uFF1A"),ra=m(),x(Xt.$$.fragment),ia=m(),Q=o("table"),Ks=o("thead"),V=o("tr"),pa=o("th"),On=m(),Zs=o("th"),Pn=a("html_url"),Fn=m(),te=o("th"),Hn=a("title"),Gn=m(),se=o("th"),Mn=a("comments"),Ln=m(),ee=o("th"),zn=a("body"),Un=m(),rt=o("tbody"),X=o("tr"),ae=o("th"),Yn=a("0"),Bn=m(),ne=o("td"),Wn=a("https://github.com/huggingface/datasets/issues/2787"),Vn=m(),le=o("td"),Jn=a("ConnectionError: Couldn't reach https://raw.githubusercontent.com"),Qn=m(),oe=o("td"),Xn=a("the bug code locate in \uFF1A\\r\\n    if data_args.task_name is not None..."),Kn=m(),re=o("td"),Zn=a("Hello,\\r\\nI am trying to run run_glue.py and it gives me this error..."),tl=m(),K=o("tr"),ie=o("th"),sl=a("1"),el=m(),pe=o("td"),al=a("https://github.com/huggingface/datasets/issues/2787"),nl=m(),me=o("td"),ll=a("ConnectionError: Couldn't reach https://raw.githubusercontent.com"),ol=m(),ue=o("td"),rl=a("Hi @jinec,\\r\\n\\r\\nFrom time to time we get this kind of `ConnectionError` coming from the github.com website: https://raw.githubusercontent.com..."),il=m(),de=o("td"),pl=a("Hello,\\r\\nI am trying to run run_glue.py and it gives me this error..."),ml=m(),Z=o("tr"),fe=o("th"),ul=a("2"),dl=m(),ce=o("td"),fl=a("https://github.com/huggingface/datasets/issues/2787"),cl=m(),_e=o("td"),_l=a("ConnectionError: Couldn't reach https://raw.githubusercontent.com"),hl=m(),he=o("td"),gl=a("cannot connect\uFF0Ceven by Web browser\uFF0Cplease check that  there is some  problems\u3002"),bl=m(),ge=o("td"),$l=a("Hello,\\r\\nI am trying to run run_glue.py and it gives me this error..."),wl=m(),tt=o("tr"),be=o("th"),xl=a("3"),kl=m(),$e=o("td"),vl=a("https://github.com/huggingface/datasets/issues/2787"),jl=m(),we=o("td"),yl=a("ConnectionError: Couldn't reach https://raw.githubusercontent.com"),El=m(),xe=o("td"),Tl=a("I can access https://raw.githubusercontent.com/huggingface/datasets/1.7.0/datasets/glue/glue.py without problem..."),ql=m(),ke=o("td"),Sl=a("Hello,\\r\\nI am trying to run run_glue.py and it gives me this error..."),ma=m(),et=o("p"),Dl=a("\u592A\u597D\u4E86\uFF0C\u6211\u4EEC\u53EF\u4EE5\u770B\u5230\u8BC4\u8BBA\u6210\u529F\u88AB\u6269\u5145\uFF0C "),ve=o("strong"),Al=a("comments"),Rl=a(" \u662F\u5305\u542B\u4E2A\u4EBA\u8BC4\u8BBA\u7684\u5217\uFF01\u73B0\u5728\u6211\u4EEC\u5DF2\u7ECF\u5B8C\u6210\u4E86 Pandas\u8981\u5B8C\u6210\u7684\u90E8\u5206\u529F\u80FD\uFF0C\u6211\u4EEC\u53EF\u4EE5\u5FEB\u901F\u5207\u6362\u56DE "),je=o("strong"),Il=a("Dataset"),Cl=a(" \u901A\u8FC7\u52A0\u8F7D "),ye=o("strong"),Nl=a("DataFrame"),Ol=a(" \u5728\u5185\u5B58\u4E2D\uFF1A"),ua=m(),x(Kt.$$.fragment),da=m(),x(Zt.$$.fragment),fa=m(),js=o("p"),Pl=a("\u592A\u597D\u4E86\uFF0C\u6211\u4EEC\u83B7\u53D6\u5230\u4E86\u51E0\u5343\u6761\u7684\u8BC4\u8BBA\uFF01"),ca=m(),x(qt.$$.fragment),_a=m(),St=o("p"),Fl=a("\u73B0\u5728\u6211\u4EEC\u6BCF\u884C\u6709\u4E00\u4E2A\u8BC4\u8BBA\uFF0C\u8BA9\u6211\u4EEC\u521B\u5EFA\u4E00\u4E2A\u65B0\u7684 "),Ee=o("strong"),Hl=a("comments_length"),Gl=a(" \u5217\u6765\u5B58\u653E\u6BCF\u6761\u8BC4\u8BBA\u7684\u5B57\u6570\uFF1A"),ha=m(),x(ts.$$.fragment),ga=m(),ys=o("p"),Ml=a("\u6211\u4EEC\u53EF\u4EE5\u4F7F\u7528\u8FD9\u4E2A\u65B0\u5217\u6765\u8FC7\u6EE4\u6389\u7B80\u77ED\u7684\u8BC4\u8BBA\uFF0C\u5176\u4E2D\u901A\u5E38\u5305\u62EC\u201Ccc @lewtun\u201D\u6216\u201C\u8C22\u8C22\uFF01\u201D\u4E4B\u7C7B\u4E0E\u6211\u4EEC\u7684\u641C\u7D22\u5F15\u64CE\u65E0\u5173\u7684\u5185\u5BB9\u3002\u867D\u7136\u65E0\u6CD5\u4E3A\u8FC7\u6EE4\u5668\u9009\u62E9\u7684\u7CBE\u786E\u6570\u5B57\uFF0C\u4F46\u5927\u7EA6\u5927\u4E8E15 \u4E2A\u5355\u8BCD\u4F3C\u4E4E\u662F\u4E00\u4E2A\u4E0D\u9519\u7684\u9009\u62E9\uFF1A"),ba=m(),x(ss.$$.fragment),$a=m(),x(es.$$.fragment),wa=m(),_t=o("p"),Ll=a("\u7A0D\u5FAE\u6E05\u7406\u4E86\u6211\u4EEC\u7684\u6570\u636E\u96C6\u540E\uFF0C\u8BA9\u6211\u4EEC\u5C06\u95EE\u9898\u6807\u9898\u3001\u63CF\u8FF0\u548C\u8BC4\u8BBA\u8FDE\u63A5\u5230\u4E00\u4E2A\u65B0\u7684 "),Te=o("strong"),zl=a("text"),Ul=a(" \u5217\u3002\u50CF\u5F80\u5E38\u4E00\u6837\uFF0C\u6211\u4EEC\u53EF\u4EE5\u7F16\u5199\u4E00\u4E2A\u7B80\u5355\u7684\u51FD\u6570\uFF0C\u5E76\u5C06\u5176\u4F20\u9012\u7ED9 "),qe=o("strong"),Yl=a("Dataset.map()"),Bl=a("\u6765\u505A\u5230\u8FD9\u4E9B \uFF1A"),xa=m(),x(as.$$.fragment),ka=m(),Es=o("p"),Wl=a("\u6211\u4EEC\u7EC8\u4E8E\u51C6\u5907\u597D\u521B\u5EFA\u4E00\u4E9B\u5D4C\u5165\u4E86\uFF01\u8BA9\u6211\u4EEC\u6765\u770B\u770B\u3002"),va=m(),ns=o("h2"),Vl=a("\u521B\u5EFA\u6587\u672C\u5D4C\u5165"),ja=m(),G=o("p"),Jl=a("\u6211\u4EEC\u5728"),Ts=o("a"),Ql=a("\u7B2C\u4E8C\u7AE0"),Xl=a(" \u5B66\u8FC7\uFF0C\u6211\u4EEC\u53EF\u4EE5\u901A\u8FC7\u4F7F\u7528 "),Se=o("strong"),Kl=a("AutoModel"),Zl=a(" \u7C7B\u6765\u5B8C\u6210\u8BCD\u5D4C\u5165\u3002\u6211\u4EEC\u9700\u8981\u505A\u7684\u5C31\u662F\u9009\u62E9\u4E00\u4E2A\u5408\u9002\u7684\u68C0\u67E5\u70B9\u6765\u52A0\u8F7D\u6A21\u578B\u3002\u5E78\u8FD0\u7684\u662F\uFF0C\u6709\u4E00\u4E2A\u540D\u4E3A "),De=o("strong"),to=a("sentence-transformers"),so=a(" \u4E13\u95E8\u7528\u4E8E\u521B\u5EFA\u8BCD\u5D4C\u5165\u3002\u5982\u5E93\u4E2D"),ls=o("a"),eo=a("\u6587\u6863"),ao=a(", \u6240\u8FF0\u7684\uFF0C\u6211\u4EEC\u8FD9\u6B21\u8981\u5B9E\u73B0\u7684\u662F\u975E\u5BF9\u79F0\u8BED\u4E49\u641C\u7D22\uFF0C\u56E0\u4E3A\u6211\u4EEC\u6709\u4E00\u4E2A\u7B80\u77ED\u7684\u67E5\u8BE2\uFF0C\u6211\u4EEC\u5E0C\u671B\u5728\u6BD4\u5982\u95EE\u9898\u8BC4\u8BBA\u7B49\u66F4\u957F\u7684\u6587\u6863\u4E2D\u627E\u5230\u5176\u7B54\u6848\u3002\u901A\u8FC7\u67E5\u770B"),os=o("a"),no=a("\u6A21\u578B\u6982\u8FF0\u8868"),lo=a(" \u6211\u4EEC\u53EF\u4EE5\u53D1\u73B0 "),Ae=o("strong"),oo=a("multi-qa-mpnet-base-dot-v1"),ro=a(" \u68C0\u67E5\u70B9\u5728\u8BED\u4E49\u641C\u7D22\u65B9\u9762\u5177\u6709\u6700\u4F73\u6027\u80FD\uFF0C\u56E0\u6B64\u6211\u4EEC\u5C06\u5728\u6211\u4EEC\u7684\u5E94\u7528\u7A0B\u5E8F\u4E2D\u4F7F\u7528\u5B83\u3002\u6211\u4EEC\u8FD8\u5C06\u4F7F\u7528\u76F8\u540C\u7684\u68C0\u67E5\u70B9\u52A0\u8F7D\u6807\u8BB0\u5668\uFF1A"),ya=m(),pt.c(),qs=m(),Dt=o("p"),io=a("\u6B63\u5982\u6211\u4EEC\u4E4B\u524D\u63D0\u5230\u7684\uFF0C\u6211\u4EEC\u5E0C\u671B\u5C06 GitHub \u95EE\u9898\u8BED\u6599\u5E93\u4E2D\u7684\u6BCF\u4E2A\u6761\u76EE\u8868\u793A\u4E3A\u5355\u4E2A\u5411\u91CF\uFF0C\u56E0\u6B64\u6211\u4EEC\u9700\u8981\u4EE5\u67D0\u79CD\u65B9\u5F0F\u201C\u6C60\u5316\u201D\u6216\u5E73\u5747\u5316\u6211\u4EEC\u7684\u6807\u8BB0\u5D4C\u5165\u3002\u4E00\u79CD\u6D41\u884C\u7684\u65B9\u6CD5\u662F\u5728\u6211\u4EEC\u6A21\u578B\u7684\u8F93\u51FA\u4E0A\u6267\u884CCLS \u6C60\u5316\uFF0C\u6211\u4EEC\u53EA\u83B7\u53D6"),Re=o("strong"),po=a("[CLS]"),mo=a(" \u4EE4\u724C\u7684\u6700\u540E\u4E00\u4E2A\u9690\u85CF\u72B6\u6001\u3002\u4EE5\u4E0B\u51FD\u6570\u4E3A\u6211\u4EEC\u63D0\u4F9B\u4E86\u8FD9\u6837\u7684\u65B9\u6CD5\uFF1A"),Ea=m(),x(rs.$$.fragment),Ta=m(),Ss=o("p"),uo=a("\u63A5\u4E0B\u6765\uFF0C\u6211\u4EEC\u5C06\u521B\u5EFA\u4E00\u4E2A\u8F85\u52A9\u51FD\u6570\uFF0C\u8BE5\u51FD\u6570\u5C06\u6807\u8BB0\u6587\u6863\u5217\u8868\uFF0C\u5C06tensor\u653E\u5728 GPU \u4E0A\uFF0C\u7136\u540E\u63D0\u4F9B\u7ED9\u6A21\u578B\uFF0C\u6700\u540E\u5BF9\u8F93\u51FA\u4F7F\u7528CLS \u6C60\u5316\uFF1A"),qa=m(),ut.c(),Ds=m(),As=o("p"),fo=a("\u8BF7\u6CE8\u610F\uFF0C\u6211\u4EEC\u5DF2\u7ECF\u5C06\u5D4C\u5165\u8F6C\u6362\u4E3A NumPy \u6570\u7EC4\u2014\u2014\u8FD9\u662F\u56E0\u4E3A\u5F53\u6211\u4EEC\u5C1D\u8BD5\u4F7F\u7528 FAISS \u7D22\u5F15\u5B83\u4EEC\u65F6\uFF0C\u{1F917} Datasets\u9700\u8981\u8FD9\u79CD\u683C\u5F0F\uFF0C\u6211\u4EEC\u63A5\u4E0B\u6765\u4F1A\u8FD9\u6837\u505A\u3002"),Sa=m(),yt=o("h2"),At=o("a"),Ie=o("span"),x(is.$$.fragment),co=m(),Ce=o("span"),_o=a("\u4F7F\u7528 FAISS \u8FDB\u884C\u9AD8\u6548\u7684\u76F8\u4F3C\u6027\u641C\u7D22"),Da=m(),ht=o("p"),ho=a("\u73B0\u5728\u6211\u4EEC\u6709\u4E86\u4E00\u4E2A\u8BCD\u5D4C\u5165\u6570\u636E\u96C6\uFF0C\u6211\u4EEC\u9700\u8981\u4E00\u4E9B\u65B9\u6CD5\u6765\u641C\u7D22\u5B83\u4EEC\u3002\u4E3A\u6B64\uFF0C\u6211\u4EEC\u5C06\u5728 \u{1F917} Datasets\u4E2D\u4F7F\u7528\u4E00\u79CD\u7279\u6B8A\u7684\u6570\u636E\u7ED3\u6784\uFF0C\u79F0\u4E3A FAISS\u6307\u6570."),ps=o("a"),go=a("FAISS"),bo=a(" (short for Facebook AI Similarity Search) \uFF08Facebook AI Similarity Search \u7684\u7F29\u5199\uFF09\u662F\u4E00\u4E2A\u63D0\u4F9B\u9AD8\u6548\u7B97\u6CD5\u6765\u5FEB\u901F\u641C\u7D22\u548C\u805A\u7C7B\u5D4C\u5165\u5411\u91CF\u7684\u5E93\u3002FAISS \u80CC\u540E\u7684\u57FA\u672C\u601D\u60F3\u662F\u521B\u5EFA\u4E00\u4E2A\u7279\u6B8A\u7684\u6570\u636E\u7ED3\u6784\uFF0C\u79F0\u4E3A\u6307\u6570\u3002\u8FD9\u5141\u8BB8\u4EBA\u4EEC\u627E\u5230\u54EA\u4E9B\u5D4C\u5165\u8BCD\u4E0E\u8F93\u5165\u7684\u8BCD\u5D4C\u5165\u76F8\u4F3C\u3002\u5728 \u{1F917} Datasets\u4E2D\u521B\u5EFA\u4E00\u4E2A FAISS \u7D22\u5F15\u5F88\u7B80\u5355\u2014\u2014\u6211\u4EEC\u4F7F\u7528 "),Ne=o("strong"),$o=a("Dataset.add_faiss_index()"),wo=a(" \u51FD\u6570\u5E76\u6307\u5B9A\u6211\u4EEC\u8981\u7D22\u5F15\u7684\u6570\u636E\u96C6\u7684\u54EA\u4E00\u5217\uFF1A"),Aa=m(),x(ms.$$.fragment),Ra=m(),Rt=o("p"),xo=a("\u73B0\u5728\uFF0C\u6211\u4EEC\u53EF\u4EE5\u4F7F\u7528"),Oe=o("strong"),ko=a("Dataset.get_nearest_examples()"),vo=a("\u51FD\u6570\u8FDB\u884C\u6700\u8FD1\u90BB\u5C45\u67E5\u627E\u3002\u8BA9\u6211\u4EEC\u901A\u8FC7\u9996\u5148\u5D4C\u5165\u4E00\u4E2A\u95EE\u9898\u6765\u6D4B\u8BD5\u8FD9\u4E00\u70B9\uFF0C\u5982\u4E0B\u6240\u793A\uFF1A"),Ia=m(),ft.c(),Rs=m(),Is=o("p"),jo=a("\u5C31\u50CF\u6587\u6863\u4E00\u6837\uFF0C\u6211\u4EEC\u73B0\u5728\u6709\u4E00\u4E2A 768 \u7EF4\u5411\u91CF\u8868\u793A\u67E5\u8BE2\uFF0C\u6211\u4EEC\u53EF\u4EE5\u5C06\u5176\u4E0E\u6574\u4E2A\u8BED\u6599\u5E93\u8FDB\u884C\u6BD4\u8F83\u4EE5\u627E\u5230\u6700\u76F8\u4F3C\u7684\u5D4C\u5165\uFF1A"),Ca=m(),x(us.$$.fragment),Na=m(),Et=o("p"),Pe=o("strong"),yo=a("Dataset.get_nearest_examples()"),Eo=a(" \u51FD\u6570\u8FD4\u56DE\u4E00\u4E2A\u5206\u6570\u5143\u7EC4\uFF0C\u5BF9\u67E5\u8BE2\u548C\u6587\u6863\u4E4B\u95F4\u7684\u76F8\u4F3C\u5EA6\u8FDB\u884C\u6392\u5E8F\uFF0C\u4EE5\u53CA\u4E00\u7EC4\u6700\u4F73\u5339\u914D\u7684\u7ED3\u679C\uFF08\u8FD9\u91CC\u662F 5 \u4E2A\uFF09\u3002\u8BA9\u6211\u4EEC\u628A\u8FD9\u4E9B\u6536\u96C6\u5230\u4E00\u4E2A "),Fe=o("strong"),To=a("pandas.DataFrame"),qo=a(" \u4EE5\u4FBF\u6211\u4EEC\u53EF\u4EE5\u8F7B\u677E\u5730\u5BF9\u5B83\u4EEC\u8FDB\u884C\u6392\u5E8F\uFF1A"),Oa=m(),x(ds.$$.fragment),Pa=m(),Cs=o("p"),So=a("\u73B0\u5728\u6211\u4EEC\u53EF\u4EE5\u904D\u5386\u524D\u51E0\u884C\u6765\u67E5\u770B\u6211\u4EEC\u7684\u67E5\u8BE2\u4E0E\u8BC4\u8BBA\u7684\u5339\u914D\u7A0B\u5EA6\uFF1A"),Fa=m(),x(fs.$$.fragment),Ha=m(),x(cs.$$.fragment),Ga=m(),Ns=o("p"),Do=a("\u6211\u4EEC\u7684\u7B2C\u4E8C\u4E2A\u641C\u7D22\u7ED3\u679C\u4F3C\u4E4E\u4E0E\u67E5\u8BE2\u76F8\u7B26\u3002"),Ma=m(),x(It.$$.fragment),this.h()},l(t){const l=di('[data-svelte="svelte-1phssyn"]',document.head);f=r(l,"META",{name:!0,content:!0}),l.forEach(e),g=u(t),k(c.$$.fragment,t),y=u(t),_=r(t,"H1",{class:!0});var $s=i(_);h=r($s,"A",{id:!0,class:!0,href:!0});var Os=i(h);D=r(Os,"SPAN",{});var He=i(D);k(w.$$.fragment,He),He.forEach(e),Os.forEach(e),A=u($s),E=r($s,"SPAN",{});var Ps=i(E);z=n(Ps,"\u4F7F\u7528 FAISS \u8FDB\u884C\u8BED\u4E49\u641C\u7D22"),Ps.forEach(e),$s.forEach(e),N=u(t),P.l(t),F=u(t),M=r(t,"P",{});var Ct=i(M);nt=n(Ct,"\u5728"),H=r(Ct,"A",{href:!0});var Fs=i(H);lt=n(Fs,"\u7B2C\u4E94\u5C0F\u8282"),Fs.forEach(e),ot=n(Ct,", \u6211\u4EEC\u4ECE \u{1F917} Datasets \u5B58\u50A8\u5E93\u521B\u5EFA\u4E86\u4E00\u4E2A\u5305\u542B GitHub \u95EE\u9898\u548C\u8BC4\u8BBA\u7684\u6570\u636E\u96C6\u3002\u5728\u672C\u8282\u4E2D\uFF0C\u6211\u4EEC\u5C06\u4F7F\u7528\u8FD9\u4E9B\u4FE1\u606F\u6765\u6784\u5EFA\u4E00\u4E2A\u641C\u7D22\u5F15\u64CE\uFF0C\u5B83\u53EF\u4EE5\u5E2E\u52A9\u6211\u4EEC\u627E\u5230\u8FD9\u4E2A\u5E93\u6700\u7D27\u8FEB\u95EE\u9898\u7684\u7B54\u6848\uFF01"),Ct.forEach(e),U=u(t),k(I.$$.fragment,t),Y=u(t),d=r(t,"H2",{id:!0});var Ge=i(d);S=n(Ge,"\u4F7F\u7528\u5D4C\u5165\u8FDB\u884C\u8BED\u4E49\u641C\u7D22"),Ge.forEach(e),C=u(t),B=r(t,"P",{});var za=i(B);vt=n(za,"\u6B63\u5982\u6211\u4EEC\u5728"),st=r(za,"A",{href:!0});var Mo=i(st);tn=n(Mo,"\u7B2C\u4E00\u7AE0"),Mo.forEach(e),sn=n(za,",\u5B66\u4E60\u7684, \u57FA\u4E8E Transformer \u7684\u8BED\u8A00\u6A21\u578B\u4F1A\u5C06\u6587\u672C\u4E2D\u7684\u6BCF\u4E2A\u6807\u8BB0\u8F6C\u6362\u4E3A\u5D4C\u5165\u5411\u91CF.\u4E8B\u5B9E\u8BC1\u660E\uFF0C\u53EF\u4EE5\u201C\u6C47\u96C6\u201D\u5404\u4E2A\u5D4C\u5165\u5411\u91CF\u6765\u521B\u5EFA\u6574\u4E2A\u53E5\u5B50\u3001\u6BB5\u843D\u6216\u6587\u6863\uFF08\u5728\u67D0\u4E9B\u60C5\u51B5\u4E0B\uFF09\u7684\u5411\u91CF\u8868\u793A\u3002\u7136\u540E\uFF0C\u901A\u8FC7\u8BA1\u7B97\u6BCF\u4E2A\u5D4C\u5165\u4E4B\u95F4\u7684\u70B9\u79EF\u76F8\u4F3C\u5EA6\uFF08\u6216\u5176\u4ED6\u4E00\u4E9B\u76F8\u4F3C\u5EA6\u5EA6\u91CF\uFF09\u5E76\u8FD4\u56DE\u76F8\u4F3C\u5EA6\u6700\u5927\u7684\u6587\u6863\uFF0C\u8FD9\u4E9B\u5D4C\u5165\u53EF\u7528\u4E8E\u5728\u8BED\u6599\u5E93\u4E2D\u627E\u5230\u76F8\u4F3C\u7684\u6587\u6863\u3002\u5728\u672C\u8282\u4E2D\uFF0C\u6211\u4EEC\u5C06\u4F7F\u7528\u5D4C\u5165\u6765\u5F00\u53D1\u8BED\u4E49\u641C\u7D22\u5F15\u64CE\u3002\u4E0E\u57FA\u4E8E\u5C06\u67E5\u8BE2\u4E2D\u7684\u5173\u952E\u5B57\u7684\u4F20\u7EDF\u65B9\u6CD5\u76F8\u6BD4\uFF0C\u8FD9\u4E9B\u641C\u7D22\u5F15\u64CE\u5177\u6709\u591A\u79CD\u4F18\u52BF\u3002"),za.forEach(e),Le=u(t),jt=r(t,"DIV",{class:!0});var Ua=i(jt);Pt=r(Ua,"IMG",{class:!0,src:!0,alt:!0}),en=u(Ua),Ft=r(Ua,"IMG",{class:!0,src:!0,alt:!0}),Ua.forEach(e),ze=u(t),Ht=r(t,"H2",{id:!0});var Lo=i(Ht);an=n(Lo,"## \u52A0\u8F7D\u548C\u51C6\u5907\u6570\u636E\u96C6"),Lo.forEach(e),Ue=u(t),ws=r(t,"P",{});var zo=i(ws);nn=n(zo,"\u6211\u4EEC\u9700\u8981\u505A\u7684\u7B2C\u4E00\u4EF6\u4E8B\u662F\u4E0B\u8F7D\u6211\u4EEC\u7684 GitHub \u95EE\u9898\u6570\u636E\u96C6\uFF0C\u6240\u4EE5\u8BA9\u6211\u4EEC\u4F7F\u7528 \u{1F917} Hub \u5E93\u6765\u89E3\u6790\u6211\u4EEC\u7684\u6587\u4EF6\u5728 Hugging Face Hub \u4E0A\u5B58\u50A8\u7684\u6570\u636E\uFF1A"),zo.forEach(e),Ye=u(t),k(Gt.$$.fragment,t),Be=u(t),ct=r(t,"P",{});var Hs=i(ct);ln=n(Hs,"\u5C06 URL \u5B58\u50A8\u5728 "),Ls=r(Hs,"STRONG",{});var Uo=i(Ls);on=n(Uo,"data_files"),Uo.forEach(e),rn=n(Hs," \uFF0C\u7136\u540E\u6211\u4EEC\u53EF\u4EE5\u4F7F\u7528"),xs=r(Hs,"A",{href:!0});var Yo=i(xs);pn=n(Yo,"\u7B2C\u4E8C\u5C0F\u8282"),Yo.forEach(e),mn=n(Hs,"\u4ECB\u7ECD\u7684\u65B9\u6CD5\u52A0\u8F7D\u8FDC\u7A0B\u6570\u636E\u96C6:"),Hs.forEach(e),We=u(t),k(Mt.$$.fragment,t),Ve=u(t),k(Lt.$$.fragment,t),Je=u(t),ks=r(t,"P",{});var Bo=i(ks);un=n(Bo,"\u8FD9\u91CC\u6211\u4EEC\u5728load_dataset()\u4E2D\u4F7F\u7528\u4E86\u9ED8\u8BA4\u7684\u8BAD\u7EC3\u96C6\u5206\u5272\uFF0C\u6240\u4EE5\u5B83\u8FD4\u56DE\u4E00\u4E2A\u6570\u636E\u96C6\u800C\u4E0D\u662F\u6570\u636E\u96C6\u5B57\u5178\u3002\u7B2C\u4E00\u9879\u4EFB\u52A1\u662F\u8FC7\u6EE4\u6389pull\u8BF7\u6C42\uFF0C\u56E0\u4E3A\u8FD9\u4E9B\u8BF7\u6C42\u5F88\u5C11\u7528\u4E8E\u56DE\u7B54\u7528\u6237\u63D0\u51FA\u7684\u95EE\u9898\uFF0C\u800C\u4E14\u4F1A\u7ED9\u6211\u4EEC\u7684\u641C\u7D22\u5F15\u64CE\u5E26\u6765\u566A\u58F0\u3002\u73B0\u5728\u5E94\u8BE5\u5F88\u719F\u6089\u4E86\uFF0C\u6211\u4EEC\u53EF\u4EE5\u4F7F\u7528dataset.filter()\u51FD\u6570\u6765\u6392\u9664\u6570\u636E\u96C6\u4E2D\u7684\u8FD9\u4E9B\u884C\u3002\u540C\u65F6\uFF0C\u8BA9\u6211\u4EEC\u4E5F\u8FC7\u6EE4\u6389\u6CA1\u6709\u6CE8\u91CA\u7684\u884C\uFF0C\u56E0\u4E3A\u8FD9\u4E9B\u884C\u4E0D\u4F1A\u662F\u7528\u6237\u63D0\u95EE\u7684\u7B54\u6848:"),Bo.forEach(e),Qe=u(t),k(zt.$$.fragment,t),Xe=u(t),k(Ut.$$.fragment,t),Ke=u(t),L=r(t,"P",{});var at=i(L);dn=n(at,"\u6211\u4EEC\u53EF\u4EE5\u770B\u5230\u6211\u4EEC\u7684\u6570\u636E\u96C6\u4E2D\u6709\u5F88\u591A\u5217\uFF0C\u5176\u4E2D\u5927\u90E8\u5206\u6211\u4EEC\u4E0D\u9700\u8981\u6784\u5EFA\u6211\u4EEC\u7684\u641C\u7D22\u5F15\u64CE\u3002\u4ECE\u641C\u7D22\u7684\u89D2\u5EA6\u6765\u770B\uFF0C\u4FE1\u606F\u91CF\u6700\u5927\u7684\u5217\u662F "),zs=r(at,"STRONG",{});var Wo=i(zs);fn=n(Wo,"title"),Wo.forEach(e),cn=n(at," , "),Us=r(at,"STRONG",{});var Vo=i(Us);_n=n(Vo,"body"),Vo.forEach(e),hn=n(at," \uFF0C \u548C "),Ys=r(at,"STRONG",{});var Jo=i(Ys);gn=n(Jo,"comments"),Jo.forEach(e),bn=n(at," \uFF0C\u800C "),Bs=r(at,"STRONG",{});var Qo=i(Bs);$n=n(Qo,"html_url"),Qo.forEach(e),wn=n(at," \u4E3A\u6211\u4EEC\u63D0\u4F9B\u4E86\u4E00\u4E2A\u56DE\u5230\u6E90\u95EE\u9898\u7684\u94FE\u63A5\u3002\u8BA9\u6211\u4EEC\u4F7F\u7528 "),Ws=r(at,"STRONG",{});var Xo=i(Ws);xn=n(Xo,"Dataset.remove_columns()"),Xo.forEach(e),kn=n(at," \u5220\u9664\u5176\u4F59\u90E8\u5206\u7684\u529F\u80FD\uFF1A"),at.forEach(e),Ze=u(t),k(Yt.$$.fragment,t),ta=u(t),k(Bt.$$.fragment,t),sa=u(t),W=r(t,"P",{});var gt=i(W);vn=n(gt,"\u4E3A\u4E86\u521B\u5EFA\u6211\u4EEC\u7684\u5D4C\u5165\uFF0C\u6211\u4EEC\u5C06\u7528\u95EE\u9898\u7684\u6807\u9898\u548C\u6B63\u6587\u6765\u6269\u5145\u6BCF\u6761\u8BC4\u8BBA\uFF0C\u56E0\u4E3A\u8FD9\u4E9B\u5B57\u6BB5\u901A\u5E38\u5305\u542B\u6709\u7528\u7684\u4E0A\u4E0B\u6587\u4FE1\u606F\u3002\u56E0\u4E3A\u6211\u4EEC\u7684 "),Vs=r(gt,"STRONG",{});var Ko=i(Vs);jn=n(Ko,"comments"),Ko.forEach(e),yn=n(gt," \u5217\u5F53\u524D\u662F\u6BCF\u4E2A\u95EE\u9898\u7684\u8BC4\u8BBA\u5217\u8868\uFF0C\u6211\u4EEC\u9700\u8981\u201C\u91CD\u65B0\u7EC4\u5408\u201D\u5217\uFF0C\u4EE5\u4FBF\u6BCF\u4E00\u6761\u8BC4\u8BBA\u90FD\u5305\u542B\u4E00\u4E2A "),Js=r(gt,"STRONG",{});var Zo=i(Js);En=n(Zo,"(html_url, title, body, comment)"),Zo.forEach(e),Tn=n(gt," \u5143\u7EC4\u3002\u5728 Pandas \u4E2D\uFF0C\u6211\u4EEC\u53EF\u4EE5\u4F7F\u7528 "),Wt=r(gt,"A",{href:!0,rel:!0});var tr=i(Wt);qn=n(tr,"DataFrame.explode() \u51FD\u6570"),tr.forEach(e),Sn=n(gt,", \u5B83\u4E3A\u7C7B\u4F3C\u5217\u8868\u7684\u5217\u4E2D\u7684\u6BCF\u4E2A\u5143\u7D20\u521B\u5EFA\u4E00\u4E2A\u65B0\u884C\uFF0C\u540C\u65F6\u590D\u5236\u6240\u6709\u5176\u4ED6\u5217\u503C\u3002\u4E3A\u4E86\u770B\u5230\u5B83\u7684\u5B9E\u9645\u6548\u679C\uFF0C\u8BA9\u6211\u4EEC\u9996\u5148\u5207\u6362\u5230 Pandas\u7684"),Qs=r(gt,"STRONG",{});var sr=i(Qs);Dn=n(sr,"DataFrame"),sr.forEach(e),An=n(gt," \u683C\u5F0F\uFF1A"),gt.forEach(e),ea=u(t),k(Vt.$$.fragment,t),aa=u(t),Tt=r(t,"P",{});var Ya=i(Tt);Rn=n(Ya,"\u5982\u679C\u6211\u4EEC\u68C0\u67E5\u8FD9\u91CC\u7684\u7B2C\u4E00\u884C "),Xs=r(Ya,"STRONG",{});var er=i(Xs);In=n(er,"DataFrame"),er.forEach(e),Cn=n(Ya," \u6211\u4EEC\u53EF\u4EE5\u770B\u5230\u6709\u56DB\u4E2A\u8BC4\u8BBA\u4E0E\u8FD9\u4E2A\u95EE\u9898\u76F8\u5173\uFF1A"),Ya.forEach(e),na=u(t),k(Jt.$$.fragment,t),la=u(t),k(Qt.$$.fragment,t),oa=u(t),vs=r(t,"P",{});var ar=i(vs);Nn=n(ar,"\u6211\u4EEC\u5E0C\u671B\u8FD9\u4E9B\u8BC4\u8BBA\u4E2D\u7684\u6BCF\u4E00\u6761\u90FD\u5F97\u5230\u4E00\u884C\u3002\u8BA9\u6211\u4EEC\u68C0\u67E5\u662F\u5426\u662F\u8FD9\u79CD\u60C5\u51B5\uFF1A"),ar.forEach(e),ra=u(t),k(Xt.$$.fragment,t),ia=u(t),Q=r(t,"TABLE",{border:!0,class:!0,style:!0});var Ba=i(Q);Ks=r(Ba,"THEAD",{});var nr=i(Ks);V=r(nr,"TR",{style:!0});var bt=i(V);pa=r(bt,"TH",{}),i(pa).forEach(e),On=u(bt),Zs=r(bt,"TH",{});var lr=i(Zs);Pn=n(lr,"html_url"),lr.forEach(e),Fn=u(bt),te=r(bt,"TH",{});var or=i(te);Hn=n(or,"title"),or.forEach(e),Gn=u(bt),se=r(bt,"TH",{});var rr=i(se);Mn=n(rr,"comments"),rr.forEach(e),Ln=u(bt),ee=r(bt,"TH",{});var ir=i(ee);zn=n(ir,"body"),ir.forEach(e),bt.forEach(e),nr.forEach(e),Un=u(Ba),rt=r(Ba,"TBODY",{});var Nt=i(rt);X=r(Nt,"TR",{});var $t=i(X);ae=r($t,"TH",{});var pr=i(ae);Yn=n(pr,"0"),pr.forEach(e),Bn=u($t),ne=r($t,"TD",{});var mr=i(ne);Wn=n(mr,"https://github.com/huggingface/datasets/issues/2787"),mr.forEach(e),Vn=u($t),le=r($t,"TD",{});var ur=i(le);Jn=n(ur,"ConnectionError: Couldn't reach https://raw.githubusercontent.com"),ur.forEach(e),Qn=u($t),oe=r($t,"TD",{});var dr=i(oe);Xn=n(dr,"the bug code locate in \uFF1A\\r\\n    if data_args.task_name is not None..."),dr.forEach(e),Kn=u($t),re=r($t,"TD",{});var fr=i(re);Zn=n(fr,"Hello,\\r\\nI am trying to run run_glue.py and it gives me this error..."),fr.forEach(e),$t.forEach(e),tl=u(Nt),K=r(Nt,"TR",{});var wt=i(K);ie=r(wt,"TH",{});var cr=i(ie);sl=n(cr,"1"),cr.forEach(e),el=u(wt),pe=r(wt,"TD",{});var _r=i(pe);al=n(_r,"https://github.com/huggingface/datasets/issues/2787"),_r.forEach(e),nl=u(wt),me=r(wt,"TD",{});var hr=i(me);ll=n(hr,"ConnectionError: Couldn't reach https://raw.githubusercontent.com"),hr.forEach(e),ol=u(wt),ue=r(wt,"TD",{});var gr=i(ue);rl=n(gr,"Hi @jinec,\\r\\n\\r\\nFrom time to time we get this kind of `ConnectionError` coming from the github.com website: https://raw.githubusercontent.com..."),gr.forEach(e),il=u(wt),de=r(wt,"TD",{});var br=i(de);pl=n(br,"Hello,\\r\\nI am trying to run run_glue.py and it gives me this error..."),br.forEach(e),wt.forEach(e),ml=u(Nt),Z=r(Nt,"TR",{});var xt=i(Z);fe=r(xt,"TH",{});var $r=i(fe);ul=n($r,"2"),$r.forEach(e),dl=u(xt),ce=r(xt,"TD",{});var wr=i(ce);fl=n(wr,"https://github.com/huggingface/datasets/issues/2787"),wr.forEach(e),cl=u(xt),_e=r(xt,"TD",{});var xr=i(_e);_l=n(xr,"ConnectionError: Couldn't reach https://raw.githubusercontent.com"),xr.forEach(e),hl=u(xt),he=r(xt,"TD",{});var kr=i(he);gl=n(kr,"cannot connect\uFF0Ceven by Web browser\uFF0Cplease check that  there is some  problems\u3002"),kr.forEach(e),bl=u(xt),ge=r(xt,"TD",{});var vr=i(ge);$l=n(vr,"Hello,\\r\\nI am trying to run run_glue.py and it gives me this error..."),vr.forEach(e),xt.forEach(e),wl=u(Nt),tt=r(Nt,"TR",{});var kt=i(tt);be=r(kt,"TH",{});var jr=i(be);xl=n(jr,"3"),jr.forEach(e),kl=u(kt),$e=r(kt,"TD",{});var yr=i($e);vl=n(yr,"https://github.com/huggingface/datasets/issues/2787"),yr.forEach(e),jl=u(kt),we=r(kt,"TD",{});var Er=i(we);yl=n(Er,"ConnectionError: Couldn't reach https://raw.githubusercontent.com"),Er.forEach(e),El=u(kt),xe=r(kt,"TD",{});var Tr=i(xe);Tl=n(Tr,"I can access https://raw.githubusercontent.com/huggingface/datasets/1.7.0/datasets/glue/glue.py without problem..."),Tr.forEach(e),ql=u(kt),ke=r(kt,"TD",{});var qr=i(ke);Sl=n(qr,"Hello,\\r\\nI am trying to run run_glue.py and it gives me this error..."),qr.forEach(e),kt.forEach(e),Nt.forEach(e),Ba.forEach(e),ma=u(t),et=r(t,"P",{});var Ot=i(et);Dl=n(Ot,"\u592A\u597D\u4E86\uFF0C\u6211\u4EEC\u53EF\u4EE5\u770B\u5230\u8BC4\u8BBA\u6210\u529F\u88AB\u6269\u5145\uFF0C "),ve=r(Ot,"STRONG",{});var Sr=i(ve);Al=n(Sr,"comments"),Sr.forEach(e),Rl=n(Ot," \u662F\u5305\u542B\u4E2A\u4EBA\u8BC4\u8BBA\u7684\u5217\uFF01\u73B0\u5728\u6211\u4EEC\u5DF2\u7ECF\u5B8C\u6210\u4E86 Pandas\u8981\u5B8C\u6210\u7684\u90E8\u5206\u529F\u80FD\uFF0C\u6211\u4EEC\u53EF\u4EE5\u5FEB\u901F\u5207\u6362\u56DE "),je=r(Ot,"STRONG",{});var Dr=i(je);Il=n(Dr,"Dataset"),Dr.forEach(e),Cl=n(Ot," \u901A\u8FC7\u52A0\u8F7D "),ye=r(Ot,"STRONG",{});var Ar=i(ye);Nl=n(Ar,"DataFrame"),Ar.forEach(e),Ol=n(Ot," \u5728\u5185\u5B58\u4E2D\uFF1A"),Ot.forEach(e),ua=u(t),k(Kt.$$.fragment,t),da=u(t),k(Zt.$$.fragment,t),fa=u(t),js=r(t,"P",{});var Rr=i(js);Pl=n(Rr,"\u592A\u597D\u4E86\uFF0C\u6211\u4EEC\u83B7\u53D6\u5230\u4E86\u51E0\u5343\u6761\u7684\u8BC4\u8BBA\uFF01"),Rr.forEach(e),ca=u(t),k(qt.$$.fragment,t),_a=u(t),St=r(t,"P",{});var Wa=i(St);Fl=n(Wa,"\u73B0\u5728\u6211\u4EEC\u6BCF\u884C\u6709\u4E00\u4E2A\u8BC4\u8BBA\uFF0C\u8BA9\u6211\u4EEC\u521B\u5EFA\u4E00\u4E2A\u65B0\u7684 "),Ee=r(Wa,"STRONG",{});var Ir=i(Ee);Hl=n(Ir,"comments_length"),Ir.forEach(e),Gl=n(Wa," \u5217\u6765\u5B58\u653E\u6BCF\u6761\u8BC4\u8BBA\u7684\u5B57\u6570\uFF1A"),Wa.forEach(e),ha=u(t),k(ts.$$.fragment,t),ga=u(t),ys=r(t,"P",{});var Cr=i(ys);Ml=n(Cr,"\u6211\u4EEC\u53EF\u4EE5\u4F7F\u7528\u8FD9\u4E2A\u65B0\u5217\u6765\u8FC7\u6EE4\u6389\u7B80\u77ED\u7684\u8BC4\u8BBA\uFF0C\u5176\u4E2D\u901A\u5E38\u5305\u62EC\u201Ccc @lewtun\u201D\u6216\u201C\u8C22\u8C22\uFF01\u201D\u4E4B\u7C7B\u4E0E\u6211\u4EEC\u7684\u641C\u7D22\u5F15\u64CE\u65E0\u5173\u7684\u5185\u5BB9\u3002\u867D\u7136\u65E0\u6CD5\u4E3A\u8FC7\u6EE4\u5668\u9009\u62E9\u7684\u7CBE\u786E\u6570\u5B57\uFF0C\u4F46\u5927\u7EA6\u5927\u4E8E15 \u4E2A\u5355\u8BCD\u4F3C\u4E4E\u662F\u4E00\u4E2A\u4E0D\u9519\u7684\u9009\u62E9\uFF1A"),Cr.forEach(e),ba=u(t),k(ss.$$.fragment,t),$a=u(t),k(es.$$.fragment,t),wa=u(t),_t=r(t,"P",{});var Gs=i(_t);Ll=n(Gs,"\u7A0D\u5FAE\u6E05\u7406\u4E86\u6211\u4EEC\u7684\u6570\u636E\u96C6\u540E\uFF0C\u8BA9\u6211\u4EEC\u5C06\u95EE\u9898\u6807\u9898\u3001\u63CF\u8FF0\u548C\u8BC4\u8BBA\u8FDE\u63A5\u5230\u4E00\u4E2A\u65B0\u7684 "),Te=r(Gs,"STRONG",{});var Nr=i(Te);zl=n(Nr,"text"),Nr.forEach(e),Ul=n(Gs," \u5217\u3002\u50CF\u5F80\u5E38\u4E00\u6837\uFF0C\u6211\u4EEC\u53EF\u4EE5\u7F16\u5199\u4E00\u4E2A\u7B80\u5355\u7684\u51FD\u6570\uFF0C\u5E76\u5C06\u5176\u4F20\u9012\u7ED9 "),qe=r(Gs,"STRONG",{});var Or=i(qe);Yl=n(Or,"Dataset.map()"),Or.forEach(e),Bl=n(Gs,"\u6765\u505A\u5230\u8FD9\u4E9B \uFF1A"),Gs.forEach(e),xa=u(t),k(as.$$.fragment,t),ka=u(t),Es=r(t,"P",{});var Pr=i(Es);Wl=n(Pr,"\u6211\u4EEC\u7EC8\u4E8E\u51C6\u5907\u597D\u521B\u5EFA\u4E00\u4E9B\u5D4C\u5165\u4E86\uFF01\u8BA9\u6211\u4EEC\u6765\u770B\u770B\u3002"),Pr.forEach(e),va=u(t),ns=r(t,"H2",{id:!0});var Fr=i(ns);Vl=n(Fr,"\u521B\u5EFA\u6587\u672C\u5D4C\u5165"),Fr.forEach(e),ja=u(t),G=r(t,"P",{});var J=i(G);Jl=n(J,"\u6211\u4EEC\u5728"),Ts=r(J,"A",{href:!0});var Hr=i(Ts);Ql=n(Hr,"\u7B2C\u4E8C\u7AE0"),Hr.forEach(e),Xl=n(J," \u5B66\u8FC7\uFF0C\u6211\u4EEC\u53EF\u4EE5\u901A\u8FC7\u4F7F\u7528 "),Se=r(J,"STRONG",{});var Gr=i(Se);Kl=n(Gr,"AutoModel"),Gr.forEach(e),Zl=n(J," \u7C7B\u6765\u5B8C\u6210\u8BCD\u5D4C\u5165\u3002\u6211\u4EEC\u9700\u8981\u505A\u7684\u5C31\u662F\u9009\u62E9\u4E00\u4E2A\u5408\u9002\u7684\u68C0\u67E5\u70B9\u6765\u52A0\u8F7D\u6A21\u578B\u3002\u5E78\u8FD0\u7684\u662F\uFF0C\u6709\u4E00\u4E2A\u540D\u4E3A "),De=r(J,"STRONG",{});var Mr=i(De);to=n(Mr,"sentence-transformers"),Mr.forEach(e),so=n(J," \u4E13\u95E8\u7528\u4E8E\u521B\u5EFA\u8BCD\u5D4C\u5165\u3002\u5982\u5E93\u4E2D"),ls=r(J,"A",{href:!0,rel:!0});var Lr=i(ls);eo=n(Lr,"\u6587\u6863"),Lr.forEach(e),ao=n(J,", \u6240\u8FF0\u7684\uFF0C\u6211\u4EEC\u8FD9\u6B21\u8981\u5B9E\u73B0\u7684\u662F\u975E\u5BF9\u79F0\u8BED\u4E49\u641C\u7D22\uFF0C\u56E0\u4E3A\u6211\u4EEC\u6709\u4E00\u4E2A\u7B80\u77ED\u7684\u67E5\u8BE2\uFF0C\u6211\u4EEC\u5E0C\u671B\u5728\u6BD4\u5982\u95EE\u9898\u8BC4\u8BBA\u7B49\u66F4\u957F\u7684\u6587\u6863\u4E2D\u627E\u5230\u5176\u7B54\u6848\u3002\u901A\u8FC7\u67E5\u770B"),os=r(J,"A",{href:!0,rel:!0});var zr=i(os);no=n(zr,"\u6A21\u578B\u6982\u8FF0\u8868"),zr.forEach(e),lo=n(J," \u6211\u4EEC\u53EF\u4EE5\u53D1\u73B0 "),Ae=r(J,"STRONG",{});var Ur=i(Ae);oo=n(Ur,"multi-qa-mpnet-base-dot-v1"),Ur.forEach(e),ro=n(J," \u68C0\u67E5\u70B9\u5728\u8BED\u4E49\u641C\u7D22\u65B9\u9762\u5177\u6709\u6700\u4F73\u6027\u80FD\uFF0C\u56E0\u6B64\u6211\u4EEC\u5C06\u5728\u6211\u4EEC\u7684\u5E94\u7528\u7A0B\u5E8F\u4E2D\u4F7F\u7528\u5B83\u3002\u6211\u4EEC\u8FD8\u5C06\u4F7F\u7528\u76F8\u540C\u7684\u68C0\u67E5\u70B9\u52A0\u8F7D\u6807\u8BB0\u5668\uFF1A"),J.forEach(e),ya=u(t),pt.l(t),qs=u(t),Dt=r(t,"P",{});var Va=i(Dt);io=n(Va,"\u6B63\u5982\u6211\u4EEC\u4E4B\u524D\u63D0\u5230\u7684\uFF0C\u6211\u4EEC\u5E0C\u671B\u5C06 GitHub \u95EE\u9898\u8BED\u6599\u5E93\u4E2D\u7684\u6BCF\u4E2A\u6761\u76EE\u8868\u793A\u4E3A\u5355\u4E2A\u5411\u91CF\uFF0C\u56E0\u6B64\u6211\u4EEC\u9700\u8981\u4EE5\u67D0\u79CD\u65B9\u5F0F\u201C\u6C60\u5316\u201D\u6216\u5E73\u5747\u5316\u6211\u4EEC\u7684\u6807\u8BB0\u5D4C\u5165\u3002\u4E00\u79CD\u6D41\u884C\u7684\u65B9\u6CD5\u662F\u5728\u6211\u4EEC\u6A21\u578B\u7684\u8F93\u51FA\u4E0A\u6267\u884CCLS \u6C60\u5316\uFF0C\u6211\u4EEC\u53EA\u83B7\u53D6"),Re=r(Va,"STRONG",{});var Yr=i(Re);po=n(Yr,"[CLS]"),Yr.forEach(e),mo=n(Va," \u4EE4\u724C\u7684\u6700\u540E\u4E00\u4E2A\u9690\u85CF\u72B6\u6001\u3002\u4EE5\u4E0B\u51FD\u6570\u4E3A\u6211\u4EEC\u63D0\u4F9B\u4E86\u8FD9\u6837\u7684\u65B9\u6CD5\uFF1A"),Va.forEach(e),Ea=u(t),k(rs.$$.fragment,t),Ta=u(t),Ss=r(t,"P",{});var Br=i(Ss);uo=n(Br,"\u63A5\u4E0B\u6765\uFF0C\u6211\u4EEC\u5C06\u521B\u5EFA\u4E00\u4E2A\u8F85\u52A9\u51FD\u6570\uFF0C\u8BE5\u51FD\u6570\u5C06\u6807\u8BB0\u6587\u6863\u5217\u8868\uFF0C\u5C06tensor\u653E\u5728 GPU \u4E0A\uFF0C\u7136\u540E\u63D0\u4F9B\u7ED9\u6A21\u578B\uFF0C\u6700\u540E\u5BF9\u8F93\u51FA\u4F7F\u7528CLS \u6C60\u5316\uFF1A"),Br.forEach(e),qa=u(t),ut.l(t),Ds=u(t),As=r(t,"P",{});var Wr=i(As);fo=n(Wr,"\u8BF7\u6CE8\u610F\uFF0C\u6211\u4EEC\u5DF2\u7ECF\u5C06\u5D4C\u5165\u8F6C\u6362\u4E3A NumPy \u6570\u7EC4\u2014\u2014\u8FD9\u662F\u56E0\u4E3A\u5F53\u6211\u4EEC\u5C1D\u8BD5\u4F7F\u7528 FAISS \u7D22\u5F15\u5B83\u4EEC\u65F6\uFF0C\u{1F917} Datasets\u9700\u8981\u8FD9\u79CD\u683C\u5F0F\uFF0C\u6211\u4EEC\u63A5\u4E0B\u6765\u4F1A\u8FD9\u6837\u505A\u3002"),Wr.forEach(e),Sa=u(t),yt=r(t,"H2",{class:!0});var Ja=i(yt);At=r(Ja,"A",{id:!0,class:!0,href:!0});var Vr=i(At);Ie=r(Vr,"SPAN",{});var Jr=i(Ie);k(is.$$.fragment,Jr),Jr.forEach(e),Vr.forEach(e),co=u(Ja),Ce=r(Ja,"SPAN",{});var Qr=i(Ce);_o=n(Qr,"\u4F7F\u7528 FAISS \u8FDB\u884C\u9AD8\u6548\u7684\u76F8\u4F3C\u6027\u641C\u7D22"),Qr.forEach(e),Ja.forEach(e),Da=u(t),ht=r(t,"P",{});var Ms=i(ht);ho=n(Ms,"\u73B0\u5728\u6211\u4EEC\u6709\u4E86\u4E00\u4E2A\u8BCD\u5D4C\u5165\u6570\u636E\u96C6\uFF0C\u6211\u4EEC\u9700\u8981\u4E00\u4E9B\u65B9\u6CD5\u6765\u641C\u7D22\u5B83\u4EEC\u3002\u4E3A\u6B64\uFF0C\u6211\u4EEC\u5C06\u5728 \u{1F917} Datasets\u4E2D\u4F7F\u7528\u4E00\u79CD\u7279\u6B8A\u7684\u6570\u636E\u7ED3\u6784\uFF0C\u79F0\u4E3A FAISS\u6307\u6570."),ps=r(Ms,"A",{href:!0,rel:!0});var Xr=i(ps);go=n(Xr,"FAISS"),Xr.forEach(e),bo=n(Ms," (short for Facebook AI Similarity Search) \uFF08Facebook AI Similarity Search \u7684\u7F29\u5199\uFF09\u662F\u4E00\u4E2A\u63D0\u4F9B\u9AD8\u6548\u7B97\u6CD5\u6765\u5FEB\u901F\u641C\u7D22\u548C\u805A\u7C7B\u5D4C\u5165\u5411\u91CF\u7684\u5E93\u3002FAISS \u80CC\u540E\u7684\u57FA\u672C\u601D\u60F3\u662F\u521B\u5EFA\u4E00\u4E2A\u7279\u6B8A\u7684\u6570\u636E\u7ED3\u6784\uFF0C\u79F0\u4E3A\u6307\u6570\u3002\u8FD9\u5141\u8BB8\u4EBA\u4EEC\u627E\u5230\u54EA\u4E9B\u5D4C\u5165\u8BCD\u4E0E\u8F93\u5165\u7684\u8BCD\u5D4C\u5165\u76F8\u4F3C\u3002\u5728 \u{1F917} Datasets\u4E2D\u521B\u5EFA\u4E00\u4E2A FAISS \u7D22\u5F15\u5F88\u7B80\u5355\u2014\u2014\u6211\u4EEC\u4F7F\u7528 "),Ne=r(Ms,"STRONG",{});var Kr=i(Ne);$o=n(Kr,"Dataset.add_faiss_index()"),Kr.forEach(e),wo=n(Ms," \u51FD\u6570\u5E76\u6307\u5B9A\u6211\u4EEC\u8981\u7D22\u5F15\u7684\u6570\u636E\u96C6\u7684\u54EA\u4E00\u5217\uFF1A"),Ms.forEach(e),Aa=u(t),k(ms.$$.fragment,t),Ra=u(t),Rt=r(t,"P",{});var Qa=i(Rt);xo=n(Qa,"\u73B0\u5728\uFF0C\u6211\u4EEC\u53EF\u4EE5\u4F7F\u7528"),Oe=r(Qa,"STRONG",{});var Zr=i(Oe);ko=n(Zr,"Dataset.get_nearest_examples()"),Zr.forEach(e),vo=n(Qa,"\u51FD\u6570\u8FDB\u884C\u6700\u8FD1\u90BB\u5C45\u67E5\u627E\u3002\u8BA9\u6211\u4EEC\u901A\u8FC7\u9996\u5148\u5D4C\u5165\u4E00\u4E2A\u95EE\u9898\u6765\u6D4B\u8BD5\u8FD9\u4E00\u70B9\uFF0C\u5982\u4E0B\u6240\u793A\uFF1A"),Qa.forEach(e),Ia=u(t),ft.l(t),Rs=u(t),Is=r(t,"P",{});var ti=i(Is);jo=n(ti,"\u5C31\u50CF\u6587\u6863\u4E00\u6837\uFF0C\u6211\u4EEC\u73B0\u5728\u6709\u4E00\u4E2A 768 \u7EF4\u5411\u91CF\u8868\u793A\u67E5\u8BE2\uFF0C\u6211\u4EEC\u53EF\u4EE5\u5C06\u5176\u4E0E\u6574\u4E2A\u8BED\u6599\u5E93\u8FDB\u884C\u6BD4\u8F83\u4EE5\u627E\u5230\u6700\u76F8\u4F3C\u7684\u5D4C\u5165\uFF1A"),ti.forEach(e),Ca=u(t),k(us.$$.fragment,t),Na=u(t),Et=r(t,"P",{});var Me=i(Et);Pe=r(Me,"STRONG",{});var si=i(Pe);yo=n(si,"Dataset.get_nearest_examples()"),si.forEach(e),Eo=n(Me," \u51FD\u6570\u8FD4\u56DE\u4E00\u4E2A\u5206\u6570\u5143\u7EC4\uFF0C\u5BF9\u67E5\u8BE2\u548C\u6587\u6863\u4E4B\u95F4\u7684\u76F8\u4F3C\u5EA6\u8FDB\u884C\u6392\u5E8F\uFF0C\u4EE5\u53CA\u4E00\u7EC4\u6700\u4F73\u5339\u914D\u7684\u7ED3\u679C\uFF08\u8FD9\u91CC\u662F 5 \u4E2A\uFF09\u3002\u8BA9\u6211\u4EEC\u628A\u8FD9\u4E9B\u6536\u96C6\u5230\u4E00\u4E2A "),Fe=r(Me,"STRONG",{});var ei=i(Fe);To=n(ei,"pandas.DataFrame"),ei.forEach(e),qo=n(Me," \u4EE5\u4FBF\u6211\u4EEC\u53EF\u4EE5\u8F7B\u677E\u5730\u5BF9\u5B83\u4EEC\u8FDB\u884C\u6392\u5E8F\uFF1A"),Me.forEach(e),Oa=u(t),k(ds.$$.fragment,t),Pa=u(t),Cs=r(t,"P",{});var ai=i(Cs);So=n(ai,"\u73B0\u5728\u6211\u4EEC\u53EF\u4EE5\u904D\u5386\u524D\u51E0\u884C\u6765\u67E5\u770B\u6211\u4EEC\u7684\u67E5\u8BE2\u4E0E\u8BC4\u8BBA\u7684\u5339\u914D\u7A0B\u5EA6\uFF1A"),ai.forEach(e),Fa=u(t),k(fs.$$.fragment,t),Ha=u(t),k(cs.$$.fragment,t),Ga=u(t),Ns=r(t,"P",{});var ni=i(Ns);Do=n(ni,"\u6211\u4EEC\u7684\u7B2C\u4E8C\u4E2A\u641C\u7D22\u7ED3\u679C\u4F3C\u4E4E\u4E0E\u67E5\u8BE2\u76F8\u7B26\u3002"),ni.forEach(e),Ma=u(t),k(It.$$.fragment,t),this.h()},h(){q(f,"name","hf:doc:metadata"),q(f,"content",JSON.stringify(Ti)),q(h,"id","faiss"),q(h,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),q(h,"href","#faiss"),q(_,"class","relative group"),q(H,"href","/course/chapter5/5"),q(d,"id",""),q(st,"href","/course/chapter1"),q(Pt,"class","block dark:hidden"),li(Pt.src,Ao="https://huggingface.co/datasets/huggingface-course/documentation-images/resolve/main/en/chapter5/semantic-search.svg")||q(Pt,"src",Ao),q(Pt,"alt","Semantic search."),q(Ft,"class","hidden dark:block"),li(Ft.src,Ro="https://huggingface.co/datasets/huggingface-course/documentation-images/resolve/main/en/chapter5/semantic-search-dark.svg")||q(Ft,"src",Ro),q(Ft,"alt","Semantic search."),q(jt,"class","flex justify-center"),q(Ht,"id",""),q(xs,"href","/course/chapter5/2"),q(Wt,"href","https://pandas.pydata.org/pandas-docs/stable/reference/api/pandas.DataFrame.explode.html"),q(Wt,"rel","nofollow"),Xa(V,"text-align","right"),q(Q,"border","1"),q(Q,"class","dataframe"),Xa(Q,"table-layout","fixed"),Xa(Q,"word-wrap","break-word"),Xa(Q,"width","100%"),q(ns,"id",""),q(Ts,"href","/course/chapter2"),q(ls,"href","https://www.sbert.net/examples/applications/semantic-search/README.html#symmetric-vs-asymmetric-semantic-search"),q(ls,"rel","nofollow"),q(os,"href","https://www.sbert.net/docs/pretrained_models.html#model-overview"),q(os,"rel","nofollow"),q(At,"id","faiss"),q(At,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),q(At,"href","#faiss"),q(yt,"class","relative group"),q(ps,"href","https://faiss.ai/"),q(ps,"rel","nofollow")},m(t,l){s(document.head,f),p(t,g,l),v(c,t,l),p(t,y,l),p(t,_,l),s(_,h),s(h,D),v(w,D,null),s(_,A),s(_,E),s(E,z),p(t,N,l),_s[O].m(t,l),p(t,F,l),p(t,M,l),s(M,nt),s(M,H),s(H,lt),s(M,ot),p(t,U,l),v(I,t,l),p(t,Y,l),p(t,d,l),s(d,S),p(t,C,l),p(t,B,l),s(B,vt),s(B,st),s(st,tn),s(B,sn),p(t,Le,l),p(t,jt,l),s(jt,Pt),s(jt,en),s(jt,Ft),p(t,ze,l),p(t,Ht,l),s(Ht,an),p(t,Ue,l),p(t,ws,l),s(ws,nn),p(t,Ye,l),v(Gt,t,l),p(t,Be,l),p(t,ct,l),s(ct,ln),s(ct,Ls),s(Ls,on),s(ct,rn),s(ct,xs),s(xs,pn),s(ct,mn),p(t,We,l),v(Mt,t,l),p(t,Ve,l),v(Lt,t,l),p(t,Je,l),p(t,ks,l),s(ks,un),p(t,Qe,l),v(zt,t,l),p(t,Xe,l),v(Ut,t,l),p(t,Ke,l),p(t,L,l),s(L,dn),s(L,zs),s(zs,fn),s(L,cn),s(L,Us),s(Us,_n),s(L,hn),s(L,Ys),s(Ys,gn),s(L,bn),s(L,Bs),s(Bs,$n),s(L,wn),s(L,Ws),s(Ws,xn),s(L,kn),p(t,Ze,l),v(Yt,t,l),p(t,ta,l),v(Bt,t,l),p(t,sa,l),p(t,W,l),s(W,vn),s(W,Vs),s(Vs,jn),s(W,yn),s(W,Js),s(Js,En),s(W,Tn),s(W,Wt),s(Wt,qn),s(W,Sn),s(W,Qs),s(Qs,Dn),s(W,An),p(t,ea,l),v(Vt,t,l),p(t,aa,l),p(t,Tt,l),s(Tt,Rn),s(Tt,Xs),s(Xs,In),s(Tt,Cn),p(t,na,l),v(Jt,t,l),p(t,la,l),v(Qt,t,l),p(t,oa,l),p(t,vs,l),s(vs,Nn),p(t,ra,l),v(Xt,t,l),p(t,ia,l),p(t,Q,l),s(Q,Ks),s(Ks,V),s(V,pa),s(V,On),s(V,Zs),s(Zs,Pn),s(V,Fn),s(V,te),s(te,Hn),s(V,Gn),s(V,se),s(se,Mn),s(V,Ln),s(V,ee),s(ee,zn),s(Q,Un),s(Q,rt),s(rt,X),s(X,ae),s(ae,Yn),s(X,Bn),s(X,ne),s(ne,Wn),s(X,Vn),s(X,le),s(le,Jn),s(X,Qn),s(X,oe),s(oe,Xn),s(X,Kn),s(X,re),s(re,Zn),s(rt,tl),s(rt,K),s(K,ie),s(ie,sl),s(K,el),s(K,pe),s(pe,al),s(K,nl),s(K,me),s(me,ll),s(K,ol),s(K,ue),s(ue,rl),s(K,il),s(K,de),s(de,pl),s(rt,ml),s(rt,Z),s(Z,fe),s(fe,ul),s(Z,dl),s(Z,ce),s(ce,fl),s(Z,cl),s(Z,_e),s(_e,_l),s(Z,hl),s(Z,he),s(he,gl),s(Z,bl),s(Z,ge),s(ge,$l),s(rt,wl),s(rt,tt),s(tt,be),s(be,xl),s(tt,kl),s(tt,$e),s($e,vl),s(tt,jl),s(tt,we),s(we,yl),s(tt,El),s(tt,xe),s(xe,Tl),s(tt,ql),s(tt,ke),s(ke,Sl),p(t,ma,l),p(t,et,l),s(et,Dl),s(et,ve),s(ve,Al),s(et,Rl),s(et,je),s(je,Il),s(et,Cl),s(et,ye),s(ye,Nl),s(et,Ol),p(t,ua,l),v(Kt,t,l),p(t,da,l),v(Zt,t,l),p(t,fa,l),p(t,js,l),s(js,Pl),p(t,ca,l),v(qt,t,l),p(t,_a,l),p(t,St,l),s(St,Fl),s(St,Ee),s(Ee,Hl),s(St,Gl),p(t,ha,l),v(ts,t,l),p(t,ga,l),p(t,ys,l),s(ys,Ml),p(t,ba,l),v(ss,t,l),p(t,$a,l),v(es,t,l),p(t,wa,l),p(t,_t,l),s(_t,Ll),s(_t,Te),s(Te,zl),s(_t,Ul),s(_t,qe),s(qe,Yl),s(_t,Bl),p(t,xa,l),v(as,t,l),p(t,ka,l),p(t,Es,l),s(Es,Wl),p(t,va,l),p(t,ns,l),s(ns,Vl),p(t,ja,l),p(t,G,l),s(G,Jl),s(G,Ts),s(Ts,Ql),s(G,Xl),s(G,Se),s(Se,Kl),s(G,Zl),s(G,De),s(De,to),s(G,so),s(G,ls),s(ls,eo),s(G,ao),s(G,os),s(os,no),s(G,lo),s(G,Ae),s(Ae,oo),s(G,ro),p(t,ya,l),hs[it].m(t,l),p(t,qs,l),p(t,Dt,l),s(Dt,io),s(Dt,Re),s(Re,po),s(Dt,mo),p(t,Ea,l),v(rs,t,l),p(t,Ta,l),p(t,Ss,l),s(Ss,uo),p(t,qa,l),gs[mt].m(t,l),p(t,Ds,l),p(t,As,l),s(As,fo),p(t,Sa,l),p(t,yt,l),s(yt,At),s(At,Ie),v(is,Ie,null),s(yt,co),s(yt,Ce),s(Ce,_o),p(t,Da,l),p(t,ht,l),s(ht,ho),s(ht,ps),s(ps,go),s(ht,bo),s(ht,Ne),s(Ne,$o),s(ht,wo),p(t,Aa,l),v(ms,t,l),p(t,Ra,l),p(t,Rt,l),s(Rt,xo),s(Rt,Oe),s(Oe,ko),s(Rt,vo),p(t,Ia,l),bs[dt].m(t,l),p(t,Rs,l),p(t,Is,l),s(Is,jo),p(t,Ca,l),v(us,t,l),p(t,Na,l),p(t,Et,l),s(Et,Pe),s(Pe,yo),s(Et,Eo),s(Et,Fe),s(Fe,To),s(Et,qo),p(t,Oa,l),v(ds,t,l),p(t,Pa,l),p(t,Cs,l),s(Cs,So),p(t,Fa,l),v(fs,t,l),p(t,Ha,l),v(cs,t,l),p(t,Ga,l),p(t,Ns,l),s(Ns,Do),p(t,Ma,l),v(It,t,l),La=!0},p(t,[l]){const $s={};l&1&&($s.fw=t[0]),c.$set($s);let Os=O;O=Co(t),O!==Os&&(Za(),b(_s[Os],1,1,()=>{_s[Os]=null}),Ka(),P=_s[O],P||(P=_s[O]=Io[O](t),P.c()),$(P,1),P.m(F.parentNode,F));const He={};l&2&&(He.$$scope={dirty:l,ctx:t}),qt.$set(He);let Ps=it;it=Oo(t),it!==Ps&&(Za(),b(hs[Ps],1,1,()=>{hs[Ps]=null}),Ka(),pt=hs[it],pt||(pt=hs[it]=No[it](t),pt.c()),$(pt,1),pt.m(qs.parentNode,qs));let Ct=mt;mt=Fo(t),mt!==Ct&&(Za(),b(gs[Ct],1,1,()=>{gs[Ct]=null}),Ka(),ut=gs[mt],ut||(ut=gs[mt]=Po[mt](t),ut.c()),$(ut,1),ut.m(Ds.parentNode,Ds));let Fs=dt;dt=Go(t),dt!==Fs&&(Za(),b(bs[Fs],1,1,()=>{bs[Fs]=null}),Ka(),ft=bs[dt],ft||(ft=bs[dt]=Ho[dt](t),ft.c()),$(ft,1),ft.m(Rs.parentNode,Rs));const Ge={};l&2&&(Ge.$$scope={dirty:l,ctx:t}),It.$set(Ge)},i(t){La||($(c.$$.fragment,t),$(w.$$.fragment,t),$(P),$(I.$$.fragment,t),$(Gt.$$.fragment,t),$(Mt.$$.fragment,t),$(Lt.$$.fragment,t),$(zt.$$.fragment,t),$(Ut.$$.fragment,t),$(Yt.$$.fragment,t),$(Bt.$$.fragment,t),$(Vt.$$.fragment,t),$(Jt.$$.fragment,t),$(Qt.$$.fragment,t),$(Xt.$$.fragment,t),$(Kt.$$.fragment,t),$(Zt.$$.fragment,t),$(qt.$$.fragment,t),$(ts.$$.fragment,t),$(ss.$$.fragment,t),$(es.$$.fragment,t),$(as.$$.fragment,t),$(pt),$(rs.$$.fragment,t),$(ut),$(is.$$.fragment,t),$(ms.$$.fragment,t),$(ft),$(us.$$.fragment,t),$(ds.$$.fragment,t),$(fs.$$.fragment,t),$(cs.$$.fragment,t),$(It.$$.fragment,t),La=!0)},o(t){b(c.$$.fragment,t),b(w.$$.fragment,t),b(P),b(I.$$.fragment,t),b(Gt.$$.fragment,t),b(Mt.$$.fragment,t),b(Lt.$$.fragment,t),b(zt.$$.fragment,t),b(Ut.$$.fragment,t),b(Yt.$$.fragment,t),b(Bt.$$.fragment,t),b(Vt.$$.fragment,t),b(Jt.$$.fragment,t),b(Qt.$$.fragment,t),b(Xt.$$.fragment,t),b(Kt.$$.fragment,t),b(Zt.$$.fragment,t),b(qt.$$.fragment,t),b(ts.$$.fragment,t),b(ss.$$.fragment,t),b(es.$$.fragment,t),b(as.$$.fragment,t),b(pt),b(rs.$$.fragment,t),b(ut),b(is.$$.fragment,t),b(ms.$$.fragment,t),b(ft),b(us.$$.fragment,t),b(ds.$$.fragment,t),b(fs.$$.fragment,t),b(cs.$$.fragment,t),b(It.$$.fragment,t),La=!1},d(t){e(f),t&&e(g),j(c,t),t&&e(y),t&&e(_),j(w),t&&e(N),_s[O].d(t),t&&e(F),t&&e(M),t&&e(U),j(I,t),t&&e(Y),t&&e(d),t&&e(C),t&&e(B),t&&e(Le),t&&e(jt),t&&e(ze),t&&e(Ht),t&&e(Ue),t&&e(ws),t&&e(Ye),j(Gt,t),t&&e(Be),t&&e(ct),t&&e(We),j(Mt,t),t&&e(Ve),j(Lt,t),t&&e(Je),t&&e(ks),t&&e(Qe),j(zt,t),t&&e(Xe),j(Ut,t),t&&e(Ke),t&&e(L),t&&e(Ze),j(Yt,t),t&&e(ta),j(Bt,t),t&&e(sa),t&&e(W),t&&e(ea),j(Vt,t),t&&e(aa),t&&e(Tt),t&&e(na),j(Jt,t),t&&e(la),j(Qt,t),t&&e(oa),t&&e(vs),t&&e(ra),j(Xt,t),t&&e(ia),t&&e(Q),t&&e(ma),t&&e(et),t&&e(ua),j(Kt,t),t&&e(da),j(Zt,t),t&&e(fa),t&&e(js),t&&e(ca),j(qt,t),t&&e(_a),t&&e(St),t&&e(ha),j(ts,t),t&&e(ga),t&&e(ys),t&&e(ba),j(ss,t),t&&e($a),j(es,t),t&&e(wa),t&&e(_t),t&&e(xa),j(as,t),t&&e(ka),t&&e(Es),t&&e(va),t&&e(ns),t&&e(ja),t&&e(G),t&&e(ya),hs[it].d(t),t&&e(qs),t&&e(Dt),t&&e(Ea),j(rs,t),t&&e(Ta),t&&e(Ss),t&&e(qa),gs[mt].d(t),t&&e(Ds),t&&e(As),t&&e(Sa),t&&e(yt),j(is),t&&e(Da),t&&e(ht),t&&e(Aa),j(ms,t),t&&e(Ra),t&&e(Rt),t&&e(Ia),bs[dt].d(t),t&&e(Rs),t&&e(Is),t&&e(Ca),j(us,t),t&&e(Na),t&&e(Et),t&&e(Oa),j(ds,t),t&&e(Pa),t&&e(Cs),t&&e(Fa),j(fs,t),t&&e(Ha),j(cs,t),t&&e(Ga),t&&e(Ns),t&&e(Ma),j(It,t)}}}const Ti={local:"faiss",sections:[{local:"",title:"\u4F7F\u7528\u5D4C\u5165\u8FDB\u884C\u8BED\u4E49\u641C\u7D22"},{local:"",title:"## \u52A0\u8F7D\u548C\u51C6\u5907\u6570\u636E\u96C6"},{local:"",title:"\u521B\u5EFA\u6587\u672C\u5D4C\u5165"},{local:"faiss",title:"\u4F7F\u7528 FAISS \u8FDB\u884C\u9AD8\u6548\u7684\u76F8\u4F3C\u6027\u641C\u7D22"}],title:"\u4F7F\u7528 FAISS \u8FDB\u884C\u8BED\u4E49\u641C\u7D22"};function qi(R,f,g){let c="pt";return fi(()=>{const y=new URLSearchParams(window.location.search);g(0,c=y.get("fw")||"pt")}),[c]}class Pi extends pi{constructor(f){super();mi(this,f,qi,Ei,ui,{})}}export{Pi as default,Ti as metadata};
