import{S as oi,i as pi,s as ii,e as p,t as n,k as _,w as E,c as i,a as f,h as a,d as e,m,x as w,g as c,G as t,y as T,q as g,o as j,B as S,l as ai,M as fi,b as q,p as vt,v as ci,n as Et}from"../../chunks/vendor-hf-doc-builder.js";import{T as wo}from"../../chunks/Tip-hf-doc-builder.js";import{Y as nn}from"../../chunks/Youtube-hf-doc-builder.js";import{I as ui}from"../../chunks/IconCopyLink-hf-doc-builder.js";import{C as P}from"../../chunks/CodeBlock-hf-doc-builder.js";import{D as ri}from"../../chunks/DocNotebookDropdown-hf-doc-builder.js";import{F as _i}from"../../chunks/FrameworkSwitchCourse-hf-doc-builder.js";function mi(v){let l,u;return l=new ri({props:{classNames:"absolute z-10 right-0 top-0",options:[{label:"Google Colab",value:"https://colab.research.google.com/github/huggingface/notebooks/blob/master/course/chapter3/section2_tf.ipynb"},{label:"Aws Studio",value:"https://studiolab.sagemaker.aws/import/github/huggingface/notebooks/blob/master/course/chapter3/section2_tf.ipynb"}]}}),{c(){E(l.$$.fragment)},l(r){w(l.$$.fragment,r)},m(r,b){T(l,r,b),u=!0},i(r){u||(g(l.$$.fragment,r),u=!0)},o(r){j(l.$$.fragment,r),u=!1},d(r){S(l,r)}}}function hi(v){let l,u;return l=new ri({props:{classNames:"absolute z-10 right-0 top-0",options:[{label:"Google Colab",value:"https://colab.research.google.com/github/huggingface/notebooks/blob/master/course/chapter3/section2_pt.ipynb"},{label:"Aws Studio",value:"https://studiolab.sagemaker.aws/import/github/huggingface/notebooks/blob/master/course/chapter3/section2_pt.ipynb"}]}}),{c(){E(l.$$.fragment)},l(r){w(l.$$.fragment,r)},m(r,b){T(l,r,b),u=!0},i(r){u||(g(l.$$.fragment,r),u=!0)},o(r){j(l.$$.fragment,r),u=!1},d(r){S(l,r)}}}function di(v){let l,u,r,b,k,h,d,x;return d=new P({props:{code:`
`,highlighted:`<span class="hljs-keyword">import</span> tensorflow <span class="hljs-keyword">as</span> tf
<span class="hljs-keyword">import</span> numpy <span class="hljs-keyword">as</span> np
<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoTokenizer, TFAutoModelForSequenceClassification

<span class="hljs-comment"># Same as before</span>
checkpoint = <span class="hljs-string">&quot;bert-base-uncased&quot;</span>
tokenizer = AutoTokenizer.from_pretrained(checkpoint)
model = TFAutoModelForSequenceClassification.from_pretrained(checkpoint)
sequences = [
    <span class="hljs-string">&quot;I&#x27;ve been waiting for a HuggingFace course my whole life.&quot;</span>,
    <span class="hljs-string">&quot;This course is amazing!&quot;</span>,
]
batch = <span class="hljs-built_in">dict</span>(tokenizer(sequences, padding=<span class="hljs-literal">True</span>, truncation=<span class="hljs-literal">True</span>, return_tensors=<span class="hljs-string">&quot;tf&quot;</span>))

<span class="hljs-comment"># This is new</span>
model.<span class="hljs-built_in">compile</span>(optimizer=<span class="hljs-string">&quot;adam&quot;</span>, loss=<span class="hljs-string">&quot;sparse_categorical_crossentropy&quot;</span>)
labels = tf.convert_to_tensor([<span class="hljs-number">1</span>, <span class="hljs-number">1</span>])
model.train_on_batch(batch, labels)`}}),{c(){l=p("p"),u=n("\u8FD9\u4E00\u5C0F\u8282\u5B66\u4E60"),r=p("a"),b=n("\u7B2C\u4E00\u5C0F\u8282"),k=n("\u4E2D\u63D0\u5230\u7684\u201C\u5982\u4F55\u4F7F\u7528\u6A21\u578B\u4E2D\u5FC3\uFF08hub\uFF09\u5927\u578B\u6570\u636E\u96C6\u201D\uFF0C\u4E0B\u9762\u662F\u6211\u4EEC\u7528\u6A21\u578B\u4E2D\u5FC3\u7684\u6570\u636E\u5728TensorFlow\u4E0A\u8BAD\u7EC3\u53E5\u5B50\u5206\u7C7B\u5668\u7684\u4E00\u4E2A\u4F8B\u5B50\uFF1A"),h=_(),E(d.$$.fragment),this.h()},l($){l=i($,"P",{});var y=f(l);u=a(y,"\u8FD9\u4E00\u5C0F\u8282\u5B66\u4E60"),r=i(y,"A",{href:!0});var R=f(r);b=a(R,"\u7B2C\u4E00\u5C0F\u8282"),R.forEach(e),k=a(y,"\u4E2D\u63D0\u5230\u7684\u201C\u5982\u4F55\u4F7F\u7528\u6A21\u578B\u4E2D\u5FC3\uFF08hub\uFF09\u5927\u578B\u6570\u636E\u96C6\u201D\uFF0C\u4E0B\u9762\u662F\u6211\u4EEC\u7528\u6A21\u578B\u4E2D\u5FC3\u7684\u6570\u636E\u5728TensorFlow\u4E0A\u8BAD\u7EC3\u53E5\u5B50\u5206\u7C7B\u5668\u7684\u4E00\u4E2A\u4F8B\u5B50\uFF1A"),y.forEach(e),h=m($),w(d.$$.fragment,$),this.h()},h(){q(r,"href","/course/chapter2")},m($,y){c($,l,y),t(l,u),t(l,r),t(r,b),t(l,k),c($,h,y),T(d,$,y),x=!0},i($){x||(g(d.$$.fragment,$),x=!0)},o($){j(d.$$.fragment,$),x=!1},d($){$&&e(l),$&&e(h),S(d,$)}}}function bi(v){let l,u,r,b,k,h,d,x;return d=new P({props:{code:`

`,highlighted:`<span class="hljs-keyword">import</span> torch
<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AdamW, AutoTokenizer, AutoModelForSequenceClassification

<span class="hljs-comment"># Same as before</span>
checkpoint = <span class="hljs-string">&quot;bert-base-uncased&quot;</span>
tokenizer = AutoTokenizer.from_pretrained(checkpoint)
model = AutoModelForSequenceClassification.from_pretrained(checkpoint)
sequences = [
    <span class="hljs-string">&quot;I&#x27;ve been waiting for a HuggingFace course my whole life.&quot;</span>,
    <span class="hljs-string">&quot;This course is amazing!&quot;</span>,
]
batch = tokenizer(sequences, padding=<span class="hljs-literal">True</span>, truncation=<span class="hljs-literal">True</span>, return_tensors=<span class="hljs-string">&quot;pt&quot;</span>)

<span class="hljs-comment"># This is new</span>
batch[<span class="hljs-string">&quot;labels&quot;</span>] = torch.tensor([<span class="hljs-number">1</span>, <span class="hljs-number">1</span>])

optimizer = AdamW(model.parameters())
loss = model(**batch).loss
loss.backward()
optimizer.step()`}}),{c(){l=p("p"),u=n("\u8FD9\u4E00\u5C0F\u8282\u5B66\u4E60"),r=p("a"),b=n("\u7B2C\u4E00\u5C0F\u8282"),k=n("\u4E2D\u63D0\u5230\u7684\u201C\u5982\u4F55\u4F7F\u7528\u6A21\u578B\u4E2D\u5FC3\uFF08hub\uFF09\u5927\u578B\u6570\u636E\u96C6\u201D\uFF0C\u4E0B\u9762\u662F\u6211\u4EEC\u7528\u6A21\u578B\u4E2D\u5FC3\u7684\u6570\u636E\u5728PyTorch\u4E0A\u8BAD\u7EC3\u53E5\u5B50\u5206\u7C7B\u5668\u7684\u4E00\u4E2A\u4F8B\u5B50\uFF1A"),h=_(),E(d.$$.fragment),this.h()},l($){l=i($,"P",{});var y=f(l);u=a(y,"\u8FD9\u4E00\u5C0F\u8282\u5B66\u4E60"),r=i(y,"A",{href:!0});var R=f(r);b=a(R,"\u7B2C\u4E00\u5C0F\u8282"),R.forEach(e),k=a(y,"\u4E2D\u63D0\u5230\u7684\u201C\u5982\u4F55\u4F7F\u7528\u6A21\u578B\u4E2D\u5FC3\uFF08hub\uFF09\u5927\u578B\u6570\u636E\u96C6\u201D\uFF0C\u4E0B\u9762\u662F\u6211\u4EEC\u7528\u6A21\u578B\u4E2D\u5FC3\u7684\u6570\u636E\u5728PyTorch\u4E0A\u8BAD\u7EC3\u53E5\u5B50\u5206\u7C7B\u5668\u7684\u4E00\u4E2A\u4F8B\u5B50\uFF1A"),y.forEach(e),h=m($),w(d.$$.fragment,$),this.h()},h(){q(r,"href","/course/chapter2")},m($,y){c($,l,y),t(l,u),t(l,r),t(r,b),t(l,k),c($,h,y),T(d,$,y),x=!0},i($){x||(g(d.$$.fragment,$),x=!0)},o($){j(d.$$.fragment,$),x=!1},d($){$&&e(l),$&&e(h),S(d,$)}}}function $i(v){let l,u;return l=new nn({props:{id:"W_gMJF0xomE"}}),{c(){E(l.$$.fragment)},l(r){w(l.$$.fragment,r)},m(r,b){T(l,r,b),u=!0},i(r){u||(g(l.$$.fragment,r),u=!0)},o(r){j(l.$$.fragment,r),u=!1},d(r){S(l,r)}}}function gi(v){let l,u;return l=new nn({props:{id:"_BZearw7f0w"}}),{c(){E(l.$$.fragment)},l(r){w(l.$$.fragment,r)},m(r,b){T(l,r,b),u=!0},i(r){u||(g(l.$$.fragment,r),u=!0)},o(r){j(l.$$.fragment,r),u=!1},d(r){S(l,r)}}}function ji(v){let l,u,r,b,k;return{c(){l=p("p"),u=n("\u270F\uFE0F "),r=p("strong"),b=n("\u8BD5\u8BD5\u770B\uFF01"),k=n(" \u67E5\u770B\u8BAD\u7EC3\u96C6\u7684\u7B2C15\u884C\u5143\u7D20\u548C\u9A8C\u8BC1\u96C6\u768487\u884C\u5143\u7D20\u3002\u4ED6\u4EEC\u7684\u6807\u7B7E\u662F\u4EC0\u4E48\uFF1F")},l(h){l=i(h,"P",{});var d=f(l);u=a(d,"\u270F\uFE0F "),r=i(d,"STRONG",{});var x=f(r);b=a(x,"\u8BD5\u8BD5\u770B\uFF01"),x.forEach(e),k=a(d," \u67E5\u770B\u8BAD\u7EC3\u96C6\u7684\u7B2C15\u884C\u5143\u7D20\u548C\u9A8C\u8BC1\u96C6\u768487\u884C\u5143\u7D20\u3002\u4ED6\u4EEC\u7684\u6807\u7B7E\u662F\u4EC0\u4E48\uFF1F"),d.forEach(e)},m(h,d){c(h,l,d),t(l,u),t(l,r),t(r,b),t(l,k)},d(h){h&&e(l)}}}function ki(v){let l,u;return l=new nn({props:{id:"P-rZWqcB6CE"}}),{c(){E(l.$$.fragment)},l(r){w(l.$$.fragment,r)},m(r,b){T(l,r,b),u=!0},i(r){u||(g(l.$$.fragment,r),u=!0)},o(r){j(l.$$.fragment,r),u=!1},d(r){S(l,r)}}}function xi(v){let l,u;return l=new nn({props:{id:"0u3ioSwev3s"}}),{c(){E(l.$$.fragment)},l(r){w(l.$$.fragment,r)},m(r,b){T(l,r,b),u=!0},i(r){u||(g(l.$$.fragment,r),u=!0)},o(r){j(l.$$.fragment,r),u=!1},d(r){S(l,r)}}}function vi(v){let l,u,r,b,k;return{c(){l=p("p"),u=n("\u270F\uFE0F "),r=p("strong"),b=n("\u8BD5\u8BD5\u770B\uFF01"),k=n(" \u9009\u53D6\u8BAD\u7EC3\u96C6\u4E2D\u7684\u7B2C15\u4E2A\u5143\u7D20\uFF0C\u5C06\u4E24\u53E5\u8BDD\u5206\u522B\u6807\u8BB0\u4E3A\u4E00\u5BF9\u3002\u7ED3\u679C\u548C\u4E0A\u65B9\u7684\u4F8B\u5B50\u6709\u4EC0\u4E48\u4E0D\u540C\uFF1F")},l(h){l=i(h,"P",{});var d=f(l);u=a(d,"\u270F\uFE0F "),r=i(d,"STRONG",{});var x=f(r);b=a(x,"\u8BD5\u8BD5\u770B\uFF01"),x.forEach(e),k=a(d," \u9009\u53D6\u8BAD\u7EC3\u96C6\u4E2D\u7684\u7B2C15\u4E2A\u5143\u7D20\uFF0C\u5C06\u4E24\u53E5\u8BDD\u5206\u522B\u6807\u8BB0\u4E3A\u4E00\u5BF9\u3002\u7ED3\u679C\u548C\u4E0A\u65B9\u7684\u4F8B\u5B50\u6709\u4EC0\u4E48\u4E0D\u540C\uFF1F"),d.forEach(e)},m(h,d){c(h,l,d),t(l,u),t(l,r),t(r,b),t(l,k)},d(h){h&&e(l)}}}function Ei(v){let l,u,r,b,k;return{c(){l=p("p"),u=n("\u8D1F\u8D23\u5728\u6279\u5904\u7406\u4E2D\u5C06\u6570\u636E\u6574\u7406\u4E3A\u4E00\u4E2Abatch\u7684\u51FD\u6570\u79F0\u4E3A"),r=p("em"),b=n("collate\u51FD\u6570"),k=n("\u3002\u5B83\u53EA\u4F1A\u5C06\u60A8\u7684\u6837\u672C\u8F6C\u6362\u4E3A tf.Tensor\u5E76\u5C06\u5B83\u4EEC\u62FC\u63A5\u8D77\u6765(\u5982\u679C\u4F60\u7684\u5143\u7D20\u662F\u5217\u8868\u3001\u5143\u7EC4\u6216\u5B57\u5178\uFF0C\u5219\u4F1A\u4F7F\u7528\u9012\u5F52)\u3002\u8FD9\u5728\u6211\u4EEC\u7684\u8FD9\u4E2A\u4F8B\u5B50\u4E2D\u4E0B\u662F\u4E0D\u53EF\u884C\u7684\uFF0C\u56E0\u4E3A\u6211\u4EEC\u7684\u8F93\u5165\u4E0D\u662F\u90FD\u662F\u76F8\u540C\u5927\u5C0F\u7684\u3002\u6211\u4EEC\u6545\u610F\u5728\u4E4B\u540E\u6BCF\u4E2Abatch\u4E0A\u8FDB\u884C\u586B\u5145\uFF0C\u907F\u514D\u6709\u592A\u591A\u586B\u5145\u7684\u8FC7\u957F\u7684\u8F93\u5165\u3002\u8FD9\u5C06\u5927\u5927\u52A0\u5FEB\u8BAD\u7EC3\u901F\u5EA6\uFF0C\u4F46\u8BF7\u6CE8\u610F\uFF0C\u5982\u679C\u4F60\u5728TPU\u4E0A\u8BAD\u7EC3\uFF0C\u8FD9\u53EF\u80FD\u4F1A\u5BFC\u81F4\u95EE\u9898\u2014\u2014TPU\u559C\u6B22\u56FA\u5B9A\u7684\u5F62\u72B6\uFF0C\u5373\u4F7F\u8FD9\u9700\u8981\u989D\u5916\u7684\u586B\u5145\u3002")},l(h){l=i(h,"P",{});var d=f(l);u=a(d,"\u8D1F\u8D23\u5728\u6279\u5904\u7406\u4E2D\u5C06\u6570\u636E\u6574\u7406\u4E3A\u4E00\u4E2Abatch\u7684\u51FD\u6570\u79F0\u4E3A"),r=i(d,"EM",{});var x=f(r);b=a(x,"collate\u51FD\u6570"),x.forEach(e),k=a(d,"\u3002\u5B83\u53EA\u4F1A\u5C06\u60A8\u7684\u6837\u672C\u8F6C\u6362\u4E3A tf.Tensor\u5E76\u5C06\u5B83\u4EEC\u62FC\u63A5\u8D77\u6765(\u5982\u679C\u4F60\u7684\u5143\u7D20\u662F\u5217\u8868\u3001\u5143\u7EC4\u6216\u5B57\u5178\uFF0C\u5219\u4F1A\u4F7F\u7528\u9012\u5F52)\u3002\u8FD9\u5728\u6211\u4EEC\u7684\u8FD9\u4E2A\u4F8B\u5B50\u4E2D\u4E0B\u662F\u4E0D\u53EF\u884C\u7684\uFF0C\u56E0\u4E3A\u6211\u4EEC\u7684\u8F93\u5165\u4E0D\u662F\u90FD\u662F\u76F8\u540C\u5927\u5C0F\u7684\u3002\u6211\u4EEC\u6545\u610F\u5728\u4E4B\u540E\u6BCF\u4E2Abatch\u4E0A\u8FDB\u884C\u586B\u5145\uFF0C\u907F\u514D\u6709\u592A\u591A\u586B\u5145\u7684\u8FC7\u957F\u7684\u8F93\u5165\u3002\u8FD9\u5C06\u5927\u5927\u52A0\u5FEB\u8BAD\u7EC3\u901F\u5EA6\uFF0C\u4F46\u8BF7\u6CE8\u610F\uFF0C\u5982\u679C\u4F60\u5728TPU\u4E0A\u8BAD\u7EC3\uFF0C\u8FD9\u53EF\u80FD\u4F1A\u5BFC\u81F4\u95EE\u9898\u2014\u2014TPU\u559C\u6B22\u56FA\u5B9A\u7684\u5F62\u72B6\uFF0C\u5373\u4F7F\u8FD9\u9700\u8981\u989D\u5916\u7684\u586B\u5145\u3002"),d.forEach(e)},m(h,d){c(h,l,d),t(l,u),t(l,r),t(r,b),t(l,k)},d(h){h&&e(l)}}}function wi(v){let l,u,r,b,k,h,d,x;return{c(){l=p("p"),u=n("\u8D1F\u8D23\u5728\u6279\u5904\u7406\u4E2D\u5C06\u6570\u636E\u6574\u7406\u4E3A\u4E00\u4E2Abatch\u7684\u51FD\u6570\u79F0\u4E3A"),r=p("em"),b=n("collate\u51FD\u6570"),k=n("\u3002\u5B83\u662F\u4F60\u53EF\u4EE5\u5728\u6784\u5EFA"),h=p("strong"),d=n("DataLoader"),x=n("\u65F6\u4F20\u9012\u7684\u4E00\u4E2A\u53C2\u6570\uFF0C\u9ED8\u8BA4\u662F\u4E00\u4E2A\u51FD\u6570\uFF0C\u5B83\u5C06\u628A\u4F60\u7684\u6570\u636E\u96C6\u8F6C\u6362\u4E3APyTorch\u5F20\u91CF\uFF0C\u5E76\u5C06\u5B83\u4EEC\u62FC\u63A5\u8D77\u6765(\u5982\u679C\u4F60\u7684\u5143\u7D20\u662F\u5217\u8868\u3001\u5143\u7EC4\u6216\u5B57\u5178\uFF0C\u5219\u4F1A\u4F7F\u7528\u9012\u5F52)\u3002\u8FD9\u5728\u6211\u4EEC\u7684\u8FD9\u4E2A\u4F8B\u5B50\u4E2D\u4E0B\u662F\u4E0D\u53EF\u884C\u7684\uFF0C\u56E0\u4E3A\u6211\u4EEC\u7684\u8F93\u5165\u4E0D\u662F\u90FD\u662F\u76F8\u540C\u5927\u5C0F\u7684\u3002\u6211\u4EEC\u6545\u610F\u5728\u4E4B\u540E\u6BCF\u4E2Abatch\u4E0A\u8FDB\u884C\u586B\u5145\uFF0C\u907F\u514D\u6709\u592A\u591A\u586B\u5145\u7684\u8FC7\u957F\u7684\u8F93\u5165\u3002\u8FD9\u5C06\u5927\u5927\u52A0\u5FEB\u8BAD\u7EC3\u901F\u5EA6\uFF0C\u4F46\u8BF7\u6CE8\u610F\uFF0C\u5982\u679C\u4F60\u5728TPU\u4E0A\u8BAD\u7EC3\uFF0C\u8FD9\u53EF\u80FD\u4F1A\u5BFC\u81F4\u95EE\u9898\u2014\u2014TPU\u559C\u6B22\u56FA\u5B9A\u7684\u5F62\u72B6\uFF0C\u5373\u4F7F\u8FD9\u9700\u8981\u989D\u5916\u7684\u586B\u5145\u3002")},l($){l=i($,"P",{});var y=f(l);u=a(y,"\u8D1F\u8D23\u5728\u6279\u5904\u7406\u4E2D\u5C06\u6570\u636E\u6574\u7406\u4E3A\u4E00\u4E2Abatch\u7684\u51FD\u6570\u79F0\u4E3A"),r=i(y,"EM",{});var R=f(r);b=a(R,"collate\u51FD\u6570"),R.forEach(e),k=a(y,"\u3002\u5B83\u662F\u4F60\u53EF\u4EE5\u5728\u6784\u5EFA"),h=i(y,"STRONG",{});var z=f(h);d=a(z,"DataLoader"),z.forEach(e),x=a(y,"\u65F6\u4F20\u9012\u7684\u4E00\u4E2A\u53C2\u6570\uFF0C\u9ED8\u8BA4\u662F\u4E00\u4E2A\u51FD\u6570\uFF0C\u5B83\u5C06\u628A\u4F60\u7684\u6570\u636E\u96C6\u8F6C\u6362\u4E3APyTorch\u5F20\u91CF\uFF0C\u5E76\u5C06\u5B83\u4EEC\u62FC\u63A5\u8D77\u6765(\u5982\u679C\u4F60\u7684\u5143\u7D20\u662F\u5217\u8868\u3001\u5143\u7EC4\u6216\u5B57\u5178\uFF0C\u5219\u4F1A\u4F7F\u7528\u9012\u5F52)\u3002\u8FD9\u5728\u6211\u4EEC\u7684\u8FD9\u4E2A\u4F8B\u5B50\u4E2D\u4E0B\u662F\u4E0D\u53EF\u884C\u7684\uFF0C\u56E0\u4E3A\u6211\u4EEC\u7684\u8F93\u5165\u4E0D\u662F\u90FD\u662F\u76F8\u540C\u5927\u5C0F\u7684\u3002\u6211\u4EEC\u6545\u610F\u5728\u4E4B\u540E\u6BCF\u4E2Abatch\u4E0A\u8FDB\u884C\u586B\u5145\uFF0C\u907F\u514D\u6709\u592A\u591A\u586B\u5145\u7684\u8FC7\u957F\u7684\u8F93\u5165\u3002\u8FD9\u5C06\u5927\u5927\u52A0\u5FEB\u8BAD\u7EC3\u901F\u5EA6\uFF0C\u4F46\u8BF7\u6CE8\u610F\uFF0C\u5982\u679C\u4F60\u5728TPU\u4E0A\u8BAD\u7EC3\uFF0C\u8FD9\u53EF\u80FD\u4F1A\u5BFC\u81F4\u95EE\u9898\u2014\u2014TPU\u559C\u6B22\u56FA\u5B9A\u7684\u5F62\u72B6\uFF0C\u5373\u4F7F\u8FD9\u9700\u8981\u989D\u5916\u7684\u586B\u5145\u3002"),y.forEach(e)},m($,y){c($,l,y),t(l,u),t(l,r),t(r,b),t(l,k),t(l,h),t(h,d),t(l,x)},d($){$&&e(l)}}}function Ti(v){let l,u;return l=new P({props:{code:"",highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> DataCollatorWithPadding

data_collator = DataCollatorWithPadding(tokenizer=tokenizer, return_tensors=<span class="hljs-string">&quot;tf&quot;</span>)`}}),{c(){E(l.$$.fragment)},l(r){w(l.$$.fragment,r)},m(r,b){T(l,r,b),u=!0},i(r){u||(g(l.$$.fragment,r),u=!0)},o(r){j(l.$$.fragment,r),u=!1},d(r){S(l,r)}}}function Si(v){let l,u;return l=new P({props:{code:"",highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> DataCollatorWithPadding

data_collator = DataCollatorWithPadding(tokenizer=tokenizer)`}}),{c(){E(l.$$.fragment)},l(r){w(l.$$.fragment,r)},m(r,b){T(l,r,b),u=!0},i(r){u||(g(l.$$.fragment,r),u=!0)},o(r){j(l.$$.fragment,r),u=!1},d(r){S(l,r)}}}function yi(v){let l,u,r,b,k;return l=new P({props:{code:`{'attention_mask': torch.Size([8, 67]),
 'input_ids': torch.Size([8, 67]),
 'token_type_ids': torch.Size([8, 67]),
 'labels': torch.Size([8])}`,highlighted:`{<span class="hljs-string">&#x27;attention_mask&#x27;</span>: torch.Size([<span class="hljs-number">8</span>, <span class="hljs-number">67</span>]),
 <span class="hljs-string">&#x27;input_ids&#x27;</span>: torch.Size([<span class="hljs-number">8</span>, <span class="hljs-number">67</span>]),
 <span class="hljs-string">&#x27;token_type_ids&#x27;</span>: torch.Size([<span class="hljs-number">8</span>, <span class="hljs-number">67</span>]),
 <span class="hljs-string">&#x27;labels&#x27;</span>: torch.Size([<span class="hljs-number">8</span>])}`}}),{c(){E(l.$$.fragment),u=_(),r=p("p"),b=n("\u770B\u8D77\u6765\u4E0D\u9519\uFF01\u73B0\u5728\uFF0C\u6211\u4EEC\u5DF2\u7ECF\u5C06\u539F\u59CB\u6587\u672C\u8F6C\u5316\u4E3A\u4E86\u6A21\u578B\u53EF\u4EE5\u5904\u7406\u7684\u6570\u636E\uFF0C\u6211\u4EEC\u5DF2\u51C6\u5907\u597D\u5BF9\u5176\u8FDB\u884C\u5FAE\u8C03\uFF01")},l(h){w(l.$$.fragment,h),u=m(h),r=i(h,"P",{});var d=f(r);b=a(d,"\u770B\u8D77\u6765\u4E0D\u9519\uFF01\u73B0\u5728\uFF0C\u6211\u4EEC\u5DF2\u7ECF\u5C06\u539F\u59CB\u6587\u672C\u8F6C\u5316\u4E3A\u4E86\u6A21\u578B\u53EF\u4EE5\u5904\u7406\u7684\u6570\u636E\uFF0C\u6211\u4EEC\u5DF2\u51C6\u5907\u597D\u5BF9\u5176\u8FDB\u884C\u5FAE\u8C03\uFF01"),d.forEach(e)},m(h,d){T(l,h,d),c(h,u,d),c(h,r,d),t(r,b),k=!0},i(h){k||(g(l.$$.fragment,h),k=!0)},o(h){j(l.$$.fragment,h),k=!1},d(h){S(l,h),h&&e(u),h&&e(r)}}}function qi(v){let l,u;return l=new P({props:{code:`{'attention_mask': TensorShape([8, 67]),
 'input_ids': TensorShape([8, 67]),
 'token_type_ids': TensorShape([8, 67]),
 'labels': TensorShape([8])}`,highlighted:`{<span class="hljs-string">&#x27;attention_mask&#x27;</span>: TensorShape([<span class="hljs-number">8</span>, <span class="hljs-number">67</span>]),
 <span class="hljs-string">&#x27;input_ids&#x27;</span>: TensorShape([<span class="hljs-number">8</span>, <span class="hljs-number">67</span>]),
 <span class="hljs-string">&#x27;token_type_ids&#x27;</span>: TensorShape([<span class="hljs-number">8</span>, <span class="hljs-number">67</span>]),
 <span class="hljs-string">&#x27;labels&#x27;</span>: TensorShape([<span class="hljs-number">8</span>])}`}}),{c(){E(l.$$.fragment)},l(r){w(l.$$.fragment,r)},m(r,b){T(l,r,b),u=!0},i(r){u||(g(l.$$.fragment,r),u=!0)},o(r){j(l.$$.fragment,r),u=!1},d(r){S(l,r)}}}function Pi(v){let l,u,r,b,k;return{c(){l=p("p"),u=n("\u270F\uFE0F "),r=p("strong"),b=n("\u8BD5\u8BD5\u770B\uFF01"),k=n(" \u5728GLUE SST-2\u6570\u636E\u96C6\u4E0A\u5E94\u7528\u9884\u5904\u7406\u3002\u5B83\u6709\u70B9\u4E0D\u540C\uFF0C\u56E0\u4E3A\u5B83\u662F\u7531\u5355\u4E2A\u53E5\u5B50\u800C\u4E0D\u662F\u6210\u5BF9\u7684\u53E5\u5B50\u7EC4\u6210\u7684\uFF0C\u4F46\u662F\u6211\u4EEC\u6240\u505A\u7684\u5176\u4ED6\u4E8B\u60C5\u770B\u8D77\u6765\u5E94\u8BE5\u662F\u4E00\u6837\u7684\u3002\u53E6\u4E00\u4E2A\u66F4\u96BE\u7684\u6311\u6218\uFF0C\u8BF7\u5C1D\u8BD5\u7F16\u5199\u4E00\u4E2A\u53EF\u7528\u4E8E\u4EFB\u4F55GLUE\u4EFB\u52A1\u7684\u9884\u5904\u7406\u51FD\u6570\u3002")},l(h){l=i(h,"P",{});var d=f(l);u=a(d,"\u270F\uFE0F "),r=i(d,"STRONG",{});var x=f(r);b=a(x,"\u8BD5\u8BD5\u770B\uFF01"),x.forEach(e),k=a(d," \u5728GLUE SST-2\u6570\u636E\u96C6\u4E0A\u5E94\u7528\u9884\u5904\u7406\u3002\u5B83\u6709\u70B9\u4E0D\u540C\uFF0C\u56E0\u4E3A\u5B83\u662F\u7531\u5355\u4E2A\u53E5\u5B50\u800C\u4E0D\u662F\u6210\u5BF9\u7684\u53E5\u5B50\u7EC4\u6210\u7684\uFF0C\u4F46\u662F\u6211\u4EEC\u6240\u505A\u7684\u5176\u4ED6\u4E8B\u60C5\u770B\u8D77\u6765\u5E94\u8BE5\u662F\u4E00\u6837\u7684\u3002\u53E6\u4E00\u4E2A\u66F4\u96BE\u7684\u6311\u6218\uFF0C\u8BF7\u5C1D\u8BD5\u7F16\u5199\u4E00\u4E2A\u53EF\u7528\u4E8E\u4EFB\u4F55GLUE\u4EFB\u52A1\u7684\u9884\u5904\u7406\u51FD\u6570\u3002"),d.forEach(e)},m(h,d){c(h,l,d),t(l,u),t(l,r),t(r,b),t(l,k)},d(h){h&&e(l)}}}function li(v){let l,u,r,b,k,h,d,x,$,y,R,z,cs,us,vs,W,B,_s,ts,Es;return W=new P({props:{code:"",highlighted:`tf_train_dataset = tokenized_datasets[<span class="hljs-string">&quot;train&quot;</span>].to_tf_dataset(
    columns=[<span class="hljs-string">&quot;attention_mask&quot;</span>, <span class="hljs-string">&quot;input_ids&quot;</span>, <span class="hljs-string">&quot;token_type_ids&quot;</span>],
    label_cols=[<span class="hljs-string">&quot;labels&quot;</span>],
    shuffle=<span class="hljs-literal">True</span>,
    collate_fn=data_collator,
    batch_size=<span class="hljs-number">8</span>,
)

tf_validation_dataset = tokenized_datasets[<span class="hljs-string">&quot;validation&quot;</span>].to_tf_dataset(
    columns=[<span class="hljs-string">&quot;attention_mask&quot;</span>, <span class="hljs-string">&quot;input_ids&quot;</span>, <span class="hljs-string">&quot;token_type_ids&quot;</span>],
    label_cols=[<span class="hljs-string">&quot;labels&quot;</span>],
    shuffle=<span class="hljs-literal">False</span>,
    collate_fn=data_collator,
    batch_size=<span class="hljs-number">8</span>,
)`}}),{c(){l=p("p"),u=n("\u73B0\u5728\u6211\u4EEC\u6709\u4E86dataset\u548Cdata collator\uFF0C\u6211\u4EEC\u9700\u8981\u5C06dataset\u6279\u91CF\u5730\u5E94\u7528data collator\u3002 \u6211\u4EEC\u53EF\u4EE5\u624B\u52A8\u52A0\u8F7D\u6279\u6B21\u5E76\u6574\u7406\u5B83\u4EEC\uFF0C\u4F46\u8FD9\u9700\u8981\u5927\u91CF\u5DE5\u4F5C\uFF0C\u800C\u4E14\u53EF\u80FD\u6027\u80FD\u4E5F\u4E0D\u662F\u5F88\u597D\u3002 \u76F8\u53CD\uFF0C\u6709\u4E00\u4E2A\u7B80\u5355\u7684\u65B9\u6CD5\u53EF\u4EE5\u4E3A\u8FD9\u4E2A\u95EE\u9898\u63D0\u4F9B\u9AD8\u6548\u7684\u89E3\u51B3\u65B9\u6848\uFF1A"),r=p("code"),b=n("to_tf_dataset()"),k=n("\u3002 \u8FD9\u5C06\u5728\u60A8\u7684\u6570\u636E\u96C6\u4E0A\u8C03\u7528\u4E00\u4E2A "),h=p("code"),d=n("tf.data.Dataset"),x=n("\u7684\u65B9\u6CD5\uFF0C\u8FD9\u4E2A\u65B9\u6CD5\u5E26\u6709\u4E00\u4E2A\u53EF\u9009\u7684data collator\u529F\u80FD\u3002 "),$=p("code"),y=n("tf.data.Dataset"),R=n(" \u662F Keras \u53EF\u7528\u4E8E "),z=p("code"),cs=n("model.fit()"),us=n(" \u7684\u539F\u751F TensorFlow \u683C\u5F0F\uFF0C\u56E0\u6B64\u8FD9\u79CD\u65B9\u6CD5\u4F1A\u7ACB\u5373\u5C06\u{1F917} Dataset \u8F6C\u6362\u4E3A\u53EF\u7528\u4E8E\u8BAD\u7EC3\u7684\u683C\u5F0F\u3002 \u8BA9\u6211\u4EEC\u770B\u770B\u5B83\u5728\u6211\u4EEC\u7684\u6570\u636E\u96C6\u4E0A\u662F\u5982\u4F55\u4F7F\u7528\u7684\uFF01"),vs=_(),E(W.$$.fragment),B=_(),_s=p("p"),ts=n("\u5C31\u662F\u8FD9\u6837\uFF01 \u6211\u4EEC\u53EF\u4EE5\u5C06\u8FD9\u4E9B\u6570\u636E\u96C6\u5E26\u5165\u4E0B\u4E00\u8282\uFF0C\u5728\u7ECF\u8FC7\u6240\u6709\u8270\u82E6\u7684\u6570\u636E\u9884\u5904\u7406\u5DE5\u4F5C\u4E4B\u540E\uFF0C\u8BAD\u7EC3\u5C06\u53D8\u5F97\u975E\u5E38\u7B80\u5355\u3002")},l(N){l=i(N,"P",{});var G=f(l);u=a(G,"\u73B0\u5728\u6211\u4EEC\u6709\u4E86dataset\u548Cdata collator\uFF0C\u6211\u4EEC\u9700\u8981\u5C06dataset\u6279\u91CF\u5730\u5E94\u7528data collator\u3002 \u6211\u4EEC\u53EF\u4EE5\u624B\u52A8\u52A0\u8F7D\u6279\u6B21\u5E76\u6574\u7406\u5B83\u4EEC\uFF0C\u4F46\u8FD9\u9700\u8981\u5927\u91CF\u5DE5\u4F5C\uFF0C\u800C\u4E14\u53EF\u80FD\u6027\u80FD\u4E5F\u4E0D\u662F\u5F88\u597D\u3002 \u76F8\u53CD\uFF0C\u6709\u4E00\u4E2A\u7B80\u5355\u7684\u65B9\u6CD5\u53EF\u4EE5\u4E3A\u8FD9\u4E2A\u95EE\u9898\u63D0\u4F9B\u9AD8\u6548\u7684\u89E3\u51B3\u65B9\u6848\uFF1A"),r=i(G,"CODE",{});var V=f(r);b=a(V,"to_tf_dataset()"),V.forEach(e),k=a(G,"\u3002 \u8FD9\u5C06\u5728\u60A8\u7684\u6570\u636E\u96C6\u4E0A\u8C03\u7528\u4E00\u4E2A "),h=i(G,"CODE",{});var es=f(h);d=a(es,"tf.data.Dataset"),es.forEach(e),x=a(G,"\u7684\u65B9\u6CD5\uFF0C\u8FD9\u4E2A\u65B9\u6CD5\u5E26\u6709\u4E00\u4E2A\u53EF\u9009\u7684data collator\u529F\u80FD\u3002 "),$=i(G,"CODE",{});var ws=f($);y=a(ws,"tf.data.Dataset"),ws.forEach(e),R=a(G," \u662F Keras \u53EF\u7528\u4E8E "),z=i(G,"CODE",{});var ms=f(z);cs=a(ms,"model.fit()"),ms.forEach(e),us=a(G," \u7684\u539F\u751F TensorFlow \u683C\u5F0F\uFF0C\u56E0\u6B64\u8FD9\u79CD\u65B9\u6CD5\u4F1A\u7ACB\u5373\u5C06\u{1F917} Dataset \u8F6C\u6362\u4E3A\u53EF\u7528\u4E8E\u8BAD\u7EC3\u7684\u683C\u5F0F\u3002 \u8BA9\u6211\u4EEC\u770B\u770B\u5B83\u5728\u6211\u4EEC\u7684\u6570\u636E\u96C6\u4E0A\u662F\u5982\u4F55\u4F7F\u7528\u7684\uFF01"),G.forEach(e),vs=m(N),w(W.$$.fragment,N),B=m(N),_s=i(N,"P",{});var wt=f(_s);ts=a(wt,"\u5C31\u662F\u8FD9\u6837\uFF01 \u6211\u4EEC\u53EF\u4EE5\u5C06\u8FD9\u4E9B\u6570\u636E\u96C6\u5E26\u5165\u4E0B\u4E00\u8282\uFF0C\u5728\u7ECF\u8FC7\u6240\u6709\u8270\u82E6\u7684\u6570\u636E\u9884\u5904\u7406\u5DE5\u4F5C\u4E4B\u540E\uFF0C\u8BAD\u7EC3\u5C06\u53D8\u5F97\u975E\u5E38\u7B80\u5355\u3002"),wt.forEach(e)},m(N,G){c(N,l,G),t(l,u),t(l,r),t(r,b),t(l,k),t(l,h),t(h,d),t(l,x),t(l,$),t($,y),t(l,R),t(l,z),t(z,cs),t(l,us),c(N,vs,G),T(W,N,G),c(N,B,G),c(N,_s,G),t(_s,ts),Es=!0},i(N){Es||(g(W.$$.fragment,N),Es=!0)},o(N){j(W.$$.fragment,N),Es=!1},d(N){N&&e(l),N&&e(vs),S(W,N),N&&e(B),N&&e(_s)}}}function Ni(v){let l,u,r,b,k,h,d,x,$,y,R,z,cs,us,vs,W,B,_s,ts,Es,N,G,V,es,ws,ms,wt,Qt,ja,an,ns,as,Tt,J,ka,Fs,xa,va,Us,Ea,wa,Hs,Ta,Sa,ln,St,ya,rn,Bs,on,Ws,pn,I,qa,Xt,Pa,Na,se,Ra,Da,te,Ga,za,ee,Oa,Ca,ne,Aa,Ia,fn,hs,La,ae,Ma,Fa,le,Ua,Ha,cn,Ts,Ba,re,Wa,Va,un,Vs,_n,Js,mn,ds,Ja,oe,Ka,Ya,pe,Za,Qa,hn,Ks,dn,Ys,bn,C,Xa,ie,sl,tl,fe,el,nl,ce,al,ll,ue,rl,ol,_e,pl,il,me,fl,cl,$n,Ss,gn,Zs,ul,jn,ls,rs,yt,ys,_l,qt,ml,hl,kn,Qs,xn,Pt,dl,vn,Xs,En,st,wn,L,bl,Nt,$l,gl,he,jl,kl,de,xl,vl,be,El,wl,$e,Tl,Sl,Tn,qs,Sn,Ps,yl,ge,ql,Pl,yn,tt,qn,Rt,Nl,Pn,et,Nn,bs,Rl,je,Dl,Gl,ke,zl,Ol,Rn,nt,Dn,F,Cl,xe,Al,Il,ve,Ll,Ml,Ee,Fl,Ul,we,Hl,Bl,Gn,Ns,Wl,Te,Vl,Jl,zn,Rs,Kl,Dt,Yl,Zl,On,Gt,Ql,Cn,Ds,Xl,Se,sr,tr,An,$s,er,zt,nr,ar,Ot,lr,rr,In,at,Ln,U,or,ye,pr,ir,qe,fr,cr,Pe,ur,_r,lt,mr,hr,Mn,Gs,dr,rt,br,$r,Fn,ot,Un,D,gr,Ne,jr,kr,Re,xr,vr,De,Er,wr,Ge,Tr,Sr,ze,yr,qr,Oe,Pr,Nr,Ce,Rr,Dr,pt,Gr,zr,Hn,zs,Or,Ae,Cr,Ar,Bn,gs,Ir,Ie,Lr,Mr,Le,Fr,Ur,Wn,it,Vn,Ct,Hr,Jn,ft,Kn,K,Br,Me,Wr,Vr,Fe,Jr,Kr,Ue,Yr,Zr,Yn,M,Qr,He,Xr,so,Be,to,eo,We,no,ao,Ve,lo,ro,Je,oo,po,Zn,At,io,Qn,ct,fo,Xn,ut,sa,It,Os,co,Ke,uo,_o,ta,os,ps,Lt,Y,mo,Ye,ho,bo,Ze,$o,go,Qe,jo,ko,ea,_t,na,mt,aa,Cs,xo,Xe,vo,Eo,la,ht,ra,is,fs,Mt,As,oa,Ft,pa;r=new _i({props:{fw:v[0]}});const To=[hi,mi],dt=[];function So(s,o){return s[0]==="pt"?0:1}x=So(v),$=dt[x]=To[x](v);const yo=[bi,di],bt=[];function qo(s,o){return s[0]==="pt"?0:1}R=qo(v),z=bt[R]=yo[R](v),ms=new ui({});const Po=[gi,$i],$t=[];function No(s,o){return s[0]==="pt"?0:1}ns=No(v),as=$t[ns]=Po[ns](v),Bs=new P({props:{code:"",highlighted:`<span class="hljs-keyword">from</span> datasets <span class="hljs-keyword">import</span> load_dataset

raw_datasets = load_dataset(<span class="hljs-string">&quot;glue&quot;</span>, <span class="hljs-string">&quot;mrpc&quot;</span>)
raw_datasets`}}),Ws=new P({props:{code:`DatasetDict({
    train: Dataset({
        features: ['sentence1', 'sentence2', 'label', 'idx'],
        num_rows: 3668
    })
    validation: Dataset({
        features: ['sentence1', 'sentence2', 'label', 'idx'],
        num_rows: 408
    })
    test: Dataset({
        features: ['sentence1', 'sentence2', 'label', 'idx'],
        num_rows: 1725
    })
})`,highlighted:`DatasetDict({
    train: Dataset({
        features: [<span class="hljs-string">&#x27;sentence1&#x27;</span>, <span class="hljs-string">&#x27;sentence2&#x27;</span>, <span class="hljs-string">&#x27;label&#x27;</span>, <span class="hljs-string">&#x27;idx&#x27;</span>],
        num_rows: <span class="hljs-number">3668</span>
    })
    validation: Dataset({
        features: [<span class="hljs-string">&#x27;sentence1&#x27;</span>, <span class="hljs-string">&#x27;sentence2&#x27;</span>, <span class="hljs-string">&#x27;label&#x27;</span>, <span class="hljs-string">&#x27;idx&#x27;</span>],
        num_rows: <span class="hljs-number">408</span>
    })
    test: Dataset({
        features: [<span class="hljs-string">&#x27;sentence1&#x27;</span>, <span class="hljs-string">&#x27;sentence2&#x27;</span>, <span class="hljs-string">&#x27;label&#x27;</span>, <span class="hljs-string">&#x27;idx&#x27;</span>],
        num_rows: <span class="hljs-number">1725</span>
    })
})`}}),Vs=new P({props:{code:`raw_train_dataset = raw_datasets["train"]
raw_train_dataset[0]`,highlighted:`raw_train_dataset = raw_datasets[<span class="hljs-string">&quot;train&quot;</span>]
raw_train_dataset[<span class="hljs-number">0</span>]`}}),Js=new P({props:{code:`{'idx': 0,
 'label': 1,
 'sentence1': 'Amrozi accused his brother , whom he called " the witness " , of deliberately distorting his evidence .',
 'sentence2': 'Referring to him as only " the witness " , Amrozi accused his brother of deliberately distorting his evidence .'}`,highlighted:`{<span class="hljs-string">&#x27;idx&#x27;</span>: <span class="hljs-number">0</span>,
 <span class="hljs-string">&#x27;label&#x27;</span>: <span class="hljs-number">1</span>,
 <span class="hljs-string">&#x27;sentence1&#x27;</span>: <span class="hljs-string">&#x27;Amrozi accused his brother , whom he called &quot; the witness &quot; , of deliberately distorting his evidence .&#x27;</span>,
 <span class="hljs-string">&#x27;sentence2&#x27;</span>: <span class="hljs-string">&#x27;Referring to him as only &quot; the witness &quot; , Amrozi accused his brother of deliberately distorting his evidence .&#x27;</span>}`}}),Ks=new P({props:{code:"raw_train_dataset.features",highlighted:"raw_train_dataset.features"}}),Ys=new P({props:{code:`{'sentence1': Value(dtype='string', id=None),
 'sentence2': Value(dtype='string', id=None),
 'label': ClassLabel(num_classes=2, names=['not_equivalent', 'equivalent'], names_file=None, id=None),
 'idx': Value(dtype='int32', id=None)}`,highlighted:`{<span class="hljs-string">&#x27;sentence1&#x27;</span>: Value(dtype=<span class="hljs-string">&#x27;string&#x27;</span>, <span class="hljs-built_in">id</span>=<span class="hljs-literal">None</span>),
 <span class="hljs-string">&#x27;sentence2&#x27;</span>: Value(dtype=<span class="hljs-string">&#x27;string&#x27;</span>, <span class="hljs-built_in">id</span>=<span class="hljs-literal">None</span>),
 <span class="hljs-string">&#x27;label&#x27;</span>: ClassLabel(num_classes=<span class="hljs-number">2</span>, names=[<span class="hljs-string">&#x27;not_equivalent&#x27;</span>, <span class="hljs-string">&#x27;equivalent&#x27;</span>], names_file=<span class="hljs-literal">None</span>, <span class="hljs-built_in">id</span>=<span class="hljs-literal">None</span>),
 <span class="hljs-string">&#x27;idx&#x27;</span>: Value(dtype=<span class="hljs-string">&#x27;int32&#x27;</span>, <span class="hljs-built_in">id</span>=<span class="hljs-literal">None</span>)}`}}),Ss=new wo({props:{$$slots:{default:[ji]},$$scope:{ctx:v}}});const Ro=[xi,ki],gt=[];function Do(s,o){return s[0]==="pt"?0:1}ls=Do(v),rs=gt[ls]=Ro[ls](v),Qs=new P({props:{code:"",highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoTokenizer

checkpoint = <span class="hljs-string">&quot;bert-base-uncased&quot;</span>
tokenizer = AutoTokenizer.from_pretrained(checkpoint)
tokenized_sentences_1 = tokenizer(raw_datasets[<span class="hljs-string">&quot;train&quot;</span>][<span class="hljs-string">&quot;sentence1&quot;</span>])
tokenized_sentences_2 = tokenizer(raw_datasets[<span class="hljs-string">&quot;train&quot;</span>][<span class="hljs-string">&quot;sentence2&quot;</span>])`}}),Xs=new P({props:{code:`inputs = tokenizer("This is the first sentence.", "This is the second one.")
inputs`,highlighted:`inputs = tokenizer(<span class="hljs-string">&quot;This is the first sentence.&quot;</span>, <span class="hljs-string">&quot;This is the second one.&quot;</span>)
inputs`}}),st=new P({props:{code:`{ 
  'input_ids': [101, 2023, 2003, 1996, 2034, 6251, 1012, 102, 2023, 2003, 1996, 2117, 2028, 1012, 102],
  'token_type_ids': [0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1],
  'attention_mask': [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]
}`,highlighted:`{ 
  <span class="hljs-string">&#x27;input_ids&#x27;</span>: [<span class="hljs-number">101</span>, <span class="hljs-number">2023</span>, <span class="hljs-number">2003</span>, <span class="hljs-number">1996</span>, <span class="hljs-number">2034</span>, <span class="hljs-number">6251</span>, <span class="hljs-number">1012</span>, <span class="hljs-number">102</span>, <span class="hljs-number">2023</span>, <span class="hljs-number">2003</span>, <span class="hljs-number">1996</span>, <span class="hljs-number">2117</span>, <span class="hljs-number">2028</span>, <span class="hljs-number">1012</span>, <span class="hljs-number">102</span>],
  <span class="hljs-string">&#x27;token_type_ids&#x27;</span>: [<span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>],
  <span class="hljs-string">&#x27;attention_mask&#x27;</span>: [<span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>]
}`}}),qs=new wo({props:{$$slots:{default:[vi]},$$scope:{ctx:v}}}),tt=new P({props:{code:'tokenizer.convert_ids_to_tokens(inputs["input_ids"])',highlighted:'tokenizer.convert_ids_to_tokens(inputs[<span class="hljs-string">&quot;input_ids&quot;</span>])'}}),et=new P({props:{code:"['[CLS]', 'this', 'is', 'the', 'first', 'sentence', '.', '[SEP]', 'this', 'is', 'the', 'second', 'one', '.', '[SEP]']",highlighted:'[<span class="hljs-string">&#x27;[CLS]&#x27;</span>, <span class="hljs-string">&#x27;this&#x27;</span>, <span class="hljs-string">&#x27;is&#x27;</span>, <span class="hljs-string">&#x27;the&#x27;</span>, <span class="hljs-string">&#x27;first&#x27;</span>, <span class="hljs-string">&#x27;sentence&#x27;</span>, <span class="hljs-string">&#x27;.&#x27;</span>, <span class="hljs-string">&#x27;[SEP]&#x27;</span>, <span class="hljs-string">&#x27;this&#x27;</span>, <span class="hljs-string">&#x27;is&#x27;</span>, <span class="hljs-string">&#x27;the&#x27;</span>, <span class="hljs-string">&#x27;second&#x27;</span>, <span class="hljs-string">&#x27;one&#x27;</span>, <span class="hljs-string">&#x27;.&#x27;</span>, <span class="hljs-string">&#x27;[SEP]&#x27;</span>]'}}),nt=new P({props:{code:`['[CLS]', 'this', 'is', 'the', 'first', 'sentence', '.', '[SEP]', 'this', 'is', 'the', 'second', 'one', '.', '[SEP]']
[      0,      0,    0,     0,       0,          0,   0,       0,      1,    1,     1,        1,     1,   1,       1]`,highlighted:`[<span class="hljs-string">&#x27;[CLS]&#x27;</span>, <span class="hljs-string">&#x27;this&#x27;</span>, <span class="hljs-string">&#x27;is&#x27;</span>, <span class="hljs-string">&#x27;the&#x27;</span>, <span class="hljs-string">&#x27;first&#x27;</span>, <span class="hljs-string">&#x27;sentence&#x27;</span>, <span class="hljs-string">&#x27;.&#x27;</span>, <span class="hljs-string">&#x27;[SEP]&#x27;</span>, <span class="hljs-string">&#x27;this&#x27;</span>, <span class="hljs-string">&#x27;is&#x27;</span>, <span class="hljs-string">&#x27;the&#x27;</span>, <span class="hljs-string">&#x27;second&#x27;</span>, <span class="hljs-string">&#x27;one&#x27;</span>, <span class="hljs-string">&#x27;.&#x27;</span>, <span class="hljs-string">&#x27;[SEP]&#x27;</span>]
[      <span class="hljs-number">0</span>,      <span class="hljs-number">0</span>,    <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,       <span class="hljs-number">0</span>,          <span class="hljs-number">0</span>,   <span class="hljs-number">0</span>,       <span class="hljs-number">0</span>,      <span class="hljs-number">1</span>,    <span class="hljs-number">1</span>,     <span class="hljs-number">1</span>,        <span class="hljs-number">1</span>,     <span class="hljs-number">1</span>,   <span class="hljs-number">1</span>,       <span class="hljs-number">1</span>]`}}),at=new P({props:{code:`tokenized_dataset = tokenizer(
    raw_datasets["train"]["sentence1"],
    raw_datasets["train"]["sentence2"],
    padding=True,
    truncation=True,
)`,highlighted:`tokenized_dataset = tokenizer(
    raw_datasets[<span class="hljs-string">&quot;train&quot;</span>][<span class="hljs-string">&quot;sentence1&quot;</span>],
    raw_datasets[<span class="hljs-string">&quot;train&quot;</span>][<span class="hljs-string">&quot;sentence2&quot;</span>],
    padding=<span class="hljs-literal">True</span>,
    truncation=<span class="hljs-literal">True</span>,
)`}}),ot=new P({props:{code:`def tokenize_function(example):
    return tokenizer(example["sentence1"], example["sentence2"], truncation=True)`,highlighted:`<span class="hljs-keyword">def</span> <span class="hljs-title function_">tokenize_function</span>(<span class="hljs-params">example</span>):
    <span class="hljs-keyword">return</span> tokenizer(example[<span class="hljs-string">&quot;sentence1&quot;</span>], example[<span class="hljs-string">&quot;sentence2&quot;</span>], truncation=<span class="hljs-literal">True</span>)`}}),it=new P({props:{code:`tokenized_datasets = raw_datasets.map(tokenize_function, batched=True)
tokenized_datasets`,highlighted:`tokenized_datasets = raw_datasets.<span class="hljs-built_in">map</span>(tokenize_function, batched=<span class="hljs-literal">True</span>)
tokenized_datasets`}}),ft=new P({props:{code:`DatasetDict({
    train: Dataset({
        features: ['attention_mask', 'idx', 'input_ids', 'label', 'sentence1', 'sentence2', 'token_type_ids'],
        num_rows: 3668
    })
    validation: Dataset({
        features: ['attention_mask', 'idx', 'input_ids', 'label', 'sentence1', 'sentence2', 'token_type_ids'],
        num_rows: 408
    })
    test: Dataset({
        features: ['attention_mask', 'idx', 'input_ids', 'label', 'sentence1', 'sentence2', 'token_type_ids'],
        num_rows: 1725
    })
})`,highlighted:`DatasetDict({
    train: Dataset({
        features: [<span class="hljs-string">&#x27;attention_mask&#x27;</span>, <span class="hljs-string">&#x27;idx&#x27;</span>, <span class="hljs-string">&#x27;input_ids&#x27;</span>, <span class="hljs-string">&#x27;label&#x27;</span>, <span class="hljs-string">&#x27;sentence1&#x27;</span>, <span class="hljs-string">&#x27;sentence2&#x27;</span>, <span class="hljs-string">&#x27;token_type_ids&#x27;</span>],
        num_rows: <span class="hljs-number">3668</span>
    })
    validation: Dataset({
        features: [<span class="hljs-string">&#x27;attention_mask&#x27;</span>, <span class="hljs-string">&#x27;idx&#x27;</span>, <span class="hljs-string">&#x27;input_ids&#x27;</span>, <span class="hljs-string">&#x27;label&#x27;</span>, <span class="hljs-string">&#x27;sentence1&#x27;</span>, <span class="hljs-string">&#x27;sentence2&#x27;</span>, <span class="hljs-string">&#x27;token_type_ids&#x27;</span>],
        num_rows: <span class="hljs-number">408</span>
    })
    test: Dataset({
        features: [<span class="hljs-string">&#x27;attention_mask&#x27;</span>, <span class="hljs-string">&#x27;idx&#x27;</span>, <span class="hljs-string">&#x27;input_ids&#x27;</span>, <span class="hljs-string">&#x27;label&#x27;</span>, <span class="hljs-string">&#x27;sentence1&#x27;</span>, <span class="hljs-string">&#x27;sentence2&#x27;</span>, <span class="hljs-string">&#x27;token_type_ids&#x27;</span>],
        num_rows: <span class="hljs-number">1725</span>
    })
})`}}),ut=new nn({props:{id:"7q5NyFT8REg"}});function Go(s,o){return s[0]==="pt"?wi:Ei}let ia=Go(v),xs=ia(v);const zo=[Si,Ti],jt=[];function Oo(s,o){return s[0]==="pt"?0:1}os=Oo(v),ps=jt[os]=zo[os](v),_t=new P({props:{code:`samples = tokenized_datasets["train"][:8]
samples = {k: v for k, v in samples.items() if k not in ["idx", "sentence1", "sentence2"]}
[len(x) for x in samples["input_ids"]]`,highlighted:`samples = tokenized_datasets[<span class="hljs-string">&quot;train&quot;</span>][:<span class="hljs-number">8</span>]
samples = {k: v <span class="hljs-keyword">for</span> k, v <span class="hljs-keyword">in</span> samples.items() <span class="hljs-keyword">if</span> k <span class="hljs-keyword">not</span> <span class="hljs-keyword">in</span> [<span class="hljs-string">&quot;idx&quot;</span>, <span class="hljs-string">&quot;sentence1&quot;</span>, <span class="hljs-string">&quot;sentence2&quot;</span>]}
[<span class="hljs-built_in">len</span>(x) <span class="hljs-keyword">for</span> x <span class="hljs-keyword">in</span> samples[<span class="hljs-string">&quot;input_ids&quot;</span>]]`}}),mt=new P({props:{code:"[50, 59, 47, 67, 59, 50, 62, 32]",highlighted:'[<span class="hljs-number">50</span>, <span class="hljs-number">59</span>, <span class="hljs-number">47</span>, <span class="hljs-number">67</span>, <span class="hljs-number">59</span>, <span class="hljs-number">50</span>, <span class="hljs-number">62</span>, <span class="hljs-number">32</span>]'}}),ht=new P({props:{code:"",highlighted:'\n```py\nbatch = dat<span class="hljs-built_in">a_collator</span>(samples)\n<span class="hljs-title">{k:</span> v.shape for k, v in batch.items()}'}});const Co=[qi,yi],kt=[];function Ao(s,o){return s[0]==="tf"?0:1}is=Ao(v),fs=kt[is]=Co[is](v),As=new wo({props:{$$slots:{default:[Pi]},$$scope:{ctx:v}}});let O=v[0]==="tf"&&li();return{c(){l=p("meta"),u=_(),E(r.$$.fragment),b=_(),k=p("h1"),h=n("\u5904\u7406\u6570\u636E"),d=_(),$.c(),y=_(),z.c(),cs=_(),us=p("p"),vs=n("\u5F53\u7136\uFF0C\u4EC5\u4EC5\u7528\u4E24\u53E5\u8BDD\u8BAD\u7EC3\u6A21\u578B\u4E0D\u4F1A\u4EA7\u751F\u5F88\u597D\u7684\u6548\u679C\u3002\u4E3A\u4E86\u83B7\u5F97\u66F4\u597D\u7684\u7ED3\u679C\uFF0C\u60A8\u9700\u8981\u51C6\u5907\u4E00\u4E2A\u66F4\u5927\u7684\u6570\u636E\u96C6\u3002"),W=_(),B=p("p"),_s=n("\u5728\u672C\u8282\u4E2D\uFF0C\u6211\u4EEC\u5C06\u4F7F\u7528MRPC\uFF08\u5FAE\u8F6F\u7814\u7A76\u91CA\u4E49\u8BED\u6599\u5E93\uFF09\u6570\u636E\u96C6\u4F5C\u4E3A\u793A\u4F8B\uFF0C\u8BE5\u6570\u636E\u96C6\u7531\u5A01\u5EC9\xB7\u591A\u5170\u548C\u514B\u91CC\u65AF\xB7\u5E03\u7F57\u514B\u7279\u5728"),ts=p("a"),Es=n("\u8FD9\u7BC7\u6587\u7AE0"),N=n("\u53D1\u5E03\u3002\u8BE5\u6570\u636E\u96C6\u75315801\u5BF9\u53E5\u5B50\u7EC4\u6210\uFF0C\u6BCF\u4E2A\u53E5\u5B50\u5BF9\u5E26\u6709\u4E00\u4E2A\u6807\u7B7E\uFF0C\u6307\u793A\u5B83\u4EEC\u662F\u5426\u4E3A\u540C\u4E49\uFF08\u5373\uFF0C\u5982\u679C\u4E24\u4E2A\u53E5\u5B50\u7684\u610F\u601D\u76F8\u540C\uFF09\u3002\u6211\u4EEC\u5728\u672C\u7AE0\u4E2D\u9009\u62E9\u4E86\u5B83\uFF0C\u56E0\u4E3A\u5B83\u662F\u4E00\u4E2A\u5C0F\u6570\u636E\u96C6\uFF0C\u6240\u4EE5\u5F88\u5BB9\u6613\u5BF9\u5B83\u8FDB\u884C\u8BAD\u7EC3\u3002"),G=_(),V=p("h3"),es=p("a"),ws=p("span"),E(ms.$$.fragment),wt=_(),Qt=p("span"),ja=n("\u4ECE\u6A21\u578B\u4E2D\u5FC3\uFF08Hub\uFF09\u52A0\u8F7D\u6570\u636E\u96C6"),an=_(),as.c(),Tt=_(),J=p("p"),ka=n("\u6A21\u578B\u4E2D\u5FC3\uFF08hub\uFF09\u4E0D\u53EA\u662F\u5305\u542B\u6A21\u578B\uFF1B\u5B83\u4E5F\u6709\u8BB8\u591A\u4E0D\u540C\u8BED\u8A00\u7684\u591A\u4E2A\u6570\u636E\u96C6\u3002\u70B9\u51FB"),Fs=p("a"),xa=n("\u6570\u636E\u96C6"),va=n("\u7684\u94FE\u63A5\u5373\u53EF\u8FDB\u884C\u6D4F\u89C8\u3002\u6211\u4EEC\u5EFA\u8BAE\u60A8\u5728\u9605\u8BFB\u672C\u8282\u540E\u9605\u8BFB\u4E00\u4E0B"),Us=p("a"),Ea=n("\u52A0\u8F7D\u548C\u5904\u7406\u65B0\u7684\u6570\u636E\u96C6"),wa=n("\u8FD9\u7BC7\u6587\u7AE0\uFF0C\u8FD9\u4F1A\u8BA9\u60A8\u5BF9huggingface\u7684darasets\u66F4\u52A0\u6E05\u6670\u3002\u4F46\u73B0\u5728\uFF0C\u8BA9\u6211\u4EEC\u4F7F\u7528MRPC\u6570\u636E\u96C6\u4E2D\u7684"),Hs=p("a"),Ta=n("GLUE \u57FA\u51C6\u6D4B\u8BD5\u6570\u636E\u96C6"),Sa=n("\uFF0C\u5B83\u662F\u6784\u6210MRPC\u6570\u636E\u96C6\u768410\u4E2A\u6570\u636E\u96C6\u4E4B\u4E00\uFF0C\u8FD9\u662F\u4E00\u4E2A\u5B66\u672F\u57FA\u51C6\uFF0C\u7528\u4E8E\u8861\u91CF\u673A\u5668\u5B66\u4E60\u6A21\u578B\u572810\u4E2A\u4E0D\u540C\u6587\u672C\u5206\u7C7B\u4EFB\u52A1\u4E2D\u7684\u6027\u80FD\u3002"),ln=_(),St=p("p"),ya=n("\u{1F917} Datasets\u5E93\u63D0\u4F9B\u4E86\u4E00\u4E2A\u975E\u5E38\u4FBF\u6377\u7684\u547D\u4EE4\uFF0C\u53EF\u4EE5\u5728\u6A21\u578B\u4E2D\u5FC3\uFF08hub\uFF09\u4E0A\u4E0B\u8F7D\u548C\u7F13\u5B58\u6570\u636E\u96C6\u3002\u6211\u4EEC\u53EF\u4EE5\u901A\u8FC7\u4EE5\u4E0B\u7684\u4EE3\u7801\u4E0B\u8F7DMRPC\u6570\u636E\u96C6\uFF1A"),rn=_(),E(Bs.$$.fragment),on=_(),E(Ws.$$.fragment),pn=_(),I=p("p"),qa=n("\u6B63\u5982\u4F60\u6240\u770B\u5230\u7684\uFF0C\u6211\u4EEC\u83B7\u5F97\u4E86\u4E00\u4E2A"),Xt=p("strong"),Pa=n("DatasetDict"),Na=n("\u5BF9\u8C61\uFF0C\u5176\u4E2D\u5305\u542B\u8BAD\u7EC3\u96C6\u3001\u9A8C\u8BC1\u96C6\u548C\u6D4B\u8BD5\u96C6\u3002\u6BCF\u4E00\u4E2A\u96C6\u5408\u90FD\u5305\u542B\u51E0\u4E2A\u5217("),se=p("strong"),Ra=n("sentence1"),Da=n(", "),te=p("strong"),Ga=n("sentence2"),za=n(", "),ee=p("strong"),Oa=n("label"),Ca=n(", and "),ne=p("strong"),Aa=n("idx"),Ia=n(")\u4EE5\u53CA\u4E00\u4E2A\u4EE3\u8868\u884C\u6570\u7684\u53D8\u91CF\uFF0C\u5373\u6BCF\u4E2A\u96C6\u5408\u4E2D\u7684\u884C\u7684\u4E2A\u6570\uFF08\u56E0\u6B64\uFF0C\u8BAD\u7EC3\u96C6\u4E2D\u67093668\u5BF9\u53E5\u5B50\uFF0C\u9A8C\u8BC1\u96C6\u4E2D\u6709408\u5BF9\uFF0C\u6D4B\u8BD5\u96C6\u4E2D\u67091725\u5BF9\uFF09\u3002"),fn=_(),hs=p("p"),La=n("\u9ED8\u8BA4\u60C5\u51B5\u4E0B\uFF0C\u6B64\u547D\u4EE4\u5728\u4E0B\u8F7D\u6570\u636E\u96C6\u5E76\u7F13\u5B58\u5230 "),ae=p("strong"),Ma=n("~/.cache/huggingface/dataset"),Fa=n(". \u56DE\u60F3\u4E00\u4E0B\u7B2C2\u7AE0\uFF0C\u60A8\u53EF\u4EE5\u901A\u8FC7\u8BBE\u7F6E"),le=p("strong"),Ua=n("HF_HOME"),Ha=n("\u73AF\u5883\u53D8\u91CF\u6765\u81EA\u5B9A\u4E49\u7F13\u5B58\u7684\u6587\u4EF6\u5939\u3002"),cn=_(),Ts=p("p"),Ba=n("\u6211\u4EEC\u53EF\u4EE5\u8BBF\u95EE\u6211\u4EEC\u6570\u636E\u96C6\u4E2D\u7684\u6BCF\u4E00\u4E2A"),re=p("strong"),Wa=n("raw_train_dataset"),Va=n("\u5BF9\u8C61\uFF0C\u5982\u4F7F\u7528\u5B57\u5178\uFF1A"),un=_(),E(Vs.$$.fragment),_n=_(),E(Js.$$.fragment),mn=_(),ds=p("p"),Ja=n("\u6211\u4EEC\u53EF\u4EE5\u770B\u5230\u6807\u7B7E\u5DF2\u7ECF\u662F\u6574\u6570\u4E86\uFF0C\u6240\u4EE5\u6211\u4EEC\u4E0D\u9700\u8981\u5BF9\u6807\u7B7E\u505A\u4EFB\u4F55\u9884\u5904\u7406\u3002\u8981\u77E5\u9053\u54EA\u4E2A\u6570\u5B57\u5BF9\u5E94\u4E8E\u54EA\u4E2A\u6807\u7B7E\uFF0C\u6211\u4EEC\u53EF\u4EE5\u67E5\u770B"),oe=p("strong"),Ka=n("raw_train_dataset"),Ya=n("\u7684"),pe=p("strong"),Za=n("features"),Qa=n(". \u8FD9\u5C06\u544A\u8BC9\u6211\u4EEC\u6BCF\u5217\u7684\u7C7B\u578B\uFF1A"),hn=_(),E(Ks.$$.fragment),dn=_(),E(Ys.$$.fragment),bn=_(),C=p("p"),Xa=n("\u5728\u4E0A\u9762\u7684\u4F8B\u5B50\u4E4B\u4E2D,"),ie=p("strong"),sl=n("Label\uFF08\u6807\u7B7E\uFF09"),tl=n(" \u662F\u4E00\u79CD"),fe=p("strong"),el=n("ClassLabel\uFF08\u5206\u7C7B\u6807\u7B7E\uFF09"),nl=n("\uFF0C\u4F7F\u7528\u6574\u6570\u5EFA\u7ACB\u8D77\u5230\u7C7B\u522B\u6807\u7B7E\u7684\u6620\u5C04\u5173\u7CFB\u3002"),ce=p("strong"),al=n("0"),ll=n("\u5BF9\u5E94\u4E8E"),ue=p("strong"),rl=n("not_equivalent"),ol=n("\uFF0C"),_e=p("strong"),pl=n("1"),il=n("\u5BF9\u5E94\u4E8E"),me=p("strong"),fl=n("equivalent"),cl=n("\u3002"),$n=_(),E(Ss.$$.fragment),gn=_(),Zs=p("h3"),ul=n("\u9884\u5904\u7406\u6570\u636E\u96C6"),jn=_(),rs.c(),yt=_(),ys=p("p"),_l=n("\u4E3A\u4E86\u9884\u5904\u7406\u6570\u636E\u96C6\uFF0C\u6211\u4EEC\u9700\u8981\u5C06\u6587\u672C\u8F6C\u6362\u4E3A\u6A21\u578B\u80FD\u591F\u7406\u89E3\u7684\u6570\u5B57\u3002\u6B63\u5982\u4F60\u5728"),qt=p("a"),ml=n("\u7B2C\u4E8C\u7AE0"),hl=n("\u4E0A\u770B\u5230\u7684\u90A3\u6837"),kn=_(),E(Qs.$$.fragment),xn=_(),Pt=p("p"),dl=n("\u7136\u800C\uFF0C\u5728\u4E24\u53E5\u8BDD\u4F20\u9012\u7ED9\u6A21\u578B\uFF0C\u9884\u6D4B\u8FD9\u4E24\u53E5\u8BDD\u662F\u5426\u662F\u540C\u4E49\u4E4B\u524D\u3002\u6211\u4EEC\u9700\u8981\u8FD9\u4E24\u53E5\u8BDD\u4F9D\u6B21\u8FDB\u884C\u9002\u5F53\u7684\u9884\u5904\u7406\u3002\u5E78\u8FD0\u7684\u662F\uFF0C\u6807\u8BB0\u5668\u4E0D\u4EC5\u4EC5\u53EF\u4EE5\u8F93\u5165\u5355\u4E2A\u53E5\u5B50\u8FD8\u53EF\u4EE5\u8F93\u5165\u4E00\u7EC4\u53E5\u5B50\uFF0C\u5E76\u6309\u7167\u6211\u4EEC\u7684BERT\u6A21\u578B\u6240\u671F\u671B\u7684\u8F93\u5165\u8FDB\u884C\u5904\u7406\uFF1A"),vn=_(),E(Xs.$$.fragment),En=_(),E(st.$$.fragment),wn=_(),L=p("p"),bl=n("\u6211\u4EEC\u5728"),Nt=p("a"),$l=n("\u7B2C\u4E8C\u7AE0"),gl=n(" \u8BA8\u8BBA\u4E86"),he=p("strong"),jl=n("\u8F93\u5165\u8BCDid(input_ids)"),kl=n(" \u548C "),de=p("strong"),xl=n("\u6CE8\u610F\u529B\u906E\u7F69(attention_mask)"),vl=n(" \uFF0C\u4F46\u6211\u4EEC\u5728\u90A3\u4E2A\u65F6\u5019\u6CA1\u6709\u8BA8\u8BBA"),be=p("strong"),El=n("\u7C7B\u578B\u6807\u8BB0ID(token_type_ids)"),wl=n("\u3002\u5728\u8FD9\u4E2A\u4F8B\u5B50\u4E2D\uFF0C"),$e=p("strong"),Tl=n("\u7C7B\u578B\u6807\u8BB0ID(token_type_ids)"),Sl=n("\u7684\u4F5C\u7528\u5C31\u662F\u544A\u8BC9\u6A21\u578B\u8F93\u5165\u7684\u54EA\u4E00\u90E8\u5206\u662F\u7B2C\u4E00\u53E5\uFF0C\u54EA\u4E00\u90E8\u5206\u662F\u7B2C\u4E8C\u53E5\u3002"),Tn=_(),E(qs.$$.fragment),Sn=_(),Ps=p("p"),yl=n("\u5982\u679C\u6211\u4EEC\u5C06"),ge=p("strong"),ql=n("input_ids"),Pl=n("\u4E2D\u7684id\u8F6C\u6362\u56DE\u6587\u5B57:"),yn=_(),E(tt.$$.fragment),qn=_(),Rt=p("p"),Nl=n("\u6211\u4EEC\u5C06\u5F97\u5230\uFF1A"),Pn=_(),E(et.$$.fragment),Nn=_(),bs=p("p"),Rl=n("\u6240\u4EE5\u6211\u4EEC\u770B\u5230\u6A21\u578B\u9700\u8981\u8F93\u5165\u7684\u5F62\u5F0F\u662F "),je=p("strong"),Dl=n("[CLS] sentence1 [SEP] sentence2 [SEP]"),Gl=n("\u3002\u56E0\u6B64\uFF0C\u5F53\u6709\u4E24\u53E5\u8BDD\u7684\u65F6\u5019\u3002"),ke=p("strong"),zl=n("\u7C7B\u578B\u6807\u8BB0ID(token_type_ids)"),Ol=n(" \u7684\u503C\u662F\uFF1A"),Rn=_(),E(nt.$$.fragment),Dn=_(),F=p("p"),Cl=n("\u5982\u60A8\u6240\u89C1\uFF0C\u8F93\u5165\u4E2D "),xe=p("strong"),Al=n("[CLS] sentence1 [SEP]"),Il=n(" \u5B83\u4EEC\u7684\u7C7B\u578B\u6807\u8BB0ID\u5747\u4E3A"),ve=p("strong"),Ll=n("0"),Ml=n("\uFF0C\u800C\u5176\u4ED6\u90E8\u5206\uFF0C\u5BF9\u5E94\u4E8E"),Ee=p("strong"),Fl=n("sentence2 [SEP]"),Ul=n("\uFF0C\u6240\u6709\u7684\u7C7B\u578B\u6807\u8BB0ID\u5747\u4E3A"),we=p("strong"),Hl=n("1"),Bl=n("."),Gn=_(),Ns=p("p"),Wl=n("\u8BF7\u6CE8\u610F\uFF0C\u5982\u679C\u9009\u62E9\u5176\u4ED6\u7684\u68C0\u67E5\u70B9\uFF0C\u5219\u4E0D\u4E00\u5B9A\u5177\u6709"),Te=p("strong"),Vl=n("\u7C7B\u578B\u6807\u8BB0ID(token_type_ids)"),Jl=n("\uFF08\u4F8B\u5982\uFF0C\u5982\u679C\u4F7F\u7528DistilBERT\u6A21\u578B\uFF0C\u5C31\u4E0D\u4F1A\u8FD4\u56DE\u5B83\u4EEC\uFF09\u3002\u53EA\u6709\u5F53\u5B83\u5728\u9884\u8BAD\u7EC3\u671F\u95F4\u4F7F\u7528\u8FC7\u8FD9\u4E00\u5C42\uFF0C\u6A21\u578B\u5728\u6784\u5EFA\u65F6\u4F9D\u8D56\u5B83\u4EEC\uFF0C\u624D\u4F1A\u8FD4\u56DE\u5B83\u4EEC\u3002"),zn=_(),Rs=p("p"),Kl=n("\u7528\u7C7B\u578B\u6807\u8BB0ID\u5BF9BERT\u8FDB\u884C\u9884\u8BAD\u7EC3,\u5E76\u4E14\u4F7F\u7528"),Dt=p("a"),Yl=n("\u7B2C\u4E00\u7AE0"),Zl=n("\u7684\u906E\u7F69\u8BED\u8A00\u6A21\u578B\uFF0C\u8FD8\u6709\u4E00\u4E2A\u989D\u5916\u7684\u5E94\u7528\u7C7B\u578B\uFF0C\u53EB\u505A\u4E0B\u4E00\u53E5\u9884\u6D4B. \u8FD9\u9879\u4EFB\u52A1\u7684\u76EE\u6807\u662F\u5EFA\u7ACB\u6210\u5BF9\u53E5\u5B50\u4E4B\u95F4\u5173\u7CFB\u7684\u6A21\u578B\u3002"),On=_(),Gt=p("p"),Ql=n("\u5728\u4E0B\u4E00\u4E2A\u53E5\u5B50\u9884\u6D4B\u4EFB\u52A1\u4E2D\uFF0C\u4F1A\u7ED9\u6A21\u578B\u8F93\u5165\u6210\u5BF9\u7684\u53E5\u5B50\uFF08\u5E26\u6709\u968F\u673A\u906E\u7F69\u7684\u6807\u8BB0\uFF09\uFF0C\u5E76\u88AB\u8981\u6C42\u9884\u6D4B\u7B2C\u4E8C\u4E2A\u53E5\u5B50\u662F\u5426\u7D27\u8DDF\u7B2C\u4E00\u4E2A\u53E5\u5B50\u3002\u4E3A\u4E86\u63D0\u9AD8\u6A21\u578B\u7684\u6CDB\u5316\u80FD\u529B\uFF0C\u6570\u636E\u96C6\u4E2D\u4E00\u534A\u7684\u4E24\u4E2A\u53E5\u5B50\u5728\u539F\u59CB\u6587\u6863\u4E2D\u6328\u5728\u4E00\u8D77\uFF0C\u53E6\u4E00\u534A\u7684\u4E24\u4E2A\u53E5\u5B50\u6765\u81EA\u4E24\u4E2A\u4E0D\u540C\u7684\u6587\u6863\u3002"),Cn=_(),Ds=p("p"),Xl=n("\u4E00\u822C\u6765\u8BF4\uFF0C\u4F60\u4E0D\u9700\u8981\u62C5\u5FC3\u662F\u5426\u6709"),Se=p("strong"),sr=n("\u7C7B\u578B\u6807\u8BB0ID(token_type_ids)"),tr=n("\u3002\u5728\u60A8\u7684\u6807\u8F93\u5165\u4E2D\uFF1A\u53EA\u8981\u60A8\u5BF9\u6807\u8BB0\u5668\u548C\u6A21\u578B\u4F7F\u7528\u76F8\u540C\u7684\u68C0\u67E5\u70B9\uFF0C\u4E00\u5207\u90FD\u4F1A\u5F88\u597D\uFF0C\u56E0\u4E3A\u6807\u8BB0\u5668\u77E5\u9053\u5411\u5176\u6A21\u578B\u63D0\u4F9B\u4EC0\u4E48\u3002"),An=_(),$s=p("p"),er=n("\u73B0\u5728\u6211\u4EEC\u5DF2\u7ECF\u4E86\u89E3\u4E86\u6807\u8BB0\u5668\u5982\u4F55\u5904\u7406\u4E00\u5BF9\u53E5\u5B50\uFF0C\u6211\u4EEC\u53EF\u4EE5\u4F7F\u7528\u5B83\u5BF9\u6574\u4E2A\u6570\u636E\u96C6\u8FDB\u884C\u5904\u7406\uFF1A\u5982"),zt=p("a"),nr=n("\u4E4B\u524D\u7684\u7AE0\u8282"),ar=n("\uFF0C\u6211\u4EEC\u53EF\u4EE5\u7ED9\u6807\u8BB0\u5668\u63D0\u4F9B\u4E00\u7EC4\u53E5\u5B50\uFF0C\u7B2C\u4E00\u4E2A\u53C2\u6570\u662F\u5B83\u7B2C\u4E00\u4E2A\u53E5\u5B50\u7684\u5217\u8868\uFF0C\u7B2C\u4E8C\u4E2A\u53C2\u6570\u662F\u7B2C\u4E8C\u4E2A\u53E5\u5B50\u7684\u5217\u8868\u3002\u8FD9\u4E5F\u4E0E\u6211\u4EEC\u5728"),Ot=p("a"),lr=n("\u7B2C\u4E8C\u7AE0"),rr=n("\u4E2D\u770B\u5230\u7684\u586B\u5145\u548C\u622A\u65AD\u9009\u9879\u517C\u5BB9. \u56E0\u6B64\uFF0C\u9884\u5904\u7406\u8BAD\u7EC3\u6570\u636E\u96C6\u7684\u4E00\u79CD\u65B9\u6CD5\u662F\uFF1A"),In=_(),E(at.$$.fragment),Ln=_(),U=p("p"),or=n("\u8FD9\u5F88\u6709\u6548\uFF0C\u4F46\u5B83\u7684\u7F3A\u70B9\u662F\u8FD4\u56DE\u5B57\u5178\uFF08\u5B57\u5178\u7684\u952E\u662F"),ye=p("strong"),pr=n("\u8F93\u5165\u8BCDid(input_ids)"),ir=n(" \uFF0C "),qe=p("strong"),fr=n("\u6CE8\u610F\u529B\u906E\u7F69(attention_mask)"),cr=n(" \u548C "),Pe=p("strong"),ur=n("\u7C7B\u578B\u6807\u8BB0ID(token_type_ids)"),_r=n("\uFF0C\u5B57\u5178\u7684\u503C\u662F\u952E\u6240\u5BF9\u5E94\u503C\u7684\u5217\u8868\uFF09\u3002\u800C\u4E14\u53EA\u6709\u5F53\u60A8\u5728\u8F6C\u6362\u8FC7\u7A0B\u4E2D\u6709\u8DB3\u591F\u7684\u5185\u5B58\u6765\u5B58\u50A8\u6574\u4E2A\u6570\u636E\u96C6\u65F6\u624D\u4E0D\u4F1A\u51FA\u9519\uFF08\u800C\u{1F917}\u6570\u636E\u96C6\u5E93\u4E2D\u7684\u6570\u636E\u96C6\u662F\u4EE5"),lt=p("a"),mr=n("Apache Arrow"),hr=n("\u6587\u4EF6\u5B58\u50A8\u5728\u78C1\u76D8\u4E0A\uFF0C\u56E0\u6B64\u60A8\u53EA\u9700\u5C06\u63A5\u4E0B\u6765\u8981\u7528\u7684\u6570\u636E\u52A0\u8F7D\u5728\u5185\u5B58\u4E2D\uFF0C\u56E0\u6B64\u4F1A\u5BF9\u5185\u5B58\u5BB9\u91CF\u7684\u9700\u6C42\u8981\u4F4E\u4E00\u4E9B\uFF09\u3002"),Mn=_(),Gs=p("p"),dr=n("\u4E3A\u4E86\u5C06\u6570\u636E\u4FDD\u5B58\u4E3A\u6570\u636E\u96C6\uFF0C\u6211\u4EEC\u5C06\u4F7F\u7528"),rt=p("a"),br=n("Dataset.map()"),$r=n("\u65B9\u6CD5\uFF0C\u5982\u679C\u6211\u4EEC\u9700\u8981\u505A\u66F4\u591A\u7684\u9884\u5904\u7406\u800C\u4E0D\u4EC5\u4EC5\u662F\u6807\u8BB0\u5316\uFF0C\u90A3\u4E48\u8FD9\u4E5F\u7ED9\u4E86\u6211\u4EEC\u4E00\u4E9B\u989D\u5916\u7684\u81EA\u5B9A\u4E49\u7684\u65B9\u6CD5\u3002\u8FD9\u4E2A\u65B9\u6CD5\u7684\u5DE5\u4F5C\u539F\u7406\u662F\u5728\u6570\u636E\u96C6\u7684\u6BCF\u4E2A\u5143\u7D20\u4E0A\u5E94\u7528\u4E00\u4E2A\u51FD\u6570\uFF0C\u56E0\u6B64\u8BA9\u6211\u4EEC\u5B9A\u4E49\u4E00\u4E2A\u6807\u8BB0\u8F93\u5165\u7684\u51FD\u6570\uFF1A"),Fn=_(),E(ot.$$.fragment),Un=_(),D=p("p"),gr=n("\u6B64\u51FD\u6570\u7684\u8F93\u5165\u662F\u4E00\u4E2A\u5B57\u5178\uFF08\u4E0E\u6570\u636E\u96C6\u7684\u9879\u7C7B\u4F3C\uFF09\uFF0C\u5E76\u8FD4\u56DE\u4E00\u4E2A\u5305\u542B"),Ne=p("strong"),jr=n("\u8F93\u5165\u8BCDid(input_ids)"),kr=n(" \uFF0C "),Re=p("strong"),xr=n("\u6CE8\u610F\u529B\u906E\u7F69(attention_mask)"),vr=n(" \u548C "),De=p("strong"),Er=n("\u7C7B\u578B\u6807\u8BB0ID(token_type_ids)"),wr=n(" \u952E\u7684\u65B0\u5B57\u5178\u3002\u8BF7\u6CE8\u610F\uFF0C\u5982\u679C\u50CF\u4E0A\u9762\u7684"),Ge=p("strong"),Tr=n("\u793A\u4F8B"),Sr=n("\u4E00\u6837\uFF0C\u5982\u679C\u952E\u6240\u5BF9\u5E94\u7684\u503C\u5305\u542B\u591A\u4E2A\u53E5\u5B50\uFF08\u6BCF\u4E2A\u952E\u4F5C\u4E3A\u4E00\u4E2A\u53E5\u5B50\u5217\u8868\uFF09\uFF0C\u90A3\u4E48\u5B83\u4F9D\u7136\u53EF\u4EE5\u5DE5\u4F5C\uFF0C\u5C31\u50CF\u524D\u9762\u7684\u4F8B\u5B50\u4E00\u6837\u6807\u8BB0\u5668\u53EF\u4EE5\u5904\u7406\u6210\u5BF9\u7684\u53E5\u5B50\u5217\u8868\u3002\u8FD9\u6837\u7684\u8BDD\u6211\u4EEC\u53EF\u4EE5\u5728\u8C03\u7528"),ze=p("strong"),yr=n("map()"),qr=n("\u4F7F\u7528\u8BE5\u9009\u9879 "),Oe=p("strong"),Pr=n("batched=True"),Nr=n(" \uFF0C\u8FD9\u5C06\u663E\u8457\u52A0\u5FEB\u6807\u8BB0\u4E0E\u6807\u8BB0\u7684\u901F\u5EA6\u3002\u8FD9\u4E2A"),Ce=p("strong"),Rr=n("\u6807\u8BB0\u5668"),Dr=n("\u6765\u81EA"),pt=p("a"),Gr=n("\u{1F917} Tokenizers"),zr=n("\u5E93\u7531Rust\u7F16\u5199\u800C\u6210\u3002\u5F53\u6211\u4EEC\u4E00\u6B21\u7ED9\u5B83\u5927\u91CF\u7684\u8F93\u5165\u65F6\uFF0C\u8FD9\u4E2A\u6807\u8BB0\u5668\u53EF\u4EE5\u975E\u5E38\u5FEB\u3002"),Hn=_(),zs=p("p"),Or=n("\u8BF7\u6CE8\u610F\uFF0C\u6211\u4EEC\u73B0\u5728\u5728\u6807\u8BB0\u51FD\u6570\u4E2D\u7701\u7565\u4E86"),Ae=p("strong"),Cr=n("padding"),Ar=n("\u53C2\u6570\u3002\u8FD9\u662F\u56E0\u4E3A\u5728\u6807\u8BB0\u7684\u65F6\u5019\u5C06\u6240\u6709\u6837\u672C\u586B\u5145\u5230\u6700\u5927\u957F\u5EA6\u7684\u6548\u7387\u4E0D\u9AD8\u3002\u4E00\u4E2A\u66F4\u597D\u7684\u505A\u6CD5\uFF1A\u5728\u6784\u5EFA\u6279\u5904\u7406\u65F6\u586B\u5145\u6837\u672C\u66F4\u597D\uFF0C\u56E0\u4E3A\u8FD9\u6837\u6211\u4EEC\u53EA\u9700\u8981\u586B\u5145\u5230\u8BE5\u6279\u5904\u7406\u4E2D\u7684\u6700\u5927\u957F\u5EA6\uFF0C\u800C\u4E0D\u662F\u6574\u4E2A\u6570\u636E\u96C6\u7684\u6700\u5927\u957F\u5EA6\u3002\u5F53\u8F93\u5165\u957F\u5EA6\u53D8\u5316\u5F88\u5927\u65F6\uFF0C\u8FD9\u53EF\u4EE5\u8282\u7701\u5927\u91CF\u65F6\u95F4\u548C\u5904\u7406\u80FD\u529B!"),Bn=_(),gs=p("p"),Ir=n("\u4E0B\u9762\u662F\u6211\u4EEC\u5982\u4F55\u5728\u6240\u6709\u6570\u636E\u96C6\u4E0A\u540C\u65F6\u5E94\u7528\u6807\u8BB0\u51FD\u6570\u3002\u6211\u4EEC\u5728\u8C03\u7528"),Ie=p("strong"),Lr=n("map"),Mr=n("\u65F6\u4F7F\u7528\u4E86"),Le=p("strong"),Fr=n("batch =True"),Ur=n("\uFF0C\u8FD9\u6837\u51FD\u6570\u5C31\u53EF\u4EE5\u540C\u65F6\u5E94\u7528\u5230\u6570\u636E\u96C6\u7684\u591A\u4E2A\u5143\u7D20\u4E0A\uFF0C\u800C\u4E0D\u662F\u5206\u522B\u5E94\u7528\u5230\u6BCF\u4E2A\u5143\u7D20\u4E0A\u3002\u8FD9\u5C06\u4F7F\u6211\u4EEC\u7684\u9884\u5904\u7406\u5FEB\u8BB8\u591A"),Wn=_(),E(it.$$.fragment),Vn=_(),Ct=p("p"),Hr=n("\u{1F917}Datasets\u5E93\u5E94\u7528\u8FD9\u79CD\u5904\u7406\u7684\u65B9\u5F0F\u662F\u5411\u6570\u636E\u96C6\u6DFB\u52A0\u65B0\u7684\u5B57\u6BB5\uFF0C\u6BCF\u4E2A\u5B57\u6BB5\u5BF9\u5E94\u9884\u5904\u7406\u51FD\u6570\u8FD4\u56DE\u7684\u5B57\u5178\u4E2D\u7684\u6BCF\u4E2A\u952E:"),Jn=_(),E(ft.$$.fragment),Kn=_(),K=p("p"),Br=n("\u5728\u4F7F\u7528\u9884\u5904\u7406\u51FD\u6570"),Me=p("strong"),Wr=n("map()"),Vr=n("\u65F6\uFF0C\u751A\u81F3\u53EF\u4EE5\u901A\u8FC7\u4F20\u9012"),Fe=p("strong"),Jr=n("num_proc"),Kr=n("\u53C2\u6570\u4F7F\u7528\u5E76\u884C\u5904\u7406\u3002\u6211\u4EEC\u5728\u8FD9\u91CC\u6CA1\u6709\u8FD9\u6837\u505A\uFF0C\u56E0\u4E3A\u{1F917}\u6807\u8BB0\u5668\u5E93\u5DF2\u7ECF\u4F7F\u7528\u591A\u4E2A\u7EBF\u7A0B\u6765\u66F4\u5FEB\u5730\u6807\u8BB0\u6211\u4EEC\u7684\u6837\u672C\uFF0C\u4F46\u662F\u5982\u679C\u60A8\u6CA1\u6709\u4F7F\u7528\u8BE5\u5E93\u652F\u6301\u7684\u5FEB\u901F\u6807\u8BB0\u5668\uFF0C\u4F7F\u7528"),Ue=p("strong"),Yr=n("num_proc"),Zr=n("\u53EF\u80FD\u4F1A\u52A0\u5FEB\u9884\u5904\u7406\u3002"),Yn=_(),M=p("p"),Qr=n("\u6211\u4EEC\u7684"),He=p("strong"),Xr=n("\u6807\u8BB0\u51FD\u6570(tokenize_function)"),so=n("\u8FD4\u56DE\u5305\u542B"),Be=p("strong"),to=n("\u8F93\u5165\u8BCDid(input_ids)"),eo=n(" \uFF0C "),We=p("strong"),no=n("\u6CE8\u610F\u529B\u906E\u7F69(attention_mask)"),ao=n(" \u548C "),Ve=p("strong"),lo=n("\u7C7B\u578B\u6807\u8BB0ID(token_type_ids)"),ro=n(" \u952E\u7684\u5B57\u5178,\u6240\u4EE5\u8FD9\u4E09\u4E2A\u5B57\u6BB5\u88AB\u6DFB\u52A0\u5230\u6570\u636E\u96C6\u7684\u6807\u8BB0\u7684\u7ED3\u679C\u4E2D\u3002\u6CE8\u610F\uFF0C\u5982\u679C\u9884\u5904\u7406\u51FD\u6570"),Je=p("strong"),oo=n("map()"),po=n("\u4E3A\u73B0\u6709\u952E\u8FD4\u56DE\u4E00\u4E2A\u65B0\u503C\uFF0C\u90A3\u5C06\u4F1A\u4FEE\u6539\u539F\u6709\u952E\u7684\u503C\u3002"),Zn=_(),At=p("p"),io=n("\u6700\u540E\u4E00\u4EF6\u6211\u4EEC\u9700\u8981\u505A\u7684\u4E8B\u60C5\u662F\uFF0C\u5F53\u6211\u4EEC\u4E00\u8D77\u6279\u5904\u7406\u5143\u7D20\u65F6\uFF0C\u5C06\u6240\u6709\u793A\u4F8B\u586B\u5145\u5230\u6700\u957F\u5143\u7D20\u7684\u957F\u5EA6\u2014\u2014\u6211\u4EEC\u79F0\u4E4B\u4E3A\u52A8\u6001\u586B\u5145\u3002"),Qn=_(),ct=p("h3"),fo=n("\u52A8\u6001\u586B\u5145"),Xn=_(),E(ut.$$.fragment),sa=_(),xs.c(),It=_(),Os=p("p"),co=n("\u4E3A\u4E86\u89E3\u51B3\u53E5\u5B50\u957F\u5EA6\u7EDF\u4E00\u7684\u95EE\u9898\uFF0C\u6211\u4EEC\u5FC5\u987B\u5B9A\u4E49\u4E00\u4E2Acollate\u51FD\u6570\uFF0C\u8BE5\u51FD\u6570\u4F1A\u5C06\u6BCF\u4E2Abatch\u53E5\u5B50\u586B\u5145\u5230\u6B63\u786E\u7684\u957F\u5EA6\u3002\u5E78\u8FD0\u7684\u662F\uFF0C\u{1F917}transformer\u5E93\u901A\u8FC7"),Ke=p("strong"),uo=n("DataCollatorWithPadding"),_o=n("\u4E3A\u6211\u4EEC\u63D0\u4F9B\u4E86\u8FD9\u6837\u4E00\u4E2A\u51FD\u6570\u3002\u5F53\u4F60\u5B9E\u4F8B\u5316\u5B83\u65F6\uFF0C\u9700\u8981\u4E00\u4E2A\u6807\u8BB0\u5668(\u7528\u6765\u77E5\u9053\u4F7F\u7528\u54EA\u4E2A\u8BCD\u6765\u586B\u5145\uFF0C\u4EE5\u53CA\u6A21\u578B\u671F\u671B\u586B\u5145\u5728\u5DE6\u8FB9\u8FD8\u662F\u53F3\u8FB9)\uFF0C\u5E76\u5C06\u505A\u4F60\u9700\u8981\u7684\u4E00\u5207:"),ta=_(),ps.c(),Lt=_(),Y=p("p"),mo=n("\u4E3A\u4E86\u6D4B\u8BD5\u8FD9\u4E2A\u65B0\u73A9\u5177\uFF0C\u8BA9\u6211\u4EEC\u4ECE\u6211\u4EEC\u7684\u8BAD\u7EC3\u96C6\u4E2D\u62BD\u53D6\u51E0\u4E2A\u6837\u672C\u3002\u8FD9\u91CC\uFF0C\u6211\u4EEC\u5220\u9664\u5217"),Ye=p("strong"),ho=n("idx"),bo=n(", "),Ze=p("strong"),$o=n("sentence1"),go=n("\u548C"),Qe=p("strong"),jo=n("sentence2"),ko=n("\uFF0C\u56E0\u4E3A\u4E0D\u9700\u8981\u5B83\u4EEC\uFF0C\u5E76\u67E5\u770B\u4E00\u4E2Abatch\u4E2D\u6BCF\u4E2A\u6761\u76EE\u7684\u957F\u5EA6:"),ea=_(),E(_t.$$.fragment),na=_(),E(mt.$$.fragment),aa=_(),Cs=p("p"),xo=n("\u6BEB\u65E0\u7591\u95EE\uFF0C\u6211\u4EEC\u5F97\u5230\u4E86\u4E0D\u540C\u957F\u5EA6\u7684\u6837\u672C\uFF0C\u4ECE32\u523067\u3002\u52A8\u6001\u586B\u5145\u610F\u5473\u7740\u8BE5\u6279\u4E2D\u7684\u6240\u6709\u6837\u672C\u90FD\u5E94\u8BE5\u586B\u5145\u5230\u957F\u5EA6\u4E3A67\uFF0C\u8FD9\u662F\u8BE5\u6279\u4E2D\u7684\u6700\u5927\u957F\u5EA6\u3002\u5982\u679C\u6CA1\u6709\u52A8\u6001\u586B\u5145\uFF0C\u6240\u6709\u7684\u6837\u672C\u90FD\u5FC5\u987B\u586B\u5145\u5230\u6574\u4E2A\u6570\u636E\u96C6\u4E2D\u7684\u6700\u5927\u957F\u5EA6\uFF0C\u6216\u8005\u6A21\u578B\u53EF\u4EE5\u63A5\u53D7\u7684\u6700\u5927\u957F\u5EA6\u3002\u8BA9\u6211\u4EEC\u518D\u6B21\u68C0\u67E5"),Xe=p("strong"),vo=n("data_collator"),Eo=n("\u662F\u5426\u6B63\u786E\u5730\u52A8\u6001\u586B\u5145\u4E86\u8FD9\u6279\u6837\u672C\uFF1A"),la=_(),E(ht.$$.fragment),ra=_(),fs.c(),Mt=_(),E(As.$$.fragment),oa=_(),O&&O.c(),Ft=ai(),this.h()},l(s){const o=fi('[data-svelte="svelte-1phssyn"]',document.head);l=i(o,"META",{name:!0,content:!0}),o.forEach(e),u=m(s),w(r.$$.fragment,s),b=m(s),k=i(s,"H1",{id:!0});var sn=f(k);h=a(sn,"\u5904\u7406\u6570\u636E"),sn.forEach(e),d=m(s),$.l(s),y=m(s),z.l(s),cs=m(s),us=i(s,"P",{});var Ut=f(us);vs=a(Ut,"\u5F53\u7136\uFF0C\u4EC5\u4EC5\u7528\u4E24\u53E5\u8BDD\u8BAD\u7EC3\u6A21\u578B\u4E0D\u4F1A\u4EA7\u751F\u5F88\u597D\u7684\u6548\u679C\u3002\u4E3A\u4E86\u83B7\u5F97\u66F4\u597D\u7684\u7ED3\u679C\uFF0C\u60A8\u9700\u8981\u51C6\u5907\u4E00\u4E2A\u66F4\u5927\u7684\u6570\u636E\u96C6\u3002"),Ut.forEach(e),W=m(s),B=i(s,"P",{});var Is=f(B);_s=a(Is,"\u5728\u672C\u8282\u4E2D\uFF0C\u6211\u4EEC\u5C06\u4F7F\u7528MRPC\uFF08\u5FAE\u8F6F\u7814\u7A76\u91CA\u4E49\u8BED\u6599\u5E93\uFF09\u6570\u636E\u96C6\u4F5C\u4E3A\u793A\u4F8B\uFF0C\u8BE5\u6570\u636E\u96C6\u7531\u5A01\u5EC9\xB7\u591A\u5170\u548C\u514B\u91CC\u65AF\xB7\u5E03\u7F57\u514B\u7279\u5728"),ts=i(Is,"A",{href:!0,rel:!0});var Ht=f(ts);Es=a(Ht,"\u8FD9\u7BC7\u6587\u7AE0"),Ht.forEach(e),N=a(Is,"\u53D1\u5E03\u3002\u8BE5\u6570\u636E\u96C6\u75315801\u5BF9\u53E5\u5B50\u7EC4\u6210\uFF0C\u6BCF\u4E2A\u53E5\u5B50\u5BF9\u5E26\u6709\u4E00\u4E2A\u6807\u7B7E\uFF0C\u6307\u793A\u5B83\u4EEC\u662F\u5426\u4E3A\u540C\u4E49\uFF08\u5373\uFF0C\u5982\u679C\u4E24\u4E2A\u53E5\u5B50\u7684\u610F\u601D\u76F8\u540C\uFF09\u3002\u6211\u4EEC\u5728\u672C\u7AE0\u4E2D\u9009\u62E9\u4E86\u5B83\uFF0C\u56E0\u4E3A\u5B83\u662F\u4E00\u4E2A\u5C0F\u6570\u636E\u96C6\uFF0C\u6240\u4EE5\u5F88\u5BB9\u6613\u5BF9\u5B83\u8FDB\u884C\u8BAD\u7EC3\u3002"),Is.forEach(e),G=m(s),V=i(s,"H3",{class:!0});var xt=f(V);es=i(xt,"A",{id:!0,class:!0,href:!0});var Bt=f(es);ws=i(Bt,"SPAN",{});var tn=f(ws);w(ms.$$.fragment,tn),tn.forEach(e),Bt.forEach(e),wt=m(xt),Qt=i(xt,"SPAN",{});var Wt=f(Qt);ja=a(Wt,"\u4ECE\u6A21\u578B\u4E2D\u5FC3\uFF08Hub\uFF09\u52A0\u8F7D\u6570\u636E\u96C6"),Wt.forEach(e),xt.forEach(e),an=m(s),as.l(s),Tt=m(s),J=i(s,"P",{});var Z=f(J);ka=a(Z,"\u6A21\u578B\u4E2D\u5FC3\uFF08hub\uFF09\u4E0D\u53EA\u662F\u5305\u542B\u6A21\u578B\uFF1B\u5B83\u4E5F\u6709\u8BB8\u591A\u4E0D\u540C\u8BED\u8A00\u7684\u591A\u4E2A\u6570\u636E\u96C6\u3002\u70B9\u51FB"),Fs=i(Z,"A",{href:!0,rel:!0});var en=f(Fs);xa=a(en,"\u6570\u636E\u96C6"),en.forEach(e),va=a(Z,"\u7684\u94FE\u63A5\u5373\u53EF\u8FDB\u884C\u6D4F\u89C8\u3002\u6211\u4EEC\u5EFA\u8BAE\u60A8\u5728\u9605\u8BFB\u672C\u8282\u540E\u9605\u8BFB\u4E00\u4E0B"),Us=i(Z,"A",{href:!0,rel:!0});var Io=f(Us);Ea=a(Io,"\u52A0\u8F7D\u548C\u5904\u7406\u65B0\u7684\u6570\u636E\u96C6"),Io.forEach(e),wa=a(Z,"\u8FD9\u7BC7\u6587\u7AE0\uFF0C\u8FD9\u4F1A\u8BA9\u60A8\u5BF9huggingface\u7684darasets\u66F4\u52A0\u6E05\u6670\u3002\u4F46\u73B0\u5728\uFF0C\u8BA9\u6211\u4EEC\u4F7F\u7528MRPC\u6570\u636E\u96C6\u4E2D\u7684"),Hs=i(Z,"A",{href:!0,rel:!0});var Lo=f(Hs);Ta=a(Lo,"GLUE \u57FA\u51C6\u6D4B\u8BD5\u6570\u636E\u96C6"),Lo.forEach(e),Sa=a(Z,"\uFF0C\u5B83\u662F\u6784\u6210MRPC\u6570\u636E\u96C6\u768410\u4E2A\u6570\u636E\u96C6\u4E4B\u4E00\uFF0C\u8FD9\u662F\u4E00\u4E2A\u5B66\u672F\u57FA\u51C6\uFF0C\u7528\u4E8E\u8861\u91CF\u673A\u5668\u5B66\u4E60\u6A21\u578B\u572810\u4E2A\u4E0D\u540C\u6587\u672C\u5206\u7C7B\u4EFB\u52A1\u4E2D\u7684\u6027\u80FD\u3002"),Z.forEach(e),ln=m(s),St=i(s,"P",{});var Mo=f(St);ya=a(Mo,"\u{1F917} Datasets\u5E93\u63D0\u4F9B\u4E86\u4E00\u4E2A\u975E\u5E38\u4FBF\u6377\u7684\u547D\u4EE4\uFF0C\u53EF\u4EE5\u5728\u6A21\u578B\u4E2D\u5FC3\uFF08hub\uFF09\u4E0A\u4E0B\u8F7D\u548C\u7F13\u5B58\u6570\u636E\u96C6\u3002\u6211\u4EEC\u53EF\u4EE5\u901A\u8FC7\u4EE5\u4E0B\u7684\u4EE3\u7801\u4E0B\u8F7DMRPC\u6570\u636E\u96C6\uFF1A"),Mo.forEach(e),rn=m(s),w(Bs.$$.fragment,s),on=m(s),w(Ws.$$.fragment,s),pn=m(s),I=i(s,"P",{});var Q=f(I);qa=a(Q,"\u6B63\u5982\u4F60\u6240\u770B\u5230\u7684\uFF0C\u6211\u4EEC\u83B7\u5F97\u4E86\u4E00\u4E2A"),Xt=i(Q,"STRONG",{});var Fo=f(Xt);Pa=a(Fo,"DatasetDict"),Fo.forEach(e),Na=a(Q,"\u5BF9\u8C61\uFF0C\u5176\u4E2D\u5305\u542B\u8BAD\u7EC3\u96C6\u3001\u9A8C\u8BC1\u96C6\u548C\u6D4B\u8BD5\u96C6\u3002\u6BCF\u4E00\u4E2A\u96C6\u5408\u90FD\u5305\u542B\u51E0\u4E2A\u5217("),se=i(Q,"STRONG",{});var Uo=f(se);Ra=a(Uo,"sentence1"),Uo.forEach(e),Da=a(Q,", "),te=i(Q,"STRONG",{});var Ho=f(te);Ga=a(Ho,"sentence2"),Ho.forEach(e),za=a(Q,", "),ee=i(Q,"STRONG",{});var Bo=f(ee);Oa=a(Bo,"label"),Bo.forEach(e),Ca=a(Q,", and "),ne=i(Q,"STRONG",{});var Wo=f(ne);Aa=a(Wo,"idx"),Wo.forEach(e),Ia=a(Q,")\u4EE5\u53CA\u4E00\u4E2A\u4EE3\u8868\u884C\u6570\u7684\u53D8\u91CF\uFF0C\u5373\u6BCF\u4E2A\u96C6\u5408\u4E2D\u7684\u884C\u7684\u4E2A\u6570\uFF08\u56E0\u6B64\uFF0C\u8BAD\u7EC3\u96C6\u4E2D\u67093668\u5BF9\u53E5\u5B50\uFF0C\u9A8C\u8BC1\u96C6\u4E2D\u6709408\u5BF9\uFF0C\u6D4B\u8BD5\u96C6\u4E2D\u67091725\u5BF9\uFF09\u3002"),Q.forEach(e),fn=m(s),hs=i(s,"P",{});var Vt=f(hs);La=a(Vt,"\u9ED8\u8BA4\u60C5\u51B5\u4E0B\uFF0C\u6B64\u547D\u4EE4\u5728\u4E0B\u8F7D\u6570\u636E\u96C6\u5E76\u7F13\u5B58\u5230 "),ae=i(Vt,"STRONG",{});var Vo=f(ae);Ma=a(Vo,"~/.cache/huggingface/dataset"),Vo.forEach(e),Fa=a(Vt,". \u56DE\u60F3\u4E00\u4E0B\u7B2C2\u7AE0\uFF0C\u60A8\u53EF\u4EE5\u901A\u8FC7\u8BBE\u7F6E"),le=i(Vt,"STRONG",{});var Jo=f(le);Ua=a(Jo,"HF_HOME"),Jo.forEach(e),Ha=a(Vt,"\u73AF\u5883\u53D8\u91CF\u6765\u81EA\u5B9A\u4E49\u7F13\u5B58\u7684\u6587\u4EF6\u5939\u3002"),Vt.forEach(e),cn=m(s),Ts=i(s,"P",{});var fa=f(Ts);Ba=a(fa,"\u6211\u4EEC\u53EF\u4EE5\u8BBF\u95EE\u6211\u4EEC\u6570\u636E\u96C6\u4E2D\u7684\u6BCF\u4E00\u4E2A"),re=i(fa,"STRONG",{});var Ko=f(re);Wa=a(Ko,"raw_train_dataset"),Ko.forEach(e),Va=a(fa,"\u5BF9\u8C61\uFF0C\u5982\u4F7F\u7528\u5B57\u5178\uFF1A"),fa.forEach(e),un=m(s),w(Vs.$$.fragment,s),_n=m(s),w(Js.$$.fragment,s),mn=m(s),ds=i(s,"P",{});var Jt=f(ds);Ja=a(Jt,"\u6211\u4EEC\u53EF\u4EE5\u770B\u5230\u6807\u7B7E\u5DF2\u7ECF\u662F\u6574\u6570\u4E86\uFF0C\u6240\u4EE5\u6211\u4EEC\u4E0D\u9700\u8981\u5BF9\u6807\u7B7E\u505A\u4EFB\u4F55\u9884\u5904\u7406\u3002\u8981\u77E5\u9053\u54EA\u4E2A\u6570\u5B57\u5BF9\u5E94\u4E8E\u54EA\u4E2A\u6807\u7B7E\uFF0C\u6211\u4EEC\u53EF\u4EE5\u67E5\u770B"),oe=i(Jt,"STRONG",{});var Yo=f(oe);Ka=a(Yo,"raw_train_dataset"),Yo.forEach(e),Ya=a(Jt,"\u7684"),pe=i(Jt,"STRONG",{});var Zo=f(pe);Za=a(Zo,"features"),Zo.forEach(e),Qa=a(Jt,". \u8FD9\u5C06\u544A\u8BC9\u6211\u4EEC\u6BCF\u5217\u7684\u7C7B\u578B\uFF1A"),Jt.forEach(e),hn=m(s),w(Ks.$$.fragment,s),dn=m(s),w(Ys.$$.fragment,s),bn=m(s),C=i(s,"P",{});var H=f(C);Xa=a(H,"\u5728\u4E0A\u9762\u7684\u4F8B\u5B50\u4E4B\u4E2D,"),ie=i(H,"STRONG",{});var Qo=f(ie);sl=a(Qo,"Label\uFF08\u6807\u7B7E\uFF09"),Qo.forEach(e),tl=a(H," \u662F\u4E00\u79CD"),fe=i(H,"STRONG",{});var Xo=f(fe);el=a(Xo,"ClassLabel\uFF08\u5206\u7C7B\u6807\u7B7E\uFF09"),Xo.forEach(e),nl=a(H,"\uFF0C\u4F7F\u7528\u6574\u6570\u5EFA\u7ACB\u8D77\u5230\u7C7B\u522B\u6807\u7B7E\u7684\u6620\u5C04\u5173\u7CFB\u3002"),ce=i(H,"STRONG",{});var sp=f(ce);al=a(sp,"0"),sp.forEach(e),ll=a(H,"\u5BF9\u5E94\u4E8E"),ue=i(H,"STRONG",{});var tp=f(ue);rl=a(tp,"not_equivalent"),tp.forEach(e),ol=a(H,"\uFF0C"),_e=i(H,"STRONG",{});var ep=f(_e);pl=a(ep,"1"),ep.forEach(e),il=a(H,"\u5BF9\u5E94\u4E8E"),me=i(H,"STRONG",{});var np=f(me);fl=a(np,"equivalent"),np.forEach(e),cl=a(H,"\u3002"),H.forEach(e),$n=m(s),w(Ss.$$.fragment,s),gn=m(s),Zs=i(s,"H3",{id:!0});var ap=f(Zs);ul=a(ap,"\u9884\u5904\u7406\u6570\u636E\u96C6"),ap.forEach(e),jn=m(s),rs.l(s),yt=m(s),ys=i(s,"P",{});var ca=f(ys);_l=a(ca,"\u4E3A\u4E86\u9884\u5904\u7406\u6570\u636E\u96C6\uFF0C\u6211\u4EEC\u9700\u8981\u5C06\u6587\u672C\u8F6C\u6362\u4E3A\u6A21\u578B\u80FD\u591F\u7406\u89E3\u7684\u6570\u5B57\u3002\u6B63\u5982\u4F60\u5728"),qt=i(ca,"A",{href:!0});var lp=f(qt);ml=a(lp,"\u7B2C\u4E8C\u7AE0"),lp.forEach(e),hl=a(ca,"\u4E0A\u770B\u5230\u7684\u90A3\u6837"),ca.forEach(e),kn=m(s),w(Qs.$$.fragment,s),xn=m(s),Pt=i(s,"P",{});var rp=f(Pt);dl=a(rp,"\u7136\u800C\uFF0C\u5728\u4E24\u53E5\u8BDD\u4F20\u9012\u7ED9\u6A21\u578B\uFF0C\u9884\u6D4B\u8FD9\u4E24\u53E5\u8BDD\u662F\u5426\u662F\u540C\u4E49\u4E4B\u524D\u3002\u6211\u4EEC\u9700\u8981\u8FD9\u4E24\u53E5\u8BDD\u4F9D\u6B21\u8FDB\u884C\u9002\u5F53\u7684\u9884\u5904\u7406\u3002\u5E78\u8FD0\u7684\u662F\uFF0C\u6807\u8BB0\u5668\u4E0D\u4EC5\u4EC5\u53EF\u4EE5\u8F93\u5165\u5355\u4E2A\u53E5\u5B50\u8FD8\u53EF\u4EE5\u8F93\u5165\u4E00\u7EC4\u53E5\u5B50\uFF0C\u5E76\u6309\u7167\u6211\u4EEC\u7684BERT\u6A21\u578B\u6240\u671F\u671B\u7684\u8F93\u5165\u8FDB\u884C\u5904\u7406\uFF1A"),rp.forEach(e),vn=m(s),w(Xs.$$.fragment,s),En=m(s),w(st.$$.fragment,s),wn=m(s),L=i(s,"P",{});var X=f(L);bl=a(X,"\u6211\u4EEC\u5728"),Nt=i(X,"A",{href:!0});var op=f(Nt);$l=a(op,"\u7B2C\u4E8C\u7AE0"),op.forEach(e),gl=a(X," \u8BA8\u8BBA\u4E86"),he=i(X,"STRONG",{});var pp=f(he);jl=a(pp,"\u8F93\u5165\u8BCDid(input_ids)"),pp.forEach(e),kl=a(X," \u548C "),de=i(X,"STRONG",{});var ip=f(de);xl=a(ip,"\u6CE8\u610F\u529B\u906E\u7F69(attention_mask)"),ip.forEach(e),vl=a(X," \uFF0C\u4F46\u6211\u4EEC\u5728\u90A3\u4E2A\u65F6\u5019\u6CA1\u6709\u8BA8\u8BBA"),be=i(X,"STRONG",{});var fp=f(be);El=a(fp,"\u7C7B\u578B\u6807\u8BB0ID(token_type_ids)"),fp.forEach(e),wl=a(X,"\u3002\u5728\u8FD9\u4E2A\u4F8B\u5B50\u4E2D\uFF0C"),$e=i(X,"STRONG",{});var cp=f($e);Tl=a(cp,"\u7C7B\u578B\u6807\u8BB0ID(token_type_ids)"),cp.forEach(e),Sl=a(X,"\u7684\u4F5C\u7528\u5C31\u662F\u544A\u8BC9\u6A21\u578B\u8F93\u5165\u7684\u54EA\u4E00\u90E8\u5206\u662F\u7B2C\u4E00\u53E5\uFF0C\u54EA\u4E00\u90E8\u5206\u662F\u7B2C\u4E8C\u53E5\u3002"),X.forEach(e),Tn=m(s),w(qs.$$.fragment,s),Sn=m(s),Ps=i(s,"P",{});var ua=f(Ps);yl=a(ua,"\u5982\u679C\u6211\u4EEC\u5C06"),ge=i(ua,"STRONG",{});var up=f(ge);ql=a(up,"input_ids"),up.forEach(e),Pl=a(ua,"\u4E2D\u7684id\u8F6C\u6362\u56DE\u6587\u5B57:"),ua.forEach(e),yn=m(s),w(tt.$$.fragment,s),qn=m(s),Rt=i(s,"P",{});var _p=f(Rt);Nl=a(_p,"\u6211\u4EEC\u5C06\u5F97\u5230\uFF1A"),_p.forEach(e),Pn=m(s),w(et.$$.fragment,s),Nn=m(s),bs=i(s,"P",{});var Kt=f(bs);Rl=a(Kt,"\u6240\u4EE5\u6211\u4EEC\u770B\u5230\u6A21\u578B\u9700\u8981\u8F93\u5165\u7684\u5F62\u5F0F\u662F "),je=i(Kt,"STRONG",{});var mp=f(je);Dl=a(mp,"[CLS] sentence1 [SEP] sentence2 [SEP]"),mp.forEach(e),Gl=a(Kt,"\u3002\u56E0\u6B64\uFF0C\u5F53\u6709\u4E24\u53E5\u8BDD\u7684\u65F6\u5019\u3002"),ke=i(Kt,"STRONG",{});var hp=f(ke);zl=a(hp,"\u7C7B\u578B\u6807\u8BB0ID(token_type_ids)"),hp.forEach(e),Ol=a(Kt," \u7684\u503C\u662F\uFF1A"),Kt.forEach(e),Rn=m(s),w(nt.$$.fragment,s),Dn=m(s),F=i(s,"P",{});var js=f(F);Cl=a(js,"\u5982\u60A8\u6240\u89C1\uFF0C\u8F93\u5165\u4E2D "),xe=i(js,"STRONG",{});var dp=f(xe);Al=a(dp,"[CLS] sentence1 [SEP]"),dp.forEach(e),Il=a(js," \u5B83\u4EEC\u7684\u7C7B\u578B\u6807\u8BB0ID\u5747\u4E3A"),ve=i(js,"STRONG",{});var bp=f(ve);Ll=a(bp,"0"),bp.forEach(e),Ml=a(js,"\uFF0C\u800C\u5176\u4ED6\u90E8\u5206\uFF0C\u5BF9\u5E94\u4E8E"),Ee=i(js,"STRONG",{});var $p=f(Ee);Fl=a($p,"sentence2 [SEP]"),$p.forEach(e),Ul=a(js,"\uFF0C\u6240\u6709\u7684\u7C7B\u578B\u6807\u8BB0ID\u5747\u4E3A"),we=i(js,"STRONG",{});var gp=f(we);Hl=a(gp,"1"),gp.forEach(e),Bl=a(js,"."),js.forEach(e),Gn=m(s),Ns=i(s,"P",{});var _a=f(Ns);Wl=a(_a,"\u8BF7\u6CE8\u610F\uFF0C\u5982\u679C\u9009\u62E9\u5176\u4ED6\u7684\u68C0\u67E5\u70B9\uFF0C\u5219\u4E0D\u4E00\u5B9A\u5177\u6709"),Te=i(_a,"STRONG",{});var jp=f(Te);Vl=a(jp,"\u7C7B\u578B\u6807\u8BB0ID(token_type_ids)"),jp.forEach(e),Jl=a(_a,"\uFF08\u4F8B\u5982\uFF0C\u5982\u679C\u4F7F\u7528DistilBERT\u6A21\u578B\uFF0C\u5C31\u4E0D\u4F1A\u8FD4\u56DE\u5B83\u4EEC\uFF09\u3002\u53EA\u6709\u5F53\u5B83\u5728\u9884\u8BAD\u7EC3\u671F\u95F4\u4F7F\u7528\u8FC7\u8FD9\u4E00\u5C42\uFF0C\u6A21\u578B\u5728\u6784\u5EFA\u65F6\u4F9D\u8D56\u5B83\u4EEC\uFF0C\u624D\u4F1A\u8FD4\u56DE\u5B83\u4EEC\u3002"),_a.forEach(e),zn=m(s),Rs=i(s,"P",{});var ma=f(Rs);Kl=a(ma,"\u7528\u7C7B\u578B\u6807\u8BB0ID\u5BF9BERT\u8FDB\u884C\u9884\u8BAD\u7EC3,\u5E76\u4E14\u4F7F\u7528"),Dt=i(ma,"A",{href:!0});var kp=f(Dt);Yl=a(kp,"\u7B2C\u4E00\u7AE0"),kp.forEach(e),Zl=a(ma,"\u7684\u906E\u7F69\u8BED\u8A00\u6A21\u578B\uFF0C\u8FD8\u6709\u4E00\u4E2A\u989D\u5916\u7684\u5E94\u7528\u7C7B\u578B\uFF0C\u53EB\u505A\u4E0B\u4E00\u53E5\u9884\u6D4B. \u8FD9\u9879\u4EFB\u52A1\u7684\u76EE\u6807\u662F\u5EFA\u7ACB\u6210\u5BF9\u53E5\u5B50\u4E4B\u95F4\u5173\u7CFB\u7684\u6A21\u578B\u3002"),ma.forEach(e),On=m(s),Gt=i(s,"P",{});var xp=f(Gt);Ql=a(xp,"\u5728\u4E0B\u4E00\u4E2A\u53E5\u5B50\u9884\u6D4B\u4EFB\u52A1\u4E2D\uFF0C\u4F1A\u7ED9\u6A21\u578B\u8F93\u5165\u6210\u5BF9\u7684\u53E5\u5B50\uFF08\u5E26\u6709\u968F\u673A\u906E\u7F69\u7684\u6807\u8BB0\uFF09\uFF0C\u5E76\u88AB\u8981\u6C42\u9884\u6D4B\u7B2C\u4E8C\u4E2A\u53E5\u5B50\u662F\u5426\u7D27\u8DDF\u7B2C\u4E00\u4E2A\u53E5\u5B50\u3002\u4E3A\u4E86\u63D0\u9AD8\u6A21\u578B\u7684\u6CDB\u5316\u80FD\u529B\uFF0C\u6570\u636E\u96C6\u4E2D\u4E00\u534A\u7684\u4E24\u4E2A\u53E5\u5B50\u5728\u539F\u59CB\u6587\u6863\u4E2D\u6328\u5728\u4E00\u8D77\uFF0C\u53E6\u4E00\u534A\u7684\u4E24\u4E2A\u53E5\u5B50\u6765\u81EA\u4E24\u4E2A\u4E0D\u540C\u7684\u6587\u6863\u3002"),xp.forEach(e),Cn=m(s),Ds=i(s,"P",{});var ha=f(Ds);Xl=a(ha,"\u4E00\u822C\u6765\u8BF4\uFF0C\u4F60\u4E0D\u9700\u8981\u62C5\u5FC3\u662F\u5426\u6709"),Se=i(ha,"STRONG",{});var vp=f(Se);sr=a(vp,"\u7C7B\u578B\u6807\u8BB0ID(token_type_ids)"),vp.forEach(e),tr=a(ha,"\u3002\u5728\u60A8\u7684\u6807\u8F93\u5165\u4E2D\uFF1A\u53EA\u8981\u60A8\u5BF9\u6807\u8BB0\u5668\u548C\u6A21\u578B\u4F7F\u7528\u76F8\u540C\u7684\u68C0\u67E5\u70B9\uFF0C\u4E00\u5207\u90FD\u4F1A\u5F88\u597D\uFF0C\u56E0\u4E3A\u6807\u8BB0\u5668\u77E5\u9053\u5411\u5176\u6A21\u578B\u63D0\u4F9B\u4EC0\u4E48\u3002"),ha.forEach(e),An=m(s),$s=i(s,"P",{});var Yt=f($s);er=a(Yt,"\u73B0\u5728\u6211\u4EEC\u5DF2\u7ECF\u4E86\u89E3\u4E86\u6807\u8BB0\u5668\u5982\u4F55\u5904\u7406\u4E00\u5BF9\u53E5\u5B50\uFF0C\u6211\u4EEC\u53EF\u4EE5\u4F7F\u7528\u5B83\u5BF9\u6574\u4E2A\u6570\u636E\u96C6\u8FDB\u884C\u5904\u7406\uFF1A\u5982"),zt=i(Yt,"A",{href:!0});var Ep=f(zt);nr=a(Ep,"\u4E4B\u524D\u7684\u7AE0\u8282"),Ep.forEach(e),ar=a(Yt,"\uFF0C\u6211\u4EEC\u53EF\u4EE5\u7ED9\u6807\u8BB0\u5668\u63D0\u4F9B\u4E00\u7EC4\u53E5\u5B50\uFF0C\u7B2C\u4E00\u4E2A\u53C2\u6570\u662F\u5B83\u7B2C\u4E00\u4E2A\u53E5\u5B50\u7684\u5217\u8868\uFF0C\u7B2C\u4E8C\u4E2A\u53C2\u6570\u662F\u7B2C\u4E8C\u4E2A\u53E5\u5B50\u7684\u5217\u8868\u3002\u8FD9\u4E5F\u4E0E\u6211\u4EEC\u5728"),Ot=i(Yt,"A",{href:!0});var wp=f(Ot);lr=a(wp,"\u7B2C\u4E8C\u7AE0"),wp.forEach(e),rr=a(Yt,"\u4E2D\u770B\u5230\u7684\u586B\u5145\u548C\u622A\u65AD\u9009\u9879\u517C\u5BB9. \u56E0\u6B64\uFF0C\u9884\u5904\u7406\u8BAD\u7EC3\u6570\u636E\u96C6\u7684\u4E00\u79CD\u65B9\u6CD5\u662F\uFF1A"),Yt.forEach(e),In=m(s),w(at.$$.fragment,s),Ln=m(s),U=i(s,"P",{});var ks=f(U);or=a(ks,"\u8FD9\u5F88\u6709\u6548\uFF0C\u4F46\u5B83\u7684\u7F3A\u70B9\u662F\u8FD4\u56DE\u5B57\u5178\uFF08\u5B57\u5178\u7684\u952E\u662F"),ye=i(ks,"STRONG",{});var Tp=f(ye);pr=a(Tp,"\u8F93\u5165\u8BCDid(input_ids)"),Tp.forEach(e),ir=a(ks," \uFF0C "),qe=i(ks,"STRONG",{});var Sp=f(qe);fr=a(Sp,"\u6CE8\u610F\u529B\u906E\u7F69(attention_mask)"),Sp.forEach(e),cr=a(ks," \u548C "),Pe=i(ks,"STRONG",{});var yp=f(Pe);ur=a(yp,"\u7C7B\u578B\u6807\u8BB0ID(token_type_ids)"),yp.forEach(e),_r=a(ks,"\uFF0C\u5B57\u5178\u7684\u503C\u662F\u952E\u6240\u5BF9\u5E94\u503C\u7684\u5217\u8868\uFF09\u3002\u800C\u4E14\u53EA\u6709\u5F53\u60A8\u5728\u8F6C\u6362\u8FC7\u7A0B\u4E2D\u6709\u8DB3\u591F\u7684\u5185\u5B58\u6765\u5B58\u50A8\u6574\u4E2A\u6570\u636E\u96C6\u65F6\u624D\u4E0D\u4F1A\u51FA\u9519\uFF08\u800C\u{1F917}\u6570\u636E\u96C6\u5E93\u4E2D\u7684\u6570\u636E\u96C6\u662F\u4EE5"),lt=i(ks,"A",{href:!0,rel:!0});var qp=f(lt);mr=a(qp,"Apache Arrow"),qp.forEach(e),hr=a(ks,"\u6587\u4EF6\u5B58\u50A8\u5728\u78C1\u76D8\u4E0A\uFF0C\u56E0\u6B64\u60A8\u53EA\u9700\u5C06\u63A5\u4E0B\u6765\u8981\u7528\u7684\u6570\u636E\u52A0\u8F7D\u5728\u5185\u5B58\u4E2D\uFF0C\u56E0\u6B64\u4F1A\u5BF9\u5185\u5B58\u5BB9\u91CF\u7684\u9700\u6C42\u8981\u4F4E\u4E00\u4E9B\uFF09\u3002"),ks.forEach(e),Mn=m(s),Gs=i(s,"P",{});var da=f(Gs);dr=a(da,"\u4E3A\u4E86\u5C06\u6570\u636E\u4FDD\u5B58\u4E3A\u6570\u636E\u96C6\uFF0C\u6211\u4EEC\u5C06\u4F7F\u7528"),rt=i(da,"A",{href:!0,rel:!0});var Pp=f(rt);br=a(Pp,"Dataset.map()"),Pp.forEach(e),$r=a(da,"\u65B9\u6CD5\uFF0C\u5982\u679C\u6211\u4EEC\u9700\u8981\u505A\u66F4\u591A\u7684\u9884\u5904\u7406\u800C\u4E0D\u4EC5\u4EC5\u662F\u6807\u8BB0\u5316\uFF0C\u90A3\u4E48\u8FD9\u4E5F\u7ED9\u4E86\u6211\u4EEC\u4E00\u4E9B\u989D\u5916\u7684\u81EA\u5B9A\u4E49\u7684\u65B9\u6CD5\u3002\u8FD9\u4E2A\u65B9\u6CD5\u7684\u5DE5\u4F5C\u539F\u7406\u662F\u5728\u6570\u636E\u96C6\u7684\u6BCF\u4E2A\u5143\u7D20\u4E0A\u5E94\u7528\u4E00\u4E2A\u51FD\u6570\uFF0C\u56E0\u6B64\u8BA9\u6211\u4EEC\u5B9A\u4E49\u4E00\u4E2A\u6807\u8BB0\u8F93\u5165\u7684\u51FD\u6570\uFF1A"),da.forEach(e),Fn=m(s),w(ot.$$.fragment,s),Un=m(s),D=i(s,"P",{});var A=f(D);gr=a(A,"\u6B64\u51FD\u6570\u7684\u8F93\u5165\u662F\u4E00\u4E2A\u5B57\u5178\uFF08\u4E0E\u6570\u636E\u96C6\u7684\u9879\u7C7B\u4F3C\uFF09\uFF0C\u5E76\u8FD4\u56DE\u4E00\u4E2A\u5305\u542B"),Ne=i(A,"STRONG",{});var Np=f(Ne);jr=a(Np,"\u8F93\u5165\u8BCDid(input_ids)"),Np.forEach(e),kr=a(A," \uFF0C "),Re=i(A,"STRONG",{});var Rp=f(Re);xr=a(Rp,"\u6CE8\u610F\u529B\u906E\u7F69(attention_mask)"),Rp.forEach(e),vr=a(A," \u548C "),De=i(A,"STRONG",{});var Dp=f(De);Er=a(Dp,"\u7C7B\u578B\u6807\u8BB0ID(token_type_ids)"),Dp.forEach(e),wr=a(A," \u952E\u7684\u65B0\u5B57\u5178\u3002\u8BF7\u6CE8\u610F\uFF0C\u5982\u679C\u50CF\u4E0A\u9762\u7684"),Ge=i(A,"STRONG",{});var Gp=f(Ge);Tr=a(Gp,"\u793A\u4F8B"),Gp.forEach(e),Sr=a(A,"\u4E00\u6837\uFF0C\u5982\u679C\u952E\u6240\u5BF9\u5E94\u7684\u503C\u5305\u542B\u591A\u4E2A\u53E5\u5B50\uFF08\u6BCF\u4E2A\u952E\u4F5C\u4E3A\u4E00\u4E2A\u53E5\u5B50\u5217\u8868\uFF09\uFF0C\u90A3\u4E48\u5B83\u4F9D\u7136\u53EF\u4EE5\u5DE5\u4F5C\uFF0C\u5C31\u50CF\u524D\u9762\u7684\u4F8B\u5B50\u4E00\u6837\u6807\u8BB0\u5668\u53EF\u4EE5\u5904\u7406\u6210\u5BF9\u7684\u53E5\u5B50\u5217\u8868\u3002\u8FD9\u6837\u7684\u8BDD\u6211\u4EEC\u53EF\u4EE5\u5728\u8C03\u7528"),ze=i(A,"STRONG",{});var zp=f(ze);yr=a(zp,"map()"),zp.forEach(e),qr=a(A,"\u4F7F\u7528\u8BE5\u9009\u9879 "),Oe=i(A,"STRONG",{});var Op=f(Oe);Pr=a(Op,"batched=True"),Op.forEach(e),Nr=a(A," \uFF0C\u8FD9\u5C06\u663E\u8457\u52A0\u5FEB\u6807\u8BB0\u4E0E\u6807\u8BB0\u7684\u901F\u5EA6\u3002\u8FD9\u4E2A"),Ce=i(A,"STRONG",{});var Cp=f(Ce);Rr=a(Cp,"\u6807\u8BB0\u5668"),Cp.forEach(e),Dr=a(A,"\u6765\u81EA"),pt=i(A,"A",{href:!0,rel:!0});var Ap=f(pt);Gr=a(Ap,"\u{1F917} Tokenizers"),Ap.forEach(e),zr=a(A,"\u5E93\u7531Rust\u7F16\u5199\u800C\u6210\u3002\u5F53\u6211\u4EEC\u4E00\u6B21\u7ED9\u5B83\u5927\u91CF\u7684\u8F93\u5165\u65F6\uFF0C\u8FD9\u4E2A\u6807\u8BB0\u5668\u53EF\u4EE5\u975E\u5E38\u5FEB\u3002"),A.forEach(e),Hn=m(s),zs=i(s,"P",{});var ba=f(zs);Or=a(ba,"\u8BF7\u6CE8\u610F\uFF0C\u6211\u4EEC\u73B0\u5728\u5728\u6807\u8BB0\u51FD\u6570\u4E2D\u7701\u7565\u4E86"),Ae=i(ba,"STRONG",{});var Ip=f(Ae);Cr=a(Ip,"padding"),Ip.forEach(e),Ar=a(ba,"\u53C2\u6570\u3002\u8FD9\u662F\u56E0\u4E3A\u5728\u6807\u8BB0\u7684\u65F6\u5019\u5C06\u6240\u6709\u6837\u672C\u586B\u5145\u5230\u6700\u5927\u957F\u5EA6\u7684\u6548\u7387\u4E0D\u9AD8\u3002\u4E00\u4E2A\u66F4\u597D\u7684\u505A\u6CD5\uFF1A\u5728\u6784\u5EFA\u6279\u5904\u7406\u65F6\u586B\u5145\u6837\u672C\u66F4\u597D\uFF0C\u56E0\u4E3A\u8FD9\u6837\u6211\u4EEC\u53EA\u9700\u8981\u586B\u5145\u5230\u8BE5\u6279\u5904\u7406\u4E2D\u7684\u6700\u5927\u957F\u5EA6\uFF0C\u800C\u4E0D\u662F\u6574\u4E2A\u6570\u636E\u96C6\u7684\u6700\u5927\u957F\u5EA6\u3002\u5F53\u8F93\u5165\u957F\u5EA6\u53D8\u5316\u5F88\u5927\u65F6\uFF0C\u8FD9\u53EF\u4EE5\u8282\u7701\u5927\u91CF\u65F6\u95F4\u548C\u5904\u7406\u80FD\u529B!"),ba.forEach(e),Bn=m(s),gs=i(s,"P",{});var Zt=f(gs);Ir=a(Zt,"\u4E0B\u9762\u662F\u6211\u4EEC\u5982\u4F55\u5728\u6240\u6709\u6570\u636E\u96C6\u4E0A\u540C\u65F6\u5E94\u7528\u6807\u8BB0\u51FD\u6570\u3002\u6211\u4EEC\u5728\u8C03\u7528"),Ie=i(Zt,"STRONG",{});var Lp=f(Ie);Lr=a(Lp,"map"),Lp.forEach(e),Mr=a(Zt,"\u65F6\u4F7F\u7528\u4E86"),Le=i(Zt,"STRONG",{});var Mp=f(Le);Fr=a(Mp,"batch =True"),Mp.forEach(e),Ur=a(Zt,"\uFF0C\u8FD9\u6837\u51FD\u6570\u5C31\u53EF\u4EE5\u540C\u65F6\u5E94\u7528\u5230\u6570\u636E\u96C6\u7684\u591A\u4E2A\u5143\u7D20\u4E0A\uFF0C\u800C\u4E0D\u662F\u5206\u522B\u5E94\u7528\u5230\u6BCF\u4E2A\u5143\u7D20\u4E0A\u3002\u8FD9\u5C06\u4F7F\u6211\u4EEC\u7684\u9884\u5904\u7406\u5FEB\u8BB8\u591A"),Zt.forEach(e),Wn=m(s),w(it.$$.fragment,s),Vn=m(s),Ct=i(s,"P",{});var Fp=f(Ct);Hr=a(Fp,"\u{1F917}Datasets\u5E93\u5E94\u7528\u8FD9\u79CD\u5904\u7406\u7684\u65B9\u5F0F\u662F\u5411\u6570\u636E\u96C6\u6DFB\u52A0\u65B0\u7684\u5B57\u6BB5\uFF0C\u6BCF\u4E2A\u5B57\u6BB5\u5BF9\u5E94\u9884\u5904\u7406\u51FD\u6570\u8FD4\u56DE\u7684\u5B57\u5178\u4E2D\u7684\u6BCF\u4E2A\u952E:"),Fp.forEach(e),Jn=m(s),w(ft.$$.fragment,s),Kn=m(s),K=i(s,"P",{});var Ls=f(K);Br=a(Ls,"\u5728\u4F7F\u7528\u9884\u5904\u7406\u51FD\u6570"),Me=i(Ls,"STRONG",{});var Up=f(Me);Wr=a(Up,"map()"),Up.forEach(e),Vr=a(Ls,"\u65F6\uFF0C\u751A\u81F3\u53EF\u4EE5\u901A\u8FC7\u4F20\u9012"),Fe=i(Ls,"STRONG",{});var Hp=f(Fe);Jr=a(Hp,"num_proc"),Hp.forEach(e),Kr=a(Ls,"\u53C2\u6570\u4F7F\u7528\u5E76\u884C\u5904\u7406\u3002\u6211\u4EEC\u5728\u8FD9\u91CC\u6CA1\u6709\u8FD9\u6837\u505A\uFF0C\u56E0\u4E3A\u{1F917}\u6807\u8BB0\u5668\u5E93\u5DF2\u7ECF\u4F7F\u7528\u591A\u4E2A\u7EBF\u7A0B\u6765\u66F4\u5FEB\u5730\u6807\u8BB0\u6211\u4EEC\u7684\u6837\u672C\uFF0C\u4F46\u662F\u5982\u679C\u60A8\u6CA1\u6709\u4F7F\u7528\u8BE5\u5E93\u652F\u6301\u7684\u5FEB\u901F\u6807\u8BB0\u5668\uFF0C\u4F7F\u7528"),Ue=i(Ls,"STRONG",{});var Bp=f(Ue);Yr=a(Bp,"num_proc"),Bp.forEach(e),Zr=a(Ls,"\u53EF\u80FD\u4F1A\u52A0\u5FEB\u9884\u5904\u7406\u3002"),Ls.forEach(e),Yn=m(s),M=i(s,"P",{});var ss=f(M);Qr=a(ss,"\u6211\u4EEC\u7684"),He=i(ss,"STRONG",{});var Wp=f(He);Xr=a(Wp,"\u6807\u8BB0\u51FD\u6570(tokenize_function)"),Wp.forEach(e),so=a(ss,"\u8FD4\u56DE\u5305\u542B"),Be=i(ss,"STRONG",{});var Vp=f(Be);to=a(Vp,"\u8F93\u5165\u8BCDid(input_ids)"),Vp.forEach(e),eo=a(ss," \uFF0C "),We=i(ss,"STRONG",{});var Jp=f(We);no=a(Jp,"\u6CE8\u610F\u529B\u906E\u7F69(attention_mask)"),Jp.forEach(e),ao=a(ss," \u548C "),Ve=i(ss,"STRONG",{});var Kp=f(Ve);lo=a(Kp,"\u7C7B\u578B\u6807\u8BB0ID(token_type_ids)"),Kp.forEach(e),ro=a(ss," \u952E\u7684\u5B57\u5178,\u6240\u4EE5\u8FD9\u4E09\u4E2A\u5B57\u6BB5\u88AB\u6DFB\u52A0\u5230\u6570\u636E\u96C6\u7684\u6807\u8BB0\u7684\u7ED3\u679C\u4E2D\u3002\u6CE8\u610F\uFF0C\u5982\u679C\u9884\u5904\u7406\u51FD\u6570"),Je=i(ss,"STRONG",{});var Yp=f(Je);oo=a(Yp,"map()"),Yp.forEach(e),po=a(ss,"\u4E3A\u73B0\u6709\u952E\u8FD4\u56DE\u4E00\u4E2A\u65B0\u503C\uFF0C\u90A3\u5C06\u4F1A\u4FEE\u6539\u539F\u6709\u952E\u7684\u503C\u3002"),ss.forEach(e),Zn=m(s),At=i(s,"P",{});var Zp=f(At);io=a(Zp,"\u6700\u540E\u4E00\u4EF6\u6211\u4EEC\u9700\u8981\u505A\u7684\u4E8B\u60C5\u662F\uFF0C\u5F53\u6211\u4EEC\u4E00\u8D77\u6279\u5904\u7406\u5143\u7D20\u65F6\uFF0C\u5C06\u6240\u6709\u793A\u4F8B\u586B\u5145\u5230\u6700\u957F\u5143\u7D20\u7684\u957F\u5EA6\u2014\u2014\u6211\u4EEC\u79F0\u4E4B\u4E3A\u52A8\u6001\u586B\u5145\u3002"),Zp.forEach(e),Qn=m(s),ct=i(s,"H3",{id:!0});var Qp=f(ct);fo=a(Qp,"\u52A8\u6001\u586B\u5145"),Qp.forEach(e),Xn=m(s),w(ut.$$.fragment,s),sa=m(s),xs.l(s),It=m(s),Os=i(s,"P",{});var $a=f(Os);co=a($a,"\u4E3A\u4E86\u89E3\u51B3\u53E5\u5B50\u957F\u5EA6\u7EDF\u4E00\u7684\u95EE\u9898\uFF0C\u6211\u4EEC\u5FC5\u987B\u5B9A\u4E49\u4E00\u4E2Acollate\u51FD\u6570\uFF0C\u8BE5\u51FD\u6570\u4F1A\u5C06\u6BCF\u4E2Abatch\u53E5\u5B50\u586B\u5145\u5230\u6B63\u786E\u7684\u957F\u5EA6\u3002\u5E78\u8FD0\u7684\u662F\uFF0C\u{1F917}transformer\u5E93\u901A\u8FC7"),Ke=i($a,"STRONG",{});var Xp=f(Ke);uo=a(Xp,"DataCollatorWithPadding"),Xp.forEach(e),_o=a($a,"\u4E3A\u6211\u4EEC\u63D0\u4F9B\u4E86\u8FD9\u6837\u4E00\u4E2A\u51FD\u6570\u3002\u5F53\u4F60\u5B9E\u4F8B\u5316\u5B83\u65F6\uFF0C\u9700\u8981\u4E00\u4E2A\u6807\u8BB0\u5668(\u7528\u6765\u77E5\u9053\u4F7F\u7528\u54EA\u4E2A\u8BCD\u6765\u586B\u5145\uFF0C\u4EE5\u53CA\u6A21\u578B\u671F\u671B\u586B\u5145\u5728\u5DE6\u8FB9\u8FD8\u662F\u53F3\u8FB9)\uFF0C\u5E76\u5C06\u505A\u4F60\u9700\u8981\u7684\u4E00\u5207:"),$a.forEach(e),ta=m(s),ps.l(s),Lt=m(s),Y=i(s,"P",{});var Ms=f(Y);mo=a(Ms,"\u4E3A\u4E86\u6D4B\u8BD5\u8FD9\u4E2A\u65B0\u73A9\u5177\uFF0C\u8BA9\u6211\u4EEC\u4ECE\u6211\u4EEC\u7684\u8BAD\u7EC3\u96C6\u4E2D\u62BD\u53D6\u51E0\u4E2A\u6837\u672C\u3002\u8FD9\u91CC\uFF0C\u6211\u4EEC\u5220\u9664\u5217"),Ye=i(Ms,"STRONG",{});var si=f(Ye);ho=a(si,"idx"),si.forEach(e),bo=a(Ms,", "),Ze=i(Ms,"STRONG",{});var ti=f(Ze);$o=a(ti,"sentence1"),ti.forEach(e),go=a(Ms,"\u548C"),Qe=i(Ms,"STRONG",{});var ei=f(Qe);jo=a(ei,"sentence2"),ei.forEach(e),ko=a(Ms,"\uFF0C\u56E0\u4E3A\u4E0D\u9700\u8981\u5B83\u4EEC\uFF0C\u5E76\u67E5\u770B\u4E00\u4E2Abatch\u4E2D\u6BCF\u4E2A\u6761\u76EE\u7684\u957F\u5EA6:"),Ms.forEach(e),ea=m(s),w(_t.$$.fragment,s),na=m(s),w(mt.$$.fragment,s),aa=m(s),Cs=i(s,"P",{});var ga=f(Cs);xo=a(ga,"\u6BEB\u65E0\u7591\u95EE\uFF0C\u6211\u4EEC\u5F97\u5230\u4E86\u4E0D\u540C\u957F\u5EA6\u7684\u6837\u672C\uFF0C\u4ECE32\u523067\u3002\u52A8\u6001\u586B\u5145\u610F\u5473\u7740\u8BE5\u6279\u4E2D\u7684\u6240\u6709\u6837\u672C\u90FD\u5E94\u8BE5\u586B\u5145\u5230\u957F\u5EA6\u4E3A67\uFF0C\u8FD9\u662F\u8BE5\u6279\u4E2D\u7684\u6700\u5927\u957F\u5EA6\u3002\u5982\u679C\u6CA1\u6709\u52A8\u6001\u586B\u5145\uFF0C\u6240\u6709\u7684\u6837\u672C\u90FD\u5FC5\u987B\u586B\u5145\u5230\u6574\u4E2A\u6570\u636E\u96C6\u4E2D\u7684\u6700\u5927\u957F\u5EA6\uFF0C\u6216\u8005\u6A21\u578B\u53EF\u4EE5\u63A5\u53D7\u7684\u6700\u5927\u957F\u5EA6\u3002\u8BA9\u6211\u4EEC\u518D\u6B21\u68C0\u67E5"),Xe=i(ga,"STRONG",{});var ni=f(Xe);vo=a(ni,"data_collator"),ni.forEach(e),Eo=a(ga,"\u662F\u5426\u6B63\u786E\u5730\u52A8\u6001\u586B\u5145\u4E86\u8FD9\u6279\u6837\u672C\uFF1A"),ga.forEach(e),la=m(s),w(ht.$$.fragment,s),ra=m(s),fs.l(s),Mt=m(s),w(As.$$.fragment,s),oa=m(s),O&&O.l(s),Ft=ai(),this.h()},h(){q(l,"name","hf:doc:metadata"),q(l,"content",JSON.stringify(Ri)),q(k,"id",""),q(ts,"href","https://www.aclweb.org/anthology/I05-5002.pdf"),q(ts,"rel","nofollow"),q(es,"id","hub"),q(es,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),q(es,"href","#hub"),q(V,"class","relative group"),q(Fs,"href","https://huggingface.co/datasets"),q(Fs,"rel","nofollow"),q(Us,"href","https://huggingface.co/docs/datasets/loading_datasets.html#from-the-huggingface-hub"),q(Us,"rel","nofollow"),q(Hs,"href","https://gluebenchmark.com/"),q(Hs,"rel","nofollow"),q(Zs,"id",""),q(qt,"href","/course/chapter2"),q(Nt,"href","/course/chapter2"),q(Dt,"href","/course/chapter1"),q(zt,"href","/course/chapter2"),q(Ot,"href","/course/chapter2"),q(lt,"href","https://arrow.apache.org/"),q(lt,"rel","nofollow"),q(rt,"href","https://huggingface.co/docs/datasets/package_reference/main_classes.html#datasets.Dataset.map"),q(rt,"rel","nofollow"),q(pt,"href","https://github.com/huggingface/tokenizers"),q(pt,"rel","nofollow"),q(ct,"id","")},m(s,o){t(document.head,l),c(s,u,o),T(r,s,o),c(s,b,o),c(s,k,o),t(k,h),c(s,d,o),dt[x].m(s,o),c(s,y,o),bt[R].m(s,o),c(s,cs,o),c(s,us,o),t(us,vs),c(s,W,o),c(s,B,o),t(B,_s),t(B,ts),t(ts,Es),t(B,N),c(s,G,o),c(s,V,o),t(V,es),t(es,ws),T(ms,ws,null),t(V,wt),t(V,Qt),t(Qt,ja),c(s,an,o),$t[ns].m(s,o),c(s,Tt,o),c(s,J,o),t(J,ka),t(J,Fs),t(Fs,xa),t(J,va),t(J,Us),t(Us,Ea),t(J,wa),t(J,Hs),t(Hs,Ta),t(J,Sa),c(s,ln,o),c(s,St,o),t(St,ya),c(s,rn,o),T(Bs,s,o),c(s,on,o),T(Ws,s,o),c(s,pn,o),c(s,I,o),t(I,qa),t(I,Xt),t(Xt,Pa),t(I,Na),t(I,se),t(se,Ra),t(I,Da),t(I,te),t(te,Ga),t(I,za),t(I,ee),t(ee,Oa),t(I,Ca),t(I,ne),t(ne,Aa),t(I,Ia),c(s,fn,o),c(s,hs,o),t(hs,La),t(hs,ae),t(ae,Ma),t(hs,Fa),t(hs,le),t(le,Ua),t(hs,Ha),c(s,cn,o),c(s,Ts,o),t(Ts,Ba),t(Ts,re),t(re,Wa),t(Ts,Va),c(s,un,o),T(Vs,s,o),c(s,_n,o),T(Js,s,o),c(s,mn,o),c(s,ds,o),t(ds,Ja),t(ds,oe),t(oe,Ka),t(ds,Ya),t(ds,pe),t(pe,Za),t(ds,Qa),c(s,hn,o),T(Ks,s,o),c(s,dn,o),T(Ys,s,o),c(s,bn,o),c(s,C,o),t(C,Xa),t(C,ie),t(ie,sl),t(C,tl),t(C,fe),t(fe,el),t(C,nl),t(C,ce),t(ce,al),t(C,ll),t(C,ue),t(ue,rl),t(C,ol),t(C,_e),t(_e,pl),t(C,il),t(C,me),t(me,fl),t(C,cl),c(s,$n,o),T(Ss,s,o),c(s,gn,o),c(s,Zs,o),t(Zs,ul),c(s,jn,o),gt[ls].m(s,o),c(s,yt,o),c(s,ys,o),t(ys,_l),t(ys,qt),t(qt,ml),t(ys,hl),c(s,kn,o),T(Qs,s,o),c(s,xn,o),c(s,Pt,o),t(Pt,dl),c(s,vn,o),T(Xs,s,o),c(s,En,o),T(st,s,o),c(s,wn,o),c(s,L,o),t(L,bl),t(L,Nt),t(Nt,$l),t(L,gl),t(L,he),t(he,jl),t(L,kl),t(L,de),t(de,xl),t(L,vl),t(L,be),t(be,El),t(L,wl),t(L,$e),t($e,Tl),t(L,Sl),c(s,Tn,o),T(qs,s,o),c(s,Sn,o),c(s,Ps,o),t(Ps,yl),t(Ps,ge),t(ge,ql),t(Ps,Pl),c(s,yn,o),T(tt,s,o),c(s,qn,o),c(s,Rt,o),t(Rt,Nl),c(s,Pn,o),T(et,s,o),c(s,Nn,o),c(s,bs,o),t(bs,Rl),t(bs,je),t(je,Dl),t(bs,Gl),t(bs,ke),t(ke,zl),t(bs,Ol),c(s,Rn,o),T(nt,s,o),c(s,Dn,o),c(s,F,o),t(F,Cl),t(F,xe),t(xe,Al),t(F,Il),t(F,ve),t(ve,Ll),t(F,Ml),t(F,Ee),t(Ee,Fl),t(F,Ul),t(F,we),t(we,Hl),t(F,Bl),c(s,Gn,o),c(s,Ns,o),t(Ns,Wl),t(Ns,Te),t(Te,Vl),t(Ns,Jl),c(s,zn,o),c(s,Rs,o),t(Rs,Kl),t(Rs,Dt),t(Dt,Yl),t(Rs,Zl),c(s,On,o),c(s,Gt,o),t(Gt,Ql),c(s,Cn,o),c(s,Ds,o),t(Ds,Xl),t(Ds,Se),t(Se,sr),t(Ds,tr),c(s,An,o),c(s,$s,o),t($s,er),t($s,zt),t(zt,nr),t($s,ar),t($s,Ot),t(Ot,lr),t($s,rr),c(s,In,o),T(at,s,o),c(s,Ln,o),c(s,U,o),t(U,or),t(U,ye),t(ye,pr),t(U,ir),t(U,qe),t(qe,fr),t(U,cr),t(U,Pe),t(Pe,ur),t(U,_r),t(U,lt),t(lt,mr),t(U,hr),c(s,Mn,o),c(s,Gs,o),t(Gs,dr),t(Gs,rt),t(rt,br),t(Gs,$r),c(s,Fn,o),T(ot,s,o),c(s,Un,o),c(s,D,o),t(D,gr),t(D,Ne),t(Ne,jr),t(D,kr),t(D,Re),t(Re,xr),t(D,vr),t(D,De),t(De,Er),t(D,wr),t(D,Ge),t(Ge,Tr),t(D,Sr),t(D,ze),t(ze,yr),t(D,qr),t(D,Oe),t(Oe,Pr),t(D,Nr),t(D,Ce),t(Ce,Rr),t(D,Dr),t(D,pt),t(pt,Gr),t(D,zr),c(s,Hn,o),c(s,zs,o),t(zs,Or),t(zs,Ae),t(Ae,Cr),t(zs,Ar),c(s,Bn,o),c(s,gs,o),t(gs,Ir),t(gs,Ie),t(Ie,Lr),t(gs,Mr),t(gs,Le),t(Le,Fr),t(gs,Ur),c(s,Wn,o),T(it,s,o),c(s,Vn,o),c(s,Ct,o),t(Ct,Hr),c(s,Jn,o),T(ft,s,o),c(s,Kn,o),c(s,K,o),t(K,Br),t(K,Me),t(Me,Wr),t(K,Vr),t(K,Fe),t(Fe,Jr),t(K,Kr),t(K,Ue),t(Ue,Yr),t(K,Zr),c(s,Yn,o),c(s,M,o),t(M,Qr),t(M,He),t(He,Xr),t(M,so),t(M,Be),t(Be,to),t(M,eo),t(M,We),t(We,no),t(M,ao),t(M,Ve),t(Ve,lo),t(M,ro),t(M,Je),t(Je,oo),t(M,po),c(s,Zn,o),c(s,At,o),t(At,io),c(s,Qn,o),c(s,ct,o),t(ct,fo),c(s,Xn,o),T(ut,s,o),c(s,sa,o),xs.m(s,o),c(s,It,o),c(s,Os,o),t(Os,co),t(Os,Ke),t(Ke,uo),t(Os,_o),c(s,ta,o),jt[os].m(s,o),c(s,Lt,o),c(s,Y,o),t(Y,mo),t(Y,Ye),t(Ye,ho),t(Y,bo),t(Y,Ze),t(Ze,$o),t(Y,go),t(Y,Qe),t(Qe,jo),t(Y,ko),c(s,ea,o),T(_t,s,o),c(s,na,o),T(mt,s,o),c(s,aa,o),c(s,Cs,o),t(Cs,xo),t(Cs,Xe),t(Xe,vo),t(Cs,Eo),c(s,la,o),T(ht,s,o),c(s,ra,o),kt[is].m(s,o),c(s,Mt,o),T(As,s,o),c(s,oa,o),O&&O.m(s,o),c(s,Ft,o),pa=!0},p(s,[o]){const sn={};o&1&&(sn.fw=s[0]),r.$set(sn);let Ut=x;x=So(s),x!==Ut&&(Et(),j(dt[Ut],1,1,()=>{dt[Ut]=null}),vt(),$=dt[x],$||($=dt[x]=To[x](s),$.c()),g($,1),$.m(y.parentNode,y));let Is=R;R=qo(s),R!==Is&&(Et(),j(bt[Is],1,1,()=>{bt[Is]=null}),vt(),z=bt[R],z||(z=bt[R]=yo[R](s),z.c()),g(z,1),z.m(cs.parentNode,cs));let Ht=ns;ns=No(s),ns!==Ht&&(Et(),j($t[Ht],1,1,()=>{$t[Ht]=null}),vt(),as=$t[ns],as||(as=$t[ns]=Po[ns](s),as.c()),g(as,1),as.m(Tt.parentNode,Tt));const xt={};o&2&&(xt.$$scope={dirty:o,ctx:s}),Ss.$set(xt);let Bt=ls;ls=Do(s),ls!==Bt&&(Et(),j(gt[Bt],1,1,()=>{gt[Bt]=null}),vt(),rs=gt[ls],rs||(rs=gt[ls]=Ro[ls](s),rs.c()),g(rs,1),rs.m(yt.parentNode,yt));const tn={};o&2&&(tn.$$scope={dirty:o,ctx:s}),qs.$set(tn),ia!==(ia=Go(s))&&(xs.d(1),xs=ia(s),xs&&(xs.c(),xs.m(It.parentNode,It)));let Wt=os;os=Oo(s),os!==Wt&&(Et(),j(jt[Wt],1,1,()=>{jt[Wt]=null}),vt(),ps=jt[os],ps||(ps=jt[os]=zo[os](s),ps.c()),g(ps,1),ps.m(Lt.parentNode,Lt));let Z=is;is=Ao(s),is!==Z&&(Et(),j(kt[Z],1,1,()=>{kt[Z]=null}),vt(),fs=kt[is],fs||(fs=kt[is]=Co[is](s),fs.c()),g(fs,1),fs.m(Mt.parentNode,Mt));const en={};o&2&&(en.$$scope={dirty:o,ctx:s}),As.$set(en),s[0]==="tf"?O?o&1&&g(O,1):(O=li(),O.c(),g(O,1),O.m(Ft.parentNode,Ft)):O&&(Et(),j(O,1,1,()=>{O=null}),vt())},i(s){pa||(g(r.$$.fragment,s),g($),g(z),g(ms.$$.fragment,s),g(as),g(Bs.$$.fragment,s),g(Ws.$$.fragment,s),g(Vs.$$.fragment,s),g(Js.$$.fragment,s),g(Ks.$$.fragment,s),g(Ys.$$.fragment,s),g(Ss.$$.fragment,s),g(rs),g(Qs.$$.fragment,s),g(Xs.$$.fragment,s),g(st.$$.fragment,s),g(qs.$$.fragment,s),g(tt.$$.fragment,s),g(et.$$.fragment,s),g(nt.$$.fragment,s),g(at.$$.fragment,s),g(ot.$$.fragment,s),g(it.$$.fragment,s),g(ft.$$.fragment,s),g(ut.$$.fragment,s),g(ps),g(_t.$$.fragment,s),g(mt.$$.fragment,s),g(ht.$$.fragment,s),g(fs),g(As.$$.fragment,s),g(O),pa=!0)},o(s){j(r.$$.fragment,s),j($),j(z),j(ms.$$.fragment,s),j(as),j(Bs.$$.fragment,s),j(Ws.$$.fragment,s),j(Vs.$$.fragment,s),j(Js.$$.fragment,s),j(Ks.$$.fragment,s),j(Ys.$$.fragment,s),j(Ss.$$.fragment,s),j(rs),j(Qs.$$.fragment,s),j(Xs.$$.fragment,s),j(st.$$.fragment,s),j(qs.$$.fragment,s),j(tt.$$.fragment,s),j(et.$$.fragment,s),j(nt.$$.fragment,s),j(at.$$.fragment,s),j(ot.$$.fragment,s),j(it.$$.fragment,s),j(ft.$$.fragment,s),j(ut.$$.fragment,s),j(ps),j(_t.$$.fragment,s),j(mt.$$.fragment,s),j(ht.$$.fragment,s),j(fs),j(As.$$.fragment,s),j(O),pa=!1},d(s){e(l),s&&e(u),S(r,s),s&&e(b),s&&e(k),s&&e(d),dt[x].d(s),s&&e(y),bt[R].d(s),s&&e(cs),s&&e(us),s&&e(W),s&&e(B),s&&e(G),s&&e(V),S(ms),s&&e(an),$t[ns].d(s),s&&e(Tt),s&&e(J),s&&e(ln),s&&e(St),s&&e(rn),S(Bs,s),s&&e(on),S(Ws,s),s&&e(pn),s&&e(I),s&&e(fn),s&&e(hs),s&&e(cn),s&&e(Ts),s&&e(un),S(Vs,s),s&&e(_n),S(Js,s),s&&e(mn),s&&e(ds),s&&e(hn),S(Ks,s),s&&e(dn),S(Ys,s),s&&e(bn),s&&e(C),s&&e($n),S(Ss,s),s&&e(gn),s&&e(Zs),s&&e(jn),gt[ls].d(s),s&&e(yt),s&&e(ys),s&&e(kn),S(Qs,s),s&&e(xn),s&&e(Pt),s&&e(vn),S(Xs,s),s&&e(En),S(st,s),s&&e(wn),s&&e(L),s&&e(Tn),S(qs,s),s&&e(Sn),s&&e(Ps),s&&e(yn),S(tt,s),s&&e(qn),s&&e(Rt),s&&e(Pn),S(et,s),s&&e(Nn),s&&e(bs),s&&e(Rn),S(nt,s),s&&e(Dn),s&&e(F),s&&e(Gn),s&&e(Ns),s&&e(zn),s&&e(Rs),s&&e(On),s&&e(Gt),s&&e(Cn),s&&e(Ds),s&&e(An),s&&e($s),s&&e(In),S(at,s),s&&e(Ln),s&&e(U),s&&e(Mn),s&&e(Gs),s&&e(Fn),S(ot,s),s&&e(Un),s&&e(D),s&&e(Hn),s&&e(zs),s&&e(Bn),s&&e(gs),s&&e(Wn),S(it,s),s&&e(Vn),s&&e(Ct),s&&e(Jn),S(ft,s),s&&e(Kn),s&&e(K),s&&e(Yn),s&&e(M),s&&e(Zn),s&&e(At),s&&e(Qn),s&&e(ct),s&&e(Xn),S(ut,s),s&&e(sa),xs.d(s),s&&e(It),s&&e(Os),s&&e(ta),jt[os].d(s),s&&e(Lt),s&&e(Y),s&&e(ea),S(_t,s),s&&e(na),S(mt,s),s&&e(aa),s&&e(Cs),s&&e(la),S(ht,s),s&&e(ra),kt[is].d(s),s&&e(Mt),S(As,s),s&&e(oa),O&&O.d(s),s&&e(Ft)}}}const Ri={local:"",sections:[{local:"hub",title:"\u4ECE\u6A21\u578B\u4E2D\u5FC3\uFF08Hub\uFF09\u52A0\u8F7D\u6570\u636E\u96C6"},{local:"",title:"\u9884\u5904\u7406\u6570\u636E\u96C6"},{local:"",title:"\u52A8\u6001\u586B\u5145"}],title:"\u5904\u7406\u6570\u636E"};function Di(v,l,u){let r="pt";return ci(()=>{const b=new URLSearchParams(window.location.search);u(0,r=b.get("fw")||"pt")}),[r]}class Mi extends oi{constructor(l){super();pi(this,l,Di,Ni,ii,{})}}export{Mi as default,Ri as metadata};
