import{S as oi,i as pi,s as ii,e as p,t as a,k as _,w,c as i,a as c,h as n,d as e,m,x as E,g as f,G as t,y as T,q as $,o as j,B as S,l as ni,M as ci,b as y,p as vt,v as fi,n as wt}from"../../chunks/vendor-hf-doc-builder.js";import{T as Eo}from"../../chunks/Tip-hf-doc-builder.js";import{Y as aa}from"../../chunks/Youtube-hf-doc-builder.js";import{I as ui}from"../../chunks/IconCopyLink-hf-doc-builder.js";import{C as z}from"../../chunks/CodeBlock-hf-doc-builder.js";import{D as ri}from"../../chunks/DocNotebookDropdown-hf-doc-builder.js";import{F as _i}from"../../chunks/FrameworkSwitchCourse-hf-doc-builder.js";function mi(v){let l,u;return l=new ri({props:{classNames:"absolute z-10 right-0 top-0",options:[{label:"Google Colab",value:"https://colab.research.google.com/github/huggingface/notebooks/blob/master/course/chapter3/section2_tf.ipynb"},{label:"Aws Studio",value:"https://studiolab.sagemaker.aws/import/github/huggingface/notebooks/blob/master/course/chapter3/section2_tf.ipynb"}]}}),{c(){w(l.$$.fragment)},l(r){E(l.$$.fragment,r)},m(r,b){T(l,r,b),u=!0},i(r){u||($(l.$$.fragment,r),u=!0)},o(r){j(l.$$.fragment,r),u=!1},d(r){S(l,r)}}}function hi(v){let l,u;return l=new ri({props:{classNames:"absolute z-10 right-0 top-0",options:[{label:"Google Colab",value:"https://colab.research.google.com/github/huggingface/notebooks/blob/master/course/chapter3/section2_pt.ipynb"},{label:"Aws Studio",value:"https://studiolab.sagemaker.aws/import/github/huggingface/notebooks/blob/master/course/chapter3/section2_pt.ipynb"}]}}),{c(){w(l.$$.fragment)},l(r){E(l.$$.fragment,r)},m(r,b){T(l,r,b),u=!0},i(r){u||($(l.$$.fragment,r),u=!0)},o(r){j(l.$$.fragment,r),u=!1},d(r){S(l,r)}}}function di(v){let l,u,r,b,k,h,d,x;return d=new z({props:{code:`import tensorflow as tf
import numpy as np
from transformers import AutoTokenizer, TFAutoModelForSequenceClassification

# Same as before
checkpoint = "bert-base-uncased"
tokenizer = AutoTokenizer.from_pretrained(checkpoint)
model = TFAutoModelForSequenceClassification.from_pretrained(checkpoint)
sequences = [
    "I've been waiting for a HuggingFace course my whole life.",
    "This course is amazing!",
]
batch = dict(tokenizer(sequences, padding=True, truncation=True, return_tensors="tf"))

# This is new
model.compile(optimizer="adam", loss="sparse_categorical_crossentropy")
labels = tf.convert_to_tensor([1, 1])
model.train_on_batch(batch, labels)`,highlighted:`<span class="hljs-keyword">import</span> tensorflow <span class="hljs-keyword">as</span> tf
<span class="hljs-keyword">import</span> numpy <span class="hljs-keyword">as</span> np
<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoTokenizer, TFAutoModelForSequenceClassification

<span class="hljs-comment"># Same as before</span>
checkpoint = <span class="hljs-string">&quot;bert-base-uncased&quot;</span>
tokenizer = AutoTokenizer.from_pretrained(checkpoint)
model = TFAutoModelForSequenceClassification.from_pretrained(checkpoint)
sequences = [
    <span class="hljs-string">&quot;I&#x27;ve been waiting for a HuggingFace course my whole life.&quot;</span>,
    <span class="hljs-string">&quot;This course is amazing!&quot;</span>,
]
batch = <span class="hljs-built_in">dict</span>(tokenizer(sequences, padding=<span class="hljs-literal">True</span>, truncation=<span class="hljs-literal">True</span>, return_tensors=<span class="hljs-string">&quot;tf&quot;</span>))

<span class="hljs-comment"># This is new</span>
model.<span class="hljs-built_in">compile</span>(optimizer=<span class="hljs-string">&quot;adam&quot;</span>, loss=<span class="hljs-string">&quot;sparse_categorical_crossentropy&quot;</span>)
labels = tf.convert_to_tensor([<span class="hljs-number">1</span>, <span class="hljs-number">1</span>])
model.train_on_batch(batch, labels)`}}),{c(){l=p("p"),u=a("\u8FD9\u4E00\u5C0F\u8282\u5B66\u4E60"),r=p("a"),b=a("\u7B2C\u4E00\u5C0F\u8282"),k=a("\u4E2D\u63D0\u5230\u7684\u201C\u5982\u4F55\u4F7F\u7528\u6A21\u578B\u4E2D\u5FC3\uFF08hub\uFF09\u5927\u578B\u6570\u636E\u96C6\u201D\uFF0C\u4E0B\u9762\u662F\u6211\u4EEC\u7528\u6A21\u578B\u4E2D\u5FC3\u7684\u6570\u636E\u5728TensorFlow\u4E0A\u8BAD\u7EC3\u53E5\u5B50\u5206\u7C7B\u5668\u7684\u4E00\u4E2A\u4F8B\u5B50\uFF1A"),h=_(),w(d.$$.fragment),this.h()},l(g){l=i(g,"P",{});var q=c(l);u=n(q,"\u8FD9\u4E00\u5C0F\u8282\u5B66\u4E60"),r=i(q,"A",{href:!0});var N=c(r);b=n(N,"\u7B2C\u4E00\u5C0F\u8282"),N.forEach(e),k=n(q,"\u4E2D\u63D0\u5230\u7684\u201C\u5982\u4F55\u4F7F\u7528\u6A21\u578B\u4E2D\u5FC3\uFF08hub\uFF09\u5927\u578B\u6570\u636E\u96C6\u201D\uFF0C\u4E0B\u9762\u662F\u6211\u4EEC\u7528\u6A21\u578B\u4E2D\u5FC3\u7684\u6570\u636E\u5728TensorFlow\u4E0A\u8BAD\u7EC3\u53E5\u5B50\u5206\u7C7B\u5668\u7684\u4E00\u4E2A\u4F8B\u5B50\uFF1A"),q.forEach(e),h=m(g),E(d.$$.fragment,g),this.h()},h(){y(r,"href","/course/chapter2")},m(g,q){f(g,l,q),t(l,u),t(l,r),t(r,b),t(l,k),f(g,h,q),T(d,g,q),x=!0},i(g){x||($(d.$$.fragment,g),x=!0)},o(g){j(d.$$.fragment,g),x=!1},d(g){g&&e(l),g&&e(h),S(d,g)}}}function bi(v){let l,u,r,b,k,h,d,x;return d=new z({props:{code:`import torch
from transformers import AdamW, AutoTokenizer, AutoModelForSequenceClassification

# Same as before
checkpoint = "bert-base-uncased"
tokenizer = AutoTokenizer.from_pretrained(checkpoint)
model = AutoModelForSequenceClassification.from_pretrained(checkpoint)
sequences = [
    "I've been waiting for a HuggingFace course my whole life.",
    "This course is amazing!",
]
batch = tokenizer(sequences, padding=True, truncation=True, return_tensors="pt")

# This is new
batch["labels"] = torch.tensor([1, 1])

optimizer = AdamW(model.parameters())
loss = model(**batch).loss
loss.backward()
optimizer.step()`,highlighted:`<span class="hljs-keyword">import</span> torch
<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AdamW, AutoTokenizer, AutoModelForSequenceClassification

<span class="hljs-comment"># Same as before</span>
checkpoint = <span class="hljs-string">&quot;bert-base-uncased&quot;</span>
tokenizer = AutoTokenizer.from_pretrained(checkpoint)
model = AutoModelForSequenceClassification.from_pretrained(checkpoint)
sequences = [
    <span class="hljs-string">&quot;I&#x27;ve been waiting for a HuggingFace course my whole life.&quot;</span>,
    <span class="hljs-string">&quot;This course is amazing!&quot;</span>,
]
batch = tokenizer(sequences, padding=<span class="hljs-literal">True</span>, truncation=<span class="hljs-literal">True</span>, return_tensors=<span class="hljs-string">&quot;pt&quot;</span>)

<span class="hljs-comment"># This is new</span>
batch[<span class="hljs-string">&quot;labels&quot;</span>] = torch.tensor([<span class="hljs-number">1</span>, <span class="hljs-number">1</span>])

optimizer = AdamW(model.parameters())
loss = model(**batch).loss
loss.backward()
optimizer.step()`}}),{c(){l=p("p"),u=a("\u8FD9\u4E00\u5C0F\u8282\u5B66\u4E60"),r=p("a"),b=a("\u7B2C\u4E00\u5C0F\u8282"),k=a("\u4E2D\u63D0\u5230\u7684\u201C\u5982\u4F55\u4F7F\u7528\u6A21\u578B\u4E2D\u5FC3\uFF08hub\uFF09\u5927\u578B\u6570\u636E\u96C6\u201D\uFF0C\u4E0B\u9762\u662F\u6211\u4EEC\u7528\u6A21\u578B\u4E2D\u5FC3\u7684\u6570\u636E\u5728PyTorch\u4E0A\u8BAD\u7EC3\u53E5\u5B50\u5206\u7C7B\u5668\u7684\u4E00\u4E2A\u4F8B\u5B50\uFF1A"),h=_(),w(d.$$.fragment),this.h()},l(g){l=i(g,"P",{});var q=c(l);u=n(q,"\u8FD9\u4E00\u5C0F\u8282\u5B66\u4E60"),r=i(q,"A",{href:!0});var N=c(r);b=n(N,"\u7B2C\u4E00\u5C0F\u8282"),N.forEach(e),k=n(q,"\u4E2D\u63D0\u5230\u7684\u201C\u5982\u4F55\u4F7F\u7528\u6A21\u578B\u4E2D\u5FC3\uFF08hub\uFF09\u5927\u578B\u6570\u636E\u96C6\u201D\uFF0C\u4E0B\u9762\u662F\u6211\u4EEC\u7528\u6A21\u578B\u4E2D\u5FC3\u7684\u6570\u636E\u5728PyTorch\u4E0A\u8BAD\u7EC3\u53E5\u5B50\u5206\u7C7B\u5668\u7684\u4E00\u4E2A\u4F8B\u5B50\uFF1A"),q.forEach(e),h=m(g),E(d.$$.fragment,g),this.h()},h(){y(r,"href","/course/chapter2")},m(g,q){f(g,l,q),t(l,u),t(l,r),t(r,b),t(l,k),f(g,h,q),T(d,g,q),x=!0},i(g){x||($(d.$$.fragment,g),x=!0)},o(g){j(d.$$.fragment,g),x=!1},d(g){g&&e(l),g&&e(h),S(d,g)}}}function gi(v){let l,u;return l=new aa({props:{id:"W_gMJF0xomE"}}),{c(){w(l.$$.fragment)},l(r){E(l.$$.fragment,r)},m(r,b){T(l,r,b),u=!0},i(r){u||($(l.$$.fragment,r),u=!0)},o(r){j(l.$$.fragment,r),u=!1},d(r){S(l,r)}}}function $i(v){let l,u;return l=new aa({props:{id:"_BZearw7f0w"}}),{c(){w(l.$$.fragment)},l(r){E(l.$$.fragment,r)},m(r,b){T(l,r,b),u=!0},i(r){u||($(l.$$.fragment,r),u=!0)},o(r){j(l.$$.fragment,r),u=!1},d(r){S(l,r)}}}function ji(v){let l,u,r,b,k;return{c(){l=p("p"),u=a("\u270F\uFE0F "),r=p("strong"),b=a("\u8BD5\u8BD5\u770B\uFF01"),k=a(" \u67E5\u770B\u8BAD\u7EC3\u96C6\u7684\u7B2C15\u884C\u5143\u7D20\u548C\u9A8C\u8BC1\u96C6\u768487\u884C\u5143\u7D20\u3002\u4ED6\u4EEC\u7684\u6807\u7B7E\u662F\u4EC0\u4E48\uFF1F")},l(h){l=i(h,"P",{});var d=c(l);u=n(d,"\u270F\uFE0F "),r=i(d,"STRONG",{});var x=c(r);b=n(x,"\u8BD5\u8BD5\u770B\uFF01"),x.forEach(e),k=n(d," \u67E5\u770B\u8BAD\u7EC3\u96C6\u7684\u7B2C15\u884C\u5143\u7D20\u548C\u9A8C\u8BC1\u96C6\u768487\u884C\u5143\u7D20\u3002\u4ED6\u4EEC\u7684\u6807\u7B7E\u662F\u4EC0\u4E48\uFF1F"),d.forEach(e)},m(h,d){f(h,l,d),t(l,u),t(l,r),t(r,b),t(l,k)},d(h){h&&e(l)}}}function ki(v){let l,u;return l=new aa({props:{id:"P-rZWqcB6CE"}}),{c(){w(l.$$.fragment)},l(r){E(l.$$.fragment,r)},m(r,b){T(l,r,b),u=!0},i(r){u||($(l.$$.fragment,r),u=!0)},o(r){j(l.$$.fragment,r),u=!1},d(r){S(l,r)}}}function xi(v){let l,u;return l=new aa({props:{id:"0u3ioSwev3s"}}),{c(){w(l.$$.fragment)},l(r){E(l.$$.fragment,r)},m(r,b){T(l,r,b),u=!0},i(r){u||($(l.$$.fragment,r),u=!0)},o(r){j(l.$$.fragment,r),u=!1},d(r){S(l,r)}}}function vi(v){let l,u,r,b,k;return{c(){l=p("p"),u=a("\u270F\uFE0F "),r=p("strong"),b=a("\u8BD5\u8BD5\u770B\uFF01"),k=a(" \u9009\u53D6\u8BAD\u7EC3\u96C6\u4E2D\u7684\u7B2C15\u4E2A\u5143\u7D20\uFF0C\u5C06\u4E24\u53E5\u8BDD\u5206\u522B\u6807\u8BB0\u4E3A\u4E00\u5BF9\u3002\u7ED3\u679C\u548C\u4E0A\u65B9\u7684\u4F8B\u5B50\u6709\u4EC0\u4E48\u4E0D\u540C\uFF1F")},l(h){l=i(h,"P",{});var d=c(l);u=n(d,"\u270F\uFE0F "),r=i(d,"STRONG",{});var x=c(r);b=n(x,"\u8BD5\u8BD5\u770B\uFF01"),x.forEach(e),k=n(d," \u9009\u53D6\u8BAD\u7EC3\u96C6\u4E2D\u7684\u7B2C15\u4E2A\u5143\u7D20\uFF0C\u5C06\u4E24\u53E5\u8BDD\u5206\u522B\u6807\u8BB0\u4E3A\u4E00\u5BF9\u3002\u7ED3\u679C\u548C\u4E0A\u65B9\u7684\u4F8B\u5B50\u6709\u4EC0\u4E48\u4E0D\u540C\uFF1F"),d.forEach(e)},m(h,d){f(h,l,d),t(l,u),t(l,r),t(r,b),t(l,k)},d(h){h&&e(l)}}}function wi(v){let l,u,r,b,k;return{c(){l=p("p"),u=a("\u8D1F\u8D23\u5728\u6279\u5904\u7406\u4E2D\u5C06\u6570\u636E\u6574\u7406\u4E3A\u4E00\u4E2Abatch\u7684\u51FD\u6570\u79F0\u4E3A"),r=p("em"),b=a("collate\u51FD\u6570"),k=a("\u3002\u5B83\u53EA\u4F1A\u5C06\u60A8\u7684\u6837\u672C\u8F6C\u6362\u4E3A tf.Tensor\u5E76\u5C06\u5B83\u4EEC\u62FC\u63A5\u8D77\u6765(\u5982\u679C\u4F60\u7684\u5143\u7D20\u662F\u5217\u8868\u3001\u5143\u7EC4\u6216\u5B57\u5178\uFF0C\u5219\u4F1A\u4F7F\u7528\u9012\u5F52)\u3002\u8FD9\u5728\u6211\u4EEC\u7684\u8FD9\u4E2A\u4F8B\u5B50\u4E2D\u4E0B\u662F\u4E0D\u53EF\u884C\u7684\uFF0C\u56E0\u4E3A\u6211\u4EEC\u7684\u8F93\u5165\u4E0D\u662F\u90FD\u662F\u76F8\u540C\u5927\u5C0F\u7684\u3002\u6211\u4EEC\u6545\u610F\u5728\u4E4B\u540E\u6BCF\u4E2Abatch\u4E0A\u8FDB\u884C\u586B\u5145\uFF0C\u907F\u514D\u6709\u592A\u591A\u586B\u5145\u7684\u8FC7\u957F\u7684\u8F93\u5165\u3002\u8FD9\u5C06\u5927\u5927\u52A0\u5FEB\u8BAD\u7EC3\u901F\u5EA6\uFF0C\u4F46\u8BF7\u6CE8\u610F\uFF0C\u5982\u679C\u4F60\u5728TPU\u4E0A\u8BAD\u7EC3\uFF0C\u8FD9\u53EF\u80FD\u4F1A\u5BFC\u81F4\u95EE\u9898\u2014\u2014TPU\u559C\u6B22\u56FA\u5B9A\u7684\u5F62\u72B6\uFF0C\u5373\u4F7F\u8FD9\u9700\u8981\u989D\u5916\u7684\u586B\u5145\u3002")},l(h){l=i(h,"P",{});var d=c(l);u=n(d,"\u8D1F\u8D23\u5728\u6279\u5904\u7406\u4E2D\u5C06\u6570\u636E\u6574\u7406\u4E3A\u4E00\u4E2Abatch\u7684\u51FD\u6570\u79F0\u4E3A"),r=i(d,"EM",{});var x=c(r);b=n(x,"collate\u51FD\u6570"),x.forEach(e),k=n(d,"\u3002\u5B83\u53EA\u4F1A\u5C06\u60A8\u7684\u6837\u672C\u8F6C\u6362\u4E3A tf.Tensor\u5E76\u5C06\u5B83\u4EEC\u62FC\u63A5\u8D77\u6765(\u5982\u679C\u4F60\u7684\u5143\u7D20\u662F\u5217\u8868\u3001\u5143\u7EC4\u6216\u5B57\u5178\uFF0C\u5219\u4F1A\u4F7F\u7528\u9012\u5F52)\u3002\u8FD9\u5728\u6211\u4EEC\u7684\u8FD9\u4E2A\u4F8B\u5B50\u4E2D\u4E0B\u662F\u4E0D\u53EF\u884C\u7684\uFF0C\u56E0\u4E3A\u6211\u4EEC\u7684\u8F93\u5165\u4E0D\u662F\u90FD\u662F\u76F8\u540C\u5927\u5C0F\u7684\u3002\u6211\u4EEC\u6545\u610F\u5728\u4E4B\u540E\u6BCF\u4E2Abatch\u4E0A\u8FDB\u884C\u586B\u5145\uFF0C\u907F\u514D\u6709\u592A\u591A\u586B\u5145\u7684\u8FC7\u957F\u7684\u8F93\u5165\u3002\u8FD9\u5C06\u5927\u5927\u52A0\u5FEB\u8BAD\u7EC3\u901F\u5EA6\uFF0C\u4F46\u8BF7\u6CE8\u610F\uFF0C\u5982\u679C\u4F60\u5728TPU\u4E0A\u8BAD\u7EC3\uFF0C\u8FD9\u53EF\u80FD\u4F1A\u5BFC\u81F4\u95EE\u9898\u2014\u2014TPU\u559C\u6B22\u56FA\u5B9A\u7684\u5F62\u72B6\uFF0C\u5373\u4F7F\u8FD9\u9700\u8981\u989D\u5916\u7684\u586B\u5145\u3002"),d.forEach(e)},m(h,d){f(h,l,d),t(l,u),t(l,r),t(r,b),t(l,k)},d(h){h&&e(l)}}}function Ei(v){let l,u,r,b,k,h,d,x;return{c(){l=p("p"),u=a("\u8D1F\u8D23\u5728\u6279\u5904\u7406\u4E2D\u5C06\u6570\u636E\u6574\u7406\u4E3A\u4E00\u4E2Abatch\u7684\u51FD\u6570\u79F0\u4E3A"),r=p("em"),b=a("collate\u51FD\u6570"),k=a("\u3002\u5B83\u662F\u4F60\u53EF\u4EE5\u5728\u6784\u5EFA"),h=p("strong"),d=a("DataLoader"),x=a("\u65F6\u4F20\u9012\u7684\u4E00\u4E2A\u53C2\u6570\uFF0C\u9ED8\u8BA4\u662F\u4E00\u4E2A\u51FD\u6570\uFF0C\u5B83\u5C06\u628A\u4F60\u7684\u6570\u636E\u96C6\u8F6C\u6362\u4E3APyTorch\u5F20\u91CF\uFF0C\u5E76\u5C06\u5B83\u4EEC\u62FC\u63A5\u8D77\u6765(\u5982\u679C\u4F60\u7684\u5143\u7D20\u662F\u5217\u8868\u3001\u5143\u7EC4\u6216\u5B57\u5178\uFF0C\u5219\u4F1A\u4F7F\u7528\u9012\u5F52)\u3002\u8FD9\u5728\u6211\u4EEC\u7684\u8FD9\u4E2A\u4F8B\u5B50\u4E2D\u4E0B\u662F\u4E0D\u53EF\u884C\u7684\uFF0C\u56E0\u4E3A\u6211\u4EEC\u7684\u8F93\u5165\u4E0D\u662F\u90FD\u662F\u76F8\u540C\u5927\u5C0F\u7684\u3002\u6211\u4EEC\u6545\u610F\u5728\u4E4B\u540E\u6BCF\u4E2Abatch\u4E0A\u8FDB\u884C\u586B\u5145\uFF0C\u907F\u514D\u6709\u592A\u591A\u586B\u5145\u7684\u8FC7\u957F\u7684\u8F93\u5165\u3002\u8FD9\u5C06\u5927\u5927\u52A0\u5FEB\u8BAD\u7EC3\u901F\u5EA6\uFF0C\u4F46\u8BF7\u6CE8\u610F\uFF0C\u5982\u679C\u4F60\u5728TPU\u4E0A\u8BAD\u7EC3\uFF0C\u8FD9\u53EF\u80FD\u4F1A\u5BFC\u81F4\u95EE\u9898\u2014\u2014TPU\u559C\u6B22\u56FA\u5B9A\u7684\u5F62\u72B6\uFF0C\u5373\u4F7F\u8FD9\u9700\u8981\u989D\u5916\u7684\u586B\u5145\u3002")},l(g){l=i(g,"P",{});var q=c(l);u=n(q,"\u8D1F\u8D23\u5728\u6279\u5904\u7406\u4E2D\u5C06\u6570\u636E\u6574\u7406\u4E3A\u4E00\u4E2Abatch\u7684\u51FD\u6570\u79F0\u4E3A"),r=i(q,"EM",{});var N=c(r);b=n(N,"collate\u51FD\u6570"),N.forEach(e),k=n(q,"\u3002\u5B83\u662F\u4F60\u53EF\u4EE5\u5728\u6784\u5EFA"),h=i(q,"STRONG",{});var G=c(h);d=n(G,"DataLoader"),G.forEach(e),x=n(q,"\u65F6\u4F20\u9012\u7684\u4E00\u4E2A\u53C2\u6570\uFF0C\u9ED8\u8BA4\u662F\u4E00\u4E2A\u51FD\u6570\uFF0C\u5B83\u5C06\u628A\u4F60\u7684\u6570\u636E\u96C6\u8F6C\u6362\u4E3APyTorch\u5F20\u91CF\uFF0C\u5E76\u5C06\u5B83\u4EEC\u62FC\u63A5\u8D77\u6765(\u5982\u679C\u4F60\u7684\u5143\u7D20\u662F\u5217\u8868\u3001\u5143\u7EC4\u6216\u5B57\u5178\uFF0C\u5219\u4F1A\u4F7F\u7528\u9012\u5F52)\u3002\u8FD9\u5728\u6211\u4EEC\u7684\u8FD9\u4E2A\u4F8B\u5B50\u4E2D\u4E0B\u662F\u4E0D\u53EF\u884C\u7684\uFF0C\u56E0\u4E3A\u6211\u4EEC\u7684\u8F93\u5165\u4E0D\u662F\u90FD\u662F\u76F8\u540C\u5927\u5C0F\u7684\u3002\u6211\u4EEC\u6545\u610F\u5728\u4E4B\u540E\u6BCF\u4E2Abatch\u4E0A\u8FDB\u884C\u586B\u5145\uFF0C\u907F\u514D\u6709\u592A\u591A\u586B\u5145\u7684\u8FC7\u957F\u7684\u8F93\u5165\u3002\u8FD9\u5C06\u5927\u5927\u52A0\u5FEB\u8BAD\u7EC3\u901F\u5EA6\uFF0C\u4F46\u8BF7\u6CE8\u610F\uFF0C\u5982\u679C\u4F60\u5728TPU\u4E0A\u8BAD\u7EC3\uFF0C\u8FD9\u53EF\u80FD\u4F1A\u5BFC\u81F4\u95EE\u9898\u2014\u2014TPU\u559C\u6B22\u56FA\u5B9A\u7684\u5F62\u72B6\uFF0C\u5373\u4F7F\u8FD9\u9700\u8981\u989D\u5916\u7684\u586B\u5145\u3002"),q.forEach(e)},m(g,q){f(g,l,q),t(l,u),t(l,r),t(r,b),t(l,k),t(l,h),t(h,d),t(l,x)},d(g){g&&e(l)}}}function Ti(v){let l,u;return l=new z({props:{code:`from transformers import DataCollatorWithPadding

data_collator = DataCollatorWithPadding(tokenizer=tokenizer, return_tensors="tf")`,highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> DataCollatorWithPadding

data_collator = DataCollatorWithPadding(tokenizer=tokenizer, return_tensors=<span class="hljs-string">&quot;tf&quot;</span>)`}}),{c(){w(l.$$.fragment)},l(r){E(l.$$.fragment,r)},m(r,b){T(l,r,b),u=!0},i(r){u||($(l.$$.fragment,r),u=!0)},o(r){j(l.$$.fragment,r),u=!1},d(r){S(l,r)}}}function Si(v){let l,u;return l=new z({props:{code:`from transformers import DataCollatorWithPadding

data_collator = DataCollatorWithPadding(tokenizer=tokenizer)`,highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> DataCollatorWithPadding

data_collator = DataCollatorWithPadding(tokenizer=tokenizer)`}}),{c(){w(l.$$.fragment)},l(r){E(l.$$.fragment,r)},m(r,b){T(l,r,b),u=!0},i(r){u||($(l.$$.fragment,r),u=!0)},o(r){j(l.$$.fragment,r),u=!1},d(r){S(l,r)}}}function qi(v){let l,u,r,b,k;return l=new z({props:{code:`{'attention_mask': torch.Size([8, 67]),
 'input_ids': torch.Size([8, 67]),
 'token_type_ids': torch.Size([8, 67]),
 'labels': torch.Size([8])}`,highlighted:`{<span class="hljs-string">&#x27;attention_mask&#x27;</span>: torch.Size([<span class="hljs-number">8</span>, <span class="hljs-number">67</span>]),
 <span class="hljs-string">&#x27;input_ids&#x27;</span>: torch.Size([<span class="hljs-number">8</span>, <span class="hljs-number">67</span>]),
 <span class="hljs-string">&#x27;token_type_ids&#x27;</span>: torch.Size([<span class="hljs-number">8</span>, <span class="hljs-number">67</span>]),
 <span class="hljs-string">&#x27;labels&#x27;</span>: torch.Size([<span class="hljs-number">8</span>])}`}}),{c(){w(l.$$.fragment),u=_(),r=p("p"),b=a("\u770B\u8D77\u6765\u4E0D\u9519\uFF01\u73B0\u5728\uFF0C\u6211\u4EEC\u5DF2\u7ECF\u5C06\u539F\u59CB\u6587\u672C\u8F6C\u5316\u4E3A\u4E86\u6A21\u578B\u53EF\u4EE5\u5904\u7406\u7684\u6570\u636E\uFF0C\u6211\u4EEC\u5DF2\u51C6\u5907\u597D\u5BF9\u5176\u8FDB\u884C\u5FAE\u8C03\uFF01")},l(h){E(l.$$.fragment,h),u=m(h),r=i(h,"P",{});var d=c(r);b=n(d,"\u770B\u8D77\u6765\u4E0D\u9519\uFF01\u73B0\u5728\uFF0C\u6211\u4EEC\u5DF2\u7ECF\u5C06\u539F\u59CB\u6587\u672C\u8F6C\u5316\u4E3A\u4E86\u6A21\u578B\u53EF\u4EE5\u5904\u7406\u7684\u6570\u636E\uFF0C\u6211\u4EEC\u5DF2\u51C6\u5907\u597D\u5BF9\u5176\u8FDB\u884C\u5FAE\u8C03\uFF01"),d.forEach(e)},m(h,d){T(l,h,d),f(h,u,d),f(h,r,d),t(r,b),k=!0},i(h){k||($(l.$$.fragment,h),k=!0)},o(h){j(l.$$.fragment,h),k=!1},d(h){S(l,h),h&&e(u),h&&e(r)}}}function yi(v){let l,u;return l=new z({props:{code:`{'attention_mask': TensorShape([8, 67]),
 'input_ids': TensorShape([8, 67]),
 'token_type_ids': TensorShape([8, 67]),
 'labels': TensorShape([8])}`,highlighted:`{<span class="hljs-string">&#x27;attention_mask&#x27;</span>: TensorShape([<span class="hljs-number">8</span>, <span class="hljs-number">67</span>]),
 <span class="hljs-string">&#x27;input_ids&#x27;</span>: TensorShape([<span class="hljs-number">8</span>, <span class="hljs-number">67</span>]),
 <span class="hljs-string">&#x27;token_type_ids&#x27;</span>: TensorShape([<span class="hljs-number">8</span>, <span class="hljs-number">67</span>]),
 <span class="hljs-string">&#x27;labels&#x27;</span>: TensorShape([<span class="hljs-number">8</span>])}`}}),{c(){w(l.$$.fragment)},l(r){E(l.$$.fragment,r)},m(r,b){T(l,r,b),u=!0},i(r){u||($(l.$$.fragment,r),u=!0)},o(r){j(l.$$.fragment,r),u=!1},d(r){S(l,r)}}}function zi(v){let l,u,r,b,k;return{c(){l=p("p"),u=a("\u270F\uFE0F "),r=p("strong"),b=a("\u8BD5\u8BD5\u770B\uFF01"),k=a(" \u5728GLUE SST-2\u6570\u636E\u96C6\u4E0A\u5E94\u7528\u9884\u5904\u7406\u3002\u5B83\u6709\u70B9\u4E0D\u540C\uFF0C\u56E0\u4E3A\u5B83\u662F\u7531\u5355\u4E2A\u53E5\u5B50\u800C\u4E0D\u662F\u6210\u5BF9\u7684\u53E5\u5B50\u7EC4\u6210\u7684\uFF0C\u4F46\u662F\u6211\u4EEC\u6240\u505A\u7684\u5176\u4ED6\u4E8B\u60C5\u770B\u8D77\u6765\u5E94\u8BE5\u662F\u4E00\u6837\u7684\u3002\u53E6\u4E00\u4E2A\u66F4\u96BE\u7684\u6311\u6218\uFF0C\u8BF7\u5C1D\u8BD5\u7F16\u5199\u4E00\u4E2A\u53EF\u7528\u4E8E\u4EFB\u4F55GLUE\u4EFB\u52A1\u7684\u9884\u5904\u7406\u51FD\u6570\u3002")},l(h){l=i(h,"P",{});var d=c(l);u=n(d,"\u270F\uFE0F "),r=i(d,"STRONG",{});var x=c(r);b=n(x,"\u8BD5\u8BD5\u770B\uFF01"),x.forEach(e),k=n(d," \u5728GLUE SST-2\u6570\u636E\u96C6\u4E0A\u5E94\u7528\u9884\u5904\u7406\u3002\u5B83\u6709\u70B9\u4E0D\u540C\uFF0C\u56E0\u4E3A\u5B83\u662F\u7531\u5355\u4E2A\u53E5\u5B50\u800C\u4E0D\u662F\u6210\u5BF9\u7684\u53E5\u5B50\u7EC4\u6210\u7684\uFF0C\u4F46\u662F\u6211\u4EEC\u6240\u505A\u7684\u5176\u4ED6\u4E8B\u60C5\u770B\u8D77\u6765\u5E94\u8BE5\u662F\u4E00\u6837\u7684\u3002\u53E6\u4E00\u4E2A\u66F4\u96BE\u7684\u6311\u6218\uFF0C\u8BF7\u5C1D\u8BD5\u7F16\u5199\u4E00\u4E2A\u53EF\u7528\u4E8E\u4EFB\u4F55GLUE\u4EFB\u52A1\u7684\u9884\u5904\u7406\u51FD\u6570\u3002"),d.forEach(e)},m(h,d){f(h,l,d),t(l,u),t(l,r),t(r,b),t(l,k)},d(h){h&&e(l)}}}function li(v){let l,u,r,b,k,h,d,x,g,q,N,G,fs,us,vs,B,U,_s,ts,ws;return B=new z({props:{code:`tf_train_dataset = tokenized_datasets["train"].to_tf_dataset(
    columns=["attention_mask", "input_ids", "token_type_ids"],
    label_cols=["labels"],
    shuffle=True,
    collate_fn=data_collator,
    batch_size=8,
)

tf_validation_dataset = tokenized_datasets["validation"].to_tf_dataset(
    columns=["attention_mask", "input_ids", "token_type_ids"],
    label_cols=["labels"],
    shuffle=False,
    collate_fn=data_collator,
    batch_size=8,
)`,highlighted:`tf_train_dataset = tokenized_datasets[<span class="hljs-string">&quot;train&quot;</span>].to_tf_dataset(
    columns=[<span class="hljs-string">&quot;attention_mask&quot;</span>, <span class="hljs-string">&quot;input_ids&quot;</span>, <span class="hljs-string">&quot;token_type_ids&quot;</span>],
    label_cols=[<span class="hljs-string">&quot;labels&quot;</span>],
    shuffle=<span class="hljs-literal">True</span>,
    collate_fn=data_collator,
    batch_size=<span class="hljs-number">8</span>,
)

tf_validation_dataset = tokenized_datasets[<span class="hljs-string">&quot;validation&quot;</span>].to_tf_dataset(
    columns=[<span class="hljs-string">&quot;attention_mask&quot;</span>, <span class="hljs-string">&quot;input_ids&quot;</span>, <span class="hljs-string">&quot;token_type_ids&quot;</span>],
    label_cols=[<span class="hljs-string">&quot;labels&quot;</span>],
    shuffle=<span class="hljs-literal">False</span>,
    collate_fn=data_collator,
    batch_size=<span class="hljs-number">8</span>,
)`}}),{c(){l=p("p"),u=a("\u73B0\u5728\u6211\u4EEC\u6709\u4E86dataset\u548Cdata collator\uFF0C\u6211\u4EEC\u9700\u8981\u5C06dataset\u6279\u91CF\u5730\u5E94\u7528data collator\u3002 \u6211\u4EEC\u53EF\u4EE5\u624B\u52A8\u52A0\u8F7D\u6279\u6B21\u5E76\u6574\u7406\u5B83\u4EEC\uFF0C\u4F46\u8FD9\u9700\u8981\u5927\u91CF\u5DE5\u4F5C\uFF0C\u800C\u4E14\u53EF\u80FD\u6027\u80FD\u4E5F\u4E0D\u662F\u5F88\u597D\u3002 \u76F8\u53CD\uFF0C\u6709\u4E00\u4E2A\u7B80\u5355\u7684\u65B9\u6CD5\u53EF\u4EE5\u4E3A\u8FD9\u4E2A\u95EE\u9898\u63D0\u4F9B\u9AD8\u6548\u7684\u89E3\u51B3\u65B9\u6848\uFF1A"),r=p("code"),b=a("to_tf_dataset()"),k=a("\u3002 \u8FD9\u5C06\u5728\u60A8\u7684\u6570\u636E\u96C6\u4E0A\u8C03\u7528\u4E00\u4E2A "),h=p("code"),d=a("tf.data.Dataset"),x=a("\u7684\u65B9\u6CD5\uFF0C\u8FD9\u4E2A\u65B9\u6CD5\u5E26\u6709\u4E00\u4E2A\u53EF\u9009\u7684data collator\u529F\u80FD\u3002 "),g=p("code"),q=a("tf.data.Dataset"),N=a(" \u662F Keras \u53EF\u7528\u4E8E "),G=p("code"),fs=a("model.fit()"),us=a(" \u7684\u539F\u751F TensorFlow \u683C\u5F0F\uFF0C\u56E0\u6B64\u8FD9\u79CD\u65B9\u6CD5\u4F1A\u7ACB\u5373\u5C06\u{1F917} Dataset \u8F6C\u6362\u4E3A\u53EF\u7528\u4E8E\u8BAD\u7EC3\u7684\u683C\u5F0F\u3002 \u8BA9\u6211\u4EEC\u770B\u770B\u5B83\u5728\u6211\u4EEC\u7684\u6570\u636E\u96C6\u4E0A\u662F\u5982\u4F55\u4F7F\u7528\u7684\uFF01"),vs=_(),w(B.$$.fragment),U=_(),_s=p("p"),ts=a("\u5C31\u662F\u8FD9\u6837\uFF01 \u6211\u4EEC\u53EF\u4EE5\u5C06\u8FD9\u4E9B\u6570\u636E\u96C6\u5E26\u5165\u4E0B\u4E00\u8282\uFF0C\u5728\u7ECF\u8FC7\u6240\u6709\u8270\u82E6\u7684\u6570\u636E\u9884\u5904\u7406\u5DE5\u4F5C\u4E4B\u540E\uFF0C\u8BAD\u7EC3\u5C06\u53D8\u5F97\u975E\u5E38\u7B80\u5355\u3002")},l(P){l=i(P,"P",{});var R=c(l);u=n(R,"\u73B0\u5728\u6211\u4EEC\u6709\u4E86dataset\u548Cdata collator\uFF0C\u6211\u4EEC\u9700\u8981\u5C06dataset\u6279\u91CF\u5730\u5E94\u7528data collator\u3002 \u6211\u4EEC\u53EF\u4EE5\u624B\u52A8\u52A0\u8F7D\u6279\u6B21\u5E76\u6574\u7406\u5B83\u4EEC\uFF0C\u4F46\u8FD9\u9700\u8981\u5927\u91CF\u5DE5\u4F5C\uFF0C\u800C\u4E14\u53EF\u80FD\u6027\u80FD\u4E5F\u4E0D\u662F\u5F88\u597D\u3002 \u76F8\u53CD\uFF0C\u6709\u4E00\u4E2A\u7B80\u5355\u7684\u65B9\u6CD5\u53EF\u4EE5\u4E3A\u8FD9\u4E2A\u95EE\u9898\u63D0\u4F9B\u9AD8\u6548\u7684\u89E3\u51B3\u65B9\u6848\uFF1A"),r=i(R,"CODE",{});var V=c(r);b=n(V,"to_tf_dataset()"),V.forEach(e),k=n(R,"\u3002 \u8FD9\u5C06\u5728\u60A8\u7684\u6570\u636E\u96C6\u4E0A\u8C03\u7528\u4E00\u4E2A "),h=i(R,"CODE",{});var es=c(h);d=n(es,"tf.data.Dataset"),es.forEach(e),x=n(R,"\u7684\u65B9\u6CD5\uFF0C\u8FD9\u4E2A\u65B9\u6CD5\u5E26\u6709\u4E00\u4E2A\u53EF\u9009\u7684data collator\u529F\u80FD\u3002 "),g=i(R,"CODE",{});var Es=c(g);q=n(Es,"tf.data.Dataset"),Es.forEach(e),N=n(R," \u662F Keras \u53EF\u7528\u4E8E "),G=i(R,"CODE",{});var ms=c(G);fs=n(ms,"model.fit()"),ms.forEach(e),us=n(R," \u7684\u539F\u751F TensorFlow \u683C\u5F0F\uFF0C\u56E0\u6B64\u8FD9\u79CD\u65B9\u6CD5\u4F1A\u7ACB\u5373\u5C06\u{1F917} Dataset \u8F6C\u6362\u4E3A\u53EF\u7528\u4E8E\u8BAD\u7EC3\u7684\u683C\u5F0F\u3002 \u8BA9\u6211\u4EEC\u770B\u770B\u5B83\u5728\u6211\u4EEC\u7684\u6570\u636E\u96C6\u4E0A\u662F\u5982\u4F55\u4F7F\u7528\u7684\uFF01"),R.forEach(e),vs=m(P),E(B.$$.fragment,P),U=m(P),_s=i(P,"P",{});var Et=c(_s);ts=n(Et,"\u5C31\u662F\u8FD9\u6837\uFF01 \u6211\u4EEC\u53EF\u4EE5\u5C06\u8FD9\u4E9B\u6570\u636E\u96C6\u5E26\u5165\u4E0B\u4E00\u8282\uFF0C\u5728\u7ECF\u8FC7\u6240\u6709\u8270\u82E6\u7684\u6570\u636E\u9884\u5904\u7406\u5DE5\u4F5C\u4E4B\u540E\uFF0C\u8BAD\u7EC3\u5C06\u53D8\u5F97\u975E\u5E38\u7B80\u5355\u3002"),Et.forEach(e)},m(P,R){f(P,l,R),t(l,u),t(l,r),t(r,b),t(l,k),t(l,h),t(h,d),t(l,x),t(l,g),t(g,q),t(l,N),t(l,G),t(G,fs),t(l,us),f(P,vs,R),T(B,P,R),f(P,U,R),f(P,_s,R),t(_s,ts),ws=!0},i(P){ws||($(B.$$.fragment,P),ws=!0)},o(P){j(B.$$.fragment,P),ws=!1},d(P){P&&e(l),P&&e(vs),S(B,P),P&&e(U),P&&e(_s)}}}function Pi(v){let l,u,r,b,k,h,d,x,g,q,N,G,fs,us,vs,B,U,_s,ts,ws,P,R,V,es,Es,ms,Et,Qt,jn,na,as,ns,Tt,J,kn,Ms,xn,vn,Ws,wn,En,Hs,Tn,Sn,la,St,qn,ra,Us,oa,Bs,pa,I,yn,Xt,zn,Pn,se,Nn,Dn,te,Rn,Gn,ee,On,An,ae,Cn,In,ia,hs,Fn,ne,Ln,Mn,le,Wn,Hn,ca,Ts,Un,re,Bn,Vn,fa,Vs,ua,Js,_a,ds,Jn,oe,Kn,Yn,pe,Zn,Qn,ma,Ks,ha,Ys,da,A,Xn,ie,sl,tl,ce,el,al,fe,nl,ll,ue,rl,ol,_e,pl,il,me,cl,fl,ba,Ss,ga,Zs,ul,$a,ls,rs,qt,qs,_l,yt,ml,hl,ja,Qs,ka,zt,dl,xa,Xs,va,st,wa,F,bl,Pt,gl,$l,he,jl,kl,de,xl,vl,be,wl,El,ge,Tl,Sl,Ea,ys,Ta,zs,ql,$e,yl,zl,Sa,tt,qa,Nt,Pl,ya,et,za,bs,Nl,je,Dl,Rl,ke,Gl,Ol,Pa,at,Na,M,Al,xe,Cl,Il,ve,Fl,Ll,we,Ml,Wl,Ee,Hl,Ul,Da,Ps,Bl,Te,Vl,Jl,Ra,Ns,Kl,Dt,Yl,Zl,Ga,Rt,Ql,Oa,Ds,Xl,Se,sr,tr,Aa,gs,er,Gt,ar,nr,Ot,lr,rr,Ca,nt,Ia,W,or,qe,pr,ir,ye,cr,fr,ze,ur,_r,lt,mr,hr,Fa,Rs,dr,rt,br,gr,La,ot,Ma,D,$r,Pe,jr,kr,Ne,xr,vr,De,wr,Er,Re,Tr,Sr,Ge,qr,yr,Oe,zr,Pr,Ae,Nr,Dr,pt,Rr,Gr,Wa,Gs,Or,Ce,Ar,Cr,Ha,$s,Ir,Ie,Fr,Lr,Fe,Mr,Wr,Ua,it,Ba,At,Hr,Va,ct,Ja,K,Ur,Le,Br,Vr,Me,Jr,Kr,We,Yr,Zr,Ka,L,Qr,He,Xr,so,Ue,to,eo,Be,ao,no,Ve,lo,ro,Je,oo,po,Ya,Ct,io,Za,ft,co,Qa,ut,Xa,It,Os,fo,Ke,uo,_o,sn,os,ps,Ft,Y,mo,Ye,ho,bo,Ze,go,$o,Qe,jo,ko,tn,_t,en,mt,an,As,xo,Xe,vo,wo,nn,ht,ln,is,cs,Lt,Cs,rn,Mt,on;r=new _i({props:{fw:v[0]}});const To=[hi,mi],dt=[];function So(s,o){return s[0]==="pt"?0:1}x=So(v),g=dt[x]=To[x](v);const qo=[bi,di],bt=[];function yo(s,o){return s[0]==="pt"?0:1}N=yo(v),G=bt[N]=qo[N](v),ms=new ui({});const zo=[$i,gi],gt=[];function Po(s,o){return s[0]==="pt"?0:1}as=Po(v),ns=gt[as]=zo[as](v),Us=new z({props:{code:`from datasets import load_dataset

raw_datasets = load_dataset("glue", "mrpc")
raw_datasets`,highlighted:`<span class="hljs-keyword">from</span> datasets <span class="hljs-keyword">import</span> load_dataset

raw_datasets = load_dataset(<span class="hljs-string">&quot;glue&quot;</span>, <span class="hljs-string">&quot;mrpc&quot;</span>)
raw_datasets`}}),Bs=new z({props:{code:`DatasetDict({
    train: Dataset({
        features: ['sentence1', 'sentence2', 'label', 'idx'],
        num_rows: 3668
    })
    validation: Dataset({
        features: ['sentence1', 'sentence2', 'label', 'idx'],
        num_rows: 408
    })
    test: Dataset({
        features: ['sentence1', 'sentence2', 'label', 'idx'],
        num_rows: 1725
    })
})`,highlighted:`DatasetDict({
    train: Dataset({
        features: [<span class="hljs-string">&#x27;sentence1&#x27;</span>, <span class="hljs-string">&#x27;sentence2&#x27;</span>, <span class="hljs-string">&#x27;label&#x27;</span>, <span class="hljs-string">&#x27;idx&#x27;</span>],
        num_rows: <span class="hljs-number">3668</span>
    })
    validation: Dataset({
        features: [<span class="hljs-string">&#x27;sentence1&#x27;</span>, <span class="hljs-string">&#x27;sentence2&#x27;</span>, <span class="hljs-string">&#x27;label&#x27;</span>, <span class="hljs-string">&#x27;idx&#x27;</span>],
        num_rows: <span class="hljs-number">408</span>
    })
    test: Dataset({
        features: [<span class="hljs-string">&#x27;sentence1&#x27;</span>, <span class="hljs-string">&#x27;sentence2&#x27;</span>, <span class="hljs-string">&#x27;label&#x27;</span>, <span class="hljs-string">&#x27;idx&#x27;</span>],
        num_rows: <span class="hljs-number">1725</span>
    })
})`}}),Vs=new z({props:{code:`raw_train_dataset = raw_datasets["train"]
raw_train_dataset[0]`,highlighted:`raw_train_dataset = raw_datasets[<span class="hljs-string">&quot;train&quot;</span>]
raw_train_dataset[<span class="hljs-number">0</span>]`}}),Js=new z({props:{code:`{'idx': 0,
 'label': 1,
 'sentence1': 'Amrozi accused his brother , whom he called " the witness " , of deliberately distorting his evidence .',
 'sentence2': 'Referring to him as only " the witness " , Amrozi accused his brother of deliberately distorting his evidence .'}`,highlighted:`{<span class="hljs-string">&#x27;idx&#x27;</span>: <span class="hljs-number">0</span>,
 <span class="hljs-string">&#x27;label&#x27;</span>: <span class="hljs-number">1</span>,
 <span class="hljs-string">&#x27;sentence1&#x27;</span>: <span class="hljs-string">&#x27;Amrozi accused his brother , whom he called &quot; the witness &quot; , of deliberately distorting his evidence .&#x27;</span>,
 <span class="hljs-string">&#x27;sentence2&#x27;</span>: <span class="hljs-string">&#x27;Referring to him as only &quot; the witness &quot; , Amrozi accused his brother of deliberately distorting his evidence .&#x27;</span>}`}}),Ks=new z({props:{code:"raw_train_dataset.features",highlighted:"raw_train_dataset.features"}}),Ys=new z({props:{code:`{'sentence1': Value(dtype='string', id=None),
 'sentence2': Value(dtype='string', id=None),
 'label': ClassLabel(num_classes=2, names=['not_equivalent', 'equivalent'], names_file=None, id=None),
 'idx': Value(dtype='int32', id=None)}`,highlighted:`{<span class="hljs-string">&#x27;sentence1&#x27;</span>: Value(dtype=<span class="hljs-string">&#x27;string&#x27;</span>, <span class="hljs-built_in">id</span>=<span class="hljs-literal">None</span>),
 <span class="hljs-string">&#x27;sentence2&#x27;</span>: Value(dtype=<span class="hljs-string">&#x27;string&#x27;</span>, <span class="hljs-built_in">id</span>=<span class="hljs-literal">None</span>),
 <span class="hljs-string">&#x27;label&#x27;</span>: ClassLabel(num_classes=<span class="hljs-number">2</span>, names=[<span class="hljs-string">&#x27;not_equivalent&#x27;</span>, <span class="hljs-string">&#x27;equivalent&#x27;</span>], names_file=<span class="hljs-literal">None</span>, <span class="hljs-built_in">id</span>=<span class="hljs-literal">None</span>),
 <span class="hljs-string">&#x27;idx&#x27;</span>: Value(dtype=<span class="hljs-string">&#x27;int32&#x27;</span>, <span class="hljs-built_in">id</span>=<span class="hljs-literal">None</span>)}`}}),Ss=new Eo({props:{$$slots:{default:[ji]},$$scope:{ctx:v}}});const No=[xi,ki],$t=[];function Do(s,o){return s[0]==="pt"?0:1}ls=Do(v),rs=$t[ls]=No[ls](v),Qs=new z({props:{code:`from transformers import AutoTokenizer

checkpoint = "bert-base-uncased"
tokenizer = AutoTokenizer.from_pretrained(checkpoint)
tokenized_sentences_1 = tokenizer(raw_datasets["train"]["sentence1"])
tokenized_sentences_2 = tokenizer(raw_datasets["train"]["sentence2"])`,highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoTokenizer

checkpoint = <span class="hljs-string">&quot;bert-base-uncased&quot;</span>
tokenizer = AutoTokenizer.from_pretrained(checkpoint)
tokenized_sentences_1 = tokenizer(raw_datasets[<span class="hljs-string">&quot;train&quot;</span>][<span class="hljs-string">&quot;sentence1&quot;</span>])
tokenized_sentences_2 = tokenizer(raw_datasets[<span class="hljs-string">&quot;train&quot;</span>][<span class="hljs-string">&quot;sentence2&quot;</span>])`}}),Xs=new z({props:{code:`inputs = tokenizer("This is the first sentence.", "This is the second one.")
inputs`,highlighted:`inputs = tokenizer(<span class="hljs-string">&quot;This is the first sentence.&quot;</span>, <span class="hljs-string">&quot;This is the second one.&quot;</span>)
inputs`}}),st=new z({props:{code:`{ 
  'input_ids': [101, 2023, 2003, 1996, 2034, 6251, 1012, 102, 2023, 2003, 1996, 2117, 2028, 1012, 102],
  'token_type_ids': [0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1],
  'attention_mask': [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]
}`,highlighted:`{ 
  <span class="hljs-string">&#x27;input_ids&#x27;</span>: [<span class="hljs-number">101</span>, <span class="hljs-number">2023</span>, <span class="hljs-number">2003</span>, <span class="hljs-number">1996</span>, <span class="hljs-number">2034</span>, <span class="hljs-number">6251</span>, <span class="hljs-number">1012</span>, <span class="hljs-number">102</span>, <span class="hljs-number">2023</span>, <span class="hljs-number">2003</span>, <span class="hljs-number">1996</span>, <span class="hljs-number">2117</span>, <span class="hljs-number">2028</span>, <span class="hljs-number">1012</span>, <span class="hljs-number">102</span>],
  <span class="hljs-string">&#x27;token_type_ids&#x27;</span>: [<span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>],
  <span class="hljs-string">&#x27;attention_mask&#x27;</span>: [<span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>]
}`}}),ys=new Eo({props:{$$slots:{default:[vi]},$$scope:{ctx:v}}}),tt=new z({props:{code:'tokenizer.convert_ids_to_tokens(inputs["input_ids"])',highlighted:'tokenizer.convert_ids_to_tokens(inputs[<span class="hljs-string">&quot;input_ids&quot;</span>])'}}),et=new z({props:{code:"['[CLS]', 'this', 'is', 'the', 'first', 'sentence', '.', '[SEP]', 'this', 'is', 'the', 'second', 'one', '.', '[SEP]']",highlighted:'[<span class="hljs-string">&#x27;[CLS]&#x27;</span>, <span class="hljs-string">&#x27;this&#x27;</span>, <span class="hljs-string">&#x27;is&#x27;</span>, <span class="hljs-string">&#x27;the&#x27;</span>, <span class="hljs-string">&#x27;first&#x27;</span>, <span class="hljs-string">&#x27;sentence&#x27;</span>, <span class="hljs-string">&#x27;.&#x27;</span>, <span class="hljs-string">&#x27;[SEP]&#x27;</span>, <span class="hljs-string">&#x27;this&#x27;</span>, <span class="hljs-string">&#x27;is&#x27;</span>, <span class="hljs-string">&#x27;the&#x27;</span>, <span class="hljs-string">&#x27;second&#x27;</span>, <span class="hljs-string">&#x27;one&#x27;</span>, <span class="hljs-string">&#x27;.&#x27;</span>, <span class="hljs-string">&#x27;[SEP]&#x27;</span>]'}}),at=new z({props:{code:`['[CLS]', 'this', 'is', 'the', 'first', 'sentence', '.', '[SEP]', 'this', 'is', 'the', 'second', 'one', '.', '[SEP]']
[      0,      0,    0,     0,       0,          0,   0,       0,      1,    1,     1,        1,     1,   1,       1]`,highlighted:`[<span class="hljs-string">&#x27;[CLS]&#x27;</span>, <span class="hljs-string">&#x27;this&#x27;</span>, <span class="hljs-string">&#x27;is&#x27;</span>, <span class="hljs-string">&#x27;the&#x27;</span>, <span class="hljs-string">&#x27;first&#x27;</span>, <span class="hljs-string">&#x27;sentence&#x27;</span>, <span class="hljs-string">&#x27;.&#x27;</span>, <span class="hljs-string">&#x27;[SEP]&#x27;</span>, <span class="hljs-string">&#x27;this&#x27;</span>, <span class="hljs-string">&#x27;is&#x27;</span>, <span class="hljs-string">&#x27;the&#x27;</span>, <span class="hljs-string">&#x27;second&#x27;</span>, <span class="hljs-string">&#x27;one&#x27;</span>, <span class="hljs-string">&#x27;.&#x27;</span>, <span class="hljs-string">&#x27;[SEP]&#x27;</span>]
[      <span class="hljs-number">0</span>,      <span class="hljs-number">0</span>,    <span class="hljs-number">0</span>,     <span class="hljs-number">0</span>,       <span class="hljs-number">0</span>,          <span class="hljs-number">0</span>,   <span class="hljs-number">0</span>,       <span class="hljs-number">0</span>,      <span class="hljs-number">1</span>,    <span class="hljs-number">1</span>,     <span class="hljs-number">1</span>,        <span class="hljs-number">1</span>,     <span class="hljs-number">1</span>,   <span class="hljs-number">1</span>,       <span class="hljs-number">1</span>]`}}),nt=new z({props:{code:`tokenized_dataset = tokenizer(
    raw_datasets["train"]["sentence1"],
    raw_datasets["train"]["sentence2"],
    padding=True,
    truncation=True,
)`,highlighted:`tokenized_dataset = tokenizer(
    raw_datasets[<span class="hljs-string">&quot;train&quot;</span>][<span class="hljs-string">&quot;sentence1&quot;</span>],
    raw_datasets[<span class="hljs-string">&quot;train&quot;</span>][<span class="hljs-string">&quot;sentence2&quot;</span>],
    padding=<span class="hljs-literal">True</span>,
    truncation=<span class="hljs-literal">True</span>,
)`}}),ot=new z({props:{code:`def tokenize_function(example):
    return tokenizer(example["sentence1"], example["sentence2"], truncation=True)`,highlighted:`<span class="hljs-keyword">def</span> <span class="hljs-title function_">tokenize_function</span>(<span class="hljs-params">example</span>):
    <span class="hljs-keyword">return</span> tokenizer(example[<span class="hljs-string">&quot;sentence1&quot;</span>], example[<span class="hljs-string">&quot;sentence2&quot;</span>], truncation=<span class="hljs-literal">True</span>)`}}),it=new z({props:{code:`tokenized_datasets = raw_datasets.map(tokenize_function, batched=True)
tokenized_datasets`,highlighted:`tokenized_datasets = raw_datasets.<span class="hljs-built_in">map</span>(tokenize_function, batched=<span class="hljs-literal">True</span>)
tokenized_datasets`}}),ct=new z({props:{code:`DatasetDict({
    train: Dataset({
        features: ['attention_mask', 'idx', 'input_ids', 'label', 'sentence1', 'sentence2', 'token_type_ids'],
        num_rows: 3668
    })
    validation: Dataset({
        features: ['attention_mask', 'idx', 'input_ids', 'label', 'sentence1', 'sentence2', 'token_type_ids'],
        num_rows: 408
    })
    test: Dataset({
        features: ['attention_mask', 'idx', 'input_ids', 'label', 'sentence1', 'sentence2', 'token_type_ids'],
        num_rows: 1725
    })
})`,highlighted:`DatasetDict({
    train: Dataset({
        features: [<span class="hljs-string">&#x27;attention_mask&#x27;</span>, <span class="hljs-string">&#x27;idx&#x27;</span>, <span class="hljs-string">&#x27;input_ids&#x27;</span>, <span class="hljs-string">&#x27;label&#x27;</span>, <span class="hljs-string">&#x27;sentence1&#x27;</span>, <span class="hljs-string">&#x27;sentence2&#x27;</span>, <span class="hljs-string">&#x27;token_type_ids&#x27;</span>],
        num_rows: <span class="hljs-number">3668</span>
    })
    validation: Dataset({
        features: [<span class="hljs-string">&#x27;attention_mask&#x27;</span>, <span class="hljs-string">&#x27;idx&#x27;</span>, <span class="hljs-string">&#x27;input_ids&#x27;</span>, <span class="hljs-string">&#x27;label&#x27;</span>, <span class="hljs-string">&#x27;sentence1&#x27;</span>, <span class="hljs-string">&#x27;sentence2&#x27;</span>, <span class="hljs-string">&#x27;token_type_ids&#x27;</span>],
        num_rows: <span class="hljs-number">408</span>
    })
    test: Dataset({
        features: [<span class="hljs-string">&#x27;attention_mask&#x27;</span>, <span class="hljs-string">&#x27;idx&#x27;</span>, <span class="hljs-string">&#x27;input_ids&#x27;</span>, <span class="hljs-string">&#x27;label&#x27;</span>, <span class="hljs-string">&#x27;sentence1&#x27;</span>, <span class="hljs-string">&#x27;sentence2&#x27;</span>, <span class="hljs-string">&#x27;token_type_ids&#x27;</span>],
        num_rows: <span class="hljs-number">1725</span>
    })
})`}}),ut=new aa({props:{id:"7q5NyFT8REg"}});function Ro(s,o){return s[0]==="pt"?Ei:wi}let pn=Ro(v),xs=pn(v);const Go=[Si,Ti],jt=[];function Oo(s,o){return s[0]==="pt"?0:1}os=Oo(v),ps=jt[os]=Go[os](v),_t=new z({props:{code:`samples = tokenized_datasets["train"][:8]
samples = {k: v for k, v in samples.items() if k not in ["idx", "sentence1", "sentence2"]}
[len(x) for x in samples["input_ids"]]`,highlighted:`samples = tokenized_datasets[<span class="hljs-string">&quot;train&quot;</span>][:<span class="hljs-number">8</span>]
samples = {k: v <span class="hljs-keyword">for</span> k, v <span class="hljs-keyword">in</span> samples.items() <span class="hljs-keyword">if</span> k <span class="hljs-keyword">not</span> <span class="hljs-keyword">in</span> [<span class="hljs-string">&quot;idx&quot;</span>, <span class="hljs-string">&quot;sentence1&quot;</span>, <span class="hljs-string">&quot;sentence2&quot;</span>]}
[<span class="hljs-built_in">len</span>(x) <span class="hljs-keyword">for</span> x <span class="hljs-keyword">in</span> samples[<span class="hljs-string">&quot;input_ids&quot;</span>]]`}}),mt=new z({props:{code:"[50, 59, 47, 67, 59, 50, 62, 32]",highlighted:'[<span class="hljs-number">50</span>, <span class="hljs-number">59</span>, <span class="hljs-number">47</span>, <span class="hljs-number">67</span>, <span class="hljs-number">59</span>, <span class="hljs-number">50</span>, <span class="hljs-number">62</span>, <span class="hljs-number">32</span>]'}}),ht=new z({props:{code:"\n```py\nbatch = data_collator(samples)\n{k: v.shape for k, v in batch.items()}",highlighted:'\n```py\nbatch = dat<span class="hljs-built_in">a_collator</span>(samples)\n<span class="hljs-title">{k:</span> v.shape for k, v in batch.items()}'}});const Ao=[yi,qi],kt=[];function Co(s,o){return s[0]==="tf"?0:1}is=Co(v),cs=kt[is]=Ao[is](v),Cs=new Eo({props:{$$slots:{default:[zi]},$$scope:{ctx:v}}});let O=v[0]==="tf"&&li();return{c(){l=p("meta"),u=_(),w(r.$$.fragment),b=_(),k=p("h1"),h=a("\u5904\u7406\u6570\u636E"),d=_(),g.c(),q=_(),G.c(),fs=_(),us=p("p"),vs=a("\u5F53\u7136\uFF0C\u4EC5\u4EC5\u7528\u4E24\u53E5\u8BDD\u8BAD\u7EC3\u6A21\u578B\u4E0D\u4F1A\u4EA7\u751F\u5F88\u597D\u7684\u6548\u679C\u3002\u4E3A\u4E86\u83B7\u5F97\u66F4\u597D\u7684\u7ED3\u679C\uFF0C\u60A8\u9700\u8981\u51C6\u5907\u4E00\u4E2A\u66F4\u5927\u7684\u6570\u636E\u96C6\u3002"),B=_(),U=p("p"),_s=a("\u5728\u672C\u8282\u4E2D\uFF0C\u6211\u4EEC\u5C06\u4F7F\u7528MRPC\uFF08\u5FAE\u8F6F\u7814\u7A76\u91CA\u4E49\u8BED\u6599\u5E93\uFF09\u6570\u636E\u96C6\u4F5C\u4E3A\u793A\u4F8B\uFF0C\u8BE5\u6570\u636E\u96C6\u7531\u5A01\u5EC9\xB7\u591A\u5170\u548C\u514B\u91CC\u65AF\xB7\u5E03\u7F57\u514B\u7279\u5728"),ts=p("a"),ws=a("\u8FD9\u7BC7\u6587\u7AE0"),P=a("\u53D1\u5E03\u3002\u8BE5\u6570\u636E\u96C6\u75315801\u5BF9\u53E5\u5B50\u7EC4\u6210\uFF0C\u6BCF\u4E2A\u53E5\u5B50\u5BF9\u5E26\u6709\u4E00\u4E2A\u6807\u7B7E\uFF0C\u6307\u793A\u5B83\u4EEC\u662F\u5426\u4E3A\u540C\u4E49\uFF08\u5373\uFF0C\u5982\u679C\u4E24\u4E2A\u53E5\u5B50\u7684\u610F\u601D\u76F8\u540C\uFF09\u3002\u6211\u4EEC\u5728\u672C\u7AE0\u4E2D\u9009\u62E9\u4E86\u5B83\uFF0C\u56E0\u4E3A\u5B83\u662F\u4E00\u4E2A\u5C0F\u6570\u636E\u96C6\uFF0C\u6240\u4EE5\u5F88\u5BB9\u6613\u5BF9\u5B83\u8FDB\u884C\u8BAD\u7EC3\u3002"),R=_(),V=p("h3"),es=p("a"),Es=p("span"),w(ms.$$.fragment),Et=_(),Qt=p("span"),jn=a("\u4ECE\u6A21\u578B\u4E2D\u5FC3\uFF08Hub\uFF09\u52A0\u8F7D\u6570\u636E\u96C6"),na=_(),ns.c(),Tt=_(),J=p("p"),kn=a("\u6A21\u578B\u4E2D\u5FC3\uFF08hub\uFF09\u4E0D\u53EA\u662F\u5305\u542B\u6A21\u578B\uFF1B\u5B83\u4E5F\u6709\u8BB8\u591A\u4E0D\u540C\u8BED\u8A00\u7684\u591A\u4E2A\u6570\u636E\u96C6\u3002\u70B9\u51FB"),Ms=p("a"),xn=a("\u6570\u636E\u96C6"),vn=a("\u7684\u94FE\u63A5\u5373\u53EF\u8FDB\u884C\u6D4F\u89C8\u3002\u6211\u4EEC\u5EFA\u8BAE\u60A8\u5728\u9605\u8BFB\u672C\u8282\u540E\u9605\u8BFB\u4E00\u4E0B"),Ws=p("a"),wn=a("\u52A0\u8F7D\u548C\u5904\u7406\u65B0\u7684\u6570\u636E\u96C6"),En=a("\u8FD9\u7BC7\u6587\u7AE0\uFF0C\u8FD9\u4F1A\u8BA9\u60A8\u5BF9huggingface\u7684darasets\u66F4\u52A0\u6E05\u6670\u3002\u4F46\u73B0\u5728\uFF0C\u8BA9\u6211\u4EEC\u4F7F\u7528MRPC\u6570\u636E\u96C6\u4E2D\u7684"),Hs=p("a"),Tn=a("GLUE \u57FA\u51C6\u6D4B\u8BD5\u6570\u636E\u96C6"),Sn=a("\uFF0C\u5B83\u662F\u6784\u6210MRPC\u6570\u636E\u96C6\u768410\u4E2A\u6570\u636E\u96C6\u4E4B\u4E00\uFF0C\u8FD9\u662F\u4E00\u4E2A\u5B66\u672F\u57FA\u51C6\uFF0C\u7528\u4E8E\u8861\u91CF\u673A\u5668\u5B66\u4E60\u6A21\u578B\u572810\u4E2A\u4E0D\u540C\u6587\u672C\u5206\u7C7B\u4EFB\u52A1\u4E2D\u7684\u6027\u80FD\u3002"),la=_(),St=p("p"),qn=a("\u{1F917} Datasets\u5E93\u63D0\u4F9B\u4E86\u4E00\u4E2A\u975E\u5E38\u4FBF\u6377\u7684\u547D\u4EE4\uFF0C\u53EF\u4EE5\u5728\u6A21\u578B\u4E2D\u5FC3\uFF08hub\uFF09\u4E0A\u4E0B\u8F7D\u548C\u7F13\u5B58\u6570\u636E\u96C6\u3002\u6211\u4EEC\u53EF\u4EE5\u901A\u8FC7\u4EE5\u4E0B\u7684\u4EE3\u7801\u4E0B\u8F7DMRPC\u6570\u636E\u96C6\uFF1A"),ra=_(),w(Us.$$.fragment),oa=_(),w(Bs.$$.fragment),pa=_(),I=p("p"),yn=a("\u6B63\u5982\u4F60\u6240\u770B\u5230\u7684\uFF0C\u6211\u4EEC\u83B7\u5F97\u4E86\u4E00\u4E2A"),Xt=p("strong"),zn=a("DatasetDict"),Pn=a("\u5BF9\u8C61\uFF0C\u5176\u4E2D\u5305\u542B\u8BAD\u7EC3\u96C6\u3001\u9A8C\u8BC1\u96C6\u548C\u6D4B\u8BD5\u96C6\u3002\u6BCF\u4E00\u4E2A\u96C6\u5408\u90FD\u5305\u542B\u51E0\u4E2A\u5217("),se=p("strong"),Nn=a("sentence1"),Dn=a(", "),te=p("strong"),Rn=a("sentence2"),Gn=a(", "),ee=p("strong"),On=a("label"),An=a(", and "),ae=p("strong"),Cn=a("idx"),In=a(")\u4EE5\u53CA\u4E00\u4E2A\u4EE3\u8868\u884C\u6570\u7684\u53D8\u91CF\uFF0C\u5373\u6BCF\u4E2A\u96C6\u5408\u4E2D\u7684\u884C\u7684\u4E2A\u6570\uFF08\u56E0\u6B64\uFF0C\u8BAD\u7EC3\u96C6\u4E2D\u67093668\u5BF9\u53E5\u5B50\uFF0C\u9A8C\u8BC1\u96C6\u4E2D\u6709408\u5BF9\uFF0C\u6D4B\u8BD5\u96C6\u4E2D\u67091725\u5BF9\uFF09\u3002"),ia=_(),hs=p("p"),Fn=a("\u9ED8\u8BA4\u60C5\u51B5\u4E0B\uFF0C\u6B64\u547D\u4EE4\u5728\u4E0B\u8F7D\u6570\u636E\u96C6\u5E76\u7F13\u5B58\u5230 "),ne=p("strong"),Ln=a("~/.cache/huggingface/dataset"),Mn=a(". \u56DE\u60F3\u4E00\u4E0B\u7B2C2\u7AE0\uFF0C\u60A8\u53EF\u4EE5\u901A\u8FC7\u8BBE\u7F6E"),le=p("strong"),Wn=a("HF_HOME"),Hn=a("\u73AF\u5883\u53D8\u91CF\u6765\u81EA\u5B9A\u4E49\u7F13\u5B58\u7684\u6587\u4EF6\u5939\u3002"),ca=_(),Ts=p("p"),Un=a("\u6211\u4EEC\u53EF\u4EE5\u8BBF\u95EE\u6211\u4EEC\u6570\u636E\u96C6\u4E2D\u7684\u6BCF\u4E00\u4E2A"),re=p("strong"),Bn=a("raw_train_dataset"),Vn=a("\u5BF9\u8C61\uFF0C\u5982\u4F7F\u7528\u5B57\u5178\uFF1A"),fa=_(),w(Vs.$$.fragment),ua=_(),w(Js.$$.fragment),_a=_(),ds=p("p"),Jn=a("\u6211\u4EEC\u53EF\u4EE5\u770B\u5230\u6807\u7B7E\u5DF2\u7ECF\u662F\u6574\u6570\u4E86\uFF0C\u6240\u4EE5\u6211\u4EEC\u4E0D\u9700\u8981\u5BF9\u6807\u7B7E\u505A\u4EFB\u4F55\u9884\u5904\u7406\u3002\u8981\u77E5\u9053\u54EA\u4E2A\u6570\u5B57\u5BF9\u5E94\u4E8E\u54EA\u4E2A\u6807\u7B7E\uFF0C\u6211\u4EEC\u53EF\u4EE5\u67E5\u770B"),oe=p("strong"),Kn=a("raw_train_dataset"),Yn=a("\u7684"),pe=p("strong"),Zn=a("features"),Qn=a(". \u8FD9\u5C06\u544A\u8BC9\u6211\u4EEC\u6BCF\u5217\u7684\u7C7B\u578B\uFF1A"),ma=_(),w(Ks.$$.fragment),ha=_(),w(Ys.$$.fragment),da=_(),A=p("p"),Xn=a("\u5728\u4E0A\u9762\u7684\u4F8B\u5B50\u4E4B\u4E2D,"),ie=p("strong"),sl=a("Label\uFF08\u6807\u7B7E\uFF09"),tl=a(" \u662F\u4E00\u79CD"),ce=p("strong"),el=a("ClassLabel\uFF08\u5206\u7C7B\u6807\u7B7E\uFF09"),al=a("\uFF0C\u4F7F\u7528\u6574\u6570\u5EFA\u7ACB\u8D77\u5230\u7C7B\u522B\u6807\u7B7E\u7684\u6620\u5C04\u5173\u7CFB\u3002"),fe=p("strong"),nl=a("0"),ll=a("\u5BF9\u5E94\u4E8E"),ue=p("strong"),rl=a("not_equivalent"),ol=a("\uFF0C"),_e=p("strong"),pl=a("1"),il=a("\u5BF9\u5E94\u4E8E"),me=p("strong"),cl=a("equivalent"),fl=a("\u3002"),ba=_(),w(Ss.$$.fragment),ga=_(),Zs=p("h3"),ul=a("\u9884\u5904\u7406\u6570\u636E\u96C6"),$a=_(),rs.c(),qt=_(),qs=p("p"),_l=a("\u4E3A\u4E86\u9884\u5904\u7406\u6570\u636E\u96C6\uFF0C\u6211\u4EEC\u9700\u8981\u5C06\u6587\u672C\u8F6C\u6362\u4E3A\u6A21\u578B\u80FD\u591F\u7406\u89E3\u7684\u6570\u5B57\u3002\u6B63\u5982\u4F60\u5728"),yt=p("a"),ml=a("\u7B2C\u4E8C\u7AE0"),hl=a("\u4E0A\u770B\u5230\u7684\u90A3\u6837"),ja=_(),w(Qs.$$.fragment),ka=_(),zt=p("p"),dl=a("\u7136\u800C\uFF0C\u5728\u4E24\u53E5\u8BDD\u4F20\u9012\u7ED9\u6A21\u578B\uFF0C\u9884\u6D4B\u8FD9\u4E24\u53E5\u8BDD\u662F\u5426\u662F\u540C\u4E49\u4E4B\u524D\u3002\u6211\u4EEC\u9700\u8981\u8FD9\u4E24\u53E5\u8BDD\u4F9D\u6B21\u8FDB\u884C\u9002\u5F53\u7684\u9884\u5904\u7406\u3002\u5E78\u8FD0\u7684\u662F\uFF0C\u6807\u8BB0\u5668\u4E0D\u4EC5\u4EC5\u53EF\u4EE5\u8F93\u5165\u5355\u4E2A\u53E5\u5B50\u8FD8\u53EF\u4EE5\u8F93\u5165\u4E00\u7EC4\u53E5\u5B50\uFF0C\u5E76\u6309\u7167\u6211\u4EEC\u7684BERT\u6A21\u578B\u6240\u671F\u671B\u7684\u8F93\u5165\u8FDB\u884C\u5904\u7406\uFF1A"),xa=_(),w(Xs.$$.fragment),va=_(),w(st.$$.fragment),wa=_(),F=p("p"),bl=a("\u6211\u4EEC\u5728"),Pt=p("a"),gl=a("\u7B2C\u4E8C\u7AE0"),$l=a(" \u8BA8\u8BBA\u4E86"),he=p("strong"),jl=a("\u8F93\u5165\u8BCDid(input_ids)"),kl=a(" \u548C "),de=p("strong"),xl=a("\u6CE8\u610F\u529B\u906E\u7F69(attention_mask)"),vl=a(" \uFF0C\u4F46\u6211\u4EEC\u5728\u90A3\u4E2A\u65F6\u5019\u6CA1\u6709\u8BA8\u8BBA"),be=p("strong"),wl=a("\u7C7B\u578B\u6807\u8BB0ID(token_type_ids)"),El=a("\u3002\u5728\u8FD9\u4E2A\u4F8B\u5B50\u4E2D\uFF0C"),ge=p("strong"),Tl=a("\u7C7B\u578B\u6807\u8BB0ID(token_type_ids)"),Sl=a("\u7684\u4F5C\u7528\u5C31\u662F\u544A\u8BC9\u6A21\u578B\u8F93\u5165\u7684\u54EA\u4E00\u90E8\u5206\u662F\u7B2C\u4E00\u53E5\uFF0C\u54EA\u4E00\u90E8\u5206\u662F\u7B2C\u4E8C\u53E5\u3002"),Ea=_(),w(ys.$$.fragment),Ta=_(),zs=p("p"),ql=a("\u5982\u679C\u6211\u4EEC\u5C06"),$e=p("strong"),yl=a("input_ids"),zl=a("\u4E2D\u7684id\u8F6C\u6362\u56DE\u6587\u5B57:"),Sa=_(),w(tt.$$.fragment),qa=_(),Nt=p("p"),Pl=a("\u6211\u4EEC\u5C06\u5F97\u5230\uFF1A"),ya=_(),w(et.$$.fragment),za=_(),bs=p("p"),Nl=a("\u6240\u4EE5\u6211\u4EEC\u770B\u5230\u6A21\u578B\u9700\u8981\u8F93\u5165\u7684\u5F62\u5F0F\u662F "),je=p("strong"),Dl=a("[CLS] sentence1 [SEP] sentence2 [SEP]"),Rl=a("\u3002\u56E0\u6B64\uFF0C\u5F53\u6709\u4E24\u53E5\u8BDD\u7684\u65F6\u5019\u3002"),ke=p("strong"),Gl=a("\u7C7B\u578B\u6807\u8BB0ID(token_type_ids)"),Ol=a(" \u7684\u503C\u662F\uFF1A"),Pa=_(),w(at.$$.fragment),Na=_(),M=p("p"),Al=a("\u5982\u60A8\u6240\u89C1\uFF0C\u8F93\u5165\u4E2D "),xe=p("strong"),Cl=a("[CLS] sentence1 [SEP]"),Il=a(" \u5B83\u4EEC\u7684\u7C7B\u578B\u6807\u8BB0ID\u5747\u4E3A"),ve=p("strong"),Fl=a("0"),Ll=a("\uFF0C\u800C\u5176\u4ED6\u90E8\u5206\uFF0C\u5BF9\u5E94\u4E8E"),we=p("strong"),Ml=a("sentence2 [SEP]"),Wl=a("\uFF0C\u6240\u6709\u7684\u7C7B\u578B\u6807\u8BB0ID\u5747\u4E3A"),Ee=p("strong"),Hl=a("1"),Ul=a("."),Da=_(),Ps=p("p"),Bl=a("\u8BF7\u6CE8\u610F\uFF0C\u5982\u679C\u9009\u62E9\u5176\u4ED6\u7684\u68C0\u67E5\u70B9\uFF0C\u5219\u4E0D\u4E00\u5B9A\u5177\u6709"),Te=p("strong"),Vl=a("\u7C7B\u578B\u6807\u8BB0ID(token_type_ids)"),Jl=a("\uFF08\u4F8B\u5982\uFF0C\u5982\u679C\u4F7F\u7528DistilBERT\u6A21\u578B\uFF0C\u5C31\u4E0D\u4F1A\u8FD4\u56DE\u5B83\u4EEC\uFF09\u3002\u53EA\u6709\u5F53\u5B83\u5728\u9884\u8BAD\u7EC3\u671F\u95F4\u4F7F\u7528\u8FC7\u8FD9\u4E00\u5C42\uFF0C\u6A21\u578B\u5728\u6784\u5EFA\u65F6\u4F9D\u8D56\u5B83\u4EEC\uFF0C\u624D\u4F1A\u8FD4\u56DE\u5B83\u4EEC\u3002"),Ra=_(),Ns=p("p"),Kl=a("\u7528\u7C7B\u578B\u6807\u8BB0ID\u5BF9BERT\u8FDB\u884C\u9884\u8BAD\u7EC3,\u5E76\u4E14\u4F7F\u7528"),Dt=p("a"),Yl=a("\u7B2C\u4E00\u7AE0"),Zl=a("\u7684\u906E\u7F69\u8BED\u8A00\u6A21\u578B\uFF0C\u8FD8\u6709\u4E00\u4E2A\u989D\u5916\u7684\u5E94\u7528\u7C7B\u578B\uFF0C\u53EB\u505A\u4E0B\u4E00\u53E5\u9884\u6D4B. \u8FD9\u9879\u4EFB\u52A1\u7684\u76EE\u6807\u662F\u5EFA\u7ACB\u6210\u5BF9\u53E5\u5B50\u4E4B\u95F4\u5173\u7CFB\u7684\u6A21\u578B\u3002"),Ga=_(),Rt=p("p"),Ql=a("\u5728\u4E0B\u4E00\u4E2A\u53E5\u5B50\u9884\u6D4B\u4EFB\u52A1\u4E2D\uFF0C\u4F1A\u7ED9\u6A21\u578B\u8F93\u5165\u6210\u5BF9\u7684\u53E5\u5B50\uFF08\u5E26\u6709\u968F\u673A\u906E\u7F69\u7684\u6807\u8BB0\uFF09\uFF0C\u5E76\u88AB\u8981\u6C42\u9884\u6D4B\u7B2C\u4E8C\u4E2A\u53E5\u5B50\u662F\u5426\u7D27\u8DDF\u7B2C\u4E00\u4E2A\u53E5\u5B50\u3002\u4E3A\u4E86\u63D0\u9AD8\u6A21\u578B\u7684\u6CDB\u5316\u80FD\u529B\uFF0C\u6570\u636E\u96C6\u4E2D\u4E00\u534A\u7684\u4E24\u4E2A\u53E5\u5B50\u5728\u539F\u59CB\u6587\u6863\u4E2D\u6328\u5728\u4E00\u8D77\uFF0C\u53E6\u4E00\u534A\u7684\u4E24\u4E2A\u53E5\u5B50\u6765\u81EA\u4E24\u4E2A\u4E0D\u540C\u7684\u6587\u6863\u3002"),Oa=_(),Ds=p("p"),Xl=a("\u4E00\u822C\u6765\u8BF4\uFF0C\u4F60\u4E0D\u9700\u8981\u62C5\u5FC3\u662F\u5426\u6709"),Se=p("strong"),sr=a("\u7C7B\u578B\u6807\u8BB0ID(token_type_ids)"),tr=a("\u3002\u5728\u60A8\u7684\u6807\u8F93\u5165\u4E2D\uFF1A\u53EA\u8981\u60A8\u5BF9\u6807\u8BB0\u5668\u548C\u6A21\u578B\u4F7F\u7528\u76F8\u540C\u7684\u68C0\u67E5\u70B9\uFF0C\u4E00\u5207\u90FD\u4F1A\u5F88\u597D\uFF0C\u56E0\u4E3A\u6807\u8BB0\u5668\u77E5\u9053\u5411\u5176\u6A21\u578B\u63D0\u4F9B\u4EC0\u4E48\u3002"),Aa=_(),gs=p("p"),er=a("\u73B0\u5728\u6211\u4EEC\u5DF2\u7ECF\u4E86\u89E3\u4E86\u6807\u8BB0\u5668\u5982\u4F55\u5904\u7406\u4E00\u5BF9\u53E5\u5B50\uFF0C\u6211\u4EEC\u53EF\u4EE5\u4F7F\u7528\u5B83\u5BF9\u6574\u4E2A\u6570\u636E\u96C6\u8FDB\u884C\u5904\u7406\uFF1A\u5982"),Gt=p("a"),ar=a("\u4E4B\u524D\u7684\u7AE0\u8282"),nr=a("\uFF0C\u6211\u4EEC\u53EF\u4EE5\u7ED9\u6807\u8BB0\u5668\u63D0\u4F9B\u4E00\u7EC4\u53E5\u5B50\uFF0C\u7B2C\u4E00\u4E2A\u53C2\u6570\u662F\u5B83\u7B2C\u4E00\u4E2A\u53E5\u5B50\u7684\u5217\u8868\uFF0C\u7B2C\u4E8C\u4E2A\u53C2\u6570\u662F\u7B2C\u4E8C\u4E2A\u53E5\u5B50\u7684\u5217\u8868\u3002\u8FD9\u4E5F\u4E0E\u6211\u4EEC\u5728"),Ot=p("a"),lr=a("\u7B2C\u4E8C\u7AE0"),rr=a("\u4E2D\u770B\u5230\u7684\u586B\u5145\u548C\u622A\u65AD\u9009\u9879\u517C\u5BB9. \u56E0\u6B64\uFF0C\u9884\u5904\u7406\u8BAD\u7EC3\u6570\u636E\u96C6\u7684\u4E00\u79CD\u65B9\u6CD5\u662F\uFF1A"),Ca=_(),w(nt.$$.fragment),Ia=_(),W=p("p"),or=a("\u8FD9\u5F88\u6709\u6548\uFF0C\u4F46\u5B83\u7684\u7F3A\u70B9\u662F\u8FD4\u56DE\u5B57\u5178\uFF08\u5B57\u5178\u7684\u952E\u662F"),qe=p("strong"),pr=a("\u8F93\u5165\u8BCDid(input_ids)"),ir=a(" \uFF0C "),ye=p("strong"),cr=a("\u6CE8\u610F\u529B\u906E\u7F69(attention_mask)"),fr=a(" \u548C "),ze=p("strong"),ur=a("\u7C7B\u578B\u6807\u8BB0ID(token_type_ids)"),_r=a("\uFF0C\u5B57\u5178\u7684\u503C\u662F\u952E\u6240\u5BF9\u5E94\u503C\u7684\u5217\u8868\uFF09\u3002\u800C\u4E14\u53EA\u6709\u5F53\u60A8\u5728\u8F6C\u6362\u8FC7\u7A0B\u4E2D\u6709\u8DB3\u591F\u7684\u5185\u5B58\u6765\u5B58\u50A8\u6574\u4E2A\u6570\u636E\u96C6\u65F6\u624D\u4E0D\u4F1A\u51FA\u9519\uFF08\u800C\u{1F917}\u6570\u636E\u96C6\u5E93\u4E2D\u7684\u6570\u636E\u96C6\u662F\u4EE5"),lt=p("a"),mr=a("Apache Arrow"),hr=a("\u6587\u4EF6\u5B58\u50A8\u5728\u78C1\u76D8\u4E0A\uFF0C\u56E0\u6B64\u60A8\u53EA\u9700\u5C06\u63A5\u4E0B\u6765\u8981\u7528\u7684\u6570\u636E\u52A0\u8F7D\u5728\u5185\u5B58\u4E2D\uFF0C\u56E0\u6B64\u4F1A\u5BF9\u5185\u5B58\u5BB9\u91CF\u7684\u9700\u6C42\u8981\u4F4E\u4E00\u4E9B\uFF09\u3002"),Fa=_(),Rs=p("p"),dr=a("\u4E3A\u4E86\u5C06\u6570\u636E\u4FDD\u5B58\u4E3A\u6570\u636E\u96C6\uFF0C\u6211\u4EEC\u5C06\u4F7F\u7528"),rt=p("a"),br=a("Dataset.map()"),gr=a("\u65B9\u6CD5\uFF0C\u5982\u679C\u6211\u4EEC\u9700\u8981\u505A\u66F4\u591A\u7684\u9884\u5904\u7406\u800C\u4E0D\u4EC5\u4EC5\u662F\u6807\u8BB0\u5316\uFF0C\u90A3\u4E48\u8FD9\u4E5F\u7ED9\u4E86\u6211\u4EEC\u4E00\u4E9B\u989D\u5916\u7684\u81EA\u5B9A\u4E49\u7684\u65B9\u6CD5\u3002\u8FD9\u4E2A\u65B9\u6CD5\u7684\u5DE5\u4F5C\u539F\u7406\u662F\u5728\u6570\u636E\u96C6\u7684\u6BCF\u4E2A\u5143\u7D20\u4E0A\u5E94\u7528\u4E00\u4E2A\u51FD\u6570\uFF0C\u56E0\u6B64\u8BA9\u6211\u4EEC\u5B9A\u4E49\u4E00\u4E2A\u6807\u8BB0\u8F93\u5165\u7684\u51FD\u6570\uFF1A"),La=_(),w(ot.$$.fragment),Ma=_(),D=p("p"),$r=a("\u6B64\u51FD\u6570\u7684\u8F93\u5165\u662F\u4E00\u4E2A\u5B57\u5178\uFF08\u4E0E\u6570\u636E\u96C6\u7684\u9879\u7C7B\u4F3C\uFF09\uFF0C\u5E76\u8FD4\u56DE\u4E00\u4E2A\u5305\u542B"),Pe=p("strong"),jr=a("\u8F93\u5165\u8BCDid(input_ids)"),kr=a(" \uFF0C "),Ne=p("strong"),xr=a("\u6CE8\u610F\u529B\u906E\u7F69(attention_mask)"),vr=a(" \u548C "),De=p("strong"),wr=a("\u7C7B\u578B\u6807\u8BB0ID(token_type_ids)"),Er=a(" \u952E\u7684\u65B0\u5B57\u5178\u3002\u8BF7\u6CE8\u610F\uFF0C\u5982\u679C\u50CF\u4E0A\u9762\u7684"),Re=p("strong"),Tr=a("\u793A\u4F8B"),Sr=a("\u4E00\u6837\uFF0C\u5982\u679C\u952E\u6240\u5BF9\u5E94\u7684\u503C\u5305\u542B\u591A\u4E2A\u53E5\u5B50\uFF08\u6BCF\u4E2A\u952E\u4F5C\u4E3A\u4E00\u4E2A\u53E5\u5B50\u5217\u8868\uFF09\uFF0C\u90A3\u4E48\u5B83\u4F9D\u7136\u53EF\u4EE5\u5DE5\u4F5C\uFF0C\u5C31\u50CF\u524D\u9762\u7684\u4F8B\u5B50\u4E00\u6837\u6807\u8BB0\u5668\u53EF\u4EE5\u5904\u7406\u6210\u5BF9\u7684\u53E5\u5B50\u5217\u8868\u3002\u8FD9\u6837\u7684\u8BDD\u6211\u4EEC\u53EF\u4EE5\u5728\u8C03\u7528"),Ge=p("strong"),qr=a("map()"),yr=a("\u4F7F\u7528\u8BE5\u9009\u9879 "),Oe=p("strong"),zr=a("batched=True"),Pr=a(" \uFF0C\u8FD9\u5C06\u663E\u8457\u52A0\u5FEB\u6807\u8BB0\u4E0E\u6807\u8BB0\u7684\u901F\u5EA6\u3002\u8FD9\u4E2A"),Ae=p("strong"),Nr=a("\u6807\u8BB0\u5668"),Dr=a("\u6765\u81EA"),pt=p("a"),Rr=a("\u{1F917} Tokenizers"),Gr=a("\u5E93\u7531Rust\u7F16\u5199\u800C\u6210\u3002\u5F53\u6211\u4EEC\u4E00\u6B21\u7ED9\u5B83\u5927\u91CF\u7684\u8F93\u5165\u65F6\uFF0C\u8FD9\u4E2A\u6807\u8BB0\u5668\u53EF\u4EE5\u975E\u5E38\u5FEB\u3002"),Wa=_(),Gs=p("p"),Or=a("\u8BF7\u6CE8\u610F\uFF0C\u6211\u4EEC\u73B0\u5728\u5728\u6807\u8BB0\u51FD\u6570\u4E2D\u7701\u7565\u4E86"),Ce=p("strong"),Ar=a("padding"),Cr=a("\u53C2\u6570\u3002\u8FD9\u662F\u56E0\u4E3A\u5728\u6807\u8BB0\u7684\u65F6\u5019\u5C06\u6240\u6709\u6837\u672C\u586B\u5145\u5230\u6700\u5927\u957F\u5EA6\u7684\u6548\u7387\u4E0D\u9AD8\u3002\u4E00\u4E2A\u66F4\u597D\u7684\u505A\u6CD5\uFF1A\u5728\u6784\u5EFA\u6279\u5904\u7406\u65F6\u586B\u5145\u6837\u672C\u66F4\u597D\uFF0C\u56E0\u4E3A\u8FD9\u6837\u6211\u4EEC\u53EA\u9700\u8981\u586B\u5145\u5230\u8BE5\u6279\u5904\u7406\u4E2D\u7684\u6700\u5927\u957F\u5EA6\uFF0C\u800C\u4E0D\u662F\u6574\u4E2A\u6570\u636E\u96C6\u7684\u6700\u5927\u957F\u5EA6\u3002\u5F53\u8F93\u5165\u957F\u5EA6\u53D8\u5316\u5F88\u5927\u65F6\uFF0C\u8FD9\u53EF\u4EE5\u8282\u7701\u5927\u91CF\u65F6\u95F4\u548C\u5904\u7406\u80FD\u529B!"),Ha=_(),$s=p("p"),Ir=a("\u4E0B\u9762\u662F\u6211\u4EEC\u5982\u4F55\u5728\u6240\u6709\u6570\u636E\u96C6\u4E0A\u540C\u65F6\u5E94\u7528\u6807\u8BB0\u51FD\u6570\u3002\u6211\u4EEC\u5728\u8C03\u7528"),Ie=p("strong"),Fr=a("map"),Lr=a("\u65F6\u4F7F\u7528\u4E86"),Fe=p("strong"),Mr=a("batch =True"),Wr=a("\uFF0C\u8FD9\u6837\u51FD\u6570\u5C31\u53EF\u4EE5\u540C\u65F6\u5E94\u7528\u5230\u6570\u636E\u96C6\u7684\u591A\u4E2A\u5143\u7D20\u4E0A\uFF0C\u800C\u4E0D\u662F\u5206\u522B\u5E94\u7528\u5230\u6BCF\u4E2A\u5143\u7D20\u4E0A\u3002\u8FD9\u5C06\u4F7F\u6211\u4EEC\u7684\u9884\u5904\u7406\u5FEB\u8BB8\u591A"),Ua=_(),w(it.$$.fragment),Ba=_(),At=p("p"),Hr=a("\u{1F917}Datasets\u5E93\u5E94\u7528\u8FD9\u79CD\u5904\u7406\u7684\u65B9\u5F0F\u662F\u5411\u6570\u636E\u96C6\u6DFB\u52A0\u65B0\u7684\u5B57\u6BB5\uFF0C\u6BCF\u4E2A\u5B57\u6BB5\u5BF9\u5E94\u9884\u5904\u7406\u51FD\u6570\u8FD4\u56DE\u7684\u5B57\u5178\u4E2D\u7684\u6BCF\u4E2A\u952E:"),Va=_(),w(ct.$$.fragment),Ja=_(),K=p("p"),Ur=a("\u5728\u4F7F\u7528\u9884\u5904\u7406\u51FD\u6570"),Le=p("strong"),Br=a("map()"),Vr=a("\u65F6\uFF0C\u751A\u81F3\u53EF\u4EE5\u901A\u8FC7\u4F20\u9012"),Me=p("strong"),Jr=a("num_proc"),Kr=a("\u53C2\u6570\u4F7F\u7528\u5E76\u884C\u5904\u7406\u3002\u6211\u4EEC\u5728\u8FD9\u91CC\u6CA1\u6709\u8FD9\u6837\u505A\uFF0C\u56E0\u4E3A\u{1F917}\u6807\u8BB0\u5668\u5E93\u5DF2\u7ECF\u4F7F\u7528\u591A\u4E2A\u7EBF\u7A0B\u6765\u66F4\u5FEB\u5730\u6807\u8BB0\u6211\u4EEC\u7684\u6837\u672C\uFF0C\u4F46\u662F\u5982\u679C\u60A8\u6CA1\u6709\u4F7F\u7528\u8BE5\u5E93\u652F\u6301\u7684\u5FEB\u901F\u6807\u8BB0\u5668\uFF0C\u4F7F\u7528"),We=p("strong"),Yr=a("num_proc"),Zr=a("\u53EF\u80FD\u4F1A\u52A0\u5FEB\u9884\u5904\u7406\u3002"),Ka=_(),L=p("p"),Qr=a("\u6211\u4EEC\u7684"),He=p("strong"),Xr=a("\u6807\u8BB0\u51FD\u6570(tokenize_function)"),so=a("\u8FD4\u56DE\u5305\u542B"),Ue=p("strong"),to=a("\u8F93\u5165\u8BCDid(input_ids)"),eo=a(" \uFF0C "),Be=p("strong"),ao=a("\u6CE8\u610F\u529B\u906E\u7F69(attention_mask)"),no=a(" \u548C "),Ve=p("strong"),lo=a("\u7C7B\u578B\u6807\u8BB0ID(token_type_ids)"),ro=a(" \u952E\u7684\u5B57\u5178,\u6240\u4EE5\u8FD9\u4E09\u4E2A\u5B57\u6BB5\u88AB\u6DFB\u52A0\u5230\u6570\u636E\u96C6\u7684\u6807\u8BB0\u7684\u7ED3\u679C\u4E2D\u3002\u6CE8\u610F\uFF0C\u5982\u679C\u9884\u5904\u7406\u51FD\u6570"),Je=p("strong"),oo=a("map()"),po=a("\u4E3A\u73B0\u6709\u952E\u8FD4\u56DE\u4E00\u4E2A\u65B0\u503C\uFF0C\u90A3\u5C06\u4F1A\u4FEE\u6539\u539F\u6709\u952E\u7684\u503C\u3002"),Ya=_(),Ct=p("p"),io=a("\u6700\u540E\u4E00\u4EF6\u6211\u4EEC\u9700\u8981\u505A\u7684\u4E8B\u60C5\u662F\uFF0C\u5F53\u6211\u4EEC\u4E00\u8D77\u6279\u5904\u7406\u5143\u7D20\u65F6\uFF0C\u5C06\u6240\u6709\u793A\u4F8B\u586B\u5145\u5230\u6700\u957F\u5143\u7D20\u7684\u957F\u5EA6\u2014\u2014\u6211\u4EEC\u79F0\u4E4B\u4E3A\u52A8\u6001\u586B\u5145\u3002"),Za=_(),ft=p("h3"),co=a("\u52A8\u6001\u586B\u5145"),Qa=_(),w(ut.$$.fragment),Xa=_(),xs.c(),It=_(),Os=p("p"),fo=a("\u4E3A\u4E86\u89E3\u51B3\u53E5\u5B50\u957F\u5EA6\u7EDF\u4E00\u7684\u95EE\u9898\uFF0C\u6211\u4EEC\u5FC5\u987B\u5B9A\u4E49\u4E00\u4E2Acollate\u51FD\u6570\uFF0C\u8BE5\u51FD\u6570\u4F1A\u5C06\u6BCF\u4E2Abatch\u53E5\u5B50\u586B\u5145\u5230\u6B63\u786E\u7684\u957F\u5EA6\u3002\u5E78\u8FD0\u7684\u662F\uFF0C\u{1F917}transformer\u5E93\u901A\u8FC7"),Ke=p("strong"),uo=a("DataCollatorWithPadding"),_o=a("\u4E3A\u6211\u4EEC\u63D0\u4F9B\u4E86\u8FD9\u6837\u4E00\u4E2A\u51FD\u6570\u3002\u5F53\u4F60\u5B9E\u4F8B\u5316\u5B83\u65F6\uFF0C\u9700\u8981\u4E00\u4E2A\u6807\u8BB0\u5668(\u7528\u6765\u77E5\u9053\u4F7F\u7528\u54EA\u4E2A\u8BCD\u6765\u586B\u5145\uFF0C\u4EE5\u53CA\u6A21\u578B\u671F\u671B\u586B\u5145\u5728\u5DE6\u8FB9\u8FD8\u662F\u53F3\u8FB9)\uFF0C\u5E76\u5C06\u505A\u4F60\u9700\u8981\u7684\u4E00\u5207:"),sn=_(),ps.c(),Ft=_(),Y=p("p"),mo=a("\u4E3A\u4E86\u6D4B\u8BD5\u8FD9\u4E2A\u65B0\u73A9\u5177\uFF0C\u8BA9\u6211\u4EEC\u4ECE\u6211\u4EEC\u7684\u8BAD\u7EC3\u96C6\u4E2D\u62BD\u53D6\u51E0\u4E2A\u6837\u672C\u3002\u8FD9\u91CC\uFF0C\u6211\u4EEC\u5220\u9664\u5217"),Ye=p("strong"),ho=a("idx"),bo=a(", "),Ze=p("strong"),go=a("sentence1"),$o=a("\u548C"),Qe=p("strong"),jo=a("sentence2"),ko=a("\uFF0C\u56E0\u4E3A\u4E0D\u9700\u8981\u5B83\u4EEC\uFF0C\u5E76\u67E5\u770B\u4E00\u4E2Abatch\u4E2D\u6BCF\u4E2A\u6761\u76EE\u7684\u957F\u5EA6:"),tn=_(),w(_t.$$.fragment),en=_(),w(mt.$$.fragment),an=_(),As=p("p"),xo=a("\u6BEB\u65E0\u7591\u95EE\uFF0C\u6211\u4EEC\u5F97\u5230\u4E86\u4E0D\u540C\u957F\u5EA6\u7684\u6837\u672C\uFF0C\u4ECE32\u523067\u3002\u52A8\u6001\u586B\u5145\u610F\u5473\u7740\u8BE5\u6279\u4E2D\u7684\u6240\u6709\u6837\u672C\u90FD\u5E94\u8BE5\u586B\u5145\u5230\u957F\u5EA6\u4E3A67\uFF0C\u8FD9\u662F\u8BE5\u6279\u4E2D\u7684\u6700\u5927\u957F\u5EA6\u3002\u5982\u679C\u6CA1\u6709\u52A8\u6001\u586B\u5145\uFF0C\u6240\u6709\u7684\u6837\u672C\u90FD\u5FC5\u987B\u586B\u5145\u5230\u6574\u4E2A\u6570\u636E\u96C6\u4E2D\u7684\u6700\u5927\u957F\u5EA6\uFF0C\u6216\u8005\u6A21\u578B\u53EF\u4EE5\u63A5\u53D7\u7684\u6700\u5927\u957F\u5EA6\u3002\u8BA9\u6211\u4EEC\u518D\u6B21\u68C0\u67E5"),Xe=p("strong"),vo=a("data_collator"),wo=a("\u662F\u5426\u6B63\u786E\u5730\u52A8\u6001\u586B\u5145\u4E86\u8FD9\u6279\u6837\u672C\uFF1A"),nn=_(),w(ht.$$.fragment),ln=_(),cs.c(),Lt=_(),w(Cs.$$.fragment),rn=_(),O&&O.c(),Mt=ni(),this.h()},l(s){const o=ci('[data-svelte="svelte-1phssyn"]',document.head);l=i(o,"META",{name:!0,content:!0}),o.forEach(e),u=m(s),E(r.$$.fragment,s),b=m(s),k=i(s,"H1",{id:!0});var sa=c(k);h=n(sa,"\u5904\u7406\u6570\u636E"),sa.forEach(e),d=m(s),g.l(s),q=m(s),G.l(s),fs=m(s),us=i(s,"P",{});var Wt=c(us);vs=n(Wt,"\u5F53\u7136\uFF0C\u4EC5\u4EC5\u7528\u4E24\u53E5\u8BDD\u8BAD\u7EC3\u6A21\u578B\u4E0D\u4F1A\u4EA7\u751F\u5F88\u597D\u7684\u6548\u679C\u3002\u4E3A\u4E86\u83B7\u5F97\u66F4\u597D\u7684\u7ED3\u679C\uFF0C\u60A8\u9700\u8981\u51C6\u5907\u4E00\u4E2A\u66F4\u5927\u7684\u6570\u636E\u96C6\u3002"),Wt.forEach(e),B=m(s),U=i(s,"P",{});var Is=c(U);_s=n(Is,"\u5728\u672C\u8282\u4E2D\uFF0C\u6211\u4EEC\u5C06\u4F7F\u7528MRPC\uFF08\u5FAE\u8F6F\u7814\u7A76\u91CA\u4E49\u8BED\u6599\u5E93\uFF09\u6570\u636E\u96C6\u4F5C\u4E3A\u793A\u4F8B\uFF0C\u8BE5\u6570\u636E\u96C6\u7531\u5A01\u5EC9\xB7\u591A\u5170\u548C\u514B\u91CC\u65AF\xB7\u5E03\u7F57\u514B\u7279\u5728"),ts=i(Is,"A",{href:!0,rel:!0});var Ht=c(ts);ws=n(Ht,"\u8FD9\u7BC7\u6587\u7AE0"),Ht.forEach(e),P=n(Is,"\u53D1\u5E03\u3002\u8BE5\u6570\u636E\u96C6\u75315801\u5BF9\u53E5\u5B50\u7EC4\u6210\uFF0C\u6BCF\u4E2A\u53E5\u5B50\u5BF9\u5E26\u6709\u4E00\u4E2A\u6807\u7B7E\uFF0C\u6307\u793A\u5B83\u4EEC\u662F\u5426\u4E3A\u540C\u4E49\uFF08\u5373\uFF0C\u5982\u679C\u4E24\u4E2A\u53E5\u5B50\u7684\u610F\u601D\u76F8\u540C\uFF09\u3002\u6211\u4EEC\u5728\u672C\u7AE0\u4E2D\u9009\u62E9\u4E86\u5B83\uFF0C\u56E0\u4E3A\u5B83\u662F\u4E00\u4E2A\u5C0F\u6570\u636E\u96C6\uFF0C\u6240\u4EE5\u5F88\u5BB9\u6613\u5BF9\u5B83\u8FDB\u884C\u8BAD\u7EC3\u3002"),Is.forEach(e),R=m(s),V=i(s,"H3",{class:!0});var xt=c(V);es=i(xt,"A",{id:!0,class:!0,href:!0});var Ut=c(es);Es=i(Ut,"SPAN",{});var ta=c(Es);E(ms.$$.fragment,ta),ta.forEach(e),Ut.forEach(e),Et=m(xt),Qt=i(xt,"SPAN",{});var Bt=c(Qt);jn=n(Bt,"\u4ECE\u6A21\u578B\u4E2D\u5FC3\uFF08Hub\uFF09\u52A0\u8F7D\u6570\u636E\u96C6"),Bt.forEach(e),xt.forEach(e),na=m(s),ns.l(s),Tt=m(s),J=i(s,"P",{});var Z=c(J);kn=n(Z,"\u6A21\u578B\u4E2D\u5FC3\uFF08hub\uFF09\u4E0D\u53EA\u662F\u5305\u542B\u6A21\u578B\uFF1B\u5B83\u4E5F\u6709\u8BB8\u591A\u4E0D\u540C\u8BED\u8A00\u7684\u591A\u4E2A\u6570\u636E\u96C6\u3002\u70B9\u51FB"),Ms=i(Z,"A",{href:!0,rel:!0});var ea=c(Ms);xn=n(ea,"\u6570\u636E\u96C6"),ea.forEach(e),vn=n(Z,"\u7684\u94FE\u63A5\u5373\u53EF\u8FDB\u884C\u6D4F\u89C8\u3002\u6211\u4EEC\u5EFA\u8BAE\u60A8\u5728\u9605\u8BFB\u672C\u8282\u540E\u9605\u8BFB\u4E00\u4E0B"),Ws=i(Z,"A",{href:!0,rel:!0});var Io=c(Ws);wn=n(Io,"\u52A0\u8F7D\u548C\u5904\u7406\u65B0\u7684\u6570\u636E\u96C6"),Io.forEach(e),En=n(Z,"\u8FD9\u7BC7\u6587\u7AE0\uFF0C\u8FD9\u4F1A\u8BA9\u60A8\u5BF9huggingface\u7684darasets\u66F4\u52A0\u6E05\u6670\u3002\u4F46\u73B0\u5728\uFF0C\u8BA9\u6211\u4EEC\u4F7F\u7528MRPC\u6570\u636E\u96C6\u4E2D\u7684"),Hs=i(Z,"A",{href:!0,rel:!0});var Fo=c(Hs);Tn=n(Fo,"GLUE \u57FA\u51C6\u6D4B\u8BD5\u6570\u636E\u96C6"),Fo.forEach(e),Sn=n(Z,"\uFF0C\u5B83\u662F\u6784\u6210MRPC\u6570\u636E\u96C6\u768410\u4E2A\u6570\u636E\u96C6\u4E4B\u4E00\uFF0C\u8FD9\u662F\u4E00\u4E2A\u5B66\u672F\u57FA\u51C6\uFF0C\u7528\u4E8E\u8861\u91CF\u673A\u5668\u5B66\u4E60\u6A21\u578B\u572810\u4E2A\u4E0D\u540C\u6587\u672C\u5206\u7C7B\u4EFB\u52A1\u4E2D\u7684\u6027\u80FD\u3002"),Z.forEach(e),la=m(s),St=i(s,"P",{});var Lo=c(St);qn=n(Lo,"\u{1F917} Datasets\u5E93\u63D0\u4F9B\u4E86\u4E00\u4E2A\u975E\u5E38\u4FBF\u6377\u7684\u547D\u4EE4\uFF0C\u53EF\u4EE5\u5728\u6A21\u578B\u4E2D\u5FC3\uFF08hub\uFF09\u4E0A\u4E0B\u8F7D\u548C\u7F13\u5B58\u6570\u636E\u96C6\u3002\u6211\u4EEC\u53EF\u4EE5\u901A\u8FC7\u4EE5\u4E0B\u7684\u4EE3\u7801\u4E0B\u8F7DMRPC\u6570\u636E\u96C6\uFF1A"),Lo.forEach(e),ra=m(s),E(Us.$$.fragment,s),oa=m(s),E(Bs.$$.fragment,s),pa=m(s),I=i(s,"P",{});var Q=c(I);yn=n(Q,"\u6B63\u5982\u4F60\u6240\u770B\u5230\u7684\uFF0C\u6211\u4EEC\u83B7\u5F97\u4E86\u4E00\u4E2A"),Xt=i(Q,"STRONG",{});var Mo=c(Xt);zn=n(Mo,"DatasetDict"),Mo.forEach(e),Pn=n(Q,"\u5BF9\u8C61\uFF0C\u5176\u4E2D\u5305\u542B\u8BAD\u7EC3\u96C6\u3001\u9A8C\u8BC1\u96C6\u548C\u6D4B\u8BD5\u96C6\u3002\u6BCF\u4E00\u4E2A\u96C6\u5408\u90FD\u5305\u542B\u51E0\u4E2A\u5217("),se=i(Q,"STRONG",{});var Wo=c(se);Nn=n(Wo,"sentence1"),Wo.forEach(e),Dn=n(Q,", "),te=i(Q,"STRONG",{});var Ho=c(te);Rn=n(Ho,"sentence2"),Ho.forEach(e),Gn=n(Q,", "),ee=i(Q,"STRONG",{});var Uo=c(ee);On=n(Uo,"label"),Uo.forEach(e),An=n(Q,", and "),ae=i(Q,"STRONG",{});var Bo=c(ae);Cn=n(Bo,"idx"),Bo.forEach(e),In=n(Q,")\u4EE5\u53CA\u4E00\u4E2A\u4EE3\u8868\u884C\u6570\u7684\u53D8\u91CF\uFF0C\u5373\u6BCF\u4E2A\u96C6\u5408\u4E2D\u7684\u884C\u7684\u4E2A\u6570\uFF08\u56E0\u6B64\uFF0C\u8BAD\u7EC3\u96C6\u4E2D\u67093668\u5BF9\u53E5\u5B50\uFF0C\u9A8C\u8BC1\u96C6\u4E2D\u6709408\u5BF9\uFF0C\u6D4B\u8BD5\u96C6\u4E2D\u67091725\u5BF9\uFF09\u3002"),Q.forEach(e),ia=m(s),hs=i(s,"P",{});var Vt=c(hs);Fn=n(Vt,"\u9ED8\u8BA4\u60C5\u51B5\u4E0B\uFF0C\u6B64\u547D\u4EE4\u5728\u4E0B\u8F7D\u6570\u636E\u96C6\u5E76\u7F13\u5B58\u5230 "),ne=i(Vt,"STRONG",{});var Vo=c(ne);Ln=n(Vo,"~/.cache/huggingface/dataset"),Vo.forEach(e),Mn=n(Vt,". \u56DE\u60F3\u4E00\u4E0B\u7B2C2\u7AE0\uFF0C\u60A8\u53EF\u4EE5\u901A\u8FC7\u8BBE\u7F6E"),le=i(Vt,"STRONG",{});var Jo=c(le);Wn=n(Jo,"HF_HOME"),Jo.forEach(e),Hn=n(Vt,"\u73AF\u5883\u53D8\u91CF\u6765\u81EA\u5B9A\u4E49\u7F13\u5B58\u7684\u6587\u4EF6\u5939\u3002"),Vt.forEach(e),ca=m(s),Ts=i(s,"P",{});var cn=c(Ts);Un=n(cn,"\u6211\u4EEC\u53EF\u4EE5\u8BBF\u95EE\u6211\u4EEC\u6570\u636E\u96C6\u4E2D\u7684\u6BCF\u4E00\u4E2A"),re=i(cn,"STRONG",{});var Ko=c(re);Bn=n(Ko,"raw_train_dataset"),Ko.forEach(e),Vn=n(cn,"\u5BF9\u8C61\uFF0C\u5982\u4F7F\u7528\u5B57\u5178\uFF1A"),cn.forEach(e),fa=m(s),E(Vs.$$.fragment,s),ua=m(s),E(Js.$$.fragment,s),_a=m(s),ds=i(s,"P",{});var Jt=c(ds);Jn=n(Jt,"\u6211\u4EEC\u53EF\u4EE5\u770B\u5230\u6807\u7B7E\u5DF2\u7ECF\u662F\u6574\u6570\u4E86\uFF0C\u6240\u4EE5\u6211\u4EEC\u4E0D\u9700\u8981\u5BF9\u6807\u7B7E\u505A\u4EFB\u4F55\u9884\u5904\u7406\u3002\u8981\u77E5\u9053\u54EA\u4E2A\u6570\u5B57\u5BF9\u5E94\u4E8E\u54EA\u4E2A\u6807\u7B7E\uFF0C\u6211\u4EEC\u53EF\u4EE5\u67E5\u770B"),oe=i(Jt,"STRONG",{});var Yo=c(oe);Kn=n(Yo,"raw_train_dataset"),Yo.forEach(e),Yn=n(Jt,"\u7684"),pe=i(Jt,"STRONG",{});var Zo=c(pe);Zn=n(Zo,"features"),Zo.forEach(e),Qn=n(Jt,". \u8FD9\u5C06\u544A\u8BC9\u6211\u4EEC\u6BCF\u5217\u7684\u7C7B\u578B\uFF1A"),Jt.forEach(e),ma=m(s),E(Ks.$$.fragment,s),ha=m(s),E(Ys.$$.fragment,s),da=m(s),A=i(s,"P",{});var H=c(A);Xn=n(H,"\u5728\u4E0A\u9762\u7684\u4F8B\u5B50\u4E4B\u4E2D,"),ie=i(H,"STRONG",{});var Qo=c(ie);sl=n(Qo,"Label\uFF08\u6807\u7B7E\uFF09"),Qo.forEach(e),tl=n(H," \u662F\u4E00\u79CD"),ce=i(H,"STRONG",{});var Xo=c(ce);el=n(Xo,"ClassLabel\uFF08\u5206\u7C7B\u6807\u7B7E\uFF09"),Xo.forEach(e),al=n(H,"\uFF0C\u4F7F\u7528\u6574\u6570\u5EFA\u7ACB\u8D77\u5230\u7C7B\u522B\u6807\u7B7E\u7684\u6620\u5C04\u5173\u7CFB\u3002"),fe=i(H,"STRONG",{});var sp=c(fe);nl=n(sp,"0"),sp.forEach(e),ll=n(H,"\u5BF9\u5E94\u4E8E"),ue=i(H,"STRONG",{});var tp=c(ue);rl=n(tp,"not_equivalent"),tp.forEach(e),ol=n(H,"\uFF0C"),_e=i(H,"STRONG",{});var ep=c(_e);pl=n(ep,"1"),ep.forEach(e),il=n(H,"\u5BF9\u5E94\u4E8E"),me=i(H,"STRONG",{});var ap=c(me);cl=n(ap,"equivalent"),ap.forEach(e),fl=n(H,"\u3002"),H.forEach(e),ba=m(s),E(Ss.$$.fragment,s),ga=m(s),Zs=i(s,"H3",{id:!0});var np=c(Zs);ul=n(np,"\u9884\u5904\u7406\u6570\u636E\u96C6"),np.forEach(e),$a=m(s),rs.l(s),qt=m(s),qs=i(s,"P",{});var fn=c(qs);_l=n(fn,"\u4E3A\u4E86\u9884\u5904\u7406\u6570\u636E\u96C6\uFF0C\u6211\u4EEC\u9700\u8981\u5C06\u6587\u672C\u8F6C\u6362\u4E3A\u6A21\u578B\u80FD\u591F\u7406\u89E3\u7684\u6570\u5B57\u3002\u6B63\u5982\u4F60\u5728"),yt=i(fn,"A",{href:!0});var lp=c(yt);ml=n(lp,"\u7B2C\u4E8C\u7AE0"),lp.forEach(e),hl=n(fn,"\u4E0A\u770B\u5230\u7684\u90A3\u6837"),fn.forEach(e),ja=m(s),E(Qs.$$.fragment,s),ka=m(s),zt=i(s,"P",{});var rp=c(zt);dl=n(rp,"\u7136\u800C\uFF0C\u5728\u4E24\u53E5\u8BDD\u4F20\u9012\u7ED9\u6A21\u578B\uFF0C\u9884\u6D4B\u8FD9\u4E24\u53E5\u8BDD\u662F\u5426\u662F\u540C\u4E49\u4E4B\u524D\u3002\u6211\u4EEC\u9700\u8981\u8FD9\u4E24\u53E5\u8BDD\u4F9D\u6B21\u8FDB\u884C\u9002\u5F53\u7684\u9884\u5904\u7406\u3002\u5E78\u8FD0\u7684\u662F\uFF0C\u6807\u8BB0\u5668\u4E0D\u4EC5\u4EC5\u53EF\u4EE5\u8F93\u5165\u5355\u4E2A\u53E5\u5B50\u8FD8\u53EF\u4EE5\u8F93\u5165\u4E00\u7EC4\u53E5\u5B50\uFF0C\u5E76\u6309\u7167\u6211\u4EEC\u7684BERT\u6A21\u578B\u6240\u671F\u671B\u7684\u8F93\u5165\u8FDB\u884C\u5904\u7406\uFF1A"),rp.forEach(e),xa=m(s),E(Xs.$$.fragment,s),va=m(s),E(st.$$.fragment,s),wa=m(s),F=i(s,"P",{});var X=c(F);bl=n(X,"\u6211\u4EEC\u5728"),Pt=i(X,"A",{href:!0});var op=c(Pt);gl=n(op,"\u7B2C\u4E8C\u7AE0"),op.forEach(e),$l=n(X," \u8BA8\u8BBA\u4E86"),he=i(X,"STRONG",{});var pp=c(he);jl=n(pp,"\u8F93\u5165\u8BCDid(input_ids)"),pp.forEach(e),kl=n(X," \u548C "),de=i(X,"STRONG",{});var ip=c(de);xl=n(ip,"\u6CE8\u610F\u529B\u906E\u7F69(attention_mask)"),ip.forEach(e),vl=n(X," \uFF0C\u4F46\u6211\u4EEC\u5728\u90A3\u4E2A\u65F6\u5019\u6CA1\u6709\u8BA8\u8BBA"),be=i(X,"STRONG",{});var cp=c(be);wl=n(cp,"\u7C7B\u578B\u6807\u8BB0ID(token_type_ids)"),cp.forEach(e),El=n(X,"\u3002\u5728\u8FD9\u4E2A\u4F8B\u5B50\u4E2D\uFF0C"),ge=i(X,"STRONG",{});var fp=c(ge);Tl=n(fp,"\u7C7B\u578B\u6807\u8BB0ID(token_type_ids)"),fp.forEach(e),Sl=n(X,"\u7684\u4F5C\u7528\u5C31\u662F\u544A\u8BC9\u6A21\u578B\u8F93\u5165\u7684\u54EA\u4E00\u90E8\u5206\u662F\u7B2C\u4E00\u53E5\uFF0C\u54EA\u4E00\u90E8\u5206\u662F\u7B2C\u4E8C\u53E5\u3002"),X.forEach(e),Ea=m(s),E(ys.$$.fragment,s),Ta=m(s),zs=i(s,"P",{});var un=c(zs);ql=n(un,"\u5982\u679C\u6211\u4EEC\u5C06"),$e=i(un,"STRONG",{});var up=c($e);yl=n(up,"input_ids"),up.forEach(e),zl=n(un,"\u4E2D\u7684id\u8F6C\u6362\u56DE\u6587\u5B57:"),un.forEach(e),Sa=m(s),E(tt.$$.fragment,s),qa=m(s),Nt=i(s,"P",{});var _p=c(Nt);Pl=n(_p,"\u6211\u4EEC\u5C06\u5F97\u5230\uFF1A"),_p.forEach(e),ya=m(s),E(et.$$.fragment,s),za=m(s),bs=i(s,"P",{});var Kt=c(bs);Nl=n(Kt,"\u6240\u4EE5\u6211\u4EEC\u770B\u5230\u6A21\u578B\u9700\u8981\u8F93\u5165\u7684\u5F62\u5F0F\u662F "),je=i(Kt,"STRONG",{});var mp=c(je);Dl=n(mp,"[CLS] sentence1 [SEP] sentence2 [SEP]"),mp.forEach(e),Rl=n(Kt,"\u3002\u56E0\u6B64\uFF0C\u5F53\u6709\u4E24\u53E5\u8BDD\u7684\u65F6\u5019\u3002"),ke=i(Kt,"STRONG",{});var hp=c(ke);Gl=n(hp,"\u7C7B\u578B\u6807\u8BB0ID(token_type_ids)"),hp.forEach(e),Ol=n(Kt," \u7684\u503C\u662F\uFF1A"),Kt.forEach(e),Pa=m(s),E(at.$$.fragment,s),Na=m(s),M=i(s,"P",{});var js=c(M);Al=n(js,"\u5982\u60A8\u6240\u89C1\uFF0C\u8F93\u5165\u4E2D "),xe=i(js,"STRONG",{});var dp=c(xe);Cl=n(dp,"[CLS] sentence1 [SEP]"),dp.forEach(e),Il=n(js," \u5B83\u4EEC\u7684\u7C7B\u578B\u6807\u8BB0ID\u5747\u4E3A"),ve=i(js,"STRONG",{});var bp=c(ve);Fl=n(bp,"0"),bp.forEach(e),Ll=n(js,"\uFF0C\u800C\u5176\u4ED6\u90E8\u5206\uFF0C\u5BF9\u5E94\u4E8E"),we=i(js,"STRONG",{});var gp=c(we);Ml=n(gp,"sentence2 [SEP]"),gp.forEach(e),Wl=n(js,"\uFF0C\u6240\u6709\u7684\u7C7B\u578B\u6807\u8BB0ID\u5747\u4E3A"),Ee=i(js,"STRONG",{});var $p=c(Ee);Hl=n($p,"1"),$p.forEach(e),Ul=n(js,"."),js.forEach(e),Da=m(s),Ps=i(s,"P",{});var _n=c(Ps);Bl=n(_n,"\u8BF7\u6CE8\u610F\uFF0C\u5982\u679C\u9009\u62E9\u5176\u4ED6\u7684\u68C0\u67E5\u70B9\uFF0C\u5219\u4E0D\u4E00\u5B9A\u5177\u6709"),Te=i(_n,"STRONG",{});var jp=c(Te);Vl=n(jp,"\u7C7B\u578B\u6807\u8BB0ID(token_type_ids)"),jp.forEach(e),Jl=n(_n,"\uFF08\u4F8B\u5982\uFF0C\u5982\u679C\u4F7F\u7528DistilBERT\u6A21\u578B\uFF0C\u5C31\u4E0D\u4F1A\u8FD4\u56DE\u5B83\u4EEC\uFF09\u3002\u53EA\u6709\u5F53\u5B83\u5728\u9884\u8BAD\u7EC3\u671F\u95F4\u4F7F\u7528\u8FC7\u8FD9\u4E00\u5C42\uFF0C\u6A21\u578B\u5728\u6784\u5EFA\u65F6\u4F9D\u8D56\u5B83\u4EEC\uFF0C\u624D\u4F1A\u8FD4\u56DE\u5B83\u4EEC\u3002"),_n.forEach(e),Ra=m(s),Ns=i(s,"P",{});var mn=c(Ns);Kl=n(mn,"\u7528\u7C7B\u578B\u6807\u8BB0ID\u5BF9BERT\u8FDB\u884C\u9884\u8BAD\u7EC3,\u5E76\u4E14\u4F7F\u7528"),Dt=i(mn,"A",{href:!0});var kp=c(Dt);Yl=n(kp,"\u7B2C\u4E00\u7AE0"),kp.forEach(e),Zl=n(mn,"\u7684\u906E\u7F69\u8BED\u8A00\u6A21\u578B\uFF0C\u8FD8\u6709\u4E00\u4E2A\u989D\u5916\u7684\u5E94\u7528\u7C7B\u578B\uFF0C\u53EB\u505A\u4E0B\u4E00\u53E5\u9884\u6D4B. \u8FD9\u9879\u4EFB\u52A1\u7684\u76EE\u6807\u662F\u5EFA\u7ACB\u6210\u5BF9\u53E5\u5B50\u4E4B\u95F4\u5173\u7CFB\u7684\u6A21\u578B\u3002"),mn.forEach(e),Ga=m(s),Rt=i(s,"P",{});var xp=c(Rt);Ql=n(xp,"\u5728\u4E0B\u4E00\u4E2A\u53E5\u5B50\u9884\u6D4B\u4EFB\u52A1\u4E2D\uFF0C\u4F1A\u7ED9\u6A21\u578B\u8F93\u5165\u6210\u5BF9\u7684\u53E5\u5B50\uFF08\u5E26\u6709\u968F\u673A\u906E\u7F69\u7684\u6807\u8BB0\uFF09\uFF0C\u5E76\u88AB\u8981\u6C42\u9884\u6D4B\u7B2C\u4E8C\u4E2A\u53E5\u5B50\u662F\u5426\u7D27\u8DDF\u7B2C\u4E00\u4E2A\u53E5\u5B50\u3002\u4E3A\u4E86\u63D0\u9AD8\u6A21\u578B\u7684\u6CDB\u5316\u80FD\u529B\uFF0C\u6570\u636E\u96C6\u4E2D\u4E00\u534A\u7684\u4E24\u4E2A\u53E5\u5B50\u5728\u539F\u59CB\u6587\u6863\u4E2D\u6328\u5728\u4E00\u8D77\uFF0C\u53E6\u4E00\u534A\u7684\u4E24\u4E2A\u53E5\u5B50\u6765\u81EA\u4E24\u4E2A\u4E0D\u540C\u7684\u6587\u6863\u3002"),xp.forEach(e),Oa=m(s),Ds=i(s,"P",{});var hn=c(Ds);Xl=n(hn,"\u4E00\u822C\u6765\u8BF4\uFF0C\u4F60\u4E0D\u9700\u8981\u62C5\u5FC3\u662F\u5426\u6709"),Se=i(hn,"STRONG",{});var vp=c(Se);sr=n(vp,"\u7C7B\u578B\u6807\u8BB0ID(token_type_ids)"),vp.forEach(e),tr=n(hn,"\u3002\u5728\u60A8\u7684\u6807\u8F93\u5165\u4E2D\uFF1A\u53EA\u8981\u60A8\u5BF9\u6807\u8BB0\u5668\u548C\u6A21\u578B\u4F7F\u7528\u76F8\u540C\u7684\u68C0\u67E5\u70B9\uFF0C\u4E00\u5207\u90FD\u4F1A\u5F88\u597D\uFF0C\u56E0\u4E3A\u6807\u8BB0\u5668\u77E5\u9053\u5411\u5176\u6A21\u578B\u63D0\u4F9B\u4EC0\u4E48\u3002"),hn.forEach(e),Aa=m(s),gs=i(s,"P",{});var Yt=c(gs);er=n(Yt,"\u73B0\u5728\u6211\u4EEC\u5DF2\u7ECF\u4E86\u89E3\u4E86\u6807\u8BB0\u5668\u5982\u4F55\u5904\u7406\u4E00\u5BF9\u53E5\u5B50\uFF0C\u6211\u4EEC\u53EF\u4EE5\u4F7F\u7528\u5B83\u5BF9\u6574\u4E2A\u6570\u636E\u96C6\u8FDB\u884C\u5904\u7406\uFF1A\u5982"),Gt=i(Yt,"A",{href:!0});var wp=c(Gt);ar=n(wp,"\u4E4B\u524D\u7684\u7AE0\u8282"),wp.forEach(e),nr=n(Yt,"\uFF0C\u6211\u4EEC\u53EF\u4EE5\u7ED9\u6807\u8BB0\u5668\u63D0\u4F9B\u4E00\u7EC4\u53E5\u5B50\uFF0C\u7B2C\u4E00\u4E2A\u53C2\u6570\u662F\u5B83\u7B2C\u4E00\u4E2A\u53E5\u5B50\u7684\u5217\u8868\uFF0C\u7B2C\u4E8C\u4E2A\u53C2\u6570\u662F\u7B2C\u4E8C\u4E2A\u53E5\u5B50\u7684\u5217\u8868\u3002\u8FD9\u4E5F\u4E0E\u6211\u4EEC\u5728"),Ot=i(Yt,"A",{href:!0});var Ep=c(Ot);lr=n(Ep,"\u7B2C\u4E8C\u7AE0"),Ep.forEach(e),rr=n(Yt,"\u4E2D\u770B\u5230\u7684\u586B\u5145\u548C\u622A\u65AD\u9009\u9879\u517C\u5BB9. \u56E0\u6B64\uFF0C\u9884\u5904\u7406\u8BAD\u7EC3\u6570\u636E\u96C6\u7684\u4E00\u79CD\u65B9\u6CD5\u662F\uFF1A"),Yt.forEach(e),Ca=m(s),E(nt.$$.fragment,s),Ia=m(s),W=i(s,"P",{});var ks=c(W);or=n(ks,"\u8FD9\u5F88\u6709\u6548\uFF0C\u4F46\u5B83\u7684\u7F3A\u70B9\u662F\u8FD4\u56DE\u5B57\u5178\uFF08\u5B57\u5178\u7684\u952E\u662F"),qe=i(ks,"STRONG",{});var Tp=c(qe);pr=n(Tp,"\u8F93\u5165\u8BCDid(input_ids)"),Tp.forEach(e),ir=n(ks," \uFF0C "),ye=i(ks,"STRONG",{});var Sp=c(ye);cr=n(Sp,"\u6CE8\u610F\u529B\u906E\u7F69(attention_mask)"),Sp.forEach(e),fr=n(ks," \u548C "),ze=i(ks,"STRONG",{});var qp=c(ze);ur=n(qp,"\u7C7B\u578B\u6807\u8BB0ID(token_type_ids)"),qp.forEach(e),_r=n(ks,"\uFF0C\u5B57\u5178\u7684\u503C\u662F\u952E\u6240\u5BF9\u5E94\u503C\u7684\u5217\u8868\uFF09\u3002\u800C\u4E14\u53EA\u6709\u5F53\u60A8\u5728\u8F6C\u6362\u8FC7\u7A0B\u4E2D\u6709\u8DB3\u591F\u7684\u5185\u5B58\u6765\u5B58\u50A8\u6574\u4E2A\u6570\u636E\u96C6\u65F6\u624D\u4E0D\u4F1A\u51FA\u9519\uFF08\u800C\u{1F917}\u6570\u636E\u96C6\u5E93\u4E2D\u7684\u6570\u636E\u96C6\u662F\u4EE5"),lt=i(ks,"A",{href:!0,rel:!0});var yp=c(lt);mr=n(yp,"Apache Arrow"),yp.forEach(e),hr=n(ks,"\u6587\u4EF6\u5B58\u50A8\u5728\u78C1\u76D8\u4E0A\uFF0C\u56E0\u6B64\u60A8\u53EA\u9700\u5C06\u63A5\u4E0B\u6765\u8981\u7528\u7684\u6570\u636E\u52A0\u8F7D\u5728\u5185\u5B58\u4E2D\uFF0C\u56E0\u6B64\u4F1A\u5BF9\u5185\u5B58\u5BB9\u91CF\u7684\u9700\u6C42\u8981\u4F4E\u4E00\u4E9B\uFF09\u3002"),ks.forEach(e),Fa=m(s),Rs=i(s,"P",{});var dn=c(Rs);dr=n(dn,"\u4E3A\u4E86\u5C06\u6570\u636E\u4FDD\u5B58\u4E3A\u6570\u636E\u96C6\uFF0C\u6211\u4EEC\u5C06\u4F7F\u7528"),rt=i(dn,"A",{href:!0,rel:!0});var zp=c(rt);br=n(zp,"Dataset.map()"),zp.forEach(e),gr=n(dn,"\u65B9\u6CD5\uFF0C\u5982\u679C\u6211\u4EEC\u9700\u8981\u505A\u66F4\u591A\u7684\u9884\u5904\u7406\u800C\u4E0D\u4EC5\u4EC5\u662F\u6807\u8BB0\u5316\uFF0C\u90A3\u4E48\u8FD9\u4E5F\u7ED9\u4E86\u6211\u4EEC\u4E00\u4E9B\u989D\u5916\u7684\u81EA\u5B9A\u4E49\u7684\u65B9\u6CD5\u3002\u8FD9\u4E2A\u65B9\u6CD5\u7684\u5DE5\u4F5C\u539F\u7406\u662F\u5728\u6570\u636E\u96C6\u7684\u6BCF\u4E2A\u5143\u7D20\u4E0A\u5E94\u7528\u4E00\u4E2A\u51FD\u6570\uFF0C\u56E0\u6B64\u8BA9\u6211\u4EEC\u5B9A\u4E49\u4E00\u4E2A\u6807\u8BB0\u8F93\u5165\u7684\u51FD\u6570\uFF1A"),dn.forEach(e),La=m(s),E(ot.$$.fragment,s),Ma=m(s),D=i(s,"P",{});var C=c(D);$r=n(C,"\u6B64\u51FD\u6570\u7684\u8F93\u5165\u662F\u4E00\u4E2A\u5B57\u5178\uFF08\u4E0E\u6570\u636E\u96C6\u7684\u9879\u7C7B\u4F3C\uFF09\uFF0C\u5E76\u8FD4\u56DE\u4E00\u4E2A\u5305\u542B"),Pe=i(C,"STRONG",{});var Pp=c(Pe);jr=n(Pp,"\u8F93\u5165\u8BCDid(input_ids)"),Pp.forEach(e),kr=n(C," \uFF0C "),Ne=i(C,"STRONG",{});var Np=c(Ne);xr=n(Np,"\u6CE8\u610F\u529B\u906E\u7F69(attention_mask)"),Np.forEach(e),vr=n(C," \u548C "),De=i(C,"STRONG",{});var Dp=c(De);wr=n(Dp,"\u7C7B\u578B\u6807\u8BB0ID(token_type_ids)"),Dp.forEach(e),Er=n(C," \u952E\u7684\u65B0\u5B57\u5178\u3002\u8BF7\u6CE8\u610F\uFF0C\u5982\u679C\u50CF\u4E0A\u9762\u7684"),Re=i(C,"STRONG",{});var Rp=c(Re);Tr=n(Rp,"\u793A\u4F8B"),Rp.forEach(e),Sr=n(C,"\u4E00\u6837\uFF0C\u5982\u679C\u952E\u6240\u5BF9\u5E94\u7684\u503C\u5305\u542B\u591A\u4E2A\u53E5\u5B50\uFF08\u6BCF\u4E2A\u952E\u4F5C\u4E3A\u4E00\u4E2A\u53E5\u5B50\u5217\u8868\uFF09\uFF0C\u90A3\u4E48\u5B83\u4F9D\u7136\u53EF\u4EE5\u5DE5\u4F5C\uFF0C\u5C31\u50CF\u524D\u9762\u7684\u4F8B\u5B50\u4E00\u6837\u6807\u8BB0\u5668\u53EF\u4EE5\u5904\u7406\u6210\u5BF9\u7684\u53E5\u5B50\u5217\u8868\u3002\u8FD9\u6837\u7684\u8BDD\u6211\u4EEC\u53EF\u4EE5\u5728\u8C03\u7528"),Ge=i(C,"STRONG",{});var Gp=c(Ge);qr=n(Gp,"map()"),Gp.forEach(e),yr=n(C,"\u4F7F\u7528\u8BE5\u9009\u9879 "),Oe=i(C,"STRONG",{});var Op=c(Oe);zr=n(Op,"batched=True"),Op.forEach(e),Pr=n(C," \uFF0C\u8FD9\u5C06\u663E\u8457\u52A0\u5FEB\u6807\u8BB0\u4E0E\u6807\u8BB0\u7684\u901F\u5EA6\u3002\u8FD9\u4E2A"),Ae=i(C,"STRONG",{});var Ap=c(Ae);Nr=n(Ap,"\u6807\u8BB0\u5668"),Ap.forEach(e),Dr=n(C,"\u6765\u81EA"),pt=i(C,"A",{href:!0,rel:!0});var Cp=c(pt);Rr=n(Cp,"\u{1F917} Tokenizers"),Cp.forEach(e),Gr=n(C,"\u5E93\u7531Rust\u7F16\u5199\u800C\u6210\u3002\u5F53\u6211\u4EEC\u4E00\u6B21\u7ED9\u5B83\u5927\u91CF\u7684\u8F93\u5165\u65F6\uFF0C\u8FD9\u4E2A\u6807\u8BB0\u5668\u53EF\u4EE5\u975E\u5E38\u5FEB\u3002"),C.forEach(e),Wa=m(s),Gs=i(s,"P",{});var bn=c(Gs);Or=n(bn,"\u8BF7\u6CE8\u610F\uFF0C\u6211\u4EEC\u73B0\u5728\u5728\u6807\u8BB0\u51FD\u6570\u4E2D\u7701\u7565\u4E86"),Ce=i(bn,"STRONG",{});var Ip=c(Ce);Ar=n(Ip,"padding"),Ip.forEach(e),Cr=n(bn,"\u53C2\u6570\u3002\u8FD9\u662F\u56E0\u4E3A\u5728\u6807\u8BB0\u7684\u65F6\u5019\u5C06\u6240\u6709\u6837\u672C\u586B\u5145\u5230\u6700\u5927\u957F\u5EA6\u7684\u6548\u7387\u4E0D\u9AD8\u3002\u4E00\u4E2A\u66F4\u597D\u7684\u505A\u6CD5\uFF1A\u5728\u6784\u5EFA\u6279\u5904\u7406\u65F6\u586B\u5145\u6837\u672C\u66F4\u597D\uFF0C\u56E0\u4E3A\u8FD9\u6837\u6211\u4EEC\u53EA\u9700\u8981\u586B\u5145\u5230\u8BE5\u6279\u5904\u7406\u4E2D\u7684\u6700\u5927\u957F\u5EA6\uFF0C\u800C\u4E0D\u662F\u6574\u4E2A\u6570\u636E\u96C6\u7684\u6700\u5927\u957F\u5EA6\u3002\u5F53\u8F93\u5165\u957F\u5EA6\u53D8\u5316\u5F88\u5927\u65F6\uFF0C\u8FD9\u53EF\u4EE5\u8282\u7701\u5927\u91CF\u65F6\u95F4\u548C\u5904\u7406\u80FD\u529B!"),bn.forEach(e),Ha=m(s),$s=i(s,"P",{});var Zt=c($s);Ir=n(Zt,"\u4E0B\u9762\u662F\u6211\u4EEC\u5982\u4F55\u5728\u6240\u6709\u6570\u636E\u96C6\u4E0A\u540C\u65F6\u5E94\u7528\u6807\u8BB0\u51FD\u6570\u3002\u6211\u4EEC\u5728\u8C03\u7528"),Ie=i(Zt,"STRONG",{});var Fp=c(Ie);Fr=n(Fp,"map"),Fp.forEach(e),Lr=n(Zt,"\u65F6\u4F7F\u7528\u4E86"),Fe=i(Zt,"STRONG",{});var Lp=c(Fe);Mr=n(Lp,"batch =True"),Lp.forEach(e),Wr=n(Zt,"\uFF0C\u8FD9\u6837\u51FD\u6570\u5C31\u53EF\u4EE5\u540C\u65F6\u5E94\u7528\u5230\u6570\u636E\u96C6\u7684\u591A\u4E2A\u5143\u7D20\u4E0A\uFF0C\u800C\u4E0D\u662F\u5206\u522B\u5E94\u7528\u5230\u6BCF\u4E2A\u5143\u7D20\u4E0A\u3002\u8FD9\u5C06\u4F7F\u6211\u4EEC\u7684\u9884\u5904\u7406\u5FEB\u8BB8\u591A"),Zt.forEach(e),Ua=m(s),E(it.$$.fragment,s),Ba=m(s),At=i(s,"P",{});var Mp=c(At);Hr=n(Mp,"\u{1F917}Datasets\u5E93\u5E94\u7528\u8FD9\u79CD\u5904\u7406\u7684\u65B9\u5F0F\u662F\u5411\u6570\u636E\u96C6\u6DFB\u52A0\u65B0\u7684\u5B57\u6BB5\uFF0C\u6BCF\u4E2A\u5B57\u6BB5\u5BF9\u5E94\u9884\u5904\u7406\u51FD\u6570\u8FD4\u56DE\u7684\u5B57\u5178\u4E2D\u7684\u6BCF\u4E2A\u952E:"),Mp.forEach(e),Va=m(s),E(ct.$$.fragment,s),Ja=m(s),K=i(s,"P",{});var Fs=c(K);Ur=n(Fs,"\u5728\u4F7F\u7528\u9884\u5904\u7406\u51FD\u6570"),Le=i(Fs,"STRONG",{});var Wp=c(Le);Br=n(Wp,"map()"),Wp.forEach(e),Vr=n(Fs,"\u65F6\uFF0C\u751A\u81F3\u53EF\u4EE5\u901A\u8FC7\u4F20\u9012"),Me=i(Fs,"STRONG",{});var Hp=c(Me);Jr=n(Hp,"num_proc"),Hp.forEach(e),Kr=n(Fs,"\u53C2\u6570\u4F7F\u7528\u5E76\u884C\u5904\u7406\u3002\u6211\u4EEC\u5728\u8FD9\u91CC\u6CA1\u6709\u8FD9\u6837\u505A\uFF0C\u56E0\u4E3A\u{1F917}\u6807\u8BB0\u5668\u5E93\u5DF2\u7ECF\u4F7F\u7528\u591A\u4E2A\u7EBF\u7A0B\u6765\u66F4\u5FEB\u5730\u6807\u8BB0\u6211\u4EEC\u7684\u6837\u672C\uFF0C\u4F46\u662F\u5982\u679C\u60A8\u6CA1\u6709\u4F7F\u7528\u8BE5\u5E93\u652F\u6301\u7684\u5FEB\u901F\u6807\u8BB0\u5668\uFF0C\u4F7F\u7528"),We=i(Fs,"STRONG",{});var Up=c(We);Yr=n(Up,"num_proc"),Up.forEach(e),Zr=n(Fs,"\u53EF\u80FD\u4F1A\u52A0\u5FEB\u9884\u5904\u7406\u3002"),Fs.forEach(e),Ka=m(s),L=i(s,"P",{});var ss=c(L);Qr=n(ss,"\u6211\u4EEC\u7684"),He=i(ss,"STRONG",{});var Bp=c(He);Xr=n(Bp,"\u6807\u8BB0\u51FD\u6570(tokenize_function)"),Bp.forEach(e),so=n(ss,"\u8FD4\u56DE\u5305\u542B"),Ue=i(ss,"STRONG",{});var Vp=c(Ue);to=n(Vp,"\u8F93\u5165\u8BCDid(input_ids)"),Vp.forEach(e),eo=n(ss," \uFF0C "),Be=i(ss,"STRONG",{});var Jp=c(Be);ao=n(Jp,"\u6CE8\u610F\u529B\u906E\u7F69(attention_mask)"),Jp.forEach(e),no=n(ss," \u548C "),Ve=i(ss,"STRONG",{});var Kp=c(Ve);lo=n(Kp,"\u7C7B\u578B\u6807\u8BB0ID(token_type_ids)"),Kp.forEach(e),ro=n(ss," \u952E\u7684\u5B57\u5178,\u6240\u4EE5\u8FD9\u4E09\u4E2A\u5B57\u6BB5\u88AB\u6DFB\u52A0\u5230\u6570\u636E\u96C6\u7684\u6807\u8BB0\u7684\u7ED3\u679C\u4E2D\u3002\u6CE8\u610F\uFF0C\u5982\u679C\u9884\u5904\u7406\u51FD\u6570"),Je=i(ss,"STRONG",{});var Yp=c(Je);oo=n(Yp,"map()"),Yp.forEach(e),po=n(ss,"\u4E3A\u73B0\u6709\u952E\u8FD4\u56DE\u4E00\u4E2A\u65B0\u503C\uFF0C\u90A3\u5C06\u4F1A\u4FEE\u6539\u539F\u6709\u952E\u7684\u503C\u3002"),ss.forEach(e),Ya=m(s),Ct=i(s,"P",{});var Zp=c(Ct);io=n(Zp,"\u6700\u540E\u4E00\u4EF6\u6211\u4EEC\u9700\u8981\u505A\u7684\u4E8B\u60C5\u662F\uFF0C\u5F53\u6211\u4EEC\u4E00\u8D77\u6279\u5904\u7406\u5143\u7D20\u65F6\uFF0C\u5C06\u6240\u6709\u793A\u4F8B\u586B\u5145\u5230\u6700\u957F\u5143\u7D20\u7684\u957F\u5EA6\u2014\u2014\u6211\u4EEC\u79F0\u4E4B\u4E3A\u52A8\u6001\u586B\u5145\u3002"),Zp.forEach(e),Za=m(s),ft=i(s,"H3",{id:!0});var Qp=c(ft);co=n(Qp,"\u52A8\u6001\u586B\u5145"),Qp.forEach(e),Qa=m(s),E(ut.$$.fragment,s),Xa=m(s),xs.l(s),It=m(s),Os=i(s,"P",{});var gn=c(Os);fo=n(gn,"\u4E3A\u4E86\u89E3\u51B3\u53E5\u5B50\u957F\u5EA6\u7EDF\u4E00\u7684\u95EE\u9898\uFF0C\u6211\u4EEC\u5FC5\u987B\u5B9A\u4E49\u4E00\u4E2Acollate\u51FD\u6570\uFF0C\u8BE5\u51FD\u6570\u4F1A\u5C06\u6BCF\u4E2Abatch\u53E5\u5B50\u586B\u5145\u5230\u6B63\u786E\u7684\u957F\u5EA6\u3002\u5E78\u8FD0\u7684\u662F\uFF0C\u{1F917}transformer\u5E93\u901A\u8FC7"),Ke=i(gn,"STRONG",{});var Xp=c(Ke);uo=n(Xp,"DataCollatorWithPadding"),Xp.forEach(e),_o=n(gn,"\u4E3A\u6211\u4EEC\u63D0\u4F9B\u4E86\u8FD9\u6837\u4E00\u4E2A\u51FD\u6570\u3002\u5F53\u4F60\u5B9E\u4F8B\u5316\u5B83\u65F6\uFF0C\u9700\u8981\u4E00\u4E2A\u6807\u8BB0\u5668(\u7528\u6765\u77E5\u9053\u4F7F\u7528\u54EA\u4E2A\u8BCD\u6765\u586B\u5145\uFF0C\u4EE5\u53CA\u6A21\u578B\u671F\u671B\u586B\u5145\u5728\u5DE6\u8FB9\u8FD8\u662F\u53F3\u8FB9)\uFF0C\u5E76\u5C06\u505A\u4F60\u9700\u8981\u7684\u4E00\u5207:"),gn.forEach(e),sn=m(s),ps.l(s),Ft=m(s),Y=i(s,"P",{});var Ls=c(Y);mo=n(Ls,"\u4E3A\u4E86\u6D4B\u8BD5\u8FD9\u4E2A\u65B0\u73A9\u5177\uFF0C\u8BA9\u6211\u4EEC\u4ECE\u6211\u4EEC\u7684\u8BAD\u7EC3\u96C6\u4E2D\u62BD\u53D6\u51E0\u4E2A\u6837\u672C\u3002\u8FD9\u91CC\uFF0C\u6211\u4EEC\u5220\u9664\u5217"),Ye=i(Ls,"STRONG",{});var si=c(Ye);ho=n(si,"idx"),si.forEach(e),bo=n(Ls,", "),Ze=i(Ls,"STRONG",{});var ti=c(Ze);go=n(ti,"sentence1"),ti.forEach(e),$o=n(Ls,"\u548C"),Qe=i(Ls,"STRONG",{});var ei=c(Qe);jo=n(ei,"sentence2"),ei.forEach(e),ko=n(Ls,"\uFF0C\u56E0\u4E3A\u4E0D\u9700\u8981\u5B83\u4EEC\uFF0C\u5E76\u67E5\u770B\u4E00\u4E2Abatch\u4E2D\u6BCF\u4E2A\u6761\u76EE\u7684\u957F\u5EA6:"),Ls.forEach(e),tn=m(s),E(_t.$$.fragment,s),en=m(s),E(mt.$$.fragment,s),an=m(s),As=i(s,"P",{});var $n=c(As);xo=n($n,"\u6BEB\u65E0\u7591\u95EE\uFF0C\u6211\u4EEC\u5F97\u5230\u4E86\u4E0D\u540C\u957F\u5EA6\u7684\u6837\u672C\uFF0C\u4ECE32\u523067\u3002\u52A8\u6001\u586B\u5145\u610F\u5473\u7740\u8BE5\u6279\u4E2D\u7684\u6240\u6709\u6837\u672C\u90FD\u5E94\u8BE5\u586B\u5145\u5230\u957F\u5EA6\u4E3A67\uFF0C\u8FD9\u662F\u8BE5\u6279\u4E2D\u7684\u6700\u5927\u957F\u5EA6\u3002\u5982\u679C\u6CA1\u6709\u52A8\u6001\u586B\u5145\uFF0C\u6240\u6709\u7684\u6837\u672C\u90FD\u5FC5\u987B\u586B\u5145\u5230\u6574\u4E2A\u6570\u636E\u96C6\u4E2D\u7684\u6700\u5927\u957F\u5EA6\uFF0C\u6216\u8005\u6A21\u578B\u53EF\u4EE5\u63A5\u53D7\u7684\u6700\u5927\u957F\u5EA6\u3002\u8BA9\u6211\u4EEC\u518D\u6B21\u68C0\u67E5"),Xe=i($n,"STRONG",{});var ai=c(Xe);vo=n(ai,"data_collator"),ai.forEach(e),wo=n($n,"\u662F\u5426\u6B63\u786E\u5730\u52A8\u6001\u586B\u5145\u4E86\u8FD9\u6279\u6837\u672C\uFF1A"),$n.forEach(e),nn=m(s),E(ht.$$.fragment,s),ln=m(s),cs.l(s),Lt=m(s),E(Cs.$$.fragment,s),rn=m(s),O&&O.l(s),Mt=ni(),this.h()},h(){y(l,"name","hf:doc:metadata"),y(l,"content",JSON.stringify(Ni)),y(k,"id",""),y(ts,"href","https://www.aclweb.org/anthology/I05-5002.pdf"),y(ts,"rel","nofollow"),y(es,"id","hub"),y(es,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),y(es,"href","#hub"),y(V,"class","relative group"),y(Ms,"href","https://huggingface.co/datasets"),y(Ms,"rel","nofollow"),y(Ws,"href","https://huggingface.co/docs/datasets/loading_datasets.html#from-the-huggingface-hub"),y(Ws,"rel","nofollow"),y(Hs,"href","https://gluebenchmark.com/"),y(Hs,"rel","nofollow"),y(Zs,"id",""),y(yt,"href","/course/chapter2"),y(Pt,"href","/course/chapter2"),y(Dt,"href","/course/chapter1"),y(Gt,"href","/course/chapter2"),y(Ot,"href","/course/chapter2"),y(lt,"href","https://arrow.apache.org/"),y(lt,"rel","nofollow"),y(rt,"href","https://huggingface.co/docs/datasets/package_reference/main_classes.html#datasets.Dataset.map"),y(rt,"rel","nofollow"),y(pt,"href","https://github.com/huggingface/tokenizers"),y(pt,"rel","nofollow"),y(ft,"id","")},m(s,o){t(document.head,l),f(s,u,o),T(r,s,o),f(s,b,o),f(s,k,o),t(k,h),f(s,d,o),dt[x].m(s,o),f(s,q,o),bt[N].m(s,o),f(s,fs,o),f(s,us,o),t(us,vs),f(s,B,o),f(s,U,o),t(U,_s),t(U,ts),t(ts,ws),t(U,P),f(s,R,o),f(s,V,o),t(V,es),t(es,Es),T(ms,Es,null),t(V,Et),t(V,Qt),t(Qt,jn),f(s,na,o),gt[as].m(s,o),f(s,Tt,o),f(s,J,o),t(J,kn),t(J,Ms),t(Ms,xn),t(J,vn),t(J,Ws),t(Ws,wn),t(J,En),t(J,Hs),t(Hs,Tn),t(J,Sn),f(s,la,o),f(s,St,o),t(St,qn),f(s,ra,o),T(Us,s,o),f(s,oa,o),T(Bs,s,o),f(s,pa,o),f(s,I,o),t(I,yn),t(I,Xt),t(Xt,zn),t(I,Pn),t(I,se),t(se,Nn),t(I,Dn),t(I,te),t(te,Rn),t(I,Gn),t(I,ee),t(ee,On),t(I,An),t(I,ae),t(ae,Cn),t(I,In),f(s,ia,o),f(s,hs,o),t(hs,Fn),t(hs,ne),t(ne,Ln),t(hs,Mn),t(hs,le),t(le,Wn),t(hs,Hn),f(s,ca,o),f(s,Ts,o),t(Ts,Un),t(Ts,re),t(re,Bn),t(Ts,Vn),f(s,fa,o),T(Vs,s,o),f(s,ua,o),T(Js,s,o),f(s,_a,o),f(s,ds,o),t(ds,Jn),t(ds,oe),t(oe,Kn),t(ds,Yn),t(ds,pe),t(pe,Zn),t(ds,Qn),f(s,ma,o),T(Ks,s,o),f(s,ha,o),T(Ys,s,o),f(s,da,o),f(s,A,o),t(A,Xn),t(A,ie),t(ie,sl),t(A,tl),t(A,ce),t(ce,el),t(A,al),t(A,fe),t(fe,nl),t(A,ll),t(A,ue),t(ue,rl),t(A,ol),t(A,_e),t(_e,pl),t(A,il),t(A,me),t(me,cl),t(A,fl),f(s,ba,o),T(Ss,s,o),f(s,ga,o),f(s,Zs,o),t(Zs,ul),f(s,$a,o),$t[ls].m(s,o),f(s,qt,o),f(s,qs,o),t(qs,_l),t(qs,yt),t(yt,ml),t(qs,hl),f(s,ja,o),T(Qs,s,o),f(s,ka,o),f(s,zt,o),t(zt,dl),f(s,xa,o),T(Xs,s,o),f(s,va,o),T(st,s,o),f(s,wa,o),f(s,F,o),t(F,bl),t(F,Pt),t(Pt,gl),t(F,$l),t(F,he),t(he,jl),t(F,kl),t(F,de),t(de,xl),t(F,vl),t(F,be),t(be,wl),t(F,El),t(F,ge),t(ge,Tl),t(F,Sl),f(s,Ea,o),T(ys,s,o),f(s,Ta,o),f(s,zs,o),t(zs,ql),t(zs,$e),t($e,yl),t(zs,zl),f(s,Sa,o),T(tt,s,o),f(s,qa,o),f(s,Nt,o),t(Nt,Pl),f(s,ya,o),T(et,s,o),f(s,za,o),f(s,bs,o),t(bs,Nl),t(bs,je),t(je,Dl),t(bs,Rl),t(bs,ke),t(ke,Gl),t(bs,Ol),f(s,Pa,o),T(at,s,o),f(s,Na,o),f(s,M,o),t(M,Al),t(M,xe),t(xe,Cl),t(M,Il),t(M,ve),t(ve,Fl),t(M,Ll),t(M,we),t(we,Ml),t(M,Wl),t(M,Ee),t(Ee,Hl),t(M,Ul),f(s,Da,o),f(s,Ps,o),t(Ps,Bl),t(Ps,Te),t(Te,Vl),t(Ps,Jl),f(s,Ra,o),f(s,Ns,o),t(Ns,Kl),t(Ns,Dt),t(Dt,Yl),t(Ns,Zl),f(s,Ga,o),f(s,Rt,o),t(Rt,Ql),f(s,Oa,o),f(s,Ds,o),t(Ds,Xl),t(Ds,Se),t(Se,sr),t(Ds,tr),f(s,Aa,o),f(s,gs,o),t(gs,er),t(gs,Gt),t(Gt,ar),t(gs,nr),t(gs,Ot),t(Ot,lr),t(gs,rr),f(s,Ca,o),T(nt,s,o),f(s,Ia,o),f(s,W,o),t(W,or),t(W,qe),t(qe,pr),t(W,ir),t(W,ye),t(ye,cr),t(W,fr),t(W,ze),t(ze,ur),t(W,_r),t(W,lt),t(lt,mr),t(W,hr),f(s,Fa,o),f(s,Rs,o),t(Rs,dr),t(Rs,rt),t(rt,br),t(Rs,gr),f(s,La,o),T(ot,s,o),f(s,Ma,o),f(s,D,o),t(D,$r),t(D,Pe),t(Pe,jr),t(D,kr),t(D,Ne),t(Ne,xr),t(D,vr),t(D,De),t(De,wr),t(D,Er),t(D,Re),t(Re,Tr),t(D,Sr),t(D,Ge),t(Ge,qr),t(D,yr),t(D,Oe),t(Oe,zr),t(D,Pr),t(D,Ae),t(Ae,Nr),t(D,Dr),t(D,pt),t(pt,Rr),t(D,Gr),f(s,Wa,o),f(s,Gs,o),t(Gs,Or),t(Gs,Ce),t(Ce,Ar),t(Gs,Cr),f(s,Ha,o),f(s,$s,o),t($s,Ir),t($s,Ie),t(Ie,Fr),t($s,Lr),t($s,Fe),t(Fe,Mr),t($s,Wr),f(s,Ua,o),T(it,s,o),f(s,Ba,o),f(s,At,o),t(At,Hr),f(s,Va,o),T(ct,s,o),f(s,Ja,o),f(s,K,o),t(K,Ur),t(K,Le),t(Le,Br),t(K,Vr),t(K,Me),t(Me,Jr),t(K,Kr),t(K,We),t(We,Yr),t(K,Zr),f(s,Ka,o),f(s,L,o),t(L,Qr),t(L,He),t(He,Xr),t(L,so),t(L,Ue),t(Ue,to),t(L,eo),t(L,Be),t(Be,ao),t(L,no),t(L,Ve),t(Ve,lo),t(L,ro),t(L,Je),t(Je,oo),t(L,po),f(s,Ya,o),f(s,Ct,o),t(Ct,io),f(s,Za,o),f(s,ft,o),t(ft,co),f(s,Qa,o),T(ut,s,o),f(s,Xa,o),xs.m(s,o),f(s,It,o),f(s,Os,o),t(Os,fo),t(Os,Ke),t(Ke,uo),t(Os,_o),f(s,sn,o),jt[os].m(s,o),f(s,Ft,o),f(s,Y,o),t(Y,mo),t(Y,Ye),t(Ye,ho),t(Y,bo),t(Y,Ze),t(Ze,go),t(Y,$o),t(Y,Qe),t(Qe,jo),t(Y,ko),f(s,tn,o),T(_t,s,o),f(s,en,o),T(mt,s,o),f(s,an,o),f(s,As,o),t(As,xo),t(As,Xe),t(Xe,vo),t(As,wo),f(s,nn,o),T(ht,s,o),f(s,ln,o),kt[is].m(s,o),f(s,Lt,o),T(Cs,s,o),f(s,rn,o),O&&O.m(s,o),f(s,Mt,o),on=!0},p(s,[o]){const sa={};o&1&&(sa.fw=s[0]),r.$set(sa);let Wt=x;x=So(s),x!==Wt&&(wt(),j(dt[Wt],1,1,()=>{dt[Wt]=null}),vt(),g=dt[x],g||(g=dt[x]=To[x](s),g.c()),$(g,1),g.m(q.parentNode,q));let Is=N;N=yo(s),N!==Is&&(wt(),j(bt[Is],1,1,()=>{bt[Is]=null}),vt(),G=bt[N],G||(G=bt[N]=qo[N](s),G.c()),$(G,1),G.m(fs.parentNode,fs));let Ht=as;as=Po(s),as!==Ht&&(wt(),j(gt[Ht],1,1,()=>{gt[Ht]=null}),vt(),ns=gt[as],ns||(ns=gt[as]=zo[as](s),ns.c()),$(ns,1),ns.m(Tt.parentNode,Tt));const xt={};o&2&&(xt.$$scope={dirty:o,ctx:s}),Ss.$set(xt);let Ut=ls;ls=Do(s),ls!==Ut&&(wt(),j($t[Ut],1,1,()=>{$t[Ut]=null}),vt(),rs=$t[ls],rs||(rs=$t[ls]=No[ls](s),rs.c()),$(rs,1),rs.m(qt.parentNode,qt));const ta={};o&2&&(ta.$$scope={dirty:o,ctx:s}),ys.$set(ta),pn!==(pn=Ro(s))&&(xs.d(1),xs=pn(s),xs&&(xs.c(),xs.m(It.parentNode,It)));let Bt=os;os=Oo(s),os!==Bt&&(wt(),j(jt[Bt],1,1,()=>{jt[Bt]=null}),vt(),ps=jt[os],ps||(ps=jt[os]=Go[os](s),ps.c()),$(ps,1),ps.m(Ft.parentNode,Ft));let Z=is;is=Co(s),is!==Z&&(wt(),j(kt[Z],1,1,()=>{kt[Z]=null}),vt(),cs=kt[is],cs||(cs=kt[is]=Ao[is](s),cs.c()),$(cs,1),cs.m(Lt.parentNode,Lt));const ea={};o&2&&(ea.$$scope={dirty:o,ctx:s}),Cs.$set(ea),s[0]==="tf"?O?o&1&&$(O,1):(O=li(),O.c(),$(O,1),O.m(Mt.parentNode,Mt)):O&&(wt(),j(O,1,1,()=>{O=null}),vt())},i(s){on||($(r.$$.fragment,s),$(g),$(G),$(ms.$$.fragment,s),$(ns),$(Us.$$.fragment,s),$(Bs.$$.fragment,s),$(Vs.$$.fragment,s),$(Js.$$.fragment,s),$(Ks.$$.fragment,s),$(Ys.$$.fragment,s),$(Ss.$$.fragment,s),$(rs),$(Qs.$$.fragment,s),$(Xs.$$.fragment,s),$(st.$$.fragment,s),$(ys.$$.fragment,s),$(tt.$$.fragment,s),$(et.$$.fragment,s),$(at.$$.fragment,s),$(nt.$$.fragment,s),$(ot.$$.fragment,s),$(it.$$.fragment,s),$(ct.$$.fragment,s),$(ut.$$.fragment,s),$(ps),$(_t.$$.fragment,s),$(mt.$$.fragment,s),$(ht.$$.fragment,s),$(cs),$(Cs.$$.fragment,s),$(O),on=!0)},o(s){j(r.$$.fragment,s),j(g),j(G),j(ms.$$.fragment,s),j(ns),j(Us.$$.fragment,s),j(Bs.$$.fragment,s),j(Vs.$$.fragment,s),j(Js.$$.fragment,s),j(Ks.$$.fragment,s),j(Ys.$$.fragment,s),j(Ss.$$.fragment,s),j(rs),j(Qs.$$.fragment,s),j(Xs.$$.fragment,s),j(st.$$.fragment,s),j(ys.$$.fragment,s),j(tt.$$.fragment,s),j(et.$$.fragment,s),j(at.$$.fragment,s),j(nt.$$.fragment,s),j(ot.$$.fragment,s),j(it.$$.fragment,s),j(ct.$$.fragment,s),j(ut.$$.fragment,s),j(ps),j(_t.$$.fragment,s),j(mt.$$.fragment,s),j(ht.$$.fragment,s),j(cs),j(Cs.$$.fragment,s),j(O),on=!1},d(s){e(l),s&&e(u),S(r,s),s&&e(b),s&&e(k),s&&e(d),dt[x].d(s),s&&e(q),bt[N].d(s),s&&e(fs),s&&e(us),s&&e(B),s&&e(U),s&&e(R),s&&e(V),S(ms),s&&e(na),gt[as].d(s),s&&e(Tt),s&&e(J),s&&e(la),s&&e(St),s&&e(ra),S(Us,s),s&&e(oa),S(Bs,s),s&&e(pa),s&&e(I),s&&e(ia),s&&e(hs),s&&e(ca),s&&e(Ts),s&&e(fa),S(Vs,s),s&&e(ua),S(Js,s),s&&e(_a),s&&e(ds),s&&e(ma),S(Ks,s),s&&e(ha),S(Ys,s),s&&e(da),s&&e(A),s&&e(ba),S(Ss,s),s&&e(ga),s&&e(Zs),s&&e($a),$t[ls].d(s),s&&e(qt),s&&e(qs),s&&e(ja),S(Qs,s),s&&e(ka),s&&e(zt),s&&e(xa),S(Xs,s),s&&e(va),S(st,s),s&&e(wa),s&&e(F),s&&e(Ea),S(ys,s),s&&e(Ta),s&&e(zs),s&&e(Sa),S(tt,s),s&&e(qa),s&&e(Nt),s&&e(ya),S(et,s),s&&e(za),s&&e(bs),s&&e(Pa),S(at,s),s&&e(Na),s&&e(M),s&&e(Da),s&&e(Ps),s&&e(Ra),s&&e(Ns),s&&e(Ga),s&&e(Rt),s&&e(Oa),s&&e(Ds),s&&e(Aa),s&&e(gs),s&&e(Ca),S(nt,s),s&&e(Ia),s&&e(W),s&&e(Fa),s&&e(Rs),s&&e(La),S(ot,s),s&&e(Ma),s&&e(D),s&&e(Wa),s&&e(Gs),s&&e(Ha),s&&e($s),s&&e(Ua),S(it,s),s&&e(Ba),s&&e(At),s&&e(Va),S(ct,s),s&&e(Ja),s&&e(K),s&&e(Ka),s&&e(L),s&&e(Ya),s&&e(Ct),s&&e(Za),s&&e(ft),s&&e(Qa),S(ut,s),s&&e(Xa),xs.d(s),s&&e(It),s&&e(Os),s&&e(sn),jt[os].d(s),s&&e(Ft),s&&e(Y),s&&e(tn),S(_t,s),s&&e(en),S(mt,s),s&&e(an),s&&e(As),s&&e(nn),S(ht,s),s&&e(ln),kt[is].d(s),s&&e(Lt),S(Cs,s),s&&e(rn),O&&O.d(s),s&&e(Mt)}}}const Ni={local:"",sections:[{local:"hub",title:"\u4ECE\u6A21\u578B\u4E2D\u5FC3\uFF08Hub\uFF09\u52A0\u8F7D\u6570\u636E\u96C6"},{local:"",title:"\u9884\u5904\u7406\u6570\u636E\u96C6"},{local:"",title:"\u52A8\u6001\u586B\u5145"}],title:"\u5904\u7406\u6570\u636E"};function Di(v,l,u){let r="pt";return fi(()=>{const b=new URLSearchParams(window.location.search);u(0,r=b.get("fw")||"pt")}),[r]}class Li extends oi{constructor(l){super();pi(this,l,Di,Pi,ii,{})}}export{Li as default,Ni as metadata};
