import{S as vf,i as wf,s as gf,e as o,t,k as f,w,c as p,a as i,h as a,d as s,m as _,x as g,b as P,g as n,G as l,y as x,q as j,o as k,B as O,M as xf,N as vi,p as Yr,v as Of,n as Vr,L as bf}from"../../chunks/vendor-hf-doc-builder.js";import{T as Jr}from"../../chunks/Tip-hf-doc-builder.js";import{Y as $f}from"../../chunks/Youtube-hf-doc-builder.js";import{I as wi}from"../../chunks/IconCopyLink-hf-doc-builder.js";import{C as T}from"../../chunks/CodeBlock-hf-doc-builder.js";import{C as Ef}from"../../chunks/CourseFloatingBanner-hf-doc-builder.js";import{F as yf}from"../../chunks/FrameworkSwitchCourse-hf-doc-builder.js";function Cf(H){let u,$;return u=new Ef({props:{chapter:7,classNames:"absolute z-10 right-0 top-0",notebooks:[{label:"Google Colab",value:"https://colab.research.google.com/github/huggingface/notebooks/blob/master/course/ja/chapter7/section2_tf.ipynb"},{label:"Aws Studio",value:"https://studiolab.sagemaker.aws/import/github/huggingface/notebooks/blob/master/course/ja/chapter7/section2_tf.ipynb"}]}}),{c(){w(u.$$.fragment)},l(d){g(u.$$.fragment,d)},m(d,b){x(u,d,b),$=!0},i(d){$||(j(u.$$.fragment,d),$=!0)},o(d){k(u.$$.fragment,d),$=!1},d(d){O(u,d)}}}function Df(H){let u,$;return u=new Ef({props:{chapter:7,classNames:"absolute z-10 right-0 top-0",notebooks:[{label:"Google Colab",value:"https://colab.research.google.com/github/huggingface/notebooks/blob/master/course/ja/chapter7/section2_pt.ipynb"},{label:"Aws Studio",value:"https://studiolab.sagemaker.aws/import/github/huggingface/notebooks/blob/master/course/ja/chapter7/section2_pt.ipynb"}]}}),{c(){w(u.$$.fragment)},l(d){g(u.$$.fragment,d)},m(d,b){x(u,d,b),$=!0},i(d){$||(j(u.$$.fragment,d),$=!0)},o(d){k(u.$$.fragment,d),$=!1},d(d){O(u,d)}}}function qf(H){let u,$,d,b,D,E,v,q;return{c(){u=o("p"),$=t("\u{1F4A1} \u5358\u8A9E\u3068\u305D\u308C\u306B\u5BFE\u5FDC\u3059\u308B\u30E9\u30D9\u30EB\u306B\u5206\u5272\u3055\u308C\u305F\u30C6\u30AD\u30B9\u30C8\u304B\u3089\u306A\u308B\u30C7\u30FC\u30BF\u30BB\u30C3\u30C8\u3067\u3042\u308C\u3070\u3001\u3053\u3053\u3067\u8AAC\u660E\u3059\u308B\u30C7\u30FC\u30BF\u51E6\u7406\u3092\u81EA\u5206\u306E\u30C7\u30FC\u30BF\u30BB\u30C3\u30C8\u306B\u9069\u7528\u3059\u308B\u3053\u3068\u304C\u3067\u304D\u307E\u3059\u3002\u72EC\u81EA\u306E\u30C7\u30FC\u30BF\u3092 "),d=o("code"),b=t("Dataset"),D=t(" \u306B\u30ED\u30FC\u30C9\u3059\u308B\u65B9\u6CD5\u306B\u3064\u3044\u3066\u5FA9\u7FD2\u304C\u5FC5\u8981\u306A\u5834\u5408\u306F\u3001"),E=o("a"),v=t("\u7B2C5\u7AE0"),q=t(" \u3092\u53C2\u7167\u3057\u3066\u304F\u3060\u3055\u3044\u3002"),this.h()},l(C){u=p(C,"P",{});var z=i(u);$=a(z,"\u{1F4A1} \u5358\u8A9E\u3068\u305D\u308C\u306B\u5BFE\u5FDC\u3059\u308B\u30E9\u30D9\u30EB\u306B\u5206\u5272\u3055\u308C\u305F\u30C6\u30AD\u30B9\u30C8\u304B\u3089\u306A\u308B\u30C7\u30FC\u30BF\u30BB\u30C3\u30C8\u3067\u3042\u308C\u3070\u3001\u3053\u3053\u3067\u8AAC\u660E\u3059\u308B\u30C7\u30FC\u30BF\u51E6\u7406\u3092\u81EA\u5206\u306E\u30C7\u30FC\u30BF\u30BB\u30C3\u30C8\u306B\u9069\u7528\u3059\u308B\u3053\u3068\u304C\u3067\u304D\u307E\u3059\u3002\u72EC\u81EA\u306E\u30C7\u30FC\u30BF\u3092 "),d=p(z,"CODE",{});var B=i(d);b=a(B,"Dataset"),B.forEach(s),D=a(z," \u306B\u30ED\u30FC\u30C9\u3059\u308B\u65B9\u6CD5\u306B\u3064\u3044\u3066\u5FA9\u7FD2\u304C\u5FC5\u8981\u306A\u5834\u5408\u306F\u3001"),E=p(z,"A",{href:!0});var A=i(E);v=a(A,"\u7B2C5\u7AE0"),A.forEach(s),q=a(z," \u3092\u53C2\u7167\u3057\u3066\u304F\u3060\u3055\u3044\u3002"),z.forEach(s),this.h()},h(){P(E,"href","/course/ja/chapter5")},m(C,z){n(C,u,z),l(u,$),l(u,d),l(d,b),l(u,D),l(u,E),l(E,v),l(u,q)},d(C){C&&s(u)}}}function Pf(H){let u,$,d,b,D;return{c(){u=o("p"),$=t("\u270F\uFE0F "),d=o("strong"),b=t("\u3042\u306A\u305F\u306E\u756A\u3067\u3059\uFF01"),D=t(" \u540C\u30582\u3064\u306E\u6587\u3092POS\u30E9\u30D9\u30EB\u307E\u305F\u306F\u30C1\u30E3\u30F3\u30AD\u30F3\u30B0\u30E9\u30D9\u30EB\u3068\u4E00\u7DD2\u306B\u51FA\u529B\u3057\u3066\u304F\u3060\u3055\u3044\u3002")},l(E){u=p(E,"P",{});var v=i(u);$=a(v,"\u270F\uFE0F "),d=p(v,"STRONG",{});var q=i(d);b=a(q,"\u3042\u306A\u305F\u306E\u756A\u3067\u3059\uFF01"),q.forEach(s),D=a(v," \u540C\u30582\u3064\u306E\u6587\u3092POS\u30E9\u30D9\u30EB\u307E\u305F\u306F\u30C1\u30E3\u30F3\u30AD\u30F3\u30B0\u30E9\u30D9\u30EB\u3068\u4E00\u7DD2\u306B\u51FA\u529B\u3057\u3066\u304F\u3060\u3055\u3044\u3002"),v.forEach(s)},m(E,v){n(E,u,v),l(u,$),l(u,d),l(d,b),l(u,D)},d(E){E&&s(u)}}}function zf(H){let u,$,d,b,D,E,v,q;return{c(){u=o("p"),$=t("\u270F\uFE0F "),d=o("strong"),b=t("\u3042\u306A\u305F\u306E\u756A\u3067\u3059\uFF01"),D=t(" \u7814\u7A76\u8005\u306E\u4E2D\u306B\u306F\u30011\u3064\u306E\u5358\u8A9E\u306B\u306F1\u3064\u306E\u30E9\u30D9\u30EB\u3057\u304B\u4ED8\u3051\u305A\u3001\u4E0E\u3048\u3089\u308C\u305F\u5358\u8A9E\u5185\u306E\u4ED6\u306E\u30B5\u30D6\u30C8\u30FC\u30AF\u30F3\u306B"),E=o("code"),v=t("-100"),q=t("\u3092\u5272\u308A\u5F53\u3066\u308B\u3053\u3068\u3092\u597D\u3080\u4EBA\u3082\u3044\u307E\u3059\u3002\u3053\u308C\u306F\u3001\u591A\u304F\u306E\u30B5\u30D6\u30C8\u30FC\u30AF\u30F3\u306B\u5206\u5272\u3055\u308C\u308B\u9577\u3044\u5358\u8A9E\u304C\u5B66\u7FD2\u6642\u306E\u640D\u5931\u306B\u5927\u304D\u304F\u5BC4\u4E0E\u3059\u308B\u306E\u3092\u907F\u3051\u308B\u305F\u3081\u3067\u3059\u3002\u3053\u306E\u30EB\u30FC\u30EB\u306B\u5F93\u3063\u3066\u3001\u30E9\u30D9\u30EB\u3068\u5165\u529BID\u3092\u4E00\u81F4\u3055\u305B\u308B\u3088\u3046\u306B\u3001\u524D\u306E\u95A2\u6570\u3092\u5909\u66F4\u3057\u3066\u307F\u307E\u3057\u3087\u3046\u3002")},l(C){u=p(C,"P",{});var z=i(u);$=a(z,"\u270F\uFE0F "),d=p(z,"STRONG",{});var B=i(d);b=a(B,"\u3042\u306A\u305F\u306E\u756A\u3067\u3059\uFF01"),B.forEach(s),D=a(z," \u7814\u7A76\u8005\u306E\u4E2D\u306B\u306F\u30011\u3064\u306E\u5358\u8A9E\u306B\u306F1\u3064\u306E\u30E9\u30D9\u30EB\u3057\u304B\u4ED8\u3051\u305A\u3001\u4E0E\u3048\u3089\u308C\u305F\u5358\u8A9E\u5185\u306E\u4ED6\u306E\u30B5\u30D6\u30C8\u30FC\u30AF\u30F3\u306B"),E=p(z,"CODE",{});var A=i(E);v=a(A,"-100"),A.forEach(s),q=a(z,"\u3092\u5272\u308A\u5F53\u3066\u308B\u3053\u3068\u3092\u597D\u3080\u4EBA\u3082\u3044\u307E\u3059\u3002\u3053\u308C\u306F\u3001\u591A\u304F\u306E\u30B5\u30D6\u30C8\u30FC\u30AF\u30F3\u306B\u5206\u5272\u3055\u308C\u308B\u9577\u3044\u5358\u8A9E\u304C\u5B66\u7FD2\u6642\u306E\u640D\u5931\u306B\u5927\u304D\u304F\u5BC4\u4E0E\u3059\u308B\u306E\u3092\u907F\u3051\u308B\u305F\u3081\u3067\u3059\u3002\u3053\u306E\u30EB\u30FC\u30EB\u306B\u5F93\u3063\u3066\u3001\u30E9\u30D9\u30EB\u3068\u5165\u529BID\u3092\u4E00\u81F4\u3055\u305B\u308B\u3088\u3046\u306B\u3001\u524D\u306E\u95A2\u6570\u3092\u5909\u66F4\u3057\u3066\u307F\u307E\u3057\u3087\u3046\u3002"),z.forEach(s)},m(C,z){n(C,u,z),l(u,$),l(u,d),l(d,b),l(u,D),l(u,E),l(E,v),l(u,q)},d(C){C&&s(u)}}}function Tf(H){let u,$,d,b,D,E,v,q,C,z,B;return b=new wi({}),{c(){u=o("h2"),$=o("a"),d=o("span"),w(b.$$.fragment),D=f(),E=o("span"),v=t("Keras \u3092\u4F7F\u3063\u3066\u30E2\u30C7\u30EB\u3092\u5FAE\u8ABF\u6574\u3059\u308B"),q=f(),C=o("p"),z=t("Keras\u3092\u4F7F\u3063\u305F\u5B9F\u969B\u306E\u30B3\u30FC\u30C9\u306F\u3001\u3053\u308C\u307E\u3067\u3068\u307B\u3068\u3093\u3069\u540C\u3058\u3067\u3001\u30C7\u30FC\u30BF\u3092\u30D0\u30C3\u30C1\u5316\u3059\u308B\u65B9\u6CD5\u3068\u3001\u6307\u6A19\u8A08\u7B97\u306E\u95A2\u6570\u304C\u5909\u308F\u308B\u3060\u3051\u3067\u3059\u3002"),this.h()},l(A){u=p(A,"H2",{class:!0});var R=i(u);$=p(R,"A",{id:!0,class:!0,href:!0});var U=i($);d=p(U,"SPAN",{});var y=i(d);g(b.$$.fragment,y),y.forEach(s),U.forEach(s),D=_(R),E=p(R,"SPAN",{});var M=i(E);v=a(M,"Keras \u3092\u4F7F\u3063\u3066\u30E2\u30C7\u30EB\u3092\u5FAE\u8ABF\u6574\u3059\u308B"),M.forEach(s),R.forEach(s),q=_(A),C=p(A,"P",{});var K=i(C);z=a(K,"Keras\u3092\u4F7F\u3063\u305F\u5B9F\u969B\u306E\u30B3\u30FC\u30C9\u306F\u3001\u3053\u308C\u307E\u3067\u3068\u307B\u3068\u3093\u3069\u540C\u3058\u3067\u3001\u30C7\u30FC\u30BF\u3092\u30D0\u30C3\u30C1\u5316\u3059\u308B\u65B9\u6CD5\u3068\u3001\u6307\u6A19\u8A08\u7B97\u306E\u95A2\u6570\u304C\u5909\u308F\u308B\u3060\u3051\u3067\u3059\u3002"),K.forEach(s),this.h()},h(){P($,"id","keras"),P($,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),P($,"href","#keras"),P(u,"class","relative group")},m(A,R){n(A,u,R),l(u,$),l($,d),x(b,d,null),l(u,D),l(u,E),l(E,v),n(A,q,R),n(A,C,R),l(C,z),B=!0},i(A){B||(j(b.$$.fragment,A),B=!0)},o(A){k(b.$$.fragment,A),B=!1},d(A){A&&s(u),O(b),A&&s(q),A&&s(C)}}}function If(H){let u,$,d,b,D,E,v,q,C,z,B,A,R,U;return b=new wi({}),{c(){u=o("h2"),$=o("a"),d=o("span"),w(b.$$.fragment),D=f(),E=o("span"),v=t("Trainer API \u3067\u30E2\u30C7\u30EB\u3092\u5FAE\u8ABF\u6574\u3059\u308B"),q=f(),C=o("p"),z=t("\u5B9F\u969B\u306B "),B=o("code"),A=t("Trainer"),R=t(" \u3092\u4F7F\u7528\u3059\u308B\u30B3\u30FC\u30C9\u306F\u3001\u3053\u308C\u307E\u3067\u3068\u540C\u3058\u3067\u3059\u3002\u5909\u66F4\u70B9\u306F\u3001\u30C7\u30FC\u30BF\u3092\u30D0\u30C3\u30C1\u5316\u3059\u308B\u65B9\u6CD5\u3068\u3001\u6307\u6A19\u3092\u8A08\u7B97\u3059\u308B\u95A2\u6570\u3060\u3051\u3067\u3059\u3002"),this.h()},l(y){u=p(y,"H2",{class:!0});var M=i(u);$=p(M,"A",{id:!0,class:!0,href:!0});var K=i($);d=p(K,"SPAN",{});var Z=i(d);g(b.$$.fragment,Z),Z.forEach(s),K.forEach(s),D=_(M),E=p(M,"SPAN",{});var W=i(E);v=a(W,"Trainer API \u3067\u30E2\u30C7\u30EB\u3092\u5FAE\u8ABF\u6574\u3059\u308B"),W.forEach(s),M.forEach(s),q=_(y),C=p(y,"P",{});var I=i(C);z=a(I,"\u5B9F\u969B\u306B "),B=p(I,"CODE",{});var F=i(B);A=a(F,"Trainer"),F.forEach(s),R=a(I," \u3092\u4F7F\u7528\u3059\u308B\u30B3\u30FC\u30C9\u306F\u3001\u3053\u308C\u307E\u3067\u3068\u540C\u3058\u3067\u3059\u3002\u5909\u66F4\u70B9\u306F\u3001\u30C7\u30FC\u30BF\u3092\u30D0\u30C3\u30C1\u5316\u3059\u308B\u65B9\u6CD5\u3068\u3001\u6307\u6A19\u3092\u8A08\u7B97\u3059\u308B\u95A2\u6570\u3060\u3051\u3067\u3059\u3002"),I.forEach(s),this.h()},h(){P($,"id","trainer-api"),P($,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),P($,"href","#trainer-api"),P(u,"class","relative group")},m(y,M){n(y,u,M),l(u,$),l($,d),x(b,d,null),l(u,D),l(u,E),l(E,v),n(y,q,M),n(y,C,M),l(C,z),l(C,B),l(B,A),l(C,R),U=!0},i(y){U||(j(b.$$.fragment,y),U=!0)},o(y){k(b.$$.fragment,y),U=!1},d(y){y&&s(u),O(b),y&&s(q),y&&s(C)}}}function Af(H){let u,$;return u=new T({props:{code:`from transformers import DataCollatorForTokenClassification

data_collator = DataCollatorForTokenClassification(
    tokenizer=tokenizer, return_tensors="tf"
)`,highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> DataCollatorForTokenClassification

data_collator = DataCollatorForTokenClassification(
    tokenizer=tokenizer, return_tensors=<span class="hljs-string">&quot;tf&quot;</span>
)`}}),{c(){w(u.$$.fragment)},l(d){g(u.$$.fragment,d)},m(d,b){x(u,d,b),$=!0},i(d){$||(j(u.$$.fragment,d),$=!0)},o(d){k(u.$$.fragment,d),$=!1},d(d){O(u,d)}}}function Rf(H){let u,$;return u=new T({props:{code:`from transformers import DataCollatorForTokenClassification

data_collator = DataCollatorForTokenClassification(tokenizer=tokenizer)`,highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> DataCollatorForTokenClassification

data_collator = DataCollatorForTokenClassification(tokenizer=tokenizer)`}}),{c(){w(u.$$.fragment)},l(d){g(u.$$.fragment,d)},m(d,b){x(u,d,b),$=!0},i(d){$||(j(u.$$.fragment,d),$=!0)},o(d){k(u.$$.fragment,d),$=!1},d(d){O(u,d)}}}function Sf(H){let u,$,d,b,D,E,v,q,C,z,B,A,R,U;return z=new T({props:{code:`tf_train_dataset = tokenized_datasets["train"].to_tf_dataset(
    columns=["attention_mask", "input_ids", "labels", "token_type_ids"],
    collate_fn=data_collator,
    shuffle=True,
    batch_size=16,
)

tf_eval_dataset = tokenized_datasets["validation"].to_tf_dataset(
    columns=["attention_mask", "input_ids", "labels", "token_type_ids"],
    collate_fn=data_collator,
    shuffle=False,
    batch_size=16,
)`,highlighted:`tf_train_dataset = tokenized_datasets[<span class="hljs-string">&quot;train&quot;</span>].to_tf_dataset(
    columns=[<span class="hljs-string">&quot;attention_mask&quot;</span>, <span class="hljs-string">&quot;input_ids&quot;</span>, <span class="hljs-string">&quot;labels&quot;</span>, <span class="hljs-string">&quot;token_type_ids&quot;</span>],
    collate_fn=data_collator,
    shuffle=<span class="hljs-literal">True</span>,
    batch_size=<span class="hljs-number">16</span>,
)

tf_eval_dataset = tokenized_datasets[<span class="hljs-string">&quot;validation&quot;</span>].to_tf_dataset(
    columns=[<span class="hljs-string">&quot;attention_mask&quot;</span>, <span class="hljs-string">&quot;input_ids&quot;</span>, <span class="hljs-string">&quot;labels&quot;</span>, <span class="hljs-string">&quot;token_type_ids&quot;</span>],
    collate_fn=data_collator,
    shuffle=<span class="hljs-literal">False</span>,
    batch_size=<span class="hljs-number">16</span>,
)`}}),{c(){u=o("p"),$=t(`\u30C7\u30FC\u30BF\u7167\u5408\u306E\u6E96\u5099\u304C\u3067\u304D\u307E\u3057\u305F\uFF01
\u3067\u306F\u3001\u3053\u308C\u3092\u4F7F\u3063\u3066 `),d=o("code"),b=t("tf.data.Dataset"),D=t(" \u3092 "),E=o("code"),v=t("to_tf_dataset()"),q=t(" \u30E1\u30BD\u30C3\u30C9\u3092\u4F7F\u3063\u3066\u4F5C\u3063\u3066\u307F\u307E\u3057\u3087\u3046\u3002"),C=f(),w(z.$$.fragment),B=f(),A=o("p"),R=t("\u6B21\u306F\u3001\u30E2\u30C7\u30EB\u672C\u4F53\u3067\u3059\u3002")},l(y){u=p(y,"P",{});var M=i(u);$=a(M,`\u30C7\u30FC\u30BF\u7167\u5408\u306E\u6E96\u5099\u304C\u3067\u304D\u307E\u3057\u305F\uFF01
\u3067\u306F\u3001\u3053\u308C\u3092\u4F7F\u3063\u3066 `),d=p(M,"CODE",{});var K=i(d);b=a(K,"tf.data.Dataset"),K.forEach(s),D=a(M," \u3092 "),E=p(M,"CODE",{});var Z=i(E);v=a(Z,"to_tf_dataset()"),Z.forEach(s),q=a(M," \u30E1\u30BD\u30C3\u30C9\u3092\u4F7F\u3063\u3066\u4F5C\u3063\u3066\u307F\u307E\u3057\u3087\u3046\u3002"),M.forEach(s),C=_(y),g(z.$$.fragment,y),B=_(y),A=p(y,"P",{});var W=i(A);R=a(W,"\u6B21\u306F\u3001\u30E2\u30C7\u30EB\u672C\u4F53\u3067\u3059\u3002"),W.forEach(s)},m(y,M){n(y,u,M),l(u,$),l(u,d),l(d,b),l(u,D),l(u,E),l(E,v),l(u,q),n(y,C,M),x(z,y,M),n(y,B,M),n(y,A,M),l(A,R),U=!0},i(y){U||(j(z.$$.fragment,y),U=!0)},o(y){k(z.$$.fragment,y),U=!1},d(y){y&&s(u),y&&s(C),O(z,y),y&&s(B),y&&s(A)}}}function Bf(H){let u,$,d,b,D;return{c(){u=o("p"),$=t("\u898B\u3066\u306E\u901A\u308A\u30012\u3064\u76EE\u306E\u30E9\u30D9\u30EB\u306E\u30BB\u30C3\u30C8\u306F\u6700\u521D\u306E\u30E9\u30D9\u30EB\u306E\u9577\u3055\u306B "),d=o("code"),b=t("-100"),D=t("s \u3092\u4F7F\u3063\u3066\u30D1\u30C7\u30A3\u30F3\u30B0\u3055\u308C\u3066\u3044\u307E\u3059\uFF0E")},l(E){u=p(E,"P",{});var v=i(u);$=a(v,"\u898B\u3066\u306E\u901A\u308A\u30012\u3064\u76EE\u306E\u30E9\u30D9\u30EB\u306E\u30BB\u30C3\u30C8\u306F\u6700\u521D\u306E\u30E9\u30D9\u30EB\u306E\u9577\u3055\u306B "),d=p(v,"CODE",{});var q=i(d);b=a(q,"-100"),q.forEach(s),D=a(v,"s \u3092\u4F7F\u3063\u3066\u30D1\u30C7\u30A3\u30F3\u30B0\u3055\u308C\u3066\u3044\u307E\u3059\uFF0E"),v.forEach(s)},m(E,v){n(E,u,v),l(u,$),l(u,d),l(d,b),l(u,D)},i:bf,o:bf,d(E){E&&s(u)}}}function jf(H){let u,$,d,b,D,E,v,q,C,z,B,A,R,U,y,M,K,Z,W,I,F,L,X,Q,V,le,J,ce,Y,G,te,$e,N,ee,je,Te,se,fe,ae,ge,re,fs,ue,ke,Ml,xe,Le,Ne,Gs,Ll,Ie,$l,jl,Oe,Cs,ss,He,Nl,_s,Ge,Hl,de,ye,Us,Ce,it,Ds,Gl,us,qs,Ee,ls,ts,Ae,kl,ne,ct,ms,ds,ft,hs,he,_t,Ps,De,Ul,me,El,Ws,Wl,zs,Ts,oe,Xs,as,Xl,ve,Zs,Ue,ut,vl,Ks,Ys,Zl,We,bs,rs,$s,Xe,Is,Vs,wl,qe,js,Ze,gl,Pe,ks,Js;return W=new T({props:{code:`id2label = {i: label for i, label in enumerate(label_names)}
label2id = {v: k for k, v in id2label.items()}`,highlighted:`id2label = {i: label <span class="hljs-keyword">for</span> i, label <span class="hljs-keyword">in</span> <span class="hljs-built_in">enumerate</span>(label_names)}
label2id = {v: k <span class="hljs-keyword">for</span> k, v <span class="hljs-keyword">in</span> id2label.items()}`}}),J=new T({props:{code:`from transformers import TFAutoModelForTokenClassification

model = TFAutoModelForTokenClassification.from_pretrained(
    model_checkpoint,
    id2label=id2label,
    label2id=label2id,
)`,highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> TFAutoModelForTokenClassification

model = TFAutoModelForTokenClassification.from_pretrained(
    model_checkpoint,
    id2label=id2label,
    label2id=label2id,
)`}}),se=new T({props:{code:"model.config.num_labels",highlighted:"model.config.num_labels"}}),ae=new T({props:{code:"9",highlighted:'<span class="hljs-number">9</span>'}}),re=new Jr({props:{warning:!0,$$slots:{default:[Ff]},$$scope:{ctx:H}}}),Gs=new T({props:{code:`from huggingface_hub import notebook_login

notebook_login()`,highlighted:`<span class="hljs-keyword">from</span> huggingface_hub <span class="hljs-keyword">import</span> notebook_login

notebook_login()`}}),He=new T({props:{code:"huggingface-cli login",highlighted:"huggingface-cli login"}}),Ae=new T({props:{code:`from transformers import create_optimizer
import tensorflow as tf

# Train in mixed-precision float16
# Comment this line out if you're using a GPU that will not benefit from this
tf.keras.mixed_precision.set_global_policy("mixed_float16")

# The number of training steps is the number of samples in the dataset, divided by the batch size then multiplied
# by the total number of epochs. Note that the tf_train_dataset here is a batched tf.data.Dataset,
# not the original Hugging Face Dataset, so its len() is already num_samples // batch_size.
num_epochs = 3
num_train_steps = len(tf_train_dataset) * num_epochs

optimizer, schedule = create_optimizer(
    init_lr=2e-5,
    num_warmup_steps=0,
    num_train_steps=num_train_steps,
    weight_decay_rate=0.01,
)
model.compile(optimizer=optimizer)`,highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> create_optimizer
<span class="hljs-keyword">import</span> tensorflow <span class="hljs-keyword">as</span> tf

<span class="hljs-comment"># Train in mixed-precision float16</span>
<span class="hljs-comment"># Comment this line out if you&#x27;re using a GPU that will not benefit from this</span>
tf.keras.mixed_precision.set_global_policy(<span class="hljs-string">&quot;mixed_float16&quot;</span>)

<span class="hljs-comment"># The number of training steps is the number of samples in the dataset, divided by the batch size then multiplied</span>
<span class="hljs-comment"># by the total number of epochs. Note that the tf_train_dataset here is a batched tf.data.Dataset,</span>
<span class="hljs-comment"># not the original Hugging Face Dataset, so its len() is already num_samples // batch_size.</span>
num_epochs = <span class="hljs-number">3</span>
num_train_steps = <span class="hljs-built_in">len</span>(tf_train_dataset) * num_epochs

optimizer, schedule = create_optimizer(
    init_lr=<span class="hljs-number">2e-5</span>,
    num_warmup_steps=<span class="hljs-number">0</span>,
    num_train_steps=num_train_steps,
    weight_decay_rate=<span class="hljs-number">0.01</span>,
)
model.<span class="hljs-built_in">compile</span>(optimizer=optimizer)`}}),zs=new T({props:{code:`from transformers.keras_callbacks import PushToHubCallback

callback = PushToHubCallback(output_dir="bert-finetuned-ner", tokenizer=tokenizer)

model.fit(
    tf_train_dataset,
    validation_data=tf_eval_dataset,
    callbacks=[callback],
    epochs=num_epochs,
)`,highlighted:`<span class="hljs-keyword">from</span> transformers.keras_callbacks <span class="hljs-keyword">import</span> PushToHubCallback

callback = PushToHubCallback(output_dir=<span class="hljs-string">&quot;bert-finetuned-ner&quot;</span>, tokenizer=tokenizer)

model.fit(
    tf_train_dataset,
    validation_data=tf_eval_dataset,
    callbacks=[callback],
    epochs=num_epochs,
)`}}),Xe=new Jr({props:{$$slots:{default:[Mf]},$$scope:{ctx:H}}}),{c(){u=o("h3"),$=t("\u30E2\u30C7\u30EB\u306E\u5B9A\u7FA9"),d=f(),b=o("p"),D=t("\u4ECA\u56DE\u306F\u30C8\u30FC\u30AF\u30F3\u5206\u985E\u306E\u554F\u984C\u3092\u6271\u3046\u306E\u3067\u3001 "),E=o("code"),v=t("TFAutoModelForTokenClassification"),q=t(" \u30AF\u30E9\u30B9\u3092\u4F7F\u7528\u3057\u307E\u3059\u3002\u3053\u306E\u30E2\u30C7\u30EB\u3092\u5B9A\u7FA9\u3059\u308B\u969B\u306B\u899A\u3048\u3066\u304A\u304F\u3079\u304D\u3053\u3068\u306F\u3001\u30E9\u30D9\u30EB\u306E\u6570\u306B\u95A2\u3059\u308B\u60C5\u5831\u3092\u6E21\u3059\u3053\u3068\u3067\u3059\u3002\u6700\u3082\u7C21\u5358\u306A\u65B9\u6CD5\u306F "),C=o("code"),z=t("num_labels"),B=t(" \u5F15\u6570\u3067\u305D\u306E\u6570\u3092\u6E21\u3059\u3053\u3068\u3067\u3059\u304C\u3001\u3053\u306E\u30BB\u30AF\u30B7\u30E7\u30F3\u306E\u6700\u521D\u306B\u898B\u305F\u3088\u3046\u306A\u7D20\u6575\u306A\u63A8\u8AD6\u30A6\u30A3\u30B8\u30A7\u30C3\u30C8\u3092\u52D5\u4F5C\u3055\u305B\u305F\u3044\u5834\u5408\u306F\u3001\u4EE3\u308F\u308A\u306B\u6B63\u3057\u3044\u30E9\u30D9\u30EB\u306E\u5BFE\u5FDC\u95A2\u4FC2\u3092\u8A2D\u5B9A\u3057\u305F\u65B9\u304C\u826F\u3044\u3067\u3057\u3087\u3046\u3002"),A=f(),R=o("p"),U=t("id2label"),y=o("code"),M=t("\u3068"),K=t("label2id` \u3068\u3044\u3046 2 \u3064\u306E\u8F9E\u66F8\u578B\u30C7\u30FC\u30BF\u304C\u3042\u308A\u3001ID \u304B\u3089\u30E9\u30D9\u30EB\u3001\u30E9\u30D9\u30EB\u304B\u3089 ID \u3078\u306E\u30DE\u30C3\u30D4\u30F3\u30B0\u3092\u8A2D\u5B9A\u3059\u308B\u3053\u3068\u304C\u3067\u304D\u307E\u3059\u3002"),Z=f(),w(W.$$.fragment),I=f(),F=o("p"),L=t("\u3042\u3068\u306F\u305D\u308C\u3089\u3092 "),X=o("code"),Q=t("TFAutoModelForTokenClassification.from_pretrained()"),V=t(" \u30E1\u30BD\u30C3\u30C9\u306B\u6E21\u305B\u3070\u3001\u30E2\u30C7\u30EB\u306E\u8A2D\u5B9A\u306B\u30BB\u30C3\u30C8\u3055\u308C\u3001\u9069\u5207\u306B\u4FDD\u5B58\u3055\u308C\u3066Hub\u306B\u30A2\u30C3\u30D7\u30ED\u30FC\u30C9\u3055\u308C\u308B\u3067\u3057\u3087\u3046\u3002"),le=f(),w(J.$$.fragment),ce=f(),Y=o("p"),G=o("a"),te=t("\u7B2C3\u7AE0"),$e=t(" \u3067 "),N=o("code"),ee=t("TFAutoModelForSequenceClassification"),je=t(" \u3092\u5B9A\u7FA9\u3057\u305F\u3068\u304D\u306E\u3088\u3046\u306B\u3001\u30E2\u30C7\u30EB\u3092\u4F5C\u6210\u3059\u308B\u3068\u3001\u3044\u304F\u3064\u304B\u306E\u91CD\u307F\u304C\u4F7F\u308F\u308C\u3066\u3044\u306A\u3044\uFF08\u4E8B\u524D\u5B66\u7FD2\u6E08\u307F\u30E2\u30C7\u30EB\u306E\u30D8\u30C3\u30C9\u90E8\u306E\u91CD\u307F\uFF09\u3001\u4ED6\u306E\u3044\u304F\u3064\u304B\u306E\u91CD\u307F\u304C\u30E9\u30F3\u30C0\u30E0\u306B\u521D\u671F\u5316\u3055\u308C\u3066\u3044\u308B\uFF08\u65B0\u3057\u304F\u63A5\u7D9A\u3057\u305F\u30C8\u30FC\u30AF\u30F3\u5206\u985E\u30D8\u30C3\u30C9\u306E\u91CD\u307F\uFF09\u3001\u3053\u306E\u30E2\u30C7\u30EB\u306F\u30C8\u30EC\u30FC\u30CB\u30F3\u30B0\u3059\u308B\u5FC5\u8981\u304C\u3042\u308B\u3068\u3044\u3046\u8B66\u544A\u304C\u8868\u793A\u3055\u308C\u307E\u3059\u3002\u30C8\u30EC\u30FC\u30CB\u30F3\u30B0\u306F\u3059\u3050\u306B\u3067\u3082\u5B9F\u884C\u3067\u304D\u307E\u3059\u304C\u3001\u307E\u305A\u306F\u3053\u306E\u30E2\u30C7\u30EB\u304C\u6B63\u3057\u3044\u6570\u306E\u30E9\u30D9\u30EB\u3092\u6301\u3064\u3053\u3068\u3092\u518D\u78BA\u8A8D\u3057\u307E\u3057\u3087\u3046\u3002"),Te=f(),w(se.$$.fragment),fe=f(),w(ae.$$.fragment),ge=f(),w(re.$$.fragment),fs=f(),ue=o("h3"),ke=t("\u30E2\u30C7\u30EB\u306E\u5FAE\u8ABF\u6574"),Ml=f(),xe=o("p"),Le=t(`\u3053\u308C\u3067\u30E2\u30C7\u30EB\u3092\u5B66\u7FD2\u3059\u308B\u6E96\u5099\u304C\u6574\u3044\u307E\u3057\u305F\uFF01
\u3057\u304B\u3057\u3001\u305D\u306E\u524D\u306B\u3082\u3046\u5C11\u3057\u3060\u3051\u3084\u308B\u3079\u304D\u3053\u3068\u304C\u3042\u308A\u307E\u3059\u3002Hugging Face\u306B\u30ED\u30B0\u30A4\u30F3\u3057\u3001\u5B66\u7FD2\u7528\u30CF\u30A4\u30D1\u30FC\u30D1\u30E9\u30E1\u30FC\u30BF\u3092\u5B9A\u7FA9\u3059\u308B\u5FC5\u8981\u304C\u3042\u308A\u307E\u3059\u3002\u3082\u3057Notebook\u3067\u4F5C\u696D\u3057\u3066\u3044\u308B\u306A\u3089\u3001\u3053\u308C\u3092\u52A9\u3051\u308B\u4FBF\u5229\u306A\u95A2\u6570\u304C\u3042\u308A\u307E\u3059\u3002`),Ne=f(),w(Gs.$$.fragment),Ll=f(),Ie=o("p"),$l=t("\u3053\u308C\u306B\u3088\u308A\u3001Hugging Face\u306E\u30ED\u30B0\u30A4\u30F3\u60C5\u5831\u3092\u5165\u529B\u3059\u308B\u30A6\u30A3\u30B8\u30A7\u30C3\u30C8\u304C\u8868\u793A\u3055\u308C\u307E\u3059\u3002"),jl=f(),Oe=o("p"),Cs=t("Notebook\u3067\u4F5C\u696D\u3057\u3066\u3044\u306A\u3044\u5834\u5408\u306F\u3001\u30BF\u30FC\u30DF\u30CA\u30EB\u306B\u6B21\u306E\u884C\u3092\u5165\u529B\u3059\u308B\u3060\u3051\u3067\u3059\u3002"),ss=f(),w(He.$$.fragment),Nl=f(),_s=o("p"),Ge=t("\u30ED\u30B0\u30A4\u30F3\u3057\u305F\u3089\u3001\u30E2\u30C7\u30EB\u3092\u30B3\u30F3\u30D1\u30A4\u30EB\u3059\u308B\u305F\u3081\u306B\u5FC5\u8981\u306A\u3082\u306E\u3092\u3059\u3079\u3066\u6E96\u5099\u3057\u307E\u3059\u3002"),Hl=f(),de=o("p"),ye=t("\u{1F917} Transformers \u306F\u4FBF\u5229\u306A "),Us=o("code"),Ce=t("create_optimizer()"),it=t(" \u95A2\u6570\u3092\u63D0\u4F9B\u3057\u3066\u304A\u308A\u3001\u91CD\u307F\u306E\u6E1B\u8870\u3068\u5B66\u7FD2\u7387\u306E\u6E1B\u8870\u3092\u9069\u5207\u306B\u8A2D\u5B9A\u3057\u305F "),Ds=o("code"),Gl=t("AdamW"),us=t(" \u30AA\u30D7\u30C6\u30A3\u30DE\u30A4\u30B6\u304C\u5F97\u3089\u308C\u307E\u3059\u3002\u3053\u306E2\u3064\u306E\u8A2D\u5B9A\u306F\u3001\u7D44\u307F\u8FBC\u307F\u306E "),qs=o("code"),Ee=t("Adam"),ls=t(" \u30AA\u30D7\u30C6\u30A3\u30DE\u30A4\u30B6\u3068\u6BD4\u8F03\u3057\u3066\u3042\u306A\u305F\u306E\u30E2\u30C7\u30EB\u306E\u6027\u80FD\u3092\u5411\u4E0A\u3055\u305B\u308B\u3067\u3057\u3087\u3046\u3002"),ts=f(),w(Ae.$$.fragment),kl=f(),ne=o("p"),ct=t("\u307E\u305F\u3001"),ms=o("code"),ds=t("compile()"),ft=t("\u306B "),hs=o("code"),he=t("loss"),_t=t("\u5F15\u6570\u3092\u4E0E\u3048\u306A\u3044\u3053\u3068\u306B\u3082\u6CE8\u610F\u3057\u3066\u304F\u3060\u3055\u3044\u3002\u3053\u308C\u306F\u3001\u30E2\u30C7\u30EB\u304C\u5185\u90E8\u7684\u306B\u640D\u5931\u3092\u8A08\u7B97\u3059\u308B\u3053\u3068\u304C\u3067\u304D\u308B\u304B\u3089\u3067\u3059\u3002\u640D\u5931\u306A\u3057\u3067\u30B3\u30F3\u30D1\u30A4\u30EB\u3057\u3066\u3001\u5165\u529B\u8F9E\u66F8\u306B\u30E9\u30D9\u30EB\u3092\u6307\u5B9A\u3059\u308B\u3068\uFF08\u79C1\u305F\u3061\u306E\u30C7\u30FC\u30BF\u30BB\u30C3\u30C8\u3067\u884C\u3063\u3066\u3044\u308B\u3088\u3046\u306B\uFF09\u3001\u30E2\u30C7\u30EB\u306F\u305D\u306E\u5185\u90E8\u640D\u5931\u3092\u4F7F\u7528\u3057\u3066\u5B66\u7FD2\u3057\u3001\u305D\u308C\u306F\u9078\u3093\u3060\u30BF\u30B9\u30AF\u3068\u30E2\u30C7\u30EB\u30BF\u30A4\u30D7\u306B\u9069\u3057\u305F\u3082\u306E\u306B\u306A\u308B\u3067\u3057\u3087\u3046\u3002"),Ps=f(),De=o("p"),Ul=t("\u6B21\u306B\u3001\u5B66\u7FD2\u4E2D\u306B\u30E2\u30C7\u30EB\u3092Hub\u306B\u30A2\u30C3\u30D7\u30ED\u30FC\u30C9\u3059\u308B\u305F\u3081\u306E"),me=o("code"),El=t("PushToHubCallback"),Ws=t("\u3092\u5B9A\u7FA9\u3057\u3001\u305D\u306E\u30B3\u30FC\u30EB\u30D0\u30C3\u30AF\u3067\u30E2\u30C7\u30EB\u3092\u30D5\u30A3\u30C3\u30C8\u3055\u305B\u307E\u3059\u3002"),Wl=f(),w(zs.$$.fragment),Ts=f(),oe=o("p"),Xs=o("code"),as=t("hub_model_id"),Xl=t(" \u5F15\u6570\u306B\u306F\u3001\u30D7\u30C3\u30B7\u30E5\u3057\u305F\u3044\u30EA\u30DD\u30B8\u30C8\u30EA\u306E\u30D5\u30EB\u30CD\u30FC\u30E0\u3092\u6307\u5B9A\u3067\u304D\u307E\u3059\u3002 (\u7279\u306B\u3001\u7279\u5B9A\u306E\u7D44\u7E54\uFF08organization\uFF09\u306B\u30D7\u30C3\u30B7\u30E5\u3059\u308B\u5834\u5408\u306F\u3053\u306E\u5F15\u6570\u3092\u4F7F\u7528\u3059\u308B\u5FC5\u8981\u304C\u3042\u308A\u307E\u3059)\u3002\u4F8B\u3048\u3070\u3001\u30E2\u30C7\u30EB\u3092 "),ve=o("a"),Zs=o("code"),Ue=t("huggingface-course"),ut=t(" organization"),vl=t(" \u306B\u30D7\u30C3\u30B7\u30E5\u3059\u308B\u5834\u5408\u3001 "),Ks=o("code"),Ys=t('hub_model_id="huggingface-course/bert-finetuned-ner"'),Zl=t(" \u3092\u8FFD\u52A0\u3057\u307E\u3059\u3002\u30C7\u30D5\u30A9\u30EB\u30C8\u3067\u306F\u3001\u4F7F\u7528\u3055\u308C\u308B\u30EA\u30DD\u30B8\u30C8\u30EA\u306F\u3042\u306A\u305F\u306E\u540D\u524D\u304C\u4F7F\u308F\u308C\u3001\u8A2D\u5B9A\u3057\u305F\u51FA\u529B\u30C7\u30A3\u30EC\u30AF\u30C8\u30EA\u306B\u3061\u306A\u3093\u3060\u540D\u524D\u3001\u4F8B\u3048\u3070 "),We=o("code"),bs=t('"cool_huggingface_user/bert-finetuned-ner"'),rs=t(" \u3068\u306A\u308A\u307E\u3059\u3002"),$s=f(),w(Xe.$$.fragment),Is=f(),Vs=o("p"),wl=t("\u5B66\u7FD2\u304C\u884C\u308F\u308C\u3066\u3044\u308B\u9593\u3001\u30E2\u30C7\u30EB\u304C\u4FDD\u5B58\u3055\u308C\u308B\u305F\u3073\u306B\uFF08\u4ECA\u56DE\u306E\u4F8B\u3067\u306F\u30A8\u30DD\u30C3\u30AF\u3054\u3068\u306B\uFF09\u30D0\u30C3\u30AF\u30B0\u30E9\u30A6\u30F3\u30C9\u3067Hub\u306B\u30A2\u30C3\u30D7\u30ED\u30FC\u30C9\u3055\u308C\u308B\u3053\u3068\u306B\u6CE8\u610F\u3057\u3066\u304F\u3060\u3055\u3044\u3002\u3053\u306E\u3088\u3046\u306B\u3057\u3066\u304A\u3051\u3070\u3001\u5FC5\u8981\u306B\u5FDC\u3058\u3066\u5225\u306E\u30DE\u30B7\u30F3\u3067\u5B66\u7FD2\u3092\u518D\u958B\u3059\u308B\u3053\u3068\u304C\u3067\u304D\u307E\u3059\u3002\u3053\u306E\u6BB5\u968E\u3067\u3001Model Hub\u4E0A\u306E\u63A8\u8AD6\u30A6\u30A3\u30B8\u30A7\u30C3\u30C8\u3092\u4F7F\u3063\u3066\u30E2\u30C7\u30EB\u3092\u30C6\u30B9\u30C8\u3057\u3001\u53CB\u4EBA\u3068\u5171\u6709\u3059\u308B\u3053\u3068\u304C\u3067\u304D\u307E\u3059\u3002"),qe=f(),js=o("p"),Ze=t("\u3053\u308C\u3067\u30C8\u30FC\u30AF\u30F3\u5206\u985E\u30BF\u30B9\u30AF\u306E\u30E2\u30C7\u30EB\u5FAE\u8ABF\u6574\u306B\u6210\u529F\u3057\u307E\u3057\u305F\u3002\u304A\u3081\u3067\u3068\u3046\u3054\u3056\u3044\u307E\u3059\uFF01"),gl=f(),Pe=o("p"),ks=t("\u3057\u304B\u3057\u3001\u3053\u306E\u30E2\u30C7\u30EB\u306E\u5B9F\u529B\u306F\u3044\u304B\u307B\u3069\u3067\u3057\u3087\u3046\u304B\uFF1F\u305D\u308C\u3092\u77E5\u308B\u305F\u3081\u306B\u3001\u3044\u304F\u3064\u304B\u306E\u6307\u6A19\u3092\u4F7F\u3063\u3066\u8A55\u4FA1\u3059\u308B\u5FC5\u8981\u304C\u3042\u308A\u307E\u3059\u3002"),this.h()},l(m){u=p(m,"H3",{id:!0});var S=i(u);$=a(S,"\u30E2\u30C7\u30EB\u306E\u5B9A\u7FA9"),S.forEach(s),d=_(m),b=p(m,"P",{});var As=i(b);D=a(As,"\u4ECA\u56DE\u306F\u30C8\u30FC\u30AF\u30F3\u5206\u985E\u306E\u554F\u984C\u3092\u6271\u3046\u306E\u3067\u3001 "),E=p(As,"CODE",{});var Rs=i(E);v=a(Rs,"TFAutoModelForTokenClassification"),Rs.forEach(s),q=a(As," \u30AF\u30E9\u30B9\u3092\u4F7F\u7528\u3057\u307E\u3059\u3002\u3053\u306E\u30E2\u30C7\u30EB\u3092\u5B9A\u7FA9\u3059\u308B\u969B\u306B\u899A\u3048\u3066\u304A\u304F\u3079\u304D\u3053\u3068\u306F\u3001\u30E9\u30D9\u30EB\u306E\u6570\u306B\u95A2\u3059\u308B\u60C5\u5831\u3092\u6E21\u3059\u3053\u3068\u3067\u3059\u3002\u6700\u3082\u7C21\u5358\u306A\u65B9\u6CD5\u306F "),C=p(As,"CODE",{});var Ss=i(C);z=a(Ss,"num_labels"),Ss.forEach(s),B=a(As," \u5F15\u6570\u3067\u305D\u306E\u6570\u3092\u6E21\u3059\u3053\u3068\u3067\u3059\u304C\u3001\u3053\u306E\u30BB\u30AF\u30B7\u30E7\u30F3\u306E\u6700\u521D\u306B\u898B\u305F\u3088\u3046\u306A\u7D20\u6575\u306A\u63A8\u8AD6\u30A6\u30A3\u30B8\u30A7\u30C3\u30C8\u3092\u52D5\u4F5C\u3055\u305B\u305F\u3044\u5834\u5408\u306F\u3001\u4EE3\u308F\u308A\u306B\u6B63\u3057\u3044\u30E9\u30D9\u30EB\u306E\u5BFE\u5FDC\u95A2\u4FC2\u3092\u8A2D\u5B9A\u3057\u305F\u65B9\u304C\u826F\u3044\u3067\u3057\u3087\u3046\u3002"),As.forEach(s),A=_(m),R=p(m,"P",{});var ns=i(R);U=a(ns,"id2label"),y=p(ns,"CODE",{});var Qs=i(y);M=a(Qs,"\u3068"),Qs.forEach(s),K=a(ns,"label2id` \u3068\u3044\u3046 2 \u3064\u306E\u8F9E\u66F8\u578B\u30C7\u30FC\u30BF\u304C\u3042\u308A\u3001ID \u304B\u3089\u30E9\u30D9\u30EB\u3001\u30E9\u30D9\u30EB\u304B\u3089 ID \u3078\u306E\u30DE\u30C3\u30D4\u30F3\u30B0\u3092\u8A2D\u5B9A\u3059\u308B\u3053\u3068\u304C\u3067\u304D\u307E\u3059\u3002"),ns.forEach(s),Z=_(m),g(W.$$.fragment,m),I=_(m),F=p(m,"P",{});var pe=i(F);L=a(pe,"\u3042\u3068\u306F\u305D\u308C\u3089\u3092 "),X=p(pe,"CODE",{});var mt=i(X);Q=a(mt,"TFAutoModelForTokenClassification.from_pretrained()"),mt.forEach(s),V=a(pe," \u30E1\u30BD\u30C3\u30C9\u306B\u6E21\u305B\u3070\u3001\u30E2\u30C7\u30EB\u306E\u8A2D\u5B9A\u306B\u30BB\u30C3\u30C8\u3055\u308C\u3001\u9069\u5207\u306B\u4FDD\u5B58\u3055\u308C\u3066Hub\u306B\u30A2\u30C3\u30D7\u30ED\u30FC\u30C9\u3055\u308C\u308B\u3067\u3057\u3087\u3046\u3002"),pe.forEach(s),le=_(m),g(J.$$.fragment,m),ce=_(m),Y=p(m,"P",{});var Re=i(Y);G=p(Re,"A",{href:!0});var dt=i(G);te=a(dt,"\u7B2C3\u7AE0"),dt.forEach(s),$e=a(Re," \u3067 "),N=p(Re,"CODE",{});var Bs=i(N);ee=a(Bs,"TFAutoModelForSequenceClassification"),Bs.forEach(s),je=a(Re," \u3092\u5B9A\u7FA9\u3057\u305F\u3068\u304D\u306E\u3088\u3046\u306B\u3001\u30E2\u30C7\u30EB\u3092\u4F5C\u6210\u3059\u308B\u3068\u3001\u3044\u304F\u3064\u304B\u306E\u91CD\u307F\u304C\u4F7F\u308F\u308C\u3066\u3044\u306A\u3044\uFF08\u4E8B\u524D\u5B66\u7FD2\u6E08\u307F\u30E2\u30C7\u30EB\u306E\u30D8\u30C3\u30C9\u90E8\u306E\u91CD\u307F\uFF09\u3001\u4ED6\u306E\u3044\u304F\u3064\u304B\u306E\u91CD\u307F\u304C\u30E9\u30F3\u30C0\u30E0\u306B\u521D\u671F\u5316\u3055\u308C\u3066\u3044\u308B\uFF08\u65B0\u3057\u304F\u63A5\u7D9A\u3057\u305F\u30C8\u30FC\u30AF\u30F3\u5206\u985E\u30D8\u30C3\u30C9\u306E\u91CD\u307F\uFF09\u3001\u3053\u306E\u30E2\u30C7\u30EB\u306F\u30C8\u30EC\u30FC\u30CB\u30F3\u30B0\u3059\u308B\u5FC5\u8981\u304C\u3042\u308B\u3068\u3044\u3046\u8B66\u544A\u304C\u8868\u793A\u3055\u308C\u307E\u3059\u3002\u30C8\u30EC\u30FC\u30CB\u30F3\u30B0\u306F\u3059\u3050\u306B\u3067\u3082\u5B9F\u884C\u3067\u304D\u307E\u3059\u304C\u3001\u307E\u305A\u306F\u3053\u306E\u30E2\u30C7\u30EB\u304C\u6B63\u3057\u3044\u6570\u306E\u30E9\u30D9\u30EB\u3092\u6301\u3064\u3053\u3068\u3092\u518D\u78BA\u8A8D\u3057\u307E\u3057\u3087\u3046\u3002"),Re.forEach(s),Te=_(m),g(se.$$.fragment,m),fe=_(m),g(ae.$$.fragment,m),ge=_(m),g(re.$$.fragment,m),fs=_(m),ue=p(m,"H3",{id:!0});var Kl=i(ue);ke=a(Kl,"\u30E2\u30C7\u30EB\u306E\u5FAE\u8ABF\u6574"),Kl.forEach(s),Ml=_(m),xe=p(m,"P",{});var Yl=i(xe);Le=a(Yl,`\u3053\u308C\u3067\u30E2\u30C7\u30EB\u3092\u5B66\u7FD2\u3059\u308B\u6E96\u5099\u304C\u6574\u3044\u307E\u3057\u305F\uFF01
\u3057\u304B\u3057\u3001\u305D\u306E\u524D\u306B\u3082\u3046\u5C11\u3057\u3060\u3051\u3084\u308B\u3079\u304D\u3053\u3068\u304C\u3042\u308A\u307E\u3059\u3002Hugging Face\u306B\u30ED\u30B0\u30A4\u30F3\u3057\u3001\u5B66\u7FD2\u7528\u30CF\u30A4\u30D1\u30FC\u30D1\u30E9\u30E1\u30FC\u30BF\u3092\u5B9A\u7FA9\u3059\u308B\u5FC5\u8981\u304C\u3042\u308A\u307E\u3059\u3002\u3082\u3057Notebook\u3067\u4F5C\u696D\u3057\u3066\u3044\u308B\u306A\u3089\u3001\u3053\u308C\u3092\u52A9\u3051\u308B\u4FBF\u5229\u306A\u95A2\u6570\u304C\u3042\u308A\u307E\u3059\u3002`),Yl.forEach(s),Ne=_(m),g(Gs.$$.fragment,m),Ll=_(m),Ie=p(m,"P",{});var ea=i(Ie);$l=a(ea,"\u3053\u308C\u306B\u3088\u308A\u3001Hugging Face\u306E\u30ED\u30B0\u30A4\u30F3\u60C5\u5831\u3092\u5165\u529B\u3059\u308B\u30A6\u30A3\u30B8\u30A7\u30C3\u30C8\u304C\u8868\u793A\u3055\u308C\u307E\u3059\u3002"),ea.forEach(s),jl=_(m),Oe=p(m,"P",{});var Vl=i(Oe);Cs=a(Vl,"Notebook\u3067\u4F5C\u696D\u3057\u3066\u3044\u306A\u3044\u5834\u5408\u306F\u3001\u30BF\u30FC\u30DF\u30CA\u30EB\u306B\u6B21\u306E\u884C\u3092\u5165\u529B\u3059\u308B\u3060\u3051\u3067\u3059\u3002"),Vl.forEach(s),ss=_(m),g(He.$$.fragment,m),Nl=_(m),_s=p(m,"P",{});var ht=i(_s);Ge=a(ht,"\u30ED\u30B0\u30A4\u30F3\u3057\u305F\u3089\u3001\u30E2\u30C7\u30EB\u3092\u30B3\u30F3\u30D1\u30A4\u30EB\u3059\u308B\u305F\u3081\u306B\u5FC5\u8981\u306A\u3082\u306E\u3092\u3059\u3079\u3066\u6E96\u5099\u3057\u307E\u3059\u3002"),ht.forEach(s),Hl=_(m),de=p(m,"P",{});var Se=i(de);ye=a(Se,"\u{1F917} Transformers \u306F\u4FBF\u5229\u306A "),Us=p(Se,"CODE",{});var Jl=i(Us);Ce=a(Jl,"create_optimizer()"),Jl.forEach(s),it=a(Se," \u95A2\u6570\u3092\u63D0\u4F9B\u3057\u3066\u304A\u308A\u3001\u91CD\u307F\u306E\u6E1B\u8870\u3068\u5B66\u7FD2\u7387\u306E\u6E1B\u8870\u3092\u9069\u5207\u306B\u8A2D\u5B9A\u3057\u305F "),Ds=p(Se,"CODE",{});var bt=i(Ds);Gl=a(bt,"AdamW"),bt.forEach(s),us=a(Se," \u30AA\u30D7\u30C6\u30A3\u30DE\u30A4\u30B6\u304C\u5F97\u3089\u308C\u307E\u3059\u3002\u3053\u306E2\u3064\u306E\u8A2D\u5B9A\u306F\u3001\u7D44\u307F\u8FBC\u307F\u306E "),qs=p(Se,"CODE",{});var el=i(qs);Ee=a(el,"Adam"),el.forEach(s),ls=a(Se," \u30AA\u30D7\u30C6\u30A3\u30DE\u30A4\u30B6\u3068\u6BD4\u8F03\u3057\u3066\u3042\u306A\u305F\u306E\u30E2\u30C7\u30EB\u306E\u6027\u80FD\u3092\u5411\u4E0A\u3055\u305B\u308B\u3067\u3057\u3087\u3046\u3002"),Se.forEach(s),ts=_(m),g(Ae.$$.fragment,m),kl=_(m),ne=p(m,"P",{});var Es=i(ne);ct=a(Es,"\u307E\u305F\u3001"),ms=p(Es,"CODE",{});var $t=i(ms);ds=a($t,"compile()"),$t.forEach(s),ft=a(Es,"\u306B "),hs=p(Es,"CODE",{});var Fs=i(hs);he=a(Fs,"loss"),Fs.forEach(s),_t=a(Es,"\u5F15\u6570\u3092\u4E0E\u3048\u306A\u3044\u3053\u3068\u306B\u3082\u6CE8\u610F\u3057\u3066\u304F\u3060\u3055\u3044\u3002\u3053\u308C\u306F\u3001\u30E2\u30C7\u30EB\u304C\u5185\u90E8\u7684\u306B\u640D\u5931\u3092\u8A08\u7B97\u3059\u308B\u3053\u3068\u304C\u3067\u304D\u308B\u304B\u3089\u3067\u3059\u3002\u640D\u5931\u306A\u3057\u3067\u30B3\u30F3\u30D1\u30A4\u30EB\u3057\u3066\u3001\u5165\u529B\u8F9E\u66F8\u306B\u30E9\u30D9\u30EB\u3092\u6307\u5B9A\u3059\u308B\u3068\uFF08\u79C1\u305F\u3061\u306E\u30C7\u30FC\u30BF\u30BB\u30C3\u30C8\u3067\u884C\u3063\u3066\u3044\u308B\u3088\u3046\u306B\uFF09\u3001\u30E2\u30C7\u30EB\u306F\u305D\u306E\u5185\u90E8\u640D\u5931\u3092\u4F7F\u7528\u3057\u3066\u5B66\u7FD2\u3057\u3001\u305D\u308C\u306F\u9078\u3093\u3060\u30BF\u30B9\u30AF\u3068\u30E2\u30C7\u30EB\u30BF\u30A4\u30D7\u306B\u9069\u3057\u305F\u3082\u306E\u306B\u306A\u308B\u3067\u3057\u3087\u3046\u3002"),Es.forEach(s),Ps=_(m),De=p(m,"P",{});var xl=i(De);Ul=a(xl,"\u6B21\u306B\u3001\u5B66\u7FD2\u4E2D\u306B\u30E2\u30C7\u30EB\u3092Hub\u306B\u30A2\u30C3\u30D7\u30ED\u30FC\u30C9\u3059\u308B\u305F\u3081\u306E"),me=p(xl,"CODE",{});var os=i(me);El=a(os,"PushToHubCallback"),os.forEach(s),Ws=a(xl,"\u3092\u5B9A\u7FA9\u3057\u3001\u305D\u306E\u30B3\u30FC\u30EB\u30D0\u30C3\u30AF\u3067\u30E2\u30C7\u30EB\u3092\u30D5\u30A3\u30C3\u30C8\u3055\u305B\u307E\u3059\u3002"),xl.forEach(s),Wl=_(m),g(zs.$$.fragment,m),Ts=_(m),oe=p(m,"P",{});var Ke=i(oe);Xs=p(Ke,"CODE",{});var sl=i(Xs);as=a(sl,"hub_model_id"),sl.forEach(s),Xl=a(Ke," \u5F15\u6570\u306B\u306F\u3001\u30D7\u30C3\u30B7\u30E5\u3057\u305F\u3044\u30EA\u30DD\u30B8\u30C8\u30EA\u306E\u30D5\u30EB\u30CD\u30FC\u30E0\u3092\u6307\u5B9A\u3067\u304D\u307E\u3059\u3002 (\u7279\u306B\u3001\u7279\u5B9A\u306E\u7D44\u7E54\uFF08organization\uFF09\u306B\u30D7\u30C3\u30B7\u30E5\u3059\u308B\u5834\u5408\u306F\u3053\u306E\u5F15\u6570\u3092\u4F7F\u7528\u3059\u308B\u5FC5\u8981\u304C\u3042\u308A\u307E\u3059)\u3002\u4F8B\u3048\u3070\u3001\u30E2\u30C7\u30EB\u3092 "),ve=p(Ke,"A",{href:!0,rel:!0});var Ol=i(ve);Zs=p(Ol,"CODE",{});var we=i(Zs);Ue=a(we,"huggingface-course"),we.forEach(s),ut=a(Ol," organization"),Ol.forEach(s),vl=a(Ke," \u306B\u30D7\u30C3\u30B7\u30E5\u3059\u308B\u5834\u5408\u3001 "),Ks=p(Ke,"CODE",{});var sa=i(Ks);Ys=a(sa,'hub_model_id="huggingface-course/bert-finetuned-ner"'),sa.forEach(s),Zl=a(Ke," \u3092\u8FFD\u52A0\u3057\u307E\u3059\u3002\u30C7\u30D5\u30A9\u30EB\u30C8\u3067\u306F\u3001\u4F7F\u7528\u3055\u308C\u308B\u30EA\u30DD\u30B8\u30C8\u30EA\u306F\u3042\u306A\u305F\u306E\u540D\u524D\u304C\u4F7F\u308F\u308C\u3001\u8A2D\u5B9A\u3057\u305F\u51FA\u529B\u30C7\u30A3\u30EC\u30AF\u30C8\u30EA\u306B\u3061\u306A\u3093\u3060\u540D\u524D\u3001\u4F8B\u3048\u3070 "),We=p(Ke,"CODE",{});var ll=i(We);bs=a(ll,'"cool_huggingface_user/bert-finetuned-ner"'),ll.forEach(s),rs=a(Ke," \u3068\u306A\u308A\u307E\u3059\u3002"),Ke.forEach(s),$s=_(m),g(Xe.$$.fragment,m),Is=_(m),Vs=p(m,"P",{});var Be=i(Vs);wl=a(Be,"\u5B66\u7FD2\u304C\u884C\u308F\u308C\u3066\u3044\u308B\u9593\u3001\u30E2\u30C7\u30EB\u304C\u4FDD\u5B58\u3055\u308C\u308B\u305F\u3073\u306B\uFF08\u4ECA\u56DE\u306E\u4F8B\u3067\u306F\u30A8\u30DD\u30C3\u30AF\u3054\u3068\u306B\uFF09\u30D0\u30C3\u30AF\u30B0\u30E9\u30A6\u30F3\u30C9\u3067Hub\u306B\u30A2\u30C3\u30D7\u30ED\u30FC\u30C9\u3055\u308C\u308B\u3053\u3068\u306B\u6CE8\u610F\u3057\u3066\u304F\u3060\u3055\u3044\u3002\u3053\u306E\u3088\u3046\u306B\u3057\u3066\u304A\u3051\u3070\u3001\u5FC5\u8981\u306B\u5FDC\u3058\u3066\u5225\u306E\u30DE\u30B7\u30F3\u3067\u5B66\u7FD2\u3092\u518D\u958B\u3059\u308B\u3053\u3068\u304C\u3067\u304D\u307E\u3059\u3002\u3053\u306E\u6BB5\u968E\u3067\u3001Model Hub\u4E0A\u306E\u63A8\u8AD6\u30A6\u30A3\u30B8\u30A7\u30C3\u30C8\u3092\u4F7F\u3063\u3066\u30E2\u30C7\u30EB\u3092\u30C6\u30B9\u30C8\u3057\u3001\u53CB\u4EBA\u3068\u5171\u6709\u3059\u308B\u3053\u3068\u304C\u3067\u304D\u307E\u3059\u3002"),Be.forEach(s),qe=_(m),js=p(m,"P",{});var la=i(js);Ze=a(la,"\u3053\u308C\u3067\u30C8\u30FC\u30AF\u30F3\u5206\u985E\u30BF\u30B9\u30AF\u306E\u30E2\u30C7\u30EB\u5FAE\u8ABF\u6574\u306B\u6210\u529F\u3057\u307E\u3057\u305F\u3002\u304A\u3081\u3067\u3068\u3046\u3054\u3056\u3044\u307E\u3059\uFF01"),la.forEach(s),gl=_(m),Pe=p(m,"P",{});var tl=i(Pe);ks=a(tl,"\u3057\u304B\u3057\u3001\u3053\u306E\u30E2\u30C7\u30EB\u306E\u5B9F\u529B\u306F\u3044\u304B\u307B\u3069\u3067\u3057\u3087\u3046\u304B\uFF1F\u305D\u308C\u3092\u77E5\u308B\u305F\u3081\u306B\u3001\u3044\u304F\u3064\u304B\u306E\u6307\u6A19\u3092\u4F7F\u3063\u3066\u8A55\u4FA1\u3059\u308B\u5FC5\u8981\u304C\u3042\u308A\u307E\u3059\u3002"),tl.forEach(s),this.h()},h(){P(u,"id",""),P(G,"href","/course/ja/chapter3"),P(ue,"id",""),P(ve,"href","https://huggingface.co/huggingface-course"),P(ve,"rel","nofollow")},m(m,S){n(m,u,S),l(u,$),n(m,d,S),n(m,b,S),l(b,D),l(b,E),l(E,v),l(b,q),l(b,C),l(C,z),l(b,B),n(m,A,S),n(m,R,S),l(R,U),l(R,y),l(y,M),l(R,K),n(m,Z,S),x(W,m,S),n(m,I,S),n(m,F,S),l(F,L),l(F,X),l(X,Q),l(F,V),n(m,le,S),x(J,m,S),n(m,ce,S),n(m,Y,S),l(Y,G),l(G,te),l(Y,$e),l(Y,N),l(N,ee),l(Y,je),n(m,Te,S),x(se,m,S),n(m,fe,S),x(ae,m,S),n(m,ge,S),x(re,m,S),n(m,fs,S),n(m,ue,S),l(ue,ke),n(m,Ml,S),n(m,xe,S),l(xe,Le),n(m,Ne,S),x(Gs,m,S),n(m,Ll,S),n(m,Ie,S),l(Ie,$l),n(m,jl,S),n(m,Oe,S),l(Oe,Cs),n(m,ss,S),x(He,m,S),n(m,Nl,S),n(m,_s,S),l(_s,Ge),n(m,Hl,S),n(m,de,S),l(de,ye),l(de,Us),l(Us,Ce),l(de,it),l(de,Ds),l(Ds,Gl),l(de,us),l(de,qs),l(qs,Ee),l(de,ls),n(m,ts,S),x(Ae,m,S),n(m,kl,S),n(m,ne,S),l(ne,ct),l(ne,ms),l(ms,ds),l(ne,ft),l(ne,hs),l(hs,he),l(ne,_t),n(m,Ps,S),n(m,De,S),l(De,Ul),l(De,me),l(me,El),l(De,Ws),n(m,Wl,S),x(zs,m,S),n(m,Ts,S),n(m,oe,S),l(oe,Xs),l(Xs,as),l(oe,Xl),l(oe,ve),l(ve,Zs),l(Zs,Ue),l(ve,ut),l(oe,vl),l(oe,Ks),l(Ks,Ys),l(oe,Zl),l(oe,We),l(We,bs),l(oe,rs),n(m,$s,S),x(Xe,m,S),n(m,Is,S),n(m,Vs,S),l(Vs,wl),n(m,qe,S),n(m,js,S),l(js,Ze),n(m,gl,S),n(m,Pe,S),l(Pe,ks),Js=!0},i(m){Js||(j(W.$$.fragment,m),j(J.$$.fragment,m),j(se.$$.fragment,m),j(ae.$$.fragment,m),j(re.$$.fragment,m),j(Gs.$$.fragment,m),j(He.$$.fragment,m),j(Ae.$$.fragment,m),j(zs.$$.fragment,m),j(Xe.$$.fragment,m),Js=!0)},o(m){k(W.$$.fragment,m),k(J.$$.fragment,m),k(se.$$.fragment,m),k(ae.$$.fragment,m),k(re.$$.fragment,m),k(Gs.$$.fragment,m),k(He.$$.fragment,m),k(Ae.$$.fragment,m),k(zs.$$.fragment,m),k(Xe.$$.fragment,m),Js=!1},d(m){m&&s(u),m&&s(d),m&&s(b),m&&s(A),m&&s(R),m&&s(Z),O(W,m),m&&s(I),m&&s(F),m&&s(le),O(J,m),m&&s(ce),m&&s(Y),m&&s(Te),O(se,m),m&&s(fe),O(ae,m),m&&s(ge),O(re,m),m&&s(fs),m&&s(ue),m&&s(Ml),m&&s(xe),m&&s(Ne),O(Gs,m),m&&s(Ll),m&&s(Ie),m&&s(jl),m&&s(Oe),m&&s(ss),O(He,m),m&&s(Nl),m&&s(_s),m&&s(Hl),m&&s(de),m&&s(ts),O(Ae,m),m&&s(kl),m&&s(ne),m&&s(Ps),m&&s(De),m&&s(Wl),O(zs,m),m&&s(Ts),m&&s(oe),m&&s($s),O(Xe,m),m&&s(Is),m&&s(Vs),m&&s(qe),m&&s(js),m&&s(gl),m&&s(Pe)}}}function Ff(H){let u,$,d,b,D;return{c(){u=o("p"),$=t("\u26A0\uFE0F \u30E9\u30D9\u30EB\u306E\u6570\u304C\u9593\u9055\u3063\u3066\u3044\u308B\u30E2\u30C7\u30EB\u304C\u3042\u308B\u3068\u3001\u5F8C\u3067 "),d=o("code"),b=t("model.fit()"),D=t(" \u3092\u547C\u3073\u51FA\u3059\u3068\u304D\u306B\u3088\u304F\u308F\u304B\u3089\u306A\u3044\u30A8\u30E9\u30FC\u304C\u767A\u751F\u3057\u307E\u3059\u3002\u3053\u306E\u30A8\u30E9\u30FC\u306F\u30C7\u30D0\u30C3\u30B0\u306E\u969B\u306B\u5384\u4ECB\u306A\u306E\u3067\u3001\u3053\u306E\u30C1\u30A7\u30C3\u30AF\u3092\u5FC5\u305A\u884C\u3044\u3001\u671F\u5F85\u901A\u308A\u306E\u30E9\u30D9\u30EB\u6570\u3067\u3042\u308B\u3053\u3068\u3092\u78BA\u8A8D\u3057\u3066\u304F\u3060\u3055\u3044\u3002")},l(E){u=p(E,"P",{});var v=i(u);$=a(v,"\u26A0\uFE0F \u30E9\u30D9\u30EB\u306E\u6570\u304C\u9593\u9055\u3063\u3066\u3044\u308B\u30E2\u30C7\u30EB\u304C\u3042\u308B\u3068\u3001\u5F8C\u3067 "),d=p(v,"CODE",{});var q=i(d);b=a(q,"model.fit()"),q.forEach(s),D=a(v," \u3092\u547C\u3073\u51FA\u3059\u3068\u304D\u306B\u3088\u304F\u308F\u304B\u3089\u306A\u3044\u30A8\u30E9\u30FC\u304C\u767A\u751F\u3057\u307E\u3059\u3002\u3053\u306E\u30A8\u30E9\u30FC\u306F\u30C7\u30D0\u30C3\u30B0\u306E\u969B\u306B\u5384\u4ECB\u306A\u306E\u3067\u3001\u3053\u306E\u30C1\u30A7\u30C3\u30AF\u3092\u5FC5\u305A\u884C\u3044\u3001\u671F\u5F85\u901A\u308A\u306E\u30E9\u30D9\u30EB\u6570\u3067\u3042\u308B\u3053\u3068\u3092\u78BA\u8A8D\u3057\u3066\u304F\u3060\u3055\u3044\u3002"),v.forEach(s)},m(E,v){n(E,u,v),l(u,$),l(u,d),l(d,b),l(u,D)},d(E){E&&s(u)}}}function Mf(H){let u,$,d,b,D;return{c(){u=o("p"),$=t("\u{1F4A1} \u4F7F\u7528\u3057\u3066\u3044\u308B\u51FA\u529B\u30C7\u30A3\u30EC\u30AF\u30C8\u30EA\u304C\u3059\u3067\u306B\u5B58\u5728\u3059\u308B\u5834\u5408\u306F\u3001\u30D7\u30C3\u30B7\u30E5\u3057\u305F\u3044\u30EA\u30DD\u30B8\u30C8\u30EA\u306E\u30ED\u30FC\u30AB\u30EB\u30AF\u30ED\u30FC\u30F3\u3067\u3042\u308B\u5FC5\u8981\u304C\u3042\u308A\u307E\u3059\u3002\u305D\u3046\u3067\u306A\u3044\u5834\u5408\u306F\u3001"),d=o("code"),b=t("model.fit()"),D=t(" \u3092\u547C\u3073\u51FA\u3059\u3068\u304D\u306B\u30A8\u30E9\u30FC\u304C\u767A\u751F\u3057\u3001\u65B0\u3057\u3044\u540D\u524D\u3092\u8A2D\u5B9A\u3059\u308B\u5FC5\u8981\u304C\u3042\u308A\u307E\u3059\u3002")},l(E){u=p(E,"P",{});var v=i(u);$=a(v,"\u{1F4A1} \u4F7F\u7528\u3057\u3066\u3044\u308B\u51FA\u529B\u30C7\u30A3\u30EC\u30AF\u30C8\u30EA\u304C\u3059\u3067\u306B\u5B58\u5728\u3059\u308B\u5834\u5408\u306F\u3001\u30D7\u30C3\u30B7\u30E5\u3057\u305F\u3044\u30EA\u30DD\u30B8\u30C8\u30EA\u306E\u30ED\u30FC\u30AB\u30EB\u30AF\u30ED\u30FC\u30F3\u3067\u3042\u308B\u5FC5\u8981\u304C\u3042\u308A\u307E\u3059\u3002\u305D\u3046\u3067\u306A\u3044\u5834\u5408\u306F\u3001"),d=p(v,"CODE",{});var q=i(d);b=a(q,"model.fit()"),q.forEach(s),D=a(v," \u3092\u547C\u3073\u51FA\u3059\u3068\u304D\u306B\u30A8\u30E9\u30FC\u304C\u767A\u751F\u3057\u3001\u65B0\u3057\u3044\u540D\u524D\u3092\u8A2D\u5B9A\u3059\u308B\u5FC5\u8981\u304C\u3042\u308A\u307E\u3059\u3002"),v.forEach(s)},m(E,v){n(E,u,v),l(u,$),l(u,d),l(d,b),l(u,D)},d(E){E&&s(u)}}}function Lf(H){let u,$,d,b,D,E,v,q,C,z,B,A,R,U,y,M,K,Z,W,I,F;return B=new T({props:{code:"!pip install seqeval",highlighted:"!pip install seqeval"}}),{c(){u=o("p"),$=t("\u30C8\u30FC\u30AF\u30F3\u5206\u985E\u4E88\u6E2C\u306E\u8A55\u4FA1\u306B\u4F7F\u308F\u308C\u308B\u4F1D\u7D71\u7684\u306A\u67A0\u7D44\u307F\u306F"),d=o("a"),b=o("em"),D=t("seqeval"),E=t("\u3067\u3059\u3002\u3053\u306E\u8A55\u4FA1\u6307\u6A19\u3092\u4F7F\u3046\u306B\u306F\u3001\u307E\u305A"),v=o("em"),q=t("seqeval"),C=t("\u30E9\u30A4\u30D6\u30E9\u30EA\u3092\u30A4\u30F3\u30B9\u30C8\u30FC\u30EB\u3059\u308B\u5FC5\u8981\u304C\u3042\u308A\u307E\u3059\u3002"),z=f(),w(B.$$.fragment),A=f(),R=o("p"),U=t("\u305D\u3057\u3066\u3001"),y=o("a"),M=t("\u7B2C3\u7AE0"),K=t(" \u3067\u884C\u3063\u305F\u3088\u3046\u306B "),Z=o("code"),W=t("evaluate.load()"),I=t(" \u95A2\u6570\u3067\u8AAD\u307F\u8FBC\u3080\u3053\u3068\u304C\u3067\u304D\u308B\u3088\u3046\u306B\u306A\u308A\u307E\u3059\u3002"),this.h()},l(L){u=p(L,"P",{});var X=i(u);$=a(X,"\u30C8\u30FC\u30AF\u30F3\u5206\u985E\u4E88\u6E2C\u306E\u8A55\u4FA1\u306B\u4F7F\u308F\u308C\u308B\u4F1D\u7D71\u7684\u306A\u67A0\u7D44\u307F\u306F"),d=p(X,"A",{href:!0,rel:!0});var Q=i(d);b=p(Q,"EM",{});var V=i(b);D=a(V,"seqeval"),V.forEach(s),Q.forEach(s),E=a(X,"\u3067\u3059\u3002\u3053\u306E\u8A55\u4FA1\u6307\u6A19\u3092\u4F7F\u3046\u306B\u306F\u3001\u307E\u305A"),v=p(X,"EM",{});var le=i(v);q=a(le,"seqeval"),le.forEach(s),C=a(X,"\u30E9\u30A4\u30D6\u30E9\u30EA\u3092\u30A4\u30F3\u30B9\u30C8\u30FC\u30EB\u3059\u308B\u5FC5\u8981\u304C\u3042\u308A\u307E\u3059\u3002"),X.forEach(s),z=_(L),g(B.$$.fragment,L),A=_(L),R=p(L,"P",{});var J=i(R);U=a(J,"\u305D\u3057\u3066\u3001"),y=p(J,"A",{href:!0});var ce=i(y);M=a(ce,"\u7B2C3\u7AE0"),ce.forEach(s),K=a(J," \u3067\u884C\u3063\u305F\u3088\u3046\u306B "),Z=p(J,"CODE",{});var Y=i(Z);W=a(Y,"evaluate.load()"),Y.forEach(s),I=a(J," \u95A2\u6570\u3067\u8AAD\u307F\u8FBC\u3080\u3053\u3068\u304C\u3067\u304D\u308B\u3088\u3046\u306B\u306A\u308A\u307E\u3059\u3002"),J.forEach(s),this.h()},h(){P(d,"href","https://github.com/chakki-works/seqeval"),P(d,"rel","nofollow"),P(y,"href","/course/ja/chapter3")},m(L,X){n(L,u,X),l(u,$),l(u,d),l(d,b),l(b,D),l(u,E),l(u,v),l(v,q),l(u,C),n(L,z,X),x(B,L,X),n(L,A,X),n(L,R,X),l(R,U),l(R,y),l(y,M),l(R,K),l(R,Z),l(Z,W),l(R,I),F=!0},i(L){F||(j(B.$$.fragment,L),F=!0)},o(L){k(B.$$.fragment,L),F=!1},d(L){L&&s(u),L&&s(z),O(B,L),L&&s(A),L&&s(R)}}}function Nf(H){let u,$,d,b,D,E,v,q,C,z,B,A,R,U,y,M,K,Z,W,I,F,L,X,Q,V,le,J,ce,Y;return W=new T({props:{code:"!pip install seqeval",highlighted:"!pip install seqeval"}}),{c(){u=o("p"),$=o("code"),d=t("Trainer"),b=t(" \u306B\u30A8\u30DD\u30C3\u30AF\u6BCE\u306B\u6307\u6A19\u3092\u8A08\u7B97\u3055\u305B\u308B\u305F\u3081\u306B\u306F\u3001"),D=o("code"),E=t("compute_metrics()"),v=t(" \u95A2\u6570\u3092\u5B9A\u7FA9\u3059\u308B\u5FC5\u8981\u304C\u3042\u308A\u307E\u3059\u3002\u3053\u308C\u306F\u4E88\u6E2C\u3068\u30E9\u30D9\u30EB\u306E\u914D\u5217\u3092\u53D7\u3051\u53D6\u308A\u3001\u6307\u6A19\u306E\u540D\u524D\u3068\u5024\u3092\u542B\u3080\u8F9E\u66F8\u3092\u8FD4\u3059\u95A2\u6570\u3067\u3059\u3002"),q=f(),C=o("p"),z=t("\u30C8\u30FC\u30AF\u30F3\u5206\u985E\u4E88\u6E2C\u306E\u8A55\u4FA1\u306B\u4F7F\u308F\u308C\u308B\u4F1D\u7D71\u7684\u306A\u67A0\u7D44\u307F\u306F"),B=o("a"),A=o("em"),R=t("seqeval"),U=t("\u3067\u3059\u3002\u3053\u306E\u6307\u6A19\u3092\u4F7F\u3046\u306B\u306F\u3001\u307E\u305A "),y=o("em"),M=t("seqeval"),K=t(" \u30E9\u30A4\u30D6\u30E9\u30EA\u3092\u30A4\u30F3\u30B9\u30C8\u30FC\u30EB\u3059\u308B\u5FC5\u8981\u304C\u3042\u308A\u307E\u3059\u3002"),Z=f(),w(W.$$.fragment),I=f(),F=o("p"),L=t("\u305D\u3057\u3066\u3001"),X=o("a"),Q=t("\u7B2C3\u7AE0"),V=t(" \u3067\u884C\u3063\u305F\u3088\u3046\u306B "),le=o("code"),J=t("evaluate.load()"),ce=t(" \u95A2\u6570\u3067\u8AAD\u307F\u8FBC\u3080\u3053\u3068\u304C\u3067\u304D\u308B\u3088\u3046\u306B\u306A\u308A\u307E\u3059\u3002"),this.h()},l(G){u=p(G,"P",{});var te=i(u);$=p(te,"CODE",{});var $e=i($);d=a($e,"Trainer"),$e.forEach(s),b=a(te," \u306B\u30A8\u30DD\u30C3\u30AF\u6BCE\u306B\u6307\u6A19\u3092\u8A08\u7B97\u3055\u305B\u308B\u305F\u3081\u306B\u306F\u3001"),D=p(te,"CODE",{});var N=i(D);E=a(N,"compute_metrics()"),N.forEach(s),v=a(te," \u95A2\u6570\u3092\u5B9A\u7FA9\u3059\u308B\u5FC5\u8981\u304C\u3042\u308A\u307E\u3059\u3002\u3053\u308C\u306F\u4E88\u6E2C\u3068\u30E9\u30D9\u30EB\u306E\u914D\u5217\u3092\u53D7\u3051\u53D6\u308A\u3001\u6307\u6A19\u306E\u540D\u524D\u3068\u5024\u3092\u542B\u3080\u8F9E\u66F8\u3092\u8FD4\u3059\u95A2\u6570\u3067\u3059\u3002"),te.forEach(s),q=_(G),C=p(G,"P",{});var ee=i(C);z=a(ee,"\u30C8\u30FC\u30AF\u30F3\u5206\u985E\u4E88\u6E2C\u306E\u8A55\u4FA1\u306B\u4F7F\u308F\u308C\u308B\u4F1D\u7D71\u7684\u306A\u67A0\u7D44\u307F\u306F"),B=p(ee,"A",{href:!0,rel:!0});var je=i(B);A=p(je,"EM",{});var Te=i(A);R=a(Te,"seqeval"),Te.forEach(s),je.forEach(s),U=a(ee,"\u3067\u3059\u3002\u3053\u306E\u6307\u6A19\u3092\u4F7F\u3046\u306B\u306F\u3001\u307E\u305A "),y=p(ee,"EM",{});var se=i(y);M=a(se,"seqeval"),se.forEach(s),K=a(ee," \u30E9\u30A4\u30D6\u30E9\u30EA\u3092\u30A4\u30F3\u30B9\u30C8\u30FC\u30EB\u3059\u308B\u5FC5\u8981\u304C\u3042\u308A\u307E\u3059\u3002"),ee.forEach(s),Z=_(G),g(W.$$.fragment,G),I=_(G),F=p(G,"P",{});var fe=i(F);L=a(fe,"\u305D\u3057\u3066\u3001"),X=p(fe,"A",{href:!0});var ae=i(X);Q=a(ae,"\u7B2C3\u7AE0"),ae.forEach(s),V=a(fe," \u3067\u884C\u3063\u305F\u3088\u3046\u306B "),le=p(fe,"CODE",{});var ge=i(le);J=a(ge,"evaluate.load()"),ge.forEach(s),ce=a(fe," \u95A2\u6570\u3067\u8AAD\u307F\u8FBC\u3080\u3053\u3068\u304C\u3067\u304D\u308B\u3088\u3046\u306B\u306A\u308A\u307E\u3059\u3002"),fe.forEach(s),this.h()},h(){P(B,"href","https://github.com/chakki-works/seqeval"),P(B,"rel","nofollow"),P(X,"href","/course/ja/chapter3")},m(G,te){n(G,u,te),l(u,$),l($,d),l(u,b),l(u,D),l(D,E),l(u,v),n(G,q,te),n(G,C,te),l(C,z),l(C,B),l(B,A),l(A,R),l(C,U),l(C,y),l(y,M),l(C,K),n(G,Z,te),x(W,G,te),n(G,I,te),n(G,F,te),l(F,L),l(F,X),l(X,Q),l(F,V),l(F,le),l(le,J),l(F,ce),Y=!0},i(G){Y||(j(W.$$.fragment,G),Y=!0)},o(G){k(W.$$.fragment,G),Y=!1},d(G){G&&s(u),G&&s(q),G&&s(C),G&&s(Z),O(W,G),G&&s(I),G&&s(F)}}}function Hf(H){let u,$,d,b,D,E,v,q,C,z,B,A,R,U,y,M,K,Z,W;return R=new T({props:{code:`import numpy as np

all_predictions = []
all_labels = []
for batch in tf_eval_dataset:
    logits = model.predict(batch)["logits"]
    labels = batch["labels"]
    predictions = np.argmax(logits, axis=-1)
    for prediction, label in zip(predictions, labels):
        for predicted_idx, label_idx in zip(prediction, label):
            if label_idx == -100:
                continue
            all_predictions.append(label_names[predicted_idx])
            all_labels.append(label_names[label_idx])
metric.compute(predictions=[all_predictions], references=[all_labels])`,highlighted:`<span class="hljs-keyword">import</span> numpy <span class="hljs-keyword">as</span> np

all_predictions = []
all_labels = []
<span class="hljs-keyword">for</span> batch <span class="hljs-keyword">in</span> tf_eval_dataset:
    logits = model.predict(batch)[<span class="hljs-string">&quot;logits&quot;</span>]
    labels = batch[<span class="hljs-string">&quot;labels&quot;</span>]
    predictions = np.argmax(logits, axis=-<span class="hljs-number">1</span>)
    <span class="hljs-keyword">for</span> prediction, label <span class="hljs-keyword">in</span> <span class="hljs-built_in">zip</span>(predictions, labels):
        <span class="hljs-keyword">for</span> predicted_idx, label_idx <span class="hljs-keyword">in</span> <span class="hljs-built_in">zip</span>(prediction, label):
            <span class="hljs-keyword">if</span> label_idx == -<span class="hljs-number">100</span>:
                <span class="hljs-keyword">continue</span>
            all_predictions.append(label_names[predicted_idx])
            all_labels.append(label_names[label_idx])
metric.compute(predictions=[all_predictions], references=[all_labels])`}}),y=new T({props:{code:`{'LOC': {'precision': 0.91, 'recall': 0.92, 'f1': 0.91, 'number': 1668},
 'MISC': {'precision': 0.70, 'recall': 0.79, 'f1': 0.74, 'number': 702},
 'ORG': {'precision': 0.85, 'recall': 0.90, 'f1': 0.88, 'number': 1661},
 'PER': {'precision': 0.95, 'recall': 0.95, 'f1': 0.95, 'number': 1617},
 'overall_precision': 0.87,
 'overall_recall': 0.91,
 'overall_f1': 0.89,
 'overall_accuracy': 0.97}`,highlighted:`{<span class="hljs-string">&#x27;LOC&#x27;</span>: {<span class="hljs-string">&#x27;precision&#x27;</span>: <span class="hljs-number">0.91</span>, <span class="hljs-string">&#x27;recall&#x27;</span>: <span class="hljs-number">0.92</span>, <span class="hljs-string">&#x27;f1&#x27;</span>: <span class="hljs-number">0.91</span>, <span class="hljs-string">&#x27;number&#x27;</span>: <span class="hljs-number">1668</span>},
 <span class="hljs-string">&#x27;MISC&#x27;</span>: {<span class="hljs-string">&#x27;precision&#x27;</span>: <span class="hljs-number">0.70</span>, <span class="hljs-string">&#x27;recall&#x27;</span>: <span class="hljs-number">0.79</span>, <span class="hljs-string">&#x27;f1&#x27;</span>: <span class="hljs-number">0.74</span>, <span class="hljs-string">&#x27;number&#x27;</span>: <span class="hljs-number">702</span>},
 <span class="hljs-string">&#x27;ORG&#x27;</span>: {<span class="hljs-string">&#x27;precision&#x27;</span>: <span class="hljs-number">0.85</span>, <span class="hljs-string">&#x27;recall&#x27;</span>: <span class="hljs-number">0.90</span>, <span class="hljs-string">&#x27;f1&#x27;</span>: <span class="hljs-number">0.88</span>, <span class="hljs-string">&#x27;number&#x27;</span>: <span class="hljs-number">1661</span>},
 <span class="hljs-string">&#x27;PER&#x27;</span>: {<span class="hljs-string">&#x27;precision&#x27;</span>: <span class="hljs-number">0.95</span>, <span class="hljs-string">&#x27;recall&#x27;</span>: <span class="hljs-number">0.95</span>, <span class="hljs-string">&#x27;f1&#x27;</span>: <span class="hljs-number">0.95</span>, <span class="hljs-string">&#x27;number&#x27;</span>: <span class="hljs-number">1617</span>},
 <span class="hljs-string">&#x27;overall_precision&#x27;</span>: <span class="hljs-number">0.87</span>,
 <span class="hljs-string">&#x27;overall_recall&#x27;</span>: <span class="hljs-number">0.91</span>,
 <span class="hljs-string">&#x27;overall_f1&#x27;</span>: <span class="hljs-number">0.89</span>,
 <span class="hljs-string">&#x27;overall_accuracy&#x27;</span>: <span class="hljs-number">0.97</span>}`}}),{c(){u=o("p"),$=t(`\u3068\u3066\u3082\u591A\u304F\u306E\u60C5\u5831\u3092\u53D6\u5F97\u3057\u3066\u3044\u307E\u3059\uFF01
\u5404\u30A8\u30F3\u30C6\u30A3\u30C6\u30A3\u306E\u7CBE\u5EA6\u3001\u518D\u73FE\u7387\u3001F1\u30B9\u30B3\u30A2\u3001\u305D\u3057\u3066\u7DCF\u5408\u7684\u306A\u30B9\u30B3\u30A2\u3067\u3059\u3002
\u3067\u306F\u3001\u5B9F\u969B\u306B\u30E2\u30C7\u30EB\u306E\u4E88\u6E2C\u5024\u3092\u4F7F\u3063\u3066\u30B9\u30B3\u30A2\u3092\u8A08\u7B97\u3057\u3066\u307F\u308B\u3068\u3069\u3046\u306A\u308B\u304B\u898B\u3066\u307F\u307E\u3057\u3087\u3046\u3002`),d=f(),b=o("p"),D=t("TensorFlow\u306F\u4E88\u6E2C\u5024\u3092\u9023\u7D50\u3059\u308B\u3053\u3068\u3092\u597D\u307F\u307E\u305B\u3093\u3002\u306A\u305C\u306A\u3089\u3001\u4E88\u6E2C\u5024\u306F\u53EF\u5909\u9577\u3067\u3042\u308B\u304B\u3089\u3067\u3059\u3002\u3064\u307E\u308A\u3001"),E=o("code"),v=t("model.predict()"),q=t("\u3092\u305D\u306E\u307E\u307E\u4F7F\u3046\u3053\u3068\u306F\u3067\u304D\u306A\u3044\u306E\u3067\u3059\u3002\u3057\u304B\u3057\u3001\u3060\u304B\u3089\u3068\u3044\u3063\u3066\u3053\u3053\u3067\u6B62\u3081\u308B\u3064\u3082\u308A\u306F\u3042\u308A\u307E\u305B\u3093\u3002\u4E00\u5EA6\u306B\u3044\u304F\u3064\u304B\u306E\u4E88\u6E2C\u3092\u30D0\u30C3\u30C1\u3067\u5B9F\u884C\u3057\u3066\u53D6\u5F97\u3057\u3001\u305D\u308C\u3089\u3092\u4E00\u3064\u306E\u5927\u304D\u306A\u9577\u3044\u30EA\u30B9\u30C8\u306B\u9023\u7D50\u3057\u3001\u30DE\u30B9\u30AD\u30F3\u30B0\u3084\u30D1\u30C7\u30A3\u30F3\u30B0\u3092\u793A\u3059 "),C=o("code"),z=t("-100"),B=t(" \u30C8\u30FC\u30AF\u30F3\u3092\u524A\u9664\u3057\u307E\u3059\u3002"),A=f(),w(R.$$.fragment),U=f(),w(y.$$.fragment),M=f(),K=o("p"),Z=t(`\u3042\u306A\u305F\u306E\u30E2\u30C7\u30EB\u306F\u3001\u79C1\u305F\u3061\u306E\u30E2\u30C7\u30EB\u3068\u6BD4\u8F03\u3057\u3066\u3001\u3069\u3046\u3067\u3057\u305F\u304B\uFF1F
\u3082\u3057\u3001\u540C\u3058\u3088\u3046\u306A\u6570\u5024\u304C\u51FA\u305F\u306E\u306A\u3089\u3001\u30C8\u30EC\u30FC\u30CB\u30F3\u30B0\u306F\u6210\u529F\u3067\u3059\u3002`)},l(I){u=p(I,"P",{});var F=i(u);$=a(F,`\u3068\u3066\u3082\u591A\u304F\u306E\u60C5\u5831\u3092\u53D6\u5F97\u3057\u3066\u3044\u307E\u3059\uFF01
\u5404\u30A8\u30F3\u30C6\u30A3\u30C6\u30A3\u306E\u7CBE\u5EA6\u3001\u518D\u73FE\u7387\u3001F1\u30B9\u30B3\u30A2\u3001\u305D\u3057\u3066\u7DCF\u5408\u7684\u306A\u30B9\u30B3\u30A2\u3067\u3059\u3002
\u3067\u306F\u3001\u5B9F\u969B\u306B\u30E2\u30C7\u30EB\u306E\u4E88\u6E2C\u5024\u3092\u4F7F\u3063\u3066\u30B9\u30B3\u30A2\u3092\u8A08\u7B97\u3057\u3066\u307F\u308B\u3068\u3069\u3046\u306A\u308B\u304B\u898B\u3066\u307F\u307E\u3057\u3087\u3046\u3002`),F.forEach(s),d=_(I),b=p(I,"P",{});var L=i(b);D=a(L,"TensorFlow\u306F\u4E88\u6E2C\u5024\u3092\u9023\u7D50\u3059\u308B\u3053\u3068\u3092\u597D\u307F\u307E\u305B\u3093\u3002\u306A\u305C\u306A\u3089\u3001\u4E88\u6E2C\u5024\u306F\u53EF\u5909\u9577\u3067\u3042\u308B\u304B\u3089\u3067\u3059\u3002\u3064\u307E\u308A\u3001"),E=p(L,"CODE",{});var X=i(E);v=a(X,"model.predict()"),X.forEach(s),q=a(L,"\u3092\u305D\u306E\u307E\u307E\u4F7F\u3046\u3053\u3068\u306F\u3067\u304D\u306A\u3044\u306E\u3067\u3059\u3002\u3057\u304B\u3057\u3001\u3060\u304B\u3089\u3068\u3044\u3063\u3066\u3053\u3053\u3067\u6B62\u3081\u308B\u3064\u3082\u308A\u306F\u3042\u308A\u307E\u305B\u3093\u3002\u4E00\u5EA6\u306B\u3044\u304F\u3064\u304B\u306E\u4E88\u6E2C\u3092\u30D0\u30C3\u30C1\u3067\u5B9F\u884C\u3057\u3066\u53D6\u5F97\u3057\u3001\u305D\u308C\u3089\u3092\u4E00\u3064\u306E\u5927\u304D\u306A\u9577\u3044\u30EA\u30B9\u30C8\u306B\u9023\u7D50\u3057\u3001\u30DE\u30B9\u30AD\u30F3\u30B0\u3084\u30D1\u30C7\u30A3\u30F3\u30B0\u3092\u793A\u3059 "),C=p(L,"CODE",{});var Q=i(C);z=a(Q,"-100"),Q.forEach(s),B=a(L," \u30C8\u30FC\u30AF\u30F3\u3092\u524A\u9664\u3057\u307E\u3059\u3002"),L.forEach(s),A=_(I),g(R.$$.fragment,I),U=_(I),g(y.$$.fragment,I),M=_(I),K=p(I,"P",{});var V=i(K);Z=a(V,`\u3042\u306A\u305F\u306E\u30E2\u30C7\u30EB\u306F\u3001\u79C1\u305F\u3061\u306E\u30E2\u30C7\u30EB\u3068\u6BD4\u8F03\u3057\u3066\u3001\u3069\u3046\u3067\u3057\u305F\u304B\uFF1F
\u3082\u3057\u3001\u540C\u3058\u3088\u3046\u306A\u6570\u5024\u304C\u51FA\u305F\u306E\u306A\u3089\u3001\u30C8\u30EC\u30FC\u30CB\u30F3\u30B0\u306F\u6210\u529F\u3067\u3059\u3002`),V.forEach(s)},m(I,F){n(I,u,F),l(u,$),n(I,d,F),n(I,b,F),l(b,D),l(b,E),l(E,v),l(b,q),l(b,C),l(C,z),l(b,B),n(I,A,F),x(R,I,F),n(I,U,F),x(y,I,F),n(I,M,F),n(I,K,F),l(K,Z),W=!0},i(I){W||(j(R.$$.fragment,I),j(y.$$.fragment,I),W=!0)},o(I){k(R.$$.fragment,I),k(y.$$.fragment,I),W=!1},d(I){I&&s(u),I&&s(d),I&&s(b),I&&s(A),O(R,I),I&&s(U),O(y,I),I&&s(M),I&&s(K)}}}function Gf(H){let u,$,d,b,D,E,v,q,C,z,B,A,R,U,y,M,K,Z,W,I,F,L,X,Q,V,le,J,ce,Y,G,te,$e;return L=new T({props:{code:`import numpy as np


def compute_metrics(eval_preds):
    logits, labels = eval_preds
    predictions = np.argmax(logits, axis=-1)

    # Remove ignored index (special tokens) and convert to labels
    true_labels = [[label_names[l] for l in label if l != -100] for label in labels]
    true_predictions = [
        [label_names[p] for (p, l) in zip(prediction, label) if l != -100]
        for prediction, label in zip(predictions, labels)
    ]
    all_metrics = metric.compute(predictions=true_predictions, references=true_labels)
    return {
        "precision": all_metrics["overall_precision"],
        "recall": all_metrics["overall_recall"],
        "f1": all_metrics["overall_f1"],
        "accuracy": all_metrics["overall_accuracy"],
    }`,highlighted:`<span class="hljs-keyword">import</span> numpy <span class="hljs-keyword">as</span> np


<span class="hljs-keyword">def</span> <span class="hljs-title function_">compute_metrics</span>(<span class="hljs-params">eval_preds</span>):
    logits, labels = eval_preds
    predictions = np.argmax(logits, axis=-<span class="hljs-number">1</span>)

    <span class="hljs-comment"># Remove ignored index (special tokens) and convert to labels</span>
    true_labels = [[label_names[l] <span class="hljs-keyword">for</span> l <span class="hljs-keyword">in</span> label <span class="hljs-keyword">if</span> l != -<span class="hljs-number">100</span>] <span class="hljs-keyword">for</span> label <span class="hljs-keyword">in</span> labels]
    true_predictions = [
        [label_names[p] <span class="hljs-keyword">for</span> (p, l) <span class="hljs-keyword">in</span> <span class="hljs-built_in">zip</span>(prediction, label) <span class="hljs-keyword">if</span> l != -<span class="hljs-number">100</span>]
        <span class="hljs-keyword">for</span> prediction, label <span class="hljs-keyword">in</span> <span class="hljs-built_in">zip</span>(predictions, labels)
    ]
    all_metrics = metric.compute(predictions=true_predictions, references=true_labels)
    <span class="hljs-keyword">return</span> {
        <span class="hljs-string">&quot;precision&quot;</span>: all_metrics[<span class="hljs-string">&quot;overall_precision&quot;</span>],
        <span class="hljs-string">&quot;recall&quot;</span>: all_metrics[<span class="hljs-string">&quot;overall_recall&quot;</span>],
        <span class="hljs-string">&quot;f1&quot;</span>: all_metrics[<span class="hljs-string">&quot;overall_f1&quot;</span>],
        <span class="hljs-string">&quot;accuracy&quot;</span>: all_metrics[<span class="hljs-string">&quot;overall_accuracy&quot;</span>],
    }`}}),{c(){u=o("p"),$=t(`\u3068\u3066\u3082\u591A\u304F\u306E\u60C5\u5831\u3092\u53D6\u5F97\u3057\u3066\u3044\u307E\u3059\uFF01
\u5404\u30A8\u30F3\u30C6\u30A3\u30C6\u30A3\u306E\u7CBE\u5EA6\u3001\u518D\u73FE\u7387\u3001F1\u30B9\u30B3\u30A2\u3001\u305D\u3057\u3066\u7DCF\u5408\u7684\u306A\u30B9\u30B3\u30A2\u3067\u3059\u3002`),d=f(),b=o("p"),D=t(`\u79C1\u9054\u306E\u6307\u6A19\u8A08\u7B97\u3067\u306F\u3001\u7DCF\u5408\u7684\u306A\u30B9\u30B3\u30A2\u306E\u307F\u3092\u4FDD\u6301\u3059\u308B\u3053\u3068\u306B\u3057\u307E\u3059\u3002
\u3057\u304B\u3057\u3001\u304A\u671B\u307F\u306A\u3089`),E=o("code"),v=t("compute_metrics()"),q=t("\u95A2\u6570\u3092\u5FAE\u8ABF\u6574\u3057\u3066\u3001\u5831\u544A\u3055\u305B\u305F\u3044\u3059\u3079\u3066\u306E\u6307\u6A19\u3092\u8FD4\u3059\u3053\u3068\u3082\u3067\u304D\u307E\u3059\u3002"),C=f(),z=o("p"),B=t("\u3053\u306E "),A=o("code"),R=t("compute_metrics()"),U=t(" \u95A2\u6570\u306F\u3001\u307E\u305A \u6700\u7D42\u30EC\u30A4\u30E4\u30FC\u304C\u51FA\u529B\u3059\u308B\u30D9\u30AF\u30C8\u30EB\u306E\u6700\u5927\u5024\u3092\u4E88\u6E2C\u5024\u306B\u5909\u63DB\u3057\u307E\u3059\uFF08\u6700\u7D42\u30EC\u30A4\u30E4\u30FC\u304C\u51FA\u529B\u3059\u308B\u751F\u306E\u5024\u306F\u901A\u5E38\u306F\u78BA\u7387\u306B\u5909\u63DB\u3055\u308C\u307E\u3059\u304C\u3001\u6700\u5927\u5024\u306F\u78BA\u7387\u306B\u5909\u63DB\u3057\u306A\u304F\u3068\u3082\u540C\u3058\u306A\u306E\u3067\u3001softmax \u3067\u78BA\u7387\u306B\u5909\u63DB\u3055\u305B\u308B\u5FC5\u8981\u306F\u3042\u308A\u307E\u305B\u3093\uFF09\u3002\u6B21\u306B\u3001\u30E9\u30D9\u30EB\u3068\u4E88\u6E2C\u5024\u306E\u4E21\u65B9\u3092\u6574\u6570\u304B\u3089\u6587\u5B57\u5217\u306B\u5909\u63DB\u3059\u308B\u5FC5\u8981\u304C\u3042\u308A\u307E\u3059\u3002\u30E9\u30D9\u30EB\u304C "),y=o("code"),M=t("-100"),K=t(" \u3067\u3042\u308B\u5024\u3092\u3059\u3079\u3066\u524A\u9664\u3057\u3001\u305D\u306E\u7D50\u679C\u3092 "),Z=o("code"),W=t("metric.compute()"),I=t(" \u30E1\u30BD\u30C3\u30C9\u306B\u6E21\u3057\u307E\u3059\u3002"),F=f(),w(L.$$.fragment),X=f(),Q=o("p"),V=t("\u3053\u308C\u3067\u3001"),le=o("code"),J=t("Trainer"),ce=t(" \u3092\u5B9A\u7FA9\u3059\u308B\u6E96\u5099\u306F\u307B\u307C\u6574\u3044\u307E\u3057\u305F\u3002\u3042\u3068\u306F\u5FAE\u8ABF\u6574\u3092\u3059\u308B\u305F\u3081\u306E "),Y=o("code"),G=t("model"),te=t(" \u304C\u5FC5\u8981\u3067\u3059\u3002")},l(N){u=p(N,"P",{});var ee=i(u);$=a(ee,`\u3068\u3066\u3082\u591A\u304F\u306E\u60C5\u5831\u3092\u53D6\u5F97\u3057\u3066\u3044\u307E\u3059\uFF01
\u5404\u30A8\u30F3\u30C6\u30A3\u30C6\u30A3\u306E\u7CBE\u5EA6\u3001\u518D\u73FE\u7387\u3001F1\u30B9\u30B3\u30A2\u3001\u305D\u3057\u3066\u7DCF\u5408\u7684\u306A\u30B9\u30B3\u30A2\u3067\u3059\u3002`),ee.forEach(s),d=_(N),b=p(N,"P",{});var je=i(b);D=a(je,`\u79C1\u9054\u306E\u6307\u6A19\u8A08\u7B97\u3067\u306F\u3001\u7DCF\u5408\u7684\u306A\u30B9\u30B3\u30A2\u306E\u307F\u3092\u4FDD\u6301\u3059\u308B\u3053\u3068\u306B\u3057\u307E\u3059\u3002
\u3057\u304B\u3057\u3001\u304A\u671B\u307F\u306A\u3089`),E=p(je,"CODE",{});var Te=i(E);v=a(Te,"compute_metrics()"),Te.forEach(s),q=a(je,"\u95A2\u6570\u3092\u5FAE\u8ABF\u6574\u3057\u3066\u3001\u5831\u544A\u3055\u305B\u305F\u3044\u3059\u3079\u3066\u306E\u6307\u6A19\u3092\u8FD4\u3059\u3053\u3068\u3082\u3067\u304D\u307E\u3059\u3002"),je.forEach(s),C=_(N),z=p(N,"P",{});var se=i(z);B=a(se,"\u3053\u306E "),A=p(se,"CODE",{});var fe=i(A);R=a(fe,"compute_metrics()"),fe.forEach(s),U=a(se," \u95A2\u6570\u306F\u3001\u307E\u305A \u6700\u7D42\u30EC\u30A4\u30E4\u30FC\u304C\u51FA\u529B\u3059\u308B\u30D9\u30AF\u30C8\u30EB\u306E\u6700\u5927\u5024\u3092\u4E88\u6E2C\u5024\u306B\u5909\u63DB\u3057\u307E\u3059\uFF08\u6700\u7D42\u30EC\u30A4\u30E4\u30FC\u304C\u51FA\u529B\u3059\u308B\u751F\u306E\u5024\u306F\u901A\u5E38\u306F\u78BA\u7387\u306B\u5909\u63DB\u3055\u308C\u307E\u3059\u304C\u3001\u6700\u5927\u5024\u306F\u78BA\u7387\u306B\u5909\u63DB\u3057\u306A\u304F\u3068\u3082\u540C\u3058\u306A\u306E\u3067\u3001softmax \u3067\u78BA\u7387\u306B\u5909\u63DB\u3055\u305B\u308B\u5FC5\u8981\u306F\u3042\u308A\u307E\u305B\u3093\uFF09\u3002\u6B21\u306B\u3001\u30E9\u30D9\u30EB\u3068\u4E88\u6E2C\u5024\u306E\u4E21\u65B9\u3092\u6574\u6570\u304B\u3089\u6587\u5B57\u5217\u306B\u5909\u63DB\u3059\u308B\u5FC5\u8981\u304C\u3042\u308A\u307E\u3059\u3002\u30E9\u30D9\u30EB\u304C "),y=p(se,"CODE",{});var ae=i(y);M=a(ae,"-100"),ae.forEach(s),K=a(se," \u3067\u3042\u308B\u5024\u3092\u3059\u3079\u3066\u524A\u9664\u3057\u3001\u305D\u306E\u7D50\u679C\u3092 "),Z=p(se,"CODE",{});var ge=i(Z);W=a(ge,"metric.compute()"),ge.forEach(s),I=a(se," \u30E1\u30BD\u30C3\u30C9\u306B\u6E21\u3057\u307E\u3059\u3002"),se.forEach(s),F=_(N),g(L.$$.fragment,N),X=_(N),Q=p(N,"P",{});var re=i(Q);V=a(re,"\u3053\u308C\u3067\u3001"),le=p(re,"CODE",{});var fs=i(le);J=a(fs,"Trainer"),fs.forEach(s),ce=a(re," \u3092\u5B9A\u7FA9\u3059\u308B\u6E96\u5099\u306F\u307B\u307C\u6574\u3044\u307E\u3057\u305F\u3002\u3042\u3068\u306F\u5FAE\u8ABF\u6574\u3092\u3059\u308B\u305F\u3081\u306E "),Y=p(re,"CODE",{});var ue=i(Y);G=a(ue,"model"),ue.forEach(s),te=a(re," \u304C\u5FC5\u8981\u3067\u3059\u3002"),re.forEach(s)},m(N,ee){n(N,u,ee),l(u,$),n(N,d,ee),n(N,b,ee),l(b,D),l(b,E),l(E,v),l(b,q),n(N,C,ee),n(N,z,ee),l(z,B),l(z,A),l(A,R),l(z,U),l(z,y),l(y,M),l(z,K),l(z,Z),l(Z,W),l(z,I),n(N,F,ee),x(L,N,ee),n(N,X,ee),n(N,Q,ee),l(Q,V),l(Q,le),l(le,J),l(Q,ce),l(Q,Y),l(Y,G),l(Q,te),$e=!0},i(N){$e||(j(L.$$.fragment,N),$e=!0)},o(N){k(L.$$.fragment,N),$e=!1},d(N){N&&s(u),N&&s(d),N&&s(b),N&&s(C),N&&s(z),N&&s(F),O(L,N),N&&s(X),N&&s(Q)}}}function kf(H){let u,$,d,b,D,E,v,q,C,z,B,A,R,U,y,M,K,Z,W,I,F,L,X,Q,V,le,J,ce,Y,G,te,$e,N,ee,je,Te,se,fe,ae,ge,re,fs,ue,ke,Ml,xe,Le,Ne,Gs,Ll,Ie,$l,jl,Oe,Cs,ss,He,Nl,_s,Ge,Hl,de,ye,Us,Ce,it,Ds,Gl,us,qs,Ee,ls,ts,Ae,kl,ne,ct,ms,ds,ft,hs,he,_t,Ps,De,Ul,me,El,Ws,Wl,zs,Ts,oe,Xs,as,Xl,ve,Zs,Ue,ut,vl,Ks,Ys,Zl,We,bs,rs,$s,Xe,Is,Vs,wl,qe,js,Ze,gl,Pe,ks,Js,m,S,As,Rs,Ss,ns,Qs,pe,mt,Re,dt,Bs,Kl,Yl,ea,Vl,ht,Se,Jl,bt,el,Es,$t,Fs,xl,os,Ke,sl,Ol,we,sa,ll,Be,la,tl,Qr,en,ta,Ye,aa,jt,sn,ln,ur,Ve,ra,na,tn,oa,kt,mr,vs,an,Aa,rn,al,Et,nn,on,pa,vt,dr,rl,pn,Ra,cn,nl,ia,wt,hr,yl,br,ol,Sa,Ba,fn,_n,ws,Fa,un,$r,Cl,jr,gs,Ma,La,mn,kr,Ql,dn,Er,Dl,vr,gt,Na,Je,wr,ql,gr,ca,Ha,xr,xt,Ga,Qe,Or,xs,hn,Ua,Wa,bn,Xa,yr,Ot,Cr,Pl,Ms,yt,Za,Dr,Ct,fa,pl,$n,Dt,qt,qr,et,jn,Pr,Ls,il,Pt,Ka,Ya,kn,zr,zt,_a,_e,En,Tt,vn,wn,zl,gn,xn,ua,Os,On,Tl,yn,Cn,Va,Ja,Il,Qa,er,Dn,Tr,Al,Ir,It,sr,ps,qn,ma,At,Ar,st,Pn,Rr,lt,tt,ie,Rt,da,ze,zn,St,Tn,In,Bt,An,Rn,Ft,Sn,Bn,Mt,Fn,Mn,Lt,Ln,Nn,ha,at,Hn,Nt,cl,Gn,ba,Ht,Sr,Ns,$a,es,Un,ja;return W=new T({props:{code:`id2label = {i: label for i, label in enumerate(label_names)}
label2id = {v: k for k, v in id2label.items()}`,highlighted:`id2label = {i: label <span class="hljs-keyword">for</span> i, label <span class="hljs-keyword">in</span> <span class="hljs-built_in">enumerate</span>(label_names)}
label2id = {v: k <span class="hljs-keyword">for</span> k, v <span class="hljs-keyword">in</span> id2label.items()}`}}),J=new T({props:{code:`from transformers import AutoModelForTokenClassification

model = AutoModelForTokenClassification.from_pretrained(
    model_checkpoint,
    id2label=id2label,
    label2id=label2id,
)`,highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoModelForTokenClassification

model = AutoModelForTokenClassification.from_pretrained(
    model_checkpoint,
    id2label=id2label,
    label2id=label2id,
)`}}),se=new T({props:{code:"model.config.num_labels",highlighted:"model.config.num_labels"}}),ae=new T({props:{code:"9",highlighted:'<span class="hljs-number">9</span>'}}),re=new Jr({props:{warning:!0,$$slots:{default:[Uf]},$$scope:{ctx:H}}}),Cs=new T({props:{code:`from huggingface_hub import notebook_login

notebook_login()`,highlighted:`<span class="hljs-keyword">from</span> huggingface_hub <span class="hljs-keyword">import</span> notebook_login

notebook_login()`}}),ye=new T({props:{code:"huggingface-cli login",highlighted:"huggingface-cli login"}}),Ee=new T({props:{code:`from transformers import TrainingArguments

args = TrainingArguments(
    "bert-finetuned-ner",
    evaluation_strategy="epoch",
    save_strategy="epoch",
    learning_rate=2e-5,
    num_train_epochs=3,
    weight_decay=0.01,
    push_to_hub=True,
)`,highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> TrainingArguments

args = TrainingArguments(
    <span class="hljs-string">&quot;bert-finetuned-ner&quot;</span>,
    evaluation_strategy=<span class="hljs-string">&quot;epoch&quot;</span>,
    save_strategy=<span class="hljs-string">&quot;epoch&quot;</span>,
    learning_rate=<span class="hljs-number">2e-5</span>,
    num_train_epochs=<span class="hljs-number">3</span>,
    weight_decay=<span class="hljs-number">0.01</span>,
    push_to_hub=<span class="hljs-literal">True</span>,
)`}}),We=new Jr({props:{$$slots:{default:[Wf]},$$scope:{ctx:H}}}),qe=new T({props:{code:`from transformers import Trainer

trainer = Trainer(
    model=model,
    args=args,
    train_dataset=tokenized_datasets["train"],
    eval_dataset=tokenized_datasets["validation"],
    data_collator=data_collator,
    compute_metrics=compute_metrics,
    tokenizer=tokenizer,
)
trainer.train()`,highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> Trainer

trainer = Trainer(
    model=model,
    args=args,
    train_dataset=tokenized_datasets[<span class="hljs-string">&quot;train&quot;</span>],
    eval_dataset=tokenized_datasets[<span class="hljs-string">&quot;validation&quot;</span>],
    data_collator=data_collator,
    compute_metrics=compute_metrics,
    tokenizer=tokenizer,
)
trainer.train()`}}),Ss=new T({props:{code:'trainer.push_to_hub(commit_message="Training complete")',highlighted:'trainer.push_to_hub(commit_message=<span class="hljs-string">&quot;Training complete&quot;</span>)'}}),Re=new T({props:{code:"'https://huggingface.co/sgugger/bert-finetuned-ner/commit/26ab21e5b1568f9afeccdaed2d8715f571d786ed'",highlighted:'<span class="hljs-string">&#x27;https://huggingface.co/sgugger/bert-finetuned-ner/commit/26ab21e5b1568f9afeccdaed2d8715f571d786ed&#x27;</span>'}}),Ve=new T({props:{code:`from torch.utils.data import DataLoader

train_dataloader = DataLoader(
    tokenized_datasets["train"],
    shuffle=True,
    collate_fn=data_collator,
    batch_size=8,
)
eval_dataloader = DataLoader(
    tokenized_datasets["validation"], collate_fn=data_collator, batch_size=8
)`,highlighted:`<span class="hljs-keyword">from</span> torch.utils.data <span class="hljs-keyword">import</span> DataLoader

train_dataloader = DataLoader(
    tokenized_datasets[<span class="hljs-string">&quot;train&quot;</span>],
    shuffle=<span class="hljs-literal">True</span>,
    collate_fn=data_collator,
    batch_size=<span class="hljs-number">8</span>,
)
eval_dataloader = DataLoader(
    tokenized_datasets[<span class="hljs-string">&quot;validation&quot;</span>], collate_fn=data_collator, batch_size=<span class="hljs-number">8</span>
)`}}),kt=new T({props:{code:`model = AutoModelForTokenClassification.from_pretrained(
    model_checkpoint,
    id2label=id2label,
    label2id=label2id,
)`,highlighted:`model = AutoModelForTokenClassification.from_pretrained(
    model_checkpoint,
    id2label=id2label,
    label2id=label2id,
)`}}),vt=new T({props:{code:`from torch.optim import AdamW

optimizer = AdamW(model.parameters(), lr=2e-5)`,highlighted:`<span class="hljs-keyword">from</span> torch.optim <span class="hljs-keyword">import</span> AdamW

optimizer = AdamW(model.parameters(), lr=<span class="hljs-number">2e-5</span>)`}}),wt=new T({props:{code:`from accelerate import Accelerator

accelerator = Accelerator()
model, optimizer, train_dataloader, eval_dataloader = accelerator.prepare(
    model, optimizer, train_dataloader, eval_dataloader
)`,highlighted:`<span class="hljs-keyword">from</span> accelerate <span class="hljs-keyword">import</span> Accelerator

accelerator = Accelerator()
model, optimizer, train_dataloader, eval_dataloader = accelerator.prepare(
    model, optimizer, train_dataloader, eval_dataloader
)`}}),yl=new Jr({props:{$$slots:{default:[Xf]},$$scope:{ctx:H}}}),Cl=new T({props:{code:`from transformers import get_scheduler

num_train_epochs = 3
num_update_steps_per_epoch = len(train_dataloader)
num_training_steps = num_train_epochs * num_update_steps_per_epoch

lr_scheduler = get_scheduler(
    "linear",
    optimizer=optimizer,
    num_warmup_steps=0,
    num_training_steps=num_training_steps,
)`,highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> get_scheduler

num_train_epochs = <span class="hljs-number">3</span>
num_update_steps_per_epoch = <span class="hljs-built_in">len</span>(train_dataloader)
num_training_steps = num_train_epochs * num_update_steps_per_epoch

lr_scheduler = get_scheduler(
    <span class="hljs-string">&quot;linear&quot;</span>,
    optimizer=optimizer,
    num_warmup_steps=<span class="hljs-number">0</span>,
    num_training_steps=num_training_steps,
)`}}),Je=new T({props:{code:`from huggingface_hub import Repository, get_full_repo_name

model_name = "bert-finetuned-ner-accelerate"
repo_name = get_full_repo_name(model_name)
repo_name`,highlighted:`<span class="hljs-keyword">from</span> huggingface_hub <span class="hljs-keyword">import</span> Repository, get_full_repo_name

model_name = <span class="hljs-string">&quot;bert-finetuned-ner-accelerate&quot;</span>
repo_name = get_full_repo_name(model_name)
repo_name`}}),ql=new T({props:{code:"'sgugger/bert-finetuned-ner-accelerate'",highlighted:'<span class="hljs-string">&#x27;sgugger/bert-finetuned-ner-accelerate&#x27;</span>'}}),xt=new T({props:{code:`output_dir = "bert-finetuned-ner-accelerate"
repo = Repository(output_dir, clone_from=repo_name)`,highlighted:`output_dir = <span class="hljs-string">&quot;bert-finetuned-ner-accelerate&quot;</span>
repo = Repository(output_dir, clone_from=repo_name)`}}),qt=new T({props:{code:`def postprocess(predictions, labels):
    predictions = predictions.detach().cpu().clone().numpy()
    labels = labels.detach().cpu().clone().numpy()

    # Remove ignored index (special tokens) and convert to labels
    true_labels = [[label_names[l] for l in label if l != -100] for label in labels]
    true_predictions = [
        [label_names[p] for (p, l) in zip(prediction, label) if l != -100]
        for prediction, label in zip(predictions, labels)
    ]
    return true_labels, true_predictions`,highlighted:`<span class="hljs-keyword">def</span> <span class="hljs-title function_">postprocess</span>(<span class="hljs-params">predictions, labels</span>):
    predictions = predictions.detach().cpu().clone().numpy()
    labels = labels.detach().cpu().clone().numpy()

    <span class="hljs-comment"># Remove ignored index (special tokens) and convert to labels</span>
    true_labels = [[label_names[l] <span class="hljs-keyword">for</span> l <span class="hljs-keyword">in</span> label <span class="hljs-keyword">if</span> l != -<span class="hljs-number">100</span>] <span class="hljs-keyword">for</span> label <span class="hljs-keyword">in</span> labels]
    true_predictions = [
        [label_names[p] <span class="hljs-keyword">for</span> (p, l) <span class="hljs-keyword">in</span> <span class="hljs-built_in">zip</span>(prediction, label) <span class="hljs-keyword">if</span> l != -<span class="hljs-number">100</span>]
        <span class="hljs-keyword">for</span> prediction, label <span class="hljs-keyword">in</span> <span class="hljs-built_in">zip</span>(predictions, labels)
    ]
    <span class="hljs-keyword">return</span> true_labels, true_predictions`}}),At=new T({props:{code:`from tqdm.auto import tqdm
import torch

progress_bar = tqdm(range(num_training_steps))

for epoch in range(num_train_epochs):
    # Training
    model.train()
    for batch in train_dataloader:
        outputs = model(**batch)
        loss = outputs.loss
        accelerator.backward(loss)

        optimizer.step()
        lr_scheduler.step()
        optimizer.zero_grad()
        progress_bar.update(1)

    # Evaluation
    model.eval()
    for batch in eval_dataloader:
        with torch.no_grad():
            outputs = model(**batch)

        predictions = outputs.logits.argmax(dim=-1)
        labels = batch["labels"]

        # Necessary to pad predictions and labels for being gathered
        predictions = accelerator.pad_across_processes(predictions, dim=1, pad_index=-100)
        labels = accelerator.pad_across_processes(labels, dim=1, pad_index=-100)

        predictions_gathered = accelerator.gather(predictions)
        labels_gathered = accelerator.gather(labels)

        true_predictions, true_labels = postprocess(predictions_gathered, labels_gathered)
        metric.add_batch(predictions=true_predictions, references=true_labels)

    results = metric.compute()
    print(
        f"epoch {epoch}:",
        {
            key: results[f"overall_{key}"]
            for key in ["precision", "recall", "f1", "accuracy"]
        },
    )

    # Save and upload
    accelerator.wait_for_everyone()
    unwrapped_model = accelerator.unwrap_model(model)
    unwrapped_model.save_pretrained(output_dir, save_function=accelerator.save)
    if accelerator.is_main_process:
        tokenizer.save_pretrained(output_dir)
        repo.push_to_hub(
            commit_message=f"Training in progress epoch {epoch}", blocking=False
        )`,highlighted:`<span class="hljs-keyword">from</span> tqdm.auto <span class="hljs-keyword">import</span> tqdm
<span class="hljs-keyword">import</span> torch

progress_bar = tqdm(<span class="hljs-built_in">range</span>(num_training_steps))

<span class="hljs-keyword">for</span> epoch <span class="hljs-keyword">in</span> <span class="hljs-built_in">range</span>(num_train_epochs):
    <span class="hljs-comment"># Training</span>
    model.train()
    <span class="hljs-keyword">for</span> batch <span class="hljs-keyword">in</span> train_dataloader:
        outputs = model(**batch)
        loss = outputs.loss
        accelerator.backward(loss)

        optimizer.step()
        lr_scheduler.step()
        optimizer.zero_grad()
        progress_bar.update(<span class="hljs-number">1</span>)

    <span class="hljs-comment"># Evaluation</span>
    model.<span class="hljs-built_in">eval</span>()
    <span class="hljs-keyword">for</span> batch <span class="hljs-keyword">in</span> eval_dataloader:
        <span class="hljs-keyword">with</span> torch.no_grad():
            outputs = model(**batch)

        predictions = outputs.logits.argmax(dim=-<span class="hljs-number">1</span>)
        labels = batch[<span class="hljs-string">&quot;labels&quot;</span>]

        <span class="hljs-comment"># Necessary to pad predictions and labels for being gathered</span>
        predictions = accelerator.pad_across_processes(predictions, dim=<span class="hljs-number">1</span>, pad_index=-<span class="hljs-number">100</span>)
        labels = accelerator.pad_across_processes(labels, dim=<span class="hljs-number">1</span>, pad_index=-<span class="hljs-number">100</span>)

        predictions_gathered = accelerator.gather(predictions)
        labels_gathered = accelerator.gather(labels)

        true_predictions, true_labels = postprocess(predictions_gathered, labels_gathered)
        metric.add_batch(predictions=true_predictions, references=true_labels)

    results = metric.compute()
    <span class="hljs-built_in">print</span>(
        <span class="hljs-string">f&quot;epoch <span class="hljs-subst">{epoch}</span>:&quot;</span>,
        {
            key: results[<span class="hljs-string">f&quot;overall_<span class="hljs-subst">{key}</span>&quot;</span>]
            <span class="hljs-keyword">for</span> key <span class="hljs-keyword">in</span> [<span class="hljs-string">&quot;precision&quot;</span>, <span class="hljs-string">&quot;recall&quot;</span>, <span class="hljs-string">&quot;f1&quot;</span>, <span class="hljs-string">&quot;accuracy&quot;</span>]
        },
    )

    <span class="hljs-comment"># Save and upload</span>
    accelerator.wait_for_everyone()
    unwrapped_model = accelerator.unwrap_model(model)
    unwrapped_model.save_pretrained(output_dir, save_function=accelerator.save)
    <span class="hljs-keyword">if</span> accelerator.is_main_process:
        tokenizer.save_pretrained(output_dir)
        repo.push_to_hub(
            commit_message=<span class="hljs-string">f&quot;Training in progress epoch <span class="hljs-subst">{epoch}</span>&quot;</span>, blocking=<span class="hljs-literal">False</span>
        )`}}),lt=new T({props:{code:`accelerator.wait_for_everyone()
unwrapped_model = accelerator.unwrap_model(model)
unwrapped_model.save_pretrained(output_dir, save_function=accelerator.save)`,highlighted:`accelerator.wait_for_everyone()
unwrapped_model = accelerator.unwrap_model(model)
unwrapped_model.save_pretrained(output_dir, save_function=accelerator.save)`}}),{c(){u=o("h3"),$=t("\u30E2\u30C7\u30EB\u3092\u5B9A\u7FA9"),d=f(),b=o("p"),D=t("\u4ECA\u56DE\u306F\u30C8\u30FC\u30AF\u30F3\u5206\u985E\u306E\u554F\u984C\u3092\u6271\u3046\u306E\u3067\u3001 "),E=o("code"),v=t("AutoModelForTokenClassification"),q=t(" \u30AF\u30E9\u30B9\u3092\u4F7F\u7528\u3057\u307E\u3059\u3002\u3053\u306E\u30E2\u30C7\u30EB\u3092\u5B9A\u7FA9\u3059\u308B\u969B\u306B\u899A\u3048\u3066\u304A\u304F\u3079\u304D\u3053\u3068\u306F\u3001\u30E9\u30D9\u30EB\u306E\u6570\u306B\u95A2\u3059\u308B\u60C5\u5831\u3092\u6E21\u3059\u3053\u3068\u3067\u3059\u3002\u6700\u3082\u7C21\u5358\u306A\u65B9\u6CD5\u306F "),C=o("code"),z=t("num_labels"),B=t(" \u5F15\u6570\u3067\u305D\u306E\u6570\u3092\u6E21\u3059\u3053\u3068\u3067\u3059\u304C\u3001\u3053\u306E\u30BB\u30AF\u30B7\u30E7\u30F3\u306E\u6700\u521D\u306B\u898B\u305F\u3088\u3046\u306A\u7D20\u6575\u306A\u63A8\u8AD6\u30A6\u30A3\u30B8\u30A7\u30C3\u30C8\u3092\u52D5\u4F5C\u3055\u305B\u305F\u3044\u5834\u5408\u306F\u3001\u4EE3\u308F\u308A\u306B\u6B63\u3057\u3044\u30E9\u30D9\u30EB\u306E\u5BFE\u5FDC\u95A2\u4FC2\u3092\u8A2D\u5B9A\u3057\u305F\u65B9\u304C\u826F\u3044\u3067\u3057\u3087\u3046\u3002"),A=f(),R=o("p"),U=t("id2label"),y=o("code"),M=t("\u3068"),K=t("label2id` \u3068\u3044\u3046 2 \u3064\u306E\u8F9E\u66F8\u578B\u30C7\u30FC\u30BF\u304C\u3042\u308A\u3001ID \u304B\u3089\u30E9\u30D9\u30EB\u3001\u30E9\u30D9\u30EB\u304B\u3089 ID \u3078\u306E\u30DE\u30C3\u30D4\u30F3\u30B0\u3092\u8A2D\u5B9A\u3059\u308B\u3053\u3068\u304C\u3067\u304D\u307E\u3059\u3002"),Z=f(),w(W.$$.fragment),I=f(),F=o("p"),L=t("\u3042\u3068\u306F\u305D\u308C\u3089\u3092 "),X=o("code"),Q=t("AutoModelForTokenClassification.from_pretrained()"),V=t(" \u30E1\u30BD\u30C3\u30C9\u306B\u6E21\u305B\u3070\u3001\u30E2\u30C7\u30EB\u306E\u69CB\u6210\u306B\u8A2D\u5B9A\u3055\u308C\u3001\u9069\u5207\u306B\u4FDD\u5B58\u3055\u308C\u3066 Hub \u306B\u30A2\u30C3\u30D7\u30ED\u30FC\u30C9\u3055\u308C\u307E\u3059\u3002"),le=f(),w(J.$$.fragment),ce=f(),Y=o("p"),G=o("a"),te=t("\u7B2C3\u7AE0"),$e=t(" \u3067 "),N=o("code"),ee=t("AutoModelForSequenceClassification"),je=t(" \u3092\u5B9A\u7FA9\u3057\u305F\u3068\u304D\u306E\u3088\u3046\u306B\u3001\u30E2\u30C7\u30EB\u3092\u4F5C\u6210\u3059\u308B\u3068\u3001\u3044\u304F\u3064\u304B\u306E\u91CD\u307F\u304C\u4F7F\u308F\u308C\u3066\u3044\u306A\u3044\uFF08\u4E8B\u524D\u5B66\u7FD2\u6E08\u307F\u30E2\u30C7\u30EB\u306E\u30D8\u30C3\u30C9\u90E8\u306E\u91CD\u307F\uFF09\u3001\u4ED6\u306E\u3044\u304F\u3064\u304B\u306E\u91CD\u307F\u304C\u30E9\u30F3\u30C0\u30E0\u306B\u521D\u671F\u5316\u3055\u308C\u3066\u3044\u308B\uFF08\u65B0\u3057\u304F\u63A5\u7D9A\u3057\u305F\u30C8\u30FC\u30AF\u30F3\u5206\u985E\u30D8\u30C3\u30C9\u306E\u91CD\u307F\uFF09\u3001\u3053\u306E\u30E2\u30C7\u30EB\u306F\u30C8\u30EC\u30FC\u30CB\u30F3\u30B0\u3059\u308B\u5FC5\u8981\u304C\u3042\u308B\u3068\u3044\u3046\u8B66\u544A\u304C\u8868\u793A\u3055\u308C\u307E\u3059\u3002\u30C8\u30EC\u30FC\u30CB\u30F3\u30B0\u306F\u3059\u3050\u306B\u3067\u3082\u5B9F\u884C\u3067\u304D\u307E\u3059\u304C\u3001\u307E\u305A\u306F\u3053\u306E\u30E2\u30C7\u30EB\u304C\u6B63\u3057\u3044\u6570\u306E\u30E9\u30D9\u30EB\u3092\u6301\u3064\u3053\u3068\u3092\u518D\u78BA\u8A8D\u3057\u307E\u3057\u3087\u3046\u3002"),Te=f(),w(se.$$.fragment),fe=f(),w(ae.$$.fragment),ge=f(),w(re.$$.fragment),fs=f(),ue=o("h3"),ke=t("\u30E2\u30C7\u30EB\u306E\u5FAE\u8ABF\u6574"),Ml=f(),xe=o("p"),Le=t(`\u3053\u308C\u3067\u30E2\u30C7\u30EB\u3092\u5B66\u7FD2\u3059\u308B\u6E96\u5099\u304C\u6574\u3044\u307E\u3057\u305F\uFF01
\u3057\u304B\u3057\u3001`),Ne=o("code"),Gs=t("Trainer"),Ll=t("\u3092\u5B9A\u7FA9\u3059\u308B\u524D\u306B\u3001\u6700\u5F8C\u306B2\u3064\u306E\u3053\u3068\u3092\u3059\u308B\u5FC5\u8981\u304C\u3042\u308A\u307E\u3059\u3002"),Ie=f(),$l=o("p"),jl=t("Hugging Face\u306B\u30ED\u30B0\u30A4\u30F3\u3057\u3001\u5B66\u7FD2\u7528\u30CF\u30A4\u30D1\u30FC\u30D1\u30E9\u30E1\u30FC\u30BF\u3092\u5B9A\u7FA9\u3059\u308B\u5FC5\u8981\u304C\u3042\u308A\u307E\u3059\u3002\u3082\u3057Notebook\u3067\u4F5C\u696D\u3057\u3066\u3044\u308B\u306A\u3089\u3001\u3053\u308C\u3092\u52A9\u3051\u308B\u4FBF\u5229\u306A\u95A2\u6570\u304C\u3042\u308A\u307E\u3059\u3002"),Oe=f(),w(Cs.$$.fragment),ss=f(),He=o("p"),Nl=t("\u3053\u308C\u306B\u3088\u308A\u3001Hugging Face\u306E\u30ED\u30B0\u30A4\u30F3\u60C5\u5831\u3092\u5165\u529B\u3059\u308B\u30A6\u30A3\u30B8\u30A7\u30C3\u30C8\u304C\u8868\u793A\u3055\u308C\u307E\u3059\u3002"),_s=f(),Ge=o("p"),Hl=t("Notebook\u3067\u4F5C\u696D\u3057\u3066\u3044\u306A\u3044\u5834\u5408\u306F\u3001\u30BF\u30FC\u30DF\u30CA\u30EB\u306B\u6B21\u306E\u884C\u3092\u5165\u529B\u3059\u308B\u3060\u3051\u3067\u3059\u3002"),de=f(),w(ye.$$.fragment),Us=f(),Ce=o("p"),it=t("\u5B8C\u4E86\u3057\u305F\u3089\u3001"),Ds=o("code"),Gl=t("TrainingArguments"),us=t("\u3092\u5B9A\u7FA9\u3059\u308B\u4E8B\u304C\u3067\u304D\u308B\u3088\u3046\u306B\u306A\u308A\u307E\u3059\u3002"),qs=f(),w(Ee.$$.fragment),ls=f(),ts=o("p"),Ae=t("\u3053\u308C\u3089\u306E\u30D1\u30E9\u30E1\u30FC\u30BF\u306E\u307B\u3068\u3093\u3069\u306F\u4EE5\u524D\u306B\u898B\u305F\u3053\u3068\u304C\u3042\u308B\u3068\u601D\u3044\u307E\u3059\u3002"),kl=f(),ne=o("p"),ct=t("\u30CF\u30A4\u30D1\u30FC\u30D1\u30E9\u30E1\u30FC\u30BF\uFF08\u5B66\u7FD2\u7387\u3001\u5B66\u7FD2\u30A8\u30DD\u30C3\u30AF\u6570\u3001\u30A6\u30A7\u30A4\u30C8\u6E1B\u8870\u306A\u3069\uFF09\u3092\u8A2D\u5B9A\u3057\u3001"),ms=o("code"),ds=t("push_to_hub=True"),ft=t(" \u3092\u6307\u5B9A\u3057\u3066\u3001\u30E2\u30C7\u30EB\u3092\u4FDD\u5B58\u3057\u3066\u5404\u30A8\u30DD\u30C3\u30AF\u7D42\u4E86\u6642\u306B\u8A55\u4FA1\u3057\u3001\u305D\u306E\u7D50\u679C\u3092\u30E2\u30C7\u30EB\u30CF\u30D6\u306B\u30A2\u30C3\u30D7\u30ED\u30FC\u30C9\u3059\u308B\u3053\u3068\u3092\u6307\u793A\u3057\u307E\u3059\u3002"),hs=f(),he=o("p"),_t=t("\u306A\u304A\u3001 "),Ps=o("code"),De=t("hub_model_id"),Ul=t(" \u5F15\u6570\u3067\u30D7\u30C3\u30B7\u30E5\u5148\u306E\u30EA\u30DD\u30B8\u30C8\u30EA\u540D\u3092\u6307\u5B9A\u3067\u304D\u307E\u3059 (\u7279\u306B\u3001\u7279\u5B9A\u306E\u7D44\u7E54\uFF08organization\uFF09\u306B\u30D7\u30C3\u30B7\u30E5\u3059\u308B\u5834\u5408\u306F\u3001\u3053\u306E\u5F15\u6570\u3092\u4F7F\u7528\u3059\u308B\u5FC5\u8981\u304C\u3042\u308A\u307E\u3059)\u3002\u4F8B\u3048\u3070\u3001"),me=o("a"),El=o("code"),Ws=t("huggingface-course"),Wl=t(" organization"),zs=t(" \u306B\u30E2\u30C7\u30EB\u3092\u30D7\u30C3\u30B7\u30E5\u3059\u308B\u5834\u5408\u3001"),Ts=o("code"),oe=t("TrainingArguments"),Xs=t(" \u306B "),as=o("code"),Xl=t('hub_model_id="huggingface-course/bert-finetuned-ner"'),ve=t(" \u3092\u8FFD\u52A0\u3057\u307E\u3057\u305F\u3002"),Zs=f(),Ue=o("p"),ut=t("\u30C7\u30D5\u30A9\u30EB\u30C8\u3067\u306F\u3001\u4F7F\u7528\u3055\u308C\u308B\u30EA\u30DD\u30B8\u30C8\u30EA\u306F\u3042\u306A\u305F\u306E\u540D\u524D\u304C\u4F7F\u308F\u308C\u3001\u8A2D\u5B9A\u3057\u305F\u51FA\u529B\u30C7\u30A3\u30EC\u30AF\u30C8\u30EA\u3061\u306A\u3093\u3060\u540D\u524D\u3001\u4F8B\u3048\u3070\u4ECA\u56DE\u306E\u4F8B\u3067\u306F "),vl=o("code"),Ks=t('"sgugger/bert-finetuned-ner"'),Ys=t(" \u3068\u306A\u308A\u307E\u3059\u3002"),Zl=f(),w(We.$$.fragment),bs=f(),rs=o("p"),$s=t("\u6700\u5F8C\u306B\u3001\u3059\u3079\u3066\u3092 "),Xe=o("code"),Is=t("Trainer"),Vs=t(" \u306B\u6E21\u3057\u3066\u3001\u30C8\u30EC\u30FC\u30CB\u30F3\u30B0\u3092\u958B\u59CB\u3059\u308B\u3060\u3051\u3067\u3059\u3002"),wl=f(),w(qe.$$.fragment),js=f(),Ze=o("p"),gl=t("\u5B66\u7FD2\u304C\u884C\u308F\u308C\u3066\u3044\u308B\u9593\u3001\u30E2\u30C7\u30EB\u304C\u4FDD\u5B58\u3055\u308C\u308B\u305F\u3073\u306B\uFF08\u3053\u3053\u3067\u306F\u30A8\u30DD\u30C3\u30AF\u3054\u3068\u306B\uFF09\u30D0\u30C3\u30AF\u30B0\u30E9\u30A6\u30F3\u30C9\u3067Hub\u306B\u30A2\u30C3\u30D7\u30ED\u30FC\u30C9\u3055\u308C\u308B\u3053\u3068\u306B\u6CE8\u610F\u3057\u3066\u304F\u3060\u3055\u3044\u3002\u3053\u306E\u3088\u3046\u306B\u3057\u3066\u3001\u5FC5\u8981\u306B\u5FDC\u3058\u3066\u5225\u306E\u30DE\u30B7\u30F3\u3067\u5B66\u7FD2\u3092\u518D\u958B\u3059\u308B\u3053\u3068\u304C\u3067\u304D\u307E\u3059\u3002"),Pe=f(),ks=o("p"),Js=t("\u5B66\u7FD2\u304C\u5B8C\u4E86\u3057\u305F\u3089\u3001"),m=o("code"),S=t("push_to_hub()"),As=t(" \u30E1\u30BD\u30C3\u30C9\u3092\u4F7F\u7528\u3057\u3066\u3001\u6700\u65B0\u30D0\u30FC\u30B8\u30E7\u30F3\u306E\u30E2\u30C7\u30EB\u3092\u30A2\u30C3\u30D7\u30ED\u30FC\u30C9\u3059\u308B\u3088\u3046\u306B\u3057\u307E\u3059\u3002"),Rs=f(),w(Ss.$$.fragment),ns=f(),Qs=o("p"),pe=t("\u3053\u306E\u30B3\u30DE\u30F3\u30C9\u306F\u3001\u4ECA\u884C\u3063\u305F\u30B3\u30DF\u30C3\u30C8\u306E URL \u3092\u8FD4\u3059\u306E\u3067\u3001\u305D\u308C\u3092\u691C\u67FB\u3057\u305F\u3044\u5834\u5408\u306F\u3001\u3053\u306E\u30B3\u30DE\u30F3\u30C9\u3092\u4F7F\u7528\u3057\u307E\u3059\u3002"),mt=f(),w(Re.$$.fragment),dt=f(),Bs=o("p"),Kl=t("\u307E\u305F\u3001"),Yl=o("code"),ea=t("Trainer"),Vl=t(`\u306F\u3059\u3079\u3066\u306E\u8A55\u4FA1\u7D50\u679C\u3092\u542B\u3080\u30E2\u30C7\u30EB\u30AB\u30FC\u30C9\u3092\u8D77\u8349\u3057\u3001\u30A2\u30C3\u30D7\u30ED\u30FC\u30C9\u3057\u307E\u3059\u3002\u3053\u306E\u6BB5\u968E\u3067\u3001Model Hub\u4E0A\u306E\u63A8\u8AD6\u30A6\u30A3\u30B8\u30A7\u30C3\u30C8\u3092\u4F7F\u3063\u3066\u30E2\u30C7\u30EB\u3092\u30C6\u30B9\u30C8\u3057\u3001\u53CB\u4EBA\u3068\u5171\u6709\u3059\u308B\u3053\u3068\u304C\u3067\u304D\u307E\u3059\u3002\u3053\u308C\u3067\u3001\u30C8\u30FC\u30AF\u30F3\u5206\u985E\u30BF\u30B9\u30AF\u306E\u30E2\u30C7\u30EB\u5FAE\u8ABF\u6574\u306B\u6210\u529F\u3057\u307E\u3057\u305F\u3002
\u304A\u3081\u3067\u3068\u3046\u3054\u3056\u3044\u307E\u3059\uFF01`),ht=f(),Se=o("p"),Jl=t("\u3082\u3046\u5C11\u3057\u6DF1\u304F\u5B66\u7FD2\u30EB\u30FC\u30D7\u306B\u3064\u3044\u3066\u5B66\u3073\u305F\u3044\u5834\u5408\u306F\u3001\u{1F917} Accelerate \u3092\u4F7F\u3063\u3066\u540C\u3058\u3053\u3068\u3092\u3059\u308B\u65B9\u6CD5\u3092\u7D39\u4ECB\u3057\u307E\u3059\u3002"),bt=f(),el=o("h2"),Es=t("\u30AB\u30B9\u30BF\u30E0\u30C8\u30EC\u30FC\u30CB\u30F3\u30B0\u30EB\u30FC\u30D7"),$t=f(),Fs=o("p"),xl=t("\u305D\u308C\u3067\u306F\u3001\u5FC5\u8981\u306A\u90E8\u5206\u3092\u7C21\u5358\u306B\u30AB\u30B9\u30BF\u30DE\u30A4\u30BA\u3067\u304D\u308B\u3088\u3046\u306B\u3001\u30C8\u30EC\u30FC\u30CB\u30F3\u30B0\u30EB\u30FC\u30D7\u306E\u5168\u4F53\u50CF\u3092\u898B\u3066\u307F\u307E\u3057\u3087\u3046\u3002\u3053\u308C\u306F\u3001"),os=o("a"),Ke=t("\u7B2C3\u7AE0"),sl=t(" \u3067\u884C\u3063\u305F\u3053\u3068\u3068\u3088\u304F\u4F3C\u3066\u3044\u307E\u3059\u304C\u3001\u8A55\u4FA1\u306E\u305F\u3081\u306B\u5C11\u3057\u5909\u66F4\u304C\u52A0\u3048\u3089\u308C\u3066\u3044\u307E\u3059\u3002"),Ol=f(),we=o("h3"),sa=t("\u30C8\u30EC\u30FC\u30CB\u30F3\u30B0\u306E\u305F\u3081\u306E\u6E96\u5099"),ll=f(),Be=o("p"),la=t("\u307E\u305A\u3001\u30C7\u30FC\u30BF\u30BB\u30C3\u30C8\u304B\u3089 "),tl=o("code"),Qr=t("DataLoader"),en=t(" \u3092\u4F5C\u6210\u3059\u308B\u5FC5\u8981\u304C\u3042\u308A\u307E\u3059\u3002\u3053\u3053\u3067\u306F\u3001"),ta=o("code"),Ye=t("data_collator"),aa=t(" \u3092 "),jt=o("code"),sn=t("collate_fn"),ln=t(" \u3068\u3057\u3066\u518D\u5229\u7528\u3057\u3001\u30C8\u30EC\u30FC\u30CB\u30F3\u30B0\u30BB\u30C3\u30C8\u3092\u30B7\u30E3\u30C3\u30D5\u30EB\u3057\u307E\u3059\u3002\u305F\u3060\u3057\u3001\u691C\u8A3C\u30BB\u30C3\u30C8\u306F\u30B7\u30E3\u30C3\u30D5\u30EB\u3057\u307E\u305B\u3093\u3002"),ur=f(),w(Ve.$$.fragment),ra=f(),na=o("p"),tn=t("\u6B21\u306B\u3001\u30E2\u30C7\u30EB\u306E\u518D\u5B9A\u7FA9\u3092\u884C\u3044\u307E\u3059\u3002\u3053\u308C\u306F\u3001\u4EE5\u524D\u306E\u5FAE\u8ABF\u6574\u3092\u7D99\u7D9A\u3059\u308B\u306E\u3067\u306F\u306A\u304F\u3001BERT\u3067\u4E8B\u524D\u5B66\u7FD2\u3057\u305F\u30E2\u30C7\u30EB\u304B\u3089\u518D\u3073\u958B\u59CB\u3059\u308B\u3053\u3068\u3092\u78BA\u8A8D\u3059\u308B\u305F\u3081\u3067\u3059\u3002"),oa=f(),w(kt.$$.fragment),mr=f(),vs=o("p"),an=t("\u305D\u308C\u304B\u3089\u3001\u30AA\u30D7\u30C6\u30A3\u30DE\u30A4\u30B6\u304C\u5FC5\u8981\u306B\u306A\u308A\u307E\u3059\u3002\u3053\u3053\u3067\u306F\u3001\u53E4\u5178\u7684\u306A "),Aa=o("code"),rn=t("AdamW"),al=t(" \u3092\u4F7F\u7528\u3057\u307E\u3059\u3002\u3053\u308C\u306F "),Et=o("code"),nn=t("Adam"),on=t(" \u306E\u3088\u3046\u306A\u3082\u306E\u3067\u3059\u304C\u3001\u91CD\u307F\u306E\u6E1B\u8870\u306E\u9069\u7528\u65B9\u6CD5\u3092\u4FEE\u6B63\u3057\u305F\u3082\u306E\u3067\u3059\u3002"),pa=f(),w(vt.$$.fragment),dr=f(),rl=o("p"),pn=t("\u3053\u308C\u3089\u306E\u30AA\u30D6\u30B8\u30A7\u30AF\u30C8\u3092\u3059\u3079\u3066\u53D6\u5F97\u3057\u305F\u3089\u3001\u305D\u308C\u3089\u3092 "),Ra=o("code"),cn=t("accelerator.prepare()"),nl=t(" \u30E1\u30BD\u30C3\u30C9\u306B\u9001\u308A\u307E\u3059\u3002"),ia=f(),w(wt.$$.fragment),hr=f(),w(yl.$$.fragment),br=f(),ol=o("p"),Sa=t("\u3053\u308C\u3067 "),Ba=o("code"),fn=t("train_dataloader"),_n=t(" \u3092 "),ws=o("code"),Fa=t("accelerator.prepare()"),un=t(" \u306B\u9001\u3063\u305F\u306E\u3067\u3001\u305D\u306E\u30C7\u30FC\u30BF\u9577\u3092\u7528\u3044\u3066\u5B66\u7FD2\u30B9\u30C6\u30C3\u30D7\u6570\u3092\u8A08\u7B97\u3059\u308B\u3053\u3068\u304C\u3067\u304D\u307E\u3059\u3002\u3053\u306E\u30E1\u30BD\u30C3\u30C9\u306Fdataloader\u306E\u9577\u3055\u3092\u5909\u66F4\u3059\u308B\u306E\u3067\u3001\u5E38\u306Bdataloader\u3092\u6E96\u5099\u3057\u305F\u5F8C\u306B\u884C\u3046\u5FC5\u8981\u304C\u3042\u308B\u3053\u3068\u3092\u5FD8\u308C\u306A\u3044\u3067\u304F\u3060\u3055\u3044\u3002\u3053\u3053\u3067\u306F\u3001\u5B66\u7FD2\u7387\u304B\u30890\u307E\u3067\u53E4\u5178\u7684\u306A\u7DDA\u5F62\u30B9\u30B1\u30B8\u30E5\u30FC\u30EB\u3092\u4F7F\u7528\u3057\u307E\u3059\u3002"),$r=f(),w(Cl.$$.fragment),jr=f(),gs=o("p"),Ma=t("\u6700\u5F8C\u306B\u3001\u30E2\u30C7\u30EB\u3092Hub\u306B\u30D7\u30C3\u30B7\u30E5\u3059\u308B\u305F\u3081\u306B\u3001\u4F5C\u696D\u30D5\u30A9\u30EB\u30C0\u306B "),La=o("code"),mn=t("Repository"),kr=t(" \u30AA\u30D6\u30B8\u30A7\u30AF\u30C8\u3092\u4F5C\u696D\u30D5\u30A9\u30EB\u30C0\u306B\u4F5C\u6210\u3059\u308B\u5FC5\u8981\u304C\u3042\u308A\u307E\u3059\u3002\u307E\u305A\u3001\u307E\u3060\u30ED\u30B0\u30A4\u30F3\u3057\u3066\u3044\u306A\u3051\u308C\u3070Hugging Face\u306B\u30ED\u30B0\u30A4\u30F3\u3057\u3066\u304F\u3060\u3055\u3044\u3002\u30E2\u30C7\u30EB\u306B\u4ED8\u4E0E\u3057\u305F\u3044\u30E2\u30C7\u30EBID\u304B\u3089\u30EA\u30DD\u30B8\u30C8\u30EA\u540D\u3092\u6C7A\u5B9A\u3057\u307E\u3059\u3002\uFF08"),Ql=o("code"),dn=t("repo_name"),Er=t("\u306F\u81EA\u7531\u306B\u7F6E\u304D\u63DB\u3048\u3066\u304F\u3060\u3055\u3044\uFF1B\u30E6\u30FC\u30B6\u30FC\u540D\u3092\u542B\u3080\u5FC5\u8981\u304C\u3042\u308B\u3060\u3051\u3067\u3001\u3053\u308C\u306F\u95A2\u6570 "),Dl=o("code"),vr=t("get_full_repo_name()"),gt=t(" \u304C\u884C\u3063\u3066\u3044\u308B\u4E8B\u3067\u3059\uFF09\u3002"),Na=f(),w(Je.$$.fragment),wr=f(),w(ql.$$.fragment),gr=f(),ca=o("p"),Ha=t("\u305D\u3057\u3066\u3001\u305D\u306E\u30EA\u30DD\u30B8\u30C8\u30EA\u3092\u30ED\u30FC\u30AB\u30EB\u30D5\u30A9\u30EB\u30C0\u30FC\u306B\u30AF\u30ED\u30FC\u30F3\u3059\u308B\u3053\u3068\u304C\u3067\u304D\u307E\u3059\u3002\u3059\u3067\u306B\u5B58\u5728\u3059\u308B\u306E\u3067\u3042\u308C\u3070\u3001\u3053\u306E\u30ED\u30FC\u30AB\u30EB\u30D5\u30A9\u30EB\u30C0\u30FC\u306F\u4F5C\u696D\u4E2D\u306E\u30EA\u30DD\u30B8\u30C8\u30EA\u306E\u65E2\u5B58\u306E\u30AF\u30ED\u30FC\u30F3\u3067\u3042\u308B\u3079\u304D\u3067\u3059\u3002"),xr=f(),w(xt.$$.fragment),Ga=f(),Qe=o("p"),Or=t("\u3053\u308C\u3067 "),xs=o("code"),hn=t("repo.push_to_hub()"),Ua=t(" \u30E1\u30BD\u30C3\u30C9\u3092\u547C\u3073\u51FA\u3059\u3053\u3068\u3067\u3001"),Wa=o("code"),bn=t("output_dir"),Xa=t(" \u306B\u4FDD\u5B58\u3057\u305F\u3082\u306E\u3092\u30A2\u30C3\u30D7\u30ED\u30FC\u30C9\u3067\u304D\u308B\u3088\u3046\u306B\u306A\u308A\u307E\u3057\u305F\u3002\u3053\u308C\u306B\u3088\u308A\u3001\u5404\u30A8\u30DD\u30C3\u30AF\u7D42\u4E86\u6642\u306B\u4E2D\u9593\u30E2\u30C7\u30EB\u3092\u30A2\u30C3\u30D7\u30ED\u30FC\u30C9\u3059\u308B\u3053\u3068\u304C\u3067\u304D\u307E\u3059\u3002"),yr=f(),Ot=o("h3"),Cr=t("\u5B66\u7FD2\u30EB\u30FC\u30D7"),Pl=f(),Ms=o("p"),yt=t(`\u3053\u308C\u3067\u5B66\u7FD2\u30EB\u30FC\u30D7\u3092\u66F8\u304F\u6E96\u5099\u304C\u3067\u304D\u307E\u3057\u305F\u3002
\u8A55\u4FA1\u90E8\u5206\u3092\u7C21\u7565\u5316\u3059\u308B\u305F\u3081\u3001`),Za=o("code"),Dr=t("postprocess()"),Ct=t(` \u95A2\u6570\u3092\u7C21\u5358\u306B\u5B9A\u7FA9\u3057\u307E\u3059\u3002
\u3053\u306E\u95A2\u6570\u306F\u4E88\u6E2C\u5024\u3068\u30E9\u30D9\u30EB\u3092\u53D7\u3051\u53D6\u3063\u3066 `),fa=o("code"),pl=t("metric"),$n=t(" \u30AA\u30D6\u30B8\u30A7\u30AF\u30C8\u304C\u671F\u5F85\u3059\u308B\u3088\u3046\u306A\u6587\u5B57\u5217\u306E\u30EA\u30B9\u30C8\u306B\u5909\u63DB\u3057\u307E\u3059\u3002"),Dt=f(),w(qt.$$.fragment),qr=f(),et=o("p"),jn=t("\u6B21\u306B\u3001\u30C8\u30EC\u30FC\u30CB\u30F3\u30B0\u30EB\u30FC\u30D7\u3092\u66F8\u304D\u307E\u3059\u3002\u30C8\u30EC\u30FC\u30CB\u30F3\u30B0\u306E\u9032\u6357\u3092\u78BA\u8A8D\u3059\u308B\u305F\u3081\u306E\u30D7\u30ED\u30B0\u30EC\u30B9\u30D0\u30FC\u3092\u5B9A\u7FA9\u3057\u305F\u5F8C\u3001\u30EB\u30FC\u30D7\u306F3\u3064\u306E\u30D1\u30FC\u30C8\u306B\u5206\u304B\u308C\u307E\u3059\u3002"),Pr=f(),Ls=o("ul"),il=o("li"),Pt=o("p"),Ka=t("\u5B66\u7FD2\u305D\u306E\u3082\u306E\u3002"),Ya=o("code"),kn=t("train_dataloader"),zr=t("\u306B\u5BFE\u3059\u308B\u53E4\u5178\u7684\u306A\u7E70\u308A\u8FD4\u3057\u3067\u3001\u30E2\u30C7\u30EB\u3092\u524D\u65B9\u306B\u4F1D\u64AD\u3055\u305B\u3001\u5F8C\u65B9\u306B\u9006\u4F1D\u64AD\u3055\u305B\u3001\u6700\u9069\u5316\u306E\u30B9\u30C6\u30C3\u30D7\u3092\u884C\u3044\u307E\u3059\u3002"),zt=f(),_a=o("li"),_e=o("p"),En=t("\u8A55\u4FA1\u3002\u30E2\u30C7\u30EB\u306E\u51FA\u529B\u3092\u30D0\u30C3\u30C1\u3067\u53D6\u5F97\u3057\u305F\u5F8C\u306B\u3001\u65B0\u3057\u3044\u4E8B\u3092\u3057\u307E\u3059\u30022\u3064\u306E\u30D7\u30ED\u30BB\u30B9\u3067\u5165\u529B\u3068\u30E9\u30D9\u30EB\u3092\u7570\u306A\u308B\u5F62\u72B6\u306B\u30D1\u30C7\u30A3\u30F3\u30B0\u3057\u3066\u3044\u308B\u304B\u3082\u3057\u308C\u306A\u3044\u306E\u3067\u3001"),Tt=o("code"),vn=t("gather()"),wn=t("\u30E1\u30BD\u30C3\u30C9\u3092\u547C\u3076\u524D\u306B "),zl=o("code"),gn=t("accelerator.pad_across_processes()"),xn=t(" \u3092\u4F7F\u3063\u3066\u4E88\u6E2C\u5024\u3068\u30E9\u30D9\u30EB\u3092\u540C\u3058\u5F62\u72B6\u306B\u3059\u308B\u5FC5\u8981\u304C\u3042\u308B\u306E\u3067\u3059\u3002\u3053\u308C\u3092\u884C\u308F\u306A\u3044\u3068\u3001\u8A55\u4FA1\u304C\u30A8\u30E9\u30FC\u306B\u306A\u308B\u304B\u3001\u6C38\u9060\u306B\u30CF\u30F3\u30B0\u30A2\u30C3\u30D7\u3057\u307E\u3059\u3002\u305D\u3057\u3066\u3001\u7D50\u679C\u3092 "),ua=o("code"),Os=t("metric.add_batch()"),On=t(" \u306B\u9001\u308A\u3001\u8A55\u4FA1\u30EB\u30FC\u30D7\u304C\u7D42\u4E86\u3057\u305F\u3089 "),Tl=o("code"),yn=t("metric.compute()"),Cn=t(" \u3092\u547C\u3073\u51FA\u3057\u307E\u3059\u3002"),Va=f(),Ja=o("li"),Il=o("p"),Qa=t("\u4FDD\u5B58\u3068\u30A2\u30C3\u30D7\u30ED\u30FC\u30C9\u3002\u307E\u305A\u30E2\u30C7\u30EB\u3068tokenizer\u3092\u4FDD\u5B58\u3057\u3001\u6B21\u306B "),er=o("code"),Dn=t("repo.push_to_hub()"),Tr=t(" \u3092\u547C\u3073\u51FA\u3057\u307E\u3059\u3002\u5F15\u6570 "),Al=o("code"),Ir=t("blocking=False"),It=t(" \u3092\u4F7F\u3063\u3066\u3001\u{1F917} Hub library\u306B\u975E\u540C\u671F\u51E6\u7406\u3067\u30D7\u30C3\u30B7\u30E5\u3059\u308B\u3088\u3046\u306B\u6307\u793A\u3057\u3066\u3044\u308B\u3053\u3068\u306B\u6CE8\u610F\u3057\u3066\u304F\u3060\u3055\u3044\u3002\u3053\u306E\u6307\u5B9A\u3092\u3059\u308B\u3068\u3001\u30C8\u30EC\u30FC\u30CB\u30F3\u30B0\u306F\u901A\u5E38\u901A\u308A\u884C\u308F\u308C\u3001\u3053\u306E\uFF08\u9577\u3044\u6642\u9593\u306E\u304B\u304B\u308B\uFF09\u547D\u4EE4\u306F\u30D0\u30C3\u30AF\u30B0\u30E9\u30A6\u30F3\u30C9\u3067\u5B9F\u884C\u3055\u308C\u307E\u3059\u3002"),sr=f(),ps=o("p"),qn=t("\u4EE5\u4E0B\u306F\u3001\u30C8\u30EC\u30FC\u30CB\u30F3\u30B0\u30EB\u30FC\u30D7\u306E\u5B8C\u5168\u306A\u30B3\u30FC\u30C9\u3067\u3059\u3002"),ma=f(),w(At.$$.fragment),Ar=f(),st=o("p"),Pn=t("\u4ECA\u56DE\u521D\u3081\u3066\u{1F917} Accelerate\u3067\u4FDD\u5B58\u3055\u308C\u305F\u30E2\u30C7\u30EB\u3092\u3054\u89A7\u306B\u306A\u308B\u65B9\u306E\u305F\u3081\u306B\u3001\u305D\u308C\u306B\u4ED8\u968F\u3059\u308B3\u884C\u306E\u30B3\u30FC\u30C9\u3092\u5C11\u3057\u70B9\u691C\u3057\u3066\u307F\u307E\u3057\u3087\u3046\u3002"),Rr=f(),w(lt.$$.fragment),tt=f(),ie=o("p"),Rt=t("\u6700\u521D\u306E\u884C\u306F\u660E\u3089\u304B\u3067\u3059\u3002\u3053\u306E\u884C\u306F\u3001\u3059\u3079\u3066\u306E\u30D7\u30ED\u30BB\u30B9\u306B\u3001\u5168\u30D7\u30ED\u30BB\u30B9\u304C\u305D\u306E\u884C\u306B\u9054\u3059\u308B\u307E\u3067\u3001\u51E6\u7406\u3092\u5F85\u3064\u3088\u3046\u6307\u793A\u3057\u307E\u3059\u3002\u3053\u308C\u306F\u3001\u4FDD\u5B58\u3059\u308B\u524D\u306B\u3001\u3059\u3079\u3066\u306E\u30D7\u30ED\u30BB\u30B9\u304C\u540C\u3058\u30E2\u30C7\u30EB\u306B\u306A\u3063\u3066\u3044\u308B\u4E8B\u3092\u78BA\u8A8D\u3059\u308B\u305F\u3081\u3067\u3059\u3002\u6B21\u306B "),da=o("code"),ze=t("unwrapped_model"),zn=t(" \u3092\u53D6\u5F97\u3057\u307E\u3059\u3002\u3053\u308C\u306F\u5B9A\u7FA9\u3057\u305F\u30D9\u30FC\u30B9\u30E2\u30C7\u30EB\u3067\u3059\u3002"),St=o("code"),Tn=t("accelerator.prepare()"),In=t(" \u30E1\u30BD\u30C3\u30C9\u306F\u5206\u6563\u3057\u3066\u5B66\u7FD2\u3059\u308B\u3088\u3046\u306B\u30E2\u30C7\u30EB\u3092\u5909\u66F4\u3059\u308B\u306E\u3067\u3001"),Bt=o("code"),An=t("save_pretrained()"),Rn=t(" \u30E1\u30BD\u30C3\u30C9\u3092\u6301\u305F\u306A\u304F\u306A\u308A\u307E\u3059\u3002"),Ft=o("code"),Sn=t("accelerator.unwrap_model()"),Bn=t(" \u30E1\u30BD\u30C3\u30C9\u306F\u305D\u306E\u30B9\u30C6\u30C3\u30D7\u3092\u5143\u306B\u623B\u3057\u307E\u3059\u3002\u6700\u5F8C\u306B\u3001"),Mt=o("code"),Fn=t("save_pretrained()"),Mn=t(" \u3092\u547C\u3073\u51FA\u3057\u307E\u3059\u304C\u3001\u3053\u306E\u30E1\u30BD\u30C3\u30C9\u306B\u306F "),Lt=o("code"),Ln=t("torch.save()"),Nn=t(" \u306E\u4EE3\u308F\u308A\u306B "),ha=o("code"),at=t("accelerator.save()"),Hn=t(" \u3092\u4F7F\u7528\u3059\u308B\u3088\u3046\u306B\u6307\u793A\u3057\u307E\u3059\u3002"),Nt=f(),cl=o("p"),Gn=t("\u3053\u308C\u304C\u5B8C\u4E86\u3059\u308B\u3068\u3001"),ba=o("code"),Ht=t("Trainer"),Sr=t(" \u3067\u5B66\u7FD2\u3057\u305F\u3082\u306E\u3068\u307B\u307C\u540C\u3058\u7D50\u679C\u3092\u5F97\u308B\u3053\u3068\u304C\u3067\u304D\u308B\u30E2\u30C7\u30EB\u304C\u3067\u304D\u3042\u304C\u308A\u307E\u3059\u3002\u3053\u306E\u30B3\u30FC\u30C9\u3092\u4F7F\u3063\u3066\u5B66\u7FD2\u3057\u305F\u30E2\u30C7\u30EB\u306F "),Ns=o("a"),$a=o("em"),es=t("huggingface-course/bert-finetuned-ner-accelerate"),Un=t(" \u3067\u78BA\u8A8D\u3059\u308B\u3053\u3068\u304C\u3067\u304D\u307E\u3059\u3002\u307E\u305F\u3001\u5B66\u7FD2\u30EB\u30FC\u30D7\u306E\u5FAE\u8ABF\u6574\u3092\u8A66\u3057\u305F\u3044\u5834\u5408\u306F\u3001\u4E0A\u306B\u793A\u3057\u305F\u30B3\u30FC\u30C9\u3092\u7DE8\u96C6\u3059\u308B\u3053\u3068\u3067\u76F4\u63A5\u5B9F\u88C5\u3059\u308B\u3053\u3068\u304C\u3067\u304D\u307E\u3059!"),this.h()},l(r){u=p(r,"H3",{id:!0});var h=i(u);$=a(h,"\u30E2\u30C7\u30EB\u3092\u5B9A\u7FA9"),h.forEach(s),d=_(r),b=p(r,"P",{});var rt=i(b);D=a(rt,"\u4ECA\u56DE\u306F\u30C8\u30FC\u30AF\u30F3\u5206\u985E\u306E\u554F\u984C\u3092\u6271\u3046\u306E\u3067\u3001 "),E=p(rt,"CODE",{});var qo=i(E);v=a(qo,"AutoModelForTokenClassification"),qo.forEach(s),q=a(rt," \u30AF\u30E9\u30B9\u3092\u4F7F\u7528\u3057\u307E\u3059\u3002\u3053\u306E\u30E2\u30C7\u30EB\u3092\u5B9A\u7FA9\u3059\u308B\u969B\u306B\u899A\u3048\u3066\u304A\u304F\u3079\u304D\u3053\u3068\u306F\u3001\u30E9\u30D9\u30EB\u306E\u6570\u306B\u95A2\u3059\u308B\u60C5\u5831\u3092\u6E21\u3059\u3053\u3068\u3067\u3059\u3002\u6700\u3082\u7C21\u5358\u306A\u65B9\u6CD5\u306F "),C=p(rt,"CODE",{});var Po=i(C);z=a(Po,"num_labels"),Po.forEach(s),B=a(rt," \u5F15\u6570\u3067\u305D\u306E\u6570\u3092\u6E21\u3059\u3053\u3068\u3067\u3059\u304C\u3001\u3053\u306E\u30BB\u30AF\u30B7\u30E7\u30F3\u306E\u6700\u521D\u306B\u898B\u305F\u3088\u3046\u306A\u7D20\u6575\u306A\u63A8\u8AD6\u30A6\u30A3\u30B8\u30A7\u30C3\u30C8\u3092\u52D5\u4F5C\u3055\u305B\u305F\u3044\u5834\u5408\u306F\u3001\u4EE3\u308F\u308A\u306B\u6B63\u3057\u3044\u30E9\u30D9\u30EB\u306E\u5BFE\u5FDC\u95A2\u4FC2\u3092\u8A2D\u5B9A\u3057\u305F\u65B9\u304C\u826F\u3044\u3067\u3057\u3087\u3046\u3002"),rt.forEach(s),A=_(r),R=p(r,"P",{});var ka=i(R);U=a(ka,"id2label"),y=p(ka,"CODE",{});var zo=i(y);M=a(zo,"\u3068"),zo.forEach(s),K=a(ka,"label2id` \u3068\u3044\u3046 2 \u3064\u306E\u8F9E\u66F8\u578B\u30C7\u30FC\u30BF\u304C\u3042\u308A\u3001ID \u304B\u3089\u30E9\u30D9\u30EB\u3001\u30E9\u30D9\u30EB\u304B\u3089 ID \u3078\u306E\u30DE\u30C3\u30D4\u30F3\u30B0\u3092\u8A2D\u5B9A\u3059\u308B\u3053\u3068\u304C\u3067\u304D\u307E\u3059\u3002"),ka.forEach(s),Z=_(r),g(W.$$.fragment,r),I=_(r),F=p(r,"P",{});var Br=i(F);L=a(Br,"\u3042\u3068\u306F\u305D\u308C\u3089\u3092 "),X=p(Br,"CODE",{});var Fr=i(X);Q=a(Fr,"AutoModelForTokenClassification.from_pretrained()"),Fr.forEach(s),V=a(Br," \u30E1\u30BD\u30C3\u30C9\u306B\u6E21\u305B\u3070\u3001\u30E2\u30C7\u30EB\u306E\u69CB\u6210\u306B\u8A2D\u5B9A\u3055\u308C\u3001\u9069\u5207\u306B\u4FDD\u5B58\u3055\u308C\u3066 Hub \u306B\u30A2\u30C3\u30D7\u30ED\u30FC\u30C9\u3055\u308C\u307E\u3059\u3002"),Br.forEach(s),le=_(r),g(J.$$.fragment,r),ce=_(r),Y=p(r,"P",{});var lr=i(Y);G=p(lr,"A",{href:!0});var To=i(G);te=a(To,"\u7B2C3\u7AE0"),To.forEach(s),$e=a(lr," \u3067 "),N=p(lr,"CODE",{});var Wn=i(N);ee=a(Wn,"AutoModelForSequenceClassification"),Wn.forEach(s),je=a(lr," \u3092\u5B9A\u7FA9\u3057\u305F\u3068\u304D\u306E\u3088\u3046\u306B\u3001\u30E2\u30C7\u30EB\u3092\u4F5C\u6210\u3059\u308B\u3068\u3001\u3044\u304F\u3064\u304B\u306E\u91CD\u307F\u304C\u4F7F\u308F\u308C\u3066\u3044\u306A\u3044\uFF08\u4E8B\u524D\u5B66\u7FD2\u6E08\u307F\u30E2\u30C7\u30EB\u306E\u30D8\u30C3\u30C9\u90E8\u306E\u91CD\u307F\uFF09\u3001\u4ED6\u306E\u3044\u304F\u3064\u304B\u306E\u91CD\u307F\u304C\u30E9\u30F3\u30C0\u30E0\u306B\u521D\u671F\u5316\u3055\u308C\u3066\u3044\u308B\uFF08\u65B0\u3057\u304F\u63A5\u7D9A\u3057\u305F\u30C8\u30FC\u30AF\u30F3\u5206\u985E\u30D8\u30C3\u30C9\u306E\u91CD\u307F\uFF09\u3001\u3053\u306E\u30E2\u30C7\u30EB\u306F\u30C8\u30EC\u30FC\u30CB\u30F3\u30B0\u3059\u308B\u5FC5\u8981\u304C\u3042\u308B\u3068\u3044\u3046\u8B66\u544A\u304C\u8868\u793A\u3055\u308C\u307E\u3059\u3002\u30C8\u30EC\u30FC\u30CB\u30F3\u30B0\u306F\u3059\u3050\u306B\u3067\u3082\u5B9F\u884C\u3067\u304D\u307E\u3059\u304C\u3001\u307E\u305A\u306F\u3053\u306E\u30E2\u30C7\u30EB\u304C\u6B63\u3057\u3044\u6570\u306E\u30E9\u30D9\u30EB\u3092\u6301\u3064\u3053\u3068\u3092\u518D\u78BA\u8A8D\u3057\u307E\u3057\u3087\u3046\u3002"),lr.forEach(s),Te=_(r),g(se.$$.fragment,r),fe=_(r),g(ae.$$.fragment,r),ge=_(r),g(re.$$.fragment,r),fs=_(r),ue=p(r,"H3",{id:!0});var Ea=i(ue);ke=a(Ea,"\u30E2\u30C7\u30EB\u306E\u5FAE\u8ABF\u6574"),Ea.forEach(s),Ml=_(r),xe=p(r,"P",{});var tr=i(xe);Le=a(tr,`\u3053\u308C\u3067\u30E2\u30C7\u30EB\u3092\u5B66\u7FD2\u3059\u308B\u6E96\u5099\u304C\u6574\u3044\u307E\u3057\u305F\uFF01
\u3057\u304B\u3057\u3001`),Ne=p(tr,"CODE",{});var ar=i(Ne);Gs=a(ar,"Trainer"),ar.forEach(s),Ll=a(tr,"\u3092\u5B9A\u7FA9\u3059\u308B\u524D\u306B\u3001\u6700\u5F8C\u306B2\u3064\u306E\u3053\u3068\u3092\u3059\u308B\u5FC5\u8981\u304C\u3042\u308A\u307E\u3059\u3002"),tr.forEach(s),Ie=_(r),$l=p(r,"P",{});var Io=i($l);jl=a(Io,"Hugging Face\u306B\u30ED\u30B0\u30A4\u30F3\u3057\u3001\u5B66\u7FD2\u7528\u30CF\u30A4\u30D1\u30FC\u30D1\u30E9\u30E1\u30FC\u30BF\u3092\u5B9A\u7FA9\u3059\u308B\u5FC5\u8981\u304C\u3042\u308A\u307E\u3059\u3002\u3082\u3057Notebook\u3067\u4F5C\u696D\u3057\u3066\u3044\u308B\u306A\u3089\u3001\u3053\u308C\u3092\u52A9\u3051\u308B\u4FBF\u5229\u306A\u95A2\u6570\u304C\u3042\u308A\u307E\u3059\u3002"),Io.forEach(s),Oe=_(r),g(Cs.$$.fragment,r),ss=_(r),He=p(r,"P",{});var Xn=i(He);Nl=a(Xn,"\u3053\u308C\u306B\u3088\u308A\u3001Hugging Face\u306E\u30ED\u30B0\u30A4\u30F3\u60C5\u5831\u3092\u5165\u529B\u3059\u308B\u30A6\u30A3\u30B8\u30A7\u30C3\u30C8\u304C\u8868\u793A\u3055\u308C\u307E\u3059\u3002"),Xn.forEach(s),_s=_(r),Ge=p(r,"P",{});var va=i(Ge);Hl=a(va,"Notebook\u3067\u4F5C\u696D\u3057\u3066\u3044\u306A\u3044\u5834\u5408\u306F\u3001\u30BF\u30FC\u30DF\u30CA\u30EB\u306B\u6B21\u306E\u884C\u3092\u5165\u529B\u3059\u308B\u3060\u3051\u3067\u3059\u3002"),va.forEach(s),de=_(r),g(ye.$$.fragment,r),Us=_(r),Ce=p(r,"P",{});var rr=i(Ce);it=a(rr,"\u5B8C\u4E86\u3057\u305F\u3089\u3001"),Ds=p(rr,"CODE",{});var wa=i(Ds);Gl=a(wa,"TrainingArguments"),wa.forEach(s),us=a(rr,"\u3092\u5B9A\u7FA9\u3059\u308B\u4E8B\u304C\u3067\u304D\u308B\u3088\u3046\u306B\u306A\u308A\u307E\u3059\u3002"),rr.forEach(s),qs=_(r),g(Ee.$$.fragment,r),ls=_(r),ts=p(r,"P",{});var Zn=i(ts);Ae=a(Zn,"\u3053\u308C\u3089\u306E\u30D1\u30E9\u30E1\u30FC\u30BF\u306E\u307B\u3068\u3093\u3069\u306F\u4EE5\u524D\u306B\u898B\u305F\u3053\u3068\u304C\u3042\u308B\u3068\u601D\u3044\u307E\u3059\u3002"),Zn.forEach(s),kl=_(r),ne=p(r,"P",{});var Hs=i(ne);ct=a(Hs,"\u30CF\u30A4\u30D1\u30FC\u30D1\u30E9\u30E1\u30FC\u30BF\uFF08\u5B66\u7FD2\u7387\u3001\u5B66\u7FD2\u30A8\u30DD\u30C3\u30AF\u6570\u3001\u30A6\u30A7\u30A4\u30C8\u6E1B\u8870\u306A\u3069\uFF09\u3092\u8A2D\u5B9A\u3057\u3001"),ms=p(Hs,"CODE",{});var Ao=i(ms);ds=a(Ao,"push_to_hub=True"),Ao.forEach(s),ft=a(Hs," \u3092\u6307\u5B9A\u3057\u3066\u3001\u30E2\u30C7\u30EB\u3092\u4FDD\u5B58\u3057\u3066\u5404\u30A8\u30DD\u30C3\u30AF\u7D42\u4E86\u6642\u306B\u8A55\u4FA1\u3057\u3001\u305D\u306E\u7D50\u679C\u3092\u30E2\u30C7\u30EB\u30CF\u30D6\u306B\u30A2\u30C3\u30D7\u30ED\u30FC\u30C9\u3059\u308B\u3053\u3068\u3092\u6307\u793A\u3057\u307E\u3059\u3002"),Hs.forEach(s),hs=_(r),he=p(r,"P",{});var ys=i(he);_t=a(ys,"\u306A\u304A\u3001 "),Ps=p(ys,"CODE",{});var Ro=i(Ps);De=a(Ro,"hub_model_id"),Ro.forEach(s),Ul=a(ys," \u5F15\u6570\u3067\u30D7\u30C3\u30B7\u30E5\u5148\u306E\u30EA\u30DD\u30B8\u30C8\u30EA\u540D\u3092\u6307\u5B9A\u3067\u304D\u307E\u3059 (\u7279\u306B\u3001\u7279\u5B9A\u306E\u7D44\u7E54\uFF08organization\uFF09\u306B\u30D7\u30C3\u30B7\u30E5\u3059\u308B\u5834\u5408\u306F\u3001\u3053\u306E\u5F15\u6570\u3092\u4F7F\u7528\u3059\u308B\u5FC5\u8981\u304C\u3042\u308A\u307E\u3059)\u3002\u4F8B\u3048\u3070\u3001"),me=p(ys,"A",{href:!0,rel:!0});var Kn=i(me);El=p(Kn,"CODE",{});var Mr=i(El);Ws=a(Mr,"huggingface-course"),Mr.forEach(s),Wl=a(Kn," organization"),Kn.forEach(s),zs=a(ys," \u306B\u30E2\u30C7\u30EB\u3092\u30D7\u30C3\u30B7\u30E5\u3059\u308B\u5834\u5408\u3001"),Ts=p(ys,"CODE",{});var So=i(Ts);oe=a(So,"TrainingArguments"),So.forEach(s),Xs=a(ys," \u306B "),as=p(ys,"CODE",{});var Bo=i(as);Xl=a(Bo,'hub_model_id="huggingface-course/bert-finetuned-ner"'),Bo.forEach(s),ve=a(ys," \u3092\u8FFD\u52A0\u3057\u307E\u3057\u305F\u3002"),ys.forEach(s),Zs=_(r),Ue=p(r,"P",{});var nr=i(Ue);ut=a(nr,"\u30C7\u30D5\u30A9\u30EB\u30C8\u3067\u306F\u3001\u4F7F\u7528\u3055\u308C\u308B\u30EA\u30DD\u30B8\u30C8\u30EA\u306F\u3042\u306A\u305F\u306E\u540D\u524D\u304C\u4F7F\u308F\u308C\u3001\u8A2D\u5B9A\u3057\u305F\u51FA\u529B\u30C7\u30A3\u30EC\u30AF\u30C8\u30EA\u3061\u306A\u3093\u3060\u540D\u524D\u3001\u4F8B\u3048\u3070\u4ECA\u56DE\u306E\u4F8B\u3067\u306F "),vl=p(nr,"CODE",{});var Gt=i(vl);Ks=a(Gt,'"sgugger/bert-finetuned-ner"'),Gt.forEach(s),Ys=a(nr," \u3068\u306A\u308A\u307E\u3059\u3002"),nr.forEach(s),Zl=_(r),g(We.$$.fragment,r),bs=_(r),rs=p(r,"P",{});var or=i(rs);$s=a(or,"\u6700\u5F8C\u306B\u3001\u3059\u3079\u3066\u3092 "),Xe=p(or,"CODE",{});var is=i(Xe);Is=a(is,"Trainer"),is.forEach(s),Vs=a(or," \u306B\u6E21\u3057\u3066\u3001\u30C8\u30EC\u30FC\u30CB\u30F3\u30B0\u3092\u958B\u59CB\u3059\u308B\u3060\u3051\u3067\u3059\u3002"),or.forEach(s),wl=_(r),g(qe.$$.fragment,r),js=_(r),Ze=p(r,"P",{});var Fo=i(Ze);gl=a(Fo,"\u5B66\u7FD2\u304C\u884C\u308F\u308C\u3066\u3044\u308B\u9593\u3001\u30E2\u30C7\u30EB\u304C\u4FDD\u5B58\u3055\u308C\u308B\u305F\u3073\u306B\uFF08\u3053\u3053\u3067\u306F\u30A8\u30DD\u30C3\u30AF\u3054\u3068\u306B\uFF09\u30D0\u30C3\u30AF\u30B0\u30E9\u30A6\u30F3\u30C9\u3067Hub\u306B\u30A2\u30C3\u30D7\u30ED\u30FC\u30C9\u3055\u308C\u308B\u3053\u3068\u306B\u6CE8\u610F\u3057\u3066\u304F\u3060\u3055\u3044\u3002\u3053\u306E\u3088\u3046\u306B\u3057\u3066\u3001\u5FC5\u8981\u306B\u5FDC\u3058\u3066\u5225\u306E\u30DE\u30B7\u30F3\u3067\u5B66\u7FD2\u3092\u518D\u958B\u3059\u308B\u3053\u3068\u304C\u3067\u304D\u307E\u3059\u3002"),Fo.forEach(s),Pe=_(r),ks=p(r,"P",{});var ga=i(ks);Js=a(ga,"\u5B66\u7FD2\u304C\u5B8C\u4E86\u3057\u305F\u3089\u3001"),m=p(ga,"CODE",{});var Mo=i(m);S=a(Mo,"push_to_hub()"),Mo.forEach(s),As=a(ga," \u30E1\u30BD\u30C3\u30C9\u3092\u4F7F\u7528\u3057\u3066\u3001\u6700\u65B0\u30D0\u30FC\u30B8\u30E7\u30F3\u306E\u30E2\u30C7\u30EB\u3092\u30A2\u30C3\u30D7\u30ED\u30FC\u30C9\u3059\u308B\u3088\u3046\u306B\u3057\u307E\u3059\u3002"),ga.forEach(s),Rs=_(r),g(Ss.$$.fragment,r),ns=_(r),Qs=p(r,"P",{});var Lo=i(Qs);pe=a(Lo,"\u3053\u306E\u30B3\u30DE\u30F3\u30C9\u306F\u3001\u4ECA\u884C\u3063\u305F\u30B3\u30DF\u30C3\u30C8\u306E URL \u3092\u8FD4\u3059\u306E\u3067\u3001\u305D\u308C\u3092\u691C\u67FB\u3057\u305F\u3044\u5834\u5408\u306F\u3001\u3053\u306E\u30B3\u30DE\u30F3\u30C9\u3092\u4F7F\u7528\u3057\u307E\u3059\u3002"),Lo.forEach(s),mt=_(r),g(Re.$$.fragment,r),dt=_(r),Bs=p(r,"P",{});var xa=i(Bs);Kl=a(xa,"\u307E\u305F\u3001"),Yl=p(xa,"CODE",{});var No=i(Yl);ea=a(No,"Trainer"),No.forEach(s),Vl=a(xa,`\u306F\u3059\u3079\u3066\u306E\u8A55\u4FA1\u7D50\u679C\u3092\u542B\u3080\u30E2\u30C7\u30EB\u30AB\u30FC\u30C9\u3092\u8D77\u8349\u3057\u3001\u30A2\u30C3\u30D7\u30ED\u30FC\u30C9\u3057\u307E\u3059\u3002\u3053\u306E\u6BB5\u968E\u3067\u3001Model Hub\u4E0A\u306E\u63A8\u8AD6\u30A6\u30A3\u30B8\u30A7\u30C3\u30C8\u3092\u4F7F\u3063\u3066\u30E2\u30C7\u30EB\u3092\u30C6\u30B9\u30C8\u3057\u3001\u53CB\u4EBA\u3068\u5171\u6709\u3059\u308B\u3053\u3068\u304C\u3067\u304D\u307E\u3059\u3002\u3053\u308C\u3067\u3001\u30C8\u30FC\u30AF\u30F3\u5206\u985E\u30BF\u30B9\u30AF\u306E\u30E2\u30C7\u30EB\u5FAE\u8ABF\u6574\u306B\u6210\u529F\u3057\u307E\u3057\u305F\u3002
\u304A\u3081\u3067\u3068\u3046\u3054\u3056\u3044\u307E\u3059\uFF01`),xa.forEach(s),ht=_(r),Se=p(r,"P",{});var Ho=i(Se);Jl=a(Ho,"\u3082\u3046\u5C11\u3057\u6DF1\u304F\u5B66\u7FD2\u30EB\u30FC\u30D7\u306B\u3064\u3044\u3066\u5B66\u3073\u305F\u3044\u5834\u5408\u306F\u3001\u{1F917} Accelerate \u3092\u4F7F\u3063\u3066\u540C\u3058\u3053\u3068\u3092\u3059\u308B\u65B9\u6CD5\u3092\u7D39\u4ECB\u3057\u307E\u3059\u3002"),Ho.forEach(s),bt=_(r),el=p(r,"H2",{id:!0});var Lr=i(el);Es=a(Lr,"\u30AB\u30B9\u30BF\u30E0\u30C8\u30EC\u30FC\u30CB\u30F3\u30B0\u30EB\u30FC\u30D7"),Lr.forEach(s),$t=_(r),Fs=p(r,"P",{});var Nr=i(Fs);xl=a(Nr,"\u305D\u308C\u3067\u306F\u3001\u5FC5\u8981\u306A\u90E8\u5206\u3092\u7C21\u5358\u306B\u30AB\u30B9\u30BF\u30DE\u30A4\u30BA\u3067\u304D\u308B\u3088\u3046\u306B\u3001\u30C8\u30EC\u30FC\u30CB\u30F3\u30B0\u30EB\u30FC\u30D7\u306E\u5168\u4F53\u50CF\u3092\u898B\u3066\u307F\u307E\u3057\u3087\u3046\u3002\u3053\u308C\u306F\u3001"),os=p(Nr,"A",{href:!0});var Go=i(os);Ke=a(Go,"\u7B2C3\u7AE0"),Go.forEach(s),sl=a(Nr," \u3067\u884C\u3063\u305F\u3053\u3068\u3068\u3088\u304F\u4F3C\u3066\u3044\u307E\u3059\u304C\u3001\u8A55\u4FA1\u306E\u305F\u3081\u306B\u5C11\u3057\u5909\u66F4\u304C\u52A0\u3048\u3089\u308C\u3066\u3044\u307E\u3059\u3002"),Nr.forEach(s),Ol=_(r),we=p(r,"H3",{id:!0});var Hr=i(we);sa=a(Hr,"\u30C8\u30EC\u30FC\u30CB\u30F3\u30B0\u306E\u305F\u3081\u306E\u6E96\u5099"),Hr.forEach(s),ll=_(r),Be=p(r,"P",{});var nt=i(Be);la=a(nt,"\u307E\u305A\u3001\u30C7\u30FC\u30BF\u30BB\u30C3\u30C8\u304B\u3089 "),tl=p(nt,"CODE",{});var Uo=i(tl);Qr=a(Uo,"DataLoader"),Uo.forEach(s),en=a(nt," \u3092\u4F5C\u6210\u3059\u308B\u5FC5\u8981\u304C\u3042\u308A\u307E\u3059\u3002\u3053\u3053\u3067\u306F\u3001"),ta=p(nt,"CODE",{});var Yn=i(ta);Ye=a(Yn,"data_collator"),Yn.forEach(s),aa=a(nt," \u3092 "),jt=p(nt,"CODE",{});var Oa=i(jt);sn=a(Oa,"collate_fn"),Oa.forEach(s),ln=a(nt," \u3068\u3057\u3066\u518D\u5229\u7528\u3057\u3001\u30C8\u30EC\u30FC\u30CB\u30F3\u30B0\u30BB\u30C3\u30C8\u3092\u30B7\u30E3\u30C3\u30D5\u30EB\u3057\u307E\u3059\u3002\u305F\u3060\u3057\u3001\u691C\u8A3C\u30BB\u30C3\u30C8\u306F\u30B7\u30E3\u30C3\u30D5\u30EB\u3057\u307E\u305B\u3093\u3002"),nt.forEach(s),ur=_(r),g(Ve.$$.fragment,r),ra=_(r),na=p(r,"P",{});var Vn=i(na);tn=a(Vn,"\u6B21\u306B\u3001\u30E2\u30C7\u30EB\u306E\u518D\u5B9A\u7FA9\u3092\u884C\u3044\u307E\u3059\u3002\u3053\u308C\u306F\u3001\u4EE5\u524D\u306E\u5FAE\u8ABF\u6574\u3092\u7D99\u7D9A\u3059\u308B\u306E\u3067\u306F\u306A\u304F\u3001BERT\u3067\u4E8B\u524D\u5B66\u7FD2\u3057\u305F\u30E2\u30C7\u30EB\u304B\u3089\u518D\u3073\u958B\u59CB\u3059\u308B\u3053\u3068\u3092\u78BA\u8A8D\u3059\u308B\u305F\u3081\u3067\u3059\u3002"),Vn.forEach(s),oa=_(r),g(kt.$$.fragment,r),mr=_(r),vs=p(r,"P",{});var Rl=i(vs);an=a(Rl,"\u305D\u308C\u304B\u3089\u3001\u30AA\u30D7\u30C6\u30A3\u30DE\u30A4\u30B6\u304C\u5FC5\u8981\u306B\u306A\u308A\u307E\u3059\u3002\u3053\u3053\u3067\u306F\u3001\u53E4\u5178\u7684\u306A "),Aa=p(Rl,"CODE",{});var Wo=i(Aa);rn=a(Wo,"AdamW"),Wo.forEach(s),al=a(Rl," \u3092\u4F7F\u7528\u3057\u307E\u3059\u3002\u3053\u308C\u306F "),Et=p(Rl,"CODE",{});var Jn=i(Et);nn=a(Jn,"Adam"),Jn.forEach(s),on=a(Rl," \u306E\u3088\u3046\u306A\u3082\u306E\u3067\u3059\u304C\u3001\u91CD\u307F\u306E\u6E1B\u8870\u306E\u9069\u7528\u65B9\u6CD5\u3092\u4FEE\u6B63\u3057\u305F\u3082\u306E\u3067\u3059\u3002"),Rl.forEach(s),pa=_(r),g(vt.$$.fragment,r),dr=_(r),rl=p(r,"P",{});var Ut=i(rl);pn=a(Ut,"\u3053\u308C\u3089\u306E\u30AA\u30D6\u30B8\u30A7\u30AF\u30C8\u3092\u3059\u3079\u3066\u53D6\u5F97\u3057\u305F\u3089\u3001\u305D\u308C\u3089\u3092 "),Ra=p(Ut,"CODE",{});var Xo=i(Ra);cn=a(Xo,"accelerator.prepare()"),Xo.forEach(s),nl=a(Ut," \u30E1\u30BD\u30C3\u30C9\u306B\u9001\u308A\u307E\u3059\u3002"),Ut.forEach(s),ia=_(r),g(wt.$$.fragment,r),hr=_(r),g(yl.$$.fragment,r),br=_(r),ol=p(r,"P",{});var Wt=i(ol);Sa=a(Wt,"\u3053\u308C\u3067 "),Ba=p(Wt,"CODE",{});var ya=i(Ba);fn=a(ya,"train_dataloader"),ya.forEach(s),_n=a(Wt," \u3092 "),ws=p(Wt,"CODE",{});var Qn=i(ws);Fa=a(Qn,"accelerator.prepare()"),Qn.forEach(s),un=a(Wt," \u306B\u9001\u3063\u305F\u306E\u3067\u3001\u305D\u306E\u30C7\u30FC\u30BF\u9577\u3092\u7528\u3044\u3066\u5B66\u7FD2\u30B9\u30C6\u30C3\u30D7\u6570\u3092\u8A08\u7B97\u3059\u308B\u3053\u3068\u304C\u3067\u304D\u307E\u3059\u3002\u3053\u306E\u30E1\u30BD\u30C3\u30C9\u306Fdataloader\u306E\u9577\u3055\u3092\u5909\u66F4\u3059\u308B\u306E\u3067\u3001\u5E38\u306Bdataloader\u3092\u6E96\u5099\u3057\u305F\u5F8C\u306B\u884C\u3046\u5FC5\u8981\u304C\u3042\u308B\u3053\u3068\u3092\u5FD8\u308C\u306A\u3044\u3067\u304F\u3060\u3055\u3044\u3002\u3053\u3053\u3067\u306F\u3001\u5B66\u7FD2\u7387\u304B\u30890\u307E\u3067\u53E4\u5178\u7684\u306A\u7DDA\u5F62\u30B9\u30B1\u30B8\u30E5\u30FC\u30EB\u3092\u4F7F\u7528\u3057\u307E\u3059\u3002"),Wt.forEach(s),$r=_(r),g(Cl.$$.fragment,r),jr=_(r),gs=p(r,"P",{});var cs=i(gs);Ma=a(cs,"\u6700\u5F8C\u306B\u3001\u30E2\u30C7\u30EB\u3092Hub\u306B\u30D7\u30C3\u30B7\u30E5\u3059\u308B\u305F\u3081\u306B\u3001\u4F5C\u696D\u30D5\u30A9\u30EB\u30C0\u306B "),La=p(cs,"CODE",{});var Zo=i(La);mn=a(Zo,"Repository"),Zo.forEach(s),kr=a(cs," \u30AA\u30D6\u30B8\u30A7\u30AF\u30C8\u3092\u4F5C\u696D\u30D5\u30A9\u30EB\u30C0\u306B\u4F5C\u6210\u3059\u308B\u5FC5\u8981\u304C\u3042\u308A\u307E\u3059\u3002\u307E\u305A\u3001\u307E\u3060\u30ED\u30B0\u30A4\u30F3\u3057\u3066\u3044\u306A\u3051\u308C\u3070Hugging Face\u306B\u30ED\u30B0\u30A4\u30F3\u3057\u3066\u304F\u3060\u3055\u3044\u3002\u30E2\u30C7\u30EB\u306B\u4ED8\u4E0E\u3057\u305F\u3044\u30E2\u30C7\u30EBID\u304B\u3089\u30EA\u30DD\u30B8\u30C8\u30EA\u540D\u3092\u6C7A\u5B9A\u3057\u307E\u3059\u3002\uFF08"),Ql=p(cs,"CODE",{});var pr=i(Ql);dn=a(pr,"repo_name"),pr.forEach(s),Er=a(cs,"\u306F\u81EA\u7531\u306B\u7F6E\u304D\u63DB\u3048\u3066\u304F\u3060\u3055\u3044\uFF1B\u30E6\u30FC\u30B6\u30FC\u540D\u3092\u542B\u3080\u5FC5\u8981\u304C\u3042\u308B\u3060\u3051\u3067\u3001\u3053\u308C\u306F\u95A2\u6570 "),Dl=p(cs,"CODE",{});var Ko=i(Dl);vr=a(Ko,"get_full_repo_name()"),Ko.forEach(s),gt=a(cs," \u304C\u884C\u3063\u3066\u3044\u308B\u4E8B\u3067\u3059\uFF09\u3002"),cs.forEach(s),Na=_(r),g(Je.$$.fragment,r),wr=_(r),g(ql.$$.fragment,r),gr=_(r),ca=p(r,"P",{});var Yo=i(ca);Ha=a(Yo,"\u305D\u3057\u3066\u3001\u305D\u306E\u30EA\u30DD\u30B8\u30C8\u30EA\u3092\u30ED\u30FC\u30AB\u30EB\u30D5\u30A9\u30EB\u30C0\u30FC\u306B\u30AF\u30ED\u30FC\u30F3\u3059\u308B\u3053\u3068\u304C\u3067\u304D\u307E\u3059\u3002\u3059\u3067\u306B\u5B58\u5728\u3059\u308B\u306E\u3067\u3042\u308C\u3070\u3001\u3053\u306E\u30ED\u30FC\u30AB\u30EB\u30D5\u30A9\u30EB\u30C0\u30FC\u306F\u4F5C\u696D\u4E2D\u306E\u30EA\u30DD\u30B8\u30C8\u30EA\u306E\u65E2\u5B58\u306E\u30AF\u30ED\u30FC\u30F3\u3067\u3042\u308B\u3079\u304D\u3067\u3059\u3002"),Yo.forEach(s),xr=_(r),g(xt.$$.fragment,r),Ga=_(r),Qe=p(r,"P",{});var Xt=i(Qe);Or=a(Xt,"\u3053\u308C\u3067 "),xs=p(Xt,"CODE",{});var fl=i(xs);hn=a(fl,"repo.push_to_hub()"),fl.forEach(s),Ua=a(Xt," \u30E1\u30BD\u30C3\u30C9\u3092\u547C\u3073\u51FA\u3059\u3053\u3068\u3067\u3001"),Wa=p(Xt,"CODE",{});var _l=i(Wa);bn=a(_l,"output_dir"),_l.forEach(s),Xa=a(Xt," \u306B\u4FDD\u5B58\u3057\u305F\u3082\u306E\u3092\u30A2\u30C3\u30D7\u30ED\u30FC\u30C9\u3067\u304D\u308B\u3088\u3046\u306B\u306A\u308A\u307E\u3057\u305F\u3002\u3053\u308C\u306B\u3088\u308A\u3001\u5404\u30A8\u30DD\u30C3\u30AF\u7D42\u4E86\u6642\u306B\u4E2D\u9593\u30E2\u30C7\u30EB\u3092\u30A2\u30C3\u30D7\u30ED\u30FC\u30C9\u3059\u308B\u3053\u3068\u304C\u3067\u304D\u307E\u3059\u3002"),Xt.forEach(s),yr=_(r),Ot=p(r,"H3",{id:!0});var ir=i(Ot);Cr=a(ir,"\u5B66\u7FD2\u30EB\u30FC\u30D7"),ir.forEach(s),Pl=_(r),Ms=p(r,"P",{});var ul=i(Ms);yt=a(ul,`\u3053\u308C\u3067\u5B66\u7FD2\u30EB\u30FC\u30D7\u3092\u66F8\u304F\u6E96\u5099\u304C\u3067\u304D\u307E\u3057\u305F\u3002
\u8A55\u4FA1\u90E8\u5206\u3092\u7C21\u7565\u5316\u3059\u308B\u305F\u3081\u3001`),Za=p(ul,"CODE",{});var Vo=i(Za);Dr=a(Vo,"postprocess()"),Vo.forEach(s),Ct=a(ul,` \u95A2\u6570\u3092\u7C21\u5358\u306B\u5B9A\u7FA9\u3057\u307E\u3059\u3002
\u3053\u306E\u95A2\u6570\u306F\u4E88\u6E2C\u5024\u3068\u30E9\u30D9\u30EB\u3092\u53D7\u3051\u53D6\u3063\u3066 `),fa=p(ul,"CODE",{});var eo=i(fa);pl=a(eo,"metric"),eo.forEach(s),$n=a(ul," \u30AA\u30D6\u30B8\u30A7\u30AF\u30C8\u304C\u671F\u5F85\u3059\u308B\u3088\u3046\u306A\u6587\u5B57\u5217\u306E\u30EA\u30B9\u30C8\u306B\u5909\u63DB\u3057\u307E\u3059\u3002"),ul.forEach(s),Dt=_(r),g(qt.$$.fragment,r),qr=_(r),et=p(r,"P",{});var ml=i(et);jn=a(ml,"\u6B21\u306B\u3001\u30C8\u30EC\u30FC\u30CB\u30F3\u30B0\u30EB\u30FC\u30D7\u3092\u66F8\u304D\u307E\u3059\u3002\u30C8\u30EC\u30FC\u30CB\u30F3\u30B0\u306E\u9032\u6357\u3092\u78BA\u8A8D\u3059\u308B\u305F\u3081\u306E\u30D7\u30ED\u30B0\u30EC\u30B9\u30D0\u30FC\u3092\u5B9A\u7FA9\u3057\u305F\u5F8C\u3001\u30EB\u30FC\u30D7\u306F3\u3064\u306E\u30D1\u30FC\u30C8\u306B\u5206\u304B\u308C\u307E\u3059\u3002"),ml.forEach(s),Pr=_(r),Ls=p(r,"UL",{});var Sl=i(Ls);il=p(Sl,"LI",{});var Jo=i(il);Pt=p(Jo,"P",{});var Gr=i(Pt);Ka=a(Gr,"\u5B66\u7FD2\u305D\u306E\u3082\u306E\u3002"),Ya=p(Gr,"CODE",{});var Ur=i(Ya);kn=a(Ur,"train_dataloader"),Ur.forEach(s),zr=a(Gr,"\u306B\u5BFE\u3059\u308B\u53E4\u5178\u7684\u306A\u7E70\u308A\u8FD4\u3057\u3067\u3001\u30E2\u30C7\u30EB\u3092\u524D\u65B9\u306B\u4F1D\u64AD\u3055\u305B\u3001\u5F8C\u65B9\u306B\u9006\u4F1D\u64AD\u3055\u305B\u3001\u6700\u9069\u5316\u306E\u30B9\u30C6\u30C3\u30D7\u3092\u884C\u3044\u307E\u3059\u3002"),Gr.forEach(s),Jo.forEach(s),zt=_(Sl),_a=p(Sl,"LI",{});var Qo=i(_a);_e=p(Qo,"P",{});var dl=i(_e);En=a(dl,"\u8A55\u4FA1\u3002\u30E2\u30C7\u30EB\u306E\u51FA\u529B\u3092\u30D0\u30C3\u30C1\u3067\u53D6\u5F97\u3057\u305F\u5F8C\u306B\u3001\u65B0\u3057\u3044\u4E8B\u3092\u3057\u307E\u3059\u30022\u3064\u306E\u30D7\u30ED\u30BB\u30B9\u3067\u5165\u529B\u3068\u30E9\u30D9\u30EB\u3092\u7570\u306A\u308B\u5F62\u72B6\u306B\u30D1\u30C7\u30A3\u30F3\u30B0\u3057\u3066\u3044\u308B\u304B\u3082\u3057\u308C\u306A\u3044\u306E\u3067\u3001"),Tt=p(dl,"CODE",{});var Wr=i(Tt);vn=a(Wr,"gather()"),Wr.forEach(s),wn=a(dl,"\u30E1\u30BD\u30C3\u30C9\u3092\u547C\u3076\u524D\u306B "),zl=p(dl,"CODE",{});var ep=i(zl);gn=a(ep,"accelerator.pad_across_processes()"),ep.forEach(s),xn=a(dl," \u3092\u4F7F\u3063\u3066\u4E88\u6E2C\u5024\u3068\u30E9\u30D9\u30EB\u3092\u540C\u3058\u5F62\u72B6\u306B\u3059\u308B\u5FC5\u8981\u304C\u3042\u308B\u306E\u3067\u3059\u3002\u3053\u308C\u3092\u884C\u308F\u306A\u3044\u3068\u3001\u8A55\u4FA1\u304C\u30A8\u30E9\u30FC\u306B\u306A\u308B\u304B\u3001\u6C38\u9060\u306B\u30CF\u30F3\u30B0\u30A2\u30C3\u30D7\u3057\u307E\u3059\u3002\u305D\u3057\u3066\u3001\u7D50\u679C\u3092 "),ua=p(dl,"CODE",{});var sp=i(ua);Os=a(sp,"metric.add_batch()"),sp.forEach(s),On=a(dl," \u306B\u9001\u308A\u3001\u8A55\u4FA1\u30EB\u30FC\u30D7\u304C\u7D42\u4E86\u3057\u305F\u3089 "),Tl=p(dl,"CODE",{});var so=i(Tl);yn=a(so,"metric.compute()"),so.forEach(s),Cn=a(dl," \u3092\u547C\u3073\u51FA\u3057\u307E\u3059\u3002"),dl.forEach(s),Qo.forEach(s),Va=_(Sl),Ja=p(Sl,"LI",{});var Bl=i(Ja);Il=p(Bl,"P",{});var Ca=i(Il);Qa=a(Ca,"\u4FDD\u5B58\u3068\u30A2\u30C3\u30D7\u30ED\u30FC\u30C9\u3002\u307E\u305A\u30E2\u30C7\u30EB\u3068tokenizer\u3092\u4FDD\u5B58\u3057\u3001\u6B21\u306B "),er=p(Ca,"CODE",{});var Da=i(er);Dn=a(Da,"repo.push_to_hub()"),Da.forEach(s),Tr=a(Ca," \u3092\u547C\u3073\u51FA\u3057\u307E\u3059\u3002\u5F15\u6570 "),Al=p(Ca,"CODE",{});var Xr=i(Al);Ir=a(Xr,"blocking=False"),Xr.forEach(s),It=a(Ca," \u3092\u4F7F\u3063\u3066\u3001\u{1F917} Hub library\u306B\u975E\u540C\u671F\u51E6\u7406\u3067\u30D7\u30C3\u30B7\u30E5\u3059\u308B\u3088\u3046\u306B\u6307\u793A\u3057\u3066\u3044\u308B\u3053\u3068\u306B\u6CE8\u610F\u3057\u3066\u304F\u3060\u3055\u3044\u3002\u3053\u306E\u6307\u5B9A\u3092\u3059\u308B\u3068\u3001\u30C8\u30EC\u30FC\u30CB\u30F3\u30B0\u306F\u901A\u5E38\u901A\u308A\u884C\u308F\u308C\u3001\u3053\u306E\uFF08\u9577\u3044\u6642\u9593\u306E\u304B\u304B\u308B\uFF09\u547D\u4EE4\u306F\u30D0\u30C3\u30AF\u30B0\u30E9\u30A6\u30F3\u30C9\u3067\u5B9F\u884C\u3055\u308C\u307E\u3059\u3002"),Ca.forEach(s),Bl.forEach(s),Sl.forEach(s),sr=_(r),ps=p(r,"P",{});var lp=i(ps);qn=a(lp,"\u4EE5\u4E0B\u306F\u3001\u30C8\u30EC\u30FC\u30CB\u30F3\u30B0\u30EB\u30FC\u30D7\u306E\u5B8C\u5168\u306A\u30B3\u30FC\u30C9\u3067\u3059\u3002"),lp.forEach(s),ma=_(r),g(At.$$.fragment,r),Ar=_(r),st=p(r,"P",{});var tp=i(st);Pn=a(tp,"\u4ECA\u56DE\u521D\u3081\u3066\u{1F917} Accelerate\u3067\u4FDD\u5B58\u3055\u308C\u305F\u30E2\u30C7\u30EB\u3092\u3054\u89A7\u306B\u306A\u308B\u65B9\u306E\u305F\u3081\u306B\u3001\u305D\u308C\u306B\u4ED8\u968F\u3059\u308B3\u884C\u306E\u30B3\u30FC\u30C9\u3092\u5C11\u3057\u70B9\u691C\u3057\u3066\u307F\u307E\u3057\u3087\u3046\u3002"),tp.forEach(s),Rr=_(r),g(lt.$$.fragment,r),tt=_(r),ie=p(r,"P",{});var be=i(ie);Rt=a(be,"\u6700\u521D\u306E\u884C\u306F\u660E\u3089\u304B\u3067\u3059\u3002\u3053\u306E\u884C\u306F\u3001\u3059\u3079\u3066\u306E\u30D7\u30ED\u30BB\u30B9\u306B\u3001\u5168\u30D7\u30ED\u30BB\u30B9\u304C\u305D\u306E\u884C\u306B\u9054\u3059\u308B\u307E\u3067\u3001\u51E6\u7406\u3092\u5F85\u3064\u3088\u3046\u6307\u793A\u3057\u307E\u3059\u3002\u3053\u308C\u306F\u3001\u4FDD\u5B58\u3059\u308B\u524D\u306B\u3001\u3059\u3079\u3066\u306E\u30D7\u30ED\u30BB\u30B9\u304C\u540C\u3058\u30E2\u30C7\u30EB\u306B\u306A\u3063\u3066\u3044\u308B\u4E8B\u3092\u78BA\u8A8D\u3059\u308B\u305F\u3081\u3067\u3059\u3002\u6B21\u306B "),da=p(be,"CODE",{});var ap=i(da);ze=a(ap,"unwrapped_model"),ap.forEach(s),zn=a(be," \u3092\u53D6\u5F97\u3057\u307E\u3059\u3002\u3053\u308C\u306F\u5B9A\u7FA9\u3057\u305F\u30D9\u30FC\u30B9\u30E2\u30C7\u30EB\u3067\u3059\u3002"),St=p(be,"CODE",{});var rp=i(St);Tn=a(rp,"accelerator.prepare()"),rp.forEach(s),In=a(be," \u30E1\u30BD\u30C3\u30C9\u306F\u5206\u6563\u3057\u3066\u5B66\u7FD2\u3059\u308B\u3088\u3046\u306B\u30E2\u30C7\u30EB\u3092\u5909\u66F4\u3059\u308B\u306E\u3067\u3001"),Bt=p(be,"CODE",{});var lo=i(Bt);An=a(lo,"save_pretrained()"),lo.forEach(s),Rn=a(be," \u30E1\u30BD\u30C3\u30C9\u3092\u6301\u305F\u306A\u304F\u306A\u308A\u307E\u3059\u3002"),Ft=p(be,"CODE",{});var hl=i(Ft);Sn=a(hl,"accelerator.unwrap_model()"),hl.forEach(s),Bn=a(be," \u30E1\u30BD\u30C3\u30C9\u306F\u305D\u306E\u30B9\u30C6\u30C3\u30D7\u3092\u5143\u306B\u623B\u3057\u307E\u3059\u3002\u6700\u5F8C\u306B\u3001"),Mt=p(be,"CODE",{});var bl=i(Mt);Fn=a(bl,"save_pretrained()"),bl.forEach(s),Mn=a(be," \u3092\u547C\u3073\u51FA\u3057\u307E\u3059\u304C\u3001\u3053\u306E\u30E1\u30BD\u30C3\u30C9\u306B\u306F "),Lt=p(be,"CODE",{});var cr=i(Lt);Ln=a(cr,"torch.save()"),cr.forEach(s),Nn=a(be," \u306E\u4EE3\u308F\u308A\u306B "),ha=p(be,"CODE",{});var fr=i(ha);at=a(fr,"accelerator.save()"),fr.forEach(s),Hn=a(be," \u3092\u4F7F\u7528\u3059\u308B\u3088\u3046\u306B\u6307\u793A\u3057\u307E\u3059\u3002"),be.forEach(s),Nt=_(r),cl=p(r,"P",{});var qa=i(cl);Gn=a(qa,"\u3053\u308C\u304C\u5B8C\u4E86\u3059\u308B\u3068\u3001"),ba=p(qa,"CODE",{});var to=i(ba);Ht=a(to,"Trainer"),to.forEach(s),Sr=a(qa," \u3067\u5B66\u7FD2\u3057\u305F\u3082\u306E\u3068\u307B\u307C\u540C\u3058\u7D50\u679C\u3092\u5F97\u308B\u3053\u3068\u304C\u3067\u304D\u308B\u30E2\u30C7\u30EB\u304C\u3067\u304D\u3042\u304C\u308A\u307E\u3059\u3002\u3053\u306E\u30B3\u30FC\u30C9\u3092\u4F7F\u3063\u3066\u5B66\u7FD2\u3057\u305F\u30E2\u30C7\u30EB\u306F "),Ns=p(qa,"A",{href:!0,rel:!0});var Pa=i(Ns);$a=p(Pa,"EM",{});var ao=i($a);es=a(ao,"huggingface-course/bert-finetuned-ner-accelerate"),ao.forEach(s),Pa.forEach(s),Un=a(qa," \u3067\u78BA\u8A8D\u3059\u308B\u3053\u3068\u304C\u3067\u304D\u307E\u3059\u3002\u307E\u305F\u3001\u5B66\u7FD2\u30EB\u30FC\u30D7\u306E\u5FAE\u8ABF\u6574\u3092\u8A66\u3057\u305F\u3044\u5834\u5408\u306F\u3001\u4E0A\u306B\u793A\u3057\u305F\u30B3\u30FC\u30C9\u3092\u7DE8\u96C6\u3059\u308B\u3053\u3068\u3067\u76F4\u63A5\u5B9F\u88C5\u3059\u308B\u3053\u3068\u304C\u3067\u304D\u307E\u3059!"),qa.forEach(s),this.h()},h(){P(u,"id",""),P(G,"href","/course/ja/chapter3"),P(ue,"id",""),P(me,"href","https://huggingface.co/huggingface-course"),P(me,"rel","nofollow"),P(el,"id",""),P(os,"href","/course/ja/chapter3/4"),P(we,"id",""),P(Ot,"id",""),P(Ns,"href","https://huggingface.co/huggingface-course/bert-finetuned-ner-accelerate"),P(Ns,"rel","nofollow")},m(r,h){n(r,u,h),l(u,$),n(r,d,h),n(r,b,h),l(b,D),l(b,E),l(E,v),l(b,q),l(b,C),l(C,z),l(b,B),n(r,A,h),n(r,R,h),l(R,U),l(R,y),l(y,M),l(R,K),n(r,Z,h),x(W,r,h),n(r,I,h),n(r,F,h),l(F,L),l(F,X),l(X,Q),l(F,V),n(r,le,h),x(J,r,h),n(r,ce,h),n(r,Y,h),l(Y,G),l(G,te),l(Y,$e),l(Y,N),l(N,ee),l(Y,je),n(r,Te,h),x(se,r,h),n(r,fe,h),x(ae,r,h),n(r,ge,h),x(re,r,h),n(r,fs,h),n(r,ue,h),l(ue,ke),n(r,Ml,h),n(r,xe,h),l(xe,Le),l(xe,Ne),l(Ne,Gs),l(xe,Ll),n(r,Ie,h),n(r,$l,h),l($l,jl),n(r,Oe,h),x(Cs,r,h),n(r,ss,h),n(r,He,h),l(He,Nl),n(r,_s,h),n(r,Ge,h),l(Ge,Hl),n(r,de,h),x(ye,r,h),n(r,Us,h),n(r,Ce,h),l(Ce,it),l(Ce,Ds),l(Ds,Gl),l(Ce,us),n(r,qs,h),x(Ee,r,h),n(r,ls,h),n(r,ts,h),l(ts,Ae),n(r,kl,h),n(r,ne,h),l(ne,ct),l(ne,ms),l(ms,ds),l(ne,ft),n(r,hs,h),n(r,he,h),l(he,_t),l(he,Ps),l(Ps,De),l(he,Ul),l(he,me),l(me,El),l(El,Ws),l(me,Wl),l(he,zs),l(he,Ts),l(Ts,oe),l(he,Xs),l(he,as),l(as,Xl),l(he,ve),n(r,Zs,h),n(r,Ue,h),l(Ue,ut),l(Ue,vl),l(vl,Ks),l(Ue,Ys),n(r,Zl,h),x(We,r,h),n(r,bs,h),n(r,rs,h),l(rs,$s),l(rs,Xe),l(Xe,Is),l(rs,Vs),n(r,wl,h),x(qe,r,h),n(r,js,h),n(r,Ze,h),l(Ze,gl),n(r,Pe,h),n(r,ks,h),l(ks,Js),l(ks,m),l(m,S),l(ks,As),n(r,Rs,h),x(Ss,r,h),n(r,ns,h),n(r,Qs,h),l(Qs,pe),n(r,mt,h),x(Re,r,h),n(r,dt,h),n(r,Bs,h),l(Bs,Kl),l(Bs,Yl),l(Yl,ea),l(Bs,Vl),n(r,ht,h),n(r,Se,h),l(Se,Jl),n(r,bt,h),n(r,el,h),l(el,Es),n(r,$t,h),n(r,Fs,h),l(Fs,xl),l(Fs,os),l(os,Ke),l(Fs,sl),n(r,Ol,h),n(r,we,h),l(we,sa),n(r,ll,h),n(r,Be,h),l(Be,la),l(Be,tl),l(tl,Qr),l(Be,en),l(Be,ta),l(ta,Ye),l(Be,aa),l(Be,jt),l(jt,sn),l(Be,ln),n(r,ur,h),x(Ve,r,h),n(r,ra,h),n(r,na,h),l(na,tn),n(r,oa,h),x(kt,r,h),n(r,mr,h),n(r,vs,h),l(vs,an),l(vs,Aa),l(Aa,rn),l(vs,al),l(vs,Et),l(Et,nn),l(vs,on),n(r,pa,h),x(vt,r,h),n(r,dr,h),n(r,rl,h),l(rl,pn),l(rl,Ra),l(Ra,cn),l(rl,nl),n(r,ia,h),x(wt,r,h),n(r,hr,h),x(yl,r,h),n(r,br,h),n(r,ol,h),l(ol,Sa),l(ol,Ba),l(Ba,fn),l(ol,_n),l(ol,ws),l(ws,Fa),l(ol,un),n(r,$r,h),x(Cl,r,h),n(r,jr,h),n(r,gs,h),l(gs,Ma),l(gs,La),l(La,mn),l(gs,kr),l(gs,Ql),l(Ql,dn),l(gs,Er),l(gs,Dl),l(Dl,vr),l(gs,gt),n(r,Na,h),x(Je,r,h),n(r,wr,h),x(ql,r,h),n(r,gr,h),n(r,ca,h),l(ca,Ha),n(r,xr,h),x(xt,r,h),n(r,Ga,h),n(r,Qe,h),l(Qe,Or),l(Qe,xs),l(xs,hn),l(Qe,Ua),l(Qe,Wa),l(Wa,bn),l(Qe,Xa),n(r,yr,h),n(r,Ot,h),l(Ot,Cr),n(r,Pl,h),n(r,Ms,h),l(Ms,yt),l(Ms,Za),l(Za,Dr),l(Ms,Ct),l(Ms,fa),l(fa,pl),l(Ms,$n),n(r,Dt,h),x(qt,r,h),n(r,qr,h),n(r,et,h),l(et,jn),n(r,Pr,h),n(r,Ls,h),l(Ls,il),l(il,Pt),l(Pt,Ka),l(Pt,Ya),l(Ya,kn),l(Pt,zr),l(Ls,zt),l(Ls,_a),l(_a,_e),l(_e,En),l(_e,Tt),l(Tt,vn),l(_e,wn),l(_e,zl),l(zl,gn),l(_e,xn),l(_e,ua),l(ua,Os),l(_e,On),l(_e,Tl),l(Tl,yn),l(_e,Cn),l(Ls,Va),l(Ls,Ja),l(Ja,Il),l(Il,Qa),l(Il,er),l(er,Dn),l(Il,Tr),l(Il,Al),l(Al,Ir),l(Il,It),n(r,sr,h),n(r,ps,h),l(ps,qn),n(r,ma,h),x(At,r,h),n(r,Ar,h),n(r,st,h),l(st,Pn),n(r,Rr,h),x(lt,r,h),n(r,tt,h),n(r,ie,h),l(ie,Rt),l(ie,da),l(da,ze),l(ie,zn),l(ie,St),l(St,Tn),l(ie,In),l(ie,Bt),l(Bt,An),l(ie,Rn),l(ie,Ft),l(Ft,Sn),l(ie,Bn),l(ie,Mt),l(Mt,Fn),l(ie,Mn),l(ie,Lt),l(Lt,Ln),l(ie,Nn),l(ie,ha),l(ha,at),l(ie,Hn),n(r,Nt,h),n(r,cl,h),l(cl,Gn),l(cl,ba),l(ba,Ht),l(cl,Sr),l(cl,Ns),l(Ns,$a),l($a,es),l(cl,Un),ja=!0},i(r){ja||(j(W.$$.fragment,r),j(J.$$.fragment,r),j(se.$$.fragment,r),j(ae.$$.fragment,r),j(re.$$.fragment,r),j(Cs.$$.fragment,r),j(ye.$$.fragment,r),j(Ee.$$.fragment,r),j(We.$$.fragment,r),j(qe.$$.fragment,r),j(Ss.$$.fragment,r),j(Re.$$.fragment,r),j(Ve.$$.fragment,r),j(kt.$$.fragment,r),j(vt.$$.fragment,r),j(wt.$$.fragment,r),j(yl.$$.fragment,r),j(Cl.$$.fragment,r),j(Je.$$.fragment,r),j(ql.$$.fragment,r),j(xt.$$.fragment,r),j(qt.$$.fragment,r),j(At.$$.fragment,r),j(lt.$$.fragment,r),ja=!0)},o(r){k(W.$$.fragment,r),k(J.$$.fragment,r),k(se.$$.fragment,r),k(ae.$$.fragment,r),k(re.$$.fragment,r),k(Cs.$$.fragment,r),k(ye.$$.fragment,r),k(Ee.$$.fragment,r),k(We.$$.fragment,r),k(qe.$$.fragment,r),k(Ss.$$.fragment,r),k(Re.$$.fragment,r),k(Ve.$$.fragment,r),k(kt.$$.fragment,r),k(vt.$$.fragment,r),k(wt.$$.fragment,r),k(yl.$$.fragment,r),k(Cl.$$.fragment,r),k(Je.$$.fragment,r),k(ql.$$.fragment,r),k(xt.$$.fragment,r),k(qt.$$.fragment,r),k(At.$$.fragment,r),k(lt.$$.fragment,r),ja=!1},d(r){r&&s(u),r&&s(d),r&&s(b),r&&s(A),r&&s(R),r&&s(Z),O(W,r),r&&s(I),r&&s(F),r&&s(le),O(J,r),r&&s(ce),r&&s(Y),r&&s(Te),O(se,r),r&&s(fe),O(ae,r),r&&s(ge),O(re,r),r&&s(fs),r&&s(ue),r&&s(Ml),r&&s(xe),r&&s(Ie),r&&s($l),r&&s(Oe),O(Cs,r),r&&s(ss),r&&s(He),r&&s(_s),r&&s(Ge),r&&s(de),O(ye,r),r&&s(Us),r&&s(Ce),r&&s(qs),O(Ee,r),r&&s(ls),r&&s(ts),r&&s(kl),r&&s(ne),r&&s(hs),r&&s(he),r&&s(Zs),r&&s(Ue),r&&s(Zl),O(We,r),r&&s(bs),r&&s(rs),r&&s(wl),O(qe,r),r&&s(js),r&&s(Ze),r&&s(Pe),r&&s(ks),r&&s(Rs),O(Ss,r),r&&s(ns),r&&s(Qs),r&&s(mt),O(Re,r),r&&s(dt),r&&s(Bs),r&&s(ht),r&&s(Se),r&&s(bt),r&&s(el),r&&s($t),r&&s(Fs),r&&s(Ol),r&&s(we),r&&s(ll),r&&s(Be),r&&s(ur),O(Ve,r),r&&s(ra),r&&s(na),r&&s(oa),O(kt,r),r&&s(mr),r&&s(vs),r&&s(pa),O(vt,r),r&&s(dr),r&&s(rl),r&&s(ia),O(wt,r),r&&s(hr),O(yl,r),r&&s(br),r&&s(ol),r&&s($r),O(Cl,r),r&&s(jr),r&&s(gs),r&&s(Na),O(Je,r),r&&s(wr),O(ql,r),r&&s(gr),r&&s(ca),r&&s(xr),O(xt,r),r&&s(Ga),r&&s(Qe),r&&s(yr),r&&s(Ot),r&&s(Pl),r&&s(Ms),r&&s(Dt),O(qt,r),r&&s(qr),r&&s(et),r&&s(Pr),r&&s(Ls),r&&s(sr),r&&s(ps),r&&s(ma),O(At,r),r&&s(Ar),r&&s(st),r&&s(Rr),O(lt,r),r&&s(tt),r&&s(ie),r&&s(Nt),r&&s(cl)}}}function Uf(H){let u,$,d,b,D;return{c(){u=o("p"),$=t("\u26A0\uFE0F \u30E9\u30D9\u30EB\u306E\u6570\u304C\u9593\u9055\u3063\u3066\u3044\u308B\u30E2\u30C7\u30EB\u304C\u3042\u308B\u3068\u3001\u5F8C\u3067 "),d=o("code"),b=t("model.fit()"),D=t(" \u3092\u547C\u3073\u51FA\u3059\u3068\u304D\u306B\u3088\u304F\u308F\u304B\u3089\u306A\u3044\u30A8\u30E9\u30FC\uFF08\u201CCUDA error: device-side assert triggered\u201D\u306E\u3088\u3046\u306A\u30A8\u30E9\u30FC\uFF09\u304C\u767A\u751F\u3057\u307E\u3059\u3002\u3053\u306E\u3088\u3046\u306A\u30A8\u30E9\u30FC\u306F\u30E6\u30FC\u30B6\u30FC\u304B\u3089\u5831\u544A\u3055\u308C\u308B\u30D0\u30B0\u306E\u539F\u56E0\u3068\u3057\u3066\u4E00\u756A\u591A\u3044\u3082\u306E\u3067\u3059\u3002\u3053\u306E\u30C1\u30A7\u30C3\u30AF\u3092\u5FC5\u305A\u884C\u3044\u3001\u671F\u5F85\u901A\u308A\u306E\u30E9\u30D9\u30EB\u6570\u3067\u3042\u308B\u3053\u3068\u3092\u78BA\u8A8D\u3057\u3066\u304F\u3060\u3055\u3044\u3002")},l(E){u=p(E,"P",{});var v=i(u);$=a(v,"\u26A0\uFE0F \u30E9\u30D9\u30EB\u306E\u6570\u304C\u9593\u9055\u3063\u3066\u3044\u308B\u30E2\u30C7\u30EB\u304C\u3042\u308B\u3068\u3001\u5F8C\u3067 "),d=p(v,"CODE",{});var q=i(d);b=a(q,"model.fit()"),q.forEach(s),D=a(v," \u3092\u547C\u3073\u51FA\u3059\u3068\u304D\u306B\u3088\u304F\u308F\u304B\u3089\u306A\u3044\u30A8\u30E9\u30FC\uFF08\u201CCUDA error: device-side assert triggered\u201D\u306E\u3088\u3046\u306A\u30A8\u30E9\u30FC\uFF09\u304C\u767A\u751F\u3057\u307E\u3059\u3002\u3053\u306E\u3088\u3046\u306A\u30A8\u30E9\u30FC\u306F\u30E6\u30FC\u30B6\u30FC\u304B\u3089\u5831\u544A\u3055\u308C\u308B\u30D0\u30B0\u306E\u539F\u56E0\u3068\u3057\u3066\u4E00\u756A\u591A\u3044\u3082\u306E\u3067\u3059\u3002\u3053\u306E\u30C1\u30A7\u30C3\u30AF\u3092\u5FC5\u305A\u884C\u3044\u3001\u671F\u5F85\u901A\u308A\u306E\u30E9\u30D9\u30EB\u6570\u3067\u3042\u308B\u3053\u3068\u3092\u78BA\u8A8D\u3057\u3066\u304F\u3060\u3055\u3044\u3002"),v.forEach(s)},m(E,v){n(E,u,v),l(u,$),l(u,d),l(d,b),l(u,D)},d(E){E&&s(u)}}}function Wf(H){let u;return{c(){u=t("\u{1F4A1} \u4F7F\u7528\u3059\u308B\u51FA\u529B\u30C7\u30A3\u30EC\u30AF\u30C8\u30EA\u304C\u65E2\u306B\u5B58\u5728\u3059\u308B\u5834\u5408\u306F\u3001\u30D7\u30C3\u30B7\u30E5\u3057\u305F\u3044\u30EA\u30DD\u30B8\u30C8\u30EA\u306E\u30ED\u30FC\u30AB\u30EB\u30AF\u30ED\u30FC\u30F3\u3067\u3042\u308B\u5FC5\u8981\u304C\u3042\u308A\u307E\u3059\u3002\u305D\u3046\u3067\u306A\u3044\u5834\u5408\u306F\u3001`Trainer` \u3092\u5B9A\u7FA9\u3059\u308B\u969B\u306B\u30A8\u30E9\u30FC\u304C\u767A\u751F\u3057\u3001\u65B0\u3057\u3044\u540D\u524D\u3092\u8A2D\u5B9A\u3059\u308B\u5FC5\u8981\u304C\u3042\u308A\u307E\u3059\u3002")},l($){u=a($,"\u{1F4A1} \u4F7F\u7528\u3059\u308B\u51FA\u529B\u30C7\u30A3\u30EC\u30AF\u30C8\u30EA\u304C\u65E2\u306B\u5B58\u5728\u3059\u308B\u5834\u5408\u306F\u3001\u30D7\u30C3\u30B7\u30E5\u3057\u305F\u3044\u30EA\u30DD\u30B8\u30C8\u30EA\u306E\u30ED\u30FC\u30AB\u30EB\u30AF\u30ED\u30FC\u30F3\u3067\u3042\u308B\u5FC5\u8981\u304C\u3042\u308A\u307E\u3059\u3002\u305D\u3046\u3067\u306A\u3044\u5834\u5408\u306F\u3001`Trainer` \u3092\u5B9A\u7FA9\u3059\u308B\u969B\u306B\u30A8\u30E9\u30FC\u304C\u767A\u751F\u3057\u3001\u65B0\u3057\u3044\u540D\u524D\u3092\u8A2D\u5B9A\u3059\u308B\u5FC5\u8981\u304C\u3042\u308A\u307E\u3059\u3002")},m($,d){n($,u,d)},d($){$&&s(u)}}}function Xf(H){let u,$,d,b,D;return{c(){u=o("p"),$=t("\u{1F6A8} TPU\u3067\u30C8\u30EC\u30FC\u30CB\u30F3\u30B0\u3059\u308B\u5834\u5408\u306F\u3001\u4E0A\u306E\u30BB\u30EB\u304B\u3089\u59CB\u307E\u308B\u30B3\u30FC\u30C9\u3092\u5168\u3066\u5C02\u7528\u306E\u30C8\u30EC\u30FC\u30CB\u30F3\u30B0\u95A2\u6570\u306B\u79FB\u52D5\u3059\u308B\u5FC5\u8981\u304C\u3042\u308A\u307E\u3059\u3002\u8A73\u3057\u304F\u306F"),d=o("a"),b=t("\u7B2C3\u7AE0"),D=t("\u3092\u53C2\u7167\u3057\u3066\u304F\u3060\u3055\u3044\u3002"),this.h()},l(E){u=p(E,"P",{});var v=i(u);$=a(v,"\u{1F6A8} TPU\u3067\u30C8\u30EC\u30FC\u30CB\u30F3\u30B0\u3059\u308B\u5834\u5408\u306F\u3001\u4E0A\u306E\u30BB\u30EB\u304B\u3089\u59CB\u307E\u308B\u30B3\u30FC\u30C9\u3092\u5168\u3066\u5C02\u7528\u306E\u30C8\u30EC\u30FC\u30CB\u30F3\u30B0\u95A2\u6570\u306B\u79FB\u52D5\u3059\u308B\u5FC5\u8981\u304C\u3042\u308A\u307E\u3059\u3002\u8A73\u3057\u304F\u306F"),d=p(v,"A",{href:!0});var q=i(d);b=a(q,"\u7B2C3\u7AE0"),q.forEach(s),D=a(v,"\u3092\u53C2\u7167\u3057\u3066\u304F\u3060\u3055\u3044\u3002"),v.forEach(s),this.h()},h(){P(d,"href","/course/ja/chapter3")},m(E,v){n(E,u,v),l(u,$),l(u,d),l(d,b),l(u,D)},d(E){E&&s(u)}}}function Zf(H){let u,$,d,b,D,E,v,q,C,z,B,A,R,U,y,M,K,Z,W,I,F,L,X,Q,V,le,J,ce,Y,G,te,$e,N,ee,je,Te,se,fe,ae,ge,re,fs,ue,ke,Ml,xe,Le,Ne,Gs,Ll,Ie,$l,jl,Oe,Cs,ss,He,Nl,_s,Ge,Hl,de,ye,Us,Ce,it,Ds,Gl,us,qs,Ee,ls,ts,Ae,kl,ne,ct,ms,ds,ft,hs,he,_t,Ps,De,Ul,me,El,Ws,Wl,zs,Ts,oe,Xs,as,Xl,ve,Zs,Ue,ut,vl,Ks,Ys,Zl,We,bs,rs,$s,Xe,Is,Vs,wl,qe,js,Ze,gl,Pe,ks,Js,m,S,As,Rs,Ss,ns,Qs,pe,mt,Re,dt,Bs,Kl,Yl,ea,Vl,ht,Se,Jl,bt,el,Es,$t,Fs,xl,os,Ke,sl,Ol,we,sa,ll,Be,la,tl,Qr,en,ta,Ye,aa,jt,sn,ln,ur,Ve,ra,na,tn,oa,kt,mr,vs,an,Aa,rn,al,Et,nn,on,pa,vt,dr,rl,pn,Ra,cn,nl,ia,wt,hr,yl,br,ol,Sa,Ba,fn,_n,ws,Fa,un,$r,Cl,jr,gs,Ma,La,mn,kr,Ql,dn,Er,Dl,vr,gt,Na,Je,wr,ql,gr,ca,Ha,xr,xt,Ga,Qe,Or,xs,hn,Ua,Wa,bn,Xa,yr,Ot,Cr,Pl,Ms,yt,Za,Dr,Ct,fa,pl,$n,Dt,qt,qr,et,jn,Pr,Ls,il,Pt,Ka,Ya,kn,zr,zt,_a,_e,En,Tt,vn,wn,zl,gn,xn,ua,Os,On,Tl,yn,Cn,Va,Ja,Il,Qa,er,Dn,Tr,Al,Ir,It,sr,ps,qn,ma,At,Ar,st,Pn,Rr,lt,tt,ie,Rt,da,ze,zn,St,Tn,In,Bt,An,Rn,Ft,Sn,Bn,Mt,Fn,Mn,Lt,Ln,Nn,ha,at,Hn,Nt,cl,Gn,ba,Ht,Sr,Ns,$a,es,Un,ja,r,h,rt,qo,Po,ka,zo,Br,Fr,lr,To,Wn,Ea,tr,ar,Io,Xn,va,rr,wa,Zn,Hs,Ao,ys,Ro,Kn,Mr,So,Bo,nr,Gt,or,is,Fo,ga,Mo,Lo,xa,No,Ho,Lr,Nr,Go,Hr,nt,Uo,Yn,Oa,Vn,Rl,Wo,Jn,Ut,Xo,Wt,ya,Qn,cs,Zo,pr,Ko,Yo,Xt,fl,_l,ir,ul,Vo,eo,ml,Sl,Jo,Gr,Ur,Qo,dl,Wr,ep,sp,so,Bl,Ca,Da,Xr,lp,tp,be,ap,rp,lo,hl,bl,cr,fr,qa,to,Pa,ao,ro,Tp,np,ci,Ip,no,Ap,oo,Rp,Zt,Kt,op,pp,po,fi,Sp,Yt,Vt,ip,io,Bp,cp,_i,Fp,fp,ui,Mp,co,Lp,fo,Np,_p,mi,Hp,_o,Gp,up,di,Up,uo,Wp,Jt,Qt,mp,dp,mo,hi,Xp,Zr,bi,qp,$i,ji,Zp,ho,Kp,bo,Yp,hp,ki,Vp;d=new yf({props:{fw:H[0]}});const gi=[Df,Cf],$o=[];function xi(e,c){return e[0]==="pt"?0:1}q=xi(H),C=$o[q]=gi[q](H),ae=new $f({props:{id:"wVHdVlPScxA"}}),us=new Jr({props:{$$slots:{default:[qf]},$$scope:{ctx:H}}}),Ae=new wi({}),De=new T({props:{code:`from datasets import load_dataset

raw_datasets = load_dataset("conll2003")`,highlighted:`<span class="hljs-keyword">from</span> datasets <span class="hljs-keyword">import</span> load_dataset

raw_datasets = load_dataset(<span class="hljs-string">&quot;conll2003&quot;</span>)`}}),oe=new T({props:{code:"raw_datasets",highlighted:"raw_datasets"}}),as=new T({props:{code:`DatasetDict({
    train: Dataset({
        features: ['chunk_tags', 'id', 'ner_tags', 'pos_tags', 'tokens'],
        num_rows: 14041
    })
    validation: Dataset({
        features: ['chunk_tags', 'id', 'ner_tags', 'pos_tags', 'tokens'],
        num_rows: 3250
    })
    test: Dataset({
        features: ['chunk_tags', 'id', 'ner_tags', 'pos_tags', 'tokens'],
        num_rows: 3453
    })
})`,highlighted:`DatasetDict({
    train: Dataset({
        features: [<span class="hljs-string">&#x27;chunk_tags&#x27;</span>, <span class="hljs-string">&#x27;id&#x27;</span>, <span class="hljs-string">&#x27;ner_tags&#x27;</span>, <span class="hljs-string">&#x27;pos_tags&#x27;</span>, <span class="hljs-string">&#x27;tokens&#x27;</span>],
        num_rows: <span class="hljs-number">14041</span>
    })
    validation: Dataset({
        features: [<span class="hljs-string">&#x27;chunk_tags&#x27;</span>, <span class="hljs-string">&#x27;id&#x27;</span>, <span class="hljs-string">&#x27;ner_tags&#x27;</span>, <span class="hljs-string">&#x27;pos_tags&#x27;</span>, <span class="hljs-string">&#x27;tokens&#x27;</span>],
        num_rows: <span class="hljs-number">3250</span>
    })
    test: Dataset({
        features: [<span class="hljs-string">&#x27;chunk_tags&#x27;</span>, <span class="hljs-string">&#x27;id&#x27;</span>, <span class="hljs-string">&#x27;ner_tags&#x27;</span>, <span class="hljs-string">&#x27;pos_tags&#x27;</span>, <span class="hljs-string">&#x27;tokens&#x27;</span>],
        num_rows: <span class="hljs-number">3453</span>
    })
})`}}),bs=new T({props:{code:'raw_datasets["train"][0]["tokens"]',highlighted:'raw_datasets[<span class="hljs-string">&quot;train&quot;</span>][<span class="hljs-number">0</span>][<span class="hljs-string">&quot;tokens&quot;</span>]'}}),$s=new T({props:{code:"['EU', 'rejects', 'German', 'call', 'to', 'boycott', 'British', 'lamb', '.']",highlighted:'[<span class="hljs-string">&#x27;EU&#x27;</span>, <span class="hljs-string">&#x27;rejects&#x27;</span>, <span class="hljs-string">&#x27;German&#x27;</span>, <span class="hljs-string">&#x27;call&#x27;</span>, <span class="hljs-string">&#x27;to&#x27;</span>, <span class="hljs-string">&#x27;boycott&#x27;</span>, <span class="hljs-string">&#x27;British&#x27;</span>, <span class="hljs-string">&#x27;lamb&#x27;</span>, <span class="hljs-string">&#x27;.&#x27;</span>]'}}),qe=new T({props:{code:'raw_datasets["train"][0]["ner_tags"]',highlighted:'raw_datasets[<span class="hljs-string">&quot;train&quot;</span>][<span class="hljs-number">0</span>][<span class="hljs-string">&quot;ner_tags&quot;</span>]'}}),Ze=new T({props:{code:"[3, 0, 7, 0, 0, 0, 7, 0, 0]",highlighted:'[<span class="hljs-number">3</span>, <span class="hljs-number">0</span>, <span class="hljs-number">7</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">7</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>]'}}),Rs=new T({props:{code:`ner_feature = raw_datasets["train"].features["ner_tags"]
ner_feature`,highlighted:`ner_feature = raw_datasets[<span class="hljs-string">&quot;train&quot;</span>].features[<span class="hljs-string">&quot;ner_tags&quot;</span>]
ner_feature`}}),ns=new T({props:{code:"Sequence(feature=ClassLabel(num_classes=9, names=['O', 'B-PER', 'I-PER', 'B-ORG', 'I-ORG', 'B-LOC', 'I-LOC', 'B-MISC', 'I-MISC'], names_file=None, id=None), length=-1, id=None)",highlighted:'<span class="hljs-type">Sequence</span>(feature=ClassLabel(num_classes=<span class="hljs-number">9</span>, names=[<span class="hljs-string">&#x27;O&#x27;</span>, <span class="hljs-string">&#x27;B-PER&#x27;</span>, <span class="hljs-string">&#x27;I-PER&#x27;</span>, <span class="hljs-string">&#x27;B-ORG&#x27;</span>, <span class="hljs-string">&#x27;I-ORG&#x27;</span>, <span class="hljs-string">&#x27;B-LOC&#x27;</span>, <span class="hljs-string">&#x27;I-LOC&#x27;</span>, <span class="hljs-string">&#x27;B-MISC&#x27;</span>, <span class="hljs-string">&#x27;I-MISC&#x27;</span>], names_file=<span class="hljs-literal">None</span>, <span class="hljs-built_in">id</span>=<span class="hljs-literal">None</span>), length=-<span class="hljs-number">1</span>, <span class="hljs-built_in">id</span>=<span class="hljs-literal">None</span>)'}}),os=new T({props:{code:`label_names = ner_feature.feature.names
label_names`,highlighted:`label_names = ner_feature.feature.names
label_names`}}),sl=new T({props:{code:"['O', 'B-PER', 'I-PER', 'B-ORG', 'I-ORG', 'B-LOC', 'I-LOC', 'B-MISC', 'I-MISC']",highlighted:'[<span class="hljs-string">&#x27;O&#x27;</span>, <span class="hljs-string">&#x27;B-PER&#x27;</span>, <span class="hljs-string">&#x27;I-PER&#x27;</span>, <span class="hljs-string">&#x27;B-ORG&#x27;</span>, <span class="hljs-string">&#x27;I-ORG&#x27;</span>, <span class="hljs-string">&#x27;B-LOC&#x27;</span>, <span class="hljs-string">&#x27;I-LOC&#x27;</span>, <span class="hljs-string">&#x27;B-MISC&#x27;</span>, <span class="hljs-string">&#x27;I-MISC&#x27;</span>]'}}),Dl=new T({props:{code:`words = raw_datasets["train"][0]["tokens"]
labels = raw_datasets["train"][0]["ner_tags"]
line1 = ""
line2 = ""
for word, label in zip(words, labels):
    full_label = label_names[label]
    max_length = max(len(word), len(full_label))
    line1 += word + " " * (max_length - len(word) + 1)
    line2 += full_label + " " * (max_length - len(full_label) + 1)

print(line1)
print(line2)`,highlighted:`words = raw_datasets[<span class="hljs-string">&quot;train&quot;</span>][<span class="hljs-number">0</span>][<span class="hljs-string">&quot;tokens&quot;</span>]
labels = raw_datasets[<span class="hljs-string">&quot;train&quot;</span>][<span class="hljs-number">0</span>][<span class="hljs-string">&quot;ner_tags&quot;</span>]
line1 = <span class="hljs-string">&quot;&quot;</span>
line2 = <span class="hljs-string">&quot;&quot;</span>
<span class="hljs-keyword">for</span> word, label <span class="hljs-keyword">in</span> <span class="hljs-built_in">zip</span>(words, labels):
    full_label = label_names[label]
    max_length = <span class="hljs-built_in">max</span>(<span class="hljs-built_in">len</span>(word), <span class="hljs-built_in">len</span>(full_label))
    line1 += word + <span class="hljs-string">&quot; &quot;</span> * (max_length - <span class="hljs-built_in">len</span>(word) + <span class="hljs-number">1</span>)
    line2 += full_label + <span class="hljs-string">&quot; &quot;</span> * (max_length - <span class="hljs-built_in">len</span>(full_label) + <span class="hljs-number">1</span>)

<span class="hljs-built_in">print</span>(line1)
<span class="hljs-built_in">print</span>(line2)`}}),gt=new T({props:{code:`'EU    rejects German call to boycott British lamb .'
'B-ORG O       B-MISC O    O  O       B-MISC  O    O'`,highlighted:`<span class="hljs-string">&#x27;EU    rejects German call to boycott British lamb .&#x27;</span>
<span class="hljs-string">&#x27;B-ORG O       B-MISC O    O  O       B-MISC  O    O&#x27;</span>`}}),Qe=new T({props:{code:`'Germany \\'s representative to the European Union \\'s veterinary committee Werner Zwingmann said on Wednesday consumers should buy sheepmeat from countries other than Britain until the scientific advice was clearer .'
'B-LOC   O  O              O  O   B-ORG    I-ORG O  O          O         B-PER  I-PER     O    O  O         O         O      O   O         O    O         O     O    B-LOC   O     O   O          O      O   O       O'`,highlighted:`<span class="hljs-string">&#x27;Germany \\&#x27;s representative to the European Union \\&#x27;s veterinary committee Werner Zwingmann said on Wednesday consumers should buy sheepmeat from countries other than Britain until the scientific advice was clearer .&#x27;</span>
<span class="hljs-string">&#x27;B-LOC   O  O              O  O   B-ORG    I-ORG O  O          O         B-PER  I-PER     O    O  O         O         O      O   O         O    O         O     O    B-LOC   O     O   O          O      O   O       O&#x27;</span>`}}),Pl=new Jr({props:{$$slots:{default:[Pf]},$$scope:{ctx:H}}}),Ct=new $f({props:{id:"iY2AZYdZAr0"}}),zt=new T({props:{code:`from transformers import AutoTokenizer

model_checkpoint = "bert-base-cased"
tokenizer = AutoTokenizer.from_pretrained(model_checkpoint)`,highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoTokenizer

model_checkpoint = <span class="hljs-string">&quot;bert-base-cased&quot;</span>
tokenizer = AutoTokenizer.from_pretrained(model_checkpoint)`}}),Al=new T({props:{code:"tokenizer.is_fast",highlighted:"tokenizer.is_fast"}}),It=new T({props:{code:"True",highlighted:'<span class="hljs-literal">True</span>'}}),tt=new T({props:{code:`inputs = tokenizer(raw_datasets["train"][0]["tokens"], is_split_into_words=True)
inputs.tokens()`,highlighted:`inputs = tokenizer(raw_datasets[<span class="hljs-string">&quot;train&quot;</span>][<span class="hljs-number">0</span>][<span class="hljs-string">&quot;tokens&quot;</span>], is_split_into_words=<span class="hljs-literal">True</span>)
inputs.tokens()`}}),Rt=new T({props:{code:"['[CLS]', 'EU', 'rejects', 'German', 'call', 'to', 'boycott', 'British', 'la', '##mb', '.', '[SEP]']",highlighted:'[<span class="hljs-string">&#x27;[CLS]&#x27;</span>, <span class="hljs-string">&#x27;EU&#x27;</span>, <span class="hljs-string">&#x27;rejects&#x27;</span>, <span class="hljs-string">&#x27;German&#x27;</span>, <span class="hljs-string">&#x27;call&#x27;</span>, <span class="hljs-string">&#x27;to&#x27;</span>, <span class="hljs-string">&#x27;boycott&#x27;</span>, <span class="hljs-string">&#x27;British&#x27;</span>, <span class="hljs-string">&#x27;la&#x27;</span>, <span class="hljs-string">&#x27;##mb&#x27;</span>, <span class="hljs-string">&#x27;.&#x27;</span>, <span class="hljs-string">&#x27;[SEP]&#x27;</span>]'}}),Ht=new T({props:{code:"inputs.word_ids()",highlighted:"inputs.word_ids()"}}),Ns=new T({props:{code:"[None, 0, 1, 2, 3, 4, 5, 6, 7, 7, 8, None]",highlighted:'[<span class="hljs-literal">None</span>, <span class="hljs-number">0</span>, <span class="hljs-number">1</span>, <span class="hljs-number">2</span>, <span class="hljs-number">3</span>, <span class="hljs-number">4</span>, <span class="hljs-number">5</span>, <span class="hljs-number">6</span>, <span class="hljs-number">7</span>, <span class="hljs-number">7</span>, <span class="hljs-number">8</span>, <span class="hljs-literal">None</span>]'}}),Ea=new T({props:{code:`def align_labels_with_tokens(labels, word_ids):
    new_labels = []
    current_word = None
    for word_id in word_ids:
        if word_id != current_word:
            # Start of a new word!
            current_word = word_id
            label = -100 if word_id is None else labels[word_id]
            new_labels.append(label)
        elif word_id is None:
            # Special token
            new_labels.append(-100)
        else:
            # Same word as previous token
            label = labels[word_id]
            # If the label is B-XXX we change it to I-XXX
            if label % 2 == 1:
                label += 1
            new_labels.append(label)

    return new_labels`,highlighted:`<span class="hljs-keyword">def</span> <span class="hljs-title function_">align_labels_with_tokens</span>(<span class="hljs-params">labels, word_ids</span>):
    new_labels = []
    current_word = <span class="hljs-literal">None</span>
    <span class="hljs-keyword">for</span> word_id <span class="hljs-keyword">in</span> word_ids:
        <span class="hljs-keyword">if</span> word_id != current_word:
            <span class="hljs-comment"># Start of a new word!</span>
            current_word = word_id
            label = -<span class="hljs-number">100</span> <span class="hljs-keyword">if</span> word_id <span class="hljs-keyword">is</span> <span class="hljs-literal">None</span> <span class="hljs-keyword">else</span> labels[word_id]
            new_labels.append(label)
        <span class="hljs-keyword">elif</span> word_id <span class="hljs-keyword">is</span> <span class="hljs-literal">None</span>:
            <span class="hljs-comment"># Special token</span>
            new_labels.append(-<span class="hljs-number">100</span>)
        <span class="hljs-keyword">else</span>:
            <span class="hljs-comment"># Same word as previous token</span>
            label = labels[word_id]
            <span class="hljs-comment"># If the label is B-XXX we change it to I-XXX</span>
            <span class="hljs-keyword">if</span> label % <span class="hljs-number">2</span> == <span class="hljs-number">1</span>:
                label += <span class="hljs-number">1</span>
            new_labels.append(label)

    <span class="hljs-keyword">return</span> new_labels`}}),va=new T({props:{code:`labels = raw_datasets["train"][0]["ner_tags"]
word_ids = inputs.word_ids()
print(labels)
print(align_labels_with_tokens(labels, word_ids))`,highlighted:`labels = raw_datasets[<span class="hljs-string">&quot;train&quot;</span>][<span class="hljs-number">0</span>][<span class="hljs-string">&quot;ner_tags&quot;</span>]
word_ids = inputs.word_ids()
<span class="hljs-built_in">print</span>(labels)
<span class="hljs-built_in">print</span>(align_labels_with_tokens(labels, word_ids))`}}),wa=new T({props:{code:`[3, 0, 7, 0, 0, 0, 7, 0, 0]
[-100, 3, 0, 7, 0, 0, 0, 7, 0, 0, 0, -100]`,highlighted:`[<span class="hljs-number">3</span>, <span class="hljs-number">0</span>, <span class="hljs-number">7</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">7</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>]
[-<span class="hljs-number">100</span>, <span class="hljs-number">3</span>, <span class="hljs-number">0</span>, <span class="hljs-number">7</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">7</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, -<span class="hljs-number">100</span>]`}}),Gt=new Jr({props:{$$slots:{default:[zf]},$$scope:{ctx:H}}}),Oa=new T({props:{code:`def tokenize_and_align_labels(examples):
    tokenized_inputs = tokenizer(
        examples["tokens"], truncation=True, is_split_into_words=True
    )
    all_labels = examples["ner_tags"]
    new_labels = []
    for i, labels in enumerate(all_labels):
        word_ids = tokenized_inputs.word_ids(i)
        new_labels.append(align_labels_with_tokens(labels, word_ids))

    tokenized_inputs["labels"] = new_labels
    return tokenized_inputs`,highlighted:`<span class="hljs-keyword">def</span> <span class="hljs-title function_">tokenize_and_align_labels</span>(<span class="hljs-params">examples</span>):
    tokenized_inputs = tokenizer(
        examples[<span class="hljs-string">&quot;tokens&quot;</span>], truncation=<span class="hljs-literal">True</span>, is_split_into_words=<span class="hljs-literal">True</span>
    )
    all_labels = examples[<span class="hljs-string">&quot;ner_tags&quot;</span>]
    new_labels = []
    <span class="hljs-keyword">for</span> i, labels <span class="hljs-keyword">in</span> <span class="hljs-built_in">enumerate</span>(all_labels):
        word_ids = tokenized_inputs.word_ids(i)
        new_labels.append(align_labels_with_tokens(labels, word_ids))

    tokenized_inputs[<span class="hljs-string">&quot;labels&quot;</span>] = new_labels
    <span class="hljs-keyword">return</span> tokenized_inputs`}}),ya=new T({props:{code:`tokenized_datasets = raw_datasets.map(
    tokenize_and_align_labels,
    batched=True,
    remove_columns=raw_datasets["train"].column_names,
)`,highlighted:`tokenized_datasets = raw_datasets.<span class="hljs-built_in">map</span>(
    tokenize_and_align_labels,
    batched=<span class="hljs-literal">True</span>,
    remove_columns=raw_datasets[<span class="hljs-string">&quot;train&quot;</span>].column_names,
)`}});const Oi=[If,Tf],jo=[];function yi(e,c){return e[0]==="pt"?0:1}fl=yi(H),_l=jo[fl]=Oi[fl](H);const Ci=[Rf,Af],ko=[];function Di(e,c){return e[0]==="pt"?0:1}hl=Di(H),bl=ko[hl]=Ci[hl](H),Pa=new T({props:{code:`batch = data_collator([tokenized_datasets["train"][i] for i in range(2)])
batch["labels"]`,highlighted:`batch = data_collator([tokenized_datasets[<span class="hljs-string">&quot;train&quot;</span>][i] <span class="hljs-keyword">for</span> i <span class="hljs-keyword">in</span> <span class="hljs-built_in">range</span>(<span class="hljs-number">2</span>)])
batch[<span class="hljs-string">&quot;labels&quot;</span>]`}}),ro=new T({props:{code:`tensor([[-100,    3,    0,    7,    0,    0,    0,    7,    0,    0,    0, -100],
        [-100,    1,    2, -100, -100, -100, -100, -100, -100, -100, -100, -100]])`,highlighted:`tensor([[-<span class="hljs-number">100</span>,    <span class="hljs-number">3</span>,    <span class="hljs-number">0</span>,    <span class="hljs-number">7</span>,    <span class="hljs-number">0</span>,    <span class="hljs-number">0</span>,    <span class="hljs-number">0</span>,    <span class="hljs-number">7</span>,    <span class="hljs-number">0</span>,    <span class="hljs-number">0</span>,    <span class="hljs-number">0</span>, -<span class="hljs-number">100</span>],
        [-<span class="hljs-number">100</span>,    <span class="hljs-number">1</span>,    <span class="hljs-number">2</span>, -<span class="hljs-number">100</span>, -<span class="hljs-number">100</span>, -<span class="hljs-number">100</span>, -<span class="hljs-number">100</span>, -<span class="hljs-number">100</span>, -<span class="hljs-number">100</span>, -<span class="hljs-number">100</span>, -<span class="hljs-number">100</span>, -<span class="hljs-number">100</span>]])`}}),no=new T({props:{code:`for i in range(2):
    print(tokenized_datasets["train"][i]["labels"])`,highlighted:`<span class="hljs-keyword">for</span> i <span class="hljs-keyword">in</span> <span class="hljs-built_in">range</span>(<span class="hljs-number">2</span>):
    <span class="hljs-built_in">print</span>(tokenized_datasets[<span class="hljs-string">&quot;train&quot;</span>][i][<span class="hljs-string">&quot;labels&quot;</span>])`}}),oo=new T({props:{code:`[-100, 3, 0, 7, 0, 0, 0, 7, 0, 0, 0, -100]
[-100, 1, 2, -100]`,highlighted:`[-<span class="hljs-number">100</span>, <span class="hljs-number">3</span>, <span class="hljs-number">0</span>, <span class="hljs-number">7</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">7</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, -<span class="hljs-number">100</span>]
[-<span class="hljs-number">100</span>, <span class="hljs-number">1</span>, <span class="hljs-number">2</span>, -<span class="hljs-number">100</span>]`}});const qi=[Bf,Sf],Eo=[];function Pi(e,c){return e[0]==="pt"?0:1}Zt=Pi(H),Kt=Eo[Zt]=qi[Zt](H);let Fe=H[0]==="tf"&&jf(H);const zi=[Nf,Lf],vo=[];function Ti(e,c){return e[0]==="pt"?0:1}Yt=Ti(H),Vt=vo[Yt]=zi[Yt](H),io=new T({props:{code:`import evaluate

metric = evaluate.load("seqeval")`,highlighted:`<span class="hljs-keyword">import</span> evaluate

metric = evaluate.load(<span class="hljs-string">&quot;seqeval&quot;</span>)`}}),co=new T({props:{code:`labels = raw_datasets["train"][0]["ner_tags"]
labels = [label_names[i] for i in labels]
labels`,highlighted:`labels = raw_datasets[<span class="hljs-string">&quot;train&quot;</span>][<span class="hljs-number">0</span>][<span class="hljs-string">&quot;ner_tags&quot;</span>]
labels = [label_names[i] <span class="hljs-keyword">for</span> i <span class="hljs-keyword">in</span> labels]
labels`}}),fo=new T({props:{code:"['B-ORG', 'O', 'B-MISC', 'O', 'O', 'O', 'B-MISC', 'O', 'O']",highlighted:'[<span class="hljs-string">&#x27;B-ORG&#x27;</span>, <span class="hljs-string">&#x27;O&#x27;</span>, <span class="hljs-string">&#x27;B-MISC&#x27;</span>, <span class="hljs-string">&#x27;O&#x27;</span>, <span class="hljs-string">&#x27;O&#x27;</span>, <span class="hljs-string">&#x27;O&#x27;</span>, <span class="hljs-string">&#x27;B-MISC&#x27;</span>, <span class="hljs-string">&#x27;O&#x27;</span>, <span class="hljs-string">&#x27;O&#x27;</span>]'}}),_o=new T({props:{code:`predictions = labels.copy()
predictions[2] = "O"
metric.compute(predictions=[predictions], references=[labels])`,highlighted:`predictions = labels.copy()
predictions[<span class="hljs-number">2</span>] = <span class="hljs-string">&quot;O&quot;</span>
metric.compute(predictions=[predictions], references=[labels])`}}),uo=new T({props:{code:`{'MISC': {'precision': 1.0, 'recall': 0.5, 'f1': 0.67, 'number': 2},
 'ORG': {'precision': 1.0, 'recall': 1.0, 'f1': 1.0, 'number': 1},
 'overall_precision': 1.0,
 'overall_recall': 0.67,
 'overall_f1': 0.8,
 'overall_accuracy': 0.89}`,highlighted:`{<span class="hljs-string">&#x27;MISC&#x27;</span>: {<span class="hljs-string">&#x27;precision&#x27;</span>: <span class="hljs-number">1.0</span>, <span class="hljs-string">&#x27;recall&#x27;</span>: <span class="hljs-number">0.5</span>, <span class="hljs-string">&#x27;f1&#x27;</span>: <span class="hljs-number">0.67</span>, <span class="hljs-string">&#x27;number&#x27;</span>: <span class="hljs-number">2</span>},
 <span class="hljs-string">&#x27;ORG&#x27;</span>: {<span class="hljs-string">&#x27;precision&#x27;</span>: <span class="hljs-number">1.0</span>, <span class="hljs-string">&#x27;recall&#x27;</span>: <span class="hljs-number">1.0</span>, <span class="hljs-string">&#x27;f1&#x27;</span>: <span class="hljs-number">1.0</span>, <span class="hljs-string">&#x27;number&#x27;</span>: <span class="hljs-number">1</span>},
 <span class="hljs-string">&#x27;overall_precision&#x27;</span>: <span class="hljs-number">1.0</span>,
 <span class="hljs-string">&#x27;overall_recall&#x27;</span>: <span class="hljs-number">0.67</span>,
 <span class="hljs-string">&#x27;overall_f1&#x27;</span>: <span class="hljs-number">0.8</span>,
 <span class="hljs-string">&#x27;overall_accuracy&#x27;</span>: <span class="hljs-number">0.89</span>}`}});const Ii=[Gf,Hf],wo=[];function Ai(e,c){return e[0]==="pt"?0:1}Jt=Ai(H),Qt=wo[Jt]=Ii[Jt](H);let Me=H[0]==="pt"&&kf(H);return ho=new T({props:{code:`from transformers import pipeline

# Replace this with your own checkpoint
model_checkpoint = "huggingface-course/bert-finetuned-ner"
token_classifier = pipeline(
    "token-classification", model=model_checkpoint, aggregation_strategy="simple"
)
token_classifier("My name is Sylvain and I work at Hugging Face in Brooklyn.")`,highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> pipeline

<span class="hljs-comment"># Replace this with your own checkpoint</span>
model_checkpoint = <span class="hljs-string">&quot;huggingface-course/bert-finetuned-ner&quot;</span>
token_classifier = pipeline(
    <span class="hljs-string">&quot;token-classification&quot;</span>, model=model_checkpoint, aggregation_strategy=<span class="hljs-string">&quot;simple&quot;</span>
)
token_classifier(<span class="hljs-string">&quot;My name is Sylvain and I work at Hugging Face in Brooklyn.&quot;</span>)`}}),bo=new T({props:{code:`[{'entity_group': 'PER', 'score': 0.9988506, 'word': 'Sylvain', 'start': 11, 'end': 18},
 {'entity_group': 'ORG', 'score': 0.9647625, 'word': 'Hugging Face', 'start': 33, 'end': 45},
 {'entity_group': 'LOC', 'score': 0.9986118, 'word': 'Brooklyn', 'start': 49, 'end': 57}]`,highlighted:`[{<span class="hljs-string">&#x27;entity_group&#x27;</span>: <span class="hljs-string">&#x27;PER&#x27;</span>, <span class="hljs-string">&#x27;score&#x27;</span>: <span class="hljs-number">0.9988506</span>, <span class="hljs-string">&#x27;word&#x27;</span>: <span class="hljs-string">&#x27;Sylvain&#x27;</span>, <span class="hljs-string">&#x27;start&#x27;</span>: <span class="hljs-number">11</span>, <span class="hljs-string">&#x27;end&#x27;</span>: <span class="hljs-number">18</span>},
 {<span class="hljs-string">&#x27;entity_group&#x27;</span>: <span class="hljs-string">&#x27;ORG&#x27;</span>, <span class="hljs-string">&#x27;score&#x27;</span>: <span class="hljs-number">0.9647625</span>, <span class="hljs-string">&#x27;word&#x27;</span>: <span class="hljs-string">&#x27;Hugging Face&#x27;</span>, <span class="hljs-string">&#x27;start&#x27;</span>: <span class="hljs-number">33</span>, <span class="hljs-string">&#x27;end&#x27;</span>: <span class="hljs-number">45</span>},
 {<span class="hljs-string">&#x27;entity_group&#x27;</span>: <span class="hljs-string">&#x27;LOC&#x27;</span>, <span class="hljs-string">&#x27;score&#x27;</span>: <span class="hljs-number">0.9986118</span>, <span class="hljs-string">&#x27;word&#x27;</span>: <span class="hljs-string">&#x27;Brooklyn&#x27;</span>, <span class="hljs-string">&#x27;start&#x27;</span>: <span class="hljs-number">49</span>, <span class="hljs-string">&#x27;end&#x27;</span>: <span class="hljs-number">57</span>}]`}}),{c(){u=o("meta"),$=f(),w(d.$$.fragment),b=f(),D=o("h1"),E=t("\u30C8\u30FC\u30AF\u30F3\u5206\u985E"),v=f(),C.c(),z=f(),B=o("p"),A=t("\u6700\u521D\u306B\u7D39\u4ECB\u3059\u308B\u30A2\u30D7\u30EA\u30B1\u30FC\u30B7\u30E7\u30F3\u306F\u3001\u30C8\u30FC\u30AF\u30F3\u5206\u985E\u3067\u3059\u3002\u3053\u306E\u6C4E\u7528\u7684\u306A\u30BF\u30B9\u30AF\u306F\u300C\u6587\u4E2D\u306E\u5404\u30C8\u30FC\u30AF\u30F3\u306B\u30E9\u30D9\u30EB\u3092\u4ED8\u3051\u308B\u300D\u3068\u5B9A\u7FA9\u53EF\u80FD\u306A\u3001\u4EE5\u4E0B\u306E\u3088\u3046\u306A\u554F\u984C\u3092\u542B\u307F\u307E\u3059\u3002"),R=f(),U=o("ul"),y=o("li"),M=o("strong"),K=t("\u56FA\u6709\u8868\u73FE\u8A8D\u8B58(NER)"),Z=t(": \u6587\u4E2D\u306B\u542B\u307E\u308C\u308B\u4EBA\u540D\u3001\u5730\u540D\u3001\u7D44\u7E54\u540D\u306A\u3069\u306E\u56FA\u6709\u306E\u30A8\u30F3\u30C6\u30A3\u30C6\u30A3\u3092\u691C\u51FA\u3057\u307E\u3059\u3002\u3053\u308C\u306F\u3001\u56FA\u6709\u30A8\u30F3\u30C6\u30A3\u30C6\u30A3\u30921\u30AF\u30E9\u30B9\u3001\u56FA\u6709\u30A8\u30F3\u30C6\u30A3\u30C6\u30A3\u306A\u3057\u30921\u30AF\u30E9\u30B9\u3068\u3057\u3066\u3001\u5404\u30C8\u30FC\u30AF\u30F3\u306B\u30E9\u30D9\u30EB\u3092\u4ED8\u4E0E\u3059\u308B\u30BF\u30B9\u30AF\u3068\u5B9A\u7FA9\u3067\u304D\u307E\u3059\u3002"),W=f(),I=o("li"),F=o("strong"),L=t("\u54C1\u8A5E\u30BF\u30B0\u4ED8\u3051(POS)"),X=t(": \u6587\u4E2D\u306E\u5404\u5358\u8A9E\u3092\u7279\u5B9A\u306E\u54C1\u8A5E\uFF08\u540D\u8A5E\u3001\u52D5\u8A5E\u3001\u5F62\u5BB9\u8A5E\u306A\u3069\uFF09\u306B\u5BFE\u5FDC\u3059\u308B\u3082\u306E\u3068\u3057\u3066\u30DE\u30FC\u30AF\u3057\u307E\u3059\u3002"),Q=f(),V=o("li"),le=o("strong"),J=t("\u30C1\u30E3\u30F3\u30AD\u30F3\u30B0(chunking)"),ce=t(": \u540C\u3058\u30A8\u30F3\u30C6\u30A3\u30C6\u30A3\u306B\u5C5E\u3059\u308B\u30C8\u30FC\u30AF\u30F3\u3092\u898B\u3064\u3051\u307E\u3059\u3002\u3053\u306E\u30BF\u30B9\u30AF\uFF08POS\u3084NER\u3068\u7D44\u307F\u5408\u308F\u305B\u308B\u3053\u3068\u304C\u3067\u304D\u307E\u3059\uFF09\u306F\u3001\u30C1\u30E3\u30F3\u30AF\u306E\u5148\u982D\u306B\u3042\u308B\u30C8\u30FC\u30AF\u30F3\u306B\u306F\u4E00\u3064\u306E\u30E9\u30D9\u30EB\uFF08\u901A\u5E38 "),Y=o("code"),G=t("B-"),te=t("\uFF09\u3001\u30C1\u30E3\u30F3\u30AF\u306E\u4E2D\u306B\u3042\u308B\u30C8\u30FC\u30AF\u30F3\u306B\u306F\u5225\u306E\u30E9\u30D9\u30EB\uFF08\u901A\u5E38 "),$e=o("code"),N=t("I-"),ee=t("\uFF09\u3001\u3069\u306E\u30C1\u30E3\u30F3\u30AF\u306B\u3082\u5C5E\u3055\u306A\u3044\u30C8\u30FC\u30AF\u30F3\u306B\u306F\u4E09\u3064\u76EE\u306E\u30E9\u30D9\u30EB\uFF08\u901A\u5E38 "),je=o("code"),Te=t("O"),se=t("\uFF09\u3092\u4ED8\u3051\u308B\u3053\u3068\u3068\u5B9A\u7FA9\u3067\u304D\u307E\u3059\u3002\u3002"),fe=f(),w(ae.$$.fragment),ge=f(),re=o("p"),fs=t("\u3082\u3061\u308D\u3093\u3001\u30C8\u30FC\u30AF\u30F3\u5206\u985E\u554F\u984C\u306B\u306F\u4ED6\u306B\u3082\u591A\u304F\u306E\u554F\u984C\u304C\u3042\u308A\u3001\u3053\u308C\u3089\u306F\u4EE3\u8868\u7684\u306A\u4F8B\u306B\u904E\u304E\u307E\u305B\u3093\u3002\u3053\u306E\u30BB\u30AF\u30B7\u30E7\u30F3\u3067\u306F\u3001NER\u30BF\u30B9\u30AF\u3067\u30E2\u30C7\u30EB\uFF08BERT\uFF09\u3092\u5FAE\u8ABF\u6574\u3057\u3001\u4EE5\u4E0B\u306E\u3088\u3046\u306A\u4E88\u6E2C\u8A08\u7B97\u304C\u3067\u304D\u308B\u3088\u3046\u306B\u3057\u307E\u3059\u3002"),ue=f(),ke=o("iframe"),xe=f(),Le=o("a"),Ne=o("img"),Ll=f(),Ie=o("img"),jl=f(),Oe=o("p"),Cs=t("\u3042\u306A\u305F\u306F\u5B66\u7FD2\u6E08\u307F\u30E2\u30C7\u30EB\u3092Hub\u3067\u63A2\u3057\u305F\u308A\u3001Hub\u306B\u30A2\u30C3\u30D7\u30ED\u30FC\u30C9\u3057\u3001\u305D\u306E\u4E88\u6E2C\u5024\u3092"),ss=o("a"),He=t("\u3053\u3053\u3067"),Nl=t("\u518D\u78BA\u8A8D\u3059\u308B\u3053\u3068\u304C\u3067\u304D\u307E\u3059 \u3002"),_s=f(),Ge=o("h2"),Hl=t("\u30C7\u30FC\u30BF\u306E\u6E96\u5099"),de=f(),ye=o("p"),Us=t("\u307E\u305A\u6700\u521D\u306B\u3001\u30C8\u30FC\u30AF\u30F3\u5206\u985E\u306B\u9069\u3057\u305F\u30C7\u30FC\u30BF\u30BB\u30C3\u30C8\u304C\u5FC5\u8981\u3067\u3059\u3002\u3053\u306E\u30BB\u30AF\u30B7\u30E7\u30F3\u3067\u306F\u3001"),Ce=o("a"),it=t("CoNLL-2003 dataset"),Ds=t("\u3092\u4F7F\u3044\u307E\u3059\u3002\u3053\u306E\u30C7\u30FC\u30BF\u30BB\u30C3\u30C8\u306F\u30ED\u30A4\u30BF\u30FC\u304C\u914D\u4FE1\u3059\u308B\u30CB\u30E5\u30FC\u30B9\u8A18\u4E8B\u3092\u542B\u307F\u307E\u3059\u3002"),Gl=f(),w(us.$$.fragment),qs=f(),Ee=o("h3"),ls=o("a"),ts=o("span"),w(Ae.$$.fragment),kl=f(),ne=o("span"),ct=t("The CoNLL-2003 dataset"),ms=f(),ds=o("p"),ft=t("CoNLL-2003\u306E\u30C7\u30FC\u30BF\u30BB\u30C3\u30C8\u3092\u30ED\u30FC\u30C9\u3059\u308B\u305F\u3081\u306B\u3001\u{1F917} Datasets\u30E9\u30A4\u30D6\u30E9\u30EA\u306E "),hs=o("code"),he=t("load_dataset()"),_t=t(" \u30E1\u30BD\u30C3\u30C9\u3092\u4F7F\u7528\u3057\u307E\u3059\u3002"),Ps=f(),w(De.$$.fragment),Ul=f(),me=o("p"),El=t("\u3053\u308C\u306F\u30C7\u30FC\u30BF\u30BB\u30C3\u30C8\u3092\u30C0\u30A6\u30F3\u30ED\u30FC\u30C9\u3057\u30AD\u30E3\u30C3\u30B7\u30E5\u3057\u307E\u3059\u3002"),Ws=o("a"),Wl=t("\u7B2C3\u7AE0"),zs=t("\u3067GLUE MRPC dataset\u3092\u6271\u3063\u305F\u3068\u304D\u3068\u540C\u3058\u3067\u3059\u3002\u3053\u306E\u30AA\u30D6\u30B8\u30A7\u30AF\u30C8\u3092\u8ABF\u3079\u308B\u3068\u3001\u5B9A\u7FA9\u3055\u308C\u305F\u5217\u3068\u3001\u30C8\u30EC\u30FC\u30CB\u30F3\u30B0\u30BB\u30C3\u30C8\u3001\u691C\u8A3C\u30BB\u30C3\u30C8\u3001\u30C6\u30B9\u30C8\u30BB\u30C3\u30C8\u306E3\u3064\u306B\u5206\u5272\u3055\u308C\u3066\u3044\u308B\u4E8B\u304C\u308F\u304B\u308A\u307E\u3059\u3002"),Ts=f(),w(oe.$$.fragment),Xs=f(),w(as.$$.fragment),Xl=f(),ve=o("p"),Zs=t("\u7279\u306B\u3001\u3053\u306E\u30C7\u30FC\u30BF\u30BB\u30C3\u30C8\u306B\u306F\u3001\u5148\u306B\u8FF0\u3079\u305F3\u3064\u306E\u30BF\u30B9\u30AF\u7528\u306E\u30E9\u30D9\u30EB\u3001NER\u3001POS\u3001\u30C1\u30E3\u30F3\u30AD\u30F3\u30B0\u304C\u542B\u307E\u308C\u3066\u3044\u308B\u3053\u3068\u304C\u308F\u304B\u308A\u307E\u3059\u3002\u4ED6\u306E\u30C7\u30FC\u30BF\u30BB\u30C3\u30C8\u3068\u306E\u5927\u304D\u306A\u9055\u3044\u306F\u3001\u5165\u529B\u30C6\u30AD\u30B9\u30C8\u304C\u6587\u3084\u6587\u66F8\u3068\u3057\u3066\u3067\u306F\u306A\u304F\u3001\u5358\u8A9E\u306E\u30EA\u30B9\u30C8\u3068\u3057\u3066\u8868\u793A\u3055\u308C\u3066\u3044\u308B\u3053\u3068\u3067\u3059\uFF08\u6700\u5F8C\u306E\u5217\u306F"),Ue=o("code"),ut=t("tokens"),vl=t("\u547C\u3070\u308C\u3066\u3044\u307E\u3059\u304C\u3001\u3053\u308C\u306F\u30C8\u30FC\u30AF\u30F3\u5316\u524D\u306E\u5165\u529B\u3067\u3001\u307E\u3060\u30B5\u30D6\u30EF\u30FC\u30C9 \u30C8\u30FC\u30AF\u30F3\u5316\u306E\u305F\u3081\u306Btokenizer\u51E6\u7406\u3059\u308B\u5FC5\u8981\u304C\u3042\u308B\u3068\u3044\u3046\u610F\u5473\u3067\u5358\u8A9E\u3092\u542B\u3093\u3067\u3044\u307E\u3059\uFF09\u3002"),Ks=f(),Ys=o("p"),Zl=t("\u305D\u308C\u3067\u306F\u3001\u5B66\u7FD2\u30BB\u30C3\u30C8\u306E\u6700\u521D\u306E\u8981\u7D20\u3092\u898B\u3066\u307F\u307E\u3057\u3087\u3046\u3002"),We=f(),w(bs.$$.fragment),rs=f(),w($s.$$.fragment),Xe=f(),Is=o("p"),Vs=t("\u4ECA\u56DE\u306F\u56FA\u6709\u8868\u73FE\u8A8D\u8B58\u3092\u884C\u3044\u305F\u3044\u306E\u3067\u3001NER\u30BF\u30B0\u3092\u898B\u308B\u3053\u3068\u306B\u3057\u307E\u3059\u3002"),wl=f(),w(qe.$$.fragment),js=f(),w(Ze.$$.fragment),gl=f(),Pe=o("p"),ks=t("\u3053\u308C\u306F\u5B66\u7FD2\u6642\u306B\u4F7F\u308F\u308C\u308B\u30E9\u30D9\u30EB\u306E\u305F\u3081\u6574\u6570\u5024\u3067\u683C\u7D0D\u3055\u308C\u3066\u3044\u307E\u3059\u3002\u30C7\u30FC\u30BF\u3092\u8ABF\u3079\u308B\u3068\u304D\u306B\u306F\u5FC5\u305A\u3057\u3082\u4FBF\u5229\u3067\u306F\u3042\u308A\u307E\u305B\u3093\u3002\u30C6\u30AD\u30B9\u30C8\u5206\u985E\u306E\u3088\u3046\u306B\u3001\u30C7\u30FC\u30BF\u30BB\u30C3\u30C8\u306E "),Js=o("code"),m=t("features"),S=t(" \u5C5E\u6027\u3092\u898B\u308C\u3070\u3001\u3053\u308C\u3089\u306E\u6574\u6570\u5024\u304C\u4F55\u306E\u30E9\u30D9\u30EB\u3067\u3042\u308B\u304B\u8ABF\u3079\u308B\u4E8B\u304C\u3067\u304D\u307E\u3059\u3002"),As=f(),w(Rs.$$.fragment),Ss=f(),w(ns.$$.fragment),Qs=f(),pe=o("p"),mt=t("\u3064\u307E\u308A\u3001\u3053\u306E\u30AB\u30E9\u30E0\u306F "),Re=o("code"),dt=t("ClassLabel"),Bs=t(" \u306E\u8981\u7D20\u3092\u542B\u3093\u3067\u3044\u307E\u3059\u3002\u5404\u8981\u7D20\u306E\u578B\u306F\u3001\u3053\u306E "),Kl=o("code"),Yl=t("ner_feature"),ea=t(" \u306E "),Vl=o("code"),ht=t("feature"),Se=t(" \u5C5E\u6027\u306B\u3042\u308A\u3001\u305D\u306E "),Jl=o("code"),bt=t("feature"),el=t(" \u306E "),Es=o("code"),$t=t("names"),Fs=t(" \u5C5E\u6027\u3092\u898B\u308B\u3053\u3068\u3067\u540D\u524D\u306E\u30EA\u30B9\u30C8\u3092\u78BA\u8A8D\u3059\u308B\u4E8B\u304C\u3067\u304D\u307E\u3059\u3002"),xl=f(),w(os.$$.fragment),Ke=f(),w(sl.$$.fragment),Ol=f(),we=o("p"),sa=t("\u3053\u308C\u3089\u306E\u30E9\u30D9\u30EB\u306F"),ll=o("a"),Be=t("\u7B2C6\u7AE0"),la=t("\u3067\u3001"),tl=o("code"),Qr=t("token-classification"),en=t("\u30D1\u30A4\u30D7\u30E9\u30A4\u30F3\u3092\u5B66\u3093\u3060\u3068\u304D\u306B\u6398\u308A\u4E0B\u3052\u307E\u3057\u305F\u304C\u3001\u7C21\u5358\u306B\u5FA9\u7FD2\u3057\u3066\u304A\u304D\u307E\u3057\u3087\u3046\u3002"),ta=f(),Ye=o("ul"),aa=o("li"),jt=o("code"),sn=t("O"),ln=t(" \u306F\u305D\u306E\u5358\u8A9E\u304C\u3069\u306E\u30A8\u30F3\u30C6\u30A3\u30C6\u30A3\u306B\u3082\u5BFE\u5FDC\u3057\u306A\u3044\u3053\u3068\u3092\u610F\u5473\u3057\u307E\u3059\u3002"),ur=f(),Ve=o("li"),ra=o("code"),na=t("B-PER"),tn=t("/"),oa=o("code"),kt=t("I-PER"),mr=t(" \u306F\u3001\u305D\u306E\u5358\u8A9E\u304C "),vs=o("em"),an=t("\u4EBA(person)"),Aa=t(" \u30A8\u30F3\u30C6\u30A3\u30C6\u30A3\u306E\u5148\u982D\u3001\u307E\u305F\u306F\u5185\u90E8\u3067\u3042\u308B\u3053\u3068\u3092\u610F\u5473\u3057\u307E\u3059\u3002"),rn=f(),al=o("li"),Et=o("code"),nn=t("B-ORG"),on=t("/"),pa=o("code"),vt=t("I-ORG"),dr=t(" \u306F\u3001\u305D\u306E\u5358\u8A9E\u304C "),rl=o("em"),pn=t("\u7D44\u7E54(organization)"),Ra=t(" \u30A8\u30F3\u30C6\u30A3\u30C6\u30A3\u306E\u5148\u982D\u3001\u307E\u305F\u306F\u5185\u90E8\u3067\u3042\u308B\u3053\u3068\u3092\u610F\u5473\u3057\u307E\u3059\u3002"),cn=f(),nl=o("li"),ia=o("code"),wt=t("B-LOC"),hr=t("/"),yl=o("code"),br=t("I-LOC"),ol=t(" \u306F\u3001\u305D\u306E\u5358\u8A9E\u304C "),Sa=o("em"),Ba=t("\u5834\u6240(location)"),fn=t(" \u30A8\u30F3\u30C6\u30A3\u30C6\u30A3\u306E\u5148\u982D\u3001\u307E\u305F\u306F\u5185\u90E8\u3067\u3042\u308B\u3053\u3068\u3092\u610F\u5473\u3057\u307E\u3059\u3002"),_n=f(),ws=o("li"),Fa=o("code"),un=t("B-MISC"),$r=t("/"),Cl=o("code"),jr=t("I-MISC"),gs=t(" \u306F\u3001\u305D\u306E\u5358\u8A9E\u304C "),Ma=o("em"),La=t("\u305D\u306E\u4ED6(miscellaneous)"),mn=t(" \u30A8\u30F3\u30C6\u30A3\u30C6\u30A3\u306E\u5148\u982D\u3001\u307E\u305F\u306F\u5185\u90E8\u3067\u3042\u308B\u3053\u3068\u3092\u610F\u5473\u3057\u307E\u3059\u3002"),kr=f(),Ql=o("p"),dn=t("\u3055\u3066\u3001\u5148\u307B\u3069\u306E\u30E9\u30D9\u30EB\u3092\u30C7\u30B3\u30FC\u30C9\u3059\u308B\u3068\u3001\u4EE5\u4E0B\u306E\u3088\u3046\u306B\u306A\u308A\u307E\u3059\u3002"),Er=f(),w(Dl.$$.fragment),vr=f(),w(gt.$$.fragment),Na=f(),Je=o("p"),wr=t("\u307E\u305F\u3001"),ql=o("code"),gr=t("B-"),ca=t(" \u3068 "),Ha=o("code"),xr=t("I-"),xt=t("\u306E\u30E9\u30D9\u30EB\u3092\u6DF7\u5728\u3055\u305B\u305F\u4F8B\u3068\u3057\u3066\u3001\u5B66\u7FD2\u30BB\u30C3\u30C8\u306E4\u756A\u76EE\u306E\u8981\u7D20\u306B\u3064\u3044\u3066\u540C\u3058\u30B3\u30FC\u30C9\u3092\u5B9F\u884C\u3059\u308B\u3068\u3001\u4EE5\u4E0B\u306E\u3088\u3046\u306B\u306A\u308A\u307E\u3059\u3002"),Ga=f(),w(Qe.$$.fragment),Or=f(),xs=o("p"),hn=t("\u3053\u306E\u3088\u3046\u306B\u3001\u201CEuropean Union\u201D \u3068 \u201CWerner Zwingmann\u201D \u306E\u3088\u3046\u306B2\u3064\u306E\u5358\u8A9E\u306B\u307E\u305F\u304C\u308B\u30A8\u30F3\u30C6\u30A3\u30C6\u30A3\u306F\u3001\u6700\u521D\u306E\u5358\u8A9E\u306B\u306F "),Ua=o("code"),Wa=t("B-"),bn=t(" \u30E9\u30D9\u30EB\u304C\u30012\u756A\u76EE\u306E\u5358\u8A9E\u306B\u306F "),Xa=o("code"),yr=t("I-"),Ot=t(" \u30E9\u30D9\u30EB\u304C\u4ED8\u4E0E\u3055\u308C\u307E\u3059\u3002"),Cr=f(),w(Pl.$$.fragment),Ms=f(),yt=o("h3"),Za=t("\u30C7\u30FC\u30BF\u306E\u51E6\u7406"),Dr=f(),w(Ct.$$.fragment),fa=f(),pl=o("p"),$n=t("\u3044\u3064\u3082\u306E\u3088\u3046\u306B\u3001\u30E2\u30C7\u30EB\u304C\u610F\u5473\u3092\u7406\u89E3\u3067\u304D\u308B\u3088\u3046\u306B\u3059\u308B\u305F\u3081\u306B\u3001\u30C6\u30AD\u30B9\u30C8\u306F\u30C8\u30FC\u30AF\u30F3ID\u306B\u5909\u63DB\u3055\u308C\u308B\u5FC5\u8981\u304C\u3042\u308A\u307E\u3059\u3002"),Dt=o("a"),qt=t("\u7B2C6\u7AE0"),qr=t("\u3067\u898B\u305F\u3088\u3046\u306B\u3001\u30C8\u30FC\u30AF\u30F3\u5206\u985E\u30BF\u30B9\u30AF\u306E\u5834\u5408\u306E\u5927\u304D\u306A\u9055\u3044\u306F\u3001\u5165\u529B\u304C\u3042\u3089\u304B\u3058\u3081\u30C8\u30FC\u30AF\u30F3\u5316\u3055\u308C\u3066\u3044\u308B\u3068\u8A00\u3046\u4E8B\u3067\u3059\u3002\u5E78\u3044\u306A\u3053\u3068\u306B\u3001tokenizer API \u306F\u3053\u306E\u70B9\u3092\u304B\u306A\u308A\u7C21\u5358\u306B\u51E6\u7406\u3067\u304D\u307E\u3059\u3002\u7279\u5225\u306A\u30D5\u30E9\u30B0\u3092\u6307\u5B9A\u3057\u3066 "),et=o("code"),jn=t("tokenizer"),Pr=t(" \u306B\u8B66\u544A\u3059\u308B\u3060\u3051\u3067\u3059\u3002"),Ls=f(),il=o("p"),Pt=t("\u307E\u305A\u6700\u521D\u306B\u3001"),Ka=o("code"),Ya=t("tokenizer"),kn=t(" \u30AA\u30D6\u30B8\u30A7\u30AF\u30C8\u3092\u4F5C\u6210\u3057\u307E\u3057\u3087\u3046\u3002\u524D\u306B\u8FF0\u3079\u305F\u3088\u3046\u306B\u3001\u4E8B\u524D\u5B66\u7FD2\u6E08\u307FBERT\u30E2\u30C7\u30EB\u3092\u4F7F\u7528\u3059\u308B\u4E88\u5B9A\u306A\u306E\u3067\u3001\u95A2\u9023\u3059\u308Btokenizer\u3092\u30C0\u30A6\u30F3\u30ED\u30FC\u30C9\u3057\u3066\u30AD\u30E3\u30C3\u30B7\u30E5\u3059\u308B\u3053\u3068\u304B\u3089\u59CB\u3081\u307E\u3059\u3002"),zr=f(),w(zt.$$.fragment),_a=f(),_e=o("p"),En=t("\u3042\u306A\u305F\u306F"),Tt=o("code"),vn=t("model_checkpoint"),wn=t(" \u3092\u81EA\u7531\u306B\u7F6E\u304D\u63DB\u3048\u308B\u4E8B\u304C\u3067\u304D\u307E\u3059\u3002"),zl=o("a"),gn=t("Hub"),xn=t(" \u306B\u3042\u308B\u597D\u304D\u306A\u30E2\u30C7\u30EB\u3084\u3001\u81EA\u5206\u306E\u7AEF\u672B\u306B\u4FDD\u5B58\u3057\u305F\u4E8B\u524D\u5B66\u7FD2\u6E08\u307F\u30E2\u30C7\u30EB\u3084tokenizer\u3092\u3067\u7F6E\u304D\u63DB\u3048\u308B\u3053\u3068\u304C\u3067\u304D\u307E\u3059\u3002"),ua=f(),Os=o("p"),On=t("\u552F\u4E00\u306E\u5236\u7D04\u306F\u3001tokenizer\u304C \u{1F917} Tokenizers \u30E9\u30A4\u30D6\u30E9\u30EA\u306B\u3088\u3063\u3066\u30D0\u30C3\u30AF\u30A2\u30C3\u30D7\u3055\u308C\u308B\u5FC5\u8981\u304C\u3042\u308B\u3053\u3068\u3067\u3059\u3002\u3053\u308C\u306B\u3088\u308A\u300C\u9AD8\u901F\u300D\u30D0\u30FC\u30B8\u30E7\u30F3\u304C\u7528\u610F\u3055\u308C\u307E\u3059\u3002"),Tl=o("a"),yn=t("\u3053\u306E\u5927\u304D\u306A\u30C6\u30FC\u30D6\u30EB"),Cn=t(" \u3067\u9AD8\u901F\u30D0\u30FC\u30B8\u30E7\u30F3\u3092\u6301\u3064\u5168\u3066\u306E\u30A2\u30FC\u30AD\u30C6\u30AF\u30C1\u30E3\u3092\u898B\u308B\u3053\u3068\u304C\u3067\u304D\u307E\u3059\u3002\u4F7F\u7528\u3057\u3066\u3044\u308B "),Va=o("code"),Ja=t("tokenizer"),Il=t(" \u30AA\u30D6\u30B8\u30A7\u30AF\u30C8\u304C\u672C\u5F53\u306B \u{1F917} Tokenizers \u3067\u30D0\u30C3\u30AF\u30A2\u30C3\u30D7\u3055\u308C\u3066\u3044\u308B\u304B\u3069\u3046\u304B\u3092\u78BA\u8A8D\u3059\u308B\u306B\u306F\u3001"),Qa=o("code"),er=t("is_fast"),Dn=t(" \u5C5E\u6027\u3092\u898B\u308B\u4E8B\u304C\u78BA\u8A8D\u3067\u304D\u307E\u3059\u3002"),Tr=f(),w(Al.$$.fragment),Ir=f(),w(It.$$.fragment),sr=f(),ps=o("p"),qn=t("\u30C8\u30FC\u30AF\u30F3\u5316\u524D\u306E\u5165\u529B\u3092\u30C8\u30FC\u30AF\u30F3\u5316\u3059\u308B\u306B\u306F\u3001\u666E\u6BB5\u901A\u308A "),ma=o("code"),At=t("tokenizer"),Ar=t(" \u3092\u4F7F\u7528\u3057\u3066\u3001 "),st=o("code"),Pn=t("is_split_into_words=True"),Rr=t(" \u3092\u8FFD\u52A0\u3059\u308B\u3060\u3051\u3067\u3059\u3002"),lt=f(),w(tt.$$.fragment),ie=f(),w(Rt.$$.fragment),da=f(),ze=o("p"),zn=t("\u898B\u3066\u306E\u901A\u308A\u3001\u30C8\u30FC\u30AF\u30F3\u5316\u306F\u30E2\u30C7\u30EB\u304C\u4F7F\u7528\u3059\u308B\u7279\u6B8A\u306A\u30C8\u30FC\u30AF\u30F3\uFF08\u5148\u982D\u306E "),St=o("code"),Tn=t("[CLS]"),In=t(" \u3068\u672B\u5C3E\u306E "),Bt=o("code"),An=t("[SEP]"),Rn=t(" \uFF09\u3092\u8FFD\u52A0\u3057\u3001\u307B\u3068\u3093\u3069\u306E\u5358\u8A9E\u306F\u305D\u306E\u307E\u307E\u306B\u3057\u307E\u3057\u305F\u3002\u3057\u304B\u3057\u3001"),Ft=o("code"),Sn=t("lamb"),Bn=t("\u3068\u3044\u3046\u5358\u8A9E\u306F"),Mt=o("code"),Fn=t("la"),Mn=t("\u3068"),Lt=o("code"),Ln=t("##mb"),Nn=t("\u3068\u3044\u30462\u3064\u306E\u30B5\u30D6\u30EF\u30FC\u30C9\u306B\u30C8\u30FC\u30AF\u30F3\u5316\u3055\u308C\u307E\u3057\u305F\u3002\u3053\u306E\u305F\u3081\u3001\u5165\u529B\u3068\u30E9\u30D9\u30EB\u306E\u9593\u306B\u30DF\u30B9\u30DE\u30C3\u30C1\u304C\u751F\u3058\u307E\u3059\u3002\u30E9\u30D9\u30EB\u306E\u30EA\u30B9\u30C8\u306B\u306F9\u3064\u306E\u8981\u7D20\u3057\u304B\u3042\u308A\u307E\u305B\u3093\u304C\u3001\u5165\u529B\u306E\u30EA\u30B9\u30C8\u306B\u306F12\u306E\u30C8\u30FC\u30AF\u30F3\u304C\u3042\u308A\u307E\u3059\u3002\u7279\u6B8A\u306A\u30C8\u30FC\u30AF\u30F3\u3092\u8003\u616E\u3059\u308B\u306E\u306F\u7C21\u5358\u3067\u3059\u304C\uFF08\u6700\u521D\u3068\u6700\u5F8C\u306B\u3042\u308B\u3053\u3068\u304C\u5206\u304B\u3063\u3066\u3044\u307E\u3059\uFF09\u3001\u3059\u3079\u3066\u306E\u30E9\u30D9\u30EB\u3092\u9069\u5207\u306A\u5358\u8A9E\u306B\u63C3\u3048\u308B\u3053\u3068\u3082\u5FC5\u8981\u3067\u3059\u3002"),ha=f(),at=o("p"),Hn=t("\u5E78\u3044\u3001\u9AD8\u901F\u306Atokenizer\u3092\u4F7F\u3063\u3066\u3044\u308B\u306E\u3067\u3001\u{1F917} Tokenizers \u306E\u30B9\u30FC\u30D1\u30FC\u30D1\u30EF\u30FC\u306B\u30A2\u30AF\u30BB\u30B9\u3059\u308B\u3053\u3068\u304C\u3067\u304D\u3001\u305D\u308C\u305E\u308C\u306E\u30C8\u30FC\u30AF\u30F3\u3092\u5BFE\u5FDC\u3059\u308B\u5358\u8A9E\u306B\u7C21\u5358\u306B\u30DE\u30C3\u30D4\u30F3\u30B0\u3059\u308B\u3053\u3068\u304C\u3067\u304D\u307E\u3059 (\u3053\u308C\u306F "),Nt=o("a"),cl=t("\u7B2C\uFF16\u7AE0"),Gn=t(" \u3067\u898B\u305F\u3068\u304A\u308A\u3067\u3059)\u3002"),ba=f(),w(Ht.$$.fragment),Sr=f(),w(Ns.$$.fragment),$a=f(),es=o("p"),Un=t("\u307B\u3093\u306E\u5C11\u3057\u306E\u4F5C\u696D\u3067\u3001\u30C8\u30FC\u30AF\u30F3\u306B\u30DE\u30C3\u30C1\u3059\u308B\u3088\u3046\u306B\u30E9\u30D9\u30EB\u30EA\u30B9\u30C8\u3092\u62E1\u5F35\u3059\u308B\u3053\u3068\u304C\u3067\u304D\u307E\u3059\u3002\u6700\u521D\u306B\u9069\u7528\u3059\u308B\u30EB\u30FC\u30EB\u306F\u3001\u7279\u6B8A\u306A\u30C8\u30FC\u30AF\u30F3\u306B\u306F "),ja=o("code"),r=t("-100"),h=t(" \u3068\u3044\u3046\u30E9\u30D9\u30EB\u3092\u4ED8\u3051\u308B\u3068\u3044\u3046\u3082\u306E\u3067\u3059\u3002\u3053\u308C\u306F\u30C7\u30D5\u30A9\u30EB\u30C8\u3067 "),rt=o("code"),qo=t("-100"),Po=t(" \u304C\u3053\u308C\u304B\u3089\u4F7F\u3046\u640D\u5931\u95A2\u6570\uFF08\u30AF\u30ED\u30B9\u30A8\u30F3\u30C8\u30ED\u30D4\u30FC\uFF09\u3067\u7121\u8996\u3055\u308C\u308B\u6570\u3060\u304B\u3089\u3067\u3059\u3002\u6B21\u306B\u3001\u5358\u8A9E\u5185\u306E\u5404\u30C8\u30FC\u30AF\u30F3\u306F\u5358\u8A9E\u306E\u5148\u982D\u306E\u30C8\u30FC\u30AF\u30F3\u3068\u540C\u3058\u30E9\u30D9\u30EB\u304C\u4ED8\u4E0E\u3055\u308C\u307E\u3059\u3002\u3053\u308C\u306F\u540C\u3058\u30A8\u30F3\u30C6\u30A3\u30C6\u30A3\u306E\u4E00\u90E8\u3067\u3042\u308B\u305F\u3081\u3067\u3059\u3002\u5358\u8A9E\u306E\u5185\u90E8\u306B\u3042\u308A\u3001\u304B\u3064\u5148\u982D\u306B\u306A\u3044\u30C8\u30FC\u30AF\u30F3\u306B\u3064\u3044\u3066\u306F\u3001"),ka=o("code"),zo=t("B-"),Br=t(" \u3092 "),Fr=o("code"),lr=t("I-"),To=t(" \u306B\u7F6E\u304D\u63DB\u3048\u307E\u3059\uFF08\u305D\u306E\u30C8\u30FC\u30AF\u30F3\u306F\u30A8\u30F3\u30C6\u30A3\u30C6\u30A3\u3092\u958B\u59CB\u3057\u306A\u3044\u305F\u3081\u3067\u3059\uFF09\u3002"),Wn=f(),w(Ea.$$.fragment),tr=f(),ar=o("p"),Io=t("\u305D\u308C\u3067\u306F\u3001\u6700\u521D\u306E\u6587\u7AE0\u3067\u8A66\u3057\u3066\u307F\u307E\u3057\u3087\u3046\u3002"),Xn=f(),w(va.$$.fragment),rr=f(),w(wa.$$.fragment),Zn=f(),Hs=o("p"),Ao=t("\u898B\u3066\u308F\u304B\u308B\u3088\u3046\u306B\u3001\u3053\u306E\u95A2\u6570\u306F\u6700\u521D\u3068\u6700\u5F8C\u306E2\u3064\u306E\u7279\u5225\u306A\u30C8\u30FC\u30AF\u30F3\u306B\u5BFE\u3057\u3066 "),ys=o("code"),Ro=t("-100"),Kn=t(" \u3092\u8FFD\u52A0\u3057\u30012\u3064\u306E\u30C8\u30FC\u30AF\u30F3\u306B\u5206\u5272\u3055\u308C\u305F\u5358\u8A9E\u306B\u5BFE\u3057\u3066\u65B0\u305F\u306B "),Mr=o("code"),So=t("0"),Bo=t(" \u3092\u8FFD\u52A0\u3057\u3066\u3044\u307E\u3059\u3002"),nr=f(),w(Gt.$$.fragment),or=f(),is=o("p"),Fo=t("\u30C7\u30FC\u30BF\u30BB\u30C3\u30C8\u5168\u4F53\u306E\u524D\u51E6\u7406\u3068\u3057\u3066\u3001\u3059\u3079\u3066\u306E\u5165\u529B\u3092\u30C8\u30FC\u30AF\u30F3\u5316\u3057\u3001\u3059\u3079\u3066\u306E\u30E9\u30D9\u30EB\u306B\u5BFE\u3057\u3066 "),ga=o("code"),Mo=t("align_labels_with_tokens()"),Lo=t(" \u3092\u9069\u7528\u3059\u308B\u5FC5\u8981\u304C\u3042\u308A\u307E\u3059\u3002\u9AD8\u901F\u306Atokenizer\u306E\u901F\u5EA6\u3092\u6D3B\u304B\u3059\u306B\u306F\u3001\u305F\u304F\u3055\u3093\u306E\u30C6\u30AD\u30B9\u30C8\u3092\u540C\u6642\u306B\u30C8\u30FC\u30AF\u30F3\u5316\u3059\u308B\u306E\u304C\u3088\u3044\u3067\u3057\u3087\u3046\u3002\u305D\u3053\u3067\u3001\u30B5\u30F3\u30D7\u30EB\u306E\u30EA\u30B9\u30C8\u3092\u51E6\u7406\u3059\u308B\u95A2\u6570\u3092\u66F8\u3044\u3066\u3001 "),xa=o("code"),No=t("Dataset.map()"),Ho=t(" \u30E1\u30BD\u30C3\u30C9\u306B "),Lr=o("code"),Nr=t("batched=True"),Go=t(" \u30AA\u30D7\u30B7\u30E7\u30F3\u3092\u4ED8\u3051\u3066\u4F7F\u7528\u3059\u308B\u3053\u3068\u306B\u3057\u307E\u3057\u3087\u3046\u3002\u4EE5\u524D\u306E\u4F8B\u3068\u552F\u4E00\u9055\u3046\u306E\u306F\u3001tokenizer\u3078\u306E\u5165\u529B\u304C\u30C6\u30AD\u30B9\u30C8\u306E\u30EA\u30B9\u30C8\uFF08\u3053\u306E\u5834\u5408\u306F\u5358\u8A9E\u306E\u30EA\u30B9\u30C8\u306E\u30EA\u30B9\u30C8\uFF09\u3067\u3042\u308B\u5834\u5408\u3001 "),Hr=o("code"),nt=t("word_ids()"),Uo=t(" \u95A2\u6570\u306F\u5358\u8A9EID\u304C\u6B32\u3057\u3044\u30EA\u30B9\u30C8\u306E\u30A4\u30F3\u30C7\u30C3\u30AF\u30B9\u3092\u5FC5\u8981\u3068\u3059\u308B\u306E\u3067\u3001\u3053\u308C\u3082\u8FFD\u52A0\u3057\u307E\u3059\u3002"),Yn=f(),w(Oa.$$.fragment),Vn=f(),Rl=o("p"),Wo=t(`\u307E\u3060\u3001\u5165\u529B\u3092\u30D1\u30C7\u30A3\u30F3\u30B0\u3057\u3066\u3044\u306A\u3044\u3053\u3068\u306B\u6CE8\u610F\u3057\u3066\u304F\u3060\u3055\u3044\u3002
\u3053\u308C\u306F\u5F8C\u3067\u30C7\u30FC\u30BF\u7167\u5408\u30C4\u30FC\u30EB\u3067\u30D0\u30C3\u30C1\u3092\u4F5C\u6210\u3059\u308B\u3068\u304D\u306B\u3084\u308B\u3053\u3068\u306B\u3057\u307E\u3059\u3002`),Jn=f(),Ut=o("p"),Xo=t("\u3053\u308C\u3067\u3001\u30C7\u30FC\u30BF\u30BB\u30C3\u30C8\u306E\u5206\u5272\u306B\u5BFE\u3057\u3066\u3001\u3059\u3079\u3066\u306E\u524D\u51E6\u7406\u3092\u4E00\u5EA6\u306B\u9069\u7528\u3059\u308B\u3053\u3068\u304C\u3067\u304D\u308B\u3088\u3046\u306B\u306A\u308A\u307E\u3057\u305F\u3002"),Wt=f(),w(ya.$$.fragment),Qn=f(),cs=o("p"),Zo=t(`\u4E00\u756A\u5927\u5909\u306A\u3068\u3053\u308D\u3092\u3084\u308A\u307E\u3057\u305F\u306D\uFF01
\u30C7\u30FC\u30BF\u306E\u524D\u51E6\u7406\u304C\u7D42\u308F\u3063\u305F\u306E\u3067\u3001\u5B9F\u969B\u306E\u5B66\u7FD2\u306F`),pr=o("a"),Ko=t("\u7B2C3\u7AE0"),Yo=t("\u3067\u3084\u3063\u305F\u3088\u3046\u306A\u611F\u3058\u306B\u306A\u308A\u307E\u3059\u306D\u3002"),Xt=f(),_l.c(),ir=f(),ul=o("h3"),Vo=t("\u30C7\u30FC\u30BF\u7167\u5408"),eo=f(),ml=o("p"),Sl=o("a"),Jo=t("\u7B2C3\u7AE0"),Gr=t(" \u306B\u3042\u308B\u3088\u3046\u306A "),Ur=o("code"),Qo=t("DataCollatorWithPadding"),dl=t(" \u306F\u5165\u529B (\u5165\u529B ID\u3001\u30A2\u30C6\u30F3\u30B7\u30E7\u30F3\u30DE\u30B9\u30AF\u3001\u30C8\u30FC\u30AF\u30F3\u30BF\u30A4\u30D7 ID) \u306E\u307F\u3092\u30D1\u30C7\u30A3\u30F3\u30B0\u3059\u308B\u306E\u3067\u4F7F\u3048\u307E\u305B\u3093\u3002\u3053\u3053\u3067\u306F\u3001\u30E9\u30D9\u30EB\u306E\u30B5\u30A4\u30BA\u304C\u5909\u308F\u3089\u306A\u3044\u3088\u3046\u306B\u3001\u5165\u529B\u3068\u5168\u304F\u540C\u3058\u65B9\u6CD5\u3067\u30D1\u30C7\u30A3\u30F3\u30B0\u3092\u884C\u3044\u307E\u3059\u3002\u5024\u3068\u3057\u3066 "),Wr=o("code"),ep=t("-100"),sp=t(" \u3092\u4F7F\u7528\u3057\u3001\u5BFE\u5FDC\u3059\u308B\u4E88\u6E2C\u5024\u304C\u640D\u5931\u8A08\u7B97\u3067\u7121\u8996\u3055\u308C\u308B\u3088\u3046\u306B\u3057\u307E\u3059\u3002"),so=f(),Bl=o("p"),Ca=t("\u3053\u308C\u306F\u5168\u3066 "),Da=o("a"),Xr=o("code"),lp=t("DataCollatorForTokenClassification"),tp=t(" \u306B\u3088\u3063\u3066\u884C\u308F\u308C\u307E\u3059\u3002DataCollatorWithPadding"),be=o("code"),ap=t("\u3068\u540C\u69D8\u306B\u3001\u5165\u529B\u306E\u524D\u51E6\u7406\u306B\u4F7F\u7528\u3055\u308C\u308B"),rp=t("tokenizer` \u3092\u53D7\u3051\u53D6\u308A\u307E\u3059\u3002"),lo=f(),bl.c(),cr=f(),fr=o("p"),qa=t("\u3053\u308C\u3092\u3044\u304F\u3064\u304B\u306E\u30B5\u30F3\u30D7\u30EB\u3067\u30C6\u30B9\u30C8\u3059\u308B\u306B\u306F\u3001\u30C8\u30FC\u30AF\u30F3\u5316\u3055\u308C\u305F\u30C8\u30EC\u30FC\u30CB\u30F3\u30B0\u30BB\u30C3\u30C8\u304B\u3089\u30B5\u30F3\u30D7\u30EB\u306E\u30EA\u30B9\u30C8\u306B\u5BFE\u3057\u3066\u547C\u3073\u51FA\u3059\u3060\u3051\u3067\u3088\u3044\u306E\u3067\u3059\u3002"),to=f(),w(Pa.$$.fragment),ao=f(),w(ro.$$.fragment),Tp=f(),np=o("p"),ci=t("\u3053\u308C\u3092\u30C7\u30FC\u30BF\u30BB\u30C3\u30C8\u306E1\u756A\u76EE\u30682\u756A\u76EE\u306E\u8981\u7D20\u306E\u30E9\u30D9\u30EB\u3068\u6BD4\u8F03\u3057\u3066\u307F\u307E\u3057\u3087\u3046\u3002"),Ip=f(),w(no.$$.fragment),Ap=f(),w(oo.$$.fragment),Rp=f(),Kt.c(),op=f(),Fe&&Fe.c(),pp=f(),po=o("h3"),fi=t("\u6307\u6A19"),Sp=f(),Vt.c(),ip=f(),w(io.$$.fragment),Bp=f(),cp=o("p"),_i=t("\u3053\u306E\u6307\u6A19\u306F\u6A19\u6E96\u7684\u306A\u7CBE\u5EA6\u6307\u6A19\u306E\u3088\u3046\u306B\u52D5\u4F5C\u3057\u307E\u305B\u3093\uFF1A\u5B9F\u969B\u306B\u306F\u30E9\u30D9\u30EB\u306E\u30EA\u30B9\u30C8\u3092\u6574\u6570\u3067\u306F\u306A\u304F\u6587\u5B57\u5217\u3068\u3057\u3066\u53D7\u3051\u53D6\u308B\u306E\u3067\u3001\u4E88\u6E2C\u5024\u3068\u30E9\u30D9\u30EB\u3092\u6307\u6A19\u306B\u6E21\u3059\u524D\u306B\u5B8C\u5168\u306B\u30C7\u30B3\u30FC\u30C9\u3059\u308B\u5FC5\u8981\u304C\u3042\u308A\u307E\u3059\u3002"),Fp=f(),fp=o("p"),ui=t("\u305D\u308C\u3067\u306F\u3001\u3069\u306E\u3088\u3046\u306B\u52D5\u4F5C\u3059\u308B\u304B\u898B\u3066\u307F\u307E\u3057\u3087\u3046\u3002\u307E\u305A\u3001\u6700\u521D\u306E\u5B66\u7FD2\u30B5\u30F3\u30D7\u30EB\u306B\u5BFE\u3059\u308B\u30E9\u30D9\u30EB\u3092\u53D6\u5F97\u3057\u307E\u3059\u3002"),Mp=f(),w(co.$$.fragment),Lp=f(),w(fo.$$.fragment),Np=f(),_p=o("p"),mi=t("\u305D\u3057\u3066\u3001\u30A4\u30F3\u30C7\u30C3\u30AF\u30B92\u306E\u5024\u3092\u5909\u66F4\u3059\u308B\u3060\u3051\u3067\u3001\u305D\u308C\u3089\u306E\u7591\u4F3C\u4E88\u6E2C\u3092\u4F5C\u6210\u3059\u308B\u3053\u3068\u304C\u3067\u304D\u307E\u3059\u3002"),Hp=f(),w(_o.$$.fragment),Gp=f(),up=o("p"),di=t("\u3053\u306E\u6307\u6A19\u306F\u4E88\u6E2C\u5024\u306E\u30EA\u30B9\u30C8\uFF081\u3064\u3060\u3051\u3067\u306F\u306A\u3044\uFF09\u3068\u30E9\u30D9\u30EB\u306E\u30EA\u30B9\u30C8\u3092\u53D7\u3051\u53D6\u308B\u3053\u3068\u306B\u6CE8\u610F\u3057\u3066\u304F\u3060\u3055\u3044\u3002\u4EE5\u4E0B\u306F\u305D\u306E\u51FA\u529B\u3067\u3059\u3002"),Up=f(),w(uo.$$.fragment),Wp=f(),Qt.c(),mp=f(),Me&&Me.c(),dp=f(),mo=o("h2"),hi=t("\u5FAE\u8ABF\u6574\u3057\u305F\u30E2\u30C7\u30EB\u3092\u4F7F\u3046"),Xp=f(),Zr=o("p"),bi=t("Model Hub\u3067\u5FAE\u8ABF\u6574\u3057\u305F\u30E2\u30C7\u30EB\u3092\u63A8\u8AD6\u30A6\u30A3\u30B8\u30A7\u30C3\u30C8\u3067\u4F7F\u7528\u3059\u308B\u65B9\u6CD5\u306F\u65E2\u306B\u7D39\u4ECB\u3057\u307E\u3057\u305F\u3002\u30ED\u30FC\u30AB\u30EB\u74B0\u5883\u306E"),qp=o("code"),$i=t("pipeline"),ji=t("\u3067\u4F7F\u7528\u3059\u308B\u5834\u5408\u306F\u3001\u30E2\u30C7\u30EB\u8B58\u5225\u5B50\u3092\u6307\u5B9A\u3057\u307E\u3059\u3002"),Zp=f(),w(ho.$$.fragment),Kp=f(),w(bo.$$.fragment),Yp=f(),hp=o("p"),ki=t(`\u7D20\u6674\u3089\u3057\u3044\uFF01
\u79C1\u305F\u3061\u306E\u30E2\u30C7\u30EB\u306F\u3001\u3053\u306E\u30D1\u30A4\u30D7\u30E9\u30A4\u30F3\u306E\u30C7\u30D5\u30A9\u30EB\u30C8\u306E\u3082\u306E\u3068\u540C\u3058\u3088\u3046\u306B\u52D5\u4F5C\u3057\u3066\u3044\u307E\u3059\uFF01`),this.h()},l(e){const c=xf('[data-svelte="svelte-1phssyn"]',document.head);u=p(c,"META",{name:!0,content:!0}),c.forEach(s),$=_(e),g(d.$$.fragment,e),b=_(e),D=p(e,"H1",{id:!0});var Pp=i(D);E=a(Pp,"\u30C8\u30FC\u30AF\u30F3\u5206\u985E"),Pp.forEach(s),v=_(e),C.l(e),z=_(e),B=p(e,"P",{});var bp=i(B);A=a(bp,"\u6700\u521D\u306B\u7D39\u4ECB\u3059\u308B\u30A2\u30D7\u30EA\u30B1\u30FC\u30B7\u30E7\u30F3\u306F\u3001\u30C8\u30FC\u30AF\u30F3\u5206\u985E\u3067\u3059\u3002\u3053\u306E\u6C4E\u7528\u7684\u306A\u30BF\u30B9\u30AF\u306F\u300C\u6587\u4E2D\u306E\u5404\u30C8\u30FC\u30AF\u30F3\u306B\u30E9\u30D9\u30EB\u3092\u4ED8\u3051\u308B\u300D\u3068\u5B9A\u7FA9\u53EF\u80FD\u306A\u3001\u4EE5\u4E0B\u306E\u3088\u3046\u306A\u554F\u984C\u3092\u542B\u307F\u307E\u3059\u3002"),bp.forEach(s),R=_(e),U=p(e,"UL",{});var _r=i(U);y=p(_r,"LI",{});var $p=i(y);M=p($p,"STRONG",{});var zp=i(M);K=a(zp,"\u56FA\u6709\u8868\u73FE\u8A8D\u8B58(NER)"),zp.forEach(s),Z=a($p,": \u6587\u4E2D\u306B\u542B\u307E\u308C\u308B\u4EBA\u540D\u3001\u5730\u540D\u3001\u7D44\u7E54\u540D\u306A\u3069\u306E\u56FA\u6709\u306E\u30A8\u30F3\u30C6\u30A3\u30C6\u30A3\u3092\u691C\u51FA\u3057\u307E\u3059\u3002\u3053\u308C\u306F\u3001\u56FA\u6709\u30A8\u30F3\u30C6\u30A3\u30C6\u30A3\u30921\u30AF\u30E9\u30B9\u3001\u56FA\u6709\u30A8\u30F3\u30C6\u30A3\u30C6\u30A3\u306A\u3057\u30921\u30AF\u30E9\u30B9\u3068\u3057\u3066\u3001\u5404\u30C8\u30FC\u30AF\u30F3\u306B\u30E9\u30D9\u30EB\u3092\u4ED8\u4E0E\u3059\u308B\u30BF\u30B9\u30AF\u3068\u5B9A\u7FA9\u3067\u304D\u307E\u3059\u3002"),$p.forEach(s),W=_(_r),I=p(_r,"LI",{});var go=i(I);F=p(go,"STRONG",{});var jp=i(F);L=a(jp,"\u54C1\u8A5E\u30BF\u30B0\u4ED8\u3051(POS)"),jp.forEach(s),X=a(go,": \u6587\u4E2D\u306E\u5404\u5358\u8A9E\u3092\u7279\u5B9A\u306E\u54C1\u8A5E\uFF08\u540D\u8A5E\u3001\u52D5\u8A5E\u3001\u5F62\u5BB9\u8A5E\u306A\u3069\uFF09\u306B\u5BFE\u5FDC\u3059\u308B\u3082\u306E\u3068\u3057\u3066\u30DE\u30FC\u30AF\u3057\u307E\u3059\u3002"),go.forEach(s),Q=_(_r),V=p(_r,"LI",{});var Fl=i(V);le=p(Fl,"STRONG",{});var kp=i(le);J=a(kp,"\u30C1\u30E3\u30F3\u30AD\u30F3\u30B0(chunking)"),kp.forEach(s),ce=a(Fl,": \u540C\u3058\u30A8\u30F3\u30C6\u30A3\u30C6\u30A3\u306B\u5C5E\u3059\u308B\u30C8\u30FC\u30AF\u30F3\u3092\u898B\u3064\u3051\u307E\u3059\u3002\u3053\u306E\u30BF\u30B9\u30AF\uFF08POS\u3084NER\u3068\u7D44\u307F\u5408\u308F\u305B\u308B\u3053\u3068\u304C\u3067\u304D\u307E\u3059\uFF09\u306F\u3001\u30C1\u30E3\u30F3\u30AF\u306E\u5148\u982D\u306B\u3042\u308B\u30C8\u30FC\u30AF\u30F3\u306B\u306F\u4E00\u3064\u306E\u30E9\u30D9\u30EB\uFF08\u901A\u5E38 "),Y=p(Fl,"CODE",{});var Ep=i(Y);G=a(Ep,"B-"),Ep.forEach(s),te=a(Fl,"\uFF09\u3001\u30C1\u30E3\u30F3\u30AF\u306E\u4E2D\u306B\u3042\u308B\u30C8\u30FC\u30AF\u30F3\u306B\u306F\u5225\u306E\u30E9\u30D9\u30EB\uFF08\u901A\u5E38 "),$e=p(Fl,"CODE",{});var Ri=i($e);N=a(Ri,"I-"),Ri.forEach(s),ee=a(Fl,"\uFF09\u3001\u3069\u306E\u30C1\u30E3\u30F3\u30AF\u306B\u3082\u5C5E\u3055\u306A\u3044\u30C8\u30FC\u30AF\u30F3\u306B\u306F\u4E09\u3064\u76EE\u306E\u30E9\u30D9\u30EB\uFF08\u901A\u5E38 "),je=p(Fl,"CODE",{});var Si=i(je);Te=a(Si,"O"),Si.forEach(s),se=a(Fl,"\uFF09\u3092\u4ED8\u3051\u308B\u3053\u3068\u3068\u5B9A\u7FA9\u3067\u304D\u307E\u3059\u3002\u3002"),Fl.forEach(s),_r.forEach(s),fe=_(e),g(ae.$$.fragment,e),ge=_(e),re=p(e,"P",{});var Bi=i(re);fs=a(Bi,"\u3082\u3061\u308D\u3093\u3001\u30C8\u30FC\u30AF\u30F3\u5206\u985E\u554F\u984C\u306B\u306F\u4ED6\u306B\u3082\u591A\u304F\u306E\u554F\u984C\u304C\u3042\u308A\u3001\u3053\u308C\u3089\u306F\u4EE3\u8868\u7684\u306A\u4F8B\u306B\u904E\u304E\u307E\u305B\u3093\u3002\u3053\u306E\u30BB\u30AF\u30B7\u30E7\u30F3\u3067\u306F\u3001NER\u30BF\u30B9\u30AF\u3067\u30E2\u30C7\u30EB\uFF08BERT\uFF09\u3092\u5FAE\u8ABF\u6574\u3057\u3001\u4EE5\u4E0B\u306E\u3088\u3046\u306A\u4E88\u6E2C\u8A08\u7B97\u304C\u3067\u304D\u308B\u3088\u3046\u306B\u3057\u307E\u3059\u3002"),Bi.forEach(s),ue=_(e),ke=p(e,"IFRAME",{src:!0,frameborder:!0,height:!0,title:!0,class:!0,allow:!0,sandbox:!0}),i(ke).forEach(s),xe=_(e),Le=p(e,"A",{class:!0,href:!0});var Jp=i(Le);Ne=p(Jp,"IMG",{class:!0,src:!0,alt:!0}),Ll=_(Jp),Ie=p(Jp,"IMG",{class:!0,src:!0,alt:!0}),Jp.forEach(s),jl=_(e),Oe=p(e,"P",{});var Qp=i(Oe);Cs=a(Qp,"\u3042\u306A\u305F\u306F\u5B66\u7FD2\u6E08\u307F\u30E2\u30C7\u30EB\u3092Hub\u3067\u63A2\u3057\u305F\u308A\u3001Hub\u306B\u30A2\u30C3\u30D7\u30ED\u30FC\u30C9\u3057\u3001\u305D\u306E\u4E88\u6E2C\u5024\u3092"),ss=p(Qp,"A",{href:!0,rel:!0});var Fi=i(ss);He=a(Fi,"\u3053\u3053\u3067"),Fi.forEach(s),Nl=a(Qp,"\u518D\u78BA\u8A8D\u3059\u308B\u3053\u3068\u304C\u3067\u304D\u307E\u3059 \u3002"),Qp.forEach(s),_s=_(e),Ge=p(e,"H2",{id:!0});var Mi=i(Ge);Hl=a(Mi,"\u30C7\u30FC\u30BF\u306E\u6E96\u5099"),Mi.forEach(s),de=_(e),ye=p(e,"P",{});var ei=i(ye);Us=a(ei,"\u307E\u305A\u6700\u521D\u306B\u3001\u30C8\u30FC\u30AF\u30F3\u5206\u985E\u306B\u9069\u3057\u305F\u30C7\u30FC\u30BF\u30BB\u30C3\u30C8\u304C\u5FC5\u8981\u3067\u3059\u3002\u3053\u306E\u30BB\u30AF\u30B7\u30E7\u30F3\u3067\u306F\u3001"),Ce=p(ei,"A",{href:!0,rel:!0});var Li=i(Ce);it=a(Li,"CoNLL-2003 dataset"),Li.forEach(s),Ds=a(ei,"\u3092\u4F7F\u3044\u307E\u3059\u3002\u3053\u306E\u30C7\u30FC\u30BF\u30BB\u30C3\u30C8\u306F\u30ED\u30A4\u30BF\u30FC\u304C\u914D\u4FE1\u3059\u308B\u30CB\u30E5\u30FC\u30B9\u8A18\u4E8B\u3092\u542B\u307F\u307E\u3059\u3002"),ei.forEach(s),Gl=_(e),g(us.$$.fragment,e),qs=_(e),Ee=p(e,"H3",{class:!0});var si=i(Ee);ls=p(si,"A",{id:!0,class:!0,href:!0});var Ni=i(ls);ts=p(Ni,"SPAN",{});var Hi=i(ts);g(Ae.$$.fragment,Hi),Hi.forEach(s),Ni.forEach(s),kl=_(si),ne=p(si,"SPAN",{});var Gi=i(ne);ct=a(Gi,"The CoNLL-2003 dataset"),Gi.forEach(s),si.forEach(s),ms=_(e),ds=p(e,"P",{});var li=i(ds);ft=a(li,"CoNLL-2003\u306E\u30C7\u30FC\u30BF\u30BB\u30C3\u30C8\u3092\u30ED\u30FC\u30C9\u3059\u308B\u305F\u3081\u306B\u3001\u{1F917} Datasets\u30E9\u30A4\u30D6\u30E9\u30EA\u306E "),hs=p(li,"CODE",{});var Ui=i(hs);he=a(Ui,"load_dataset()"),Ui.forEach(s),_t=a(li," \u30E1\u30BD\u30C3\u30C9\u3092\u4F7F\u7528\u3057\u307E\u3059\u3002"),li.forEach(s),Ps=_(e),g(De.$$.fragment,e),Ul=_(e),me=p(e,"P",{});var ti=i(me);El=a(ti,"\u3053\u308C\u306F\u30C7\u30FC\u30BF\u30BB\u30C3\u30C8\u3092\u30C0\u30A6\u30F3\u30ED\u30FC\u30C9\u3057\u30AD\u30E3\u30C3\u30B7\u30E5\u3057\u307E\u3059\u3002"),Ws=p(ti,"A",{href:!0});var Wi=i(Ws);Wl=a(Wi,"\u7B2C3\u7AE0"),Wi.forEach(s),zs=a(ti,"\u3067GLUE MRPC dataset\u3092\u6271\u3063\u305F\u3068\u304D\u3068\u540C\u3058\u3067\u3059\u3002\u3053\u306E\u30AA\u30D6\u30B8\u30A7\u30AF\u30C8\u3092\u8ABF\u3079\u308B\u3068\u3001\u5B9A\u7FA9\u3055\u308C\u305F\u5217\u3068\u3001\u30C8\u30EC\u30FC\u30CB\u30F3\u30B0\u30BB\u30C3\u30C8\u3001\u691C\u8A3C\u30BB\u30C3\u30C8\u3001\u30C6\u30B9\u30C8\u30BB\u30C3\u30C8\u306E3\u3064\u306B\u5206\u5272\u3055\u308C\u3066\u3044\u308B\u4E8B\u304C\u308F\u304B\u308A\u307E\u3059\u3002"),ti.forEach(s),Ts=_(e),g(oe.$$.fragment,e),Xs=_(e),g(as.$$.fragment,e),Xl=_(e),ve=p(e,"P",{});var ai=i(ve);Zs=a(ai,"\u7279\u306B\u3001\u3053\u306E\u30C7\u30FC\u30BF\u30BB\u30C3\u30C8\u306B\u306F\u3001\u5148\u306B\u8FF0\u3079\u305F3\u3064\u306E\u30BF\u30B9\u30AF\u7528\u306E\u30E9\u30D9\u30EB\u3001NER\u3001POS\u3001\u30C1\u30E3\u30F3\u30AD\u30F3\u30B0\u304C\u542B\u307E\u308C\u3066\u3044\u308B\u3053\u3068\u304C\u308F\u304B\u308A\u307E\u3059\u3002\u4ED6\u306E\u30C7\u30FC\u30BF\u30BB\u30C3\u30C8\u3068\u306E\u5927\u304D\u306A\u9055\u3044\u306F\u3001\u5165\u529B\u30C6\u30AD\u30B9\u30C8\u304C\u6587\u3084\u6587\u66F8\u3068\u3057\u3066\u3067\u306F\u306A\u304F\u3001\u5358\u8A9E\u306E\u30EA\u30B9\u30C8\u3068\u3057\u3066\u8868\u793A\u3055\u308C\u3066\u3044\u308B\u3053\u3068\u3067\u3059\uFF08\u6700\u5F8C\u306E\u5217\u306F"),Ue=p(ai,"CODE",{});var Xi=i(Ue);ut=a(Xi,"tokens"),Xi.forEach(s),vl=a(ai,"\u547C\u3070\u308C\u3066\u3044\u307E\u3059\u304C\u3001\u3053\u308C\u306F\u30C8\u30FC\u30AF\u30F3\u5316\u524D\u306E\u5165\u529B\u3067\u3001\u307E\u3060\u30B5\u30D6\u30EF\u30FC\u30C9 \u30C8\u30FC\u30AF\u30F3\u5316\u306E\u305F\u3081\u306Btokenizer\u51E6\u7406\u3059\u308B\u5FC5\u8981\u304C\u3042\u308B\u3068\u3044\u3046\u610F\u5473\u3067\u5358\u8A9E\u3092\u542B\u3093\u3067\u3044\u307E\u3059\uFF09\u3002"),ai.forEach(s),Ks=_(e),Ys=p(e,"P",{});var Zi=i(Ys);Zl=a(Zi,"\u305D\u308C\u3067\u306F\u3001\u5B66\u7FD2\u30BB\u30C3\u30C8\u306E\u6700\u521D\u306E\u8981\u7D20\u3092\u898B\u3066\u307F\u307E\u3057\u3087\u3046\u3002"),Zi.forEach(s),We=_(e),g(bs.$$.fragment,e),rs=_(e),g($s.$$.fragment,e),Xe=_(e),Is=p(e,"P",{});var Ki=i(Is);Vs=a(Ki,"\u4ECA\u56DE\u306F\u56FA\u6709\u8868\u73FE\u8A8D\u8B58\u3092\u884C\u3044\u305F\u3044\u306E\u3067\u3001NER\u30BF\u30B0\u3092\u898B\u308B\u3053\u3068\u306B\u3057\u307E\u3059\u3002"),Ki.forEach(s),wl=_(e),g(qe.$$.fragment,e),js=_(e),g(Ze.$$.fragment,e),gl=_(e),Pe=p(e,"P",{});var ri=i(Pe);ks=a(ri,"\u3053\u308C\u306F\u5B66\u7FD2\u6642\u306B\u4F7F\u308F\u308C\u308B\u30E9\u30D9\u30EB\u306E\u305F\u3081\u6574\u6570\u5024\u3067\u683C\u7D0D\u3055\u308C\u3066\u3044\u307E\u3059\u3002\u30C7\u30FC\u30BF\u3092\u8ABF\u3079\u308B\u3068\u304D\u306B\u306F\u5FC5\u305A\u3057\u3082\u4FBF\u5229\u3067\u306F\u3042\u308A\u307E\u305B\u3093\u3002\u30C6\u30AD\u30B9\u30C8\u5206\u985E\u306E\u3088\u3046\u306B\u3001\u30C7\u30FC\u30BF\u30BB\u30C3\u30C8\u306E "),Js=p(ri,"CODE",{});var Yi=i(Js);m=a(Yi,"features"),Yi.forEach(s),S=a(ri," \u5C5E\u6027\u3092\u898B\u308C\u3070\u3001\u3053\u308C\u3089\u306E\u6574\u6570\u5024\u304C\u4F55\u306E\u30E9\u30D9\u30EB\u3067\u3042\u308B\u304B\u8ABF\u3079\u308B\u4E8B\u304C\u3067\u304D\u307E\u3059\u3002"),ri.forEach(s),As=_(e),g(Rs.$$.fragment,e),Ss=_(e),g(ns.$$.fragment,e),Qs=_(e),pe=p(e,"P",{});var ot=i(pe);mt=a(ot,"\u3064\u307E\u308A\u3001\u3053\u306E\u30AB\u30E9\u30E0\u306F "),Re=p(ot,"CODE",{});var Vi=i(Re);dt=a(Vi,"ClassLabel"),Vi.forEach(s),Bs=a(ot," \u306E\u8981\u7D20\u3092\u542B\u3093\u3067\u3044\u307E\u3059\u3002\u5404\u8981\u7D20\u306E\u578B\u306F\u3001\u3053\u306E "),Kl=p(ot,"CODE",{});var Ji=i(Kl);Yl=a(Ji,"ner_feature"),Ji.forEach(s),ea=a(ot," \u306E "),Vl=p(ot,"CODE",{});var Qi=i(Vl);ht=a(Qi,"feature"),Qi.forEach(s),Se=a(ot," \u5C5E\u6027\u306B\u3042\u308A\u3001\u305D\u306E "),Jl=p(ot,"CODE",{});var ec=i(Jl);bt=a(ec,"feature"),ec.forEach(s),el=a(ot," \u306E "),Es=p(ot,"CODE",{});var sc=i(Es);$t=a(sc,"names"),sc.forEach(s),Fs=a(ot," \u5C5E\u6027\u3092\u898B\u308B\u3053\u3068\u3067\u540D\u524D\u306E\u30EA\u30B9\u30C8\u3092\u78BA\u8A8D\u3059\u308B\u4E8B\u304C\u3067\u304D\u307E\u3059\u3002"),ot.forEach(s),xl=_(e),g(os.$$.fragment,e),Ke=_(e),g(sl.$$.fragment,e),Ol=_(e),we=p(e,"P",{});var vp=i(we);sa=a(vp,"\u3053\u308C\u3089\u306E\u30E9\u30D9\u30EB\u306F"),ll=p(vp,"A",{href:!0});var lc=i(ll);Be=a(lc,"\u7B2C6\u7AE0"),lc.forEach(s),la=a(vp,"\u3067\u3001"),tl=p(vp,"CODE",{});var tc=i(tl);Qr=a(tc,"token-classification"),tc.forEach(s),en=a(vp,"\u30D1\u30A4\u30D7\u30E9\u30A4\u30F3\u3092\u5B66\u3093\u3060\u3068\u304D\u306B\u6398\u308A\u4E0B\u3052\u307E\u3057\u305F\u304C\u3001\u7C21\u5358\u306B\u5FA9\u7FD2\u3057\u3066\u304A\u304D\u307E\u3057\u3087\u3046\u3002"),vp.forEach(s),ta=_(e),Ye=p(e,"UL",{});var za=i(Ye);aa=p(za,"LI",{});var Ei=i(aa);jt=p(Ei,"CODE",{});var ac=i(jt);sn=a(ac,"O"),ac.forEach(s),ln=a(Ei," \u306F\u305D\u306E\u5358\u8A9E\u304C\u3069\u306E\u30A8\u30F3\u30C6\u30A3\u30C6\u30A3\u306B\u3082\u5BFE\u5FDC\u3057\u306A\u3044\u3053\u3068\u3092\u610F\u5473\u3057\u307E\u3059\u3002"),Ei.forEach(s),ur=_(za),Ve=p(za,"LI",{});var xo=i(Ve);ra=p(xo,"CODE",{});var rc=i(ra);na=a(rc,"B-PER"),rc.forEach(s),tn=a(xo,"/"),oa=p(xo,"CODE",{});var nc=i(oa);kt=a(nc,"I-PER"),nc.forEach(s),mr=a(xo," \u306F\u3001\u305D\u306E\u5358\u8A9E\u304C "),vs=p(xo,"EM",{});var oc=i(vs);an=a(oc,"\u4EBA(person)"),oc.forEach(s),Aa=a(xo," \u30A8\u30F3\u30C6\u30A3\u30C6\u30A3\u306E\u5148\u982D\u3001\u307E\u305F\u306F\u5185\u90E8\u3067\u3042\u308B\u3053\u3068\u3092\u610F\u5473\u3057\u307E\u3059\u3002"),xo.forEach(s),rn=_(za),al=p(za,"LI",{});var Oo=i(al);Et=p(Oo,"CODE",{});var pc=i(Et);nn=a(pc,"B-ORG"),pc.forEach(s),on=a(Oo,"/"),pa=p(Oo,"CODE",{});var ic=i(pa);vt=a(ic,"I-ORG"),ic.forEach(s),dr=a(Oo," \u306F\u3001\u305D\u306E\u5358\u8A9E\u304C "),rl=p(Oo,"EM",{});var cc=i(rl);pn=a(cc,"\u7D44\u7E54(organization)"),cc.forEach(s),Ra=a(Oo," \u30A8\u30F3\u30C6\u30A3\u30C6\u30A3\u306E\u5148\u982D\u3001\u307E\u305F\u306F\u5185\u90E8\u3067\u3042\u308B\u3053\u3068\u3092\u610F\u5473\u3057\u307E\u3059\u3002"),Oo.forEach(s),cn=_(za),nl=p(za,"LI",{});var yo=i(nl);ia=p(yo,"CODE",{});var fc=i(ia);wt=a(fc,"B-LOC"),fc.forEach(s),hr=a(yo,"/"),yl=p(yo,"CODE",{});var _c=i(yl);br=a(_c,"I-LOC"),_c.forEach(s),ol=a(yo," \u306F\u3001\u305D\u306E\u5358\u8A9E\u304C "),Sa=p(yo,"EM",{});var uc=i(Sa);Ba=a(uc,"\u5834\u6240(location)"),uc.forEach(s),fn=a(yo," \u30A8\u30F3\u30C6\u30A3\u30C6\u30A3\u306E\u5148\u982D\u3001\u307E\u305F\u306F\u5185\u90E8\u3067\u3042\u308B\u3053\u3068\u3092\u610F\u5473\u3057\u307E\u3059\u3002"),yo.forEach(s),_n=_(za),ws=p(za,"LI",{});var Co=i(ws);Fa=p(Co,"CODE",{});var mc=i(Fa);un=a(mc,"B-MISC"),mc.forEach(s),$r=a(Co,"/"),Cl=p(Co,"CODE",{});var dc=i(Cl);jr=a(dc,"I-MISC"),dc.forEach(s),gs=a(Co," \u306F\u3001\u305D\u306E\u5358\u8A9E\u304C "),Ma=p(Co,"EM",{});var hc=i(Ma);La=a(hc,"\u305D\u306E\u4ED6(miscellaneous)"),hc.forEach(s),mn=a(Co," \u30A8\u30F3\u30C6\u30A3\u30C6\u30A3\u306E\u5148\u982D\u3001\u307E\u305F\u306F\u5185\u90E8\u3067\u3042\u308B\u3053\u3068\u3092\u610F\u5473\u3057\u307E\u3059\u3002"),Co.forEach(s),za.forEach(s),kr=_(e),Ql=p(e,"P",{});var bc=i(Ql);dn=a(bc,"\u3055\u3066\u3001\u5148\u307B\u3069\u306E\u30E9\u30D9\u30EB\u3092\u30C7\u30B3\u30FC\u30C9\u3059\u308B\u3068\u3001\u4EE5\u4E0B\u306E\u3088\u3046\u306B\u306A\u308A\u307E\u3059\u3002"),bc.forEach(s),Er=_(e),g(Dl.$$.fragment,e),vr=_(e),g(gt.$$.fragment,e),Na=_(e),Je=p(e,"P",{});var wp=i(Je);wr=a(wp,"\u307E\u305F\u3001"),ql=p(wp,"CODE",{});var $c=i(ql);gr=a($c,"B-"),$c.forEach(s),ca=a(wp," \u3068 "),Ha=p(wp,"CODE",{});var jc=i(Ha);xr=a(jc,"I-"),jc.forEach(s),xt=a(wp,"\u306E\u30E9\u30D9\u30EB\u3092\u6DF7\u5728\u3055\u305B\u305F\u4F8B\u3068\u3057\u3066\u3001\u5B66\u7FD2\u30BB\u30C3\u30C8\u306E4\u756A\u76EE\u306E\u8981\u7D20\u306B\u3064\u3044\u3066\u540C\u3058\u30B3\u30FC\u30C9\u3092\u5B9F\u884C\u3059\u308B\u3068\u3001\u4EE5\u4E0B\u306E\u3088\u3046\u306B\u306A\u308A\u307E\u3059\u3002"),wp.forEach(s),Ga=_(e),g(Qe.$$.fragment,e),Or=_(e),xs=p(e,"P",{});var gp=i(xs);hn=a(gp,"\u3053\u306E\u3088\u3046\u306B\u3001\u201CEuropean Union\u201D \u3068 \u201CWerner Zwingmann\u201D \u306E\u3088\u3046\u306B2\u3064\u306E\u5358\u8A9E\u306B\u307E\u305F\u304C\u308B\u30A8\u30F3\u30C6\u30A3\u30C6\u30A3\u306F\u3001\u6700\u521D\u306E\u5358\u8A9E\u306B\u306F "),Ua=p(gp,"CODE",{});var kc=i(Ua);Wa=a(kc,"B-"),kc.forEach(s),bn=a(gp," \u30E9\u30D9\u30EB\u304C\u30012\u756A\u76EE\u306E\u5358\u8A9E\u306B\u306F "),Xa=p(gp,"CODE",{});var Ec=i(Xa);yr=a(Ec,"I-"),Ec.forEach(s),Ot=a(gp," \u30E9\u30D9\u30EB\u304C\u4ED8\u4E0E\u3055\u308C\u307E\u3059\u3002"),gp.forEach(s),Cr=_(e),g(Pl.$$.fragment,e),Ms=_(e),yt=p(e,"H3",{id:!0});var vc=i(yt);Za=a(vc,"\u30C7\u30FC\u30BF\u306E\u51E6\u7406"),vc.forEach(s),Dr=_(e),g(Ct.$$.fragment,e),fa=_(e),pl=p(e,"P",{});var xp=i(pl);$n=a(xp,"\u3044\u3064\u3082\u306E\u3088\u3046\u306B\u3001\u30E2\u30C7\u30EB\u304C\u610F\u5473\u3092\u7406\u89E3\u3067\u304D\u308B\u3088\u3046\u306B\u3059\u308B\u305F\u3081\u306B\u3001\u30C6\u30AD\u30B9\u30C8\u306F\u30C8\u30FC\u30AF\u30F3ID\u306B\u5909\u63DB\u3055\u308C\u308B\u5FC5\u8981\u304C\u3042\u308A\u307E\u3059\u3002"),Dt=p(xp,"A",{href:!0});var wc=i(Dt);qt=a(wc,"\u7B2C6\u7AE0"),wc.forEach(s),qr=a(xp,"\u3067\u898B\u305F\u3088\u3046\u306B\u3001\u30C8\u30FC\u30AF\u30F3\u5206\u985E\u30BF\u30B9\u30AF\u306E\u5834\u5408\u306E\u5927\u304D\u306A\u9055\u3044\u306F\u3001\u5165\u529B\u304C\u3042\u3089\u304B\u3058\u3081\u30C8\u30FC\u30AF\u30F3\u5316\u3055\u308C\u3066\u3044\u308B\u3068\u8A00\u3046\u4E8B\u3067\u3059\u3002\u5E78\u3044\u306A\u3053\u3068\u306B\u3001tokenizer API \u306F\u3053\u306E\u70B9\u3092\u304B\u306A\u308A\u7C21\u5358\u306B\u51E6\u7406\u3067\u304D\u307E\u3059\u3002\u7279\u5225\u306A\u30D5\u30E9\u30B0\u3092\u6307\u5B9A\u3057\u3066 "),et=p(xp,"CODE",{});var gc=i(et);jn=a(gc,"tokenizer"),gc.forEach(s),Pr=a(xp," \u306B\u8B66\u544A\u3059\u308B\u3060\u3051\u3067\u3059\u3002"),xp.forEach(s),Ls=_(e),il=p(e,"P",{});var ni=i(il);Pt=a(ni,"\u307E\u305A\u6700\u521D\u306B\u3001"),Ka=p(ni,"CODE",{});var xc=i(Ka);Ya=a(xc,"tokenizer"),xc.forEach(s),kn=a(ni," \u30AA\u30D6\u30B8\u30A7\u30AF\u30C8\u3092\u4F5C\u6210\u3057\u307E\u3057\u3087\u3046\u3002\u524D\u306B\u8FF0\u3079\u305F\u3088\u3046\u306B\u3001\u4E8B\u524D\u5B66\u7FD2\u6E08\u307FBERT\u30E2\u30C7\u30EB\u3092\u4F7F\u7528\u3059\u308B\u4E88\u5B9A\u306A\u306E\u3067\u3001\u95A2\u9023\u3059\u308Btokenizer\u3092\u30C0\u30A6\u30F3\u30ED\u30FC\u30C9\u3057\u3066\u30AD\u30E3\u30C3\u30B7\u30E5\u3059\u308B\u3053\u3068\u304B\u3089\u59CB\u3081\u307E\u3059\u3002"),ni.forEach(s),zr=_(e),g(zt.$$.fragment,e),_a=_(e),_e=p(e,"P",{});var Op=i(_e);En=a(Op,"\u3042\u306A\u305F\u306F"),Tt=p(Op,"CODE",{});var Oc=i(Tt);vn=a(Oc,"model_checkpoint"),Oc.forEach(s),wn=a(Op," \u3092\u81EA\u7531\u306B\u7F6E\u304D\u63DB\u3048\u308B\u4E8B\u304C\u3067\u304D\u307E\u3059\u3002"),zl=p(Op,"A",{href:!0,rel:!0});var yc=i(zl);gn=a(yc,"Hub"),yc.forEach(s),xn=a(Op," \u306B\u3042\u308B\u597D\u304D\u306A\u30E2\u30C7\u30EB\u3084\u3001\u81EA\u5206\u306E\u7AEF\u672B\u306B\u4FDD\u5B58\u3057\u305F\u4E8B\u524D\u5B66\u7FD2\u6E08\u307F\u30E2\u30C7\u30EB\u3084tokenizer\u3092\u3067\u7F6E\u304D\u63DB\u3048\u308B\u3053\u3068\u304C\u3067\u304D\u307E\u3059\u3002"),Op.forEach(s),ua=_(e),Os=p(e,"P",{});var Kr=i(Os);On=a(Kr,"\u552F\u4E00\u306E\u5236\u7D04\u306F\u3001tokenizer\u304C \u{1F917} Tokenizers \u30E9\u30A4\u30D6\u30E9\u30EA\u306B\u3088\u3063\u3066\u30D0\u30C3\u30AF\u30A2\u30C3\u30D7\u3055\u308C\u308B\u5FC5\u8981\u304C\u3042\u308B\u3053\u3068\u3067\u3059\u3002\u3053\u308C\u306B\u3088\u308A\u300C\u9AD8\u901F\u300D\u30D0\u30FC\u30B8\u30E7\u30F3\u304C\u7528\u610F\u3055\u308C\u307E\u3059\u3002"),Tl=p(Kr,"A",{href:!0,rel:!0});var Cc=i(Tl);yn=a(Cc,"\u3053\u306E\u5927\u304D\u306A\u30C6\u30FC\u30D6\u30EB"),Cc.forEach(s),Cn=a(Kr," \u3067\u9AD8\u901F\u30D0\u30FC\u30B8\u30E7\u30F3\u3092\u6301\u3064\u5168\u3066\u306E\u30A2\u30FC\u30AD\u30C6\u30AF\u30C1\u30E3\u3092\u898B\u308B\u3053\u3068\u304C\u3067\u304D\u307E\u3059\u3002\u4F7F\u7528\u3057\u3066\u3044\u308B "),Va=p(Kr,"CODE",{});var Dc=i(Va);Ja=a(Dc,"tokenizer"),Dc.forEach(s),Il=a(Kr," \u30AA\u30D6\u30B8\u30A7\u30AF\u30C8\u304C\u672C\u5F53\u306B \u{1F917} Tokenizers \u3067\u30D0\u30C3\u30AF\u30A2\u30C3\u30D7\u3055\u308C\u3066\u3044\u308B\u304B\u3069\u3046\u304B\u3092\u78BA\u8A8D\u3059\u308B\u306B\u306F\u3001"),Qa=p(Kr,"CODE",{});var qc=i(Qa);er=a(qc,"is_fast"),qc.forEach(s),Dn=a(Kr," \u5C5E\u6027\u3092\u898B\u308B\u4E8B\u304C\u78BA\u8A8D\u3067\u304D\u307E\u3059\u3002"),Kr.forEach(s),Tr=_(e),g(Al.$$.fragment,e),Ir=_(e),g(It.$$.fragment,e),sr=_(e),ps=p(e,"P",{});var yp=i(ps);qn=a(yp,"\u30C8\u30FC\u30AF\u30F3\u5316\u524D\u306E\u5165\u529B\u3092\u30C8\u30FC\u30AF\u30F3\u5316\u3059\u308B\u306B\u306F\u3001\u666E\u6BB5\u901A\u308A "),ma=p(yp,"CODE",{});var Pc=i(ma);At=a(Pc,"tokenizer"),Pc.forEach(s),Ar=a(yp," \u3092\u4F7F\u7528\u3057\u3066\u3001 "),st=p(yp,"CODE",{});var zc=i(st);Pn=a(zc,"is_split_into_words=True"),zc.forEach(s),Rr=a(yp," \u3092\u8FFD\u52A0\u3059\u308B\u3060\u3051\u3067\u3059\u3002"),yp.forEach(s),lt=_(e),g(tt.$$.fragment,e),ie=_(e),g(Rt.$$.fragment,e),da=_(e),ze=p(e,"P",{});var pt=i(ze);zn=a(pt,"\u898B\u3066\u306E\u901A\u308A\u3001\u30C8\u30FC\u30AF\u30F3\u5316\u306F\u30E2\u30C7\u30EB\u304C\u4F7F\u7528\u3059\u308B\u7279\u6B8A\u306A\u30C8\u30FC\u30AF\u30F3\uFF08\u5148\u982D\u306E "),St=p(pt,"CODE",{});var Tc=i(St);Tn=a(Tc,"[CLS]"),Tc.forEach(s),In=a(pt," \u3068\u672B\u5C3E\u306E "),Bt=p(pt,"CODE",{});var Ic=i(Bt);An=a(Ic,"[SEP]"),Ic.forEach(s),Rn=a(pt," \uFF09\u3092\u8FFD\u52A0\u3057\u3001\u307B\u3068\u3093\u3069\u306E\u5358\u8A9E\u306F\u305D\u306E\u307E\u307E\u306B\u3057\u307E\u3057\u305F\u3002\u3057\u304B\u3057\u3001"),Ft=p(pt,"CODE",{});var Ac=i(Ft);Sn=a(Ac,"lamb"),Ac.forEach(s),Bn=a(pt,"\u3068\u3044\u3046\u5358\u8A9E\u306F"),Mt=p(pt,"CODE",{});var Rc=i(Mt);Fn=a(Rc,"la"),Rc.forEach(s),Mn=a(pt,"\u3068"),Lt=p(pt,"CODE",{});var Sc=i(Lt);Ln=a(Sc,"##mb"),Sc.forEach(s),Nn=a(pt,"\u3068\u3044\u30462\u3064\u306E\u30B5\u30D6\u30EF\u30FC\u30C9\u306B\u30C8\u30FC\u30AF\u30F3\u5316\u3055\u308C\u307E\u3057\u305F\u3002\u3053\u306E\u305F\u3081\u3001\u5165\u529B\u3068\u30E9\u30D9\u30EB\u306E\u9593\u306B\u30DF\u30B9\u30DE\u30C3\u30C1\u304C\u751F\u3058\u307E\u3059\u3002\u30E9\u30D9\u30EB\u306E\u30EA\u30B9\u30C8\u306B\u306F9\u3064\u306E\u8981\u7D20\u3057\u304B\u3042\u308A\u307E\u305B\u3093\u304C\u3001\u5165\u529B\u306E\u30EA\u30B9\u30C8\u306B\u306F12\u306E\u30C8\u30FC\u30AF\u30F3\u304C\u3042\u308A\u307E\u3059\u3002\u7279\u6B8A\u306A\u30C8\u30FC\u30AF\u30F3\u3092\u8003\u616E\u3059\u308B\u306E\u306F\u7C21\u5358\u3067\u3059\u304C\uFF08\u6700\u521D\u3068\u6700\u5F8C\u306B\u3042\u308B\u3053\u3068\u304C\u5206\u304B\u3063\u3066\u3044\u307E\u3059\uFF09\u3001\u3059\u3079\u3066\u306E\u30E9\u30D9\u30EB\u3092\u9069\u5207\u306A\u5358\u8A9E\u306B\u63C3\u3048\u308B\u3053\u3068\u3082\u5FC5\u8981\u3067\u3059\u3002"),pt.forEach(s),ha=_(e),at=p(e,"P",{});var oi=i(at);Hn=a(oi,"\u5E78\u3044\u3001\u9AD8\u901F\u306Atokenizer\u3092\u4F7F\u3063\u3066\u3044\u308B\u306E\u3067\u3001\u{1F917} Tokenizers \u306E\u30B9\u30FC\u30D1\u30FC\u30D1\u30EF\u30FC\u306B\u30A2\u30AF\u30BB\u30B9\u3059\u308B\u3053\u3068\u304C\u3067\u304D\u3001\u305D\u308C\u305E\u308C\u306E\u30C8\u30FC\u30AF\u30F3\u3092\u5BFE\u5FDC\u3059\u308B\u5358\u8A9E\u306B\u7C21\u5358\u306B\u30DE\u30C3\u30D4\u30F3\u30B0\u3059\u308B\u3053\u3068\u304C\u3067\u304D\u307E\u3059 (\u3053\u308C\u306F "),Nt=p(oi,"A",{href:!0});var Bc=i(Nt);cl=a(Bc,"\u7B2C\uFF16\u7AE0"),Bc.forEach(s),Gn=a(oi," \u3067\u898B\u305F\u3068\u304A\u308A\u3067\u3059)\u3002"),oi.forEach(s),ba=_(e),g(Ht.$$.fragment,e),Sr=_(e),g(Ns.$$.fragment,e),$a=_(e),es=p(e,"P",{});var Ta=i(es);Un=a(Ta,"\u307B\u3093\u306E\u5C11\u3057\u306E\u4F5C\u696D\u3067\u3001\u30C8\u30FC\u30AF\u30F3\u306B\u30DE\u30C3\u30C1\u3059\u308B\u3088\u3046\u306B\u30E9\u30D9\u30EB\u30EA\u30B9\u30C8\u3092\u62E1\u5F35\u3059\u308B\u3053\u3068\u304C\u3067\u304D\u307E\u3059\u3002\u6700\u521D\u306B\u9069\u7528\u3059\u308B\u30EB\u30FC\u30EB\u306F\u3001\u7279\u6B8A\u306A\u30C8\u30FC\u30AF\u30F3\u306B\u306F "),ja=p(Ta,"CODE",{});var Fc=i(ja);r=a(Fc,"-100"),Fc.forEach(s),h=a(Ta," \u3068\u3044\u3046\u30E9\u30D9\u30EB\u3092\u4ED8\u3051\u308B\u3068\u3044\u3046\u3082\u306E\u3067\u3059\u3002\u3053\u308C\u306F\u30C7\u30D5\u30A9\u30EB\u30C8\u3067 "),rt=p(Ta,"CODE",{});var Mc=i(rt);qo=a(Mc,"-100"),Mc.forEach(s),Po=a(Ta," \u304C\u3053\u308C\u304B\u3089\u4F7F\u3046\u640D\u5931\u95A2\u6570\uFF08\u30AF\u30ED\u30B9\u30A8\u30F3\u30C8\u30ED\u30D4\u30FC\uFF09\u3067\u7121\u8996\u3055\u308C\u308B\u6570\u3060\u304B\u3089\u3067\u3059\u3002\u6B21\u306B\u3001\u5358\u8A9E\u5185\u306E\u5404\u30C8\u30FC\u30AF\u30F3\u306F\u5358\u8A9E\u306E\u5148\u982D\u306E\u30C8\u30FC\u30AF\u30F3\u3068\u540C\u3058\u30E9\u30D9\u30EB\u304C\u4ED8\u4E0E\u3055\u308C\u307E\u3059\u3002\u3053\u308C\u306F\u540C\u3058\u30A8\u30F3\u30C6\u30A3\u30C6\u30A3\u306E\u4E00\u90E8\u3067\u3042\u308B\u305F\u3081\u3067\u3059\u3002\u5358\u8A9E\u306E\u5185\u90E8\u306B\u3042\u308A\u3001\u304B\u3064\u5148\u982D\u306B\u306A\u3044\u30C8\u30FC\u30AF\u30F3\u306B\u3064\u3044\u3066\u306F\u3001"),ka=p(Ta,"CODE",{});var Lc=i(ka);zo=a(Lc,"B-"),Lc.forEach(s),Br=a(Ta," \u3092 "),Fr=p(Ta,"CODE",{});var Nc=i(Fr);lr=a(Nc,"I-"),Nc.forEach(s),To=a(Ta," \u306B\u7F6E\u304D\u63DB\u3048\u307E\u3059\uFF08\u305D\u306E\u30C8\u30FC\u30AF\u30F3\u306F\u30A8\u30F3\u30C6\u30A3\u30C6\u30A3\u3092\u958B\u59CB\u3057\u306A\u3044\u305F\u3081\u3067\u3059\uFF09\u3002"),Ta.forEach(s),Wn=_(e),g(Ea.$$.fragment,e),tr=_(e),ar=p(e,"P",{});var Hc=i(ar);Io=a(Hc,"\u305D\u308C\u3067\u306F\u3001\u6700\u521D\u306E\u6587\u7AE0\u3067\u8A66\u3057\u3066\u307F\u307E\u3057\u3087\u3046\u3002"),Hc.forEach(s),Xn=_(e),g(va.$$.fragment,e),rr=_(e),g(wa.$$.fragment,e),Zn=_(e),Hs=p(e,"P",{});var Cp=i(Hs);Ao=a(Cp,"\u898B\u3066\u308F\u304B\u308B\u3088\u3046\u306B\u3001\u3053\u306E\u95A2\u6570\u306F\u6700\u521D\u3068\u6700\u5F8C\u306E2\u3064\u306E\u7279\u5225\u306A\u30C8\u30FC\u30AF\u30F3\u306B\u5BFE\u3057\u3066 "),ys=p(Cp,"CODE",{});var Gc=i(ys);Ro=a(Gc,"-100"),Gc.forEach(s),Kn=a(Cp," \u3092\u8FFD\u52A0\u3057\u30012\u3064\u306E\u30C8\u30FC\u30AF\u30F3\u306B\u5206\u5272\u3055\u308C\u305F\u5358\u8A9E\u306B\u5BFE\u3057\u3066\u65B0\u305F\u306B "),Mr=p(Cp,"CODE",{});var Uc=i(Mr);So=a(Uc,"0"),Uc.forEach(s),Bo=a(Cp," \u3092\u8FFD\u52A0\u3057\u3066\u3044\u307E\u3059\u3002"),Cp.forEach(s),nr=_(e),g(Gt.$$.fragment,e),or=_(e),is=p(e,"P",{});var Ia=i(is);Fo=a(Ia,"\u30C7\u30FC\u30BF\u30BB\u30C3\u30C8\u5168\u4F53\u306E\u524D\u51E6\u7406\u3068\u3057\u3066\u3001\u3059\u3079\u3066\u306E\u5165\u529B\u3092\u30C8\u30FC\u30AF\u30F3\u5316\u3057\u3001\u3059\u3079\u3066\u306E\u30E9\u30D9\u30EB\u306B\u5BFE\u3057\u3066 "),ga=p(Ia,"CODE",{});var Wc=i(ga);Mo=a(Wc,"align_labels_with_tokens()"),Wc.forEach(s),Lo=a(Ia," \u3092\u9069\u7528\u3059\u308B\u5FC5\u8981\u304C\u3042\u308A\u307E\u3059\u3002\u9AD8\u901F\u306Atokenizer\u306E\u901F\u5EA6\u3092\u6D3B\u304B\u3059\u306B\u306F\u3001\u305F\u304F\u3055\u3093\u306E\u30C6\u30AD\u30B9\u30C8\u3092\u540C\u6642\u306B\u30C8\u30FC\u30AF\u30F3\u5316\u3059\u308B\u306E\u304C\u3088\u3044\u3067\u3057\u3087\u3046\u3002\u305D\u3053\u3067\u3001\u30B5\u30F3\u30D7\u30EB\u306E\u30EA\u30B9\u30C8\u3092\u51E6\u7406\u3059\u308B\u95A2\u6570\u3092\u66F8\u3044\u3066\u3001 "),xa=p(Ia,"CODE",{});var Xc=i(xa);No=a(Xc,"Dataset.map()"),Xc.forEach(s),Ho=a(Ia," \u30E1\u30BD\u30C3\u30C9\u306B "),Lr=p(Ia,"CODE",{});var Zc=i(Lr);Nr=a(Zc,"batched=True"),Zc.forEach(s),Go=a(Ia," \u30AA\u30D7\u30B7\u30E7\u30F3\u3092\u4ED8\u3051\u3066\u4F7F\u7528\u3059\u308B\u3053\u3068\u306B\u3057\u307E\u3057\u3087\u3046\u3002\u4EE5\u524D\u306E\u4F8B\u3068\u552F\u4E00\u9055\u3046\u306E\u306F\u3001tokenizer\u3078\u306E\u5165\u529B\u304C\u30C6\u30AD\u30B9\u30C8\u306E\u30EA\u30B9\u30C8\uFF08\u3053\u306E\u5834\u5408\u306F\u5358\u8A9E\u306E\u30EA\u30B9\u30C8\u306E\u30EA\u30B9\u30C8\uFF09\u3067\u3042\u308B\u5834\u5408\u3001 "),Hr=p(Ia,"CODE",{});var Kc=i(Hr);nt=a(Kc,"word_ids()"),Kc.forEach(s),Uo=a(Ia," \u95A2\u6570\u306F\u5358\u8A9EID\u304C\u6B32\u3057\u3044\u30EA\u30B9\u30C8\u306E\u30A4\u30F3\u30C7\u30C3\u30AF\u30B9\u3092\u5FC5\u8981\u3068\u3059\u308B\u306E\u3067\u3001\u3053\u308C\u3082\u8FFD\u52A0\u3057\u307E\u3059\u3002"),Ia.forEach(s),Yn=_(e),g(Oa.$$.fragment,e),Vn=_(e),Rl=p(e,"P",{});var Yc=i(Rl);Wo=a(Yc,`\u307E\u3060\u3001\u5165\u529B\u3092\u30D1\u30C7\u30A3\u30F3\u30B0\u3057\u3066\u3044\u306A\u3044\u3053\u3068\u306B\u6CE8\u610F\u3057\u3066\u304F\u3060\u3055\u3044\u3002
\u3053\u308C\u306F\u5F8C\u3067\u30C7\u30FC\u30BF\u7167\u5408\u30C4\u30FC\u30EB\u3067\u30D0\u30C3\u30C1\u3092\u4F5C\u6210\u3059\u308B\u3068\u304D\u306B\u3084\u308B\u3053\u3068\u306B\u3057\u307E\u3059\u3002`),Yc.forEach(s),Jn=_(e),Ut=p(e,"P",{});var Vc=i(Ut);Xo=a(Vc,"\u3053\u308C\u3067\u3001\u30C7\u30FC\u30BF\u30BB\u30C3\u30C8\u306E\u5206\u5272\u306B\u5BFE\u3057\u3066\u3001\u3059\u3079\u3066\u306E\u524D\u51E6\u7406\u3092\u4E00\u5EA6\u306B\u9069\u7528\u3059\u308B\u3053\u3068\u304C\u3067\u304D\u308B\u3088\u3046\u306B\u306A\u308A\u307E\u3057\u305F\u3002"),Vc.forEach(s),Wt=_(e),g(ya.$$.fragment,e),Qn=_(e),cs=p(e,"P",{});var pi=i(cs);Zo=a(pi,`\u4E00\u756A\u5927\u5909\u306A\u3068\u3053\u308D\u3092\u3084\u308A\u307E\u3057\u305F\u306D\uFF01
\u30C7\u30FC\u30BF\u306E\u524D\u51E6\u7406\u304C\u7D42\u308F\u3063\u305F\u306E\u3067\u3001\u5B9F\u969B\u306E\u5B66\u7FD2\u306F`),pr=p(pi,"A",{href:!0});var Jc=i(pr);Ko=a(Jc,"\u7B2C3\u7AE0"),Jc.forEach(s),Yo=a(pi,"\u3067\u3084\u3063\u305F\u3088\u3046\u306A\u611F\u3058\u306B\u306A\u308A\u307E\u3059\u306D\u3002"),pi.forEach(s),Xt=_(e),_l.l(e),ir=_(e),ul=p(e,"H3",{id:!0});var Qc=i(ul);Vo=a(Qc,"\u30C7\u30FC\u30BF\u7167\u5408"),Qc.forEach(s),eo=_(e),ml=p(e,"P",{});var Do=i(ml);Sl=p(Do,"A",{href:!0});var ef=i(Sl);Jo=a(ef,"\u7B2C3\u7AE0"),ef.forEach(s),Gr=a(Do," \u306B\u3042\u308B\u3088\u3046\u306A "),Ur=p(Do,"CODE",{});var sf=i(Ur);Qo=a(sf,"DataCollatorWithPadding"),sf.forEach(s),dl=a(Do," \u306F\u5165\u529B (\u5165\u529B ID\u3001\u30A2\u30C6\u30F3\u30B7\u30E7\u30F3\u30DE\u30B9\u30AF\u3001\u30C8\u30FC\u30AF\u30F3\u30BF\u30A4\u30D7 ID) \u306E\u307F\u3092\u30D1\u30C7\u30A3\u30F3\u30B0\u3059\u308B\u306E\u3067\u4F7F\u3048\u307E\u305B\u3093\u3002\u3053\u3053\u3067\u306F\u3001\u30E9\u30D9\u30EB\u306E\u30B5\u30A4\u30BA\u304C\u5909\u308F\u3089\u306A\u3044\u3088\u3046\u306B\u3001\u5165\u529B\u3068\u5168\u304F\u540C\u3058\u65B9\u6CD5\u3067\u30D1\u30C7\u30A3\u30F3\u30B0\u3092\u884C\u3044\u307E\u3059\u3002\u5024\u3068\u3057\u3066 "),Wr=p(Do,"CODE",{});var lf=i(Wr);ep=a(lf,"-100"),lf.forEach(s),sp=a(Do," \u3092\u4F7F\u7528\u3057\u3001\u5BFE\u5FDC\u3059\u308B\u4E88\u6E2C\u5024\u304C\u640D\u5931\u8A08\u7B97\u3067\u7121\u8996\u3055\u308C\u308B\u3088\u3046\u306B\u3057\u307E\u3059\u3002"),Do.forEach(s),so=_(e),Bl=p(e,"P",{});var Dp=i(Bl);Ca=a(Dp,"\u3053\u308C\u306F\u5168\u3066 "),Da=p(Dp,"A",{href:!0,rel:!0});var tf=i(Da);Xr=p(tf,"CODE",{});var af=i(Xr);lp=a(af,"DataCollatorForTokenClassification"),af.forEach(s),tf.forEach(s),tp=a(Dp," \u306B\u3088\u3063\u3066\u884C\u308F\u308C\u307E\u3059\u3002DataCollatorWithPadding"),be=p(Dp,"CODE",{});var rf=i(be);ap=a(rf,"\u3068\u540C\u69D8\u306B\u3001\u5165\u529B\u306E\u524D\u51E6\u7406\u306B\u4F7F\u7528\u3055\u308C\u308B"),rf.forEach(s),rp=a(Dp,"tokenizer` \u3092\u53D7\u3051\u53D6\u308A\u307E\u3059\u3002"),Dp.forEach(s),lo=_(e),bl.l(e),cr=_(e),fr=p(e,"P",{});var nf=i(fr);qa=a(nf,"\u3053\u308C\u3092\u3044\u304F\u3064\u304B\u306E\u30B5\u30F3\u30D7\u30EB\u3067\u30C6\u30B9\u30C8\u3059\u308B\u306B\u306F\u3001\u30C8\u30FC\u30AF\u30F3\u5316\u3055\u308C\u305F\u30C8\u30EC\u30FC\u30CB\u30F3\u30B0\u30BB\u30C3\u30C8\u304B\u3089\u30B5\u30F3\u30D7\u30EB\u306E\u30EA\u30B9\u30C8\u306B\u5BFE\u3057\u3066\u547C\u3073\u51FA\u3059\u3060\u3051\u3067\u3088\u3044\u306E\u3067\u3059\u3002"),nf.forEach(s),to=_(e),g(Pa.$$.fragment,e),ao=_(e),g(ro.$$.fragment,e),Tp=_(e),np=p(e,"P",{});var of=i(np);ci=a(of,"\u3053\u308C\u3092\u30C7\u30FC\u30BF\u30BB\u30C3\u30C8\u306E1\u756A\u76EE\u30682\u756A\u76EE\u306E\u8981\u7D20\u306E\u30E9\u30D9\u30EB\u3068\u6BD4\u8F03\u3057\u3066\u307F\u307E\u3057\u3087\u3046\u3002"),of.forEach(s),Ip=_(e),g(no.$$.fragment,e),Ap=_(e),g(oo.$$.fragment,e),Rp=_(e),Kt.l(e),op=_(e),Fe&&Fe.l(e),pp=_(e),po=p(e,"H3",{id:!0});var pf=i(po);fi=a(pf,"\u6307\u6A19"),pf.forEach(s),Sp=_(e),Vt.l(e),ip=_(e),g(io.$$.fragment,e),Bp=_(e),cp=p(e,"P",{});var cf=i(cp);_i=a(cf,"\u3053\u306E\u6307\u6A19\u306F\u6A19\u6E96\u7684\u306A\u7CBE\u5EA6\u6307\u6A19\u306E\u3088\u3046\u306B\u52D5\u4F5C\u3057\u307E\u305B\u3093\uFF1A\u5B9F\u969B\u306B\u306F\u30E9\u30D9\u30EB\u306E\u30EA\u30B9\u30C8\u3092\u6574\u6570\u3067\u306F\u306A\u304F\u6587\u5B57\u5217\u3068\u3057\u3066\u53D7\u3051\u53D6\u308B\u306E\u3067\u3001\u4E88\u6E2C\u5024\u3068\u30E9\u30D9\u30EB\u3092\u6307\u6A19\u306B\u6E21\u3059\u524D\u306B\u5B8C\u5168\u306B\u30C7\u30B3\u30FC\u30C9\u3059\u308B\u5FC5\u8981\u304C\u3042\u308A\u307E\u3059\u3002"),cf.forEach(s),Fp=_(e),fp=p(e,"P",{});var ff=i(fp);ui=a(ff,"\u305D\u308C\u3067\u306F\u3001\u3069\u306E\u3088\u3046\u306B\u52D5\u4F5C\u3059\u308B\u304B\u898B\u3066\u307F\u307E\u3057\u3087\u3046\u3002\u307E\u305A\u3001\u6700\u521D\u306E\u5B66\u7FD2\u30B5\u30F3\u30D7\u30EB\u306B\u5BFE\u3059\u308B\u30E9\u30D9\u30EB\u3092\u53D6\u5F97\u3057\u307E\u3059\u3002"),ff.forEach(s),Mp=_(e),g(co.$$.fragment,e),Lp=_(e),g(fo.$$.fragment,e),Np=_(e),_p=p(e,"P",{});var _f=i(_p);mi=a(_f,"\u305D\u3057\u3066\u3001\u30A4\u30F3\u30C7\u30C3\u30AF\u30B92\u306E\u5024\u3092\u5909\u66F4\u3059\u308B\u3060\u3051\u3067\u3001\u305D\u308C\u3089\u306E\u7591\u4F3C\u4E88\u6E2C\u3092\u4F5C\u6210\u3059\u308B\u3053\u3068\u304C\u3067\u304D\u307E\u3059\u3002"),_f.forEach(s),Hp=_(e),g(_o.$$.fragment,e),Gp=_(e),up=p(e,"P",{});var uf=i(up);di=a(uf,"\u3053\u306E\u6307\u6A19\u306F\u4E88\u6E2C\u5024\u306E\u30EA\u30B9\u30C8\uFF081\u3064\u3060\u3051\u3067\u306F\u306A\u3044\uFF09\u3068\u30E9\u30D9\u30EB\u306E\u30EA\u30B9\u30C8\u3092\u53D7\u3051\u53D6\u308B\u3053\u3068\u306B\u6CE8\u610F\u3057\u3066\u304F\u3060\u3055\u3044\u3002\u4EE5\u4E0B\u306F\u305D\u306E\u51FA\u529B\u3067\u3059\u3002"),uf.forEach(s),Up=_(e),g(uo.$$.fragment,e),Wp=_(e),Qt.l(e),mp=_(e),Me&&Me.l(e),dp=_(e),mo=p(e,"H2",{id:!0});var mf=i(mo);hi=a(mf,"\u5FAE\u8ABF\u6574\u3057\u305F\u30E2\u30C7\u30EB\u3092\u4F7F\u3046"),mf.forEach(s),Xp=_(e),Zr=p(e,"P",{});var ii=i(Zr);bi=a(ii,"Model Hub\u3067\u5FAE\u8ABF\u6574\u3057\u305F\u30E2\u30C7\u30EB\u3092\u63A8\u8AD6\u30A6\u30A3\u30B8\u30A7\u30C3\u30C8\u3067\u4F7F\u7528\u3059\u308B\u65B9\u6CD5\u306F\u65E2\u306B\u7D39\u4ECB\u3057\u307E\u3057\u305F\u3002\u30ED\u30FC\u30AB\u30EB\u74B0\u5883\u306E"),qp=p(ii,"CODE",{});var df=i(qp);$i=a(df,"pipeline"),df.forEach(s),ji=a(ii,"\u3067\u4F7F\u7528\u3059\u308B\u5834\u5408\u306F\u3001\u30E2\u30C7\u30EB\u8B58\u5225\u5B50\u3092\u6307\u5B9A\u3057\u307E\u3059\u3002"),ii.forEach(s),Zp=_(e),g(ho.$$.fragment,e),Kp=_(e),g(bo.$$.fragment,e),Yp=_(e),hp=p(e,"P",{});var hf=i(hp);ki=a(hf,`\u7D20\u6674\u3089\u3057\u3044\uFF01
\u79C1\u305F\u3061\u306E\u30E2\u30C7\u30EB\u306F\u3001\u3053\u306E\u30D1\u30A4\u30D7\u30E9\u30A4\u30F3\u306E\u30C7\u30D5\u30A9\u30EB\u30C8\u306E\u3082\u306E\u3068\u540C\u3058\u3088\u3046\u306B\u52D5\u4F5C\u3057\u3066\u3044\u307E\u3059\uFF01`),hf.forEach(s),this.h()},h(){P(u,"name","hf:doc:metadata"),P(u,"content",JSON.stringify(Kf)),P(D,"id",""),vi(ke.src,Ml="https://hf.space/gradioiframe/course-demos/bert-finetuned-ner/+")||P(ke,"src",Ml),P(ke,"frameborder","0"),P(ke,"height","350"),P(ke,"title","Gradio app"),P(ke,"class","block dark:hidden container p-0 flex-grow space-iframe"),P(ke,"allow","accelerometer; ambient-light-sensor; autoplay; battery; camera; document-domain; encrypted-media; fullscreen; geolocation; gyroscope; layout-animations; legacy-image-formats; magnetometer; microphone; midi; oversized-images; payment; picture-in-picture; publickey-credentials-get; sync-xhr; usb; vr ; wake-lock; xr-spatial-tracking"),P(ke,"sandbox","allow-forms allow-modals allow-popups allow-popups-to-escape-sandbox allow-same-origin allow-scripts allow-downloads"),P(Ne,"class","block dark:hidden lg:w-3/5"),vi(Ne.src,Gs="https://huggingface.co/datasets/huggingface-course/documentation-images/resolve/main/en/chapter7/model-eval-bert-finetuned-ner.png")||P(Ne,"src",Gs),P(Ne,"alt","One-hot encoded labels for question answering."),P(Ie,"class","hidden dark:block lg:w-3/5"),vi(Ie.src,$l="https://huggingface.co/datasets/huggingface-course/documentation-images/resolve/main/en/chapter7/model-eval-bert-finetuned-ner-dark.png")||P(Ie,"src",$l),P(Ie,"alt","One-hot encoded labels for question answering."),P(Le,"class","flex justify-center"),P(Le,"href","/huggingface-course/bert-finetuned-ner"),P(ss,"href","https://huggingface.co/huggingface-course/bert-finetuned-ner?text=My+name+is+Sylvain+and+I+work+at+Hugging+Face+in+Brooklyn"),P(ss,"rel","nofollow"),P(Ge,"id",""),P(Ce,"href","https://huggingface.co/datasets/conll2003"),P(Ce,"rel","nofollow"),P(ls,"id","the-conll2003-dataset"),P(ls,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),P(ls,"href","#the-conll2003-dataset"),P(Ee,"class","relative group"),P(Ws,"href","/course/ja/chapter3"),P(ll,"href","/course/ja/chapter6/3"),P(yt,"id",""),P(Dt,"href","/course/ja/chapter6/"),P(zl,"href","https://huggingface.co/models"),P(zl,"rel","nofollow"),P(Tl,"href","https://huggingface.co/transformers/#supported-frameworks"),P(Tl,"rel","nofollow"),P(Nt,"href","/course/ja/chapter6/3"),P(pr,"href","/course/ja/chapter3"),P(ul,"id",""),P(Sl,"href","/course/ja/chapter3"),P(Da,"href","https://huggingface.co/transformers/main_classes/data_collator.html#datacollatorfortokenclassification"),P(Da,"rel","nofollow"),P(po,"id",""),P(mo,"id","")},m(e,c){l(document.head,u),n(e,$,c),x(d,e,c),n(e,b,c),n(e,D,c),l(D,E),n(e,v,c),$o[q].m(e,c),n(e,z,c),n(e,B,c),l(B,A),n(e,R,c),n(e,U,c),l(U,y),l(y,M),l(M,K),l(y,Z),l(U,W),l(U,I),l(I,F),l(F,L),l(I,X),l(U,Q),l(U,V),l(V,le),l(le,J),l(V,ce),l(V,Y),l(Y,G),l(V,te),l(V,$e),l($e,N),l(V,ee),l(V,je),l(je,Te),l(V,se),n(e,fe,c),x(ae,e,c),n(e,ge,c),n(e,re,c),l(re,fs),n(e,ue,c),n(e,ke,c),n(e,xe,c),n(e,Le,c),l(Le,Ne),l(Le,Ll),l(Le,Ie),n(e,jl,c),n(e,Oe,c),l(Oe,Cs),l(Oe,ss),l(ss,He),l(Oe,Nl),n(e,_s,c),n(e,Ge,c),l(Ge,Hl),n(e,de,c),n(e,ye,c),l(ye,Us),l(ye,Ce),l(Ce,it),l(ye,Ds),n(e,Gl,c),x(us,e,c),n(e,qs,c),n(e,Ee,c),l(Ee,ls),l(ls,ts),x(Ae,ts,null),l(Ee,kl),l(Ee,ne),l(ne,ct),n(e,ms,c),n(e,ds,c),l(ds,ft),l(ds,hs),l(hs,he),l(ds,_t),n(e,Ps,c),x(De,e,c),n(e,Ul,c),n(e,me,c),l(me,El),l(me,Ws),l(Ws,Wl),l(me,zs),n(e,Ts,c),x(oe,e,c),n(e,Xs,c),x(as,e,c),n(e,Xl,c),n(e,ve,c),l(ve,Zs),l(ve,Ue),l(Ue,ut),l(ve,vl),n(e,Ks,c),n(e,Ys,c),l(Ys,Zl),n(e,We,c),x(bs,e,c),n(e,rs,c),x($s,e,c),n(e,Xe,c),n(e,Is,c),l(Is,Vs),n(e,wl,c),x(qe,e,c),n(e,js,c),x(Ze,e,c),n(e,gl,c),n(e,Pe,c),l(Pe,ks),l(Pe,Js),l(Js,m),l(Pe,S),n(e,As,c),x(Rs,e,c),n(e,Ss,c),x(ns,e,c),n(e,Qs,c),n(e,pe,c),l(pe,mt),l(pe,Re),l(Re,dt),l(pe,Bs),l(pe,Kl),l(Kl,Yl),l(pe,ea),l(pe,Vl),l(Vl,ht),l(pe,Se),l(pe,Jl),l(Jl,bt),l(pe,el),l(pe,Es),l(Es,$t),l(pe,Fs),n(e,xl,c),x(os,e,c),n(e,Ke,c),x(sl,e,c),n(e,Ol,c),n(e,we,c),l(we,sa),l(we,ll),l(ll,Be),l(we,la),l(we,tl),l(tl,Qr),l(we,en),n(e,ta,c),n(e,Ye,c),l(Ye,aa),l(aa,jt),l(jt,sn),l(aa,ln),l(Ye,ur),l(Ye,Ve),l(Ve,ra),l(ra,na),l(Ve,tn),l(Ve,oa),l(oa,kt),l(Ve,mr),l(Ve,vs),l(vs,an),l(Ve,Aa),l(Ye,rn),l(Ye,al),l(al,Et),l(Et,nn),l(al,on),l(al,pa),l(pa,vt),l(al,dr),l(al,rl),l(rl,pn),l(al,Ra),l(Ye,cn),l(Ye,nl),l(nl,ia),l(ia,wt),l(nl,hr),l(nl,yl),l(yl,br),l(nl,ol),l(nl,Sa),l(Sa,Ba),l(nl,fn),l(Ye,_n),l(Ye,ws),l(ws,Fa),l(Fa,un),l(ws,$r),l(ws,Cl),l(Cl,jr),l(ws,gs),l(ws,Ma),l(Ma,La),l(ws,mn),n(e,kr,c),n(e,Ql,c),l(Ql,dn),n(e,Er,c),x(Dl,e,c),n(e,vr,c),x(gt,e,c),n(e,Na,c),n(e,Je,c),l(Je,wr),l(Je,ql),l(ql,gr),l(Je,ca),l(Je,Ha),l(Ha,xr),l(Je,xt),n(e,Ga,c),x(Qe,e,c),n(e,Or,c),n(e,xs,c),l(xs,hn),l(xs,Ua),l(Ua,Wa),l(xs,bn),l(xs,Xa),l(Xa,yr),l(xs,Ot),n(e,Cr,c),x(Pl,e,c),n(e,Ms,c),n(e,yt,c),l(yt,Za),n(e,Dr,c),x(Ct,e,c),n(e,fa,c),n(e,pl,c),l(pl,$n),l(pl,Dt),l(Dt,qt),l(pl,qr),l(pl,et),l(et,jn),l(pl,Pr),n(e,Ls,c),n(e,il,c),l(il,Pt),l(il,Ka),l(Ka,Ya),l(il,kn),n(e,zr,c),x(zt,e,c),n(e,_a,c),n(e,_e,c),l(_e,En),l(_e,Tt),l(Tt,vn),l(_e,wn),l(_e,zl),l(zl,gn),l(_e,xn),n(e,ua,c),n(e,Os,c),l(Os,On),l(Os,Tl),l(Tl,yn),l(Os,Cn),l(Os,Va),l(Va,Ja),l(Os,Il),l(Os,Qa),l(Qa,er),l(Os,Dn),n(e,Tr,c),x(Al,e,c),n(e,Ir,c),x(It,e,c),n(e,sr,c),n(e,ps,c),l(ps,qn),l(ps,ma),l(ma,At),l(ps,Ar),l(ps,st),l(st,Pn),l(ps,Rr),n(e,lt,c),x(tt,e,c),n(e,ie,c),x(Rt,e,c),n(e,da,c),n(e,ze,c),l(ze,zn),l(ze,St),l(St,Tn),l(ze,In),l(ze,Bt),l(Bt,An),l(ze,Rn),l(ze,Ft),l(Ft,Sn),l(ze,Bn),l(ze,Mt),l(Mt,Fn),l(ze,Mn),l(ze,Lt),l(Lt,Ln),l(ze,Nn),n(e,ha,c),n(e,at,c),l(at,Hn),l(at,Nt),l(Nt,cl),l(at,Gn),n(e,ba,c),x(Ht,e,c),n(e,Sr,c),x(Ns,e,c),n(e,$a,c),n(e,es,c),l(es,Un),l(es,ja),l(ja,r),l(es,h),l(es,rt),l(rt,qo),l(es,Po),l(es,ka),l(ka,zo),l(es,Br),l(es,Fr),l(Fr,lr),l(es,To),n(e,Wn,c),x(Ea,e,c),n(e,tr,c),n(e,ar,c),l(ar,Io),n(e,Xn,c),x(va,e,c),n(e,rr,c),x(wa,e,c),n(e,Zn,c),n(e,Hs,c),l(Hs,Ao),l(Hs,ys),l(ys,Ro),l(Hs,Kn),l(Hs,Mr),l(Mr,So),l(Hs,Bo),n(e,nr,c),x(Gt,e,c),n(e,or,c),n(e,is,c),l(is,Fo),l(is,ga),l(ga,Mo),l(is,Lo),l(is,xa),l(xa,No),l(is,Ho),l(is,Lr),l(Lr,Nr),l(is,Go),l(is,Hr),l(Hr,nt),l(is,Uo),n(e,Yn,c),x(Oa,e,c),n(e,Vn,c),n(e,Rl,c),l(Rl,Wo),n(e,Jn,c),n(e,Ut,c),l(Ut,Xo),n(e,Wt,c),x(ya,e,c),n(e,Qn,c),n(e,cs,c),l(cs,Zo),l(cs,pr),l(pr,Ko),l(cs,Yo),n(e,Xt,c),jo[fl].m(e,c),n(e,ir,c),n(e,ul,c),l(ul,Vo),n(e,eo,c),n(e,ml,c),l(ml,Sl),l(Sl,Jo),l(ml,Gr),l(ml,Ur),l(Ur,Qo),l(ml,dl),l(ml,Wr),l(Wr,ep),l(ml,sp),n(e,so,c),n(e,Bl,c),l(Bl,Ca),l(Bl,Da),l(Da,Xr),l(Xr,lp),l(Bl,tp),l(Bl,be),l(be,ap),l(Bl,rp),n(e,lo,c),ko[hl].m(e,c),n(e,cr,c),n(e,fr,c),l(fr,qa),n(e,to,c),x(Pa,e,c),n(e,ao,c),x(ro,e,c),n(e,Tp,c),n(e,np,c),l(np,ci),n(e,Ip,c),x(no,e,c),n(e,Ap,c),x(oo,e,c),n(e,Rp,c),Eo[Zt].m(e,c),n(e,op,c),Fe&&Fe.m(e,c),n(e,pp,c),n(e,po,c),l(po,fi),n(e,Sp,c),vo[Yt].m(e,c),n(e,ip,c),x(io,e,c),n(e,Bp,c),n(e,cp,c),l(cp,_i),n(e,Fp,c),n(e,fp,c),l(fp,ui),n(e,Mp,c),x(co,e,c),n(e,Lp,c),x(fo,e,c),n(e,Np,c),n(e,_p,c),l(_p,mi),n(e,Hp,c),x(_o,e,c),n(e,Gp,c),n(e,up,c),l(up,di),n(e,Up,c),x(uo,e,c),n(e,Wp,c),wo[Jt].m(e,c),n(e,mp,c),Me&&Me.m(e,c),n(e,dp,c),n(e,mo,c),l(mo,hi),n(e,Xp,c),n(e,Zr,c),l(Zr,bi),l(Zr,qp),l(qp,$i),l(Zr,ji),n(e,Zp,c),x(ho,e,c),n(e,Kp,c),x(bo,e,c),n(e,Yp,c),n(e,hp,c),l(hp,ki),Vp=!0},p(e,[c]){const Pp={};c&1&&(Pp.fw=e[0]),d.$set(Pp);let bp=q;q=xi(e),q!==bp&&(Vr(),k($o[bp],1,1,()=>{$o[bp]=null}),Yr(),C=$o[q],C||(C=$o[q]=gi[q](e),C.c()),j(C,1),C.m(z.parentNode,z));const _r={};c&2&&(_r.$$scope={dirty:c,ctx:e}),us.$set(_r);const $p={};c&2&&($p.$$scope={dirty:c,ctx:e}),Pl.$set($p);const zp={};c&2&&(zp.$$scope={dirty:c,ctx:e}),Gt.$set(zp);let go=fl;fl=yi(e),fl!==go&&(Vr(),k(jo[go],1,1,()=>{jo[go]=null}),Yr(),_l=jo[fl],_l||(_l=jo[fl]=Oi[fl](e),_l.c()),j(_l,1),_l.m(ir.parentNode,ir));let jp=hl;hl=Di(e),hl!==jp&&(Vr(),k(ko[jp],1,1,()=>{ko[jp]=null}),Yr(),bl=ko[hl],bl||(bl=ko[hl]=Ci[hl](e),bl.c()),j(bl,1),bl.m(cr.parentNode,cr));let Fl=Zt;Zt=Pi(e),Zt!==Fl&&(Vr(),k(Eo[Fl],1,1,()=>{Eo[Fl]=null}),Yr(),Kt=Eo[Zt],Kt||(Kt=Eo[Zt]=qi[Zt](e),Kt.c()),j(Kt,1),Kt.m(op.parentNode,op)),e[0]==="tf"?Fe?c&1&&j(Fe,1):(Fe=jf(e),Fe.c(),j(Fe,1),Fe.m(pp.parentNode,pp)):Fe&&(Vr(),k(Fe,1,1,()=>{Fe=null}),Yr());let kp=Yt;Yt=Ti(e),Yt!==kp&&(Vr(),k(vo[kp],1,1,()=>{vo[kp]=null}),Yr(),Vt=vo[Yt],Vt||(Vt=vo[Yt]=zi[Yt](e),Vt.c()),j(Vt,1),Vt.m(ip.parentNode,ip));let Ep=Jt;Jt=Ai(e),Jt!==Ep&&(Vr(),k(wo[Ep],1,1,()=>{wo[Ep]=null}),Yr(),Qt=wo[Jt],Qt||(Qt=wo[Jt]=Ii[Jt](e),Qt.c()),j(Qt,1),Qt.m(mp.parentNode,mp)),e[0]==="pt"?Me?c&1&&j(Me,1):(Me=kf(e),Me.c(),j(Me,1),Me.m(dp.parentNode,dp)):Me&&(Vr(),k(Me,1,1,()=>{Me=null}),Yr())},i(e){Vp||(j(d.$$.fragment,e),j(C),j(ae.$$.fragment,e),j(us.$$.fragment,e),j(Ae.$$.fragment,e),j(De.$$.fragment,e),j(oe.$$.fragment,e),j(as.$$.fragment,e),j(bs.$$.fragment,e),j($s.$$.fragment,e),j(qe.$$.fragment,e),j(Ze.$$.fragment,e),j(Rs.$$.fragment,e),j(ns.$$.fragment,e),j(os.$$.fragment,e),j(sl.$$.fragment,e),j(Dl.$$.fragment,e),j(gt.$$.fragment,e),j(Qe.$$.fragment,e),j(Pl.$$.fragment,e),j(Ct.$$.fragment,e),j(zt.$$.fragment,e),j(Al.$$.fragment,e),j(It.$$.fragment,e),j(tt.$$.fragment,e),j(Rt.$$.fragment,e),j(Ht.$$.fragment,e),j(Ns.$$.fragment,e),j(Ea.$$.fragment,e),j(va.$$.fragment,e),j(wa.$$.fragment,e),j(Gt.$$.fragment,e),j(Oa.$$.fragment,e),j(ya.$$.fragment,e),j(_l),j(bl),j(Pa.$$.fragment,e),j(ro.$$.fragment,e),j(no.$$.fragment,e),j(oo.$$.fragment,e),j(Kt),j(Fe),j(Vt),j(io.$$.fragment,e),j(co.$$.fragment,e),j(fo.$$.fragment,e),j(_o.$$.fragment,e),j(uo.$$.fragment,e),j(Qt),j(Me),j(ho.$$.fragment,e),j(bo.$$.fragment,e),Vp=!0)},o(e){k(d.$$.fragment,e),k(C),k(ae.$$.fragment,e),k(us.$$.fragment,e),k(Ae.$$.fragment,e),k(De.$$.fragment,e),k(oe.$$.fragment,e),k(as.$$.fragment,e),k(bs.$$.fragment,e),k($s.$$.fragment,e),k(qe.$$.fragment,e),k(Ze.$$.fragment,e),k(Rs.$$.fragment,e),k(ns.$$.fragment,e),k(os.$$.fragment,e),k(sl.$$.fragment,e),k(Dl.$$.fragment,e),k(gt.$$.fragment,e),k(Qe.$$.fragment,e),k(Pl.$$.fragment,e),k(Ct.$$.fragment,e),k(zt.$$.fragment,e),k(Al.$$.fragment,e),k(It.$$.fragment,e),k(tt.$$.fragment,e),k(Rt.$$.fragment,e),k(Ht.$$.fragment,e),k(Ns.$$.fragment,e),k(Ea.$$.fragment,e),k(va.$$.fragment,e),k(wa.$$.fragment,e),k(Gt.$$.fragment,e),k(Oa.$$.fragment,e),k(ya.$$.fragment,e),k(_l),k(bl),k(Pa.$$.fragment,e),k(ro.$$.fragment,e),k(no.$$.fragment,e),k(oo.$$.fragment,e),k(Kt),k(Fe),k(Vt),k(io.$$.fragment,e),k(co.$$.fragment,e),k(fo.$$.fragment,e),k(_o.$$.fragment,e),k(uo.$$.fragment,e),k(Qt),k(Me),k(ho.$$.fragment,e),k(bo.$$.fragment,e),Vp=!1},d(e){s(u),e&&s($),O(d,e),e&&s(b),e&&s(D),e&&s(v),$o[q].d(e),e&&s(z),e&&s(B),e&&s(R),e&&s(U),e&&s(fe),O(ae,e),e&&s(ge),e&&s(re),e&&s(ue),e&&s(ke),e&&s(xe),e&&s(Le),e&&s(jl),e&&s(Oe),e&&s(_s),e&&s(Ge),e&&s(de),e&&s(ye),e&&s(Gl),O(us,e),e&&s(qs),e&&s(Ee),O(Ae),e&&s(ms),e&&s(ds),e&&s(Ps),O(De,e),e&&s(Ul),e&&s(me),e&&s(Ts),O(oe,e),e&&s(Xs),O(as,e),e&&s(Xl),e&&s(ve),e&&s(Ks),e&&s(Ys),e&&s(We),O(bs,e),e&&s(rs),O($s,e),e&&s(Xe),e&&s(Is),e&&s(wl),O(qe,e),e&&s(js),O(Ze,e),e&&s(gl),e&&s(Pe),e&&s(As),O(Rs,e),e&&s(Ss),O(ns,e),e&&s(Qs),e&&s(pe),e&&s(xl),O(os,e),e&&s(Ke),O(sl,e),e&&s(Ol),e&&s(we),e&&s(ta),e&&s(Ye),e&&s(kr),e&&s(Ql),e&&s(Er),O(Dl,e),e&&s(vr),O(gt,e),e&&s(Na),e&&s(Je),e&&s(Ga),O(Qe,e),e&&s(Or),e&&s(xs),e&&s(Cr),O(Pl,e),e&&s(Ms),e&&s(yt),e&&s(Dr),O(Ct,e),e&&s(fa),e&&s(pl),e&&s(Ls),e&&s(il),e&&s(zr),O(zt,e),e&&s(_a),e&&s(_e),e&&s(ua),e&&s(Os),e&&s(Tr),O(Al,e),e&&s(Ir),O(It,e),e&&s(sr),e&&s(ps),e&&s(lt),O(tt,e),e&&s(ie),O(Rt,e),e&&s(da),e&&s(ze),e&&s(ha),e&&s(at),e&&s(ba),O(Ht,e),e&&s(Sr),O(Ns,e),e&&s($a),e&&s(es),e&&s(Wn),O(Ea,e),e&&s(tr),e&&s(ar),e&&s(Xn),O(va,e),e&&s(rr),O(wa,e),e&&s(Zn),e&&s(Hs),e&&s(nr),O(Gt,e),e&&s(or),e&&s(is),e&&s(Yn),O(Oa,e),e&&s(Vn),e&&s(Rl),e&&s(Jn),e&&s(Ut),e&&s(Wt),O(ya,e),e&&s(Qn),e&&s(cs),e&&s(Xt),jo[fl].d(e),e&&s(ir),e&&s(ul),e&&s(eo),e&&s(ml),e&&s(so),e&&s(Bl),e&&s(lo),ko[hl].d(e),e&&s(cr),e&&s(fr),e&&s(to),O(Pa,e),e&&s(ao),O(ro,e),e&&s(Tp),e&&s(np),e&&s(Ip),O(no,e),e&&s(Ap),O(oo,e),e&&s(Rp),Eo[Zt].d(e),e&&s(op),Fe&&Fe.d(e),e&&s(pp),e&&s(po),e&&s(Sp),vo[Yt].d(e),e&&s(ip),O(io,e),e&&s(Bp),e&&s(cp),e&&s(Fp),e&&s(fp),e&&s(Mp),O(co,e),e&&s(Lp),O(fo,e),e&&s(Np),e&&s(_p),e&&s(Hp),O(_o,e),e&&s(Gp),e&&s(up),e&&s(Up),O(uo,e),e&&s(Wp),wo[Jt].d(e),e&&s(mp),Me&&Me.d(e),e&&s(dp),e&&s(mo),e&&s(Xp),e&&s(Zr),e&&s(Zp),O(ho,e),e&&s(Kp),O(bo,e),e&&s(Yp),e&&s(hp)}}}const Kf={local:"",sections:[{local:"",sections:[{local:"the-conll2003-dataset",title:"The CoNLL-2003 dataset"},{local:"",title:"\u30C7\u30FC\u30BF\u306E\u51E6\u7406"}],title:"\u30C7\u30FC\u30BF\u306E\u6E96\u5099"},{local:"trainer-api",title:"Trainer API \u3067\u30E2\u30C7\u30EB\u3092\u5FAE\u8ABF\u6574\u3059\u308B"},{local:"keras",sections:[{local:"",title:"\u30C7\u30FC\u30BF\u7167\u5408"},{local:"",title:"\u30E2\u30C7\u30EB\u306E\u5B9A\u7FA9"},{local:"",title:"\u30E2\u30C7\u30EB\u306E\u5FAE\u8ABF\u6574"},{local:"",title:"\u6307\u6A19"},{local:"",title:"\u30E2\u30C7\u30EB\u3092\u5B9A\u7FA9"},{local:"",title:"\u30E2\u30C7\u30EB\u306E\u5FAE\u8ABF\u6574"}],title:"Keras \u3092\u4F7F\u3063\u3066\u30E2\u30C7\u30EB\u3092\u5FAE\u8ABF\u6574\u3059\u308B"},{local:"",sections:[{local:"",title:"\u30C8\u30EC\u30FC\u30CB\u30F3\u30B0\u306E\u305F\u3081\u306E\u6E96\u5099"},{local:"",title:"\u5B66\u7FD2\u30EB\u30FC\u30D7"}],title:"\u30AB\u30B9\u30BF\u30E0\u30C8\u30EC\u30FC\u30CB\u30F3\u30B0\u30EB\u30FC\u30D7"},{local:"",title:"\u5FAE\u8ABF\u6574\u3057\u305F\u30E2\u30C7\u30EB\u3092\u4F7F\u3046"}],title:"\u30C8\u30FC\u30AF\u30F3\u5206\u985E"};function Yf(H,u,$){let d="pt";return Of(()=>{const b=new URLSearchParams(window.location.search);$(0,d=b.get("fw")||"pt")}),[d]}class a_ extends vf{constructor(u){super();wf(this,u,Yf,Zf,gf,{})}}export{a_ as default,Kf as metadata};
