import{S as Mn,i as Qn,s as Rn,e as s,k as u,w as b,t as c,M as Vn,c as n,d as a,m,a as r,x as w,h as d,b as h,G as e,g as $,y as q,q as y,o as E,B as k,v as Bn,L as Ce}from"../../chunks/vendor-hf-doc-builder.js";import{T as Gs}from"../../chunks/Tip-hf-doc-builder.js";import{D as N}from"../../chunks/Docstring-hf-doc-builder.js";import{C as Pe}from"../../chunks/CodeBlock-hf-doc-builder.js";import{I as lt}from"../../chunks/IconCopyLink-hf-doc-builder.js";import{E as Te}from"../../chunks/ExampleCodeBlock-hf-doc-builder.js";function Gn(x){let l,v,i,p,_;return p=new Pe({props:{code:`from evaluate import evaluator
# Sentiment analysis evaluator
evaluator("sentiment-analysis")`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> evaluate <span class="hljs-keyword">import</span> evaluator
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-comment"># Sentiment analysis evaluator</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>evaluator(<span class="hljs-string">&quot;sentiment-analysis&quot;</span>)`}}),{c(){l=s("p"),v=c("Examples:"),i=u(),b(p.$$.fragment)},l(o){l=n(o,"P",{});var f=r(l);v=d(f,"Examples:"),f.forEach(a),i=m(o),w(p.$$.fragment,o)},m(o,f){$(o,l,f),e(l,v),$(o,i,f),q(p,o,f),_=!0},p:Ce,i(o){_||(y(p.$$.fragment,o),_=!0)},o(o){E(p.$$.fragment,o),_=!1},d(o){o&&a(l),o&&a(i),k(p,o)}}}function Hn(x){let l,v,i,p,_;return p=new Pe({props:{code:`from evaluate import evaluator
from datasets import load_dataset
task_evaluator = evaluator("image-classification")
data = load_dataset("beans", split="test[:40]")
results = task_evaluator.compute(
    model_or_pipeline="nateraw/vit-base-beans",
    data=data,
    label_column="labels",
    metric="accuracy",
    label_mapping={'angular_leaf_spot': 0, 'bean_rust': 1, 'healthy': 2},
    strategy="bootstrap"
)`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> evaluate <span class="hljs-keyword">import</span> evaluator
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> datasets <span class="hljs-keyword">import</span> load_dataset
<span class="hljs-meta">&gt;&gt;&gt; </span>task_evaluator = evaluator(<span class="hljs-string">&quot;image-classification&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>data = load_dataset(<span class="hljs-string">&quot;beans&quot;</span>, split=<span class="hljs-string">&quot;test[:40]&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>results = task_evaluator.compute(
<span class="hljs-meta">&gt;&gt;&gt; </span>    model_or_pipeline=<span class="hljs-string">&quot;nateraw/vit-base-beans&quot;</span>,
<span class="hljs-meta">&gt;&gt;&gt; </span>    data=data,
<span class="hljs-meta">&gt;&gt;&gt; </span>    label_column=<span class="hljs-string">&quot;labels&quot;</span>,
<span class="hljs-meta">&gt;&gt;&gt; </span>    metric=<span class="hljs-string">&quot;accuracy&quot;</span>,
<span class="hljs-meta">&gt;&gt;&gt; </span>    label_mapping={<span class="hljs-string">&#x27;angular_leaf_spot&#x27;</span>: <span class="hljs-number">0</span>, <span class="hljs-string">&#x27;bean_rust&#x27;</span>: <span class="hljs-number">1</span>, <span class="hljs-string">&#x27;healthy&#x27;</span>: <span class="hljs-number">2</span>},
<span class="hljs-meta">&gt;&gt;&gt; </span>    strategy=<span class="hljs-string">&quot;bootstrap&quot;</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>)`}}),{c(){l=s("p"),v=c("Examples:"),i=u(),b(p.$$.fragment)},l(o){l=n(o,"P",{});var f=r(l);v=d(f,"Examples:"),f.forEach(a),i=m(o),w(p.$$.fragment,o)},m(o,f){$(o,l,f),e(l,v),$(o,i,f),q(p,o,f),_=!0},p:Ce,i(o){_||(y(p.$$.fragment,o),_=!0)},o(o){E(p.$$.fragment,o),_=!1},d(o){o&&a(l),o&&a(i),k(p,o)}}}function Yn(x){let l,v,i,p,_;return p=new Pe({props:{code:`from evaluate import evaluator
from datasets import load_dataset
task_evaluator = evaluator("question-answering")
data = load_dataset("squad", split="validation[:2]")
results = task_evaluator.compute(
    model_or_pipeline="sshleifer/tiny-distilbert-base-cased-distilled-squad",
    data=data,
    metric="squad",
)`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> evaluate <span class="hljs-keyword">import</span> evaluator
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> datasets <span class="hljs-keyword">import</span> load_dataset
<span class="hljs-meta">&gt;&gt;&gt; </span>task_evaluator = evaluator(<span class="hljs-string">&quot;question-answering&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>data = load_dataset(<span class="hljs-string">&quot;squad&quot;</span>, split=<span class="hljs-string">&quot;validation[:2]&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>results = task_evaluator.compute(
<span class="hljs-meta">&gt;&gt;&gt; </span>    model_or_pipeline=<span class="hljs-string">&quot;sshleifer/tiny-distilbert-base-cased-distilled-squad&quot;</span>,
<span class="hljs-meta">&gt;&gt;&gt; </span>    data=data,
<span class="hljs-meta">&gt;&gt;&gt; </span>    metric=<span class="hljs-string">&quot;squad&quot;</span>,
<span class="hljs-meta">&gt;&gt;&gt; </span>)`}}),{c(){l=s("p"),v=c("Examples:"),i=u(),b(p.$$.fragment)},l(o){l=n(o,"P",{});var f=r(l);v=d(f,"Examples:"),f.forEach(a),i=m(o),w(p.$$.fragment,o)},m(o,f){$(o,l,f),e(l,v),$(o,i,f),q(p,o,f),_=!0},p:Ce,i(o){_||(y(p.$$.fragment,o),_=!0)},o(o){E(p.$$.fragment,o),_=!1},d(o){o&&a(l),o&&a(i),k(p,o)}}}function Wn(x){let l,v,i,p,_;return{c(){l=s("p"),v=c("Datasets where the answer may be missing in the context are supported, for example SQuAD v2 dataset. In this case, it is safer to pass "),i=s("code"),p=c("squad_v2_format=True"),_=c(` to
the compute() call.`)},l(o){l=n(o,"P",{});var f=r(l);v=d(f,"Datasets where the answer may be missing in the context are supported, for example SQuAD v2 dataset. In this case, it is safer to pass "),i=n(f,"CODE",{});var F=r(i);p=d(F,"squad_v2_format=True"),F.forEach(a),_=d(f,` to
the compute() call.`),f.forEach(a)},m(o,f){$(o,l,f),e(l,v),e(l,i),e(i,p),e(l,_)},d(o){o&&a(l)}}}function Jn(x){let l,v;return l=new Pe({props:{code:`from evaluate import evaluator
from datasets import load_dataset
task_evaluator = evaluator("question-answering")
data = load_dataset("squad_v2", split="validation[:2]")
results = task_evaluator.compute(
    model_or_pipeline="mrm8488/bert-tiny-finetuned-squadv2",
    data=data,
    metric="squad_v2",
    squad_v2_format=True,
)`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> evaluate <span class="hljs-keyword">import</span> evaluator
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> datasets <span class="hljs-keyword">import</span> load_dataset
<span class="hljs-meta">&gt;&gt;&gt; </span>task_evaluator = evaluator(<span class="hljs-string">&quot;question-answering&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>data = load_dataset(<span class="hljs-string">&quot;squad_v2&quot;</span>, split=<span class="hljs-string">&quot;validation[:2]&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>results = task_evaluator.compute(
<span class="hljs-meta">&gt;&gt;&gt; </span>    model_or_pipeline=<span class="hljs-string">&quot;mrm8488/bert-tiny-finetuned-squadv2&quot;</span>,
<span class="hljs-meta">&gt;&gt;&gt; </span>    data=data,
<span class="hljs-meta">&gt;&gt;&gt; </span>    metric=<span class="hljs-string">&quot;squad_v2&quot;</span>,
<span class="hljs-meta">&gt;&gt;&gt; </span>    squad_v2_format=<span class="hljs-literal">True</span>,
<span class="hljs-meta">&gt;&gt;&gt; </span>)`}}),{c(){b(l.$$.fragment)},l(i){w(l.$$.fragment,i)},m(i,p){q(l,i,p),v=!0},p:Ce,i(i){v||(y(l.$$.fragment,i),v=!0)},o(i){E(l.$$.fragment,i),v=!1},d(i){k(l,i)}}}function Kn(x){let l,v,i,p,_;return p=new Pe({props:{code:`from evaluate import evaluator
from datasets import load_dataset
task_evaluator = evaluator("text-classification")
data = load_dataset("imdb", split="test[:2]")
results = task_evaluator.compute(
    model_or_pipeline="huggingface/prunebert-base-uncased-6-finepruned-w-distil-mnli",
    data=data,
    metric="accuracy",
    label_mapping={"LABEL_0": 0.0, "LABEL_1": 1.0},
    strategy="bootstrap",
    n_resamples=10,
    random_state=0
)`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> evaluate <span class="hljs-keyword">import</span> evaluator
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> datasets <span class="hljs-keyword">import</span> load_dataset
<span class="hljs-meta">&gt;&gt;&gt; </span>task_evaluator = evaluator(<span class="hljs-string">&quot;text-classification&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>data = load_dataset(<span class="hljs-string">&quot;imdb&quot;</span>, split=<span class="hljs-string">&quot;test[:2]&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>results = task_evaluator.compute(
<span class="hljs-meta">&gt;&gt;&gt; </span>    model_or_pipeline=<span class="hljs-string">&quot;huggingface/prunebert-base-uncased-6-finepruned-w-distil-mnli&quot;</span>,
<span class="hljs-meta">&gt;&gt;&gt; </span>    data=data,
<span class="hljs-meta">&gt;&gt;&gt; </span>    metric=<span class="hljs-string">&quot;accuracy&quot;</span>,
<span class="hljs-meta">&gt;&gt;&gt; </span>    label_mapping={<span class="hljs-string">&quot;LABEL_0&quot;</span>: <span class="hljs-number">0.0</span>, <span class="hljs-string">&quot;LABEL_1&quot;</span>: <span class="hljs-number">1.0</span>},
<span class="hljs-meta">&gt;&gt;&gt; </span>    strategy=<span class="hljs-string">&quot;bootstrap&quot;</span>,
<span class="hljs-meta">&gt;&gt;&gt; </span>    n_resamples=<span class="hljs-number">10</span>,
<span class="hljs-meta">&gt;&gt;&gt; </span>    random_state=<span class="hljs-number">0</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>)`}}),{c(){l=s("p"),v=c("Examples:"),i=u(),b(p.$$.fragment)},l(o){l=n(o,"P",{});var f=r(l);v=d(f,"Examples:"),f.forEach(a),i=m(o),w(p.$$.fragment,o)},m(o,f){$(o,l,f),e(l,v),$(o,i,f),q(p,o,f),_=!0},p:Ce,i(o){_||(y(p.$$.fragment,o),_=!0)},o(o){E(p.$$.fragment,o),_=!1},d(o){o&&a(l),o&&a(i),k(p,o)}}}function Xn(x){let l,v,i,p,_;return p=new Pe({props:{code:`from evaluate import evaluator
from datasets import load_dataset
task_evaluator = evaluator("token-classification")
data = load_dataset("conll2003", split="validation[:2]")
results = task_evaluator.compute(
    model_or_pipeline="elastic/distilbert-base-uncased-finetuned-conll03-english",
    data=data,
    metric="seqeval",
)`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> evaluate <span class="hljs-keyword">import</span> evaluator
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> datasets <span class="hljs-keyword">import</span> load_dataset
<span class="hljs-meta">&gt;&gt;&gt; </span>task_evaluator = evaluator(<span class="hljs-string">&quot;token-classification&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>data = load_dataset(<span class="hljs-string">&quot;conll2003&quot;</span>, split=<span class="hljs-string">&quot;validation[:2]&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>results = task_evaluator.compute(
<span class="hljs-meta">&gt;&gt;&gt; </span>    model_or_pipeline=<span class="hljs-string">&quot;elastic/distilbert-base-uncased-finetuned-conll03-english&quot;</span>,
<span class="hljs-meta">&gt;&gt;&gt; </span>    data=data,
<span class="hljs-meta">&gt;&gt;&gt; </span>    metric=<span class="hljs-string">&quot;seqeval&quot;</span>,
<span class="hljs-meta">&gt;&gt;&gt; </span>)`}}),{c(){l=s("p"),v=c("Examples:"),i=u(),b(p.$$.fragment)},l(o){l=n(o,"P",{});var f=r(l);v=d(f,"Examples:"),f.forEach(a),i=m(o),w(p.$$.fragment,o)},m(o,f){$(o,l,f),e(l,v),$(o,i,f),q(p,o,f),_=!0},p:Ce,i(o){_||(y(p.$$.fragment,o),_=!0)},o(o){E(p.$$.fragment,o),_=!1},d(o){o&&a(l),o&&a(i),k(p,o)}}}function Zn(x){let l,v,i,p,_;return p=new Pe({props:{code:`dataset = Dataset.from_dict(
    mapping={
        "tokens": [["New", "York", "is", "a", "city", "and", "Felix", "a", "person", "."]],
        "ner_tags": [[1, 2, 0, 0, 0, 0, 3, 0, 0, 0]],
    },
    features=Features({
        "tokens": Sequence(feature=Value(dtype="string")),
        "ner_tags": Sequence(feature=ClassLabel(names=["O", "B-LOC", "I-LOC", "B-PER", "I-PER"])),
        }),
)`,highlighted:`dataset = Dataset.from_dict(
    mapping={
        <span class="hljs-string">&quot;tokens&quot;</span>: [[<span class="hljs-string">&quot;New&quot;</span>, <span class="hljs-string">&quot;York&quot;</span>, <span class="hljs-string">&quot;is&quot;</span>, <span class="hljs-string">&quot;a&quot;</span>, <span class="hljs-string">&quot;city&quot;</span>, <span class="hljs-string">&quot;and&quot;</span>, <span class="hljs-string">&quot;Felix&quot;</span>, <span class="hljs-string">&quot;a&quot;</span>, <span class="hljs-string">&quot;person&quot;</span>, <span class="hljs-string">&quot;.&quot;</span>]],
        <span class="hljs-string">&quot;ner_tags&quot;</span>: [[<span class="hljs-number">1</span>, <span class="hljs-number">2</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">3</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>]],
    },
    features=Features({
        <span class="hljs-string">&quot;tokens&quot;</span>: <span class="hljs-type">Sequence</span>(feature=Value(dtype=<span class="hljs-string">&quot;string&quot;</span>)),
        <span class="hljs-string">&quot;ner_tags&quot;</span>: <span class="hljs-type">Sequence</span>(feature=ClassLabel(names=[<span class="hljs-string">&quot;O&quot;</span>, <span class="hljs-string">&quot;B-LOC&quot;</span>, <span class="hljs-string">&quot;I-LOC&quot;</span>, <span class="hljs-string">&quot;B-PER&quot;</span>, <span class="hljs-string">&quot;I-PER&quot;</span>])),
        }),
)`}}),{c(){l=s("p"),v=c("For example, the following dataset format is accepted by the evaluator:"),i=u(),b(p.$$.fragment)},l(o){l=n(o,"P",{});var f=r(l);v=d(f,"For example, the following dataset format is accepted by the evaluator:"),f.forEach(a),i=m(o),w(p.$$.fragment,o)},m(o,f){$(o,l,f),e(l,v),$(o,i,f),q(p,o,f),_=!0},p:Ce,i(o){_||(y(p.$$.fragment,o),_=!0)},o(o){E(p.$$.fragment,o),_=!1},d(o){o&&a(l),o&&a(i),k(p,o)}}}function er(x){let l,v;return l=new Te({props:{anchor:"evaluate.TokenClassificationEvaluator.compute.example-2",$$slots:{default:[Zn]},$$scope:{ctx:x}}}),{c(){b(l.$$.fragment)},l(i){w(l.$$.fragment,i)},m(i,p){q(l,i,p),v=!0},p(i,p){const _={};p&2&&(_.$$scope={dirty:p,ctx:i}),l.$set(_)},i(i){v||(y(l.$$.fragment,i),v=!0)},o(i){E(l.$$.fragment,i),v=!1},d(i){k(l,i)}}}function tr(x){let l,v,i,p,_,o,f,F;return f=new Pe({props:{code:`dataset = Dataset.from_dict(
    mapping={
        "tokens": [["New York is a city and Felix a person."]],
        "starts": [[0, 23]],
        "ends": [[7, 27]],
        "ner_tags": [["LOC", "PER"]],
    },
    features=Features({
        "tokens": Value(dtype="string"),
        "starts": Sequence(feature=Value(dtype="int32")),
        "ends": Sequence(feature=Value(dtype="int32")),
        "ner_tags": Sequence(feature=Value(dtype="string")),
    }),
)`,highlighted:`dataset = Dataset.from_dict(
    mapping={
        <span class="hljs-string">&quot;tokens&quot;</span>: [[<span class="hljs-string">&quot;New York is a city and Felix a person.&quot;</span>]],
        <span class="hljs-string">&quot;starts&quot;</span>: [[<span class="hljs-number">0</span>, <span class="hljs-number">23</span>]],
        <span class="hljs-string">&quot;ends&quot;</span>: [[<span class="hljs-number">7</span>, <span class="hljs-number">27</span>]],
        <span class="hljs-string">&quot;ner_tags&quot;</span>: [[<span class="hljs-string">&quot;LOC&quot;</span>, <span class="hljs-string">&quot;PER&quot;</span>]],
    },
    features=Features({
        <span class="hljs-string">&quot;tokens&quot;</span>: Value(dtype=<span class="hljs-string">&quot;string&quot;</span>),
        <span class="hljs-string">&quot;starts&quot;</span>: <span class="hljs-type">Sequence</span>(feature=Value(dtype=<span class="hljs-string">&quot;int32&quot;</span>)),
        <span class="hljs-string">&quot;ends&quot;</span>: <span class="hljs-type">Sequence</span>(feature=Value(dtype=<span class="hljs-string">&quot;int32&quot;</span>)),
        <span class="hljs-string">&quot;ner_tags&quot;</span>: <span class="hljs-type">Sequence</span>(feature=Value(dtype=<span class="hljs-string">&quot;string&quot;</span>)),
    }),
)`}}),{c(){l=s("p"),v=c("For example, the following dataset format is "),i=s("strong"),p=c("not"),_=c(" accepted by the evaluator:"),o=u(),b(f.$$.fragment)},l(j){l=n(j,"P",{});var S=r(l);v=d(S,"For example, the following dataset format is "),i=n(S,"STRONG",{});var J=r(i);p=d(J,"not"),J.forEach(a),_=d(S," accepted by the evaluator:"),S.forEach(a),o=m(j),w(f.$$.fragment,j)},m(j,S){$(j,l,S),e(l,v),e(l,i),e(i,p),e(l,_),$(j,o,S),q(f,j,S),F=!0},p:Ce,i(j){F||(y(f.$$.fragment,j),F=!0)},o(j){E(f.$$.fragment,j),F=!1},d(j){j&&a(l),j&&a(o),k(f,j)}}}function ar(x){let l,v;return l=new Te({props:{anchor:"evaluate.TokenClassificationEvaluator.compute.example-3",$$slots:{default:[tr]},$$scope:{ctx:x}}}),{c(){b(l.$$.fragment)},l(i){w(l.$$.fragment,i)},m(i,p){q(l,i,p),v=!0},p(i,p){const _={};p&2&&(_.$$scope={dirty:p,ctx:i}),l.$set(_)},i(i){v||(y(l.$$.fragment,i),v=!0)},o(i){E(l.$$.fragment,i),v=!1},d(i){k(l,i)}}}function or(x){let l,v,i,p,_,o,f,F,j,S,J,La,da,K,ne,qt,De,Ma,yt,Qa,pa,it,Ra,ua,U,Ne,Va,L,Ba,ct,Ga,Ha,Et,Ya,Wa,kt,Ja,Ka,Xa,re,ma,dt,Za,fa,T,Ie,eo,xt,to,ao,le,Ae,oo,jt,so,no,ie,Fe,ro,Se,lo,Tt,io,co,po,ce,Oe,uo,Ct,mo,fo,de,ze,ho,Pt,go,vo,pe,Ue,_o,Dt,$o,ha,X,ue,Nt,Le,bo,It,wo,ga,Z,me,At,Me,qo,Ft,yo,va,M,Qe,Eo,Q,ko,pt,xo,jo,St,To,Co,Ot,Po,Do,No,V,Re,Io,zt,Ao,Fo,fe,_a,ee,he,Ut,Ve,So,Lt,Oo,$a,P,Be,zo,Ge,Uo,ge,Mt,Lo,Mo,Qo,Ro,te,Vo,ut,Bo,Go,Qt,Ho,Yo,Wo,He,Jo,Ye,Rt,Ko,Xo,Zo,I,We,es,Vt,ts,as,ve,os,_e,ss,$e,ba,ae,be,Bt,Je,ns,Gt,rs,wa,R,Ke,ls,O,is,mt,cs,ds,Ht,ps,us,Yt,ms,fs,Wt,hs,gs,vs,B,Xe,_s,Jt,$s,bs,we,qa,oe,qe,Kt,Ze,ws,Xt,qs,ya,D,et,ys,Zt,Es,ks,se,xs,ft,js,Ts,ea,Cs,Ps,Ds,tt,Ns,ta,Is,As,Fs,C,at,Ss,aa,Os,zs,ot,Us,st,Ls,Ms,Qs,ye,Rs,Ee,Vs,ke,Ea;return o=new lt({}),De=new lt({}),Ne=new N({props:{name:"evaluate.evaluator",anchor:"evaluate.evaluator",parameters:[{name:"task",val:": str = None"}],parametersDescription:[{anchor:"evaluate.evaluator.task",description:`<strong>task</strong> (<code>str</code>) &#x2014;
The task defining which evaluator will be returned. Currently accepted tasks are:<ul>
<li><code>&quot;image-classification&quot;</code>: will return a <a href="/docs/evaluate/pr_199/en/package_reference/evaluator_classes#evaluate.ImageClassificationEvaluator">ImageClassificationEvaluator</a>.</li>
<li><code>&quot;question-answering&quot;</code>: will return a <a href="/docs/evaluate/pr_199/en/package_reference/evaluator_classes#evaluate.QuestionAnsweringEvaluator">QuestionAnsweringEvaluator</a>.</li>
<li><code>&quot;text-classification&quot;</code> (alias <code>&quot;sentiment-analysis&quot;</code> available): will return a <a href="/docs/evaluate/pr_199/en/package_reference/evaluator_classes#evaluate.TextClassificationEvaluator">TextClassificationEvaluator</a>.</li>
<li><code>&quot;token-classification&quot;</code>: will return a <a href="/docs/evaluate/pr_199/en/package_reference/evaluator_classes#evaluate.TokenClassificationEvaluator">TokenClassificationEvaluator</a>.</li>
</ul>`,name:"task"}],source:"https://github.com/huggingface/evaluate/blob/vr_199/src/evaluate/evaluator/__init__.py#L85",returnDescription:`
<p>An evaluator suitable for the task.</p>
`,returnType:`
<p><a
  href="/docs/evaluate/pr_199/en/package_reference/evaluator_classes#evaluate.Evaluator"
>Evaluator</a></p>
`}}),re=new Te({props:{anchor:"evaluate.evaluator.example",$$slots:{default:[Gn]},$$scope:{ctx:x}}}),Ie=new N({props:{name:"class evaluate.Evaluator",anchor:"evaluate.Evaluator",parameters:[{name:"task",val:": str"},{name:"default_metric_name",val:": str = None"}],source:"https://github.com/huggingface/evaluate/blob/vr_199/src/evaluate/evaluator/base.py#L51"}}),Ae=new N({props:{name:"compute_metric",anchor:"evaluate.Evaluator.compute_metric",parameters:[{name:"metric",val:": EvaluationModule"},{name:"metric_inputs",val:": typing.Dict"},{name:"strategy",val:": typing.Literal['simple', 'bootstrap'] = 'simple'"},{name:"confidence_level",val:": float = 0.95"},{name:"n_resamples",val:": int = 9999"},{name:"random_state",val:": typing.Optional[int] = None"}],source:"https://github.com/huggingface/evaluate/blob/vr_199/src/evaluate/evaluator/base.py#L333"}}),Fe=new N({props:{name:"predictions_processor",anchor:"evaluate.Evaluator.predictions_processor",parameters:[{name:"*args",val:""},{name:"**kwargs",val:""}],source:"https://github.com/huggingface/evaluate/blob/vr_199/src/evaluate/evaluator/base.py#L160"}}),Oe=new N({props:{name:"prepare_data",anchor:"evaluate.Evaluator.prepare_data",parameters:[{name:"data",val:": typing.Union[str, datasets.arrow_dataset.Dataset]"},{name:"input_column",val:": str"},{name:"label_column",val:": str"}],parametersDescription:[{anchor:"evaluate.Evaluator.prepare_data.data",description:"<strong>data</strong> (<code>str</code> or <code>Dataset</code>, defaults to <code>None) -- Specifies the dataset we will run evaluation on. If it is of type </code>str`, we treat it as the dataset\nname, and load it. Otherwise we assume it represents a pre-loaded dataset.",name:"data"},{anchor:"evaluate.Evaluator.prepare_data.input_column",description:`<strong>input_column</strong> (<code>str</code>, defaults to <code>&quot;text&quot;</code>) &#x2014;
the name of the column containing the text feature in the dataset specified by <code>data</code>.`,name:"input_column"},{anchor:"evaluate.Evaluator.prepare_data.label_column",description:`<strong>label_column</strong> (<code>str</code>, defaults to <code>&quot;label&quot;</code>) &#x2014;
the name of the column containing the labels in the dataset specified by <code>data</code>.`,name:"label_column"}],source:"https://github.com/huggingface/evaluate/blob/vr_199/src/evaluate/evaluator/base.py#L219",returnDescription:`
<p>metric inputs.
<code>list</code>:  pipeline inputs.</p>
`,returnType:`
<p><code>dict</code></p>
`}}),ze=new N({props:{name:"prepare_metric",anchor:"evaluate.Evaluator.prepare_metric",parameters:[{name:"metric",val:": typing.Union[str, evaluate.module.EvaluationModule]"}],parametersDescription:[{anchor:"evaluate.Evaluator.prepare_metric.metric",description:`<strong>metric</strong> (<code>str</code> or <code>EvaluationModule</code>, defaults to <code>None</code>) &#x2014;
Specifies the metric we use in evaluator. If it is of type <code>str</code>, we treat it as the metric name, and
load it. Otherwise we assume it represents a pre-loaded metric.`,name:"metric"}],source:"https://github.com/huggingface/evaluate/blob/vr_199/src/evaluate/evaluator/base.py#L303",returnDescription:`
<p>The loaded metric.</p>
`}}),Ue=new N({props:{name:"prepare_pipeline",anchor:"evaluate.Evaluator.prepare_pipeline",parameters:[{name:"model_or_pipeline",val:": typing.Union[str, ForwardRef('Pipeline'), typing.Callable, ForwardRef('PreTrainedModel'), ForwardRef('TFPreTrainedModel')]"},{name:"tokenizer",val:": typing.Union[ForwardRef('PreTrainedTokenizerBase'), ForwardRef('FeatureExtractionMixin')] = None"},{name:"feature_extractor",val:": typing.Union[ForwardRef('PreTrainedTokenizerBase'), ForwardRef('FeatureExtractionMixin')] = None"},{name:"device",val:": int = None"}],parametersDescription:[{anchor:"evaluate.Evaluator.prepare_pipeline.model_or_pipeline",description:"<strong>model_or_pipeline</strong> (<code>str</code> or <code>Pipeline</code> or <code>Callable</code> or <code>PreTrainedModel</code> or <code>TFPreTrainedModel</code>, &#x2014;",name:"model_or_pipeline"},{anchor:"evaluate.Evaluator.prepare_pipeline.defaults",description:`<strong>defaults</strong> to <code>None</code>) &#x2014;
If the argument in not specified, we initialize the default pipeline for the task. If the argument is of the type <code>str</code> or
is a model instance, we use it to initialize a new <code>Pipeline</code> with the given model. Otherwise we assume the
argument specifies a pre-initialized pipeline.`,name:"defaults"},{anchor:"evaluate.Evaluator.prepare_pipeline.preprocessor",description:`<strong>preprocessor</strong> (<code>PreTrainedTokenizerBase</code> or <code>FeatureExtractionMixin</code>, <em>optional</em>, defaults to <code>None</code>) &#x2014;
Argument can be used to overwrite a default preprocessor if <code>model_or_pipeline</code> represents a model for
which we build a pipeline. If <code>model_or_pipeline</code> is <code>None</code> or a pre-initialized pipeline, we ignore
this argument.`,name:"preprocessor"}],source:"https://github.com/huggingface/evaluate/blob/vr_199/src/evaluate/evaluator/base.py#L251",returnDescription:`
<p>The initialized pipeline.</p>
`}}),Le=new lt({}),Me=new lt({}),Qe=new N({props:{name:"class evaluate.ImageClassificationEvaluator",anchor:"evaluate.ImageClassificationEvaluator",parameters:[{name:"task",val:" = 'image-classification'"},{name:"default_metric_name",val:" = None"}],source:"https://github.com/huggingface/evaluate/blob/vr_199/src/evaluate/evaluator/image_classification.py#L20"}}),Re=new N({props:{name:"compute",anchor:"evaluate.ImageClassificationEvaluator.compute",parameters:[{name:"input_column",val:": str = 'image'"},{name:"*args",val:""},{name:"**kwargs",val:""}],parametersDescription:[{anchor:"evaluate.ImageClassificationEvaluator.compute.model_or_pipeline",description:`<strong>model_or_pipeline</strong> (<code>str</code> or <code>Pipeline</code> or <code>Callable</code> or <code>PreTrainedModel</code> or <code>TFPreTrainedModel</code>, defaults to <code>None</code>) &#x2014;
If the argument in not specified, we initialize the default pipeline for the task (in this case
<code>image-classification</code>. If the argument is of the type <code>str</code> or
is a model instance, we use it to initialize a new <code>Pipeline</code> with the given model. Otherwise we assume the
argument specifies a pre-initialized pipeline.`,name:"model_or_pipeline"},{anchor:"evaluate.ImageClassificationEvaluator.compute.data",description:`<strong>data</strong> (<code>str</code> or <code>Dataset</code>, defaults to <code>None</code>) &#x2014;
Specifies the dataset we will run evaluation on. If it is of type <code>str</code>, we treat it as the dataset
name, and load it. Otherwise we assume it represents a pre-loaded dataset.`,name:"data"},{anchor:"evaluate.ImageClassificationEvaluator.compute.metric",description:`<strong>metric</strong> (<code>str</code> or <code>EvaluationModule</code>, defaults to <code>None</code>) &#x2014;
Specifies the metric we use in evaluator. If it is of type <code>str</code>, we treat it as the metric name, and
load it. Otherwise we assume it represents a pre-loaded metric.`,name:"metric"},{anchor:"evaluate.ImageClassificationEvaluator.compute.feature_extractor",description:`<strong>feature_extractor</strong> (<code>str</code> or <code>FeatureExtractionMixin</code>, <em>optional</em>, defaults to <code>None</code>) &#x2014;
Argument can be used to overwrite a default feature extractor if <code>model_or_pipeline</code> represents a model for
which we build a pipeline. If <code>model_or_pipeline</code> is <code>None</code> or a pre-initialized pipeline, we ignore
this argument.`,name:"feature_extractor"},{anchor:"evaluate.ImageClassificationEvaluator.compute.strategy",description:`<strong>strategy</strong> (<code>Literal[&quot;simple&quot;, &quot;bootstrap&quot;]</code>, defaults to &#x201C;simple&#x201D;) &#x2014;
specifies the evaluation strategy. Possible values are:<ul>
<li><code>&quot;simple&quot;</code> - we evaluate the metric and return the scores.</li>
<li><code>&quot;bootstrap&quot;</code> - on top of computing the metric scores, we calculate the confidence interval for each
of the returned metric keys, using <code>scipy</code>&#x2019;s <code>bootstrap</code> method
<a href="https://docs.scipy.org/doc/scipy/reference/generated/scipy.stats.bootstrap.html" rel="nofollow">https://docs.scipy.org/doc/scipy/reference/generated/scipy.stats.bootstrap.html</a>.</li>
</ul>`,name:"strategy"},{anchor:"evaluate.ImageClassificationEvaluator.compute.confidence_level",description:`<strong>confidence_level</strong> (<code>float</code>, defaults to <code>0.95</code>) &#x2014;
The <code>confidence_level</code> value passed to <code>bootstrap</code> if <code>&quot;bootstrap&quot;</code> strategy is chosen.`,name:"confidence_level"},{anchor:"evaluate.ImageClassificationEvaluator.compute.n_resamples",description:`<strong>n_resamples</strong> (<code>int</code>, defaults to <code>9999</code>) &#x2014;
The <code>n_resamples</code> value passed to <code>bootstrap</code> if <code>&quot;bootstrap&quot;</code> strategy is chosen.`,name:"n_resamples"},{anchor:"evaluate.ImageClassificationEvaluator.compute.device",description:`<strong>device</strong> (<code>int</code>, defaults to <code>None</code>) &#x2014;
Device ordinal for CPU/GPU support of the pipeline. Setting this to -1 will leverage CPU, a positive
integer will run the model on the associated CUDA device ID. If<code>None</code> is provided it will be inferred and
CUDA:0 used if available, CPU otherwise.`,name:"device"},{anchor:"evaluate.ImageClassificationEvaluator.compute.random_state",description:`<strong>random_state</strong> (<code>int</code>, <em>optional</em>, defaults to <code>None</code>) &#x2014;
The <code>random_state</code> value passed to <code>bootstrap</code> if <code>&quot;bootstrap&quot;</code> strategy is chosen. Useful for
debugging.`,name:"random_state"},{anchor:"evaluate.ImageClassificationEvaluator.compute.input_column",description:`<strong>input_column</strong> (<code>str</code>, defaults to <code>&quot;image&quot;</code>) &#x2014;
the name of the column containing the images as PIL ImageFile in the dataset specified by <code>data</code>.`,name:"input_column"},{anchor:"evaluate.ImageClassificationEvaluator.compute.label_column",description:`<strong>label_column</strong> (<code>str</code>, defaults to <code>&quot;label&quot;</code>) &#x2014;
the name of the column containing the labels in the dataset specified by <code>data</code>.`,name:"label_column"},{anchor:"evaluate.ImageClassificationEvaluator.compute.label_mapping",description:`<strong>label_mapping</strong> (<code>Dict[str, Number]</code>, <em>optional</em>, defaults to <code>None</code>) &#x2014;
We want to map class labels defined by the model in the pipeline to values consistent with those
defined in the <code>label_column</code> of the <code>data</code> dataset.`,name:"label_mapping"}],source:"https://github.com/huggingface/evaluate/blob/vr_199/src/evaluate/evaluator/image_classification.py#L37",returnDescription:`
<p>A <code>Dict</code>. The keys represent metric keys calculated for the <code>metric</code> spefied in function arguments. For the
<code>"simple"</code> strategy, the value is the metric score. For the <code>"bootstrap"</code> strategy, the value is a <code>Dict</code>
containing the score, the confidence interval and the standard error calculated for each metric key.</p>
`}}),fe=new Te({props:{anchor:"evaluate.ImageClassificationEvaluator.compute.example",$$slots:{default:[Hn]},$$scope:{ctx:x}}}),Ve=new lt({}),Be=new N({props:{name:"class evaluate.QuestionAnsweringEvaluator",anchor:"evaluate.QuestionAnsweringEvaluator",parameters:[{name:"task",val:" = 'question-answering'"},{name:"default_metric_name",val:" = None"}],source:"https://github.com/huggingface/evaluate/blob/vr_199/src/evaluate/evaluator/question_answering.py#L39"}}),We=new N({props:{name:"compute",anchor:"evaluate.QuestionAnsweringEvaluator.compute",parameters:[{name:"model_or_pipeline",val:": typing.Union[str, ForwardRef('Pipeline'), typing.Callable, ForwardRef('PreTrainedModel'), ForwardRef('TFPreTrainedModel')] = None"},{name:"data",val:": typing.Union[str, datasets.arrow_dataset.Dataset] = None"},{name:"metric",val:": typing.Union[str, evaluate.module.EvaluationModule] = None"},{name:"tokenizer",val:": typing.Union[str, ForwardRef('PreTrainedTokenizer'), NoneType] = None"},{name:"strategy",val:": typing.Literal['simple', 'bootstrap'] = 'simple'"},{name:"confidence_level",val:": float = 0.95"},{name:"n_resamples",val:": int = 9999"},{name:"device",val:": int = None"},{name:"random_state",val:": typing.Optional[int] = None"},{name:"question_column",val:": str = 'question'"},{name:"context_column",val:": str = 'context'"},{name:"id_column",val:": str = 'id'"},{name:"label_column",val:": str = 'answers'"},{name:"squad_v2_format",val:": typing.Optional[bool] = None"}],parametersDescription:[{anchor:"evaluate.QuestionAnsweringEvaluator.compute.model_or_pipeline",description:`<strong>model_or_pipeline</strong> (<code>str</code> or <code>Pipeline</code> or <code>Callable</code> or <code>PreTrainedModel</code> or <code>TFPreTrainedModel</code>, defaults to <code>None</code>) &#x2014;
If the argument in not specified, we initialize the default pipeline for the task (in this case
<code>question-answering</code>). If the argument is of the type <code>str</code> or
is a model instance, we use it to initialize a new <code>Pipeline</code> with the given model. Otherwise we assume the
argument specifies a pre-initialized pipeline.`,name:"model_or_pipeline"},{anchor:"evaluate.QuestionAnsweringEvaluator.compute.data",description:`<strong>data</strong> (<code>str</code> or <code>Dataset</code>, defaults to <code>None</code>) &#x2014;
Specifies the dataset we will run evaluation on. If it is of type <code>str</code>, we treat it as the dataset
name, and load it. Otherwise we assume it represents a pre-loaded dataset.`,name:"data"},{anchor:"evaluate.QuestionAnsweringEvaluator.compute.metric",description:`<strong>metric</strong> (<code>str</code> or <code>EvaluationModule</code>, defaults to <code>None</code>) &#x2014;
Specifies the metric we use in evaluator. If it is of type <code>str</code>, we treat it as the metric name, and
load it. Otherwise we assume it represents a pre-loaded metric.`,name:"metric"},{anchor:"evaluate.QuestionAnsweringEvaluator.compute.tokenizer",description:`<strong>tokenizer</strong> (<code>str</code> or <code>PreTrainedTokenizer</code>, <em>optional</em>, defaults to <code>None</code>) &#x2014;
Argument can be used to overwrite a default tokenizer if <code>model_or_pipeline</code> represents a model for
which we build a pipeline. If <code>model_or_pipeline</code> is <code>None</code> or a pre-initialized pipeline, we ignore
this argument.`,name:"tokenizer"},{anchor:"evaluate.QuestionAnsweringEvaluator.compute.strategy",description:`<strong>strategy</strong> (<code>Literal[&quot;simple&quot;, &quot;bootstrap&quot;]</code>, defaults to &#x201C;simple&#x201D;) &#x2014;
specifies the evaluation strategy. Possible values are:<ul>
<li><code>&quot;simple&quot;</code> - we evaluate the metric and return the scores.</li>
<li><code>&quot;bootstrap&quot;</code> - on top of computing the metric scores, we calculate the confidence interval for each
of the returned metric keys, using <code>scipy</code>&#x2019;s <code>bootstrap</code> method
<a href="https://docs.scipy.org/doc/scipy/reference/generated/scipy.stats.bootstrap.html" rel="nofollow">https://docs.scipy.org/doc/scipy/reference/generated/scipy.stats.bootstrap.html</a>.</li>
</ul>`,name:"strategy"},{anchor:"evaluate.QuestionAnsweringEvaluator.compute.confidence_level",description:`<strong>confidence_level</strong> (<code>float</code>, defaults to <code>0.95</code>) &#x2014;
The <code>confidence_level</code> value passed to <code>bootstrap</code> if <code>&quot;bootstrap&quot;</code> strategy is chosen.`,name:"confidence_level"},{anchor:"evaluate.QuestionAnsweringEvaluator.compute.n_resamples",description:`<strong>n_resamples</strong> (<code>int</code>, defaults to <code>9999</code>) &#x2014;
The <code>n_resamples</code> value passed to <code>bootstrap</code> if <code>&quot;bootstrap&quot;</code> strategy is chosen.`,name:"n_resamples"},{anchor:"evaluate.QuestionAnsweringEvaluator.compute.device",description:`<strong>device</strong> (<code>int</code>, defaults to <code>None</code>) &#x2014;
Device ordinal for CPU/GPU support of the pipeline. Setting this to -1 will leverage CPU, a positive
will run the model on the associated CUDA device ID. If<code>None</code> is provided it will be inferred and
CUDA:0 used if available, CPU otherwise.`,name:"device"},{anchor:"evaluate.QuestionAnsweringEvaluator.compute.random_state",description:`<strong>random_state</strong> (<code>int</code>, <em>optional</em>, defaults to <code>None</code>) &#x2014;
The <code>random_state</code> value passed to <code>bootstrap</code> if <code>&quot;bootstrap&quot;</code> strategy is chosen. Useful for
debugging.`,name:"random_state"},{anchor:"evaluate.QuestionAnsweringEvaluator.compute.question_column",description:`<strong>question_column</strong> (<code>str</code>, defaults to <code>&quot;question&quot;</code>) &#x2014;
the name of the column containing the question in the dataset specified by <code>data</code>.`,name:"question_column"},{anchor:"evaluate.QuestionAnsweringEvaluator.compute.context_column",description:`<strong>context_column</strong> (<code>str</code>, defaults to <code>&quot;context&quot;</code>) &#x2014;
the name of the column containing the context in the dataset specified by <code>data</code>.`,name:"context_column"},{anchor:"evaluate.QuestionAnsweringEvaluator.compute.id_column",description:`<strong>id_column</strong> (<code>str</code>, defaults to <code>&quot;id&quot;</code>) &#x2014;
the name of the column cointaing the identification field of the question and answer pair in the
dataset specified by <code>data</code>.`,name:"id_column"},{anchor:"evaluate.QuestionAnsweringEvaluator.compute.label_column",description:`<strong>label_column</strong> (<code>str</code>, defaults to <code>&quot;answers&quot;</code>) &#x2014;
the name of the column containing the answers in the dataset specified by <code>data</code>.`,name:"label_column"},{anchor:"evaluate.QuestionAnsweringEvaluator.compute.squad_v2_format",description:`<strong>squad_v2_format</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>None</code>) &#x2014;
whether the dataset follows the format of squad_v2 dataset where a question may have no answer in the context. If this parameter is not provided,
the format will be automatically inferred.`,name:"squad_v2_format"}],source:"https://github.com/huggingface/evaluate/blob/vr_199/src/evaluate/evaluator/question_answering.py#L121",returnDescription:`
<p>A <code>Dict</code>. The keys represent metric keys calculated for the <code>metric</code> spefied in function arguments. For the
<code>"simple"</code> strategy, the value is the metric score. For the <code>"bootstrap"</code> strategy, the value is a <code>Dict</code>
containing the score, the confidence interval and the standard error calculated for each metric key.</p>
`}}),ve=new Te({props:{anchor:"evaluate.QuestionAnsweringEvaluator.compute.example",$$slots:{default:[Yn]},$$scope:{ctx:x}}}),_e=new Gs({props:{$$slots:{default:[Wn]},$$scope:{ctx:x}}}),$e=new Te({props:{anchor:"evaluate.QuestionAnsweringEvaluator.compute.example-2",$$slots:{default:[Jn]},$$scope:{ctx:x}}}),Je=new lt({}),Ke=new N({props:{name:"class evaluate.TextClassificationEvaluator",anchor:"evaluate.TextClassificationEvaluator",parameters:[{name:"task",val:" = 'text-classification'"},{name:"default_metric_name",val:" = None"}],source:"https://github.com/huggingface/evaluate/blob/vr_199/src/evaluate/evaluator/text_classification.py#L20"}}),Xe=new N({props:{name:"compute",anchor:"evaluate.TextClassificationEvaluator.compute",parameters:[{name:"*args",val:""},{name:"**kwargs",val:""}],parametersDescription:[{anchor:"evaluate.TextClassificationEvaluator.compute.model_or_pipeline",description:`<strong>model_or_pipeline</strong> (<code>str</code> or <code>Pipeline</code> or <code>Callable</code> or <code>PreTrainedModel</code> or <code>TFPreTrainedModel</code>, defaults to <code>None</code>) &#x2014;
If the argument in not specified, we initialize the default pipeline for the task (in this case
<code>text-classification</code> or its alias - <code>sentiment-analysis</code>). If the argument is of the type <code>str</code> or
is a model instance, we use it to initialize a new <code>Pipeline</code> with the given model. Otherwise we assume the
argument specifies a pre-initialized pipeline.`,name:"model_or_pipeline"},{anchor:"evaluate.TextClassificationEvaluator.compute.data",description:`<strong>data</strong> (<code>str</code> or <code>Dataset</code>, defaults to <code>None</code>) &#x2014;
Specifies the dataset we will run evaluation on. If it is of type <code>str</code>, we treat it as the dataset
name, and load it. Otherwise we assume it represents a pre-loaded dataset.`,name:"data"},{anchor:"evaluate.TextClassificationEvaluator.compute.metric",description:`<strong>metric</strong> (<code>str</code> or <code>EvaluationModule</code>, defaults to <code>None</code>) &#x2014;
Specifies the metric we use in evaluator. If it is of type <code>str</code>, we treat it as the metric name, and
load it. Otherwise we assume it represents a pre-loaded metric.`,name:"metric"},{anchor:"evaluate.TextClassificationEvaluator.compute.tokenizer",description:`<strong>tokenizer</strong> (<code>str</code> or <code>PreTrainedTokenizer</code>, <em>optional</em>, defaults to <code>None</code>) &#x2014;
Argument can be used to overwrite a default tokenizer if <code>model_or_pipeline</code> represents a model for
which we build a pipeline. If <code>model_or_pipeline</code> is <code>None</code> or a pre-initialized pipeline, we ignore
this argument.`,name:"tokenizer"},{anchor:"evaluate.TextClassificationEvaluator.compute.strategy",description:`<strong>strategy</strong> (<code>Literal[&quot;simple&quot;, &quot;bootstrap&quot;]</code>, defaults to &#x201C;simple&#x201D;) &#x2014;
specifies the evaluation strategy. Possible values are:<ul>
<li><code>&quot;simple&quot;</code> - we evaluate the metric and return the scores.</li>
<li><code>&quot;bootstrap&quot;</code> - on top of computing the metric scores, we calculate the confidence interval for each
of the returned metric keys, using <code>scipy</code>&#x2019;s <code>bootstrap</code> method
<a href="https://docs.scipy.org/doc/scipy/reference/generated/scipy.stats.bootstrap.html" rel="nofollow">https://docs.scipy.org/doc/scipy/reference/generated/scipy.stats.bootstrap.html</a>.</li>
</ul>`,name:"strategy"},{anchor:"evaluate.TextClassificationEvaluator.compute.confidence_level",description:`<strong>confidence_level</strong> (<code>float</code>, defaults to <code>0.95</code>) &#x2014;
The <code>confidence_level</code> value passed to <code>bootstrap</code> if <code>&quot;bootstrap&quot;</code> strategy is chosen.`,name:"confidence_level"},{anchor:"evaluate.TextClassificationEvaluator.compute.n_resamples",description:`<strong>n_resamples</strong> (<code>int</code>, defaults to <code>9999</code>) &#x2014;
The <code>n_resamples</code> value passed to <code>bootstrap</code> if <code>&quot;bootstrap&quot;</code> strategy is chosen.`,name:"n_resamples"},{anchor:"evaluate.TextClassificationEvaluator.compute.device",description:`<strong>device</strong> (<code>int</code>, defaults to <code>None</code>) &#x2014;
Device ordinal for CPU/GPU support of the pipeline. Setting this to -1 will leverage CPU, a positive
integer will run the model on the associated CUDA device ID. If<code>None</code> is provided it will be inferred and
CUDA:0 used if available, CPU otherwise.`,name:"device"},{anchor:"evaluate.TextClassificationEvaluator.compute.random_state",description:`<strong>random_state</strong> (<code>int</code>, <em>optional</em>, defaults to <code>None</code>) &#x2014;
The <code>random_state</code> value passed to <code>bootstrap</code> if <code>&quot;bootstrap&quot;</code> strategy is chosen. Useful for
debugging.`,name:"random_state"},{anchor:"evaluate.TextClassificationEvaluator.compute.input_column",description:`<strong>input_column</strong> (<code>str</code>, defaults to <code>&quot;text&quot;</code>) &#x2014;
the name of the column containing the text feature in the dataset specified by <code>data</code>.`,name:"input_column"},{anchor:"evaluate.TextClassificationEvaluator.compute.label_column",description:`<strong>label_column</strong> (<code>str</code>, defaults to <code>&quot;label&quot;</code>) &#x2014;
the name of the column containing the labels in the dataset specified by <code>data</code>.`,name:"label_column"},{anchor:"evaluate.TextClassificationEvaluator.compute.label_mapping",description:`<strong>label_mapping</strong> (<code>Dict[str, Number]</code>, <em>optional</em>, defaults to <code>None</code>) &#x2014;
We want to map class labels defined by the model in the pipeline to values consistent with those
defined in the <code>label_column</code> of the <code>data</code> dataset.`,name:"label_mapping"}],source:"https://github.com/huggingface/evaluate/blob/vr_199/src/evaluate/evaluator/text_classification.py#L41",returnDescription:`
<p>A <code>Dict</code>. The keys represent metric keys calculated for the <code>metric</code> spefied in function arguments. For the
<code>"simple"</code> strategy, the value is the metric score. For the <code>"bootstrap"</code> strategy, the value is a <code>Dict</code>
containing the score, the confidence interval and the standard error calculated for each metric key.</p>
`}}),we=new Te({props:{anchor:"evaluate.TextClassificationEvaluator.compute.example",$$slots:{default:[Kn]},$$scope:{ctx:x}}}),Ze=new lt({}),et=new N({props:{name:"class evaluate.TokenClassificationEvaluator",anchor:"evaluate.TokenClassificationEvaluator",parameters:[{name:"task",val:" = 'token-classification'"},{name:"default_metric_name",val:" = None"}],source:"https://github.com/huggingface/evaluate/blob/vr_199/src/evaluate/evaluator/token_classification.py#L25"}}),at=new N({props:{name:"compute",anchor:"evaluate.TokenClassificationEvaluator.compute",parameters:[{name:"model_or_pipeline",val:": typing.Union[str, ForwardRef('Pipeline'), typing.Callable, ForwardRef('PreTrainedModel'), ForwardRef('TFPreTrainedModel')] = None"},{name:"data",val:": typing.Union[str, datasets.arrow_dataset.Dataset] = None"},{name:"metric",val:": typing.Union[str, ForwardRef('EvaluationModule')] = None"},{name:"tokenizer",val:": typing.Union[str, ForwardRef('PreTrainedTokenizer'), NoneType] = None"},{name:"strategy",val:": typing.Literal['simple', 'bootstrap'] = 'simple'"},{name:"confidence_level",val:": float = 0.95"},{name:"n_resamples",val:": int = 9999"},{name:"device",val:": typing.Optional[int] = None"},{name:"random_state",val:": typing.Optional[int] = None"},{name:"input_column",val:": str = 'tokens'"},{name:"label_column",val:": str = 'ner_tags'"},{name:"join_by",val:": typing.Optional[str] = ' '"}],parametersDescription:[{anchor:"evaluate.TokenClassificationEvaluator.compute.model_or_pipeline",description:`<strong>model_or_pipeline</strong> (<code>str</code> or <code>Pipeline</code> or <code>Callable</code> or <code>PreTrainedModel</code> or <code>TFPreTrainedModel</code>, defaults to <code>None</code>) &#x2014;
If the argument in not specified, we initialize the default pipeline for the task (in this case
<code>token-classification</code>). If the argument is of the type <code>str</code> or
is a model instance, we use it to initialize a new <code>Pipeline</code> with the given model. Otherwise we assume the
argument specifies a pre-initialized pipeline.`,name:"model_or_pipeline"},{anchor:"evaluate.TokenClassificationEvaluator.compute.data",description:`<strong>data</strong> (<code>str</code> or <code>Dataset</code>, defaults to <code>None</code>) &#x2014;
Specifies the dataset we will run evaluation on. If it is of type <code>str</code>, we treat it as the dataset
name, and load it. Otherwise we assume it represents a pre-loaded dataset.`,name:"data"},{anchor:"evaluate.TokenClassificationEvaluator.compute.metric",description:`<strong>metric</strong> (<code>str</code> or <code>EvaluationModule</code>, defaults to <code>None</code>) &#x2014;
Specifies the metric we use in evaluator. If it is of type <code>str</code>, we treat it as the metric name, and
load it. Otherwise we assume it represents a pre-loaded metric.`,name:"metric"},{anchor:"evaluate.TokenClassificationEvaluator.compute.tokenizer",description:`<strong>tokenizer</strong> (<code>str</code> or <code>PreTrainedTokenizer</code>, <em>optional</em>, defaults to <code>None</code>) &#x2014;
Argument can be used to overwrite a default tokenizer if <code>model_or_pipeline</code> represents a model for
which we build a pipeline. If <code>model_or_pipeline</code> is <code>None</code> or a pre-initialized pipeline, we ignore
this argument.`,name:"tokenizer"},{anchor:"evaluate.TokenClassificationEvaluator.compute.strategy",description:`<strong>strategy</strong> (<code>Literal[&quot;simple&quot;, &quot;bootstrap&quot;]</code>, defaults to &#x201C;simple&#x201D;) &#x2014;
specifies the evaluation strategy. Possible values are:</p>
<ul>
<li><code>&quot;simple&quot;</code> - we evaluate the metric and return the scores.</li>
<li><code>&quot;bootstrap&quot;</code> - on top of computing the metric scores, we calculate the confidence interval for each
of the returned metric keys, using <code>scipy</code>&#x2019;s <code>bootstrap</code> method
<a href="https://docs.scipy.org/doc/scipy/reference/generated/scipy.stats.bootstrap.html" rel="nofollow">https://docs.scipy.org/doc/scipy/reference/generated/scipy.stats.bootstrap.html</a>.</li>
</ul>`,name:"strategy"},{anchor:"evaluate.TokenClassificationEvaluator.compute.confidence_level",description:`<strong>confidence_level</strong> (<code>float</code>, defaults to <code>0.95</code>) &#x2014;
The <code>confidence_level</code> value passed to <code>bootstrap</code> if <code>&quot;bootstrap&quot;</code> strategy is chosen.`,name:"confidence_level"},{anchor:"evaluate.TokenClassificationEvaluator.compute.n_resamples",description:`<strong>n_resamples</strong> (<code>int</code>, defaults to <code>9999</code>) &#x2014;
The <code>n_resamples</code> value passed to <code>bootstrap</code> if <code>&quot;bootstrap&quot;</code> strategy is chosen.`,name:"n_resamples"},{anchor:"evaluate.TokenClassificationEvaluator.compute.device",description:`<strong>device</strong> (<code>int</code>, defaults to <code>None</code>) &#x2014;
Device ordinal for CPU/GPU support of the pipeline. Setting this to -1 will leverage CPU, a positive
integer will run the model on the associated CUDA device ID. If<code>None</code> is provided it will be inferred and
CUDA:0 used if available, CPU otherwise.`,name:"device"},{anchor:"evaluate.TokenClassificationEvaluator.compute.random_state",description:`<strong>random_state</strong> (<code>int</code>, <em>optional</em>, defaults to <code>None</code>) &#x2014;
The <code>random_state</code> value passed to <code>bootstrap</code> if <code>&quot;bootstrap&quot;</code> strategy is chosen. Useful for
debugging.`,name:"random_state"},{anchor:"evaluate.TokenClassificationEvaluator.compute.input_column",description:`<strong>input_column</strong> (<code>str</code>, defaults to <code>&quot;tokens&quot;</code>) &#x2014;
the name of the column containing the tokens feature in the dataset specified by <code>data</code>.`,name:"input_column"},{anchor:"evaluate.TokenClassificationEvaluator.compute.label_column",description:`<strong>label_column</strong> (<code>str</code>, defaults to <code>&quot;label&quot;</code>) &#x2014;
the name of the column containing the labels in the dataset specified by <code>data</code>.`,name:"label_column"},{anchor:"evaluate.TokenClassificationEvaluator.compute.join_by",description:`<strong>join_by</strong> (<code>str</code>, <em>optional</em>, defaults to <code>&quot; &quot;</code>) &#x2014;
This evaluator supports dataset whose input column is a list of words. This parameter specifies how to join
words to generate a string input. This is especially useful for languages that do not separate words by a space.`,name:"join_by"}],source:"https://github.com/huggingface/evaluate/blob/vr_199/src/evaluate/evaluator/token_classification.py#L148",returnDescription:`
<p>A <code>Dict</code>. The keys represent metric keys calculated for the <code>metric</code> spefied in function arguments. For the
<code>"simple"</code> strategy, the value is the metric score. For the <code>"bootstrap"</code> strategy, the value is a <code>Dict</code>
containing the score, the confidence interval and the standard error calculated for each metric key.</p>
`}}),ye=new Te({props:{anchor:"evaluate.TokenClassificationEvaluator.compute.example",$$slots:{default:[Xn]},$$scope:{ctx:x}}}),Ee=new Gs({props:{$$slots:{default:[er]},$$scope:{ctx:x}}}),ke=new Gs({props:{warning:!0,$$slots:{default:[ar]},$$scope:{ctx:x}}}),{c(){l=s("meta"),v=u(),i=s("h1"),p=s("a"),_=s("span"),b(o.$$.fragment),f=u(),F=s("span"),j=c("Evaluator"),S=u(),J=s("p"),La=c("The evaluator classes for automatic evaluation."),da=u(),K=s("h2"),ne=s("a"),qt=s("span"),b(De.$$.fragment),Ma=u(),yt=s("span"),Qa=c("Evaluator classes"),pa=u(),it=s("p"),Ra=c("The main entry point for using the evaluator:"),ua=u(),U=s("div"),b(Ne.$$.fragment),Va=u(),L=s("p"),Ba=c("Utility factory method to build an "),ct=s("a"),Ga=c("Evaluator"),Ha=c(`.
Evaluators encapsulate a task and a default metric name. They leverage `),Et=s("code"),Ya=c("pipeline"),Wa=c(" functionalify from "),kt=s("code"),Ja=c("transformers"),Ka=c(`
to simplify the evaluation of multiple combinations of models, datasets and metrics for a given task.`),Xa=u(),b(re.$$.fragment),ma=u(),dt=s("p"),Za=c("The base class for all evaluator classes:"),fa=u(),T=s("div"),b(Ie.$$.fragment),eo=u(),xt=s("p"),to=c(`The Evaluator class is the class from which all evaluators inherit. Refer to this class for methods shared across
different evaluators.
Base class implementing evaluator operations.`),ao=u(),le=s("div"),b(Ae.$$.fragment),oo=u(),jt=s("p"),so=c("Compute and return metrics."),no=u(),ie=s("div"),b(Fe.$$.fragment),ro=u(),Se=s("p"),lo=c("A core method of the "),Tt=s("code"),io=c("Evaluator"),co=c(" class, which processes the pipeline outputs for compatibility with the metric."),po=u(),ce=s("div"),b(Oe.$$.fragment),uo=u(),Ct=s("p"),mo=c("Prepare data."),fo=u(),de=s("div"),b(ze.$$.fragment),ho=u(),Pt=s("p"),go=c("Prepare metric."),vo=u(),pe=s("div"),b(Ue.$$.fragment),_o=u(),Dt=s("p"),$o=c("Prepare pipeline."),ha=u(),X=s("h2"),ue=s("a"),Nt=s("span"),b(Le.$$.fragment),bo=u(),It=s("span"),wo=c("The task specific evaluators"),ga=u(),Z=s("h3"),me=s("a"),At=s("span"),b(Me.$$.fragment),qo=u(),Ft=s("span"),yo=c("ImageClassificationEvaluator"),va=u(),M=s("div"),b(Qe.$$.fragment),Eo=u(),Q=s("p"),ko=c(`Image classification evaluator.
This image classification evaluator can currently be loaded from `),pt=s("a"),xo=c("evaluator()"),jo=c(` using the default task name
`),St=s("code"),To=c("image-classification"),Co=c(`.
Methods in this class assume a data format compatible with the `),Ot=s("code"),Po=c("ImageClassificationPipeline"),Do=c("."),No=u(),V=s("div"),b(Re.$$.fragment),Io=u(),zt=s("p"),Ao=c("Compute the metric for a given pipeline and dataset combination."),Fo=u(),b(fe.$$.fragment),_a=u(),ee=s("h3"),he=s("a"),Ut=s("span"),b(Ve.$$.fragment),So=u(),Lt=s("span"),Oo=c("QuestionAnsweringEvaluator"),$a=u(),P=s("div"),b(Be.$$.fragment),zo=u(),Ge=s("p"),Uo=c(`Question answering evaluator. This evaluator handles
`),ge=s("a"),Mt=s("strong"),Lo=c("extractive"),Mo=c(" question answering"),Qo=c(`,
where the answer to the question is extracted from a context.`),Ro=u(),te=s("p"),Vo=c("This question answering evaluator can currently be loaded from "),ut=s("a"),Bo=c("evaluator()"),Go=c(` using the default task name
`),Qt=s("code"),Ho=c("question-answering"),Yo=c("."),Wo=u(),He=s("p"),Jo=c(`Methods in this class assume a data format compatible with the
`),Ye=s("a"),Rt=s("code"),Ko=c("QuestionAnsweringPipeline"),Xo=c("."),Zo=u(),I=s("div"),b(We.$$.fragment),es=u(),Vt=s("p"),ts=c("Compute the metric for a given pipeline and dataset combination."),as=u(),b(ve.$$.fragment),os=u(),b(_e.$$.fragment),ss=u(),b($e.$$.fragment),ba=u(),ae=s("h3"),be=s("a"),Bt=s("span"),b(Je.$$.fragment),ns=u(),Gt=s("span"),rs=c("TextClassificationEvaluator"),wa=u(),R=s("div"),b(Ke.$$.fragment),ls=u(),O=s("p"),is=c(`Text classification evaluator.
This text classification evaluator can currently be loaded from `),mt=s("a"),cs=c("evaluator()"),ds=c(` using the default task name
`),Ht=s("code"),ps=c("text-classification"),us=c(" or with a "),Yt=s("code"),ms=c('"sentiment-analysis"'),fs=c(` alias.
Methods in this class assume a data format compatible with the `),Wt=s("code"),hs=c("TextClassificationPipeline"),gs=c(` - a single textual
feature as input and a categorical label as output.`),vs=u(),B=s("div"),b(Xe.$$.fragment),_s=u(),Jt=s("p"),$s=c("Compute the metric for a given pipeline and dataset combination."),bs=u(),b(we.$$.fragment),qa=u(),oe=s("h3"),qe=s("a"),Kt=s("span"),b(Ze.$$.fragment),ws=u(),Xt=s("span"),qs=c("TokenClassificationEvaluator"),ya=u(),D=s("div"),b(et.$$.fragment),ys=u(),Zt=s("p"),Es=c("Token classification evaluator."),ks=u(),se=s("p"),xs=c("This token classification evaluator can currently be loaded from "),ft=s("a"),js=c("evaluator()"),Ts=c(` using the default task name
`),ea=s("code"),Cs=c("token-classification"),Ps=c("."),Ds=u(),tt=s("p"),Ns=c("Methods in this class assume a data format compatible with the "),ta=s("code"),Is=c("TokenClassificationPipeline"),As=c("."),Fs=u(),C=s("div"),b(at.$$.fragment),Ss=u(),aa=s("p"),Os=c("Compute the metric for a given pipeline and dataset combination."),zs=u(),ot=s("p"),Us=c("The dataset input and label columns are expected to be formatted as a list of words and a list of labels respectively, following "),st=s("a"),Ls=c("conll2003 dataset"),Ms=c(". Datasets whose inputs are single strings, and labels are a list of offset are not supported."),Qs=u(),b(ye.$$.fragment),Rs=u(),b(Ee.$$.fragment),Vs=u(),b(ke.$$.fragment),this.h()},l(t){const g=Vn('[data-svelte="svelte-1phssyn"]',document.head);l=n(g,"META",{name:!0,content:!0}),g.forEach(a),v=m(t),i=n(t,"H1",{class:!0});var nt=r(i);p=n(nt,"A",{id:!0,class:!0,href:!0});var oa=r(p);_=n(oa,"SPAN",{});var sa=r(_);w(o.$$.fragment,sa),sa.forEach(a),oa.forEach(a),f=m(nt),F=n(nt,"SPAN",{});var na=r(F);j=d(na,"Evaluator"),na.forEach(a),nt.forEach(a),S=m(t),J=n(t,"P",{});var ra=r(J);La=d(ra,"The evaluator classes for automatic evaluation."),ra.forEach(a),da=m(t),K=n(t,"H2",{class:!0});var rt=r(K);ne=n(rt,"A",{id:!0,class:!0,href:!0});var la=r(ne);qt=n(la,"SPAN",{});var ia=r(qt);w(De.$$.fragment,ia),ia.forEach(a),la.forEach(a),Ma=m(rt),yt=n(rt,"SPAN",{});var ca=r(yt);Qa=d(ca,"Evaluator classes"),ca.forEach(a),rt.forEach(a),pa=m(t),it=n(t,"P",{});var Hs=r(it);Ra=d(Hs,"The main entry point for using the evaluator:"),Hs.forEach(a),ua=m(t),U=n(t,"DIV",{class:!0});var ht=r(U);w(Ne.$$.fragment,ht),Va=m(ht),L=n(ht,"P",{});var xe=r(L);Ba=d(xe,"Utility factory method to build an "),ct=n(xe,"A",{href:!0});var Ys=r(ct);Ga=d(Ys,"Evaluator"),Ys.forEach(a),Ha=d(xe,`.
Evaluators encapsulate a task and a default metric name. They leverage `),Et=n(xe,"CODE",{});var Ws=r(Et);Ya=d(Ws,"pipeline"),Ws.forEach(a),Wa=d(xe," functionalify from "),kt=n(xe,"CODE",{});var Js=r(kt);Ja=d(Js,"transformers"),Js.forEach(a),Ka=d(xe,`
to simplify the evaluation of multiple combinations of models, datasets and metrics for a given task.`),xe.forEach(a),Xa=m(ht),w(re.$$.fragment,ht),ht.forEach(a),ma=m(t),dt=n(t,"P",{});var Ks=r(dt);Za=d(Ks,"The base class for all evaluator classes:"),Ks.forEach(a),fa=m(t),T=n(t,"DIV",{class:!0});var A=r(T);w(Ie.$$.fragment,A),eo=m(A),xt=n(A,"P",{});var Xs=r(xt);to=d(Xs,`The Evaluator class is the class from which all evaluators inherit. Refer to this class for methods shared across
different evaluators.
Base class implementing evaluator operations.`),Xs.forEach(a),ao=m(A),le=n(A,"DIV",{class:!0});var ka=r(le);w(Ae.$$.fragment,ka),oo=m(ka),jt=n(ka,"P",{});var Zs=r(jt);so=d(Zs,"Compute and return metrics."),Zs.forEach(a),ka.forEach(a),no=m(A),ie=n(A,"DIV",{class:!0});var xa=r(ie);w(Fe.$$.fragment,xa),ro=m(xa),Se=n(xa,"P",{});var ja=r(Se);lo=d(ja,"A core method of the "),Tt=n(ja,"CODE",{});var en=r(Tt);io=d(en,"Evaluator"),en.forEach(a),co=d(ja," class, which processes the pipeline outputs for compatibility with the metric."),ja.forEach(a),xa.forEach(a),po=m(A),ce=n(A,"DIV",{class:!0});var Ta=r(ce);w(Oe.$$.fragment,Ta),uo=m(Ta),Ct=n(Ta,"P",{});var tn=r(Ct);mo=d(tn,"Prepare data."),tn.forEach(a),Ta.forEach(a),fo=m(A),de=n(A,"DIV",{class:!0});var Ca=r(de);w(ze.$$.fragment,Ca),ho=m(Ca),Pt=n(Ca,"P",{});var an=r(Pt);go=d(an,"Prepare metric."),an.forEach(a),Ca.forEach(a),vo=m(A),pe=n(A,"DIV",{class:!0});var Pa=r(pe);w(Ue.$$.fragment,Pa),_o=m(Pa),Dt=n(Pa,"P",{});var on=r(Dt);$o=d(on,"Prepare pipeline."),on.forEach(a),Pa.forEach(a),A.forEach(a),ha=m(t),X=n(t,"H2",{class:!0});var Da=r(X);ue=n(Da,"A",{id:!0,class:!0,href:!0});var sn=r(ue);Nt=n(sn,"SPAN",{});var nn=r(Nt);w(Le.$$.fragment,nn),nn.forEach(a),sn.forEach(a),bo=m(Da),It=n(Da,"SPAN",{});var rn=r(It);wo=d(rn,"The task specific evaluators"),rn.forEach(a),Da.forEach(a),ga=m(t),Z=n(t,"H3",{class:!0});var Na=r(Z);me=n(Na,"A",{id:!0,class:!0,href:!0});var ln=r(me);At=n(ln,"SPAN",{});var cn=r(At);w(Me.$$.fragment,cn),cn.forEach(a),ln.forEach(a),qo=m(Na),Ft=n(Na,"SPAN",{});var dn=r(Ft);yo=d(dn,"ImageClassificationEvaluator"),dn.forEach(a),Na.forEach(a),va=m(t),M=n(t,"DIV",{class:!0});var gt=r(M);w(Qe.$$.fragment,gt),Eo=m(gt),Q=n(gt,"P",{});var je=r(Q);ko=d(je,`Image classification evaluator.
This image classification evaluator can currently be loaded from `),pt=n(je,"A",{href:!0});var pn=r(pt);xo=d(pn,"evaluator()"),pn.forEach(a),jo=d(je,` using the default task name
`),St=n(je,"CODE",{});var un=r(St);To=d(un,"image-classification"),un.forEach(a),Co=d(je,`.
Methods in this class assume a data format compatible with the `),Ot=n(je,"CODE",{});var mn=r(Ot);Po=d(mn,"ImageClassificationPipeline"),mn.forEach(a),Do=d(je,"."),je.forEach(a),No=m(gt),V=n(gt,"DIV",{class:!0});var vt=r(V);w(Re.$$.fragment,vt),Io=m(vt),zt=n(vt,"P",{});var fn=r(zt);Ao=d(fn,"Compute the metric for a given pipeline and dataset combination."),fn.forEach(a),Fo=m(vt),w(fe.$$.fragment,vt),vt.forEach(a),gt.forEach(a),_a=m(t),ee=n(t,"H3",{class:!0});var Ia=r(ee);he=n(Ia,"A",{id:!0,class:!0,href:!0});var hn=r(he);Ut=n(hn,"SPAN",{});var gn=r(Ut);w(Ve.$$.fragment,gn),gn.forEach(a),hn.forEach(a),So=m(Ia),Lt=n(Ia,"SPAN",{});var vn=r(Lt);Oo=d(vn,"QuestionAnsweringEvaluator"),vn.forEach(a),Ia.forEach(a),$a=m(t),P=n(t,"DIV",{class:!0});var G=r(P);w(Be.$$.fragment,G),zo=m(G),Ge=n(G,"P",{});var Aa=r(Ge);Uo=d(Aa,`Question answering evaluator. This evaluator handles
`),ge=n(Aa,"A",{href:!0,rel:!0});var Bs=r(ge);Mt=n(Bs,"STRONG",{});var _n=r(Mt);Lo=d(_n,"extractive"),_n.forEach(a),Mo=d(Bs," question answering"),Bs.forEach(a),Qo=d(Aa,`,
where the answer to the question is extracted from a context.`),Aa.forEach(a),Ro=m(G),te=n(G,"P",{});var _t=r(te);Vo=d(_t,"This question answering evaluator can currently be loaded from "),ut=n(_t,"A",{href:!0});var $n=r(ut);Bo=d($n,"evaluator()"),$n.forEach(a),Go=d(_t,` using the default task name
`),Qt=n(_t,"CODE",{});var bn=r(Qt);Ho=d(bn,"question-answering"),bn.forEach(a),Yo=d(_t,"."),_t.forEach(a),Wo=m(G),He=n(G,"P",{});var Fa=r(He);Jo=d(Fa,`Methods in this class assume a data format compatible with the
`),Ye=n(Fa,"A",{href:!0,rel:!0});var wn=r(Ye);Rt=n(wn,"CODE",{});var qn=r(Rt);Ko=d(qn,"QuestionAnsweringPipeline"),qn.forEach(a),wn.forEach(a),Xo=d(Fa,"."),Fa.forEach(a),Zo=m(G),I=n(G,"DIV",{class:!0});var H=r(I);w(We.$$.fragment,H),es=m(H),Vt=n(H,"P",{});var yn=r(Vt);ts=d(yn,"Compute the metric for a given pipeline and dataset combination."),yn.forEach(a),as=m(H),w(ve.$$.fragment,H),os=m(H),w(_e.$$.fragment,H),ss=m(H),w($e.$$.fragment,H),H.forEach(a),G.forEach(a),ba=m(t),ae=n(t,"H3",{class:!0});var Sa=r(ae);be=n(Sa,"A",{id:!0,class:!0,href:!0});var En=r(be);Bt=n(En,"SPAN",{});var kn=r(Bt);w(Je.$$.fragment,kn),kn.forEach(a),En.forEach(a),ns=m(Sa),Gt=n(Sa,"SPAN",{});var xn=r(Gt);rs=d(xn,"TextClassificationEvaluator"),xn.forEach(a),Sa.forEach(a),wa=m(t),R=n(t,"DIV",{class:!0});var $t=r(R);w(Ke.$$.fragment,$t),ls=m($t),O=n($t,"P",{});var Y=r(O);is=d(Y,`Text classification evaluator.
This text classification evaluator can currently be loaded from `),mt=n(Y,"A",{href:!0});var jn=r(mt);cs=d(jn,"evaluator()"),jn.forEach(a),ds=d(Y,` using the default task name
`),Ht=n(Y,"CODE",{});var Tn=r(Ht);ps=d(Tn,"text-classification"),Tn.forEach(a),us=d(Y," or with a "),Yt=n(Y,"CODE",{});var Cn=r(Yt);ms=d(Cn,'"sentiment-analysis"'),Cn.forEach(a),fs=d(Y,` alias.
Methods in this class assume a data format compatible with the `),Wt=n(Y,"CODE",{});var Pn=r(Wt);hs=d(Pn,"TextClassificationPipeline"),Pn.forEach(a),gs=d(Y,` - a single textual
feature as input and a categorical label as output.`),Y.forEach(a),vs=m($t),B=n($t,"DIV",{class:!0});var bt=r(B);w(Xe.$$.fragment,bt),_s=m(bt),Jt=n(bt,"P",{});var Dn=r(Jt);$s=d(Dn,"Compute the metric for a given pipeline and dataset combination."),Dn.forEach(a),bs=m(bt),w(we.$$.fragment,bt),bt.forEach(a),$t.forEach(a),qa=m(t),oe=n(t,"H3",{class:!0});var Oa=r(oe);qe=n(Oa,"A",{id:!0,class:!0,href:!0});var Nn=r(qe);Kt=n(Nn,"SPAN",{});var In=r(Kt);w(Ze.$$.fragment,In),In.forEach(a),Nn.forEach(a),ws=m(Oa),Xt=n(Oa,"SPAN",{});var An=r(Xt);qs=d(An,"TokenClassificationEvaluator"),An.forEach(a),Oa.forEach(a),ya=m(t),D=n(t,"DIV",{class:!0});var W=r(D);w(et.$$.fragment,W),ys=m(W),Zt=n(W,"P",{});var Fn=r(Zt);Es=d(Fn,"Token classification evaluator."),Fn.forEach(a),ks=m(W),se=n(W,"P",{});var wt=r(se);xs=d(wt,"This token classification evaluator can currently be loaded from "),ft=n(wt,"A",{href:!0});var Sn=r(ft);js=d(Sn,"evaluator()"),Sn.forEach(a),Ts=d(wt,` using the default task name
`),ea=n(wt,"CODE",{});var On=r(ea);Cs=d(On,"token-classification"),On.forEach(a),Ps=d(wt,"."),wt.forEach(a),Ds=m(W),tt=n(W,"P",{});var za=r(tt);Ns=d(za,"Methods in this class assume a data format compatible with the "),ta=n(za,"CODE",{});var zn=r(ta);Is=d(zn,"TokenClassificationPipeline"),zn.forEach(a),As=d(za,"."),za.forEach(a),Fs=m(W),C=n(W,"DIV",{class:!0});var z=r(C);w(at.$$.fragment,z),Ss=m(z),aa=n(z,"P",{});var Un=r(aa);Os=d(Un,"Compute the metric for a given pipeline and dataset combination."),Un.forEach(a),zs=m(z),ot=n(z,"P",{});var Ua=r(ot);Us=d(Ua,"The dataset input and label columns are expected to be formatted as a list of words and a list of labels respectively, following "),st=n(Ua,"A",{href:!0,rel:!0});var Ln=r(st);Ls=d(Ln,"conll2003 dataset"),Ln.forEach(a),Ms=d(Ua,". Datasets whose inputs are single strings, and labels are a list of offset are not supported."),Ua.forEach(a),Qs=m(z),w(ye.$$.fragment,z),Rs=m(z),w(Ee.$$.fragment,z),Vs=m(z),w(ke.$$.fragment,z),z.forEach(a),W.forEach(a),this.h()},h(){h(l,"name","hf:doc:metadata"),h(l,"content",JSON.stringify(sr)),h(p,"id","evaluator"),h(p,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),h(p,"href","#evaluator"),h(i,"class","relative group"),h(ne,"id","evaluate.evaluator"),h(ne,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),h(ne,"href","#evaluate.evaluator"),h(K,"class","relative group"),h(ct,"href","/docs/evaluate/pr_199/en/package_reference/evaluator_classes#evaluate.Evaluator"),h(U,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),h(le,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),h(ie,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),h(ce,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),h(de,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),h(pe,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),h(T,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),h(ue,"id","the-task-specific-evaluators"),h(ue,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),h(ue,"href","#the-task-specific-evaluators"),h(X,"class","relative group"),h(me,"id","evaluate.ImageClassificationEvaluator"),h(me,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),h(me,"href","#evaluate.ImageClassificationEvaluator"),h(Z,"class","relative group"),h(pt,"href","/docs/evaluate/pr_199/en/package_reference/evaluator_classes#evaluate.evaluator"),h(V,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),h(M,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),h(he,"id","evaluate.QuestionAnsweringEvaluator"),h(he,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),h(he,"href","#evaluate.QuestionAnsweringEvaluator"),h(ee,"class","relative group"),h(ge,"href","https://huggingface.co/docs/transformers/task_summary#extractive-question-answering"),h(ge,"rel","nofollow"),h(ut,"href","/docs/evaluate/pr_199/en/package_reference/evaluator_classes#evaluate.evaluator"),h(Ye,"href","https://huggingface.co/docs/transformers/en/main_classes/pipelines#transformers.QuestionAnsweringPipeline"),h(Ye,"rel","nofollow"),h(I,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),h(P,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),h(be,"id","evaluate.TextClassificationEvaluator"),h(be,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),h(be,"href","#evaluate.TextClassificationEvaluator"),h(ae,"class","relative group"),h(mt,"href","/docs/evaluate/pr_199/en/package_reference/evaluator_classes#evaluate.evaluator"),h(B,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),h(R,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),h(qe,"id","evaluate.TokenClassificationEvaluator"),h(qe,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),h(qe,"href","#evaluate.TokenClassificationEvaluator"),h(oe,"class","relative group"),h(ft,"href","/docs/evaluate/pr_199/en/package_reference/evaluator_classes#evaluate.evaluator"),h(st,"href","https://huggingface.co/datasets/conll2003"),h(st,"rel","nofollow"),h(C,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),h(D,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8")},m(t,g){e(document.head,l),$(t,v,g),$(t,i,g),e(i,p),e(p,_),q(o,_,null),e(i,f),e(i,F),e(F,j),$(t,S,g),$(t,J,g),e(J,La),$(t,da,g),$(t,K,g),e(K,ne),e(ne,qt),q(De,qt,null),e(K,Ma),e(K,yt),e(yt,Qa),$(t,pa,g),$(t,it,g),e(it,Ra),$(t,ua,g),$(t,U,g),q(Ne,U,null),e(U,Va),e(U,L),e(L,Ba),e(L,ct),e(ct,Ga),e(L,Ha),e(L,Et),e(Et,Ya),e(L,Wa),e(L,kt),e(kt,Ja),e(L,Ka),e(U,Xa),q(re,U,null),$(t,ma,g),$(t,dt,g),e(dt,Za),$(t,fa,g),$(t,T,g),q(Ie,T,null),e(T,eo),e(T,xt),e(xt,to),e(T,ao),e(T,le),q(Ae,le,null),e(le,oo),e(le,jt),e(jt,so),e(T,no),e(T,ie),q(Fe,ie,null),e(ie,ro),e(ie,Se),e(Se,lo),e(Se,Tt),e(Tt,io),e(Se,co),e(T,po),e(T,ce),q(Oe,ce,null),e(ce,uo),e(ce,Ct),e(Ct,mo),e(T,fo),e(T,de),q(ze,de,null),e(de,ho),e(de,Pt),e(Pt,go),e(T,vo),e(T,pe),q(Ue,pe,null),e(pe,_o),e(pe,Dt),e(Dt,$o),$(t,ha,g),$(t,X,g),e(X,ue),e(ue,Nt),q(Le,Nt,null),e(X,bo),e(X,It),e(It,wo),$(t,ga,g),$(t,Z,g),e(Z,me),e(me,At),q(Me,At,null),e(Z,qo),e(Z,Ft),e(Ft,yo),$(t,va,g),$(t,M,g),q(Qe,M,null),e(M,Eo),e(M,Q),e(Q,ko),e(Q,pt),e(pt,xo),e(Q,jo),e(Q,St),e(St,To),e(Q,Co),e(Q,Ot),e(Ot,Po),e(Q,Do),e(M,No),e(M,V),q(Re,V,null),e(V,Io),e(V,zt),e(zt,Ao),e(V,Fo),q(fe,V,null),$(t,_a,g),$(t,ee,g),e(ee,he),e(he,Ut),q(Ve,Ut,null),e(ee,So),e(ee,Lt),e(Lt,Oo),$(t,$a,g),$(t,P,g),q(Be,P,null),e(P,zo),e(P,Ge),e(Ge,Uo),e(Ge,ge),e(ge,Mt),e(Mt,Lo),e(ge,Mo),e(Ge,Qo),e(P,Ro),e(P,te),e(te,Vo),e(te,ut),e(ut,Bo),e(te,Go),e(te,Qt),e(Qt,Ho),e(te,Yo),e(P,Wo),e(P,He),e(He,Jo),e(He,Ye),e(Ye,Rt),e(Rt,Ko),e(He,Xo),e(P,Zo),e(P,I),q(We,I,null),e(I,es),e(I,Vt),e(Vt,ts),e(I,as),q(ve,I,null),e(I,os),q(_e,I,null),e(I,ss),q($e,I,null),$(t,ba,g),$(t,ae,g),e(ae,be),e(be,Bt),q(Je,Bt,null),e(ae,ns),e(ae,Gt),e(Gt,rs),$(t,wa,g),$(t,R,g),q(Ke,R,null),e(R,ls),e(R,O),e(O,is),e(O,mt),e(mt,cs),e(O,ds),e(O,Ht),e(Ht,ps),e(O,us),e(O,Yt),e(Yt,ms),e(O,fs),e(O,Wt),e(Wt,hs),e(O,gs),e(R,vs),e(R,B),q(Xe,B,null),e(B,_s),e(B,Jt),e(Jt,$s),e(B,bs),q(we,B,null),$(t,qa,g),$(t,oe,g),e(oe,qe),e(qe,Kt),q(Ze,Kt,null),e(oe,ws),e(oe,Xt),e(Xt,qs),$(t,ya,g),$(t,D,g),q(et,D,null),e(D,ys),e(D,Zt),e(Zt,Es),e(D,ks),e(D,se),e(se,xs),e(se,ft),e(ft,js),e(se,Ts),e(se,ea),e(ea,Cs),e(se,Ps),e(D,Ds),e(D,tt),e(tt,Ns),e(tt,ta),e(ta,Is),e(tt,As),e(D,Fs),e(D,C),q(at,C,null),e(C,Ss),e(C,aa),e(aa,Os),e(C,zs),e(C,ot),e(ot,Us),e(ot,st),e(st,Ls),e(ot,Ms),e(C,Qs),q(ye,C,null),e(C,Rs),q(Ee,C,null),e(C,Vs),q(ke,C,null),Ea=!0},p(t,[g]){const nt={};g&2&&(nt.$$scope={dirty:g,ctx:t}),re.$set(nt);const oa={};g&2&&(oa.$$scope={dirty:g,ctx:t}),fe.$set(oa);const sa={};g&2&&(sa.$$scope={dirty:g,ctx:t}),ve.$set(sa);const na={};g&2&&(na.$$scope={dirty:g,ctx:t}),_e.$set(na);const ra={};g&2&&(ra.$$scope={dirty:g,ctx:t}),$e.$set(ra);const rt={};g&2&&(rt.$$scope={dirty:g,ctx:t}),we.$set(rt);const la={};g&2&&(la.$$scope={dirty:g,ctx:t}),ye.$set(la);const ia={};g&2&&(ia.$$scope={dirty:g,ctx:t}),Ee.$set(ia);const ca={};g&2&&(ca.$$scope={dirty:g,ctx:t}),ke.$set(ca)},i(t){Ea||(y(o.$$.fragment,t),y(De.$$.fragment,t),y(Ne.$$.fragment,t),y(re.$$.fragment,t),y(Ie.$$.fragment,t),y(Ae.$$.fragment,t),y(Fe.$$.fragment,t),y(Oe.$$.fragment,t),y(ze.$$.fragment,t),y(Ue.$$.fragment,t),y(Le.$$.fragment,t),y(Me.$$.fragment,t),y(Qe.$$.fragment,t),y(Re.$$.fragment,t),y(fe.$$.fragment,t),y(Ve.$$.fragment,t),y(Be.$$.fragment,t),y(We.$$.fragment,t),y(ve.$$.fragment,t),y(_e.$$.fragment,t),y($e.$$.fragment,t),y(Je.$$.fragment,t),y(Ke.$$.fragment,t),y(Xe.$$.fragment,t),y(we.$$.fragment,t),y(Ze.$$.fragment,t),y(et.$$.fragment,t),y(at.$$.fragment,t),y(ye.$$.fragment,t),y(Ee.$$.fragment,t),y(ke.$$.fragment,t),Ea=!0)},o(t){E(o.$$.fragment,t),E(De.$$.fragment,t),E(Ne.$$.fragment,t),E(re.$$.fragment,t),E(Ie.$$.fragment,t),E(Ae.$$.fragment,t),E(Fe.$$.fragment,t),E(Oe.$$.fragment,t),E(ze.$$.fragment,t),E(Ue.$$.fragment,t),E(Le.$$.fragment,t),E(Me.$$.fragment,t),E(Qe.$$.fragment,t),E(Re.$$.fragment,t),E(fe.$$.fragment,t),E(Ve.$$.fragment,t),E(Be.$$.fragment,t),E(We.$$.fragment,t),E(ve.$$.fragment,t),E(_e.$$.fragment,t),E($e.$$.fragment,t),E(Je.$$.fragment,t),E(Ke.$$.fragment,t),E(Xe.$$.fragment,t),E(we.$$.fragment,t),E(Ze.$$.fragment,t),E(et.$$.fragment,t),E(at.$$.fragment,t),E(ye.$$.fragment,t),E(Ee.$$.fragment,t),E(ke.$$.fragment,t),Ea=!1},d(t){a(l),t&&a(v),t&&a(i),k(o),t&&a(S),t&&a(J),t&&a(da),t&&a(K),k(De),t&&a(pa),t&&a(it),t&&a(ua),t&&a(U),k(Ne),k(re),t&&a(ma),t&&a(dt),t&&a(fa),t&&a(T),k(Ie),k(Ae),k(Fe),k(Oe),k(ze),k(Ue),t&&a(ha),t&&a(X),k(Le),t&&a(ga),t&&a(Z),k(Me),t&&a(va),t&&a(M),k(Qe),k(Re),k(fe),t&&a(_a),t&&a(ee),k(Ve),t&&a($a),t&&a(P),k(Be),k(We),k(ve),k(_e),k($e),t&&a(ba),t&&a(ae),k(Je),t&&a(wa),t&&a(R),k(Ke),k(Xe),k(we),t&&a(qa),t&&a(oe),k(Ze),t&&a(ya),t&&a(D),k(et),k(at),k(ye),k(Ee),k(ke)}}}const sr={local:"evaluator",sections:[{local:"evaluate.evaluator",title:"Evaluator classes"},{local:"the-task-specific-evaluators",sections:[{local:"evaluate.ImageClassificationEvaluator",title:"ImageClassificationEvaluator"},{local:"evaluate.QuestionAnsweringEvaluator",title:"QuestionAnsweringEvaluator"},{local:"evaluate.TextClassificationEvaluator",title:"TextClassificationEvaluator"},{local:"evaluate.TokenClassificationEvaluator",title:"TokenClassificationEvaluator"}],title:"The task specific evaluators"}],title:"Evaluator"};function nr(x){return Bn(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class ur extends Mn{constructor(l){super();Qn(this,l,nr,or,Rn,{})}}export{ur as default,sr as metadata};
