import{S as Jn,i as Kn,s as Wn,e as o,k as u,w as b,t as c,M as Xn,c as n,d as a,m,a as r,x as w,h as p,b as g,G as e,g as $,y,q as E,o as q,B as k,v as Zn,L as Pe}from"../../chunks/vendor-hf-doc-builder.js";import{T as Zo}from"../../chunks/Tip-hf-doc-builder.js";import{D as N}from"../../chunks/Docstring-hf-doc-builder.js";import{C as Ne}from"../../chunks/CodeBlock-hf-doc-builder.js";import{I as ct}from"../../chunks/IconCopyLink-hf-doc-builder.js";import{E as Ce}from"../../chunks/ExampleCodeBlock-hf-doc-builder.js";function er(x){let l,v,i,d,_;return d=new Ne({props:{code:`from evaluate import evaluator
# Sentiment analysis evaluator
evaluator("sentiment-analysis")`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> evaluate <span class="hljs-keyword">import</span> evaluator
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-comment"># Sentiment analysis evaluator</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>evaluator(<span class="hljs-string">&quot;sentiment-analysis&quot;</span>)`}}),{c(){l=o("p"),v=c("Examples:"),i=u(),b(d.$$.fragment)},l(s){l=n(s,"P",{});var f=r(l);v=p(f,"Examples:"),f.forEach(a),i=m(s),w(d.$$.fragment,s)},m(s,f){$(s,l,f),e(l,v),$(s,i,f),y(d,s,f),_=!0},p:Pe,i(s){_||(E(d.$$.fragment,s),_=!0)},o(s){q(d.$$.fragment,s),_=!1},d(s){s&&a(l),s&&a(i),k(d,s)}}}function tr(x){let l,v,i,d,_;return d=new Ne({props:{code:`from evaluate import evaluator
from datasets import load_dataset
task_evaluator = evaluator("image-classification")
data = load_dataset("beans", split="test[:40]")
results = task_evaluator.compute(
    model_or_pipeline="nateraw/vit-base-beans",
    data=data,
    label_column="labels",
    metric="accuracy",
    label_mapping={'angular_leaf_spot': 0, 'bean_rust': 1, 'healthy': 2},
    strategy="bootstrap"
)`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> evaluate <span class="hljs-keyword">import</span> evaluator
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> datasets <span class="hljs-keyword">import</span> load_dataset
<span class="hljs-meta">&gt;&gt;&gt; </span>task_evaluator = evaluator(<span class="hljs-string">&quot;image-classification&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>data = load_dataset(<span class="hljs-string">&quot;beans&quot;</span>, split=<span class="hljs-string">&quot;test[:40]&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>results = task_evaluator.compute(
<span class="hljs-meta">&gt;&gt;&gt; </span>    model_or_pipeline=<span class="hljs-string">&quot;nateraw/vit-base-beans&quot;</span>,
<span class="hljs-meta">&gt;&gt;&gt; </span>    data=data,
<span class="hljs-meta">&gt;&gt;&gt; </span>    label_column=<span class="hljs-string">&quot;labels&quot;</span>,
<span class="hljs-meta">&gt;&gt;&gt; </span>    metric=<span class="hljs-string">&quot;accuracy&quot;</span>,
<span class="hljs-meta">&gt;&gt;&gt; </span>    label_mapping={<span class="hljs-string">&#x27;angular_leaf_spot&#x27;</span>: <span class="hljs-number">0</span>, <span class="hljs-string">&#x27;bean_rust&#x27;</span>: <span class="hljs-number">1</span>, <span class="hljs-string">&#x27;healthy&#x27;</span>: <span class="hljs-number">2</span>},
<span class="hljs-meta">&gt;&gt;&gt; </span>    strategy=<span class="hljs-string">&quot;bootstrap&quot;</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>)`}}),{c(){l=o("p"),v=c("Examples:"),i=u(),b(d.$$.fragment)},l(s){l=n(s,"P",{});var f=r(l);v=p(f,"Examples:"),f.forEach(a),i=m(s),w(d.$$.fragment,s)},m(s,f){$(s,l,f),e(l,v),$(s,i,f),y(d,s,f),_=!0},p:Pe,i(s){_||(E(d.$$.fragment,s),_=!0)},o(s){q(d.$$.fragment,s),_=!1},d(s){s&&a(l),s&&a(i),k(d,s)}}}function ar(x){let l,v,i,d,_;return d=new Ne({props:{code:`from evaluate import evaluator
from datasets import load_dataset
task_evaluator = evaluator("question-answering")
data = load_dataset("squad", split="validation[:2]")
results = task_evaluator.compute(
    model_or_pipeline="sshleifer/tiny-distilbert-base-cased-distilled-squad",
    data=data,
    metric="squad",
)`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> evaluate <span class="hljs-keyword">import</span> evaluator
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> datasets <span class="hljs-keyword">import</span> load_dataset
<span class="hljs-meta">&gt;&gt;&gt; </span>task_evaluator = evaluator(<span class="hljs-string">&quot;question-answering&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>data = load_dataset(<span class="hljs-string">&quot;squad&quot;</span>, split=<span class="hljs-string">&quot;validation[:2]&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>results = task_evaluator.compute(
<span class="hljs-meta">&gt;&gt;&gt; </span>    model_or_pipeline=<span class="hljs-string">&quot;sshleifer/tiny-distilbert-base-cased-distilled-squad&quot;</span>,
<span class="hljs-meta">&gt;&gt;&gt; </span>    data=data,
<span class="hljs-meta">&gt;&gt;&gt; </span>    metric=<span class="hljs-string">&quot;squad&quot;</span>,
<span class="hljs-meta">&gt;&gt;&gt; </span>)`}}),{c(){l=o("p"),v=c("Examples:"),i=u(),b(d.$$.fragment)},l(s){l=n(s,"P",{});var f=r(l);v=p(f,"Examples:"),f.forEach(a),i=m(s),w(d.$$.fragment,s)},m(s,f){$(s,l,f),e(l,v),$(s,i,f),y(d,s,f),_=!0},p:Pe,i(s){_||(E(d.$$.fragment,s),_=!0)},o(s){q(d.$$.fragment,s),_=!1},d(s){s&&a(l),s&&a(i),k(d,s)}}}function sr(x){let l,v,i,d,_;return{c(){l=o("p"),v=c("Datasets where the answer may be missing in the context are supported, for example SQuAD v2 dataset. In this case, it is safer to pass "),i=o("code"),d=c("squad_v2_format=True"),_=c(` to
the compute() call.`)},l(s){l=n(s,"P",{});var f=r(l);v=p(f,"Datasets where the answer may be missing in the context are supported, for example SQuAD v2 dataset. In this case, it is safer to pass "),i=n(f,"CODE",{});var F=r(i);d=p(F,"squad_v2_format=True"),F.forEach(a),_=p(f,` to
the compute() call.`),f.forEach(a)},m(s,f){$(s,l,f),e(l,v),e(l,i),e(i,d),e(l,_)},d(s){s&&a(l)}}}function or(x){let l,v;return l=new Ne({props:{code:`from evaluate import evaluator
from datasets import load_dataset
task_evaluator = evaluator("question-answering")
data = load_dataset("squad_v2", split="validation[:2]")
results = task_evaluator.compute(
    model_or_pipeline="mrm8488/bert-tiny-finetuned-squadv2",
    data=data,
    metric="squad_v2",
    squad_v2_format=True,
)`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> evaluate <span class="hljs-keyword">import</span> evaluator
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> datasets <span class="hljs-keyword">import</span> load_dataset
<span class="hljs-meta">&gt;&gt;&gt; </span>task_evaluator = evaluator(<span class="hljs-string">&quot;question-answering&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>data = load_dataset(<span class="hljs-string">&quot;squad_v2&quot;</span>, split=<span class="hljs-string">&quot;validation[:2]&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>results = task_evaluator.compute(
<span class="hljs-meta">&gt;&gt;&gt; </span>    model_or_pipeline=<span class="hljs-string">&quot;mrm8488/bert-tiny-finetuned-squadv2&quot;</span>,
<span class="hljs-meta">&gt;&gt;&gt; </span>    data=data,
<span class="hljs-meta">&gt;&gt;&gt; </span>    metric=<span class="hljs-string">&quot;squad_v2&quot;</span>,
<span class="hljs-meta">&gt;&gt;&gt; </span>    squad_v2_format=<span class="hljs-literal">True</span>,
<span class="hljs-meta">&gt;&gt;&gt; </span>)`}}),{c(){b(l.$$.fragment)},l(i){w(l.$$.fragment,i)},m(i,d){y(l,i,d),v=!0},p:Pe,i(i){v||(E(l.$$.fragment,i),v=!0)},o(i){q(l.$$.fragment,i),v=!1},d(i){k(l,i)}}}function nr(x){let l,v,i,d,_;return d=new Ne({props:{code:`from evaluate import evaluator
from datasets import load_dataset
task_evaluator = evaluator("text-classification")
data = load_dataset("imdb", split="test[:2]")
results = task_evaluator.compute(
    model_or_pipeline="huggingface/prunebert-base-uncased-6-finepruned-w-distil-mnli",
    data=data,
    metric="accuracy",
    label_mapping={"LABEL_0": 0.0, "LABEL_1": 1.0},
    strategy="bootstrap",
    n_resamples=10,
    random_state=0
)`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> evaluate <span class="hljs-keyword">import</span> evaluator
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> datasets <span class="hljs-keyword">import</span> load_dataset
<span class="hljs-meta">&gt;&gt;&gt; </span>task_evaluator = evaluator(<span class="hljs-string">&quot;text-classification&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>data = load_dataset(<span class="hljs-string">&quot;imdb&quot;</span>, split=<span class="hljs-string">&quot;test[:2]&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>results = task_evaluator.compute(
<span class="hljs-meta">&gt;&gt;&gt; </span>    model_or_pipeline=<span class="hljs-string">&quot;huggingface/prunebert-base-uncased-6-finepruned-w-distil-mnli&quot;</span>,
<span class="hljs-meta">&gt;&gt;&gt; </span>    data=data,
<span class="hljs-meta">&gt;&gt;&gt; </span>    metric=<span class="hljs-string">&quot;accuracy&quot;</span>,
<span class="hljs-meta">&gt;&gt;&gt; </span>    label_mapping={<span class="hljs-string">&quot;LABEL_0&quot;</span>: <span class="hljs-number">0.0</span>, <span class="hljs-string">&quot;LABEL_1&quot;</span>: <span class="hljs-number">1.0</span>},
<span class="hljs-meta">&gt;&gt;&gt; </span>    strategy=<span class="hljs-string">&quot;bootstrap&quot;</span>,
<span class="hljs-meta">&gt;&gt;&gt; </span>    n_resamples=<span class="hljs-number">10</span>,
<span class="hljs-meta">&gt;&gt;&gt; </span>    random_state=<span class="hljs-number">0</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>)`}}),{c(){l=o("p"),v=c("Examples:"),i=u(),b(d.$$.fragment)},l(s){l=n(s,"P",{});var f=r(l);v=p(f,"Examples:"),f.forEach(a),i=m(s),w(d.$$.fragment,s)},m(s,f){$(s,l,f),e(l,v),$(s,i,f),y(d,s,f),_=!0},p:Pe,i(s){_||(E(d.$$.fragment,s),_=!0)},o(s){q(d.$$.fragment,s),_=!1},d(s){s&&a(l),s&&a(i),k(d,s)}}}function rr(x){let l,v,i,d,_;return d=new Ne({props:{code:`from evaluate import evaluator
from datasets import load_dataset
task_evaluator = evaluator("token-classification")
data = load_dataset("conll2003", split="validation[:2]")
results = task_evaluator.compute(
    model_or_pipeline="elastic/distilbert-base-uncased-finetuned-conll03-english",
    data=data,
    metric="seqeval",
)`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> evaluate <span class="hljs-keyword">import</span> evaluator
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> datasets <span class="hljs-keyword">import</span> load_dataset
<span class="hljs-meta">&gt;&gt;&gt; </span>task_evaluator = evaluator(<span class="hljs-string">&quot;token-classification&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>data = load_dataset(<span class="hljs-string">&quot;conll2003&quot;</span>, split=<span class="hljs-string">&quot;validation[:2]&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>results = task_evaluator.compute(
<span class="hljs-meta">&gt;&gt;&gt; </span>    model_or_pipeline=<span class="hljs-string">&quot;elastic/distilbert-base-uncased-finetuned-conll03-english&quot;</span>,
<span class="hljs-meta">&gt;&gt;&gt; </span>    data=data,
<span class="hljs-meta">&gt;&gt;&gt; </span>    metric=<span class="hljs-string">&quot;seqeval&quot;</span>,
<span class="hljs-meta">&gt;&gt;&gt; </span>)`}}),{c(){l=o("p"),v=c("Examples:"),i=u(),b(d.$$.fragment)},l(s){l=n(s,"P",{});var f=r(l);v=p(f,"Examples:"),f.forEach(a),i=m(s),w(d.$$.fragment,s)},m(s,f){$(s,l,f),e(l,v),$(s,i,f),y(d,s,f),_=!0},p:Pe,i(s){_||(E(d.$$.fragment,s),_=!0)},o(s){q(d.$$.fragment,s),_=!1},d(s){s&&a(l),s&&a(i),k(d,s)}}}function lr(x){let l,v,i,d,_;return d=new Ne({props:{code:`dataset = Dataset.from_dict(
    mapping={
        "tokens": [["New", "York", "is", "a", "city", "and", "Felix", "a", "person", "."]],
        "ner_tags": [[1, 2, 0, 0, 0, 0, 3, 0, 0, 0]],
    },
    features=Features({
        "tokens": Sequence(feature=Value(dtype="string")),
        "ner_tags": Sequence(feature=ClassLabel(names=["O", "B-LOC", "I-LOC", "B-PER", "I-PER"])),
        }),
)`,highlighted:`dataset = Dataset.from_dict(
    mapping={
        <span class="hljs-string">&quot;tokens&quot;</span>: [[<span class="hljs-string">&quot;New&quot;</span>, <span class="hljs-string">&quot;York&quot;</span>, <span class="hljs-string">&quot;is&quot;</span>, <span class="hljs-string">&quot;a&quot;</span>, <span class="hljs-string">&quot;city&quot;</span>, <span class="hljs-string">&quot;and&quot;</span>, <span class="hljs-string">&quot;Felix&quot;</span>, <span class="hljs-string">&quot;a&quot;</span>, <span class="hljs-string">&quot;person&quot;</span>, <span class="hljs-string">&quot;.&quot;</span>]],
        <span class="hljs-string">&quot;ner_tags&quot;</span>: [[<span class="hljs-number">1</span>, <span class="hljs-number">2</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">3</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>]],
    },
    features=Features({
        <span class="hljs-string">&quot;tokens&quot;</span>: <span class="hljs-type">Sequence</span>(feature=Value(dtype=<span class="hljs-string">&quot;string&quot;</span>)),
        <span class="hljs-string">&quot;ner_tags&quot;</span>: <span class="hljs-type">Sequence</span>(feature=ClassLabel(names=[<span class="hljs-string">&quot;O&quot;</span>, <span class="hljs-string">&quot;B-LOC&quot;</span>, <span class="hljs-string">&quot;I-LOC&quot;</span>, <span class="hljs-string">&quot;B-PER&quot;</span>, <span class="hljs-string">&quot;I-PER&quot;</span>])),
        }),
)`}}),{c(){l=o("p"),v=c("For example, the following dataset format is accepted by the evaluator:"),i=u(),b(d.$$.fragment)},l(s){l=n(s,"P",{});var f=r(l);v=p(f,"For example, the following dataset format is accepted by the evaluator:"),f.forEach(a),i=m(s),w(d.$$.fragment,s)},m(s,f){$(s,l,f),e(l,v),$(s,i,f),y(d,s,f),_=!0},p:Pe,i(s){_||(E(d.$$.fragment,s),_=!0)},o(s){q(d.$$.fragment,s),_=!1},d(s){s&&a(l),s&&a(i),k(d,s)}}}function ir(x){let l,v;return l=new Ce({props:{anchor:"evaluate.TokenClassificationEvaluator.compute.example-2",$$slots:{default:[lr]},$$scope:{ctx:x}}}),{c(){b(l.$$.fragment)},l(i){w(l.$$.fragment,i)},m(i,d){y(l,i,d),v=!0},p(i,d){const _={};d&2&&(_.$$scope={dirty:d,ctx:i}),l.$set(_)},i(i){v||(E(l.$$.fragment,i),v=!0)},o(i){q(l.$$.fragment,i),v=!1},d(i){k(l,i)}}}function cr(x){let l,v,i,d,_,s,f,F;return f=new Ne({props:{code:`dataset = Dataset.from_dict(
    mapping={
        "tokens": [["New York is a city and Felix a person."]],
        "starts": [[0, 23]],
        "ends": [[7, 27]],
        "ner_tags": [["LOC", "PER"]],
    },
    features=Features({
        "tokens": Value(dtype="string"),
        "starts": Sequence(feature=Value(dtype="int32")),
        "ends": Sequence(feature=Value(dtype="int32")),
        "ner_tags": Sequence(feature=Value(dtype="string")),
    }),
)`,highlighted:`dataset = Dataset.from_dict(
    mapping={
        <span class="hljs-string">&quot;tokens&quot;</span>: [[<span class="hljs-string">&quot;New York is a city and Felix a person.&quot;</span>]],
        <span class="hljs-string">&quot;starts&quot;</span>: [[<span class="hljs-number">0</span>, <span class="hljs-number">23</span>]],
        <span class="hljs-string">&quot;ends&quot;</span>: [[<span class="hljs-number">7</span>, <span class="hljs-number">27</span>]],
        <span class="hljs-string">&quot;ner_tags&quot;</span>: [[<span class="hljs-string">&quot;LOC&quot;</span>, <span class="hljs-string">&quot;PER&quot;</span>]],
    },
    features=Features({
        <span class="hljs-string">&quot;tokens&quot;</span>: Value(dtype=<span class="hljs-string">&quot;string&quot;</span>),
        <span class="hljs-string">&quot;starts&quot;</span>: <span class="hljs-type">Sequence</span>(feature=Value(dtype=<span class="hljs-string">&quot;int32&quot;</span>)),
        <span class="hljs-string">&quot;ends&quot;</span>: <span class="hljs-type">Sequence</span>(feature=Value(dtype=<span class="hljs-string">&quot;int32&quot;</span>)),
        <span class="hljs-string">&quot;ner_tags&quot;</span>: <span class="hljs-type">Sequence</span>(feature=Value(dtype=<span class="hljs-string">&quot;string&quot;</span>)),
    }),
)`}}),{c(){l=o("p"),v=c("For example, the following dataset format is "),i=o("strong"),d=c("not"),_=c(" accepted by the evaluator:"),s=u(),b(f.$$.fragment)},l(T){l=n(T,"P",{});var U=r(l);v=p(U,"For example, the following dataset format is "),i=n(U,"STRONG",{});var K=r(i);d=p(K,"not"),K.forEach(a),_=p(U," accepted by the evaluator:"),U.forEach(a),s=m(T),w(f.$$.fragment,T)},m(T,U){$(T,l,U),e(l,v),e(l,i),e(i,d),e(l,_),$(T,s,U),y(f,T,U),F=!0},p:Pe,i(T){F||(E(f.$$.fragment,T),F=!0)},o(T){q(f.$$.fragment,T),F=!1},d(T){T&&a(l),T&&a(s),k(f,T)}}}function pr(x){let l,v;return l=new Ce({props:{anchor:"evaluate.TokenClassificationEvaluator.compute.example-3",$$slots:{default:[cr]},$$scope:{ctx:x}}}),{c(){b(l.$$.fragment)},l(i){w(l.$$.fragment,i)},m(i,d){y(l,i,d),v=!0},p(i,d){const _={};d&2&&(_.$$scope={dirty:d,ctx:i}),l.$set(_)},i(i){v||(E(l.$$.fragment,i),v=!0)},o(i){q(l.$$.fragment,i),v=!1},d(i){k(l,i)}}}function dr(x){let l,v,i,d,_,s,f,F,T,U,K,Va,ma,W,ne,qt,De,Ba,kt,Ga,fa,pt,Ha,ga,L,Ie,Ya,O,Ja,dt,Ka,Wa,xt,Xa,Za,jt,es,ts,as,re,ha,ut,ss,va,j,Ae,os,Tt,ns,rs,le,Fe,ls,Ct,is,cs,ie,Ue,ps,Pt,ds,us,ce,ze,ms,Se,fs,Nt,gs,hs,vs,pe,Le,_s,Dt,$s,bs,de,Oe,ws,It,ys,Es,ue,Me,qs,At,ks,_a,X,me,Ft,Re,xs,Ut,js,$a,Z,fe,zt,Qe,Ts,St,Cs,ba,M,Ve,Ps,R,Ns,mt,Ds,Is,Lt,As,Fs,Ot,Us,zs,Ss,V,Be,Ls,Mt,Os,Ms,ge,wa,ee,he,Rt,Ge,Rs,Qt,Qs,ya,D,He,Vs,Ye,Bs,ve,Vt,Gs,Hs,Ys,Js,te,Ks,ft,Ws,Xs,Bt,Zs,eo,to,Je,ao,Ke,Gt,so,oo,no,A,We,ro,Ht,lo,io,_e,co,$e,po,be,Ea,ae,we,Yt,Xe,uo,Jt,mo,qa,Q,Ze,fo,z,go,gt,ho,vo,Kt,_o,$o,Wt,bo,wo,Xt,yo,Eo,qo,B,et,ko,Zt,xo,jo,ye,ka,se,Ee,ea,tt,To,ta,Co,xa,I,at,Po,aa,No,Do,oe,Io,ht,Ao,Fo,sa,Uo,zo,So,st,Lo,oa,Oo,Mo,Ro,C,ot,Qo,na,Vo,Bo,nt,Go,rt,Ho,Yo,Jo,qe,Ko,ke,Wo,xe,ja;return s=new ct({}),De=new ct({}),Ie=new N({props:{name:"evaluate.evaluator",anchor:"evaluate.evaluator",parameters:[{name:"task",val:": str = None"}],parametersDescription:[{anchor:"evaluate.evaluator.task",description:`<strong>task</strong> (<code>str</code>) &#x2014;
The task defining which evaluator will be returned. Currently accepted tasks are:<ul>
<li><code>&quot;image-classification&quot;</code>: will return a <a href="/docs/evaluate/pr_36/en/package_reference/evaluator_classes#evaluate.ImageClassificationEvaluator">ImageClassificationEvaluator</a>.</li>
<li><code>&quot;question-answering&quot;</code>: will return a <a href="/docs/evaluate/pr_36/en/package_reference/evaluator_classes#evaluate.QuestionAnsweringEvaluator">QuestionAnsweringEvaluator</a>.</li>
<li><code>&quot;text-classification&quot;</code> (alias <code>&quot;sentiment-analysis&quot;</code> available): will return a <a href="/docs/evaluate/pr_36/en/package_reference/evaluator_classes#evaluate.TextClassificationEvaluator">TextClassificationEvaluator</a>.</li>
<li><code>&quot;token-classification&quot;</code>: will return a <a href="/docs/evaluate/pr_36/en/package_reference/evaluator_classes#evaluate.TokenClassificationEvaluator">TokenClassificationEvaluator</a>.</li>
</ul>`,name:"task"}],source:"https://github.com/huggingface/evaluate/blob/vr_36/src/evaluate/evaluator/__init__.py#L85",returnDescription:`
<p>An evaluator suitable for the task.</p>
`,returnType:`
<p><a
  href="/docs/evaluate/pr_36/en/package_reference/evaluator_classes#evaluate.Evaluator"
>Evaluator</a></p>
`}}),re=new Ce({props:{anchor:"evaluate.evaluator.example",$$slots:{default:[er]},$$scope:{ctx:x}}}),Ae=new N({props:{name:"class evaluate.Evaluator",anchor:"evaluate.Evaluator",parameters:[{name:"task",val:": str"},{name:"default_metric_name",val:": str = None"}],source:"https://github.com/huggingface/evaluate/blob/vr_36/src/evaluate/evaluator/base.py#L96"}}),Fe=new N({props:{name:"check_required_columns",anchor:"evaluate.Evaluator.check_required_columns",parameters:[{name:"data",val:": typing.Union[str, datasets.arrow_dataset.Dataset]"},{name:"columns_names",val:": typing.Dict[str, str]"}],parametersDescription:[{anchor:"evaluate.Evaluator.check_required_columns.data",description:`<strong>data</strong> (<code>str</code> or <code>Dataset</code>) &#x2014;
Specifies the dataset we will run evaluation on.`,name:"data"},{anchor:"evaluate.Evaluator.check_required_columns.columns_names",description:"<strong>columns_names</strong> (<code>List[str]</code>) &#x2014;",name:"columns_names"},{anchor:"evaluate.Evaluator.check_required_columns.List",description:"<strong>List</strong> of column names to check in the dataset. The keys are the arguments to the compute() method, &#x2014;",name:"List"},{anchor:"evaluate.Evaluator.check_required_columns.while",description:"<strong>while</strong> the values are the column names to check. &#x2014;",name:"while"}],source:"https://github.com/huggingface/evaluate/blob/vr_36/src/evaluate/evaluator/base.py#L264"}}),Ue=new N({props:{name:"compute_metric",anchor:"evaluate.Evaluator.compute_metric",parameters:[{name:"metric",val:": EvaluationModule"},{name:"metric_inputs",val:": typing.Dict"},{name:"strategy",val:": typing.Literal['simple', 'bootstrap'] = 'simple'"},{name:"confidence_level",val:": float = 0.95"},{name:"n_resamples",val:": int = 9999"},{name:"random_state",val:": typing.Optional[int] = None"}],source:"https://github.com/huggingface/evaluate/blob/vr_36/src/evaluate/evaluator/base.py#L389"}}),ze=new N({props:{name:"predictions_processor",anchor:"evaluate.Evaluator.predictions_processor",parameters:[{name:"*args",val:""},{name:"**kwargs",val:""}],source:"https://github.com/huggingface/evaluate/blob/vr_36/src/evaluate/evaluator/base.py#L205"}}),Le=new N({props:{name:"prepare_data",anchor:"evaluate.Evaluator.prepare_data",parameters:[{name:"data",val:": typing.Union[str, datasets.arrow_dataset.Dataset]"},{name:"input_column",val:": str"},{name:"label_column",val:": str"}],parametersDescription:[{anchor:"evaluate.Evaluator.prepare_data.data",description:"<strong>data</strong> (<code>str</code> or <code>Dataset</code>, defaults to <code>None) -- Specifies the dataset we will run evaluation on. If it is of type </code>str`, we treat it as the dataset\nname, and load it. Otherwise we assume it represents a pre-loaded dataset.",name:"data"},{anchor:"evaluate.Evaluator.prepare_data.input_column",description:`<strong>input_column</strong> (<code>str</code>, defaults to <code>&quot;text&quot;</code>) &#x2014;
the name of the column containing the text feature in the dataset specified by <code>data</code>.`,name:"input_column"},{anchor:"evaluate.Evaluator.prepare_data.label_column",description:`<strong>label_column</strong> (<code>str</code>, defaults to <code>&quot;label&quot;</code>) &#x2014;
the name of the column containing the labels in the dataset specified by <code>data</code>.`,name:"label_column"}],source:"https://github.com/huggingface/evaluate/blob/vr_36/src/evaluate/evaluator/base.py#L281",returnDescription:`
<p>metric inputs.
<code>list</code>:  pipeline inputs.</p>
`,returnType:`
<p><code>dict</code></p>
`}}),Oe=new N({props:{name:"prepare_metric",anchor:"evaluate.Evaluator.prepare_metric",parameters:[{name:"metric",val:": typing.Union[str, evaluate.module.EvaluationModule]"}],parametersDescription:[{anchor:"evaluate.Evaluator.prepare_metric.metric",description:`<strong>metric</strong> (<code>str</code> or <code>EvaluationModule</code>, defaults to <code>None</code>) &#x2014;
Specifies the metric we use in evaluator. If it is of type <code>str</code>, we treat it as the metric name, and
load it. Otherwise we assume it represents a pre-loaded metric.`,name:"metric"}],source:"https://github.com/huggingface/evaluate/blob/vr_36/src/evaluate/evaluator/base.py#L359",returnDescription:`
<p>The loaded metric.</p>
`}}),Me=new N({props:{name:"prepare_pipeline",anchor:"evaluate.Evaluator.prepare_pipeline",parameters:[{name:"model_or_pipeline",val:": typing.Union[str, ForwardRef('Pipeline'), typing.Callable, ForwardRef('PreTrainedModel'), ForwardRef('TFPreTrainedModel')]"},{name:"tokenizer",val:": typing.Union[ForwardRef('PreTrainedTokenizerBase'), ForwardRef('FeatureExtractionMixin')] = None"},{name:"feature_extractor",val:": typing.Union[ForwardRef('PreTrainedTokenizerBase'), ForwardRef('FeatureExtractionMixin')] = None"},{name:"device",val:": int = None"}],parametersDescription:[{anchor:"evaluate.Evaluator.prepare_pipeline.model_or_pipeline",description:"<strong>model_or_pipeline</strong> (<code>str</code> or <code>Pipeline</code> or <code>Callable</code> or <code>PreTrainedModel</code> or <code>TFPreTrainedModel</code>, &#x2014;",name:"model_or_pipeline"},{anchor:"evaluate.Evaluator.prepare_pipeline.defaults",description:`<strong>defaults</strong> to <code>None</code>) &#x2014;
If the argument in not specified, we initialize the default pipeline for the task. If the argument is of the type <code>str</code> or
is a model instance, we use it to initialize a new <code>Pipeline</code> with the given model. Otherwise we assume the
argument specifies a pre-initialized pipeline.`,name:"defaults"},{anchor:"evaluate.Evaluator.prepare_pipeline.preprocessor",description:`<strong>preprocessor</strong> (<code>PreTrainedTokenizerBase</code> or <code>FeatureExtractionMixin</code>, <em>optional</em>, defaults to <code>None</code>) &#x2014;
Argument can be used to overwrite a default preprocessor if <code>model_or_pipeline</code> represents a model for
which we build a pipeline. If <code>model_or_pipeline</code> is <code>None</code> or a pre-initialized pipeline, we ignore
this argument.`,name:"preprocessor"}],source:"https://github.com/huggingface/evaluate/blob/vr_36/src/evaluate/evaluator/base.py#L307",returnDescription:`
<p>The initialized pipeline.</p>
`}}),Re=new ct({}),Qe=new ct({}),Ve=new N({props:{name:"class evaluate.ImageClassificationEvaluator",anchor:"evaluate.ImageClassificationEvaluator",parameters:[{name:"task",val:" = 'image-classification'"},{name:"default_metric_name",val:" = None"}],source:"https://github.com/huggingface/evaluate/blob/vr_36/src/evaluate/evaluator/image_classification.py#L45"}}),Be=new N({props:{name:"compute",anchor:"evaluate.ImageClassificationEvaluator.compute",parameters:[{name:"model_or_pipeline",val:": typing.Union[str, ForwardRef('Pipeline'), typing.Callable, ForwardRef('PreTrainedModel'), ForwardRef('TFPreTrainedModel')] = None"},{name:"data",val:": typing.Union[str, datasets.arrow_dataset.Dataset] = None"},{name:"metric",val:": typing.Union[str, evaluate.module.EvaluationModule] = None"},{name:"tokenizer",val:": typing.Union[str, ForwardRef('PreTrainedTokenizer'), NoneType] = None"},{name:"feature_extractor",val:": typing.Union[str, ForwardRef('FeatureExtractionMixin'), NoneType] = None"},{name:"strategy",val:": typing.Literal['simple', 'bootstrap'] = 'simple'"},{name:"confidence_level",val:": float = 0.95"},{name:"n_resamples",val:": int = 9999"},{name:"device",val:": int = None"},{name:"random_state",val:": typing.Optional[int] = None"},{name:"input_column",val:": str = 'image'"},{name:"label_column",val:": str = 'label'"},{name:"label_mapping",val:": typing.Union[typing.Dict[str, numbers.Number], NoneType] = None"}],parametersDescription:[{anchor:"evaluate.ImageClassificationEvaluator.compute.model_or_pipeline",description:`<strong>model_or_pipeline</strong> (<code>str</code> or <code>Pipeline</code> or <code>Callable</code> or <code>PreTrainedModel</code> or <code>TFPreTrainedModel</code>, defaults to <code>None</code>) &#x2014;
If the argument in not specified, we initialize the default pipeline for the task (in this case
<code>text-classification</code> or its alias - <code>sentiment-analysis</code>). If the argument is of the type <code>str</code> or
is a model instance, we use it to initialize a new <code>Pipeline</code> with the given model. Otherwise we assume the
argument specifies a pre-initialized pipeline.`,name:"model_or_pipeline"},{anchor:"evaluate.ImageClassificationEvaluator.compute.data",description:`<strong>data</strong> (<code>str</code> or <code>Dataset</code>, defaults to <code>None</code>) &#x2014;
Specifies the dataset we will run evaluation on. If it is of type <code>str</code>, we treat it as the dataset
name, and load it. Otherwise we assume it represents a pre-loaded dataset.`,name:"data"},{anchor:"evaluate.ImageClassificationEvaluator.compute.metric",description:`<strong>metric</strong> (<code>str</code> or <code>EvaluationModule</code>, defaults to <code>None</code>) &#x2014;
Specifies the metric we use in evaluator. If it is of type <code>str</code>, we treat it as the metric name, and
load it. Otherwise we assume it represents a pre-loaded metric.`,name:"metric"},{anchor:"evaluate.ImageClassificationEvaluator.compute.tokenizer",description:`<strong>tokenizer</strong> (<code>str</code> or <code>PreTrainedTokenizer</code>, <em>optional</em>, defaults to <code>None</code>) &#x2014;
Argument can be used to overwrite a default tokenizer if <code>model_or_pipeline</code> represents a model for
which we build a pipeline. If <code>model_or_pipeline</code> is <code>None</code> or a pre-initialized pipeline, we ignore
this argument.`,name:"tokenizer"},{anchor:"evaluate.ImageClassificationEvaluator.compute.strategy",description:`<strong>strategy</strong> (<code>Literal[&quot;simple&quot;, &quot;bootstrap&quot;]</code>, defaults to &#x201C;simple&#x201D;) &#x2014;
specifies the evaluation strategy. Possible values are:<ul>
<li><code>&quot;simple&quot;</code> - we evaluate the metric and return the scores.</li>
<li><code>&quot;bootstrap&quot;</code> - on top of computing the metric scores, we calculate the confidence interval for each
of the returned metric keys, using <code>scipy</code>&#x2019;s <code>bootstrap</code> method
<a href="https://docs.scipy.org/doc/scipy/reference/generated/scipy.stats.bootstrap.html" rel="nofollow">https://docs.scipy.org/doc/scipy/reference/generated/scipy.stats.bootstrap.html</a>.</li>
</ul>`,name:"strategy"},{anchor:"evaluate.ImageClassificationEvaluator.compute.confidence_level",description:`<strong>confidence_level</strong> (<code>float</code>, defaults to <code>0.95</code>) &#x2014;
The <code>confidence_level</code> value passed to <code>bootstrap</code> if <code>&quot;bootstrap&quot;</code> strategy is chosen.`,name:"confidence_level"},{anchor:"evaluate.ImageClassificationEvaluator.compute.n_resamples",description:`<strong>n_resamples</strong> (<code>int</code>, defaults to <code>9999</code>) &#x2014;
The <code>n_resamples</code> value passed to <code>bootstrap</code> if <code>&quot;bootstrap&quot;</code> strategy is chosen.`,name:"n_resamples"},{anchor:"evaluate.ImageClassificationEvaluator.compute.device",description:`<strong>device</strong> (<code>int</code>, defaults to <code>None</code>) &#x2014;
Device ordinal for CPU/GPU support of the pipeline. Setting this to -1 will leverage CPU, a positive
integer will run the model on the associated CUDA device ID. If<code>None</code> is provided it will be inferred and
CUDA:0 used if available, CPU otherwise.`,name:"device"},{anchor:"evaluate.ImageClassificationEvaluator.compute.random_state",description:`<strong>random_state</strong> (<code>int</code>, <em>optional</em>, defaults to <code>None</code>) &#x2014;
The <code>random_state</code> value passed to <code>bootstrap</code> if <code>&quot;bootstrap&quot;</code> strategy is chosen. Useful for
debugging.`,name:"random_state"}],source:"https://github.com/huggingface/evaluate/blob/vr_36/src/evaluate/evaluator/image_classification.py#L62",returnDescription:`
<p>A <code>Dict</code>. The keys represent metric keys calculated for the <code>metric</code> spefied in function arguments. For the
<code>"simple"</code> strategy, the value is the metric score. For the <code>"bootstrap"</code> strategy, the value is a <code>Dict</code>
containing the score, the confidence interval and the standard error calculated for each metric key.</p>
`}}),ge=new Ce({props:{anchor:"evaluate.ImageClassificationEvaluator.compute.example",$$slots:{default:[tr]},$$scope:{ctx:x}}}),Ge=new ct({}),He=new N({props:{name:"class evaluate.QuestionAnsweringEvaluator",anchor:"evaluate.QuestionAnsweringEvaluator",parameters:[{name:"task",val:" = 'question-answering'"},{name:"default_metric_name",val:" = None"}],source:"https://github.com/huggingface/evaluate/blob/vr_36/src/evaluate/evaluator/question_answering.py#L74"}}),We=new N({props:{name:"compute",anchor:"evaluate.QuestionAnsweringEvaluator.compute",parameters:[{name:"model_or_pipeline",val:": typing.Union[str, ForwardRef('Pipeline'), typing.Callable, ForwardRef('PreTrainedModel'), ForwardRef('TFPreTrainedModel')] = None"},{name:"data",val:": typing.Union[str, datasets.arrow_dataset.Dataset] = None"},{name:"metric",val:": typing.Union[str, evaluate.module.EvaluationModule] = None"},{name:"tokenizer",val:": typing.Union[str, ForwardRef('PreTrainedTokenizer'), NoneType] = None"},{name:"strategy",val:": typing.Literal['simple', 'bootstrap'] = 'simple'"},{name:"confidence_level",val:": float = 0.95"},{name:"n_resamples",val:": int = 9999"},{name:"device",val:": int = None"},{name:"random_state",val:": typing.Optional[int] = None"},{name:"question_column",val:": str = 'question'"},{name:"context_column",val:": str = 'context'"},{name:"id_column",val:": str = 'id'"},{name:"label_column",val:": str = 'answers'"},{name:"squad_v2_format",val:": typing.Optional[bool] = None"}],parametersDescription:[{anchor:"evaluate.QuestionAnsweringEvaluator.compute.model_or_pipeline",description:`<strong>model_or_pipeline</strong> (<code>str</code> or <code>Pipeline</code> or <code>Callable</code> or <code>PreTrainedModel</code> or <code>TFPreTrainedModel</code>, defaults to <code>None</code>) &#x2014;
If the argument in not specified, we initialize the default pipeline for the task (in this case
<code>text-classification</code> or its alias - <code>sentiment-analysis</code>). If the argument is of the type <code>str</code> or
is a model instance, we use it to initialize a new <code>Pipeline</code> with the given model. Otherwise we assume the
argument specifies a pre-initialized pipeline.`,name:"model_or_pipeline"},{anchor:"evaluate.QuestionAnsweringEvaluator.compute.data",description:`<strong>data</strong> (<code>str</code> or <code>Dataset</code>, defaults to <code>None</code>) &#x2014;
Specifies the dataset we will run evaluation on. If it is of type <code>str</code>, we treat it as the dataset
name, and load it. Otherwise we assume it represents a pre-loaded dataset.`,name:"data"},{anchor:"evaluate.QuestionAnsweringEvaluator.compute.metric",description:`<strong>metric</strong> (<code>str</code> or <code>EvaluationModule</code>, defaults to <code>None</code>) &#x2014;
Specifies the metric we use in evaluator. If it is of type <code>str</code>, we treat it as the metric name, and
load it. Otherwise we assume it represents a pre-loaded metric.`,name:"metric"},{anchor:"evaluate.QuestionAnsweringEvaluator.compute.tokenizer",description:`<strong>tokenizer</strong> (<code>str</code> or <code>PreTrainedTokenizer</code>, <em>optional</em>, defaults to <code>None</code>) &#x2014;
Argument can be used to overwrite a default tokenizer if <code>model_or_pipeline</code> represents a model for
which we build a pipeline. If <code>model_or_pipeline</code> is <code>None</code> or a pre-initialized pipeline, we ignore
this argument.`,name:"tokenizer"},{anchor:"evaluate.QuestionAnsweringEvaluator.compute.strategy",description:`<strong>strategy</strong> (<code>Literal[&quot;simple&quot;, &quot;bootstrap&quot;]</code>, defaults to &#x201C;simple&#x201D;) &#x2014;
specifies the evaluation strategy. Possible values are:<ul>
<li><code>&quot;simple&quot;</code> - we evaluate the metric and return the scores.</li>
<li><code>&quot;bootstrap&quot;</code> - on top of computing the metric scores, we calculate the confidence interval for each
of the returned metric keys, using <code>scipy</code>&#x2019;s <code>bootstrap</code> method
<a href="https://docs.scipy.org/doc/scipy/reference/generated/scipy.stats.bootstrap.html" rel="nofollow">https://docs.scipy.org/doc/scipy/reference/generated/scipy.stats.bootstrap.html</a>.</li>
</ul>`,name:"strategy"},{anchor:"evaluate.QuestionAnsweringEvaluator.compute.confidence_level",description:`<strong>confidence_level</strong> (<code>float</code>, defaults to <code>0.95</code>) &#x2014;
The <code>confidence_level</code> value passed to <code>bootstrap</code> if <code>&quot;bootstrap&quot;</code> strategy is chosen.`,name:"confidence_level"},{anchor:"evaluate.QuestionAnsweringEvaluator.compute.n_resamples",description:`<strong>n_resamples</strong> (<code>int</code>, defaults to <code>9999</code>) &#x2014;
The <code>n_resamples</code> value passed to <code>bootstrap</code> if <code>&quot;bootstrap&quot;</code> strategy is chosen.`,name:"n_resamples"},{anchor:"evaluate.QuestionAnsweringEvaluator.compute.device",description:`<strong>device</strong> (<code>int</code>, defaults to <code>None</code>) &#x2014;
Device ordinal for CPU/GPU support of the pipeline. Setting this to -1 will leverage CPU, a positive
integer will run the model on the associated CUDA device ID. If<code>None</code> is provided it will be inferred and
CUDA:0 used if available, CPU otherwise.`,name:"device"},{anchor:"evaluate.QuestionAnsweringEvaluator.compute.random_state",description:`<strong>random_state</strong> (<code>int</code>, <em>optional</em>, defaults to <code>None</code>) &#x2014;
The <code>random_state</code> value passed to <code>bootstrap</code> if <code>&quot;bootstrap&quot;</code> strategy is chosen. Useful for
debugging.`,name:"random_state"}],source:"https://github.com/huggingface/evaluate/blob/vr_36/src/evaluate/evaluator/question_answering.py#L150",returnDescription:`
<p>A <code>Dict</code>. The keys represent metric keys calculated for the <code>metric</code> spefied in function arguments. For the
<code>"simple"</code> strategy, the value is the metric score. For the <code>"bootstrap"</code> strategy, the value is a <code>Dict</code>
containing the score, the confidence interval and the standard error calculated for each metric key.</p>
`}}),_e=new Ce({props:{anchor:"evaluate.QuestionAnsweringEvaluator.compute.example",$$slots:{default:[ar]},$$scope:{ctx:x}}}),$e=new Zo({props:{$$slots:{default:[sr]},$$scope:{ctx:x}}}),be=new Ce({props:{anchor:"evaluate.QuestionAnsweringEvaluator.compute.example-2",$$slots:{default:[or]},$$scope:{ctx:x}}}),Xe=new ct({}),Ze=new N({props:{name:"class evaluate.TextClassificationEvaluator",anchor:"evaluate.TextClassificationEvaluator",parameters:[{name:"task",val:" = 'text-classification'"},{name:"default_metric_name",val:" = None"}],source:"https://github.com/huggingface/evaluate/blob/vr_36/src/evaluate/evaluator/text_classification.py#L47"}}),et=new N({props:{name:"compute",anchor:"evaluate.TextClassificationEvaluator.compute",parameters:[{name:"model_or_pipeline",val:": typing.Union[str, ForwardRef('Pipeline'), typing.Callable, ForwardRef('PreTrainedModel'), ForwardRef('TFPreTrainedModel')] = None"},{name:"data",val:": typing.Union[str, datasets.arrow_dataset.Dataset] = None"},{name:"metric",val:": typing.Union[str, evaluate.module.EvaluationModule] = None"},{name:"tokenizer",val:": typing.Union[str, ForwardRef('PreTrainedTokenizer'), NoneType] = None"},{name:"feature_extractor",val:": typing.Union[str, ForwardRef('FeatureExtractionMixin'), NoneType] = None"},{name:"strategy",val:": typing.Literal['simple', 'bootstrap'] = 'simple'"},{name:"confidence_level",val:": float = 0.95"},{name:"n_resamples",val:": int = 9999"},{name:"device",val:": int = None"},{name:"random_state",val:": typing.Optional[int] = None"},{name:"input_column",val:": str = 'text'"},{name:"second_input_column",val:": typing.Optional[str] = None"},{name:"label_column",val:": str = 'label'"},{name:"label_mapping",val:": typing.Union[typing.Dict[str, numbers.Number], NoneType] = None"}],parametersDescription:[{anchor:"evaluate.TextClassificationEvaluator.compute.model_or_pipeline",description:`<strong>model_or_pipeline</strong> (<code>str</code> or <code>Pipeline</code> or <code>Callable</code> or <code>PreTrainedModel</code> or <code>TFPreTrainedModel</code>, defaults to <code>None</code>) &#x2014;
If the argument in not specified, we initialize the default pipeline for the task (in this case
<code>text-classification</code> or its alias - <code>sentiment-analysis</code>). If the argument is of the type <code>str</code> or
is a model instance, we use it to initialize a new <code>Pipeline</code> with the given model. Otherwise we assume the
argument specifies a pre-initialized pipeline.`,name:"model_or_pipeline"},{anchor:"evaluate.TextClassificationEvaluator.compute.data",description:`<strong>data</strong> (<code>str</code> or <code>Dataset</code>, defaults to <code>None</code>) &#x2014;
Specifies the dataset we will run evaluation on. If it is of type <code>str</code>, we treat it as the dataset
name, and load it. Otherwise we assume it represents a pre-loaded dataset.`,name:"data"},{anchor:"evaluate.TextClassificationEvaluator.compute.metric",description:`<strong>metric</strong> (<code>str</code> or <code>EvaluationModule</code>, defaults to <code>None</code>) &#x2014;
Specifies the metric we use in evaluator. If it is of type <code>str</code>, we treat it as the metric name, and
load it. Otherwise we assume it represents a pre-loaded metric.`,name:"metric"},{anchor:"evaluate.TextClassificationEvaluator.compute.tokenizer",description:`<strong>tokenizer</strong> (<code>str</code> or <code>PreTrainedTokenizer</code>, <em>optional</em>, defaults to <code>None</code>) &#x2014;
Argument can be used to overwrite a default tokenizer if <code>model_or_pipeline</code> represents a model for
which we build a pipeline. If <code>model_or_pipeline</code> is <code>None</code> or a pre-initialized pipeline, we ignore
this argument.`,name:"tokenizer"},{anchor:"evaluate.TextClassificationEvaluator.compute.strategy",description:`<strong>strategy</strong> (<code>Literal[&quot;simple&quot;, &quot;bootstrap&quot;]</code>, defaults to &#x201C;simple&#x201D;) &#x2014;
specifies the evaluation strategy. Possible values are:<ul>
<li><code>&quot;simple&quot;</code> - we evaluate the metric and return the scores.</li>
<li><code>&quot;bootstrap&quot;</code> - on top of computing the metric scores, we calculate the confidence interval for each
of the returned metric keys, using <code>scipy</code>&#x2019;s <code>bootstrap</code> method
<a href="https://docs.scipy.org/doc/scipy/reference/generated/scipy.stats.bootstrap.html" rel="nofollow">https://docs.scipy.org/doc/scipy/reference/generated/scipy.stats.bootstrap.html</a>.</li>
</ul>`,name:"strategy"},{anchor:"evaluate.TextClassificationEvaluator.compute.confidence_level",description:`<strong>confidence_level</strong> (<code>float</code>, defaults to <code>0.95</code>) &#x2014;
The <code>confidence_level</code> value passed to <code>bootstrap</code> if <code>&quot;bootstrap&quot;</code> strategy is chosen.`,name:"confidence_level"},{anchor:"evaluate.TextClassificationEvaluator.compute.n_resamples",description:`<strong>n_resamples</strong> (<code>int</code>, defaults to <code>9999</code>) &#x2014;
The <code>n_resamples</code> value passed to <code>bootstrap</code> if <code>&quot;bootstrap&quot;</code> strategy is chosen.`,name:"n_resamples"},{anchor:"evaluate.TextClassificationEvaluator.compute.device",description:`<strong>device</strong> (<code>int</code>, defaults to <code>None</code>) &#x2014;
Device ordinal for CPU/GPU support of the pipeline. Setting this to -1 will leverage CPU, a positive
integer will run the model on the associated CUDA device ID. If<code>None</code> is provided it will be inferred and
CUDA:0 used if available, CPU otherwise.`,name:"device"},{anchor:"evaluate.TextClassificationEvaluator.compute.random_state",description:`<strong>random_state</strong> (<code>int</code>, <em>optional</em>, defaults to <code>None</code>) &#x2014;
The <code>random_state</code> value passed to <code>bootstrap</code> if <code>&quot;bootstrap&quot;</code> strategy is chosen. Useful for
debugging.`,name:"random_state"}],source:"https://github.com/huggingface/evaluate/blob/vr_36/src/evaluate/evaluator/text_classification.py#L85",returnDescription:`
<p>A <code>Dict</code>. The keys represent metric keys calculated for the <code>metric</code> spefied in function arguments. For the
<code>"simple"</code> strategy, the value is the metric score. For the <code>"bootstrap"</code> strategy, the value is a <code>Dict</code>
containing the score, the confidence interval and the standard error calculated for each metric key.</p>
`}}),ye=new Ce({props:{anchor:"evaluate.TextClassificationEvaluator.compute.example",$$slots:{default:[nr]},$$scope:{ctx:x}}}),tt=new ct({}),at=new N({props:{name:"class evaluate.TokenClassificationEvaluator",anchor:"evaluate.TokenClassificationEvaluator",parameters:[{name:"task",val:" = 'token-classification'"},{name:"default_metric_name",val:" = None"}],source:"https://github.com/huggingface/evaluate/blob/vr_36/src/evaluate/evaluator/token_classification.py#L87"}}),ot=new N({props:{name:"compute",anchor:"evaluate.TokenClassificationEvaluator.compute",parameters:[{name:"model_or_pipeline",val:": typing.Union[str, ForwardRef('Pipeline'), typing.Callable, ForwardRef('PreTrainedModel'), ForwardRef('TFPreTrainedModel')] = None"},{name:"data",val:": typing.Union[str, datasets.arrow_dataset.Dataset] = None"},{name:"metric",val:": typing.Union[str, evaluate.module.EvaluationModule] = None"},{name:"tokenizer",val:": typing.Union[str, ForwardRef('PreTrainedTokenizer'), NoneType] = None"},{name:"strategy",val:": typing.Literal['simple', 'bootstrap'] = 'simple'"},{name:"confidence_level",val:": float = 0.95"},{name:"n_resamples",val:": int = 9999"},{name:"device",val:": typing.Optional[int] = None"},{name:"random_state",val:": typing.Optional[int] = None"},{name:"input_column",val:": str = 'tokens'"},{name:"label_column",val:": str = 'ner_tags'"},{name:"join_by",val:": typing.Optional[str] = ' '"}],parametersDescription:[{anchor:"evaluate.TokenClassificationEvaluator.compute.model_or_pipeline",description:`<strong>model_or_pipeline</strong> (<code>str</code> or <code>Pipeline</code> or <code>Callable</code> or <code>PreTrainedModel</code> or <code>TFPreTrainedModel</code>, defaults to <code>None</code>) &#x2014;
If the argument in not specified, we initialize the default pipeline for the task (in this case
<code>text-classification</code> or its alias - <code>sentiment-analysis</code>). If the argument is of the type <code>str</code> or
is a model instance, we use it to initialize a new <code>Pipeline</code> with the given model. Otherwise we assume the
argument specifies a pre-initialized pipeline.`,name:"model_or_pipeline"},{anchor:"evaluate.TokenClassificationEvaluator.compute.data",description:`<strong>data</strong> (<code>str</code> or <code>Dataset</code>, defaults to <code>None</code>) &#x2014;
Specifies the dataset we will run evaluation on. If it is of type <code>str</code>, we treat it as the dataset
name, and load it. Otherwise we assume it represents a pre-loaded dataset.`,name:"data"},{anchor:"evaluate.TokenClassificationEvaluator.compute.metric",description:`<strong>metric</strong> (<code>str</code> or <code>EvaluationModule</code>, defaults to <code>None</code>) &#x2014;
Specifies the metric we use in evaluator. If it is of type <code>str</code>, we treat it as the metric name, and
load it. Otherwise we assume it represents a pre-loaded metric.`,name:"metric"},{anchor:"evaluate.TokenClassificationEvaluator.compute.tokenizer",description:`<strong>tokenizer</strong> (<code>str</code> or <code>PreTrainedTokenizer</code>, <em>optional</em>, defaults to <code>None</code>) &#x2014;
Argument can be used to overwrite a default tokenizer if <code>model_or_pipeline</code> represents a model for
which we build a pipeline. If <code>model_or_pipeline</code> is <code>None</code> or a pre-initialized pipeline, we ignore
this argument.`,name:"tokenizer"},{anchor:"evaluate.TokenClassificationEvaluator.compute.strategy",description:`<strong>strategy</strong> (<code>Literal[&quot;simple&quot;, &quot;bootstrap&quot;]</code>, defaults to &#x201C;simple&#x201D;) &#x2014;
specifies the evaluation strategy. Possible values are:<ul>
<li><code>&quot;simple&quot;</code> - we evaluate the metric and return the scores.</li>
<li><code>&quot;bootstrap&quot;</code> - on top of computing the metric scores, we calculate the confidence interval for each
of the returned metric keys, using <code>scipy</code>&#x2019;s <code>bootstrap</code> method
<a href="https://docs.scipy.org/doc/scipy/reference/generated/scipy.stats.bootstrap.html" rel="nofollow">https://docs.scipy.org/doc/scipy/reference/generated/scipy.stats.bootstrap.html</a>.</li>
</ul>`,name:"strategy"},{anchor:"evaluate.TokenClassificationEvaluator.compute.confidence_level",description:`<strong>confidence_level</strong> (<code>float</code>, defaults to <code>0.95</code>) &#x2014;
The <code>confidence_level</code> value passed to <code>bootstrap</code> if <code>&quot;bootstrap&quot;</code> strategy is chosen.`,name:"confidence_level"},{anchor:"evaluate.TokenClassificationEvaluator.compute.n_resamples",description:`<strong>n_resamples</strong> (<code>int</code>, defaults to <code>9999</code>) &#x2014;
The <code>n_resamples</code> value passed to <code>bootstrap</code> if <code>&quot;bootstrap&quot;</code> strategy is chosen.`,name:"n_resamples"},{anchor:"evaluate.TokenClassificationEvaluator.compute.device",description:`<strong>device</strong> (<code>int</code>, defaults to <code>None</code>) &#x2014;
Device ordinal for CPU/GPU support of the pipeline. Setting this to -1 will leverage CPU, a positive
integer will run the model on the associated CUDA device ID. If<code>None</code> is provided it will be inferred and
CUDA:0 used if available, CPU otherwise.`,name:"device"},{anchor:"evaluate.TokenClassificationEvaluator.compute.random_state",description:`<strong>random_state</strong> (<code>int</code>, <em>optional</em>, defaults to <code>None</code>) &#x2014;
The <code>random_state</code> value passed to <code>bootstrap</code> if <code>&quot;bootstrap&quot;</code> strategy is chosen. Useful for
debugging.`,name:"random_state"}],source:"https://github.com/huggingface/evaluate/blob/vr_36/src/evaluate/evaluator/token_classification.py#L210",returnDescription:`
<p>A <code>Dict</code>. The keys represent metric keys calculated for the <code>metric</code> spefied in function arguments. For the
<code>"simple"</code> strategy, the value is the metric score. For the <code>"bootstrap"</code> strategy, the value is a <code>Dict</code>
containing the score, the confidence interval and the standard error calculated for each metric key.</p>
`}}),qe=new Ce({props:{anchor:"evaluate.TokenClassificationEvaluator.compute.example",$$slots:{default:[rr]},$$scope:{ctx:x}}}),ke=new Zo({props:{$$slots:{default:[ir]},$$scope:{ctx:x}}}),xe=new Zo({props:{warning:!0,$$slots:{default:[pr]},$$scope:{ctx:x}}}),{c(){l=o("meta"),v=u(),i=o("h1"),d=o("a"),_=o("span"),b(s.$$.fragment),f=u(),F=o("span"),T=c("Evaluator"),U=u(),K=o("p"),Va=c("The evaluator classes for automatic evaluation."),ma=u(),W=o("h2"),ne=o("a"),qt=o("span"),b(De.$$.fragment),Ba=u(),kt=o("span"),Ga=c("Evaluator classes"),fa=u(),pt=o("p"),Ha=c("The main entry point for using the evaluator:"),ga=u(),L=o("div"),b(Ie.$$.fragment),Ya=u(),O=o("p"),Ja=c("Utility factory method to build an "),dt=o("a"),Ka=c("Evaluator"),Wa=c(`.
Evaluators encapsulate a task and a default metric name. They leverage `),xt=o("code"),Xa=c("pipeline"),Za=c(" functionalify from "),jt=o("code"),es=c("transformers"),ts=c(`
to simplify the evaluation of multiple combinations of models, datasets and metrics for a given task.`),as=u(),b(re.$$.fragment),ha=u(),ut=o("p"),ss=c("The base class for all evaluator classes:"),va=u(),j=o("div"),b(Ae.$$.fragment),os=u(),Tt=o("p"),ns=c(`The Evaluator class is the class from which all evaluators inherit. Refer to this class for methods shared across
different evaluators.
Base class implementing evaluator operations.`),rs=u(),le=o("div"),b(Fe.$$.fragment),ls=u(),Ct=o("p"),is=c("Ensure the columns required for the evaluation are present in the dataset."),cs=u(),ie=o("div"),b(Ue.$$.fragment),ps=u(),Pt=o("p"),ds=c("Compute and return metrics."),us=u(),ce=o("div"),b(ze.$$.fragment),ms=u(),Se=o("p"),fs=c("A core method of the "),Nt=o("code"),gs=c("Evaluator"),hs=c(" class, which processes the pipeline outputs for compatibility with the metric."),vs=u(),pe=o("div"),b(Le.$$.fragment),_s=u(),Dt=o("p"),$s=c("Prepare data."),bs=u(),de=o("div"),b(Oe.$$.fragment),ws=u(),It=o("p"),ys=c("Prepare metric."),Es=u(),ue=o("div"),b(Me.$$.fragment),qs=u(),At=o("p"),ks=c("Prepare pipeline."),_a=u(),X=o("h2"),me=o("a"),Ft=o("span"),b(Re.$$.fragment),xs=u(),Ut=o("span"),js=c("The task specific evaluators"),$a=u(),Z=o("h3"),fe=o("a"),zt=o("span"),b(Qe.$$.fragment),Ts=u(),St=o("span"),Cs=c("ImageClassificationEvaluator"),ba=u(),M=o("div"),b(Ve.$$.fragment),Ps=u(),R=o("p"),Ns=c(`Image classification evaluator.
This image classification evaluator can currently be loaded from `),mt=o("a"),Ds=c("evaluator()"),Is=c(` using the default task name
`),Lt=o("code"),As=c("image-classification"),Fs=c(`.
Methods in this class assume a data format compatible with the `),Ot=o("code"),Us=c("ImageClassificationPipeline"),zs=c("."),Ss=u(),V=o("div"),b(Be.$$.fragment),Ls=u(),Mt=o("p"),Os=c("Compute the metric for a given pipeline and dataset combination."),Ms=u(),b(ge.$$.fragment),wa=u(),ee=o("h3"),he=o("a"),Rt=o("span"),b(Ge.$$.fragment),Rs=u(),Qt=o("span"),Qs=c("QuestionAnsweringEvaluator"),ya=u(),D=o("div"),b(He.$$.fragment),Vs=u(),Ye=o("p"),Bs=c(`Question answering evaluator. This evaluator handles
`),ve=o("a"),Vt=o("strong"),Gs=c("extractive"),Hs=c(" question answering"),Ys=c(`,
where the answer to the question is extracted from a context.`),Js=u(),te=o("p"),Ks=c("This question answering evaluator can currently be loaded from "),ft=o("a"),Ws=c("evaluator()"),Xs=c(` using the default task name
`),Bt=o("code"),Zs=c("question-answering"),eo=c("."),to=u(),Je=o("p"),ao=c(`Methods in this class assume a data format compatible with the
`),Ke=o("a"),Gt=o("code"),so=c("QuestionAnsweringPipeline"),oo=c("."),no=u(),A=o("div"),b(We.$$.fragment),ro=u(),Ht=o("p"),lo=c("Compute the metric for a given pipeline and dataset combination."),io=u(),b(_e.$$.fragment),co=u(),b($e.$$.fragment),po=u(),b(be.$$.fragment),Ea=u(),ae=o("h3"),we=o("a"),Yt=o("span"),b(Xe.$$.fragment),uo=u(),Jt=o("span"),mo=c("TextClassificationEvaluator"),qa=u(),Q=o("div"),b(Ze.$$.fragment),fo=u(),z=o("p"),go=c(`Text classification evaluator.
This text classification evaluator can currently be loaded from `),gt=o("a"),ho=c("evaluator()"),vo=c(` using the default task name
`),Kt=o("code"),_o=c("text-classification"),$o=c(" or with a "),Wt=o("code"),bo=c('"sentiment-analysis"'),wo=c(` alias.
Methods in this class assume a data format compatible with the `),Xt=o("code"),yo=c("TextClassificationPipeline"),Eo=c(` - a single textual
feature as input and a categorical label as output.`),qo=u(),B=o("div"),b(et.$$.fragment),ko=u(),Zt=o("p"),xo=c("Compute the metric for a given pipeline and dataset combination."),jo=u(),b(ye.$$.fragment),ka=u(),se=o("h3"),Ee=o("a"),ea=o("span"),b(tt.$$.fragment),To=u(),ta=o("span"),Co=c("TokenClassificationEvaluator"),xa=u(),I=o("div"),b(at.$$.fragment),Po=u(),aa=o("p"),No=c("Token classification evaluator."),Do=u(),oe=o("p"),Io=c("This token classification evaluator can currently be loaded from "),ht=o("a"),Ao=c("evaluator()"),Fo=c(` using the default task name
`),sa=o("code"),Uo=c("token-classification"),zo=c("."),So=u(),st=o("p"),Lo=c("Methods in this class assume a data format compatible with the "),oa=o("code"),Oo=c("TokenClassificationPipeline"),Mo=c("."),Ro=u(),C=o("div"),b(ot.$$.fragment),Qo=u(),na=o("p"),Vo=c("Compute the metric for a given pipeline and dataset combination."),Bo=u(),nt=o("p"),Go=c("The dataset input and label columns are expected to be formatted as a list of words and a list of labels respectively, following "),rt=o("a"),Ho=c("conll2003 dataset"),Yo=c(". Datasets whose inputs are single strings, and labels are a list of offset are not supported."),Jo=u(),b(qe.$$.fragment),Ko=u(),b(ke.$$.fragment),Wo=u(),b(xe.$$.fragment),this.h()},l(t){const h=Xn('[data-svelte="svelte-1phssyn"]',document.head);l=n(h,"META",{name:!0,content:!0}),h.forEach(a),v=m(t),i=n(t,"H1",{class:!0});var lt=r(i);d=n(lt,"A",{id:!0,class:!0,href:!0});var ra=r(d);_=n(ra,"SPAN",{});var la=r(_);w(s.$$.fragment,la),la.forEach(a),ra.forEach(a),f=m(lt),F=n(lt,"SPAN",{});var ia=r(F);T=p(ia,"Evaluator"),ia.forEach(a),lt.forEach(a),U=m(t),K=n(t,"P",{});var ca=r(K);Va=p(ca,"The evaluator classes for automatic evaluation."),ca.forEach(a),ma=m(t),W=n(t,"H2",{class:!0});var it=r(W);ne=n(it,"A",{id:!0,class:!0,href:!0});var pa=r(ne);qt=n(pa,"SPAN",{});var da=r(qt);w(De.$$.fragment,da),da.forEach(a),pa.forEach(a),Ba=m(it),kt=n(it,"SPAN",{});var ua=r(kt);Ga=p(ua,"Evaluator classes"),ua.forEach(a),it.forEach(a),fa=m(t),pt=n(t,"P",{});var en=r(pt);Ha=p(en,"The main entry point for using the evaluator:"),en.forEach(a),ga=m(t),L=n(t,"DIV",{class:!0});var vt=r(L);w(Ie.$$.fragment,vt),Ya=m(vt),O=n(vt,"P",{});var je=r(O);Ja=p(je,"Utility factory method to build an "),dt=n(je,"A",{href:!0});var tn=r(dt);Ka=p(tn,"Evaluator"),tn.forEach(a),Wa=p(je,`.
Evaluators encapsulate a task and a default metric name. They leverage `),xt=n(je,"CODE",{});var an=r(xt);Xa=p(an,"pipeline"),an.forEach(a),Za=p(je," functionalify from "),jt=n(je,"CODE",{});var sn=r(jt);es=p(sn,"transformers"),sn.forEach(a),ts=p(je,`
to simplify the evaluation of multiple combinations of models, datasets and metrics for a given task.`),je.forEach(a),as=m(vt),w(re.$$.fragment,vt),vt.forEach(a),ha=m(t),ut=n(t,"P",{});var on=r(ut);ss=p(on,"The base class for all evaluator classes:"),on.forEach(a),va=m(t),j=n(t,"DIV",{class:!0});var P=r(j);w(Ae.$$.fragment,P),os=m(P),Tt=n(P,"P",{});var nn=r(Tt);ns=p(nn,`The Evaluator class is the class from which all evaluators inherit. Refer to this class for methods shared across
different evaluators.
Base class implementing evaluator operations.`),nn.forEach(a),rs=m(P),le=n(P,"DIV",{class:!0});var Ta=r(le);w(Fe.$$.fragment,Ta),ls=m(Ta),Ct=n(Ta,"P",{});var rn=r(Ct);is=p(rn,"Ensure the columns required for the evaluation are present in the dataset."),rn.forEach(a),Ta.forEach(a),cs=m(P),ie=n(P,"DIV",{class:!0});var Ca=r(ie);w(Ue.$$.fragment,Ca),ps=m(Ca),Pt=n(Ca,"P",{});var ln=r(Pt);ds=p(ln,"Compute and return metrics."),ln.forEach(a),Ca.forEach(a),us=m(P),ce=n(P,"DIV",{class:!0});var Pa=r(ce);w(ze.$$.fragment,Pa),ms=m(Pa),Se=n(Pa,"P",{});var Na=r(Se);fs=p(Na,"A core method of the "),Nt=n(Na,"CODE",{});var cn=r(Nt);gs=p(cn,"Evaluator"),cn.forEach(a),hs=p(Na," class, which processes the pipeline outputs for compatibility with the metric."),Na.forEach(a),Pa.forEach(a),vs=m(P),pe=n(P,"DIV",{class:!0});var Da=r(pe);w(Le.$$.fragment,Da),_s=m(Da),Dt=n(Da,"P",{});var pn=r(Dt);$s=p(pn,"Prepare data."),pn.forEach(a),Da.forEach(a),bs=m(P),de=n(P,"DIV",{class:!0});var Ia=r(de);w(Oe.$$.fragment,Ia),ws=m(Ia),It=n(Ia,"P",{});var dn=r(It);ys=p(dn,"Prepare metric."),dn.forEach(a),Ia.forEach(a),Es=m(P),ue=n(P,"DIV",{class:!0});var Aa=r(ue);w(Me.$$.fragment,Aa),qs=m(Aa),At=n(Aa,"P",{});var un=r(At);ks=p(un,"Prepare pipeline."),un.forEach(a),Aa.forEach(a),P.forEach(a),_a=m(t),X=n(t,"H2",{class:!0});var Fa=r(X);me=n(Fa,"A",{id:!0,class:!0,href:!0});var mn=r(me);Ft=n(mn,"SPAN",{});var fn=r(Ft);w(Re.$$.fragment,fn),fn.forEach(a),mn.forEach(a),xs=m(Fa),Ut=n(Fa,"SPAN",{});var gn=r(Ut);js=p(gn,"The task specific evaluators"),gn.forEach(a),Fa.forEach(a),$a=m(t),Z=n(t,"H3",{class:!0});var Ua=r(Z);fe=n(Ua,"A",{id:!0,class:!0,href:!0});var hn=r(fe);zt=n(hn,"SPAN",{});var vn=r(zt);w(Qe.$$.fragment,vn),vn.forEach(a),hn.forEach(a),Ts=m(Ua),St=n(Ua,"SPAN",{});var _n=r(St);Cs=p(_n,"ImageClassificationEvaluator"),_n.forEach(a),Ua.forEach(a),ba=m(t),M=n(t,"DIV",{class:!0});var _t=r(M);w(Ve.$$.fragment,_t),Ps=m(_t),R=n(_t,"P",{});var Te=r(R);Ns=p(Te,`Image classification evaluator.
This image classification evaluator can currently be loaded from `),mt=n(Te,"A",{href:!0});var $n=r(mt);Ds=p($n,"evaluator()"),$n.forEach(a),Is=p(Te,` using the default task name
`),Lt=n(Te,"CODE",{});var bn=r(Lt);As=p(bn,"image-classification"),bn.forEach(a),Fs=p(Te,`.
Methods in this class assume a data format compatible with the `),Ot=n(Te,"CODE",{});var wn=r(Ot);Us=p(wn,"ImageClassificationPipeline"),wn.forEach(a),zs=p(Te,"."),Te.forEach(a),Ss=m(_t),V=n(_t,"DIV",{class:!0});var $t=r(V);w(Be.$$.fragment,$t),Ls=m($t),Mt=n($t,"P",{});var yn=r(Mt);Os=p(yn,"Compute the metric for a given pipeline and dataset combination."),yn.forEach(a),Ms=m($t),w(ge.$$.fragment,$t),$t.forEach(a),_t.forEach(a),wa=m(t),ee=n(t,"H3",{class:!0});var za=r(ee);he=n(za,"A",{id:!0,class:!0,href:!0});var En=r(he);Rt=n(En,"SPAN",{});var qn=r(Rt);w(Ge.$$.fragment,qn),qn.forEach(a),En.forEach(a),Rs=m(za),Qt=n(za,"SPAN",{});var kn=r(Qt);Qs=p(kn,"QuestionAnsweringEvaluator"),kn.forEach(a),za.forEach(a),ya=m(t),D=n(t,"DIV",{class:!0});var G=r(D);w(He.$$.fragment,G),Vs=m(G),Ye=n(G,"P",{});var Sa=r(Ye);Bs=p(Sa,`Question answering evaluator. This evaluator handles
`),ve=n(Sa,"A",{href:!0,rel:!0});var Xo=r(ve);Vt=n(Xo,"STRONG",{});var xn=r(Vt);Gs=p(xn,"extractive"),xn.forEach(a),Hs=p(Xo," question answering"),Xo.forEach(a),Ys=p(Sa,`,
where the answer to the question is extracted from a context.`),Sa.forEach(a),Js=m(G),te=n(G,"P",{});var bt=r(te);Ks=p(bt,"This question answering evaluator can currently be loaded from "),ft=n(bt,"A",{href:!0});var jn=r(ft);Ws=p(jn,"evaluator()"),jn.forEach(a),Xs=p(bt,` using the default task name
`),Bt=n(bt,"CODE",{});var Tn=r(Bt);Zs=p(Tn,"question-answering"),Tn.forEach(a),eo=p(bt,"."),bt.forEach(a),to=m(G),Je=n(G,"P",{});var La=r(Je);ao=p(La,`Methods in this class assume a data format compatible with the
`),Ke=n(La,"A",{href:!0,rel:!0});var Cn=r(Ke);Gt=n(Cn,"CODE",{});var Pn=r(Gt);so=p(Pn,"QuestionAnsweringPipeline"),Pn.forEach(a),Cn.forEach(a),oo=p(La,"."),La.forEach(a),no=m(G),A=n(G,"DIV",{class:!0});var H=r(A);w(We.$$.fragment,H),ro=m(H),Ht=n(H,"P",{});var Nn=r(Ht);lo=p(Nn,"Compute the metric for a given pipeline and dataset combination."),Nn.forEach(a),io=m(H),w(_e.$$.fragment,H),co=m(H),w($e.$$.fragment,H),po=m(H),w(be.$$.fragment,H),H.forEach(a),G.forEach(a),Ea=m(t),ae=n(t,"H3",{class:!0});var Oa=r(ae);we=n(Oa,"A",{id:!0,class:!0,href:!0});var Dn=r(we);Yt=n(Dn,"SPAN",{});var In=r(Yt);w(Xe.$$.fragment,In),In.forEach(a),Dn.forEach(a),uo=m(Oa),Jt=n(Oa,"SPAN",{});var An=r(Jt);mo=p(An,"TextClassificationEvaluator"),An.forEach(a),Oa.forEach(a),qa=m(t),Q=n(t,"DIV",{class:!0});var wt=r(Q);w(Ze.$$.fragment,wt),fo=m(wt),z=n(wt,"P",{});var Y=r(z);go=p(Y,`Text classification evaluator.
This text classification evaluator can currently be loaded from `),gt=n(Y,"A",{href:!0});var Fn=r(gt);ho=p(Fn,"evaluator()"),Fn.forEach(a),vo=p(Y,` using the default task name
`),Kt=n(Y,"CODE",{});var Un=r(Kt);_o=p(Un,"text-classification"),Un.forEach(a),$o=p(Y," or with a "),Wt=n(Y,"CODE",{});var zn=r(Wt);bo=p(zn,'"sentiment-analysis"'),zn.forEach(a),wo=p(Y,` alias.
Methods in this class assume a data format compatible with the `),Xt=n(Y,"CODE",{});var Sn=r(Xt);yo=p(Sn,"TextClassificationPipeline"),Sn.forEach(a),Eo=p(Y,` - a single textual
feature as input and a categorical label as output.`),Y.forEach(a),qo=m(wt),B=n(wt,"DIV",{class:!0});var yt=r(B);w(et.$$.fragment,yt),ko=m(yt),Zt=n(yt,"P",{});var Ln=r(Zt);xo=p(Ln,"Compute the metric for a given pipeline and dataset combination."),Ln.forEach(a),jo=m(yt),w(ye.$$.fragment,yt),yt.forEach(a),wt.forEach(a),ka=m(t),se=n(t,"H3",{class:!0});var Ma=r(se);Ee=n(Ma,"A",{id:!0,class:!0,href:!0});var On=r(Ee);ea=n(On,"SPAN",{});var Mn=r(ea);w(tt.$$.fragment,Mn),Mn.forEach(a),On.forEach(a),To=m(Ma),ta=n(Ma,"SPAN",{});var Rn=r(ta);Co=p(Rn,"TokenClassificationEvaluator"),Rn.forEach(a),Ma.forEach(a),xa=m(t),I=n(t,"DIV",{class:!0});var J=r(I);w(at.$$.fragment,J),Po=m(J),aa=n(J,"P",{});var Qn=r(aa);No=p(Qn,"Token classification evaluator."),Qn.forEach(a),Do=m(J),oe=n(J,"P",{});var Et=r(oe);Io=p(Et,"This token classification evaluator can currently be loaded from "),ht=n(Et,"A",{href:!0});var Vn=r(ht);Ao=p(Vn,"evaluator()"),Vn.forEach(a),Fo=p(Et,` using the default task name
`),sa=n(Et,"CODE",{});var Bn=r(sa);Uo=p(Bn,"token-classification"),Bn.forEach(a),zo=p(Et,"."),Et.forEach(a),So=m(J),st=n(J,"P",{});var Ra=r(st);Lo=p(Ra,"Methods in this class assume a data format compatible with the "),oa=n(Ra,"CODE",{});var Gn=r(oa);Oo=p(Gn,"TokenClassificationPipeline"),Gn.forEach(a),Mo=p(Ra,"."),Ra.forEach(a),Ro=m(J),C=n(J,"DIV",{class:!0});var S=r(C);w(ot.$$.fragment,S),Qo=m(S),na=n(S,"P",{});var Hn=r(na);Vo=p(Hn,"Compute the metric for a given pipeline and dataset combination."),Hn.forEach(a),Bo=m(S),nt=n(S,"P",{});var Qa=r(nt);Go=p(Qa,"The dataset input and label columns are expected to be formatted as a list of words and a list of labels respectively, following "),rt=n(Qa,"A",{href:!0,rel:!0});var Yn=r(rt);Ho=p(Yn,"conll2003 dataset"),Yn.forEach(a),Yo=p(Qa,". Datasets whose inputs are single strings, and labels are a list of offset are not supported."),Qa.forEach(a),Jo=m(S),w(qe.$$.fragment,S),Ko=m(S),w(ke.$$.fragment,S),Wo=m(S),w(xe.$$.fragment,S),S.forEach(a),J.forEach(a),this.h()},h(){g(l,"name","hf:doc:metadata"),g(l,"content",JSON.stringify(ur)),g(d,"id","evaluator"),g(d,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),g(d,"href","#evaluator"),g(i,"class","relative group"),g(ne,"id","evaluate.evaluator"),g(ne,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),g(ne,"href","#evaluate.evaluator"),g(W,"class","relative group"),g(dt,"href","/docs/evaluate/pr_36/en/package_reference/evaluator_classes#evaluate.Evaluator"),g(L,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),g(le,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),g(ie,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),g(ce,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),g(pe,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),g(de,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),g(ue,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),g(j,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),g(me,"id","the-task-specific-evaluators"),g(me,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),g(me,"href","#the-task-specific-evaluators"),g(X,"class","relative group"),g(fe,"id","evaluate.ImageClassificationEvaluator"),g(fe,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),g(fe,"href","#evaluate.ImageClassificationEvaluator"),g(Z,"class","relative group"),g(mt,"href","/docs/evaluate/pr_36/en/package_reference/evaluator_classes#evaluate.evaluator"),g(V,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),g(M,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),g(he,"id","evaluate.QuestionAnsweringEvaluator"),g(he,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),g(he,"href","#evaluate.QuestionAnsweringEvaluator"),g(ee,"class","relative group"),g(ve,"href","https://huggingface.co/docs/transformers/task_summary#extractive-question-answering"),g(ve,"rel","nofollow"),g(ft,"href","/docs/evaluate/pr_36/en/package_reference/evaluator_classes#evaluate.evaluator"),g(Ke,"href","https://huggingface.co/docs/transformers/en/main_classes/pipelines#transformers.QuestionAnsweringPipeline"),g(Ke,"rel","nofollow"),g(A,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),g(D,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),g(we,"id","evaluate.TextClassificationEvaluator"),g(we,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),g(we,"href","#evaluate.TextClassificationEvaluator"),g(ae,"class","relative group"),g(gt,"href","/docs/evaluate/pr_36/en/package_reference/evaluator_classes#evaluate.evaluator"),g(B,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),g(Q,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),g(Ee,"id","evaluate.TokenClassificationEvaluator"),g(Ee,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),g(Ee,"href","#evaluate.TokenClassificationEvaluator"),g(se,"class","relative group"),g(ht,"href","/docs/evaluate/pr_36/en/package_reference/evaluator_classes#evaluate.evaluator"),g(rt,"href","https://huggingface.co/datasets/conll2003"),g(rt,"rel","nofollow"),g(C,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),g(I,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8")},m(t,h){e(document.head,l),$(t,v,h),$(t,i,h),e(i,d),e(d,_),y(s,_,null),e(i,f),e(i,F),e(F,T),$(t,U,h),$(t,K,h),e(K,Va),$(t,ma,h),$(t,W,h),e(W,ne),e(ne,qt),y(De,qt,null),e(W,Ba),e(W,kt),e(kt,Ga),$(t,fa,h),$(t,pt,h),e(pt,Ha),$(t,ga,h),$(t,L,h),y(Ie,L,null),e(L,Ya),e(L,O),e(O,Ja),e(O,dt),e(dt,Ka),e(O,Wa),e(O,xt),e(xt,Xa),e(O,Za),e(O,jt),e(jt,es),e(O,ts),e(L,as),y(re,L,null),$(t,ha,h),$(t,ut,h),e(ut,ss),$(t,va,h),$(t,j,h),y(Ae,j,null),e(j,os),e(j,Tt),e(Tt,ns),e(j,rs),e(j,le),y(Fe,le,null),e(le,ls),e(le,Ct),e(Ct,is),e(j,cs),e(j,ie),y(Ue,ie,null),e(ie,ps),e(ie,Pt),e(Pt,ds),e(j,us),e(j,ce),y(ze,ce,null),e(ce,ms),e(ce,Se),e(Se,fs),e(Se,Nt),e(Nt,gs),e(Se,hs),e(j,vs),e(j,pe),y(Le,pe,null),e(pe,_s),e(pe,Dt),e(Dt,$s),e(j,bs),e(j,de),y(Oe,de,null),e(de,ws),e(de,It),e(It,ys),e(j,Es),e(j,ue),y(Me,ue,null),e(ue,qs),e(ue,At),e(At,ks),$(t,_a,h),$(t,X,h),e(X,me),e(me,Ft),y(Re,Ft,null),e(X,xs),e(X,Ut),e(Ut,js),$(t,$a,h),$(t,Z,h),e(Z,fe),e(fe,zt),y(Qe,zt,null),e(Z,Ts),e(Z,St),e(St,Cs),$(t,ba,h),$(t,M,h),y(Ve,M,null),e(M,Ps),e(M,R),e(R,Ns),e(R,mt),e(mt,Ds),e(R,Is),e(R,Lt),e(Lt,As),e(R,Fs),e(R,Ot),e(Ot,Us),e(R,zs),e(M,Ss),e(M,V),y(Be,V,null),e(V,Ls),e(V,Mt),e(Mt,Os),e(V,Ms),y(ge,V,null),$(t,wa,h),$(t,ee,h),e(ee,he),e(he,Rt),y(Ge,Rt,null),e(ee,Rs),e(ee,Qt),e(Qt,Qs),$(t,ya,h),$(t,D,h),y(He,D,null),e(D,Vs),e(D,Ye),e(Ye,Bs),e(Ye,ve),e(ve,Vt),e(Vt,Gs),e(ve,Hs),e(Ye,Ys),e(D,Js),e(D,te),e(te,Ks),e(te,ft),e(ft,Ws),e(te,Xs),e(te,Bt),e(Bt,Zs),e(te,eo),e(D,to),e(D,Je),e(Je,ao),e(Je,Ke),e(Ke,Gt),e(Gt,so),e(Je,oo),e(D,no),e(D,A),y(We,A,null),e(A,ro),e(A,Ht),e(Ht,lo),e(A,io),y(_e,A,null),e(A,co),y($e,A,null),e(A,po),y(be,A,null),$(t,Ea,h),$(t,ae,h),e(ae,we),e(we,Yt),y(Xe,Yt,null),e(ae,uo),e(ae,Jt),e(Jt,mo),$(t,qa,h),$(t,Q,h),y(Ze,Q,null),e(Q,fo),e(Q,z),e(z,go),e(z,gt),e(gt,ho),e(z,vo),e(z,Kt),e(Kt,_o),e(z,$o),e(z,Wt),e(Wt,bo),e(z,wo),e(z,Xt),e(Xt,yo),e(z,Eo),e(Q,qo),e(Q,B),y(et,B,null),e(B,ko),e(B,Zt),e(Zt,xo),e(B,jo),y(ye,B,null),$(t,ka,h),$(t,se,h),e(se,Ee),e(Ee,ea),y(tt,ea,null),e(se,To),e(se,ta),e(ta,Co),$(t,xa,h),$(t,I,h),y(at,I,null),e(I,Po),e(I,aa),e(aa,No),e(I,Do),e(I,oe),e(oe,Io),e(oe,ht),e(ht,Ao),e(oe,Fo),e(oe,sa),e(sa,Uo),e(oe,zo),e(I,So),e(I,st),e(st,Lo),e(st,oa),e(oa,Oo),e(st,Mo),e(I,Ro),e(I,C),y(ot,C,null),e(C,Qo),e(C,na),e(na,Vo),e(C,Bo),e(C,nt),e(nt,Go),e(nt,rt),e(rt,Ho),e(nt,Yo),e(C,Jo),y(qe,C,null),e(C,Ko),y(ke,C,null),e(C,Wo),y(xe,C,null),ja=!0},p(t,[h]){const lt={};h&2&&(lt.$$scope={dirty:h,ctx:t}),re.$set(lt);const ra={};h&2&&(ra.$$scope={dirty:h,ctx:t}),ge.$set(ra);const la={};h&2&&(la.$$scope={dirty:h,ctx:t}),_e.$set(la);const ia={};h&2&&(ia.$$scope={dirty:h,ctx:t}),$e.$set(ia);const ca={};h&2&&(ca.$$scope={dirty:h,ctx:t}),be.$set(ca);const it={};h&2&&(it.$$scope={dirty:h,ctx:t}),ye.$set(it);const pa={};h&2&&(pa.$$scope={dirty:h,ctx:t}),qe.$set(pa);const da={};h&2&&(da.$$scope={dirty:h,ctx:t}),ke.$set(da);const ua={};h&2&&(ua.$$scope={dirty:h,ctx:t}),xe.$set(ua)},i(t){ja||(E(s.$$.fragment,t),E(De.$$.fragment,t),E(Ie.$$.fragment,t),E(re.$$.fragment,t),E(Ae.$$.fragment,t),E(Fe.$$.fragment,t),E(Ue.$$.fragment,t),E(ze.$$.fragment,t),E(Le.$$.fragment,t),E(Oe.$$.fragment,t),E(Me.$$.fragment,t),E(Re.$$.fragment,t),E(Qe.$$.fragment,t),E(Ve.$$.fragment,t),E(Be.$$.fragment,t),E(ge.$$.fragment,t),E(Ge.$$.fragment,t),E(He.$$.fragment,t),E(We.$$.fragment,t),E(_e.$$.fragment,t),E($e.$$.fragment,t),E(be.$$.fragment,t),E(Xe.$$.fragment,t),E(Ze.$$.fragment,t),E(et.$$.fragment,t),E(ye.$$.fragment,t),E(tt.$$.fragment,t),E(at.$$.fragment,t),E(ot.$$.fragment,t),E(qe.$$.fragment,t),E(ke.$$.fragment,t),E(xe.$$.fragment,t),ja=!0)},o(t){q(s.$$.fragment,t),q(De.$$.fragment,t),q(Ie.$$.fragment,t),q(re.$$.fragment,t),q(Ae.$$.fragment,t),q(Fe.$$.fragment,t),q(Ue.$$.fragment,t),q(ze.$$.fragment,t),q(Le.$$.fragment,t),q(Oe.$$.fragment,t),q(Me.$$.fragment,t),q(Re.$$.fragment,t),q(Qe.$$.fragment,t),q(Ve.$$.fragment,t),q(Be.$$.fragment,t),q(ge.$$.fragment,t),q(Ge.$$.fragment,t),q(He.$$.fragment,t),q(We.$$.fragment,t),q(_e.$$.fragment,t),q($e.$$.fragment,t),q(be.$$.fragment,t),q(Xe.$$.fragment,t),q(Ze.$$.fragment,t),q(et.$$.fragment,t),q(ye.$$.fragment,t),q(tt.$$.fragment,t),q(at.$$.fragment,t),q(ot.$$.fragment,t),q(qe.$$.fragment,t),q(ke.$$.fragment,t),q(xe.$$.fragment,t),ja=!1},d(t){a(l),t&&a(v),t&&a(i),k(s),t&&a(U),t&&a(K),t&&a(ma),t&&a(W),k(De),t&&a(fa),t&&a(pt),t&&a(ga),t&&a(L),k(Ie),k(re),t&&a(ha),t&&a(ut),t&&a(va),t&&a(j),k(Ae),k(Fe),k(Ue),k(ze),k(Le),k(Oe),k(Me),t&&a(_a),t&&a(X),k(Re),t&&a($a),t&&a(Z),k(Qe),t&&a(ba),t&&a(M),k(Ve),k(Be),k(ge),t&&a(wa),t&&a(ee),k(Ge),t&&a(ya),t&&a(D),k(He),k(We),k(_e),k($e),k(be),t&&a(Ea),t&&a(ae),k(Xe),t&&a(qa),t&&a(Q),k(Ze),k(et),k(ye),t&&a(ka),t&&a(se),k(tt),t&&a(xa),t&&a(I),k(at),k(ot),k(qe),k(ke),k(xe)}}}const ur={local:"evaluator",sections:[{local:"evaluate.evaluator",title:"Evaluator classes"},{local:"the-task-specific-evaluators",sections:[{local:"evaluate.ImageClassificationEvaluator",title:"ImageClassificationEvaluator"},{local:"evaluate.QuestionAnsweringEvaluator",title:"QuestionAnsweringEvaluator"},{local:"evaluate.TextClassificationEvaluator",title:"TextClassificationEvaluator"},{local:"evaluate.TokenClassificationEvaluator",title:"TokenClassificationEvaluator"}],title:"The task specific evaluators"}],title:"Evaluator"};function mr(x){return Zn(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class br extends Jn{constructor(l){super();Kn(this,l,mr,dr,Wn,{})}}export{br as default,ur as metadata};
