import{S as Ei,i as xi,s as qi,e as o,k as d,w as _,t as l,M as ki,c as s,d as a,m as p,a as n,x as b,h as i,b as m,G as e,g as v,y as w,q as $,o as y,B as E,v as Ti,L as We}from"../../chunks/vendor-hf-doc-builder.js";import{T as nl}from"../../chunks/Tip-hf-doc-builder.js";import{D as j}from"../../chunks/Docstring-hf-doc-builder.js";import{C as Xe}from"../../chunks/CodeBlock-hf-doc-builder.js";import{I as ae}from"../../chunks/IconCopyLink-hf-doc-builder.js";import{E as Ke}from"../../chunks/ExampleCodeBlock-hf-doc-builder.js";function ji(T){let c,x,u,g,q;return g=new Xe({props:{code:`from evaluate import evaluator
# Sentiment analysis evaluator
evaluator("sentiment-analysis")`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> evaluate <span class="hljs-keyword">import</span> evaluator
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-comment"># Sentiment analysis evaluator</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>evaluator(<span class="hljs-string">&quot;sentiment-analysis&quot;</span>)`}}),{c(){c=o("p"),x=l("Examples:"),u=d(),_(g.$$.fragment)},l(r){c=s(r,"P",{});var h=n(c);x=i(h,"Examples:"),h.forEach(a),u=p(r),b(g.$$.fragment,r)},m(r,h){v(r,c,h),e(c,x),v(r,u,h),w(g,r,h),q=!0},p:We,i(r){q||($(g.$$.fragment,r),q=!0)},o(r){y(g.$$.fragment,r),q=!1},d(r){r&&a(c),r&&a(u),E(g,r)}}}function Ni(T){let c,x,u,g,q;return g=new Xe({props:{code:`from evaluate import evaluator
from datasets import load_dataset
task_evaluator = evaluator("image-classification")
data = load_dataset("beans", split="test[:40]")
results = task_evaluator.compute(
    model_or_pipeline="nateraw/vit-base-beans",
    data=data,
    label_column="labels",
    metric="accuracy",
    label_mapping={'angular_leaf_spot': 0, 'bean_rust': 1, 'healthy': 2},
    strategy="bootstrap"
)`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> evaluate <span class="hljs-keyword">import</span> evaluator
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> datasets <span class="hljs-keyword">import</span> load_dataset
<span class="hljs-meta">&gt;&gt;&gt; </span>task_evaluator = evaluator(<span class="hljs-string">&quot;image-classification&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>data = load_dataset(<span class="hljs-string">&quot;beans&quot;</span>, split=<span class="hljs-string">&quot;test[:40]&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>results = task_evaluator.compute(
<span class="hljs-meta">&gt;&gt;&gt; </span>    model_or_pipeline=<span class="hljs-string">&quot;nateraw/vit-base-beans&quot;</span>,
<span class="hljs-meta">&gt;&gt;&gt; </span>    data=data,
<span class="hljs-meta">&gt;&gt;&gt; </span>    label_column=<span class="hljs-string">&quot;labels&quot;</span>,
<span class="hljs-meta">&gt;&gt;&gt; </span>    metric=<span class="hljs-string">&quot;accuracy&quot;</span>,
<span class="hljs-meta">&gt;&gt;&gt; </span>    label_mapping={<span class="hljs-string">&#x27;angular_leaf_spot&#x27;</span>: <span class="hljs-number">0</span>, <span class="hljs-string">&#x27;bean_rust&#x27;</span>: <span class="hljs-number">1</span>, <span class="hljs-string">&#x27;healthy&#x27;</span>: <span class="hljs-number">2</span>},
<span class="hljs-meta">&gt;&gt;&gt; </span>    strategy=<span class="hljs-string">&quot;bootstrap&quot;</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>)`}}),{c(){c=o("p"),x=l("Examples:"),u=d(),_(g.$$.fragment)},l(r){c=s(r,"P",{});var h=n(c);x=i(h,"Examples:"),h.forEach(a),u=p(r),b(g.$$.fragment,r)},m(r,h){v(r,c,h),e(c,x),v(r,u,h),w(g,r,h),q=!0},p:We,i(r){q||($(g.$$.fragment,r),q=!0)},o(r){y(g.$$.fragment,r),q=!1},d(r){r&&a(c),r&&a(u),E(g,r)}}}function Pi(T){let c,x,u,g,q;return g=new Xe({props:{code:`from evaluate import evaluator
from datasets import load_dataset
task_evaluator = evaluator("question-answering")
data = load_dataset("squad", split="validation[:2]")
results = task_evaluator.compute(
    model_or_pipeline="sshleifer/tiny-distilbert-base-cased-distilled-squad",
    data=data,
    metric="squad",
)`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> evaluate <span class="hljs-keyword">import</span> evaluator
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> datasets <span class="hljs-keyword">import</span> load_dataset
<span class="hljs-meta">&gt;&gt;&gt; </span>task_evaluator = evaluator(<span class="hljs-string">&quot;question-answering&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>data = load_dataset(<span class="hljs-string">&quot;squad&quot;</span>, split=<span class="hljs-string">&quot;validation[:2]&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>results = task_evaluator.compute(
<span class="hljs-meta">&gt;&gt;&gt; </span>    model_or_pipeline=<span class="hljs-string">&quot;sshleifer/tiny-distilbert-base-cased-distilled-squad&quot;</span>,
<span class="hljs-meta">&gt;&gt;&gt; </span>    data=data,
<span class="hljs-meta">&gt;&gt;&gt; </span>    metric=<span class="hljs-string">&quot;squad&quot;</span>,
<span class="hljs-meta">&gt;&gt;&gt; </span>)`}}),{c(){c=o("p"),x=l("Examples:"),u=d(),_(g.$$.fragment)},l(r){c=s(r,"P",{});var h=n(c);x=i(h,"Examples:"),h.forEach(a),u=p(r),b(g.$$.fragment,r)},m(r,h){v(r,c,h),e(c,x),v(r,u,h),w(g,r,h),q=!0},p:We,i(r){q||($(g.$$.fragment,r),q=!0)},o(r){y(g.$$.fragment,r),q=!1},d(r){r&&a(c),r&&a(u),E(g,r)}}}function Ci(T){let c,x,u,g,q;return{c(){c=o("p"),x=l("Datasets where the answer may be missing in the context are supported, for example SQuAD v2 dataset. In this case, it is safer to pass "),u=o("code"),g=l("squad_v2_format=True"),q=l(` to
the compute() call.`)},l(r){c=s(r,"P",{});var h=n(c);x=i(h,"Datasets where the answer may be missing in the context are supported, for example SQuAD v2 dataset. In this case, it is safer to pass "),u=s(h,"CODE",{});var A=n(u);g=i(A,"squad_v2_format=True"),A.forEach(a),q=i(h,` to
the compute() call.`),h.forEach(a)},m(r,h){v(r,c,h),e(c,x),e(c,u),e(u,g),e(c,q)},d(r){r&&a(c)}}}function Di(T){let c,x;return c=new Xe({props:{code:`from evaluate import evaluator
from datasets import load_dataset
task_evaluator = evaluator("question-answering")
data = load_dataset("squad_v2", split="validation[:2]")
results = task_evaluator.compute(
    model_or_pipeline="mrm8488/bert-tiny-finetuned-squadv2",
    data=data,
    metric="squad_v2",
    squad_v2_format=True,
)`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> evaluate <span class="hljs-keyword">import</span> evaluator
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> datasets <span class="hljs-keyword">import</span> load_dataset
<span class="hljs-meta">&gt;&gt;&gt; </span>task_evaluator = evaluator(<span class="hljs-string">&quot;question-answering&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>data = load_dataset(<span class="hljs-string">&quot;squad_v2&quot;</span>, split=<span class="hljs-string">&quot;validation[:2]&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>results = task_evaluator.compute(
<span class="hljs-meta">&gt;&gt;&gt; </span>    model_or_pipeline=<span class="hljs-string">&quot;mrm8488/bert-tiny-finetuned-squadv2&quot;</span>,
<span class="hljs-meta">&gt;&gt;&gt; </span>    data=data,
<span class="hljs-meta">&gt;&gt;&gt; </span>    metric=<span class="hljs-string">&quot;squad_v2&quot;</span>,
<span class="hljs-meta">&gt;&gt;&gt; </span>    squad_v2_format=<span class="hljs-literal">True</span>,
<span class="hljs-meta">&gt;&gt;&gt; </span>)`}}),{c(){_(c.$$.fragment)},l(u){b(c.$$.fragment,u)},m(u,g){w(c,u,g),x=!0},p:We,i(u){x||($(c.$$.fragment,u),x=!0)},o(u){y(c.$$.fragment,u),x=!1},d(u){E(c,u)}}}function Ii(T){let c,x,u,g,q;return g=new Xe({props:{code:`from evaluate import evaluator
from datasets import load_dataset
task_evaluator = evaluator("text-classification")
data = load_dataset("imdb", split="test[:2]")
results = task_evaluator.compute(
    model_or_pipeline="huggingface/prunebert-base-uncased-6-finepruned-w-distil-mnli",
    data=data,
    metric="accuracy",
    label_mapping={"LABEL_0": 0.0, "LABEL_1": 1.0},
    strategy="bootstrap",
    n_resamples=10,
    random_state=0
)`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> evaluate <span class="hljs-keyword">import</span> evaluator
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> datasets <span class="hljs-keyword">import</span> load_dataset
<span class="hljs-meta">&gt;&gt;&gt; </span>task_evaluator = evaluator(<span class="hljs-string">&quot;text-classification&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>data = load_dataset(<span class="hljs-string">&quot;imdb&quot;</span>, split=<span class="hljs-string">&quot;test[:2]&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>results = task_evaluator.compute(
<span class="hljs-meta">&gt;&gt;&gt; </span>    model_or_pipeline=<span class="hljs-string">&quot;huggingface/prunebert-base-uncased-6-finepruned-w-distil-mnli&quot;</span>,
<span class="hljs-meta">&gt;&gt;&gt; </span>    data=data,
<span class="hljs-meta">&gt;&gt;&gt; </span>    metric=<span class="hljs-string">&quot;accuracy&quot;</span>,
<span class="hljs-meta">&gt;&gt;&gt; </span>    label_mapping={<span class="hljs-string">&quot;LABEL_0&quot;</span>: <span class="hljs-number">0.0</span>, <span class="hljs-string">&quot;LABEL_1&quot;</span>: <span class="hljs-number">1.0</span>},
<span class="hljs-meta">&gt;&gt;&gt; </span>    strategy=<span class="hljs-string">&quot;bootstrap&quot;</span>,
<span class="hljs-meta">&gt;&gt;&gt; </span>    n_resamples=<span class="hljs-number">10</span>,
<span class="hljs-meta">&gt;&gt;&gt; </span>    random_state=<span class="hljs-number">0</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>)`}}),{c(){c=o("p"),x=l("Examples:"),u=d(),_(g.$$.fragment)},l(r){c=s(r,"P",{});var h=n(c);x=i(h,"Examples:"),h.forEach(a),u=p(r),b(g.$$.fragment,r)},m(r,h){v(r,c,h),e(c,x),v(r,u,h),w(g,r,h),q=!0},p:We,i(r){q||($(g.$$.fragment,r),q=!0)},o(r){y(g.$$.fragment,r),q=!1},d(r){r&&a(c),r&&a(u),E(g,r)}}}function zi(T){let c,x,u,g,q;return g=new Xe({props:{code:`from evaluate import evaluator
from datasets import load_dataset
task_evaluator = evaluator("token-classification")
data = load_dataset("conll2003", split="validation[:2]")
results = task_evaluator.compute(
    model_or_pipeline="elastic/distilbert-base-uncased-finetuned-conll03-english",
    data=data,
    metric="seqeval",
)`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> evaluate <span class="hljs-keyword">import</span> evaluator
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> datasets <span class="hljs-keyword">import</span> load_dataset
<span class="hljs-meta">&gt;&gt;&gt; </span>task_evaluator = evaluator(<span class="hljs-string">&quot;token-classification&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>data = load_dataset(<span class="hljs-string">&quot;conll2003&quot;</span>, split=<span class="hljs-string">&quot;validation[:2]&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>results = task_evaluator.compute(
<span class="hljs-meta">&gt;&gt;&gt; </span>    model_or_pipeline=<span class="hljs-string">&quot;elastic/distilbert-base-uncased-finetuned-conll03-english&quot;</span>,
<span class="hljs-meta">&gt;&gt;&gt; </span>    data=data,
<span class="hljs-meta">&gt;&gt;&gt; </span>    metric=<span class="hljs-string">&quot;seqeval&quot;</span>,
<span class="hljs-meta">&gt;&gt;&gt; </span>)`}}),{c(){c=o("p"),x=l("Examples:"),u=d(),_(g.$$.fragment)},l(r){c=s(r,"P",{});var h=n(c);x=i(h,"Examples:"),h.forEach(a),u=p(r),b(g.$$.fragment,r)},m(r,h){v(r,c,h),e(c,x),v(r,u,h),w(g,r,h),q=!0},p:We,i(r){q||($(g.$$.fragment,r),q=!0)},o(r){y(g.$$.fragment,r),q=!1},d(r){r&&a(c),r&&a(u),E(g,r)}}}function Ai(T){let c,x,u,g,q;return g=new Xe({props:{code:`dataset = Dataset.from_dict(
    mapping={
        "tokens": [["New", "York", "is", "a", "city", "and", "Felix", "a", "person", "."]],
        "ner_tags": [[1, 2, 0, 0, 0, 0, 3, 0, 0, 0]],
    },
    features=Features({
        "tokens": Sequence(feature=Value(dtype="string")),
        "ner_tags": Sequence(feature=ClassLabel(names=["O", "B-LOC", "I-LOC", "B-PER", "I-PER"])),
        }),
)`,highlighted:`dataset = Dataset.from_dict(
    mapping={
        <span class="hljs-string">&quot;tokens&quot;</span>: [[<span class="hljs-string">&quot;New&quot;</span>, <span class="hljs-string">&quot;York&quot;</span>, <span class="hljs-string">&quot;is&quot;</span>, <span class="hljs-string">&quot;a&quot;</span>, <span class="hljs-string">&quot;city&quot;</span>, <span class="hljs-string">&quot;and&quot;</span>, <span class="hljs-string">&quot;Felix&quot;</span>, <span class="hljs-string">&quot;a&quot;</span>, <span class="hljs-string">&quot;person&quot;</span>, <span class="hljs-string">&quot;.&quot;</span>]],
        <span class="hljs-string">&quot;ner_tags&quot;</span>: [[<span class="hljs-number">1</span>, <span class="hljs-number">2</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">3</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>]],
    },
    features=Features({
        <span class="hljs-string">&quot;tokens&quot;</span>: <span class="hljs-type">Sequence</span>(feature=Value(dtype=<span class="hljs-string">&quot;string&quot;</span>)),
        <span class="hljs-string">&quot;ner_tags&quot;</span>: <span class="hljs-type">Sequence</span>(feature=ClassLabel(names=[<span class="hljs-string">&quot;O&quot;</span>, <span class="hljs-string">&quot;B-LOC&quot;</span>, <span class="hljs-string">&quot;I-LOC&quot;</span>, <span class="hljs-string">&quot;B-PER&quot;</span>, <span class="hljs-string">&quot;I-PER&quot;</span>])),
        }),
)`}}),{c(){c=o("p"),x=l("For example, the following dataset format is accepted by the evaluator:"),u=d(),_(g.$$.fragment)},l(r){c=s(r,"P",{});var h=n(c);x=i(h,"For example, the following dataset format is accepted by the evaluator:"),h.forEach(a),u=p(r),b(g.$$.fragment,r)},m(r,h){v(r,c,h),e(c,x),v(r,u,h),w(g,r,h),q=!0},p:We,i(r){q||($(g.$$.fragment,r),q=!0)},o(r){y(g.$$.fragment,r),q=!1},d(r){r&&a(c),r&&a(u),E(g,r)}}}function Si(T){let c,x;return c=new Ke({props:{anchor:"evaluate.TokenClassificationEvaluator.compute.example-2",$$slots:{default:[Ai]},$$scope:{ctx:T}}}),{c(){_(c.$$.fragment)},l(u){b(c.$$.fragment,u)},m(u,g){w(c,u,g),x=!0},p(u,g){const q={};g&2&&(q.$$scope={dirty:g,ctx:u}),c.$set(q)},i(u){x||($(c.$$.fragment,u),x=!0)},o(u){y(c.$$.fragment,u),x=!1},d(u){E(c,u)}}}function Ui(T){let c,x,u,g,q,r,h,A;return h=new Xe({props:{code:`dataset = Dataset.from_dict(
    mapping={
        "tokens": [["New York is a city and Felix a person."]],
        "starts": [[0, 23]],
        "ends": [[7, 27]],
        "ner_tags": [["LOC", "PER"]],
    },
    features=Features({
        "tokens": Value(dtype="string"),
        "starts": Sequence(feature=Value(dtype="int32")),
        "ends": Sequence(feature=Value(dtype="int32")),
        "ner_tags": Sequence(feature=Value(dtype="string")),
    }),
)`,highlighted:`dataset = Dataset.from_dict(
    mapping={
        <span class="hljs-string">&quot;tokens&quot;</span>: [[<span class="hljs-string">&quot;New York is a city and Felix a person.&quot;</span>]],
        <span class="hljs-string">&quot;starts&quot;</span>: [[<span class="hljs-number">0</span>, <span class="hljs-number">23</span>]],
        <span class="hljs-string">&quot;ends&quot;</span>: [[<span class="hljs-number">7</span>, <span class="hljs-number">27</span>]],
        <span class="hljs-string">&quot;ner_tags&quot;</span>: [[<span class="hljs-string">&quot;LOC&quot;</span>, <span class="hljs-string">&quot;PER&quot;</span>]],
    },
    features=Features({
        <span class="hljs-string">&quot;tokens&quot;</span>: Value(dtype=<span class="hljs-string">&quot;string&quot;</span>),
        <span class="hljs-string">&quot;starts&quot;</span>: <span class="hljs-type">Sequence</span>(feature=Value(dtype=<span class="hljs-string">&quot;int32&quot;</span>)),
        <span class="hljs-string">&quot;ends&quot;</span>: <span class="hljs-type">Sequence</span>(feature=Value(dtype=<span class="hljs-string">&quot;int32&quot;</span>)),
        <span class="hljs-string">&quot;ner_tags&quot;</span>: <span class="hljs-type">Sequence</span>(feature=Value(dtype=<span class="hljs-string">&quot;string&quot;</span>)),
    }),
)`}}),{c(){c=o("p"),x=l("For example, the following dataset format is "),u=o("strong"),g=l("not"),q=l(" accepted by the evaluator:"),r=d(),_(h.$$.fragment)},l(P){c=s(P,"P",{});var S=n(c);x=i(S,"For example, the following dataset format is "),u=s(S,"STRONG",{});var oe=n(u);g=i(oe,"not"),oe.forEach(a),q=i(S," accepted by the evaluator:"),S.forEach(a),r=p(P),b(h.$$.fragment,P)},m(P,S){v(P,c,S),e(c,x),e(c,u),e(u,g),e(c,q),v(P,r,S),w(h,P,S),A=!0},p:We,i(P){A||($(h.$$.fragment,P),A=!0)},o(P){y(h.$$.fragment,P),A=!1},d(P){P&&a(c),P&&a(r),E(h,P)}}}function Fi(T){let c,x;return c=new Ke({props:{anchor:"evaluate.TokenClassificationEvaluator.compute.example-3",$$slots:{default:[Ui]},$$scope:{ctx:T}}}),{c(){_(c.$$.fragment)},l(u){b(c.$$.fragment,u)},m(u,g){w(c,u,g),x=!0},p(u,g){const q={};g&2&&(q.$$scope={dirty:g,ctx:u}),c.$set(q)},i(u){x||($(c.$$.fragment,u),x=!0)},o(u){y(c.$$.fragment,u),x=!1},d(u){E(c,u)}}}function Oi(T){let c,x,u,g,q,r,h,A,P,S,oe,ns,go,se,fe,ia,Ze,rs,ca,ls,fo,Lt,is,ho,O,et,cs,M,ds,Rt,ps,us,da,ms,gs,pa,fs,hs,vs,he,vo,Gt,_s,_o,k,tt,bs,ua,ws,$s,ve,at,ys,ma,Es,xs,_e,ot,qs,ga,ks,Ts,be,st,js,fa,Ns,Ps,we,nt,Cs,ha,Ds,Is,$e,rt,zs,lt,As,va,Ss,Us,Fs,ye,it,Os,_a,Ms,Ls,Ee,ct,Rs,ba,Gs,Vs,xe,dt,Qs,wa,Bs,bo,ne,qe,$a,pt,Hs,ya,Ys,wo,re,ke,Ea,ut,Js,xa,Ks,$o,L,mt,Ws,R,Xs,Vt,Zs,en,qa,tn,an,ka,on,sn,nn,K,gt,rn,Ta,ln,cn,Te,yo,le,je,ja,ft,dn,Na,pn,Eo,D,ht,un,vt,mn,Ne,Pa,gn,fn,hn,vn,ie,_n,Qt,bn,wn,Ca,$n,yn,En,_t,xn,bt,Da,qn,kn,Tn,z,wt,jn,Ia,Nn,Pn,Pe,Cn,Ce,Dn,De,xo,ce,Ie,za,$t,In,Aa,zn,qo,G,yt,An,U,Sn,Bt,Un,Fn,Sa,On,Mn,Ua,Ln,Rn,Fa,Gn,Vn,Qn,W,Et,Bn,Oa,Hn,Yn,ze,ko,de,Ae,Ma,xt,Jn,La,Kn,To,I,qt,Wn,Ra,Xn,Zn,pe,er,Ht,tr,ar,Ga,or,sr,nr,kt,rr,Va,lr,ir,cr,C,Tt,dr,Qa,pr,ur,jt,mr,Nt,gr,fr,hr,Se,vr,Ue,_r,Fe,jo,ue,Oe,Ba,Pt,br,Ha,wr,No,V,Ct,$r,Q,yr,Yt,Er,xr,Ya,qr,kr,Ja,Tr,jr,Nr,Me,Dt,Pr,Ka,Cr,Po,me,Le,Wa,It,Dr,Xa,Ir,Co,B,zt,zr,H,Ar,Jt,Sr,Ur,Za,Fr,Or,Kt,Mr,Lr,Rr,Re,At,Gr,eo,Vr,Do,ge,Ge,to,St,Qr,ao,Br,Io,Y,Ut,Hr,J,Yr,Wt,Jr,Kr,oo,Wr,Xr,so,Zr,el,tl,Ve,Ft,al,no,ol,zo;return r=new ae({}),Ze=new ae({}),et=new j({props:{name:"evaluate.evaluator",anchor:"evaluate.evaluator",parameters:[{name:"task",val:": str = None"}],parametersDescription:[{anchor:"evaluate.evaluator.task",description:`<strong>task</strong> (<code>str</code>) &#x2014;
The task defining which evaluator will be returned. Currently accepted tasks are:<ul>
<li><code>&quot;image-classification&quot;</code>: will return a <a href="/docs/evaluate/pr_330/en/package_reference/evaluator_classes#evaluate.ImageClassificationEvaluator">ImageClassificationEvaluator</a>.</li>
<li><code>&quot;question-answering&quot;</code>: will return a <a href="/docs/evaluate/pr_330/en/package_reference/evaluator_classes#evaluate.QuestionAnsweringEvaluator">QuestionAnsweringEvaluator</a>.</li>
<li><code>&quot;text-classification&quot;</code> (alias <code>&quot;sentiment-analysis&quot;</code> available): will return a <a href="/docs/evaluate/pr_330/en/package_reference/evaluator_classes#evaluate.TextClassificationEvaluator">TextClassificationEvaluator</a>.</li>
<li><code>&quot;token-classification&quot;</code>: will return a <a href="/docs/evaluate/pr_330/en/package_reference/evaluator_classes#evaluate.TokenClassificationEvaluator">TokenClassificationEvaluator</a>.</li>
</ul>`,name:"task"}],source:"https://github.com/huggingface/evaluate/blob/vr_330/src/evaluate/evaluator/__init__.py#L98",returnDescription:`
<p>An evaluator suitable for the task.</p>
`,returnType:`
<p><a
  href="/docs/evaluate/pr_330/en/package_reference/evaluator_classes#evaluate.Evaluator"
>Evaluator</a></p>
`}}),he=new Ke({props:{anchor:"evaluate.evaluator.example",$$slots:{default:[ji]},$$scope:{ctx:T}}}),tt=new j({props:{name:"class evaluate.Evaluator",anchor:"evaluate.Evaluator",parameters:[{name:"task",val:": str"},{name:"default_metric_name",val:": str = None"}],source:"https://github.com/huggingface/evaluate/blob/vr_330/src/evaluate/evaluator/base.py#L102"}}),at=new j({props:{name:"check_required_columns",anchor:"evaluate.Evaluator.check_required_columns",parameters:[{name:"data",val:": typing.Union[str, datasets.arrow_dataset.Dataset]"},{name:"columns_names",val:": typing.Dict[str, str]"}],parametersDescription:[{anchor:"evaluate.Evaluator.check_required_columns.data",description:`<strong>data</strong> (<code>str</code> or <code>Dataset</code>) &#x2014;
Specifies the dataset we will run evaluation on.`,name:"data"},{anchor:"evaluate.Evaluator.check_required_columns.columns_names",description:"<strong>columns_names</strong> (<code>List[str]</code>) &#x2014;",name:"columns_names"},{anchor:"evaluate.Evaluator.check_required_columns.List",description:"<strong>List</strong> of column names to check in the dataset. The keys are the arguments to the compute() method, &#x2014;",name:"List"},{anchor:"evaluate.Evaluator.check_required_columns.while",description:"<strong>while</strong> the values are the column names to check. &#x2014;",name:"while"}],source:"https://github.com/huggingface/evaluate/blob/vr_330/src/evaluate/evaluator/base.py#L289"}}),ot=new j({props:{name:"compute_metric",anchor:"evaluate.Evaluator.compute_metric",parameters:[{name:"metric",val:": EvaluationModule"},{name:"metric_inputs",val:": typing.Dict"},{name:"strategy",val:": typing.Literal['simple', 'bootstrap'] = 'simple'"},{name:"confidence_level",val:": float = 0.95"},{name:"n_resamples",val:": int = 9999"},{name:"random_state",val:": typing.Optional[int] = None"}],source:"https://github.com/huggingface/evaluate/blob/vr_330/src/evaluate/evaluator/base.py#L451"}}),st=new j({props:{name:"get_dataset_split",anchor:"evaluate.Evaluator.get_dataset_split",parameters:[{name:"data",val:""},{name:"subset",val:" = None"},{name:"split",val:" = None"}],parametersDescription:[{anchor:"evaluate.Evaluator.get_dataset_split.data",description:"<strong>data</strong> (<code>str</code>) &#x2014; Name of dataset",name:"data"},{anchor:"evaluate.Evaluator.get_dataset_split.subset",description:"<strong>subset</strong> (<code>str</code>) &#x2014; Name of config for datasets with multiple configurations (e.g. &#x2018;glue/cola&#x2019;)",name:"subset"},{anchor:"evaluate.Evaluator.get_dataset_split.split",description:"<strong>split</strong> (<code>str</code>, defaults to None) &#x2014; Split to use",name:"split"}],source:"https://github.com/huggingface/evaluate/blob/vr_330/src/evaluate/evaluator/base.py#L306",returnDescription:`
<p><code>str</code> containing which split to use</p>
`,returnType:`
<p><code>split</code></p>
`}}),nt=new j({props:{name:"load_data",anchor:"evaluate.Evaluator.load_data",parameters:[{name:"data",val:": typing.Union[str, datasets.arrow_dataset.Dataset]"},{name:"subset",val:": str = None"},{name:"split",val:": str = None"}],parametersDescription:[{anchor:"evaluate.Evaluator.load_data.data",description:"<strong>data</strong> (<code>Dataset</code> or <code>str</code>, defaults to None) &#x2014; Specifies the dataset we will run evaluation on. If it is of",name:"data"},{anchor:"evaluate.Evaluator.load_data.type",description:"<strong>type</strong> <code>str</code>, we treat it as the dataset name, and load it. Otherwise we assume it represents a pre-loaded dataset. &#x2014;",name:"type"},{anchor:"evaluate.Evaluator.load_data.subset",description:`<strong>subset</strong> (<code>str</code>, defaults to None) &#x2014; Specifies dataset subset to be passed to <code>name</code> in <code>load_dataset</code>. To be
used with datasets with several configurations (e.g. glue/sst2).`,name:"subset"},{anchor:"evaluate.Evaluator.load_data.split",description:`<strong>split</strong> (<code>str</code>, defaults to None) &#x2014;
User-defined dataset split by name (e.g. train, validation, test). Supports slice-split (test[:n]).
If not defined and data is a <code>str</code> type, will automatically select the best one via <code>choose_split()</code>.`,name:"split"}],source:"https://github.com/huggingface/evaluate/blob/vr_330/src/evaluate/evaluator/base.py#L323",returnDescription:`
<p>Loaded dataset which will be used for evaluation.</p>
`,returnType:`
<p>data (<code>Dataset</code>)</p>
`}}),rt=new j({props:{name:"predictions_processor",anchor:"evaluate.Evaluator.predictions_processor",parameters:[{name:"*args",val:""},{name:"**kwargs",val:""}],source:"https://github.com/huggingface/evaluate/blob/vr_330/src/evaluate/evaluator/base.py#L211"}}),it=new j({props:{name:"prepare_data",anchor:"evaluate.Evaluator.prepare_data",parameters:[{name:"data",val:": Dataset"},{name:"input_column",val:": str"},{name:"label_column",val:": str"}],parametersDescription:[{anchor:"evaluate.Evaluator.prepare_data.data",description:"<strong>data</strong> (<code>Dataset</code>) &#x2014; Specifies the dataset we will run evaluation on.",name:"data"},{anchor:"evaluate.Evaluator.prepare_data.input_column",description:`<strong>input_column</strong> (<code>str</code>, defaults to <code>&quot;text&quot;</code>) &#x2014;
the name of the column containing the text feature in the dataset specified by <code>data</code>.`,name:"input_column"},{anchor:"evaluate.Evaluator.prepare_data.label_column",description:`<strong>label_column</strong> (<code>str</code>, defaults to <code>&quot;label&quot;</code>) &#x2014;
the name of the column containing the labels in the dataset specified by <code>data</code>.`,name:"label_column"}],source:"https://github.com/huggingface/evaluate/blob/vr_330/src/evaluate/evaluator/base.py#L350",returnDescription:`
<p>metric inputs.
<code>list</code>:  pipeline inputs.</p>
`,returnType:`
<p><code>dict</code></p>
`}}),ct=new j({props:{name:"prepare_metric",anchor:"evaluate.Evaluator.prepare_metric",parameters:[{name:"metric",val:": typing.Union[str, evaluate.module.EvaluationModule]"}],parametersDescription:[{anchor:"evaluate.Evaluator.prepare_metric.metric",description:`<strong>metric</strong> (<code>str</code> or <code>EvaluationModule</code>, defaults to <code>None</code>) &#x2014;
Specifies the metric we use in evaluator. If it is of type <code>str</code>, we treat it as the metric name, and
load it. Otherwise we assume it represents a pre-loaded metric.`,name:"metric"}],source:"https://github.com/huggingface/evaluate/blob/vr_330/src/evaluate/evaluator/base.py#L421",returnDescription:`
<p>The loaded metric.</p>
`}}),dt=new j({props:{name:"prepare_pipeline",anchor:"evaluate.Evaluator.prepare_pipeline",parameters:[{name:"model_or_pipeline",val:": typing.Union[str, ForwardRef('Pipeline'), typing.Callable, ForwardRef('PreTrainedModel'), ForwardRef('TFPreTrainedModel')]"},{name:"tokenizer",val:": typing.Union[ForwardRef('PreTrainedTokenizerBase'), ForwardRef('FeatureExtractionMixin')] = None"},{name:"feature_extractor",val:": typing.Union[ForwardRef('PreTrainedTokenizerBase'), ForwardRef('FeatureExtractionMixin')] = None"},{name:"device",val:": int = None"}],parametersDescription:[{anchor:"evaluate.Evaluator.prepare_pipeline.model_or_pipeline",description:"<strong>model_or_pipeline</strong> (<code>str</code> or <code>Pipeline</code> or <code>Callable</code> or <code>PreTrainedModel</code> or <code>TFPreTrainedModel</code>, &#x2014;",name:"model_or_pipeline"},{anchor:"evaluate.Evaluator.prepare_pipeline.defaults",description:`<strong>defaults</strong> to <code>None</code>) &#x2014;
If the argument in not specified, we initialize the default pipeline for the task. If the argument is of the type <code>str</code> or
is a model instance, we use it to initialize a new <code>Pipeline</code> with the given model. Otherwise we assume the
argument specifies a pre-initialized pipeline.`,name:"defaults"},{anchor:"evaluate.Evaluator.prepare_pipeline.preprocessor",description:`<strong>preprocessor</strong> (<code>PreTrainedTokenizerBase</code> or <code>FeatureExtractionMixin</code>, <em>optional</em>, defaults to <code>None</code>) &#x2014;
Argument can be used to overwrite a default preprocessor if <code>model_or_pipeline</code> represents a model for
which we build a pipeline. If <code>model_or_pipeline</code> is <code>None</code> or a pre-initialized pipeline, we ignore
this argument.`,name:"preprocessor"}],source:"https://github.com/huggingface/evaluate/blob/vr_330/src/evaluate/evaluator/base.py#L369",returnDescription:`
<p>The initialized pipeline.</p>
`}}),pt=new ae({}),ut=new ae({}),mt=new j({props:{name:"class evaluate.ImageClassificationEvaluator",anchor:"evaluate.ImageClassificationEvaluator",parameters:[{name:"task",val:" = 'image-classification'"},{name:"default_metric_name",val:" = None"}],source:"https://github.com/huggingface/evaluate/blob/vr_330/src/evaluate/evaluator/image_classification.py#L45"}}),gt=new j({props:{name:"compute",anchor:"evaluate.ImageClassificationEvaluator.compute",parameters:[{name:"model_or_pipeline",val:": typing.Union[str, ForwardRef('Pipeline'), typing.Callable, ForwardRef('PreTrainedModel'), ForwardRef('TFPreTrainedModel')] = None"},{name:"data",val:": typing.Union[str, datasets.arrow_dataset.Dataset] = None"},{name:"subset",val:": typing.Optional[str] = None"},{name:"split",val:": typing.Optional[str] = None"},{name:"metric",val:": typing.Union[str, evaluate.module.EvaluationModule] = None"},{name:"tokenizer",val:": typing.Union[str, ForwardRef('PreTrainedTokenizer'), NoneType] = None"},{name:"feature_extractor",val:": typing.Union[str, ForwardRef('FeatureExtractionMixin'), NoneType] = None"},{name:"strategy",val:": typing.Literal['simple', 'bootstrap'] = 'simple'"},{name:"confidence_level",val:": float = 0.95"},{name:"n_resamples",val:": int = 9999"},{name:"device",val:": int = None"},{name:"random_state",val:": typing.Optional[int] = None"},{name:"input_column",val:": str = 'image'"},{name:"label_column",val:": str = 'label'"},{name:"label_mapping",val:": typing.Union[typing.Dict[str, numbers.Number], NoneType] = None"}],parametersDescription:[{anchor:"evaluate.ImageClassificationEvaluator.compute.model_or_pipeline",description:`<strong>model_or_pipeline</strong> (<code>str</code> or <code>Pipeline</code> or <code>Callable</code> or <code>PreTrainedModel</code> or <code>TFPreTrainedModel</code>, defaults to <code>None</code>) &#x2014;
If the argument in not specified, we initialize the default pipeline for the task (in this case
<code>text-classification</code> or its alias - <code>sentiment-analysis</code>). If the argument is of the type <code>str</code> or
is a model instance, we use it to initialize a new <code>Pipeline</code> with the given model. Otherwise we assume the
argument specifies a pre-initialized pipeline.`,name:"model_or_pipeline"},{anchor:"evaluate.ImageClassificationEvaluator.compute.data",description:`<strong>data</strong> (<code>str</code> or <code>Dataset</code>, defaults to <code>None</code>) &#x2014;
Specifies the dataset we will run evaluation on. If it is of type <code>str</code>, we treat it as the dataset
name, and load it. Otherwise we assume it represents a pre-loaded dataset.`,name:"data"},{anchor:"evaluate.ImageClassificationEvaluator.compute.subset",description:`<strong>subset</strong> (<code>str</code>, defaults to <code>None</code>) &#x2014;
Defines which dataset subset to load. If <code>None</code> is passed the default subset is loaded.`,name:"subset"},{anchor:"evaluate.ImageClassificationEvaluator.compute.split",description:`<strong>split</strong> (<code>str</code>, defaults to <code>None</code>) &#x2014;
Defines which dataset split to load. If <code>None</code> is passed, infers based on the <code>choose_split</code> function.`,name:"split"},{anchor:"evaluate.ImageClassificationEvaluator.compute.metric",description:`<strong>metric</strong> (<code>str</code> or <code>EvaluationModule</code>, defaults to <code>None</code>) &#x2014;
Specifies the metric we use in evaluator. If it is of type <code>str</code>, we treat it as the metric name, and
load it. Otherwise we assume it represents a pre-loaded metric.`,name:"metric"},{anchor:"evaluate.ImageClassificationEvaluator.compute.tokenizer",description:`<strong>tokenizer</strong> (<code>str</code> or <code>PreTrainedTokenizer</code>, <em>optional</em>, defaults to <code>None</code>) &#x2014;
Argument can be used to overwrite a default tokenizer if <code>model_or_pipeline</code> represents a model for
which we build a pipeline. If <code>model_or_pipeline</code> is <code>None</code> or a pre-initialized pipeline, we ignore
this argument.`,name:"tokenizer"},{anchor:"evaluate.ImageClassificationEvaluator.compute.strategy",description:`<strong>strategy</strong> (<code>Literal[&quot;simple&quot;, &quot;bootstrap&quot;]</code>, defaults to &#x201C;simple&#x201D;) &#x2014;
specifies the evaluation strategy. Possible values are:<ul>
<li><code>&quot;simple&quot;</code> - we evaluate the metric and return the scores.</li>
<li><code>&quot;bootstrap&quot;</code> - on top of computing the metric scores, we calculate the confidence interval for each
of the returned metric keys, using <code>scipy</code>&#x2019;s <code>bootstrap</code> method
<a href="https://docs.scipy.org/doc/scipy/reference/generated/scipy.stats.bootstrap.html" rel="nofollow">https://docs.scipy.org/doc/scipy/reference/generated/scipy.stats.bootstrap.html</a>.</li>
</ul>`,name:"strategy"},{anchor:"evaluate.ImageClassificationEvaluator.compute.confidence_level",description:`<strong>confidence_level</strong> (<code>float</code>, defaults to <code>0.95</code>) &#x2014;
The <code>confidence_level</code> value passed to <code>bootstrap</code> if <code>&quot;bootstrap&quot;</code> strategy is chosen.`,name:"confidence_level"},{anchor:"evaluate.ImageClassificationEvaluator.compute.n_resamples",description:`<strong>n_resamples</strong> (<code>int</code>, defaults to <code>9999</code>) &#x2014;
The <code>n_resamples</code> value passed to <code>bootstrap</code> if <code>&quot;bootstrap&quot;</code> strategy is chosen.`,name:"n_resamples"},{anchor:"evaluate.ImageClassificationEvaluator.compute.device",description:`<strong>device</strong> (<code>int</code>, defaults to <code>None</code>) &#x2014;
Device ordinal for CPU/GPU support of the pipeline. Setting this to -1 will leverage CPU, a positive
integer will run the model on the associated CUDA device ID. If <code>None</code> is provided it will be inferred and
CUDA:0 used if available, CPU otherwise.`,name:"device"},{anchor:"evaluate.ImageClassificationEvaluator.compute.random_state",description:`<strong>random_state</strong> (<code>int</code>, <em>optional</em>, defaults to <code>None</code>) &#x2014;
The <code>random_state</code> value passed to <code>bootstrap</code> if <code>&quot;bootstrap&quot;</code> strategy is chosen. Useful for
debugging.`,name:"random_state"}],source:"https://github.com/huggingface/evaluate/blob/vr_330/src/evaluate/evaluator/image_classification.py#L64",returnDescription:`
<p>A <code>Dict</code>. The keys represent metric keys calculated for the <code>metric</code> spefied in function arguments. For the
<code>"simple"</code> strategy, the value is the metric score. For the <code>"bootstrap"</code> strategy, the value is a <code>Dict</code>
containing the score, the confidence interval and the standard error calculated for each metric key.</p>
`}}),Te=new Ke({props:{anchor:"evaluate.ImageClassificationEvaluator.compute.example",$$slots:{default:[Ni]},$$scope:{ctx:T}}}),ft=new ae({}),ht=new j({props:{name:"class evaluate.QuestionAnsweringEvaluator",anchor:"evaluate.QuestionAnsweringEvaluator",parameters:[{name:"task",val:" = 'question-answering'"},{name:"default_metric_name",val:" = None"}],source:"https://github.com/huggingface/evaluate/blob/vr_330/src/evaluate/evaluator/question_answering.py#L74"}}),wt=new j({props:{name:"compute",anchor:"evaluate.QuestionAnsweringEvaluator.compute",parameters:[{name:"model_or_pipeline",val:": typing.Union[str, ForwardRef('Pipeline'), typing.Callable, ForwardRef('PreTrainedModel'), ForwardRef('TFPreTrainedModel')] = None"},{name:"data",val:": typing.Union[str, datasets.arrow_dataset.Dataset] = None"},{name:"subset",val:": typing.Optional[str] = None"},{name:"split",val:": typing.Optional[str] = None"},{name:"metric",val:": typing.Union[str, evaluate.module.EvaluationModule] = None"},{name:"tokenizer",val:": typing.Union[str, ForwardRef('PreTrainedTokenizer'), NoneType] = None"},{name:"strategy",val:": typing.Literal['simple', 'bootstrap'] = 'simple'"},{name:"confidence_level",val:": float = 0.95"},{name:"n_resamples",val:": int = 9999"},{name:"device",val:": int = None"},{name:"random_state",val:": typing.Optional[int] = None"},{name:"question_column",val:": str = 'question'"},{name:"context_column",val:": str = 'context'"},{name:"id_column",val:": str = 'id'"},{name:"label_column",val:": str = 'answers'"},{name:"squad_v2_format",val:": typing.Optional[bool] = None"}],parametersDescription:[{anchor:"evaluate.QuestionAnsweringEvaluator.compute.model_or_pipeline",description:`<strong>model_or_pipeline</strong> (<code>str</code> or <code>Pipeline</code> or <code>Callable</code> or <code>PreTrainedModel</code> or <code>TFPreTrainedModel</code>, defaults to <code>None</code>) &#x2014;
If the argument in not specified, we initialize the default pipeline for the task (in this case
<code>text-classification</code> or its alias - <code>sentiment-analysis</code>). If the argument is of the type <code>str</code> or
is a model instance, we use it to initialize a new <code>Pipeline</code> with the given model. Otherwise we assume the
argument specifies a pre-initialized pipeline.`,name:"model_or_pipeline"},{anchor:"evaluate.QuestionAnsweringEvaluator.compute.data",description:`<strong>data</strong> (<code>str</code> or <code>Dataset</code>, defaults to <code>None</code>) &#x2014;
Specifies the dataset we will run evaluation on. If it is of type <code>str</code>, we treat it as the dataset
name, and load it. Otherwise we assume it represents a pre-loaded dataset.`,name:"data"},{anchor:"evaluate.QuestionAnsweringEvaluator.compute.subset",description:`<strong>subset</strong> (<code>str</code>, defaults to <code>None</code>) &#x2014;
Defines which dataset subset to load. If <code>None</code> is passed the default subset is loaded.`,name:"subset"},{anchor:"evaluate.QuestionAnsweringEvaluator.compute.split",description:`<strong>split</strong> (<code>str</code>, defaults to <code>None</code>) &#x2014;
Defines which dataset split to load. If <code>None</code> is passed, infers based on the <code>choose_split</code> function.`,name:"split"},{anchor:"evaluate.QuestionAnsweringEvaluator.compute.metric",description:`<strong>metric</strong> (<code>str</code> or <code>EvaluationModule</code>, defaults to <code>None</code>) &#x2014;
Specifies the metric we use in evaluator. If it is of type <code>str</code>, we treat it as the metric name, and
load it. Otherwise we assume it represents a pre-loaded metric.`,name:"metric"},{anchor:"evaluate.QuestionAnsweringEvaluator.compute.tokenizer",description:`<strong>tokenizer</strong> (<code>str</code> or <code>PreTrainedTokenizer</code>, <em>optional</em>, defaults to <code>None</code>) &#x2014;
Argument can be used to overwrite a default tokenizer if <code>model_or_pipeline</code> represents a model for
which we build a pipeline. If <code>model_or_pipeline</code> is <code>None</code> or a pre-initialized pipeline, we ignore
this argument.`,name:"tokenizer"},{anchor:"evaluate.QuestionAnsweringEvaluator.compute.strategy",description:`<strong>strategy</strong> (<code>Literal[&quot;simple&quot;, &quot;bootstrap&quot;]</code>, defaults to &#x201C;simple&#x201D;) &#x2014;
specifies the evaluation strategy. Possible values are:<ul>
<li><code>&quot;simple&quot;</code> - we evaluate the metric and return the scores.</li>
<li><code>&quot;bootstrap&quot;</code> - on top of computing the metric scores, we calculate the confidence interval for each
of the returned metric keys, using <code>scipy</code>&#x2019;s <code>bootstrap</code> method
<a href="https://docs.scipy.org/doc/scipy/reference/generated/scipy.stats.bootstrap.html" rel="nofollow">https://docs.scipy.org/doc/scipy/reference/generated/scipy.stats.bootstrap.html</a>.</li>
</ul>`,name:"strategy"},{anchor:"evaluate.QuestionAnsweringEvaluator.compute.confidence_level",description:`<strong>confidence_level</strong> (<code>float</code>, defaults to <code>0.95</code>) &#x2014;
The <code>confidence_level</code> value passed to <code>bootstrap</code> if <code>&quot;bootstrap&quot;</code> strategy is chosen.`,name:"confidence_level"},{anchor:"evaluate.QuestionAnsweringEvaluator.compute.n_resamples",description:`<strong>n_resamples</strong> (<code>int</code>, defaults to <code>9999</code>) &#x2014;
The <code>n_resamples</code> value passed to <code>bootstrap</code> if <code>&quot;bootstrap&quot;</code> strategy is chosen.`,name:"n_resamples"},{anchor:"evaluate.QuestionAnsweringEvaluator.compute.device",description:`<strong>device</strong> (<code>int</code>, defaults to <code>None</code>) &#x2014;
Device ordinal for CPU/GPU support of the pipeline. Setting this to -1 will leverage CPU, a positive
integer will run the model on the associated CUDA device ID. If <code>None</code> is provided it will be inferred and
CUDA:0 used if available, CPU otherwise.`,name:"device"},{anchor:"evaluate.QuestionAnsweringEvaluator.compute.random_state",description:`<strong>random_state</strong> (<code>int</code>, <em>optional</em>, defaults to <code>None</code>) &#x2014;
The <code>random_state</code> value passed to <code>bootstrap</code> if <code>&quot;bootstrap&quot;</code> strategy is chosen. Useful for
debugging.`,name:"random_state"}],source:"https://github.com/huggingface/evaluate/blob/vr_330/src/evaluate/evaluator/question_answering.py#L143",returnDescription:`
<p>A <code>Dict</code>. The keys represent metric keys calculated for the <code>metric</code> spefied in function arguments. For the
<code>"simple"</code> strategy, the value is the metric score. For the <code>"bootstrap"</code> strategy, the value is a <code>Dict</code>
containing the score, the confidence interval and the standard error calculated for each metric key.</p>
`}}),Pe=new Ke({props:{anchor:"evaluate.QuestionAnsweringEvaluator.compute.example",$$slots:{default:[Pi]},$$scope:{ctx:T}}}),Ce=new nl({props:{$$slots:{default:[Ci]},$$scope:{ctx:T}}}),De=new Ke({props:{anchor:"evaluate.QuestionAnsweringEvaluator.compute.example-2",$$slots:{default:[Di]},$$scope:{ctx:T}}}),$t=new ae({}),yt=new j({props:{name:"class evaluate.TextClassificationEvaluator",anchor:"evaluate.TextClassificationEvaluator",parameters:[{name:"task",val:" = 'text-classification'"},{name:"default_metric_name",val:" = None"}],source:"https://github.com/huggingface/evaluate/blob/vr_330/src/evaluate/evaluator/text_classification.py#L47"}}),Et=new j({props:{name:"compute",anchor:"evaluate.TextClassificationEvaluator.compute",parameters:[{name:"model_or_pipeline",val:": typing.Union[str, ForwardRef('Pipeline'), typing.Callable, ForwardRef('PreTrainedModel'), ForwardRef('TFPreTrainedModel')] = None"},{name:"data",val:": typing.Union[str, datasets.arrow_dataset.Dataset] = None"},{name:"subset",val:": typing.Optional[str] = None"},{name:"split",val:": typing.Optional[str] = None"},{name:"metric",val:": typing.Union[str, evaluate.module.EvaluationModule] = None"},{name:"tokenizer",val:": typing.Union[str, ForwardRef('PreTrainedTokenizer'), NoneType] = None"},{name:"feature_extractor",val:": typing.Union[str, ForwardRef('FeatureExtractionMixin'), NoneType] = None"},{name:"strategy",val:": typing.Literal['simple', 'bootstrap'] = 'simple'"},{name:"confidence_level",val:": float = 0.95"},{name:"n_resamples",val:": int = 9999"},{name:"device",val:": int = None"},{name:"random_state",val:": typing.Optional[int] = None"},{name:"input_column",val:": str = 'text'"},{name:"second_input_column",val:": typing.Optional[str] = None"},{name:"label_column",val:": str = 'label'"},{name:"label_mapping",val:": typing.Union[typing.Dict[str, numbers.Number], NoneType] = None"}],parametersDescription:[{anchor:"evaluate.TextClassificationEvaluator.compute.model_or_pipeline",description:`<strong>model_or_pipeline</strong> (<code>str</code> or <code>Pipeline</code> or <code>Callable</code> or <code>PreTrainedModel</code> or <code>TFPreTrainedModel</code>, defaults to <code>None</code>) &#x2014;
If the argument in not specified, we initialize the default pipeline for the task (in this case
<code>text-classification</code> or its alias - <code>sentiment-analysis</code>). If the argument is of the type <code>str</code> or
is a model instance, we use it to initialize a new <code>Pipeline</code> with the given model. Otherwise we assume the
argument specifies a pre-initialized pipeline.`,name:"model_or_pipeline"},{anchor:"evaluate.TextClassificationEvaluator.compute.data",description:`<strong>data</strong> (<code>str</code> or <code>Dataset</code>, defaults to <code>None</code>) &#x2014;
Specifies the dataset we will run evaluation on. If it is of type <code>str</code>, we treat it as the dataset
name, and load it. Otherwise we assume it represents a pre-loaded dataset.`,name:"data"},{anchor:"evaluate.TextClassificationEvaluator.compute.subset",description:`<strong>subset</strong> (<code>str</code>, defaults to <code>None</code>) &#x2014;
Defines which dataset subset to load. If <code>None</code> is passed the default subset is loaded.`,name:"subset"},{anchor:"evaluate.TextClassificationEvaluator.compute.split",description:`<strong>split</strong> (<code>str</code>, defaults to <code>None</code>) &#x2014;
Defines which dataset split to load. If <code>None</code> is passed, infers based on the <code>choose_split</code> function.`,name:"split"},{anchor:"evaluate.TextClassificationEvaluator.compute.metric",description:`<strong>metric</strong> (<code>str</code> or <code>EvaluationModule</code>, defaults to <code>None</code>) &#x2014;
Specifies the metric we use in evaluator. If it is of type <code>str</code>, we treat it as the metric name, and
load it. Otherwise we assume it represents a pre-loaded metric.`,name:"metric"},{anchor:"evaluate.TextClassificationEvaluator.compute.tokenizer",description:`<strong>tokenizer</strong> (<code>str</code> or <code>PreTrainedTokenizer</code>, <em>optional</em>, defaults to <code>None</code>) &#x2014;
Argument can be used to overwrite a default tokenizer if <code>model_or_pipeline</code> represents a model for
which we build a pipeline. If <code>model_or_pipeline</code> is <code>None</code> or a pre-initialized pipeline, we ignore
this argument.`,name:"tokenizer"},{anchor:"evaluate.TextClassificationEvaluator.compute.strategy",description:`<strong>strategy</strong> (<code>Literal[&quot;simple&quot;, &quot;bootstrap&quot;]</code>, defaults to &#x201C;simple&#x201D;) &#x2014;
specifies the evaluation strategy. Possible values are:<ul>
<li><code>&quot;simple&quot;</code> - we evaluate the metric and return the scores.</li>
<li><code>&quot;bootstrap&quot;</code> - on top of computing the metric scores, we calculate the confidence interval for each
of the returned metric keys, using <code>scipy</code>&#x2019;s <code>bootstrap</code> method
<a href="https://docs.scipy.org/doc/scipy/reference/generated/scipy.stats.bootstrap.html" rel="nofollow">https://docs.scipy.org/doc/scipy/reference/generated/scipy.stats.bootstrap.html</a>.</li>
</ul>`,name:"strategy"},{anchor:"evaluate.TextClassificationEvaluator.compute.confidence_level",description:`<strong>confidence_level</strong> (<code>float</code>, defaults to <code>0.95</code>) &#x2014;
The <code>confidence_level</code> value passed to <code>bootstrap</code> if <code>&quot;bootstrap&quot;</code> strategy is chosen.`,name:"confidence_level"},{anchor:"evaluate.TextClassificationEvaluator.compute.n_resamples",description:`<strong>n_resamples</strong> (<code>int</code>, defaults to <code>9999</code>) &#x2014;
The <code>n_resamples</code> value passed to <code>bootstrap</code> if <code>&quot;bootstrap&quot;</code> strategy is chosen.`,name:"n_resamples"},{anchor:"evaluate.TextClassificationEvaluator.compute.device",description:`<strong>device</strong> (<code>int</code>, defaults to <code>None</code>) &#x2014;
Device ordinal for CPU/GPU support of the pipeline. Setting this to -1 will leverage CPU, a positive
integer will run the model on the associated CUDA device ID. If <code>None</code> is provided it will be inferred and
CUDA:0 used if available, CPU otherwise.`,name:"device"},{anchor:"evaluate.TextClassificationEvaluator.compute.random_state",description:`<strong>random_state</strong> (<code>int</code>, <em>optional</em>, defaults to <code>None</code>) &#x2014;
The <code>random_state</code> value passed to <code>bootstrap</code> if <code>&quot;bootstrap&quot;</code> strategy is chosen. Useful for
debugging.`,name:"random_state"}],source:"https://github.com/huggingface/evaluate/blob/vr_330/src/evaluate/evaluator/text_classification.py#L85",returnDescription:`
<p>A <code>Dict</code>. The keys represent metric keys calculated for the <code>metric</code> spefied in function arguments. For the
<code>"simple"</code> strategy, the value is the metric score. For the <code>"bootstrap"</code> strategy, the value is a <code>Dict</code>
containing the score, the confidence interval and the standard error calculated for each metric key.</p>
`}}),ze=new Ke({props:{anchor:"evaluate.TextClassificationEvaluator.compute.example",$$slots:{default:[Ii]},$$scope:{ctx:T}}}),xt=new ae({}),qt=new j({props:{name:"class evaluate.TokenClassificationEvaluator",anchor:"evaluate.TokenClassificationEvaluator",parameters:[{name:"task",val:" = 'token-classification'"},{name:"default_metric_name",val:" = None"}],source:"https://github.com/huggingface/evaluate/blob/vr_330/src/evaluate/evaluator/token_classification.py#L86"}}),Tt=new j({props:{name:"compute",anchor:"evaluate.TokenClassificationEvaluator.compute",parameters:[{name:"model_or_pipeline",val:": typing.Union[str, ForwardRef('Pipeline'), typing.Callable, ForwardRef('PreTrainedModel'), ForwardRef('TFPreTrainedModel')] = None"},{name:"data",val:": typing.Union[str, datasets.arrow_dataset.Dataset] = None"},{name:"subset",val:": typing.Optional[str] = None"},{name:"split",val:": str = None"},{name:"metric",val:": typing.Union[str, evaluate.module.EvaluationModule] = None"},{name:"tokenizer",val:": typing.Union[str, ForwardRef('PreTrainedTokenizer'), NoneType] = None"},{name:"strategy",val:": typing.Literal['simple', 'bootstrap'] = 'simple'"},{name:"confidence_level",val:": float = 0.95"},{name:"n_resamples",val:": int = 9999"},{name:"device",val:": typing.Optional[int] = None"},{name:"random_state",val:": typing.Optional[int] = None"},{name:"input_column",val:": str = 'tokens'"},{name:"label_column",val:": str = 'ner_tags'"},{name:"join_by",val:": typing.Optional[str] = ' '"}],parametersDescription:[{anchor:"evaluate.TokenClassificationEvaluator.compute.model_or_pipeline",description:`<strong>model_or_pipeline</strong> (<code>str</code> or <code>Pipeline</code> or <code>Callable</code> or <code>PreTrainedModel</code> or <code>TFPreTrainedModel</code>, defaults to <code>None</code>) &#x2014;
If the argument in not specified, we initialize the default pipeline for the task (in this case
<code>text-classification</code> or its alias - <code>sentiment-analysis</code>). If the argument is of the type <code>str</code> or
is a model instance, we use it to initialize a new <code>Pipeline</code> with the given model. Otherwise we assume the
argument specifies a pre-initialized pipeline.`,name:"model_or_pipeline"},{anchor:"evaluate.TokenClassificationEvaluator.compute.data",description:`<strong>data</strong> (<code>str</code> or <code>Dataset</code>, defaults to <code>None</code>) &#x2014;
Specifies the dataset we will run evaluation on. If it is of type <code>str</code>, we treat it as the dataset
name, and load it. Otherwise we assume it represents a pre-loaded dataset.`,name:"data"},{anchor:"evaluate.TokenClassificationEvaluator.compute.subset",description:`<strong>subset</strong> (<code>str</code>, defaults to <code>None</code>) &#x2014;
Defines which dataset subset to load. If <code>None</code> is passed the default subset is loaded.`,name:"subset"},{anchor:"evaluate.TokenClassificationEvaluator.compute.split",description:`<strong>split</strong> (<code>str</code>, defaults to <code>None</code>) &#x2014;
Defines which dataset split to load. If <code>None</code> is passed, infers based on the <code>choose_split</code> function.`,name:"split"},{anchor:"evaluate.TokenClassificationEvaluator.compute.metric",description:`<strong>metric</strong> (<code>str</code> or <code>EvaluationModule</code>, defaults to <code>None</code>) &#x2014;
Specifies the metric we use in evaluator. If it is of type <code>str</code>, we treat it as the metric name, and
load it. Otherwise we assume it represents a pre-loaded metric.`,name:"metric"},{anchor:"evaluate.TokenClassificationEvaluator.compute.tokenizer",description:`<strong>tokenizer</strong> (<code>str</code> or <code>PreTrainedTokenizer</code>, <em>optional</em>, defaults to <code>None</code>) &#x2014;
Argument can be used to overwrite a default tokenizer if <code>model_or_pipeline</code> represents a model for
which we build a pipeline. If <code>model_or_pipeline</code> is <code>None</code> or a pre-initialized pipeline, we ignore
this argument.`,name:"tokenizer"},{anchor:"evaluate.TokenClassificationEvaluator.compute.strategy",description:`<strong>strategy</strong> (<code>Literal[&quot;simple&quot;, &quot;bootstrap&quot;]</code>, defaults to &#x201C;simple&#x201D;) &#x2014;
specifies the evaluation strategy. Possible values are:<ul>
<li><code>&quot;simple&quot;</code> - we evaluate the metric and return the scores.</li>
<li><code>&quot;bootstrap&quot;</code> - on top of computing the metric scores, we calculate the confidence interval for each
of the returned metric keys, using <code>scipy</code>&#x2019;s <code>bootstrap</code> method
<a href="https://docs.scipy.org/doc/scipy/reference/generated/scipy.stats.bootstrap.html" rel="nofollow">https://docs.scipy.org/doc/scipy/reference/generated/scipy.stats.bootstrap.html</a>.</li>
</ul>`,name:"strategy"},{anchor:"evaluate.TokenClassificationEvaluator.compute.confidence_level",description:`<strong>confidence_level</strong> (<code>float</code>, defaults to <code>0.95</code>) &#x2014;
The <code>confidence_level</code> value passed to <code>bootstrap</code> if <code>&quot;bootstrap&quot;</code> strategy is chosen.`,name:"confidence_level"},{anchor:"evaluate.TokenClassificationEvaluator.compute.n_resamples",description:`<strong>n_resamples</strong> (<code>int</code>, defaults to <code>9999</code>) &#x2014;
The <code>n_resamples</code> value passed to <code>bootstrap</code> if <code>&quot;bootstrap&quot;</code> strategy is chosen.`,name:"n_resamples"},{anchor:"evaluate.TokenClassificationEvaluator.compute.device",description:`<strong>device</strong> (<code>int</code>, defaults to <code>None</code>) &#x2014;
Device ordinal for CPU/GPU support of the pipeline. Setting this to -1 will leverage CPU, a positive
integer will run the model on the associated CUDA device ID. If <code>None</code> is provided it will be inferred and
CUDA:0 used if available, CPU otherwise.`,name:"device"},{anchor:"evaluate.TokenClassificationEvaluator.compute.random_state",description:`<strong>random_state</strong> (<code>int</code>, <em>optional</em>, defaults to <code>None</code>) &#x2014;
The <code>random_state</code> value passed to <code>bootstrap</code> if <code>&quot;bootstrap&quot;</code> strategy is chosen. Useful for
debugging.`,name:"random_state"}],source:"https://github.com/huggingface/evaluate/blob/vr_330/src/evaluate/evaluator/token_classification.py#L209",returnDescription:`
<p>A <code>Dict</code>. The keys represent metric keys calculated for the <code>metric</code> spefied in function arguments. For the
<code>"simple"</code> strategy, the value is the metric score. For the <code>"bootstrap"</code> strategy, the value is a <code>Dict</code>
containing the score, the confidence interval and the standard error calculated for each metric key.</p>
`}}),Se=new Ke({props:{anchor:"evaluate.TokenClassificationEvaluator.compute.example",$$slots:{default:[zi]},$$scope:{ctx:T}}}),Ue=new nl({props:{$$slots:{default:[Si]},$$scope:{ctx:T}}}),Fe=new nl({props:{warning:!0,$$slots:{default:[Fi]},$$scope:{ctx:T}}}),Pt=new ae({}),Ct=new j({props:{name:"class evaluate.Text2TextGenerationEvaluator",anchor:"evaluate.Text2TextGenerationEvaluator",parameters:[{name:"task",val:" = 'text2text-generation'"},{name:"default_metric_name",val:" = None"}],source:"https://github.com/huggingface/evaluate/blob/vr_330/src/evaluate/evaluator/text2text_generation.py#L35"}}),Dt=new j({props:{name:"compute",anchor:"evaluate.Text2TextGenerationEvaluator.compute",parameters:[{name:"model_or_pipeline",val:": typing.Union[str, ForwardRef('Pipeline'), typing.Callable, ForwardRef('PreTrainedModel'), ForwardRef('TFPreTrainedModel')] = None"},{name:"data",val:": typing.Union[str, datasets.arrow_dataset.Dataset] = None"},{name:"subset",val:": typing.Optional[str] = None"},{name:"split",val:": typing.Optional[str] = None"},{name:"metric",val:": typing.Union[str, evaluate.module.EvaluationModule] = None"},{name:"tokenizer",val:": typing.Union[str, ForwardRef('PreTrainedTokenizer'), NoneType] = None"},{name:"strategy",val:": typing.Literal['simple', 'bootstrap'] = 'simple'"},{name:"confidence_level",val:": float = 0.95"},{name:"n_resamples",val:": int = 9999"},{name:"device",val:": int = None"},{name:"random_state",val:": typing.Optional[int] = None"},{name:"input_column",val:": str = 'text'"},{name:"label_column",val:": str = 'label'"},{name:"generation_kwargs",val:": dict = None"}],parametersDescription:[{anchor:"evaluate.Text2TextGenerationEvaluator.compute.model_or_pipeline",description:`<strong>model_or_pipeline</strong> (<code>str</code> or <code>Pipeline</code> or <code>Callable</code> or <code>PreTrainedModel</code> or <code>TFPreTrainedModel</code>, defaults to <code>None</code>) &#x2014;
If the argument in not specified, we initialize the default pipeline for the task (in this case
<code>text-classification</code> or its alias - <code>sentiment-analysis</code>). If the argument is of the type <code>str</code> or
is a model instance, we use it to initialize a new <code>Pipeline</code> with the given model. Otherwise we assume the
argument specifies a pre-initialized pipeline.`,name:"model_or_pipeline"},{anchor:"evaluate.Text2TextGenerationEvaluator.compute.data",description:`<strong>data</strong> (<code>str</code> or <code>Dataset</code>, defaults to <code>None</code>) &#x2014;
Specifies the dataset we will run evaluation on. If it is of type <code>str</code>, we treat it as the dataset
name, and load it. Otherwise we assume it represents a pre-loaded dataset.`,name:"data"},{anchor:"evaluate.Text2TextGenerationEvaluator.compute.subset",description:`<strong>subset</strong> (<code>str</code>, defaults to <code>None</code>) &#x2014;
Defines which dataset subset to load. If <code>None</code> is passed the default subset is loaded.`,name:"subset"},{anchor:"evaluate.Text2TextGenerationEvaluator.compute.split",description:`<strong>split</strong> (<code>str</code>, defaults to <code>None</code>) &#x2014;
Defines which dataset split to load. If <code>None</code> is passed, infers based on the <code>choose_split</code> function.`,name:"split"},{anchor:"evaluate.Text2TextGenerationEvaluator.compute.metric",description:`<strong>metric</strong> (<code>str</code> or <code>EvaluationModule</code>, defaults to <code>None</code>) &#x2014;
Specifies the metric we use in evaluator. If it is of type <code>str</code>, we treat it as the metric name, and
load it. Otherwise we assume it represents a pre-loaded metric.`,name:"metric"},{anchor:"evaluate.Text2TextGenerationEvaluator.compute.tokenizer",description:`<strong>tokenizer</strong> (<code>str</code> or <code>PreTrainedTokenizer</code>, <em>optional</em>, defaults to <code>None</code>) &#x2014;
Argument can be used to overwrite a default tokenizer if <code>model_or_pipeline</code> represents a model for
which we build a pipeline. If <code>model_or_pipeline</code> is <code>None</code> or a pre-initialized pipeline, we ignore
this argument.`,name:"tokenizer"},{anchor:"evaluate.Text2TextGenerationEvaluator.compute.strategy",description:`<strong>strategy</strong> (<code>Literal[&quot;simple&quot;, &quot;bootstrap&quot;]</code>, defaults to &#x201C;simple&#x201D;) &#x2014;
specifies the evaluation strategy. Possible values are:</p>
<ul>
<li><code>&quot;simple&quot;</code> - we evaluate the metric and return the scores.</li>
<li><code>&quot;bootstrap&quot;</code> - on top of computing the metric scores, we calculate the confidence interval for each
of the returned metric keys, using <code>scipy</code>&#x2019;s <code>bootstrap</code> method
<a href="https://docs.scipy.org/doc/scipy/reference/generated/scipy.stats.bootstrap.html" rel="nofollow">https://docs.scipy.org/doc/scipy/reference/generated/scipy.stats.bootstrap.html</a>.</li>
</ul>`,name:"strategy"},{anchor:"evaluate.Text2TextGenerationEvaluator.compute.confidence_level",description:`<strong>confidence_level</strong> (<code>float</code>, defaults to <code>0.95</code>) &#x2014;
The <code>confidence_level</code> value passed to <code>bootstrap</code> if <code>&quot;bootstrap&quot;</code> strategy is chosen.`,name:"confidence_level"},{anchor:"evaluate.Text2TextGenerationEvaluator.compute.n_resamples",description:`<strong>n_resamples</strong> (<code>int</code>, defaults to <code>9999</code>) &#x2014;
The <code>n_resamples</code> value passed to <code>bootstrap</code> if <code>&quot;bootstrap&quot;</code> strategy is chosen.`,name:"n_resamples"},{anchor:"evaluate.Text2TextGenerationEvaluator.compute.device",description:`<strong>device</strong> (<code>int</code>, defaults to <code>None</code>) &#x2014;
Device ordinal for CPU/GPU support of the pipeline. Setting this to -1 will leverage CPU, a positive
integer will run the model on the associated CUDA device ID. If <code>None</code> is provided it will be inferred and
CUDA:0 used if available, CPU otherwise.`,name:"device"},{anchor:"evaluate.Text2TextGenerationEvaluator.compute.random_state",description:`<strong>random_state</strong> (<code>int</code>, <em>optional</em>, defaults to <code>None</code>) &#x2014;
The <code>random_state</code> value passed to <code>bootstrap</code> if <code>&quot;bootstrap&quot;</code> strategy is chosen. Useful for
debugging.`,name:"random_state"},{anchor:"evaluate.Text2TextGenerationEvaluator.compute.input_column",description:`<strong>input_column</strong> (<code>str</code>, defaults to <code>&quot;text&quot;</code>) &#x2014;
the name of the column containing the input text in the dataset specified by <code>data</code>.`,name:"input_column"},{anchor:"evaluate.Text2TextGenerationEvaluator.compute.label_column",description:`<strong>label_column</strong> (<code>str</code>, defaults to <code>&quot;label&quot;</code>) &#x2014;
the name of the column containing the labels in the dataset specified by <code>data</code>.`,name:"label_column"},{anchor:"evaluate.Text2TextGenerationEvaluator.compute.generation_kwargs",description:`<strong>generation_kwargs</strong> (<code>Dict</code>, <em>optional</em>, defaults to <code>None</code>) &#x2014;
The generation kwargs are passed to the pipeline and set the text generation strategy.`,name:"generation_kwargs"}],source:"https://github.com/huggingface/evaluate/blob/vr_330/src/evaluate/evaluator/text2text_generation.py#L52",returnDescription:`
<p>A <code>Dict</code>. The keys represent metric keys calculated for the <code>metric</code> spefied in function arguments. For the
<code>"simple"</code> strategy, the value is the metric score. For the <code>"bootstrap"</code> strategy, the value is a <code>Dict</code>
containing the score, the confidence interval and the standard error calculated for each metric key.</p>
<p>Examples:</p>

	<CodeBlock 
		code={\`from evaluate import evaluator
from datasets import load_dataset
task_evaluator = evaluator("text2text-generation")
data = load_dataset("cnn_dailymail", "3.0.0", split="validation[:40]")
results = task_evaluator.compute(
    model_or_pipeline="facebook/bart-large-cnn",
    data=data,
    input_column="article",
    label_column="highlights",
    metric="rouge",
)\`}
		highlighted={\`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> evaluate <span class="hljs-keyword">import</span> evaluator
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> datasets <span class="hljs-keyword">import</span> load_dataset
<span class="hljs-meta">&gt;&gt;&gt; </span>task_evaluator = evaluator(<span class="hljs-string">&quot;text2text-generation&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>data = load_dataset(<span class="hljs-string">&quot;cnn_dailymail&quot;</span>, <span class="hljs-string">&quot;3.0.0&quot;</span>, split=<span class="hljs-string">&quot;validation[:40]&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>results = task_evaluator.compute(
<span class="hljs-meta">&gt;&gt;&gt; </span>    model_or_pipeline=<span class="hljs-string">&quot;facebook/bart-large-cnn&quot;</span>,
<span class="hljs-meta">&gt;&gt;&gt; </span>    data=data,
<span class="hljs-meta">&gt;&gt;&gt; </span>    input_column=<span class="hljs-string">&quot;article&quot;</span>,
<span class="hljs-meta">&gt;&gt;&gt; </span>    label_column=<span class="hljs-string">&quot;highlights&quot;</span>,
<span class="hljs-meta">&gt;&gt;&gt; </span>    metric=<span class="hljs-string">&quot;rouge&quot;</span>,
<span class="hljs-meta">&gt;&gt;&gt; </span>)\`}
	/>
`}}),It=new ae({}),zt=new j({props:{name:"class evaluate.SummarizationEvaluator",anchor:"evaluate.SummarizationEvaluator",parameters:[{name:"task",val:" = 'summarization'"},{name:"default_metric_name",val:" = None"}],source:"https://github.com/huggingface/evaluate/blob/vr_330/src/evaluate/evaluator/text2text_generation.py#L113"}}),At=new j({props:{name:"compute",anchor:"evaluate.SummarizationEvaluator.compute",parameters:[{name:"model_or_pipeline",val:": typing.Union[str, ForwardRef('Pipeline'), typing.Callable, ForwardRef('PreTrainedModel'), ForwardRef('TFPreTrainedModel')] = None"},{name:"data",val:": typing.Union[str, datasets.arrow_dataset.Dataset] = None"},{name:"subset",val:": typing.Optional[str] = None"},{name:"split",val:": typing.Optional[str] = None"},{name:"metric",val:": typing.Union[str, evaluate.module.EvaluationModule] = None"},{name:"tokenizer",val:": typing.Union[str, ForwardRef('PreTrainedTokenizer'), NoneType] = None"},{name:"strategy",val:": typing.Literal['simple', 'bootstrap'] = 'simple'"},{name:"confidence_level",val:": float = 0.95"},{name:"n_resamples",val:": int = 9999"},{name:"device",val:": int = None"},{name:"random_state",val:": typing.Optional[int] = None"},{name:"input_column",val:": str = 'text'"},{name:"label_column",val:": str = 'label'"},{name:"generation_kwargs",val:": dict = None"}],parametersDescription:[{anchor:"evaluate.SummarizationEvaluator.compute.model_or_pipeline",description:`<strong>model_or_pipeline</strong> (<code>str</code> or <code>Pipeline</code> or <code>Callable</code> or <code>PreTrainedModel</code> or <code>TFPreTrainedModel</code>, defaults to <code>None</code>) &#x2014;
If the argument in not specified, we initialize the default pipeline for the task (in this case
<code>text-classification</code> or its alias - <code>sentiment-analysis</code>). If the argument is of the type <code>str</code> or
is a model instance, we use it to initialize a new <code>Pipeline</code> with the given model. Otherwise we assume the
argument specifies a pre-initialized pipeline.`,name:"model_or_pipeline"},{anchor:"evaluate.SummarizationEvaluator.compute.data",description:`<strong>data</strong> (<code>str</code> or <code>Dataset</code>, defaults to <code>None</code>) &#x2014;
Specifies the dataset we will run evaluation on. If it is of type <code>str</code>, we treat it as the dataset
name, and load it. Otherwise we assume it represents a pre-loaded dataset.`,name:"data"},{anchor:"evaluate.SummarizationEvaluator.compute.subset",description:`<strong>subset</strong> (<code>str</code>, defaults to <code>None</code>) &#x2014;
Defines which dataset subset to load. If <code>None</code> is passed the default subset is loaded.`,name:"subset"},{anchor:"evaluate.SummarizationEvaluator.compute.split",description:`<strong>split</strong> (<code>str</code>, defaults to <code>None</code>) &#x2014;
Defines which dataset split to load. If <code>None</code> is passed, infers based on the <code>choose_split</code> function.`,name:"split"},{anchor:"evaluate.SummarizationEvaluator.compute.metric",description:`<strong>metric</strong> (<code>str</code> or <code>EvaluationModule</code>, defaults to <code>None</code>) &#x2014;
Specifies the metric we use in evaluator. If it is of type <code>str</code>, we treat it as the metric name, and
load it. Otherwise we assume it represents a pre-loaded metric.`,name:"metric"},{anchor:"evaluate.SummarizationEvaluator.compute.tokenizer",description:`<strong>tokenizer</strong> (<code>str</code> or <code>PreTrainedTokenizer</code>, <em>optional</em>, defaults to <code>None</code>) &#x2014;
Argument can be used to overwrite a default tokenizer if <code>model_or_pipeline</code> represents a model for
which we build a pipeline. If <code>model_or_pipeline</code> is <code>None</code> or a pre-initialized pipeline, we ignore
this argument.`,name:"tokenizer"},{anchor:"evaluate.SummarizationEvaluator.compute.strategy",description:`<strong>strategy</strong> (<code>Literal[&quot;simple&quot;, &quot;bootstrap&quot;]</code>, defaults to &#x201C;simple&#x201D;) &#x2014;
specifies the evaluation strategy. Possible values are:</p>
<ul>
<li><code>&quot;simple&quot;</code> - we evaluate the metric and return the scores.</li>
<li><code>&quot;bootstrap&quot;</code> - on top of computing the metric scores, we calculate the confidence interval for each
of the returned metric keys, using <code>scipy</code>&#x2019;s <code>bootstrap</code> method
<a href="https://docs.scipy.org/doc/scipy/reference/generated/scipy.stats.bootstrap.html" rel="nofollow">https://docs.scipy.org/doc/scipy/reference/generated/scipy.stats.bootstrap.html</a>.</li>
</ul>`,name:"strategy"},{anchor:"evaluate.SummarizationEvaluator.compute.confidence_level",description:`<strong>confidence_level</strong> (<code>float</code>, defaults to <code>0.95</code>) &#x2014;
The <code>confidence_level</code> value passed to <code>bootstrap</code> if <code>&quot;bootstrap&quot;</code> strategy is chosen.`,name:"confidence_level"},{anchor:"evaluate.SummarizationEvaluator.compute.n_resamples",description:`<strong>n_resamples</strong> (<code>int</code>, defaults to <code>9999</code>) &#x2014;
The <code>n_resamples</code> value passed to <code>bootstrap</code> if <code>&quot;bootstrap&quot;</code> strategy is chosen.`,name:"n_resamples"},{anchor:"evaluate.SummarizationEvaluator.compute.device",description:`<strong>device</strong> (<code>int</code>, defaults to <code>None</code>) &#x2014;
Device ordinal for CPU/GPU support of the pipeline. Setting this to -1 will leverage CPU, a positive
integer will run the model on the associated CUDA device ID. If <code>None</code> is provided it will be inferred and
CUDA:0 used if available, CPU otherwise.`,name:"device"},{anchor:"evaluate.SummarizationEvaluator.compute.random_state",description:`<strong>random_state</strong> (<code>int</code>, <em>optional</em>, defaults to <code>None</code>) &#x2014;
The <code>random_state</code> value passed to <code>bootstrap</code> if <code>&quot;bootstrap&quot;</code> strategy is chosen. Useful for
debugging.`,name:"random_state"},{anchor:"evaluate.SummarizationEvaluator.compute.input_column",description:`<strong>input_column</strong> (<code>str</code>, defaults to <code>&quot;text&quot;</code>) &#x2014;
the name of the column containing the input text in the dataset specified by <code>data</code>.`,name:"input_column"},{anchor:"evaluate.SummarizationEvaluator.compute.label_column",description:`<strong>label_column</strong> (<code>str</code>, defaults to <code>&quot;label&quot;</code>) &#x2014;
the name of the column containing the labels in the dataset specified by <code>data</code>.`,name:"label_column"},{anchor:"evaluate.SummarizationEvaluator.compute.generation_kwargs",description:`<strong>generation_kwargs</strong> (<code>Dict</code>, <em>optional</em>, defaults to <code>None</code>) &#x2014;
The generation kwargs are passed to the pipeline and set the text generation strategy.`,name:"generation_kwargs"}],source:"https://github.com/huggingface/evaluate/blob/vr_330/src/evaluate/evaluator/text2text_generation.py#L127",returnDescription:`
<p>A <code>Dict</code>. The keys represent metric keys calculated for the <code>metric</code> spefied in function arguments. For the
<code>"simple"</code> strategy, the value is the metric score. For the <code>"bootstrap"</code> strategy, the value is a <code>Dict</code>
containing the score, the confidence interval and the standard error calculated for each metric key.</p>
<p>Examples:</p>

	<CodeBlock 
		code={\`from evaluate import evaluator
from datasets import load_dataset
task_evaluator = evaluator("summarization")
data = load_dataset("cnn_dailymail", "3.0.0", split="validation[:40]")
results = task_evaluator.compute(
    model_or_pipeline="facebook/bart-large-cnn",
    data=data,
    input_column="article",
    label_column="highlights",
)\`}
		highlighted={\`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> evaluate <span class="hljs-keyword">import</span> evaluator
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> datasets <span class="hljs-keyword">import</span> load_dataset
<span class="hljs-meta">&gt;&gt;&gt; </span>task_evaluator = evaluator(<span class="hljs-string">&quot;summarization&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>data = load_dataset(<span class="hljs-string">&quot;cnn_dailymail&quot;</span>, <span class="hljs-string">&quot;3.0.0&quot;</span>, split=<span class="hljs-string">&quot;validation[:40]&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>results = task_evaluator.compute(
<span class="hljs-meta">&gt;&gt;&gt; </span>    model_or_pipeline=<span class="hljs-string">&quot;facebook/bart-large-cnn&quot;</span>,
<span class="hljs-meta">&gt;&gt;&gt; </span>    data=data,
<span class="hljs-meta">&gt;&gt;&gt; </span>    input_column=<span class="hljs-string">&quot;article&quot;</span>,
<span class="hljs-meta">&gt;&gt;&gt; </span>    label_column=<span class="hljs-string">&quot;highlights&quot;</span>,
<span class="hljs-meta">&gt;&gt;&gt; </span>)\`}
	/>
`}}),St=new ae({}),Ut=new j({props:{name:"class evaluate.TranslationEvaluator",anchor:"evaluate.TranslationEvaluator",parameters:[{name:"task",val:" = 'translation'"},{name:"default_metric_name",val:" = None"}],source:"https://github.com/huggingface/evaluate/blob/vr_330/src/evaluate/evaluator/text2text_generation.py#L184"}}),Ft=new j({props:{name:"compute",anchor:"evaluate.TranslationEvaluator.compute",parameters:[{name:"model_or_pipeline",val:": typing.Union[str, ForwardRef('Pipeline'), typing.Callable, ForwardRef('PreTrainedModel'), ForwardRef('TFPreTrainedModel')] = None"},{name:"data",val:": typing.Union[str, datasets.arrow_dataset.Dataset] = None"},{name:"subset",val:": typing.Optional[str] = None"},{name:"split",val:": typing.Optional[str] = None"},{name:"metric",val:": typing.Union[str, evaluate.module.EvaluationModule] = None"},{name:"tokenizer",val:": typing.Union[str, ForwardRef('PreTrainedTokenizer'), NoneType] = None"},{name:"strategy",val:": typing.Literal['simple', 'bootstrap'] = 'simple'"},{name:"confidence_level",val:": float = 0.95"},{name:"n_resamples",val:": int = 9999"},{name:"device",val:": int = None"},{name:"random_state",val:": typing.Optional[int] = None"},{name:"input_column",val:": str = 'text'"},{name:"label_column",val:": str = 'label'"},{name:"generation_kwargs",val:": dict = None"}],parametersDescription:[{anchor:"evaluate.TranslationEvaluator.compute.model_or_pipeline",description:`<strong>model_or_pipeline</strong> (<code>str</code> or <code>Pipeline</code> or <code>Callable</code> or <code>PreTrainedModel</code> or <code>TFPreTrainedModel</code>, defaults to <code>None</code>) &#x2014;
If the argument in not specified, we initialize the default pipeline for the task (in this case
<code>text-classification</code> or its alias - <code>sentiment-analysis</code>). If the argument is of the type <code>str</code> or
is a model instance, we use it to initialize a new <code>Pipeline</code> with the given model. Otherwise we assume the
argument specifies a pre-initialized pipeline.`,name:"model_or_pipeline"},{anchor:"evaluate.TranslationEvaluator.compute.data",description:`<strong>data</strong> (<code>str</code> or <code>Dataset</code>, defaults to <code>None</code>) &#x2014;
Specifies the dataset we will run evaluation on. If it is of type <code>str</code>, we treat it as the dataset
name, and load it. Otherwise we assume it represents a pre-loaded dataset.`,name:"data"},{anchor:"evaluate.TranslationEvaluator.compute.subset",description:`<strong>subset</strong> (<code>str</code>, defaults to <code>None</code>) &#x2014;
Defines which dataset subset to load. If <code>None</code> is passed the default subset is loaded.`,name:"subset"},{anchor:"evaluate.TranslationEvaluator.compute.split",description:`<strong>split</strong> (<code>str</code>, defaults to <code>None</code>) &#x2014;
Defines which dataset split to load. If <code>None</code> is passed, infers based on the <code>choose_split</code> function.`,name:"split"},{anchor:"evaluate.TranslationEvaluator.compute.metric",description:`<strong>metric</strong> (<code>str</code> or <code>EvaluationModule</code>, defaults to <code>None</code>) &#x2014;
Specifies the metric we use in evaluator. If it is of type <code>str</code>, we treat it as the metric name, and
load it. Otherwise we assume it represents a pre-loaded metric.`,name:"metric"},{anchor:"evaluate.TranslationEvaluator.compute.tokenizer",description:`<strong>tokenizer</strong> (<code>str</code> or <code>PreTrainedTokenizer</code>, <em>optional</em>, defaults to <code>None</code>) &#x2014;
Argument can be used to overwrite a default tokenizer if <code>model_or_pipeline</code> represents a model for
which we build a pipeline. If <code>model_or_pipeline</code> is <code>None</code> or a pre-initialized pipeline, we ignore
this argument.`,name:"tokenizer"},{anchor:"evaluate.TranslationEvaluator.compute.strategy",description:`<strong>strategy</strong> (<code>Literal[&quot;simple&quot;, &quot;bootstrap&quot;]</code>, defaults to &#x201C;simple&#x201D;) &#x2014;
specifies the evaluation strategy. Possible values are:</p>
<ul>
<li><code>&quot;simple&quot;</code> - we evaluate the metric and return the scores.</li>
<li><code>&quot;bootstrap&quot;</code> - on top of computing the metric scores, we calculate the confidence interval for each
of the returned metric keys, using <code>scipy</code>&#x2019;s <code>bootstrap</code> method
<a href="https://docs.scipy.org/doc/scipy/reference/generated/scipy.stats.bootstrap.html" rel="nofollow">https://docs.scipy.org/doc/scipy/reference/generated/scipy.stats.bootstrap.html</a>.</li>
</ul>`,name:"strategy"},{anchor:"evaluate.TranslationEvaluator.compute.confidence_level",description:`<strong>confidence_level</strong> (<code>float</code>, defaults to <code>0.95</code>) &#x2014;
The <code>confidence_level</code> value passed to <code>bootstrap</code> if <code>&quot;bootstrap&quot;</code> strategy is chosen.`,name:"confidence_level"},{anchor:"evaluate.TranslationEvaluator.compute.n_resamples",description:`<strong>n_resamples</strong> (<code>int</code>, defaults to <code>9999</code>) &#x2014;
The <code>n_resamples</code> value passed to <code>bootstrap</code> if <code>&quot;bootstrap&quot;</code> strategy is chosen.`,name:"n_resamples"},{anchor:"evaluate.TranslationEvaluator.compute.device",description:`<strong>device</strong> (<code>int</code>, defaults to <code>None</code>) &#x2014;
Device ordinal for CPU/GPU support of the pipeline. Setting this to -1 will leverage CPU, a positive
integer will run the model on the associated CUDA device ID. If <code>None</code> is provided it will be inferred and
CUDA:0 used if available, CPU otherwise.`,name:"device"},{anchor:"evaluate.TranslationEvaluator.compute.random_state",description:`<strong>random_state</strong> (<code>int</code>, <em>optional</em>, defaults to <code>None</code>) &#x2014;
The <code>random_state</code> value passed to <code>bootstrap</code> if <code>&quot;bootstrap&quot;</code> strategy is chosen. Useful for
debugging.`,name:"random_state"},{anchor:"evaluate.TranslationEvaluator.compute.input_column",description:`<strong>input_column</strong> (<code>str</code>, defaults to <code>&quot;text&quot;</code>) &#x2014;
the name of the column containing the input text in the dataset specified by <code>data</code>.`,name:"input_column"},{anchor:"evaluate.TranslationEvaluator.compute.label_column",description:`<strong>label_column</strong> (<code>str</code>, defaults to <code>&quot;label&quot;</code>) &#x2014;
the name of the column containing the labels in the dataset specified by <code>data</code>.`,name:"label_column"},{anchor:"evaluate.TranslationEvaluator.compute.generation_kwargs",description:`<strong>generation_kwargs</strong> (<code>Dict</code>, <em>optional</em>, defaults to <code>None</code>) &#x2014;
The generation kwargs are passed to the pipeline and set the text generation strategy.`,name:"generation_kwargs"}],source:"https://github.com/huggingface/evaluate/blob/vr_330/src/evaluate/evaluator/text2text_generation.py#L198",returnDescription:`
<p>A <code>Dict</code>. The keys represent metric keys calculated for the <code>metric</code> spefied in function arguments. For the
<code>"simple"</code> strategy, the value is the metric score. For the <code>"bootstrap"</code> strategy, the value is a <code>Dict</code>
containing the score, the confidence interval and the standard error calculated for each metric key.</p>
<p>Examples:</p>

	<CodeBlock 
		code={\`from evaluate import evaluator
from datasets import load_dataset
task_evaluator = evaluator("translation")
data = load_dataset("wmt19", "fr-de", split="validation[:40]")
data = data.map(lambda x: {"text": x["translation"]["de"], "label": x["translation"]["fr"]\\})
results = task_evaluator.compute(
    model_or_pipeline="Helsinki-NLP/opus-mt-de-fr",
    data=data,
)\`}
		highlighted={\`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> evaluate <span class="hljs-keyword">import</span> evaluator
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> datasets <span class="hljs-keyword">import</span> load_dataset
<span class="hljs-meta">&gt;&gt;&gt; </span>task_evaluator = evaluator(<span class="hljs-string">&quot;translation&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>data = load_dataset(<span class="hljs-string">&quot;wmt19&quot;</span>, <span class="hljs-string">&quot;fr-de&quot;</span>, split=<span class="hljs-string">&quot;validation[:40]&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>data = data.<span class="hljs-built_in">map</span>(<span class="hljs-keyword">lambda</span> x: {<span class="hljs-string">&quot;text&quot;</span>: x[<span class="hljs-string">&quot;translation&quot;</span>][<span class="hljs-string">&quot;de&quot;</span>], <span class="hljs-string">&quot;label&quot;</span>: x[<span class="hljs-string">&quot;translation&quot;</span>][<span class="hljs-string">&quot;fr&quot;</span>]\\})
<span class="hljs-meta">&gt;&gt;&gt; </span>results = task_evaluator.compute(
<span class="hljs-meta">&gt;&gt;&gt; </span>    model_or_pipeline=<span class="hljs-string">&quot;Helsinki-NLP/opus-mt-de-fr&quot;</span>,
<span class="hljs-meta">&gt;&gt;&gt; </span>    data=data,
<span class="hljs-meta">&gt;&gt;&gt; </span>)\`}
	/>
`}}),{c(){c=o("meta"),x=d(),u=o("h1"),g=o("a"),q=o("span"),_(r.$$.fragment),h=d(),A=o("span"),P=l("Evaluator"),S=d(),oe=o("p"),ns=l("The evaluator classes for automatic evaluation."),go=d(),se=o("h2"),fe=o("a"),ia=o("span"),_(Ze.$$.fragment),rs=d(),ca=o("span"),ls=l("Evaluator classes"),fo=d(),Lt=o("p"),is=l("The main entry point for using the evaluator:"),ho=d(),O=o("div"),_(et.$$.fragment),cs=d(),M=o("p"),ds=l("Utility factory method to build an "),Rt=o("a"),ps=l("Evaluator"),us=l(`.
Evaluators encapsulate a task and a default metric name. They leverage `),da=o("code"),ms=l("pipeline"),gs=l(" functionalify from "),pa=o("code"),fs=l("transformers"),hs=l(`
to simplify the evaluation of multiple combinations of models, datasets and metrics for a given task.`),vs=d(),_(he.$$.fragment),vo=d(),Gt=o("p"),_s=l("The base class for all evaluator classes:"),_o=d(),k=o("div"),_(tt.$$.fragment),bs=d(),ua=o("p"),ws=l(`The Evaluator class is the class from which all evaluators inherit. Refer to this class for methods shared across
different evaluators.
Base class implementing evaluator operations.`),$s=d(),ve=o("div"),_(at.$$.fragment),ys=d(),ma=o("p"),Es=l("Ensure the columns required for the evaluation are present in the dataset."),xs=d(),_e=o("div"),_(ot.$$.fragment),qs=d(),ga=o("p"),ks=l("Compute and return metrics."),Ts=d(),be=o("div"),_(st.$$.fragment),js=d(),fa=o("p"),Ns=l("Infers which split to use if None is given."),Ps=d(),we=o("div"),_(nt.$$.fragment),Cs=d(),ha=o("p"),Ds=l("Load dataset with given subset and split."),Is=d(),$e=o("div"),_(rt.$$.fragment),zs=d(),lt=o("p"),As=l("A core method of the "),va=o("code"),Ss=l("Evaluator"),Us=l(" class, which processes the pipeline outputs for compatibility with the metric."),Fs=d(),ye=o("div"),_(it.$$.fragment),Os=d(),_a=o("p"),Ms=l("Prepare data."),Ls=d(),Ee=o("div"),_(ct.$$.fragment),Rs=d(),ba=o("p"),Gs=l("Prepare metric."),Vs=d(),xe=o("div"),_(dt.$$.fragment),Qs=d(),wa=o("p"),Bs=l("Prepare pipeline."),bo=d(),ne=o("h2"),qe=o("a"),$a=o("span"),_(pt.$$.fragment),Hs=d(),ya=o("span"),Ys=l("The task specific evaluators"),wo=d(),re=o("h3"),ke=o("a"),Ea=o("span"),_(ut.$$.fragment),Js=d(),xa=o("span"),Ks=l("ImageClassificationEvaluator"),$o=d(),L=o("div"),_(mt.$$.fragment),Ws=d(),R=o("p"),Xs=l(`Image classification evaluator.
This image classification evaluator can currently be loaded from `),Vt=o("a"),Zs=l("evaluator()"),en=l(` using the default task name
`),qa=o("code"),tn=l("image-classification"),an=l(`.
Methods in this class assume a data format compatible with the `),ka=o("code"),on=l("ImageClassificationPipeline"),sn=l("."),nn=d(),K=o("div"),_(gt.$$.fragment),rn=d(),Ta=o("p"),ln=l("Compute the metric for a given pipeline and dataset combination."),cn=d(),_(Te.$$.fragment),yo=d(),le=o("h3"),je=o("a"),ja=o("span"),_(ft.$$.fragment),dn=d(),Na=o("span"),pn=l("QuestionAnsweringEvaluator"),Eo=d(),D=o("div"),_(ht.$$.fragment),un=d(),vt=o("p"),mn=l(`Question answering evaluator. This evaluator handles
`),Ne=o("a"),Pa=o("strong"),gn=l("extractive"),fn=l(" question answering"),hn=l(`,
where the answer to the question is extracted from a context.`),vn=d(),ie=o("p"),_n=l("This question answering evaluator can currently be loaded from "),Qt=o("a"),bn=l("evaluator()"),wn=l(` using the default task name
`),Ca=o("code"),$n=l("question-answering"),yn=l("."),En=d(),_t=o("p"),xn=l(`Methods in this class assume a data format compatible with the
`),bt=o("a"),Da=o("code"),qn=l("QuestionAnsweringPipeline"),kn=l("."),Tn=d(),z=o("div"),_(wt.$$.fragment),jn=d(),Ia=o("p"),Nn=l("Compute the metric for a given pipeline and dataset combination."),Pn=d(),_(Pe.$$.fragment),Cn=d(),_(Ce.$$.fragment),Dn=d(),_(De.$$.fragment),xo=d(),ce=o("h3"),Ie=o("a"),za=o("span"),_($t.$$.fragment),In=d(),Aa=o("span"),zn=l("TextClassificationEvaluator"),qo=d(),G=o("div"),_(yt.$$.fragment),An=d(),U=o("p"),Sn=l(`Text classification evaluator.
This text classification evaluator can currently be loaded from `),Bt=o("a"),Un=l("evaluator()"),Fn=l(` using the default task name
`),Sa=o("code"),On=l("text-classification"),Mn=l(" or with a "),Ua=o("code"),Ln=l('"sentiment-analysis"'),Rn=l(` alias.
Methods in this class assume a data format compatible with the `),Fa=o("code"),Gn=l("TextClassificationPipeline"),Vn=l(` - a single textual
feature as input and a categorical label as output.`),Qn=d(),W=o("div"),_(Et.$$.fragment),Bn=d(),Oa=o("p"),Hn=l("Compute the metric for a given pipeline and dataset combination."),Yn=d(),_(ze.$$.fragment),ko=d(),de=o("h3"),Ae=o("a"),Ma=o("span"),_(xt.$$.fragment),Jn=d(),La=o("span"),Kn=l("TokenClassificationEvaluator"),To=d(),I=o("div"),_(qt.$$.fragment),Wn=d(),Ra=o("p"),Xn=l("Token classification evaluator."),Zn=d(),pe=o("p"),er=l("This token classification evaluator can currently be loaded from "),Ht=o("a"),tr=l("evaluator()"),ar=l(` using the default task name
`),Ga=o("code"),or=l("token-classification"),sr=l("."),nr=d(),kt=o("p"),rr=l("Methods in this class assume a data format compatible with the "),Va=o("code"),lr=l("TokenClassificationPipeline"),ir=l("."),cr=d(),C=o("div"),_(Tt.$$.fragment),dr=d(),Qa=o("p"),pr=l("Compute the metric for a given pipeline and dataset combination."),ur=d(),jt=o("p"),mr=l("The dataset input and label columns are expected to be formatted as a list of words and a list of labels respectively, following "),Nt=o("a"),gr=l("conll2003 dataset"),fr=l(". Datasets whose inputs are single strings, and labels are a list of offset are not supported."),hr=d(),_(Se.$$.fragment),vr=d(),_(Ue.$$.fragment),_r=d(),_(Fe.$$.fragment),jo=d(),ue=o("h3"),Oe=o("a"),Ba=o("span"),_(Pt.$$.fragment),br=d(),Ha=o("span"),wr=l("Text2TextGenerationEvaluator"),No=d(),V=o("div"),_(Ct.$$.fragment),$r=d(),Q=o("p"),yr=l(`Text2Text generation evaluator.
This Text2Text generation evaluator can currently be loaded from `),Yt=o("a"),Er=l("evaluator()"),xr=l(` using the default task name
`),Ya=o("code"),qr=l("text2text-generation"),kr=l(`.
Methods in this class assume a data format compatible with the `),Ja=o("code"),Tr=l("Text2TextGenerationPipeline"),jr=l("."),Nr=d(),Me=o("div"),_(Dt.$$.fragment),Pr=d(),Ka=o("p"),Cr=l("Compute the metric for a given pipeline and dataset combination."),Po=d(),me=o("h3"),Le=o("a"),Wa=o("span"),_(It.$$.fragment),Dr=d(),Xa=o("span"),Ir=l("SummarizationEvaluator"),Co=d(),B=o("div"),_(zt.$$.fragment),zr=d(),H=o("p"),Ar=l(`Text summarization evaluator.
This text summarization evaluator can currently be loaded from `),Jt=o("a"),Sr=l("evaluator()"),Ur=l(` using the default task name
`),Za=o("code"),Fr=l("summarization"),Or=l(`.
Methods in this class assume a data format compatible with the `),Kt=o("a"),Mr=l("SummarizationEvaluator"),Lr=l("."),Rr=d(),Re=o("div"),_(At.$$.fragment),Gr=d(),eo=o("p"),Vr=l("Compute the metric for a given pipeline and dataset combination."),Do=d(),ge=o("h3"),Ge=o("a"),to=o("span"),_(St.$$.fragment),Qr=d(),ao=o("span"),Br=l("TranslationEvaluator"),Io=d(),Y=o("div"),_(Ut.$$.fragment),Hr=d(),J=o("p"),Yr=l(`Translation evaluator.
This translation generation evaluator can currently be loaded from `),Wt=o("a"),Jr=l("evaluator()"),Kr=l(` using the default task name
`),oo=o("code"),Wr=l("translation"),Xr=l(`.
Methods in this class assume a data format compatible with the `),so=o("code"),Zr=l("TranslationPipeline"),el=l("."),tl=d(),Ve=o("div"),_(Ft.$$.fragment),al=d(),no=o("p"),ol=l("Compute the metric for a given pipeline and dataset combination."),this.h()},l(t){const f=ki('[data-svelte="svelte-1phssyn"]',document.head);c=s(f,"META",{name:!0,content:!0}),f.forEach(a),x=p(t),u=s(t,"H1",{class:!0});var Ot=n(u);g=s(Ot,"A",{id:!0,class:!0,href:!0});var ro=n(g);q=s(ro,"SPAN",{});var lo=n(q);b(r.$$.fragment,lo),lo.forEach(a),ro.forEach(a),h=p(Ot),A=s(Ot,"SPAN",{});var io=n(A);P=i(io,"Evaluator"),io.forEach(a),Ot.forEach(a),S=p(t),oe=s(t,"P",{});var co=n(oe);ns=i(co,"The evaluator classes for automatic evaluation."),co.forEach(a),go=p(t),se=s(t,"H2",{class:!0});var Mt=n(se);fe=s(Mt,"A",{id:!0,class:!0,href:!0});var po=n(fe);ia=s(po,"SPAN",{});var uo=n(ia);b(Ze.$$.fragment,uo),uo.forEach(a),po.forEach(a),rs=p(Mt),ca=s(Mt,"SPAN",{});var mo=n(ca);ls=i(mo,"Evaluator classes"),mo.forEach(a),Mt.forEach(a),fo=p(t),Lt=s(t,"P",{});var rl=n(Lt);is=i(rl,"The main entry point for using the evaluator:"),rl.forEach(a),ho=p(t),O=s(t,"DIV",{class:!0});var Xt=n(O);b(et.$$.fragment,Xt),cs=p(Xt),M=s(Xt,"P",{});var Qe=n(M);ds=i(Qe,"Utility factory method to build an "),Rt=s(Qe,"A",{href:!0});var ll=n(Rt);ps=i(ll,"Evaluator"),ll.forEach(a),us=i(Qe,`.
Evaluators encapsulate a task and a default metric name. They leverage `),da=s(Qe,"CODE",{});var il=n(da);ms=i(il,"pipeline"),il.forEach(a),gs=i(Qe," functionalify from "),pa=s(Qe,"CODE",{});var cl=n(pa);fs=i(cl,"transformers"),cl.forEach(a),hs=i(Qe,`
to simplify the evaluation of multiple combinations of models, datasets and metrics for a given task.`),Qe.forEach(a),vs=p(Xt),b(he.$$.fragment,Xt),Xt.forEach(a),vo=p(t),Gt=s(t,"P",{});var dl=n(Gt);_s=i(dl,"The base class for all evaluator classes:"),dl.forEach(a),_o=p(t),k=s(t,"DIV",{class:!0});var N=n(k);b(tt.$$.fragment,N),bs=p(N),ua=s(N,"P",{});var pl=n(ua);ws=i(pl,`The Evaluator class is the class from which all evaluators inherit. Refer to this class for methods shared across
different evaluators.
Base class implementing evaluator operations.`),pl.forEach(a),$s=p(N),ve=s(N,"DIV",{class:!0});var Ao=n(ve);b(at.$$.fragment,Ao),ys=p(Ao),ma=s(Ao,"P",{});var ul=n(ma);Es=i(ul,"Ensure the columns required for the evaluation are present in the dataset."),ul.forEach(a),Ao.forEach(a),xs=p(N),_e=s(N,"DIV",{class:!0});var So=n(_e);b(ot.$$.fragment,So),qs=p(So),ga=s(So,"P",{});var ml=n(ga);ks=i(ml,"Compute and return metrics."),ml.forEach(a),So.forEach(a),Ts=p(N),be=s(N,"DIV",{class:!0});var Uo=n(be);b(st.$$.fragment,Uo),js=p(Uo),fa=s(Uo,"P",{});var gl=n(fa);Ns=i(gl,"Infers which split to use if None is given."),gl.forEach(a),Uo.forEach(a),Ps=p(N),we=s(N,"DIV",{class:!0});var Fo=n(we);b(nt.$$.fragment,Fo),Cs=p(Fo),ha=s(Fo,"P",{});var fl=n(ha);Ds=i(fl,"Load dataset with given subset and split."),fl.forEach(a),Fo.forEach(a),Is=p(N),$e=s(N,"DIV",{class:!0});var Oo=n($e);b(rt.$$.fragment,Oo),zs=p(Oo),lt=s(Oo,"P",{});var Mo=n(lt);As=i(Mo,"A core method of the "),va=s(Mo,"CODE",{});var hl=n(va);Ss=i(hl,"Evaluator"),hl.forEach(a),Us=i(Mo," class, which processes the pipeline outputs for compatibility with the metric."),Mo.forEach(a),Oo.forEach(a),Fs=p(N),ye=s(N,"DIV",{class:!0});var Lo=n(ye);b(it.$$.fragment,Lo),Os=p(Lo),_a=s(Lo,"P",{});var vl=n(_a);Ms=i(vl,"Prepare data."),vl.forEach(a),Lo.forEach(a),Ls=p(N),Ee=s(N,"DIV",{class:!0});var Ro=n(Ee);b(ct.$$.fragment,Ro),Rs=p(Ro),ba=s(Ro,"P",{});var _l=n(ba);Gs=i(_l,"Prepare metric."),_l.forEach(a),Ro.forEach(a),Vs=p(N),xe=s(N,"DIV",{class:!0});var Go=n(xe);b(dt.$$.fragment,Go),Qs=p(Go),wa=s(Go,"P",{});var bl=n(wa);Bs=i(bl,"Prepare pipeline."),bl.forEach(a),Go.forEach(a),N.forEach(a),bo=p(t),ne=s(t,"H2",{class:!0});var Vo=n(ne);qe=s(Vo,"A",{id:!0,class:!0,href:!0});var wl=n(qe);$a=s(wl,"SPAN",{});var $l=n($a);b(pt.$$.fragment,$l),$l.forEach(a),wl.forEach(a),Hs=p(Vo),ya=s(Vo,"SPAN",{});var yl=n(ya);Ys=i(yl,"The task specific evaluators"),yl.forEach(a),Vo.forEach(a),wo=p(t),re=s(t,"H3",{class:!0});var Qo=n(re);ke=s(Qo,"A",{id:!0,class:!0,href:!0});var El=n(ke);Ea=s(El,"SPAN",{});var xl=n(Ea);b(ut.$$.fragment,xl),xl.forEach(a),El.forEach(a),Js=p(Qo),xa=s(Qo,"SPAN",{});var ql=n(xa);Ks=i(ql,"ImageClassificationEvaluator"),ql.forEach(a),Qo.forEach(a),$o=p(t),L=s(t,"DIV",{class:!0});var Zt=n(L);b(mt.$$.fragment,Zt),Ws=p(Zt),R=s(Zt,"P",{});var Be=n(R);Xs=i(Be,`Image classification evaluator.
This image classification evaluator can currently be loaded from `),Vt=s(Be,"A",{href:!0});var kl=n(Vt);Zs=i(kl,"evaluator()"),kl.forEach(a),en=i(Be,` using the default task name
`),qa=s(Be,"CODE",{});var Tl=n(qa);tn=i(Tl,"image-classification"),Tl.forEach(a),an=i(Be,`.
Methods in this class assume a data format compatible with the `),ka=s(Be,"CODE",{});var jl=n(ka);on=i(jl,"ImageClassificationPipeline"),jl.forEach(a),sn=i(Be,"."),Be.forEach(a),nn=p(Zt),K=s(Zt,"DIV",{class:!0});var ea=n(K);b(gt.$$.fragment,ea),rn=p(ea),Ta=s(ea,"P",{});var Nl=n(Ta);ln=i(Nl,"Compute the metric for a given pipeline and dataset combination."),Nl.forEach(a),cn=p(ea),b(Te.$$.fragment,ea),ea.forEach(a),Zt.forEach(a),yo=p(t),le=s(t,"H3",{class:!0});var Bo=n(le);je=s(Bo,"A",{id:!0,class:!0,href:!0});var Pl=n(je);ja=s(Pl,"SPAN",{});var Cl=n(ja);b(ft.$$.fragment,Cl),Cl.forEach(a),Pl.forEach(a),dn=p(Bo),Na=s(Bo,"SPAN",{});var Dl=n(Na);pn=i(Dl,"QuestionAnsweringEvaluator"),Dl.forEach(a),Bo.forEach(a),Eo=p(t),D=s(t,"DIV",{class:!0});var X=n(D);b(ht.$$.fragment,X),un=p(X),vt=s(X,"P",{});var Ho=n(vt);mn=i(Ho,`Question answering evaluator. This evaluator handles
`),Ne=s(Ho,"A",{href:!0,rel:!0});var sl=n(Ne);Pa=s(sl,"STRONG",{});var Il=n(Pa);gn=i(Il,"extractive"),Il.forEach(a),fn=i(sl," question answering"),sl.forEach(a),hn=i(Ho,`,
where the answer to the question is extracted from a context.`),Ho.forEach(a),vn=p(X),ie=s(X,"P",{});var ta=n(ie);_n=i(ta,"This question answering evaluator can currently be loaded from "),Qt=s(ta,"A",{href:!0});var zl=n(Qt);bn=i(zl,"evaluator()"),zl.forEach(a),wn=i(ta,` using the default task name
`),Ca=s(ta,"CODE",{});var Al=n(Ca);$n=i(Al,"question-answering"),Al.forEach(a),yn=i(ta,"."),ta.forEach(a),En=p(X),_t=s(X,"P",{});var Yo=n(_t);xn=i(Yo,`Methods in this class assume a data format compatible with the
`),bt=s(Yo,"A",{href:!0,rel:!0});var Sl=n(bt);Da=s(Sl,"CODE",{});var Ul=n(Da);qn=i(Ul,"QuestionAnsweringPipeline"),Ul.forEach(a),Sl.forEach(a),kn=i(Yo,"."),Yo.forEach(a),Tn=p(X),z=s(X,"DIV",{class:!0});var Z=n(z);b(wt.$$.fragment,Z),jn=p(Z),Ia=s(Z,"P",{});var Fl=n(Ia);Nn=i(Fl,"Compute the metric for a given pipeline and dataset combination."),Fl.forEach(a),Pn=p(Z),b(Pe.$$.fragment,Z),Cn=p(Z),b(Ce.$$.fragment,Z),Dn=p(Z),b(De.$$.fragment,Z),Z.forEach(a),X.forEach(a),xo=p(t),ce=s(t,"H3",{class:!0});var Jo=n(ce);Ie=s(Jo,"A",{id:!0,class:!0,href:!0});var Ol=n(Ie);za=s(Ol,"SPAN",{});var Ml=n(za);b($t.$$.fragment,Ml),Ml.forEach(a),Ol.forEach(a),In=p(Jo),Aa=s(Jo,"SPAN",{});var Ll=n(Aa);zn=i(Ll,"TextClassificationEvaluator"),Ll.forEach(a),Jo.forEach(a),qo=p(t),G=s(t,"DIV",{class:!0});var aa=n(G);b(yt.$$.fragment,aa),An=p(aa),U=s(aa,"P",{});var ee=n(U);Sn=i(ee,`Text classification evaluator.
This text classification evaluator can currently be loaded from `),Bt=s(ee,"A",{href:!0});var Rl=n(Bt);Un=i(Rl,"evaluator()"),Rl.forEach(a),Fn=i(ee,` using the default task name
`),Sa=s(ee,"CODE",{});var Gl=n(Sa);On=i(Gl,"text-classification"),Gl.forEach(a),Mn=i(ee," or with a "),Ua=s(ee,"CODE",{});var Vl=n(Ua);Ln=i(Vl,'"sentiment-analysis"'),Vl.forEach(a),Rn=i(ee,` alias.
Methods in this class assume a data format compatible with the `),Fa=s(ee,"CODE",{});var Ql=n(Fa);Gn=i(Ql,"TextClassificationPipeline"),Ql.forEach(a),Vn=i(ee,` - a single textual
feature as input and a categorical label as output.`),ee.forEach(a),Qn=p(aa),W=s(aa,"DIV",{class:!0});var oa=n(W);b(Et.$$.fragment,oa),Bn=p(oa),Oa=s(oa,"P",{});var Bl=n(Oa);Hn=i(Bl,"Compute the metric for a given pipeline and dataset combination."),Bl.forEach(a),Yn=p(oa),b(ze.$$.fragment,oa),oa.forEach(a),aa.forEach(a),ko=p(t),de=s(t,"H3",{class:!0});var Ko=n(de);Ae=s(Ko,"A",{id:!0,class:!0,href:!0});var Hl=n(Ae);Ma=s(Hl,"SPAN",{});var Yl=n(Ma);b(xt.$$.fragment,Yl),Yl.forEach(a),Hl.forEach(a),Jn=p(Ko),La=s(Ko,"SPAN",{});var Jl=n(La);Kn=i(Jl,"TokenClassificationEvaluator"),Jl.forEach(a),Ko.forEach(a),To=p(t),I=s(t,"DIV",{class:!0});var te=n(I);b(qt.$$.fragment,te),Wn=p(te),Ra=s(te,"P",{});var Kl=n(Ra);Xn=i(Kl,"Token classification evaluator."),Kl.forEach(a),Zn=p(te),pe=s(te,"P",{});var sa=n(pe);er=i(sa,"This token classification evaluator can currently be loaded from "),Ht=s(sa,"A",{href:!0});var Wl=n(Ht);tr=i(Wl,"evaluator()"),Wl.forEach(a),ar=i(sa,` using the default task name
`),Ga=s(sa,"CODE",{});var Xl=n(Ga);or=i(Xl,"token-classification"),Xl.forEach(a),sr=i(sa,"."),sa.forEach(a),nr=p(te),kt=s(te,"P",{});var Wo=n(kt);rr=i(Wo,"Methods in this class assume a data format compatible with the "),Va=s(Wo,"CODE",{});var Zl=n(Va);lr=i(Zl,"TokenClassificationPipeline"),Zl.forEach(a),ir=i(Wo,"."),Wo.forEach(a),cr=p(te),C=s(te,"DIV",{class:!0});var F=n(C);b(Tt.$$.fragment,F),dr=p(F),Qa=s(F,"P",{});var ei=n(Qa);pr=i(ei,"Compute the metric for a given pipeline and dataset combination."),ei.forEach(a),ur=p(F),jt=s(F,"P",{});var Xo=n(jt);mr=i(Xo,"The dataset input and label columns are expected to be formatted as a list of words and a list of labels respectively, following "),Nt=s(Xo,"A",{href:!0,rel:!0});var ti=n(Nt);gr=i(ti,"conll2003 dataset"),ti.forEach(a),fr=i(Xo,". Datasets whose inputs are single strings, and labels are a list of offset are not supported."),Xo.forEach(a),hr=p(F),b(Se.$$.fragment,F),vr=p(F),b(Ue.$$.fragment,F),_r=p(F),b(Fe.$$.fragment,F),F.forEach(a),te.forEach(a),jo=p(t),ue=s(t,"H3",{class:!0});var Zo=n(ue);Oe=s(Zo,"A",{id:!0,class:!0,href:!0});var ai=n(Oe);Ba=s(ai,"SPAN",{});var oi=n(Ba);b(Pt.$$.fragment,oi),oi.forEach(a),ai.forEach(a),br=p(Zo),Ha=s(Zo,"SPAN",{});var si=n(Ha);wr=i(si,"Text2TextGenerationEvaluator"),si.forEach(a),Zo.forEach(a),No=p(t),V=s(t,"DIV",{class:!0});var na=n(V);b(Ct.$$.fragment,na),$r=p(na),Q=s(na,"P",{});var He=n(Q);yr=i(He,`Text2Text generation evaluator.
This Text2Text generation evaluator can currently be loaded from `),Yt=s(He,"A",{href:!0});var ni=n(Yt);Er=i(ni,"evaluator()"),ni.forEach(a),xr=i(He,` using the default task name
`),Ya=s(He,"CODE",{});var ri=n(Ya);qr=i(ri,"text2text-generation"),ri.forEach(a),kr=i(He,`.
Methods in this class assume a data format compatible with the `),Ja=s(He,"CODE",{});var li=n(Ja);Tr=i(li,"Text2TextGenerationPipeline"),li.forEach(a),jr=i(He,"."),He.forEach(a),Nr=p(na),Me=s(na,"DIV",{class:!0});var es=n(Me);b(Dt.$$.fragment,es),Pr=p(es),Ka=s(es,"P",{});var ii=n(Ka);Cr=i(ii,"Compute the metric for a given pipeline and dataset combination."),ii.forEach(a),es.forEach(a),na.forEach(a),Po=p(t),me=s(t,"H3",{class:!0});var ts=n(me);Le=s(ts,"A",{id:!0,class:!0,href:!0});var ci=n(Le);Wa=s(ci,"SPAN",{});var di=n(Wa);b(It.$$.fragment,di),di.forEach(a),ci.forEach(a),Dr=p(ts),Xa=s(ts,"SPAN",{});var pi=n(Xa);Ir=i(pi,"SummarizationEvaluator"),pi.forEach(a),ts.forEach(a),Co=p(t),B=s(t,"DIV",{class:!0});var ra=n(B);b(zt.$$.fragment,ra),zr=p(ra),H=s(ra,"P",{});var Ye=n(H);Ar=i(Ye,`Text summarization evaluator.
This text summarization evaluator can currently be loaded from `),Jt=s(Ye,"A",{href:!0});var ui=n(Jt);Sr=i(ui,"evaluator()"),ui.forEach(a),Ur=i(Ye,` using the default task name
`),Za=s(Ye,"CODE",{});var mi=n(Za);Fr=i(mi,"summarization"),mi.forEach(a),Or=i(Ye,`.
Methods in this class assume a data format compatible with the `),Kt=s(Ye,"A",{href:!0});var gi=n(Kt);Mr=i(gi,"SummarizationEvaluator"),gi.forEach(a),Lr=i(Ye,"."),Ye.forEach(a),Rr=p(ra),Re=s(ra,"DIV",{class:!0});var as=n(Re);b(At.$$.fragment,as),Gr=p(as),eo=s(as,"P",{});var fi=n(eo);Vr=i(fi,"Compute the metric for a given pipeline and dataset combination."),fi.forEach(a),as.forEach(a),ra.forEach(a),Do=p(t),ge=s(t,"H3",{class:!0});var os=n(ge);Ge=s(os,"A",{id:!0,class:!0,href:!0});var hi=n(Ge);to=s(hi,"SPAN",{});var vi=n(to);b(St.$$.fragment,vi),vi.forEach(a),hi.forEach(a),Qr=p(os),ao=s(os,"SPAN",{});var _i=n(ao);Br=i(_i,"TranslationEvaluator"),_i.forEach(a),os.forEach(a),Io=p(t),Y=s(t,"DIV",{class:!0});var la=n(Y);b(Ut.$$.fragment,la),Hr=p(la),J=s(la,"P",{});var Je=n(J);Yr=i(Je,`Translation evaluator.
This translation generation evaluator can currently be loaded from `),Wt=s(Je,"A",{href:!0});var bi=n(Wt);Jr=i(bi,"evaluator()"),bi.forEach(a),Kr=i(Je,` using the default task name
`),oo=s(Je,"CODE",{});var wi=n(oo);Wr=i(wi,"translation"),wi.forEach(a),Xr=i(Je,`.
Methods in this class assume a data format compatible with the `),so=s(Je,"CODE",{});var $i=n(so);Zr=i($i,"TranslationPipeline"),$i.forEach(a),el=i(Je,"."),Je.forEach(a),tl=p(la),Ve=s(la,"DIV",{class:!0});var ss=n(Ve);b(Ft.$$.fragment,ss),al=p(ss),no=s(ss,"P",{});var yi=n(no);ol=i(yi,"Compute the metric for a given pipeline and dataset combination."),yi.forEach(a),ss.forEach(a),la.forEach(a),this.h()},h(){m(c,"name","hf:doc:metadata"),m(c,"content",JSON.stringify(Mi)),m(g,"id","evaluator"),m(g,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),m(g,"href","#evaluator"),m(u,"class","relative group"),m(fe,"id","evaluate.evaluator"),m(fe,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),m(fe,"href","#evaluate.evaluator"),m(se,"class","relative group"),m(Rt,"href","/docs/evaluate/pr_330/en/package_reference/evaluator_classes#evaluate.Evaluator"),m(O,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),m(ve,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),m(_e,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),m(be,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),m(we,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),m($e,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),m(ye,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),m(Ee,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),m(xe,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),m(k,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),m(qe,"id","the-task-specific-evaluators"),m(qe,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),m(qe,"href","#the-task-specific-evaluators"),m(ne,"class","relative group"),m(ke,"id","evaluate.ImageClassificationEvaluator"),m(ke,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),m(ke,"href","#evaluate.ImageClassificationEvaluator"),m(re,"class","relative group"),m(Vt,"href","/docs/evaluate/pr_330/en/package_reference/evaluator_classes#evaluate.evaluator"),m(K,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),m(L,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),m(je,"id","evaluate.QuestionAnsweringEvaluator"),m(je,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),m(je,"href","#evaluate.QuestionAnsweringEvaluator"),m(le,"class","relative group"),m(Ne,"href","https://huggingface.co/docs/transformers/task_summary#extractive-question-answering"),m(Ne,"rel","nofollow"),m(Qt,"href","/docs/evaluate/pr_330/en/package_reference/evaluator_classes#evaluate.evaluator"),m(bt,"href","https://huggingface.co/docs/transformers/en/main_classes/pipelines#transformers.QuestionAnsweringPipeline"),m(bt,"rel","nofollow"),m(z,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),m(D,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),m(Ie,"id","evaluate.TextClassificationEvaluator"),m(Ie,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),m(Ie,"href","#evaluate.TextClassificationEvaluator"),m(ce,"class","relative group"),m(Bt,"href","/docs/evaluate/pr_330/en/package_reference/evaluator_classes#evaluate.evaluator"),m(W,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),m(G,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),m(Ae,"id","evaluate.TokenClassificationEvaluator"),m(Ae,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),m(Ae,"href","#evaluate.TokenClassificationEvaluator"),m(de,"class","relative group"),m(Ht,"href","/docs/evaluate/pr_330/en/package_reference/evaluator_classes#evaluate.evaluator"),m(Nt,"href","https://huggingface.co/datasets/conll2003"),m(Nt,"rel","nofollow"),m(C,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),m(I,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),m(Oe,"id","evaluate.Text2TextGenerationEvaluator"),m(Oe,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),m(Oe,"href","#evaluate.Text2TextGenerationEvaluator"),m(ue,"class","relative group"),m(Yt,"href","/docs/evaluate/pr_330/en/package_reference/evaluator_classes#evaluate.evaluator"),m(Me,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),m(V,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),m(Le,"id","evaluate.SummarizationEvaluator"),m(Le,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),m(Le,"href","#evaluate.SummarizationEvaluator"),m(me,"class","relative group"),m(Jt,"href","/docs/evaluate/pr_330/en/package_reference/evaluator_classes#evaluate.evaluator"),m(Kt,"href","/docs/evaluate/pr_330/en/package_reference/evaluator_classes#evaluate.SummarizationEvaluator"),m(Re,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),m(B,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),m(Ge,"id","evaluate.TranslationEvaluator"),m(Ge,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),m(Ge,"href","#evaluate.TranslationEvaluator"),m(ge,"class","relative group"),m(Wt,"href","/docs/evaluate/pr_330/en/package_reference/evaluator_classes#evaluate.evaluator"),m(Ve,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),m(Y,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8")},m(t,f){e(document.head,c),v(t,x,f),v(t,u,f),e(u,g),e(g,q),w(r,q,null),e(u,h),e(u,A),e(A,P),v(t,S,f),v(t,oe,f),e(oe,ns),v(t,go,f),v(t,se,f),e(se,fe),e(fe,ia),w(Ze,ia,null),e(se,rs),e(se,ca),e(ca,ls),v(t,fo,f),v(t,Lt,f),e(Lt,is),v(t,ho,f),v(t,O,f),w(et,O,null),e(O,cs),e(O,M),e(M,ds),e(M,Rt),e(Rt,ps),e(M,us),e(M,da),e(da,ms),e(M,gs),e(M,pa),e(pa,fs),e(M,hs),e(O,vs),w(he,O,null),v(t,vo,f),v(t,Gt,f),e(Gt,_s),v(t,_o,f),v(t,k,f),w(tt,k,null),e(k,bs),e(k,ua),e(ua,ws),e(k,$s),e(k,ve),w(at,ve,null),e(ve,ys),e(ve,ma),e(ma,Es),e(k,xs),e(k,_e),w(ot,_e,null),e(_e,qs),e(_e,ga),e(ga,ks),e(k,Ts),e(k,be),w(st,be,null),e(be,js),e(be,fa),e(fa,Ns),e(k,Ps),e(k,we),w(nt,we,null),e(we,Cs),e(we,ha),e(ha,Ds),e(k,Is),e(k,$e),w(rt,$e,null),e($e,zs),e($e,lt),e(lt,As),e(lt,va),e(va,Ss),e(lt,Us),e(k,Fs),e(k,ye),w(it,ye,null),e(ye,Os),e(ye,_a),e(_a,Ms),e(k,Ls),e(k,Ee),w(ct,Ee,null),e(Ee,Rs),e(Ee,ba),e(ba,Gs),e(k,Vs),e(k,xe),w(dt,xe,null),e(xe,Qs),e(xe,wa),e(wa,Bs),v(t,bo,f),v(t,ne,f),e(ne,qe),e(qe,$a),w(pt,$a,null),e(ne,Hs),e(ne,ya),e(ya,Ys),v(t,wo,f),v(t,re,f),e(re,ke),e(ke,Ea),w(ut,Ea,null),e(re,Js),e(re,xa),e(xa,Ks),v(t,$o,f),v(t,L,f),w(mt,L,null),e(L,Ws),e(L,R),e(R,Xs),e(R,Vt),e(Vt,Zs),e(R,en),e(R,qa),e(qa,tn),e(R,an),e(R,ka),e(ka,on),e(R,sn),e(L,nn),e(L,K),w(gt,K,null),e(K,rn),e(K,Ta),e(Ta,ln),e(K,cn),w(Te,K,null),v(t,yo,f),v(t,le,f),e(le,je),e(je,ja),w(ft,ja,null),e(le,dn),e(le,Na),e(Na,pn),v(t,Eo,f),v(t,D,f),w(ht,D,null),e(D,un),e(D,vt),e(vt,mn),e(vt,Ne),e(Ne,Pa),e(Pa,gn),e(Ne,fn),e(vt,hn),e(D,vn),e(D,ie),e(ie,_n),e(ie,Qt),e(Qt,bn),e(ie,wn),e(ie,Ca),e(Ca,$n),e(ie,yn),e(D,En),e(D,_t),e(_t,xn),e(_t,bt),e(bt,Da),e(Da,qn),e(_t,kn),e(D,Tn),e(D,z),w(wt,z,null),e(z,jn),e(z,Ia),e(Ia,Nn),e(z,Pn),w(Pe,z,null),e(z,Cn),w(Ce,z,null),e(z,Dn),w(De,z,null),v(t,xo,f),v(t,ce,f),e(ce,Ie),e(Ie,za),w($t,za,null),e(ce,In),e(ce,Aa),e(Aa,zn),v(t,qo,f),v(t,G,f),w(yt,G,null),e(G,An),e(G,U),e(U,Sn),e(U,Bt),e(Bt,Un),e(U,Fn),e(U,Sa),e(Sa,On),e(U,Mn),e(U,Ua),e(Ua,Ln),e(U,Rn),e(U,Fa),e(Fa,Gn),e(U,Vn),e(G,Qn),e(G,W),w(Et,W,null),e(W,Bn),e(W,Oa),e(Oa,Hn),e(W,Yn),w(ze,W,null),v(t,ko,f),v(t,de,f),e(de,Ae),e(Ae,Ma),w(xt,Ma,null),e(de,Jn),e(de,La),e(La,Kn),v(t,To,f),v(t,I,f),w(qt,I,null),e(I,Wn),e(I,Ra),e(Ra,Xn),e(I,Zn),e(I,pe),e(pe,er),e(pe,Ht),e(Ht,tr),e(pe,ar),e(pe,Ga),e(Ga,or),e(pe,sr),e(I,nr),e(I,kt),e(kt,rr),e(kt,Va),e(Va,lr),e(kt,ir),e(I,cr),e(I,C),w(Tt,C,null),e(C,dr),e(C,Qa),e(Qa,pr),e(C,ur),e(C,jt),e(jt,mr),e(jt,Nt),e(Nt,gr),e(jt,fr),e(C,hr),w(Se,C,null),e(C,vr),w(Ue,C,null),e(C,_r),w(Fe,C,null),v(t,jo,f),v(t,ue,f),e(ue,Oe),e(Oe,Ba),w(Pt,Ba,null),e(ue,br),e(ue,Ha),e(Ha,wr),v(t,No,f),v(t,V,f),w(Ct,V,null),e(V,$r),e(V,Q),e(Q,yr),e(Q,Yt),e(Yt,Er),e(Q,xr),e(Q,Ya),e(Ya,qr),e(Q,kr),e(Q,Ja),e(Ja,Tr),e(Q,jr),e(V,Nr),e(V,Me),w(Dt,Me,null),e(Me,Pr),e(Me,Ka),e(Ka,Cr),v(t,Po,f),v(t,me,f),e(me,Le),e(Le,Wa),w(It,Wa,null),e(me,Dr),e(me,Xa),e(Xa,Ir),v(t,Co,f),v(t,B,f),w(zt,B,null),e(B,zr),e(B,H),e(H,Ar),e(H,Jt),e(Jt,Sr),e(H,Ur),e(H,Za),e(Za,Fr),e(H,Or),e(H,Kt),e(Kt,Mr),e(H,Lr),e(B,Rr),e(B,Re),w(At,Re,null),e(Re,Gr),e(Re,eo),e(eo,Vr),v(t,Do,f),v(t,ge,f),e(ge,Ge),e(Ge,to),w(St,to,null),e(ge,Qr),e(ge,ao),e(ao,Br),v(t,Io,f),v(t,Y,f),w(Ut,Y,null),e(Y,Hr),e(Y,J),e(J,Yr),e(J,Wt),e(Wt,Jr),e(J,Kr),e(J,oo),e(oo,Wr),e(J,Xr),e(J,so),e(so,Zr),e(J,el),e(Y,tl),e(Y,Ve),w(Ft,Ve,null),e(Ve,al),e(Ve,no),e(no,ol),zo=!0},p(t,[f]){const Ot={};f&2&&(Ot.$$scope={dirty:f,ctx:t}),he.$set(Ot);const ro={};f&2&&(ro.$$scope={dirty:f,ctx:t}),Te.$set(ro);const lo={};f&2&&(lo.$$scope={dirty:f,ctx:t}),Pe.$set(lo);const io={};f&2&&(io.$$scope={dirty:f,ctx:t}),Ce.$set(io);const co={};f&2&&(co.$$scope={dirty:f,ctx:t}),De.$set(co);const Mt={};f&2&&(Mt.$$scope={dirty:f,ctx:t}),ze.$set(Mt);const po={};f&2&&(po.$$scope={dirty:f,ctx:t}),Se.$set(po);const uo={};f&2&&(uo.$$scope={dirty:f,ctx:t}),Ue.$set(uo);const mo={};f&2&&(mo.$$scope={dirty:f,ctx:t}),Fe.$set(mo)},i(t){zo||($(r.$$.fragment,t),$(Ze.$$.fragment,t),$(et.$$.fragment,t),$(he.$$.fragment,t),$(tt.$$.fragment,t),$(at.$$.fragment,t),$(ot.$$.fragment,t),$(st.$$.fragment,t),$(nt.$$.fragment,t),$(rt.$$.fragment,t),$(it.$$.fragment,t),$(ct.$$.fragment,t),$(dt.$$.fragment,t),$(pt.$$.fragment,t),$(ut.$$.fragment,t),$(mt.$$.fragment,t),$(gt.$$.fragment,t),$(Te.$$.fragment,t),$(ft.$$.fragment,t),$(ht.$$.fragment,t),$(wt.$$.fragment,t),$(Pe.$$.fragment,t),$(Ce.$$.fragment,t),$(De.$$.fragment,t),$($t.$$.fragment,t),$(yt.$$.fragment,t),$(Et.$$.fragment,t),$(ze.$$.fragment,t),$(xt.$$.fragment,t),$(qt.$$.fragment,t),$(Tt.$$.fragment,t),$(Se.$$.fragment,t),$(Ue.$$.fragment,t),$(Fe.$$.fragment,t),$(Pt.$$.fragment,t),$(Ct.$$.fragment,t),$(Dt.$$.fragment,t),$(It.$$.fragment,t),$(zt.$$.fragment,t),$(At.$$.fragment,t),$(St.$$.fragment,t),$(Ut.$$.fragment,t),$(Ft.$$.fragment,t),zo=!0)},o(t){y(r.$$.fragment,t),y(Ze.$$.fragment,t),y(et.$$.fragment,t),y(he.$$.fragment,t),y(tt.$$.fragment,t),y(at.$$.fragment,t),y(ot.$$.fragment,t),y(st.$$.fragment,t),y(nt.$$.fragment,t),y(rt.$$.fragment,t),y(it.$$.fragment,t),y(ct.$$.fragment,t),y(dt.$$.fragment,t),y(pt.$$.fragment,t),y(ut.$$.fragment,t),y(mt.$$.fragment,t),y(gt.$$.fragment,t),y(Te.$$.fragment,t),y(ft.$$.fragment,t),y(ht.$$.fragment,t),y(wt.$$.fragment,t),y(Pe.$$.fragment,t),y(Ce.$$.fragment,t),y(De.$$.fragment,t),y($t.$$.fragment,t),y(yt.$$.fragment,t),y(Et.$$.fragment,t),y(ze.$$.fragment,t),y(xt.$$.fragment,t),y(qt.$$.fragment,t),y(Tt.$$.fragment,t),y(Se.$$.fragment,t),y(Ue.$$.fragment,t),y(Fe.$$.fragment,t),y(Pt.$$.fragment,t),y(Ct.$$.fragment,t),y(Dt.$$.fragment,t),y(It.$$.fragment,t),y(zt.$$.fragment,t),y(At.$$.fragment,t),y(St.$$.fragment,t),y(Ut.$$.fragment,t),y(Ft.$$.fragment,t),zo=!1},d(t){a(c),t&&a(x),t&&a(u),E(r),t&&a(S),t&&a(oe),t&&a(go),t&&a(se),E(Ze),t&&a(fo),t&&a(Lt),t&&a(ho),t&&a(O),E(et),E(he),t&&a(vo),t&&a(Gt),t&&a(_o),t&&a(k),E(tt),E(at),E(ot),E(st),E(nt),E(rt),E(it),E(ct),E(dt),t&&a(bo),t&&a(ne),E(pt),t&&a(wo),t&&a(re),E(ut),t&&a($o),t&&a(L),E(mt),E(gt),E(Te),t&&a(yo),t&&a(le),E(ft),t&&a(Eo),t&&a(D),E(ht),E(wt),E(Pe),E(Ce),E(De),t&&a(xo),t&&a(ce),E($t),t&&a(qo),t&&a(G),E(yt),E(Et),E(ze),t&&a(ko),t&&a(de),E(xt),t&&a(To),t&&a(I),E(qt),E(Tt),E(Se),E(Ue),E(Fe),t&&a(jo),t&&a(ue),E(Pt),t&&a(No),t&&a(V),E(Ct),E(Dt),t&&a(Po),t&&a(me),E(It),t&&a(Co),t&&a(B),E(zt),E(At),t&&a(Do),t&&a(ge),E(St),t&&a(Io),t&&a(Y),E(Ut),E(Ft)}}}const Mi={local:"evaluator",sections:[{local:"evaluate.evaluator",title:"Evaluator classes"},{local:"the-task-specific-evaluators",sections:[{local:"evaluate.ImageClassificationEvaluator",title:"ImageClassificationEvaluator"},{local:"evaluate.QuestionAnsweringEvaluator",title:"QuestionAnsweringEvaluator"},{local:"evaluate.TextClassificationEvaluator",title:"TextClassificationEvaluator"},{local:"evaluate.TokenClassificationEvaluator",title:"TokenClassificationEvaluator"},{local:"evaluate.Text2TextGenerationEvaluator",title:"Text2TextGenerationEvaluator"},{local:"evaluate.SummarizationEvaluator",title:"SummarizationEvaluator"},{local:"evaluate.TranslationEvaluator",title:"TranslationEvaluator"}],title:"The task specific evaluators"}],title:"Evaluator"};function Li(T){return Ti(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class Yi extends Ei{constructor(c){super();xi(this,c,Li,Oi,qi,{})}}export{Yi as default,Mi as metadata};
