import{S as Pu,i as Au,s as Iu,e as l,k as r,w as m,t as n,M as Ou,c as s,d as a,m as c,a as o,x as v,h as i,b as d,G as e,g as h,y as _,q as g,o as E,B as b,v as Su}from"../chunks/vendor-hf-doc-builder.js";import{T as Cu}from"../chunks/Tip-hf-doc-builder.js";import{I as oe}from"../chunks/IconCopyLink-hf-doc-builder.js";import{C as Me}from"../chunks/CodeBlock-hf-doc-builder.js";function Nu(mt){let u,G,f,k,ne,H,Fe,te;return{c(){u=l("p"),G=n("Without specifying a device, the default for model inference will be the first GPU on the machine if one is available, and else CPU. If you want to use a specific device you can pass "),f=l("code"),k=n("device"),ne=n(" to "),H=l("code"),Fe=n("compute"),te=n(" where -1 will use the GPU and a positive integer (starting with 0) will use the associated CUDA device.")},l(ie){u=s(ie,"P",{});var O=o(u);G=i(O,"Without specifying a device, the default for model inference will be the first GPU on the machine if one is available, and else CPU. If you want to use a specific device you can pass "),f=s(O,"CODE",{});var vt=o(f);k=i(vt,"device"),vt.forEach(a),ne=i(O," to "),H=s(O,"CODE",{});var Qe=o(H);Fe=i(Qe,"compute"),Qe.forEach(a),te=i(O," where -1 will use the GPU and a positive integer (starting with 0) will use the associated CUDA device."),O.forEach(a)},m(ie,O){h(ie,u,O),e(u,G),e(u,f),e(f,k),e(u,ne),e(u,H),e(H,Fe),e(u,te)},d(ie){ie&&a(u)}}}function Lu(mt){let u,G;return{c(){u=l("p"),G=n("The time performances can give useful indication on model speed for inference but should be taken with a grain of salt: they include all the processing that goes on in the pipeline. This may include tokenizing, post-processing, that may be different depending on the model. Furthermore, it depends a lot on the hardware you are running the evaluation on and you may be able to improve the performance by optimizing things like the batch size.")},l(f){u=s(f,"P",{});var k=o(u);G=i(k,"The time performances can give useful indication on model speed for inference but should be taken with a grain of salt: they include all the processing that goes on in the pipeline. This may include tokenizing, post-processing, that may be different depending on the model. Furthermore, it depends a lot on the hardware you are running the evaluation on and you may be able to improve the performance by optimizing things like the batch size."),k.forEach(a)},m(f,k){h(f,u,k),e(u,G)},d(f){f&&a(u)}}}function Hu(mt){let u,G,f,k,ne,H,Fe,te,ie,O,vt,Qe,W,Do,Ka,Co,Po,Xa,Ao,Io,_e,Oo,Za,So,No,Lo,vs,_t,Ho,_s,w,ge,el,Bo,zo,gt,Go,Wo,Ro,Ee,tl,Uo,Vo,Et,Mo,Fo,Qo,be,al,Jo,Yo,bt,Ko,Xo,Zo,we,ll,en,tn,wt,an,ln,sn,ke,sl,on,nn,kt,rn,cn,pn,ye,ol,dn,hn,yt,un,fn,mn,$e,nl,vn,_n,$t,gn,En,gs,R,bn,il,wn,kn,qt,yn,$n,rl,qn,jn,Es,jt,Tn,bs,re,qe,cl,Je,xn,pl,Dn,ws,Tt,Cn,ks,ae,xt,dl,Pn,An,In,Dt,hl,On,Sn,Nn,$,ul,Ln,Hn,fl,Bn,zn,ml,Gn,Wn,vl,Rn,Un,_l,Vn,Mn,gl,Fn,Qn,ys,je,Jn,El,Yn,Kn,$s,ce,Te,bl,Ye,Xn,wl,Zn,qs,U,ei,kl,ti,ai,yl,li,si,$l,oi,ni,js,Ct,ii,Ts,Ke,xs,xe,Ds,Pt,ri,Cs,Xe,Ps,At,ci,As,De,Is,pe,Ce,ql,Ze,pi,jl,di,Os,Pe,hi,It,ui,fi,Ss,et,Ns,Ot,mi,Ls,tt,Hs,St,vi,Bs,de,Ae,Tl,at,_i,xl,gi,zs,Nt,Ei,Gs,V,Lt,Dl,bi,wi,ki,Ht,Cl,yi,$i,qi,q,Pl,ji,Ti,Al,xi,Di,Il,Ci,Pi,Ol,Ai,Ii,Sl,Oi,Si,Nl,Ni,Li,Hi,Bt,Ll,Bi,zi,Ws,zt,Gi,Rs,he,Ie,Hl,lt,Wi,Bl,Ri,Us,Oe,Ui,zl,Vi,Mi,Vs,st,Ms,Gt,Fi,Fs,Se,Gl,j,Wt,Qi,Ji,Rt,Yi,Ki,Ut,Xi,Zi,Vt,er,tr,Mt,ar,lr,Ft,sr,or,y,T,Qt,nr,ir,Jt,rr,cr,Yt,pr,dr,Kt,hr,ur,Xt,fr,mr,Zt,vr,_r,x,ea,gr,Er,ta,br,wr,aa,kr,yr,la,$r,qr,sa,jr,Tr,oa,xr,Dr,D,na,Cr,Pr,ia,Ar,Ir,ra,Or,Sr,ca,Nr,Lr,pa,Hr,Br,da,zr,Gr,C,ha,Wr,Rr,ua,Ur,Vr,fa,Mr,Fr,ma,Qr,Jr,va,Yr,Kr,_a,Xr,Zr,P,ga,ec,tc,Ea,ac,lc,ba,sc,oc,wa,nc,ic,ka,rc,cc,ya,pc,dc,A,$a,hc,uc,qa,fc,mc,ja,vc,_c,Ta,gc,Ec,xa,bc,wc,Da,kc,yc,I,Ca,$c,qc,Pa,jc,Tc,Aa,xc,Dc,Ia,Cc,Pc,Oa,Ac,Ic,Sa,Oc,Qs,ue,Ne,Wl,ot,Sc,Rl,Nc,Js,Na,Lc,Ys,S,La,Ul,Hc,Bc,zc,Ha,Vl,Gc,Wc,Rc,Ba,Ml,Uc,Vc,Mc,za,Fl,Fc,Qc,Jc,Ga,Ql,Yc,Kc,Ks,Wa,Xc,Xs,fe,Le,Jl,nt,Zc,Yl,ep,Zs,M,tp,it,ap,lp,Kl,sp,op,Xl,np,ip,eo,rt,to,Ra,rp,ao,ct,lo,me,He,Zl,pt,cp,es,pp,so,Ua,dp,oo,le,Va,ts,hp,up,fp,Ma,as,mp,vp,_p,dt,ls,gp,Ep,ss,bp,no,Fa,wp,io,ve,Be,os,ht,kp,ns,yp,ro,Qa,$p,co,ut,po,ze,qp,is,jp,Tp,ho;return H=new oe({}),Je=new oe({}),Ye=new oe({}),Ke=new Me({props:{code:`from datasets import load_dataset
from evaluate import evaluator
from transformers import AutoModelForSequenceClassification, pipeline

data = load_dataset("imdb", split="test").shuffle(seed=42).select(range(1000))
task_evaluator = evaluator("text-classification")

# 1. Pass a model name or path
eval_results = task_evaluator.compute(
    model_or_pipeline="lvwerra/distilbert-imdb",
    data=data,
    label_mapping={"NEGATIVE": 0, "POSITIVE": 1}
)

# 2. Pass an instantiated model
model = AutoModelForSequenceClassification.from_pretrained("lvwerra/distilbert-imdb")

eval_results = task_evaluator.compute(
    model_or_pipeline=model,
    data=data,
    label_mapping={"NEGATIVE": 0, "POSITIVE": 1}
)

# 3. Pass an instantiated pipeline 
pipe = pipeline("text-classification", model="lvwerra/distilbert-imdb")

eval_results = task_evaluator.compute(
    model_or_pipeline=pipe,
    data=data,
    label_mapping={"NEGATIVE": 0, "POSITIVE": 1}
)
print(eval_results)`,highlighted:`<span class="hljs-keyword">from</span> datasets <span class="hljs-keyword">import</span> load_dataset
<span class="hljs-keyword">from</span> evaluate <span class="hljs-keyword">import</span> evaluator
<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoModelForSequenceClassification, pipeline

data = load_dataset(<span class="hljs-string">&quot;imdb&quot;</span>, split=<span class="hljs-string">&quot;test&quot;</span>).shuffle(seed=<span class="hljs-number">42</span>).select(<span class="hljs-built_in">range</span>(<span class="hljs-number">1000</span>))
task_evaluator = evaluator(<span class="hljs-string">&quot;text-classification&quot;</span>)

<span class="hljs-comment"># 1. Pass a model name or path</span>
eval_results = task_evaluator.compute(
    model_or_pipeline=<span class="hljs-string">&quot;lvwerra/distilbert-imdb&quot;</span>,
    data=data,
    label_mapping={<span class="hljs-string">&quot;NEGATIVE&quot;</span>: <span class="hljs-number">0</span>, <span class="hljs-string">&quot;POSITIVE&quot;</span>: <span class="hljs-number">1</span>}
)

<span class="hljs-comment"># 2. Pass an instantiated model</span>
model = AutoModelForSequenceClassification.from_pretrained(<span class="hljs-string">&quot;lvwerra/distilbert-imdb&quot;</span>)

eval_results = task_evaluator.compute(
    model_or_pipeline=model,
    data=data,
    label_mapping={<span class="hljs-string">&quot;NEGATIVE&quot;</span>: <span class="hljs-number">0</span>, <span class="hljs-string">&quot;POSITIVE&quot;</span>: <span class="hljs-number">1</span>}
)

<span class="hljs-comment"># 3. Pass an instantiated pipeline </span>
pipe = pipeline(<span class="hljs-string">&quot;text-classification&quot;</span>, model=<span class="hljs-string">&quot;lvwerra/distilbert-imdb&quot;</span>)

eval_results = task_evaluator.compute(
    model_or_pipeline=pipe,
    data=data,
    label_mapping={<span class="hljs-string">&quot;NEGATIVE&quot;</span>: <span class="hljs-number">0</span>, <span class="hljs-string">&quot;POSITIVE&quot;</span>: <span class="hljs-number">1</span>}
)
<span class="hljs-built_in">print</span>(eval_results)`}}),xe=new Cu({props:{$$slots:{default:[Nu]},$$scope:{ctx:mt}}}),Xe=new Me({props:{code:`{
    'accuracy': 0.918,
    'latency_in_seconds': 0.013,
    'samples_per_second': 78.887,
    'total_time_in_seconds': 12.676
}`,highlighted:`{
    <span class="hljs-string">&#x27;accuracy&#x27;</span>: <span class="hljs-number">0.918</span>,
    <span class="hljs-string">&#x27;latency_in_seconds&#x27;</span>: <span class="hljs-number">0.013</span>,
    <span class="hljs-string">&#x27;samples_per_second&#x27;</span>: <span class="hljs-number">78.887</span>,
    <span class="hljs-string">&#x27;total_time_in_seconds&#x27;</span>: <span class="hljs-number">12.676</span>
}`}}),De=new Cu({props:{$$slots:{default:[Lu]},$$scope:{ctx:mt}}}),Ze=new oe({}),et=new Me({props:{code:`import evaluate

eval_results = task_evaluator.compute(
    model_or_pipeline="lvwerra/distilbert-imdb",
    data=data,
    metric=evaluate.combine(["accuracy", "recall", "precision", "f1"]),
    label_mapping={"NEGATIVE": 0, "POSITIVE": 1}
)
print(eval_results)
`,highlighted:`<span class="hljs-keyword">import</span> evaluate

eval_results = task_evaluator.compute(
    model_or_pipeline=<span class="hljs-string">&quot;lvwerra/distilbert-imdb&quot;</span>,
    data=data,
    metric=evaluate.combine([<span class="hljs-string">&quot;accuracy&quot;</span>, <span class="hljs-string">&quot;recall&quot;</span>, <span class="hljs-string">&quot;precision&quot;</span>, <span class="hljs-string">&quot;f1&quot;</span>]),
    label_mapping={<span class="hljs-string">&quot;NEGATIVE&quot;</span>: <span class="hljs-number">0</span>, <span class="hljs-string">&quot;POSITIVE&quot;</span>: <span class="hljs-number">1</span>}
)
<span class="hljs-built_in">print</span>(eval_results)
`}}),tt=new Me({props:{code:`{
    'accuracy': 0.918,
    'f1': 0.916,
    'precision': 0.9147,
    'recall': 0.9187,
    'latency_in_seconds': 0.013,
    'samples_per_second': 78.887,
    'total_time_in_seconds': 12.676
}`,highlighted:`{
    <span class="hljs-string">&#x27;accuracy&#x27;</span>: <span class="hljs-number">0.918</span>,
    <span class="hljs-string">&#x27;f1&#x27;</span>: <span class="hljs-number">0.916</span>,
    <span class="hljs-string">&#x27;precision&#x27;</span>: <span class="hljs-number">0.9147</span>,
    <span class="hljs-string">&#x27;recall&#x27;</span>: <span class="hljs-number">0.9187</span>,
    <span class="hljs-string">&#x27;latency_in_seconds&#x27;</span>: <span class="hljs-number">0.013</span>,
    <span class="hljs-string">&#x27;samples_per_second&#x27;</span>: <span class="hljs-number">78.887</span>,
    <span class="hljs-string">&#x27;total_time_in_seconds&#x27;</span>: <span class="hljs-number">12.676</span>
}`}}),at=new oe({}),lt=new oe({}),st=new Me({props:{code:`import pandas as pd
from datasets import load_dataset
from evaluate import evaluator
from transformers import pipeline

models = [
    "xlm-roberta-large-finetuned-conll03-english",
    "dbmdz/bert-large-cased-finetuned-conll03-english",
    "elastic/distilbert-base-uncased-finetuned-conll03-english",
    "dbmdz/electra-large-discriminator-finetuned-conll03-english",
    "gunghio/distilbert-base-multilingual-cased-finetuned-conll2003-ner",
    "philschmid/distilroberta-base-ner-conll2003",
    "Jorgeutd/albert-base-v2-finetuned-ner",
]

data = load_dataset("conll2003", split="validation").shuffle().select(1000)
task_evaluator = evaluator("token-classification")

results = []
for model in models:
    results.append(
        task_evaluator.compute(
            model_or_pipeline=model, data=data, metric="seqeval"
            )
        )

df = pd.DataFrame(results, index=models)
df[["overall_f1", "overall_accuracy", "total_time_in_seconds", "samples_per_second", "latency_in_seconds"]]`,highlighted:`<span class="hljs-keyword">import</span> pandas <span class="hljs-keyword">as</span> pd
<span class="hljs-keyword">from</span> datasets <span class="hljs-keyword">import</span> load_dataset
<span class="hljs-keyword">from</span> evaluate <span class="hljs-keyword">import</span> evaluator
<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> pipeline

models = [
    <span class="hljs-string">&quot;xlm-roberta-large-finetuned-conll03-english&quot;</span>,
    <span class="hljs-string">&quot;dbmdz/bert-large-cased-finetuned-conll03-english&quot;</span>,
    <span class="hljs-string">&quot;elastic/distilbert-base-uncased-finetuned-conll03-english&quot;</span>,
    <span class="hljs-string">&quot;dbmdz/electra-large-discriminator-finetuned-conll03-english&quot;</span>,
    <span class="hljs-string">&quot;gunghio/distilbert-base-multilingual-cased-finetuned-conll2003-ner&quot;</span>,
    <span class="hljs-string">&quot;philschmid/distilroberta-base-ner-conll2003&quot;</span>,
    <span class="hljs-string">&quot;Jorgeutd/albert-base-v2-finetuned-ner&quot;</span>,
]

data = load_dataset(<span class="hljs-string">&quot;conll2003&quot;</span>, split=<span class="hljs-string">&quot;validation&quot;</span>).shuffle().select(<span class="hljs-number">1000</span>)
task_evaluator = evaluator(<span class="hljs-string">&quot;token-classification&quot;</span>)

results = []
<span class="hljs-keyword">for</span> model <span class="hljs-keyword">in</span> models:
    results.append(
        task_evaluator.compute(
            model_or_pipeline=model, data=data, metric=<span class="hljs-string">&quot;seqeval&quot;</span>
            )
        )

df = pd.DataFrame(results, index=models)
df[[<span class="hljs-string">&quot;overall_f1&quot;</span>, <span class="hljs-string">&quot;overall_accuracy&quot;</span>, <span class="hljs-string">&quot;total_time_in_seconds&quot;</span>, <span class="hljs-string">&quot;samples_per_second&quot;</span>, <span class="hljs-string">&quot;latency_in_seconds&quot;</span>]]`}}),ot=new oe({}),nt=new oe({}),rt=new Me({props:{code:`from datasets import load_dataset
from evaluate import evaluator

task_evaluator = evaluator("question-answering")

data = load_dataset("squad", split="validation[:1000]")
eval_results = task_evaluator.compute(
    model_or_pipeline="distilbert-base-uncased-distilled-squad",
    data=data,
    metric="squad",
    strategy="bootstrap",
    n_resamples=30
)`,highlighted:`<span class="hljs-keyword">from</span> datasets <span class="hljs-keyword">import</span> load_dataset
<span class="hljs-keyword">from</span> evaluate <span class="hljs-keyword">import</span> evaluator

task_evaluator = evaluator(<span class="hljs-string">&quot;question-answering&quot;</span>)

data = load_dataset(<span class="hljs-string">&quot;squad&quot;</span>, split=<span class="hljs-string">&quot;validation[:1000]&quot;</span>)
eval_results = task_evaluator.compute(
    model_or_pipeline=<span class="hljs-string">&quot;distilbert-base-uncased-distilled-squad&quot;</span>,
    data=data,
    metric=<span class="hljs-string">&quot;squad&quot;</span>,
    strategy=<span class="hljs-string">&quot;bootstrap&quot;</span>,
    n_resamples=<span class="hljs-number">30</span>
)`}}),ct=new Me({props:{code:`{
    'exact_match': 
    {
        'confidence_interval': (79.67, 84.54),
        'score': 82.30,
        'standard_error': 1.28
    },
    'f1': 
    {
        'confidence_interval': (85.30, 88.88),
        'score': 87.23,
        'standard_error': 0.97
    },
    'latency_in_seconds': 0.0085,
    'samples_per_second': 117.31,
    'total_time_in_seconds': 8.52
 }`,highlighted:`{
    <span class="hljs-string">&#x27;exact_match&#x27;</span>: 
    {
        <span class="hljs-string">&#x27;confidence_interval&#x27;</span>: (<span class="hljs-number">79.67</span>, <span class="hljs-number">84.54</span>),
        <span class="hljs-string">&#x27;score&#x27;</span>: <span class="hljs-number">82.30</span>,
        <span class="hljs-string">&#x27;standard_error&#x27;</span>: <span class="hljs-number">1.28</span>
    },
    <span class="hljs-string">&#x27;f1&#x27;</span>: 
    {
        <span class="hljs-string">&#x27;confidence_interval&#x27;</span>: (<span class="hljs-number">85.30</span>, <span class="hljs-number">88.88</span>),
        <span class="hljs-string">&#x27;score&#x27;</span>: <span class="hljs-number">87.23</span>,
        <span class="hljs-string">&#x27;standard_error&#x27;</span>: <span class="hljs-number">0.97</span>
    },
    <span class="hljs-string">&#x27;latency_in_seconds&#x27;</span>: <span class="hljs-number">0.0085</span>,
    <span class="hljs-string">&#x27;samples_per_second&#x27;</span>: <span class="hljs-number">117.31</span>,
    <span class="hljs-string">&#x27;total_time_in_seconds&#x27;</span>: <span class="hljs-number">8.52</span>
 }`}}),pt=new oe({}),ht=new oe({}),ut=new Me({props:{code:`data = load_dataset("imagenet-1k", split="validation", use_auth_token=True)

pipe = pipeline(
    task="image-classification",
    model="facebook/deit-small-distilled-patch16-224"
)

task_evaluator = evaluator("image-classification")
eval_results = task_evaluator.compute(
    model_or_pipeline=pipe,
    data=data,
    metric="accuracy",
    label_mapping=pipe.model.config.label2id
)`,highlighted:`data = load_dataset(<span class="hljs-string">&quot;imagenet-1k&quot;</span>, split=<span class="hljs-string">&quot;validation&quot;</span>, use_auth_token=<span class="hljs-literal">True</span>)

pipe = pipeline(
    task=<span class="hljs-string">&quot;image-classification&quot;</span>,
    model=<span class="hljs-string">&quot;facebook/deit-small-distilled-patch16-224&quot;</span>
)

task_evaluator = evaluator(<span class="hljs-string">&quot;image-classification&quot;</span>)
eval_results = task_evaluator.compute(
    model_or_pipeline=pipe,
    data=data,
    metric=<span class="hljs-string">&quot;accuracy&quot;</span>,
    label_mapping=pipe.model.config.label2id
)`}}),{c(){u=l("meta"),G=r(),f=l("h1"),k=l("a"),ne=l("span"),m(H.$$.fragment),Fe=r(),te=l("span"),ie=n("Using the "),O=l("code"),vt=n("evaluator"),Qe=r(),W=l("p"),Do=n("The "),Ka=l("code"),Co=n("Evaluator"),Po=n(" classes allow to evaluate a  triplet of model, dataset, and metric. The models wrapped in a pipeline, responsible for handling all preprocessing and post-processing and out-of-the-box, "),Xa=l("code"),Ao=n("Evaluator"),Io=n("s support transformers pipelines for the supported tasks, but custom pipelines can be passed, as showcased in the section "),_e=l("a"),Oo=n("Using the "),Za=l("code"),So=n("evaluator"),No=n(" with custom pipelines"),Lo=n("."),vs=r(),_t=l("p"),Ho=n("Currently supported tasks are:"),_s=r(),w=l("ul"),ge=l("li"),el=l("code"),Bo=n('"text-classification"'),zo=n(": will use the "),gt=l("a"),Go=n("TextClassificationEvaluator"),Wo=n("."),Ro=r(),Ee=l("li"),tl=l("code"),Uo=n('"token-classification"'),Vo=n(": will use the "),Et=l("a"),Mo=n("TokenClassificationEvaluator"),Fo=n("."),Qo=r(),be=l("li"),al=l("code"),Jo=n('"question-answering"'),Yo=n(": will use the "),bt=l("a"),Ko=n("QuestionAnsweringEvaluator"),Xo=n("."),Zo=r(),we=l("li"),ll=l("code"),en=n('"image-classification"'),tn=n(": will use the "),wt=l("a"),an=n("ImageClassificationEvaluator"),ln=n("."),sn=r(),ke=l("li"),sl=l("code"),on=n('"text2text-generation"'),nn=n(": will use the "),kt=l("a"),rn=n("Text2TextGenerationEvaluator"),cn=n("."),pn=r(),ye=l("li"),ol=l("code"),dn=n('"summarization"'),hn=n(": will use the "),yt=l("a"),un=n("SummarizationEvaluator"),fn=n("."),mn=r(),$e=l("li"),nl=l("code"),vn=n('"translation"'),_n=n(": will use the "),$t=l("a"),gn=n("TranslationEvaluator"),En=n("."),gs=r(),R=l("p"),bn=n("To run an "),il=l("code"),wn=n("Evaluator"),kn=n(" with several tasks in a single call, use the "),qt=l("a"),yn=n("EvaluationSuite"),$n=n(", which runs evaluation on a collection of "),rl=l("code"),qn=n("SubTask"),jn=n("s."),Es=r(),jt=l("p"),Tn=n("Each task has its own set of requirements for the dataset format and pipeline output, make sure to check them out for your custom use case. Let\u2019s have a look at some of them and see how you can use the evaluator to evalute a single or multiple of models, datasets, and metrics at the same time."),bs=r(),re=l("h2"),qe=l("a"),cl=l("span"),m(Je.$$.fragment),xn=r(),pl=l("span"),Dn=n("Text classification"),ws=r(),Tt=l("p"),Cn=n("The text classification evaluator can be used to evaluate text models on classification datasets such as IMDb. Beside the model, data, and metric inputs it takes the following optional inputs:"),ks=r(),ae=l("ul"),xt=l("li"),dl=l("code"),Pn=n('input_column="text"'),An=n(": with this argument the column with the data for the pipeline can be specified."),In=r(),Dt=l("li"),hl=l("code"),On=n('label_column="label"'),Sn=n(": with this argument the column with the labels for the evaluation can be specified."),Nn=r(),$=l("li"),ul=l("code"),Ln=n("label_mapping=None"),Hn=n(": the label mapping aligns the labels in the pipeline output with the labels need for evaluation. E.g. the labels in "),fl=l("code"),Bn=n("label_column"),zn=n(" can be integers ("),ml=l("code"),Gn=n("0"),Wn=n("/"),vl=l("code"),Rn=n("1"),Un=n(") whereas the pipeline can produce label names such as "),_l=l("code"),Vn=n('"positive"'),Mn=n("/"),gl=l("code"),Fn=n('"negative"'),Qn=n(". With that dictionary the pipeline outputs are mapped to the labels."),ys=r(),je=l("p"),Jn=n("By default the "),El=l("code"),Yn=n('"accuracy"'),Kn=n(" metric is computed."),$s=r(),ce=l("h3"),Te=l("a"),bl=l("span"),m(Ye.$$.fragment),Xn=r(),wl=l("span"),Zn=n("Evaluate models on the Hub"),qs=r(),U=l("p"),ei=n("There are several ways to pass a model to the evaluator: you can pass the name of a model on the Hub, you can load a "),kl=l("code"),ti=n("transformers"),ai=n(" model and pass it to the evaluator or you can pass an initialized "),yl=l("code"),li=n("transformers.Pipeline"),si=n(". Alternatively you can pass any callable function that behaves like a "),$l=l("code"),oi=n("pipeline"),ni=n(" call for the task in any framework."),js=r(),Ct=l("p"),ii=n("So any of the following works:"),Ts=r(),m(Ke.$$.fragment),xs=r(),m(xe.$$.fragment),Ds=r(),Pt=l("p"),ri=n("The results will look as follows:"),Cs=r(),m(Xe.$$.fragment),Ps=r(),At=l("p"),ci=n("Note that evaluation results include both the requested metric, and information about the time it took to obtain predictions through the pipeline."),As=r(),m(De.$$.fragment),Is=r(),pe=l("h3"),Ce=l("a"),ql=l("span"),m(Ze.$$.fragment),pi=r(),jl=l("span"),di=n("Evaluate multiple metrics"),Os=r(),Pe=l("p"),hi=n("With the "),It=l("a"),ui=n("combine()"),fi=n(" function one can bundle several metrics into an object that behaves like a single metric. We can use this to evaluate several metrics at once with the evaluator:"),Ss=r(),m(et.$$.fragment),Ns=r(),Ot=l("p"),mi=n("The results will look as follows:"),Ls=r(),m(tt.$$.fragment),Hs=r(),St=l("p"),vi=n("Next let\u2019s have a look at token classification."),Bs=r(),de=l("h2"),Ae=l("a"),Tl=l("span"),m(at.$$.fragment),_i=r(),xl=l("span"),gi=n("Token Classification"),zs=r(),Nt=l("p"),Ei=n("With the token classification evaluator one can evaluate models for tasks such as NER or POS tagging. It has the following specific arguments:"),Gs=r(),V=l("ul"),Lt=l("li"),Dl=l("code"),bi=n('input_column="text"'),wi=n(": with this argument the column with the data for the pipeline can be specified."),ki=r(),Ht=l("li"),Cl=l("code"),yi=n('label_column="label"'),$i=n(": with this argument the column with the labels for the evaluation can be specified."),qi=r(),q=l("li"),Pl=l("code"),ji=n("label_mapping=None"),Ti=n(": the label mapping aligns the labels in the pipeline output with the labels need for evaluation. E.g. the labels in "),Al=l("code"),xi=n("label_column"),Di=n(" can be integers ("),Il=l("code"),Ci=n("0"),Pi=n("/"),Ol=l("code"),Ai=n("1"),Ii=n(") whereas the pipeline can produce label names such as "),Sl=l("code"),Oi=n('"positive"'),Si=n("/"),Nl=l("code"),Ni=n('"negative"'),Li=n(". With that dictionary the pipeline outputs are mapped to the labels."),Hi=r(),Bt=l("li"),Ll=l("code"),Bi=n('join_by=" "'),zi=n(": While most datasets are already tokenized the pipeline expects a string. Thus the tokens need to be joined before passing to the pipeline. By default they are joined with a whitespace."),Ws=r(),zt=l("p"),Gi=n("Let\u2019s have a look how we can use the evaluator to benchmark several models."),Rs=r(),he=l("h3"),Ie=l("a"),Hl=l("span"),m(lt.$$.fragment),Wi=r(),Bl=l("span"),Ri=n("Benchmarking several models"),Us=r(),Oe=l("p"),Ui=n("Here is an example where several models can be compared thanks to the "),zl=l("code"),Vi=n("evaluator"),Mi=n(" in only a few lines of code, abstracting away the preprocessing, inference, postprocessing, metric computation:"),Vs=r(),m(st.$$.fragment),Ms=r(),Gt=l("p"),Fi=n("The result is a table that looks like this:"),Fs=r(),Se=l("table"),Gl=l("thead"),j=l("tr"),Wt=l("th"),Qi=n("model"),Ji=r(),Rt=l("th"),Yi=n("overall_f1"),Ki=r(),Ut=l("th"),Xi=n("overall_accuracy"),Zi=r(),Vt=l("th"),er=n("total_time_in_seconds"),tr=r(),Mt=l("th"),ar=n("samples_per_second"),lr=r(),Ft=l("th"),sr=n("latency_in_seconds"),or=r(),y=l("tbody"),T=l("tr"),Qt=l("td"),nr=n("Jorgeutd/albert-base-v2-finetuned-ner"),ir=r(),Jt=l("td"),rr=n("0.941"),cr=r(),Yt=l("td"),pr=n("0.989"),dr=r(),Kt=l("td"),hr=n("4.515"),ur=r(),Xt=l("td"),fr=n("221.468"),mr=r(),Zt=l("td"),vr=n("0.005"),_r=r(),x=l("tr"),ea=l("td"),gr=n("dbmdz/bert-large-cased-finetuned-conll03-english"),Er=r(),ta=l("td"),br=n("0.962"),wr=r(),aa=l("td"),kr=n("0.881"),yr=r(),la=l("td"),$r=n("11.648"),qr=r(),sa=l("td"),jr=n("85.850"),Tr=r(),oa=l("td"),xr=n("0.012"),Dr=r(),D=l("tr"),na=l("td"),Cr=n("dbmdz/electra-large-discriminator-finetuned-conll03-english"),Pr=r(),ia=l("td"),Ar=n("0.965"),Ir=r(),ra=l("td"),Or=n("0.881"),Sr=r(),ca=l("td"),Nr=n("11.456"),Lr=r(),pa=l("td"),Hr=n("87.292"),Br=r(),da=l("td"),zr=n("0.011"),Gr=r(),C=l("tr"),ha=l("td"),Wr=n("elastic/distilbert-base-uncased-finetuned-conll03-english"),Rr=r(),ua=l("td"),Ur=n("0.940"),Vr=r(),fa=l("td"),Mr=n("0.989"),Fr=r(),ma=l("td"),Qr=n("2.318"),Jr=r(),va=l("td"),Yr=n("431.378"),Kr=r(),_a=l("td"),Xr=n("0.002"),Zr=r(),P=l("tr"),ga=l("td"),ec=n("gunghio/distilbert-base-multilingual-cased-finetuned-conll2003-ner"),tc=r(),Ea=l("td"),ac=n("0.947"),lc=r(),ba=l("td"),sc=n("0.991"),oc=r(),wa=l("td"),nc=n("2.376"),ic=r(),ka=l("td"),rc=n("420.873"),cc=r(),ya=l("td"),pc=n("0.002"),dc=r(),A=l("tr"),$a=l("td"),hc=n("philschmid/distilroberta-base-ner-conll2003"),uc=r(),qa=l("td"),fc=n("0.961"),mc=r(),ja=l("td"),vc=n("0.994"),_c=r(),Ta=l("td"),gc=n("2.436"),Ec=r(),xa=l("td"),bc=n("410.579"),wc=r(),Da=l("td"),kc=n("0.002"),yc=r(),I=l("tr"),Ca=l("td"),$c=n("xlm-roberta-large-finetuned-conll03-english"),qc=r(),Pa=l("td"),jc=n("0.969"),Tc=r(),Aa=l("td"),xc=n("0.882"),Dc=r(),Ia=l("td"),Cc=n("11.996"),Pc=r(),Oa=l("td"),Ac=n("83.359"),Ic=r(),Sa=l("td"),Oc=n("0.012"),Qs=r(),ue=l("h2"),Ne=l("a"),Wl=l("span"),m(ot.$$.fragment),Sc=r(),Rl=l("span"),Nc=n("Question Answering"),Js=r(),Na=l("p"),Lc=n("With the question-answering evaluator one can evaluate models for QA without needing to worry about the complicated pre- and post-processing that\u2019s required for these models. It has the following specific arguments:"),Ys=r(),S=l("ul"),La=l("li"),Ul=l("code"),Hc=n('question_column="question"'),Bc=n(": the name of the column containing the question in the dataset"),zc=r(),Ha=l("li"),Vl=l("code"),Gc=n('context_column="context"'),Wc=n(": the name of the column containing the context"),Rc=r(),Ba=l("li"),Ml=l("code"),Uc=n('id_column="id"'),Vc=n(": the name of the column cointaing the identification field of the question and answer pair"),Mc=r(),za=l("li"),Fl=l("code"),Fc=n('label_column="answers"'),Qc=n(": the name of the column containing the answers"),Jc=r(),Ga=l("li"),Ql=l("code"),Yc=n("squad_v2_format=None"),Kc=n(": whether the dataset follows the format of squad_v2 dataset where a question may have no answer in the context. If this parameter is not provided, the format will be automatically inferred."),Ks=r(),Wa=l("p"),Xc=n("Let\u2019s have a look how we can evaluate QA models and compute confidence intervals at the same time."),Xs=r(),fe=l("h3"),Le=l("a"),Jl=l("span"),m(nt.$$.fragment),Zc=r(),Yl=l("span"),ep=n("Confidence intervals"),Zs=r(),M=l("p"),tp=n("Every evaluator comes with the options to compute confidence intervals using "),it=l("a"),ap=n("bootstrapping"),lp=n(". Simply pass "),Kl=l("code"),sp=n('strategy="bootstrap"'),op=n(" and set the number of resanmples with "),Xl=l("code"),np=n("n_resamples"),ip=n("."),eo=r(),m(rt.$$.fragment),to=r(),Ra=l("p"),rp=n("Results include confidence intervals as well as error estimates as follows:"),ao=r(),m(ct.$$.fragment),lo=r(),me=l("h2"),He=l("a"),Zl=l("span"),m(pt.$$.fragment),cp=r(),es=l("span"),pp=n("Image classification"),so=r(),Ua=l("p"),dp=n("With the image classification evaluator we can evaluate any image classifier. It uses the same keyword arguments at the text classifier:"),oo=r(),le=l("ul"),Va=l("li"),ts=l("code"),hp=n('input_column="image"'),up=n(": the name of the column containing the images as PIL ImageFile"),fp=r(),Ma=l("li"),as=l("code"),mp=n('label_column="label"'),vp=n(": the name of the column containing the labels"),_p=r(),dt=l("li"),ls=l("code"),gp=n("label_mapping=None"),Ep=n(": We want to map class labels defined by the model in the pipeline to values consistent with those defined in the "),ss=l("code"),bp=n("label_column"),no=r(),Fa=l("p"),wp=n("Let\u2019s have a look at how can evaluate image classification models on large datasets."),io=r(),ve=l("h3"),Be=l("a"),os=l("span"),m(ht.$$.fragment),kp=r(),ns=l("span"),yp=n("Handling large datasets"),ro=r(),Qa=l("p"),$p=n("The evaluator can be used on large datasets! Below, an example shows how to use it on ImageNet-1k for image classification. Beware that this example will require to download ~150 GB."),co=r(),m(ut.$$.fragment),po=r(),ze=l("p"),qp=n("Since we are using "),is=l("code"),jp=n("datasets"),Tp=n(" to store data we make use of a technique called memory mappings. This means that the dataset is never fully loaded into memory which saves a lot of RAM. Running the above code only uses roughly 1.5 GB of RAM while the validation split is more than 30 GB big."),this.h()},l(t){const p=Ou('[data-svelte="svelte-1phssyn"]',document.head);u=s(p,"META",{name:!0,content:!0}),p.forEach(a),G=c(t),f=s(t,"H1",{class:!0});var ft=o(f);k=s(ft,"A",{id:!0,class:!0,href:!0});var rs=o(k);ne=s(rs,"SPAN",{});var Gp=o(ne);v(H.$$.fragment,Gp),Gp.forEach(a),rs.forEach(a),Fe=c(ft),te=s(ft,"SPAN",{});var xp=o(te);ie=i(xp,"Using the "),O=s(xp,"CODE",{});var Wp=o(O);vt=i(Wp,"evaluator"),Wp.forEach(a),xp.forEach(a),ft.forEach(a),Qe=c(t),W=s(t,"P",{});var Ge=o(W);Do=i(Ge,"The "),Ka=s(Ge,"CODE",{});var Rp=o(Ka);Co=i(Rp,"Evaluator"),Rp.forEach(a),Po=i(Ge," classes allow to evaluate a  triplet of model, dataset, and metric. The models wrapped in a pipeline, responsible for handling all preprocessing and post-processing and out-of-the-box, "),Xa=s(Ge,"CODE",{});var Up=o(Xa);Ao=i(Up,"Evaluator"),Up.forEach(a),Io=i(Ge,"s support transformers pipelines for the supported tasks, but custom pipelines can be passed, as showcased in the section "),_e=s(Ge,"A",{href:!0});var uo=o(_e);Oo=i(uo,"Using the "),Za=s(uo,"CODE",{});var Vp=o(Za);So=i(Vp,"evaluator"),Vp.forEach(a),No=i(uo," with custom pipelines"),uo.forEach(a),Lo=i(Ge,"."),Ge.forEach(a),vs=c(t),_t=s(t,"P",{});var Mp=o(_t);Ho=i(Mp,"Currently supported tasks are:"),Mp.forEach(a),_s=c(t),w=s(t,"UL",{});var N=o(w);ge=s(N,"LI",{});var cs=o(ge);el=s(cs,"CODE",{});var Fp=o(el);Bo=i(Fp,'"text-classification"'),Fp.forEach(a),zo=i(cs,": will use the "),gt=s(cs,"A",{href:!0});var Qp=o(gt);Go=i(Qp,"TextClassificationEvaluator"),Qp.forEach(a),Wo=i(cs,"."),cs.forEach(a),Ro=c(N),Ee=s(N,"LI",{});var ps=o(Ee);tl=s(ps,"CODE",{});var Jp=o(tl);Uo=i(Jp,'"token-classification"'),Jp.forEach(a),Vo=i(ps,": will use the "),Et=s(ps,"A",{href:!0});var Yp=o(Et);Mo=i(Yp,"TokenClassificationEvaluator"),Yp.forEach(a),Fo=i(ps,"."),ps.forEach(a),Qo=c(N),be=s(N,"LI",{});var ds=o(be);al=s(ds,"CODE",{});var Kp=o(al);Jo=i(Kp,'"question-answering"'),Kp.forEach(a),Yo=i(ds,": will use the "),bt=s(ds,"A",{href:!0});var Xp=o(bt);Ko=i(Xp,"QuestionAnsweringEvaluator"),Xp.forEach(a),Xo=i(ds,"."),ds.forEach(a),Zo=c(N),we=s(N,"LI",{});var hs=o(we);ll=s(hs,"CODE",{});var Zp=o(ll);en=i(Zp,'"image-classification"'),Zp.forEach(a),tn=i(hs,": will use the "),wt=s(hs,"A",{href:!0});var ed=o(wt);an=i(ed,"ImageClassificationEvaluator"),ed.forEach(a),ln=i(hs,"."),hs.forEach(a),sn=c(N),ke=s(N,"LI",{});var us=o(ke);sl=s(us,"CODE",{});var td=o(sl);on=i(td,'"text2text-generation"'),td.forEach(a),nn=i(us,": will use the "),kt=s(us,"A",{href:!0});var ad=o(kt);rn=i(ad,"Text2TextGenerationEvaluator"),ad.forEach(a),cn=i(us,"."),us.forEach(a),pn=c(N),ye=s(N,"LI",{});var fs=o(ye);ol=s(fs,"CODE",{});var ld=o(ol);dn=i(ld,'"summarization"'),ld.forEach(a),hn=i(fs,": will use the "),yt=s(fs,"A",{href:!0});var sd=o(yt);un=i(sd,"SummarizationEvaluator"),sd.forEach(a),fn=i(fs,"."),fs.forEach(a),mn=c(N),$e=s(N,"LI",{});var ms=o($e);nl=s(ms,"CODE",{});var od=o(nl);vn=i(od,'"translation"'),od.forEach(a),_n=i(ms,": will use the "),$t=s(ms,"A",{href:!0});var nd=o($t);gn=i(nd,"TranslationEvaluator"),nd.forEach(a),En=i(ms,"."),ms.forEach(a),N.forEach(a),gs=c(t),R=s(t,"P",{});var We=o(R);bn=i(We,"To run an "),il=s(We,"CODE",{});var id=o(il);wn=i(id,"Evaluator"),id.forEach(a),kn=i(We," with several tasks in a single call, use the "),qt=s(We,"A",{href:!0});var rd=o(qt);yn=i(rd,"EvaluationSuite"),rd.forEach(a),$n=i(We,", which runs evaluation on a collection of "),rl=s(We,"CODE",{});var cd=o(rl);qn=i(cd,"SubTask"),cd.forEach(a),jn=i(We,"s."),We.forEach(a),Es=c(t),jt=s(t,"P",{});var pd=o(jt);Tn=i(pd,"Each task has its own set of requirements for the dataset format and pipeline output, make sure to check them out for your custom use case. Let\u2019s have a look at some of them and see how you can use the evaluator to evalute a single or multiple of models, datasets, and metrics at the same time."),pd.forEach(a),bs=c(t),re=s(t,"H2",{class:!0});var fo=o(re);qe=s(fo,"A",{id:!0,class:!0,href:!0});var dd=o(qe);cl=s(dd,"SPAN",{});var hd=o(cl);v(Je.$$.fragment,hd),hd.forEach(a),dd.forEach(a),xn=c(fo),pl=s(fo,"SPAN",{});var ud=o(pl);Dn=i(ud,"Text classification"),ud.forEach(a),fo.forEach(a),ws=c(t),Tt=s(t,"P",{});var fd=o(Tt);Cn=i(fd,"The text classification evaluator can be used to evaluate text models on classification datasets such as IMDb. Beside the model, data, and metric inputs it takes the following optional inputs:"),fd.forEach(a),ks=c(t),ae=s(t,"UL",{});var Ja=o(ae);xt=s(Ja,"LI",{});var Dp=o(xt);dl=s(Dp,"CODE",{});var md=o(dl);Pn=i(md,'input_column="text"'),md.forEach(a),An=i(Dp,": with this argument the column with the data for the pipeline can be specified."),Dp.forEach(a),In=c(Ja),Dt=s(Ja,"LI",{});var Cp=o(Dt);hl=s(Cp,"CODE",{});var vd=o(hl);On=i(vd,'label_column="label"'),vd.forEach(a),Sn=i(Cp,": with this argument the column with the labels for the evaluation can be specified."),Cp.forEach(a),Nn=c(Ja),$=s(Ja,"LI",{});var B=o($);ul=s(B,"CODE",{});var _d=o(ul);Ln=i(_d,"label_mapping=None"),_d.forEach(a),Hn=i(B,": the label mapping aligns the labels in the pipeline output with the labels need for evaluation. E.g. the labels in "),fl=s(B,"CODE",{});var gd=o(fl);Bn=i(gd,"label_column"),gd.forEach(a),zn=i(B," can be integers ("),ml=s(B,"CODE",{});var Ed=o(ml);Gn=i(Ed,"0"),Ed.forEach(a),Wn=i(B,"/"),vl=s(B,"CODE",{});var bd=o(vl);Rn=i(bd,"1"),bd.forEach(a),Un=i(B,") whereas the pipeline can produce label names such as "),_l=s(B,"CODE",{});var wd=o(_l);Vn=i(wd,'"positive"'),wd.forEach(a),Mn=i(B,"/"),gl=s(B,"CODE",{});var kd=o(gl);Fn=i(kd,'"negative"'),kd.forEach(a),Qn=i(B,". With that dictionary the pipeline outputs are mapped to the labels."),B.forEach(a),Ja.forEach(a),ys=c(t),je=s(t,"P",{});var mo=o(je);Jn=i(mo,"By default the "),El=s(mo,"CODE",{});var yd=o(El);Yn=i(yd,'"accuracy"'),yd.forEach(a),Kn=i(mo," metric is computed."),mo.forEach(a),$s=c(t),ce=s(t,"H3",{class:!0});var vo=o(ce);Te=s(vo,"A",{id:!0,class:!0,href:!0});var $d=o(Te);bl=s($d,"SPAN",{});var qd=o(bl);v(Ye.$$.fragment,qd),qd.forEach(a),$d.forEach(a),Xn=c(vo),wl=s(vo,"SPAN",{});var jd=o(wl);Zn=i(jd,"Evaluate models on the Hub"),jd.forEach(a),vo.forEach(a),qs=c(t),U=s(t,"P",{});var Re=o(U);ei=i(Re,"There are several ways to pass a model to the evaluator: you can pass the name of a model on the Hub, you can load a "),kl=s(Re,"CODE",{});var Td=o(kl);ti=i(Td,"transformers"),Td.forEach(a),ai=i(Re," model and pass it to the evaluator or you can pass an initialized "),yl=s(Re,"CODE",{});var xd=o(yl);li=i(xd,"transformers.Pipeline"),xd.forEach(a),si=i(Re,". Alternatively you can pass any callable function that behaves like a "),$l=s(Re,"CODE",{});var Dd=o($l);oi=i(Dd,"pipeline"),Dd.forEach(a),ni=i(Re," call for the task in any framework."),Re.forEach(a),js=c(t),Ct=s(t,"P",{});var Cd=o(Ct);ii=i(Cd,"So any of the following works:"),Cd.forEach(a),Ts=c(t),v(Ke.$$.fragment,t),xs=c(t),v(xe.$$.fragment,t),Ds=c(t),Pt=s(t,"P",{});var Pd=o(Pt);ri=i(Pd,"The results will look as follows:"),Pd.forEach(a),Cs=c(t),v(Xe.$$.fragment,t),Ps=c(t),At=s(t,"P",{});var Ad=o(At);ci=i(Ad,"Note that evaluation results include both the requested metric, and information about the time it took to obtain predictions through the pipeline."),Ad.forEach(a),As=c(t),v(De.$$.fragment,t),Is=c(t),pe=s(t,"H3",{class:!0});var _o=o(pe);Ce=s(_o,"A",{id:!0,class:!0,href:!0});var Id=o(Ce);ql=s(Id,"SPAN",{});var Od=o(ql);v(Ze.$$.fragment,Od),Od.forEach(a),Id.forEach(a),pi=c(_o),jl=s(_o,"SPAN",{});var Sd=o(jl);di=i(Sd,"Evaluate multiple metrics"),Sd.forEach(a),_o.forEach(a),Os=c(t),Pe=s(t,"P",{});var go=o(Pe);hi=i(go,"With the "),It=s(go,"A",{href:!0});var Nd=o(It);ui=i(Nd,"combine()"),Nd.forEach(a),fi=i(go," function one can bundle several metrics into an object that behaves like a single metric. We can use this to evaluate several metrics at once with the evaluator:"),go.forEach(a),Ss=c(t),v(et.$$.fragment,t),Ns=c(t),Ot=s(t,"P",{});var Ld=o(Ot);mi=i(Ld,"The results will look as follows:"),Ld.forEach(a),Ls=c(t),v(tt.$$.fragment,t),Hs=c(t),St=s(t,"P",{});var Hd=o(St);vi=i(Hd,"Next let\u2019s have a look at token classification."),Hd.forEach(a),Bs=c(t),de=s(t,"H2",{class:!0});var Eo=o(de);Ae=s(Eo,"A",{id:!0,class:!0,href:!0});var Bd=o(Ae);Tl=s(Bd,"SPAN",{});var zd=o(Tl);v(at.$$.fragment,zd),zd.forEach(a),Bd.forEach(a),_i=c(Eo),xl=s(Eo,"SPAN",{});var Gd=o(xl);gi=i(Gd,"Token Classification"),Gd.forEach(a),Eo.forEach(a),zs=c(t),Nt=s(t,"P",{});var Wd=o(Nt);Ei=i(Wd,"With the token classification evaluator one can evaluate models for tasks such as NER or POS tagging. It has the following specific arguments:"),Wd.forEach(a),Gs=c(t),V=s(t,"UL",{});var Ue=o(V);Lt=s(Ue,"LI",{});var Pp=o(Lt);Dl=s(Pp,"CODE",{});var Rd=o(Dl);bi=i(Rd,'input_column="text"'),Rd.forEach(a),wi=i(Pp,": with this argument the column with the data for the pipeline can be specified."),Pp.forEach(a),ki=c(Ue),Ht=s(Ue,"LI",{});var Ap=o(Ht);Cl=s(Ap,"CODE",{});var Ud=o(Cl);yi=i(Ud,'label_column="label"'),Ud.forEach(a),$i=i(Ap,": with this argument the column with the labels for the evaluation can be specified."),Ap.forEach(a),qi=c(Ue),q=s(Ue,"LI",{});var z=o(q);Pl=s(z,"CODE",{});var Vd=o(Pl);ji=i(Vd,"label_mapping=None"),Vd.forEach(a),Ti=i(z,": the label mapping aligns the labels in the pipeline output with the labels need for evaluation. E.g. the labels in "),Al=s(z,"CODE",{});var Md=o(Al);xi=i(Md,"label_column"),Md.forEach(a),Di=i(z," can be integers ("),Il=s(z,"CODE",{});var Fd=o(Il);Ci=i(Fd,"0"),Fd.forEach(a),Pi=i(z,"/"),Ol=s(z,"CODE",{});var Qd=o(Ol);Ai=i(Qd,"1"),Qd.forEach(a),Ii=i(z,") whereas the pipeline can produce label names such as "),Sl=s(z,"CODE",{});var Jd=o(Sl);Oi=i(Jd,'"positive"'),Jd.forEach(a),Si=i(z,"/"),Nl=s(z,"CODE",{});var Yd=o(Nl);Ni=i(Yd,'"negative"'),Yd.forEach(a),Li=i(z,". With that dictionary the pipeline outputs are mapped to the labels."),z.forEach(a),Hi=c(Ue),Bt=s(Ue,"LI",{});var Ip=o(Bt);Ll=s(Ip,"CODE",{});var Kd=o(Ll);Bi=i(Kd,'join_by=" "'),Kd.forEach(a),zi=i(Ip,": While most datasets are already tokenized the pipeline expects a string. Thus the tokens need to be joined before passing to the pipeline. By default they are joined with a whitespace."),Ip.forEach(a),Ue.forEach(a),Ws=c(t),zt=s(t,"P",{});var Xd=o(zt);Gi=i(Xd,"Let\u2019s have a look how we can use the evaluator to benchmark several models."),Xd.forEach(a),Rs=c(t),he=s(t,"H3",{class:!0});var bo=o(he);Ie=s(bo,"A",{id:!0,class:!0,href:!0});var Zd=o(Ie);Hl=s(Zd,"SPAN",{});var eh=o(Hl);v(lt.$$.fragment,eh),eh.forEach(a),Zd.forEach(a),Wi=c(bo),Bl=s(bo,"SPAN",{});var th=o(Bl);Ri=i(th,"Benchmarking several models"),th.forEach(a),bo.forEach(a),Us=c(t),Oe=s(t,"P",{});var wo=o(Oe);Ui=i(wo,"Here is an example where several models can be compared thanks to the "),zl=s(wo,"CODE",{});var ah=o(zl);Vi=i(ah,"evaluator"),ah.forEach(a),Mi=i(wo," in only a few lines of code, abstracting away the preprocessing, inference, postprocessing, metric computation:"),wo.forEach(a),Vs=c(t),v(st.$$.fragment,t),Ms=c(t),Gt=s(t,"P",{});var lh=o(Gt);Fi=i(lh,"The result is a table that looks like this:"),lh.forEach(a),Fs=c(t),Se=s(t,"TABLE",{});var ko=o(Se);Gl=s(ko,"THEAD",{});var sh=o(Gl);j=s(sh,"TR",{});var F=o(j);Wt=s(F,"TH",{align:!0});var oh=o(Wt);Qi=i(oh,"model"),oh.forEach(a),Ji=c(F),Rt=s(F,"TH",{align:!0});var nh=o(Rt);Yi=i(nh,"overall_f1"),nh.forEach(a),Ki=c(F),Ut=s(F,"TH",{align:!0});var ih=o(Ut);Xi=i(ih,"overall_accuracy"),ih.forEach(a),Zi=c(F),Vt=s(F,"TH",{align:!0});var rh=o(Vt);er=i(rh,"total_time_in_seconds"),rh.forEach(a),tr=c(F),Mt=s(F,"TH",{align:!0});var ch=o(Mt);ar=i(ch,"samples_per_second"),ch.forEach(a),lr=c(F),Ft=s(F,"TH",{align:!0});var ph=o(Ft);sr=i(ph,"latency_in_seconds"),ph.forEach(a),F.forEach(a),sh.forEach(a),or=c(ko),y=s(ko,"TBODY",{});var L=o(y);T=s(L,"TR",{});var Q=o(T);Qt=s(Q,"TD",{align:!0});var dh=o(Qt);nr=i(dh,"Jorgeutd/albert-base-v2-finetuned-ner"),dh.forEach(a),ir=c(Q),Jt=s(Q,"TD",{align:!0});var hh=o(Jt);rr=i(hh,"0.941"),hh.forEach(a),cr=c(Q),Yt=s(Q,"TD",{align:!0});var uh=o(Yt);pr=i(uh,"0.989"),uh.forEach(a),dr=c(Q),Kt=s(Q,"TD",{align:!0});var fh=o(Kt);hr=i(fh,"4.515"),fh.forEach(a),ur=c(Q),Xt=s(Q,"TD",{align:!0});var mh=o(Xt);fr=i(mh,"221.468"),mh.forEach(a),mr=c(Q),Zt=s(Q,"TD",{align:!0});var vh=o(Zt);vr=i(vh,"0.005"),vh.forEach(a),Q.forEach(a),_r=c(L),x=s(L,"TR",{});var J=o(x);ea=s(J,"TD",{align:!0});var _h=o(ea);gr=i(_h,"dbmdz/bert-large-cased-finetuned-conll03-english"),_h.forEach(a),Er=c(J),ta=s(J,"TD",{align:!0});var gh=o(ta);br=i(gh,"0.962"),gh.forEach(a),wr=c(J),aa=s(J,"TD",{align:!0});var Eh=o(aa);kr=i(Eh,"0.881"),Eh.forEach(a),yr=c(J),la=s(J,"TD",{align:!0});var bh=o(la);$r=i(bh,"11.648"),bh.forEach(a),qr=c(J),sa=s(J,"TD",{align:!0});var wh=o(sa);jr=i(wh,"85.850"),wh.forEach(a),Tr=c(J),oa=s(J,"TD",{align:!0});var kh=o(oa);xr=i(kh,"0.012"),kh.forEach(a),J.forEach(a),Dr=c(L),D=s(L,"TR",{});var Y=o(D);na=s(Y,"TD",{align:!0});var yh=o(na);Cr=i(yh,"dbmdz/electra-large-discriminator-finetuned-conll03-english"),yh.forEach(a),Pr=c(Y),ia=s(Y,"TD",{align:!0});var $h=o(ia);Ar=i($h,"0.965"),$h.forEach(a),Ir=c(Y),ra=s(Y,"TD",{align:!0});var qh=o(ra);Or=i(qh,"0.881"),qh.forEach(a),Sr=c(Y),ca=s(Y,"TD",{align:!0});var jh=o(ca);Nr=i(jh,"11.456"),jh.forEach(a),Lr=c(Y),pa=s(Y,"TD",{align:!0});var Th=o(pa);Hr=i(Th,"87.292"),Th.forEach(a),Br=c(Y),da=s(Y,"TD",{align:!0});var xh=o(da);zr=i(xh,"0.011"),xh.forEach(a),Y.forEach(a),Gr=c(L),C=s(L,"TR",{});var K=o(C);ha=s(K,"TD",{align:!0});var Dh=o(ha);Wr=i(Dh,"elastic/distilbert-base-uncased-finetuned-conll03-english"),Dh.forEach(a),Rr=c(K),ua=s(K,"TD",{align:!0});var Ch=o(ua);Ur=i(Ch,"0.940"),Ch.forEach(a),Vr=c(K),fa=s(K,"TD",{align:!0});var Ph=o(fa);Mr=i(Ph,"0.989"),Ph.forEach(a),Fr=c(K),ma=s(K,"TD",{align:!0});var Ah=o(ma);Qr=i(Ah,"2.318"),Ah.forEach(a),Jr=c(K),va=s(K,"TD",{align:!0});var Ih=o(va);Yr=i(Ih,"431.378"),Ih.forEach(a),Kr=c(K),_a=s(K,"TD",{align:!0});var Oh=o(_a);Xr=i(Oh,"0.002"),Oh.forEach(a),K.forEach(a),Zr=c(L),P=s(L,"TR",{});var X=o(P);ga=s(X,"TD",{align:!0});var Sh=o(ga);ec=i(Sh,"gunghio/distilbert-base-multilingual-cased-finetuned-conll2003-ner"),Sh.forEach(a),tc=c(X),Ea=s(X,"TD",{align:!0});var Nh=o(Ea);ac=i(Nh,"0.947"),Nh.forEach(a),lc=c(X),ba=s(X,"TD",{align:!0});var Lh=o(ba);sc=i(Lh,"0.991"),Lh.forEach(a),oc=c(X),wa=s(X,"TD",{align:!0});var Hh=o(wa);nc=i(Hh,"2.376"),Hh.forEach(a),ic=c(X),ka=s(X,"TD",{align:!0});var Bh=o(ka);rc=i(Bh,"420.873"),Bh.forEach(a),cc=c(X),ya=s(X,"TD",{align:!0});var zh=o(ya);pc=i(zh,"0.002"),zh.forEach(a),X.forEach(a),dc=c(L),A=s(L,"TR",{});var Z=o(A);$a=s(Z,"TD",{align:!0});var Gh=o($a);hc=i(Gh,"philschmid/distilroberta-base-ner-conll2003"),Gh.forEach(a),uc=c(Z),qa=s(Z,"TD",{align:!0});var Wh=o(qa);fc=i(Wh,"0.961"),Wh.forEach(a),mc=c(Z),ja=s(Z,"TD",{align:!0});var Rh=o(ja);vc=i(Rh,"0.994"),Rh.forEach(a),_c=c(Z),Ta=s(Z,"TD",{align:!0});var Uh=o(Ta);gc=i(Uh,"2.436"),Uh.forEach(a),Ec=c(Z),xa=s(Z,"TD",{align:!0});var Vh=o(xa);bc=i(Vh,"410.579"),Vh.forEach(a),wc=c(Z),Da=s(Z,"TD",{align:!0});var Mh=o(Da);kc=i(Mh,"0.002"),Mh.forEach(a),Z.forEach(a),yc=c(L),I=s(L,"TR",{});var ee=o(I);Ca=s(ee,"TD",{align:!0});var Fh=o(Ca);$c=i(Fh,"xlm-roberta-large-finetuned-conll03-english"),Fh.forEach(a),qc=c(ee),Pa=s(ee,"TD",{align:!0});var Qh=o(Pa);jc=i(Qh,"0.969"),Qh.forEach(a),Tc=c(ee),Aa=s(ee,"TD",{align:!0});var Jh=o(Aa);xc=i(Jh,"0.882"),Jh.forEach(a),Dc=c(ee),Ia=s(ee,"TD",{align:!0});var Yh=o(Ia);Cc=i(Yh,"11.996"),Yh.forEach(a),Pc=c(ee),Oa=s(ee,"TD",{align:!0});var Kh=o(Oa);Ac=i(Kh,"83.359"),Kh.forEach(a),Ic=c(ee),Sa=s(ee,"TD",{align:!0});var Xh=o(Sa);Oc=i(Xh,"0.012"),Xh.forEach(a),ee.forEach(a),L.forEach(a),ko.forEach(a),Qs=c(t),ue=s(t,"H2",{class:!0});var yo=o(ue);Ne=s(yo,"A",{id:!0,class:!0,href:!0});var Zh=o(Ne);Wl=s(Zh,"SPAN",{});var eu=o(Wl);v(ot.$$.fragment,eu),eu.forEach(a),Zh.forEach(a),Sc=c(yo),Rl=s(yo,"SPAN",{});var tu=o(Rl);Nc=i(tu,"Question Answering"),tu.forEach(a),yo.forEach(a),Js=c(t),Na=s(t,"P",{});var au=o(Na);Lc=i(au,"With the question-answering evaluator one can evaluate models for QA without needing to worry about the complicated pre- and post-processing that\u2019s required for these models. It has the following specific arguments:"),au.forEach(a),Ys=c(t),S=s(t,"UL",{});var se=o(S);La=s(se,"LI",{});var Op=o(La);Ul=s(Op,"CODE",{});var lu=o(Ul);Hc=i(lu,'question_column="question"'),lu.forEach(a),Bc=i(Op,": the name of the column containing the question in the dataset"),Op.forEach(a),zc=c(se),Ha=s(se,"LI",{});var Sp=o(Ha);Vl=s(Sp,"CODE",{});var su=o(Vl);Gc=i(su,'context_column="context"'),su.forEach(a),Wc=i(Sp,": the name of the column containing the context"),Sp.forEach(a),Rc=c(se),Ba=s(se,"LI",{});var Np=o(Ba);Ml=s(Np,"CODE",{});var ou=o(Ml);Uc=i(ou,'id_column="id"'),ou.forEach(a),Vc=i(Np,": the name of the column cointaing the identification field of the question and answer pair"),Np.forEach(a),Mc=c(se),za=s(se,"LI",{});var Lp=o(za);Fl=s(Lp,"CODE",{});var nu=o(Fl);Fc=i(nu,'label_column="answers"'),nu.forEach(a),Qc=i(Lp,": the name of the column containing the answers"),Lp.forEach(a),Jc=c(se),Ga=s(se,"LI",{});var Hp=o(Ga);Ql=s(Hp,"CODE",{});var iu=o(Ql);Yc=i(iu,"squad_v2_format=None"),iu.forEach(a),Kc=i(Hp,": whether the dataset follows the format of squad_v2 dataset where a question may have no answer in the context. If this parameter is not provided, the format will be automatically inferred."),Hp.forEach(a),se.forEach(a),Ks=c(t),Wa=s(t,"P",{});var ru=o(Wa);Xc=i(ru,"Let\u2019s have a look how we can evaluate QA models and compute confidence intervals at the same time."),ru.forEach(a),Xs=c(t),fe=s(t,"H3",{class:!0});var $o=o(fe);Le=s($o,"A",{id:!0,class:!0,href:!0});var cu=o(Le);Jl=s(cu,"SPAN",{});var pu=o(Jl);v(nt.$$.fragment,pu),pu.forEach(a),cu.forEach(a),Zc=c($o),Yl=s($o,"SPAN",{});var du=o(Yl);ep=i(du,"Confidence intervals"),du.forEach(a),$o.forEach(a),Zs=c(t),M=s(t,"P",{});var Ve=o(M);tp=i(Ve,"Every evaluator comes with the options to compute confidence intervals using "),it=s(Ve,"A",{href:!0,rel:!0});var hu=o(it);ap=i(hu,"bootstrapping"),hu.forEach(a),lp=i(Ve,". Simply pass "),Kl=s(Ve,"CODE",{});var uu=o(Kl);sp=i(uu,'strategy="bootstrap"'),uu.forEach(a),op=i(Ve," and set the number of resanmples with "),Xl=s(Ve,"CODE",{});var fu=o(Xl);np=i(fu,"n_resamples"),fu.forEach(a),ip=i(Ve,"."),Ve.forEach(a),eo=c(t),v(rt.$$.fragment,t),to=c(t),Ra=s(t,"P",{});var mu=o(Ra);rp=i(mu,"Results include confidence intervals as well as error estimates as follows:"),mu.forEach(a),ao=c(t),v(ct.$$.fragment,t),lo=c(t),me=s(t,"H2",{class:!0});var qo=o(me);He=s(qo,"A",{id:!0,class:!0,href:!0});var vu=o(He);Zl=s(vu,"SPAN",{});var _u=o(Zl);v(pt.$$.fragment,_u),_u.forEach(a),vu.forEach(a),cp=c(qo),es=s(qo,"SPAN",{});var gu=o(es);pp=i(gu,"Image classification"),gu.forEach(a),qo.forEach(a),so=c(t),Ua=s(t,"P",{});var Eu=o(Ua);dp=i(Eu,"With the image classification evaluator we can evaluate any image classifier. It uses the same keyword arguments at the text classifier:"),Eu.forEach(a),oo=c(t),le=s(t,"UL",{});var Ya=o(le);Va=s(Ya,"LI",{});var Bp=o(Va);ts=s(Bp,"CODE",{});var bu=o(ts);hp=i(bu,'input_column="image"'),bu.forEach(a),up=i(Bp,": the name of the column containing the images as PIL ImageFile"),Bp.forEach(a),fp=c(Ya),Ma=s(Ya,"LI",{});var zp=o(Ma);as=s(zp,"CODE",{});var wu=o(as);mp=i(wu,'label_column="label"'),wu.forEach(a),vp=i(zp,": the name of the column containing the labels"),zp.forEach(a),_p=c(Ya),dt=s(Ya,"LI",{});var jo=o(dt);ls=s(jo,"CODE",{});var ku=o(ls);gp=i(ku,"label_mapping=None"),ku.forEach(a),Ep=i(jo,": We want to map class labels defined by the model in the pipeline to values consistent with those defined in the "),ss=s(jo,"CODE",{});var yu=o(ss);bp=i(yu,"label_column"),yu.forEach(a),jo.forEach(a),Ya.forEach(a),no=c(t),Fa=s(t,"P",{});var $u=o(Fa);wp=i($u,"Let\u2019s have a look at how can evaluate image classification models on large datasets."),$u.forEach(a),io=c(t),ve=s(t,"H3",{class:!0});var To=o(ve);Be=s(To,"A",{id:!0,class:!0,href:!0});var qu=o(Be);os=s(qu,"SPAN",{});var ju=o(os);v(ht.$$.fragment,ju),ju.forEach(a),qu.forEach(a),kp=c(To),ns=s(To,"SPAN",{});var Tu=o(ns);yp=i(Tu,"Handling large datasets"),Tu.forEach(a),To.forEach(a),ro=c(t),Qa=s(t,"P",{});var xu=o(Qa);$p=i(xu,"The evaluator can be used on large datasets! Below, an example shows how to use it on ImageNet-1k for image classification. Beware that this example will require to download ~150 GB."),xu.forEach(a),co=c(t),v(ut.$$.fragment,t),po=c(t),ze=s(t,"P",{});var xo=o(ze);qp=i(xo,"Since we are using "),is=s(xo,"CODE",{});var Du=o(is);jp=i(Du,"datasets"),Du.forEach(a),Tp=i(xo," to store data we make use of a technique called memory mappings. This means that the dataset is never fully loaded into memory which saves a lot of RAM. Running the above code only uses roughly 1.5 GB of RAM while the validation split is more than 30 GB big."),xo.forEach(a),this.h()},h(){d(u,"name","hf:doc:metadata"),d(u,"content",JSON.stringify(Bu)),d(k,"id","using-the-evaluator"),d(k,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),d(k,"href","#using-the-evaluator"),d(f,"class","relative group"),d(_e,"href","custom_evaluator"),d(gt,"href","/docs/evaluate/pr_340/en/package_reference/evaluator_classes#evaluate.TextClassificationEvaluator"),d(Et,"href","/docs/evaluate/pr_340/en/package_reference/evaluator_classes#evaluate.TokenClassificationEvaluator"),d(bt,"href","/docs/evaluate/pr_340/en/package_reference/evaluator_classes#evaluate.QuestionAnsweringEvaluator"),d(wt,"href","/docs/evaluate/pr_340/en/package_reference/evaluator_classes#evaluate.ImageClassificationEvaluator"),d(kt,"href","/docs/evaluate/pr_340/en/package_reference/evaluator_classes#evaluate.Text2TextGenerationEvaluator"),d(yt,"href","/docs/evaluate/pr_340/en/package_reference/evaluator_classes#evaluate.SummarizationEvaluator"),d($t,"href","/docs/evaluate/pr_340/en/package_reference/evaluator_classes#evaluate.TranslationEvaluator"),d(qt,"href","evaluation_suite"),d(qe,"id","text-classification"),d(qe,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),d(qe,"href","#text-classification"),d(re,"class","relative group"),d(Te,"id","evaluate-models-on-the-hub"),d(Te,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),d(Te,"href","#evaluate-models-on-the-hub"),d(ce,"class","relative group"),d(Ce,"id","evaluate-multiple-metrics"),d(Ce,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),d(Ce,"href","#evaluate-multiple-metrics"),d(pe,"class","relative group"),d(It,"href","/docs/evaluate/pr_340/en/package_reference/main_classes#evaluate.combine"),d(Ae,"id","token-classification"),d(Ae,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),d(Ae,"href","#token-classification"),d(de,"class","relative group"),d(Ie,"id","benchmarking-several-models"),d(Ie,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),d(Ie,"href","#benchmarking-several-models"),d(he,"class","relative group"),d(Wt,"align","left"),d(Rt,"align","right"),d(Ut,"align","right"),d(Vt,"align","right"),d(Mt,"align","right"),d(Ft,"align","right"),d(Qt,"align","left"),d(Jt,"align","right"),d(Yt,"align","right"),d(Kt,"align","right"),d(Xt,"align","right"),d(Zt,"align","right"),d(ea,"align","left"),d(ta,"align","right"),d(aa,"align","right"),d(la,"align","right"),d(sa,"align","right"),d(oa,"align","right"),d(na,"align","left"),d(ia,"align","right"),d(ra,"align","right"),d(ca,"align","right"),d(pa,"align","right"),d(da,"align","right"),d(ha,"align","left"),d(ua,"align","right"),d(fa,"align","right"),d(ma,"align","right"),d(va,"align","right"),d(_a,"align","right"),d(ga,"align","left"),d(Ea,"align","right"),d(ba,"align","right"),d(wa,"align","right"),d(ka,"align","right"),d(ya,"align","right"),d($a,"align","left"),d(qa,"align","right"),d(ja,"align","right"),d(Ta,"align","right"),d(xa,"align","right"),d(Da,"align","right"),d(Ca,"align","left"),d(Pa,"align","right"),d(Aa,"align","right"),d(Ia,"align","right"),d(Oa,"align","right"),d(Sa,"align","right"),d(Ne,"id","question-answering"),d(Ne,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),d(Ne,"href","#question-answering"),d(ue,"class","relative group"),d(Le,"id","confidence-intervals"),d(Le,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),d(Le,"href","#confidence-intervals"),d(fe,"class","relative group"),d(it,"href","https://docs.scipy.org/doc/scipy/reference/generated/scipy.stats.bootstrap.html"),d(it,"rel","nofollow"),d(He,"id","image-classification"),d(He,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),d(He,"href","#image-classification"),d(me,"class","relative group"),d(Be,"id","handling-large-datasets"),d(Be,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),d(Be,"href","#handling-large-datasets"),d(ve,"class","relative group")},m(t,p){e(document.head,u),h(t,G,p),h(t,f,p),e(f,k),e(k,ne),_(H,ne,null),e(f,Fe),e(f,te),e(te,ie),e(te,O),e(O,vt),h(t,Qe,p),h(t,W,p),e(W,Do),e(W,Ka),e(Ka,Co),e(W,Po),e(W,Xa),e(Xa,Ao),e(W,Io),e(W,_e),e(_e,Oo),e(_e,Za),e(Za,So),e(_e,No),e(W,Lo),h(t,vs,p),h(t,_t,p),e(_t,Ho),h(t,_s,p),h(t,w,p),e(w,ge),e(ge,el),e(el,Bo),e(ge,zo),e(ge,gt),e(gt,Go),e(ge,Wo),e(w,Ro),e(w,Ee),e(Ee,tl),e(tl,Uo),e(Ee,Vo),e(Ee,Et),e(Et,Mo),e(Ee,Fo),e(w,Qo),e(w,be),e(be,al),e(al,Jo),e(be,Yo),e(be,bt),e(bt,Ko),e(be,Xo),e(w,Zo),e(w,we),e(we,ll),e(ll,en),e(we,tn),e(we,wt),e(wt,an),e(we,ln),e(w,sn),e(w,ke),e(ke,sl),e(sl,on),e(ke,nn),e(ke,kt),e(kt,rn),e(ke,cn),e(w,pn),e(w,ye),e(ye,ol),e(ol,dn),e(ye,hn),e(ye,yt),e(yt,un),e(ye,fn),e(w,mn),e(w,$e),e($e,nl),e(nl,vn),e($e,_n),e($e,$t),e($t,gn),e($e,En),h(t,gs,p),h(t,R,p),e(R,bn),e(R,il),e(il,wn),e(R,kn),e(R,qt),e(qt,yn),e(R,$n),e(R,rl),e(rl,qn),e(R,jn),h(t,Es,p),h(t,jt,p),e(jt,Tn),h(t,bs,p),h(t,re,p),e(re,qe),e(qe,cl),_(Je,cl,null),e(re,xn),e(re,pl),e(pl,Dn),h(t,ws,p),h(t,Tt,p),e(Tt,Cn),h(t,ks,p),h(t,ae,p),e(ae,xt),e(xt,dl),e(dl,Pn),e(xt,An),e(ae,In),e(ae,Dt),e(Dt,hl),e(hl,On),e(Dt,Sn),e(ae,Nn),e(ae,$),e($,ul),e(ul,Ln),e($,Hn),e($,fl),e(fl,Bn),e($,zn),e($,ml),e(ml,Gn),e($,Wn),e($,vl),e(vl,Rn),e($,Un),e($,_l),e(_l,Vn),e($,Mn),e($,gl),e(gl,Fn),e($,Qn),h(t,ys,p),h(t,je,p),e(je,Jn),e(je,El),e(El,Yn),e(je,Kn),h(t,$s,p),h(t,ce,p),e(ce,Te),e(Te,bl),_(Ye,bl,null),e(ce,Xn),e(ce,wl),e(wl,Zn),h(t,qs,p),h(t,U,p),e(U,ei),e(U,kl),e(kl,ti),e(U,ai),e(U,yl),e(yl,li),e(U,si),e(U,$l),e($l,oi),e(U,ni),h(t,js,p),h(t,Ct,p),e(Ct,ii),h(t,Ts,p),_(Ke,t,p),h(t,xs,p),_(xe,t,p),h(t,Ds,p),h(t,Pt,p),e(Pt,ri),h(t,Cs,p),_(Xe,t,p),h(t,Ps,p),h(t,At,p),e(At,ci),h(t,As,p),_(De,t,p),h(t,Is,p),h(t,pe,p),e(pe,Ce),e(Ce,ql),_(Ze,ql,null),e(pe,pi),e(pe,jl),e(jl,di),h(t,Os,p),h(t,Pe,p),e(Pe,hi),e(Pe,It),e(It,ui),e(Pe,fi),h(t,Ss,p),_(et,t,p),h(t,Ns,p),h(t,Ot,p),e(Ot,mi),h(t,Ls,p),_(tt,t,p),h(t,Hs,p),h(t,St,p),e(St,vi),h(t,Bs,p),h(t,de,p),e(de,Ae),e(Ae,Tl),_(at,Tl,null),e(de,_i),e(de,xl),e(xl,gi),h(t,zs,p),h(t,Nt,p),e(Nt,Ei),h(t,Gs,p),h(t,V,p),e(V,Lt),e(Lt,Dl),e(Dl,bi),e(Lt,wi),e(V,ki),e(V,Ht),e(Ht,Cl),e(Cl,yi),e(Ht,$i),e(V,qi),e(V,q),e(q,Pl),e(Pl,ji),e(q,Ti),e(q,Al),e(Al,xi),e(q,Di),e(q,Il),e(Il,Ci),e(q,Pi),e(q,Ol),e(Ol,Ai),e(q,Ii),e(q,Sl),e(Sl,Oi),e(q,Si),e(q,Nl),e(Nl,Ni),e(q,Li),e(V,Hi),e(V,Bt),e(Bt,Ll),e(Ll,Bi),e(Bt,zi),h(t,Ws,p),h(t,zt,p),e(zt,Gi),h(t,Rs,p),h(t,he,p),e(he,Ie),e(Ie,Hl),_(lt,Hl,null),e(he,Wi),e(he,Bl),e(Bl,Ri),h(t,Us,p),h(t,Oe,p),e(Oe,Ui),e(Oe,zl),e(zl,Vi),e(Oe,Mi),h(t,Vs,p),_(st,t,p),h(t,Ms,p),h(t,Gt,p),e(Gt,Fi),h(t,Fs,p),h(t,Se,p),e(Se,Gl),e(Gl,j),e(j,Wt),e(Wt,Qi),e(j,Ji),e(j,Rt),e(Rt,Yi),e(j,Ki),e(j,Ut),e(Ut,Xi),e(j,Zi),e(j,Vt),e(Vt,er),e(j,tr),e(j,Mt),e(Mt,ar),e(j,lr),e(j,Ft),e(Ft,sr),e(Se,or),e(Se,y),e(y,T),e(T,Qt),e(Qt,nr),e(T,ir),e(T,Jt),e(Jt,rr),e(T,cr),e(T,Yt),e(Yt,pr),e(T,dr),e(T,Kt),e(Kt,hr),e(T,ur),e(T,Xt),e(Xt,fr),e(T,mr),e(T,Zt),e(Zt,vr),e(y,_r),e(y,x),e(x,ea),e(ea,gr),e(x,Er),e(x,ta),e(ta,br),e(x,wr),e(x,aa),e(aa,kr),e(x,yr),e(x,la),e(la,$r),e(x,qr),e(x,sa),e(sa,jr),e(x,Tr),e(x,oa),e(oa,xr),e(y,Dr),e(y,D),e(D,na),e(na,Cr),e(D,Pr),e(D,ia),e(ia,Ar),e(D,Ir),e(D,ra),e(ra,Or),e(D,Sr),e(D,ca),e(ca,Nr),e(D,Lr),e(D,pa),e(pa,Hr),e(D,Br),e(D,da),e(da,zr),e(y,Gr),e(y,C),e(C,ha),e(ha,Wr),e(C,Rr),e(C,ua),e(ua,Ur),e(C,Vr),e(C,fa),e(fa,Mr),e(C,Fr),e(C,ma),e(ma,Qr),e(C,Jr),e(C,va),e(va,Yr),e(C,Kr),e(C,_a),e(_a,Xr),e(y,Zr),e(y,P),e(P,ga),e(ga,ec),e(P,tc),e(P,Ea),e(Ea,ac),e(P,lc),e(P,ba),e(ba,sc),e(P,oc),e(P,wa),e(wa,nc),e(P,ic),e(P,ka),e(ka,rc),e(P,cc),e(P,ya),e(ya,pc),e(y,dc),e(y,A),e(A,$a),e($a,hc),e(A,uc),e(A,qa),e(qa,fc),e(A,mc),e(A,ja),e(ja,vc),e(A,_c),e(A,Ta),e(Ta,gc),e(A,Ec),e(A,xa),e(xa,bc),e(A,wc),e(A,Da),e(Da,kc),e(y,yc),e(y,I),e(I,Ca),e(Ca,$c),e(I,qc),e(I,Pa),e(Pa,jc),e(I,Tc),e(I,Aa),e(Aa,xc),e(I,Dc),e(I,Ia),e(Ia,Cc),e(I,Pc),e(I,Oa),e(Oa,Ac),e(I,Ic),e(I,Sa),e(Sa,Oc),h(t,Qs,p),h(t,ue,p),e(ue,Ne),e(Ne,Wl),_(ot,Wl,null),e(ue,Sc),e(ue,Rl),e(Rl,Nc),h(t,Js,p),h(t,Na,p),e(Na,Lc),h(t,Ys,p),h(t,S,p),e(S,La),e(La,Ul),e(Ul,Hc),e(La,Bc),e(S,zc),e(S,Ha),e(Ha,Vl),e(Vl,Gc),e(Ha,Wc),e(S,Rc),e(S,Ba),e(Ba,Ml),e(Ml,Uc),e(Ba,Vc),e(S,Mc),e(S,za),e(za,Fl),e(Fl,Fc),e(za,Qc),e(S,Jc),e(S,Ga),e(Ga,Ql),e(Ql,Yc),e(Ga,Kc),h(t,Ks,p),h(t,Wa,p),e(Wa,Xc),h(t,Xs,p),h(t,fe,p),e(fe,Le),e(Le,Jl),_(nt,Jl,null),e(fe,Zc),e(fe,Yl),e(Yl,ep),h(t,Zs,p),h(t,M,p),e(M,tp),e(M,it),e(it,ap),e(M,lp),e(M,Kl),e(Kl,sp),e(M,op),e(M,Xl),e(Xl,np),e(M,ip),h(t,eo,p),_(rt,t,p),h(t,to,p),h(t,Ra,p),e(Ra,rp),h(t,ao,p),_(ct,t,p),h(t,lo,p),h(t,me,p),e(me,He),e(He,Zl),_(pt,Zl,null),e(me,cp),e(me,es),e(es,pp),h(t,so,p),h(t,Ua,p),e(Ua,dp),h(t,oo,p),h(t,le,p),e(le,Va),e(Va,ts),e(ts,hp),e(Va,up),e(le,fp),e(le,Ma),e(Ma,as),e(as,mp),e(Ma,vp),e(le,_p),e(le,dt),e(dt,ls),e(ls,gp),e(dt,Ep),e(dt,ss),e(ss,bp),h(t,no,p),h(t,Fa,p),e(Fa,wp),h(t,io,p),h(t,ve,p),e(ve,Be),e(Be,os),_(ht,os,null),e(ve,kp),e(ve,ns),e(ns,yp),h(t,ro,p),h(t,Qa,p),e(Qa,$p),h(t,co,p),_(ut,t,p),h(t,po,p),h(t,ze,p),e(ze,qp),e(ze,is),e(is,jp),e(ze,Tp),ho=!0},p(t,[p]){const ft={};p&2&&(ft.$$scope={dirty:p,ctx:t}),xe.$set(ft);const rs={};p&2&&(rs.$$scope={dirty:p,ctx:t}),De.$set(rs)},i(t){ho||(g(H.$$.fragment,t),g(Je.$$.fragment,t),g(Ye.$$.fragment,t),g(Ke.$$.fragment,t),g(xe.$$.fragment,t),g(Xe.$$.fragment,t),g(De.$$.fragment,t),g(Ze.$$.fragment,t),g(et.$$.fragment,t),g(tt.$$.fragment,t),g(at.$$.fragment,t),g(lt.$$.fragment,t),g(st.$$.fragment,t),g(ot.$$.fragment,t),g(nt.$$.fragment,t),g(rt.$$.fragment,t),g(ct.$$.fragment,t),g(pt.$$.fragment,t),g(ht.$$.fragment,t),g(ut.$$.fragment,t),ho=!0)},o(t){E(H.$$.fragment,t),E(Je.$$.fragment,t),E(Ye.$$.fragment,t),E(Ke.$$.fragment,t),E(xe.$$.fragment,t),E(Xe.$$.fragment,t),E(De.$$.fragment,t),E(Ze.$$.fragment,t),E(et.$$.fragment,t),E(tt.$$.fragment,t),E(at.$$.fragment,t),E(lt.$$.fragment,t),E(st.$$.fragment,t),E(ot.$$.fragment,t),E(nt.$$.fragment,t),E(rt.$$.fragment,t),E(ct.$$.fragment,t),E(pt.$$.fragment,t),E(ht.$$.fragment,t),E(ut.$$.fragment,t),ho=!1},d(t){a(u),t&&a(G),t&&a(f),b(H),t&&a(Qe),t&&a(W),t&&a(vs),t&&a(_t),t&&a(_s),t&&a(w),t&&a(gs),t&&a(R),t&&a(Es),t&&a(jt),t&&a(bs),t&&a(re),b(Je),t&&a(ws),t&&a(Tt),t&&a(ks),t&&a(ae),t&&a(ys),t&&a(je),t&&a($s),t&&a(ce),b(Ye),t&&a(qs),t&&a(U),t&&a(js),t&&a(Ct),t&&a(Ts),b(Ke,t),t&&a(xs),b(xe,t),t&&a(Ds),t&&a(Pt),t&&a(Cs),b(Xe,t),t&&a(Ps),t&&a(At),t&&a(As),b(De,t),t&&a(Is),t&&a(pe),b(Ze),t&&a(Os),t&&a(Pe),t&&a(Ss),b(et,t),t&&a(Ns),t&&a(Ot),t&&a(Ls),b(tt,t),t&&a(Hs),t&&a(St),t&&a(Bs),t&&a(de),b(at),t&&a(zs),t&&a(Nt),t&&a(Gs),t&&a(V),t&&a(Ws),t&&a(zt),t&&a(Rs),t&&a(he),b(lt),t&&a(Us),t&&a(Oe),t&&a(Vs),b(st,t),t&&a(Ms),t&&a(Gt),t&&a(Fs),t&&a(Se),t&&a(Qs),t&&a(ue),b(ot),t&&a(Js),t&&a(Na),t&&a(Ys),t&&a(S),t&&a(Ks),t&&a(Wa),t&&a(Xs),t&&a(fe),b(nt),t&&a(Zs),t&&a(M),t&&a(eo),b(rt,t),t&&a(to),t&&a(Ra),t&&a(ao),b(ct,t),t&&a(lo),t&&a(me),b(pt),t&&a(so),t&&a(Ua),t&&a(oo),t&&a(le),t&&a(no),t&&a(Fa),t&&a(io),t&&a(ve),b(ht),t&&a(ro),t&&a(Qa),t&&a(co),b(ut,t),t&&a(po),t&&a(ze)}}}const Bu={local:"using-the-evaluator",sections:[{local:"text-classification",sections:[{local:"evaluate-models-on-the-hub",title:"Evaluate models on the Hub"},{local:"evaluate-multiple-metrics",title:"Evaluate multiple metrics"}],title:"Text classification"},{local:"token-classification",sections:[{local:"benchmarking-several-models",title:"Benchmarking several models"}],title:"Token Classification"},{local:"question-answering",sections:[{local:"confidence-intervals",title:"Confidence intervals"}],title:"Question Answering"},{local:"image-classification",sections:[{local:"handling-large-datasets",title:"Handling large datasets"}],title:"Image classification"}],title:"Using the `evaluator`"};function zu(mt){return Su(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class Vu extends Pu{constructor(u){super();Au(this,u,zu,Hu,Iu,{})}}export{Vu as default,Bu as metadata};
