import{S as Sh,i as Lh,s as Hh,e as l,k as r,w as m,t as n,M as Bh,c as s,d as a,m as c,a as o,x as v,h as i,b as d,G as e,g as h,y as _,q as g,o as b,B as E,v as Wh}from"../chunks/vendor-hf-doc-builder.js";import{T as Nh}from"../chunks/Tip-hf-doc-builder.js";import{I as le}from"../chunks/IconCopyLink-hf-doc-builder.js";import{C as Ge}from"../chunks/CodeBlock-hf-doc-builder.js";function Gh(pt){let u,B,f,w,se,S,Re,Z;return{c(){u=l("p"),B=n("Without specifying a device, the default for model inference will be the first GPU on the machine if one is available, and else CPU. If you want to use a specific device you can pass "),f=l("code"),w=n("device"),se=n(" to "),S=l("code"),Re=n("compute"),Z=n(" where -1 will use the GPU and a positive integer (starting with 0) will use the associated CUDA device.")},l(oe){u=s(oe,"P",{});var I=o(u);B=i(I,"Without specifying a device, the default for model inference will be the first GPU on the machine if one is available, and else CPU. If you want to use a specific device you can pass "),f=s(I,"CODE",{});var dt=o(f);w=i(dt,"device"),dt.forEach(a),se=i(I," to "),S=s(I,"CODE",{});var Ue=o(S);Re=i(Ue,"compute"),Ue.forEach(a),Z=i(I," where -1 will use the GPU and a positive integer (starting with 0) will use the associated CUDA device."),I.forEach(a)},m(oe,I){h(oe,u,I),e(u,B),e(u,f),e(f,w),e(u,se),e(u,S),e(S,Re),e(u,Z)},d(oe){oe&&a(u)}}}function Rh(pt){let u,B;return{c(){u=l("p"),B=n("The time performances can give useful indication on model speed for inference but should be taken with a grain of salt: they include all the processing that goes on in the pipeline. This may include tokenizing, post-processing, that may be different depending on the model. Furthermore, it depends a lot on the hardware you are running the evaluation on and you may be able to improve the performance by optimizing things like the batch size.")},l(f){u=s(f,"P",{});var w=o(u);B=i(w,"The time performances can give useful indication on model speed for inference but should be taken with a grain of salt: they include all the processing that goes on in the pipeline. This may include tokenizing, post-processing, that may be different depending on the model. Furthermore, it depends a lot on the hardware you are running the evaluation on and you may be able to improve the performance by optimizing things like the batch size."),w.forEach(a)},m(f,w){h(f,u,w),e(u,B)},d(f){f&&a(u)}}}function Uh(pt){let u,B,f,w,se,S,Re,Z,oe,I,dt,Ue,W,po,Ra,ho,uo,Ua,fo,mo,me,vo,za,_o,go,bo,Zl,ht,Eo,es,G,ve,Va,wo,ko,ut,yo,$o,qo,_e,Ma,jo,To,ft,xo,Do,Po,ge,Fa,Co,Ao,mt,Io,Oo,No,be,Qa,So,Lo,vt,Ho,Bo,ts,_t,Wo,as,ne,Ee,Ja,ze,Go,Ya,Ro,ls,gt,Uo,ss,ee,bt,Ka,zo,Vo,Mo,Et,Xa,Fo,Qo,Jo,y,Za,Yo,Ko,el,Xo,Zo,tl,en,tn,al,an,ln,ll,sn,on,sl,nn,rn,os,we,cn,ol,pn,dn,ns,ie,ke,nl,Ve,hn,il,un,is,R,fn,rl,mn,vn,cl,_n,gn,pl,bn,En,rs,wt,wn,cs,Me,ps,ye,ds,kt,kn,hs,Fe,us,yt,yn,fs,$e,ms,re,qe,dl,Qe,$n,hl,qn,vs,je,jn,$t,Tn,xn,_s,Je,gs,qt,Dn,bs,Ye,Es,jt,Pn,ws,ce,Te,ul,Ke,Cn,fl,An,ks,Tt,In,ys,U,xt,ml,On,Nn,Sn,Dt,vl,Ln,Hn,Bn,$,_l,Wn,Gn,gl,Rn,Un,bl,zn,Vn,El,Mn,Fn,wl,Qn,Jn,kl,Yn,Kn,Xn,Pt,yl,Zn,ei,$s,Ct,ti,qs,pe,xe,$l,Xe,ai,ql,li,js,De,si,jl,oi,ni,Ts,Ze,xs,At,ii,Ds,Pe,Tl,q,It,ri,ci,Ot,pi,di,Nt,hi,ui,St,fi,mi,Lt,vi,_i,Ht,gi,bi,k,j,Bt,Ei,wi,Wt,ki,yi,Gt,$i,qi,Rt,ji,Ti,Ut,xi,Di,zt,Pi,Ci,T,Vt,Ai,Ii,Mt,Oi,Ni,Ft,Si,Li,Qt,Hi,Bi,Jt,Wi,Gi,Yt,Ri,Ui,x,Kt,zi,Vi,Xt,Mi,Fi,Zt,Qi,Ji,ea,Yi,Ki,ta,Xi,Zi,aa,er,tr,D,la,ar,lr,sa,sr,or,oa,nr,ir,na,rr,cr,ia,pr,dr,ra,hr,ur,P,ca,fr,mr,pa,vr,_r,da,gr,br,ha,Er,wr,ua,kr,yr,fa,$r,qr,C,ma,jr,Tr,va,xr,Dr,_a,Pr,Cr,ga,Ar,Ir,ba,Or,Nr,Ea,Sr,Lr,A,wa,Hr,Br,ka,Wr,Gr,ya,Rr,Ur,$a,zr,Vr,qa,Mr,Fr,ja,Qr,Ps,de,Ce,xl,et,Jr,Dl,Yr,Cs,Ta,Kr,As,O,xa,Pl,Xr,Zr,ec,Da,Cl,tc,ac,lc,Pa,Al,sc,oc,nc,Ca,Il,ic,rc,cc,Aa,Ol,pc,dc,Is,Ia,hc,Os,he,Ae,Nl,tt,uc,Sl,fc,Ns,z,mc,at,vc,_c,Ll,gc,bc,Hl,Ec,wc,Ss,lt,Ls,Oa,kc,Hs,st,Bs,ue,Ie,Bl,ot,yc,Wl,$c,Ws,Na,qc,Gs,te,Sa,Gl,jc,Tc,xc,La,Rl,Dc,Pc,Cc,nt,Ul,Ac,Ic,zl,Oc,Rs,Ha,Nc,Us,fe,Oe,Vl,it,Sc,Ml,Lc,zs,Ba,Hc,Vs,rt,Ms,Ne,Bc,Fl,Wc,Gc,Fs;return S=new le({}),ze=new le({}),Ve=new le({}),Me=new Ge({props:{code:`from datasets import load_dataset
from evaluate import evaluator
from transformers import AutoModelForSequenceClassification, pipeline

data = load_dataset("imdb", split="test").shuffle(seed=42).select(range(1000))
task_evaluator = evaluator("text-classification")

# 1. Pass a model name or path
eval_results = task_evaluator.compute(
    model_or_pipeline="lvwerra/distilbert-imdb",
    data=data,
    label_mapping={"NEGATIVE": 0, "POSITIVE": 1}
)

# 2. Pass an instantiated model
model = AutoModelForSequenceClassification.from_pretrained("lvwerra/distilbert-imdb")

eval_results = task_evaluator.compute(
    model_or_pipeline=model,
    data=data,
    label_mapping={"NEGATIVE": 0, "POSITIVE": 1}
)

# 3. Pass an instantiated pipeline 
pipe = pipeline("text-classification", model="lvwerra/distilbert-imdb")

eval_results = task_evaluator.compute(
    model_or_pipeline=pipe,
    data=data,
    label_mapping={"NEGATIVE": 0, "POSITIVE": 1}
)
print(eval_results)`,highlighted:`<span class="hljs-keyword">from</span> datasets <span class="hljs-keyword">import</span> load_dataset
<span class="hljs-keyword">from</span> evaluate <span class="hljs-keyword">import</span> evaluator
<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoModelForSequenceClassification, pipeline

data = load_dataset(<span class="hljs-string">&quot;imdb&quot;</span>, split=<span class="hljs-string">&quot;test&quot;</span>).shuffle(seed=<span class="hljs-number">42</span>).select(<span class="hljs-built_in">range</span>(<span class="hljs-number">1000</span>))
task_evaluator = evaluator(<span class="hljs-string">&quot;text-classification&quot;</span>)

<span class="hljs-comment"># 1. Pass a model name or path</span>
eval_results = task_evaluator.compute(
    model_or_pipeline=<span class="hljs-string">&quot;lvwerra/distilbert-imdb&quot;</span>,
    data=data,
    label_mapping={<span class="hljs-string">&quot;NEGATIVE&quot;</span>: <span class="hljs-number">0</span>, <span class="hljs-string">&quot;POSITIVE&quot;</span>: <span class="hljs-number">1</span>}
)

<span class="hljs-comment"># 2. Pass an instantiated model</span>
model = AutoModelForSequenceClassification.from_pretrained(<span class="hljs-string">&quot;lvwerra/distilbert-imdb&quot;</span>)

eval_results = task_evaluator.compute(
    model_or_pipeline=model,
    data=data,
    label_mapping={<span class="hljs-string">&quot;NEGATIVE&quot;</span>: <span class="hljs-number">0</span>, <span class="hljs-string">&quot;POSITIVE&quot;</span>: <span class="hljs-number">1</span>}
)

<span class="hljs-comment"># 3. Pass an instantiated pipeline </span>
pipe = pipeline(<span class="hljs-string">&quot;text-classification&quot;</span>, model=<span class="hljs-string">&quot;lvwerra/distilbert-imdb&quot;</span>)

eval_results = task_evaluator.compute(
    model_or_pipeline=pipe,
    data=data,
    label_mapping={<span class="hljs-string">&quot;NEGATIVE&quot;</span>: <span class="hljs-number">0</span>, <span class="hljs-string">&quot;POSITIVE&quot;</span>: <span class="hljs-number">1</span>}
)
<span class="hljs-built_in">print</span>(eval_results)`}}),ye=new Nh({props:{$$slots:{default:[Gh]},$$scope:{ctx:pt}}}),Fe=new Ge({props:{code:`{
    'accuracy': 0.918,
    'latency_in_seconds': 0.013,
    'samples_per_second': 78.887,
    'total_time_in_seconds': 12.676
}`,highlighted:`{
    <span class="hljs-string">&#x27;accuracy&#x27;</span>: <span class="hljs-number">0.918</span>,
    <span class="hljs-string">&#x27;latency_in_seconds&#x27;</span>: <span class="hljs-number">0.013</span>,
    <span class="hljs-string">&#x27;samples_per_second&#x27;</span>: <span class="hljs-number">78.887</span>,
    <span class="hljs-string">&#x27;total_time_in_seconds&#x27;</span>: <span class="hljs-number">12.676</span>
}`}}),$e=new Nh({props:{$$slots:{default:[Rh]},$$scope:{ctx:pt}}}),Qe=new le({}),Je=new Ge({props:{code:`import evaluate

eval_results = task_evaluator.compute(
    model_or_pipeline="lvwerra/distilbert-imdb",
    data=data,
    metric=evaluate.combine(["accuracy", "recall", "precision", "f1"]),
    label_mapping={"NEGATIVE": 0, "POSITIVE": 1}
)
print(eval_results)
`,highlighted:`<span class="hljs-keyword">import</span> evaluate

eval_results = task_evaluator.compute(
    model_or_pipeline=<span class="hljs-string">&quot;lvwerra/distilbert-imdb&quot;</span>,
    data=data,
    metric=evaluate.combine([<span class="hljs-string">&quot;accuracy&quot;</span>, <span class="hljs-string">&quot;recall&quot;</span>, <span class="hljs-string">&quot;precision&quot;</span>, <span class="hljs-string">&quot;f1&quot;</span>]),
    label_mapping={<span class="hljs-string">&quot;NEGATIVE&quot;</span>: <span class="hljs-number">0</span>, <span class="hljs-string">&quot;POSITIVE&quot;</span>: <span class="hljs-number">1</span>}
)
<span class="hljs-built_in">print</span>(eval_results)
`}}),Ye=new Ge({props:{code:`{
    'accuracy': 0.918,
    'f1': 0.916,
    'precision': 0.9147,
    'recall': 0.9187,
    'latency_in_seconds': 0.013,
    'samples_per_second': 78.887,
    'total_time_in_seconds': 12.676
}`,highlighted:`{
    <span class="hljs-string">&#x27;accuracy&#x27;</span>: <span class="hljs-number">0.918</span>,
    <span class="hljs-string">&#x27;f1&#x27;</span>: <span class="hljs-number">0.916</span>,
    <span class="hljs-string">&#x27;precision&#x27;</span>: <span class="hljs-number">0.9147</span>,
    <span class="hljs-string">&#x27;recall&#x27;</span>: <span class="hljs-number">0.9187</span>,
    <span class="hljs-string">&#x27;latency_in_seconds&#x27;</span>: <span class="hljs-number">0.013</span>,
    <span class="hljs-string">&#x27;samples_per_second&#x27;</span>: <span class="hljs-number">78.887</span>,
    <span class="hljs-string">&#x27;total_time_in_seconds&#x27;</span>: <span class="hljs-number">12.676</span>
}`}}),Ke=new le({}),Xe=new le({}),Ze=new Ge({props:{code:`import pandas as pd
from datasets import load_dataset
from evaluate import evaluator
from transformers import pipeline

models = [
    "xlm-roberta-large-finetuned-conll03-english",
    "dbmdz/bert-large-cased-finetuned-conll03-english",
    "elastic/distilbert-base-uncased-finetuned-conll03-english",
    "dbmdz/electra-large-discriminator-finetuned-conll03-english",
    "gunghio/distilbert-base-multilingual-cased-finetuned-conll2003-ner",
    "philschmid/distilroberta-base-ner-conll2003",
    "Jorgeutd/albert-base-v2-finetuned-ner",
]

data = load_dataset("conll2003", split="validation").shuffle().select(1000)
task_evaluator = evaluator("token-classification")

results = []
for model in models:
    results.append(
        task_evaluator.compute(
            model_or_pipeline=model, data=data, metric="seqeval"
            )
        )

df = pd.DataFrame(results, index=models)
df[["overall_f1", "overall_accuracy", "total_time_in_seconds", "samples_per_second", "latency_in_seconds"]]`,highlighted:`<span class="hljs-keyword">import</span> pandas <span class="hljs-keyword">as</span> pd
<span class="hljs-keyword">from</span> datasets <span class="hljs-keyword">import</span> load_dataset
<span class="hljs-keyword">from</span> evaluate <span class="hljs-keyword">import</span> evaluator
<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> pipeline

models = [
    <span class="hljs-string">&quot;xlm-roberta-large-finetuned-conll03-english&quot;</span>,
    <span class="hljs-string">&quot;dbmdz/bert-large-cased-finetuned-conll03-english&quot;</span>,
    <span class="hljs-string">&quot;elastic/distilbert-base-uncased-finetuned-conll03-english&quot;</span>,
    <span class="hljs-string">&quot;dbmdz/electra-large-discriminator-finetuned-conll03-english&quot;</span>,
    <span class="hljs-string">&quot;gunghio/distilbert-base-multilingual-cased-finetuned-conll2003-ner&quot;</span>,
    <span class="hljs-string">&quot;philschmid/distilroberta-base-ner-conll2003&quot;</span>,
    <span class="hljs-string">&quot;Jorgeutd/albert-base-v2-finetuned-ner&quot;</span>,
]

data = load_dataset(<span class="hljs-string">&quot;conll2003&quot;</span>, split=<span class="hljs-string">&quot;validation&quot;</span>).shuffle().select(<span class="hljs-number">1000</span>)
task_evaluator = evaluator(<span class="hljs-string">&quot;token-classification&quot;</span>)

results = []
<span class="hljs-keyword">for</span> model <span class="hljs-keyword">in</span> models:
    results.append(
        task_evaluator.compute(
            model_or_pipeline=model, data=data, metric=<span class="hljs-string">&quot;seqeval&quot;</span>
            )
        )

df = pd.DataFrame(results, index=models)
df[[<span class="hljs-string">&quot;overall_f1&quot;</span>, <span class="hljs-string">&quot;overall_accuracy&quot;</span>, <span class="hljs-string">&quot;total_time_in_seconds&quot;</span>, <span class="hljs-string">&quot;samples_per_second&quot;</span>, <span class="hljs-string">&quot;latency_in_seconds&quot;</span>]]`}}),et=new le({}),tt=new le({}),lt=new Ge({props:{code:`from datasets import load_dataset
from evaluate import evaluator

task_evaluator = evaluator("question-answering")

data = load_dataset("squad", split="validation[:1000]")
eval_results = task_evaluator.compute(
    model_or_pipeline="distilbert-base-uncased-distilled-squad",
    data=data,
    metric="squad",
    strategy="bootstrap",
    n_resamples=30
)`,highlighted:`<span class="hljs-keyword">from</span> datasets <span class="hljs-keyword">import</span> load_dataset
<span class="hljs-keyword">from</span> evaluate <span class="hljs-keyword">import</span> evaluator

task_evaluator = evaluator(<span class="hljs-string">&quot;question-answering&quot;</span>)

data = load_dataset(<span class="hljs-string">&quot;squad&quot;</span>, split=<span class="hljs-string">&quot;validation[:1000]&quot;</span>)
eval_results = task_evaluator.compute(
    model_or_pipeline=<span class="hljs-string">&quot;distilbert-base-uncased-distilled-squad&quot;</span>,
    data=data,
    metric=<span class="hljs-string">&quot;squad&quot;</span>,
    strategy=<span class="hljs-string">&quot;bootstrap&quot;</span>,
    n_resamples=<span class="hljs-number">30</span>
)`}}),st=new Ge({props:{code:`{
    'exact_match': 
    {
        'confidence_interval': (79.67, 84.54),
        'score': 82.30,
        'standard_error': 1.28
    },
    'f1': 
    {
        'confidence_interval': (85.30, 88.88),
        'score': 87.23,
        'standard_error': 0.97
    },
    'latency_in_seconds': 0.0085,
    'samples_per_second': 117.31,
    'total_time_in_seconds': 8.52
 }`,highlighted:`{
    <span class="hljs-string">&#x27;exact_match&#x27;</span>: 
    {
        <span class="hljs-string">&#x27;confidence_interval&#x27;</span>: (<span class="hljs-number">79.67</span>, <span class="hljs-number">84.54</span>),
        <span class="hljs-string">&#x27;score&#x27;</span>: <span class="hljs-number">82.30</span>,
        <span class="hljs-string">&#x27;standard_error&#x27;</span>: <span class="hljs-number">1.28</span>
    },
    <span class="hljs-string">&#x27;f1&#x27;</span>: 
    {
        <span class="hljs-string">&#x27;confidence_interval&#x27;</span>: (<span class="hljs-number">85.30</span>, <span class="hljs-number">88.88</span>),
        <span class="hljs-string">&#x27;score&#x27;</span>: <span class="hljs-number">87.23</span>,
        <span class="hljs-string">&#x27;standard_error&#x27;</span>: <span class="hljs-number">0.97</span>
    },
    <span class="hljs-string">&#x27;latency_in_seconds&#x27;</span>: <span class="hljs-number">0.0085</span>,
    <span class="hljs-string">&#x27;samples_per_second&#x27;</span>: <span class="hljs-number">117.31</span>,
    <span class="hljs-string">&#x27;total_time_in_seconds&#x27;</span>: <span class="hljs-number">8.52</span>
 }`}}),ot=new le({}),it=new le({}),rt=new Ge({props:{code:`data = load_dataset("imagenet-1k", split="validation", use_auth_token=True)

pipe = pipeline(
    task="image-classification",
    model="facebook/deit-small-distilled-patch16-224"
)

task_evaluator = evaluator("image-classification")
eval_results = task_evaluator.compute(
    model_or_pipeline=pipe,
    data=data,
    metric="accuracy",
    label_mapping=pipe.model.config.label2id
)`,highlighted:`data = load_dataset(<span class="hljs-string">&quot;imagenet-1k&quot;</span>, split=<span class="hljs-string">&quot;validation&quot;</span>, use_auth_token=<span class="hljs-literal">True</span>)

pipe = pipeline(
    task=<span class="hljs-string">&quot;image-classification&quot;</span>,
    model=<span class="hljs-string">&quot;facebook/deit-small-distilled-patch16-224&quot;</span>
)

task_evaluator = evaluator(<span class="hljs-string">&quot;image-classification&quot;</span>)
eval_results = task_evaluator.compute(
    model_or_pipeline=pipe,
    data=data,
    metric=<span class="hljs-string">&quot;accuracy&quot;</span>,
    label_mapping=pipe.model.config.label2id
)`}}),{c(){u=l("meta"),B=r(),f=l("h1"),w=l("a"),se=l("span"),m(S.$$.fragment),Re=r(),Z=l("span"),oe=n("Using the "),I=l("code"),dt=n("evaluator"),Ue=r(),W=l("p"),po=n("The "),Ra=l("code"),ho=n("Evaluator"),uo=n(" classes allow to evaluate a  triplet of model, dataset, and metric. The models wrapped in a pipeline, responsible for handling all preprocessing and post-processing and out-of-the-box, "),Ua=l("code"),fo=n("Evaluator"),mo=n("s support transformers pipelines for the supported tasks, but custom pipelines can be passed, as showcased in the section "),me=l("a"),vo=n("Using the "),za=l("code"),_o=n("evaluator"),go=n(" with custom pipelines"),bo=n("."),Zl=r(),ht=l("p"),Eo=n("Currently supported tasks are:"),es=r(),G=l("ul"),ve=l("li"),Va=l("code"),wo=n('"text-classification"'),ko=n(": will use the "),ut=l("a"),yo=n("TextClassificationEvaluator"),$o=n("."),qo=r(),_e=l("li"),Ma=l("code"),jo=n('"token-classification"'),To=n(": will use the "),ft=l("a"),xo=n("TokenClassificationEvaluator"),Do=n("."),Po=r(),ge=l("li"),Fa=l("code"),Co=n('"question-answering"'),Ao=n(": will use the "),mt=l("a"),Io=n("QuestionAnsweringEvaluator"),Oo=n("."),No=r(),be=l("li"),Qa=l("code"),So=n('"image-classification"'),Lo=n(": will use the "),vt=l("a"),Ho=n("ImageClassificationEvaluator"),Bo=n("."),ts=r(),_t=l("p"),Wo=n("Each task has its own set of requirements for the dataset format and pipeline output, make sure to check them out for your custom use case. Let\u2019s have a look at each one of them and see how you can use the evaluator to evalute a single or multiple of models, datasets, and metrics at the same time."),as=r(),ne=l("h2"),Ee=l("a"),Ja=l("span"),m(ze.$$.fragment),Go=r(),Ya=l("span"),Ro=n("Text classification"),ls=r(),gt=l("p"),Uo=n("The text classification evaluator can be used to evaluate text models on classification datasets such as IMDb. Beside the model, data, and metric inputs it takes the following optional inputs:"),ss=r(),ee=l("ul"),bt=l("li"),Ka=l("code"),zo=n('input_column="text"'),Vo=n(": with this argument the column with the data for the pipeline can be specified."),Mo=r(),Et=l("li"),Xa=l("code"),Fo=n('label_column="label"'),Qo=n(": with this argument the column with the labels for the evaluation can be specified."),Jo=r(),y=l("li"),Za=l("code"),Yo=n("label_mapping=None"),Ko=n(": the label mapping aligns the labels in the pipeline output with the labels need for evaluation. E.g. the labels in "),el=l("code"),Xo=n("label_column"),Zo=n(" can be integers ("),tl=l("code"),en=n("0"),tn=n("/"),al=l("code"),an=n("1"),ln=n(") whereas the pipeline can produce label names such as "),ll=l("code"),sn=n('"positive"'),on=n("/"),sl=l("code"),nn=n('"negative"'),rn=n(". With that dictionary the pipeline outputs are mapped to the labels."),os=r(),we=l("p"),cn=n("By default the "),ol=l("code"),pn=n('"accuracy"'),dn=n(" metric is computed."),ns=r(),ie=l("h3"),ke=l("a"),nl=l("span"),m(Ve.$$.fragment),hn=r(),il=l("span"),un=n("Evaluate models on the Hub"),is=r(),R=l("p"),fn=n("There are several ways to pass a model to the evaluator: you can pass the name of a model on the Hub, you can load a "),rl=l("code"),mn=n("transformers"),vn=n(" model and pass it to the evaluator or you can pass an initialized "),cl=l("code"),_n=n("transformers.Pipeline"),gn=n(". Alternatively you can pass any callable function that behaves like a "),pl=l("code"),bn=n("pipeline"),En=n(" call for the task in any framework."),rs=r(),wt=l("p"),wn=n("So any of the following works:"),cs=r(),m(Me.$$.fragment),ps=r(),m(ye.$$.fragment),ds=r(),kt=l("p"),kn=n("The results will look as follows:"),hs=r(),m(Fe.$$.fragment),us=r(),yt=l("p"),yn=n("Note that evaluation results include both the requested metric, and information about the time it took to obtain predictions through the pipeline."),fs=r(),m($e.$$.fragment),ms=r(),re=l("h3"),qe=l("a"),dl=l("span"),m(Qe.$$.fragment),$n=r(),hl=l("span"),qn=n("Evaluate multiple metrics"),vs=r(),je=l("p"),jn=n("With the "),$t=l("a"),Tn=n("combine()"),xn=n(" function one can bundle several metrics into an object that behaves like a single metric. We can use this to evaluate several metrics at once with the evaluator:"),_s=r(),m(Je.$$.fragment),gs=r(),qt=l("p"),Dn=n("The results will look as follows:"),bs=r(),m(Ye.$$.fragment),Es=r(),jt=l("p"),Pn=n("Next let\u2019s have a look at token classification."),ws=r(),ce=l("h2"),Te=l("a"),ul=l("span"),m(Ke.$$.fragment),Cn=r(),fl=l("span"),An=n("Token Classification"),ks=r(),Tt=l("p"),In=n("With the token classification evaluator one can evaluate models for tasks such as NER or POS tagging. It has the following specific arguments:"),ys=r(),U=l("ul"),xt=l("li"),ml=l("code"),On=n('input_column="text"'),Nn=n(": with this argument the column with the data for the pipeline can be specified."),Sn=r(),Dt=l("li"),vl=l("code"),Ln=n('label_column="label"'),Hn=n(": with this argument the column with the labels for the evaluation can be specified."),Bn=r(),$=l("li"),_l=l("code"),Wn=n("label_mapping=None"),Gn=n(": the label mapping aligns the labels in the pipeline output with the labels need for evaluation. E.g. the labels in "),gl=l("code"),Rn=n("label_column"),Un=n(" can be integers ("),bl=l("code"),zn=n("0"),Vn=n("/"),El=l("code"),Mn=n("1"),Fn=n(") whereas the pipeline can produce label names such as "),wl=l("code"),Qn=n('"positive"'),Jn=n("/"),kl=l("code"),Yn=n('"negative"'),Kn=n(". With that dictionary the pipeline outputs are mapped to the labels."),Xn=r(),Pt=l("li"),yl=l("code"),Zn=n('join_by=" "'),ei=n(": While most datasets are already tokenized the pipeline expects a string. Thus the tokens need to be joined before passing to the pipeline. By default they are joined with a whitespace."),$s=r(),Ct=l("p"),ti=n("Let\u2019s have a look how we can use the evaluator to benchmark several models."),qs=r(),pe=l("h3"),xe=l("a"),$l=l("span"),m(Xe.$$.fragment),ai=r(),ql=l("span"),li=n("Benchmarking several models"),js=r(),De=l("p"),si=n("Here is an example where several models can be compared thanks to the "),jl=l("code"),oi=n("evaluator"),ni=n(" in only a few lines of code, abstracting away the preprocessing, inference, postprocessing, metric computation:"),Ts=r(),m(Ze.$$.fragment),xs=r(),At=l("p"),ii=n("The result is a table that looks like this:"),Ds=r(),Pe=l("table"),Tl=l("thead"),q=l("tr"),It=l("th"),ri=n("model"),ci=r(),Ot=l("th"),pi=n("overall_f1"),di=r(),Nt=l("th"),hi=n("overall_accuracy"),ui=r(),St=l("th"),fi=n("total_time_in_seconds"),mi=r(),Lt=l("th"),vi=n("samples_per_second"),_i=r(),Ht=l("th"),gi=n("latency_in_seconds"),bi=r(),k=l("tbody"),j=l("tr"),Bt=l("td"),Ei=n("Jorgeutd/albert-base-v2-finetuned-ner"),wi=r(),Wt=l("td"),ki=n("0.941"),yi=r(),Gt=l("td"),$i=n("0.989"),qi=r(),Rt=l("td"),ji=n("4.515"),Ti=r(),Ut=l("td"),xi=n("221.468"),Di=r(),zt=l("td"),Pi=n("0.005"),Ci=r(),T=l("tr"),Vt=l("td"),Ai=n("dbmdz/bert-large-cased-finetuned-conll03-english"),Ii=r(),Mt=l("td"),Oi=n("0.962"),Ni=r(),Ft=l("td"),Si=n("0.881"),Li=r(),Qt=l("td"),Hi=n("11.648"),Bi=r(),Jt=l("td"),Wi=n("85.850"),Gi=r(),Yt=l("td"),Ri=n("0.012"),Ui=r(),x=l("tr"),Kt=l("td"),zi=n("dbmdz/electra-large-discriminator-finetuned-conll03-english"),Vi=r(),Xt=l("td"),Mi=n("0.965"),Fi=r(),Zt=l("td"),Qi=n("0.881"),Ji=r(),ea=l("td"),Yi=n("11.456"),Ki=r(),ta=l("td"),Xi=n("87.292"),Zi=r(),aa=l("td"),er=n("0.011"),tr=r(),D=l("tr"),la=l("td"),ar=n("elastic/distilbert-base-uncased-finetuned-conll03-english"),lr=r(),sa=l("td"),sr=n("0.940"),or=r(),oa=l("td"),nr=n("0.989"),ir=r(),na=l("td"),rr=n("2.318"),cr=r(),ia=l("td"),pr=n("431.378"),dr=r(),ra=l("td"),hr=n("0.002"),ur=r(),P=l("tr"),ca=l("td"),fr=n("gunghio/distilbert-base-multilingual-cased-finetuned-conll2003-ner"),mr=r(),pa=l("td"),vr=n("0.947"),_r=r(),da=l("td"),gr=n("0.991"),br=r(),ha=l("td"),Er=n("2.376"),wr=r(),ua=l("td"),kr=n("420.873"),yr=r(),fa=l("td"),$r=n("0.002"),qr=r(),C=l("tr"),ma=l("td"),jr=n("philschmid/distilroberta-base-ner-conll2003"),Tr=r(),va=l("td"),xr=n("0.961"),Dr=r(),_a=l("td"),Pr=n("0.994"),Cr=r(),ga=l("td"),Ar=n("2.436"),Ir=r(),ba=l("td"),Or=n("410.579"),Nr=r(),Ea=l("td"),Sr=n("0.002"),Lr=r(),A=l("tr"),wa=l("td"),Hr=n("xlm-roberta-large-finetuned-conll03-english"),Br=r(),ka=l("td"),Wr=n("0.969"),Gr=r(),ya=l("td"),Rr=n("0.882"),Ur=r(),$a=l("td"),zr=n("11.996"),Vr=r(),qa=l("td"),Mr=n("83.359"),Fr=r(),ja=l("td"),Qr=n("0.012"),Ps=r(),de=l("h2"),Ce=l("a"),xl=l("span"),m(et.$$.fragment),Jr=r(),Dl=l("span"),Yr=n("Question Answering"),Cs=r(),Ta=l("p"),Kr=n("With the question-answering evaluator one can evaluate models for QA without needing to worry about the complicated pre- and post-processing that\u2019s required for these models. It has the following specific arguments:"),As=r(),O=l("ul"),xa=l("li"),Pl=l("code"),Xr=n('question_column="question"'),Zr=n(": the name of the column containing the question in the dataset"),ec=r(),Da=l("li"),Cl=l("code"),tc=n('context_column="context"'),ac=n(": the name of the column containing the context"),lc=r(),Pa=l("li"),Al=l("code"),sc=n('id_column="id"'),oc=n(": the name of the column cointaing the identification field of the question and answer pair"),nc=r(),Ca=l("li"),Il=l("code"),ic=n('label_column="answers"'),rc=n(": the name of the column containing the answers"),cc=r(),Aa=l("li"),Ol=l("code"),pc=n("squad_v2_format=None"),dc=n(": whether the dataset follows the format of squad_v2 dataset where a question may have no answer in the context. If this parameter is not provided, the format will be automatically inferred."),Is=r(),Ia=l("p"),hc=n("Let\u2019s have a look how we can evaluate QA models and compute confidence intervals at the same time."),Os=r(),he=l("h3"),Ae=l("a"),Nl=l("span"),m(tt.$$.fragment),uc=r(),Sl=l("span"),fc=n("Confidence intervals"),Ns=r(),z=l("p"),mc=n("Every evaluator comes with the options to compute confidence intervals using "),at=l("a"),vc=n("bootstrapping"),_c=n(". Simply pass "),Ll=l("code"),gc=n('strategy="bootstrap"'),bc=n(" and set the number of resanmples with "),Hl=l("code"),Ec=n("n_resamples"),wc=n("."),Ss=r(),m(lt.$$.fragment),Ls=r(),Oa=l("p"),kc=n("Results include confidence intervals as well as error estimates as follows:"),Hs=r(),m(st.$$.fragment),Bs=r(),ue=l("h2"),Ie=l("a"),Bl=l("span"),m(ot.$$.fragment),yc=r(),Wl=l("span"),$c=n("Image classification"),Ws=r(),Na=l("p"),qc=n("With the image classification evaluator we can evaluate any image classifier. It uses the same keyword arguments at the text classifier:"),Gs=r(),te=l("ul"),Sa=l("li"),Gl=l("code"),jc=n('input_column="image"'),Tc=n(": the name of the column containing the images as PIL ImageFile"),xc=r(),La=l("li"),Rl=l("code"),Dc=n('label_column="label"'),Pc=n(": the name of the column containing the labels"),Cc=r(),nt=l("li"),Ul=l("code"),Ac=n("label_mapping=None"),Ic=n(": We want to map class labels defined by the model in the pipeline to values consistent with those defined in the "),zl=l("code"),Oc=n("label_column"),Rs=r(),Ha=l("p"),Nc=n("Let\u2019s have a look at how can evaluate image classification models on large datasets."),Us=r(),fe=l("h3"),Oe=l("a"),Vl=l("span"),m(it.$$.fragment),Sc=r(),Ml=l("span"),Lc=n("Handling large datasets"),zs=r(),Ba=l("p"),Hc=n("The evaluator can be used on large datasets! Below, an example shows how to use it on ImageNet-1k for image classification. Beware that this example will require to download ~150 GB."),Vs=r(),m(rt.$$.fragment),Ms=r(),Ne=l("p"),Bc=n("Since we are using "),Fl=l("code"),Wc=n("datasets"),Gc=n(" to store data we make use of a technique called memory mappings. This means that the dataset is never fully loaded into memory which saves a lot of RAM. Running the above code only uses roughly 1.5 GB of RAM while the validation split is more than 30 GB big."),this.h()},l(t){const p=Bh('[data-svelte="svelte-1phssyn"]',document.head);u=s(p,"META",{name:!0,content:!0}),p.forEach(a),B=c(t),f=s(t,"H1",{class:!0});var ct=o(f);w=s(ct,"A",{id:!0,class:!0,href:!0});var Ql=o(w);se=s(Ql,"SPAN",{});var tp=o(se);v(S.$$.fragment,tp),tp.forEach(a),Ql.forEach(a),Re=c(ct),Z=s(ct,"SPAN",{});var Rc=o(Z);oe=i(Rc,"Using the "),I=s(Rc,"CODE",{});var ap=o(I);dt=i(ap,"evaluator"),ap.forEach(a),Rc.forEach(a),ct.forEach(a),Ue=c(t),W=s(t,"P",{});var Se=o(W);po=i(Se,"The "),Ra=s(Se,"CODE",{});var lp=o(Ra);ho=i(lp,"Evaluator"),lp.forEach(a),uo=i(Se," classes allow to evaluate a  triplet of model, dataset, and metric. The models wrapped in a pipeline, responsible for handling all preprocessing and post-processing and out-of-the-box, "),Ua=s(Se,"CODE",{});var sp=o(Ua);fo=i(sp,"Evaluator"),sp.forEach(a),mo=i(Se,"s support transformers pipelines for the supported tasks, but custom pipelines can be passed, as showcased in the section "),me=s(Se,"A",{href:!0});var Qs=o(me);vo=i(Qs,"Using the "),za=s(Qs,"CODE",{});var op=o(za);_o=i(op,"evaluator"),op.forEach(a),go=i(Qs," with custom pipelines"),Qs.forEach(a),bo=i(Se,"."),Se.forEach(a),Zl=c(t),ht=s(t,"P",{});var np=o(ht);Eo=i(np,"Currently supported tasks are:"),np.forEach(a),es=c(t),G=s(t,"UL",{});var Le=o(G);ve=s(Le,"LI",{});var Jl=o(ve);Va=s(Jl,"CODE",{});var ip=o(Va);wo=i(ip,'"text-classification"'),ip.forEach(a),ko=i(Jl,": will use the "),ut=s(Jl,"A",{href:!0});var rp=o(ut);yo=i(rp,"TextClassificationEvaluator"),rp.forEach(a),$o=i(Jl,"."),Jl.forEach(a),qo=c(Le),_e=s(Le,"LI",{});var Yl=o(_e);Ma=s(Yl,"CODE",{});var cp=o(Ma);jo=i(cp,'"token-classification"'),cp.forEach(a),To=i(Yl,": will use the "),ft=s(Yl,"A",{href:!0});var pp=o(ft);xo=i(pp,"TokenClassificationEvaluator"),pp.forEach(a),Do=i(Yl,"."),Yl.forEach(a),Po=c(Le),ge=s(Le,"LI",{});var Kl=o(ge);Fa=s(Kl,"CODE",{});var dp=o(Fa);Co=i(dp,'"question-answering"'),dp.forEach(a),Ao=i(Kl,": will use the "),mt=s(Kl,"A",{href:!0});var hp=o(mt);Io=i(hp,"QuestionAnsweringEvaluator"),hp.forEach(a),Oo=i(Kl,"."),Kl.forEach(a),No=c(Le),be=s(Le,"LI",{});var Xl=o(be);Qa=s(Xl,"CODE",{});var up=o(Qa);So=i(up,'"image-classification"'),up.forEach(a),Lo=i(Xl,": will use the "),vt=s(Xl,"A",{href:!0});var fp=o(vt);Ho=i(fp,"ImageClassificationEvaluator"),fp.forEach(a),Bo=i(Xl,"."),Xl.forEach(a),Le.forEach(a),ts=c(t),_t=s(t,"P",{});var mp=o(_t);Wo=i(mp,"Each task has its own set of requirements for the dataset format and pipeline output, make sure to check them out for your custom use case. Let\u2019s have a look at each one of them and see how you can use the evaluator to evalute a single or multiple of models, datasets, and metrics at the same time."),mp.forEach(a),as=c(t),ne=s(t,"H2",{class:!0});var Js=o(ne);Ee=s(Js,"A",{id:!0,class:!0,href:!0});var vp=o(Ee);Ja=s(vp,"SPAN",{});var _p=o(Ja);v(ze.$$.fragment,_p),_p.forEach(a),vp.forEach(a),Go=c(Js),Ya=s(Js,"SPAN",{});var gp=o(Ya);Ro=i(gp,"Text classification"),gp.forEach(a),Js.forEach(a),ls=c(t),gt=s(t,"P",{});var bp=o(gt);Uo=i(bp,"The text classification evaluator can be used to evaluate text models on classification datasets such as IMDb. Beside the model, data, and metric inputs it takes the following optional inputs:"),bp.forEach(a),ss=c(t),ee=s(t,"UL",{});var Wa=o(ee);bt=s(Wa,"LI",{});var Uc=o(bt);Ka=s(Uc,"CODE",{});var Ep=o(Ka);zo=i(Ep,'input_column="text"'),Ep.forEach(a),Vo=i(Uc,": with this argument the column with the data for the pipeline can be specified."),Uc.forEach(a),Mo=c(Wa),Et=s(Wa,"LI",{});var zc=o(Et);Xa=s(zc,"CODE",{});var wp=o(Xa);Fo=i(wp,'label_column="label"'),wp.forEach(a),Qo=i(zc,": with this argument the column with the labels for the evaluation can be specified."),zc.forEach(a),Jo=c(Wa),y=s(Wa,"LI",{});var L=o(y);Za=s(L,"CODE",{});var kp=o(Za);Yo=i(kp,"label_mapping=None"),kp.forEach(a),Ko=i(L,": the label mapping aligns the labels in the pipeline output with the labels need for evaluation. E.g. the labels in "),el=s(L,"CODE",{});var yp=o(el);Xo=i(yp,"label_column"),yp.forEach(a),Zo=i(L," can be integers ("),tl=s(L,"CODE",{});var $p=o(tl);en=i($p,"0"),$p.forEach(a),tn=i(L,"/"),al=s(L,"CODE",{});var qp=o(al);an=i(qp,"1"),qp.forEach(a),ln=i(L,") whereas the pipeline can produce label names such as "),ll=s(L,"CODE",{});var jp=o(ll);sn=i(jp,'"positive"'),jp.forEach(a),on=i(L,"/"),sl=s(L,"CODE",{});var Tp=o(sl);nn=i(Tp,'"negative"'),Tp.forEach(a),rn=i(L,". With that dictionary the pipeline outputs are mapped to the labels."),L.forEach(a),Wa.forEach(a),os=c(t),we=s(t,"P",{});var Ys=o(we);cn=i(Ys,"By default the "),ol=s(Ys,"CODE",{});var xp=o(ol);pn=i(xp,'"accuracy"'),xp.forEach(a),dn=i(Ys," metric is computed."),Ys.forEach(a),ns=c(t),ie=s(t,"H3",{class:!0});var Ks=o(ie);ke=s(Ks,"A",{id:!0,class:!0,href:!0});var Dp=o(ke);nl=s(Dp,"SPAN",{});var Pp=o(nl);v(Ve.$$.fragment,Pp),Pp.forEach(a),Dp.forEach(a),hn=c(Ks),il=s(Ks,"SPAN",{});var Cp=o(il);un=i(Cp,"Evaluate models on the Hub"),Cp.forEach(a),Ks.forEach(a),is=c(t),R=s(t,"P",{});var He=o(R);fn=i(He,"There are several ways to pass a model to the evaluator: you can pass the name of a model on the Hub, you can load a "),rl=s(He,"CODE",{});var Ap=o(rl);mn=i(Ap,"transformers"),Ap.forEach(a),vn=i(He," model and pass it to the evaluator or you can pass an initialized "),cl=s(He,"CODE",{});var Ip=o(cl);_n=i(Ip,"transformers.Pipeline"),Ip.forEach(a),gn=i(He,". Alternatively you can pass any callable function that behaves like a "),pl=s(He,"CODE",{});var Op=o(pl);bn=i(Op,"pipeline"),Op.forEach(a),En=i(He," call for the task in any framework."),He.forEach(a),rs=c(t),wt=s(t,"P",{});var Np=o(wt);wn=i(Np,"So any of the following works:"),Np.forEach(a),cs=c(t),v(Me.$$.fragment,t),ps=c(t),v(ye.$$.fragment,t),ds=c(t),kt=s(t,"P",{});var Sp=o(kt);kn=i(Sp,"The results will look as follows:"),Sp.forEach(a),hs=c(t),v(Fe.$$.fragment,t),us=c(t),yt=s(t,"P",{});var Lp=o(yt);yn=i(Lp,"Note that evaluation results include both the requested metric, and information about the time it took to obtain predictions through the pipeline."),Lp.forEach(a),fs=c(t),v($e.$$.fragment,t),ms=c(t),re=s(t,"H3",{class:!0});var Xs=o(re);qe=s(Xs,"A",{id:!0,class:!0,href:!0});var Hp=o(qe);dl=s(Hp,"SPAN",{});var Bp=o(dl);v(Qe.$$.fragment,Bp),Bp.forEach(a),Hp.forEach(a),$n=c(Xs),hl=s(Xs,"SPAN",{});var Wp=o(hl);qn=i(Wp,"Evaluate multiple metrics"),Wp.forEach(a),Xs.forEach(a),vs=c(t),je=s(t,"P",{});var Zs=o(je);jn=i(Zs,"With the "),$t=s(Zs,"A",{href:!0});var Gp=o($t);Tn=i(Gp,"combine()"),Gp.forEach(a),xn=i(Zs," function one can bundle several metrics into an object that behaves like a single metric. We can use this to evaluate several metrics at once with the evaluator:"),Zs.forEach(a),_s=c(t),v(Je.$$.fragment,t),gs=c(t),qt=s(t,"P",{});var Rp=o(qt);Dn=i(Rp,"The results will look as follows:"),Rp.forEach(a),bs=c(t),v(Ye.$$.fragment,t),Es=c(t),jt=s(t,"P",{});var Up=o(jt);Pn=i(Up,"Next let\u2019s have a look at token classification."),Up.forEach(a),ws=c(t),ce=s(t,"H2",{class:!0});var eo=o(ce);Te=s(eo,"A",{id:!0,class:!0,href:!0});var zp=o(Te);ul=s(zp,"SPAN",{});var Vp=o(ul);v(Ke.$$.fragment,Vp),Vp.forEach(a),zp.forEach(a),Cn=c(eo),fl=s(eo,"SPAN",{});var Mp=o(fl);An=i(Mp,"Token Classification"),Mp.forEach(a),eo.forEach(a),ks=c(t),Tt=s(t,"P",{});var Fp=o(Tt);In=i(Fp,"With the token classification evaluator one can evaluate models for tasks such as NER or POS tagging. It has the following specific arguments:"),Fp.forEach(a),ys=c(t),U=s(t,"UL",{});var Be=o(U);xt=s(Be,"LI",{});var Vc=o(xt);ml=s(Vc,"CODE",{});var Qp=o(ml);On=i(Qp,'input_column="text"'),Qp.forEach(a),Nn=i(Vc,": with this argument the column with the data for the pipeline can be specified."),Vc.forEach(a),Sn=c(Be),Dt=s(Be,"LI",{});var Mc=o(Dt);vl=s(Mc,"CODE",{});var Jp=o(vl);Ln=i(Jp,'label_column="label"'),Jp.forEach(a),Hn=i(Mc,": with this argument the column with the labels for the evaluation can be specified."),Mc.forEach(a),Bn=c(Be),$=s(Be,"LI",{});var H=o($);_l=s(H,"CODE",{});var Yp=o(_l);Wn=i(Yp,"label_mapping=None"),Yp.forEach(a),Gn=i(H,": the label mapping aligns the labels in the pipeline output with the labels need for evaluation. E.g. the labels in "),gl=s(H,"CODE",{});var Kp=o(gl);Rn=i(Kp,"label_column"),Kp.forEach(a),Un=i(H," can be integers ("),bl=s(H,"CODE",{});var Xp=o(bl);zn=i(Xp,"0"),Xp.forEach(a),Vn=i(H,"/"),El=s(H,"CODE",{});var Zp=o(El);Mn=i(Zp,"1"),Zp.forEach(a),Fn=i(H,") whereas the pipeline can produce label names such as "),wl=s(H,"CODE",{});var ed=o(wl);Qn=i(ed,'"positive"'),ed.forEach(a),Jn=i(H,"/"),kl=s(H,"CODE",{});var td=o(kl);Yn=i(td,'"negative"'),td.forEach(a),Kn=i(H,". With that dictionary the pipeline outputs are mapped to the labels."),H.forEach(a),Xn=c(Be),Pt=s(Be,"LI",{});var Fc=o(Pt);yl=s(Fc,"CODE",{});var ad=o(yl);Zn=i(ad,'join_by=" "'),ad.forEach(a),ei=i(Fc,": While most datasets are already tokenized the pipeline expects a string. Thus the tokens need to be joined before passing to the pipeline. By default they are joined with a whitespace."),Fc.forEach(a),Be.forEach(a),$s=c(t),Ct=s(t,"P",{});var ld=o(Ct);ti=i(ld,"Let\u2019s have a look how we can use the evaluator to benchmark several models."),ld.forEach(a),qs=c(t),pe=s(t,"H3",{class:!0});var to=o(pe);xe=s(to,"A",{id:!0,class:!0,href:!0});var sd=o(xe);$l=s(sd,"SPAN",{});var od=o($l);v(Xe.$$.fragment,od),od.forEach(a),sd.forEach(a),ai=c(to),ql=s(to,"SPAN",{});var nd=o(ql);li=i(nd,"Benchmarking several models"),nd.forEach(a),to.forEach(a),js=c(t),De=s(t,"P",{});var ao=o(De);si=i(ao,"Here is an example where several models can be compared thanks to the "),jl=s(ao,"CODE",{});var id=o(jl);oi=i(id,"evaluator"),id.forEach(a),ni=i(ao," in only a few lines of code, abstracting away the preprocessing, inference, postprocessing, metric computation:"),ao.forEach(a),Ts=c(t),v(Ze.$$.fragment,t),xs=c(t),At=s(t,"P",{});var rd=o(At);ii=i(rd,"The result is a table that looks like this:"),rd.forEach(a),Ds=c(t),Pe=s(t,"TABLE",{});var lo=o(Pe);Tl=s(lo,"THEAD",{});var cd=o(Tl);q=s(cd,"TR",{});var V=o(q);It=s(V,"TH",{align:!0});var pd=o(It);ri=i(pd,"model"),pd.forEach(a),ci=c(V),Ot=s(V,"TH",{align:!0});var dd=o(Ot);pi=i(dd,"overall_f1"),dd.forEach(a),di=c(V),Nt=s(V,"TH",{align:!0});var hd=o(Nt);hi=i(hd,"overall_accuracy"),hd.forEach(a),ui=c(V),St=s(V,"TH",{align:!0});var ud=o(St);fi=i(ud,"total_time_in_seconds"),ud.forEach(a),mi=c(V),Lt=s(V,"TH",{align:!0});var fd=o(Lt);vi=i(fd,"samples_per_second"),fd.forEach(a),_i=c(V),Ht=s(V,"TH",{align:!0});var md=o(Ht);gi=i(md,"latency_in_seconds"),md.forEach(a),V.forEach(a),cd.forEach(a),bi=c(lo),k=s(lo,"TBODY",{});var N=o(k);j=s(N,"TR",{});var M=o(j);Bt=s(M,"TD",{align:!0});var vd=o(Bt);Ei=i(vd,"Jorgeutd/albert-base-v2-finetuned-ner"),vd.forEach(a),wi=c(M),Wt=s(M,"TD",{align:!0});var _d=o(Wt);ki=i(_d,"0.941"),_d.forEach(a),yi=c(M),Gt=s(M,"TD",{align:!0});var gd=o(Gt);$i=i(gd,"0.989"),gd.forEach(a),qi=c(M),Rt=s(M,"TD",{align:!0});var bd=o(Rt);ji=i(bd,"4.515"),bd.forEach(a),Ti=c(M),Ut=s(M,"TD",{align:!0});var Ed=o(Ut);xi=i(Ed,"221.468"),Ed.forEach(a),Di=c(M),zt=s(M,"TD",{align:!0});var wd=o(zt);Pi=i(wd,"0.005"),wd.forEach(a),M.forEach(a),Ci=c(N),T=s(N,"TR",{});var F=o(T);Vt=s(F,"TD",{align:!0});var kd=o(Vt);Ai=i(kd,"dbmdz/bert-large-cased-finetuned-conll03-english"),kd.forEach(a),Ii=c(F),Mt=s(F,"TD",{align:!0});var yd=o(Mt);Oi=i(yd,"0.962"),yd.forEach(a),Ni=c(F),Ft=s(F,"TD",{align:!0});var $d=o(Ft);Si=i($d,"0.881"),$d.forEach(a),Li=c(F),Qt=s(F,"TD",{align:!0});var qd=o(Qt);Hi=i(qd,"11.648"),qd.forEach(a),Bi=c(F),Jt=s(F,"TD",{align:!0});var jd=o(Jt);Wi=i(jd,"85.850"),jd.forEach(a),Gi=c(F),Yt=s(F,"TD",{align:!0});var Td=o(Yt);Ri=i(Td,"0.012"),Td.forEach(a),F.forEach(a),Ui=c(N),x=s(N,"TR",{});var Q=o(x);Kt=s(Q,"TD",{align:!0});var xd=o(Kt);zi=i(xd,"dbmdz/electra-large-discriminator-finetuned-conll03-english"),xd.forEach(a),Vi=c(Q),Xt=s(Q,"TD",{align:!0});var Dd=o(Xt);Mi=i(Dd,"0.965"),Dd.forEach(a),Fi=c(Q),Zt=s(Q,"TD",{align:!0});var Pd=o(Zt);Qi=i(Pd,"0.881"),Pd.forEach(a),Ji=c(Q),ea=s(Q,"TD",{align:!0});var Cd=o(ea);Yi=i(Cd,"11.456"),Cd.forEach(a),Ki=c(Q),ta=s(Q,"TD",{align:!0});var Ad=o(ta);Xi=i(Ad,"87.292"),Ad.forEach(a),Zi=c(Q),aa=s(Q,"TD",{align:!0});var Id=o(aa);er=i(Id,"0.011"),Id.forEach(a),Q.forEach(a),tr=c(N),D=s(N,"TR",{});var J=o(D);la=s(J,"TD",{align:!0});var Od=o(la);ar=i(Od,"elastic/distilbert-base-uncased-finetuned-conll03-english"),Od.forEach(a),lr=c(J),sa=s(J,"TD",{align:!0});var Nd=o(sa);sr=i(Nd,"0.940"),Nd.forEach(a),or=c(J),oa=s(J,"TD",{align:!0});var Sd=o(oa);nr=i(Sd,"0.989"),Sd.forEach(a),ir=c(J),na=s(J,"TD",{align:!0});var Ld=o(na);rr=i(Ld,"2.318"),Ld.forEach(a),cr=c(J),ia=s(J,"TD",{align:!0});var Hd=o(ia);pr=i(Hd,"431.378"),Hd.forEach(a),dr=c(J),ra=s(J,"TD",{align:!0});var Bd=o(ra);hr=i(Bd,"0.002"),Bd.forEach(a),J.forEach(a),ur=c(N),P=s(N,"TR",{});var Y=o(P);ca=s(Y,"TD",{align:!0});var Wd=o(ca);fr=i(Wd,"gunghio/distilbert-base-multilingual-cased-finetuned-conll2003-ner"),Wd.forEach(a),mr=c(Y),pa=s(Y,"TD",{align:!0});var Gd=o(pa);vr=i(Gd,"0.947"),Gd.forEach(a),_r=c(Y),da=s(Y,"TD",{align:!0});var Rd=o(da);gr=i(Rd,"0.991"),Rd.forEach(a),br=c(Y),ha=s(Y,"TD",{align:!0});var Ud=o(ha);Er=i(Ud,"2.376"),Ud.forEach(a),wr=c(Y),ua=s(Y,"TD",{align:!0});var zd=o(ua);kr=i(zd,"420.873"),zd.forEach(a),yr=c(Y),fa=s(Y,"TD",{align:!0});var Vd=o(fa);$r=i(Vd,"0.002"),Vd.forEach(a),Y.forEach(a),qr=c(N),C=s(N,"TR",{});var K=o(C);ma=s(K,"TD",{align:!0});var Md=o(ma);jr=i(Md,"philschmid/distilroberta-base-ner-conll2003"),Md.forEach(a),Tr=c(K),va=s(K,"TD",{align:!0});var Fd=o(va);xr=i(Fd,"0.961"),Fd.forEach(a),Dr=c(K),_a=s(K,"TD",{align:!0});var Qd=o(_a);Pr=i(Qd,"0.994"),Qd.forEach(a),Cr=c(K),ga=s(K,"TD",{align:!0});var Jd=o(ga);Ar=i(Jd,"2.436"),Jd.forEach(a),Ir=c(K),ba=s(K,"TD",{align:!0});var Yd=o(ba);Or=i(Yd,"410.579"),Yd.forEach(a),Nr=c(K),Ea=s(K,"TD",{align:!0});var Kd=o(Ea);Sr=i(Kd,"0.002"),Kd.forEach(a),K.forEach(a),Lr=c(N),A=s(N,"TR",{});var X=o(A);wa=s(X,"TD",{align:!0});var Xd=o(wa);Hr=i(Xd,"xlm-roberta-large-finetuned-conll03-english"),Xd.forEach(a),Br=c(X),ka=s(X,"TD",{align:!0});var Zd=o(ka);Wr=i(Zd,"0.969"),Zd.forEach(a),Gr=c(X),ya=s(X,"TD",{align:!0});var eh=o(ya);Rr=i(eh,"0.882"),eh.forEach(a),Ur=c(X),$a=s(X,"TD",{align:!0});var th=o($a);zr=i(th,"11.996"),th.forEach(a),Vr=c(X),qa=s(X,"TD",{align:!0});var ah=o(qa);Mr=i(ah,"83.359"),ah.forEach(a),Fr=c(X),ja=s(X,"TD",{align:!0});var lh=o(ja);Qr=i(lh,"0.012"),lh.forEach(a),X.forEach(a),N.forEach(a),lo.forEach(a),Ps=c(t),de=s(t,"H2",{class:!0});var so=o(de);Ce=s(so,"A",{id:!0,class:!0,href:!0});var sh=o(Ce);xl=s(sh,"SPAN",{});var oh=o(xl);v(et.$$.fragment,oh),oh.forEach(a),sh.forEach(a),Jr=c(so),Dl=s(so,"SPAN",{});var nh=o(Dl);Yr=i(nh,"Question Answering"),nh.forEach(a),so.forEach(a),Cs=c(t),Ta=s(t,"P",{});var ih=o(Ta);Kr=i(ih,"With the question-answering evaluator one can evaluate models for QA without needing to worry about the complicated pre- and post-processing that\u2019s required for these models. It has the following specific arguments:"),ih.forEach(a),As=c(t),O=s(t,"UL",{});var ae=o(O);xa=s(ae,"LI",{});var Qc=o(xa);Pl=s(Qc,"CODE",{});var rh=o(Pl);Xr=i(rh,'question_column="question"'),rh.forEach(a),Zr=i(Qc,": the name of the column containing the question in the dataset"),Qc.forEach(a),ec=c(ae),Da=s(ae,"LI",{});var Jc=o(Da);Cl=s(Jc,"CODE",{});var ch=o(Cl);tc=i(ch,'context_column="context"'),ch.forEach(a),ac=i(Jc,": the name of the column containing the context"),Jc.forEach(a),lc=c(ae),Pa=s(ae,"LI",{});var Yc=o(Pa);Al=s(Yc,"CODE",{});var ph=o(Al);sc=i(ph,'id_column="id"'),ph.forEach(a),oc=i(Yc,": the name of the column cointaing the identification field of the question and answer pair"),Yc.forEach(a),nc=c(ae),Ca=s(ae,"LI",{});var Kc=o(Ca);Il=s(Kc,"CODE",{});var dh=o(Il);ic=i(dh,'label_column="answers"'),dh.forEach(a),rc=i(Kc,": the name of the column containing the answers"),Kc.forEach(a),cc=c(ae),Aa=s(ae,"LI",{});var Xc=o(Aa);Ol=s(Xc,"CODE",{});var hh=o(Ol);pc=i(hh,"squad_v2_format=None"),hh.forEach(a),dc=i(Xc,": whether the dataset follows the format of squad_v2 dataset where a question may have no answer in the context. If this parameter is not provided, the format will be automatically inferred."),Xc.forEach(a),ae.forEach(a),Is=c(t),Ia=s(t,"P",{});var uh=o(Ia);hc=i(uh,"Let\u2019s have a look how we can evaluate QA models and compute confidence intervals at the same time."),uh.forEach(a),Os=c(t),he=s(t,"H3",{class:!0});var oo=o(he);Ae=s(oo,"A",{id:!0,class:!0,href:!0});var fh=o(Ae);Nl=s(fh,"SPAN",{});var mh=o(Nl);v(tt.$$.fragment,mh),mh.forEach(a),fh.forEach(a),uc=c(oo),Sl=s(oo,"SPAN",{});var vh=o(Sl);fc=i(vh,"Confidence intervals"),vh.forEach(a),oo.forEach(a),Ns=c(t),z=s(t,"P",{});var We=o(z);mc=i(We,"Every evaluator comes with the options to compute confidence intervals using "),at=s(We,"A",{href:!0,rel:!0});var _h=o(at);vc=i(_h,"bootstrapping"),_h.forEach(a),_c=i(We,". Simply pass "),Ll=s(We,"CODE",{});var gh=o(Ll);gc=i(gh,'strategy="bootstrap"'),gh.forEach(a),bc=i(We," and set the number of resanmples with "),Hl=s(We,"CODE",{});var bh=o(Hl);Ec=i(bh,"n_resamples"),bh.forEach(a),wc=i(We,"."),We.forEach(a),Ss=c(t),v(lt.$$.fragment,t),Ls=c(t),Oa=s(t,"P",{});var Eh=o(Oa);kc=i(Eh,"Results include confidence intervals as well as error estimates as follows:"),Eh.forEach(a),Hs=c(t),v(st.$$.fragment,t),Bs=c(t),ue=s(t,"H2",{class:!0});var no=o(ue);Ie=s(no,"A",{id:!0,class:!0,href:!0});var wh=o(Ie);Bl=s(wh,"SPAN",{});var kh=o(Bl);v(ot.$$.fragment,kh),kh.forEach(a),wh.forEach(a),yc=c(no),Wl=s(no,"SPAN",{});var yh=o(Wl);$c=i(yh,"Image classification"),yh.forEach(a),no.forEach(a),Ws=c(t),Na=s(t,"P",{});var $h=o(Na);qc=i($h,"With the image classification evaluator we can evaluate any image classifier. It uses the same keyword arguments at the text classifier:"),$h.forEach(a),Gs=c(t),te=s(t,"UL",{});var Ga=o(te);Sa=s(Ga,"LI",{});var Zc=o(Sa);Gl=s(Zc,"CODE",{});var qh=o(Gl);jc=i(qh,'input_column="image"'),qh.forEach(a),Tc=i(Zc,": the name of the column containing the images as PIL ImageFile"),Zc.forEach(a),xc=c(Ga),La=s(Ga,"LI",{});var ep=o(La);Rl=s(ep,"CODE",{});var jh=o(Rl);Dc=i(jh,'label_column="label"'),jh.forEach(a),Pc=i(ep,": the name of the column containing the labels"),ep.forEach(a),Cc=c(Ga),nt=s(Ga,"LI",{});var io=o(nt);Ul=s(io,"CODE",{});var Th=o(Ul);Ac=i(Th,"label_mapping=None"),Th.forEach(a),Ic=i(io,": We want to map class labels defined by the model in the pipeline to values consistent with those defined in the "),zl=s(io,"CODE",{});var xh=o(zl);Oc=i(xh,"label_column"),xh.forEach(a),io.forEach(a),Ga.forEach(a),Rs=c(t),Ha=s(t,"P",{});var Dh=o(Ha);Nc=i(Dh,"Let\u2019s have a look at how can evaluate image classification models on large datasets."),Dh.forEach(a),Us=c(t),fe=s(t,"H3",{class:!0});var ro=o(fe);Oe=s(ro,"A",{id:!0,class:!0,href:!0});var Ph=o(Oe);Vl=s(Ph,"SPAN",{});var Ch=o(Vl);v(it.$$.fragment,Ch),Ch.forEach(a),Ph.forEach(a),Sc=c(ro),Ml=s(ro,"SPAN",{});var Ah=o(Ml);Lc=i(Ah,"Handling large datasets"),Ah.forEach(a),ro.forEach(a),zs=c(t),Ba=s(t,"P",{});var Ih=o(Ba);Hc=i(Ih,"The evaluator can be used on large datasets! Below, an example shows how to use it on ImageNet-1k for image classification. Beware that this example will require to download ~150 GB."),Ih.forEach(a),Vs=c(t),v(rt.$$.fragment,t),Ms=c(t),Ne=s(t,"P",{});var co=o(Ne);Bc=i(co,"Since we are using "),Fl=s(co,"CODE",{});var Oh=o(Fl);Wc=i(Oh,"datasets"),Oh.forEach(a),Gc=i(co," to store data we make use of a technique called memory mappings. This means that the dataset is never fully loaded into memory which saves a lot of RAM. Running the above code only uses roughly 1.5 GB of RAM while the validation split is more than 30 GB big."),co.forEach(a),this.h()},h(){d(u,"name","hf:doc:metadata"),d(u,"content",JSON.stringify(zh)),d(w,"id","using-the-evaluator"),d(w,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),d(w,"href","#using-the-evaluator"),d(f,"class","relative group"),d(me,"href","custom_evaluator"),d(ut,"href","/docs/evaluate/pr_290/en/package_reference/evaluator_classes#evaluate.TextClassificationEvaluator"),d(ft,"href","/docs/evaluate/pr_290/en/package_reference/evaluator_classes#evaluate.TokenClassificationEvaluator"),d(mt,"href","/docs/evaluate/pr_290/en/package_reference/evaluator_classes#evaluate.QuestionAnsweringEvaluator"),d(vt,"href","/docs/evaluate/pr_290/en/package_reference/evaluator_classes#evaluate.ImageClassificationEvaluator"),d(Ee,"id","text-classification"),d(Ee,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),d(Ee,"href","#text-classification"),d(ne,"class","relative group"),d(ke,"id","evaluate-models-on-the-hub"),d(ke,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),d(ke,"href","#evaluate-models-on-the-hub"),d(ie,"class","relative group"),d(qe,"id","evaluate-multiple-metrics"),d(qe,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),d(qe,"href","#evaluate-multiple-metrics"),d(re,"class","relative group"),d($t,"href","/docs/evaluate/pr_290/en/package_reference/main_classes#evaluate.combine"),d(Te,"id","token-classification"),d(Te,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),d(Te,"href","#token-classification"),d(ce,"class","relative group"),d(xe,"id","benchmarking-several-models"),d(xe,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),d(xe,"href","#benchmarking-several-models"),d(pe,"class","relative group"),d(It,"align","left"),d(Ot,"align","right"),d(Nt,"align","right"),d(St,"align","right"),d(Lt,"align","right"),d(Ht,"align","right"),d(Bt,"align","left"),d(Wt,"align","right"),d(Gt,"align","right"),d(Rt,"align","right"),d(Ut,"align","right"),d(zt,"align","right"),d(Vt,"align","left"),d(Mt,"align","right"),d(Ft,"align","right"),d(Qt,"align","right"),d(Jt,"align","right"),d(Yt,"align","right"),d(Kt,"align","left"),d(Xt,"align","right"),d(Zt,"align","right"),d(ea,"align","right"),d(ta,"align","right"),d(aa,"align","right"),d(la,"align","left"),d(sa,"align","right"),d(oa,"align","right"),d(na,"align","right"),d(ia,"align","right"),d(ra,"align","right"),d(ca,"align","left"),d(pa,"align","right"),d(da,"align","right"),d(ha,"align","right"),d(ua,"align","right"),d(fa,"align","right"),d(ma,"align","left"),d(va,"align","right"),d(_a,"align","right"),d(ga,"align","right"),d(ba,"align","right"),d(Ea,"align","right"),d(wa,"align","left"),d(ka,"align","right"),d(ya,"align","right"),d($a,"align","right"),d(qa,"align","right"),d(ja,"align","right"),d(Ce,"id","question-answering"),d(Ce,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),d(Ce,"href","#question-answering"),d(de,"class","relative group"),d(Ae,"id","confidence-intervals"),d(Ae,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),d(Ae,"href","#confidence-intervals"),d(he,"class","relative group"),d(at,"href","https://docs.scipy.org/doc/scipy/reference/generated/scipy.stats.bootstrap.html"),d(at,"rel","nofollow"),d(Ie,"id","image-classification"),d(Ie,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),d(Ie,"href","#image-classification"),d(ue,"class","relative group"),d(Oe,"id","handling-large-datasets"),d(Oe,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),d(Oe,"href","#handling-large-datasets"),d(fe,"class","relative group")},m(t,p){e(document.head,u),h(t,B,p),h(t,f,p),e(f,w),e(w,se),_(S,se,null),e(f,Re),e(f,Z),e(Z,oe),e(Z,I),e(I,dt),h(t,Ue,p),h(t,W,p),e(W,po),e(W,Ra),e(Ra,ho),e(W,uo),e(W,Ua),e(Ua,fo),e(W,mo),e(W,me),e(me,vo),e(me,za),e(za,_o),e(me,go),e(W,bo),h(t,Zl,p),h(t,ht,p),e(ht,Eo),h(t,es,p),h(t,G,p),e(G,ve),e(ve,Va),e(Va,wo),e(ve,ko),e(ve,ut),e(ut,yo),e(ve,$o),e(G,qo),e(G,_e),e(_e,Ma),e(Ma,jo),e(_e,To),e(_e,ft),e(ft,xo),e(_e,Do),e(G,Po),e(G,ge),e(ge,Fa),e(Fa,Co),e(ge,Ao),e(ge,mt),e(mt,Io),e(ge,Oo),e(G,No),e(G,be),e(be,Qa),e(Qa,So),e(be,Lo),e(be,vt),e(vt,Ho),e(be,Bo),h(t,ts,p),h(t,_t,p),e(_t,Wo),h(t,as,p),h(t,ne,p),e(ne,Ee),e(Ee,Ja),_(ze,Ja,null),e(ne,Go),e(ne,Ya),e(Ya,Ro),h(t,ls,p),h(t,gt,p),e(gt,Uo),h(t,ss,p),h(t,ee,p),e(ee,bt),e(bt,Ka),e(Ka,zo),e(bt,Vo),e(ee,Mo),e(ee,Et),e(Et,Xa),e(Xa,Fo),e(Et,Qo),e(ee,Jo),e(ee,y),e(y,Za),e(Za,Yo),e(y,Ko),e(y,el),e(el,Xo),e(y,Zo),e(y,tl),e(tl,en),e(y,tn),e(y,al),e(al,an),e(y,ln),e(y,ll),e(ll,sn),e(y,on),e(y,sl),e(sl,nn),e(y,rn),h(t,os,p),h(t,we,p),e(we,cn),e(we,ol),e(ol,pn),e(we,dn),h(t,ns,p),h(t,ie,p),e(ie,ke),e(ke,nl),_(Ve,nl,null),e(ie,hn),e(ie,il),e(il,un),h(t,is,p),h(t,R,p),e(R,fn),e(R,rl),e(rl,mn),e(R,vn),e(R,cl),e(cl,_n),e(R,gn),e(R,pl),e(pl,bn),e(R,En),h(t,rs,p),h(t,wt,p),e(wt,wn),h(t,cs,p),_(Me,t,p),h(t,ps,p),_(ye,t,p),h(t,ds,p),h(t,kt,p),e(kt,kn),h(t,hs,p),_(Fe,t,p),h(t,us,p),h(t,yt,p),e(yt,yn),h(t,fs,p),_($e,t,p),h(t,ms,p),h(t,re,p),e(re,qe),e(qe,dl),_(Qe,dl,null),e(re,$n),e(re,hl),e(hl,qn),h(t,vs,p),h(t,je,p),e(je,jn),e(je,$t),e($t,Tn),e(je,xn),h(t,_s,p),_(Je,t,p),h(t,gs,p),h(t,qt,p),e(qt,Dn),h(t,bs,p),_(Ye,t,p),h(t,Es,p),h(t,jt,p),e(jt,Pn),h(t,ws,p),h(t,ce,p),e(ce,Te),e(Te,ul),_(Ke,ul,null),e(ce,Cn),e(ce,fl),e(fl,An),h(t,ks,p),h(t,Tt,p),e(Tt,In),h(t,ys,p),h(t,U,p),e(U,xt),e(xt,ml),e(ml,On),e(xt,Nn),e(U,Sn),e(U,Dt),e(Dt,vl),e(vl,Ln),e(Dt,Hn),e(U,Bn),e(U,$),e($,_l),e(_l,Wn),e($,Gn),e($,gl),e(gl,Rn),e($,Un),e($,bl),e(bl,zn),e($,Vn),e($,El),e(El,Mn),e($,Fn),e($,wl),e(wl,Qn),e($,Jn),e($,kl),e(kl,Yn),e($,Kn),e(U,Xn),e(U,Pt),e(Pt,yl),e(yl,Zn),e(Pt,ei),h(t,$s,p),h(t,Ct,p),e(Ct,ti),h(t,qs,p),h(t,pe,p),e(pe,xe),e(xe,$l),_(Xe,$l,null),e(pe,ai),e(pe,ql),e(ql,li),h(t,js,p),h(t,De,p),e(De,si),e(De,jl),e(jl,oi),e(De,ni),h(t,Ts,p),_(Ze,t,p),h(t,xs,p),h(t,At,p),e(At,ii),h(t,Ds,p),h(t,Pe,p),e(Pe,Tl),e(Tl,q),e(q,It),e(It,ri),e(q,ci),e(q,Ot),e(Ot,pi),e(q,di),e(q,Nt),e(Nt,hi),e(q,ui),e(q,St),e(St,fi),e(q,mi),e(q,Lt),e(Lt,vi),e(q,_i),e(q,Ht),e(Ht,gi),e(Pe,bi),e(Pe,k),e(k,j),e(j,Bt),e(Bt,Ei),e(j,wi),e(j,Wt),e(Wt,ki),e(j,yi),e(j,Gt),e(Gt,$i),e(j,qi),e(j,Rt),e(Rt,ji),e(j,Ti),e(j,Ut),e(Ut,xi),e(j,Di),e(j,zt),e(zt,Pi),e(k,Ci),e(k,T),e(T,Vt),e(Vt,Ai),e(T,Ii),e(T,Mt),e(Mt,Oi),e(T,Ni),e(T,Ft),e(Ft,Si),e(T,Li),e(T,Qt),e(Qt,Hi),e(T,Bi),e(T,Jt),e(Jt,Wi),e(T,Gi),e(T,Yt),e(Yt,Ri),e(k,Ui),e(k,x),e(x,Kt),e(Kt,zi),e(x,Vi),e(x,Xt),e(Xt,Mi),e(x,Fi),e(x,Zt),e(Zt,Qi),e(x,Ji),e(x,ea),e(ea,Yi),e(x,Ki),e(x,ta),e(ta,Xi),e(x,Zi),e(x,aa),e(aa,er),e(k,tr),e(k,D),e(D,la),e(la,ar),e(D,lr),e(D,sa),e(sa,sr),e(D,or),e(D,oa),e(oa,nr),e(D,ir),e(D,na),e(na,rr),e(D,cr),e(D,ia),e(ia,pr),e(D,dr),e(D,ra),e(ra,hr),e(k,ur),e(k,P),e(P,ca),e(ca,fr),e(P,mr),e(P,pa),e(pa,vr),e(P,_r),e(P,da),e(da,gr),e(P,br),e(P,ha),e(ha,Er),e(P,wr),e(P,ua),e(ua,kr),e(P,yr),e(P,fa),e(fa,$r),e(k,qr),e(k,C),e(C,ma),e(ma,jr),e(C,Tr),e(C,va),e(va,xr),e(C,Dr),e(C,_a),e(_a,Pr),e(C,Cr),e(C,ga),e(ga,Ar),e(C,Ir),e(C,ba),e(ba,Or),e(C,Nr),e(C,Ea),e(Ea,Sr),e(k,Lr),e(k,A),e(A,wa),e(wa,Hr),e(A,Br),e(A,ka),e(ka,Wr),e(A,Gr),e(A,ya),e(ya,Rr),e(A,Ur),e(A,$a),e($a,zr),e(A,Vr),e(A,qa),e(qa,Mr),e(A,Fr),e(A,ja),e(ja,Qr),h(t,Ps,p),h(t,de,p),e(de,Ce),e(Ce,xl),_(et,xl,null),e(de,Jr),e(de,Dl),e(Dl,Yr),h(t,Cs,p),h(t,Ta,p),e(Ta,Kr),h(t,As,p),h(t,O,p),e(O,xa),e(xa,Pl),e(Pl,Xr),e(xa,Zr),e(O,ec),e(O,Da),e(Da,Cl),e(Cl,tc),e(Da,ac),e(O,lc),e(O,Pa),e(Pa,Al),e(Al,sc),e(Pa,oc),e(O,nc),e(O,Ca),e(Ca,Il),e(Il,ic),e(Ca,rc),e(O,cc),e(O,Aa),e(Aa,Ol),e(Ol,pc),e(Aa,dc),h(t,Is,p),h(t,Ia,p),e(Ia,hc),h(t,Os,p),h(t,he,p),e(he,Ae),e(Ae,Nl),_(tt,Nl,null),e(he,uc),e(he,Sl),e(Sl,fc),h(t,Ns,p),h(t,z,p),e(z,mc),e(z,at),e(at,vc),e(z,_c),e(z,Ll),e(Ll,gc),e(z,bc),e(z,Hl),e(Hl,Ec),e(z,wc),h(t,Ss,p),_(lt,t,p),h(t,Ls,p),h(t,Oa,p),e(Oa,kc),h(t,Hs,p),_(st,t,p),h(t,Bs,p),h(t,ue,p),e(ue,Ie),e(Ie,Bl),_(ot,Bl,null),e(ue,yc),e(ue,Wl),e(Wl,$c),h(t,Ws,p),h(t,Na,p),e(Na,qc),h(t,Gs,p),h(t,te,p),e(te,Sa),e(Sa,Gl),e(Gl,jc),e(Sa,Tc),e(te,xc),e(te,La),e(La,Rl),e(Rl,Dc),e(La,Pc),e(te,Cc),e(te,nt),e(nt,Ul),e(Ul,Ac),e(nt,Ic),e(nt,zl),e(zl,Oc),h(t,Rs,p),h(t,Ha,p),e(Ha,Nc),h(t,Us,p),h(t,fe,p),e(fe,Oe),e(Oe,Vl),_(it,Vl,null),e(fe,Sc),e(fe,Ml),e(Ml,Lc),h(t,zs,p),h(t,Ba,p),e(Ba,Hc),h(t,Vs,p),_(rt,t,p),h(t,Ms,p),h(t,Ne,p),e(Ne,Bc),e(Ne,Fl),e(Fl,Wc),e(Ne,Gc),Fs=!0},p(t,[p]){const ct={};p&2&&(ct.$$scope={dirty:p,ctx:t}),ye.$set(ct);const Ql={};p&2&&(Ql.$$scope={dirty:p,ctx:t}),$e.$set(Ql)},i(t){Fs||(g(S.$$.fragment,t),g(ze.$$.fragment,t),g(Ve.$$.fragment,t),g(Me.$$.fragment,t),g(ye.$$.fragment,t),g(Fe.$$.fragment,t),g($e.$$.fragment,t),g(Qe.$$.fragment,t),g(Je.$$.fragment,t),g(Ye.$$.fragment,t),g(Ke.$$.fragment,t),g(Xe.$$.fragment,t),g(Ze.$$.fragment,t),g(et.$$.fragment,t),g(tt.$$.fragment,t),g(lt.$$.fragment,t),g(st.$$.fragment,t),g(ot.$$.fragment,t),g(it.$$.fragment,t),g(rt.$$.fragment,t),Fs=!0)},o(t){b(S.$$.fragment,t),b(ze.$$.fragment,t),b(Ve.$$.fragment,t),b(Me.$$.fragment,t),b(ye.$$.fragment,t),b(Fe.$$.fragment,t),b($e.$$.fragment,t),b(Qe.$$.fragment,t),b(Je.$$.fragment,t),b(Ye.$$.fragment,t),b(Ke.$$.fragment,t),b(Xe.$$.fragment,t),b(Ze.$$.fragment,t),b(et.$$.fragment,t),b(tt.$$.fragment,t),b(lt.$$.fragment,t),b(st.$$.fragment,t),b(ot.$$.fragment,t),b(it.$$.fragment,t),b(rt.$$.fragment,t),Fs=!1},d(t){a(u),t&&a(B),t&&a(f),E(S),t&&a(Ue),t&&a(W),t&&a(Zl),t&&a(ht),t&&a(es),t&&a(G),t&&a(ts),t&&a(_t),t&&a(as),t&&a(ne),E(ze),t&&a(ls),t&&a(gt),t&&a(ss),t&&a(ee),t&&a(os),t&&a(we),t&&a(ns),t&&a(ie),E(Ve),t&&a(is),t&&a(R),t&&a(rs),t&&a(wt),t&&a(cs),E(Me,t),t&&a(ps),E(ye,t),t&&a(ds),t&&a(kt),t&&a(hs),E(Fe,t),t&&a(us),t&&a(yt),t&&a(fs),E($e,t),t&&a(ms),t&&a(re),E(Qe),t&&a(vs),t&&a(je),t&&a(_s),E(Je,t),t&&a(gs),t&&a(qt),t&&a(bs),E(Ye,t),t&&a(Es),t&&a(jt),t&&a(ws),t&&a(ce),E(Ke),t&&a(ks),t&&a(Tt),t&&a(ys),t&&a(U),t&&a($s),t&&a(Ct),t&&a(qs),t&&a(pe),E(Xe),t&&a(js),t&&a(De),t&&a(Ts),E(Ze,t),t&&a(xs),t&&a(At),t&&a(Ds),t&&a(Pe),t&&a(Ps),t&&a(de),E(et),t&&a(Cs),t&&a(Ta),t&&a(As),t&&a(O),t&&a(Is),t&&a(Ia),t&&a(Os),t&&a(he),E(tt),t&&a(Ns),t&&a(z),t&&a(Ss),E(lt,t),t&&a(Ls),t&&a(Oa),t&&a(Hs),E(st,t),t&&a(Bs),t&&a(ue),E(ot),t&&a(Ws),t&&a(Na),t&&a(Gs),t&&a(te),t&&a(Rs),t&&a(Ha),t&&a(Us),t&&a(fe),E(it),t&&a(zs),t&&a(Ba),t&&a(Vs),E(rt,t),t&&a(Ms),t&&a(Ne)}}}const zh={local:"using-the-evaluator",sections:[{local:"text-classification",sections:[{local:"evaluate-models-on-the-hub",title:"Evaluate models on the Hub"},{local:"evaluate-multiple-metrics",title:"Evaluate multiple metrics"}],title:"Text classification"},{local:"token-classification",sections:[{local:"benchmarking-several-models",title:"Benchmarking several models"}],title:"Token Classification"},{local:"question-answering",sections:[{local:"confidence-intervals",title:"Confidence intervals"}],title:"Question Answering"},{local:"image-classification",sections:[{local:"handling-large-datasets",title:"Handling large datasets"}],title:"Image classification"}],title:"Using the `evaluator`"};function Vh(pt){return Wh(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class Yh extends Sh{constructor(u){super();Lh(this,u,Vh,Uh,Hh,{})}}export{Yh as default,zh as metadata};
