import{S as mu,i as vu,s as _u,e as l,k as r,w as m,t as n,M as gu,c as s,d as a,m as c,a as o,x as v,h as i,b as d,G as e,g as h,y as _,q as g,o as E,B as b,v as Eu}from"../chunks/vendor-hf-doc-builder.js";import{T as fu}from"../chunks/Tip-hf-doc-builder.js";import{I as se}from"../chunks/IconCopyLink-hf-doc-builder.js";import{C as Ue}from"../chunks/CodeBlock-hf-doc-builder.js";function bu(ut){let u,G,f,k,oe,H,Ve,ee;return{c(){u=l("p"),G=n("Without specifying a device, the default for model inference will be the first GPU on the machine if one is available, and else CPU. If you want to use a specific device you can pass "),f=l("code"),k=n("device"),oe=n(" to "),H=l("code"),Ve=n("compute"),ee=n(" where -1 will use the GPU and a positive integer (starting with 0) will use the associated CUDA device.")},l(ne){u=s(ne,"P",{});var O=o(u);G=i(O,"Without specifying a device, the default for model inference will be the first GPU on the machine if one is available, and else CPU. If you want to use a specific device you can pass "),f=s(O,"CODE",{});var ft=o(f);k=i(ft,"device"),ft.forEach(a),oe=i(O," to "),H=s(O,"CODE",{});var Me=o(H);Ve=i(Me,"compute"),Me.forEach(a),ee=i(O," where -1 will use the GPU and a positive integer (starting with 0) will use the associated CUDA device."),O.forEach(a)},m(ne,O){h(ne,u,O),e(u,G),e(u,f),e(f,k),e(u,oe),e(u,H),e(H,Ve),e(u,ee)},d(ne){ne&&a(u)}}}function wu(ut){let u,G;return{c(){u=l("p"),G=n("The time performances can give useful indication on model speed for inference but should be taken with a grain of salt: they include all the processing that goes on in the pipeline. This may include tokenizing, post-processing, that may be different depending on the model. Furthermore, it depends a lot on the hardware you are running the evaluation on and you may be able to improve the performance by optimizing things like the batch size.")},l(f){u=s(f,"P",{});var k=o(u);G=i(k,"The time performances can give useful indication on model speed for inference but should be taken with a grain of salt: they include all the processing that goes on in the pipeline. This may include tokenizing, post-processing, that may be different depending on the model. Furthermore, it depends a lot on the hardware you are running the evaluation on and you may be able to improve the performance by optimizing things like the batch size."),k.forEach(a)},m(f,k){h(f,u,k),e(u,G)},d(f){f&&a(u)}}}function ku(ut){let u,G,f,k,oe,H,Ve,ee,ne,O,ft,Me,W,yo,Qa,$o,qo,Ja,jo,To,ve,xo,Ya,Do,Co,Po,ds,mt,Ao,hs,w,_e,Ka,Io,Oo,vt,So,No,Lo,ge,Xa,Ho,Bo,_t,zo,Go,Wo,Ee,Za,Ro,Uo,gt,Vo,Mo,Fo,be,el,Qo,Jo,Et,Yo,Ko,Xo,we,tl,Zo,en,bt,tn,an,ln,ke,al,sn,on,wt,nn,rn,cn,ye,ll,pn,dn,kt,hn,un,us,yt,fn,fs,ie,$e,sl,Fe,mn,ol,vn,ms,$t,_n,vs,te,qt,nl,gn,En,bn,jt,il,wn,kn,yn,$,rl,$n,qn,cl,jn,Tn,pl,xn,Dn,dl,Cn,Pn,hl,An,In,ul,On,Sn,_s,qe,Nn,fl,Ln,Hn,gs,re,je,ml,Qe,Bn,vl,zn,Es,R,Gn,_l,Wn,Rn,gl,Un,Vn,El,Mn,Fn,bs,Tt,Qn,ws,Je,ks,Te,ys,xt,Jn,$s,Ye,qs,Dt,Yn,js,xe,Ts,ce,De,bl,Ke,Kn,wl,Xn,xs,Ce,Zn,Ct,ei,ti,Ds,Xe,Cs,Pt,ai,Ps,Ze,As,At,li,Is,pe,Pe,kl,et,si,yl,oi,Os,It,ni,Ss,U,Ot,$l,ii,ri,ci,St,ql,pi,di,hi,q,jl,ui,fi,Tl,mi,vi,xl,_i,gi,Dl,Ei,bi,Cl,wi,ki,Pl,yi,$i,qi,Nt,Al,ji,Ti,Ns,Lt,xi,Ls,de,Ae,Il,tt,Di,Ol,Ci,Hs,Ie,Pi,Sl,Ai,Ii,Bs,at,zs,Ht,Oi,Gs,Oe,Nl,j,Bt,Si,Ni,zt,Li,Hi,Gt,Bi,zi,Wt,Gi,Wi,Rt,Ri,Ui,Ut,Vi,Mi,y,T,Vt,Fi,Qi,Mt,Ji,Yi,Ft,Ki,Xi,Qt,Zi,er,Jt,tr,ar,Yt,lr,sr,x,Kt,or,nr,Xt,ir,rr,Zt,cr,pr,ea,dr,hr,ta,ur,fr,aa,mr,vr,D,la,_r,gr,sa,Er,br,oa,wr,kr,na,yr,$r,ia,qr,jr,ra,Tr,xr,C,ca,Dr,Cr,pa,Pr,Ar,da,Ir,Or,ha,Sr,Nr,ua,Lr,Hr,fa,Br,zr,P,ma,Gr,Wr,va,Rr,Ur,_a,Vr,Mr,ga,Fr,Qr,Ea,Jr,Yr,ba,Kr,Xr,A,wa,Zr,ec,ka,tc,ac,ya,lc,sc,$a,oc,nc,qa,ic,rc,ja,cc,pc,I,Ta,dc,hc,xa,uc,fc,Da,mc,vc,Ca,_c,gc,Pa,Ec,bc,Aa,wc,Ws,he,Se,Ll,lt,kc,Hl,yc,Rs,Ia,$c,Us,S,Oa,Bl,qc,jc,Tc,Sa,zl,xc,Dc,Cc,Na,Gl,Pc,Ac,Ic,La,Wl,Oc,Sc,Nc,Ha,Rl,Lc,Hc,Vs,Ba,Bc,Ms,ue,Ne,Ul,st,zc,Vl,Gc,Fs,V,Wc,ot,Rc,Uc,Ml,Vc,Mc,Fl,Fc,Qc,Qs,nt,Js,za,Jc,Ys,it,Ks,fe,Le,Ql,rt,Yc,Jl,Kc,Xs,Ga,Xc,Zs,ae,Wa,Yl,Zc,ep,tp,Ra,Kl,ap,lp,sp,ct,Xl,op,np,Zl,ip,eo,Ua,rp,to,me,He,es,pt,cp,ts,pp,ao,Va,dp,lo,dt,so,Be,hp,as,up,fp,oo;return H=new se({}),Fe=new se({}),Qe=new se({}),Je=new Ue({props:{code:`from datasets import load_dataset
from evaluate import evaluator
from transformers import AutoModelForSequenceClassification, pipeline

data = load_dataset("imdb", split="test").shuffle(seed=42).select(range(1000))
task_evaluator = evaluator("text-classification")

# 1. Pass a model name or path
eval_results = task_evaluator.compute(
    model_or_pipeline="lvwerra/distilbert-imdb",
    data=data,
    label_mapping={"NEGATIVE": 0, "POSITIVE": 1}
)

# 2. Pass an instantiated model
model = AutoModelForSequenceClassification.from_pretrained("lvwerra/distilbert-imdb")

eval_results = task_evaluator.compute(
    model_or_pipeline=model,
    data=data,
    label_mapping={"NEGATIVE": 0, "POSITIVE": 1}
)

# 3. Pass an instantiated pipeline 
pipe = pipeline("text-classification", model="lvwerra/distilbert-imdb")

eval_results = task_evaluator.compute(
    model_or_pipeline=pipe,
    data=data,
    label_mapping={"NEGATIVE": 0, "POSITIVE": 1}
)
print(eval_results)`,highlighted:`<span class="hljs-keyword">from</span> datasets <span class="hljs-keyword">import</span> load_dataset
<span class="hljs-keyword">from</span> evaluate <span class="hljs-keyword">import</span> evaluator
<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoModelForSequenceClassification, pipeline

data = load_dataset(<span class="hljs-string">&quot;imdb&quot;</span>, split=<span class="hljs-string">&quot;test&quot;</span>).shuffle(seed=<span class="hljs-number">42</span>).select(<span class="hljs-built_in">range</span>(<span class="hljs-number">1000</span>))
task_evaluator = evaluator(<span class="hljs-string">&quot;text-classification&quot;</span>)

<span class="hljs-comment"># 1. Pass a model name or path</span>
eval_results = task_evaluator.compute(
    model_or_pipeline=<span class="hljs-string">&quot;lvwerra/distilbert-imdb&quot;</span>,
    data=data,
    label_mapping={<span class="hljs-string">&quot;NEGATIVE&quot;</span>: <span class="hljs-number">0</span>, <span class="hljs-string">&quot;POSITIVE&quot;</span>: <span class="hljs-number">1</span>}
)

<span class="hljs-comment"># 2. Pass an instantiated model</span>
model = AutoModelForSequenceClassification.from_pretrained(<span class="hljs-string">&quot;lvwerra/distilbert-imdb&quot;</span>)

eval_results = task_evaluator.compute(
    model_or_pipeline=model,
    data=data,
    label_mapping={<span class="hljs-string">&quot;NEGATIVE&quot;</span>: <span class="hljs-number">0</span>, <span class="hljs-string">&quot;POSITIVE&quot;</span>: <span class="hljs-number">1</span>}
)

<span class="hljs-comment"># 3. Pass an instantiated pipeline </span>
pipe = pipeline(<span class="hljs-string">&quot;text-classification&quot;</span>, model=<span class="hljs-string">&quot;lvwerra/distilbert-imdb&quot;</span>)

eval_results = task_evaluator.compute(
    model_or_pipeline=pipe,
    data=data,
    label_mapping={<span class="hljs-string">&quot;NEGATIVE&quot;</span>: <span class="hljs-number">0</span>, <span class="hljs-string">&quot;POSITIVE&quot;</span>: <span class="hljs-number">1</span>}
)
<span class="hljs-built_in">print</span>(eval_results)`}}),Te=new fu({props:{$$slots:{default:[bu]},$$scope:{ctx:ut}}}),Ye=new Ue({props:{code:`{
    'accuracy': 0.918,
    'latency_in_seconds': 0.013,
    'samples_per_second': 78.887,
    'total_time_in_seconds': 12.676
}`,highlighted:`{
    <span class="hljs-string">&#x27;accuracy&#x27;</span>: <span class="hljs-number">0.918</span>,
    <span class="hljs-string">&#x27;latency_in_seconds&#x27;</span>: <span class="hljs-number">0.013</span>,
    <span class="hljs-string">&#x27;samples_per_second&#x27;</span>: <span class="hljs-number">78.887</span>,
    <span class="hljs-string">&#x27;total_time_in_seconds&#x27;</span>: <span class="hljs-number">12.676</span>
}`}}),xe=new fu({props:{$$slots:{default:[wu]},$$scope:{ctx:ut}}}),Ke=new se({}),Xe=new Ue({props:{code:`import evaluate

eval_results = task_evaluator.compute(
    model_or_pipeline="lvwerra/distilbert-imdb",
    data=data,
    metric=evaluate.combine(["accuracy", "recall", "precision", "f1"]),
    label_mapping={"NEGATIVE": 0, "POSITIVE": 1}
)
print(eval_results)
`,highlighted:`<span class="hljs-keyword">import</span> evaluate

eval_results = task_evaluator.compute(
    model_or_pipeline=<span class="hljs-string">&quot;lvwerra/distilbert-imdb&quot;</span>,
    data=data,
    metric=evaluate.combine([<span class="hljs-string">&quot;accuracy&quot;</span>, <span class="hljs-string">&quot;recall&quot;</span>, <span class="hljs-string">&quot;precision&quot;</span>, <span class="hljs-string">&quot;f1&quot;</span>]),
    label_mapping={<span class="hljs-string">&quot;NEGATIVE&quot;</span>: <span class="hljs-number">0</span>, <span class="hljs-string">&quot;POSITIVE&quot;</span>: <span class="hljs-number">1</span>}
)
<span class="hljs-built_in">print</span>(eval_results)
`}}),Ze=new Ue({props:{code:`{
    'accuracy': 0.918,
    'f1': 0.916,
    'precision': 0.9147,
    'recall': 0.9187,
    'latency_in_seconds': 0.013,
    'samples_per_second': 78.887,
    'total_time_in_seconds': 12.676
}`,highlighted:`{
    <span class="hljs-string">&#x27;accuracy&#x27;</span>: <span class="hljs-number">0.918</span>,
    <span class="hljs-string">&#x27;f1&#x27;</span>: <span class="hljs-number">0.916</span>,
    <span class="hljs-string">&#x27;precision&#x27;</span>: <span class="hljs-number">0.9147</span>,
    <span class="hljs-string">&#x27;recall&#x27;</span>: <span class="hljs-number">0.9187</span>,
    <span class="hljs-string">&#x27;latency_in_seconds&#x27;</span>: <span class="hljs-number">0.013</span>,
    <span class="hljs-string">&#x27;samples_per_second&#x27;</span>: <span class="hljs-number">78.887</span>,
    <span class="hljs-string">&#x27;total_time_in_seconds&#x27;</span>: <span class="hljs-number">12.676</span>
}`}}),et=new se({}),tt=new se({}),at=new Ue({props:{code:`import pandas as pd
from datasets import load_dataset
from evaluate import evaluator
from transformers import pipeline

models = [
    "xlm-roberta-large-finetuned-conll03-english",
    "dbmdz/bert-large-cased-finetuned-conll03-english",
    "elastic/distilbert-base-uncased-finetuned-conll03-english",
    "dbmdz/electra-large-discriminator-finetuned-conll03-english",
    "gunghio/distilbert-base-multilingual-cased-finetuned-conll2003-ner",
    "philschmid/distilroberta-base-ner-conll2003",
    "Jorgeutd/albert-base-v2-finetuned-ner",
]

data = load_dataset("conll2003", split="validation").shuffle().select(1000)
task_evaluator = evaluator("token-classification")

results = []
for model in models:
    results.append(
        task_evaluator.compute(
            model_or_pipeline=model, data=data, metric="seqeval"
            )
        )

df = pd.DataFrame(results, index=models)
df[["overall_f1", "overall_accuracy", "total_time_in_seconds", "samples_per_second", "latency_in_seconds"]]`,highlighted:`<span class="hljs-keyword">import</span> pandas <span class="hljs-keyword">as</span> pd
<span class="hljs-keyword">from</span> datasets <span class="hljs-keyword">import</span> load_dataset
<span class="hljs-keyword">from</span> evaluate <span class="hljs-keyword">import</span> evaluator
<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> pipeline

models = [
    <span class="hljs-string">&quot;xlm-roberta-large-finetuned-conll03-english&quot;</span>,
    <span class="hljs-string">&quot;dbmdz/bert-large-cased-finetuned-conll03-english&quot;</span>,
    <span class="hljs-string">&quot;elastic/distilbert-base-uncased-finetuned-conll03-english&quot;</span>,
    <span class="hljs-string">&quot;dbmdz/electra-large-discriminator-finetuned-conll03-english&quot;</span>,
    <span class="hljs-string">&quot;gunghio/distilbert-base-multilingual-cased-finetuned-conll2003-ner&quot;</span>,
    <span class="hljs-string">&quot;philschmid/distilroberta-base-ner-conll2003&quot;</span>,
    <span class="hljs-string">&quot;Jorgeutd/albert-base-v2-finetuned-ner&quot;</span>,
]

data = load_dataset(<span class="hljs-string">&quot;conll2003&quot;</span>, split=<span class="hljs-string">&quot;validation&quot;</span>).shuffle().select(<span class="hljs-number">1000</span>)
task_evaluator = evaluator(<span class="hljs-string">&quot;token-classification&quot;</span>)

results = []
<span class="hljs-keyword">for</span> model <span class="hljs-keyword">in</span> models:
    results.append(
        task_evaluator.compute(
            model_or_pipeline=model, data=data, metric=<span class="hljs-string">&quot;seqeval&quot;</span>
            )
        )

df = pd.DataFrame(results, index=models)
df[[<span class="hljs-string">&quot;overall_f1&quot;</span>, <span class="hljs-string">&quot;overall_accuracy&quot;</span>, <span class="hljs-string">&quot;total_time_in_seconds&quot;</span>, <span class="hljs-string">&quot;samples_per_second&quot;</span>, <span class="hljs-string">&quot;latency_in_seconds&quot;</span>]]`}}),lt=new se({}),st=new se({}),nt=new Ue({props:{code:`from datasets import load_dataset
from evaluate import evaluator

task_evaluator = evaluator("question-answering")

data = load_dataset("squad", split="validation[:1000]")
eval_results = task_evaluator.compute(
    model_or_pipeline="distilbert-base-uncased-distilled-squad",
    data=data,
    metric="squad",
    strategy="bootstrap",
    n_resamples=30
)`,highlighted:`<span class="hljs-keyword">from</span> datasets <span class="hljs-keyword">import</span> load_dataset
<span class="hljs-keyword">from</span> evaluate <span class="hljs-keyword">import</span> evaluator

task_evaluator = evaluator(<span class="hljs-string">&quot;question-answering&quot;</span>)

data = load_dataset(<span class="hljs-string">&quot;squad&quot;</span>, split=<span class="hljs-string">&quot;validation[:1000]&quot;</span>)
eval_results = task_evaluator.compute(
    model_or_pipeline=<span class="hljs-string">&quot;distilbert-base-uncased-distilled-squad&quot;</span>,
    data=data,
    metric=<span class="hljs-string">&quot;squad&quot;</span>,
    strategy=<span class="hljs-string">&quot;bootstrap&quot;</span>,
    n_resamples=<span class="hljs-number">30</span>
)`}}),it=new Ue({props:{code:`{
    'exact_match': 
    {
        'confidence_interval': (79.67, 84.54),
        'score': 82.30,
        'standard_error': 1.28
    },
    'f1': 
    {
        'confidence_interval': (85.30, 88.88),
        'score': 87.23,
        'standard_error': 0.97
    },
    'latency_in_seconds': 0.0085,
    'samples_per_second': 117.31,
    'total_time_in_seconds': 8.52
 }`,highlighted:`{
    <span class="hljs-string">&#x27;exact_match&#x27;</span>: 
    {
        <span class="hljs-string">&#x27;confidence_interval&#x27;</span>: (<span class="hljs-number">79.67</span>, <span class="hljs-number">84.54</span>),
        <span class="hljs-string">&#x27;score&#x27;</span>: <span class="hljs-number">82.30</span>,
        <span class="hljs-string">&#x27;standard_error&#x27;</span>: <span class="hljs-number">1.28</span>
    },
    <span class="hljs-string">&#x27;f1&#x27;</span>: 
    {
        <span class="hljs-string">&#x27;confidence_interval&#x27;</span>: (<span class="hljs-number">85.30</span>, <span class="hljs-number">88.88</span>),
        <span class="hljs-string">&#x27;score&#x27;</span>: <span class="hljs-number">87.23</span>,
        <span class="hljs-string">&#x27;standard_error&#x27;</span>: <span class="hljs-number">0.97</span>
    },
    <span class="hljs-string">&#x27;latency_in_seconds&#x27;</span>: <span class="hljs-number">0.0085</span>,
    <span class="hljs-string">&#x27;samples_per_second&#x27;</span>: <span class="hljs-number">117.31</span>,
    <span class="hljs-string">&#x27;total_time_in_seconds&#x27;</span>: <span class="hljs-number">8.52</span>
 }`}}),rt=new se({}),pt=new se({}),dt=new Ue({props:{code:`data = load_dataset("imagenet-1k", split="validation", use_auth_token=True)

pipe = pipeline(
    task="image-classification",
    model="facebook/deit-small-distilled-patch16-224"
)

task_evaluator = evaluator("image-classification")
eval_results = task_evaluator.compute(
    model_or_pipeline=pipe,
    data=data,
    metric="accuracy",
    label_mapping=pipe.model.config.label2id
)`,highlighted:`data = load_dataset(<span class="hljs-string">&quot;imagenet-1k&quot;</span>, split=<span class="hljs-string">&quot;validation&quot;</span>, use_auth_token=<span class="hljs-literal">True</span>)

pipe = pipeline(
    task=<span class="hljs-string">&quot;image-classification&quot;</span>,
    model=<span class="hljs-string">&quot;facebook/deit-small-distilled-patch16-224&quot;</span>
)

task_evaluator = evaluator(<span class="hljs-string">&quot;image-classification&quot;</span>)
eval_results = task_evaluator.compute(
    model_or_pipeline=pipe,
    data=data,
    metric=<span class="hljs-string">&quot;accuracy&quot;</span>,
    label_mapping=pipe.model.config.label2id
)`}}),{c(){u=l("meta"),G=r(),f=l("h1"),k=l("a"),oe=l("span"),m(H.$$.fragment),Ve=r(),ee=l("span"),ne=n("Using the "),O=l("code"),ft=n("evaluator"),Me=r(),W=l("p"),yo=n("The "),Qa=l("code"),$o=n("Evaluator"),qo=n(" classes allow to evaluate a  triplet of model, dataset, and metric. The models wrapped in a pipeline, responsible for handling all preprocessing and post-processing and out-of-the-box, "),Ja=l("code"),jo=n("Evaluator"),To=n("s support transformers pipelines for the supported tasks, but custom pipelines can be passed, as showcased in the section "),ve=l("a"),xo=n("Using the "),Ya=l("code"),Do=n("evaluator"),Co=n(" with custom pipelines"),Po=n("."),ds=r(),mt=l("p"),Ao=n("Currently supported tasks are:"),hs=r(),w=l("ul"),_e=l("li"),Ka=l("code"),Io=n('"text-classification"'),Oo=n(": will use the "),vt=l("a"),So=n("TextClassificationEvaluator"),No=n("."),Lo=r(),ge=l("li"),Xa=l("code"),Ho=n('"token-classification"'),Bo=n(": will use the "),_t=l("a"),zo=n("TokenClassificationEvaluator"),Go=n("."),Wo=r(),Ee=l("li"),Za=l("code"),Ro=n('"question-answering"'),Uo=n(": will use the "),gt=l("a"),Vo=n("QuestionAnsweringEvaluator"),Mo=n("."),Fo=r(),be=l("li"),el=l("code"),Qo=n('"image-classification"'),Jo=n(": will use the "),Et=l("a"),Yo=n("ImageClassificationEvaluator"),Ko=n("."),Xo=r(),we=l("li"),tl=l("code"),Zo=n('"text2text-generation"'),en=n(": will use the "),bt=l("a"),tn=n("Text2TextGenerationEvaluator"),an=n("."),ln=r(),ke=l("li"),al=l("code"),sn=n('"summarization"'),on=n(": will use the "),wt=l("a"),nn=n("SummarizationEvaluator"),rn=n("."),cn=r(),ye=l("li"),ll=l("code"),pn=n('"translation"'),dn=n(": will use the "),kt=l("a"),hn=n("TranslationEvaluator"),un=n("."),us=r(),yt=l("p"),fn=n("Each task has its own set of requirements for the dataset format and pipeline output, make sure to check them out for your custom use case. Let\u2019s have a look at some of them and see how you can use the evaluator to evalute a single or multiple of models, datasets, and metrics at the same time."),fs=r(),ie=l("h2"),$e=l("a"),sl=l("span"),m(Fe.$$.fragment),mn=r(),ol=l("span"),vn=n("Text classification"),ms=r(),$t=l("p"),_n=n("The text classification evaluator can be used to evaluate text models on classification datasets such as IMDb. Beside the model, data, and metric inputs it takes the following optional inputs:"),vs=r(),te=l("ul"),qt=l("li"),nl=l("code"),gn=n('input_column="text"'),En=n(": with this argument the column with the data for the pipeline can be specified."),bn=r(),jt=l("li"),il=l("code"),wn=n('label_column="label"'),kn=n(": with this argument the column with the labels for the evaluation can be specified."),yn=r(),$=l("li"),rl=l("code"),$n=n("label_mapping=None"),qn=n(": the label mapping aligns the labels in the pipeline output with the labels need for evaluation. E.g. the labels in "),cl=l("code"),jn=n("label_column"),Tn=n(" can be integers ("),pl=l("code"),xn=n("0"),Dn=n("/"),dl=l("code"),Cn=n("1"),Pn=n(") whereas the pipeline can produce label names such as "),hl=l("code"),An=n('"positive"'),In=n("/"),ul=l("code"),On=n('"negative"'),Sn=n(". With that dictionary the pipeline outputs are mapped to the labels."),_s=r(),qe=l("p"),Nn=n("By default the "),fl=l("code"),Ln=n('"accuracy"'),Hn=n(" metric is computed."),gs=r(),re=l("h3"),je=l("a"),ml=l("span"),m(Qe.$$.fragment),Bn=r(),vl=l("span"),zn=n("Evaluate models on the Hub"),Es=r(),R=l("p"),Gn=n("There are several ways to pass a model to the evaluator: you can pass the name of a model on the Hub, you can load a "),_l=l("code"),Wn=n("transformers"),Rn=n(" model and pass it to the evaluator or you can pass an initialized "),gl=l("code"),Un=n("transformers.Pipeline"),Vn=n(". Alternatively you can pass any callable function that behaves like a "),El=l("code"),Mn=n("pipeline"),Fn=n(" call for the task in any framework."),bs=r(),Tt=l("p"),Qn=n("So any of the following works:"),ws=r(),m(Je.$$.fragment),ks=r(),m(Te.$$.fragment),ys=r(),xt=l("p"),Jn=n("The results will look as follows:"),$s=r(),m(Ye.$$.fragment),qs=r(),Dt=l("p"),Yn=n("Note that evaluation results include both the requested metric, and information about the time it took to obtain predictions through the pipeline."),js=r(),m(xe.$$.fragment),Ts=r(),ce=l("h3"),De=l("a"),bl=l("span"),m(Ke.$$.fragment),Kn=r(),wl=l("span"),Xn=n("Evaluate multiple metrics"),xs=r(),Ce=l("p"),Zn=n("With the "),Ct=l("a"),ei=n("combine()"),ti=n(" function one can bundle several metrics into an object that behaves like a single metric. We can use this to evaluate several metrics at once with the evaluator:"),Ds=r(),m(Xe.$$.fragment),Cs=r(),Pt=l("p"),ai=n("The results will look as follows:"),Ps=r(),m(Ze.$$.fragment),As=r(),At=l("p"),li=n("Next let\u2019s have a look at token classification."),Is=r(),pe=l("h2"),Pe=l("a"),kl=l("span"),m(et.$$.fragment),si=r(),yl=l("span"),oi=n("Token Classification"),Os=r(),It=l("p"),ni=n("With the token classification evaluator one can evaluate models for tasks such as NER or POS tagging. It has the following specific arguments:"),Ss=r(),U=l("ul"),Ot=l("li"),$l=l("code"),ii=n('input_column="text"'),ri=n(": with this argument the column with the data for the pipeline can be specified."),ci=r(),St=l("li"),ql=l("code"),pi=n('label_column="label"'),di=n(": with this argument the column with the labels for the evaluation can be specified."),hi=r(),q=l("li"),jl=l("code"),ui=n("label_mapping=None"),fi=n(": the label mapping aligns the labels in the pipeline output with the labels need for evaluation. E.g. the labels in "),Tl=l("code"),mi=n("label_column"),vi=n(" can be integers ("),xl=l("code"),_i=n("0"),gi=n("/"),Dl=l("code"),Ei=n("1"),bi=n(") whereas the pipeline can produce label names such as "),Cl=l("code"),wi=n('"positive"'),ki=n("/"),Pl=l("code"),yi=n('"negative"'),$i=n(". With that dictionary the pipeline outputs are mapped to the labels."),qi=r(),Nt=l("li"),Al=l("code"),ji=n('join_by=" "'),Ti=n(": While most datasets are already tokenized the pipeline expects a string. Thus the tokens need to be joined before passing to the pipeline. By default they are joined with a whitespace."),Ns=r(),Lt=l("p"),xi=n("Let\u2019s have a look how we can use the evaluator to benchmark several models."),Ls=r(),de=l("h3"),Ae=l("a"),Il=l("span"),m(tt.$$.fragment),Di=r(),Ol=l("span"),Ci=n("Benchmarking several models"),Hs=r(),Ie=l("p"),Pi=n("Here is an example where several models can be compared thanks to the "),Sl=l("code"),Ai=n("evaluator"),Ii=n(" in only a few lines of code, abstracting away the preprocessing, inference, postprocessing, metric computation:"),Bs=r(),m(at.$$.fragment),zs=r(),Ht=l("p"),Oi=n("The result is a table that looks like this:"),Gs=r(),Oe=l("table"),Nl=l("thead"),j=l("tr"),Bt=l("th"),Si=n("model"),Ni=r(),zt=l("th"),Li=n("overall_f1"),Hi=r(),Gt=l("th"),Bi=n("overall_accuracy"),zi=r(),Wt=l("th"),Gi=n("total_time_in_seconds"),Wi=r(),Rt=l("th"),Ri=n("samples_per_second"),Ui=r(),Ut=l("th"),Vi=n("latency_in_seconds"),Mi=r(),y=l("tbody"),T=l("tr"),Vt=l("td"),Fi=n("Jorgeutd/albert-base-v2-finetuned-ner"),Qi=r(),Mt=l("td"),Ji=n("0.941"),Yi=r(),Ft=l("td"),Ki=n("0.989"),Xi=r(),Qt=l("td"),Zi=n("4.515"),er=r(),Jt=l("td"),tr=n("221.468"),ar=r(),Yt=l("td"),lr=n("0.005"),sr=r(),x=l("tr"),Kt=l("td"),or=n("dbmdz/bert-large-cased-finetuned-conll03-english"),nr=r(),Xt=l("td"),ir=n("0.962"),rr=r(),Zt=l("td"),cr=n("0.881"),pr=r(),ea=l("td"),dr=n("11.648"),hr=r(),ta=l("td"),ur=n("85.850"),fr=r(),aa=l("td"),mr=n("0.012"),vr=r(),D=l("tr"),la=l("td"),_r=n("dbmdz/electra-large-discriminator-finetuned-conll03-english"),gr=r(),sa=l("td"),Er=n("0.965"),br=r(),oa=l("td"),wr=n("0.881"),kr=r(),na=l("td"),yr=n("11.456"),$r=r(),ia=l("td"),qr=n("87.292"),jr=r(),ra=l("td"),Tr=n("0.011"),xr=r(),C=l("tr"),ca=l("td"),Dr=n("elastic/distilbert-base-uncased-finetuned-conll03-english"),Cr=r(),pa=l("td"),Pr=n("0.940"),Ar=r(),da=l("td"),Ir=n("0.989"),Or=r(),ha=l("td"),Sr=n("2.318"),Nr=r(),ua=l("td"),Lr=n("431.378"),Hr=r(),fa=l("td"),Br=n("0.002"),zr=r(),P=l("tr"),ma=l("td"),Gr=n("gunghio/distilbert-base-multilingual-cased-finetuned-conll2003-ner"),Wr=r(),va=l("td"),Rr=n("0.947"),Ur=r(),_a=l("td"),Vr=n("0.991"),Mr=r(),ga=l("td"),Fr=n("2.376"),Qr=r(),Ea=l("td"),Jr=n("420.873"),Yr=r(),ba=l("td"),Kr=n("0.002"),Xr=r(),A=l("tr"),wa=l("td"),Zr=n("philschmid/distilroberta-base-ner-conll2003"),ec=r(),ka=l("td"),tc=n("0.961"),ac=r(),ya=l("td"),lc=n("0.994"),sc=r(),$a=l("td"),oc=n("2.436"),nc=r(),qa=l("td"),ic=n("410.579"),rc=r(),ja=l("td"),cc=n("0.002"),pc=r(),I=l("tr"),Ta=l("td"),dc=n("xlm-roberta-large-finetuned-conll03-english"),hc=r(),xa=l("td"),uc=n("0.969"),fc=r(),Da=l("td"),mc=n("0.882"),vc=r(),Ca=l("td"),_c=n("11.996"),gc=r(),Pa=l("td"),Ec=n("83.359"),bc=r(),Aa=l("td"),wc=n("0.012"),Ws=r(),he=l("h2"),Se=l("a"),Ll=l("span"),m(lt.$$.fragment),kc=r(),Hl=l("span"),yc=n("Question Answering"),Rs=r(),Ia=l("p"),$c=n("With the question-answering evaluator one can evaluate models for QA without needing to worry about the complicated pre- and post-processing that\u2019s required for these models. It has the following specific arguments:"),Us=r(),S=l("ul"),Oa=l("li"),Bl=l("code"),qc=n('question_column="question"'),jc=n(": the name of the column containing the question in the dataset"),Tc=r(),Sa=l("li"),zl=l("code"),xc=n('context_column="context"'),Dc=n(": the name of the column containing the context"),Cc=r(),Na=l("li"),Gl=l("code"),Pc=n('id_column="id"'),Ac=n(": the name of the column cointaing the identification field of the question and answer pair"),Ic=r(),La=l("li"),Wl=l("code"),Oc=n('label_column="answers"'),Sc=n(": the name of the column containing the answers"),Nc=r(),Ha=l("li"),Rl=l("code"),Lc=n("squad_v2_format=None"),Hc=n(": whether the dataset follows the format of squad_v2 dataset where a question may have no answer in the context. If this parameter is not provided, the format will be automatically inferred."),Vs=r(),Ba=l("p"),Bc=n("Let\u2019s have a look how we can evaluate QA models and compute confidence intervals at the same time."),Ms=r(),ue=l("h3"),Ne=l("a"),Ul=l("span"),m(st.$$.fragment),zc=r(),Vl=l("span"),Gc=n("Confidence intervals"),Fs=r(),V=l("p"),Wc=n("Every evaluator comes with the options to compute confidence intervals using "),ot=l("a"),Rc=n("bootstrapping"),Uc=n(". Simply pass "),Ml=l("code"),Vc=n('strategy="bootstrap"'),Mc=n(" and set the number of resanmples with "),Fl=l("code"),Fc=n("n_resamples"),Qc=n("."),Qs=r(),m(nt.$$.fragment),Js=r(),za=l("p"),Jc=n("Results include confidence intervals as well as error estimates as follows:"),Ys=r(),m(it.$$.fragment),Ks=r(),fe=l("h2"),Le=l("a"),Ql=l("span"),m(rt.$$.fragment),Yc=r(),Jl=l("span"),Kc=n("Image classification"),Xs=r(),Ga=l("p"),Xc=n("With the image classification evaluator we can evaluate any image classifier. It uses the same keyword arguments at the text classifier:"),Zs=r(),ae=l("ul"),Wa=l("li"),Yl=l("code"),Zc=n('input_column="image"'),ep=n(": the name of the column containing the images as PIL ImageFile"),tp=r(),Ra=l("li"),Kl=l("code"),ap=n('label_column="label"'),lp=n(": the name of the column containing the labels"),sp=r(),ct=l("li"),Xl=l("code"),op=n("label_mapping=None"),np=n(": We want to map class labels defined by the model in the pipeline to values consistent with those defined in the "),Zl=l("code"),ip=n("label_column"),eo=r(),Ua=l("p"),rp=n("Let\u2019s have a look at how can evaluate image classification models on large datasets."),to=r(),me=l("h3"),He=l("a"),es=l("span"),m(pt.$$.fragment),cp=r(),ts=l("span"),pp=n("Handling large datasets"),ao=r(),Va=l("p"),dp=n("The evaluator can be used on large datasets! Below, an example shows how to use it on ImageNet-1k for image classification. Beware that this example will require to download ~150 GB."),lo=r(),m(dt.$$.fragment),so=r(),Be=l("p"),hp=n("Since we are using "),as=l("code"),up=n("datasets"),fp=n(" to store data we make use of a technique called memory mappings. This means that the dataset is never fully loaded into memory which saves a lot of RAM. Running the above code only uses roughly 1.5 GB of RAM while the validation split is more than 30 GB big."),this.h()},l(t){const p=gu('[data-svelte="svelte-1phssyn"]',document.head);u=s(p,"META",{name:!0,content:!0}),p.forEach(a),G=c(t),f=s(t,"H1",{class:!0});var ht=o(f);k=s(ht,"A",{id:!0,class:!0,href:!0});var ls=o(k);oe=s(ls,"SPAN",{});var xp=o(oe);v(H.$$.fragment,xp),xp.forEach(a),ls.forEach(a),Ve=c(ht),ee=s(ht,"SPAN",{});var mp=o(ee);ne=i(mp,"Using the "),O=s(mp,"CODE",{});var Dp=o(O);ft=i(Dp,"evaluator"),Dp.forEach(a),mp.forEach(a),ht.forEach(a),Me=c(t),W=s(t,"P",{});var ze=o(W);yo=i(ze,"The "),Qa=s(ze,"CODE",{});var Cp=o(Qa);$o=i(Cp,"Evaluator"),Cp.forEach(a),qo=i(ze," classes allow to evaluate a  triplet of model, dataset, and metric. The models wrapped in a pipeline, responsible for handling all preprocessing and post-processing and out-of-the-box, "),Ja=s(ze,"CODE",{});var Pp=o(Ja);jo=i(Pp,"Evaluator"),Pp.forEach(a),To=i(ze,"s support transformers pipelines for the supported tasks, but custom pipelines can be passed, as showcased in the section "),ve=s(ze,"A",{href:!0});var no=o(ve);xo=i(no,"Using the "),Ya=s(no,"CODE",{});var Ap=o(Ya);Do=i(Ap,"evaluator"),Ap.forEach(a),Co=i(no," with custom pipelines"),no.forEach(a),Po=i(ze,"."),ze.forEach(a),ds=c(t),mt=s(t,"P",{});var Ip=o(mt);Ao=i(Ip,"Currently supported tasks are:"),Ip.forEach(a),hs=c(t),w=s(t,"UL",{});var N=o(w);_e=s(N,"LI",{});var ss=o(_e);Ka=s(ss,"CODE",{});var Op=o(Ka);Io=i(Op,'"text-classification"'),Op.forEach(a),Oo=i(ss,": will use the "),vt=s(ss,"A",{href:!0});var Sp=o(vt);So=i(Sp,"TextClassificationEvaluator"),Sp.forEach(a),No=i(ss,"."),ss.forEach(a),Lo=c(N),ge=s(N,"LI",{});var os=o(ge);Xa=s(os,"CODE",{});var Np=o(Xa);Ho=i(Np,'"token-classification"'),Np.forEach(a),Bo=i(os,": will use the "),_t=s(os,"A",{href:!0});var Lp=o(_t);zo=i(Lp,"TokenClassificationEvaluator"),Lp.forEach(a),Go=i(os,"."),os.forEach(a),Wo=c(N),Ee=s(N,"LI",{});var ns=o(Ee);Za=s(ns,"CODE",{});var Hp=o(Za);Ro=i(Hp,'"question-answering"'),Hp.forEach(a),Uo=i(ns,": will use the "),gt=s(ns,"A",{href:!0});var Bp=o(gt);Vo=i(Bp,"QuestionAnsweringEvaluator"),Bp.forEach(a),Mo=i(ns,"."),ns.forEach(a),Fo=c(N),be=s(N,"LI",{});var is=o(be);el=s(is,"CODE",{});var zp=o(el);Qo=i(zp,'"image-classification"'),zp.forEach(a),Jo=i(is,": will use the "),Et=s(is,"A",{href:!0});var Gp=o(Et);Yo=i(Gp,"ImageClassificationEvaluator"),Gp.forEach(a),Ko=i(is,"."),is.forEach(a),Xo=c(N),we=s(N,"LI",{});var rs=o(we);tl=s(rs,"CODE",{});var Wp=o(tl);Zo=i(Wp,'"text2text-generation"'),Wp.forEach(a),en=i(rs,": will use the "),bt=s(rs,"A",{href:!0});var Rp=o(bt);tn=i(Rp,"Text2TextGenerationEvaluator"),Rp.forEach(a),an=i(rs,"."),rs.forEach(a),ln=c(N),ke=s(N,"LI",{});var cs=o(ke);al=s(cs,"CODE",{});var Up=o(al);sn=i(Up,'"summarization"'),Up.forEach(a),on=i(cs,": will use the "),wt=s(cs,"A",{href:!0});var Vp=o(wt);nn=i(Vp,"SummarizationEvaluator"),Vp.forEach(a),rn=i(cs,"."),cs.forEach(a),cn=c(N),ye=s(N,"LI",{});var ps=o(ye);ll=s(ps,"CODE",{});var Mp=o(ll);pn=i(Mp,'"translation"'),Mp.forEach(a),dn=i(ps,": will use the "),kt=s(ps,"A",{href:!0});var Fp=o(kt);hn=i(Fp,"TranslationEvaluator"),Fp.forEach(a),un=i(ps,"."),ps.forEach(a),N.forEach(a),us=c(t),yt=s(t,"P",{});var Qp=o(yt);fn=i(Qp,"Each task has its own set of requirements for the dataset format and pipeline output, make sure to check them out for your custom use case. Let\u2019s have a look at some of them and see how you can use the evaluator to evalute a single or multiple of models, datasets, and metrics at the same time."),Qp.forEach(a),fs=c(t),ie=s(t,"H2",{class:!0});var io=o(ie);$e=s(io,"A",{id:!0,class:!0,href:!0});var Jp=o($e);sl=s(Jp,"SPAN",{});var Yp=o(sl);v(Fe.$$.fragment,Yp),Yp.forEach(a),Jp.forEach(a),mn=c(io),ol=s(io,"SPAN",{});var Kp=o(ol);vn=i(Kp,"Text classification"),Kp.forEach(a),io.forEach(a),ms=c(t),$t=s(t,"P",{});var Xp=o($t);_n=i(Xp,"The text classification evaluator can be used to evaluate text models on classification datasets such as IMDb. Beside the model, data, and metric inputs it takes the following optional inputs:"),Xp.forEach(a),vs=c(t),te=s(t,"UL",{});var Ma=o(te);qt=s(Ma,"LI",{});var vp=o(qt);nl=s(vp,"CODE",{});var Zp=o(nl);gn=i(Zp,'input_column="text"'),Zp.forEach(a),En=i(vp,": with this argument the column with the data for the pipeline can be specified."),vp.forEach(a),bn=c(Ma),jt=s(Ma,"LI",{});var _p=o(jt);il=s(_p,"CODE",{});var ed=o(il);wn=i(ed,'label_column="label"'),ed.forEach(a),kn=i(_p,": with this argument the column with the labels for the evaluation can be specified."),_p.forEach(a),yn=c(Ma),$=s(Ma,"LI",{});var B=o($);rl=s(B,"CODE",{});var td=o(rl);$n=i(td,"label_mapping=None"),td.forEach(a),qn=i(B,": the label mapping aligns the labels in the pipeline output with the labels need for evaluation. E.g. the labels in "),cl=s(B,"CODE",{});var ad=o(cl);jn=i(ad,"label_column"),ad.forEach(a),Tn=i(B," can be integers ("),pl=s(B,"CODE",{});var ld=o(pl);xn=i(ld,"0"),ld.forEach(a),Dn=i(B,"/"),dl=s(B,"CODE",{});var sd=o(dl);Cn=i(sd,"1"),sd.forEach(a),Pn=i(B,") whereas the pipeline can produce label names such as "),hl=s(B,"CODE",{});var od=o(hl);An=i(od,'"positive"'),od.forEach(a),In=i(B,"/"),ul=s(B,"CODE",{});var nd=o(ul);On=i(nd,'"negative"'),nd.forEach(a),Sn=i(B,". With that dictionary the pipeline outputs are mapped to the labels."),B.forEach(a),Ma.forEach(a),_s=c(t),qe=s(t,"P",{});var ro=o(qe);Nn=i(ro,"By default the "),fl=s(ro,"CODE",{});var id=o(fl);Ln=i(id,'"accuracy"'),id.forEach(a),Hn=i(ro," metric is computed."),ro.forEach(a),gs=c(t),re=s(t,"H3",{class:!0});var co=o(re);je=s(co,"A",{id:!0,class:!0,href:!0});var rd=o(je);ml=s(rd,"SPAN",{});var cd=o(ml);v(Qe.$$.fragment,cd),cd.forEach(a),rd.forEach(a),Bn=c(co),vl=s(co,"SPAN",{});var pd=o(vl);zn=i(pd,"Evaluate models on the Hub"),pd.forEach(a),co.forEach(a),Es=c(t),R=s(t,"P",{});var Ge=o(R);Gn=i(Ge,"There are several ways to pass a model to the evaluator: you can pass the name of a model on the Hub, you can load a "),_l=s(Ge,"CODE",{});var dd=o(_l);Wn=i(dd,"transformers"),dd.forEach(a),Rn=i(Ge," model and pass it to the evaluator or you can pass an initialized "),gl=s(Ge,"CODE",{});var hd=o(gl);Un=i(hd,"transformers.Pipeline"),hd.forEach(a),Vn=i(Ge,". Alternatively you can pass any callable function that behaves like a "),El=s(Ge,"CODE",{});var ud=o(El);Mn=i(ud,"pipeline"),ud.forEach(a),Fn=i(Ge," call for the task in any framework."),Ge.forEach(a),bs=c(t),Tt=s(t,"P",{});var fd=o(Tt);Qn=i(fd,"So any of the following works:"),fd.forEach(a),ws=c(t),v(Je.$$.fragment,t),ks=c(t),v(Te.$$.fragment,t),ys=c(t),xt=s(t,"P",{});var md=o(xt);Jn=i(md,"The results will look as follows:"),md.forEach(a),$s=c(t),v(Ye.$$.fragment,t),qs=c(t),Dt=s(t,"P",{});var vd=o(Dt);Yn=i(vd,"Note that evaluation results include both the requested metric, and information about the time it took to obtain predictions through the pipeline."),vd.forEach(a),js=c(t),v(xe.$$.fragment,t),Ts=c(t),ce=s(t,"H3",{class:!0});var po=o(ce);De=s(po,"A",{id:!0,class:!0,href:!0});var _d=o(De);bl=s(_d,"SPAN",{});var gd=o(bl);v(Ke.$$.fragment,gd),gd.forEach(a),_d.forEach(a),Kn=c(po),wl=s(po,"SPAN",{});var Ed=o(wl);Xn=i(Ed,"Evaluate multiple metrics"),Ed.forEach(a),po.forEach(a),xs=c(t),Ce=s(t,"P",{});var ho=o(Ce);Zn=i(ho,"With the "),Ct=s(ho,"A",{href:!0});var bd=o(Ct);ei=i(bd,"combine()"),bd.forEach(a),ti=i(ho," function one can bundle several metrics into an object that behaves like a single metric. We can use this to evaluate several metrics at once with the evaluator:"),ho.forEach(a),Ds=c(t),v(Xe.$$.fragment,t),Cs=c(t),Pt=s(t,"P",{});var wd=o(Pt);ai=i(wd,"The results will look as follows:"),wd.forEach(a),Ps=c(t),v(Ze.$$.fragment,t),As=c(t),At=s(t,"P",{});var kd=o(At);li=i(kd,"Next let\u2019s have a look at token classification."),kd.forEach(a),Is=c(t),pe=s(t,"H2",{class:!0});var uo=o(pe);Pe=s(uo,"A",{id:!0,class:!0,href:!0});var yd=o(Pe);kl=s(yd,"SPAN",{});var $d=o(kl);v(et.$$.fragment,$d),$d.forEach(a),yd.forEach(a),si=c(uo),yl=s(uo,"SPAN",{});var qd=o(yl);oi=i(qd,"Token Classification"),qd.forEach(a),uo.forEach(a),Os=c(t),It=s(t,"P",{});var jd=o(It);ni=i(jd,"With the token classification evaluator one can evaluate models for tasks such as NER or POS tagging. It has the following specific arguments:"),jd.forEach(a),Ss=c(t),U=s(t,"UL",{});var We=o(U);Ot=s(We,"LI",{});var gp=o(Ot);$l=s(gp,"CODE",{});var Td=o($l);ii=i(Td,'input_column="text"'),Td.forEach(a),ri=i(gp,": with this argument the column with the data for the pipeline can be specified."),gp.forEach(a),ci=c(We),St=s(We,"LI",{});var Ep=o(St);ql=s(Ep,"CODE",{});var xd=o(ql);pi=i(xd,'label_column="label"'),xd.forEach(a),di=i(Ep,": with this argument the column with the labels for the evaluation can be specified."),Ep.forEach(a),hi=c(We),q=s(We,"LI",{});var z=o(q);jl=s(z,"CODE",{});var Dd=o(jl);ui=i(Dd,"label_mapping=None"),Dd.forEach(a),fi=i(z,": the label mapping aligns the labels in the pipeline output with the labels need for evaluation. E.g. the labels in "),Tl=s(z,"CODE",{});var Cd=o(Tl);mi=i(Cd,"label_column"),Cd.forEach(a),vi=i(z," can be integers ("),xl=s(z,"CODE",{});var Pd=o(xl);_i=i(Pd,"0"),Pd.forEach(a),gi=i(z,"/"),Dl=s(z,"CODE",{});var Ad=o(Dl);Ei=i(Ad,"1"),Ad.forEach(a),bi=i(z,") whereas the pipeline can produce label names such as "),Cl=s(z,"CODE",{});var Id=o(Cl);wi=i(Id,'"positive"'),Id.forEach(a),ki=i(z,"/"),Pl=s(z,"CODE",{});var Od=o(Pl);yi=i(Od,'"negative"'),Od.forEach(a),$i=i(z,". With that dictionary the pipeline outputs are mapped to the labels."),z.forEach(a),qi=c(We),Nt=s(We,"LI",{});var bp=o(Nt);Al=s(bp,"CODE",{});var Sd=o(Al);ji=i(Sd,'join_by=" "'),Sd.forEach(a),Ti=i(bp,": While most datasets are already tokenized the pipeline expects a string. Thus the tokens need to be joined before passing to the pipeline. By default they are joined with a whitespace."),bp.forEach(a),We.forEach(a),Ns=c(t),Lt=s(t,"P",{});var Nd=o(Lt);xi=i(Nd,"Let\u2019s have a look how we can use the evaluator to benchmark several models."),Nd.forEach(a),Ls=c(t),de=s(t,"H3",{class:!0});var fo=o(de);Ae=s(fo,"A",{id:!0,class:!0,href:!0});var Ld=o(Ae);Il=s(Ld,"SPAN",{});var Hd=o(Il);v(tt.$$.fragment,Hd),Hd.forEach(a),Ld.forEach(a),Di=c(fo),Ol=s(fo,"SPAN",{});var Bd=o(Ol);Ci=i(Bd,"Benchmarking several models"),Bd.forEach(a),fo.forEach(a),Hs=c(t),Ie=s(t,"P",{});var mo=o(Ie);Pi=i(mo,"Here is an example where several models can be compared thanks to the "),Sl=s(mo,"CODE",{});var zd=o(Sl);Ai=i(zd,"evaluator"),zd.forEach(a),Ii=i(mo," in only a few lines of code, abstracting away the preprocessing, inference, postprocessing, metric computation:"),mo.forEach(a),Bs=c(t),v(at.$$.fragment,t),zs=c(t),Ht=s(t,"P",{});var Gd=o(Ht);Oi=i(Gd,"The result is a table that looks like this:"),Gd.forEach(a),Gs=c(t),Oe=s(t,"TABLE",{});var vo=o(Oe);Nl=s(vo,"THEAD",{});var Wd=o(Nl);j=s(Wd,"TR",{});var M=o(j);Bt=s(M,"TH",{align:!0});var Rd=o(Bt);Si=i(Rd,"model"),Rd.forEach(a),Ni=c(M),zt=s(M,"TH",{align:!0});var Ud=o(zt);Li=i(Ud,"overall_f1"),Ud.forEach(a),Hi=c(M),Gt=s(M,"TH",{align:!0});var Vd=o(Gt);Bi=i(Vd,"overall_accuracy"),Vd.forEach(a),zi=c(M),Wt=s(M,"TH",{align:!0});var Md=o(Wt);Gi=i(Md,"total_time_in_seconds"),Md.forEach(a),Wi=c(M),Rt=s(M,"TH",{align:!0});var Fd=o(Rt);Ri=i(Fd,"samples_per_second"),Fd.forEach(a),Ui=c(M),Ut=s(M,"TH",{align:!0});var Qd=o(Ut);Vi=i(Qd,"latency_in_seconds"),Qd.forEach(a),M.forEach(a),Wd.forEach(a),Mi=c(vo),y=s(vo,"TBODY",{});var L=o(y);T=s(L,"TR",{});var F=o(T);Vt=s(F,"TD",{align:!0});var Jd=o(Vt);Fi=i(Jd,"Jorgeutd/albert-base-v2-finetuned-ner"),Jd.forEach(a),Qi=c(F),Mt=s(F,"TD",{align:!0});var Yd=o(Mt);Ji=i(Yd,"0.941"),Yd.forEach(a),Yi=c(F),Ft=s(F,"TD",{align:!0});var Kd=o(Ft);Ki=i(Kd,"0.989"),Kd.forEach(a),Xi=c(F),Qt=s(F,"TD",{align:!0});var Xd=o(Qt);Zi=i(Xd,"4.515"),Xd.forEach(a),er=c(F),Jt=s(F,"TD",{align:!0});var Zd=o(Jt);tr=i(Zd,"221.468"),Zd.forEach(a),ar=c(F),Yt=s(F,"TD",{align:!0});var eh=o(Yt);lr=i(eh,"0.005"),eh.forEach(a),F.forEach(a),sr=c(L),x=s(L,"TR",{});var Q=o(x);Kt=s(Q,"TD",{align:!0});var th=o(Kt);or=i(th,"dbmdz/bert-large-cased-finetuned-conll03-english"),th.forEach(a),nr=c(Q),Xt=s(Q,"TD",{align:!0});var ah=o(Xt);ir=i(ah,"0.962"),ah.forEach(a),rr=c(Q),Zt=s(Q,"TD",{align:!0});var lh=o(Zt);cr=i(lh,"0.881"),lh.forEach(a),pr=c(Q),ea=s(Q,"TD",{align:!0});var sh=o(ea);dr=i(sh,"11.648"),sh.forEach(a),hr=c(Q),ta=s(Q,"TD",{align:!0});var oh=o(ta);ur=i(oh,"85.850"),oh.forEach(a),fr=c(Q),aa=s(Q,"TD",{align:!0});var nh=o(aa);mr=i(nh,"0.012"),nh.forEach(a),Q.forEach(a),vr=c(L),D=s(L,"TR",{});var J=o(D);la=s(J,"TD",{align:!0});var ih=o(la);_r=i(ih,"dbmdz/electra-large-discriminator-finetuned-conll03-english"),ih.forEach(a),gr=c(J),sa=s(J,"TD",{align:!0});var rh=o(sa);Er=i(rh,"0.965"),rh.forEach(a),br=c(J),oa=s(J,"TD",{align:!0});var ch=o(oa);wr=i(ch,"0.881"),ch.forEach(a),kr=c(J),na=s(J,"TD",{align:!0});var ph=o(na);yr=i(ph,"11.456"),ph.forEach(a),$r=c(J),ia=s(J,"TD",{align:!0});var dh=o(ia);qr=i(dh,"87.292"),dh.forEach(a),jr=c(J),ra=s(J,"TD",{align:!0});var hh=o(ra);Tr=i(hh,"0.011"),hh.forEach(a),J.forEach(a),xr=c(L),C=s(L,"TR",{});var Y=o(C);ca=s(Y,"TD",{align:!0});var uh=o(ca);Dr=i(uh,"elastic/distilbert-base-uncased-finetuned-conll03-english"),uh.forEach(a),Cr=c(Y),pa=s(Y,"TD",{align:!0});var fh=o(pa);Pr=i(fh,"0.940"),fh.forEach(a),Ar=c(Y),da=s(Y,"TD",{align:!0});var mh=o(da);Ir=i(mh,"0.989"),mh.forEach(a),Or=c(Y),ha=s(Y,"TD",{align:!0});var vh=o(ha);Sr=i(vh,"2.318"),vh.forEach(a),Nr=c(Y),ua=s(Y,"TD",{align:!0});var _h=o(ua);Lr=i(_h,"431.378"),_h.forEach(a),Hr=c(Y),fa=s(Y,"TD",{align:!0});var gh=o(fa);Br=i(gh,"0.002"),gh.forEach(a),Y.forEach(a),zr=c(L),P=s(L,"TR",{});var K=o(P);ma=s(K,"TD",{align:!0});var Eh=o(ma);Gr=i(Eh,"gunghio/distilbert-base-multilingual-cased-finetuned-conll2003-ner"),Eh.forEach(a),Wr=c(K),va=s(K,"TD",{align:!0});var bh=o(va);Rr=i(bh,"0.947"),bh.forEach(a),Ur=c(K),_a=s(K,"TD",{align:!0});var wh=o(_a);Vr=i(wh,"0.991"),wh.forEach(a),Mr=c(K),ga=s(K,"TD",{align:!0});var kh=o(ga);Fr=i(kh,"2.376"),kh.forEach(a),Qr=c(K),Ea=s(K,"TD",{align:!0});var yh=o(Ea);Jr=i(yh,"420.873"),yh.forEach(a),Yr=c(K),ba=s(K,"TD",{align:!0});var $h=o(ba);Kr=i($h,"0.002"),$h.forEach(a),K.forEach(a),Xr=c(L),A=s(L,"TR",{});var X=o(A);wa=s(X,"TD",{align:!0});var qh=o(wa);Zr=i(qh,"philschmid/distilroberta-base-ner-conll2003"),qh.forEach(a),ec=c(X),ka=s(X,"TD",{align:!0});var jh=o(ka);tc=i(jh,"0.961"),jh.forEach(a),ac=c(X),ya=s(X,"TD",{align:!0});var Th=o(ya);lc=i(Th,"0.994"),Th.forEach(a),sc=c(X),$a=s(X,"TD",{align:!0});var xh=o($a);oc=i(xh,"2.436"),xh.forEach(a),nc=c(X),qa=s(X,"TD",{align:!0});var Dh=o(qa);ic=i(Dh,"410.579"),Dh.forEach(a),rc=c(X),ja=s(X,"TD",{align:!0});var Ch=o(ja);cc=i(Ch,"0.002"),Ch.forEach(a),X.forEach(a),pc=c(L),I=s(L,"TR",{});var Z=o(I);Ta=s(Z,"TD",{align:!0});var Ph=o(Ta);dc=i(Ph,"xlm-roberta-large-finetuned-conll03-english"),Ph.forEach(a),hc=c(Z),xa=s(Z,"TD",{align:!0});var Ah=o(xa);uc=i(Ah,"0.969"),Ah.forEach(a),fc=c(Z),Da=s(Z,"TD",{align:!0});var Ih=o(Da);mc=i(Ih,"0.882"),Ih.forEach(a),vc=c(Z),Ca=s(Z,"TD",{align:!0});var Oh=o(Ca);_c=i(Oh,"11.996"),Oh.forEach(a),gc=c(Z),Pa=s(Z,"TD",{align:!0});var Sh=o(Pa);Ec=i(Sh,"83.359"),Sh.forEach(a),bc=c(Z),Aa=s(Z,"TD",{align:!0});var Nh=o(Aa);wc=i(Nh,"0.012"),Nh.forEach(a),Z.forEach(a),L.forEach(a),vo.forEach(a),Ws=c(t),he=s(t,"H2",{class:!0});var _o=o(he);Se=s(_o,"A",{id:!0,class:!0,href:!0});var Lh=o(Se);Ll=s(Lh,"SPAN",{});var Hh=o(Ll);v(lt.$$.fragment,Hh),Hh.forEach(a),Lh.forEach(a),kc=c(_o),Hl=s(_o,"SPAN",{});var Bh=o(Hl);yc=i(Bh,"Question Answering"),Bh.forEach(a),_o.forEach(a),Rs=c(t),Ia=s(t,"P",{});var zh=o(Ia);$c=i(zh,"With the question-answering evaluator one can evaluate models for QA without needing to worry about the complicated pre- and post-processing that\u2019s required for these models. It has the following specific arguments:"),zh.forEach(a),Us=c(t),S=s(t,"UL",{});var le=o(S);Oa=s(le,"LI",{});var wp=o(Oa);Bl=s(wp,"CODE",{});var Gh=o(Bl);qc=i(Gh,'question_column="question"'),Gh.forEach(a),jc=i(wp,": the name of the column containing the question in the dataset"),wp.forEach(a),Tc=c(le),Sa=s(le,"LI",{});var kp=o(Sa);zl=s(kp,"CODE",{});var Wh=o(zl);xc=i(Wh,'context_column="context"'),Wh.forEach(a),Dc=i(kp,": the name of the column containing the context"),kp.forEach(a),Cc=c(le),Na=s(le,"LI",{});var yp=o(Na);Gl=s(yp,"CODE",{});var Rh=o(Gl);Pc=i(Rh,'id_column="id"'),Rh.forEach(a),Ac=i(yp,": the name of the column cointaing the identification field of the question and answer pair"),yp.forEach(a),Ic=c(le),La=s(le,"LI",{});var $p=o(La);Wl=s($p,"CODE",{});var Uh=o(Wl);Oc=i(Uh,'label_column="answers"'),Uh.forEach(a),Sc=i($p,": the name of the column containing the answers"),$p.forEach(a),Nc=c(le),Ha=s(le,"LI",{});var qp=o(Ha);Rl=s(qp,"CODE",{});var Vh=o(Rl);Lc=i(Vh,"squad_v2_format=None"),Vh.forEach(a),Hc=i(qp,": whether the dataset follows the format of squad_v2 dataset where a question may have no answer in the context. If this parameter is not provided, the format will be automatically inferred."),qp.forEach(a),le.forEach(a),Vs=c(t),Ba=s(t,"P",{});var Mh=o(Ba);Bc=i(Mh,"Let\u2019s have a look how we can evaluate QA models and compute confidence intervals at the same time."),Mh.forEach(a),Ms=c(t),ue=s(t,"H3",{class:!0});var go=o(ue);Ne=s(go,"A",{id:!0,class:!0,href:!0});var Fh=o(Ne);Ul=s(Fh,"SPAN",{});var Qh=o(Ul);v(st.$$.fragment,Qh),Qh.forEach(a),Fh.forEach(a),zc=c(go),Vl=s(go,"SPAN",{});var Jh=o(Vl);Gc=i(Jh,"Confidence intervals"),Jh.forEach(a),go.forEach(a),Fs=c(t),V=s(t,"P",{});var Re=o(V);Wc=i(Re,"Every evaluator comes with the options to compute confidence intervals using "),ot=s(Re,"A",{href:!0,rel:!0});var Yh=o(ot);Rc=i(Yh,"bootstrapping"),Yh.forEach(a),Uc=i(Re,". Simply pass "),Ml=s(Re,"CODE",{});var Kh=o(Ml);Vc=i(Kh,'strategy="bootstrap"'),Kh.forEach(a),Mc=i(Re," and set the number of resanmples with "),Fl=s(Re,"CODE",{});var Xh=o(Fl);Fc=i(Xh,"n_resamples"),Xh.forEach(a),Qc=i(Re,"."),Re.forEach(a),Qs=c(t),v(nt.$$.fragment,t),Js=c(t),za=s(t,"P",{});var Zh=o(za);Jc=i(Zh,"Results include confidence intervals as well as error estimates as follows:"),Zh.forEach(a),Ys=c(t),v(it.$$.fragment,t),Ks=c(t),fe=s(t,"H2",{class:!0});var Eo=o(fe);Le=s(Eo,"A",{id:!0,class:!0,href:!0});var eu=o(Le);Ql=s(eu,"SPAN",{});var tu=o(Ql);v(rt.$$.fragment,tu),tu.forEach(a),eu.forEach(a),Yc=c(Eo),Jl=s(Eo,"SPAN",{});var au=o(Jl);Kc=i(au,"Image classification"),au.forEach(a),Eo.forEach(a),Xs=c(t),Ga=s(t,"P",{});var lu=o(Ga);Xc=i(lu,"With the image classification evaluator we can evaluate any image classifier. It uses the same keyword arguments at the text classifier:"),lu.forEach(a),Zs=c(t),ae=s(t,"UL",{});var Fa=o(ae);Wa=s(Fa,"LI",{});var jp=o(Wa);Yl=s(jp,"CODE",{});var su=o(Yl);Zc=i(su,'input_column="image"'),su.forEach(a),ep=i(jp,": the name of the column containing the images as PIL ImageFile"),jp.forEach(a),tp=c(Fa),Ra=s(Fa,"LI",{});var Tp=o(Ra);Kl=s(Tp,"CODE",{});var ou=o(Kl);ap=i(ou,'label_column="label"'),ou.forEach(a),lp=i(Tp,": the name of the column containing the labels"),Tp.forEach(a),sp=c(Fa),ct=s(Fa,"LI",{});var bo=o(ct);Xl=s(bo,"CODE",{});var nu=o(Xl);op=i(nu,"label_mapping=None"),nu.forEach(a),np=i(bo,": We want to map class labels defined by the model in the pipeline to values consistent with those defined in the "),Zl=s(bo,"CODE",{});var iu=o(Zl);ip=i(iu,"label_column"),iu.forEach(a),bo.forEach(a),Fa.forEach(a),eo=c(t),Ua=s(t,"P",{});var ru=o(Ua);rp=i(ru,"Let\u2019s have a look at how can evaluate image classification models on large datasets."),ru.forEach(a),to=c(t),me=s(t,"H3",{class:!0});var wo=o(me);He=s(wo,"A",{id:!0,class:!0,href:!0});var cu=o(He);es=s(cu,"SPAN",{});var pu=o(es);v(pt.$$.fragment,pu),pu.forEach(a),cu.forEach(a),cp=c(wo),ts=s(wo,"SPAN",{});var du=o(ts);pp=i(du,"Handling large datasets"),du.forEach(a),wo.forEach(a),ao=c(t),Va=s(t,"P",{});var hu=o(Va);dp=i(hu,"The evaluator can be used on large datasets! Below, an example shows how to use it on ImageNet-1k for image classification. Beware that this example will require to download ~150 GB."),hu.forEach(a),lo=c(t),v(dt.$$.fragment,t),so=c(t),Be=s(t,"P",{});var ko=o(Be);hp=i(ko,"Since we are using "),as=s(ko,"CODE",{});var uu=o(as);up=i(uu,"datasets"),uu.forEach(a),fp=i(ko," to store data we make use of a technique called memory mappings. This means that the dataset is never fully loaded into memory which saves a lot of RAM. Running the above code only uses roughly 1.5 GB of RAM while the validation split is more than 30 GB big."),ko.forEach(a),this.h()},h(){d(u,"name","hf:doc:metadata"),d(u,"content",JSON.stringify(yu)),d(k,"id","using-the-evaluator"),d(k,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),d(k,"href","#using-the-evaluator"),d(f,"class","relative group"),d(ve,"href","custom_evaluator"),d(vt,"href","/docs/evaluate/pr_342/en/package_reference/evaluator_classes#evaluate.TextClassificationEvaluator"),d(_t,"href","/docs/evaluate/pr_342/en/package_reference/evaluator_classes#evaluate.TokenClassificationEvaluator"),d(gt,"href","/docs/evaluate/pr_342/en/package_reference/evaluator_classes#evaluate.QuestionAnsweringEvaluator"),d(Et,"href","/docs/evaluate/pr_342/en/package_reference/evaluator_classes#evaluate.ImageClassificationEvaluator"),d(bt,"href","/docs/evaluate/pr_342/en/package_reference/evaluator_classes#evaluate.Text2TextGenerationEvaluator"),d(wt,"href","/docs/evaluate/pr_342/en/package_reference/evaluator_classes#evaluate.SummarizationEvaluator"),d(kt,"href","/docs/evaluate/pr_342/en/package_reference/evaluator_classes#evaluate.TranslationEvaluator"),d($e,"id","text-classification"),d($e,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),d($e,"href","#text-classification"),d(ie,"class","relative group"),d(je,"id","evaluate-models-on-the-hub"),d(je,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),d(je,"href","#evaluate-models-on-the-hub"),d(re,"class","relative group"),d(De,"id","evaluate-multiple-metrics"),d(De,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),d(De,"href","#evaluate-multiple-metrics"),d(ce,"class","relative group"),d(Ct,"href","/docs/evaluate/pr_342/en/package_reference/main_classes#evaluate.combine"),d(Pe,"id","token-classification"),d(Pe,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),d(Pe,"href","#token-classification"),d(pe,"class","relative group"),d(Ae,"id","benchmarking-several-models"),d(Ae,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),d(Ae,"href","#benchmarking-several-models"),d(de,"class","relative group"),d(Bt,"align","left"),d(zt,"align","right"),d(Gt,"align","right"),d(Wt,"align","right"),d(Rt,"align","right"),d(Ut,"align","right"),d(Vt,"align","left"),d(Mt,"align","right"),d(Ft,"align","right"),d(Qt,"align","right"),d(Jt,"align","right"),d(Yt,"align","right"),d(Kt,"align","left"),d(Xt,"align","right"),d(Zt,"align","right"),d(ea,"align","right"),d(ta,"align","right"),d(aa,"align","right"),d(la,"align","left"),d(sa,"align","right"),d(oa,"align","right"),d(na,"align","right"),d(ia,"align","right"),d(ra,"align","right"),d(ca,"align","left"),d(pa,"align","right"),d(da,"align","right"),d(ha,"align","right"),d(ua,"align","right"),d(fa,"align","right"),d(ma,"align","left"),d(va,"align","right"),d(_a,"align","right"),d(ga,"align","right"),d(Ea,"align","right"),d(ba,"align","right"),d(wa,"align","left"),d(ka,"align","right"),d(ya,"align","right"),d($a,"align","right"),d(qa,"align","right"),d(ja,"align","right"),d(Ta,"align","left"),d(xa,"align","right"),d(Da,"align","right"),d(Ca,"align","right"),d(Pa,"align","right"),d(Aa,"align","right"),d(Se,"id","question-answering"),d(Se,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),d(Se,"href","#question-answering"),d(he,"class","relative group"),d(Ne,"id","confidence-intervals"),d(Ne,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),d(Ne,"href","#confidence-intervals"),d(ue,"class","relative group"),d(ot,"href","https://docs.scipy.org/doc/scipy/reference/generated/scipy.stats.bootstrap.html"),d(ot,"rel","nofollow"),d(Le,"id","image-classification"),d(Le,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),d(Le,"href","#image-classification"),d(fe,"class","relative group"),d(He,"id","handling-large-datasets"),d(He,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),d(He,"href","#handling-large-datasets"),d(me,"class","relative group")},m(t,p){e(document.head,u),h(t,G,p),h(t,f,p),e(f,k),e(k,oe),_(H,oe,null),e(f,Ve),e(f,ee),e(ee,ne),e(ee,O),e(O,ft),h(t,Me,p),h(t,W,p),e(W,yo),e(W,Qa),e(Qa,$o),e(W,qo),e(W,Ja),e(Ja,jo),e(W,To),e(W,ve),e(ve,xo),e(ve,Ya),e(Ya,Do),e(ve,Co),e(W,Po),h(t,ds,p),h(t,mt,p),e(mt,Ao),h(t,hs,p),h(t,w,p),e(w,_e),e(_e,Ka),e(Ka,Io),e(_e,Oo),e(_e,vt),e(vt,So),e(_e,No),e(w,Lo),e(w,ge),e(ge,Xa),e(Xa,Ho),e(ge,Bo),e(ge,_t),e(_t,zo),e(ge,Go),e(w,Wo),e(w,Ee),e(Ee,Za),e(Za,Ro),e(Ee,Uo),e(Ee,gt),e(gt,Vo),e(Ee,Mo),e(w,Fo),e(w,be),e(be,el),e(el,Qo),e(be,Jo),e(be,Et),e(Et,Yo),e(be,Ko),e(w,Xo),e(w,we),e(we,tl),e(tl,Zo),e(we,en),e(we,bt),e(bt,tn),e(we,an),e(w,ln),e(w,ke),e(ke,al),e(al,sn),e(ke,on),e(ke,wt),e(wt,nn),e(ke,rn),e(w,cn),e(w,ye),e(ye,ll),e(ll,pn),e(ye,dn),e(ye,kt),e(kt,hn),e(ye,un),h(t,us,p),h(t,yt,p),e(yt,fn),h(t,fs,p),h(t,ie,p),e(ie,$e),e($e,sl),_(Fe,sl,null),e(ie,mn),e(ie,ol),e(ol,vn),h(t,ms,p),h(t,$t,p),e($t,_n),h(t,vs,p),h(t,te,p),e(te,qt),e(qt,nl),e(nl,gn),e(qt,En),e(te,bn),e(te,jt),e(jt,il),e(il,wn),e(jt,kn),e(te,yn),e(te,$),e($,rl),e(rl,$n),e($,qn),e($,cl),e(cl,jn),e($,Tn),e($,pl),e(pl,xn),e($,Dn),e($,dl),e(dl,Cn),e($,Pn),e($,hl),e(hl,An),e($,In),e($,ul),e(ul,On),e($,Sn),h(t,_s,p),h(t,qe,p),e(qe,Nn),e(qe,fl),e(fl,Ln),e(qe,Hn),h(t,gs,p),h(t,re,p),e(re,je),e(je,ml),_(Qe,ml,null),e(re,Bn),e(re,vl),e(vl,zn),h(t,Es,p),h(t,R,p),e(R,Gn),e(R,_l),e(_l,Wn),e(R,Rn),e(R,gl),e(gl,Un),e(R,Vn),e(R,El),e(El,Mn),e(R,Fn),h(t,bs,p),h(t,Tt,p),e(Tt,Qn),h(t,ws,p),_(Je,t,p),h(t,ks,p),_(Te,t,p),h(t,ys,p),h(t,xt,p),e(xt,Jn),h(t,$s,p),_(Ye,t,p),h(t,qs,p),h(t,Dt,p),e(Dt,Yn),h(t,js,p),_(xe,t,p),h(t,Ts,p),h(t,ce,p),e(ce,De),e(De,bl),_(Ke,bl,null),e(ce,Kn),e(ce,wl),e(wl,Xn),h(t,xs,p),h(t,Ce,p),e(Ce,Zn),e(Ce,Ct),e(Ct,ei),e(Ce,ti),h(t,Ds,p),_(Xe,t,p),h(t,Cs,p),h(t,Pt,p),e(Pt,ai),h(t,Ps,p),_(Ze,t,p),h(t,As,p),h(t,At,p),e(At,li),h(t,Is,p),h(t,pe,p),e(pe,Pe),e(Pe,kl),_(et,kl,null),e(pe,si),e(pe,yl),e(yl,oi),h(t,Os,p),h(t,It,p),e(It,ni),h(t,Ss,p),h(t,U,p),e(U,Ot),e(Ot,$l),e($l,ii),e(Ot,ri),e(U,ci),e(U,St),e(St,ql),e(ql,pi),e(St,di),e(U,hi),e(U,q),e(q,jl),e(jl,ui),e(q,fi),e(q,Tl),e(Tl,mi),e(q,vi),e(q,xl),e(xl,_i),e(q,gi),e(q,Dl),e(Dl,Ei),e(q,bi),e(q,Cl),e(Cl,wi),e(q,ki),e(q,Pl),e(Pl,yi),e(q,$i),e(U,qi),e(U,Nt),e(Nt,Al),e(Al,ji),e(Nt,Ti),h(t,Ns,p),h(t,Lt,p),e(Lt,xi),h(t,Ls,p),h(t,de,p),e(de,Ae),e(Ae,Il),_(tt,Il,null),e(de,Di),e(de,Ol),e(Ol,Ci),h(t,Hs,p),h(t,Ie,p),e(Ie,Pi),e(Ie,Sl),e(Sl,Ai),e(Ie,Ii),h(t,Bs,p),_(at,t,p),h(t,zs,p),h(t,Ht,p),e(Ht,Oi),h(t,Gs,p),h(t,Oe,p),e(Oe,Nl),e(Nl,j),e(j,Bt),e(Bt,Si),e(j,Ni),e(j,zt),e(zt,Li),e(j,Hi),e(j,Gt),e(Gt,Bi),e(j,zi),e(j,Wt),e(Wt,Gi),e(j,Wi),e(j,Rt),e(Rt,Ri),e(j,Ui),e(j,Ut),e(Ut,Vi),e(Oe,Mi),e(Oe,y),e(y,T),e(T,Vt),e(Vt,Fi),e(T,Qi),e(T,Mt),e(Mt,Ji),e(T,Yi),e(T,Ft),e(Ft,Ki),e(T,Xi),e(T,Qt),e(Qt,Zi),e(T,er),e(T,Jt),e(Jt,tr),e(T,ar),e(T,Yt),e(Yt,lr),e(y,sr),e(y,x),e(x,Kt),e(Kt,or),e(x,nr),e(x,Xt),e(Xt,ir),e(x,rr),e(x,Zt),e(Zt,cr),e(x,pr),e(x,ea),e(ea,dr),e(x,hr),e(x,ta),e(ta,ur),e(x,fr),e(x,aa),e(aa,mr),e(y,vr),e(y,D),e(D,la),e(la,_r),e(D,gr),e(D,sa),e(sa,Er),e(D,br),e(D,oa),e(oa,wr),e(D,kr),e(D,na),e(na,yr),e(D,$r),e(D,ia),e(ia,qr),e(D,jr),e(D,ra),e(ra,Tr),e(y,xr),e(y,C),e(C,ca),e(ca,Dr),e(C,Cr),e(C,pa),e(pa,Pr),e(C,Ar),e(C,da),e(da,Ir),e(C,Or),e(C,ha),e(ha,Sr),e(C,Nr),e(C,ua),e(ua,Lr),e(C,Hr),e(C,fa),e(fa,Br),e(y,zr),e(y,P),e(P,ma),e(ma,Gr),e(P,Wr),e(P,va),e(va,Rr),e(P,Ur),e(P,_a),e(_a,Vr),e(P,Mr),e(P,ga),e(ga,Fr),e(P,Qr),e(P,Ea),e(Ea,Jr),e(P,Yr),e(P,ba),e(ba,Kr),e(y,Xr),e(y,A),e(A,wa),e(wa,Zr),e(A,ec),e(A,ka),e(ka,tc),e(A,ac),e(A,ya),e(ya,lc),e(A,sc),e(A,$a),e($a,oc),e(A,nc),e(A,qa),e(qa,ic),e(A,rc),e(A,ja),e(ja,cc),e(y,pc),e(y,I),e(I,Ta),e(Ta,dc),e(I,hc),e(I,xa),e(xa,uc),e(I,fc),e(I,Da),e(Da,mc),e(I,vc),e(I,Ca),e(Ca,_c),e(I,gc),e(I,Pa),e(Pa,Ec),e(I,bc),e(I,Aa),e(Aa,wc),h(t,Ws,p),h(t,he,p),e(he,Se),e(Se,Ll),_(lt,Ll,null),e(he,kc),e(he,Hl),e(Hl,yc),h(t,Rs,p),h(t,Ia,p),e(Ia,$c),h(t,Us,p),h(t,S,p),e(S,Oa),e(Oa,Bl),e(Bl,qc),e(Oa,jc),e(S,Tc),e(S,Sa),e(Sa,zl),e(zl,xc),e(Sa,Dc),e(S,Cc),e(S,Na),e(Na,Gl),e(Gl,Pc),e(Na,Ac),e(S,Ic),e(S,La),e(La,Wl),e(Wl,Oc),e(La,Sc),e(S,Nc),e(S,Ha),e(Ha,Rl),e(Rl,Lc),e(Ha,Hc),h(t,Vs,p),h(t,Ba,p),e(Ba,Bc),h(t,Ms,p),h(t,ue,p),e(ue,Ne),e(Ne,Ul),_(st,Ul,null),e(ue,zc),e(ue,Vl),e(Vl,Gc),h(t,Fs,p),h(t,V,p),e(V,Wc),e(V,ot),e(ot,Rc),e(V,Uc),e(V,Ml),e(Ml,Vc),e(V,Mc),e(V,Fl),e(Fl,Fc),e(V,Qc),h(t,Qs,p),_(nt,t,p),h(t,Js,p),h(t,za,p),e(za,Jc),h(t,Ys,p),_(it,t,p),h(t,Ks,p),h(t,fe,p),e(fe,Le),e(Le,Ql),_(rt,Ql,null),e(fe,Yc),e(fe,Jl),e(Jl,Kc),h(t,Xs,p),h(t,Ga,p),e(Ga,Xc),h(t,Zs,p),h(t,ae,p),e(ae,Wa),e(Wa,Yl),e(Yl,Zc),e(Wa,ep),e(ae,tp),e(ae,Ra),e(Ra,Kl),e(Kl,ap),e(Ra,lp),e(ae,sp),e(ae,ct),e(ct,Xl),e(Xl,op),e(ct,np),e(ct,Zl),e(Zl,ip),h(t,eo,p),h(t,Ua,p),e(Ua,rp),h(t,to,p),h(t,me,p),e(me,He),e(He,es),_(pt,es,null),e(me,cp),e(me,ts),e(ts,pp),h(t,ao,p),h(t,Va,p),e(Va,dp),h(t,lo,p),_(dt,t,p),h(t,so,p),h(t,Be,p),e(Be,hp),e(Be,as),e(as,up),e(Be,fp),oo=!0},p(t,[p]){const ht={};p&2&&(ht.$$scope={dirty:p,ctx:t}),Te.$set(ht);const ls={};p&2&&(ls.$$scope={dirty:p,ctx:t}),xe.$set(ls)},i(t){oo||(g(H.$$.fragment,t),g(Fe.$$.fragment,t),g(Qe.$$.fragment,t),g(Je.$$.fragment,t),g(Te.$$.fragment,t),g(Ye.$$.fragment,t),g(xe.$$.fragment,t),g(Ke.$$.fragment,t),g(Xe.$$.fragment,t),g(Ze.$$.fragment,t),g(et.$$.fragment,t),g(tt.$$.fragment,t),g(at.$$.fragment,t),g(lt.$$.fragment,t),g(st.$$.fragment,t),g(nt.$$.fragment,t),g(it.$$.fragment,t),g(rt.$$.fragment,t),g(pt.$$.fragment,t),g(dt.$$.fragment,t),oo=!0)},o(t){E(H.$$.fragment,t),E(Fe.$$.fragment,t),E(Qe.$$.fragment,t),E(Je.$$.fragment,t),E(Te.$$.fragment,t),E(Ye.$$.fragment,t),E(xe.$$.fragment,t),E(Ke.$$.fragment,t),E(Xe.$$.fragment,t),E(Ze.$$.fragment,t),E(et.$$.fragment,t),E(tt.$$.fragment,t),E(at.$$.fragment,t),E(lt.$$.fragment,t),E(st.$$.fragment,t),E(nt.$$.fragment,t),E(it.$$.fragment,t),E(rt.$$.fragment,t),E(pt.$$.fragment,t),E(dt.$$.fragment,t),oo=!1},d(t){a(u),t&&a(G),t&&a(f),b(H),t&&a(Me),t&&a(W),t&&a(ds),t&&a(mt),t&&a(hs),t&&a(w),t&&a(us),t&&a(yt),t&&a(fs),t&&a(ie),b(Fe),t&&a(ms),t&&a($t),t&&a(vs),t&&a(te),t&&a(_s),t&&a(qe),t&&a(gs),t&&a(re),b(Qe),t&&a(Es),t&&a(R),t&&a(bs),t&&a(Tt),t&&a(ws),b(Je,t),t&&a(ks),b(Te,t),t&&a(ys),t&&a(xt),t&&a($s),b(Ye,t),t&&a(qs),t&&a(Dt),t&&a(js),b(xe,t),t&&a(Ts),t&&a(ce),b(Ke),t&&a(xs),t&&a(Ce),t&&a(Ds),b(Xe,t),t&&a(Cs),t&&a(Pt),t&&a(Ps),b(Ze,t),t&&a(As),t&&a(At),t&&a(Is),t&&a(pe),b(et),t&&a(Os),t&&a(It),t&&a(Ss),t&&a(U),t&&a(Ns),t&&a(Lt),t&&a(Ls),t&&a(de),b(tt),t&&a(Hs),t&&a(Ie),t&&a(Bs),b(at,t),t&&a(zs),t&&a(Ht),t&&a(Gs),t&&a(Oe),t&&a(Ws),t&&a(he),b(lt),t&&a(Rs),t&&a(Ia),t&&a(Us),t&&a(S),t&&a(Vs),t&&a(Ba),t&&a(Ms),t&&a(ue),b(st),t&&a(Fs),t&&a(V),t&&a(Qs),b(nt,t),t&&a(Js),t&&a(za),t&&a(Ys),b(it,t),t&&a(Ks),t&&a(fe),b(rt),t&&a(Xs),t&&a(Ga),t&&a(Zs),t&&a(ae),t&&a(eo),t&&a(Ua),t&&a(to),t&&a(me),b(pt),t&&a(ao),t&&a(Va),t&&a(lo),b(dt,t),t&&a(so),t&&a(Be)}}}const yu={local:"using-the-evaluator",sections:[{local:"text-classification",sections:[{local:"evaluate-models-on-the-hub",title:"Evaluate models on the Hub"},{local:"evaluate-multiple-metrics",title:"Evaluate multiple metrics"}],title:"Text classification"},{local:"token-classification",sections:[{local:"benchmarking-several-models",title:"Benchmarking several models"}],title:"Token Classification"},{local:"question-answering",sections:[{local:"confidence-intervals",title:"Confidence intervals"}],title:"Question Answering"},{local:"image-classification",sections:[{local:"handling-large-datasets",title:"Handling large datasets"}],title:"Image classification"}],title:"Using the `evaluator`"};function $u(ut){return Eu(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class Du extends mu{constructor(u){super();vu(this,u,$u,ku,_u,{})}}export{Du as default,yu as metadata};
