import{S as Ph,i as Ch,s as Dh,e as s,k as p,w as d,t as n,M as Oh,c as l,d as a,m as u,a as o,x as m,h as r,b as h,N as Ih,G as t,g as c,y as f,q as v,o as y,B as _,v as Sh}from"../chunks/vendor-hf-doc-builder.js";import{T as Nh}from"../chunks/Tip-hf-doc-builder.js";import{I as P}from"../chunks/IconCopyLink-hf-doc-builder.js";import{C as b}from"../chunks/CodeBlock-hf-doc-builder.js";function Mh(Xs){let w,ne,k,C,Y;return{c(){w=s("p"),ne=n("Note that features always describe the type of a single input element. In general we will add lists of elements so you can always think of a list around the types in "),k=s("code"),C=n("features"),Y=n(". Evaluate accepts various input formats (Python lists, NumPy arrays, PyTorch tensors, etc.) and converts them to an appropriate format for storage and computation.")},l(A){w=l(A,"P",{});var N=o(w);ne=r(N,"Note that features always describe the type of a single input element. In general we will add lists of elements so you can always think of a list around the types in "),k=l(N,"CODE",{});var re=o(k);C=r(re,"features"),re.forEach(a),Y=r(N,". Evaluate accepts various input formats (Python lists, NumPy arrays, PyTorch tensors, etc.) and converts them to an appropriate format for storage and computation."),N.forEach(a)},m(A,N){c(A,w,N),t(w,ne),t(w,k),t(k,C),t(w,Y)},d(A){A&&a(w)}}}function Hh(Xs){let w,ne,k,C,Y,A,N,re,Zo,Zs,It,en,el,J,ie,pa,Be,tn,ua,an,tl,St,sn,al,M,ce,ha,ln,on,Le,nn,rn,cn,pe,da,pn,un,Fe,hn,dn,mn,ue,ma,fn,vn,We,yn,_n,sl,he,gn,Ge,bn,wn,ll,Re,Nt,ep,ol,de,En,Mt,$n,jn,nl,z,me,fa,Ue,kn,va,xn,rl,fe,qn,ya,Tn,An,il,Ve,cl,Ht,Pn,pl,Ye,ul,K,ve,_a,Je,Cn,ga,Dn,hl,Bt,On,dl,ze,ml,ye,In,Lt,Sn,Nn,fl,Q,_e,ba,Ke,Mn,wa,Hn,vl,ge,Bn,Ft,Ln,Fn,yl,Qe,_l,X,be,Ea,Xe,Wn,$a,Gn,gl,we,Rn,Wt,Un,Vn,bl,Ee,ja,Ze,ka,Yn,Jn,xa,zn,Kn,g,et,qa,Ta,Qn,Xn,Aa,Zn,er,tt,Pa,Ca,tr,ar,Da,sr,lr,at,Oa,Ia,or,nr,st,rr,Sa,ir,cr,pr,lt,Na,Ma,ur,hr,Ha,dr,mr,ot,Ba,La,fr,vr,Fa,yr,_r,nt,Wa,Ga,gr,br,Ra,wr,Er,rt,Ua,Va,$r,jr,Ya,kr,xr,it,Ja,za,qr,Tr,Ka,Ar,wl,$e,Pr,Qa,Cr,Dr,El,ct,$l,je,Or,Xa,Ir,Sr,jl,pt,kl,Gt,Nr,xl,ut,ql,ke,Tl,Z,xe,Za,ht,Mr,es,Hr,Al,Rt,Br,Pl,qe,ts,Lr,Fr,as,Wr,Cl,x,Gr,Ut,Rr,Ur,Vt,Vr,Yr,Yt,Jr,zr,ss,Kr,Qr,Dl,ee,Te,ls,dt,Xr,os,Zr,Ol,D,ei,ns,ti,ai,rs,si,li,is,oi,ni,Il,mt,Sl,H,ri,cs,ii,ci,ps,pi,ui,Nl,te,Ae,us,ft,hi,hs,di,Ml,q,mi,ds,fi,vi,ms,yi,_i,fs,gi,bi,vs,wi,Ei,Hl,vt,Bl,O,$i,ys,ji,ki,_s,xi,qi,gs,Ti,Ai,Ll,yt,Fl,Jt,Pi,Wl,_t,Gl,ae,Pe,bs,gt,Ci,ws,Di,Rl,T,Oi,Es,Ii,Si,$s,Ni,Mi,bt,Hi,Bi,js,Li,Fi,Ul,zt,Wi,Vl,B,Gi,ks,Ri,Ui,xs,Vi,Yi,Yl,Kt,Ji,Jl,se,Ce,qs,wt,zi,Ts,Ki,zl,De,Qi,Qt,Xi,Zi,Kl,Et,Ql,L,ec,As,tc,ac,Ps,sc,lc,Xl,$t,Zl,le,Oe,Cs,jt,oc,Ds,nc,eo,Ie,rc,Xt,ic,cc,to,kt,ao,Zt,pc,so,xt,lo,ea,uc,oo,Se,hc,ta,dc,mc,no,qt,ro,oe,Ne,Os,Tt,fc,Is,vc,io,E,yc,aa,_c,gc,Ss,bc,wc,At,Ec,$c,Ns,jc,kc,Ms,xc,qc,co,F,Tc,Hs,Ac,Pc,Bs,Cc,Dc,po,Pt,uo,W,Oc,Ls,Ic,Sc,Fs,Nc,Mc,ho,Ct,mo,G,Hc,Ws,Bc,Lc,Gs,Fc,Wc,fo,Dt,vo,$,Gc,Rs,Rc,Uc,Us,Vc,Yc,Vs,Jc,zc,Ys,Kc,Qc,Js,Xc,Zc,yo;return A=new P({}),Be=new P({}),Ue=new P({}),Ve=new b({props:{code:`import evaluate
accuracy = evaluate.load("accuracy")`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">import</span> evaluate
<span class="hljs-meta">&gt;&gt;&gt; </span>accuracy = evaluate.load(<span class="hljs-string">&quot;accuracy&quot;</span>)`}}),Ye=new b({props:{code:'word_length = evaluate.load("word_length", module_type="measurement")',highlighted:'<span class="hljs-meta">&gt;&gt;&gt; </span>word_length = evaluate.load(<span class="hljs-string">&quot;word_length&quot;</span>, module_type=<span class="hljs-string">&quot;measurement&quot;</span>)'}}),Je=new P({}),ze=new b({props:{code:'element_count = evaluate.load("lvwerra/element_count", module_type="measurement")',highlighted:'<span class="hljs-meta">&gt;&gt;&gt; </span>element_count = evaluate.load(<span class="hljs-string">&quot;lvwerra/element_count&quot;</span>, module_type=<span class="hljs-string">&quot;measurement&quot;</span>)'}}),Ke=new P({}),Qe=new b({props:{code:`evaluate.list_evaluation_modules(
  module_type="comparison",
  include_community=False, 
  with_details=True)
`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>evaluate.list_evaluation_modules(
<span class="hljs-meta">... </span>  module_type=<span class="hljs-string">&quot;comparison&quot;</span>,
<span class="hljs-meta">... </span>  include_community=<span class="hljs-literal">False</span>, 
<span class="hljs-meta">... </span>  with_details=<span class="hljs-literal">True</span>)

[{<span class="hljs-string">&#x27;name&#x27;</span>: <span class="hljs-string">&#x27;mcnemar&#x27;</span>, <span class="hljs-string">&#x27;type&#x27;</span>: <span class="hljs-string">&#x27;comparison&#x27;</span>, <span class="hljs-string">&#x27;community&#x27;</span>: <span class="hljs-literal">False</span>, <span class="hljs-string">&#x27;likes&#x27;</span>: <span class="hljs-number">1</span>},
 {<span class="hljs-string">&#x27;name&#x27;</span>: <span class="hljs-string">&#x27;exact_match&#x27;</span>, <span class="hljs-string">&#x27;type&#x27;</span>: <span class="hljs-string">&#x27;comparison&#x27;</span>, <span class="hljs-string">&#x27;community&#x27;</span>: <span class="hljs-literal">False</span>, <span class="hljs-string">&#x27;likes&#x27;</span>: <span class="hljs-number">0</span>}]`}}),Xe=new P({}),ct=new b({props:{code:`accuracy = evaluate.load("accuracy")
accuracy.description`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>accuracy = evaluate.load(<span class="hljs-string">&quot;accuracy&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>accuracy.description
Accuracy <span class="hljs-keyword">is</span> the proportion of correct predictions among the total number of cases processed. It can be computed <span class="hljs-keyword">with</span>:
Accuracy = (TP + TN) / (TP + TN + FP + FN)
 Where:
TP: <span class="hljs-literal">True</span> positive
TN: <span class="hljs-literal">True</span> negative
FP: <span class="hljs-literal">False</span> positive
FN: <span class="hljs-literal">False</span> negative`}}),pt=new b({props:{code:"accuracy.citation",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>accuracy.citation
<span class="hljs-meta">@article{scikit-learn,</span>
  title={Scikit-learn: Machine Learning <span class="hljs-keyword">in</span> {P}ython},
  author={Pedregosa, F. <span class="hljs-keyword">and</span> Varoquaux, G. <span class="hljs-keyword">and</span> Gramfort, A. <span class="hljs-keyword">and</span> Michel, V.
         <span class="hljs-keyword">and</span> Thirion, B. <span class="hljs-keyword">and</span> Grisel, O. <span class="hljs-keyword">and</span> Blondel, M. <span class="hljs-keyword">and</span> Prettenhofer, P.
         <span class="hljs-keyword">and</span> Weiss, R. <span class="hljs-keyword">and</span> Dubourg, V. <span class="hljs-keyword">and</span> Vanderplas, J. <span class="hljs-keyword">and</span> Passos, A. <span class="hljs-keyword">and</span>
         Cournapeau, D. <span class="hljs-keyword">and</span> Brucher, M. <span class="hljs-keyword">and</span> Perrot, M. <span class="hljs-keyword">and</span> Duchesnay, E.},
  journal={Journal of Machine Learning Research},
  volume={<span class="hljs-number">12</span>},
  pages={<span class="hljs-number">2825</span>--<span class="hljs-number">2830</span>},
  year={<span class="hljs-number">2011</span>}
}`}}),ut=new b({props:{code:"accuracy.features",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>accuracy.features
{
    <span class="hljs-string">&#x27;predictions&#x27;</span>: Value(dtype=<span class="hljs-string">&#x27;int32&#x27;</span>, <span class="hljs-built_in">id</span>=<span class="hljs-literal">None</span>),
    <span class="hljs-string">&#x27;references&#x27;</span>: Value(dtype=<span class="hljs-string">&#x27;int32&#x27;</span>, <span class="hljs-built_in">id</span>=<span class="hljs-literal">None</span>)
}`}}),ke=new Nh({props:{$$slots:{default:[Mh]},$$scope:{ctx:Xs}}}),ht=new P({}),dt=new P({}),mt=new b({props:{code:"accuracy.compute(references=[0,1,0,1], predictions=[1,0,0,1])",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>accuracy.compute(references=[<span class="hljs-number">0</span>,<span class="hljs-number">1</span>,<span class="hljs-number">0</span>,<span class="hljs-number">1</span>], predictions=[<span class="hljs-number">1</span>,<span class="hljs-number">0</span>,<span class="hljs-number">0</span>,<span class="hljs-number">1</span>])
{<span class="hljs-string">&#x27;accuracy&#x27;</span>: <span class="hljs-number">0.5</span>}`}}),ft=new P({}),vt=new b({props:{code:`for ref, pred in zip([0,1,0,1], [1,0,0,1]):
    accuracy.add(references=ref, predictions=pred)
accuracy.compute()`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">for</span> ref, pred <span class="hljs-keyword">in</span> <span class="hljs-built_in">zip</span>([<span class="hljs-number">0</span>,<span class="hljs-number">1</span>,<span class="hljs-number">0</span>,<span class="hljs-number">1</span>], [<span class="hljs-number">1</span>,<span class="hljs-number">0</span>,<span class="hljs-number">0</span>,<span class="hljs-number">1</span>]):
<span class="hljs-meta">&gt;&gt;&gt; </span>    accuracy.add(references=ref, predictions=pred)
<span class="hljs-meta">&gt;&gt;&gt; </span>accuracy.compute()
{<span class="hljs-string">&#x27;accuracy&#x27;</span>: <span class="hljs-number">0.5</span>}`}}),yt=new b({props:{code:`for refs, preds in zip([[0,1],[0,1]], [[1,0],[0,1]]):
    accuracy.add_batch(references=refs, predictions=preds)
accuracy.compute()`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">for</span> refs, preds <span class="hljs-keyword">in</span> <span class="hljs-built_in">zip</span>([[<span class="hljs-number">0</span>,<span class="hljs-number">1</span>],[<span class="hljs-number">0</span>,<span class="hljs-number">1</span>]], [[<span class="hljs-number">1</span>,<span class="hljs-number">0</span>],[<span class="hljs-number">0</span>,<span class="hljs-number">1</span>]]):
<span class="hljs-meta">&gt;&gt;&gt; </span>    accuracy.add_batch(references=refs, predictions=preds)
<span class="hljs-meta">&gt;&gt;&gt; </span>accuracy.compute()
{<span class="hljs-string">&#x27;accuracy&#x27;</span>: <span class="hljs-number">0.5</span>}`}}),_t=new b({props:{code:`for model_inputs, gold_standards in evaluation_dataset:
    predictions = model(model_inputs)
    metric.add_batch(references=gold_standards, predictions=predictions)
metric.compute()`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">for</span> model_inputs, gold_standards <span class="hljs-keyword">in</span> evaluation_dataset:
<span class="hljs-meta">&gt;&gt;&gt; </span>    predictions = model(model_inputs)
<span class="hljs-meta">&gt;&gt;&gt; </span>    metric.add_batch(references=gold_standards, predictions=predictions)
<span class="hljs-meta">&gt;&gt;&gt; </span>metric.compute()`}}),gt=new P({}),wt=new P({}),Et=new b({props:{code:'clf_metrics = evaluate.combine(["accuracy", "f1", "precision", "recall"])',highlighted:'<span class="hljs-meta">&gt;&gt;&gt; </span>clf_metrics = evaluate.combine([<span class="hljs-string">&quot;accuracy&quot;</span>, <span class="hljs-string">&quot;f1&quot;</span>, <span class="hljs-string">&quot;precision&quot;</span>, <span class="hljs-string">&quot;recall&quot;</span>])'}}),$t=new b({props:{code:`clf_metrics.compute(predictions=[0, 1, 0], references=[0, 1, 1])
`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>clf_metrics.compute(predictions=[<span class="hljs-number">0</span>, <span class="hljs-number">1</span>, <span class="hljs-number">0</span>], references=[<span class="hljs-number">0</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>])

{
  <span class="hljs-string">&#x27;accuracy&#x27;</span>: <span class="hljs-number">0.667</span>,
  <span class="hljs-string">&#x27;f1&#x27;</span>: <span class="hljs-number">0.667</span>,
  <span class="hljs-string">&#x27;precision&#x27;</span>: <span class="hljs-number">1.0</span>,
  <span class="hljs-string">&#x27;recall&#x27;</span>: <span class="hljs-number">0.5</span>
}`}}),jt=new P({}),kt=new b({props:{code:`result = accuracy.compute(references=[0,1,0,1], predictions=[1,0,0,1])

hyperparams = {"model": "bert-base-uncased"}
evaluate.save("./results/"experiment="run 42", **result, **hyperparams)`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>result = accuracy.compute(references=[<span class="hljs-number">0</span>,<span class="hljs-number">1</span>,<span class="hljs-number">0</span>,<span class="hljs-number">1</span>], predictions=[<span class="hljs-number">1</span>,<span class="hljs-number">0</span>,<span class="hljs-number">0</span>,<span class="hljs-number">1</span>])

<span class="hljs-meta">&gt;&gt;&gt; </span>hyperparams = {<span class="hljs-string">&quot;model&quot;</span>: <span class="hljs-string">&quot;bert-base-uncased&quot;</span>}
<span class="hljs-meta">&gt;&gt;&gt; </span>evaluate.save(<span class="hljs-string">&quot;./results/&quot;</span>experiment=<span class="hljs-string">&quot;run 42&quot;</span>, **result, **hyperparams)
PosixPath(<span class="hljs-string">&#x27;results/result-2022_05_30-22_09_11.json&#x27;</span>)`}}),xt=new b({props:{code:`{
    "experiment": "run 42",
    "accuracy": 0.5,
    "model": "bert-base-uncased",
    "_timestamp": "2022-05-30T22:09:11.959469",
    "_git_commit_hash": "123456789abcdefghijkl",
    "_evaluate_version": "0.1.0",
    "_python_version": "3.9.12 (main, Mar 26 2022, 15:51:15) \\n[Clang 13.1.6 (clang-1316.0.21.2)]",
    "_interpreter_path": "/Users/leandro/git/evaluate/env/bin/python"
}`,highlighted:`<span class="hljs-punctuation">{</span>
    <span class="hljs-attr">&quot;experiment&quot;</span><span class="hljs-punctuation">:</span> <span class="hljs-string">&quot;run 42&quot;</span><span class="hljs-punctuation">,</span>
    <span class="hljs-attr">&quot;accuracy&quot;</span><span class="hljs-punctuation">:</span> <span class="hljs-number">0.5</span><span class="hljs-punctuation">,</span>
    <span class="hljs-attr">&quot;model&quot;</span><span class="hljs-punctuation">:</span> <span class="hljs-string">&quot;bert-base-uncased&quot;</span><span class="hljs-punctuation">,</span>
    <span class="hljs-attr">&quot;_timestamp&quot;</span><span class="hljs-punctuation">:</span> <span class="hljs-string">&quot;2022-05-30T22:09:11.959469&quot;</span><span class="hljs-punctuation">,</span>
    <span class="hljs-attr">&quot;_git_commit_hash&quot;</span><span class="hljs-punctuation">:</span> <span class="hljs-string">&quot;123456789abcdefghijkl&quot;</span><span class="hljs-punctuation">,</span>
    <span class="hljs-attr">&quot;_evaluate_version&quot;</span><span class="hljs-punctuation">:</span> <span class="hljs-string">&quot;0.1.0&quot;</span><span class="hljs-punctuation">,</span>
    <span class="hljs-attr">&quot;_python_version&quot;</span><span class="hljs-punctuation">:</span> <span class="hljs-string">&quot;3.9.12 (main, Mar 26 2022, 15:51:15) \\n[Clang 13.1.6 (clang-1316.0.21.2)]&quot;</span><span class="hljs-punctuation">,</span>
    <span class="hljs-attr">&quot;_interpreter_path&quot;</span><span class="hljs-punctuation">:</span> <span class="hljs-string">&quot;/Users/leandro/git/evaluate/env/bin/python&quot;</span>
<span class="hljs-punctuation">}</span>`}}),qt=new b({props:{code:`evaluate.push_to_hub(
  model_id="huggingface/gpt2-wikitext2",  # model repository on hub
  metric_value=0.5,                       # metric value
  metric_type="bleu",                     # metric name, e.g. accuracy.name
  metric_name="BLEU",                     # pretty name which is displayed
  dataset_type="wikitext",                # dataset name on the hub
  dataset_name="WikiText",                # pretty name
  dataset_split="test",                   # dataset split used
  task_type="text-generation",            # task id, see https://github.com/huggingface/datasets/blob/master/src/datasets/utils/resources/tasks.json
  task_name="Text Generation"             # pretty name for task
)`,highlighted:`evaluate.push_to_hub(
  model_id=<span class="hljs-string">&quot;huggingface/gpt2-wikitext2&quot;</span>,  <span class="hljs-comment"># model repository on hub</span>
  metric_value=<span class="hljs-number">0.5</span>,                       <span class="hljs-comment"># metric value</span>
  metric_type=<span class="hljs-string">&quot;bleu&quot;</span>,                     <span class="hljs-comment"># metric name, e.g. accuracy.name</span>
  metric_name=<span class="hljs-string">&quot;BLEU&quot;</span>,                     <span class="hljs-comment"># pretty name which is displayed</span>
  dataset_type=<span class="hljs-string">&quot;wikitext&quot;</span>,                <span class="hljs-comment"># dataset name on the hub</span>
  dataset_name=<span class="hljs-string">&quot;WikiText&quot;</span>,                <span class="hljs-comment"># pretty name</span>
  dataset_split=<span class="hljs-string">&quot;test&quot;</span>,                   <span class="hljs-comment"># dataset split used</span>
  task_type=<span class="hljs-string">&quot;text-generation&quot;</span>,            <span class="hljs-comment"># task id, see https://github.com/huggingface/datasets/blob/master/src/datasets/utils/resources/tasks.json</span>
  task_name=<span class="hljs-string">&quot;Text Generation&quot;</span>             <span class="hljs-comment"># pretty name for task</span>
)`}}),Tt=new P({}),Pt=new b({props:{code:`from transformers import pipeline
from datasets import load_dataset
from evaluate import evaluator
import evaluate

pipe = pipeline("text-classification", model="lvwerra/distilbert-imdb", device=0)
data = load_dataset("imdb", split="test").shuffle().select(range(1000))
metric = evaluate.load("accuracy")`,highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> pipeline
<span class="hljs-keyword">from</span> datasets <span class="hljs-keyword">import</span> load_dataset
<span class="hljs-keyword">from</span> evaluate <span class="hljs-keyword">import</span> evaluator
<span class="hljs-keyword">import</span> evaluate

pipe = pipeline(<span class="hljs-string">&quot;text-classification&quot;</span>, model=<span class="hljs-string">&quot;lvwerra/distilbert-imdb&quot;</span>, device=<span class="hljs-number">0</span>)
data = load_dataset(<span class="hljs-string">&quot;imdb&quot;</span>, split=<span class="hljs-string">&quot;test&quot;</span>).shuffle().select(<span class="hljs-built_in">range</span>(<span class="hljs-number">1000</span>))
metric = evaluate.load(<span class="hljs-string">&quot;accuracy&quot;</span>)`}}),Ct=new b({props:{code:`eval = evaluator("text-classification")

results = eval.compute(model_or_pipeline=pipe, data=data, metric=metric,
                       label_mapping={"NEGATIVE": 0, "POSITIVE": 1},)

print(results)`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-built_in">eval</span> = evaluator(<span class="hljs-string">&quot;text-classification&quot;</span>)

<span class="hljs-meta">&gt;&gt;&gt; </span>results = <span class="hljs-built_in">eval</span>.compute(model_or_pipeline=pipe, data=data, metric=metric,
<span class="hljs-meta">... </span>                       label_mapping={<span class="hljs-string">&quot;NEGATIVE&quot;</span>: <span class="hljs-number">0</span>, <span class="hljs-string">&quot;POSITIVE&quot;</span>: <span class="hljs-number">1</span>},)

<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-built_in">print</span>(results)
{<span class="hljs-string">&#x27;accuracy&#x27;</span>: <span class="hljs-number">0.934</span>}`}}),Dt=new b({props:{code:`results = eval.compute(model_or_pipeline=pipe, data=data, metric=metric,
                       label_mapping={"NEGATIVE": 0, "POSITIVE": 1},
                       strategy="bootstrap", n_resamples=200)

print(results)`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>results = <span class="hljs-built_in">eval</span>.compute(model_or_pipeline=pipe, data=data, metric=metric,
<span class="hljs-meta">... </span>                       label_mapping={<span class="hljs-string">&quot;NEGATIVE&quot;</span>: <span class="hljs-number">0</span>, <span class="hljs-string">&quot;POSITIVE&quot;</span>: <span class="hljs-number">1</span>},
<span class="hljs-meta">... </span>                       strategy=<span class="hljs-string">&quot;bootstrap&quot;</span>, n_resamples=<span class="hljs-number">200</span>)

<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-built_in">print</span>(results)
{<span class="hljs-string">&#x27;accuracy&#x27;</span>: 
    {
      <span class="hljs-string">&#x27;confidence_interval&#x27;</span>: (<span class="hljs-number">0.906</span>, <span class="hljs-number">0.9406749892841922</span>),
      <span class="hljs-string">&#x27;standard_error&#x27;</span>: <span class="hljs-number">0.00865213251082787</span>,
      <span class="hljs-string">&#x27;score&#x27;</span>: <span class="hljs-number">0.923</span>
    }
}`}}),{c(){w=s("meta"),ne=p(),k=s("h1"),C=s("a"),Y=s("span"),d(A.$$.fragment),N=p(),re=s("span"),Zo=n("A quick tour"),Zs=p(),It=s("p"),en=n("\u{1F917} Evaluate provides access to a wide range of evaluation tools. It covers a range of modalities such as text, computer vision, audio, etc. as well as tools to evaluate models or datasets. These tools are split into three categories."),el=p(),J=s("h2"),ie=s("a"),pa=s("span"),d(Be.$$.fragment),tn=p(),ua=s("span"),an=n("Types of evaluations"),tl=p(),St=s("p"),sn=n("There are different aspects of a typical machine learning pipeline that can be evaluated and for each aspect \u{1F917} Evaluate provides a tool:"),al=p(),M=s("ul"),ce=s("li"),ha=s("strong"),ln=n("Metric"),on=n(": A metric is used to evaluate a model\u2019s performance and usually involves the model\u2019s predictions as well as some ground truth labels. You can find all integrated metrics at "),Le=s("a"),nn=n("evaluate-metric"),rn=n("."),cn=p(),pe=s("li"),da=s("strong"),pn=n("Comparison"),un=n(": A comparison is used to compare two models. This can for example be done by comparing their predictions to ground truth labels and computing their agreement. You can find all integrated comparisons at "),Fe=s("a"),hn=n("evaluate-comparison"),dn=n("."),mn=p(),ue=s("li"),ma=s("strong"),fn=n("Measurement"),vn=n(": The dataset is as important as the model trained on it. With measurements one can investigate a dataset\u2019s properties. You can find all integrated measurements at "),We=s("a"),yn=n("evaluate-measurement"),_n=n("."),sl=p(),he=s("p"),gn=n("Each of these evaluation modules live on Hugging Face Hub as a Space. They come with an interactive widget and a documentation card documenting its use and limitations. For example "),Ge=s("a"),bn=n("accuracy"),wn=n(":"),ll=p(),Re=s("div"),Nt=s("img"),ol=p(),de=s("p"),En=n("Each metric, comparison, and measurement is a separate Python module, but for using any of them, there is a single entry point: "),Mt=s("a"),$n=n("evaluate.load()"),jn=n("!"),nl=p(),z=s("h2"),me=s("a"),fa=s("span"),d(Ue.$$.fragment),kn=p(),va=s("span"),xn=n("Load"),rl=p(),fe=s("p"),qn=n("Any metric, comparison, or measurement is loaded with the "),ya=s("code"),Tn=n("evaluate.load"),An=n(" function:"),il=p(),d(Ve.$$.fragment),cl=p(),Ht=s("p"),Pn=n("If you want to make sure you are loading the right type of evaluation (especially if there are name clashes) you can explicitly pass the type:"),pl=p(),d(Ye.$$.fragment),ul=p(),K=s("h3"),ve=s("a"),_a=s("span"),d(Je.$$.fragment),Cn=p(),ga=s("span"),Dn=n("Community modules"),hl=p(),Bt=s("p"),On=n("Besides the modules implemented in \u{1F917} Evaluate you can also load any community module by specifying the repository ID of the metric implementation:"),dl=p(),d(ze.$$.fragment),ml=p(),ye=s("p"),In=n("See the "),Lt=s("a"),Sn=n("Creating and Sharing Guide"),Nn=n(" for information about uploading custom metrics."),fl=p(),Q=s("h3"),_e=s("a"),ba=s("span"),d(Ke.$$.fragment),Mn=p(),wa=s("span"),Hn=n("List available modules"),vl=p(),ge=s("p"),Bn=n("With "),Ft=s("a"),Ln=n("list_evaluation_modules()"),Fn=n(" you can check what modules are available on the hub. You can also filter for a specific modules and skip community metrics if you want. You can also see additional information such as likes:"),yl=p(),d(Qe.$$.fragment),_l=p(),X=s("h2"),be=s("a"),Ea=s("span"),d(Xe.$$.fragment),Wn=p(),$a=s("span"),Gn=n("Module attributes"),gl=p(),we=s("p"),Rn=n("All evalution modules come with a range of useful attributes that help to use a module stored in a "),Wt=s("a"),Un=n("EvaluationModuleInfo"),Vn=n(" object."),bl=p(),Ee=s("table"),ja=s("thead"),Ze=s("tr"),ka=s("th"),Yn=n("Attribute"),Jn=p(),xa=s("th"),zn=n("Description"),Kn=p(),g=s("tbody"),et=s("tr"),qa=s("td"),Ta=s("code"),Qn=n("description"),Xn=p(),Aa=s("td"),Zn=n("A short description of the evaluation module."),er=p(),tt=s("tr"),Pa=s("td"),Ca=s("code"),tr=n("citation"),ar=p(),Da=s("td"),sr=n("A BibTex string for citation when available."),lr=p(),at=s("tr"),Oa=s("td"),Ia=s("code"),or=n("features"),nr=p(),st=s("td"),rr=n("A "),Sa=s("code"),ir=n("Features"),cr=n(" object defining the input format."),pr=p(),lt=s("tr"),Na=s("td"),Ma=s("code"),ur=n("inputs_description"),hr=p(),Ha=s("td"),dr=n("This is equivalent to the modules docstring."),mr=p(),ot=s("tr"),Ba=s("td"),La=s("code"),fr=n("homepage"),vr=p(),Fa=s("td"),yr=n("The homepage of the module."),_r=p(),nt=s("tr"),Wa=s("td"),Ga=s("code"),gr=n("license"),br=p(),Ra=s("td"),wr=n("The license of the module."),Er=p(),rt=s("tr"),Ua=s("td"),Va=s("code"),$r=n("codebase_urls"),jr=p(),Ya=s("td"),kr=n("Link to the code behind the module."),xr=p(),it=s("tr"),Ja=s("td"),za=s("code"),qr=n("reference_urls"),Tr=p(),Ka=s("td"),Ar=n("Additional reference URLs."),wl=p(),$e=s("p"),Pr=n("Let\u2019s have a look at a few examples. First, let\u2019s look at the "),Qa=s("code"),Cr=n("description"),Dr=n(" attribute of the accuracy metric:"),El=p(),d(ct.$$.fragment),$l=p(),je=s("p"),Or=n("You can see that it describes how the metric works in theory. If you use this metric for your work, especially if it is an academic publication you want to reference it properly. For that you can look at the "),Xa=s("code"),Ir=n("citation"),Sr=n(" attribute:"),jl=p(),d(pt.$$.fragment),kl=p(),Gt=s("p"),Nr=n("Before we can apply a metric or other evaluation module to a use-case, we need to know what the input format of the metric is:"),xl=p(),d(ut.$$.fragment),ql=p(),d(ke.$$.fragment),Tl=p(),Z=s("h2"),xe=s("a"),Za=s("span"),d(ht.$$.fragment),Mr=p(),es=s("span"),Hr=n("Compute"),Al=p(),Rt=s("p"),Br=n("Now that we know how the evaluation module works and what should go in there we want to actually use it! When it comes to computing the actual score there are two main ways to do it:"),Pl=p(),qe=s("ol"),ts=s("li"),Lr=n("All-in-one"),Fr=p(),as=s("li"),Wr=n("Incremental"),Cl=p(),x=s("p"),Gr=n("In the incremental approach the necessary inputs are added to the module with "),Ut=s("a"),Rr=n("EvaluationModule.add()"),Ur=n(" or "),Vt=s("a"),Vr=n("EvaluationModule.add_batch()"),Yr=n(" and the score is calculated at the end with "),Yt=s("a"),Jr=n("EvaluationModule.compute()"),zr=n(". Alternatively, one can pass all the inputs at once to "),ss=s("code"),Kr=n("compute()"),Qr=n(". Let\u2019s have a look at the two approaches."),Dl=p(),ee=s("h3"),Te=s("a"),ls=s("span"),d(dt.$$.fragment),Xr=p(),os=s("span"),Zr=n("How to compute"),Ol=p(),D=s("p"),ei=n("The simplest way to calculate the score of an evaluation module is by calling "),ns=s("code"),ti=n("compute()"),ai=n(" directly with the necessary inputs. Simply pass the inputs as seen in "),rs=s("code"),si=n("features"),li=n(" to the "),is=s("code"),oi=n("compute()"),ni=n(" method."),Il=p(),d(mt.$$.fragment),Sl=p(),H=s("p"),ri=n("Evaluation modules return the results in a dictionary. However, in some instances you build up the predictions iteratively or in a distributed fashion in which case "),cs=s("code"),ii=n("add()"),ci=n(" or "),ps=s("code"),pi=n("add_batch()"),ui=n(" are useful."),Nl=p(),te=s("h3"),Ae=s("a"),us=s("span"),d(ft.$$.fragment),hi=p(),hs=s("span"),di=n("Calculate a single metric or a batch of metrics"),Ml=p(),q=s("p"),mi=n("In many evaluation pipelines you build the predictions iteratively such as in a for-loop. In that case you could store the predictions in a list and at the end pass them to "),ds=s("code"),fi=n("compute()"),vi=n(". With "),ms=s("code"),yi=n("add()"),_i=n(" and "),fs=s("code"),gi=n("add_batch()"),bi=n(" you can circumvent the step of storing the predictions separately. If you are only creating single predictions at a time you can use "),vs=s("code"),wi=n("add()"),Ei=n(":"),Hl=p(),d(vt.$$.fragment),Bl=p(),O=s("p"),$i=n("Once you have gathered all predictions you can call "),ys=s("code"),ji=n("compute()"),ki=n(" to compute the score based on all stored values. When getting predictions and references in batches you can use "),_s=s("code"),xi=n("add_batch()"),qi=n(" which adds a list elements for later processing. The rest works as with "),gs=s("code"),Ti=n("add()"),Ai=n(":"),Ll=p(),d(yt.$$.fragment),Fl=p(),Jt=s("p"),Pi=n("This is especially useful when you need to get the predictions from your model in batches:"),Wl=p(),d(_t.$$.fragment),Gl=p(),ae=s("h3"),Pe=s("a"),bs=s("span"),d(gt.$$.fragment),Ci=p(),ws=s("span"),Di=n("Distributed evaluation"),Rl=p(),T=s("p"),Oi=n("Computing metrics in a distributed environment can be tricky. Metric evaluation is executed in separate Python processes, or nodes, on different subsets of a dataset. Typically, when a metric score is additive ("),Es=s("code"),Ii=n("f(AuB) = f(A) + f(B)"),Si=n("), you can use distributed reduce operations to gather the scores for each subset of the dataset. But when a metric is non-additive ("),$s=s("code"),Ni=n("f(AuB) \u2260 f(A) + f(B)"),Mi=n("), it\u2019s not that simple. For example, you can\u2019t take the sum of the "),bt=s("a"),Hi=n("F1"),Bi=n(" scores of each data subset as your "),js=s("strong"),Li=n("final metric"),Fi=n("."),Ul=p(),zt=s("p"),Wi=n("A common way to overcome this issue is to fallback on single process evaluation. The metrics are evaluated on a single GPU, which becomes inefficient."),Vl=p(),B=s("p"),Gi=n("\u{1F917} Evaluate solves this issue by only computing the final metric on the first node. The predictions and references are computed and provided to the metric separately for each node. These are temporarily stored in an Apache Arrow table, avoiding cluttering the GPU or CPU memory. When you are ready to "),ks=s("code"),Ri=n("compute()"),Ui=n(" the final metric, the first node is able to access the predictions and references stored on all the other nodes. Once it has gathered all the predictions and references, "),xs=s("code"),Vi=n("compute()"),Yi=n(" will perform the final metric evaluation."),Yl=p(),Kt=s("p"),Ji=n("This solution allows \u{1F917} Evaluate to perform distributed predictions, which is important for evaluation speed in distributed settings. At the same time, you can also use complex non-additive metrics without wasting valuable GPU or CPU memory."),Jl=p(),se=s("h2"),Ce=s("a"),qs=s("span"),d(wt.$$.fragment),zi=p(),Ts=s("span"),Ki=n("Combining several evaluations"),zl=p(),De=s("p"),Qi=n("Often one wants to not only evaluate a single metric but a range of different metrics capturing different aspects of a model. E.g. for classification it is usually a good idea to compute F1-score, recall, and precision in addition to accuracy to get a better picture of model performance. Naturally, you can load a bunch of metrics and call them sequentially. However, a more convenient way is to use the "),Qt=s("a"),Xi=n("combine()"),Zi=n(" function to bundle them together:"),Kl=p(),d(Et.$$.fragment),Ql=p(),L=s("p"),ec=n("The "),As=s("code"),tc=n("combine"),ac=n(" function accepts both the list of names of the metrics as well as an instantiated modules. The "),Ps=s("code"),sc=n("compute"),lc=n(" call then computes each metric:"),Xl=p(),d($t.$$.fragment),Zl=p(),le=s("h2"),Oe=s("a"),Cs=s("span"),d(jt.$$.fragment),oc=p(),Ds=s("span"),nc=n("Save and push to the Hub"),eo=p(),Ie=s("p"),rc=n("Saving and sharing evaluation results is an important step. We provide the "),Xt=s("a"),ic=n("evaluate.save()"),cc=n(" function to easily save metrics results. You can either pass a specific filename or a directory. In the latter case, the results are saved in a file with an automatically created file name. Besides the directory or file name, the function takes any key-value pairs as inputs and stores them in a JSON file."),to=p(),d(kt.$$.fragment),ao=p(),Zt=s("p"),pc=n("The content of the JSON file look like the following:"),so=p(),d(xt.$$.fragment),lo=p(),ea=s("p"),uc=n("In addition to the specified fields, it also contains useful system information for reproducing the results."),oo=p(),Se=s("p"),hc=n("Besides storing the results locally, you should report them on the model\u2019s repository on the Hub. With the "),ta=s("a"),dc=n("evaluate.push_to_hub()"),mc=n(" function, you can easily report evaluation results to the model\u2019s repository:"),no=p(),d(qt.$$.fragment),ro=p(),oe=s("h2"),Ne=s("a"),Os=s("span"),d(Tt.$$.fragment),fc=p(),Is=s("span"),vc=n("Evaluator"),io=p(),E=s("p"),yc=n("The "),aa=s("a"),_c=n("evaluate.evaluator()"),gc=n(" provides automated evaluation and only requires a model, dataset, metric in contrast to the metrics in "),Ss=s("code"),bc=n("EvaluationModule"),wc=n("s that require the model\u2019s predictions. As such it is easier to evaluate a model on a dataset with a given metric as the inference is handled internally. To make that possible it uses the "),At=s("a"),Ec=n("pipeline"),$c=n(" abstraction from "),Ns=s("code"),jc=n("transformers"),kc=n(". However, you can use your own framework as long as it follows the "),Ms=s("code"),xc=n("pipeline"),qc=n(" interface."),co=p(),F=s("p"),Tc=n("To make an evaluation with the "),Hs=s("code"),Ac=n("evaluator"),Pc=n(" let\u2019s load a "),Bs=s("code"),Cc=n("transformers"),Dc=n(" pipeline (but you can pass your own custom inference class for any framework as long as it follows the pipeline call API) with an model trained on IMDb, the IMDb test split and the accuracy metric."),po=p(),d(Pt.$$.fragment),uo=p(),W=s("p"),Oc=n("Then you can create an evaluator for text classification and pass the three objects to the "),Ls=s("code"),Ic=n("compute()"),Sc=n(" method. With the label mapping "),Fs=s("code"),Nc=n("evaluate"),Mc=n(" provides a method to align the pipeline outputs with the label column in the dataset:"),ho=p(),d(Ct.$$.fragment),mo=p(),G=s("p"),Hc=n("Calculating the value of the metric alone is often not enough to know if a model performs significantly better than another one. With "),Ws=s("em"),Bc=n("bootstrapping"),Lc=p(),Gs=s("code"),Fc=n("evaluate"),Wc=n(" computes confidence intervals and the standard error which helps estimate how stable a score is:"),fo=p(),d(Dt.$$.fragment),vo=p(),$=s("p"),Gc=n("The evaluator expects a "),Rs=s("code"),Rc=n('"text"'),Uc=n(" and "),Us=s("code"),Vc=n('"label"'),Yc=n(" column for the data input. If your dataset differs you can provide the columns with the keywords "),Vs=s("code"),Jc=n('input_column="text"'),zc=n(" and "),Ys=s("code"),Kc=n('label_column="label"'),Qc=n(". Currently only "),Js=s("code"),Xc=n('"text-classification"'),Zc=n(" is supported with more tasks being added in the future."),this.h()},l(e){const i=Oh('[data-svelte="svelte-1phssyn"]',document.head);w=l(i,"META",{name:!0,content:!0}),i.forEach(a),ne=u(e),k=l(e,"H1",{class:!0});var Ot=o(k);C=l(Ot,"A",{id:!0,class:!0,href:!0});var tp=o(C);Y=l(tp,"SPAN",{});var ap=o(Y);m(A.$$.fragment,ap),ap.forEach(a),tp.forEach(a),N=u(Ot),re=l(Ot,"SPAN",{});var sp=o(re);Zo=r(sp,"A quick tour"),sp.forEach(a),Ot.forEach(a),Zs=u(e),It=l(e,"P",{});var lp=o(It);en=r(lp,"\u{1F917} Evaluate provides access to a wide range of evaluation tools. It covers a range of modalities such as text, computer vision, audio, etc. as well as tools to evaluate models or datasets. These tools are split into three categories."),lp.forEach(a),el=u(e),J=l(e,"H2",{class:!0});var _o=o(J);ie=l(_o,"A",{id:!0,class:!0,href:!0});var op=o(ie);pa=l(op,"SPAN",{});var np=o(pa);m(Be.$$.fragment,np),np.forEach(a),op.forEach(a),tn=u(_o),ua=l(_o,"SPAN",{});var rp=o(ua);an=r(rp,"Types of evaluations"),rp.forEach(a),_o.forEach(a),tl=u(e),St=l(e,"P",{});var ip=o(St);sn=r(ip,"There are different aspects of a typical machine learning pipeline that can be evaluated and for each aspect \u{1F917} Evaluate provides a tool:"),ip.forEach(a),al=u(e),M=l(e,"UL",{});var sa=o(M);ce=l(sa,"LI",{});var zs=o(ce);ha=l(zs,"STRONG",{});var cp=o(ha);ln=r(cp,"Metric"),cp.forEach(a),on=r(zs,": A metric is used to evaluate a model\u2019s performance and usually involves the model\u2019s predictions as well as some ground truth labels. You can find all integrated metrics at "),Le=l(zs,"A",{href:!0,rel:!0});var pp=o(Le);nn=r(pp,"evaluate-metric"),pp.forEach(a),rn=r(zs,"."),zs.forEach(a),cn=u(sa),pe=l(sa,"LI",{});var Ks=o(pe);da=l(Ks,"STRONG",{});var up=o(da);pn=r(up,"Comparison"),up.forEach(a),un=r(Ks,": A comparison is used to compare two models. This can for example be done by comparing their predictions to ground truth labels and computing their agreement. You can find all integrated comparisons at "),Fe=l(Ks,"A",{href:!0,rel:!0});var hp=o(Fe);hn=r(hp,"evaluate-comparison"),hp.forEach(a),dn=r(Ks,"."),Ks.forEach(a),mn=u(sa),ue=l(sa,"LI",{});var Qs=o(ue);ma=l(Qs,"STRONG",{});var dp=o(ma);fn=r(dp,"Measurement"),dp.forEach(a),vn=r(Qs,": The dataset is as important as the model trained on it. With measurements one can investigate a dataset\u2019s properties. You can find all integrated measurements at "),We=l(Qs,"A",{href:!0,rel:!0});var mp=o(We);yn=r(mp,"evaluate-measurement"),mp.forEach(a),_n=r(Qs,"."),Qs.forEach(a),sa.forEach(a),sl=u(e),he=l(e,"P",{});var go=o(he);gn=r(go,"Each of these evaluation modules live on Hugging Face Hub as a Space. They come with an interactive widget and a documentation card documenting its use and limitations. For example "),Ge=l(go,"A",{href:!0,rel:!0});var fp=o(Ge);bn=r(fp,"accuracy"),fp.forEach(a),wn=r(go,":"),go.forEach(a),ll=u(e),Re=l(e,"DIV",{class:!0});var vp=o(Re);Nt=l(vp,"IMG",{src:!0,width:!0}),vp.forEach(a),ol=u(e),de=l(e,"P",{});var bo=o(de);En=r(bo,"Each metric, comparison, and measurement is a separate Python module, but for using any of them, there is a single entry point: "),Mt=l(bo,"A",{href:!0});var yp=o(Mt);$n=r(yp,"evaluate.load()"),yp.forEach(a),jn=r(bo,"!"),bo.forEach(a),nl=u(e),z=l(e,"H2",{class:!0});var wo=o(z);me=l(wo,"A",{id:!0,class:!0,href:!0});var _p=o(me);fa=l(_p,"SPAN",{});var gp=o(fa);m(Ue.$$.fragment,gp),gp.forEach(a),_p.forEach(a),kn=u(wo),va=l(wo,"SPAN",{});var bp=o(va);xn=r(bp,"Load"),bp.forEach(a),wo.forEach(a),rl=u(e),fe=l(e,"P",{});var Eo=o(fe);qn=r(Eo,"Any metric, comparison, or measurement is loaded with the "),ya=l(Eo,"CODE",{});var wp=o(ya);Tn=r(wp,"evaluate.load"),wp.forEach(a),An=r(Eo," function:"),Eo.forEach(a),il=u(e),m(Ve.$$.fragment,e),cl=u(e),Ht=l(e,"P",{});var Ep=o(Ht);Pn=r(Ep,"If you want to make sure you are loading the right type of evaluation (especially if there are name clashes) you can explicitly pass the type:"),Ep.forEach(a),pl=u(e),m(Ye.$$.fragment,e),ul=u(e),K=l(e,"H3",{class:!0});var $o=o(K);ve=l($o,"A",{id:!0,class:!0,href:!0});var $p=o(ve);_a=l($p,"SPAN",{});var jp=o(_a);m(Je.$$.fragment,jp),jp.forEach(a),$p.forEach(a),Cn=u($o),ga=l($o,"SPAN",{});var kp=o(ga);Dn=r(kp,"Community modules"),kp.forEach(a),$o.forEach(a),hl=u(e),Bt=l(e,"P",{});var xp=o(Bt);On=r(xp,"Besides the modules implemented in \u{1F917} Evaluate you can also load any community module by specifying the repository ID of the metric implementation:"),xp.forEach(a),dl=u(e),m(ze.$$.fragment,e),ml=u(e),ye=l(e,"P",{});var jo=o(ye);In=r(jo,"See the "),Lt=l(jo,"A",{href:!0});var qp=o(Lt);Sn=r(qp,"Creating and Sharing Guide"),qp.forEach(a),Nn=r(jo," for information about uploading custom metrics."),jo.forEach(a),fl=u(e),Q=l(e,"H3",{class:!0});var ko=o(Q);_e=l(ko,"A",{id:!0,class:!0,href:!0});var Tp=o(_e);ba=l(Tp,"SPAN",{});var Ap=o(ba);m(Ke.$$.fragment,Ap),Ap.forEach(a),Tp.forEach(a),Mn=u(ko),wa=l(ko,"SPAN",{});var Pp=o(wa);Hn=r(Pp,"List available modules"),Pp.forEach(a),ko.forEach(a),vl=u(e),ge=l(e,"P",{});var xo=o(ge);Bn=r(xo,"With "),Ft=l(xo,"A",{href:!0});var Cp=o(Ft);Ln=r(Cp,"list_evaluation_modules()"),Cp.forEach(a),Fn=r(xo," you can check what modules are available on the hub. You can also filter for a specific modules and skip community metrics if you want. You can also see additional information such as likes:"),xo.forEach(a),yl=u(e),m(Qe.$$.fragment,e),_l=u(e),X=l(e,"H2",{class:!0});var qo=o(X);be=l(qo,"A",{id:!0,class:!0,href:!0});var Dp=o(be);Ea=l(Dp,"SPAN",{});var Op=o(Ea);m(Xe.$$.fragment,Op),Op.forEach(a),Dp.forEach(a),Wn=u(qo),$a=l(qo,"SPAN",{});var Ip=o($a);Gn=r(Ip,"Module attributes"),Ip.forEach(a),qo.forEach(a),gl=u(e),we=l(e,"P",{});var To=o(we);Rn=r(To,"All evalution modules come with a range of useful attributes that help to use a module stored in a "),Wt=l(To,"A",{href:!0});var Sp=o(Wt);Un=r(Sp,"EvaluationModuleInfo"),Sp.forEach(a),Vn=r(To," object."),To.forEach(a),bl=u(e),Ee=l(e,"TABLE",{});var Ao=o(Ee);ja=l(Ao,"THEAD",{});var Np=o(ja);Ze=l(Np,"TR",{});var Po=o(Ze);ka=l(Po,"TH",{});var Mp=o(ka);Yn=r(Mp,"Attribute"),Mp.forEach(a),Jn=u(Po),xa=l(Po,"TH",{});var Hp=o(xa);zn=r(Hp,"Description"),Hp.forEach(a),Po.forEach(a),Np.forEach(a),Kn=u(Ao),g=l(Ao,"TBODY",{});var j=o(g);et=l(j,"TR",{});var Co=o(et);qa=l(Co,"TD",{});var Bp=o(qa);Ta=l(Bp,"CODE",{});var Lp=o(Ta);Qn=r(Lp,"description"),Lp.forEach(a),Bp.forEach(a),Xn=u(Co),Aa=l(Co,"TD",{});var Fp=o(Aa);Zn=r(Fp,"A short description of the evaluation module."),Fp.forEach(a),Co.forEach(a),er=u(j),tt=l(j,"TR",{});var Do=o(tt);Pa=l(Do,"TD",{});var Wp=o(Pa);Ca=l(Wp,"CODE",{});var Gp=o(Ca);tr=r(Gp,"citation"),Gp.forEach(a),Wp.forEach(a),ar=u(Do),Da=l(Do,"TD",{});var Rp=o(Da);sr=r(Rp,"A BibTex string for citation when available."),Rp.forEach(a),Do.forEach(a),lr=u(j),at=l(j,"TR",{});var Oo=o(at);Oa=l(Oo,"TD",{});var Up=o(Oa);Ia=l(Up,"CODE",{});var Vp=o(Ia);or=r(Vp,"features"),Vp.forEach(a),Up.forEach(a),nr=u(Oo),st=l(Oo,"TD",{});var Io=o(st);rr=r(Io,"A "),Sa=l(Io,"CODE",{});var Yp=o(Sa);ir=r(Yp,"Features"),Yp.forEach(a),cr=r(Io," object defining the input format."),Io.forEach(a),Oo.forEach(a),pr=u(j),lt=l(j,"TR",{});var So=o(lt);Na=l(So,"TD",{});var Jp=o(Na);Ma=l(Jp,"CODE",{});var zp=o(Ma);ur=r(zp,"inputs_description"),zp.forEach(a),Jp.forEach(a),hr=u(So),Ha=l(So,"TD",{});var Kp=o(Ha);dr=r(Kp,"This is equivalent to the modules docstring."),Kp.forEach(a),So.forEach(a),mr=u(j),ot=l(j,"TR",{});var No=o(ot);Ba=l(No,"TD",{});var Qp=o(Ba);La=l(Qp,"CODE",{});var Xp=o(La);fr=r(Xp,"homepage"),Xp.forEach(a),Qp.forEach(a),vr=u(No),Fa=l(No,"TD",{});var Zp=o(Fa);yr=r(Zp,"The homepage of the module."),Zp.forEach(a),No.forEach(a),_r=u(j),nt=l(j,"TR",{});var Mo=o(nt);Wa=l(Mo,"TD",{});var eu=o(Wa);Ga=l(eu,"CODE",{});var tu=o(Ga);gr=r(tu,"license"),tu.forEach(a),eu.forEach(a),br=u(Mo),Ra=l(Mo,"TD",{});var au=o(Ra);wr=r(au,"The license of the module."),au.forEach(a),Mo.forEach(a),Er=u(j),rt=l(j,"TR",{});var Ho=o(rt);Ua=l(Ho,"TD",{});var su=o(Ua);Va=l(su,"CODE",{});var lu=o(Va);$r=r(lu,"codebase_urls"),lu.forEach(a),su.forEach(a),jr=u(Ho),Ya=l(Ho,"TD",{});var ou=o(Ya);kr=r(ou,"Link to the code behind the module."),ou.forEach(a),Ho.forEach(a),xr=u(j),it=l(j,"TR",{});var Bo=o(it);Ja=l(Bo,"TD",{});var nu=o(Ja);za=l(nu,"CODE",{});var ru=o(za);qr=r(ru,"reference_urls"),ru.forEach(a),nu.forEach(a),Tr=u(Bo),Ka=l(Bo,"TD",{});var iu=o(Ka);Ar=r(iu,"Additional reference URLs."),iu.forEach(a),Bo.forEach(a),j.forEach(a),Ao.forEach(a),wl=u(e),$e=l(e,"P",{});var Lo=o($e);Pr=r(Lo,"Let\u2019s have a look at a few examples. First, let\u2019s look at the "),Qa=l(Lo,"CODE",{});var cu=o(Qa);Cr=r(cu,"description"),cu.forEach(a),Dr=r(Lo," attribute of the accuracy metric:"),Lo.forEach(a),El=u(e),m(ct.$$.fragment,e),$l=u(e),je=l(e,"P",{});var Fo=o(je);Or=r(Fo,"You can see that it describes how the metric works in theory. If you use this metric for your work, especially if it is an academic publication you want to reference it properly. For that you can look at the "),Xa=l(Fo,"CODE",{});var pu=o(Xa);Ir=r(pu,"citation"),pu.forEach(a),Sr=r(Fo," attribute:"),Fo.forEach(a),jl=u(e),m(pt.$$.fragment,e),kl=u(e),Gt=l(e,"P",{});var uu=o(Gt);Nr=r(uu,"Before we can apply a metric or other evaluation module to a use-case, we need to know what the input format of the metric is:"),uu.forEach(a),xl=u(e),m(ut.$$.fragment,e),ql=u(e),m(ke.$$.fragment,e),Tl=u(e),Z=l(e,"H2",{class:!0});var Wo=o(Z);xe=l(Wo,"A",{id:!0,class:!0,href:!0});var hu=o(xe);Za=l(hu,"SPAN",{});var du=o(Za);m(ht.$$.fragment,du),du.forEach(a),hu.forEach(a),Mr=u(Wo),es=l(Wo,"SPAN",{});var mu=o(es);Hr=r(mu,"Compute"),mu.forEach(a),Wo.forEach(a),Al=u(e),Rt=l(e,"P",{});var fu=o(Rt);Br=r(fu,"Now that we know how the evaluation module works and what should go in there we want to actually use it! When it comes to computing the actual score there are two main ways to do it:"),fu.forEach(a),Pl=u(e),qe=l(e,"OL",{});var Go=o(qe);ts=l(Go,"LI",{});var vu=o(ts);Lr=r(vu,"All-in-one"),vu.forEach(a),Fr=u(Go),as=l(Go,"LI",{});var yu=o(as);Wr=r(yu,"Incremental"),yu.forEach(a),Go.forEach(a),Cl=u(e),x=l(e,"P",{});var R=o(x);Gr=r(R,"In the incremental approach the necessary inputs are added to the module with "),Ut=l(R,"A",{href:!0});var _u=o(Ut);Rr=r(_u,"EvaluationModule.add()"),_u.forEach(a),Ur=r(R," or "),Vt=l(R,"A",{href:!0});var gu=o(Vt);Vr=r(gu,"EvaluationModule.add_batch()"),gu.forEach(a),Yr=r(R," and the score is calculated at the end with "),Yt=l(R,"A",{href:!0});var bu=o(Yt);Jr=r(bu,"EvaluationModule.compute()"),bu.forEach(a),zr=r(R,". Alternatively, one can pass all the inputs at once to "),ss=l(R,"CODE",{});var wu=o(ss);Kr=r(wu,"compute()"),wu.forEach(a),Qr=r(R,". Let\u2019s have a look at the two approaches."),R.forEach(a),Dl=u(e),ee=l(e,"H3",{class:!0});var Ro=o(ee);Te=l(Ro,"A",{id:!0,class:!0,href:!0});var Eu=o(Te);ls=l(Eu,"SPAN",{});var $u=o(ls);m(dt.$$.fragment,$u),$u.forEach(a),Eu.forEach(a),Xr=u(Ro),os=l(Ro,"SPAN",{});var ju=o(os);Zr=r(ju,"How to compute"),ju.forEach(a),Ro.forEach(a),Ol=u(e),D=l(e,"P",{});var Me=o(D);ei=r(Me,"The simplest way to calculate the score of an evaluation module is by calling "),ns=l(Me,"CODE",{});var ku=o(ns);ti=r(ku,"compute()"),ku.forEach(a),ai=r(Me," directly with the necessary inputs. Simply pass the inputs as seen in "),rs=l(Me,"CODE",{});var xu=o(rs);si=r(xu,"features"),xu.forEach(a),li=r(Me," to the "),is=l(Me,"CODE",{});var qu=o(is);oi=r(qu,"compute()"),qu.forEach(a),ni=r(Me," method."),Me.forEach(a),Il=u(e),m(mt.$$.fragment,e),Sl=u(e),H=l(e,"P",{});var la=o(H);ri=r(la,"Evaluation modules return the results in a dictionary. However, in some instances you build up the predictions iteratively or in a distributed fashion in which case "),cs=l(la,"CODE",{});var Tu=o(cs);ii=r(Tu,"add()"),Tu.forEach(a),ci=r(la," or "),ps=l(la,"CODE",{});var Au=o(ps);pi=r(Au,"add_batch()"),Au.forEach(a),ui=r(la," are useful."),la.forEach(a),Nl=u(e),te=l(e,"H3",{class:!0});var Uo=o(te);Ae=l(Uo,"A",{id:!0,class:!0,href:!0});var Pu=o(Ae);us=l(Pu,"SPAN",{});var Cu=o(us);m(ft.$$.fragment,Cu),Cu.forEach(a),Pu.forEach(a),hi=u(Uo),hs=l(Uo,"SPAN",{});var Du=o(hs);di=r(Du,"Calculate a single metric or a batch of metrics"),Du.forEach(a),Uo.forEach(a),Ml=u(e),q=l(e,"P",{});var U=o(q);mi=r(U,"In many evaluation pipelines you build the predictions iteratively such as in a for-loop. In that case you could store the predictions in a list and at the end pass them to "),ds=l(U,"CODE",{});var Ou=o(ds);fi=r(Ou,"compute()"),Ou.forEach(a),vi=r(U,". With "),ms=l(U,"CODE",{});var Iu=o(ms);yi=r(Iu,"add()"),Iu.forEach(a),_i=r(U," and "),fs=l(U,"CODE",{});var Su=o(fs);gi=r(Su,"add_batch()"),Su.forEach(a),bi=r(U," you can circumvent the step of storing the predictions separately. If you are only creating single predictions at a time you can use "),vs=l(U,"CODE",{});var Nu=o(vs);wi=r(Nu,"add()"),Nu.forEach(a),Ei=r(U,":"),U.forEach(a),Hl=u(e),m(vt.$$.fragment,e),Bl=u(e),O=l(e,"P",{});var He=o(O);$i=r(He,"Once you have gathered all predictions you can call "),ys=l(He,"CODE",{});var Mu=o(ys);ji=r(Mu,"compute()"),Mu.forEach(a),ki=r(He," to compute the score based on all stored values. When getting predictions and references in batches you can use "),_s=l(He,"CODE",{});var Hu=o(_s);xi=r(Hu,"add_batch()"),Hu.forEach(a),qi=r(He," which adds a list elements for later processing. The rest works as with "),gs=l(He,"CODE",{});var Bu=o(gs);Ti=r(Bu,"add()"),Bu.forEach(a),Ai=r(He,":"),He.forEach(a),Ll=u(e),m(yt.$$.fragment,e),Fl=u(e),Jt=l(e,"P",{});var Lu=o(Jt);Pi=r(Lu,"This is especially useful when you need to get the predictions from your model in batches:"),Lu.forEach(a),Wl=u(e),m(_t.$$.fragment,e),Gl=u(e),ae=l(e,"H3",{class:!0});var Vo=o(ae);Pe=l(Vo,"A",{id:!0,class:!0,href:!0});var Fu=o(Pe);bs=l(Fu,"SPAN",{});var Wu=o(bs);m(gt.$$.fragment,Wu),Wu.forEach(a),Fu.forEach(a),Ci=u(Vo),ws=l(Vo,"SPAN",{});var Gu=o(ws);Di=r(Gu,"Distributed evaluation"),Gu.forEach(a),Vo.forEach(a),Rl=u(e),T=l(e,"P",{});var V=o(T);Oi=r(V,"Computing metrics in a distributed environment can be tricky. Metric evaluation is executed in separate Python processes, or nodes, on different subsets of a dataset. Typically, when a metric score is additive ("),Es=l(V,"CODE",{});var Ru=o(Es);Ii=r(Ru,"f(AuB) = f(A) + f(B)"),Ru.forEach(a),Si=r(V,"), you can use distributed reduce operations to gather the scores for each subset of the dataset. But when a metric is non-additive ("),$s=l(V,"CODE",{});var Uu=o($s);Ni=r(Uu,"f(AuB) \u2260 f(A) + f(B)"),Uu.forEach(a),Mi=r(V,"), it\u2019s not that simple. For example, you can\u2019t take the sum of the "),bt=l(V,"A",{href:!0,rel:!0});var Vu=o(bt);Hi=r(Vu,"F1"),Vu.forEach(a),Bi=r(V," scores of each data subset as your "),js=l(V,"STRONG",{});var Yu=o(js);Li=r(Yu,"final metric"),Yu.forEach(a),Fi=r(V,"."),V.forEach(a),Ul=u(e),zt=l(e,"P",{});var Ju=o(zt);Wi=r(Ju,"A common way to overcome this issue is to fallback on single process evaluation. The metrics are evaluated on a single GPU, which becomes inefficient."),Ju.forEach(a),Vl=u(e),B=l(e,"P",{});var oa=o(B);Gi=r(oa,"\u{1F917} Evaluate solves this issue by only computing the final metric on the first node. The predictions and references are computed and provided to the metric separately for each node. These are temporarily stored in an Apache Arrow table, avoiding cluttering the GPU or CPU memory. When you are ready to "),ks=l(oa,"CODE",{});var zu=o(ks);Ri=r(zu,"compute()"),zu.forEach(a),Ui=r(oa," the final metric, the first node is able to access the predictions and references stored on all the other nodes. Once it has gathered all the predictions and references, "),xs=l(oa,"CODE",{});var Ku=o(xs);Vi=r(Ku,"compute()"),Ku.forEach(a),Yi=r(oa," will perform the final metric evaluation."),oa.forEach(a),Yl=u(e),Kt=l(e,"P",{});var Qu=o(Kt);Ji=r(Qu,"This solution allows \u{1F917} Evaluate to perform distributed predictions, which is important for evaluation speed in distributed settings. At the same time, you can also use complex non-additive metrics without wasting valuable GPU or CPU memory."),Qu.forEach(a),Jl=u(e),se=l(e,"H2",{class:!0});var Yo=o(se);Ce=l(Yo,"A",{id:!0,class:!0,href:!0});var Xu=o(Ce);qs=l(Xu,"SPAN",{});var Zu=o(qs);m(wt.$$.fragment,Zu),Zu.forEach(a),Xu.forEach(a),zi=u(Yo),Ts=l(Yo,"SPAN",{});var eh=o(Ts);Ki=r(eh,"Combining several evaluations"),eh.forEach(a),Yo.forEach(a),zl=u(e),De=l(e,"P",{});var Jo=o(De);Qi=r(Jo,"Often one wants to not only evaluate a single metric but a range of different metrics capturing different aspects of a model. E.g. for classification it is usually a good idea to compute F1-score, recall, and precision in addition to accuracy to get a better picture of model performance. Naturally, you can load a bunch of metrics and call them sequentially. However, a more convenient way is to use the "),Qt=l(Jo,"A",{href:!0});var th=o(Qt);Xi=r(th,"combine()"),th.forEach(a),Zi=r(Jo," function to bundle them together:"),Jo.forEach(a),Kl=u(e),m(Et.$$.fragment,e),Ql=u(e),L=l(e,"P",{});var na=o(L);ec=r(na,"The "),As=l(na,"CODE",{});var ah=o(As);tc=r(ah,"combine"),ah.forEach(a),ac=r(na," function accepts both the list of names of the metrics as well as an instantiated modules. The "),Ps=l(na,"CODE",{});var sh=o(Ps);sc=r(sh,"compute"),sh.forEach(a),lc=r(na," call then computes each metric:"),na.forEach(a),Xl=u(e),m($t.$$.fragment,e),Zl=u(e),le=l(e,"H2",{class:!0});var zo=o(le);Oe=l(zo,"A",{id:!0,class:!0,href:!0});var lh=o(Oe);Cs=l(lh,"SPAN",{});var oh=o(Cs);m(jt.$$.fragment,oh),oh.forEach(a),lh.forEach(a),oc=u(zo),Ds=l(zo,"SPAN",{});var nh=o(Ds);nc=r(nh,"Save and push to the Hub"),nh.forEach(a),zo.forEach(a),eo=u(e),Ie=l(e,"P",{});var Ko=o(Ie);rc=r(Ko,"Saving and sharing evaluation results is an important step. We provide the "),Xt=l(Ko,"A",{href:!0});var rh=o(Xt);ic=r(rh,"evaluate.save()"),rh.forEach(a),cc=r(Ko," function to easily save metrics results. You can either pass a specific filename or a directory. In the latter case, the results are saved in a file with an automatically created file name. Besides the directory or file name, the function takes any key-value pairs as inputs and stores them in a JSON file."),Ko.forEach(a),to=u(e),m(kt.$$.fragment,e),ao=u(e),Zt=l(e,"P",{});var ih=o(Zt);pc=r(ih,"The content of the JSON file look like the following:"),ih.forEach(a),so=u(e),m(xt.$$.fragment,e),lo=u(e),ea=l(e,"P",{});var ch=o(ea);uc=r(ch,"In addition to the specified fields, it also contains useful system information for reproducing the results."),ch.forEach(a),oo=u(e),Se=l(e,"P",{});var Qo=o(Se);hc=r(Qo,"Besides storing the results locally, you should report them on the model\u2019s repository on the Hub. With the "),ta=l(Qo,"A",{href:!0});var ph=o(ta);dc=r(ph,"evaluate.push_to_hub()"),ph.forEach(a),mc=r(Qo," function, you can easily report evaluation results to the model\u2019s repository:"),Qo.forEach(a),no=u(e),m(qt.$$.fragment,e),ro=u(e),oe=l(e,"H2",{class:!0});var Xo=o(oe);Ne=l(Xo,"A",{id:!0,class:!0,href:!0});var uh=o(Ne);Os=l(uh,"SPAN",{});var hh=o(Os);m(Tt.$$.fragment,hh),hh.forEach(a),uh.forEach(a),fc=u(Xo),Is=l(Xo,"SPAN",{});var dh=o(Is);vc=r(dh,"Evaluator"),dh.forEach(a),Xo.forEach(a),io=u(e),E=l(e,"P",{});var I=o(E);yc=r(I,"The "),aa=l(I,"A",{href:!0});var mh=o(aa);_c=r(mh,"evaluate.evaluator()"),mh.forEach(a),gc=r(I," provides automated evaluation and only requires a model, dataset, metric in contrast to the metrics in "),Ss=l(I,"CODE",{});var fh=o(Ss);bc=r(fh,"EvaluationModule"),fh.forEach(a),wc=r(I,"s that require the model\u2019s predictions. As such it is easier to evaluate a model on a dataset with a given metric as the inference is handled internally. To make that possible it uses the "),At=l(I,"A",{href:!0,rel:!0});var vh=o(At);Ec=r(vh,"pipeline"),vh.forEach(a),$c=r(I," abstraction from "),Ns=l(I,"CODE",{});var yh=o(Ns);jc=r(yh,"transformers"),yh.forEach(a),kc=r(I,". However, you can use your own framework as long as it follows the "),Ms=l(I,"CODE",{});var _h=o(Ms);xc=r(_h,"pipeline"),_h.forEach(a),qc=r(I," interface."),I.forEach(a),co=u(e),F=l(e,"P",{});var ra=o(F);Tc=r(ra,"To make an evaluation with the "),Hs=l(ra,"CODE",{});var gh=o(Hs);Ac=r(gh,"evaluator"),gh.forEach(a),Pc=r(ra," let\u2019s load a "),Bs=l(ra,"CODE",{});var bh=o(Bs);Cc=r(bh,"transformers"),bh.forEach(a),Dc=r(ra," pipeline (but you can pass your own custom inference class for any framework as long as it follows the pipeline call API) with an model trained on IMDb, the IMDb test split and the accuracy metric."),ra.forEach(a),po=u(e),m(Pt.$$.fragment,e),uo=u(e),W=l(e,"P",{});var ia=o(W);Oc=r(ia,"Then you can create an evaluator for text classification and pass the three objects to the "),Ls=l(ia,"CODE",{});var wh=o(Ls);Ic=r(wh,"compute()"),wh.forEach(a),Sc=r(ia," method. With the label mapping "),Fs=l(ia,"CODE",{});var Eh=o(Fs);Nc=r(Eh,"evaluate"),Eh.forEach(a),Mc=r(ia," provides a method to align the pipeline outputs with the label column in the dataset:"),ia.forEach(a),ho=u(e),m(Ct.$$.fragment,e),mo=u(e),G=l(e,"P",{});var ca=o(G);Hc=r(ca,"Calculating the value of the metric alone is often not enough to know if a model performs significantly better than another one. With "),Ws=l(ca,"EM",{});var $h=o(Ws);Bc=r($h,"bootstrapping"),$h.forEach(a),Lc=u(ca),Gs=l(ca,"CODE",{});var jh=o(Gs);Fc=r(jh,"evaluate"),jh.forEach(a),Wc=r(ca," computes confidence intervals and the standard error which helps estimate how stable a score is:"),ca.forEach(a),fo=u(e),m(Dt.$$.fragment,e),vo=u(e),$=l(e,"P",{});var S=o($);Gc=r(S,"The evaluator expects a "),Rs=l(S,"CODE",{});var kh=o(Rs);Rc=r(kh,'"text"'),kh.forEach(a),Uc=r(S," and "),Us=l(S,"CODE",{});var xh=o(Us);Vc=r(xh,'"label"'),xh.forEach(a),Yc=r(S," column for the data input. If your dataset differs you can provide the columns with the keywords "),Vs=l(S,"CODE",{});var qh=o(Vs);Jc=r(qh,'input_column="text"'),qh.forEach(a),zc=r(S," and "),Ys=l(S,"CODE",{});var Th=o(Ys);Kc=r(Th,'label_column="label"'),Th.forEach(a),Qc=r(S,". Currently only "),Js=l(S,"CODE",{});var Ah=o(Js);Xc=r(Ah,'"text-classification"'),Ah.forEach(a),Zc=r(S," is supported with more tasks being added in the future."),S.forEach(a),this.h()},h(){h(w,"name","hf:doc:metadata"),h(w,"content",JSON.stringify(Bh)),h(C,"id","a-quick-tour"),h(C,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),h(C,"href","#a-quick-tour"),h(k,"class","relative group"),h(ie,"id","types-of-evaluations"),h(ie,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),h(ie,"href","#types-of-evaluations"),h(J,"class","relative group"),h(Le,"href","https://huggingface.co/evaluate-metric"),h(Le,"rel","nofollow"),h(Fe,"href","https://huggingface.co/evaluate-comparison"),h(Fe,"rel","nofollow"),h(We,"href","https://huggingface.co/evaluate-measurement"),h(We,"rel","nofollow"),h(Ge,"href","https://huggingface.co/spaces/evaluate-metric/accuracy"),h(Ge,"rel","nofollow"),Ih(Nt.src,ep="https://huggingface.co/datasets/evaluate/media/resolve/main/metric-widget.png")||h(Nt,"src",ep),h(Nt,"width","400"),h(Re,"class","flex justify-center"),h(Mt,"href","/docs/evaluate/pr_225/en/package_reference/loading_methods#evaluate.load"),h(me,"id","load"),h(me,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),h(me,"href","#load"),h(z,"class","relative group"),h(ve,"id","community-modules"),h(ve,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),h(ve,"href","#community-modules"),h(K,"class","relative group"),h(Lt,"href","/docs/evaluate/main/en/creating_and_sharing"),h(_e,"id","list-available-modules"),h(_e,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),h(_e,"href","#list-available-modules"),h(Q,"class","relative group"),h(Ft,"href","/docs/evaluate/pr_225/en/package_reference/loading_methods#evaluate.list_evaluation_modules"),h(be,"id","module-attributes"),h(be,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),h(be,"href","#module-attributes"),h(X,"class","relative group"),h(Wt,"href","/docs/evaluate/pr_225/en/package_reference/main_classes#evaluate.EvaluationModuleInfo"),h(xe,"id","compute"),h(xe,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),h(xe,"href","#compute"),h(Z,"class","relative group"),h(Ut,"href","/docs/evaluate/pr_225/en/package_reference/main_classes#evaluate.EvaluationModule.add"),h(Vt,"href","/docs/evaluate/pr_225/en/package_reference/main_classes#evaluate.EvaluationModule.add_batch"),h(Yt,"href","/docs/evaluate/pr_225/en/package_reference/main_classes#evaluate.EvaluationModule.compute"),h(Te,"id","how-to-compute"),h(Te,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),h(Te,"href","#how-to-compute"),h(ee,"class","relative group"),h(Ae,"id","calculate-a-single-metric-or-a-batch-of-metrics"),h(Ae,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),h(Ae,"href","#calculate-a-single-metric-or-a-batch-of-metrics"),h(te,"class","relative group"),h(Pe,"id","distributed-evaluation"),h(Pe,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),h(Pe,"href","#distributed-evaluation"),h(ae,"class","relative group"),h(bt,"href","https://huggingface.co/spaces/evaluate-metric/f1"),h(bt,"rel","nofollow"),h(Ce,"id","combining-several-evaluations"),h(Ce,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),h(Ce,"href","#combining-several-evaluations"),h(se,"class","relative group"),h(Qt,"href","/docs/evaluate/pr_225/en/package_reference/main_classes#evaluate.combine"),h(Oe,"id","save-and-push-to-the-hub"),h(Oe,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),h(Oe,"href","#save-and-push-to-the-hub"),h(le,"class","relative group"),h(Xt,"href","/docs/evaluate/pr_225/en/package_reference/saving_methods#evaluate.save"),h(ta,"href","/docs/evaluate/pr_225/en/package_reference/hub_methods#evaluate.push_to_hub"),h(Ne,"id","evaluator"),h(Ne,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),h(Ne,"href","#evaluator"),h(oe,"class","relative group"),h(aa,"href","/docs/evaluate/pr_225/en/package_reference/evaluator_classes#evaluate.evaluator"),h(At,"href","https://huggingface.co/docs/transformers/main/en/main_classes/pipelines#transformers.pipeline"),h(At,"rel","nofollow")},m(e,i){t(document.head,w),c(e,ne,i),c(e,k,i),t(k,C),t(C,Y),f(A,Y,null),t(k,N),t(k,re),t(re,Zo),c(e,Zs,i),c(e,It,i),t(It,en),c(e,el,i),c(e,J,i),t(J,ie),t(ie,pa),f(Be,pa,null),t(J,tn),t(J,ua),t(ua,an),c(e,tl,i),c(e,St,i),t(St,sn),c(e,al,i),c(e,M,i),t(M,ce),t(ce,ha),t(ha,ln),t(ce,on),t(ce,Le),t(Le,nn),t(ce,rn),t(M,cn),t(M,pe),t(pe,da),t(da,pn),t(pe,un),t(pe,Fe),t(Fe,hn),t(pe,dn),t(M,mn),t(M,ue),t(ue,ma),t(ma,fn),t(ue,vn),t(ue,We),t(We,yn),t(ue,_n),c(e,sl,i),c(e,he,i),t(he,gn),t(he,Ge),t(Ge,bn),t(he,wn),c(e,ll,i),c(e,Re,i),t(Re,Nt),c(e,ol,i),c(e,de,i),t(de,En),t(de,Mt),t(Mt,$n),t(de,jn),c(e,nl,i),c(e,z,i),t(z,me),t(me,fa),f(Ue,fa,null),t(z,kn),t(z,va),t(va,xn),c(e,rl,i),c(e,fe,i),t(fe,qn),t(fe,ya),t(ya,Tn),t(fe,An),c(e,il,i),f(Ve,e,i),c(e,cl,i),c(e,Ht,i),t(Ht,Pn),c(e,pl,i),f(Ye,e,i),c(e,ul,i),c(e,K,i),t(K,ve),t(ve,_a),f(Je,_a,null),t(K,Cn),t(K,ga),t(ga,Dn),c(e,hl,i),c(e,Bt,i),t(Bt,On),c(e,dl,i),f(ze,e,i),c(e,ml,i),c(e,ye,i),t(ye,In),t(ye,Lt),t(Lt,Sn),t(ye,Nn),c(e,fl,i),c(e,Q,i),t(Q,_e),t(_e,ba),f(Ke,ba,null),t(Q,Mn),t(Q,wa),t(wa,Hn),c(e,vl,i),c(e,ge,i),t(ge,Bn),t(ge,Ft),t(Ft,Ln),t(ge,Fn),c(e,yl,i),f(Qe,e,i),c(e,_l,i),c(e,X,i),t(X,be),t(be,Ea),f(Xe,Ea,null),t(X,Wn),t(X,$a),t($a,Gn),c(e,gl,i),c(e,we,i),t(we,Rn),t(we,Wt),t(Wt,Un),t(we,Vn),c(e,bl,i),c(e,Ee,i),t(Ee,ja),t(ja,Ze),t(Ze,ka),t(ka,Yn),t(Ze,Jn),t(Ze,xa),t(xa,zn),t(Ee,Kn),t(Ee,g),t(g,et),t(et,qa),t(qa,Ta),t(Ta,Qn),t(et,Xn),t(et,Aa),t(Aa,Zn),t(g,er),t(g,tt),t(tt,Pa),t(Pa,Ca),t(Ca,tr),t(tt,ar),t(tt,Da),t(Da,sr),t(g,lr),t(g,at),t(at,Oa),t(Oa,Ia),t(Ia,or),t(at,nr),t(at,st),t(st,rr),t(st,Sa),t(Sa,ir),t(st,cr),t(g,pr),t(g,lt),t(lt,Na),t(Na,Ma),t(Ma,ur),t(lt,hr),t(lt,Ha),t(Ha,dr),t(g,mr),t(g,ot),t(ot,Ba),t(Ba,La),t(La,fr),t(ot,vr),t(ot,Fa),t(Fa,yr),t(g,_r),t(g,nt),t(nt,Wa),t(Wa,Ga),t(Ga,gr),t(nt,br),t(nt,Ra),t(Ra,wr),t(g,Er),t(g,rt),t(rt,Ua),t(Ua,Va),t(Va,$r),t(rt,jr),t(rt,Ya),t(Ya,kr),t(g,xr),t(g,it),t(it,Ja),t(Ja,za),t(za,qr),t(it,Tr),t(it,Ka),t(Ka,Ar),c(e,wl,i),c(e,$e,i),t($e,Pr),t($e,Qa),t(Qa,Cr),t($e,Dr),c(e,El,i),f(ct,e,i),c(e,$l,i),c(e,je,i),t(je,Or),t(je,Xa),t(Xa,Ir),t(je,Sr),c(e,jl,i),f(pt,e,i),c(e,kl,i),c(e,Gt,i),t(Gt,Nr),c(e,xl,i),f(ut,e,i),c(e,ql,i),f(ke,e,i),c(e,Tl,i),c(e,Z,i),t(Z,xe),t(xe,Za),f(ht,Za,null),t(Z,Mr),t(Z,es),t(es,Hr),c(e,Al,i),c(e,Rt,i),t(Rt,Br),c(e,Pl,i),c(e,qe,i),t(qe,ts),t(ts,Lr),t(qe,Fr),t(qe,as),t(as,Wr),c(e,Cl,i),c(e,x,i),t(x,Gr),t(x,Ut),t(Ut,Rr),t(x,Ur),t(x,Vt),t(Vt,Vr),t(x,Yr),t(x,Yt),t(Yt,Jr),t(x,zr),t(x,ss),t(ss,Kr),t(x,Qr),c(e,Dl,i),c(e,ee,i),t(ee,Te),t(Te,ls),f(dt,ls,null),t(ee,Xr),t(ee,os),t(os,Zr),c(e,Ol,i),c(e,D,i),t(D,ei),t(D,ns),t(ns,ti),t(D,ai),t(D,rs),t(rs,si),t(D,li),t(D,is),t(is,oi),t(D,ni),c(e,Il,i),f(mt,e,i),c(e,Sl,i),c(e,H,i),t(H,ri),t(H,cs),t(cs,ii),t(H,ci),t(H,ps),t(ps,pi),t(H,ui),c(e,Nl,i),c(e,te,i),t(te,Ae),t(Ae,us),f(ft,us,null),t(te,hi),t(te,hs),t(hs,di),c(e,Ml,i),c(e,q,i),t(q,mi),t(q,ds),t(ds,fi),t(q,vi),t(q,ms),t(ms,yi),t(q,_i),t(q,fs),t(fs,gi),t(q,bi),t(q,vs),t(vs,wi),t(q,Ei),c(e,Hl,i),f(vt,e,i),c(e,Bl,i),c(e,O,i),t(O,$i),t(O,ys),t(ys,ji),t(O,ki),t(O,_s),t(_s,xi),t(O,qi),t(O,gs),t(gs,Ti),t(O,Ai),c(e,Ll,i),f(yt,e,i),c(e,Fl,i),c(e,Jt,i),t(Jt,Pi),c(e,Wl,i),f(_t,e,i),c(e,Gl,i),c(e,ae,i),t(ae,Pe),t(Pe,bs),f(gt,bs,null),t(ae,Ci),t(ae,ws),t(ws,Di),c(e,Rl,i),c(e,T,i),t(T,Oi),t(T,Es),t(Es,Ii),t(T,Si),t(T,$s),t($s,Ni),t(T,Mi),t(T,bt),t(bt,Hi),t(T,Bi),t(T,js),t(js,Li),t(T,Fi),c(e,Ul,i),c(e,zt,i),t(zt,Wi),c(e,Vl,i),c(e,B,i),t(B,Gi),t(B,ks),t(ks,Ri),t(B,Ui),t(B,xs),t(xs,Vi),t(B,Yi),c(e,Yl,i),c(e,Kt,i),t(Kt,Ji),c(e,Jl,i),c(e,se,i),t(se,Ce),t(Ce,qs),f(wt,qs,null),t(se,zi),t(se,Ts),t(Ts,Ki),c(e,zl,i),c(e,De,i),t(De,Qi),t(De,Qt),t(Qt,Xi),t(De,Zi),c(e,Kl,i),f(Et,e,i),c(e,Ql,i),c(e,L,i),t(L,ec),t(L,As),t(As,tc),t(L,ac),t(L,Ps),t(Ps,sc),t(L,lc),c(e,Xl,i),f($t,e,i),c(e,Zl,i),c(e,le,i),t(le,Oe),t(Oe,Cs),f(jt,Cs,null),t(le,oc),t(le,Ds),t(Ds,nc),c(e,eo,i),c(e,Ie,i),t(Ie,rc),t(Ie,Xt),t(Xt,ic),t(Ie,cc),c(e,to,i),f(kt,e,i),c(e,ao,i),c(e,Zt,i),t(Zt,pc),c(e,so,i),f(xt,e,i),c(e,lo,i),c(e,ea,i),t(ea,uc),c(e,oo,i),c(e,Se,i),t(Se,hc),t(Se,ta),t(ta,dc),t(Se,mc),c(e,no,i),f(qt,e,i),c(e,ro,i),c(e,oe,i),t(oe,Ne),t(Ne,Os),f(Tt,Os,null),t(oe,fc),t(oe,Is),t(Is,vc),c(e,io,i),c(e,E,i),t(E,yc),t(E,aa),t(aa,_c),t(E,gc),t(E,Ss),t(Ss,bc),t(E,wc),t(E,At),t(At,Ec),t(E,$c),t(E,Ns),t(Ns,jc),t(E,kc),t(E,Ms),t(Ms,xc),t(E,qc),c(e,co,i),c(e,F,i),t(F,Tc),t(F,Hs),t(Hs,Ac),t(F,Pc),t(F,Bs),t(Bs,Cc),t(F,Dc),c(e,po,i),f(Pt,e,i),c(e,uo,i),c(e,W,i),t(W,Oc),t(W,Ls),t(Ls,Ic),t(W,Sc),t(W,Fs),t(Fs,Nc),t(W,Mc),c(e,ho,i),f(Ct,e,i),c(e,mo,i),c(e,G,i),t(G,Hc),t(G,Ws),t(Ws,Bc),t(G,Lc),t(G,Gs),t(Gs,Fc),t(G,Wc),c(e,fo,i),f(Dt,e,i),c(e,vo,i),c(e,$,i),t($,Gc),t($,Rs),t(Rs,Rc),t($,Uc),t($,Us),t(Us,Vc),t($,Yc),t($,Vs),t(Vs,Jc),t($,zc),t($,Ys),t(Ys,Kc),t($,Qc),t($,Js),t(Js,Xc),t($,Zc),yo=!0},p(e,[i]){const Ot={};i&2&&(Ot.$$scope={dirty:i,ctx:e}),ke.$set(Ot)},i(e){yo||(v(A.$$.fragment,e),v(Be.$$.fragment,e),v(Ue.$$.fragment,e),v(Ve.$$.fragment,e),v(Ye.$$.fragment,e),v(Je.$$.fragment,e),v(ze.$$.fragment,e),v(Ke.$$.fragment,e),v(Qe.$$.fragment,e),v(Xe.$$.fragment,e),v(ct.$$.fragment,e),v(pt.$$.fragment,e),v(ut.$$.fragment,e),v(ke.$$.fragment,e),v(ht.$$.fragment,e),v(dt.$$.fragment,e),v(mt.$$.fragment,e),v(ft.$$.fragment,e),v(vt.$$.fragment,e),v(yt.$$.fragment,e),v(_t.$$.fragment,e),v(gt.$$.fragment,e),v(wt.$$.fragment,e),v(Et.$$.fragment,e),v($t.$$.fragment,e),v(jt.$$.fragment,e),v(kt.$$.fragment,e),v(xt.$$.fragment,e),v(qt.$$.fragment,e),v(Tt.$$.fragment,e),v(Pt.$$.fragment,e),v(Ct.$$.fragment,e),v(Dt.$$.fragment,e),yo=!0)},o(e){y(A.$$.fragment,e),y(Be.$$.fragment,e),y(Ue.$$.fragment,e),y(Ve.$$.fragment,e),y(Ye.$$.fragment,e),y(Je.$$.fragment,e),y(ze.$$.fragment,e),y(Ke.$$.fragment,e),y(Qe.$$.fragment,e),y(Xe.$$.fragment,e),y(ct.$$.fragment,e),y(pt.$$.fragment,e),y(ut.$$.fragment,e),y(ke.$$.fragment,e),y(ht.$$.fragment,e),y(dt.$$.fragment,e),y(mt.$$.fragment,e),y(ft.$$.fragment,e),y(vt.$$.fragment,e),y(yt.$$.fragment,e),y(_t.$$.fragment,e),y(gt.$$.fragment,e),y(wt.$$.fragment,e),y(Et.$$.fragment,e),y($t.$$.fragment,e),y(jt.$$.fragment,e),y(kt.$$.fragment,e),y(xt.$$.fragment,e),y(qt.$$.fragment,e),y(Tt.$$.fragment,e),y(Pt.$$.fragment,e),y(Ct.$$.fragment,e),y(Dt.$$.fragment,e),yo=!1},d(e){a(w),e&&a(ne),e&&a(k),_(A),e&&a(Zs),e&&a(It),e&&a(el),e&&a(J),_(Be),e&&a(tl),e&&a(St),e&&a(al),e&&a(M),e&&a(sl),e&&a(he),e&&a(ll),e&&a(Re),e&&a(ol),e&&a(de),e&&a(nl),e&&a(z),_(Ue),e&&a(rl),e&&a(fe),e&&a(il),_(Ve,e),e&&a(cl),e&&a(Ht),e&&a(pl),_(Ye,e),e&&a(ul),e&&a(K),_(Je),e&&a(hl),e&&a(Bt),e&&a(dl),_(ze,e),e&&a(ml),e&&a(ye),e&&a(fl),e&&a(Q),_(Ke),e&&a(vl),e&&a(ge),e&&a(yl),_(Qe,e),e&&a(_l),e&&a(X),_(Xe),e&&a(gl),e&&a(we),e&&a(bl),e&&a(Ee),e&&a(wl),e&&a($e),e&&a(El),_(ct,e),e&&a($l),e&&a(je),e&&a(jl),_(pt,e),e&&a(kl),e&&a(Gt),e&&a(xl),_(ut,e),e&&a(ql),_(ke,e),e&&a(Tl),e&&a(Z),_(ht),e&&a(Al),e&&a(Rt),e&&a(Pl),e&&a(qe),e&&a(Cl),e&&a(x),e&&a(Dl),e&&a(ee),_(dt),e&&a(Ol),e&&a(D),e&&a(Il),_(mt,e),e&&a(Sl),e&&a(H),e&&a(Nl),e&&a(te),_(ft),e&&a(Ml),e&&a(q),e&&a(Hl),_(vt,e),e&&a(Bl),e&&a(O),e&&a(Ll),_(yt,e),e&&a(Fl),e&&a(Jt),e&&a(Wl),_(_t,e),e&&a(Gl),e&&a(ae),_(gt),e&&a(Rl),e&&a(T),e&&a(Ul),e&&a(zt),e&&a(Vl),e&&a(B),e&&a(Yl),e&&a(Kt),e&&a(Jl),e&&a(se),_(wt),e&&a(zl),e&&a(De),e&&a(Kl),_(Et,e),e&&a(Ql),e&&a(L),e&&a(Xl),_($t,e),e&&a(Zl),e&&a(le),_(jt),e&&a(eo),e&&a(Ie),e&&a(to),_(kt,e),e&&a(ao),e&&a(Zt),e&&a(so),_(xt,e),e&&a(lo),e&&a(ea),e&&a(oo),e&&a(Se),e&&a(no),_(qt,e),e&&a(ro),e&&a(oe),_(Tt),e&&a(io),e&&a(E),e&&a(co),e&&a(F),e&&a(po),_(Pt,e),e&&a(uo),e&&a(W),e&&a(ho),_(Ct,e),e&&a(mo),e&&a(G),e&&a(fo),_(Dt,e),e&&a(vo),e&&a($)}}}const Bh={local:"a-quick-tour",sections:[{local:"types-of-evaluations",title:"Types of evaluations"},{local:"load",sections:[{local:"community-modules",title:"Community modules"},{local:"list-available-modules",title:"List available modules"}],title:"Load"},{local:"module-attributes",title:"Module attributes"},{local:"compute",sections:[{local:"how-to-compute",title:"How to compute"},{local:"calculate-a-single-metric-or-a-batch-of-metrics",title:"Calculate a single metric or a batch of metrics"},{local:"distributed-evaluation",title:"Distributed evaluation"}],title:"Compute"},{local:"combining-several-evaluations",title:"Combining several evaluations"},{local:"save-and-push-to-the-hub",title:"Save and push to the Hub"},{local:"evaluator",title:"Evaluator"}],title:"A quick tour"};function Lh(Xs){return Sh(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class Uh extends Ph{constructor(w){super();Ch(this,w,Lh,Hh,Dh,{})}}export{Uh as default,Bh as metadata};
